import * as path from "path"; import { AnonymousCredential, BaseRequestPolicy, type BlobGetPropertiesResponse, BlobServiceClient, BlobUploadCommonResponse, BlockBlobClient, ContainerClient, newPipeline, } from "@azure/storage-blob"; import type { Entity } from "../../src/baseTypes"; import { FakeFs } from "../../src/fsAll"; import { arrayBufferToHex, getFolderLevels } from "../../src/misc"; import type { AzureBlobStorageConfig } from "./baseTypesPro"; export const simpleTransRemotePrefix = (x: string) => { if (x === undefined) { return ""; } let y = path.posix.normalize(x.trim()); if (y === undefined || y === "" || y === "/" || y === ".") { return ""; } if (y.startsWith("/")) { y = y.slice(1); } if (!y.endsWith("/")) { y = `${y}/`; } return y; }; export const DEFAULT_AZUREBLOBSTORAGE_CONFIG: AzureBlobStorageConfig = { containerSasUrl: "", containerName: "", remotePrefix: "", generateFolderObject: false, partsConcurrency: 5, kind: "azureblobstorage", }; const getNormPath = (fileOrFolderPath: string, remotePrefix: string) => { if (remotePrefix.startsWith("/") || !remotePrefix.endsWith("/")) { throw Error( `remotePrefix should not have leading slash but should have tailing slash: ${remotePrefix}` ); } if (!fileOrFolderPath.startsWith(remotePrefix)) { throw Error(`${fileOrFolderPath} does not start with ${remotePrefix}!`); } return fileOrFolderPath.slice(remotePrefix.length); }; const getBlobPath = (fileOrFolderPath: string, remotePrefix: string) => { if (remotePrefix.startsWith("/") || !remotePrefix.endsWith("/")) { throw Error( `remotePrefix should not have leading slash but should have tailing slash: ${remotePrefix}` ); } return `${remotePrefix}${fileOrFolderPath}`; }; const fromBlobPropsToEntity = ( name: string, props: BlobGetPropertiesResponse, remotePrefix: string ): Entity => { const key = getNormPath(name, remotePrefix); let mtimeCli = props.lastModified!.valueOf(); const mtimeStr = props.metadata?.mtime; if (mtimeStr !== undefined && mtimeStr !== "") { try { mtimeCli = new Date(mtimeStr).valueOf(); } catch {} } let hash: undefined | string = undefined; if (props.contentMD5 !== undefined) { hash = arrayBufferToHex(props.contentMD5.buffer); } const entity: Entity = { key: key, keyRaw: key, mtimeCli: mtimeCli, mtimeSvr: props.lastModified!.valueOf(), size: props.contentLength ?? 0, sizeRaw: props.contentLength ?? 0, hash: hash, }; if (key.endsWith("/")) { entity.synthesizedFolder = false; } return entity; }; export class FakeFsAzureBlobStorage extends FakeFs { kind: string; config: AzureBlobStorageConfig; vaultName: string; containerClient: ContainerClient; remotePrefix: string; synthFoldersCache: Record; constructor(config: AzureBlobStorageConfig, vaultName: string) { super(); this.kind = "azureblobstorage"; this.config = config; this.vaultName = vaultName; this.synthFoldersCache = {}; this.remotePrefix = `${vaultName}/`; const k = simpleTransRemotePrefix(this.config.remotePrefix); if (k !== "") { // we have prefix this.remotePrefix = k; } this.containerClient = new ContainerClient(this.config.containerSasUrl); } async walk(): Promise { const entities: Entity[] = []; const realEntities = new Set(); for await (const blob of this.containerClient.listBlobsFlat({ prefix: this.remotePrefix, includeMetadata: true, })) { const blockBlobClient = this.containerClient.getBlockBlobClient( blob.name ); const props = await blockBlobClient.getProperties(); // console.debug(blob.name) const entity = fromBlobPropsToEntity(blob.name, props, this.remotePrefix); entities.push(entity); // so we need to fake the folders realEntities.add(entity.key!); for (const f of getFolderLevels(entity.key!, true)) { if (realEntities.has(f)) { delete this.synthFoldersCache[f]; continue; } if ( !this.synthFoldersCache.hasOwnProperty(f) || entity.mtimeSvr! >= this.synthFoldersCache[f].mtimeSvr! ) { this.synthFoldersCache[f] = { key: f, keyRaw: f, size: 0, sizeRaw: 0, sizeEnc: 0, mtimeSvr: entity.mtimeSvr, mtimeSvrFmt: entity.mtimeSvrFmt, mtimeCli: entity.mtimeCli, mtimeCliFmt: entity.mtimeCliFmt, synthesizedFolder: true, }; } } } for (const key of Object.keys(this.synthFoldersCache)) { entities.push(this.synthFoldersCache[key]); } return entities; } async walkPartial(): Promise { const entities: Entity[] = []; for await (const blob of this.containerClient.listBlobsByHierarchy("/", { prefix: this.remotePrefix, includeMetadata: true, })) { if (blob.kind === "prefix") { continue; } const blockBlobClient = this.containerClient.getBlockBlobClient( blob.name ); const props = await blockBlobClient.getProperties(); const entity = fromBlobPropsToEntity(blob.name, props, this.remotePrefix); entities.push(entity); } return entities; } async stat(key: string): Promise { const remotePath = getBlobPath(key, this.remotePrefix); const blockBlobClient = this.containerClient.getBlockBlobClient(remotePath); const props = await blockBlobClient.getProperties(); const entity = fromBlobPropsToEntity(remotePath, props, this.remotePrefix); return entity; } async mkdir( key: string, mtime?: number | undefined, ctime?: number | undefined ): Promise { if (!key.endsWith("/")) { throw new Error(`You should not call mkdir on ${key}!`); } const generateFolderObject = this.config.generateFolderObject ?? false; if (!generateFolderObject) { const synth = { key: key, keyRaw: key, size: 0, sizeRaw: 0, sizeEnc: 0, mtimeSvr: mtime, mtimeCli: mtime, synthesizedFolder: true, }; this.synthFoldersCache[key] = synth; return synth; } return await this.writeFile( key, new ArrayBuffer(0), mtime ?? Date.now(), ctime ?? Date.now() ); } async writeFile( key: string, content: ArrayBuffer, mtime: number, ctime: number ): Promise { const blobPath = getBlobPath(key, this.remotePrefix); const blobClient = this.containerClient.getBlockBlobClient(blobPath); const metadata: Record = { mtime: new Date(mtime).toISOString(), ctime: new Date(ctime).toISOString(), }; if (key.endsWith("/")) { console.debug(`yeah we have folder upload`); const generateFolderObject = this.config.generateFolderObject ?? false; if (!generateFolderObject) { throw Error( `if not generate folder object, the func should not go here` ); } metadata["hdi_isfolder"] = "true"; } const uploadResult = await blobClient.uploadData(content, { metadata: metadata, concurrency: this.config.partsConcurrency ?? 5, }); if (key.endsWith("/")) { console.debug(`yeah we have folder upload`); console.debug(uploadResult); } if (uploadResult._response.status >= 300) { throw Error(`upload ${key} failed with ${JSON.stringify(uploadResult)}`); } return await this.stat(key); } async readFile(key: string): Promise { const blobPath = getBlobPath(key, this.remotePrefix); const blobClient = this.containerClient.getBlockBlobClient(blobPath); const rsp = await blobClient.download(); if (rsp._response.status >= 300) { throw Error(`download ${key} failed with ${JSON.stringify(rsp)}`); } return await (await rsp.blobBody)!.arrayBuffer(); } async rename(key1: string, key2: string): Promise { throw new Error("Method not implemented."); } async rm(key: string): Promise { const blobPath = getBlobPath(key, this.remotePrefix); if (key.endsWith("/")) { if (this.synthFoldersCache.hasOwnProperty(key)) { delete this.synthFoldersCache[key]; } // in blob the folder may not exist, so we make our best effort. // do NOT read this.config.generateFolderObject // because the folder might be generated by previous setting try { const blobClient = this.containerClient.getBlockBlobClient(blobPath); await blobClient.deleteIfExists(); } catch (e) {} } else { // the file should really exist const blobClient = this.containerClient.getBlockBlobClient(blobPath); const rsp = await blobClient.deleteIfExists(); if (!rsp.succeeded) { throw Error( `something goes wrong while deleting ${key}: ${JSON.stringify(rsp)}` ); } } } async checkConnect(callbackFunc?: any): Promise { // if we can walk, we can connect try { await this.walkPartial(); return true; } catch (err) { console.debug(err); callbackFunc?.(err); return false; } } async getUserDisplayName(): Promise { throw new Error("Method not implemented."); } async revokeAuth(): Promise { throw new Error("Method not implemented."); } allowEmptyFile(): boolean { return true; } }