From df7b6e18481549a4e4c21d84584c0a8ce4f45785 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 27 Apr 2024 12:01:30 +0800 Subject: [PATCH] allowing s3 synth folder --- src/baseTypes.ts | 2 ++ src/fsEncrypt.ts | 4 +++ src/fsS3.ts | 73 ++++++++++++++++++++++++++++++++++++++++---- src/langs/en.json | 4 +++ src/langs/zh_cn.json | 4 +++ src/langs/zh_tw.json | 4 +++ src/main.ts | 3 ++ src/settings.ts | 28 +++++++++++++++++ src/sync.ts | 42 ------------------------- 9 files changed, 116 insertions(+), 48 deletions(-) diff --git a/src/baseTypes.ts b/src/baseTypes.ts index caf4aa4..0b247a6 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -29,6 +29,8 @@ export interface S3Config { useAccurateMTime?: boolean; reverseProxyNoSignUrl?: string; + generateFolderObject?: boolean; + /** * @deprecated */ diff --git a/src/fsEncrypt.ts b/src/fsEncrypt.ts index be28a94..e0d6291 100644 --- a/src/fsEncrypt.ts +++ b/src/fsEncrypt.ts @@ -211,6 +211,7 @@ export class FakeFsEncrypt extends FakeFs { sizeEnc: innerEntity.size!, sizeRaw: innerEntity.sizeRaw, hash: undefined, + synthesizedFolder: innerEntity.synthesizedFolder, }); this.cacheMapOrigToEnc[key] = innerEntity.keyRaw; @@ -243,6 +244,7 @@ export class FakeFsEncrypt extends FakeFs { sizeEnc: innerEntity.size!, sizeRaw: innerEntity.sizeRaw, hash: undefined, + synthesizedFolder: innerEntity.synthesizedFolder, }; } } @@ -287,6 +289,7 @@ export class FakeFsEncrypt extends FakeFs { sizeEnc: innerEntity.size!, sizeRaw: innerEntity.sizeRaw, hash: undefined, + synthesizedFolder: innerEntity.synthesizedFolder, }; } } @@ -336,6 +339,7 @@ export class FakeFsEncrypt extends FakeFs { sizeEnc: innerEntity.size!, sizeRaw: innerEntity.sizeRaw, hash: undefined, + synthesizedFolder: innerEntity.synthesizedFolder, }; } } diff --git a/src/fsS3.ts b/src/fsS3.ts index 47a80f7..fd16843 100644 --- a/src/fsS3.ts +++ b/src/fsS3.ts @@ -26,7 +26,7 @@ import { Readable } from "stream"; import * as path from "path"; import AggregateError from "aggregate-error"; import { DEFAULT_CONTENT_TYPE, S3Config, VALID_REQURL } from "./baseTypes"; -import { bufferToArrayBuffer } from "./misc"; +import { bufferToArrayBuffer, getFolderLevels } from "./misc"; import PQueue from "p-queue"; import { Entity } from "./baseTypes"; @@ -186,6 +186,7 @@ export const DEFAULT_S3_CONFIG: S3Config = { remotePrefix: "", useAccurateMTime: false, // it causes money, disable by default reverseProxyNoSignUrl: "", + generateFolderObject: false, // new version, by default not generate folders }; /** @@ -385,11 +386,13 @@ export class FakeFsS3 extends FakeFs { s3Config: S3Config; s3Client: S3Client; kind: "s3"; + synthFoldersCache: Record; constructor(s3Config: S3Config) { super(); this.s3Config = s3Config; this.s3Client = getS3Client(s3Config); this.kind = "s3"; + this.synthFoldersCache = {}; } async walk(): Promise { @@ -484,17 +487,52 @@ export class FakeFsS3 extends FakeFs { // ensemble fake rsp // in the end, we need to transform the response list // back to the local contents-alike list - return contents.map((x) => - fromS3ObjectToEntity( - x, + const res: Entity[] = []; + const realEnrities = new Set(); + for (const remoteObj of contents) { + const remoteEntity = fromS3ObjectToEntity( + remoteObj, this.s3Config.remotePrefix ?? "", mtimeRecords, ctimeRecords - ) - ); + ); + realEnrities.add(remoteEntity.key!); + res.push(remoteEntity); + + for (const f of getFolderLevels(remoteEntity.key!, true)) { + if (realEnrities.has(f)) { + delete this.synthFoldersCache[f]; + continue; + } + if ( + !this.synthFoldersCache.hasOwnProperty(f) || + remoteEntity.mtimeSvr! >= this.synthFoldersCache[f].mtimeSvr! + ) { + this.synthFoldersCache[f] = { + key: f, + keyRaw: f, + size: 0, + sizeRaw: 0, + sizeEnc: 0, + mtimeSvr: remoteEntity.mtimeSvr, + mtimeSvrFmt: remoteEntity.mtimeSvrFmt, + mtimeCli: remoteEntity.mtimeCli, + mtimeCliFmt: remoteEntity.mtimeCliFmt, + synthesizedFolder: true, + }; + } + } + } + for (const key of Object.keys(this.synthFoldersCache)) { + res.push(this.synthFoldersCache[key]); + } + return res; } async stat(key: string): Promise { + if (this.synthFoldersCache.hasOwnProperty(key)) { + return this.synthFoldersCache[key]; + } let keyFullPath = key; keyFullPath = getRemoteWithPrefixPath( keyFullPath, @@ -529,6 +567,23 @@ export class FakeFsS3 extends FakeFs { if (!key.endsWith("/")) { throw new Error(`You should not call mkdir on ${key}!`); } + + const generateFolderObject = this.s3Config.generateFolderObject ?? false; + if (!generateFolderObject) { + const synth = { + key: key, + keyRaw: key, + size: 0, + sizeRaw: 0, + sizeEnc: 0, + mtimeSvr: mtime, + mtimeCli: mtime, + synthesizedFolder: true, + }; + this.synthFoldersCache[key] = synth; + return synth; + } + const uploadFile = getRemoteWithPrefixPath( key, this.s3Config.remotePrefix ?? "" @@ -670,6 +725,12 @@ export class FakeFsS3 extends FakeFs { if (key === "/") { return; } + + if (this.synthFoldersCache.hasOwnProperty(key)) { + delete this.synthFoldersCache[key]; + return; + } + const remoteFileName = getRemoteWithPrefixPath( key, this.s3Config.remotePrefix ?? "" diff --git a/src/langs/en.json b/src/langs/en.json index 9aa9b68..0cd2ba7 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -183,6 +183,10 @@ "settings_s3_urlstyle_desc": "Whether to force path-style URLs for S3 objects (e.g., https://s3.amazonaws.com/*/ instead of https://*.s3.amazonaws.com/).", "settings_s3_reverse_proxy_no_sign_url": "S3 Reverse Proxy (No Sign) Url (experimental)", "settings_s3_reverse_proxy_no_sign_url_desc": "S3 reverse proxy url without signature. This is useful if you use a revers proxy but do not change the original credential signature. No http(s):// prefix. Leave it blank if you don't know what it is.", + "settings_s3_generatefolderobject": "Generate Folder Object Or Not", + "settings_s3_generatefolderobject_desc": "S3 doesn't have \"real\" folder. If you set \"Generate\" here (or use old version), the plugin will upload a zero-byte object endding with \"/\" to represent the folder. In the new version, the plugin skips generating folder object by default.", + "settings_s3_generatefolderobject_notgenerate": "Not generate (default)", + "settings_s3_generatefolderobject_generate": "Generate", "settings_s3_connect_succ": "Great! The bucket can be accessed.", "settings_s3_connect_fail": "The S3 bucket cannot be reached.", "settings_dropbox": "Remote For Dropbox", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index 2b242dd..fdb2d0f 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -182,6 +182,10 @@ "settings_s3_urlstyle_desc": "是否对 S3 对象强制使用 path style URL（例如使用 https://s3.amazonaws.com/*/ 而不是 https://*.s3.amazonaws.com/）。", "settings_s3_reverse_proxy_no_sign_url": "S3 反向代理（不签名）地址（实验性质）", "settings_s3_reverse_proxy_no_sign_url_desc": "不会参与到签名的 S3 反向代理地址。如果您有一个反向代理，但是不想修改原始鉴权签名，这里就可以填写。没有 http(s):// 前缀。如果您不知道这是什么，留空即可。", + "settings_s3_generatefolderobject": "是否生成文件夹 Object", + "settings_s3_generatefolderobject_desc": "S3 不存在“真正”的文件夹。如果您设置了“生成”（或用了旧版本），那么插件会上传 0 字节的以“/”结尾的 Object 来代表文件夹。新版本插件会默认跳过生成这种文件夹 Object。", + "settings_s3_generatefolderobject_notgenerate": "不生成（默认）", + "settings_s3_generatefolderobject_generate": "生成", "settings_s3_connect_succ": "很好！可以访问到对应存储桶。", "settings_s3_connect_fail": "无法访问到对应存储桶。", "settings_dropbox": "Dropbox 设置", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index 3567893..dfcfdb0 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -181,6 +181,10 @@ "settings_s3_urlstyle_desc": "是否對 S3 物件強制使用 path style URL（例如使用 https://s3.amazonaws.com/*/ 而不是 https://*.s3.amazonaws.com/）。", "settings_s3_reverse_proxy_no_sign_url": "S3 反向代理（不簽名）地址（實驗性質）", "settings_s3_reverse_proxy_no_sign_url_desc": "不會參與到簽名的 S3 反向代理地址。如果您有一個反向代理，但是不想修改原始鑑權簽名，這裡就可以填寫。沒有 http(s):// 字首。如果您不知道這是什麼，留空即可。", + "settings_s3_generatefolderobject": "是否生成文件夾 Object", + "settings_s3_generatefolderobject_desc": "S3 不存在“真正”的文件夾。如果您設置了“生成”（或用了舊版本），那麼插件會上傳 0 字節的以“/”結尾的 Object 來代表文件夾。新版本插件會默認跳過生成這種文件夾 Object。", + "settings_s3_generatefolderobject_notgenerate": "不生成（默認）", + "settings_s3_generatefolderobject_generate": "生成", "settings_s3_connect_succ": "很好！可以訪問到對應儲存桶。", "settings_s3_connect_fail": "無法訪問到對應儲存桶。", "settings_dropbox": "Dropbox 設定", diff --git a/src/main.ts b/src/main.ts index e4b5b7d..3256290 100644 --- a/src/main.ts +++ b/src/main.ts @@ -868,6 +868,9 @@ export default class RemotelySavePlugin extends Plugin { // it causes money, so disable it by default this.settings.s3.useAccurateMTime = false; } + if (this.settings.s3.generateFolderObject === undefined) { + this.settings.s3.generateFolderObject = false; + } if (this.settings.ignorePaths === undefined) { this.settings.ignorePaths = []; } diff --git a/src/settings.ts b/src/settings.ts index adee8b4..8ec65db 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -1067,6 +1067,34 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }) ); + new Setting(s3Div) + .setName(t("settings_s3_generatefolderobject")) + .setDesc(t("settings_s3_generatefolderobject_desc")) + .addDropdown((dropdown) => { + dropdown + .addOption( + "notgenerate", + t("settings_s3_generatefolderobject_notgenerate") + ) + .addOption( + "generate", + t("settings_s3_generatefolderobject_generate") + ); + + dropdown + .setValue( + `${this.plugin.settings.s3.generateFolderObject ? "generate" : "notgenerate"}` + ) + .onChange(async (val) => { + if (val === "generate") { + this.plugin.settings.s3.generateFolderObject = true; + } else { + this.plugin.settings.s3.generateFolderObject = false; + } + await this.plugin.saveSettings(); + }); + }); + new Setting(s3Div) .setName(t("settings_checkonnectivity")) .setDesc(t("settings_checkonnectivity_desc")) diff --git a/src/sync.ts b/src/sync.ts index 1993e01..69bed26 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -20,7 +20,6 @@ import { } from "./localdb"; import { atWhichLevel, - getFolderLevels, getParentFolder, isHiddenPath, isSpecialFolderNameToSkip, @@ -161,10 +160,7 @@ const ensembleMixedEnties = async ( const finalMappings: SyncPlanType = {}; - const synthFolders: Record = {}; - // remote has to be first - // we also have to synthesize folders here for (const remote of remoteEntityList) { const remoteCopied = ensureMTimeOfRemoteEntityValid( copyEntityAndFixTimeFormat(remote, serviceType) @@ -187,48 +183,10 @@ const ensembleMixedEnties = async ( key: key, remote: remoteCopied, }; - - for (const f of getFolderLevels(key, true)) { - if (finalMappings.hasOwnProperty(f)) { - delete synthFolders[f]; - continue; - } - if ( - !synthFolders.hasOwnProperty(f) || - remoteCopied.mtimeSvr! >= synthFolders[f].mtimeSvr! - ) { - synthFolders[f] = { - key: f, - keyRaw: ``, - keyEnc: ``, - size: 0, - sizeRaw: 0, - sizeEnc: 0, - mtimeSvr: remoteCopied.mtimeSvr, - mtimeSvrFmt: remoteCopied.mtimeSvrFmt, - mtimeCli: remoteCopied.mtimeCli, - mtimeCliFmt: remoteCopied.mtimeCliFmt, - synthesizedFolder: true, - }; - } - } } profiler.insert("ensembleMixedEnties: finish remote"); - console.debug(`synthFolders:`); - console.debug(synthFolders); - - // special: add synth folders - for (const key of Object.keys(synthFolders)) { - finalMappings[key] = { - key: key, - remote: synthFolders[key], - }; - } - - profiler.insert("ensembleMixedEnties: finish synth"); - if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) { // Special checking: // if one side is totally empty,