From 29a3c76b4e656168412fa125c8b0ee33e91f8fcc Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 13 Jan 2024 19:27:08 +0800 Subject: [PATCH] s3 mtime --- src/langs/en.json | 2 + src/langs/zh_cn.json | 2 + src/langs/zh_tw.json | 2 + src/main.ts | 6 +++ src/remoteForS3.ts | 123 +++++++++++++++++++++++++++++++++++++++---- src/settings.ts | 23 ++++++++ 6 files changed, 148 insertions(+), 10 deletions(-) diff --git a/src/langs/en.json b/src/langs/en.json index 3deb17d..859017f 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -173,6 +173,8 @@ "settings_s3_bypasscorslocally_desc": "The plugin allows skipping server CORS config in new version of Obsidian ( desktop>=0.13.25 or iOS>=1.1.1 or Android>=1.2.1). If you encounter any issues, please disable this setting and config CORS on servers (allowing requests from app://obsidian.md and capacitor://localhost and http://localhost and add ETag into exposed headers).", "settings_s3_parts": "Parts Concurrency", "settings_s3_parts_desc": "Large files are split into small parts to upload in S3. How many parts do you want to upload in parallel at most?", + "settings_s3_accuratemtime": "Use Accurate MTime", + "settings_s3_accuratemtime_desc": "Read the uploaded accurate last modified time for better sync algorithm. But it causes extra api requests / time / money to the S3 endpoint.", "settings_s3_urlstyle": "S3 URL style", "settings_s3_urlstyle_desc": "Whether to force path-style URLs for S3 objects (e.g., https://s3.amazonaws.com/*/ instead of https://*.s3.amazonaws.com/).", "settings_s3_connect_succ": "Great! The bucket can be accessed.", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index eaf140e..d3635d6 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -173,6 +173,8 @@ "settings_s3_bypasscorslocally_desc": "对于 Obsidian 新版本（桌面版>=0.13.25 或 iOS>=1.1.1 或 Android>=1.2.1），本插件可以跳过服务器设置 CORS 的步骤。如果您遇到任意问题，可以关闭此设定，并在服务端设置 CORS（允许来自 app://obsidian.md 和 capacitor://localhost 和 http://localhost 的请求且增加 ETag 到暴露 headers 里）。", "settings_s3_parts": "分块并行度", "settings_s3_parts_desc": "在 S3 里，大文件会被分块上传。您希望同一时间最多有多少个分块被上传？", + "settings_s3_accuratemtime": "使用准确的文件修改时间", + "settings_s3_accuratemtime_desc": "读取（已上传的）准确的文件修改时间，有助于同步算法更加准确和稳定。但是它也会导致额外的 api 请求、时间、金钱花费。", "settings_s3_urlstyle": "S3 URL style", "settings_s3_urlstyle_desc": "是否对 S3 对象强制使用 path style URL（例如使用 https://s3.amazonaws.com/*/ 而不是 https://*.s3.amazonaws.com/）。", "settings_s3_connect_succ": "很好！可以访问到对应存储桶。", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index 66c5a98..1d1a4ef 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -173,6 +173,8 @@ "settings_s3_bypasscorslocally_desc": "對於 Obsidian 新版本（桌面版>=0.13.25 或 iOS>=1.1.1 或 Android>=1.2.1），本外掛可以跳過伺服器設定 CORS 的步驟。如果您遇到任意問題，可以關閉此設定，並在服務端設定 CORS（允許來自 app://obsidian.md 和 capacitor://localhost 和 http://localhost 的請求且增加 ETag 到暴露 headers 裡）。", "settings_s3_parts": "分塊並行度", "settings_s3_parts_desc": "在 S3 裡，大檔案會被分塊上傳。您希望同一時間最多有多少個分塊被上傳？", + "settings_s3_accuratemtime": "使用準確的檔案修改時間", + "settings_s3_accuratemtime_desc": "讀取（已上傳的）準確的檔案修改時間，有助於同步演算法更加準確和穩定。但是它也會導致額外的 api 請求、時間、金錢花費。", "settings_s3_urlstyle": "S3 URL style", "settings_s3_urlstyle_desc": "是否對 S3 物件強制使用 path style URL（例如使用 https://s3.amazonaws.com/*/ 而不是 https://*.s3.amazonaws.com/）。", "settings_s3_connect_succ": "很好！可以訪問到對應儲存桶。", diff --git a/src/main.ts b/src/main.ts index edf9c76..4a22fb3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -867,6 +867,10 @@ export default class RemotelySavePlugin extends Plugin { if (this.settings.s3.remotePrefix === undefined) { this.settings.s3.remotePrefix = ""; } + if (this.settings.s3.useAccurateMTime === undefined) { + // it causes money, so disable it by default + this.settings.s3.useAccurateMTime = false; + } if (this.settings.ignorePaths === undefined) { this.settings.ignorePaths = []; } @@ -884,6 +888,8 @@ export default class RemotelySavePlugin extends Plugin { if (requireApiVersion(API_VER_ENSURE_REQURL_OK)) { this.settings.s3.bypassCorsLocally = true; // deprecated as of 20240113 } + + await this.saveSettings(); } async checkIfPresetRulesFollowed() { diff --git a/src/remoteForS3.ts b/src/remoteForS3.ts index b060f0f..5803d50 100644 --- a/src/remoteForS3.ts +++ b/src/remoteForS3.ts @@ -42,6 +42,7 @@ import { export { S3Client } from "@aws-sdk/client-s3"; import { log } from "./moreOnLog"; +import PQueue from "p-queue"; //////////////////////////////////////////////////////////////////////////////// // special handler using Obsidian requestUrl @@ -155,7 +156,7 @@ class ObsHttpHandler extends FetchHttpHandler { // other stuffs //////////////////////////////////////////////////////////////////////////////// -export const DEFAULT_S3_CONFIG = { +export const DEFAULT_S3_CONFIG: S3Config = { s3Endpoint: "", s3Region: "", s3AccessKeyID: "", @@ -165,6 +166,7 @@ export const DEFAULT_S3_CONFIG = { partsConcurrency: 20, forcePathStyle: false, remotePrefix: "", + useAccurateMTime: false, // it causes money, disable by default }; export type S3ObjectType = _Object; @@ -218,24 +220,47 @@ const getLocalNoPrefixPath = ( return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length); }; -const fromS3ObjectToRemoteItem = (x: S3ObjectType, remotePrefix: string) => { - return { +const fromS3ObjectToRemoteItem = ( + x: S3ObjectType, + remotePrefix: string, + mtimeRecords: Record, + ctimeRecords: Record +) => { + let mtime = x.LastModified.valueOf(); + if (x.Key in mtimeRecords) { + const m2 = mtimeRecords[x.Key]; + if (m2 !== 0) { + mtime = m2; + } + } + const r: RemoteItem = { key: getLocalNoPrefixPath(x.Key, remotePrefix), - lastModified: x.LastModified.valueOf(), + lastModified: mtime, size: x.Size, remoteType: "s3", etag: x.ETag, - } as RemoteItem; + }; + return r; }; const fromS3HeadObjectToRemoteItem = ( fileOrFolderPathWithRemotePrefix: string, x: HeadObjectCommandOutput, - remotePrefix: string + remotePrefix: string, + useAccurateMTime: boolean ) => { + let mtime = x.LastModified.valueOf(); + if (useAccurateMTime && x.Metadata !== undefined) { + const m2 = Math.round( + parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0") + ); + if (m2 !== 0) { + mtime = m2; + } + } return { key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix), - lastModified: x.LastModified.valueOf(), + lastModified: mtime, size: x.ContentLength, remoteType: "s3", etag: x.ETag, @@ -307,7 +332,8 @@ export const getRemoteMeta = async ( return fromS3HeadObjectToRemoteItem( fileOrFolderPathWithRemotePrefix, res, - s3Config.remotePrefix + s3Config.remotePrefix, + s3Config.useAccurateMTime ); }; @@ -320,7 +346,9 @@ export const uploadToRemote = async ( password: string = "", remoteEncryptedKey: string = "", uploadRaw: boolean = false, - rawContent: string | ArrayBuffer = "" + rawContent: string | ArrayBuffer = "", + rawContentMTime: number = 0, + rawContentCTime: number = 0 ) => { let uploadFile = fileOrFolderPath; if (password !== "") { @@ -336,6 +364,13 @@ export const uploadToRemote = async ( throw Error(`you specify uploadRaw, but you also provide a folder key!`); } // folder + let mtime = 0; + let ctime = 0; + const s = await vault.adapter.stat(fileOrFolderPath); + if (s !== null) { + mtime = s.mtime; + ctime = s.ctime; + } const contentType = DEFAULT_CONTENT_TYPE; await s3Client.send( new PutObjectCommand({ @@ -343,6 +378,10 @@ export const uploadToRemote = async ( Key: uploadFile, Body: "", ContentType: contentType, + Metadata: { + MTime: `${mtime}`, + CTime: `${ctime}`, + }, }) ); return await getRemoteMeta(s3Client, s3Config, uploadFile); @@ -357,14 +396,23 @@ export const uploadToRemote = async ( ) || DEFAULT_CONTENT_TYPE; } let localContent = undefined; + let mtime = 0; + let ctime = 0; if (uploadRaw) { if (typeof rawContent === "string") { localContent = new TextEncoder().encode(rawContent).buffer; } else { localContent = rawContent; } + mtime = rawContentMTime; + ctime = rawContentCTime; } else { localContent = await vault.adapter.readBinary(fileOrFolderPath); + const s = await vault.adapter.stat(fileOrFolderPath); + if (s !== null) { + mtime = s.mtime; + ctime = s.ctime; + } } let remoteContent = localContent; if (password !== "") { @@ -373,6 +421,7 @@ export const uploadToRemote = async ( const bytesIn5MB = 5242880; const body = new Uint8Array(remoteContent); + const upload = new Upload({ client: s3Client, queueSize: s3Config.partsConcurrency, // concurrency @@ -383,6 +432,10 @@ export const uploadToRemote = async ( Key: uploadFile, Body: body, ContentType: contentType, + Metadata: { + MTime: `${mtime}`, + CTime: `${ctime}`, + }, }, }); upload.on("httpUploadProgress", (progress) => { @@ -407,6 +460,17 @@ const listFromRemoteRaw = async ( } const contents = [] as _Object[]; + const mtimeRecords: Record = {}; + const ctimeRecords: Record = {}; + const queueHead = new PQueue({ + concurrency: s3Config.partsConcurrency, + autoStart: true, + }); + queueHead.on("error", (error) => { + queueHead.pause(); + queueHead.clear(); + throw error; + }); let isTruncated = true; do { @@ -420,6 +484,37 @@ const listFromRemoteRaw = async ( } contents.push(...rsp.Contents); + if (s3Config.useAccurateMTime) { + // head requests of all objects, love it + for (const content of rsp.Contents) { + queueHead.add(async () => { + const rspHead = await s3Client.send( + new HeadObjectCommand({ + Bucket: s3Config.s3BucketName, + Key: content.Key, + }) + ); + if (rspHead.$metadata.httpStatusCode !== 200) { + throw Error("some thing bad while heading single object!"); + } + if (rspHead.Metadata === undefined) { + // pass + } else { + mtimeRecords[content.Key] = Math.round( + parseFloat( + rspHead.Metadata.mtime || rspHead.Metadata.MTime || "0" + ) + ); + ctimeRecords[content.Key] = Math.round( + parseFloat( + rspHead.Metadata.ctime || rspHead.Metadata.CTime || "0" + ) + ); + } + }); + } + } + isTruncated = rsp.IsTruncated; confCmd.ContinuationToken = rsp.NextContinuationToken; if ( @@ -431,12 +526,20 @@ const listFromRemoteRaw = async ( } } while (isTruncated); + // wait for any head requests + await queueHead.onIdle(); + // ensemble fake rsp // in the end, we need to transform the response list // back to the local contents-alike list return { Contents: contents.map((x) => - fromS3ObjectToRemoteItem(x, s3Config.remotePrefix) + fromS3ObjectToRemoteItem( + x, + s3Config.remotePrefix, + mtimeRecords, + ctimeRecords + ) ), }; }; diff --git a/src/settings.ts b/src/settings.ts index 910c7cd..aa94712 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -1010,6 +1010,29 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); + new Setting(s3Div) + .setName(t("settings_s3_accuratemtime")) + .setDesc(t("settings_s3_accuratemtime_desc")) + .addDropdown((dropdown) => { + dropdown + .addOption("disable", t("disable")) + .addOption("enable", t("enable")); + + dropdown + .setValue(`${ + this.plugin.settings.s3.useAccurateMTime ? "enable" : "disable" + }`) + .onChange(async (val) => { + if (val === "enable") { + this.plugin.settings.s3.useAccurateMTime = true; + } else { + this.plugin.settings.s3.useAccurateMTime = false; + } + await this.plugin.saveSettings(); + }); + }); + + let newS3RemotePrefix = this.plugin.settings.s3.remotePrefix || ""; new Setting(s3Div) .setName(t("settings_remoteprefix"))