s3 mtime

src/langs/en.json

@@ -173,6 +173,8 @@
   "settings_s3_bypasscorslocally_desc": "The plugin allows skipping server CORS config in new version of Obsidian ( desktop>=0.13.25 or iOS>=1.1.1 or Android>=1.2.1). If you encounter any issues, please disable this setting and config CORS on servers (allowing requests from app://obsidian.md and capacitor://localhost and http://localhost and add ETag into exposed headers).",
   "settings_s3_parts": "Parts Concurrency",
   "settings_s3_parts_desc": "Large files are split into small parts to upload in S3. How many parts do you want to upload in parallel at most?",
+  "settings_s3_accuratemtime": "Use Accurate MTime",
+  "settings_s3_accuratemtime_desc": "Read the uploaded accurate last modified time for better sync algorithm. But it causes extra api requests / time / money to the S3 endpoint.",
   "settings_s3_urlstyle": "S3 URL style",
   "settings_s3_urlstyle_desc": "Whether to force path-style URLs for S3 objects (e.g., https://s3.amazonaws.com/*/ instead of https://*.s3.amazonaws.com/).",
   "settings_s3_connect_succ": "Great! The bucket can be accessed.",

src/langs/zh_cn.json

@@ -173,6 +173,8 @@
   "settings_s3_bypasscorslocally_desc": "对于 Obsidian 新版本（桌面版>=0.13.25 或 iOS>=1.1.1 或 Android>=1.2.1），本插件可以跳过服务器设置 CORS 的步骤。如果您遇到任意问题，可以关闭此设定，并在服务端设置 CORS（允许来自 app://obsidian.md 和 capacitor://localhost 和 http://localhost 的请求且增加 ETag 到暴露 headers 里）。",
   "settings_s3_parts": "分块并行度",
   "settings_s3_parts_desc": "在 S3 里，大文件会被分块上传。您希望同一时间最多有多少个分块被上传？",
+  "settings_s3_accuratemtime": "使用准确的文件修改时间",
+  "settings_s3_accuratemtime_desc": "读取（已上传的）准确的文件修改时间，有助于同步算法更加准确和稳定。但是它也会导致额外的 api 请求、时间、金钱花费。",
   "settings_s3_urlstyle": "S3 URL style",
   "settings_s3_urlstyle_desc": "是否对 S3 对象强制使用 path style URL（例如使用 https://s3.amazonaws.com/*/ 而不是 https://*.s3.amazonaws.com/）。",
   "settings_s3_connect_succ": "很好！可以访问到对应存储桶。",

src/langs/zh_tw.json

@@ -173,6 +173,8 @@
   "settings_s3_bypasscorslocally_desc": "對於 Obsidian 新版本（桌面版>=0.13.25 或 iOS>=1.1.1 或 Android>=1.2.1），本外掛可以跳過伺服器設定 CORS 的步驟。如果您遇到任意問題，可以關閉此設定，並在服務端設定 CORS（允許來自 app://obsidian.md 和 capacitor://localhost 和 http://localhost 的請求且增加 ETag 到暴露 headers 裡）。",
   "settings_s3_parts": "分塊並行度",
   "settings_s3_parts_desc": "在 S3 裡，大檔案會被分塊上傳。您希望同一時間最多有多少個分塊被上傳？",
+  "settings_s3_accuratemtime": "使用準確的檔案修改時間",
+  "settings_s3_accuratemtime_desc": "讀取（已上傳的）準確的檔案修改時間，有助於同步演算法更加準確和穩定。但是它也會導致額外的 api 請求、時間、金錢花費。",
   "settings_s3_urlstyle": "S3 URL style",
   "settings_s3_urlstyle_desc": "是否對 S3 物件強制使用 path style URL（例如使用 https://s3.amazonaws.com/*/ 而不是 https://*.s3.amazonaws.com/）。",
   "settings_s3_connect_succ": "很好！可以訪問到對應儲存桶。",

src/main.ts

@@ -867,6 +867,10 @@ export default class RemotelySavePlugin extends Plugin {
     if (this.settings.s3.remotePrefix === undefined) {
       this.settings.s3.remotePrefix = "";
     }
+    if (this.settings.s3.useAccurateMTime === undefined) {
+      // it causes money, so disable it by default
+      this.settings.s3.useAccurateMTime = false;
+    }
     if (this.settings.ignorePaths === undefined) {
       this.settings.ignorePaths = [];
     }
@@ -884,6 +888,8 @@ export default class RemotelySavePlugin extends Plugin {
     if (requireApiVersion(API_VER_ENSURE_REQURL_OK)) {
       this.settings.s3.bypassCorsLocally = true; // deprecated as of 20240113
     }
+
+    await this.saveSettings();
   }
 
   async checkIfPresetRulesFollowed() {

src/remoteForS3.ts

@@ -42,6 +42,7 @@ import {
 export { S3Client } from "@aws-sdk/client-s3";
 
 import { log } from "./moreOnLog";
+import PQueue from "p-queue";
 
 ////////////////////////////////////////////////////////////////////////////////
 // special handler using Obsidian requestUrl
@@ -155,7 +156,7 @@ class ObsHttpHandler extends FetchHttpHandler {
 // other stuffs
 ////////////////////////////////////////////////////////////////////////////////
 
-export const DEFAULT_S3_CONFIG = {
+export const DEFAULT_S3_CONFIG: S3Config = {
   s3Endpoint: "",
   s3Region: "",
   s3AccessKeyID: "",
@@ -165,6 +166,7 @@ export const DEFAULT_S3_CONFIG = {
   partsConcurrency: 20,
   forcePathStyle: false,
   remotePrefix: "",
+  useAccurateMTime: false, // it causes money, disable by default
 };
 
 export type S3ObjectType = _Object;
@@ -218,24 +220,47 @@ const getLocalNoPrefixPath = (
   return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length);
 };
 
-const fromS3ObjectToRemoteItem = (x: S3ObjectType, remotePrefix: string) => {
-  return {
+const fromS3ObjectToRemoteItem = (
+  x: S3ObjectType,
+  remotePrefix: string,
+  mtimeRecords: Record<string, number>,
+  ctimeRecords: Record<string, number>
+) => {
+  let mtime = x.LastModified.valueOf();
+  if (x.Key in mtimeRecords) {
+    const m2 = mtimeRecords[x.Key];
+    if (m2 !== 0) {
+      mtime = m2;
+    }
+  }
+  const r: RemoteItem = {
     key: getLocalNoPrefixPath(x.Key, remotePrefix),
-    lastModified: x.LastModified.valueOf(),
+    lastModified: mtime,
     size: x.Size,
     remoteType: "s3",
     etag: x.ETag,
-  } as RemoteItem;
+  };
+  return r;
 };
 
 const fromS3HeadObjectToRemoteItem = (
   fileOrFolderPathWithRemotePrefix: string,
   x: HeadObjectCommandOutput,
-  remotePrefix: string
+  remotePrefix: string,
+  useAccurateMTime: boolean
 ) => {
+  let mtime = x.LastModified.valueOf();
+  if (useAccurateMTime && x.Metadata !== undefined) {
+    const m2 = Math.round(
+      parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0")
+    );
+    if (m2 !== 0) {
+      mtime = m2;
+    }
+  }
   return {
     key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix),
-    lastModified: x.LastModified.valueOf(),
+    lastModified: mtime,
     size: x.ContentLength,
     remoteType: "s3",
     etag: x.ETag,
@@ -307,7 +332,8 @@ export const getRemoteMeta = async (
   return fromS3HeadObjectToRemoteItem(
     fileOrFolderPathWithRemotePrefix,
     res,
-    s3Config.remotePrefix
+    s3Config.remotePrefix,
+    s3Config.useAccurateMTime
   );
 };
 
@@ -320,7 +346,9 @@ export const uploadToRemote = async (
   password: string = "",
   remoteEncryptedKey: string = "",
   uploadRaw: boolean = false,
-  rawContent: string | ArrayBuffer = ""
+  rawContent: string | ArrayBuffer = "",
+  rawContentMTime: number = 0,
+  rawContentCTime: number = 0
 ) => {
   let uploadFile = fileOrFolderPath;
   if (password !== "") {
@@ -336,6 +364,13 @@ export const uploadToRemote = async (
       throw Error(`you specify uploadRaw, but you also provide a folder key!`);
     }
     // folder
+    let mtime = 0;
+    let ctime = 0;
+    const s = await vault.adapter.stat(fileOrFolderPath);
+    if (s !== null) {
+      mtime = s.mtime;
+      ctime = s.ctime;
+    }
     const contentType = DEFAULT_CONTENT_TYPE;
     await s3Client.send(
       new PutObjectCommand({
@@ -343,6 +378,10 @@ export const uploadToRemote = async (
         Key: uploadFile,
         Body: "",
         ContentType: contentType,
+        Metadata: {
+          MTime: `${mtime}`,
+          CTime: `${ctime}`,
+        },
       })
     );
     return await getRemoteMeta(s3Client, s3Config, uploadFile);
@@ -357,14 +396,23 @@ export const uploadToRemote = async (
         ) || DEFAULT_CONTENT_TYPE;
     }
     let localContent = undefined;
+    let mtime = 0;
+    let ctime = 0;
     if (uploadRaw) {
       if (typeof rawContent === "string") {
         localContent = new TextEncoder().encode(rawContent).buffer;
       } else {
         localContent = rawContent;
       }
+      mtime = rawContentMTime;
+      ctime = rawContentCTime;
     } else {
       localContent = await vault.adapter.readBinary(fileOrFolderPath);
+      const s = await vault.adapter.stat(fileOrFolderPath);
+      if (s !== null) {
+        mtime = s.mtime;
+        ctime = s.ctime;
+      }
     }
     let remoteContent = localContent;
     if (password !== "") {
@@ -373,6 +421,7 @@ export const uploadToRemote = async (
 
     const bytesIn5MB = 5242880;
     const body = new Uint8Array(remoteContent);
+
     const upload = new Upload({
       client: s3Client,
       queueSize: s3Config.partsConcurrency, // concurrency
@@ -383,6 +432,10 @@ export const uploadToRemote = async (
         Key: uploadFile,
         Body: body,
         ContentType: contentType,
+        Metadata: {
+          MTime: `${mtime}`,
+          CTime: `${ctime}`,
+        },
       },
     });
     upload.on("httpUploadProgress", (progress) => {
@@ -407,6 +460,17 @@ const listFromRemoteRaw = async (
   }
 
   const contents = [] as _Object[];
+  const mtimeRecords: Record<string, number> = {};
+  const ctimeRecords: Record<string, number> = {};
+  const queueHead = new PQueue({
+    concurrency: s3Config.partsConcurrency,
+    autoStart: true,
+  });
+  queueHead.on("error", (error) => {
+    queueHead.pause();
+    queueHead.clear();
+    throw error;
+  });
 
   let isTruncated = true;
   do {
@@ -420,6 +484,37 @@ const listFromRemoteRaw = async (
     }
     contents.push(...rsp.Contents);
 
+    if (s3Config.useAccurateMTime) {
+      // head requests of all objects, love it
+      for (const content of rsp.Contents) {
+        queueHead.add(async () => {
+          const rspHead = await s3Client.send(
+            new HeadObjectCommand({
+              Bucket: s3Config.s3BucketName,
+              Key: content.Key,
+            })
+          );
+          if (rspHead.$metadata.httpStatusCode !== 200) {
+            throw Error("some thing bad while heading single object!");
+          }
+          if (rspHead.Metadata === undefined) {
+            // pass
+          } else {
+            mtimeRecords[content.Key] = Math.round(
+              parseFloat(
+                rspHead.Metadata.mtime || rspHead.Metadata.MTime || "0"
+              )
+            );
+            ctimeRecords[content.Key] = Math.round(
+              parseFloat(
+                rspHead.Metadata.ctime || rspHead.Metadata.CTime || "0"
+              )
+            );
+          }
+        });
+      }
+    }
+
     isTruncated = rsp.IsTruncated;
     confCmd.ContinuationToken = rsp.NextContinuationToken;
     if (
@@ -431,12 +526,20 @@ const listFromRemoteRaw = async (
     }
   } while (isTruncated);
 
+  // wait for any head requests
+  await queueHead.onIdle();
+
   // ensemble fake rsp
   // in the end, we need to transform the response list
   // back to the local contents-alike list
   return {
     Contents: contents.map((x) =>
-      fromS3ObjectToRemoteItem(x, s3Config.remotePrefix)
+      fromS3ObjectToRemoteItem(
+        x,
+        s3Config.remotePrefix,
+        mtimeRecords,
+        ctimeRecords
+      )
     ),
   };
 };

src/settings.ts

@@ -1010,6 +1010,29 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           });
       });
 
+    new Setting(s3Div)
+      .setName(t("settings_s3_accuratemtime"))
+      .setDesc(t("settings_s3_accuratemtime_desc"))
+      .addDropdown((dropdown) => {
+        dropdown
+            .addOption("disable", t("disable"))
+            .addOption("enable", t("enable"));
+
+        dropdown
+          .setValue(`${
+            this.plugin.settings.s3.useAccurateMTime ? "enable" : "disable"
+          }`)
+          .onChange(async (val) => {
+            if (val === "enable") {
+              this.plugin.settings.s3.useAccurateMTime = true;
+            } else {
+              this.plugin.settings.s3.useAccurateMTime = false;
+            }
+            await this.plugin.saveSettings();
+          });
+      });
+
+
     let newS3RemotePrefix = this.plugin.settings.s3.remotePrefix || "";
     new Setting(s3Div)
       .setName(t("settings_remoteprefix"))