bump to 0.4.12

fix AWS S3 user's policy configuration issue https://github.com/remotely-save/remotely-save/issues/564

docs/remote_services/s3_general/s3_user_policy.md

@@ -33,6 +33,7 @@ Using the principle of least privilege is crucial for security when allowing a t
             "Effect": "Allow",
             "Action": [
                 "s3:HeadObject",
+                "s3:ListBucket",
                 "s3:PutObject",
                 "s3:CopyObject",
                 "s3:UploadPart",
@@ -48,7 +49,10 @@ Using the principle of least privilege is crucial for security when allowing a t
                 "s3:DeleteObject",
                 "s3:DeleteObjects"
             ],
-            "Resource": "arn:aws:s3:::my-bucket/*"
+            "Resource": [
+                "arn:aws:s3:::my-bucket",
+                "arn:aws:s3:::my-bucket/*"
+            ]
         }
     ]
 }

manifest-beta.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.4.10",
+  "version": "0.4.12",
   "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.4.10",
+  "version": "0.4.12",
   "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.4.10",
+  "version": "0.4.12",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs --watch",
@@ -24,7 +24,7 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@microsoft/microsoft-graph-types": "^2.40.0",
-    "@types/chai": "^4.3.11",
+    "@types/chai": "^4.3.14",
     "@types/chai-as-promised": "^7.1.8",
     "@types/jsdom": "^21.1.6",
     "@types/lodash": "^4.14.202",
@@ -34,14 +34,14 @@
     "@types/node": "^20.10.4",
     "@types/qrcode": "^1.5.5",
     "builtin-modules": "^3.3.0",
-    "chai": "^4.3.10",
+    "chai": "^4.4.1",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
     "dotenv": "^16.3.1",
     "esbuild": "^0.19.9",
     "esbuild-plugin-inline-worker": "^0.1.1",
     "jsdom": "^23.0.1",
-    "mocha": "^10.2.0",
+    "mocha": "^10.4.0",
     "npm-check-updates": "^16.14.12",
     "obsidian": "^1.4.11",
     "prettier": "^3.1.1",

src/baseTypes.ts

@@ -165,6 +165,8 @@ export type DecisionTypeForMixedEntity =
   | "remote_is_modified_then_pull"
   | "local_is_created_then_push"
   | "remote_is_created_then_pull"
+  | "local_is_created_too_large_then_do_nothing"
+  | "remote_is_created_too_large_then_do_nothing"
   | "local_is_deleted_thus_also_delete_remote"
   | "remote_is_deleted_thus_also_delete_local"
   | "conflict_created_then_keep_local"
@@ -179,7 +181,9 @@ export type DecisionTypeForMixedEntity =
   | "folder_existed_remote_then_also_create_local"
   | "folder_to_be_created"
   | "folder_to_skip"
-  | "folder_to_be_deleted";
+  | "folder_to_be_deleted_on_both"
+  | "folder_to_be_deleted_on_remote"
+  | "folder_to_be_deleted_on_local";
 
 /**
  * uniform representation
@@ -200,6 +204,7 @@ export interface Entity {
   sizeRaw: number;
   hash?: string;
   etag?: string;
+  synthesizedFolder?: boolean;
 }
 
 export interface UploadedType {
@@ -219,6 +224,8 @@ export interface MixedEntity {
   decisionBranch?: number;
   decision?: DecisionTypeForMixedEntity;
   conflictAction?: ConflictActionType;
+
+  sideNotes?: any;
 }
 
 /**

src/encryptUnified.ts

@@ -42,9 +42,18 @@ export class Cipher {
       return content;
     }
     if (this.method === "openssl-base64") {
-      return await openssl.encryptArrayBuffer(content, this.password);
+      const res = await openssl.encryptArrayBuffer(content, this.password);
+      if (res === undefined) {
+        throw Error(`cannot encrypt content`);
+      }
+      return res;
     } else if (this.method === "rclone-base64") {
-      return await this.cipherRClone!.encryptContentByCallingWorker(content);
+      const res =
+        await this.cipherRClone!.encryptContentByCallingWorker(content);
+      if (res === undefined) {
+        throw Error(`cannot encrypt content`);
+      }
+      return res;
     } else {
       throw Error(`not supported encrypt method=${this.method}`);
     }
@@ -56,9 +65,18 @@ export class Cipher {
       return content;
     }
     if (this.method === "openssl-base64") {
-      return await openssl.decryptArrayBuffer(content, this.password);
+      const res = await openssl.decryptArrayBuffer(content, this.password);
+      if (res === undefined) {
+        throw Error(`cannot decrypt content`);
+      }
+      return res;
     } else if (this.method === "rclone-base64") {
-      return await this.cipherRClone!.decryptContentByCallingWorker(content);
+      const res =
+        await this.cipherRClone!.decryptContentByCallingWorker(content);
+      if (res === undefined) {
+        throw Error(`cannot decrypt content`);
+      }
+      return res;
     } else {
       throw Error(`not supported decrypt method=${this.method}`);
     }
@@ -70,15 +88,23 @@ export class Cipher {
       return name;
     }
     if (this.method === "openssl-base64") {
-      return await openssl.encryptStringToBase64url(name, this.password);
+      const res = await openssl.encryptStringToBase64url(name, this.password);
+      if (res === undefined) {
+        throw Error(`cannot encrypt name=${name}`);
+      }
+      return res;
     } else if (this.method === "rclone-base64") {
-      return await this.cipherRClone!.encryptNameByCallingWorker(name);
+      const res = await this.cipherRClone!.encryptNameByCallingWorker(name);
+      if (res === undefined) {
+        throw Error(`cannot encrypt name=${name}`);
+      }
+      return res;
     } else {
       throw Error(`not supported encrypt method=${this.method}`);
     }
   }
 
-  async decryptName(name: string) {
+  async decryptName(name: string): Promise<string> {
     // console.debug("start decryptName");
     if (this.password === "") {
       return name;
@@ -88,7 +114,7 @@ export class Cipher {
         // backward compitable with the openssl-base32
         try {
           const res = await openssl.decryptBase32ToString(name, this.password);
-          if (isVaildText(res)) {
+          if (res !== undefined && isVaildText(res)) {
             return res;
           } else {
             throw Error(`cannot decrypt name=${name}`);
@@ -102,7 +128,7 @@ export class Cipher {
             name,
             this.password
           );
-          if (isVaildText(res)) {
+          if (res !== undefined && isVaildText(res)) {
             return res;
           } else {
             throw Error(`cannot decrypt name=${name}`);
@@ -110,9 +136,17 @@ export class Cipher {
         } catch (error) {
           throw Error(`cannot decrypt name=${name}`);
         }
+      } else {
+        throw Error(
+          `method=${this.method} but the name=${name}, likely mismatch`
+        );
       }
     } else if (this.method === "rclone-base64") {
-      return await this.cipherRClone!.decryptNameByCallingWorker(name);
+      const res = await this.cipherRClone!.decryptNameByCallingWorker(name);
+      if (res === undefined) {
+        throw Error(`cannot decrypt name=${name}`);
+      }
+      return res;
     } else {
       throw Error(`not supported decrypt method=${this.method}`);
     }
@@ -136,7 +170,7 @@ export class Cipher {
    * @param name
    * @returns
    */
-  static isLikelyEncryptedName(name: string): boolean {
+  static isLikelyOpenSSLEncryptedName(name: string): boolean {
     if (
       name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE32) ||
       name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE64URL)
@@ -145,4 +179,37 @@ export class Cipher {
     }
     return false;
   }
+
+  /**
+   * quick guess, no actual decryption here
+   * @param name
+   * @returns
+   */
+  static isLikelyEncryptedName(name: string): boolean {
+    return Cipher.isLikelyOpenSSLEncryptedName(name);
+  }
+
+  /**
+   * quick guess, no actual decryption here, only openssl can be guessed here
+   * @param name
+   * @returns
+   */
+  static isLikelyEncryptedNameNotMatchMethod(
+    name: string,
+    method: CipherMethodType
+  ): boolean {
+    if (
+      Cipher.isLikelyOpenSSLEncryptedName(name) &&
+      method !== "openssl-base64"
+    ) {
+      return true;
+    }
+    if (
+      !Cipher.isLikelyOpenSSLEncryptedName(name) &&
+      method === "openssl-base64"
+    ) {
+      return true;
+    }
+    return false;
+  }
 }

src/misc.ts

@@ -165,6 +165,9 @@ export const base64ToBase64url = (a: string, pad: boolean = false) => {
  * @param a
  */
 export const isVaildText = (a: string) => {
+  if (a === undefined) {
+    return false;
+  }
   // If the regex matches, the string is invalid.
   return !XRegExp("\\p{Cc}|\\p{Cf}|\\p{Co}|\\p{Cn}|\\p{Zl}|\\p{Zp}", "A").test(
     a
@@ -531,3 +534,52 @@ export const changeMobileStatusBar = (op: "enable" | "disable") => {
     bar.style.removeProperty("margin-bottom");
   }
 };
+
+/**
+ * https://github.com/remotely-save/remotely-save/issues/567
+ * https://www.dropboxforum.com/t5/Dropbox-API-Support-Feedback/Case-Sensitivity-in-API-2/td-p/191279
+ * @param entities
+ */
+export const fixEntityListCasesInplace = (entities: { keyRaw: string }[]) => {
+  entities.sort((a, b) => a.keyRaw.length - b.keyRaw.length);
+  // console.log(JSON.stringify(entities,null,2));
+
+  const caseMapping: Record<string, string> = { "": "" };
+  for (const e of entities) {
+    // console.log(`looking for: ${JSON.stringify(e, null, 2)}`);
+
+    let parentFolder = getParentFolder(e.keyRaw);
+    if (parentFolder === "/") {
+      parentFolder = "";
+    }
+    const parentFolderLower = parentFolder.toLocaleLowerCase();
+    const segs = e.keyRaw.split("/");
+    if (e.keyRaw.endsWith("/")) {
+      // folder
+      if (caseMapping.hasOwnProperty(parentFolderLower)) {
+        const newKeyRaw = `${caseMapping[parentFolderLower]}${segs
+          .slice(-2)
+          .join("/")}`;
+        caseMapping[newKeyRaw.toLocaleLowerCase()] = newKeyRaw;
+        e.keyRaw = newKeyRaw;
+        // console.log(JSON.stringify(caseMapping,null,2));
+        continue;
+      } else {
+        throw Error(`${parentFolder} doesn't have cases record??`);
+      }
+    } else {
+      // file
+      if (caseMapping.hasOwnProperty(parentFolderLower)) {
+        const newKeyRaw = `${caseMapping[parentFolderLower]}${segs
+          .slice(-1)
+          .join("/")}`;
+        e.keyRaw = newKeyRaw;
+        continue;
+      } else {
+        throw Error(`${parentFolder} doesn't have cases record??`);
+      }
+    }
+  }
+
+  return entities;
+};

src/remote.ts

@@ -239,7 +239,8 @@ export class RemoteClient {
   deleteFromRemote = async (
     fileOrFolderPath: string,
     cipher: Cipher,
-    remoteEncryptedKey: string = ""
+    remoteEncryptedKey: string = "",
+    synthesizedFolder: boolean = false
   ) => {
     if (this.serviceType === "s3") {
       return await s3.deleteFromRemote(
@@ -247,7 +248,8 @@ export class RemoteClient {
         this.s3Config!,
         fileOrFolderPath,
         cipher,
-        remoteEncryptedKey
+        remoteEncryptedKey,
+        synthesizedFolder
       );
     } else if (this.serviceType === "webdav") {
       return await webdav.deleteFromRemote(

src/remoteForDropbox.ts

@@ -12,6 +12,7 @@ import {
 } from "./baseTypes";
 import {
   bufferToArrayBuffer,
+  fixEntityListCasesInplace,
   getFolderLevels,
   hasEmojiInText,
   headersToRecord,
@@ -635,6 +636,8 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => {
     unifiedContents.push(...unifiedContents2);
   }
 
+  fixEntityListCasesInplace(unifiedContents);
+
   return unifiedContents;
 };
 

src/remoteForOnedrive.ts

@@ -471,6 +471,11 @@ export class WrappedOnedriveClient {
       const pathFrag = encodeURI(pathFragOrig);
       theUrl = `${API_PREFIX}${pathFrag}`;
     }
+    // we want to support file name with hash #
+    // because every url we construct here do not contain the # symbol
+    // thus it should be safe to directly replace the character
+    theUrl = theUrl.replace(/#/g, "%23");
+    // console.debug(`building url: [${pathFragOrig}] => [${theUrl}]`)
     return theUrl;
   };
 

src/remoteForS3.ts

@@ -250,6 +250,7 @@ const fromS3ObjectToEntity = (
     mtimeCli: mtimeCli,
     sizeRaw: x.Size!,
     etag: x.ETag,
+    synthesizedFolder: false,
   };
   return r;
 };
@@ -599,7 +600,10 @@ export const listAllFromRemote = async (
   s3Client: S3Client,
   s3Config: S3Config
 ) => {
-  return await listFromRemoteRaw(s3Client, s3Config, s3Config.remotePrefix);
+  const res = (
+    await listFromRemoteRaw(s3Client, s3Config, s3Config.remotePrefix)
+  ).filter((x) => x.keyRaw !== "" && x.keyRaw !== "/");
+  return res;
 };
 
 /**
@@ -715,11 +719,15 @@ export const deleteFromRemote = async (
   s3Config: S3Config,
   fileOrFolderPath: string,
   cipher: Cipher,
-  remoteEncryptedKey: string = ""
+  remoteEncryptedKey: string = "",
+  synthesizedFolder: boolean = false
 ) => {
   if (fileOrFolderPath === "/") {
     return;
   }
+  if (synthesizedFolder) {
+    return;
+  }
   let remoteFileName = fileOrFolderPath;
   if (!cipher.isPasswordEmpty()) {
     remoteFileName = remoteEncryptedKey;
@@ -771,7 +779,7 @@ export const checkConnectivity = async (
 ) => {
   try {
     // TODO: no universal way now, just check this in connectivity
-    if (Platform.isIosApp && !s3Config.s3Endpoint.startsWith("https")) {
+    if (Platform.isIosApp && s3Config.s3Endpoint.startsWith("http://")) {
       throw Error(
         `Your s3 endpoint could only be https, not http, because of the iOS restriction.`
       );

src/settings.ts

@@ -125,15 +125,9 @@ class PasswordModal extends Modal {
 
 class EncryptionMethodModal extends Modal {
   plugin: RemotelySavePlugin;
-  newEncryptionMethod: CipherMethodType;
-  constructor(
-    app: App,
-    plugin: RemotelySavePlugin,
-    newEncryptionMethod: CipherMethodType
-  ) {
+  constructor(app: App, plugin: RemotelySavePlugin) {
     super(app);
     this.plugin = plugin;
-    this.newEncryptionMethod = newEncryptionMethod;
   }
 
   onOpen() {
@@ -153,21 +147,12 @@ class EncryptionMethodModal extends Modal {
         });
       });
 
-    new Setting(contentEl)
-      .addButton((button) => {
+    new Setting(contentEl).addButton((button) => {
       button.setButtonText(t("confirm"));
       button.onClick(async () => {
-          this.plugin.settings.encryptionMethod = this.newEncryptionMethod;
-          await this.plugin.saveSettings();
         this.close();
       });
       button.setClass("encryptionmethod-second-confirm");
-      })
-      .addButton((button) => {
-        button.setButtonText(t("goback"));
-        button.onClick(() => {
-          this.close();
-        });
     });
   }
 
@@ -1693,24 +1678,15 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
       .setName(t("settings_encryptionmethod"))
       .setDesc(stringToFragment(t("settings_encryptionmethod_desc")))
       .addDropdown((dropdown) => {
-        dropdown.addOption(
-          "rclone-base64",
-          t("settings_encryptionmethod_rclone")
-        );
-        dropdown.addOption(
-          "openssl-base64",
-          t("settings_encryptionmethod_openssl")
-        );
-        dropdown.onChange(async (val: string) => {
-          if (this.plugin.settings.password === "") {
+        dropdown
+          .addOption("rclone-base64", t("settings_encryptionmethod_rclone"))
+          .addOption("openssl-base64", t("settings_encryptionmethod_openssl"))
+          .setValue(this.plugin.settings.encryptionMethod ?? "rclone-base64")
+          .onChange(async (val: string) => {
             this.plugin.settings.encryptionMethod = val as CipherMethodType;
             await this.plugin.saveSettings();
-          } else {
-            new EncryptionMethodModal(
-              this.app,
-              this.plugin,
-              val as CipherMethodType
-            ).open();
+            if (this.plugin.settings.password !== "") {
+              new EncryptionMethodModal(this.app, this.plugin).open();
             }
           });
       });

src/sync.ts

@@ -18,6 +18,7 @@ import {
   isVaildText,
   atWhichLevel,
   mkdirpInVault,
+  getFolderLevels,
 } from "./misc";
 import {
   DEFAULT_FILE_NAME_FOR_METADATAONREMOTE,
@@ -53,8 +54,9 @@ export interface PasswordCheckType {
     | "unknown_encryption_method"
     | "remote_encrypted_local_no_password"
     | "password_matched"
-    | "password_not_matched_or_remote_not_encrypted"
-    | "likely_no_password_both_sides";
+    | "password_or_method_not_matched_or_remote_not_encrypted"
+    | "likely_no_password_both_sides"
+    | "encryption_method_not_matched";
 }
 
 export const isPasswordOk = async (
@@ -91,8 +93,19 @@ export const isPasswordOk = async (
         reason: "unknown_encryption_method",
       };
     }
+    if (
+      Cipher.isLikelyEncryptedNameNotMatchMethod(santyCheckKey, cipher.method)
+    ) {
+      return {
+        ok: false,
+        reason: "encryption_method_not_matched",
+      };
+    }
     try {
-      await cipher.decryptName(santyCheckKey);
+      const k = await cipher.decryptName(santyCheckKey);
+      if (k === undefined) {
+        throw Error(`decryption failed`);
+      }
       return {
         ok: true,
         reason: "password_matched",
@@ -100,7 +113,7 @@ export const isPasswordOk = async (
     } catch (error) {
       return {
         ok: false,
-        reason: "password_not_matched_or_remote_not_encrypted",
+        reason: "password_or_method_not_matched_or_remote_not_encrypted",
       };
     }
   }
@@ -314,7 +327,10 @@ export const ensembleMixedEnties = async (
 ): Promise<SyncPlanType> => {
   const finalMappings: SyncPlanType = {};
 
+  const synthFolders: Record<string, Entity> = {};
+
   // remote has to be first
+  // we also have to synthesize folders here
   for (const remote of remoteEntityList) {
     const remoteCopied = ensureMTimeOfRemoteEntityValid(
       await decryptRemoteEntityInplace(
@@ -340,6 +356,42 @@ export const ensembleMixedEnties = async (
       key: key,
       remote: remoteCopied,
     };
+
+    for (const f of getFolderLevels(key, true)) {
+      if (finalMappings.hasOwnProperty(f)) {
+        delete synthFolders[f];
+        continue;
+      }
+      if (
+        !synthFolders.hasOwnProperty(f) ||
+        remoteCopied.mtimeSvr! >= synthFolders[f].mtimeSvr!
+      ) {
+        synthFolders[f] = {
+          key: f,
+          keyRaw: `<synth: ${f}>`,
+          keyEnc: `<enc synth: ${f}>`,
+          size: 0,
+          sizeRaw: 0,
+          sizeEnc: 0,
+          mtimeSvr: remoteCopied.mtimeSvr,
+          mtimeSvrFmt: remoteCopied.mtimeSvrFmt,
+          mtimeCli: remoteCopied.mtimeCli,
+          mtimeCliFmt: remoteCopied.mtimeCliFmt,
+          synthesizedFolder: true,
+        };
+      }
+    }
+  }
+
+  console.debug(`synthFolders:`);
+  console.debug(synthFolders);
+
+  // special: add synth folders
+  for (const key of Object.keys(synthFolders)) {
+    finalMappings[key] = {
+      key: key,
+      remote: synthFolders[key],
+    };
   }
 
   if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) {
@@ -494,9 +546,41 @@ export const getSyncPlanInplace = async (
           mixedEntry.decisionBranch = 105;
           mixedEntry.decision = "folder_to_skip";
         } else if (howToCleanEmptyFolder === "clean_both") {
+          if (local !== undefined && remote !== undefined) {
+            if (syncDirection === "bidirectional") {
               mixedEntry.decisionBranch = 106;
-          mixedEntry.decision = "folder_to_be_deleted";
-          // TODO: what to do in different sync direction?
+              mixedEntry.decision = "folder_to_be_deleted_on_both";
+            } else {
+              // right now it does nothing because of "incremental"
+              // TODO: should we delete??
+              mixedEntry.decisionBranch = 109;
+              mixedEntry.decision = "folder_to_skip";
+            }
+          } else if (local !== undefined && remote === undefined) {
+            if (syncDirection === "bidirectional") {
+              mixedEntry.decisionBranch = 110;
+              mixedEntry.decision = "folder_to_be_deleted_on_local";
+            } else {
+              // right now it does nothing because of "incremental"
+              // TODO: should we delete??
+              mixedEntry.decisionBranch = 111;
+              mixedEntry.decision = "folder_to_skip";
+            }
+          } else if (local === undefined && remote !== undefined) {
+            if (syncDirection === "bidirectional") {
+              mixedEntry.decisionBranch = 112;
+              mixedEntry.decision = "folder_to_be_deleted_on_remote";
+            } else {
+              // right now it does nothing because of "incremental"
+              // TODO: should we delete??
+              mixedEntry.decisionBranch = 113;
+              mixedEntry.decision = "folder_to_skip";
+            }
+          } else {
+            // no folder to delete, do nothing
+            mixedEntry.decisionBranch = 114;
+            mixedEntry.decision = "folder_to_skip";
+          }
         } else {
           throw Error(
             `do not know how to deal with empty folder ${mixedEntry.key}`
@@ -694,11 +778,9 @@ export const getSyncPlanInplace = async (
               keptFolder.add(getParentFolder(key));
             }
           } else {
-            throw Error(
-              `remote is created (branch 3) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify(
-                mixedEntry
-              )}`
-            );
+            mixedEntry.decisionBranch = 36;
+            mixedEntry.decision = "remote_is_created_too_large_then_do_nothing";
+            keptFolder.add(getParentFolder(key));
           }
         } else if (
           (prevSync.mtimeSvr === remote.mtimeCli ||
@@ -757,11 +839,9 @@ export const getSyncPlanInplace = async (
               keptFolder.add(getParentFolder(key));
             }
           } else {
-            throw Error(
-              `local is created (branch 6) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify(
-                mixedEntry
-              )}`
-            );
+            mixedEntry.decisionBranch = 37;
+            mixedEntry.decision = "local_is_created_too_large_then_do_nothing";
+            keptFolder.add(getParentFolder(key));
           }
         } else if (
           (prevSync.mtimeSvr === local.mtimeCli ||
@@ -823,13 +903,27 @@ export const getSyncPlanInplace = async (
     throw Error(`unexpectedly keptFolder no decisions: ${[...keptFolder]}`);
   }
 
+  // finally we want to make our life easier
+  const currTime = Date.now();
+  const currTimeFmt = unixTimeToStr(currTime);
+  // because the path should not as / in the beginning,
+  // we should be safe to add these keys:
+  mixedEntityMappings["/$@meta"] = {
+    key: "/$@meta", // don't mess up with the types
+    sideNotes: {
+      generateTime: currTime,
+      generateTimeFmt: currTimeFmt,
+    },
+  };
+
   return mixedEntityMappings;
 };
 
-const splitThreeStepsOnEntityMappings = (
+const splitFourStepsOnEntityMappings = (
   mixedEntityMappings: Record<string, MixedEntity>
 ) => {
   type StepArrayType = MixedEntity[] | undefined | null;
+  const onlyMarkSyncedOps: StepArrayType[] = [];
   const folderCreationOps: StepArrayType[] = [];
   const deletionOps: StepArrayType[] = [];
   const uploadDownloads: StepArrayType[] = [];
@@ -845,6 +939,11 @@ const splitThreeStepsOnEntityMappings = (
 
   for (let i = 0; i < sortedKeys.length; ++i) {
     const key = sortedKeys[i];
+
+    if (key === "/$@meta") {
+      continue; // special
+    }
+
     const val = mixedEntityMappings[key];
 
     if (!key.endsWith("/")) {
@@ -852,12 +951,27 @@ const splitThreeStepsOnEntityMappings = (
     }
 
     if (
-      val.decision === "equal" ||
-      val.decision === "conflict_created_then_do_nothing" ||
-      val.decision === "folder_existed_both_then_do_nothing" ||
+      val.decision === "local_is_created_too_large_then_do_nothing" ||
+      val.decision === "remote_is_created_too_large_then_do_nothing" ||
       val.decision === "folder_to_skip"
     ) {
       // pass
+    } else if (
+      val.decision === "equal" ||
+      val.decision === "conflict_created_then_do_nothing" ||
+      val.decision === "folder_existed_both_then_do_nothing"
+    ) {
+      if (
+        onlyMarkSyncedOps.length === 0 ||
+        onlyMarkSyncedOps[0] === undefined ||
+        onlyMarkSyncedOps[0] === null
+      ) {
+        onlyMarkSyncedOps[0] = [val];
+      } else {
+        onlyMarkSyncedOps[0].push(val); // only one level is needed here
+      }
+
+      // don't need to update realTotalCount here
     } else if (
       val.decision === "folder_existed_local_then_also_create_remote" ||
       val.decision === "folder_existed_remote_then_also_create_local" ||
@@ -877,7 +991,9 @@ const splitThreeStepsOnEntityMappings = (
       val.decision === "only_history" ||
       val.decision === "local_is_deleted_thus_also_delete_remote" ||
       val.decision === "remote_is_deleted_thus_also_delete_local" ||
-      val.decision === "folder_to_be_deleted"
+      val.decision === "folder_to_be_deleted_on_both" ||
+      val.decision === "folder_to_be_deleted_on_local" ||
+      val.decision === "folder_to_be_deleted_on_remote"
     ) {
       const level = atWhichLevel(key);
       const k = deletionOps[level - 1];
@@ -888,7 +1004,11 @@ const splitThreeStepsOnEntityMappings = (
       }
       realTotalCount += 1;
 
-      if (val.decision.startsWith("deleted")) {
+      if (
+        val.decision.includes("deleted") &&
+        !val.decision.includes("folder")
+      ) {
+        // only count files here, skip folder
         realModifyDeleteCount += 1;
       }
     } else if (
@@ -915,8 +1035,8 @@ const splitThreeStepsOnEntityMappings = (
       realTotalCount += 1;
 
       if (
-        val.decision.startsWith("modified") ||
-        val.decision.startsWith("conflict")
+        val.decision.includes("modified") ||
+        val.decision.includes("conflict")
       ) {
         realModifyDeleteCount += 1;
       }
@@ -931,6 +1051,7 @@ const splitThreeStepsOnEntityMappings = (
   deletionOps.reverse(); // inplace reverse
 
   return {
+    onlyMarkSyncedOps: onlyMarkSyncedOps,
     folderCreationOps: folderCreationOps,
     deletionOps: deletionOps,
     uploadDownloads: uploadDownloads,
@@ -960,13 +1081,37 @@ const dispatchOperationToActualV3 = async (
   // );
   if (r.decision === "only_history") {
     clearPrevSyncRecordByVaultAndProfile(db, vaultRandomID, profileID, key);
+  } else if (
+    r.decision === "local_is_created_too_large_then_do_nothing" ||
+    r.decision === "remote_is_created_too_large_then_do_nothing" ||
+    r.decision === "folder_to_skip"
+  ) {
+    // !! no actual sync being kept happens,
+    // so no sync record here
+    // pass
   } else if (
     r.decision === "equal" ||
     r.decision === "conflict_created_then_do_nothing" ||
-    r.decision === "folder_to_skip" ||
     r.decision === "folder_existed_both_then_do_nothing"
   ) {
-    // pass
+    // !! we need to upsert the record,
+    // so that next time we can determine the change delta
+    const entity = r.remote ?? r.local;
+    console.debug(
+      `we are in actual operation of equal, entity=${JSON.stringify(
+        entity,
+        null,
+        2
+      )}`
+    );
+    if (entity !== undefined) {
+      await upsertPrevSyncRecordByVaultAndProfile(
+        db,
+        vaultRandomID,
+        profileID,
+        entity
+      );
+    }
   } else if (
     r.decision === "local_is_modified_then_push" ||
     r.decision === "local_is_created_then_push" ||
@@ -1024,7 +1169,12 @@ const dispatchOperationToActualV3 = async (
     );
   } else if (r.decision === "local_is_deleted_thus_also_delete_remote") {
     // local is deleted, we need to delete remote now
-    await client.deleteFromRemote(r.key, cipher, r.remote!.keyEnc);
+    await client.deleteFromRemote(
+      r.key,
+      cipher,
+      r.remote!.keyEnc,
+      r.remote!.synthesizedFolder
+    );
     await clearPrevSyncRecordByVaultAndProfile(
       db,
       vaultRandomID,
@@ -1063,9 +1213,28 @@ const dispatchOperationToActualV3 = async (
       profileID,
       entity
     );
-  } else if (r.decision === "folder_to_be_deleted") {
+  } else if (
+    r.decision === "folder_to_be_deleted_on_both" ||
+    r.decision === "folder_to_be_deleted_on_local" ||
+    r.decision === "folder_to_be_deleted_on_remote"
+  ) {
+    if (
+      r.decision === "folder_to_be_deleted_on_both" ||
+      r.decision === "folder_to_be_deleted_on_local"
+    ) {
       await localDeleteFunc(r.key);
-    await client.deleteFromRemote(r.key, cipher, r.remote!.keyEnc);
+    }
+    if (
+      r.decision === "folder_to_be_deleted_on_both" ||
+      r.decision === "folder_to_be_deleted_on_remote"
+    ) {
+      await client.deleteFromRemote(
+        r.key,
+        cipher,
+        r.remote!.keyEnc,
+        r.remote!.synthesizedFolder
+      );
+    }
     await clearPrevSyncRecordByVaultAndProfile(
       db,
       vaultRandomID,
@@ -1093,13 +1262,15 @@ export const doActualSync = async (
 ) => {
   console.debug(`concurrency === ${concurrency}`);
   const {
+    onlyMarkSyncedOps,
     folderCreationOps,
     deletionOps,
     uploadDownloads,
     allFilesCount,
     realModifyDeleteCount,
     realTotalCount,
-  } = splitThreeStepsOnEntityMappings(mixedEntityMappings);
+  } = splitFourStepsOnEntityMappings(mixedEntityMappings);
+  // console.debug(`onlyMarkSyncedOps: ${JSON.stringify(onlyMarkSyncedOps)}`);
   // console.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`);
   // console.debug(`deletionOps: ${JSON.stringify(deletionOps)}`);
   // console.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`);
@@ -1115,6 +1286,12 @@ export const doActualSync = async (
     allFilesCount > 0
   ) {
     if (
+      protectModifyPercentage === 100 &&
+      realModifyDeleteCount === allFilesCount
+    ) {
+      // special treatment for 100%
+      // let it pass, we do nothing here
+    } else if (
       realModifyDeleteCount * 100 >=
       allFilesCount * protectModifyPercentage
     ) {
@@ -1128,11 +1305,17 @@ export const doActualSync = async (
     }
   }
 
-  const nested = [folderCreationOps, deletionOps, uploadDownloads];
+  const nested = [
+    onlyMarkSyncedOps,
+    folderCreationOps,
+    deletionOps,
+    uploadDownloads,
+  ];
   const logTexts = [
-    `1. create all folders from shadowest to deepest`,
-    `2. delete files and folders from deepest to shadowest`,
-    `3. upload or download files in parallel, with the desired concurrency=${concurrency}`,
+    `1. record the items already being synced`,
+    `2. create all folders from shadowest to deepest`,
+    `3. delete files and folders from deepest to shadowest`,
+    `4. upload or download files in parallel, with the desired concurrency=${concurrency}`,
   ];
 
   let realCounter = 0;

tests/misc.test.ts

@@ -285,3 +285,56 @@ describe("Misc: special char for dir", () => {
     expect(misc.checkHasSpecialCharForDir("xxx?yyy")).to.be.true;
   });
 });
+
+describe("Misc: Dropbox: should fix the folder name cases", () => {
+  it("should do nothing on empty folders", () => {
+    const input: any[] = [];
+    expect(misc.fixEntityListCasesInplace(input)).to.be.empty;
+  });
+
+  it("should sort folders by length by side effect", () => {
+    const input = [
+      { keyRaw: "aaaa/" },
+      { keyRaw: "bbb/" },
+      { keyRaw: "c/" },
+      { keyRaw: "dd/" },
+    ];
+
+    const output = [
+      { keyRaw: "c/" },
+      { keyRaw: "dd/" },
+      { keyRaw: "bbb/" },
+      { keyRaw: "aaaa/" },
+    ];
+    expect(misc.fixEntityListCasesInplace(input)).to.deep.equal(output);
+  });
+
+  it("should fix folder names", () => {
+    const input = [
+      { keyRaw: "AAA/" },
+      { keyRaw: "aaa/bbb/CCC.md" },
+      { keyRaw: "aaa/BBB/" },
+
+      { keyRaw: "ddd/" },
+      { keyRaw: "DDD/EEE/fff.md" },
+      { keyRaw: "DDD/eee/" },
+
+      { keyRaw: "Ggg/" },
+      { keyRaw: "ggG/hHH你好/Fff世界.md" },
+      { keyRaw: "ggG/Hhh你好/" },
+    ];
+
+    const output = [
+      { keyRaw: "AAA/" },
+      { keyRaw: "ddd/" },
+      { keyRaw: "Ggg/" },
+      { keyRaw: "AAA/BBB/" },
+      { keyRaw: "ddd/eee/" },
+      { keyRaw: "Ggg/Hhh你好/" },
+      { keyRaw: "AAA/BBB/CCC.md" },
+      { keyRaw: "ddd/eee/fff.md" },
+      { keyRaw: "Ggg/Hhh你好/Fff世界.md" },
+    ];
+    expect(misc.fixEntityListCasesInplace(input)).to.deep.equal(output);
+  });
+});