new delay function to avoid the import problem

* Improve sync_on_save

* fix typo

---------

Co-authored-by: fyears <1142836+fyears@users.noreply.github.com>

docs/remote_services/s3_general/s3_user_policy.md

@@ -33,6 +33,7 @@ Using the principle of least privilege is crucial for security when allowing a t
             "Effect": "Allow",
             "Action": [
                 "s3:HeadObject",
+                "s3:ListBucket",
                 "s3:PutObject",
                 "s3:CopyObject",
                 "s3:UploadPart",
@@ -48,7 +49,10 @@ Using the principle of least privilege is crucial for security when allowing a t
                 "s3:DeleteObject",
                 "s3:DeleteObjects"
             ],
-            "Resource": "arn:aws:s3:::my-bucket/*"
+            "Resource": [
+                "arn:aws:s3:::my-bucket",
+                "arn:aws:s3:::my-bucket/*"
+            ]
         }
     ]
 }

manifest-beta.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.4.10",
+  "version": "0.4.13",
   "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.4.10",
+  "version": "0.4.13",
   "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.4.10",
+  "version": "0.4.13",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs --watch",
@@ -24,7 +24,7 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@microsoft/microsoft-graph-types": "^2.40.0",
-    "@types/chai": "^4.3.11",
+    "@types/chai": "^4.3.14",
     "@types/chai-as-promised": "^7.1.8",
     "@types/jsdom": "^21.1.6",
     "@types/lodash": "^4.14.202",
@@ -34,14 +34,14 @@
     "@types/node": "^20.10.4",
     "@types/qrcode": "^1.5.5",
     "builtin-modules": "^3.3.0",
-    "chai": "^4.3.10",
+    "chai": "^4.4.1",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
     "dotenv": "^16.3.1",
     "esbuild": "^0.19.9",
     "esbuild-plugin-inline-worker": "^0.1.1",
     "jsdom": "^23.0.1",
-    "mocha": "^10.2.0",
+    "mocha": "^10.4.0",
     "npm-check-updates": "^16.14.12",
     "obsidian": "^1.4.11",
     "prettier": "^3.1.1",

src/baseTypes.ts

@@ -165,6 +165,8 @@ export type DecisionTypeForMixedEntity =
   | "remote_is_modified_then_pull"
   | "local_is_created_then_push"
   | "remote_is_created_then_pull"
+  | "local_is_created_too_large_then_do_nothing"
+  | "remote_is_created_too_large_then_do_nothing"
   | "local_is_deleted_thus_also_delete_remote"
   | "remote_is_deleted_thus_also_delete_local"
   | "conflict_created_then_keep_local"
@@ -179,7 +181,9 @@ export type DecisionTypeForMixedEntity =
   | "folder_existed_remote_then_also_create_local"
   | "folder_to_be_created"
   | "folder_to_skip"
-  | "folder_to_be_deleted";
+  | "folder_to_be_deleted_on_both"
+  | "folder_to_be_deleted_on_remote"
+  | "folder_to_be_deleted_on_local";
 
 /**
  * uniform representation
@@ -200,6 +204,7 @@ export interface Entity {
   sizeRaw: number;
   hash?: string;
   etag?: string;
+  synthesizedFolder?: boolean;
 }
 
 export interface UploadedType {
@@ -219,6 +224,8 @@ export interface MixedEntity {
   decisionBranch?: number;
   decision?: DecisionTypeForMixedEntity;
   conflictAction?: ConflictActionType;
+
+  sideNotes?: any;
 }
 
 /**
@@ -265,9 +272,9 @@ export const DEFAULT_LOG_HISTORY_FILE_PREFIX = "log_hist_exported_on_";
 
 export type SyncTriggerSourceType =
   | "manual"
-  | "auto"
   | "dry"
-  | "autoOnceInit"
+  | "auto"
+  | "auto_once_init"
   | "auto_sync_on_save";
 
 export const REMOTELY_SAVE_VERSION_2022 = "0.3.25";

src/encryptUnified.ts

@@ -42,9 +42,18 @@ export class Cipher {
       return content;
     }
     if (this.method === "openssl-base64") {
-      return await openssl.encryptArrayBuffer(content, this.password);
+      const res = await openssl.encryptArrayBuffer(content, this.password);
+      if (res === undefined) {
+        throw Error(`cannot encrypt content`);
+      }
+      return res;
     } else if (this.method === "rclone-base64") {
-      return await this.cipherRClone!.encryptContentByCallingWorker(content);
+      const res =
+        await this.cipherRClone!.encryptContentByCallingWorker(content);
+      if (res === undefined) {
+        throw Error(`cannot encrypt content`);
+      }
+      return res;
     } else {
       throw Error(`not supported encrypt method=${this.method}`);
     }
@@ -56,9 +65,18 @@ export class Cipher {
       return content;
     }
     if (this.method === "openssl-base64") {
-      return await openssl.decryptArrayBuffer(content, this.password);
+      const res = await openssl.decryptArrayBuffer(content, this.password);
+      if (res === undefined) {
+        throw Error(`cannot decrypt content`);
+      }
+      return res;
     } else if (this.method === "rclone-base64") {
-      return await this.cipherRClone!.decryptContentByCallingWorker(content);
+      const res =
+        await this.cipherRClone!.decryptContentByCallingWorker(content);
+      if (res === undefined) {
+        throw Error(`cannot decrypt content`);
+      }
+      return res;
     } else {
       throw Error(`not supported decrypt method=${this.method}`);
     }
@@ -70,15 +88,23 @@ export class Cipher {
       return name;
     }
     if (this.method === "openssl-base64") {
-      return await openssl.encryptStringToBase64url(name, this.password);
+      const res = await openssl.encryptStringToBase64url(name, this.password);
+      if (res === undefined) {
+        throw Error(`cannot encrypt name=${name}`);
+      }
+      return res;
     } else if (this.method === "rclone-base64") {
-      return await this.cipherRClone!.encryptNameByCallingWorker(name);
+      const res = await this.cipherRClone!.encryptNameByCallingWorker(name);
+      if (res === undefined) {
+        throw Error(`cannot encrypt name=${name}`);
+      }
+      return res;
     } else {
       throw Error(`not supported encrypt method=${this.method}`);
     }
   }
 
-  async decryptName(name: string) {
+  async decryptName(name: string): Promise<string> {
     // console.debug("start decryptName");
     if (this.password === "") {
       return name;
@@ -88,7 +114,7 @@ export class Cipher {
         // backward compitable with the openssl-base32
         try {
           const res = await openssl.decryptBase32ToString(name, this.password);
-          if (isVaildText(res)) {
+          if (res !== undefined && isVaildText(res)) {
             return res;
           } else {
             throw Error(`cannot decrypt name=${name}`);
@@ -102,7 +128,7 @@ export class Cipher {
             name,
             this.password
           );
-          if (isVaildText(res)) {
+          if (res !== undefined && isVaildText(res)) {
             return res;
           } else {
             throw Error(`cannot decrypt name=${name}`);
@@ -110,9 +136,17 @@ export class Cipher {
         } catch (error) {
           throw Error(`cannot decrypt name=${name}`);
         }
+      } else {
+        throw Error(
+          `method=${this.method} but the name=${name}, likely mismatch`
+        );
       }
     } else if (this.method === "rclone-base64") {
-      return await this.cipherRClone!.decryptNameByCallingWorker(name);
+      const res = await this.cipherRClone!.decryptNameByCallingWorker(name);
+      if (res === undefined) {
+        throw Error(`cannot decrypt name=${name}`);
+      }
+      return res;
     } else {
       throw Error(`not supported decrypt method=${this.method}`);
     }
@@ -136,7 +170,7 @@ export class Cipher {
    * @param name
    * @returns
    */
-  static isLikelyEncryptedName(name: string): boolean {
+  static isLikelyOpenSSLEncryptedName(name: string): boolean {
     if (
       name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE32) ||
       name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE64URL)
@@ -145,4 +179,37 @@ export class Cipher {
     }
     return false;
   }
+
+  /**
+   * quick guess, no actual decryption here
+   * @param name
+   * @returns
+   */
+  static isLikelyEncryptedName(name: string): boolean {
+    return Cipher.isLikelyOpenSSLEncryptedName(name);
+  }
+
+  /**
+   * quick guess, no actual decryption here, only openssl can be guessed here
+   * @param name
+   * @returns
+   */
+  static isLikelyEncryptedNameNotMatchMethod(
+    name: string,
+    method: CipherMethodType
+  ): boolean {
+    if (
+      Cipher.isLikelyOpenSSLEncryptedName(name) &&
+      method !== "openssl-base64"
+    ) {
+      return true;
+    }
+    if (
+      !Cipher.isLikelyOpenSSLEncryptedName(name) &&
+      method === "openssl-base64"
+    ) {
+      return true;
+    }
+    return false;
+  }
 }

src/langs/en.json

@@ -5,7 +5,7 @@
   "goback": "Go Back",
   "submit": "Submit",
   "sometext": "Here are some texts.",
-  "syncrun_alreadyrunning": "{{pluginName}} already running in stage {{syncStatus}}!",
+  "syncrun_alreadyrunning": "New command {{newTriggerSource}} stops because {{pluginName}} is already running in stage {{syncStatus}}!",
   "syncrun_syncingribbon": "{{pluginName}}: syncing from {{triggerSource}}",
   "syncrun_step0": "0/8 Remotely Save is running in dry mode, thus not actual file changes would happen.",
   "syncrun_step1": "1/8 Remotely Save is preparing ({{serviceType}})",

src/langs/zh_cn.json

@@ -5,7 +5,7 @@
   "goback": "返回",
   "submit": "提交",
   "sometext": "这里有一段文字。",
-  "syncrun_alreadyrunning": "{{pluginName}} 正处于此阶段：{{syncStatus}}!",
+  "syncrun_alreadyrunning": "{{pluginName}} 正处于此阶段：{{syncStatus}}！中断触发 {{newTriggerSource}}。",
   "syncrun_syncingribbon": "{{pluginName}}：正在由 {{triggerSource}} 触发运行",
   "syncrun_step0": "0/8 Remotely Save 在空跑（dry run）模式，不会发生实际的文件交换。",
   "syncrun_step1": "1/8 Remotely Save 准备同步（{{serviceType}}）",

src/langs/zh_tw.json

@@ -5,7 +5,7 @@
   "goback": "返回",
   "submit": "提交",
   "sometext": "這裡有一段文字。",
-  "syncrun_alreadyrunning": "{{pluginName}} 正處於此階段：{{syncStatus}}!",
+  "syncrun_alreadyrunning": "{{pluginName}} 正處於此階段：{{syncStatus}}! 中斷觸發 {{newTriggerSource}}。",
   "syncrun_syncingribbon": "{{pluginName}}：正在由 {{triggerSource}} 觸發執行",
   "syncrun_step0": "0/8 Remotely Save 在空跑（dry run）模式，不會發生實際的檔案交換。",
   "syncrun_step1": "1/8 Remotely Save 準備同步（{{serviceType}}）",

src/main.ts

@@ -9,6 +9,7 @@ import {
   Platform,
   requestUrl,
   requireApiVersion,
+  Events,
 } from "obsidian";
 import cloneDeep from "lodash/cloneDeep";
 import { createElement, RotateCcw, RefreshCcw, FileText } from "lucide";
@@ -149,6 +150,8 @@ export default class RemotelySavePlugin extends Plugin {
   i18n!: I18n;
   vaultRandomID!: string;
   debugServerTemp?: string;
+  syncEvent?: Events;
+  appContainerObserver?: MutationObserver;
 
   async syncRun(triggerSource: SyncTriggerSourceType = "manual") {
     const t = (x: TransItemType, vars?: any) => {
@@ -165,15 +168,17 @@ export default class RemotelySavePlugin extends Plugin {
       }
     };
     if (this.syncStatus !== "idle") {
-      // here the notice is shown regardless of triggerSource
+      // really, users don't want to see this in auto mode
-      new Notice(
+      // so we use getNotice to avoid unnecessary show up
+      getNotice(
         t("syncrun_alreadyrunning", {
           pluginName: this.manifest.name,
           syncStatus: this.syncStatus,
+          newTriggerSource: triggerSource,
         })
       );
       if (this.currSyncMsg !== undefined && this.currSyncMsg !== "") {
-        new Notice(this.currSyncMsg);
+        getNotice(this.currSyncMsg);
       }
       return;
     }
@@ -376,7 +381,8 @@ export default class RemotelySavePlugin extends Plugin {
               realCounter,
               realTotalCount,
               pathName,
-              decision
+              decision,
+              triggerSource
             ),
           this.db
         );
@@ -416,6 +422,7 @@ export default class RemotelySavePlugin extends Plugin {
         this.updateLastSuccessSyncMsg(lastSuccessSyncMillis);
       }
 
+      this.syncEvent?.trigger("SYNC_DONE");
       console.info(
         `${
           this.manifest.id
@@ -465,6 +472,8 @@ export default class RemotelySavePlugin extends Plugin {
 
     this.currSyncMsg = "";
 
+    this.syncEvent = new Events();
+
     await this.loadSettings();
 
     // MUST after loadSettings and before prepareDB
@@ -812,6 +821,10 @@ export default class RemotelySavePlugin extends Plugin {
   async onunload() {
     console.info(`unloading plugin ${this.manifest.id}`);
     this.syncRibbon = undefined;
+    if (this.appContainerObserver !== undefined) {
+      this.appContainerObserver.disconnect();
+      this.appContainerObserver = undefined;
+    }
     if (this.oauth2Info !== undefined) {
       this.oauth2Info.helperModal = undefined;
       this.oauth2Info = {
@@ -1100,7 +1113,7 @@ export default class RemotelySavePlugin extends Plugin {
     ) {
       this.app.workspace.onLayoutReady(() => {
         window.setTimeout(() => {
-          this.syncRun("autoOnceInit");
+          this.syncRun("auto_once_init");
         }, this.settings.initRunAfterMilliseconds);
       });
     }
@@ -1126,52 +1139,64 @@ export default class RemotelySavePlugin extends Plugin {
         }, scheduleTimeFromNow);
       };
 
-      this.app.workspace.onLayoutReady(() => {
+      const checkCurrFileModified = async (caller: "SYNC" | "FILE_CHANGES") => {
-        const intervalID = window.setInterval(() => {
         const currentFile = this.app.workspace.getActiveFile();
 
         if (currentFile) {
           // get the last modified time of the current file
-            // if it has been modified within the last syncOnSaveAfterMilliseconds
+          // if it has modified after lastSuccessSync
           // then schedule a run for syncOnSaveAfterMilliseconds after it was modified
           const lastModified = currentFile.stat.mtime;
-            const currentTime = Date.now();
+          const lastSuccessSyncMillis = await getLastSuccessSyncTimeByVault(
+            this.db,
+            this.vaultRandomID
+          );
           if (
-              currentTime - lastModified <
+            this.syncStatus === "idle" &&
-              this.settings!.syncOnSaveAfterMilliseconds!
+            lastModified > lastSuccessSyncMillis &&
+            !runScheduled
           ) {
-              if (
+            scheduleSyncOnSave(this.settings!.syncOnSaveAfterMilliseconds!);
-                !needToRunAgain &&
-                !runScheduled &&
-                this.syncStatus === "idle"
-              ) {
-                const scheduleTimeFromNow =
-                  this.settings!.syncOnSaveAfterMilliseconds! -
-                  (currentTime - lastModified);
-                scheduleSyncOnSave(scheduleTimeFromNow);
           } else if (
+            this.syncStatus === "idle" &&
             needToRunAgain &&
-                !runScheduled &&
+            !runScheduled
-                this.syncStatus === "idle"
           ) {
             scheduleSyncOnSave(this.settings!.syncOnSaveAfterMilliseconds!);
             needToRunAgain = false;
           } else {
+            if (caller === "FILE_CHANGES") {
               needToRunAgain = true;
             }
           }
         }
-        }, this.settings.syncOnSaveAfterMilliseconds);
+      };
-        this.syncOnSaveIntervalID = intervalID;
+
-        this.registerInterval(intervalID);
+      this.app.workspace.onLayoutReady(() => {
+        // listen to sync done
+        this.registerEvent(
+          this.syncEvent?.on("SYNC_DONE", () => {
+            checkCurrFileModified("SYNC");
+          })!
+        );
+
+        // listen to current file save changes
+        this.registerEvent(
+          this.app.vault.on("modify", (x) => {
+            // console.debug(`event=modify! file=${x}`);
+            checkCurrFileModified("FILE_CHANGES");
+          })
+        );
       });
     }
   }
 
   enableMobileStatusBarIfSet() {
+    this.app.workspace.onLayoutReady(() => {
       if (Platform.isMobile && this.settings.enableMobileStatusBar) {
-      changeMobileStatusBar("enable");
+        this.appContainerObserver = changeMobileStatusBar("enable");
       }
+    });
   }
 
   async saveAgreeToUseNewSyncAlgorithm() {
@@ -1183,9 +1208,10 @@ export default class RemotelySavePlugin extends Plugin {
     i: number,
     totalCount: number,
     pathName: string,
-    decision: string
+    decision: string,
+    triggerSource: SyncTriggerSourceType
   ) {
-    const msg = `syncing progress=${i}/${totalCount},decision=${decision},path=${pathName}`;
+    const msg = `syncing progress=${i}/${totalCount},decision=${decision},path=${pathName},source=${triggerSource}`;
     this.currSyncMsg = msg;
   }
 

src/misc.ts

@@ -165,6 +165,9 @@ export const base64ToBase64url = (a: string, pad: boolean = false) => {
  * @param a
  */
 export const isVaildText = (a: string) => {
+  if (a === undefined) {
+    return false;
+  }
   // If the regex matches, the string is invalid.
   return !XRegExp("\\p{Cc}|\\p{Cf}|\\p{Co}|\\p{Cn}|\\p{Zl}|\\p{Zp}", "A").test(
     a
@@ -514,20 +517,129 @@ export const stringToFragment = (string: string) => {
  * https://forum.obsidian.md/t/css-to-show-status-bar-on-mobile-devices/77185
  * @param op
  */
-export const changeMobileStatusBar = (op: "enable" | "disable") => {
+export const changeMobileStatusBar = (
-  const bar = document.querySelector(
+  op: "enable" | "disable",
+  oldAppContainerObserver?: MutationObserver
+) => {
+  const appContainer = document.getElementsByClassName("app-container")[0] as
+    | HTMLElement
+    | undefined;
+
+  const statusbar = document.querySelector(
     ".is-mobile .app-container .status-bar"
-  ) as HTMLElement;
+  ) as HTMLElement | undefined;
+
+  if (appContainer === undefined || statusbar === undefined) {
+    // give up, exit
+    console.warn(`give up watching appContainer for statusbar`);
+    console.warn(`appContainer=${appContainer}, statusbar=${statusbar}`);
+    return undefined;
+  }
+
   if (op === "enable") {
-    bar.style.setProperty("display", "flex");
+    const callback = async (
+      mutationList: MutationRecord[],
+      observer: MutationObserver
+    ) => {
+      for (const mutation of mutationList) {
+        // console.debug(mutation);
+        if (mutation.type === "childList" && mutation.addedNodes.length > 0) {
+          const k = mutation.addedNodes[0] as Element;
+          if (
+            k.className.contains("mobile-navbar") ||
+            k.className.contains("mobile-toolbar")
+          ) {
+            // have to wait, otherwise the height is not correct??
+            await new Promise((resolve) => setTimeout(resolve, 300));
+            const height = window
+              .getComputedStyle(k as Element)
+              .getPropertyValue("height");
+
+            statusbar.style.setProperty("display", "flex");
+            statusbar.style.setProperty("margin-bottom", height);
+          }
+        }
+      }
+    };
+    const observer = new MutationObserver(callback);
+    observer.observe(appContainer, {
+      attributes: false,
+      childList: true,
+      characterData: false,
+      subtree: false,
+    });
+
+    try {
+      // init, manual call
       const navBar = document.getElementsByClassName(
         "mobile-navbar"
       )[0] as HTMLElement;
       // thanks to community's solution
       const height = window.getComputedStyle(navBar).getPropertyValue("height");
-    bar.style.setProperty("margin-bottom", height);
+      statusbar.style.setProperty("display", "flex");
+      statusbar.style.setProperty("margin-bottom", height);
+    } catch (e) {
+      // skip
+    }
+
+    return observer;
   } else {
-    bar.style.removeProperty("display");
+    if (oldAppContainerObserver !== undefined) {
-    bar.style.removeProperty("margin-bottom");
+      console.debug(`disconnect oldAppContainerObserver`);
+      oldAppContainerObserver.disconnect();
+      oldAppContainerObserver = undefined;
+    }
+    statusbar.style.removeProperty("display");
+    statusbar.style.removeProperty("margin-bottom");
+    return undefined;
   }
 };
+
+/**
+ * https://github.com/remotely-save/remotely-save/issues/567
+ * https://www.dropboxforum.com/t5/Dropbox-API-Support-Feedback/Case-Sensitivity-in-API-2/td-p/191279
+ * @param entities
+ */
+export const fixEntityListCasesInplace = (entities: { keyRaw: string }[]) => {
+  entities.sort((a, b) => a.keyRaw.length - b.keyRaw.length);
+  // console.log(JSON.stringify(entities,null,2));
+
+  const caseMapping: Record<string, string> = { "": "" };
+  for (const e of entities) {
+    // console.log(`looking for: ${JSON.stringify(e, null, 2)}`);
+
+    let parentFolder = getParentFolder(e.keyRaw);
+    if (parentFolder === "/") {
+      parentFolder = "";
+    }
+    const parentFolderLower = parentFolder.toLocaleLowerCase();
+    const segs = e.keyRaw.split("/");
+    if (e.keyRaw.endsWith("/")) {
+      // folder
+      if (caseMapping.hasOwnProperty(parentFolderLower)) {
+        const newKeyRaw = `${caseMapping[parentFolderLower]}${segs
+          .slice(-2)
+          .join("/")}`;
+        caseMapping[newKeyRaw.toLocaleLowerCase()] = newKeyRaw;
+        e.keyRaw = newKeyRaw;
+        // console.log(JSON.stringify(caseMapping,null,2));
+        continue;
+      } else {
+        throw Error(`${parentFolder} doesn't have cases record??`);
+      }
+    } else {
+      // file
+      if (caseMapping.hasOwnProperty(parentFolderLower)) {
+        const newKeyRaw = `${caseMapping[parentFolderLower]}${segs
+          .slice(-1)
+          .join("/")}`;
+        e.keyRaw = newKeyRaw;
+        continue;
+      } else {
+        throw Error(`${parentFolder} doesn't have cases record??`);
+      }
+    }
+  }
+
+  return entities;
+};

src/remote.ts

@@ -239,7 +239,8 @@ export class RemoteClient {
   deleteFromRemote = async (
     fileOrFolderPath: string,
     cipher: Cipher,
-    remoteEncryptedKey: string = ""
+    remoteEncryptedKey: string = "",
+    synthesizedFolder: boolean = false
   ) => {
     if (this.serviceType === "s3") {
       return await s3.deleteFromRemote(
@@ -247,7 +248,8 @@ export class RemoteClient {
         this.s3Config!,
         fileOrFolderPath,
         cipher,
-        remoteEncryptedKey
+        remoteEncryptedKey,
+        synthesizedFolder
       );
     } else if (this.serviceType === "webdav") {
       return await webdav.deleteFromRemote(

src/remoteForDropbox.ts

@@ -12,6 +12,7 @@ import {
 } from "./baseTypes";
 import {
   bufferToArrayBuffer,
+  fixEntityListCasesInplace,
   getFolderLevels,
   hasEmojiInText,
   headersToRecord,
@@ -635,6 +636,8 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => {
     unifiedContents.push(...unifiedContents2);
   }
 
+  fixEntityListCasesInplace(unifiedContents);
+
   return unifiedContents;
 };
 

src/remoteForOnedrive.ts

@@ -471,6 +471,11 @@ export class WrappedOnedriveClient {
       const pathFrag = encodeURI(pathFragOrig);
       theUrl = `${API_PREFIX}${pathFrag}`;
     }
+    // we want to support file name with hash #
+    // because every url we construct here do not contain the # symbol
+    // thus it should be safe to directly replace the character
+    theUrl = theUrl.replace(/#/g, "%23");
+    // console.debug(`building url: [${pathFragOrig}] => [${theUrl}]`)
     return theUrl;
   };
 

src/remoteForS3.ts

@@ -250,6 +250,7 @@ const fromS3ObjectToEntity = (
     mtimeCli: mtimeCli,
     sizeRaw: x.Size!,
     etag: x.ETag,
+    synthesizedFolder: false,
   };
   return r;
 };
@@ -599,7 +600,10 @@ export const listAllFromRemote = async (
   s3Client: S3Client,
   s3Config: S3Config
 ) => {
-  return await listFromRemoteRaw(s3Client, s3Config, s3Config.remotePrefix);
+  const res = (
+    await listFromRemoteRaw(s3Client, s3Config, s3Config.remotePrefix)
+  ).filter((x) => x.keyRaw !== "" && x.keyRaw !== "/");
+  return res;
 };
 
 /**
@@ -715,11 +719,15 @@ export const deleteFromRemote = async (
   s3Config: S3Config,
   fileOrFolderPath: string,
   cipher: Cipher,
-  remoteEncryptedKey: string = ""
+  remoteEncryptedKey: string = "",
+  synthesizedFolder: boolean = false
 ) => {
   if (fileOrFolderPath === "/") {
     return;
   }
+  if (synthesizedFolder) {
+    return;
+  }
   let remoteFileName = fileOrFolderPath;
   if (!cipher.isPasswordEmpty()) {
     remoteFileName = remoteEncryptedKey;
@@ -771,7 +779,7 @@ export const checkConnectivity = async (
 ) => {
   try {
     // TODO: no universal way now, just check this in connectivity
-    if (Platform.isIosApp && !s3Config.s3Endpoint.startsWith("https")) {
+    if (Platform.isIosApp && s3Config.s3Endpoint.startsWith("http://")) {
       throw Error(
         `Your s3 endpoint could only be https, not http, because of the iOS restriction.`
       );

src/settings.ts

@@ -125,15 +125,9 @@ class PasswordModal extends Modal {
 
 class EncryptionMethodModal extends Modal {
   plugin: RemotelySavePlugin;
-  newEncryptionMethod: CipherMethodType;
+  constructor(app: App, plugin: RemotelySavePlugin) {
-  constructor(
-    app: App,
-    plugin: RemotelySavePlugin,
-    newEncryptionMethod: CipherMethodType
-  ) {
     super(app);
     this.plugin = plugin;
-    this.newEncryptionMethod = newEncryptionMethod;
   }
 
   onOpen() {
@@ -153,21 +147,12 @@ class EncryptionMethodModal extends Modal {
         });
       });
 
-    new Setting(contentEl)
+    new Setting(contentEl).addButton((button) => {
-      .addButton((button) => {
       button.setButtonText(t("confirm"));
       button.onClick(async () => {
-          this.plugin.settings.encryptionMethod = this.newEncryptionMethod;
-          await this.plugin.saveSettings();
         this.close();
       });
       button.setClass("encryptionmethod-second-confirm");
-      })
-      .addButton((button) => {
-        button.setButtonText(t("goback"));
-        button.onClick(() => {
-          this.close();
-        });
     });
   }
 
@@ -1693,24 +1678,15 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
       .setName(t("settings_encryptionmethod"))
       .setDesc(stringToFragment(t("settings_encryptionmethod_desc")))
       .addDropdown((dropdown) => {
-        dropdown.addOption(
+        dropdown
-          "rclone-base64",
+          .addOption("rclone-base64", t("settings_encryptionmethod_rclone"))
-          t("settings_encryptionmethod_rclone")
+          .addOption("openssl-base64", t("settings_encryptionmethod_openssl"))
-        );
+          .setValue(this.plugin.settings.encryptionMethod ?? "rclone-base64")
-        dropdown.addOption(
+          .onChange(async (val: string) => {
-          "openssl-base64",
-          t("settings_encryptionmethod_openssl")
-        );
-        dropdown.onChange(async (val: string) => {
-          if (this.plugin.settings.password === "") {
             this.plugin.settings.encryptionMethod = val as CipherMethodType;
             await this.plugin.saveSettings();
-          } else {
+            if (this.plugin.settings.password !== "") {
-            new EncryptionMethodModal(
+              new EncryptionMethodModal(this.app, this.plugin).open();
-              this.app,
-              this.plugin,
-              val as CipherMethodType
-            ).open();
             }
           });
       });
@@ -2122,10 +2098,16 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
             .onChange(async (val) => {
               if (val === "enable") {
                 this.plugin.settings.enableMobileStatusBar = true;
+                this.plugin.appContainerObserver =
                   changeMobileStatusBar("enable");
               } else {
                 this.plugin.settings.enableMobileStatusBar = false;
-                changeMobileStatusBar("disable");
+                changeMobileStatusBar(
+                  "disable",
+                  this.plugin.appContainerObserver
+                );
+                this.plugin.appContainerObserver?.disconnect();
+                this.plugin.appContainerObserver = undefined;
               }
               await this.plugin.saveSettings();
             });

src/sync.ts

@@ -18,6 +18,7 @@ import {
   isVaildText,
   atWhichLevel,
   mkdirpInVault,
+  getFolderLevels,
 } from "./misc";
 import {
   DEFAULT_FILE_NAME_FOR_METADATAONREMOTE,
@@ -53,8 +54,9 @@ export interface PasswordCheckType {
     | "unknown_encryption_method"
     | "remote_encrypted_local_no_password"
     | "password_matched"
-    | "password_not_matched_or_remote_not_encrypted"
+    | "password_or_method_not_matched_or_remote_not_encrypted"
-    | "likely_no_password_both_sides";
+    | "likely_no_password_both_sides"
+    | "encryption_method_not_matched";
 }
 
 export const isPasswordOk = async (
@@ -91,8 +93,19 @@ export const isPasswordOk = async (
         reason: "unknown_encryption_method",
       };
     }
+    if (
+      Cipher.isLikelyEncryptedNameNotMatchMethod(santyCheckKey, cipher.method)
+    ) {
+      return {
+        ok: false,
+        reason: "encryption_method_not_matched",
+      };
+    }
     try {
-      await cipher.decryptName(santyCheckKey);
+      const k = await cipher.decryptName(santyCheckKey);
+      if (k === undefined) {
+        throw Error(`decryption failed`);
+      }
       return {
         ok: true,
         reason: "password_matched",
@@ -100,7 +113,7 @@ export const isPasswordOk = async (
     } catch (error) {
       return {
         ok: false,
-        reason: "password_not_matched_or_remote_not_encrypted",
+        reason: "password_or_method_not_matched_or_remote_not_encrypted",
       };
     }
   }
@@ -314,7 +327,10 @@ export const ensembleMixedEnties = async (
 ): Promise<SyncPlanType> => {
   const finalMappings: SyncPlanType = {};
 
+  const synthFolders: Record<string, Entity> = {};
+
   // remote has to be first
+  // we also have to synthesize folders here
   for (const remote of remoteEntityList) {
     const remoteCopied = ensureMTimeOfRemoteEntityValid(
       await decryptRemoteEntityInplace(
@@ -340,6 +356,42 @@ export const ensembleMixedEnties = async (
       key: key,
       remote: remoteCopied,
     };
+
+    for (const f of getFolderLevels(key, true)) {
+      if (finalMappings.hasOwnProperty(f)) {
+        delete synthFolders[f];
+        continue;
+      }
+      if (
+        !synthFolders.hasOwnProperty(f) ||
+        remoteCopied.mtimeSvr! >= synthFolders[f].mtimeSvr!
+      ) {
+        synthFolders[f] = {
+          key: f,
+          keyRaw: `<synth: ${f}>`,
+          keyEnc: `<enc synth: ${f}>`,
+          size: 0,
+          sizeRaw: 0,
+          sizeEnc: 0,
+          mtimeSvr: remoteCopied.mtimeSvr,
+          mtimeSvrFmt: remoteCopied.mtimeSvrFmt,
+          mtimeCli: remoteCopied.mtimeCli,
+          mtimeCliFmt: remoteCopied.mtimeCliFmt,
+          synthesizedFolder: true,
+        };
+      }
+    }
+  }
+
+  console.debug(`synthFolders:`);
+  console.debug(synthFolders);
+
+  // special: add synth folders
+  for (const key of Object.keys(synthFolders)) {
+    finalMappings[key] = {
+      key: key,
+      remote: synthFolders[key],
+    };
   }
 
   if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) {
@@ -494,9 +546,41 @@ export const getSyncPlanInplace = async (
           mixedEntry.decisionBranch = 105;
           mixedEntry.decision = "folder_to_skip";
         } else if (howToCleanEmptyFolder === "clean_both") {
+          if (local !== undefined && remote !== undefined) {
+            if (syncDirection === "bidirectional") {
               mixedEntry.decisionBranch = 106;
-          mixedEntry.decision = "folder_to_be_deleted";
+              mixedEntry.decision = "folder_to_be_deleted_on_both";
-          // TODO: what to do in different sync direction?
+            } else {
+              // right now it does nothing because of "incremental"
+              // TODO: should we delete??
+              mixedEntry.decisionBranch = 109;
+              mixedEntry.decision = "folder_to_skip";
+            }
+          } else if (local !== undefined && remote === undefined) {
+            if (syncDirection === "bidirectional") {
+              mixedEntry.decisionBranch = 110;
+              mixedEntry.decision = "folder_to_be_deleted_on_local";
+            } else {
+              // right now it does nothing because of "incremental"
+              // TODO: should we delete??
+              mixedEntry.decisionBranch = 111;
+              mixedEntry.decision = "folder_to_skip";
+            }
+          } else if (local === undefined && remote !== undefined) {
+            if (syncDirection === "bidirectional") {
+              mixedEntry.decisionBranch = 112;
+              mixedEntry.decision = "folder_to_be_deleted_on_remote";
+            } else {
+              // right now it does nothing because of "incremental"
+              // TODO: should we delete??
+              mixedEntry.decisionBranch = 113;
+              mixedEntry.decision = "folder_to_skip";
+            }
+          } else {
+            // no folder to delete, do nothing
+            mixedEntry.decisionBranch = 114;
+            mixedEntry.decision = "folder_to_skip";
+          }
         } else {
           throw Error(
             `do not know how to deal with empty folder ${mixedEntry.key}`
@@ -694,11 +778,9 @@ export const getSyncPlanInplace = async (
               keptFolder.add(getParentFolder(key));
             }
           } else {
-            throw Error(
+            mixedEntry.decisionBranch = 36;
-              `remote is created (branch 3) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify(
+            mixedEntry.decision = "remote_is_created_too_large_then_do_nothing";
-                mixedEntry
+            keptFolder.add(getParentFolder(key));
-              )}`
-            );
           }
         } else if (
           (prevSync.mtimeSvr === remote.mtimeCli ||
@@ -757,11 +839,9 @@ export const getSyncPlanInplace = async (
               keptFolder.add(getParentFolder(key));
             }
           } else {
-            throw Error(
+            mixedEntry.decisionBranch = 37;
-              `local is created (branch 6) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify(
+            mixedEntry.decision = "local_is_created_too_large_then_do_nothing";
-                mixedEntry
+            keptFolder.add(getParentFolder(key));
-              )}`
-            );
           }
         } else if (
           (prevSync.mtimeSvr === local.mtimeCli ||
@@ -823,13 +903,27 @@ export const getSyncPlanInplace = async (
     throw Error(`unexpectedly keptFolder no decisions: ${[...keptFolder]}`);
   }
 
+  // finally we want to make our life easier
+  const currTime = Date.now();
+  const currTimeFmt = unixTimeToStr(currTime);
+  // because the path should not as / in the beginning,
+  // we should be safe to add these keys:
+  mixedEntityMappings["/$@meta"] = {
+    key: "/$@meta", // don't mess up with the types
+    sideNotes: {
+      generateTime: currTime,
+      generateTimeFmt: currTimeFmt,
+    },
+  };
+
   return mixedEntityMappings;
 };
 
-const splitThreeStepsOnEntityMappings = (
+const splitFourStepsOnEntityMappings = (
   mixedEntityMappings: Record<string, MixedEntity>
 ) => {
   type StepArrayType = MixedEntity[] | undefined | null;
+  const onlyMarkSyncedOps: StepArrayType[] = [];
   const folderCreationOps: StepArrayType[] = [];
   const deletionOps: StepArrayType[] = [];
   const uploadDownloads: StepArrayType[] = [];
@@ -845,6 +939,11 @@ const splitThreeStepsOnEntityMappings = (
 
   for (let i = 0; i < sortedKeys.length; ++i) {
     const key = sortedKeys[i];
+
+    if (key === "/$@meta") {
+      continue; // special
+    }
+
     const val = mixedEntityMappings[key];
 
     if (!key.endsWith("/")) {
@@ -852,12 +951,27 @@ const splitThreeStepsOnEntityMappings = (
     }
 
     if (
-      val.decision === "equal" ||
+      val.decision === "local_is_created_too_large_then_do_nothing" ||
-      val.decision === "conflict_created_then_do_nothing" ||
+      val.decision === "remote_is_created_too_large_then_do_nothing" ||
-      val.decision === "folder_existed_both_then_do_nothing" ||
       val.decision === "folder_to_skip"
     ) {
       // pass
+    } else if (
+      val.decision === "equal" ||
+      val.decision === "conflict_created_then_do_nothing" ||
+      val.decision === "folder_existed_both_then_do_nothing"
+    ) {
+      if (
+        onlyMarkSyncedOps.length === 0 ||
+        onlyMarkSyncedOps[0] === undefined ||
+        onlyMarkSyncedOps[0] === null
+      ) {
+        onlyMarkSyncedOps[0] = [val];
+      } else {
+        onlyMarkSyncedOps[0].push(val); // only one level is needed here
+      }
+
+      // don't need to update realTotalCount here
     } else if (
       val.decision === "folder_existed_local_then_also_create_remote" ||
       val.decision === "folder_existed_remote_then_also_create_local" ||
@@ -877,7 +991,9 @@ const splitThreeStepsOnEntityMappings = (
       val.decision === "only_history" ||
       val.decision === "local_is_deleted_thus_also_delete_remote" ||
       val.decision === "remote_is_deleted_thus_also_delete_local" ||
-      val.decision === "folder_to_be_deleted"
+      val.decision === "folder_to_be_deleted_on_both" ||
+      val.decision === "folder_to_be_deleted_on_local" ||
+      val.decision === "folder_to_be_deleted_on_remote"
     ) {
       const level = atWhichLevel(key);
       const k = deletionOps[level - 1];
@@ -888,7 +1004,11 @@ const splitThreeStepsOnEntityMappings = (
       }
       realTotalCount += 1;
 
-      if (val.decision.startsWith("deleted")) {
+      if (
+        val.decision.includes("deleted") &&
+        !val.decision.includes("folder")
+      ) {
+        // only count files here, skip folder
         realModifyDeleteCount += 1;
       }
     } else if (
@@ -915,8 +1035,8 @@ const splitThreeStepsOnEntityMappings = (
       realTotalCount += 1;
 
       if (
-        val.decision.startsWith("modified") ||
+        val.decision.includes("modified") ||
-        val.decision.startsWith("conflict")
+        val.decision.includes("conflict")
       ) {
         realModifyDeleteCount += 1;
       }
@@ -931,6 +1051,7 @@ const splitThreeStepsOnEntityMappings = (
   deletionOps.reverse(); // inplace reverse
 
   return {
+    onlyMarkSyncedOps: onlyMarkSyncedOps,
     folderCreationOps: folderCreationOps,
     deletionOps: deletionOps,
     uploadDownloads: uploadDownloads,
@@ -960,13 +1081,37 @@ const dispatchOperationToActualV3 = async (
   // );
   if (r.decision === "only_history") {
     clearPrevSyncRecordByVaultAndProfile(db, vaultRandomID, profileID, key);
+  } else if (
+    r.decision === "local_is_created_too_large_then_do_nothing" ||
+    r.decision === "remote_is_created_too_large_then_do_nothing" ||
+    r.decision === "folder_to_skip"
+  ) {
+    // !! no actual sync being kept happens,
+    // so no sync record here
+    // pass
   } else if (
     r.decision === "equal" ||
     r.decision === "conflict_created_then_do_nothing" ||
-    r.decision === "folder_to_skip" ||
     r.decision === "folder_existed_both_then_do_nothing"
   ) {
-    // pass
+    // !! we need to upsert the record,
+    // so that next time we can determine the change delta
+    const entity = r.remote ?? r.local;
+    console.debug(
+      `we are in actual operation of equal, entity=${JSON.stringify(
+        entity,
+        null,
+        2
+      )}`
+    );
+    if (entity !== undefined) {
+      await upsertPrevSyncRecordByVaultAndProfile(
+        db,
+        vaultRandomID,
+        profileID,
+        entity
+      );
+    }
   } else if (
     r.decision === "local_is_modified_then_push" ||
     r.decision === "local_is_created_then_push" ||
@@ -1024,7 +1169,12 @@ const dispatchOperationToActualV3 = async (
     );
   } else if (r.decision === "local_is_deleted_thus_also_delete_remote") {
     // local is deleted, we need to delete remote now
-    await client.deleteFromRemote(r.key, cipher, r.remote!.keyEnc);
+    await client.deleteFromRemote(
+      r.key,
+      cipher,
+      r.remote!.keyEnc,
+      r.remote!.synthesizedFolder
+    );
     await clearPrevSyncRecordByVaultAndProfile(
       db,
       vaultRandomID,
@@ -1063,9 +1213,28 @@ const dispatchOperationToActualV3 = async (
       profileID,
       entity
     );
-  } else if (r.decision === "folder_to_be_deleted") {
+  } else if (
+    r.decision === "folder_to_be_deleted_on_both" ||
+    r.decision === "folder_to_be_deleted_on_local" ||
+    r.decision === "folder_to_be_deleted_on_remote"
+  ) {
+    if (
+      r.decision === "folder_to_be_deleted_on_both" ||
+      r.decision === "folder_to_be_deleted_on_local"
+    ) {
       await localDeleteFunc(r.key);
-    await client.deleteFromRemote(r.key, cipher, r.remote!.keyEnc);
+    }
+    if (
+      r.decision === "folder_to_be_deleted_on_both" ||
+      r.decision === "folder_to_be_deleted_on_remote"
+    ) {
+      await client.deleteFromRemote(
+        r.key,
+        cipher,
+        r.remote!.keyEnc,
+        r.remote!.synthesizedFolder
+      );
+    }
     await clearPrevSyncRecordByVaultAndProfile(
       db,
       vaultRandomID,
@@ -1093,13 +1262,15 @@ export const doActualSync = async (
 ) => {
   console.debug(`concurrency === ${concurrency}`);
   const {
+    onlyMarkSyncedOps,
     folderCreationOps,
     deletionOps,
     uploadDownloads,
     allFilesCount,
     realModifyDeleteCount,
     realTotalCount,
-  } = splitThreeStepsOnEntityMappings(mixedEntityMappings);
+  } = splitFourStepsOnEntityMappings(mixedEntityMappings);
+  // console.debug(`onlyMarkSyncedOps: ${JSON.stringify(onlyMarkSyncedOps)}`);
   // console.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`);
   // console.debug(`deletionOps: ${JSON.stringify(deletionOps)}`);
   // console.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`);
@@ -1115,6 +1286,12 @@ export const doActualSync = async (
     allFilesCount > 0
   ) {
     if (
+      protectModifyPercentage === 100 &&
+      realModifyDeleteCount === allFilesCount
+    ) {
+      // special treatment for 100%
+      // let it pass, we do nothing here
+    } else if (
       realModifyDeleteCount * 100 >=
       allFilesCount * protectModifyPercentage
     ) {
@@ -1128,11 +1305,17 @@ export const doActualSync = async (
     }
   }
 
-  const nested = [folderCreationOps, deletionOps, uploadDownloads];
+  const nested = [
+    onlyMarkSyncedOps,
+    folderCreationOps,
+    deletionOps,
+    uploadDownloads,
+  ];
   const logTexts = [
-    `1. create all folders from shadowest to deepest`,
+    `1. record the items already being synced`,
-    `2. delete files and folders from deepest to shadowest`,
+    `2. create all folders from shadowest to deepest`,
-    `3. upload or download files in parallel, with the desired concurrency=${concurrency}`,
+    `3. delete files and folders from deepest to shadowest`,
+    `4. upload or download files in parallel, with the desired concurrency=${concurrency}`,
   ];
 
   let realCounter = 0;

tests/misc.test.ts

@@ -285,3 +285,56 @@ describe("Misc: special char for dir", () => {
     expect(misc.checkHasSpecialCharForDir("xxx?yyy")).to.be.true;
   });
 });
+
+describe("Misc: Dropbox: should fix the folder name cases", () => {
+  it("should do nothing on empty folders", () => {
+    const input: any[] = [];
+    expect(misc.fixEntityListCasesInplace(input)).to.be.empty;
+  });
+
+  it("should sort folders by length by side effect", () => {
+    const input = [
+      { keyRaw: "aaaa/" },
+      { keyRaw: "bbb/" },
+      { keyRaw: "c/" },
+      { keyRaw: "dd/" },
+    ];
+
+    const output = [
+      { keyRaw: "c/" },
+      { keyRaw: "dd/" },
+      { keyRaw: "bbb/" },
+      { keyRaw: "aaaa/" },
+    ];
+    expect(misc.fixEntityListCasesInplace(input)).to.deep.equal(output);
+  });
+
+  it("should fix folder names", () => {
+    const input = [
+      { keyRaw: "AAA/" },
+      { keyRaw: "aaa/bbb/CCC.md" },
+      { keyRaw: "aaa/BBB/" },
+
+      { keyRaw: "ddd/" },
+      { keyRaw: "DDD/EEE/fff.md" },
+      { keyRaw: "DDD/eee/" },
+
+      { keyRaw: "Ggg/" },
+      { keyRaw: "ggG/hHH你好/Fff世界.md" },
+      { keyRaw: "ggG/Hhh你好/" },
+    ];
+
+    const output = [
+      { keyRaw: "AAA/" },
+      { keyRaw: "ddd/" },
+      { keyRaw: "Ggg/" },
+      { keyRaw: "AAA/BBB/" },
+      { keyRaw: "ddd/eee/" },
+      { keyRaw: "Ggg/Hhh你好/" },
+      { keyRaw: "AAA/BBB/CCC.md" },
+      { keyRaw: "ddd/eee/fff.md" },
+      { keyRaw: "Ggg/Hhh你好/Fff世界.md" },
+    ];
+    expect(misc.fixEntityListCasesInplace(input)).to.deep.equal(output);
+  });
+});