0.3.4

manifest.json

@@ -1,8 +1,8 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.3.2",
+  "version": "0.3.4",
-  "minAppVersion": "0.12.15",
+  "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",
   "authorUrl": "https://github.com/fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs",
@@ -70,7 +70,8 @@
     "loglevel": "^1.8.0",
     "mime-types": "^2.1.33",
     "nanoid": "^3.1.30",
-    "obsidian": "^0.12.0",
+    "obsidian": "^0.13.26",
+    "p-queue": "^7.2.0",
     "path-browserify": "^1.0.1",
     "process": "^0.11.10",
     "qrcode": "^1.5.0",

src/baseTypes.ts

@@ -56,7 +56,9 @@ export interface RemotelySavePluginSettings {
   currLogLevel?: string;
   vaultRandomID?: string;
   autoRunEveryMilliseconds?: number;
+  initRunAfterMilliseconds?: number;
   agreeToUploadExtraMetadata?: boolean;
+  concurrency?: number;
 }
 
 export interface RemoteItem {
@@ -113,3 +115,5 @@ export interface FileOrFolderMixedState {
   syncDone?: "done";
   remoteEncryptedKey?: string;
 }
+
+export const API_VER_STAT_FOLDER = "0.13.27";

src/main.ts

@@ -53,7 +53,9 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
   currLogLevel: "info",
   vaultRandomID: "",
   autoRunEveryMilliseconds: -1,
+  initRunAfterMilliseconds: -1,
   agreeToUploadExtraMetadata: false,
+  concurrency: 5,
 };
 
 interface OAuth2Info {
@@ -64,7 +66,7 @@ interface OAuth2Info {
   revokeAuthSetting?: Setting;
 }
 
-type SyncTriggerSourceType = "manual" | "auto" | "dry";
+type SyncTriggerSourceType = "manual" | "auto" | "dry" | "autoOnceInit";
 
 const iconNameSyncWait = `remotely-save-sync-wait`;
 const iconNameSyncRunning = `remotely-save-sync-running`;
@@ -199,6 +201,7 @@ export default class RemotelySavePlugin extends Plugin {
         origMetadataOnRemote.deletions,
         localHistory,
         client.serviceType,
+        this.app.vault,
         this.settings.password
       );
       log.info(plan.mixedStates); // for debugging
@@ -229,6 +232,7 @@ export default class RemotelySavePlugin extends Plugin {
           deletions,
           (key: string) => self.trash(key),
           this.settings.password,
+          this.settings.concurrency,
           (i: number, totalCount: number, pathName: string, decision: string) =>
             self.setCurrSyncMsg(i, totalCount, pathName, decision)
         );
@@ -531,6 +535,9 @@ export default class RemotelySavePlugin extends Plugin {
     if (!this.settings.agreeToUploadExtraMetadata) {
       const syncAlgoV2Modal = new SyncAlgoV2Modal(this.app, this);
       syncAlgoV2Modal.open();
+    } else {
+      this.enableAutoSyncIfSet();
+      this.enableInitSyncIfSet();
     }
   }
 
@@ -659,11 +666,27 @@ export default class RemotelySavePlugin extends Plugin {
       this.settings.autoRunEveryMilliseconds !== null &&
       this.settings.autoRunEveryMilliseconds > 0
     ) {
+      this.app.workspace.onLayoutReady(() => {
         const intervalID = window.setInterval(() => {
           this.syncRun("auto");
         }, this.settings.autoRunEveryMilliseconds);
         this.autoRunIntervalID = intervalID;
         this.registerInterval(intervalID);
+      });
+    }
+  }
+
+  enableInitSyncIfSet() {
+    if (
+      this.settings.initRunAfterMilliseconds !== undefined &&
+      this.settings.initRunAfterMilliseconds !== null &&
+      this.settings.initRunAfterMilliseconds > 0
+    ) {
+      this.app.workspace.onLayoutReady(() => {
+        window.setTimeout(() => {
+          this.syncRun("autoOnceInit");
+        }, this.settings.initRunAfterMilliseconds);
+      });
     }
   }
 

src/settings.ts

@@ -521,6 +521,55 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           });
       });
 
+    const runOnceStartUpDiv = generalDiv.createEl("div");
+    new Setting(runOnceStartUpDiv)
+      .setName("run once on start up automatically")
+      .setDesc(
+        `This settings allows setting running ONCE on start up automatically. This will take effect on NEXT start up after changing. This setting, is different from "schedule for auto run" which starts syncing after EVERY interval.`
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("-1", "(not set)");
+        dropdown.addOption(
+          `${1000 * 10 * 1}`,
+          "sync once after 10 seconds of start up"
+        );
+        dropdown.addOption(
+          `${1000 * 30 * 1}`,
+          "sync once after 30 seconds of start up"
+        );
+        dropdown
+          .setValue(`${this.plugin.settings.initRunAfterMilliseconds}`)
+          .onChange(async (val: string) => {
+            const realVal = parseInt(val);
+            this.plugin.settings.initRunAfterMilliseconds = realVal;
+            await this.plugin.saveSettings();
+          });
+      });
+
+    const concurrencyDiv = generalDiv.createEl("div");
+    new Setting(concurrencyDiv)
+      .setName("Concurrency")
+      .setDesc(
+        "How many files do you want to download or upload in parallel at most? By default it's set to 5. If you meet any problems such as rate limit, you can reduce the concurrency to a lower value."
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("1", "1");
+        dropdown.addOption("2", "2");
+        dropdown.addOption("3", "3");
+        dropdown.addOption("5", "5 (default)");
+        dropdown.addOption("10", "10");
+        dropdown.addOption("15", "15");
+        dropdown.addOption("20", "20");
+
+        dropdown
+          .setValue(`${this.plugin.settings.concurrency}`)
+          .onChange(async (val) => {
+            const realVal = parseInt(val);
+            this.plugin.settings.concurrency = realVal;
+            await this.plugin.saveSettings();
+          });
+      });
+
     //////////////////////////////////////////////////
     // below for general chooser (part 1/2)
     //////////////////////////////////////////////////

src/sync.ts

@@ -1,9 +1,17 @@
-import { TAbstractFile, TFile, TFolder, Vault } from "obsidian";
+import {
-import type {
+  TAbstractFile,
+  TFile,
+  TFolder,
+  Vault,
+  requireApiVersion,
+} from "obsidian";
+import PQueue from "p-queue";
+import {
   RemoteItem,
   SUPPORTED_SERVICES_TYPE,
   DecisionType,
   FileOrFolderMixedState,
+  API_VER_STAT_FOLDER,
 } from "./baseTypes";
 import {
   decryptBase32ToString,
@@ -293,7 +301,7 @@ const ensembleMixedStates = async (
       r = {
         key: entry.path,
         existLocal: true,
-        mtimeLocal: entry.stat.mtime,
+        mtimeLocal: Math.max(entry.stat.mtime, entry.stat.ctime),
         sizeLocal: entry.stat.size,
       };
     } else if (entry instanceof TFolder) {
@@ -505,9 +513,10 @@ const assignOperationToFileInplace = (
   throw Error(`no decision for ${JSON.stringify(r)}`);
 };
 
-const assignOperationToFolderInplace = (
+const assignOperationToFolderInplace = async (
   origRecord: FileOrFolderMixedState,
   keptFolder: Set<string>,
+  vault: Vault,
   password: string = ""
 ) => {
   let r = origRecord;
@@ -523,10 +532,42 @@ const assignOperationToFolderInplace = (
 
     if (r.deltimeLocal !== undefined || r.deltimeRemote !== undefined) {
       // it has some deletion "commands"
+
+      const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
+      const deltimeRemote =
+        r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
+
+      // if it was created after deletion, we should keep it as is
+      if (requireApiVersion(API_VER_STAT_FOLDER)) {
+        if (r.existLocal) {
+          try {
+            const { ctime, mtime } = await vault.adapter.stat(r.key);
+            const cmtime = Math.max(ctime, mtime);
             if (
-        r.deltimeLocal !== undefined &&
+              cmtime > 0 &&
-        r.deltimeLocal >= (r.deltimeRemote !== undefined ? r.deltimeRemote : -1)
+              cmtime >= deltimeLocal &&
+              cmtime >= deltimeRemote
             ) {
+              keptFolder.add(getParentFolder(r.key));
+              if (r.existLocal && r.existRemote) {
+                r.decision = "skipFolder";
+                r.decisionBranch = 14;
+              } else if (r.existLocal || r.existRemote) {
+                r.decision = "createFolder";
+                r.decisionBranch = 15;
+              } else {
+                throw Error(
+                  `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
+                );
+              }
+            }
+          } catch (error) {
+            // pass
+          }
+        }
+      }
+
+      if (deltimeLocal > 0 && deltimeLocal > deltimeRemote) {
         r.decision = "uploadLocalDelHistToRemoteFolder";
         r.decisionBranch = 8;
       } else {
@@ -585,6 +626,7 @@ export const getSyncPlan = async (
   remoteDeleteHistory: DeletionOnRemote[],
   localDeleteHistory: FileFolderHistoryRecord[],
   remoteType: SUPPORTED_SERVICES_TYPE,
+  vault: Vault,
   password: string = ""
 ) => {
   const mixedStates = await ensembleMixedStates(
@@ -609,7 +651,7 @@ export const getSyncPlan = async (
       // decide some folders
       // because the keys are sorted by length
       // so all the children must have been shown up before in the iteration
-      assignOperationToFolderInplace(val, keptFolder, password);
+      await assignOperationToFolderInplace(val, keptFolder, vault, password);
     } else {
       // get all operations of files
       // and at the same time get some helper info for folders
@@ -850,10 +892,10 @@ export const doActualSync = async (
   deletions: DeletionOnRemote[],
   localDeleteFunc: any,
   password: string = "",
+  concurrency: number = 1,
   callbackSyncProcess?: any
 ) => {
   const mixedStates = syncPlan.mixedStates;
-  let i = 0;
   const totalCount = sortedKeys.length || 0;
 
   log.debug(`start syncing extra data firstly`);
@@ -866,6 +908,10 @@ export const doActualSync = async (
   );
   log.debug(`finish syncing extra data firstly`);
 
+  log.debug(`concurrency === ${concurrency}`);
+  if (concurrency === 1) {
+    // run everything in sequence
+    // good old way
     for (let i = 0; i < sortedKeys.length; ++i) {
       const key = sortedKeys[i];
       const val = mixedStates[key];
@@ -887,20 +933,120 @@ export const doActualSync = async (
         password
       );
       log.debug(`finished ${key}`);
+    }
+  } else {
+    let realCounter = 0;
 
-    // await Promise.all(
+    log.debug(
-    //   Object.entries(mixedStates).map(async ([k, v]) =>
+      `1. create all folders from shadowest to deepest, also check undefined decision`
-    //     dispatchOperationToActual(
+    );
-    //       k as string,
+    for (let i = sortedKeys.length - 1; i >= 0; --i) {
-    //       vaultRandomID,
+      const key = sortedKeys[i];
-    //       v as FileOrFolderMixedState,
+      const val = mixedStates[key];
-    //       client,
+
-    //       db,
+      if (
-    //       vault,
+        val.decision === undefined ||
-    //       localDeleteFunc,
+        val.decision === "skipFolder" ||
-    //       password
+        val.decision === "createFolder"
-    //     )
+      ) {
-    //   )
+        log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
-    // );
+
+        if (callbackSyncProcess !== undefined) {
+          await callbackSyncProcess(realCounter, totalCount, key, val.decision);
+        }
+        realCounter += 1;
+
+        await dispatchOperationToActual(
+          key,
+          vaultRandomID,
+          val,
+          client,
+          db,
+          vault,
+          localDeleteFunc,
+          password
+        );
+        log.debug(`finished ${key}`);
+      }
+    }
+
+    log.debug(`2. delete files and folders from deepest to shadowest`);
+    for (let i = 0; i < sortedKeys.length; ++i) {
+      const key = sortedKeys[i];
+      const val = mixedStates[key];
+      if (
+        val.decision === "uploadLocalDelHistToRemoteFolder" ||
+        val.decision === "keepRemoteDelHistFolder"
+      ) {
+        log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
+
+        if (callbackSyncProcess !== undefined) {
+          await callbackSyncProcess(realCounter, totalCount, key, val.decision);
+        }
+        realCounter += 1;
+
+        await dispatchOperationToActual(
+          key,
+          vaultRandomID,
+          val,
+          client,
+          db,
+          vault,
+          localDeleteFunc,
+          password
+        );
+        log.debug(`finished ${key}`);
+      }
+    }
+
+    log.debug(
+      `3. upload or download files in parallel, with the desired concurrency=${concurrency}`
+    );
+    const queue = new PQueue({ concurrency: concurrency, autoStart: true });
+
+    // const commands: any[] = [];
+
+    for (let i = 0; i < sortedKeys.length; ++i) {
+      const key = sortedKeys[i];
+      const val = mixedStates[key];
+      if (
+        val.decision === "skipUploading" ||
+        val.decision === "uploadLocalDelHistToRemote" ||
+        val.decision === "keepRemoteDelHist" ||
+        val.decision === "uploadLocalToRemote" ||
+        val.decision === "downloadRemoteToLocal"
+      ) {
+        const fn = async () => {
+          log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
+
+          if (callbackSyncProcess !== undefined) {
+            await callbackSyncProcess(
+              realCounter,
+              totalCount,
+              key,
+              val.decision
+            );
+
+            realCounter += 1;
+          }
+
+          await dispatchOperationToActual(
+            key,
+            vaultRandomID,
+            val,
+            client,
+            db,
+            vault,
+            localDeleteFunc,
+            password
+          );
+
+          log.debug(`finished ${key}`);
+        };
+        queue.add(fn);
+      }
+    }
+
+    await queue.onIdle();
   }
 };

src/syncAlgoV2Notice.ts

@@ -60,6 +60,7 @@ export class SyncAlgoV2Modal extends Modal {
       log.info("agree to use the new algorithm");
       this.plugin.saveAgreeToUseNewSyncAlgorithm();
       this.plugin.enableAutoSyncIfSet();
+      this.plugin.enableInitSyncIfSet();
     } else {
       log.info("do not agree to use the new algorithm");
       this.plugin.unload();

versions.json

@@ -1,3 +1,4 @@
 {
-  "0.3.2": "0.12.15"
+  "0.3.2": "0.12.15",
+  "0.3.4": "0.13.21"
 }