0.3.4

manifest.json

@@ -1,8 +1,8 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.3.1",
-  "minAppVersion": "0.12.15",
+  "version": "0.3.4",
+  "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",
   "authorUrl": "https://github.com/fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.3.1",
+  "version": "0.3.4",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs",
@@ -70,7 +70,8 @@
     "loglevel": "^1.8.0",
     "mime-types": "^2.1.33",
     "nanoid": "^3.1.30",
-    "obsidian": "^0.12.0",
+    "obsidian": "^0.13.26",
+    "p-queue": "^7.2.0",
     "path-browserify": "^1.0.1",
     "process": "^0.11.10",
     "qrcode": "^1.5.0",

src/baseTypes.ts

@@ -56,7 +56,9 @@ export interface RemotelySavePluginSettings {
   currLogLevel?: string;
   vaultRandomID?: string;
   autoRunEveryMilliseconds?: number;
+  initRunAfterMilliseconds?: number;
   agreeToUploadExtraMetadata?: boolean;
+  concurrency?: number;
 }
 
 export interface RemoteItem {
@@ -113,3 +115,5 @@ export interface FileOrFolderMixedState {
   syncDone?: "done";
   remoteEncryptedKey?: string;
 }
+
+export const API_VER_STAT_FOLDER = "0.13.27";

src/main.ts

@@ -53,7 +53,9 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
   currLogLevel: "info",
   vaultRandomID: "",
   autoRunEveryMilliseconds: -1,
+  initRunAfterMilliseconds: -1,
   agreeToUploadExtraMetadata: false,
+  concurrency: 5,
 };
 
 interface OAuth2Info {
@@ -64,7 +66,7 @@ interface OAuth2Info {
   revokeAuthSetting?: Setting;
 }
 
-type SyncTriggerSourceType = "manual" | "auto" | "dry";
+type SyncTriggerSourceType = "manual" | "auto" | "dry" | "autoOnceInit";
 
 const iconNameSyncWait = `remotely-save-sync-wait`;
 const iconNameSyncRunning = `remotely-save-sync-running`;
@@ -199,6 +201,7 @@ export default class RemotelySavePlugin extends Plugin {
         origMetadataOnRemote.deletions,
         localHistory,
         client.serviceType,
+        this.app.vault,
         this.settings.password
       );
       log.info(plan.mixedStates); // for debugging
@@ -229,6 +232,7 @@ export default class RemotelySavePlugin extends Plugin {
           deletions,
           (key: string) => self.trash(key),
           this.settings.password,
+          this.settings.concurrency,
           (i: number, totalCount: number, pathName: string, decision: string) =>
             self.setCurrSyncMsg(i, totalCount, pathName, decision)
         );
@@ -531,6 +535,9 @@ export default class RemotelySavePlugin extends Plugin {
     if (!this.settings.agreeToUploadExtraMetadata) {
       const syncAlgoV2Modal = new SyncAlgoV2Modal(this.app, this);
       syncAlgoV2Modal.open();
+    } else {
+      this.enableAutoSyncIfSet();
+      this.enableInitSyncIfSet();
     }
   }
 
@@ -659,11 +666,27 @@ export default class RemotelySavePlugin extends Plugin {
       this.settings.autoRunEveryMilliseconds !== null &&
       this.settings.autoRunEveryMilliseconds > 0
     ) {
-      const intervalID = window.setInterval(() => {
-        this.syncRun("auto");
-      }, this.settings.autoRunEveryMilliseconds);
-      this.autoRunIntervalID = intervalID;
-      this.registerInterval(intervalID);
+      this.app.workspace.onLayoutReady(() => {
+        const intervalID = window.setInterval(() => {
+          this.syncRun("auto");
+        }, this.settings.autoRunEveryMilliseconds);
+        this.autoRunIntervalID = intervalID;
+        this.registerInterval(intervalID);
+      });
+    }
+  }
+
+  enableInitSyncIfSet() {
+    if (
+      this.settings.initRunAfterMilliseconds !== undefined &&
+      this.settings.initRunAfterMilliseconds !== null &&
+      this.settings.initRunAfterMilliseconds > 0
+    ) {
+      this.app.workspace.onLayoutReady(() => {
+        window.setTimeout(() => {
+          this.syncRun("autoOnceInit");
+        }, this.settings.initRunAfterMilliseconds);
+      });
     }
   }
 

src/remoteForDropbox.ts

@@ -156,13 +156,20 @@ const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => {
 // see https://dropbox.tech/developers/pkce--what-and-why-
 ////////////////////////////////////////////////////////////////////////////////
 
-export const getAuthUrlAndVerifier = async (appKey: string) => {
+export const getAuthUrlAndVerifier = async (
+  appKey: string,
+  needManualPatse: boolean = false
+) => {
   const auth = new DropboxAuth({
     clientId: appKey,
   });
+
+  const callback = needManualPatse
+    ? undefined
+    : `obsidian://${COMMAND_CALLBACK_DROPBOX}`;
   const authUrl = (
     await auth.getAuthenticationUrl(
-      `obsidian://${COMMAND_CALLBACK_DROPBOX}`,
+      callback,
       undefined,
       "code",
       "offline",

src/remoteForS3.ts

@@ -190,7 +190,6 @@ export const listFromRemote = async (
   const contents = [] as _Object[];
 
   let isTruncated = true;
-  let continuationToken = "";
   do {
     const rsp = await s3Client.send(new ListObjectsV2Command(confCmd));
 
@@ -206,7 +205,8 @@ export const listFromRemote = async (
     confCmd.ContinuationToken = rsp.NextContinuationToken;
     if (
       isTruncated &&
-      (continuationToken === undefined || continuationToken === "")
+      (confCmd.ContinuationToken === undefined ||
+        confCmd.ContinuationToken === "")
     ) {
       throw Error("isTruncated is true but no continuationToken provided");
     }

src/settings.ts

@@ -1,4 +1,11 @@
-import { App, Modal, Notice, PluginSettingTab, Setting } from "obsidian";
+import {
+  App,
+  Modal,
+  Notice,
+  PluginSettingTab,
+  Setting,
+  Platform,
+} from "obsidian";
 import type { SUPPORTED_SERVICES_TYPE, WebdavAuthType } from "./baseTypes";
 import { exportVaultSyncPlansToFiles } from "./debugMode";
 import { exportQrCodeUri } from "./importExport";
@@ -110,17 +117,45 @@ class DropboxAuthModal extends Modal {
   async onOpen() {
     let { contentEl } = this;
 
-    const { authUrl, verifier } = await getAuthUrlAndVerifierDropbox(
-      this.plugin.settings.dropbox.clientID
-    );
-    this.plugin.oauth2Info.verifier = verifier;
+    let needManualPatse = false;
+    const userAgent = window.navigator.userAgent.toLocaleLowerCase() || "";
+    // some users report that,
+    // the Linux would open another instance Obsidian if jumping back,
+    // so fallback to manual paste on Linux
+    if (
+      Platform.isDesktopApp &&
+      !Platform.isMacOS &&
+      (/linux/.test(userAgent) ||
+        /ubuntu/.test(userAgent) ||
+        /debian/.test(userAgent) ||
+        /fedora/.test(userAgent) ||
+        /centos/.test(userAgent))
+    ) {
+      needManualPatse = true;
+    }
 
-    contentEl.createEl("p", {
-      text: "Visit the address in a browser, and follow the steps.",
-    });
-    contentEl.createEl("p", {
-      text: "Finally you should be redirected to Obsidian.",
-    });
+    const { authUrl, verifier } = await getAuthUrlAndVerifierDropbox(
+      this.plugin.settings.dropbox.clientID,
+      needManualPatse
+    );
+
+    if (needManualPatse) {
+      contentEl.createEl("p", {
+        text: "Step 1: Visit the address in a browser, and follow the steps.",
+      });
+      contentEl.createEl("p", {
+        text: 'Step 2: In the end of the web flow, you obtain a long code. Paste it here then click "Submit".',
+      });
+    } else {
+      this.plugin.oauth2Info.verifier = verifier;
+
+      contentEl.createEl("p", {
+        text: "Visit the address in a browser, and follow the steps.",
+      });
+      contentEl.createEl("p", {
+        text: "Finally you should be redirected to Obsidian.",
+      });
+    }
 
     const div2 = contentEl.createDiv();
     div2.createEl(
@@ -140,6 +175,70 @@ class DropboxAuthModal extends Modal {
       href: authUrl,
       text: authUrl,
     });
+
+    if (needManualPatse) {
+      let authCode = "";
+      new Setting(contentEl)
+        .setName("Auth Code from web page")
+        .setDesc('You need to click "Confirm".')
+        .addText((text) =>
+          text
+            .setPlaceholder("")
+            .setValue("")
+            .onChange((val) => {
+              authCode = val.trim();
+            })
+        )
+        .addButton(async (button) => {
+          button.setButtonText("Confirm");
+          button.onClick(async () => {
+            new Notice("Trying to connect to Dropbox");
+            try {
+              const authRes = await sendAuthReqDropbox(
+                this.plugin.settings.dropbox.clientID,
+                verifier,
+                authCode
+              );
+              const self = this;
+              setConfigBySuccessfullAuthInplace(
+                this.plugin.settings.dropbox,
+                authRes,
+                () => self.plugin.saveSettings()
+              );
+              const client = new RemoteClient(
+                "dropbox",
+                undefined,
+                undefined,
+                this.plugin.settings.dropbox,
+                undefined,
+                this.app.vault.getName(),
+                () => self.plugin.saveSettings()
+              );
+              const username = await client.getUser();
+              this.plugin.settings.dropbox.username = username;
+              await this.plugin.saveSettings();
+              new Notice(
+                `Good! We've connected to Dropbox as user ${username}!`
+              );
+              this.authDiv.toggleClass(
+                "dropbox-auth-button-hide",
+                this.plugin.settings.dropbox.username !== ""
+              );
+              this.revokeAuthDiv.toggleClass(
+                "dropbox-revoke-auth-button-hide",
+                this.plugin.settings.dropbox.username === ""
+              );
+              this.revokeAuthSetting.setDesc(
+                `You've connected as user ${this.plugin.settings.dropbox.username}. If you want to disconnect, click this button.`
+              );
+              this.close();
+            } catch (err) {
+              console.error(err);
+              new Notice("Something goes wrong while connecting to Dropbox.");
+            }
+          });
+        });
+    }
   }
 
   onClose() {
@@ -422,6 +521,55 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           });
       });
 
+    const runOnceStartUpDiv = generalDiv.createEl("div");
+    new Setting(runOnceStartUpDiv)
+      .setName("run once on start up automatically")
+      .setDesc(
+        `This settings allows setting running ONCE on start up automatically. This will take effect on NEXT start up after changing. This setting, is different from "schedule for auto run" which starts syncing after EVERY interval.`
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("-1", "(not set)");
+        dropdown.addOption(
+          `${1000 * 10 * 1}`,
+          "sync once after 10 seconds of start up"
+        );
+        dropdown.addOption(
+          `${1000 * 30 * 1}`,
+          "sync once after 30 seconds of start up"
+        );
+        dropdown
+          .setValue(`${this.plugin.settings.initRunAfterMilliseconds}`)
+          .onChange(async (val: string) => {
+            const realVal = parseInt(val);
+            this.plugin.settings.initRunAfterMilliseconds = realVal;
+            await this.plugin.saveSettings();
+          });
+      });
+
+    const concurrencyDiv = generalDiv.createEl("div");
+    new Setting(concurrencyDiv)
+      .setName("Concurrency")
+      .setDesc(
+        "How many files do you want to download or upload in parallel at most? By default it's set to 5. If you meet any problems such as rate limit, you can reduce the concurrency to a lower value."
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("1", "1");
+        dropdown.addOption("2", "2");
+        dropdown.addOption("3", "3");
+        dropdown.addOption("5", "5 (default)");
+        dropdown.addOption("10", "10");
+        dropdown.addOption("15", "15");
+        dropdown.addOption("20", "20");
+
+        dropdown
+          .setValue(`${this.plugin.settings.concurrency}`)
+          .onChange(async (val) => {
+            const realVal = parseInt(val);
+            this.plugin.settings.concurrency = realVal;
+            await this.plugin.saveSettings();
+          });
+      });
+
     //////////////////////////////////////////////////
     // below for general chooser (part 1/2)
     //////////////////////////////////////////////////

src/sync.ts

@@ -1,9 +1,17 @@
-import { TAbstractFile, TFile, TFolder, Vault } from "obsidian";
-import type {
+import {
+  TAbstractFile,
+  TFile,
+  TFolder,
+  Vault,
+  requireApiVersion,
+} from "obsidian";
+import PQueue from "p-queue";
+import {
   RemoteItem,
   SUPPORTED_SERVICES_TYPE,
   DecisionType,
   FileOrFolderMixedState,
+  API_VER_STAT_FOLDER,
 } from "./baseTypes";
 import {
   decryptBase32ToString,
@@ -293,7 +301,7 @@ const ensembleMixedStates = async (
       r = {
         key: entry.path,
         existLocal: true,
-        mtimeLocal: entry.stat.mtime,
+        mtimeLocal: Math.max(entry.stat.mtime, entry.stat.ctime),
         sizeLocal: entry.stat.size,
       };
     } else if (entry instanceof TFolder) {
@@ -505,9 +513,10 @@ const assignOperationToFileInplace = (
   throw Error(`no decision for ${JSON.stringify(r)}`);
 };
 
-const assignOperationToFolderInplace = (
+const assignOperationToFolderInplace = async (
   origRecord: FileOrFolderMixedState,
   keptFolder: Set<string>,
+  vault: Vault,
   password: string = ""
 ) => {
   let r = origRecord;
@@ -523,10 +532,42 @@ const assignOperationToFolderInplace = (
 
     if (r.deltimeLocal !== undefined || r.deltimeRemote !== undefined) {
       // it has some deletion "commands"
-      if (
-        r.deltimeLocal !== undefined &&
-        r.deltimeLocal >= (r.deltimeRemote !== undefined ? r.deltimeRemote : -1)
-      ) {
+
+      const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
+      const deltimeRemote =
+        r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
+
+      // if it was created after deletion, we should keep it as is
+      if (requireApiVersion(API_VER_STAT_FOLDER)) {
+        if (r.existLocal) {
+          try {
+            const { ctime, mtime } = await vault.adapter.stat(r.key);
+            const cmtime = Math.max(ctime, mtime);
+            if (
+              cmtime > 0 &&
+              cmtime >= deltimeLocal &&
+              cmtime >= deltimeRemote
+            ) {
+              keptFolder.add(getParentFolder(r.key));
+              if (r.existLocal && r.existRemote) {
+                r.decision = "skipFolder";
+                r.decisionBranch = 14;
+              } else if (r.existLocal || r.existRemote) {
+                r.decision = "createFolder";
+                r.decisionBranch = 15;
+              } else {
+                throw Error(
+                  `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
+                );
+              }
+            }
+          } catch (error) {
+            // pass
+          }
+        }
+      }
+
+      if (deltimeLocal > 0 && deltimeLocal > deltimeRemote) {
         r.decision = "uploadLocalDelHistToRemoteFolder";
         r.decisionBranch = 8;
       } else {
@@ -585,6 +626,7 @@ export const getSyncPlan = async (
   remoteDeleteHistory: DeletionOnRemote[],
   localDeleteHistory: FileFolderHistoryRecord[],
   remoteType: SUPPORTED_SERVICES_TYPE,
+  vault: Vault,
   password: string = ""
 ) => {
   const mixedStates = await ensembleMixedStates(
@@ -609,7 +651,7 @@ export const getSyncPlan = async (
       // decide some folders
       // because the keys are sorted by length
       // so all the children must have been shown up before in the iteration
-      assignOperationToFolderInplace(val, keptFolder, password);
+      await assignOperationToFolderInplace(val, keptFolder, vault, password);
     } else {
       // get all operations of files
       // and at the same time get some helper info for folders
@@ -850,10 +892,10 @@ export const doActualSync = async (
   deletions: DeletionOnRemote[],
   localDeleteFunc: any,
   password: string = "",
+  concurrency: number = 1,
   callbackSyncProcess?: any
 ) => {
   const mixedStates = syncPlan.mixedStates;
-  let i = 0;
   const totalCount = sortedKeys.length || 0;
 
   log.debug(`start syncing extra data firstly`);
@@ -866,41 +908,145 @@ export const doActualSync = async (
   );
   log.debug(`finish syncing extra data firstly`);
 
-  for (let i = 0; i < sortedKeys.length; ++i) {
-    const key = sortedKeys[i];
-    const val = mixedStates[key];
+  log.debug(`concurrency === ${concurrency}`);
+  if (concurrency === 1) {
+    // run everything in sequence
+    // good old way
+    for (let i = 0; i < sortedKeys.length; ++i) {
+      const key = sortedKeys[i];
+      const val = mixedStates[key];
 
-    log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
+      log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
 
-    if (callbackSyncProcess !== undefined) {
-      await callbackSyncProcess(i, totalCount, key, val.decision);
+      if (callbackSyncProcess !== undefined) {
+        await callbackSyncProcess(i, totalCount, key, val.decision);
+      }
+
+      await dispatchOperationToActual(
+        key,
+        vaultRandomID,
+        val,
+        client,
+        db,
+        vault,
+        localDeleteFunc,
+        password
+      );
+      log.debug(`finished ${key}`);
+    }
+  } else {
+    let realCounter = 0;
+
+    log.debug(
+      `1. create all folders from shadowest to deepest, also check undefined decision`
+    );
+    for (let i = sortedKeys.length - 1; i >= 0; --i) {
+      const key = sortedKeys[i];
+      const val = mixedStates[key];
+
+      if (
+        val.decision === undefined ||
+        val.decision === "skipFolder" ||
+        val.decision === "createFolder"
+      ) {
+        log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
+
+        if (callbackSyncProcess !== undefined) {
+          await callbackSyncProcess(realCounter, totalCount, key, val.decision);
+        }
+        realCounter += 1;
+
+        await dispatchOperationToActual(
+          key,
+          vaultRandomID,
+          val,
+          client,
+          db,
+          vault,
+          localDeleteFunc,
+          password
+        );
+        log.debug(`finished ${key}`);
+      }
     }
 
-    await dispatchOperationToActual(
-      key,
-      vaultRandomID,
-      val,
-      client,
-      db,
-      vault,
-      localDeleteFunc,
-      password
-    );
-    log.debug(`finished ${key}`);
+    log.debug(`2. delete files and folders from deepest to shadowest`);
+    for (let i = 0; i < sortedKeys.length; ++i) {
+      const key = sortedKeys[i];
+      const val = mixedStates[key];
+      if (
+        val.decision === "uploadLocalDelHistToRemoteFolder" ||
+        val.decision === "keepRemoteDelHistFolder"
+      ) {
+        log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
 
-    // await Promise.all(
-    //   Object.entries(mixedStates).map(async ([k, v]) =>
-    //     dispatchOperationToActual(
-    //       k as string,
-    //       vaultRandomID,
-    //       v as FileOrFolderMixedState,
-    //       client,
-    //       db,
-    //       vault,
-    //       localDeleteFunc,
-    //       password
-    //     )
-    //   )
-    // );
+        if (callbackSyncProcess !== undefined) {
+          await callbackSyncProcess(realCounter, totalCount, key, val.decision);
+        }
+        realCounter += 1;
+
+        await dispatchOperationToActual(
+          key,
+          vaultRandomID,
+          val,
+          client,
+          db,
+          vault,
+          localDeleteFunc,
+          password
+        );
+        log.debug(`finished ${key}`);
+      }
+    }
+
+    log.debug(
+      `3. upload or download files in parallel, with the desired concurrency=${concurrency}`
+    );
+    const queue = new PQueue({ concurrency: concurrency, autoStart: true });
+
+    // const commands: any[] = [];
+
+    for (let i = 0; i < sortedKeys.length; ++i) {
+      const key = sortedKeys[i];
+      const val = mixedStates[key];
+      if (
+        val.decision === "skipUploading" ||
+        val.decision === "uploadLocalDelHistToRemote" ||
+        val.decision === "keepRemoteDelHist" ||
+        val.decision === "uploadLocalToRemote" ||
+        val.decision === "downloadRemoteToLocal"
+      ) {
+        const fn = async () => {
+          log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
+
+          if (callbackSyncProcess !== undefined) {
+            await callbackSyncProcess(
+              realCounter,
+              totalCount,
+              key,
+              val.decision
+            );
+
+            realCounter += 1;
+          }
+
+          await dispatchOperationToActual(
+            key,
+            vaultRandomID,
+            val,
+            client,
+            db,
+            vault,
+            localDeleteFunc,
+            password
+          );
+
+          log.debug(`finished ${key}`);
+        };
+        queue.add(fn);
+      }
+    }
+
+    await queue.onIdle();
   }
 };

src/syncAlgoV2Notice.ts

@@ -60,6 +60,7 @@ export class SyncAlgoV2Modal extends Modal {
       log.info("agree to use the new algorithm");
       this.plugin.saveAgreeToUseNewSyncAlgorithm();
       this.plugin.enableAutoSyncIfSet();
+      this.plugin.enableInitSyncIfSet();
     } else {
       log.info("do not agree to use the new algorithm");
       this.plugin.unload();

versions.json

@@ -1,3 +1,4 @@
 {
-  "0.3.1": "0.12.15"
+  "0.3.2": "0.12.15",
+  "0.3.4": "0.13.21"
 }