0.2.12

README.md

@@ -23,14 +23,13 @@ As of Jan 2022, the plugin is considered in BETA stage. **DO NOT USE IT for any
   - Webdav
 - **Obsidiain Mobile supported.** Vaults can be synced across mobile and desktop devices with the cloud service as the "broker".
 - **[End-to-end encryption](./docs/encryption.md) supported.** Files would be encrypted using openssl format before being sent to the cloud **if** user specify a password.
-- **Scheduled auto sync supported.**
+- **Scheduled auto sync supported.** Manual sync is also supported, of course.
 - **[Minimal Intrusive](./docs/minimal_intrusive_design.md).**
 - **Fully open source under [Apache-2.0 License](./LICENSE).**
 - **[Sync Algorithm open](./docs/sync_algorithm.md) for discussion.**
 
 ## Limitations
 
-- **Users have to trigger the sync manually.** This design is intentional because the plugin is in beta, and it's better for users to be exactly aware of the running of this plugin.
 - **"deletion" operation can only be triggered from local device.** It's because of the "[minimal intrusive design](./docs/minimal_intrusive_design.md)". May be changed in the future.
 - **No Conflict resolution. No content-diff-and-patch algorithm.** All files and folders are compared using their local and remote "last modified time" and those with later "last modified time" wins.
 - **Cloud services cost you money.** Always be aware of the costs and pricing.

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "minAppVersion": "0.12.15",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs",
@@ -54,6 +54,7 @@
     "@aws-sdk/lib-storage": "^3.40.1",
     "@aws-sdk/signature-v4-crt": "^3.37.0",
     "@azure/msal-node": "^1.4.0",
+    "@fyears/tsqueue": "^1.0.1",
     "@microsoft/microsoft-graph-client": "^3.0.1",
     "acorn": "^8.5.0",
     "assert": "^2.0.0",

src/baseTypes.ts

@@ -31,6 +31,7 @@ export interface WebdavConfig {
   username: string;
   password: string;
   authType: WebdavAuthType;
+  manualRecursive: boolean;
 }
 
 export interface OnedriveConfig {

src/main.ts

@@ -141,7 +141,7 @@ export default class RemotelySavePlugin extends Plugin {
         () => self.saveSettings()
       );
       const remoteRsp = await client.listFromRemote();
-      // log.info(remoteRsp);
+      log.info(remoteRsp);
 
       getNotice("3/7 Starting to fetch local meta data.");
       this.syncStatus = "getting_local_meta";
@@ -504,6 +504,9 @@ export default class RemotelySavePlugin extends Plugin {
     if (this.settings.onedrive.authority === "") {
       this.settings.onedrive.authority = DEFAULT_SETTINGS.onedrive.authority;
     }
+    if (this.settings.webdav.manualRecursive === undefined) {
+      this.settings.webdav.manualRecursive = false;
+    }
   }
 
   async saveSettings() {

src/remoteForOnedrive.ts

@@ -262,19 +262,31 @@ const fromDriveItemToRemoteItem = (
   // pure english: /drive/root:/Apps/remotely-save/${vaultName}
   // or localized, e.g.: /drive/root:/应用/remotely-save/${vaultName}
   const FIRST_COMMON_PREFIX_REGEX = /^\/drive\/root:\/[^\/]+\/remotely-save\//g;
+  // or the root is absolute path /Livefolders,
+  // e.g.: /Livefolders/应用/remotely-save/${vaultName}
+  const SECOND_COMMON_PREFIX_REGEX = /^\/Livefolders\/[^\/]+\/remotely-save\//g;
 
   // another possibile prefix
-  const SECOND_COMMON_PREFIX_RAW = `/drive/items/`;
+  const THIRD_COMMON_PREFIX_RAW = `/drive/items/`;
 
   const fullPathOriginal = `${x.parentReference.path}/${x.name}`;
   const matchFirstPrefixRes = fullPathOriginal.match(FIRST_COMMON_PREFIX_REGEX);
+  const matchSecondPrefixRes = fullPathOriginal.match(
+    SECOND_COMMON_PREFIX_REGEX
+  );
   if (
     matchFirstPrefixRes !== null &&
     fullPathOriginal.startsWith(`${matchFirstPrefixRes[0]}${vaultName}`)
   ) {
     const foundPrefix = `${matchFirstPrefixRes[0]}${vaultName}`;
     key = fullPathOriginal.substring(foundPrefix.length + 1);
-  } else if (x.parentReference.path.startsWith(SECOND_COMMON_PREFIX_RAW)) {
+  } else if (
+    matchSecondPrefixRes !== null &&
+    fullPathOriginal.startsWith(`${matchSecondPrefixRes[0]}${vaultName}`)
+  ) {
+    const foundPrefix = `${matchSecondPrefixRes[0]}${vaultName}`;
+    key = fullPathOriginal.substring(foundPrefix.length + 1);
+  } else if (x.parentReference.path.startsWith(THIRD_COMMON_PREFIX_RAW)) {
     // it's something like
     // /drive/items/<some_id>!<another_id>:/${vaultName}/<subfolder>
     // with uri encoded!

src/remoteForWebdav.ts

@@ -2,6 +2,9 @@ import { Buffer } from "buffer";
 import { Vault } from "obsidian";
 import type { FileStat, WebDAVClient } from "webdav/web";
 import { AuthType, BufferLike, createClient } from "webdav/web";
+import { Queue } from "@fyears/tsqueue";
+import chunk from "lodash/chunk";
+import flatten from "lodash/flatten";
 import type { RemoteItem, WebdavConfig } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
 import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
@@ -15,6 +18,7 @@ export const DEFAULT_WEBDAV_CONFIG = {
   username: "",
   password: "",
   authType: "basic",
+  manualRecursive: false,
 } as WebdavConfig;
 
 const getWebdavPath = (fileOrFolderPath: string, vaultName: string) => {
@@ -199,14 +203,51 @@ export const listFromRemote = async (
     throw Error("prefix not supported");
   }
   await client.init();
-  const contents = (await client.client.getDirectoryContents(
+
-    `/${client.vaultName}`,
+  let contents = [] as FileStat[];
-    {
+  if (client.webdavConfig.manualRecursive) {
-      deep: true,
+    // the remote doesn't support infinity propfind,
-      details: false /* no need for verbose details here */,
+    // we need to do a bfs here
-      glob: "/**" /* avoid dot files by using glob */,
+    const q = new Queue([`/${client.vaultName}`]);
+    const CHUNK_SIZE = 10;
+    while (q.length > 0) {
+      const itemsToFetch = [];
+      while (q.length > 0) {
+        itemsToFetch.push(q.pop());
+      }
+      const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE);
+      // log.debug(itemsToFetchChunks);
+      const subContents = [] as FileStat[];
+      for (const singleChunk of itemsToFetchChunks) {
+        const r = singleChunk.map((x) => {
+          return client.client.getDirectoryContents(x, {
+            deep: false,
+            details: false /* no need for verbose details here */,
+            glob: "/**" /* avoid dot files by using glob */,
+          }) as Promise<FileStat[]>;
+        });
+        const r2 = flatten(await Promise.all(r));
+        subContents.push(...r2);
+      }
+      for (let i = 0; i < subContents.length; ++i) {
+        const f = subContents[i];
+        contents.push(f);
+        if (f.type === "directory") {
+          q.push(f.filename);
+        }
+      }
     }
-  )) as FileStat[];
+  } else {
+    // the remote supports infinity propfind
+    contents = (await client.client.getDirectoryContents(
+      `/${client.vaultName}`,
+      {
+        deep: true,
+        details: false /* no need for verbose details here */,
+        glob: "/**" /* avoid dot files by using glob */,
+      }
+    )) as FileStat[];
+  }
   return {
     Contents: contents.map((x) =>
       fromWebdavItemToRemoteItem(x, client.vaultName)

src/settings.ts

@@ -872,6 +872,32 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           });
       });
 
+    new Setting(webdavDiv)
+      .setName("server supports infinity propfind or not")
+      .setDesc(
+        "The plugin needs to get all files and folders recursively using probfind. If your webdav server only supports depth='1' (such as NGINX), you need to adjust the setting here, then the plugin consumes more network requests, but better than not working."
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("infinity", "supports depth='infinity'");
+        dropdown.addOption("1", "only supports depth='1'");
+
+        type Depth = "1" | "infinity";
+        dropdown
+          .setValue(
+            this.plugin.settings.webdav.manualRecursive === false
+              ? "infinity"
+              : "1"
+          )
+          .onChange(async (val: Depth) => {
+            if (val === "1") {
+              this.plugin.settings.webdav.manualRecursive = true;
+            } else if (val === "infinity") {
+              this.plugin.settings.webdav.manualRecursive = false;
+            }
+            await this.plugin.saveSettings();
+          });
+      });
+
     new Setting(webdavDiv)
       .setName("check connectivity")
       .setDesc("check connectivity")

src/sync.ts

@@ -498,6 +498,18 @@ const dispatchOperationToActual = async (
     throw Error(`unknown decision in ${JSON.stringify(state)}`);
   } else if (state.decision === "skip") {
     // do nothing
+  } else if (
+    client.serviceType === "onedrive" &&
+    state.size_local === 0 &&
+    !state.key.endsWith("/") &&
+    password === "" &&
+    (state.decision === "upload" || state.decision === "upload_clearhist")
+  ) {
+    // TODO: it's ugly, any other way to deal with empty file for onedrive?
+    // do nothing, skip empty file without encryption
+    // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded.
+    // this branch should be earlier than normal upload / upload_clearhist branches.
+    log.debug(`skip empty file ${state.key} uploading for OneDrive`);
   } else if (state.decision === "download_clearhist") {
     await client.downloadFromRemote(
       state.key,

versions.json

@@ -1,3 +1,3 @@
 {
-  "0.2.11": "0.12.15"
+  "0.2.12": "0.12.15"
 }