0.2.14

README.md

@@ -4,7 +4,9 @@ This is yet another unofficial sync plugin for Obsidian. If you like it or find
 
 [![BuildCI](https://github.com/fyears/remotely-save/actions/workflows/auto-build.yml/badge.svg)](https://github.com/fyears/remotely-save/actions/workflows/auto-build.yml)
 
-[![total downloads auto count)](https://remotely-save.github.io/auto-download-stats/totalDownloads.svg)](https://github.com/fyears/remotely-save/releases)
+[![total downloads auto count](https://remotely-save.github.io/auto-download-stats/totalDownloads.svg)](https://github.com/fyears/remotely-save/releases)
+
+[![downloads of latest version](https://remotely-save.github.io/auto-download-stats/latestVersionDownloads.svg)](https://github.com/fyears/remotely-save/releases)
 
 ## Disclaimer
 
@@ -23,14 +25,13 @@ As of Jan 2022, the plugin is considered in BETA stage. **DO NOT USE IT for any
   - Webdav
 - **Obsidiain Mobile supported.** Vaults can be synced across mobile and desktop devices with the cloud service as the "broker".
 - **[End-to-end encryption](./docs/encryption.md) supported.** Files would be encrypted using openssl format before being sent to the cloud **if** user specify a password.
-- **Scheduled auto sync supported.**
+- **Scheduled auto sync supported.** You can also manually trigger the sync using sidebar ribbon, or using the command from the command palette (or even bind the hot key combination to the command then press the hot key combination).
 - **[Minimal Intrusive](./docs/minimal_intrusive_design.md).**
 - **Fully open source under [Apache-2.0 License](./LICENSE).**
 - **[Sync Algorithm open](./docs/sync_algorithm.md) for discussion.**
 
 ## Limitations
 
-- **Users have to trigger the sync manually.** This design is intentional because the plugin is in beta, and it's better for users to be exactly aware of the running of this plugin.
 - **"deletion" operation can only be triggered from local device.** It's because of the "[minimal intrusive design](./docs/minimal_intrusive_design.md)". May be changed in the future.
 - **No Conflict resolution. No content-diff-and-patch algorithm.** All files and folders are compared using their local and remote "last modified time" and those with later "last modified time" wins.
 - **Cloud services cost you money.** Always be aware of the costs and pricing.
@@ -62,7 +63,7 @@ Additionally, the plugin author may occasionally visit Obsidian official forum a
 - Download and enable this plugin.
 - Enter your information to the settings of this plugin.
 - If you want to enable end-to-end encryption, also set a password in settings. If you do not specify a password, the files and folders are synced in plain, original content to the cloud.
-- Click the new "circle arrow" icon on the ribbon (the left sidebar), **every time** you want to sync your vault between local and remote. (Or, you could configure auto sync in the settings panel (See next chapter).) While syncing, the icon becomes "two half-circle arrows".
+- Click the new "circle arrow" icon on the ribbon (the left sidebar), **every time** you want to sync your vault between local and remote. (Or, you could configure auto sync in the settings panel (See next chapter).) While syncing, the icon becomes "two half-circle arrows". Besides clicking the icon on the sidebar ribbon, you can also activate the corresponding command in the command palette.
 - **Be patient while syncing.** Especially in the first-time sync.
 
 ### Dropbox

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.2.11",
+  "version": "0.2.14",
   "minAppVersion": "0.12.15",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.2.11",
+  "version": "0.2.14",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs",
@@ -54,6 +54,7 @@
     "@aws-sdk/lib-storage": "^3.40.1",
     "@aws-sdk/signature-v4-crt": "^3.37.0",
     "@azure/msal-node": "^1.4.0",
+    "@fyears/tsqueue": "^1.0.1",
     "@microsoft/microsoft-graph-client": "^3.0.1",
     "acorn": "^8.5.0",
     "assert": "^2.0.0",

src/baseTypes.ts

@@ -31,6 +31,7 @@ export interface WebdavConfig {
   username: string;
   password: string;
   authType: WebdavAuthType;
+  manualRecursive: boolean;
 }
 
 export interface OnedriveConfig {

src/main.ts

@@ -141,7 +141,7 @@ export default class RemotelySavePlugin extends Plugin {
         () => self.saveSettings()
       );
       const remoteRsp = await client.listFromRemote();
-      // log.info(remoteRsp);
+      log.info(remoteRsp);
 
       getNotice("3/7 Starting to fetch local meta data.");
       this.syncStatus = "getting_local_meta";
@@ -465,6 +465,15 @@ export default class RemotelySavePlugin extends Plugin {
       async () => this.syncRun("manual")
     );
 
+    this.addCommand({
+      id: "start-sync",
+      name: "start sync",
+      icon: iconNameSyncWait,
+      callback: async () => {
+        this.syncRun("manual");
+      },
+    });
+
     this.addSettingTab(new RemotelySaveSettingTab(this.app, this));
 
     // this.registerDomEvent(document, "click", (evt: MouseEvent) => {
@@ -504,6 +513,9 @@ export default class RemotelySavePlugin extends Plugin {
     if (this.settings.onedrive.authority === "") {
       this.settings.onedrive.authority = DEFAULT_SETTINGS.onedrive.authority;
     }
+    if (this.settings.webdav.manualRecursive === undefined) {
+      this.settings.webdav.manualRecursive = false;
+    }
   }
 
   async saveSettings() {

src/remoteForOnedrive.ts

@@ -1,26 +1,19 @@
 import { CryptoProvider, PublicClientApplication } from "@azure/msal-node";
-import {
-  AuthenticationProvider,
-  Client,
-  FileUpload,
-  LargeFileUploadSession,
-  LargeFileUploadTask,
-  LargeFileUploadTaskOptions,
-  Range,
-  UploadEventHandlers,
-  UploadResult,
-} from "@microsoft/microsoft-graph-client";
-import type { DriveItem, User } from "@microsoft/microsoft-graph-types";
+import { AuthenticationProvider } from "@microsoft/microsoft-graph-client";
+import type {
+  DriveItem,
+  UploadSession,
+  User,
+} from "@microsoft/microsoft-graph-types";
 import cloneDeep from "lodash/cloneDeep";
+import * as origLog from "loglevel";
 import { request, Vault } from "obsidian";
-import * as path from "path";
 import {
-  DropboxConfig,
+  COMMAND_CALLBACK_ONEDRIVE,
   OAUTH2_FORCE_EXPIRE_MILLISECONDS,
   OnedriveConfig,
   RemoteItem,
 } from "./baseTypes";
-import { COMMAND_CALLBACK_ONEDRIVE } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
 import {
   getRandomArrayBuffer,
@@ -28,7 +21,6 @@ import {
   mkdirpInVault,
 } from "./misc";
 
-import * as origLog from "loglevel";
 const log = origLog.getLogger("rs-default");
 
 const SCOPES = ["User.Read", "Files.ReadWrite.AppFolder", "offline_access"];
@@ -262,19 +254,31 @@ const fromDriveItemToRemoteItem = (
   // pure english: /drive/root:/Apps/remotely-save/${vaultName}
   // or localized, e.g.: /drive/root:/应用/remotely-save/${vaultName}
   const FIRST_COMMON_PREFIX_REGEX = /^\/drive\/root:\/[^\/]+\/remotely-save\//g;
+  // or the root is absolute path /Livefolders,
+  // e.g.: /Livefolders/应用/remotely-save/${vaultName}
+  const SECOND_COMMON_PREFIX_REGEX = /^\/Livefolders\/[^\/]+\/remotely-save\//g;
 
   // another possibile prefix
-  const SECOND_COMMON_PREFIX_RAW = `/drive/items/`;
+  const THIRD_COMMON_PREFIX_RAW = `/drive/items/`;
 
   const fullPathOriginal = `${x.parentReference.path}/${x.name}`;
   const matchFirstPrefixRes = fullPathOriginal.match(FIRST_COMMON_PREFIX_REGEX);
+  const matchSecondPrefixRes = fullPathOriginal.match(
+    SECOND_COMMON_PREFIX_REGEX
+  );
   if (
     matchFirstPrefixRes !== null &&
     fullPathOriginal.startsWith(`${matchFirstPrefixRes[0]}${vaultName}`)
   ) {
     const foundPrefix = `${matchFirstPrefixRes[0]}${vaultName}`;
     key = fullPathOriginal.substring(foundPrefix.length + 1);
-  } else if (x.parentReference.path.startsWith(SECOND_COMMON_PREFIX_RAW)) {
+  } else if (
+    matchSecondPrefixRes !== null &&
+    fullPathOriginal.startsWith(`${matchSecondPrefixRes[0]}${vaultName}`)
+  ) {
+    const foundPrefix = `${matchSecondPrefixRes[0]}${vaultName}`;
+    key = fullPathOriginal.substring(foundPrefix.length + 1);
+  } else if (x.parentReference.path.startsWith(THIRD_COMMON_PREFIX_RAW)) {
     // it's something like
     // /drive/items/<some_id>!<another_id>:/${vaultName}/<subfolder>
     // with uri encoded!
@@ -309,7 +313,7 @@ const fromDriveItemToRemoteItem = (
     lastModified: Date.parse(x.fileSystemInfo.lastModifiedDateTime),
     size: isFolder ? 0 : x.size,
     remoteType: "onedrive",
-    etag: x.eTag || x.cTag || "",
+    etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes
   };
 };
 
@@ -364,8 +368,8 @@ class MyAuthProvider implements AuthenticationProvider {
 export class WrappedOnedriveClient {
   onedriveConfig: OnedriveConfig;
   vaultName: string;
-  client: Client;
   vaultFolderExists: boolean;
+  authGetter: MyAuthProvider;
   saveUpdatedConfigFunc: () => Promise<any>;
   constructor(
     onedriveConfig: OnedriveConfig,
@@ -376,9 +380,7 @@ export class WrappedOnedriveClient {
     this.vaultName = vaultName;
     this.vaultFolderExists = false;
     this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
-    this.client = Client.initWithMiddleware({
-      authProvider: new MyAuthProvider(onedriveConfig, saveUpdatedConfigFunc),
-    });
+    this.authGetter = new MyAuthProvider(onedriveConfig, saveUpdatedConfigFunc);
   }
 
   init = async () => {
@@ -395,14 +397,14 @@ export class WrappedOnedriveClient {
     if (this.vaultFolderExists) {
       // log.info(`already checked, /${this.vaultName} exist before`)
     } else {
-      const k = await this.client.api("/drive/special/approot/children").get();
-      // log.info(k);
+      const k = await this.getJson("/drive/special/approot/children");
+      log.debug(k);
       this.vaultFolderExists =
         (k.value as DriveItem[]).filter((x) => x.name === this.vaultName)
           .length > 0;
       if (!this.vaultFolderExists) {
         log.info(`remote does not have folder /${this.vaultName}`);
-        await this.client.api("/drive/special/approot/children").post({
+        await this.postJson("/drive/special/approot/children", {
           name: `${this.vaultName}`,
           folder: {},
           "@microsoft.graph.conflictBehavior": "replace",
@@ -414,6 +416,132 @@ export class WrappedOnedriveClient {
       }
     }
   };
+
+  buildUrl = (pathFragOrig: string) => {
+    const API_PREFIX = "https://graph.microsoft.com/v1.0";
+    let theUrl = "";
+    if (
+      pathFragOrig.startsWith("http://") ||
+      pathFragOrig.startsWith("https://")
+    ) {
+      theUrl = pathFragOrig;
+    } else {
+      const pathFrag = encodeURI(pathFragOrig);
+      theUrl = `${API_PREFIX}${pathFrag}`;
+    }
+    return theUrl;
+  };
+
+  getJson = async (pathFragOrig: string) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`getJson, theUrl=${theUrl}`);
+    return JSON.parse(
+      await request({
+        url: theUrl,
+        method: "GET",
+        contentType: "application/json",
+        headers: {
+          Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+        },
+      })
+    );
+  };
+
+  postJson = async (pathFragOrig: string, payload: any) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`postJson, theUrl=${theUrl}`);
+    return JSON.parse(
+      await request({
+        url: theUrl,
+        method: "POST",
+        contentType: "application/json",
+        body: JSON.stringify(payload),
+        headers: {
+          Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+        },
+      })
+    );
+  };
+
+  patchJson = async (pathFragOrig: string, payload: any) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`patchJson, theUrl=${theUrl}`);
+    return JSON.parse(
+      await request({
+        url: theUrl,
+        method: "PATCH",
+        contentType: "application/json",
+        body: JSON.stringify(payload),
+        headers: {
+          Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+        },
+      })
+    );
+  };
+
+  deleteJson = async (pathFragOrig: string) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`deleteJson, theUrl=${theUrl}`);
+    // TODO: delete does not have response, so Obsidian request may have error
+    // currently downgraded to fetch()!
+    await fetch(theUrl, {
+      method: "DELETE",
+      headers: {
+        Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+      },
+    });
+  };
+
+  putArrayBuffer = async (pathFragOrig: string, payload: ArrayBuffer) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`putArrayBuffer, theUrl=${theUrl}`);
+    // TODO: Obsidian doesn't support ArrayBuffer
+    // currently downgraded to fetch()!
+    await fetch(theUrl, {
+      method: "PUT",
+      body: payload,
+      headers: {
+        Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+      },
+    });
+  };
+
+  /**
+   * A specialized function to upload large files by parts
+   * @param pathFragOrig
+   * @param payload
+   * @param rangeMin
+   * @param rangeEnd the end, exclusive
+   * @param size
+   */
+  putUint8ArrayByRange = async (
+    pathFragOrig: string,
+    payload: Uint8Array,
+    rangeStart: number,
+    rangeEnd: number,
+    size: number
+  ) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(
+      `putUint8ArrayByRange, theUrl=${theUrl}, range=${rangeStart}-${
+        rangeEnd - 1
+      }, len=${rangeEnd - rangeStart}, size=${size}`
+    );
+    // TODO: Obsidian doesn't support ArrayBuffer
+    // currently downgraded to fetch()!
+    // AND, NO AUTH HEADER here!
+    const res = await fetch(theUrl, {
+      method: "PUT",
+      body: payload.subarray(rangeStart, rangeEnd),
+      headers: {
+        "Content-Length": `${rangeEnd - rangeStart}`,
+        "Content-Range": `bytes ${rangeStart}-${rangeEnd - 1}/${size}`,
+        "Content-Type": "application/octet-stream",
+      },
+    });
+
+    return res.json() as DriveItem | UploadSession;
+  };
 }
 
 export const getOnedriveClient = (
@@ -445,13 +573,14 @@ export const listFromRemote = async (
 
   const NEXT_LINK_KEY = "@odata.nextLink";
   const DELTA_LINK_KEY = "@odata.deltaLink";
-  let res = await client.client
-    .api(`/drive/special/approot:/${client.vaultName}:/delta`)
-    .get();
+
+  let res = await client.getJson(
+    `/drive/special/approot:/${client.vaultName}:/delta`
+  );
   let driveItems = res.value as DriveItem[];
 
   while (NEXT_LINK_KEY in res) {
-    res = await client.client.api(res[NEXT_LINK_KEY]).get();
+    res = await client.getJson(res[NEXT_LINK_KEY]);
     driveItems.push(...cloneDeep(res.value as DriveItem[]));
   }
 
@@ -461,16 +590,11 @@ export const listFromRemote = async (
     await client.saveUpdatedConfigFunc();
   }
 
-  driveItems = driveItems.map((x) => {
-    const y = cloneDeep(x);
-    y.parentReference.path = y.parentReference.path.replace("/Apps", "/应用");
-    return y;
-  });
-
   // unify everything to RemoteItem
   const unifiedContents = driveItems
     .map((x) => fromDriveItemToRemoteItem(x, client.vaultName))
     .filter((x) => x.key !== "/");
+
   return {
     Contents: unifiedContents,
   };
@@ -483,10 +607,9 @@ export const getRemoteMeta = async (
   await client.init();
   const remotePath = getOnedrivePath(fileOrFolderPath, client.vaultName);
   // log.info(`remotePath=${remotePath}`);
-  const rsp = await client.client
-    .api(remotePath)
-    .select("cTag,eTag,fileSystemInfo,folder,file,name,parentReference,size")
-    .get();
+  const rsp = await client.getJson(
+    `${remotePath}?$select=cTag,eTag,fileSystemInfo,folder,file,name,parentReference,size`
+  );
   // log.info(rsp);
   const driveItem = rsp as DriveItem;
   const res = fromDriveItemToRemoteItem(driveItem, client.vaultName);
@@ -510,7 +633,7 @@ export const uploadToRemote = async (
     uploadFile = remoteEncryptedKey;
   }
   uploadFile = getOnedrivePath(uploadFile, client.vaultName);
-  // log.info(`uploadFile=${uploadFile}`);
+  log.debug(`uploadFile=${uploadFile}`);
 
   const isFolder = fileOrFolderPath.endsWith("/");
 
@@ -525,7 +648,7 @@ export const uploadToRemote = async (
       } else {
         // https://stackoverflow.com/questions/56479865/creating-nested-folders-in-one-go-onedrive-api
         // use PATCH to create folder recursively!!!
-        await client.client.api(uploadFile).patch({
+        await client.patchJson(uploadFile, {
           folder: {},
           "@microsoft.graph.conflictBehavior": "replace",
         });
@@ -545,39 +668,13 @@ export const uploadToRemote = async (
         password
       );
 
-      const uploadSession: LargeFileUploadSession =
-        await LargeFileUploadTask.createUploadSession(
-          client.client,
-          `https://graph.microsoft.com/v1.0/me${encodeURIComponent(
-            uploadFile
-          )}:/createUploadSession`,
-          {
-            item: {
-              "@microsoft.graph.conflictBehavior": "replace",
-            },
-          }
-        );
-      const task = new LargeFileUploadTask(
-        client.client,
-        new FileUpload(
-          arrBufRandom,
-          path.posix.basename(uploadFile),
-          arrBufRandom.byteLength
-        ),
-        uploadSession,
-        {
-          rangeSize: 1024 * 1024,
-          uploadEventHandlers: {
-            progress: (range?: Range) => {
-              // Handle progress event
-              // log.info(
-              //   `uploading ${range.minValue}-${range.maxValue} of ${fileOrFolderPath}`
-              // );
-            },
-          } as UploadEventHandlers,
-        } as LargeFileUploadTaskOptions
+      // an encrypted folder is always small, we just use put here
+      await client.putArrayBuffer(
+        `${uploadFile}:/content?${new URLSearchParams({
+          "@microsoft.graph.conflictBehavior": "replace",
+        })}`,
+        arrBufRandom
       );
-      const uploadResult: UploadResult = await task.upload();
       // log.info(uploadResult)
       const res = await getRemoteMeta(client, uploadFile);
       return res;
@@ -593,48 +690,42 @@ export const uploadToRemote = async (
 
     // no need to create parent folders firstly, cool!
 
-    // we need to customize the special root folder,
-    // so use LargeFileUploadTask instead of OneDriveLargeFileUploadTask
-    const progress = (range?: Range) => {
-      // Handle progress event
-      // log.info(
-      //   `uploading ${range.minValue}-${range.maxValue} of ${fileOrFolderPath}`
-      // );
-    };
-    const uploadEventHandlers: UploadEventHandlers = {
-      progress: progress,
-    };
-    const options: LargeFileUploadTaskOptions = {
-      rangeSize: 1024 * 1024,
-      uploadEventHandlers: uploadEventHandlers,
-    };
-    const payload = {
-      item: {
-        "@microsoft.graph.conflictBehavior": "replace",
-      },
-    };
+    // upload large files!
+    // ref: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession?view=odsp-graph-online
+
+    // 1. create uploadSession
     // uploadFile already starts with /drive/special/approot:/${vaultName}
-    const uploadSession: LargeFileUploadSession =
-      await LargeFileUploadTask.createUploadSession(
-        client.client,
-        `https://graph.microsoft.com/v1.0/me${encodeURIComponent(
-          uploadFile
-        )}:/createUploadSession`,
-        payload
+    const s: UploadSession = await client.postJson(
+      `${uploadFile}:/createUploadSession`,
+      {
+        item: {
+          "@microsoft.graph.conflictBehavior": "replace",
+        },
+      }
+    );
+    const uploadUrl = s.uploadUrl;
+    log.debug("uploadSession = ");
+    log.debug(s);
+
+    // 2. upload by ranges
+    // convert to uint8
+    const uint8 = new Uint8Array(remoteContent);
+    // hard code range size
+    const MIN_UNIT = 327680; // bytes in msft doc, about 0.32768 MB
+    const RANGE_SIZE = MIN_UNIT * 20; // about 6.5536 MB
+    // upload the ranges one by one
+    let rangeStart = 0;
+    while (rangeStart < uint8.byteLength) {
+      await client.putUint8ArrayByRange(
+        uploadUrl,
+        uint8,
+        rangeStart,
+        Math.min(rangeStart + RANGE_SIZE, uint8.byteLength),
+        uint8.byteLength
       );
-    const fileObject = new FileUpload(
-      remoteContent,
-      path.posix.basename(uploadFile),
-      remoteContent.byteLength
-    );
-    const task = new LargeFileUploadTask(
-      client.client,
-      fileObject,
-      uploadSession,
-      options
-    );
-    const uploadResult: UploadResult = await task.upload();
-    // log.info(uploadResult)
+      rangeStart += RANGE_SIZE;
+    }
+
     const res = await getRemoteMeta(client, uploadFile);
     return res;
   }
@@ -646,10 +737,9 @@ const downloadFromRemoteRaw = async (
 ): Promise<ArrayBuffer> => {
   await client.init();
   const key = getOnedrivePath(fileOrFolderPath, client.vaultName);
-  const rsp = await client.client
-    .api(key)
-    .select("@microsoft.graph.downloadUrl")
-    .get();
+  const rsp = await client.getJson(
+    `${key}?$select=@microsoft.graph.downloadUrl`
+  );
   const downloadUrl: string = rsp["@microsoft.graph.downloadUrl"];
   const content = await (await fetch(downloadUrl)).arrayBuffer();
   return content;
@@ -705,7 +795,7 @@ export const deleteFromRemote = async (
   remoteFileName = getOnedrivePath(remoteFileName, client.vaultName);
 
   await client.init();
-  await client.client.api(remoteFileName).delete();
+  await client.deleteJson(remoteFileName);
 };
 
 export const checkConnectivity = async (client: WrappedOnedriveClient) => {
@@ -719,7 +809,7 @@ export const checkConnectivity = async (client: WrappedOnedriveClient) => {
 
 export const getUserDisplayName = async (client: WrappedOnedriveClient) => {
   await client.init();
-  const res: User = await client.client.api("/me").select("displayName").get();
+  const res: User = await client.getJson("/me?$select=displayName");
   return res.displayName || "<unknown display name>";
 };
 
@@ -732,7 +822,7 @@ export const getUserDisplayName = async (client: WrappedOnedriveClient) => {
  */
 // export const revokeAuth = async (client: WrappedOnedriveClient) => {
 //   await client.init();
-//   await client.client.api('/me/revokeSignInSessions').post(undefined);
+//   await client.postJson('/me/revokeSignInSessions', {});
 // };
 
 export const getRevokeAddr = async () => {

src/remoteForWebdav.ts

@@ -2,6 +2,9 @@ import { Buffer } from "buffer";
 import { Vault } from "obsidian";
 import type { FileStat, WebDAVClient } from "webdav/web";
 import { AuthType, BufferLike, createClient } from "webdav/web";
+import { Queue } from "@fyears/tsqueue";
+import chunk from "lodash/chunk";
+import flatten from "lodash/flatten";
 import type { RemoteItem, WebdavConfig } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
 import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
@@ -15,6 +18,7 @@ export const DEFAULT_WEBDAV_CONFIG = {
   username: "",
   password: "",
   authType: "basic",
+  manualRecursive: false,
 } as WebdavConfig;
 
 const getWebdavPath = (fileOrFolderPath: string, vaultName: string) => {
@@ -199,14 +203,51 @@ export const listFromRemote = async (
     throw Error("prefix not supported");
   }
   await client.init();
-  const contents = (await client.client.getDirectoryContents(
-    `/${client.vaultName}`,
-    {
-      deep: true,
-      details: false /* no need for verbose details here */,
-      glob: "/**" /* avoid dot files by using glob */,
+
+  let contents = [] as FileStat[];
+  if (client.webdavConfig.manualRecursive) {
+    // the remote doesn't support infinity propfind,
+    // we need to do a bfs here
+    const q = new Queue([`/${client.vaultName}`]);
+    const CHUNK_SIZE = 10;
+    while (q.length > 0) {
+      const itemsToFetch = [];
+      while (q.length > 0) {
+        itemsToFetch.push(q.pop());
+      }
+      const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE);
+      // log.debug(itemsToFetchChunks);
+      const subContents = [] as FileStat[];
+      for (const singleChunk of itemsToFetchChunks) {
+        const r = singleChunk.map((x) => {
+          return client.client.getDirectoryContents(x, {
+            deep: false,
+            details: false /* no need for verbose details here */,
+            glob: "/**" /* avoid dot files by using glob */,
+          }) as Promise<FileStat[]>;
+        });
+        const r2 = flatten(await Promise.all(r));
+        subContents.push(...r2);
+      }
+      for (let i = 0; i < subContents.length; ++i) {
+        const f = subContents[i];
+        contents.push(f);
+        if (f.type === "directory") {
+          q.push(f.filename);
+        }
+      }
     }
-  )) as FileStat[];
+  } else {
+    // the remote supports infinity propfind
+    contents = (await client.client.getDirectoryContents(
+      `/${client.vaultName}`,
+      {
+        deep: true,
+        details: false /* no need for verbose details here */,
+        glob: "/**" /* avoid dot files by using glob */,
+      }
+    )) as FileStat[];
+  }
   return {
     Contents: contents.map((x) =>
       fromWebdavItemToRemoteItem(x, client.vaultName)

src/settings.ts

@@ -872,6 +872,32 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           });
       });
 
+    new Setting(webdavDiv)
+      .setName("server supports infinity propfind or not")
+      .setDesc(
+        "The plugin needs to get all files and folders recursively using probfind. If your webdav server only supports depth='1' (such as NGINX), you need to adjust the setting here, then the plugin consumes more network requests, but better than not working."
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("infinity", "supports depth='infinity'");
+        dropdown.addOption("1", "only supports depth='1'");
+
+        type Depth = "1" | "infinity";
+        dropdown
+          .setValue(
+            this.plugin.settings.webdav.manualRecursive === false
+              ? "infinity"
+              : "1"
+          )
+          .onChange(async (val: Depth) => {
+            if (val === "1") {
+              this.plugin.settings.webdav.manualRecursive = true;
+            } else if (val === "infinity") {
+              this.plugin.settings.webdav.manualRecursive = false;
+            }
+            await this.plugin.saveSettings();
+          });
+      });
+
     new Setting(webdavDiv)
       .setName("check connectivity")
       .setDesc("check connectivity")

src/sync.ts

@@ -498,6 +498,18 @@ const dispatchOperationToActual = async (
     throw Error(`unknown decision in ${JSON.stringify(state)}`);
   } else if (state.decision === "skip") {
     // do nothing
+  } else if (
+    client.serviceType === "onedrive" &&
+    state.size_local === 0 &&
+    !state.key.endsWith("/") &&
+    password === "" &&
+    (state.decision === "upload" || state.decision === "upload_clearhist")
+  ) {
+    // TODO: it's ugly, any other way to deal with empty file for onedrive?
+    // do nothing, skip empty file without encryption
+    // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded.
+    // this branch should be earlier than normal upload / upload_clearhist branches.
+    log.debug(`skip empty file ${state.key} uploading for OneDrive`);
   } else if (state.decision === "download_clearhist") {
     await client.downloadFromRemote(
       state.key,

versions.json

@@ -1,3 +1,3 @@
 {
-  "0.2.11": "0.12.15"
+  "0.2.14": "0.12.15"
 }