0.2.14

README.md

@@ -4,7 +4,9 @@ This is yet another unofficial sync plugin for Obsidian. If you like it or find
 
 [![BuildCI](https://github.com/fyears/remotely-save/actions/workflows/auto-build.yml/badge.svg)](https://github.com/fyears/remotely-save/actions/workflows/auto-build.yml)
 
-[![total downloads auto count)](https://remotely-save.github.io/auto-download-stats/totalDownloads.svg)](https://github.com/fyears/remotely-save/releases)
+[![total downloads auto count](https://remotely-save.github.io/auto-download-stats/totalDownloads.svg)](https://github.com/fyears/remotely-save/releases)
+
+[![downloads of latest version](https://remotely-save.github.io/auto-download-stats/latestVersionDownloads.svg)](https://github.com/fyears/remotely-save/releases)
 
 ## Disclaimer
 
@@ -23,7 +25,7 @@ As of Jan 2022, the plugin is considered in BETA stage. **DO NOT USE IT for any
   - Webdav
 - **Obsidiain Mobile supported.** Vaults can be synced across mobile and desktop devices with the cloud service as the "broker".
 - **[End-to-end encryption](./docs/encryption.md) supported.** Files would be encrypted using openssl format before being sent to the cloud **if** user specify a password.
-- **Scheduled auto sync supported.** Manual sync is also supported, of course.
+- **Scheduled auto sync supported.** You can also manually trigger the sync using sidebar ribbon, or using the command from the command palette (or even bind the hot key combination to the command then press the hot key combination).
 - **[Minimal Intrusive](./docs/minimal_intrusive_design.md).**
 - **Fully open source under [Apache-2.0 License](./LICENSE).**
 - **[Sync Algorithm open](./docs/sync_algorithm.md) for discussion.**
@@ -61,7 +63,7 @@ Additionally, the plugin author may occasionally visit Obsidian official forum a
 - Download and enable this plugin.
 - Enter your information to the settings of this plugin.
 - If you want to enable end-to-end encryption, also set a password in settings. If you do not specify a password, the files and folders are synced in plain, original content to the cloud.
-- Click the new "circle arrow" icon on the ribbon (the left sidebar), **every time** you want to sync your vault between local and remote. (Or, you could configure auto sync in the settings panel (See next chapter).) While syncing, the icon becomes "two half-circle arrows".
+- Click the new "circle arrow" icon on the ribbon (the left sidebar), **every time** you want to sync your vault between local and remote. (Or, you could configure auto sync in the settings panel (See next chapter).) While syncing, the icon becomes "two half-circle arrows". Besides clicking the icon on the sidebar ribbon, you can also activate the corresponding command in the command palette.
 - **Be patient while syncing.** Especially in the first-time sync.
 
 ### Dropbox

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "minAppVersion": "0.12.15",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs",

src/main.ts

@@ -465,6 +465,15 @@ export default class RemotelySavePlugin extends Plugin {
       async () => this.syncRun("manual")
     );
 
+    this.addCommand({
+      id: "start-sync",
+      name: "start sync",
+      icon: iconNameSyncWait,
+      callback: async () => {
+        this.syncRun("manual");
+      },
+    });
+
     this.addSettingTab(new RemotelySaveSettingTab(this.app, this));
 
     // this.registerDomEvent(document, "click", (evt: MouseEvent) => {

src/remoteForOnedrive.ts

@@ -1,26 +1,19 @@
 import { CryptoProvider, PublicClientApplication } from "@azure/msal-node";
-import {
+import { AuthenticationProvider } from "@microsoft/microsoft-graph-client";
-  AuthenticationProvider,
+import type {
-  Client,
+  DriveItem,
-  FileUpload,
+  UploadSession,
-  LargeFileUploadSession,
+  User,
-  LargeFileUploadTask,
+} from "@microsoft/microsoft-graph-types";
-  LargeFileUploadTaskOptions,
-  Range,
-  UploadEventHandlers,
-  UploadResult,
-} from "@microsoft/microsoft-graph-client";
-import type { DriveItem, User } from "@microsoft/microsoft-graph-types";
 import cloneDeep from "lodash/cloneDeep";
+import * as origLog from "loglevel";
 import { request, Vault } from "obsidian";
-import * as path from "path";
 import {
-  DropboxConfig,
+  COMMAND_CALLBACK_ONEDRIVE,
   OAUTH2_FORCE_EXPIRE_MILLISECONDS,
   OnedriveConfig,
   RemoteItem,
 } from "./baseTypes";
-import { COMMAND_CALLBACK_ONEDRIVE } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
 import {
   getRandomArrayBuffer,
@@ -28,7 +21,6 @@ import {
   mkdirpInVault,
 } from "./misc";
 
-import * as origLog from "loglevel";
 const log = origLog.getLogger("rs-default");
 
 const SCOPES = ["User.Read", "Files.ReadWrite.AppFolder", "offline_access"];
@@ -321,7 +313,7 @@ const fromDriveItemToRemoteItem = (
     lastModified: Date.parse(x.fileSystemInfo.lastModifiedDateTime),
     size: isFolder ? 0 : x.size,
     remoteType: "onedrive",
-    etag: x.eTag || x.cTag || "",
+    etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes
   };
 };
 
@@ -376,8 +368,8 @@ class MyAuthProvider implements AuthenticationProvider {
 export class WrappedOnedriveClient {
   onedriveConfig: OnedriveConfig;
   vaultName: string;
-  client: Client;
   vaultFolderExists: boolean;
+  authGetter: MyAuthProvider;
   saveUpdatedConfigFunc: () => Promise<any>;
   constructor(
     onedriveConfig: OnedriveConfig,
@@ -388,9 +380,7 @@ export class WrappedOnedriveClient {
     this.vaultName = vaultName;
     this.vaultFolderExists = false;
     this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
-    this.client = Client.initWithMiddleware({
+    this.authGetter = new MyAuthProvider(onedriveConfig, saveUpdatedConfigFunc);
-      authProvider: new MyAuthProvider(onedriveConfig, saveUpdatedConfigFunc),
-    });
   }
 
   init = async () => {
@@ -407,14 +397,14 @@ export class WrappedOnedriveClient {
     if (this.vaultFolderExists) {
       // log.info(`already checked, /${this.vaultName} exist before`)
     } else {
-      const k = await this.client.api("/drive/special/approot/children").get();
+      const k = await this.getJson("/drive/special/approot/children");
-      // log.info(k);
+      log.debug(k);
       this.vaultFolderExists =
         (k.value as DriveItem[]).filter((x) => x.name === this.vaultName)
           .length > 0;
       if (!this.vaultFolderExists) {
         log.info(`remote does not have folder /${this.vaultName}`);
-        await this.client.api("/drive/special/approot/children").post({
+        await this.postJson("/drive/special/approot/children", {
           name: `${this.vaultName}`,
           folder: {},
           "@microsoft.graph.conflictBehavior": "replace",
@@ -426,6 +416,132 @@ export class WrappedOnedriveClient {
       }
     }
   };
+
+  buildUrl = (pathFragOrig: string) => {
+    const API_PREFIX = "https://graph.microsoft.com/v1.0";
+    let theUrl = "";
+    if (
+      pathFragOrig.startsWith("http://") ||
+      pathFragOrig.startsWith("https://")
+    ) {
+      theUrl = pathFragOrig;
+    } else {
+      const pathFrag = encodeURI(pathFragOrig);
+      theUrl = `${API_PREFIX}${pathFrag}`;
+    }
+    return theUrl;
+  };
+
+  getJson = async (pathFragOrig: string) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`getJson, theUrl=${theUrl}`);
+    return JSON.parse(
+      await request({
+        url: theUrl,
+        method: "GET",
+        contentType: "application/json",
+        headers: {
+          Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+        },
+      })
+    );
+  };
+
+  postJson = async (pathFragOrig: string, payload: any) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`postJson, theUrl=${theUrl}`);
+    return JSON.parse(
+      await request({
+        url: theUrl,
+        method: "POST",
+        contentType: "application/json",
+        body: JSON.stringify(payload),
+        headers: {
+          Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+        },
+      })
+    );
+  };
+
+  patchJson = async (pathFragOrig: string, payload: any) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`patchJson, theUrl=${theUrl}`);
+    return JSON.parse(
+      await request({
+        url: theUrl,
+        method: "PATCH",
+        contentType: "application/json",
+        body: JSON.stringify(payload),
+        headers: {
+          Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+        },
+      })
+    );
+  };
+
+  deleteJson = async (pathFragOrig: string) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`deleteJson, theUrl=${theUrl}`);
+    // TODO: delete does not have response, so Obsidian request may have error
+    // currently downgraded to fetch()!
+    await fetch(theUrl, {
+      method: "DELETE",
+      headers: {
+        Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+      },
+    });
+  };
+
+  putArrayBuffer = async (pathFragOrig: string, payload: ArrayBuffer) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(`putArrayBuffer, theUrl=${theUrl}`);
+    // TODO: Obsidian doesn't support ArrayBuffer
+    // currently downgraded to fetch()!
+    await fetch(theUrl, {
+      method: "PUT",
+      body: payload,
+      headers: {
+        Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
+      },
+    });
+  };
+
+  /**
+   * A specialized function to upload large files by parts
+   * @param pathFragOrig
+   * @param payload
+   * @param rangeMin
+   * @param rangeEnd the end, exclusive
+   * @param size
+   */
+  putUint8ArrayByRange = async (
+    pathFragOrig: string,
+    payload: Uint8Array,
+    rangeStart: number,
+    rangeEnd: number,
+    size: number
+  ) => {
+    const theUrl = this.buildUrl(pathFragOrig);
+    log.debug(
+      `putUint8ArrayByRange, theUrl=${theUrl}, range=${rangeStart}-${
+        rangeEnd - 1
+      }, len=${rangeEnd - rangeStart}, size=${size}`
+    );
+    // TODO: Obsidian doesn't support ArrayBuffer
+    // currently downgraded to fetch()!
+    // AND, NO AUTH HEADER here!
+    const res = await fetch(theUrl, {
+      method: "PUT",
+      body: payload.subarray(rangeStart, rangeEnd),
+      headers: {
+        "Content-Length": `${rangeEnd - rangeStart}`,
+        "Content-Range": `bytes ${rangeStart}-${rangeEnd - 1}/${size}`,
+        "Content-Type": "application/octet-stream",
+      },
+    });
+
+    return res.json() as DriveItem | UploadSession;
+  };
 }
 
 export const getOnedriveClient = (
@@ -457,13 +573,14 @@ export const listFromRemote = async (
 
   const NEXT_LINK_KEY = "@odata.nextLink";
   const DELTA_LINK_KEY = "@odata.deltaLink";
-  let res = await client.client
+
-    .api(`/drive/special/approot:/${client.vaultName}:/delta`)
+  let res = await client.getJson(
-    .get();
+    `/drive/special/approot:/${client.vaultName}:/delta`
+  );
   let driveItems = res.value as DriveItem[];
 
   while (NEXT_LINK_KEY in res) {
-    res = await client.client.api(res[NEXT_LINK_KEY]).get();
+    res = await client.getJson(res[NEXT_LINK_KEY]);
     driveItems.push(...cloneDeep(res.value as DriveItem[]));
   }
 
@@ -473,16 +590,11 @@ export const listFromRemote = async (
     await client.saveUpdatedConfigFunc();
   }
 
-  driveItems = driveItems.map((x) => {
-    const y = cloneDeep(x);
-    y.parentReference.path = y.parentReference.path.replace("/Apps", "/应用");
-    return y;
-  });
-
   // unify everything to RemoteItem
   const unifiedContents = driveItems
     .map((x) => fromDriveItemToRemoteItem(x, client.vaultName))
     .filter((x) => x.key !== "/");
+
   return {
     Contents: unifiedContents,
   };
@@ -495,10 +607,9 @@ export const getRemoteMeta = async (
   await client.init();
   const remotePath = getOnedrivePath(fileOrFolderPath, client.vaultName);
   // log.info(`remotePath=${remotePath}`);
-  const rsp = await client.client
+  const rsp = await client.getJson(
-    .api(remotePath)
+    `${remotePath}?$select=cTag,eTag,fileSystemInfo,folder,file,name,parentReference,size`
-    .select("cTag,eTag,fileSystemInfo,folder,file,name,parentReference,size")
+  );
-    .get();
   // log.info(rsp);
   const driveItem = rsp as DriveItem;
   const res = fromDriveItemToRemoteItem(driveItem, client.vaultName);
@@ -522,7 +633,7 @@ export const uploadToRemote = async (
     uploadFile = remoteEncryptedKey;
   }
   uploadFile = getOnedrivePath(uploadFile, client.vaultName);
-  // log.info(`uploadFile=${uploadFile}`);
+  log.debug(`uploadFile=${uploadFile}`);
 
   const isFolder = fileOrFolderPath.endsWith("/");
 
@@ -537,7 +648,7 @@ export const uploadToRemote = async (
       } else {
         // https://stackoverflow.com/questions/56479865/creating-nested-folders-in-one-go-onedrive-api
         // use PATCH to create folder recursively!!!
-        await client.client.api(uploadFile).patch({
+        await client.patchJson(uploadFile, {
           folder: {},
           "@microsoft.graph.conflictBehavior": "replace",
         });
@@ -557,39 +668,13 @@ export const uploadToRemote = async (
         password
       );
 
-      const uploadSession: LargeFileUploadSession =
+      // an encrypted folder is always small, we just use put here
-        await LargeFileUploadTask.createUploadSession(
+      await client.putArrayBuffer(
-          client.client,
+        `${uploadFile}:/content?${new URLSearchParams({
-          `https://graph.microsoft.com/v1.0/me${encodeURIComponent(
+          "@microsoft.graph.conflictBehavior": "replace",
-            uploadFile
+        })}`,
-          )}:/createUploadSession`,
+        arrBufRandom
-          {
-            item: {
-              "@microsoft.graph.conflictBehavior": "replace",
-            },
-          }
-        );
-      const task = new LargeFileUploadTask(
-        client.client,
-        new FileUpload(
-          arrBufRandom,
-          path.posix.basename(uploadFile),
-          arrBufRandom.byteLength
-        ),
-        uploadSession,
-        {
-          rangeSize: 1024 * 1024,
-          uploadEventHandlers: {
-            progress: (range?: Range) => {
-              // Handle progress event
-              // log.info(
-              //   `uploading ${range.minValue}-${range.maxValue} of ${fileOrFolderPath}`
-              // );
-            },
-          } as UploadEventHandlers,
-        } as LargeFileUploadTaskOptions
       );
-      const uploadResult: UploadResult = await task.upload();
       // log.info(uploadResult)
       const res = await getRemoteMeta(client, uploadFile);
       return res;
@@ -605,48 +690,42 @@ export const uploadToRemote = async (
 
     // no need to create parent folders firstly, cool!
 
-    // we need to customize the special root folder,
+    // upload large files!
-    // so use LargeFileUploadTask instead of OneDriveLargeFileUploadTask
+    // ref: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession?view=odsp-graph-online
-    const progress = (range?: Range) => {
+
-      // Handle progress event
+    // 1. create uploadSession
-      // log.info(
-      //   `uploading ${range.minValue}-${range.maxValue} of ${fileOrFolderPath}`
-      // );
-    };
-    const uploadEventHandlers: UploadEventHandlers = {
-      progress: progress,
-    };
-    const options: LargeFileUploadTaskOptions = {
-      rangeSize: 1024 * 1024,
-      uploadEventHandlers: uploadEventHandlers,
-    };
-    const payload = {
-      item: {
-        "@microsoft.graph.conflictBehavior": "replace",
-      },
-    };
     // uploadFile already starts with /drive/special/approot:/${vaultName}
-    const uploadSession: LargeFileUploadSession =
+    const s: UploadSession = await client.postJson(
-      await LargeFileUploadTask.createUploadSession(
+      `${uploadFile}:/createUploadSession`,
-        client.client,
+      {
-        `https://graph.microsoft.com/v1.0/me${encodeURIComponent(
+        item: {
-          uploadFile
+          "@microsoft.graph.conflictBehavior": "replace",
-        )}:/createUploadSession`,
+        },
-        payload
+      }
+    );
+    const uploadUrl = s.uploadUrl;
+    log.debug("uploadSession = ");
+    log.debug(s);
+
+    // 2. upload by ranges
+    // convert to uint8
+    const uint8 = new Uint8Array(remoteContent);
+    // hard code range size
+    const MIN_UNIT = 327680; // bytes in msft doc, about 0.32768 MB
+    const RANGE_SIZE = MIN_UNIT * 20; // about 6.5536 MB
+    // upload the ranges one by one
+    let rangeStart = 0;
+    while (rangeStart < uint8.byteLength) {
+      await client.putUint8ArrayByRange(
+        uploadUrl,
+        uint8,
+        rangeStart,
+        Math.min(rangeStart + RANGE_SIZE, uint8.byteLength),
+        uint8.byteLength
       );
-    const fileObject = new FileUpload(
+      rangeStart += RANGE_SIZE;
-      remoteContent,
+    }
-      path.posix.basename(uploadFile),
+
-      remoteContent.byteLength
-    );
-    const task = new LargeFileUploadTask(
-      client.client,
-      fileObject,
-      uploadSession,
-      options
-    );
-    const uploadResult: UploadResult = await task.upload();
-    // log.info(uploadResult)
     const res = await getRemoteMeta(client, uploadFile);
     return res;
   }
@@ -658,10 +737,9 @@ const downloadFromRemoteRaw = async (
 ): Promise<ArrayBuffer> => {
   await client.init();
   const key = getOnedrivePath(fileOrFolderPath, client.vaultName);
-  const rsp = await client.client
+  const rsp = await client.getJson(
-    .api(key)
+    `${key}?$select=@microsoft.graph.downloadUrl`
-    .select("@microsoft.graph.downloadUrl")
+  );
-    .get();
   const downloadUrl: string = rsp["@microsoft.graph.downloadUrl"];
   const content = await (await fetch(downloadUrl)).arrayBuffer();
   return content;
@@ -717,7 +795,7 @@ export const deleteFromRemote = async (
   remoteFileName = getOnedrivePath(remoteFileName, client.vaultName);
 
   await client.init();
-  await client.client.api(remoteFileName).delete();
+  await client.deleteJson(remoteFileName);
 };
 
 export const checkConnectivity = async (client: WrappedOnedriveClient) => {
@@ -731,7 +809,7 @@ export const checkConnectivity = async (client: WrappedOnedriveClient) => {
 
 export const getUserDisplayName = async (client: WrappedOnedriveClient) => {
   await client.init();
-  const res: User = await client.client.api("/me").select("displayName").get();
+  const res: User = await client.getJson("/me?$select=displayName");
   return res.displayName || "<unknown display name>";
 };
 
@@ -744,7 +822,7 @@ export const getUserDisplayName = async (client: WrappedOnedriveClient) => {
  */
 // export const revokeAuth = async (client: WrappedOnedriveClient) => {
 //   await client.init();
-//   await client.client.api('/me/revokeSignInSessions').post(undefined);
+//   await client.postJson('/me/revokeSignInSessions', {});
 // };
 
 export const getRevokeAddr = async () => {

versions.json

@@ -1,3 +1,3 @@
 {
-  "0.2.12": "0.12.15"
+  "0.2.14": "0.12.15"
 }