full word

.github/workflows/auto-build.yml

@@ -13,6 +13,8 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    environment: env-for-buildci
+
     strategy:
       matrix:
         node-version: [16.x]

README.md

@@ -4,7 +4,7 @@ This is yet another unofficial sync plugin for Obsidian.
 
 ## Disclaimer
 
-- **This is NOT the [official sync service](https://github.com/fyears/obsidian-remotely-save.git) provided by Obsidian.**
+- **This is NOT the [official sync service](https://obsidian.md/sync) provided by Obsidian.**
 
 ## !!!Caution!!!
 
@@ -29,7 +29,7 @@ As of November 2021, the plugin is considered in BETA stage. **DO NOT USE IT for
 
 ## Download and Install
 
-- Option #1: [![BuildCI](https://github.com/fyears/obsidian-remotely-save/actions/workflows/auto-build.yml/badge.svg)](https://github.com/fyears/obsidian-remotely-save/actions/workflows/auto-build.yml) Every artifacts are placed in the "Summary" under every successful builds.
+- Option #1: [![BuildCI](https://github.com/fyears/remotely-save/actions/workflows/auto-build.yml/badge.svg)](https://github.com/fyears/remotely-save/actions/workflows/auto-build.yml) Every artifacts are placed in the "Summary" under every successful builds.
 - Option #2: Besides manually downloading the files, you can also use [Obsidian42 - BRAT](https://github.com/TfTHacker/obsidian42-brat) to install this plugin.
 - Option #3: The pluin would be submitted to the official "community plugin list" in near future.
 
@@ -49,7 +49,7 @@ As of November 2021, the plugin is considered in BETA stage. **DO NOT USE IT for
 
 - **This plugin's function for Dropbox is not as mature as functions for S3.**
 - **This plugin is NOT an official Dropbox product.** The plugin just uses Dropbox's public API.
-- After the authorization, the plugin can read your name and email (which cannot be unselected on Dropbox api), and read and write files in your Dropbox's `/App/obsidian-remotely-save` folder.
+- After the authorization, the plugin can read your name and email (which cannot be unselected on Dropbox api), and read and write files in your Dropbox's `/Apps/remotely-save` folder.
 - If you decide to authorize this plugin to connect to Dropbox, please go to plugin's settings, and choose Dropbox then follow the instructions.
 - Password-based end-to-end encryption is also supported.
 

esbuild.config.mjs

@@ -24,6 +24,7 @@ esbuild
       "obsidian",
       "electron",
       "fs",
+      "crypto",
       // ...builtins
     ],
     format: "cjs",

manifest.json

@@ -1,9 +1,9 @@
 {
-  "id": "obsidian-remotely-save",
+  "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.1.7",
+  "version": "0.2.1",
   "minAppVersion": "0.12.15",
-  "description": "Yet another unofficial plugin allowing users to sync notes between local device and the cloud service.",
+  "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",
   "authorUrl": "https://github.com/fyears",
   "isDesktopOnly": false

package.json

@@ -1,10 +1,11 @@
 {
-  "name": "obsidian-remotely-save",
-  "version": "0.1.7",
+  "name": "remotely-save",
+  "version": "0.2.1",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev": "node esbuild.config.mjs",
-    "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
+    "build2": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
+    "build": "webpack --mode production",
     "format": "npx prettier --write .",
     "clean": "npx rimraf main.js",
     "test": "cross-env TS_NODE_COMPILER_OPTIONS={\\\"module\\\":\\\"commonjs\\\"} mocha -r ts-node/register 'tests/**/*.ts'"
@@ -12,10 +13,7 @@
   "browser": {
     "path": "path-browserify",
     "process": "process/browser",
-    "stream": "stream-browserify",
-    "crypto": "crypto-browserify",
-    "util": "util/",
-    "assert": "assert/"
+    "stream": "stream-browserify"
   },
   "source": "main.ts",
   "keywords": [],
@@ -39,7 +37,9 @@
     "ts-node": "^10.4.0",
     "tslib": "^2.2.0",
     "typescript": "^4.4.4",
-    "webdav-server": "^2.6.2"
+    "webdav-server": "^2.6.2",
+    "webpack": "^5.64.4",
+    "webpack-cli": "^4.9.1"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.37.0",

src/main.ts

@@ -27,13 +27,16 @@ import type { InternalDBs } from "./localdb";
 import type { SyncStatusType, PasswordCheckType } from "./sync";
 import { isPasswordOk, getSyncPlan, doActualSync } from "./sync";
 
-import { S3Config, DEFAULT_S3_CONFIG } from "./s3";
-import { WebdavConfig, DEFAULT_WEBDAV_CONFIG, WebdavAuthType } from "./webdav";
+import { S3Config, DEFAULT_S3_CONFIG } from "./remoteForS3";
+import {
+  WebdavConfig,
+  DEFAULT_WEBDAV_CONFIG,
+  WebdavAuthType,
+} from "./remoteForWebdav";
 import {
   DropboxConfig,
   DEFAULT_DROPBOX_CONFIG,
-  getCodeVerifierAndChallenge,
-  getAuthUrl,
+  getAuthUrlAndVerifier,
   sendAuthReq,
   setConfigBySuccessfullAuthInplace,
 } from "./remoteForDropbox";
@@ -67,7 +70,7 @@ export default class RemotelySavePlugin extends Plugin {
   syncStatus: SyncStatusType;
 
   async onload() {
-    console.log("loading plugin obsidian-remotely-save");
+    console.log("loading plugin remotely-save");
 
     await this.loadSettings();
 
@@ -110,6 +113,7 @@ export default class RemotelySavePlugin extends Plugin {
           this.settings.s3,
           this.settings.webdav,
           this.settings.dropbox,
+          this.app.vault.getName(),
           () => self.saveSettings()
         );
         const remoteRsp = await client.listFromRemote();
@@ -190,7 +194,7 @@ export default class RemotelySavePlugin extends Plugin {
   }
 
   onunload() {
-    console.log("unloading plugin obsidian-remotely-save");
+    console.log("unloading plugin remotely-save");
     this.destroyDBs();
   }
 
@@ -295,13 +299,11 @@ export class DropboxAuthModal extends Modal {
     this.revokeAuthSetting = revokeAuthSetting;
   }
 
-  onOpen() {
+  async onOpen() {
     let { contentEl } = this;
 
-    const k = getCodeVerifierAndChallenge();
-    const authUrl = getAuthUrl(
-      this.plugin.settings.dropbox.clientID,
-      k.challenge
+    const { authUrl, verifier } = await getAuthUrlAndVerifier(
+      this.plugin.settings.dropbox.clientID
     );
 
     contentEl.createEl("p", {
@@ -335,7 +337,7 @@ export class DropboxAuthModal extends Modal {
           try {
             const authRes = await sendAuthReq(
               this.plugin.settings.dropbox.clientID,
-              k.verifier,
+              verifier,
               authCode
             );
             const self = this;
@@ -349,6 +351,7 @@ export class DropboxAuthModal extends Modal {
               undefined,
               undefined,
               this.plugin.settings.dropbox,
+              this.app.vault.getName(),
               () => self.plugin.saveSettings()
             );
             const username = await client.getUser();
@@ -600,7 +603,7 @@ class RemotelySaveSettingTab extends PluginSettingTab {
       cls: "dropbox-disclaimer",
     });
     dropboxDiv.createEl("p", {
-      text: "We will create a folder App/obsidian-remotely-save on your Dropbox. All files/folders sync would happen inside this folder.",
+      text: "We will create a folder Apps/remotely-save on your Dropbox. All files/folders sync would happen inside this folder.",
     });
 
     const dropboxSelectAuthDiv = dropboxDiv.createDiv();
@@ -626,6 +629,7 @@ class RemotelySaveSettingTab extends PluginSettingTab {
               undefined,
               undefined,
               this.plugin.settings.dropbox,
+              this.app.vault.getName(),
               () => self.plugin.saveSettings()
             );
             await client.revokeAuth();
@@ -687,6 +691,7 @@ class RemotelySaveSettingTab extends PluginSettingTab {
             undefined,
             undefined,
             this.plugin.settings.dropbox,
+            this.app.vault.getName(),
             () => self.plugin.saveSettings()
           );
 

src/remote.ts

@@ -1,8 +1,8 @@
 import { Vault } from "obsidian";
 
 import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
-import * as s3 from "./s3";
-import * as webdav from "./webdav";
+import * as s3 from "./remoteForS3";
+import * as webdav from "./remoteForWebdav";
 import * as dropbox from "./remoteForDropbox";
 
 export class RemoteClient {
@@ -19,6 +19,7 @@ export class RemoteClient {
     s3Config?: s3.S3Config,
     webdavConfig?: webdav.WebdavConfig,
     dropboxConfig?: dropbox.DropboxConfig,
+    vaultName?: string,
     saveUpdatedConfigFunc?: () => Promise<any>
   ) {
     this.serviceType = serviceType;
@@ -31,12 +32,15 @@ export class RemoteClient {
       this.webdavConfig = webdavConfig;
       this.webdavClient = webdav.getWebdavClient(this.webdavConfig);
     } else if (serviceType === "dropbox") {
-      if (saveUpdatedConfigFunc === undefined) {
-        throw Error("remember to provide callback while init dropbox client");
+      if (vaultName === undefined || saveUpdatedConfigFunc === undefined) {
+        throw Error(
+          "remember to provide vault name and callback while init dropbox client"
+        );
       }
       this.dropboxConfig = dropboxConfig;
       this.dropboxClient = dropbox.getDropboxClient(
         this.dropboxConfig,
+        vaultName,
         saveUpdatedConfigFunc
       );
     } else {

src/remoteForDropbox.ts

@@ -1,9 +1,7 @@
 import * as path from "path";
 import { FileStats, Vault } from "obsidian";
-import { Buffer } from "buffer";
-import * as crypto from "crypto";
 
-import { Dropbox, DropboxResponse, files } from "dropbox";
+import { Dropbox, DropboxAuth, DropboxResponse, files } from "dropbox";
 export { Dropbox } from "dropbox";
 import { RemoteItem } from "./baseTypes";
 import {
@@ -15,7 +13,6 @@ import {
   setToString,
 } from "./misc";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
-import { strict as assert } from "assert";
 
 export interface DropboxConfig {
   accessToken: string;
@@ -37,10 +34,11 @@ export const DEFAULT_DROPBOX_CONFIG = {
   username: "",
 };
 
-export const getDropboxPath = (fileOrFolderPath: string) => {
+export const getDropboxPath = (fileOrFolderPath: string, vaultName: string) => {
   let key = fileOrFolderPath;
   if (!fileOrFolderPath.startsWith("/")) {
-    key = `/${fileOrFolderPath}`;
+    // then this is original path in Obsidian
+    key = `/${vaultName}/${fileOrFolderPath}`;
   }
   if (key.endsWith("/")) {
     key = key.slice(0, key.length - 1);
@@ -48,20 +46,26 @@ export const getDropboxPath = (fileOrFolderPath: string) => {
   return key;
 };
 
-const getNormPath = (fileOrFolderPath: string) => {
-  if (fileOrFolderPath.startsWith("/")) {
-    return fileOrFolderPath.slice(1);
+const getNormPath = (fileOrFolderPath: string, vaultName: string) => {
+  if (
+    !(
+      fileOrFolderPath === `/${vaultName}` ||
+      fileOrFolderPath.startsWith(`/${vaultName}/`)
+    )
+  ) {
+    throw Error(`"${fileOrFolderPath}" doesn't starts with "/${vaultName}/"`);
   }
-  return fileOrFolderPath;
+  return fileOrFolderPath.slice(`/${vaultName}/`.length);
 };
 
 const fromDropboxItemToRemoteItem = (
   x:
     | files.FileMetadataReference
     | files.FolderMetadataReference
-    | files.DeletedMetadataReference
+    | files.DeletedMetadataReference,
+  vaultName: string
 ): RemoteItem => {
-  let key = getNormPath(x.path_display);
+  let key = getNormPath(x.path_display, vaultName);
   if (x[".tag"] === "folder" && !key.endsWith("/")) {
     key = `${key}/`;
   }
@@ -139,7 +143,6 @@ const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => {
     if (item.lastModified !== undefined) {
       continue; // don't need to deal with it
     }
-    assert(!(item.key in potentialMTime));
     const parent = `${path.posix.dirname(item.key)}/`;
     if (parent in potentialMTime) {
       item.lastModified = potentialMTime[parent];
@@ -157,32 +160,28 @@ const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => {
 // see https://dropbox.tech/developers/pkce--what-and-why-
 ////////////////////////////////////////////////////////////////////////////////
 
-const specialBase64Encode = (str: Buffer) => {
-  return str
-    .toString("base64")
-    .replace(/\+/g, "-")
-    .replace(/\//g, "_")
-    .replace(/=/g, "");
-};
-const sha256 = (buffer: string) => {
-  return crypto.createHash("sha256").update(buffer).digest();
-};
-
-export const getCodeVerifierAndChallenge = () => {
-  const codeVerifier = specialBase64Encode(crypto.randomBytes(32));
-  // console.log(`Client generated code_verifier: ${codeVerifier}`);
-  const codeChallenge = specialBase64Encode(sha256(codeVerifier));
-  // console.log(`Client generated code_challenge: ${codeChallenge}`);
+export const getAuthUrlAndVerifier = async (appKey: string) => {
+  const auth = new DropboxAuth({
+    clientId: appKey,
+  });
+  const authUrl = (
+    await auth.getAuthenticationUrl(
+      undefined,
+      undefined,
+      "code",
+      "offline",
+      undefined,
+      "none",
+      true
+    )
+  ).toString();
+  const verifier = auth.getCodeVerifier();
   return {
-    verifier: codeVerifier,
-    challenge: codeChallenge,
+    authUrl: authUrl,
+    verifier: verifier,
   };
 };
 
-export const getAuthUrl = (appKey: string, challenge: string) => {
-  return `https://www.dropbox.com/oauth2/authorize?client_id=${appKey}&response_type=code&code_challenge=${challenge}&code_challenge_method=S256&token_access_type=offline`;
-};
-
 export interface DropboxSuccessAuthRes {
   access_token: string;
   token_type: "bearer";
@@ -261,17 +260,23 @@ export const setConfigBySuccessfullAuthInplace = async (
 
 export class WrappedDropboxClient {
   dropboxConfig: DropboxConfig;
+  vaultName: string;
   saveUpdatedConfigFunc: () => Promise<any>;
   dropbox: Dropbox;
+  vaultFolderExists: boolean;
   constructor(
     dropboxConfig: DropboxConfig,
+    vaultName: string,
     saveUpdatedConfigFunc: () => Promise<any>
   ) {
     this.dropboxConfig = dropboxConfig;
+    this.vaultName = vaultName;
     this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
+    this.vaultFolderExists = false;
   }
 
   init = async () => {
+    // check token
     if (
       this.dropboxConfig.accessToken === "" ||
       this.dropboxConfig.refreshToken === ""
@@ -303,6 +308,33 @@ export class WrappedDropboxClient {
         accessToken: this.dropboxConfig.accessToken,
       });
     }
+
+    // check vault folder
+    // console.log(`checking remote has folder /${this.vaultName}`);
+    if (this.vaultFolderExists) {
+      // console.log(`already checked, /${this.vaultName} exist before`)
+    } else {
+      const res = await this.dropbox.filesListFolder({
+        path: "",
+        recursive: false,
+      });
+      for (const item of res.result.entries) {
+        if (item.path_display === `/${this.vaultName}`) {
+          this.vaultFolderExists = true;
+          break;
+        }
+      }
+      if (!this.vaultFolderExists) {
+        console.log(`remote does not have folder /${this.vaultName}`);
+        await this.dropbox.filesCreateFolderV2({
+          path: `/${this.vaultName}`,
+        });
+        console.log(`remote folder /${this.vaultName} created`);
+      } else {
+        // console.log(`remote folder /${this.vaultName} exists`);
+      }
+    }
+
     return this.dropbox;
   };
 }
@@ -313,9 +345,14 @@ export class WrappedDropboxClient {
  */
 export const getDropboxClient = (
   dropboxConfig: DropboxConfig,
+  vaultName: string,
   saveUpdatedConfigFunc: () => Promise<any>
 ) => {
-  return new WrappedDropboxClient(dropboxConfig, saveUpdatedConfigFunc);
+  return new WrappedDropboxClient(
+    dropboxConfig,
+    vaultName,
+    saveUpdatedConfigFunc
+  );
 };
 
 export const getRemoteMeta = async (
@@ -328,7 +365,7 @@ export const getRemoteMeta = async (
     // we instead try to list files
     // if no error occurs, we ensemble a fake result.
     const rsp = await client.dropbox.filesListFolder({
-      path: "",
+      path: `/${client.vaultName}`,
       recursive: false, // don't need to recursive here
     });
     if (rsp.status !== 200) {
@@ -343,7 +380,7 @@ export const getRemoteMeta = async (
     } as RemoteItem;
   }
 
-  const key = getDropboxPath(fileOrFolderPath);
+  const key = getDropboxPath(fileOrFolderPath, client.vaultName);
 
   const rsp = await client.dropbox.filesGetMetadata({
     path: key,
@@ -351,7 +388,7 @@ export const getRemoteMeta = async (
   if (rsp.status !== 200) {
     throw Error(JSON.stringify(rsp));
   }
-  return fromDropboxItemToRemoteItem(rsp.result);
+  return fromDropboxItemToRemoteItem(rsp.result, client.vaultName);
 };
 
 export const uploadToRemote = async (
@@ -369,7 +406,7 @@ export const uploadToRemote = async (
   if (password !== "") {
     uploadFile = remoteEncryptedKey;
   }
-  uploadFile = getDropboxPath(uploadFile);
+  uploadFile = getDropboxPath(uploadFile, client.vaultName);
 
   const isFolder = fileOrFolderPath.endsWith("/");
 
@@ -425,7 +462,9 @@ export const uploadToRemote = async (
     });
     // we want to mark that parent folders are created
     if (foldersCreatedBefore !== undefined) {
-      const dirs = getFolderLevels(uploadFile).map(getDropboxPath);
+      const dirs = getFolderLevels(uploadFile).map((x) =>
+        getDropboxPath(x, client.vaultName)
+      );
       for (const dir of dirs) {
         foldersCreatedBefore?.add(dir);
       }
@@ -442,19 +481,41 @@ export const listFromRemote = async (
     throw Error("prefix not supported (yet)");
   }
   await client.init();
-  const res = await client.dropbox.filesListFolder({
-    path: "",
+  let res = await client.dropbox.filesListFolder({
+    path: `/${client.vaultName}`,
     recursive: true,
+    include_deleted: false,
+    limit: 1000,
   });
   if (res.status !== 200) {
     throw Error(JSON.stringify(res));
   }
   // console.log(res);
+
   const contents = res.result.entries;
   const unifiedContents = contents
     .filter((x) => x[".tag"] !== "deleted")
-    .map(fromDropboxItemToRemoteItem);
+    .filter((x) => x.path_display !== `/${client.vaultName}`)
+    .map((x) => fromDropboxItemToRemoteItem(x, client.vaultName));
+
+  while (res.result.has_more) {
+    res = await client.dropbox.filesListFolderContinue({
+      cursor: res.result.cursor,
+    });
+    if (res.status !== 200) {
+      throw Error(JSON.stringify(res));
+    }
+
+    const contents2 = res.result.entries;
+    const unifiedContents2 = contents2
+      .filter((x) => x[".tag"] !== "deleted")
+      .filter((x) => x.path_display !== `/${client.vaultName}`)
+      .map((x) => fromDropboxItemToRemoteItem(x, client.vaultName));
+    unifiedContents.push(...unifiedContents2);
+  }
+
   fixLastModifiedTimeInplace(unifiedContents);
+
   return {
     Contents: unifiedContents,
   };
@@ -465,7 +526,7 @@ const downloadFromRemoteRaw = async (
   fileOrFolderPath: string
 ) => {
   await client.init();
-  const key = getDropboxPath(fileOrFolderPath);
+  const key = getDropboxPath(fileOrFolderPath, client.vaultName);
   const rsp = await client.dropbox.filesDownload({
     path: key,
   });
@@ -505,7 +566,7 @@ export const downloadFromRemote = async (
     if (password !== "") {
       downloadFile = remoteEncryptedKey;
     }
-    downloadFile = getDropboxPath(downloadFile);
+    downloadFile = getDropboxPath(downloadFile, client.vaultName);
     const remoteContent = await downloadFromRemoteRaw(client, downloadFile);
     let localContent = remoteContent;
     if (password !== "") {
@@ -530,7 +591,7 @@ export const deleteFromRemote = async (
   if (password !== "") {
     remoteFileName = remoteEncryptedKey;
   }
-  remoteFileName = getDropboxPath(remoteFileName);
+  remoteFileName = getDropboxPath(remoteFileName, client.vaultName);
 
   await client.init();
   try {

versions.json

@@ -1,3 +1,3 @@
 {
-  "0.1.7": "0.12.15"
+  "0.2.1": "0.12.15"
 }

webpack.config.js

@@ -0,0 +1,75 @@
+require("dotenv").config();
+const path = require("path");
+const webpack = require("webpack");
+const TerserPlugin = require("terser-webpack-plugin");
+
+const DEFAULT_DROPBOX_APP_KEY = process.env.DROPBOX_APP_KEY || "";
+
+module.exports = {
+  entry: "./src/main.ts",
+  target: "web",
+  output: {
+    filename: "main.js",
+    path: __dirname,
+    libraryTarget: "commonjs",
+  },
+  plugins: [
+    new webpack.DefinePlugin({
+      "process.env.DEFAULT_DROPBOX_APP_KEY": `"${DEFAULT_DROPBOX_APP_KEY}"`,
+    }),
+    // Work around for Buffer is undefined:
+    // https://github.com/webpack/changelog-v5/issues/10
+    new webpack.ProvidePlugin({
+      Buffer: ["buffer", "Buffer"],
+    }),
+    new webpack.ProvidePlugin({
+      process: "process/browser",
+    }),
+  ],
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: "ts-loader",
+        exclude: /node_modules/,
+      },
+    ],
+  },
+  resolve: {
+    extensions: [".tsx", ".ts", ".js"],
+    mainFields: ["browser", "module", "main"],
+    fallback: {
+      // assert: require.resolve("assert"),
+      // buffer: require.resolve("buffer/"),
+      // console: require.resolve("console-browserify"),
+      // constants: require.resolve("constants-browserify"),
+      // crypto: require.resolve("crypto-browserify"),
+      crypto: false,
+      // domain: require.resolve("domain-browser"),
+      // events: require.resolve("events"),
+      // http: require.resolve("stream-http"),
+      // https: require.resolve("https-browserify"),
+      // os: require.resolve("os-browserify/browser"),
+      path: require.resolve("path-browserify"),
+      // punycode: require.resolve("punycode"),
+      process: require.resolve("process/browser"),
+      // querystring: require.resolve("querystring-es3"),
+      stream: require.resolve("stream-browserify"),
+      // string_decoder: require.resolve("string_decoder"),
+      // sys: require.resolve("util"),
+      // timers: require.resolve("timers-browserify"),
+      // tty: require.resolve("tty-browserify"),
+      // url: require.resolve("url"),
+      // util: require.resolve("util"),
+      // vm: require.resolve("vm-browserify"),
+      // zlib: require.resolve("browserify-zlib"),
+    },
+  },
+  externals: {
+    obsidian: "commonjs2 obsidian",
+  },
+  optimization: {
+    minimize: true,
+    minimizer: [new TerserPlugin({ extractComments: false })],
+  },
+};