0.0.10

merge enc to master

.editorconfig

@@ -0,0 +1,28 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+[*.{js,jsx,mjs,cjs,ts,tsx,json,py}]
+charset = utf-8
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4
+
+# Tab indentation (no size specified)
+[Makefile]
+indent_style = tab
+
+# Indentation override for all JS under lib directory
+[*.{js,jsx,mjs,cjs,ts,tsx,json,py}]
+indent_style = space
+indent_size = 2

.gitattributes

@@ -0,0 +1,5 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.enc filter=lfs diff=lfs merge=lfs -text

.github/workflows/auto-build.yml

@@ -0,0 +1,40 @@
+# This workflow will do a clean install of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: BuildCI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [16.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+
+    steps:
+      - name: Checkout codes
+        uses: actions/checkout@v2
+        with:
+          lfs: true
+      - name: Checkout LFS
+        run: git lfs checkout
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: npm install
+      - run: npm test
+      - run: npm run build
+      - uses: actions/upload-artifact@v2
+        with:
+          name: my-dist
+          path: |
+            main.js
+            manifest.json

.prettierignore

@@ -0,0 +1,2 @@
+node_modules/
+main.js

README.md

@@ -2,6 +2,14 @@
 
 This is yet another sync plugin for Obsidian.
 
+## Download and Install
+
+[![BuildCI](https://github.com/fyears/obsidian-save-remote/actions/workflows/auto-build.yml/badge.svg)](https://github.com/fyears/obsidian-save-remote/actions/workflows/auto-build.yml)
+
+Every artifacts are placed in the "Summary" under every successful builds.
+
+Besides manually download the files, you can also use [Obsidian42 - BRAT](https://github.com/TfTHacker/obsidian42-brat) to install this plugin.
+
 ## Disclaimer
 
 **This is NOT the official sync service provided by Obsidian.**
@@ -14,5 +22,5 @@ As of October 2021, the plugin is under development. **DO NOT USE IT for any ser
 
 Remote service in support list:
 
-- [ ] AWS S3
+- [x] AWS S3
 - [ ] webdav

docs/algorithm.md

@@ -0,0 +1,29 @@
+# Algorithm
+
+## Sources
+
+We have three record sources:
+
+1. Local files
+2. Remote files
+3. Local "delete-or-rename" history.
+
+Assuming all sources are reliable.
+
+## Deal with them
+
+We list all combinations mutually exclusive and collectively exhaustive.
+
+| ID   | Remote Files | Local files | Local delete rename history | Extra                                                        | Decision                                                     |
+| ---- | ------------ | ----------- | --------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1    | exist        | exist       | ignore                      | mtime_remote > mtime_local                                   | download remote file, create local folder if not exists, clear local history if exists |
+| 2    | exist        | exist       | ignore                      | mtime_remote < mtime_local                                   | upload local file, create remote folder if not exists, clear local history if exists |
+| 3    | exist        | exist       | ignore                      | mtime_remote === mtime_local && password === "" && size_remote === size_local | clear local history if exists (the file was synced and no changes after last sync) |
+| 4    | exist        | exist       | ignore                      | mtime_remote === mtime_local && password === "" && size_remote !== size_local | upload local file, clear local history if exists (we always prefer local to remote) |
+| 5    | exist        | exist       | ignore                      | mtime_remote === mtime_local && password !== ""              | clear local history if exists (in encryption mode, file sizes are unequal. we can only rely on mtime(s)) |
+| 6    | exist        | exist       | ignore                      | If local is a folder. mtime_local === undefined              | clear local history if exists. TODO: what if a folder and a previous file share the same name? |
+| 7    | exist        | not exist   | exist                       | mtime_remote >= delete_time_local                            | download remote file, create folder if not exists            |
+| 8    | exist        | not exist   | exist                       | mtime_remote < delete_time_local                             | delete remote file, clear local history                      |
+| 9    | exist        | not exist   | not exist                   |                                                              | download remote file, create folder if not exists            |
+| 10   | not exist    | exist       | ignore                      | local may be folder or file                                  | upload local files recursively, create remote folder if not exists, clear local history if exists |
+| 11   | not exist    | not exist   | ignore                      |                                                              | clear local history if exists                                |

docs/code_design.md

@@ -0,0 +1,13 @@
+# Code Design
+
+## Code Organization
+
+1. Every function except `main.ts` should be pure. Pass any stateful information in parameters.
+
+2. `misc.ts` should not depend on any other written code.
+
+3. Each storage code should not depend on `sync.ts`.
+
+## File and Folder Representation
+
+While writing sync codes, folders are always represented by a string ending with `/`.

main.ts

@@ -1,340 +0,0 @@
-import * as path from "path";
-import * as fs from "fs";
-import { Buffer } from "buffer";
-import { Readable } from "stream";
-import * as mime from "mime-types";
-import {
-  App,
-  Modal,
-  Notice,
-  Plugin,
-  PluginSettingTab,
-  Setting,
-  request,
-  Platform,
-} from "obsidian";
-import * as CodeMirror from "codemirror";
-
-import {
-  S3Client,
-  ListObjectsCommand,
-  PutObjectCommand,
-  GetObjectCommand,
-} from "@aws-sdk/client-s3";
-
-interface SaveRemotePluginSettings {
-  s3Endpoint: string;
-  s3Region: string;
-  s3AccessKeyID: string;
-  s3SecretAccessKey: string;
-  s3BucketName: string;
-}
-
-const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
-  s3Endpoint: "",
-  s3Region: "",
-  s3AccessKeyID: "",
-  s3SecretAccessKey: "",
-  s3BucketName: "",
-};
-
-const ignoreHiddenFiles = (item: string) => {
-  const basename = path.basename(item);
-  return basename === "." || basename[0] !== ".";
-};
-
-/**
- * Util func for mkdir -p based on the "path" of original file or folder
- * "a/b/c/" => ["a", "a/b", "a/b/c"]
- * "a/b/c/d/e.txt" => ["a", "a/b", "a/b/c", "a/b/c/d"]
- * @param x string
- * @returns string[] might be empty
- */
-const getFolderLevels = (x: string) => {
-  const res: string[] = [];
-
-  if (x === "" || x === "/") {
-    return res;
-  }
-
-  const y1 = x.split("/");
-  let i = 0;
-  for (let index = 0; index + 1 < y1.length; index++) {
-    res.push(y1.slice(0, index + 1).join("/"));
-  }
-  return res;
-};
-
-/**
- * https://stackoverflow.com/questions/8609289
- * @param b Buffer
- * @returns ArrayBuffer
- */
-const bufferToArrayBuffer = (b: Buffer) => {
-  return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
-};
-
-/**
- * The Body of resp of aws GetObject has mix types
- * and we want to get ArrayBuffer here.
- * See https://github.com/aws/aws-sdk-js-v3/issues/1877
- * @param b The Body of GetObject
- * @returns Promise<ArrayBuffer>
- */
-const getObjectBodyToArrayBuffer = async (
-  b: Readable | ReadableStream | Blob
-) => {
-  if (b instanceof Readable) {
-    const chunks: Uint8Array[] = [];
-    for await (let chunk of b) {
-      chunks.push(chunk);
-    }
-    const buf = Buffer.concat(chunks);
-    return bufferToArrayBuffer(buf);
-  } else if (b instanceof ReadableStream) {
-    return await new Response(b, {}).arrayBuffer();
-  } else if (b instanceof Blob) {
-    return await b.arrayBuffer();
-  } else {
-    throw TypeError(`The type of ${b} is not one of the supported types`);
-  }
-};
-
-export default class SaveRemotePlugin extends Plugin {
-  settings: SaveRemotePluginSettings;
-  cm: CodeMirror.Editor;
-
-  async onload() {
-    console.log("loading plugin obsidian-save-remote");
-
-    await this.loadSettings();
-
-    this.addRibbonIcon("right-arrow-with-tail", "Upload", async () => {
-      // console.log(this.app.vault.getFiles());
-      // console.log(this.app.vault.getAllLoadedFiles());
-      new Notice(`Upload begun.`);
-      const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
-
-      const s3Client = new S3Client({
-        region: this.settings.s3Region,
-        endpoint: this.settings.s3Endpoint,
-        credentials: {
-          accessKeyId: this.settings.s3AccessKeyID,
-          secretAccessKey: this.settings.s3SecretAccessKey,
-        },
-      });
-
-      try {
-        for (const fileOrFolder of allFilesAndFolders) {
-          if (fileOrFolder.path === "/") {
-            console.log('ignore "/"');
-          } else if ("children" in fileOrFolder) {
-            // folder
-            console.log(`folder ${fileOrFolder.path}/`);
-            new Notice(`folder ${fileOrFolder.path}/`);
-
-            const results = await s3Client.send(
-              new PutObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: `${fileOrFolder.path}/`,
-                Body: "",
-              })
-            );
-          } else {
-            // file
-            console.log(`file ${fileOrFolder.path}`);
-            const arrContent = await this.app.vault.adapter.readBinary(
-              fileOrFolder.path
-            );
-            new Notice(`file ${fileOrFolder.path}`);
-            const contentType =
-              mime.contentType(
-                mime.lookup(`${fileOrFolder.path}`) || undefined
-              ) || undefined;
-            // console.log(contentType);
-            const results = await s3Client.send(
-              new PutObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: `${fileOrFolder.path}`,
-                Body: Buffer.from(arrContent),
-                ContentType: contentType,
-              })
-            );
-          }
-        }
-        new Notice("Upload finished!");
-      } catch (err) {
-        console.log("Error", err);
-        new Notice(`${err}`);
-      }
-    });
-
-    this.addRibbonIcon("left-arrow-with-tail", "Download", async () => {
-      const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
-
-      const s3Client = new S3Client({
-        region: this.settings.s3Region,
-        endpoint: this.settings.s3Endpoint,
-        credentials: {
-          accessKeyId: this.settings.s3AccessKeyID,
-          secretAccessKey: this.settings.s3SecretAccessKey,
-        },
-      });
-
-      try {
-        const listObj = await s3Client.send(
-          new ListObjectsCommand({ Bucket: this.settings.s3BucketName })
-        );
-
-        for (const singleContent of listObj.Contents) {
-          const foldersToBuild = getFolderLevels(singleContent.Key);
-          for (const folder of foldersToBuild) {
-            const r = await this.app.vault.adapter.exists(folder);
-            if (!r) {
-              console.log(`mkdir ${folder}`);
-              new Notice(`mkdir ${folder}`);
-              await this.app.vault.adapter.mkdir(folder);
-            }
-          }
-
-          if (singleContent.Key.endsWith("/")) {
-            // kind of a folder
-            // pass
-          } else {
-            // kind of a file
-            // download
-
-            console.log(`download file ${singleContent.Key}`);
-            new Notice(`download file ${singleContent.Key}`);
-
-            const data = await s3Client.send(
-              new GetObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: singleContent.Key,
-              })
-            );
-            const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
-            await this.app.vault.adapter.writeBinary(
-              singleContent.Key,
-              bodyContents
-            );
-          }
-        }
-
-        new Notice("Download finished!");
-      } catch (err) {
-        console.log("Error", err);
-        new Notice(`${err}`);
-      }
-    });
-
-    this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
-
-    this.registerCodeMirror((cm: CodeMirror.Editor) => {
-      this.cm = cm;
-      console.log("codemirror registered.");
-    });
-
-    // this.registerDomEvent(document, "click", (evt: MouseEvent) => {
-    //   console.log("click", evt);
-    // });
-
-    // this.registerInterval(
-    //   window.setInterval(() => console.log("setInterval"), 5 * 60 * 1000)
-    // );
-  }
-
-  onunload() {
-    console.log("unloading plugin obsidian-save-remote");
-  }
-
-  async loadSettings() {
-    this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
-  }
-
-  async saveSettings() {
-    await this.saveData(this.settings);
-  }
-}
-
-class SaveRemoteSettingTab extends PluginSettingTab {
-  plugin: SaveRemotePlugin;
-
-  constructor(app: App, plugin: SaveRemotePlugin) {
-    super(app, plugin);
-    this.plugin = plugin;
-  }
-
-  display(): void {
-    let { containerEl } = this;
-
-    containerEl.empty();
-
-    containerEl.createEl("h2", { text: "Settings for Save Remote" });
-
-    new Setting(containerEl)
-      .setName("s3Endpoint")
-      .setDesc("s3Endpoint")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(this.plugin.settings.s3Endpoint)
-          .onChange(async (value) => {
-            this.plugin.settings.s3Endpoint = value;
-            await this.plugin.saveSettings();
-          })
-      );
-
-    new Setting(containerEl)
-      .setName("s3Region")
-      .setDesc("s3Region")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3Region}`)
-          .onChange(async (value) => {
-            this.plugin.settings.s3Region = value;
-            await this.plugin.saveSettings();
-          })
-      );
-
-    new Setting(containerEl)
-      .setName("s3AccessKeyID")
-      .setDesc("s3AccessKeyID")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3AccessKeyID}`)
-          .onChange(async (value) => {
-            this.plugin.settings.s3AccessKeyID = value;
-            await this.plugin.saveSettings();
-          })
-      );
-
-    new Setting(containerEl)
-      .setName("s3SecretAccessKey")
-      .setDesc("s3SecretAccessKey")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3SecretAccessKey}`)
-          .onChange(async (value) => {
-            this.plugin.settings.s3SecretAccessKey = value;
-            await this.plugin.saveSettings();
-          })
-      );
-
-    new Setting(containerEl)
-      .setName("s3BucketName")
-      .setDesc("s3BucketName")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3BucketName}`)
-          .onChange(async (value) => {
-            this.plugin.settings.s3BucketName = value;
-            await this.plugin.saveSettings();
-          })
-      );
-  }
-}

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "obsdian-save-remote",
   "name": "Save remote",
-  "version": "0.0.1",
+  "version": "0.0.10",
   "minAppVersion": "0.12.15",
   "description": "This is yet another plugin allowing users to sync notes between local device and the cloud.",
   "author": "fyears",

package.json

@@ -1,23 +1,30 @@
 {
   "name": "obsidian-save-remote",
-  "version": "0.0.1",
+  "version": "0.0.10",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev": "webpack --mode development --watch",
     "build": "webpack --mode production",
     "format": "npx prettier --write .",
-    "clean": "npx rimraf main.js"
+    "clean": "npx rimraf main.js",
+    "test": "cross-env TS_NODE_COMPILER_OPTIONS={\\\"module\\\":\\\"commonjs\\\"} mocha -r ts-node/register 'tests/**/*.ts'"
   },
   "source": "main.ts",
   "keywords": [],
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
+    "@types/chai": "^4.2.22",
+    "@types/mime-types": "^2.1.1",
+    "@types/mocha": "^9.0.0",
     "@types/node": "^14.14.37",
-    "parcel": "^2.0.0",
+    "chai": "^4.3.4",
+    "cross-env": "^7.0.3",
+    "mocha": "^9.1.3",
     "prettier": "^2.4.1",
     "terser-webpack-plugin": "^5.2.4",
     "ts-loader": "^9.2.6",
+    "ts-node": "^10.4.0",
     "tslib": "^2.2.0",
     "typescript": "^4.4.4",
     "webdav-server": "^2.6.2",
@@ -27,12 +34,18 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.37.0",
     "@aws-sdk/signature-v4-crt": "^3.37.0",
-    "@types/mime-types": "^2.1.1",
+    "acorn": "^8.5.0",
     "aws-crt": "^1.10.1",
+    "buffer": "^6.0.3",
     "codemirror": "^5.63.1",
+    "lovefield-ts": "^0.7.0",
     "mime-types": "^2.1.33",
     "obsidian": "^0.12.0",
+    "path-browserify": "^1.0.1",
+    "process": "^0.11.10",
+    "rfc4648": "^1.5.0",
     "rimraf": "^3.0.2",
+    "stream-browserify": "^3.0.0",
     "webdav": "^4.7.0",
     "webdav-fs": "^4.0.0"
   }

src/debugMode.ts

@@ -0,0 +1,37 @@
+import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
+
+import * as lf from "lovefield-ts/dist/es6/lf.js";
+
+import type { SyncPlanType } from "./sync";
+import {
+  insertSyncPlanRecord,
+  clearAllSyncPlanRecords,
+  readAllSyncPlanRecordTexts,
+} from "./localdb";
+import { mkdirpInVault } from "./misc";
+
+const DEFAULT_DEBUG_FOLDER = "_debug_save_remote/";
+const DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX = "sync_plans_hist_exported_on_";
+
+export const exportSyncPlansToFiles = async (
+  db: lf.DatabaseConnection,
+  vault: Vault
+) => {
+  console.log("exporting");
+  await mkdirpInVault(DEFAULT_DEBUG_FOLDER, vault);
+  const records = await readAllSyncPlanRecordTexts(db);
+  let md = "";
+  if (records.length === 0) {
+    md = "No sync plans history found";
+  } else {
+    md =
+      "Sync plans found:\n\n" +
+      records.map((x) => "```json\n" + x + "\n```\n").join("\n");
+  }
+  const ts = Date.now();
+  const filePath = `${DEFAULT_DEBUG_FOLDER}${DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX}${ts}.md`;
+  await vault.create(filePath, md, {
+    mtime: ts,
+  });
+  console.log("finish exporting");
+};

src/encrypt.ts

@@ -0,0 +1,135 @@
+import { base32, base64 } from "rfc4648";
+import {
+  bufferToArrayBuffer,
+  arrayBufferToBuffer,
+  hexStringToTypedArray,
+  arrayBufferToHex,
+} from "./misc";
+
+const DEFAULT_ITER = 10000;
+
+const getKeyIVFromPassword = async (
+  salt: Uint8Array,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  const k1 = await window.crypto.subtle.importKey(
+    "raw",
+    new TextEncoder().encode(password),
+    { name: "PBKDF2" },
+    false,
+    ["deriveKey", "deriveBits"]
+  );
+
+  const k2 = await window.crypto.subtle.deriveBits(
+    {
+      name: "PBKDF2",
+      salt: salt,
+      iterations: rounds,
+      hash: "SHA-256",
+    },
+    k1,
+    256 + 128
+  );
+
+  return k2;
+};
+
+export const encryptArrayBuffer = async (
+  arrBuf: ArrayBuffer,
+  password: string,
+  rounds: number = DEFAULT_ITER,
+  saltHex: string = ""
+) => {
+  let salt: Uint8Array;
+  if (saltHex !== "") {
+    salt = hexStringToTypedArray(saltHex);
+  } else {
+    salt = window.crypto.getRandomValues(new Uint8Array(8));
+  }
+
+  const derivedKey = await getKeyIVFromPassword(salt, password, rounds);
+  const key = derivedKey.slice(0, 32);
+  const iv = derivedKey.slice(32, 32 + 16);
+
+  const keyCrypt = await window.crypto.subtle.importKey(
+    "raw",
+    key,
+    { name: "AES-CBC" },
+    false,
+    ["encrypt", "decrypt"]
+  );
+
+  const enc = (await window.crypto.subtle.encrypt(
+    { name: "AES-CBC", iv },
+    keyCrypt,
+    arrBuf
+  )) as ArrayBuffer;
+
+  const prefix = new TextEncoder().encode("Salted__");
+
+  const res = new Uint8Array([...prefix, ...salt, ...new Uint8Array(enc)]);
+
+  return bufferToArrayBuffer(res);
+};
+
+export const decryptArrayBuffer = async (
+  arrBuf: ArrayBuffer,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  const prefix = arrBuf.slice(0, 8);
+  const salt = arrBuf.slice(8, 16);
+  const derivedKey = await getKeyIVFromPassword(
+    new Uint8Array(salt),
+    password,
+    rounds
+  );
+  const key = derivedKey.slice(0, 32);
+  const iv = derivedKey.slice(32, 32 + 16);
+
+  const keyCrypt = await window.crypto.subtle.importKey(
+    "raw",
+    key,
+    { name: "AES-CBC" },
+    false,
+    ["encrypt", "decrypt"]
+  );
+
+  const dec = (await window.crypto.subtle.decrypt(
+    { name: "AES-CBC", iv },
+    keyCrypt,
+    arrBuf.slice(16)
+  )) as ArrayBuffer;
+
+  return dec;
+};
+
+export const encryptStringToBase32 = async (
+  text: string,
+  password: string,
+  rounds: number = DEFAULT_ITER,
+  saltHex: string = ""
+) => {
+  const enc = await encryptArrayBuffer(
+    bufferToArrayBuffer(new TextEncoder().encode(text)),
+    password,
+    rounds,
+    saltHex
+  );
+  return base32.stringify(new Uint8Array(enc));
+};
+
+export const decryptBase32ToString = async (
+  text: string,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  return new TextDecoder().decode(
+    await decryptArrayBuffer(
+      bufferToArrayBuffer(base32.parse(text)),
+      password,
+      rounds
+    )
+  );
+};

src/localdb.ts

@@ -0,0 +1,311 @@
+import * as lf from "lovefield-ts/dist/es6/lf.js";
+import { TAbstractFile, TFile, TFolder } from "obsidian";
+
+import type { SUPPORTED_SERVICES_TYPE } from "./misc";
+import type { SyncPlanType } from "./sync";
+
+export type DatabaseConnection = lf.DatabaseConnection;
+
+export const DEFAULT_DB_NAME = "saveremotedb";
+export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory";
+export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
+export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
+
+export interface FileFolderHistoryRecord {
+  key: string;
+  ctime: number;
+  mtime: number;
+  size: number;
+  action_when: number;
+  action_type: "delete" | "rename";
+  key_type: "folder" | "file";
+  rename_to: string;
+}
+
+export interface SyncMetaMappingRecord {
+  local_key: string;
+  remote_key: string;
+  local_size: number;
+  remote_size: number;
+  local_mtime: number;
+  remote_mtime: number;
+  remote_extra_key: string;
+  remote_type: SUPPORTED_SERVICES_TYPE;
+  key_type: "folder" | "file";
+}
+
+interface SyncPlanRecord {
+  ts: number;
+  remote_type: string;
+  sync_plan: string;
+}
+
+export const prepareDBs = async () => {
+  const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1);
+  schemaBuilder
+    .createTable(DEFAULT_TBL_DELETE_HISTORY)
+    .addColumn("id", lf.Type.INTEGER)
+    .addColumn("key", lf.Type.STRING)
+    .addColumn("ctime", lf.Type.INTEGER)
+    .addColumn("mtime", lf.Type.INTEGER)
+    .addColumn("size", lf.Type.INTEGER)
+    .addColumn("action_when", lf.Type.INTEGER)
+    .addColumn("action_type", lf.Type.STRING)
+    .addColumn("key_type", lf.Type.STRING)
+    .addPrimaryKey(["id"], true)
+    .addIndex("idxKey", ["key"]);
+
+  schemaBuilder
+    .createTable(DEFAULT_TBL_SYNC_MAPPING)
+    .addColumn("id", lf.Type.INTEGER)
+    .addColumn("local_key", lf.Type.STRING)
+    .addColumn("remote_key", lf.Type.STRING)
+    .addColumn("local_size", lf.Type.INTEGER)
+    .addColumn("remote_size", lf.Type.INTEGER)
+    .addColumn("local_mtime", lf.Type.INTEGER)
+    .addColumn("remote_mtime", lf.Type.INTEGER)
+    .addColumn("key_type", lf.Type.STRING)
+    .addColumn("remote_extra_key", lf.Type.STRING)
+    .addColumn("remote_type", lf.Type.STRING)
+    .addNullable([
+      "remote_extra_key",
+      "remote_mtime",
+      "remote_size",
+      "local_mtime",
+    ])
+    .addPrimaryKey(["id"], true)
+    .addIndex("idxkey", ["local_key", "remote_key"]);
+
+  schemaBuilder
+    .createTable(DEFAULT_SYNC_PLANS_HISTORY)
+    .addColumn("id", lf.Type.INTEGER)
+    .addColumn("ts", lf.Type.INTEGER)
+    .addColumn("remote_type", lf.Type.STRING)
+    .addColumn("sync_plan", lf.Type.STRING)
+    .addPrimaryKey(["id"], true)
+    .addIndex("tskey", ["ts"]);
+
+  const db = await schemaBuilder.connect({
+    storeType: lf.DataStoreType.INDEXED_DB,
+  });
+  console.log("db connected");
+  return db;
+};
+
+export const destroyDBs = async (db: lf.DatabaseConnection) => {
+  db.close();
+  const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME);
+  req.onsuccess = (event) => {
+    console.log("db deleted");
+  };
+  req.onblocked = (event) => {
+    console.warn("trying to delete db but it was blocked");
+  };
+  req.onerror = (event) => {
+    console.error("tried to delete db but something bad!");
+    console.error(event);
+  };
+};
+
+export const loadDeleteRenameHistoryTable = async (
+  db: lf.DatabaseConnection
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+
+  const records = await db
+    .select()
+    .from(schema)
+    .orderBy(schema.col("action_when"), lf.Order.ASC)
+    .exec();
+
+  return records as FileFolderHistoryRecord[];
+};
+
+export const clearDeleteRenameHistoryOfKey = async (
+  db: lf.DatabaseConnection,
+  key: string
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+
+  await db.delete().from(tbl).where(tbl.col("key").eq(key)).exec();
+};
+
+export const insertDeleteRecord = async (
+  db: lf.DatabaseConnection,
+  fileOrFolder: TAbstractFile
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  // console.log(fileOrFolder);
+  let k: FileFolderHistoryRecord;
+  if (fileOrFolder instanceof TFile) {
+    k = {
+      key: fileOrFolder.path,
+      ctime: fileOrFolder.stat.ctime,
+      mtime: fileOrFolder.stat.mtime,
+      size: fileOrFolder.stat.size,
+      action_when: Date.now(),
+      action_type: "delete",
+      key_type: "file",
+      rename_to: "",
+    };
+  } else if (fileOrFolder instanceof TFolder) {
+    // key should endswith "/"
+    const key = fileOrFolder.path.endsWith("/")
+      ? fileOrFolder.path
+      : `${fileOrFolder.path}/`;
+    k = {
+      key: key,
+      ctime: 0,
+      mtime: 0,
+      size: 0,
+      action_when: Date.now(),
+      action_type: "delete",
+      key_type: "folder",
+      rename_to: "",
+    };
+  }
+  const row = tbl.createRow(k);
+  await db.insertOrReplace().into(tbl).values([row]).exec();
+};
+
+export const insertRenameRecord = async (
+  db: lf.DatabaseConnection,
+  fileOrFolder: TAbstractFile,
+  oldPath: string
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  // console.log(fileOrFolder);
+  let k: FileFolderHistoryRecord;
+  if (fileOrFolder instanceof TFile) {
+    k = {
+      key: oldPath,
+      ctime: fileOrFolder.stat.ctime,
+      mtime: fileOrFolder.stat.mtime,
+      size: fileOrFolder.stat.size,
+      action_when: Date.now(),
+      action_type: "rename",
+      key_type: "file",
+      rename_to: fileOrFolder.path,
+    };
+  } else if (fileOrFolder instanceof TFolder) {
+    const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
+    const renameTo = fileOrFolder.path.endsWith("/")
+      ? fileOrFolder.path
+      : `${fileOrFolder.path}/`;
+    k = {
+      key: key,
+      ctime: 0,
+      mtime: 0,
+      size: 0,
+      action_when: Date.now(),
+      action_type: "rename",
+      key_type: "folder",
+      rename_to: renameTo,
+    };
+  }
+  const row = tbl.createRow(k);
+  await db.insertOrReplace().into(tbl).values([row]).exec();
+};
+
+export const getAllDeleteRenameRecords = async (db: lf.DatabaseConnection) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const res1 = await db.select().from(schema).exec();
+  const res2 = res1 as FileFolderHistoryRecord[];
+  return res2;
+};
+
+export const upsertSyncMetaMappingDataS3 = async (
+  db: lf.DatabaseConnection,
+  localKey: string,
+  localMTime: number,
+  localSize: number,
+  remoteKey: string,
+  remoteMTime: number,
+  remoteSize: number,
+  remoteExtraKey: string /* ETag from s3 */
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
+  const aggregratedInfo: SyncMetaMappingRecord = {
+    local_key: localKey,
+    local_mtime: localMTime,
+    local_size: localSize,
+    remote_key: remoteKey,
+    remote_mtime: remoteMTime,
+    remote_size: remoteSize,
+    remote_extra_key: remoteExtraKey,
+    remote_type: "s3",
+    key_type: localKey.endsWith("/") ? "folder" : "file",
+  };
+  const row = schema.createRow(aggregratedInfo);
+  await db.insertOrReplace().into(schema).values([row]).exec();
+};
+
+export const getSyncMetaMappingByRemoteKeyS3 = async (
+  db: lf.DatabaseConnection,
+  remoteKey: string,
+  remoteMTime: number,
+  remoteExtraKey: string
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
+  const tbl = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
+  const res = (await db
+    .select()
+    .from(tbl)
+    .where(
+      lf.op.and(
+        tbl.col("remote_key").eq(remoteKey),
+        tbl.col("remote_mtime").eq(remoteMTime),
+        tbl.col("remote_extra_key").eq(remoteExtraKey),
+        tbl.col("remote_type").eq("s3")
+      )
+    )
+    .exec()) as SyncMetaMappingRecord[];
+
+  if (res.length === 1) {
+    return res[0];
+  }
+
+  if (res.length === 0) {
+    return undefined;
+  }
+
+  throw Error("something bad in sync meta mapping!");
+};
+
+export const insertSyncPlanRecord = async (
+  db: lf.DatabaseConnection,
+  syncPlan: SyncPlanType
+) => {
+  const schema = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+  const row = schema.createRow({
+    ts: syncPlan.ts,
+    remote_type: syncPlan.remoteType,
+    sync_plan: JSON.stringify(syncPlan, null, 2),
+  } as SyncPlanRecord);
+  await db.insertOrReplace().into(schema).values([row]).exec();
+};
+
+export const clearAllSyncPlanRecords = async (db: lf.DatabaseConnection) => {
+  const tbl = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+  await db.delete().from(tbl).exec();
+};
+
+export const readAllSyncPlanRecordTexts = async (db: lf.DatabaseConnection) => {
+  const schema = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+
+  const records = (await db
+    .select()
+    .from(schema)
+    .orderBy(schema.col("ts"), lf.Order.DESC)
+    .exec()) as SyncPlanRecord[];
+
+  if (records === undefined) {
+    return [] as string[];
+  } else {
+    return records.map((x) => x.sync_plan);
+  }
+};

src/main.ts

@@ -0,0 +1,288 @@
+import {
+  App,
+  Modal,
+  Notice,
+  Plugin,
+  PluginSettingTab,
+  Setting,
+  request,
+  Platform,
+  TFile,
+  TFolder,
+} from "obsidian";
+import * as CodeMirror from "codemirror";
+import { clearAllSyncPlanRecords, DatabaseConnection } from "./localdb";
+import {
+  prepareDBs,
+  destroyDBs,
+  loadDeleteRenameHistoryTable,
+  insertDeleteRecord,
+  insertRenameRecord,
+  getAllDeleteRenameRecords,
+  insertSyncPlanRecord,
+} from "./localdb";
+
+import type { SyncStatusType } from "./sync";
+import { getSyncPlan, doActualSync } from "./sync";
+import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
+import { exportSyncPlansToFiles } from "./debugMode";
+
+interface SaveRemotePluginSettings {
+  s3?: S3Config;
+  password?: string;
+}
+
+const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
+  s3: DEFAULT_S3_CONFIG,
+  password: "",
+};
+
+export default class SaveRemotePlugin extends Plugin {
+  settings: SaveRemotePluginSettings;
+  cm: CodeMirror.Editor;
+  db: DatabaseConnection;
+  syncStatus: SyncStatusType;
+
+  async onload() {
+    console.log("loading plugin obsidian-save-remote");
+
+    await this.loadSettings();
+
+    await this.prepareDB();
+
+    this.syncStatus = "idle";
+
+    this.registerEvent(
+      this.app.vault.on("delete", async (fileOrFolder) => {
+        await insertDeleteRecord(this.db, fileOrFolder);
+      })
+    );
+
+    this.registerEvent(
+      this.app.vault.on("rename", async (fileOrFolder, oldPath) => {
+        await insertRenameRecord(this.db, fileOrFolder, oldPath);
+      })
+    );
+
+    this.addRibbonIcon("switch", "Save Remote", async () => {
+      if (this.syncStatus !== "idle") {
+        new Notice(`Save Remote already running in stage ${this.syncStatus}!`);
+        return;
+      }
+
+      try {
+        new Notice("1/6 Save Remote Sync Preparing");
+        this.syncStatus = "preparing";
+
+        new Notice("2/6 Starting to fetch remote meta data.");
+        this.syncStatus = "getting_remote_meta";
+        const s3Client = getS3Client(this.settings.s3);
+        const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
+
+        new Notice("3/6 Starting to fetch local meta data.");
+        this.syncStatus = "getting_local_meta";
+        const local = this.app.vault.getAllLoadedFiles();
+        const localHistory = await loadDeleteRenameHistoryTable(this.db);
+        // console.log(remoteRsp);
+        // console.log(local);
+        // console.log(localHistory);
+
+        new Notice("4/6 Starting to generate sync plan.");
+        this.syncStatus = "generating_plan";
+        const syncPlan = await getSyncPlan(
+          remoteRsp.Contents,
+          local,
+          localHistory,
+          this.db,
+          this.settings.password
+        );
+        console.log(syncPlan.mixedStates); // for debugging
+        await insertSyncPlanRecord(this.db, syncPlan);
+
+        // The operations above are read only and kind of safe.
+        // The operations below begins to write or delete (!!!) something.
+
+        new Notice("5/6 Save Remote Sync data exchanging!");
+
+        this.syncStatus = "syncing";
+        await doActualSync(
+          s3Client,
+          this.settings.s3,
+          this.db,
+          this.app.vault,
+          syncPlan,
+          this.settings.password
+        );
+
+        new Notice("6/6 Save Remote finish!");
+        this.syncStatus = "finish";
+        this.syncStatus = "idle";
+      } catch (error) {
+        const msg = `Save Remote error while ${this.syncStatus}`;
+        console.log(msg);
+        console.log(error);
+        new Notice(msg);
+        new Notice(error);
+        this.syncStatus = "idle";
+      }
+    });
+
+    this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
+
+    this.registerCodeMirror((cm: CodeMirror.Editor) => {
+      this.cm = cm;
+      console.log("codemirror registered.");
+    });
+
+    // this.registerDomEvent(document, "click", (evt: MouseEvent) => {
+    //   console.log("click", evt);
+    // });
+
+    // this.registerInterval(
+    //   window.setInterval(() => console.log("setInterval"), 5 * 60 * 1000)
+    // );
+  }
+
+  onunload() {
+    console.log("unloading plugin obsidian-save-remote");
+    this.destroyDBs();
+  }
+
+  async loadSettings() {
+    this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
+  }
+
+  async saveSettings() {
+    await this.saveData(this.settings);
+  }
+
+  async prepareDB() {
+    this.db = await prepareDBs();
+  }
+
+  destroyDBs() {
+    /* destroyDBs(this.db); */
+  }
+}
+
+class SaveRemoteSettingTab extends PluginSettingTab {
+  plugin: SaveRemotePlugin;
+
+  constructor(app: App, plugin: SaveRemotePlugin) {
+    super(app, plugin);
+    this.plugin = plugin;
+  }
+
+  display(): void {
+    let { containerEl } = this;
+
+    containerEl.empty();
+
+    containerEl.createEl("h1", { text: "Save Remote" });
+
+    containerEl.createEl("h2", { text: "S3" });
+
+    new Setting(containerEl)
+      .setName("s3Endpoint")
+      .setDesc("s3Endpoint")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(this.plugin.settings.s3.s3Endpoint)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3Endpoint = value.trim();
+            await this.plugin.saveSettings();
+          })
+      );
+
+    new Setting(containerEl)
+      .setName("s3Region")
+      .setDesc("s3Region")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.s3.s3Region}`)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3Region = value.trim();
+            await this.plugin.saveSettings();
+          })
+      );
+
+    new Setting(containerEl)
+      .setName("s3AccessKeyID")
+      .setDesc("s3AccessKeyID")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.s3.s3AccessKeyID}`)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3AccessKeyID = value.trim();
+            await this.plugin.saveSettings();
+          })
+      );
+
+    new Setting(containerEl)
+      .setName("s3SecretAccessKey")
+      .setDesc("s3SecretAccessKey")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3SecretAccessKey = value.trim();
+            await this.plugin.saveSettings();
+          })
+      );
+
+    new Setting(containerEl)
+      .setName("s3BucketName")
+      .setDesc("s3BucketName")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.s3.s3BucketName}`)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3BucketName = value.trim();
+            await this.plugin.saveSettings();
+          })
+      );
+
+    containerEl.createEl("h2", { text: "General" });
+    new Setting(containerEl)
+      .setName("encryption password")
+      .setDesc("Password for E2E encryption. Empty for no password.")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.password}`)
+          .onChange(async (value) => {
+            this.plugin.settings.password = value.trim();
+            await this.plugin.saveSettings();
+          })
+      );
+
+    containerEl.createEl("h2", { text: "Debug" });
+
+    new Setting(containerEl)
+      .setName("export sync plans")
+      .setDesc("export sync plans")
+      .addButton(async (button) => {
+        button.setButtonText("Export");
+        button.onClick(async () => {
+          await exportSyncPlansToFiles(this.plugin.db, this.app.vault);
+          new Notice("sync plans history exported");
+        });
+      });
+
+    new Setting(containerEl)
+      .setName("delete sync plans history in db")
+      .setDesc("delete sync plans history in db")
+      .addButton(async (button) => {
+        button.setButtonText("Delete History");
+        button.onClick(async () => {
+          await clearAllSyncPlanRecords(this.plugin.db);
+          new Notice("sync plans history (in db) deleted");
+        });
+      });
+  }
+}

src/misc.ts

@@ -0,0 +1,113 @@
+import { Vault } from "obsidian";
+import * as path from "path";
+
+import { base32 } from "rfc4648";
+
+export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "ftp";
+
+/**
+ * If any part of the file starts with '.' or '_' then it's a hidden file.
+ * @param item
+ * @param loose
+ * @returns
+ */
+export const isHiddenPath = (item: string, loose: boolean = true) => {
+  const k = path.posix.normalize(item); // TODO: only unix path now
+  const k2 = k.split("/"); // TODO: only unix path now
+  // console.log(k2)
+  for (const singlePart of k2) {
+    if (singlePart === "." || singlePart === ".." || singlePart === "") {
+      continue;
+    }
+    if (singlePart[0] === ".") {
+      return true;
+    }
+    if (loose && singlePart[0] === "_") {
+      return true;
+    }
+  }
+  return false;
+};
+
+/**
+ * Util func for mkdir -p based on the "path" of original file or folder
+ * "a/b/c/" => ["a", "a/b", "a/b/c"]
+ * "a/b/c/d/e.txt" => ["a", "a/b", "a/b/c", "a/b/c/d"]
+ * @param x string
+ * @returns string[] might be empty
+ */
+export const getFolderLevels = (x: string) => {
+  const res: string[] = [];
+
+  if (x === "" || x === "/") {
+    return res;
+  }
+
+  const y1 = x.split("/");
+  let i = 0;
+  for (let index = 0; index + 1 < y1.length; index++) {
+    res.push(y1.slice(0, index + 1).join("/"));
+  }
+  return res;
+};
+
+export const mkdirpInVault = async (thePath: string, vault: Vault) => {
+  // console.log(thePath);
+  const foldersToBuild = getFolderLevels(thePath);
+  // console.log(foldersToBuild);
+  for (const folder of foldersToBuild) {
+    const r = await vault.adapter.exists(folder);
+    // console.log(r);
+    if (!r) {
+      console.log(`mkdir ${folder}`);
+      await vault.adapter.mkdir(folder);
+    }
+  }
+};
+
+/**
+ * https://stackoverflow.com/questions/8609289
+ * @param b Buffer
+ * @returns ArrayBuffer
+ */
+export const bufferToArrayBuffer = (b: Buffer | Uint8Array) => {
+  return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
+};
+
+/**
+ * Simple func.
+ * @param b
+ * @returns
+ */
+export const arrayBufferToBuffer = (b: ArrayBuffer) => {
+  return Buffer.from(b);
+};
+
+export const arrayBufferToBase64 = (b: ArrayBuffer) => {
+  return arrayBufferToBuffer(b).toString("base64");
+};
+
+export const arrayBufferToHex = (b: ArrayBuffer) => {
+  return arrayBufferToBuffer(b).toString("hex");
+};
+
+export const base64ToArrayBuffer = (b64text: string) => {
+  return bufferToArrayBuffer(Buffer.from(b64text, "base64"));
+};
+
+/**
+ * https://stackoverflow.com/questions/43131242
+ * @param hex
+ * @returns
+ */
+export const hexStringToTypedArray = (hex: string) => {
+  return new Uint8Array(
+    hex.match(/[\da-f]{2}/gi).map(function (h) {
+      return parseInt(h, 16);
+    })
+  );
+};
+
+export const base64ToBase32 = (a: string) => {
+  return base32.stringify(Buffer.from(a, "base64"));
+};

src/s3.ts

@@ -0,0 +1,268 @@
+import { Buffer } from "buffer";
+import { Readable } from "stream";
+
+import { Vault } from "obsidian";
+
+import {
+  S3Client,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  GetObjectCommand,
+  DeleteObjectCommand,
+  HeadObjectCommand,
+} from "@aws-sdk/client-s3";
+
+import type { _Object } from "@aws-sdk/client-s3";
+
+import {
+  arrayBufferToBuffer,
+  bufferToArrayBuffer,
+  mkdirpInVault,
+} from "./misc";
+import * as mime from "mime-types";
+import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
+
+export interface S3Config {
+  s3Endpoint: string;
+  s3Region: string;
+  s3AccessKeyID: string;
+  s3SecretAccessKey: string;
+  s3BucketName: string;
+}
+
+export const DEFAULT_S3_CONFIG = {
+  s3Endpoint: "",
+  s3Region: "",
+  s3AccessKeyID: "",
+  s3SecretAccessKey: "",
+  s3BucketName: "",
+};
+
+export type S3ObjectType = _Object;
+
+export const getS3Client = (s3Config: S3Config) => {
+  const s3Client = new S3Client({
+    region: s3Config.s3Region,
+    endpoint: s3Config.s3Endpoint,
+    credentials: {
+      accessKeyId: s3Config.s3AccessKeyID,
+      secretAccessKey: s3Config.s3SecretAccessKey,
+    },
+  });
+  return s3Client;
+};
+
+export const getRemoteMeta = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string
+) => {
+  return await s3Client.send(
+    new HeadObjectCommand({
+      Bucket: s3Config.s3BucketName,
+      Key: fileOrFolderPath,
+    })
+  );
+};
+
+export const uploadToRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string,
+  vault: Vault,
+  isRecursively: boolean = false,
+  password: string = "",
+  remoteEncryptedKey: string = ""
+) => {
+  let uploadFile = fileOrFolderPath;
+  if (password !== "") {
+    uploadFile = remoteEncryptedKey;
+  }
+  const isFolder = fileOrFolderPath.endsWith("/");
+
+  const DEFAULT_CONTENT_TYPE = "application/octet-stream";
+
+  if (isFolder && isRecursively) {
+    throw Error("upload function doesn't implement recursive function yet!");
+  } else if (isFolder && !isRecursively) {
+    // folder
+    const contentType = DEFAULT_CONTENT_TYPE;
+    await s3Client.send(
+      new PutObjectCommand({
+        Bucket: s3Config.s3BucketName,
+        Key: uploadFile,
+        Body: "",
+        ContentType: contentType,
+      })
+    );
+    return await getRemoteMeta(s3Client, s3Config, fileOrFolderPath);
+  } else {
+    // file
+    // we ignore isRecursively parameter here
+    let contentType = DEFAULT_CONTENT_TYPE;
+    if (password === "") {
+      contentType =
+        mime.contentType(
+          mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE
+        ) || DEFAULT_CONTENT_TYPE;
+    }
+    const localContent = await vault.adapter.readBinary(fileOrFolderPath);
+    let remoteContent = localContent;
+    if (password !== "") {
+      remoteContent = await encryptArrayBuffer(localContent, password);
+    }
+    const body = arrayBufferToBuffer(remoteContent);
+    await s3Client.send(
+      new PutObjectCommand({
+        Bucket: s3Config.s3BucketName,
+        Key: uploadFile,
+        Body: body,
+        ContentType: contentType,
+      })
+    );
+    return await getRemoteMeta(s3Client, s3Config, uploadFile);
+  }
+};
+
+export const listFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  prefix?: string
+) => {
+  if (prefix !== undefined) {
+    return await s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: s3Config.s3BucketName,
+        Prefix: prefix,
+      })
+    );
+  }
+  return await s3Client.send(
+    new ListObjectsV2Command({ Bucket: s3Config.s3BucketName })
+  );
+};
+
+/**
+ * The Body of resp of aws GetObject has mix types
+ * and we want to get ArrayBuffer here.
+ * See https://github.com/aws/aws-sdk-js-v3/issues/1877
+ * @param b The Body of GetObject
+ * @returns Promise<ArrayBuffer>
+ */
+const getObjectBodyToArrayBuffer = async (
+  b: Readable | ReadableStream | Blob
+) => {
+  if (b instanceof Readable) {
+    const chunks: Uint8Array[] = [];
+    for await (let chunk of b) {
+      chunks.push(chunk);
+    }
+    const buf = Buffer.concat(chunks);
+    return bufferToArrayBuffer(buf);
+  } else if (b instanceof ReadableStream) {
+    return await new Response(b, {}).arrayBuffer();
+  } else if (b instanceof Blob) {
+    return await b.arrayBuffer();
+  } else {
+    throw TypeError(`The type of ${b} is not one of the supported types`);
+  }
+};
+
+export const downloadFromRemoteRaw = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string
+) => {
+  const data = await s3Client.send(
+    new GetObjectCommand({
+      Bucket: s3Config.s3BucketName,
+      Key: fileOrFolderPath,
+    })
+  );
+  const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
+  return bodyContents;
+};
+
+export const downloadFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string,
+  vault: Vault,
+  mtime: number,
+  password: string = "",
+  remoteEncryptedKey: string = ""
+) => {
+  const isFolder = fileOrFolderPath.endsWith("/");
+
+  await mkdirpInVault(fileOrFolderPath, vault);
+
+  // the file is always local file
+  // we need to encrypt it
+
+  if (isFolder) {
+    // mkdirp locally is enough
+    // do nothing here
+  } else {
+    let downloadFile = fileOrFolderPath;
+    if (password !== "") {
+      downloadFile = remoteEncryptedKey;
+    }
+    const remoteContent = await downloadFromRemoteRaw(
+      s3Client,
+      s3Config,
+      downloadFile
+    );
+    let localContent = remoteContent;
+    if (password !== "") {
+      localContent = await decryptArrayBuffer(remoteContent, password);
+    }
+    await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
+      mtime: mtime,
+    });
+  }
+};
+
+/**
+ * This function deals with file normally and "folder" recursively.
+ * @param s3Client
+ * @param s3Config
+ * @param fileOrFolderPath
+ * @returns
+ */
+export const deleteFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string,
+  password: string = "",
+  remoteEncryptedKey: string = ""
+) => {
+  if (fileOrFolderPath === "/") {
+    return;
+  }
+  let remoteFileName = fileOrFolderPath;
+  if (password !== "") {
+    remoteFileName = remoteEncryptedKey;
+  }
+  await s3Client.send(
+    new DeleteObjectCommand({
+      Bucket: s3Config.s3BucketName,
+      Key: remoteFileName,
+    })
+  );
+
+  if (fileOrFolderPath.endsWith("/") && password === "") {
+    const x = await listFromRemote(s3Client, s3Config, fileOrFolderPath);
+    x.Contents.forEach(async (element) => {
+      await s3Client.send(
+        new DeleteObjectCommand({
+          Bucket: s3Config.s3BucketName,
+          Key: element.Key,
+        })
+      );
+    });
+  } else if (fileOrFolderPath.endsWith("/") && password !== "") {
+    // TODO
+  } else {
+    // pass
+  }
+};

src/sync.ts

@@ -0,0 +1,458 @@
+import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
+
+import { S3Client } from "@aws-sdk/client-s3";
+import * as lf from "lovefield-ts/dist/es6/lf.js";
+
+import {
+  clearDeleteRenameHistoryOfKey,
+  FileFolderHistoryRecord,
+  upsertSyncMetaMappingDataS3,
+  getSyncMetaMappingByRemoteKeyS3,
+} from "./localdb";
+import {
+  S3Config,
+  S3ObjectType,
+  uploadToRemote,
+  deleteFromRemote,
+  downloadFromRemote,
+} from "./s3";
+import { mkdirpInVault, SUPPORTED_SERVICES_TYPE, isHiddenPath } from "./misc";
+import { decryptBase32ToString, encryptStringToBase32 } from "./encrypt";
+
+export type SyncStatusType =
+  | "idle"
+  | "preparing"
+  | "getting_remote_meta"
+  | "getting_local_meta"
+  | "generating_plan"
+  | "syncing"
+  | "finish";
+
+type DecisionType =
+  | "undecided"
+  | "unknown"
+  | "upload_clearhist"
+  | "download_clearhist"
+  | "delremote_clearhist"
+  | "download"
+  | "upload"
+  | "clearhist"
+  | "mkdirplocal"
+  | "skip";
+
+interface FileOrFolderMixedState {
+  key: string;
+  exist_local?: boolean;
+  exist_remote?: boolean;
+  mtime_local?: number;
+  mtime_remote?: number;
+  delete_time_local?: number;
+  size_local?: number;
+  size_remote?: number;
+  decision?: DecisionType;
+  syncDone?: "done";
+  decision_branch?: number;
+  remote_encrypted_key?: string;
+}
+
+export interface SyncPlanType {
+  ts: number;
+  remoteType: SUPPORTED_SERVICES_TYPE;
+  mixedStates: Record<string, FileOrFolderMixedState>;
+}
+
+const ensembleMixedStates = async (
+  remote: S3ObjectType[],
+  local: TAbstractFile[],
+  deleteHistory: FileFolderHistoryRecord[],
+  db: lf.DatabaseConnection,
+  password: string = ""
+) => {
+  const results = {} as Record<string, FileOrFolderMixedState>;
+
+  if (remote !== undefined) {
+    for (const entry of remote) {
+      const remoteEncryptedKey = entry.Key;
+      let key = remoteEncryptedKey;
+      if (password !== "") {
+        key = await decryptBase32ToString(remoteEncryptedKey, password);
+      }
+      const backwardMapping = await getSyncMetaMappingByRemoteKeyS3(
+        db,
+        key,
+        entry.LastModified.valueOf(),
+        entry.ETag
+      );
+
+      let r = {} as FileOrFolderMixedState;
+      if (backwardMapping !== undefined) {
+        key = backwardMapping.local_key;
+        r = {
+          key: key,
+          exist_remote: true,
+          mtime_remote: backwardMapping.local_mtime,
+          size_remote: backwardMapping.local_size,
+          remote_encrypted_key: remoteEncryptedKey,
+        };
+      } else {
+        r = {
+          key: key,
+          exist_remote: true,
+          mtime_remote: entry.LastModified.valueOf(),
+          size_remote: entry.Size,
+          remote_encrypted_key: remoteEncryptedKey,
+        };
+      }
+      if (isHiddenPath(key)) {
+        continue;
+      }
+      if (results.hasOwnProperty(key)) {
+        results[key].key = r.key;
+        results[key].exist_remote = r.exist_remote;
+        results[key].mtime_remote = r.mtime_remote;
+        results[key].size_remote = r.size_remote;
+        results[key].remote_encrypted_key = r.remote_encrypted_key;
+      } else {
+        results[key] = r;
+      }
+    }
+  }
+
+  for (const entry of local) {
+    let r = {} as FileOrFolderMixedState;
+    let key = entry.path;
+
+    if (entry.path === "/") {
+      // ignore
+      continue;
+    } else if (entry instanceof TFile) {
+      r = {
+        key: entry.path,
+        exist_local: true,
+        mtime_local: entry.stat.mtime,
+        size_local: entry.stat.size,
+      };
+    } else if (entry instanceof TFolder) {
+      key = `${entry.path}/`;
+      r = {
+        key: key,
+        exist_local: true,
+        mtime_local: undefined,
+        size_local: 0,
+      };
+    } else {
+      throw Error(`unexpected ${entry}`);
+    }
+
+    if (isHiddenPath(key)) {
+      continue;
+    }
+    if (results.hasOwnProperty(key)) {
+      results[key].key = r.key;
+      results[key].exist_local = r.exist_local;
+      results[key].mtime_local = r.mtime_local;
+      results[key].size_local = r.size_local;
+    } else {
+      results[key] = r;
+    }
+  }
+
+  for (const entry of deleteHistory) {
+    let key = entry.key;
+    if (entry.key_type === "folder") {
+      if (!entry.key.endsWith("/")) {
+        key = `${entry.key}/`;
+      }
+    } else if (entry.key_type === "file") {
+      // pass
+    } else {
+      throw Error(`unexpected ${entry}`);
+    }
+
+    const r = {
+      key: key,
+      delete_time_local: entry.action_when,
+    } as FileOrFolderMixedState;
+
+    if (isHiddenPath(key)) {
+      continue;
+    }
+    if (results.hasOwnProperty(key)) {
+      results[key].key = r.key;
+      results[key].delete_time_local = r.delete_time_local;
+    } else {
+      results[key] = r;
+    }
+  }
+
+  return results;
+};
+
+const getOperation = (
+  origRecord: FileOrFolderMixedState,
+  inplace: boolean = false,
+  password: string = ""
+) => {
+  let r = origRecord;
+  if (!inplace) {
+    r = Object.assign({}, origRecord);
+  }
+
+  if (r.mtime_local === 0) {
+    r.mtime_local = undefined;
+  }
+  if (r.mtime_remote === 0) {
+    r.mtime_remote = undefined;
+  }
+  if (r.delete_time_local === 0) {
+    r.delete_time_local = undefined;
+  }
+  if (r.exist_local === undefined) {
+    r.exist_local = false;
+  }
+  if (r.exist_remote === undefined) {
+    r.exist_remote = false;
+  }
+  r.decision = "unknown";
+
+  if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote > r.mtime_local
+  ) {
+    r.decision = "download_clearhist";
+    r.decision_branch = 1;
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote < r.mtime_local
+  ) {
+    r.decision = "upload_clearhist";
+    r.decision_branch = 2;
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote === r.mtime_local &&
+    password === "" &&
+    r.size_local === r.size_remote
+  ) {
+    r.decision = "skip";
+    r.decision_branch = 3;
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote === r.mtime_local &&
+    password === "" &&
+    r.size_local !== r.size_remote
+  ) {
+    r.decision = "upload_clearhist";
+    r.decision_branch = 4;
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote === r.mtime_local &&
+    password !== ""
+  ) {
+    // if we have encryption,
+    // the size is always unequal
+    // only mtime(s) are reliable
+    r.decision = "skip";
+    r.decision_branch = 5;
+  } else if (r.exist_remote && r.exist_local && r.mtime_local === undefined) {
+    // this must be a folder!
+    if (!r.key.endsWith("/")) {
+      throw Error(`${r.key} is not a folder but lacks local mtime`);
+    }
+    r.decision = "skip";
+    r.decision_branch = 6;
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local !== undefined &&
+    r.mtime_remote >= r.delete_time_local
+  ) {
+    r.decision = "download_clearhist";
+    r.decision_branch = 7;
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local !== undefined &&
+    r.mtime_remote < r.delete_time_local
+  ) {
+    r.decision = "delremote_clearhist";
+    r.decision_branch = 8;
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local == undefined
+  ) {
+    r.decision = "download";
+    r.decision_branch = 9;
+  } else if (!r.exist_remote && r.exist_local && r.mtime_remote === undefined) {
+    r.decision = "upload_clearhist";
+    r.decision_branch = 10;
+  } else if (
+    !r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote === undefined &&
+    r.mtime_local === undefined
+  ) {
+    r.decision = "clearhist";
+    r.decision_branch = 11;
+  }
+
+  return r;
+};
+
+export const getSyncPlan = async (
+  remote: S3ObjectType[],
+  local: TAbstractFile[],
+  deleteHistory: FileFolderHistoryRecord[],
+  db: lf.DatabaseConnection,
+  password: string = ""
+) => {
+  const mixedStates = await ensembleMixedStates(
+    remote,
+    local,
+    deleteHistory,
+    db,
+    password
+  );
+  for (const [key, val] of Object.entries(mixedStates)) {
+    getOperation(val, true, password);
+  }
+  const plan = {
+    ts: Date.now(),
+    remoteType: "s3",
+    mixedStates: mixedStates,
+  } as SyncPlanType;
+  return plan;
+};
+
+export const doActualSync = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  db: lf.DatabaseConnection,
+  vault: Vault,
+  syncPlan: SyncPlanType,
+  password: string = ""
+) => {
+  const keyStates = syncPlan.mixedStates;
+  await Promise.all(
+    Object.entries(keyStates)
+      .sort((k, v) => -(k as string).length)
+      .map(async ([k, v]) => {
+        const key = k as string;
+        const state = v as FileOrFolderMixedState;
+        let remoteEncryptedKey = key;
+        if (password !== "") {
+          remoteEncryptedKey = state.remote_encrypted_key;
+          if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
+            remoteEncryptedKey = await encryptStringToBase32(key, password);
+          }
+        }
+
+        if (
+          state.decision === undefined ||
+          state.decision === "unknown" ||
+          state.decision === "undecided"
+        ) {
+          throw Error(`unknown decision in ${JSON.stringify(state)}`);
+        } else if (state.decision === "skip") {
+          // do nothing
+        } else if (state.decision === "download_clearhist") {
+          await downloadFromRemote(
+            s3Client,
+            s3Config,
+            state.key,
+            vault,
+            state.mtime_remote,
+            password,
+            remoteEncryptedKey
+          );
+          await clearDeleteRenameHistoryOfKey(db, state.key);
+        } else if (state.decision === "upload_clearhist") {
+          const remoteObjMeta = await uploadToRemote(
+            s3Client,
+            s3Config,
+            state.key,
+            vault,
+            false,
+            password,
+            remoteEncryptedKey
+          );
+          await upsertSyncMetaMappingDataS3(
+            db,
+            state.key,
+            state.mtime_local,
+            state.size_local,
+            state.key,
+            remoteObjMeta.LastModified.valueOf(),
+            remoteObjMeta.ContentLength,
+            remoteObjMeta.ETag
+          );
+          await clearDeleteRenameHistoryOfKey(db, state.key);
+        } else if (state.decision === "download") {
+          await mkdirpInVault(state.key, vault);
+          await downloadFromRemote(
+            s3Client,
+            s3Config,
+            state.key,
+            vault,
+            state.mtime_remote,
+            password,
+            remoteEncryptedKey
+          );
+        } else if (state.decision === "delremote_clearhist") {
+          await deleteFromRemote(
+            s3Client,
+            s3Config,
+            state.key,
+            password,
+            remoteEncryptedKey
+          );
+          await clearDeleteRenameHistoryOfKey(db, state.key);
+        } else if (state.decision === "upload") {
+          const remoteObjMeta = await uploadToRemote(
+            s3Client,
+            s3Config,
+            state.key,
+            vault,
+            false,
+            password,
+            remoteEncryptedKey
+          );
+          await upsertSyncMetaMappingDataS3(
+            db,
+            state.key,
+            state.mtime_local,
+            state.size_local,
+            state.key,
+            remoteObjMeta.LastModified.valueOf(),
+            remoteObjMeta.ContentLength,
+            remoteObjMeta.ETag
+          );
+        } else if (state.decision === "clearhist") {
+          await clearDeleteRenameHistoryOfKey(db, state.key);
+        } else {
+          throw Error("this should never happen!");
+        }
+      })
+  );
+};

tests/encrypt.test.ts

@@ -0,0 +1,98 @@
+import * as fs from "fs";
+import * as path from "path";
+import { expect } from "chai";
+import { base64ToBase32, bufferToArrayBuffer } from "../src/misc";
+import {
+  decryptArrayBuffer,
+  decryptBase32ToString,
+  encryptArrayBuffer,
+  encryptStringToBase32,
+} from "../src/encrypt";
+
+describe("Encryption tests", () => {
+  beforeEach(function () {
+    global.window = {
+      crypto: require("crypto").webcrypto,
+    } as any;
+  });
+
+  it("should encrypt string", async () => {
+    const k = "dkjdhkfhdkjgsdklxxd";
+    const password = "hey";
+    expect(await encryptStringToBase32(k, password)).to.not.equal(k);
+  });
+
+  it("should encrypt and decrypt string and get the same result returned", async () => {
+    const k = "jfkkjkjbce7983ycdeknkkjckooAIUHIDIBIE((*BII)njD/d/dd/d/sjxhux";
+    const password = "hfiuibec989###oiu982bj1`";
+    const enc = await encryptStringToBase32(k, password);
+    // console.log(enc);
+    const dec = await decryptBase32ToString(enc, password);
+    // console.log(dec);
+    expect(dec).equal(k);
+  });
+
+  it("should encrypt text file and get the same result as openssl", async () => {
+    const fileContent = (
+      await fs.readFileSync(
+        path.join(__dirname, "static_assets", "sometext.txt")
+      )
+    ).toString("utf-8");
+    const password = "somepassword";
+    const saltHex = "8302F586FAB491EC";
+    const enc = await encryptStringToBase32(
+      fileContent,
+      password,
+      undefined,
+      saltHex
+    );
+
+    // two command returns same result:
+    // cat ./sometext.txt | openssl enc -p -aes-256-cbc -S 8302F586FAB491EC -pbkdf2 -iter 10000 -base64 -pass pass:somepassword
+    // openssl enc -p -aes-256-cbc -S 8302F586FAB491EC -pbkdf2 -iter 10000 -base64 -pass pass:somepassword -in ./sometext.txt
+    const opensslBase64Res =
+      "U2FsdGVkX1+DAvWG+rSR7MSa+yJav1zCE7SSXiBooqwI5Q+LMpIthpk/pXkLj+25";
+    // we output base32, so we need some transformation
+    const opensslBase32Res = base64ToBase32(opensslBase64Res);
+
+    expect(enc).equal(opensslBase32Res);
+  });
+
+  it("should encrypt binary file and get the same result as openssl", async () => {
+    const testFolder = path.join(__dirname, "static_assets", "mona_lisa");
+    const testFileName =
+      "1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg";
+    const fileArrBuf = bufferToArrayBuffer(
+      await fs.readFileSync(path.join(testFolder, testFileName))
+    );
+    const password = "somepassword";
+    const saltHex = "8302F586FAB491EC";
+    const enc = await encryptArrayBuffer(
+      fileArrBuf,
+      password,
+      undefined,
+      saltHex
+    );
+    const opensslArrBuf = bufferToArrayBuffer(
+      await fs.readFileSync(path.join(testFolder, testFileName + ".enc"))
+    );
+
+    expect(Buffer.from(enc).equals(Buffer.from(opensslArrBuf))).to.be.true;
+  });
+
+  it("should descypt binary file and get the same result as openssl", async () => {
+    const testFolder = path.join(__dirname, "static_assets", "mona_lisa");
+    const testFileName =
+      "1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg";
+    const fileArrBuf = bufferToArrayBuffer(
+      await fs.readFileSync(path.join(testFolder, testFileName + ".enc"))
+    );
+    const password = "somepassword";
+    const dec = await decryptArrayBuffer(fileArrBuf, password);
+    const opensslArrBuf = bufferToArrayBuffer(
+      await fs.readFileSync(path.join(testFolder, testFileName))
+    );
+
+    expect(Buffer.from(dec).equals(Buffer.from(opensslArrBuf))).to.be.true;
+  });
+});

tests/misc.test.ts

@@ -0,0 +1,72 @@
+import * as fs from "fs";
+import * as path from "path";
+import { expect } from "chai";
+
+import * as misc from "../src/misc";
+
+describe("Misc: hidden file", () => {
+  it("should find hidden file correctly", () => {
+    let item = "";
+    expect(misc.isHiddenPath(item)).to.be.false;
+
+    item = ".";
+    expect(misc.isHiddenPath(item)).to.be.false;
+
+    item = "..";
+    expect(misc.isHiddenPath(item)).to.be.false;
+
+    item = "/x/y/z/../././../a/b/c";
+    expect(misc.isHiddenPath(item)).to.be.false;
+
+    item = ".hidden";
+    expect(misc.isHiddenPath(item)).to.be.true;
+
+    item = "_hidden_loose";
+    expect(misc.isHiddenPath(item)).to.be.true;
+    expect(misc.isHiddenPath(item, false)).to.be.false;
+
+    item = "/sdd/_hidden_loose";
+    expect(misc.isHiddenPath(item)).to.be.true;
+
+    item = "what/../_hidden_loose/what/what/what";
+    expect(misc.isHiddenPath(item)).to.be.true;
+
+    item = "what/../_hidden_loose/what/what/what";
+    expect(misc.isHiddenPath(item, false)).to.be.false;
+
+    item = "what/../_hidden_loose/../.hidden/what/what/what";
+    expect(misc.isHiddenPath(item, false)).to.be.true;
+  });
+});
+
+describe("Misc: get folder levels", () => {
+  it("should ignore empty path", () => {
+    const item = "";
+    expect(misc.getFolderLevels(item)).to.be.empty;
+  });
+
+  it("should ignore single file", () => {
+    const item = "xxx";
+    expect(misc.getFolderLevels(item)).to.be.empty;
+  });
+
+  it("should detect path ending with /", () => {
+    const item = "xxx/";
+    const res = ["xxx"];
+    expect(misc.getFolderLevels(item)).to.deep.equal(res);
+  });
+
+  it("should correctly split folders and files", () => {
+    const item = "xxx/yyy/zzz.md";
+    const res = ["xxx", "xxx/yyy"];
+    expect(misc.getFolderLevels(item)).to.deep.equal(res);
+
+    const item2 = "xxx/yyy/zzz";
+    const res2 = ["xxx", "xxx/yyy"];
+    expect(misc.getFolderLevels(item2)).to.deep.equal(res2);
+
+    const item3 = "xxx/yyy/zzz/";
+    const res3 = ["xxx", "xxx/yyy", "xxx/yyy/zzz"];
+    expect(misc.getFolderLevels(item3)).to.deep.equal(res3);
+  });
+});

tests/static_assets/mona_lisa/1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.LICENSE.txt

@@ -0,0 +1,3 @@
+The file 1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg is downloaded from Wikimedia Commons: https://commons.wikimedia.org/wiki/File:Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg .
+
+Quoted from the web address: ... This photographic reproduction is therefore also considered to be in the public domain in the United States. In other jurisdictions, re-use of this content may be restricted ...

tests/static_assets/mona_lisa/1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b332c298be98be9968715952822431a051f9d07e67460cdcf91a5a3a305b5df1
+size 1141550

tests/static_assets/mona_lisa/1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg.enc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf67abd791f1e230bce57dd47150f262424247cbefb8be16efcd57d13955576a
+size 1141568

tests/static_assets/mona_lisa/openssl_command.sh

@@ -0,0 +1,3 @@
+# The encryption file is produced by the following command.
+# A salt is explictly provided because we need reproducible output in tests.
+openssl enc -p -aes-256-cbc -S 8302F586FAB491EC -pbkdf2 -iter 10000 -pass pass:somepassword -in '1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg' -out 1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg.enc

tests/static_assets/sometext.txt

@@ -0,0 +1 @@
+A secret text 你好世界

versions.json

@@ -1,3 +1,3 @@
 {
-  "0.0.1": "0.12.15"
+  "0.0.10": "0.12.15"
 }

webpack.config.js

@@ -1,14 +1,25 @@
 const path = require("path");
+const webpack = require("webpack");
 const TerserPlugin = require("terser-webpack-plugin");
 
 module.exports = {
-  entry: "./main.ts",
+  entry: "./src/main.ts",
   target: "web",
   output: {
     filename: "main.js",
     path: __dirname,
     libraryTarget: "commonjs",
   },
+  plugins: [
+    // Work around for Buffer is undefined:
+    // https://github.com/webpack/changelog-v5/issues/10
+    new webpack.ProvidePlugin({
+      Buffer: ["buffer", "Buffer"],
+    }),
+    new webpack.ProvidePlugin({
+      process: "process/browser",
+    }),
+  ],
   module: {
     rules: [
       {
@@ -20,31 +31,31 @@ module.exports = {
   },
   resolve: {
     extensions: [".tsx", ".ts", ".js"],
-    mainFields: ["module", "main"],
+    mainFields: ["browser", "module", "main"],
     fallback: {
-      assert: require.resolve("assert"),
-      buffer: require.resolve("buffer/"),
-      console: require.resolve("console-browserify"),
-      constants: require.resolve("constants-browserify"),
-      crypto: require.resolve("crypto-browserify"),
-      domain: require.resolve("domain-browser"),
-      events: require.resolve("events"),
-      http: require.resolve("stream-http"),
-      https: require.resolve("https-browserify"),
-      os: require.resolve("os-browserify/browser"),
+      // assert: require.resolve("assert"),
+      // buffer: require.resolve("buffer/"),
+      // console: require.resolve("console-browserify"),
+      // constants: require.resolve("constants-browserify"),
+      // crypto: require.resolve("crypto-browserify"),
+      // domain: require.resolve("domain-browser"),
+      // events: require.resolve("events"),
+      // http: require.resolve("stream-http"),
+      // https: require.resolve("https-browserify"),
+      // os: require.resolve("os-browserify/browser"),
       path: require.resolve("path-browserify"),
-      punycode: require.resolve("punycode"),
+      // punycode: require.resolve("punycode"),
       process: require.resolve("process/browser"),
-      querystring: require.resolve("querystring-es3"),
+      // querystring: require.resolve("querystring-es3"),
       stream: require.resolve("stream-browserify"),
-      string_decoder: require.resolve("string_decoder"),
-      sys: require.resolve("util"),
-      timers: require.resolve("timers-browserify"),
-      tty: require.resolve("tty-browserify"),
-      url: require.resolve("url"),
-      util: require.resolve("util"),
-      vm: require.resolve("vm-browserify"),
-      zlib: require.resolve("browserify-zlib"),
+      // string_decoder: require.resolve("string_decoder"),
+      // sys: require.resolve("util"),
+      // timers: require.resolve("timers-browserify"),
+      // tty: require.resolve("tty-browserify"),
+      // url: require.resolve("url"),
+      // util: require.resolve("util"),
+      // vm: require.resolve("vm-browserify"),
+      // zlib: require.resolve("browserify-zlib"),
     },
   },
   externals: {