to 0.0.9

.gitattributes

@@ -0,0 +1,5 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.enc filter=lfs diff=lfs merge=lfs -text

.github/workflows/auto-build.yml

@@ -5,9 +5,9 @@ name: BuildCI
 
 on:
   push:
-    branches: [master, enc]
+    branches: [master]
   pull_request:
-    branches: [master, enc]
+    branches: [master]
 
 jobs:
   build:
@@ -19,12 +19,18 @@ jobs:
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout codes
+        uses: actions/checkout@v2
+        with:
+          lfs: true
+      - name: Checkout LFS
+        run: git lfs checkout
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
       - run: npm install
+      - run: npm test
       - run: npm run build
       - uses: actions/upload-artifact@v2
         with:

README.md

@@ -2,6 +2,14 @@
 
 This is yet another sync plugin for Obsidian.
 
+## Download and Install
+
+[![BuildCI](https://github.com/fyears/obsidian-save-remote/actions/workflows/auto-build.yml/badge.svg)](https://github.com/fyears/obsidian-save-remote/actions/workflows/auto-build.yml)
+
+Every artifacts are placed in the "Summary" under every successful builds.
+
+Besides manually download the files, you can also use [Obsidian42 - BRAT](https://github.com/TfTHacker/obsidian42-brat) to install this plugin.
+
 ## Disclaimer
 
 **This is NOT the official sync service provided by Obsidian.**
@@ -14,5 +22,5 @@ As of October 2021, the plugin is under development. **DO NOT USE IT for any ser
 
 Remote service in support list:
 
-- [ ] AWS S3
+- [x] AWS S3
 - [ ] webdav

docs/algorithm.md

@@ -14,16 +14,16 @@ Assuming all sources are reliable.
 
 We list all combinations mutually exclusive and collectively exhaustive.
 
-| ID  | Remote Files | Local files | Local delete rename history | Extra                                                      | Decision                                                                                          |
+| ID   | Remote Files | Local files | Local delete rename history | Extra                                                        | Decision                                                     |
-| --- | ------------ | ----------- | --------------------------- | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------- |
+| ---- | ------------ | ----------- | --------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| 1   | exist        | exist       | ignore                      | mtime_remote > mtime_local                                 | download remote file, create local folder if not exists, clear local history if exists            |
+| 1    | exist        | exist       | ignore                      | mtime_remote > mtime_local                                   | download remote file, create local folder if not exists, clear local history if exists |
-| 2   | exist        | exist       | ignore                      | mtime_remote < mtime_local                                 | upload local file, create remote folder if not exists, clear local history if exists              |
+| 2    | exist        | exist       | ignore                      | mtime_remote < mtime_local                                   | upload local file, create remote folder if not exists, clear local history if exists |
-| 3   | exist        | exist       | ignore                      | mtime_remote === mtime_local && size_remote === size_local | clear local history if exists (the file was synced and no changes after last sync)                |
+| 3    | exist        | exist       | ignore                      | mtime_remote === mtime_local && password === "" && size_remote === size_local | clear local history if exists (the file was synced and no changes after last sync) |
-| 4   | exist        | exist       | ignore                      | mtime_remote === mtime_local && size_remote !== size_local | upload local file, clear local history if exists (we always prefer local to remote)               |
+| 4    | exist        | exist       | ignore                      | mtime_remote === mtime_local && password === "" && size_remote !== size_local | upload local file, clear local history if exists (we always prefer local to remote) |
-| 5   | exist        | exist       | ignore                      | If local is a folder. mtime_local === undefined            | clear local history if exists. TODO: what if a folder and a previous file share the same name?    |
+| 5    | exist        | exist       | ignore                      | mtime_remote === mtime_local && password !== ""              | clear local history if exists (in encryption mode, file sizes are unequal. we can only rely on mtime(s)) |
-| 6   | exist        | not exist   | exist                       | mtime_remote >= delete_time_local                          | download remote file, create folder if not exists                                                 |
+| 6    | exist        | exist       | ignore                      | If local is a folder. mtime_local === undefined              | clear local history if exists. TODO: what if a folder and a previous file share the same name? |
-| 7   | exist        | not exist   | exist                       | mtime_remote < delete_time_local                           | delete remote file, clear local history                                                           |
+| 7    | exist        | not exist   | exist                       | mtime_remote >= delete_time_local                            | download remote file, create folder if not exists            |
-| 8   | exist        | not exist   | not exist                   |                                                            | download remote file, create folder if not exists                                                 |
+| 8    | exist        | not exist   | exist                       | mtime_remote < delete_time_local                             | delete remote file, clear local history                      |
-| 9   | not exist    | exist       | ignore                      | If local is a single file.                                 | upload local file, create remote folder if not exists, clear local history if exists              |
+| 9    | exist        | not exist   | not exist                   |                                                              | download remote file, create folder if not exists            |
-| 10  | not exist    | exist       | ignore                      | If local is a folder.                                      | upload local files recursively, create remote folder if not exists, clear local history if exists |
+| 10   | not exist    | exist       | ignore                      | local may be folder or file                                  | upload local files recursively, create remote folder if not exists, clear local history if exists |
-| 11  | not exist    | not exist   | ignore                      |                                                            | clear local history if exists                                                                     |
+| 11   | not exist    | not exist   | ignore                      |                                                              | clear local history if exists                                |

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "obsdian-save-remote",
   "name": "Save remote",
-  "version": "0.0.3",
+  "version": "0.0.9",
   "minAppVersion": "0.12.15",
   "description": "This is yet another plugin allowing users to sync notes between local device and the cloud.",
   "author": "fyears",

package.json

@@ -1,23 +1,30 @@
 {
   "name": "obsidian-save-remote",
-  "version": "0.0.1",
+  "version": "0.0.9",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev": "webpack --mode development --watch",
     "build": "webpack --mode production",
     "format": "npx prettier --write .",
-    "clean": "npx rimraf main.js"
+    "clean": "npx rimraf main.js",
+    "test": "cross-env TS_NODE_COMPILER_OPTIONS={\\\"module\\\":\\\"commonjs\\\"} mocha -r ts-node/register 'tests/**/*.ts'"
   },
   "source": "main.ts",
   "keywords": [],
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@types/crypto-js": "^4.0.2",
+    "@types/chai": "^4.2.22",
+    "@types/mime-types": "^2.1.1",
+    "@types/mocha": "^9.0.0",
     "@types/node": "^14.14.37",
+    "chai": "^4.3.4",
+    "cross-env": "^7.0.3",
+    "mocha": "^9.1.3",
     "prettier": "^2.4.1",
     "terser-webpack-plugin": "^5.2.4",
     "ts-loader": "^9.2.6",
+    "ts-node": "^10.4.0",
     "tslib": "^2.2.0",
     "typescript": "^4.4.4",
     "webdav-server": "^2.6.2",
@@ -27,38 +34,18 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.37.0",
     "@aws-sdk/signature-v4-crt": "^3.37.0",
-    "@types/mime-types": "^2.1.1",
     "acorn": "^8.5.0",
-    "assert": "^2.0.0",
     "aws-crt": "^1.10.1",
-    "browserify-zlib": "^0.2.0",
     "buffer": "^6.0.3",
     "codemirror": "^5.63.1",
-    "console-browserify": "^1.2.0",
-    "constants-browserify": "^1.0.0",
-    "crypto-browserify": "^3.12.0",
-    "crypto-js": "^4.1.1",
-    "domain-browser": "^4.22.0",
-    "events": "^3.3.0",
-    "hi-base32": "^0.5.1",
-    "https-browserify": "^1.0.0",
     "lovefield-ts": "^0.7.0",
     "mime-types": "^2.1.33",
     "obsidian": "^0.12.0",
-    "os-browserify": "^0.3.0",
     "path-browserify": "^1.0.1",
     "process": "^0.11.10",
-    "punycode": "^2.1.1",
+    "rfc4648": "^1.5.0",
-    "querystring-es3": "^0.2.1",
     "rimraf": "^3.0.2",
     "stream-browserify": "^3.0.0",
-    "stream-http": "^3.2.0",
-    "string_decoder": "^1.3.0",
-    "timers-browserify": "^2.0.12",
-    "tty-browserify": "0.0.1",
-    "url": "^0.11.0",
-    "util": "^0.12.4",
-    "vm-browserify": "^1.1.2",
     "webdav": "^4.7.0",
     "webdav-fs": "^4.0.0"
   }

src/debugMode.ts

@@ -0,0 +1,37 @@
+import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
+
+import * as lf from "lovefield-ts/dist/es6/lf.js";
+
+import type { SyncPlanType } from "./sync";
+import {
+  insertSyncPlanRecord,
+  clearAllSyncPlanRecords,
+  readAllSyncPlanRecordTexts,
+} from "./localdb";
+import { mkdirpInVault } from "./misc";
+
+const DEFAULT_DEBUG_FOLDER = "_debug_save_remote/";
+const DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX = "sync_plans_hist_exported_on_";
+
+export const exportSyncPlansToFiles = async (
+  db: lf.DatabaseConnection,
+  vault: Vault
+) => {
+  console.log("exporting");
+  await mkdirpInVault(DEFAULT_DEBUG_FOLDER, vault);
+  const records = await readAllSyncPlanRecordTexts(db);
+  let md = "";
+  if (records.length === 0) {
+    md = "No sync plans history found";
+  } else {
+    md =
+      "Sync plans found:\n\n" +
+      records.map((x) => "```json\n" + x + "\n```\n").join("\n");
+  }
+  const ts = Date.now();
+  const filePath = `${DEFAULT_DEBUG_FOLDER}${DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX}${ts}.md`;
+  await vault.create(filePath, md, {
+    mtime: ts,
+  });
+  console.log("finish exporting");
+};

src/encrypt.ts

@@ -1,113 +1,135 @@
-import * as CryptoJS from "crypto-js";
+import { base32, base64 } from "rfc4648";
-import * as base32 from "hi-base32";
 import {
   bufferToArrayBuffer,
   arrayBufferToBuffer,
-  arrayBufferToBase64,
+  hexStringToTypedArray,
-  base64ToArrayBuffer,
+  arrayBufferToHex,
 } from "./misc";
 
 const DEFAULT_ITER = 10000;
 
-export const encryptWordArray = (
+const getKeyIVFromPassword = async (
-  wa: CryptoJS.lib.WordArray,
+  salt: Uint8Array,
   password: string,
   rounds: number = DEFAULT_ITER
 ) => {
-  const prefix = CryptoJS.enc.Utf8.parse("Salted__");
+  const k1 = await window.crypto.subtle.importKey(
-  const salt = CryptoJS.lib.WordArray.random(8);
+    "raw",
-  const derivedKey = CryptoJS.PBKDF2(password, salt, {
+    new TextEncoder().encode(password),
-    keySize: 32 + 16,
+    { name: "PBKDF2" },
-    iterations: rounds,
+    false,
-    hasher: CryptoJS.algo.SHA256,
+    ["deriveKey", "deriveBits"]
-  });
-  const key = CryptoJS.lib.WordArray.create(derivedKey.words.slice(0, 32 / 4));
-  const iv = CryptoJS.lib.WordArray.create(
-    derivedKey.words.slice(32 / 4, (32 + 16) / 4)
   );
-  const encrypted = CryptoJS.AES.encrypt(wa, key, { iv: iv }).ciphertext;
+
-  const res = CryptoJS.lib.WordArray.create()
+  const k2 = await window.crypto.subtle.deriveBits(
-    .concat(prefix)
+    {
-    .concat(salt)
+      name: "PBKDF2",
-    .concat(encrypted);
+      salt: salt,
-  return res;
+      iterations: rounds,
+      hash: "SHA-256",
+    },
+    k1,
+    256 + 128
+  );
+
+  return k2;
 };
 
-export const decryptWordArray = (
+export const encryptArrayBuffer = async (
-  wa: CryptoJS.lib.WordArray,
+  arrBuf: ArrayBuffer,
   password: string,
-  rounds: number = DEFAULT_ITER
+  rounds: number = DEFAULT_ITER,
+  saltHex: string = ""
 ) => {
-  const prefix = CryptoJS.lib.WordArray.create(wa.words.slice(0, 8 / 4));
+  let salt: Uint8Array;
+  if (saltHex !== "") {
+    salt = hexStringToTypedArray(saltHex);
+  } else {
+    salt = window.crypto.getRandomValues(new Uint8Array(8));
+  }
 
-  const salt = CryptoJS.lib.WordArray.create(
+  const derivedKey = await getKeyIVFromPassword(salt, password, rounds);
-    wa.words.slice(8 / 4, (8 + 8) / 4)
+  const key = derivedKey.slice(0, 32);
-  );
+  const iv = derivedKey.slice(32, 32 + 16);
-  const derivedKey = CryptoJS.PBKDF2(password, salt, {
+
-    keySize: 32 + 16,
+  const keyCrypt = await window.crypto.subtle.importKey(
-    iterations: rounds,
+    "raw",
-    hasher: CryptoJS.algo.SHA256,
-  });
-  const key = CryptoJS.lib.WordArray.create(derivedKey.words.slice(0, 32 / 4));
-  const iv = CryptoJS.lib.WordArray.create(
-    derivedKey.words.slice(32 / 4, 32 / 4 + 16 / 4)
-  );
-  const decrypted = CryptoJS.AES.decrypt(
-    CryptoJS.lib.CipherParams.create({
-      ciphertext: CryptoJS.lib.WordArray.create(wa.words.slice((8 + 8) / 4)),
-    }),
     key,
-    { iv: iv }
+    { name: "AES-CBC" },
+    false,
+    ["encrypt", "decrypt"]
   );
-  return decrypted;
+
+  const enc = (await window.crypto.subtle.encrypt(
+    { name: "AES-CBC", iv },
+    keyCrypt,
+    arrBuf
+  )) as ArrayBuffer;
+
+  const prefix = new TextEncoder().encode("Salted__");
+
+  const res = new Uint8Array([...prefix, ...salt, ...new Uint8Array(enc)]);
+
+  return bufferToArrayBuffer(res);
 };
 
-export const encryptArrayBuffer = (
+export const decryptArrayBuffer = async (
   arrBuf: ArrayBuffer,
   password: string,
   rounds: number = DEFAULT_ITER
 ) => {
-  const b64 = arrayBufferToBase64(arrBuf);
+  const prefix = arrBuf.slice(0, 8);
-  const wa = CryptoJS.enc.Base64.parse(b64);
+  const salt = arrBuf.slice(8, 16);
-  const enc = encryptWordArray(wa, password, rounds);
+  const derivedKey = await getKeyIVFromPassword(
-  const resb64 = CryptoJS.enc.Base64.stringify(enc);
+    new Uint8Array(salt),
-  const res = base64ToArrayBuffer(resb64);
+    password,
-  return res;
+    rounds
+  );
+  const key = derivedKey.slice(0, 32);
+  const iv = derivedKey.slice(32, 32 + 16);
+
+  const keyCrypt = await window.crypto.subtle.importKey(
+    "raw",
+    key,
+    { name: "AES-CBC" },
+    false,
+    ["encrypt", "decrypt"]
+  );
+
+  const dec = (await window.crypto.subtle.decrypt(
+    { name: "AES-CBC", iv },
+    keyCrypt,
+    arrBuf.slice(16)
+  )) as ArrayBuffer;
+
+  return dec;
 };
 
-export const decryptArrayBuffer = (
+export const encryptStringToBase32 = async (
-  arrBuf: ArrayBuffer,
+  text: string,
   password: string,
-  rounds: number = DEFAULT_ITER
+  rounds: number = DEFAULT_ITER,
+  saltHex: string = ""
 ) => {
-  const b64 = arrayBufferToBase64(arrBuf);
+  const enc = await encryptArrayBuffer(
-  const wa = CryptoJS.enc.Base64.parse(b64);
+    bufferToArrayBuffer(new TextEncoder().encode(text)),
-  const dec = decryptWordArray(wa, password, rounds);
+    password,
-  const resb64 = CryptoJS.enc.Base64.stringify(dec);
+    rounds,
-  const res = base64ToArrayBuffer(resb64);
+    saltHex
-  return res;
+  );
+  return base32.stringify(new Uint8Array(enc));
 };
 
-export const encryptStringToBase32 = (
+export const decryptBase32ToString = async (
   text: string,
   password: string,
   rounds: number = DEFAULT_ITER
 ) => {
-  const wa = CryptoJS.enc.Utf8.parse(text);
+  return new TextDecoder().decode(
-  const enc = encryptWordArray(wa, password, rounds);
+    await decryptArrayBuffer(
-  const enctext = CryptoJS.enc.Base64.stringify(enc);
+      bufferToArrayBuffer(base32.parse(text)),
-  const res = base32.encode(base64ToArrayBuffer(enctext));
+      password,
-  return res;
+      rounds
-};
+    )
-
+  );
-export const decryptBase32ToString = (
-  text: string,
-  password: string,
-  rounds: number = DEFAULT_ITER
-) => {
-  const enc = Buffer.from(base32.decode.asBytes(text)).toString("base64");
-  const wa = CryptoJS.enc.Base64.parse(enc);
-  const dec = decryptWordArray(wa, password, rounds);
-  const dectext = CryptoJS.enc.Utf8.stringify(dec);
-  return dectext;
 };

src/localdb.ts

@@ -2,12 +2,14 @@ import * as lf from "lovefield-ts/dist/es6/lf.js";
 import { TAbstractFile, TFile, TFolder } from "obsidian";
 
 import type { SUPPORTED_SERVICES_TYPE } from "./misc";
+import type { SyncPlanType } from "./sync";
 
 export type DatabaseConnection = lf.DatabaseConnection;
 
 export const DEFAULT_DB_NAME = "saveremotedb";
 export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory";
 export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
+export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
 
 export interface FileFolderHistoryRecord {
   key: string;
@@ -32,6 +34,12 @@ export interface SyncMetaMappingRecord {
   key_type: "folder" | "file";
 }
 
+interface SyncPlanRecord {
+  ts: number;
+  remote_type: string;
+  sync_plan: string;
+}
+
 export const prepareDBs = async () => {
   const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1);
   schemaBuilder
@@ -68,6 +76,15 @@ export const prepareDBs = async () => {
     .addPrimaryKey(["id"], true)
     .addIndex("idxkey", ["local_key", "remote_key"]);
 
+  schemaBuilder
+    .createTable(DEFAULT_SYNC_PLANS_HISTORY)
+    .addColumn("id", lf.Type.INTEGER)
+    .addColumn("ts", lf.Type.INTEGER)
+    .addColumn("remote_type", lf.Type.STRING)
+    .addColumn("sync_plan", lf.Type.STRING)
+    .addPrimaryKey(["id"], true)
+    .addIndex("tskey", ["ts"]);
+
   const db = await schemaBuilder.connect({
     storeType: lf.DataStoreType.INDEXED_DB,
   });
@@ -258,3 +275,37 @@ export const getSyncMetaMappingByRemoteKeyS3 = async (
 
   throw Error("something bad in sync meta mapping!");
 };
+
+export const insertSyncPlanRecord = async (
+  db: lf.DatabaseConnection,
+  syncPlan: SyncPlanType
+) => {
+  const schema = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+  const row = schema.createRow({
+    ts: syncPlan.ts,
+    remote_type: syncPlan.remoteType,
+    sync_plan: JSON.stringify(syncPlan, null, 2),
+  } as SyncPlanRecord);
+  await db.insertOrReplace().into(schema).values([row]).exec();
+};
+
+export const clearAllSyncPlanRecords = async (db: lf.DatabaseConnection) => {
+  const tbl = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+  await db.delete().from(tbl).exec();
+};
+
+export const readAllSyncPlanRecordTexts = async (db: lf.DatabaseConnection) => {
+  const schema = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+
+  const records = (await db
+    .select()
+    .from(schema)
+    .orderBy(schema.col("ts"), lf.Order.DESC)
+    .exec()) as SyncPlanRecord[];
+
+  if (records === undefined) {
+    return [] as string[];
+  } else {
+    return records.map((x) => x.sync_plan);
+  }
+};

src/main.ts

@@ -11,7 +11,7 @@ import {
   TFolder,
 } from "obsidian";
 import * as CodeMirror from "codemirror";
-import type { DatabaseConnection } from "./localdb";
+import { clearAllSyncPlanRecords, DatabaseConnection } from "./localdb";
 import {
   prepareDBs,
   destroyDBs,
@@ -19,11 +19,13 @@ import {
   insertDeleteRecord,
   insertRenameRecord,
   getAllDeleteRenameRecords,
+  insertSyncPlanRecord,
 } from "./localdb";
 
 import type { SyncStatusType } from "./sync";
-import { ensembleMixedStates, getOperation, doActualSync } from "./sync";
+import { getSyncPlan, doActualSync } from "./sync";
 import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
+import { exportSyncPlansToFiles } from "./debugMode";
 
 interface SaveRemotePluginSettings {
   s3?: S3Config;
@@ -62,59 +64,67 @@ export default class SaveRemotePlugin extends Plugin {
       })
     );
 
-    // this.addRibbonIcon("dice", "Misc", async () => {
-    //   const a = this.app.vault.getAllLoadedFiles();
-    //   console.log(a);
-    //   const h = await getAllRecords(this.db);
-    //   console.log(h);
-    // });
-
     this.addRibbonIcon("switch", "Save Remote", async () => {
       if (this.syncStatus !== "idle") {
-        new Notice("Save Remote already running!");
+        new Notice(`Save Remote already running in stage ${this.syncStatus}!`);
         return;
       }
 
-      new Notice("Save Remote Sync Preparing");
+      try {
-      this.syncStatus = "preparing";
+        new Notice("1/6 Save Remote Sync Preparing");
-      const s3Client = getS3Client(this.settings.s3);
+        this.syncStatus = "preparing";
-      const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
-      const local = this.app.vault.getAllLoadedFiles();
-      const localHistory = await loadDeleteRenameHistoryTable(this.db);
-      // console.log(remoteRsp);
-      // console.log(local);
-      // console.log(localHistory);
 
-      const mixedStates = await ensembleMixedStates(
+        new Notice("2/6 Starting to fetch remote meta data.");
-        remoteRsp.Contents,
+        this.syncStatus = "getting_remote_meta";
-        local,
+        const s3Client = getS3Client(this.settings.s3);
-        localHistory,
+        const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
-        this.db,
-        this.settings.password
-      );
 
-      for (const [key, val] of Object.entries(mixedStates)) {
+        new Notice("3/6 Starting to fetch local meta data.");
-        getOperation(val, true);
+        this.syncStatus = "getting_local_meta";
+        const local = this.app.vault.getAllLoadedFiles();
+        const localHistory = await loadDeleteRenameHistoryTable(this.db);
+        // console.log(remoteRsp);
+        // console.log(local);
+        // console.log(localHistory);
+
+        new Notice("4/6 Starting to generate sync plan.");
+        this.syncStatus = "generating_plan";
+        const syncPlan = await getSyncPlan(
+          remoteRsp.Contents,
+          local,
+          localHistory,
+          this.db,
+          this.settings.password
+        );
+        console.log(syncPlan.mixedStates); // for debugging
+        await insertSyncPlanRecord(this.db, syncPlan);
+
+        // The operations above are read only and kind of safe.
+        // The operations below begins to write or delete (!!!) something.
+
+        new Notice("5/6 Save Remote Sync data exchanging!");
+
+        this.syncStatus = "syncing";
+        await doActualSync(
+          s3Client,
+          this.settings.s3,
+          this.db,
+          this.app.vault,
+          syncPlan,
+          this.settings.password
+        );
+
+        new Notice("6/6 Save Remote finish!");
+        this.syncStatus = "finish";
+        this.syncStatus = "idle";
+      } catch (error) {
+        const msg = `Save Remote error while ${this.syncStatus}`;
+        console.log(msg);
+        console.log(error);
+        new Notice(msg);
+        new Notice(error);
+        this.syncStatus = "idle";
       }
-
-      console.log(mixedStates);
-
-      // The operations above are read only and kind of safe.
-      // The operations below begins to write or delete (!!!) something.
-
-      new Notice("Save Remote Sync data exchanging!");
-
-      await doActualSync(
-        s3Client,
-        this.settings.s3,
-        this.db,
-        this.app.vault,
-        mixedStates,
-        this.settings.password
-      );
-
-      new Notice("Save Remote finish!");
-      this.syncStatus = "idle";
     });
 
     this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
@@ -151,7 +161,7 @@ export default class SaveRemotePlugin extends Plugin {
   }
 
   destroyDBs() {
-    destroyDBs(this.db);
+    /* destroyDBs(this.db); */
   }
 }
 
@@ -168,7 +178,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
 
     containerEl.empty();
 
-    containerEl.createEl("h2", { text: "Settings for Save Remote" });
+    containerEl.createEl("h1", { text: "Save Remote" });
+
+    containerEl.createEl("h2", { text: "S3" });
 
     new Setting(containerEl)
       .setName("s3Endpoint")
@@ -178,7 +190,7 @@ class SaveRemoteSettingTab extends PluginSettingTab {
           .setPlaceholder("")
           .setValue(this.plugin.settings.s3.s3Endpoint)
           .onChange(async (value) => {
-            this.plugin.settings.s3.s3Endpoint = value;
+            this.plugin.settings.s3.s3Endpoint = value.trim();
             await this.plugin.saveSettings();
           })
       );
@@ -191,7 +203,7 @@ class SaveRemoteSettingTab extends PluginSettingTab {
           .setPlaceholder("")
           .setValue(`${this.plugin.settings.s3.s3Region}`)
           .onChange(async (value) => {
-            this.plugin.settings.s3.s3Region = value;
+            this.plugin.settings.s3.s3Region = value.trim();
             await this.plugin.saveSettings();
           })
       );
@@ -204,7 +216,7 @@ class SaveRemoteSettingTab extends PluginSettingTab {
           .setPlaceholder("")
           .setValue(`${this.plugin.settings.s3.s3AccessKeyID}`)
           .onChange(async (value) => {
-            this.plugin.settings.s3.s3AccessKeyID = value;
+            this.plugin.settings.s3.s3AccessKeyID = value.trim();
             await this.plugin.saveSettings();
           })
       );
@@ -217,19 +229,7 @@ class SaveRemoteSettingTab extends PluginSettingTab {
           .setPlaceholder("")
           .setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`)
           .onChange(async (value) => {
-            this.plugin.settings.s3.s3SecretAccessKey = value;
+            this.plugin.settings.s3.s3SecretAccessKey = value.trim();
-            await this.plugin.saveSettings();
-          })
-      );
-    new Setting(containerEl)
-      .setName("password")
-      .setDesc("password")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.password}`)
-          .onChange(async (value) => {
-            this.plugin.settings.password = value;
             await this.plugin.saveSettings();
           })
       );
@@ -242,9 +242,47 @@ class SaveRemoteSettingTab extends PluginSettingTab {
           .setPlaceholder("")
           .setValue(`${this.plugin.settings.s3.s3BucketName}`)
           .onChange(async (value) => {
-            this.plugin.settings.s3.s3BucketName = value;
+            this.plugin.settings.s3.s3BucketName = value.trim();
             await this.plugin.saveSettings();
           })
       );
+
+    containerEl.createEl("h2", { text: "General" });
+    new Setting(containerEl)
+      .setName("encryption password")
+      .setDesc("Password for E2E encryption. Empty for no password.")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.password}`)
+          .onChange(async (value) => {
+            this.plugin.settings.password = value.trim();
+            await this.plugin.saveSettings();
+          })
+      );
+
+    containerEl.createEl("h2", { text: "Debug" });
+
+    new Setting(containerEl)
+      .setName("export sync plans")
+      .setDesc("export sync plans")
+      .addButton(async (button) => {
+        button.setButtonText("Export");
+        button.onClick(async () => {
+          await exportSyncPlansToFiles(this.plugin.db, this.app.vault);
+          new Notice("sync plans history exported");
+        });
+      });
+
+    new Setting(containerEl)
+      .setName("delete sync plans history in db")
+      .setDesc("delete sync plans history in db")
+      .addButton(async (button) => {
+        button.setButtonText("Delete History");
+        button.onClick(async () => {
+          await clearAllSyncPlanRecords(this.plugin.db);
+          new Notice("sync plans history (in db) deleted");
+        });
+      });
   }
 }

src/misc.ts

@@ -1,11 +1,32 @@
 import { Vault } from "obsidian";
 import * as path from "path";
 
+import { base32 } from "rfc4648";
+
 export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "ftp";
 
-export const ignoreHiddenFiles = (item: string) => {
+/**
-  const basename = path.basename(item);
+ * If any part of the file starts with '.' or '_' then it's a hidden file.
-  return basename === "." || basename[0] !== ".";
+ * @param item
+ * @param loose
+ * @returns
+ */
+export const isHiddenPath = (item: string, loose: boolean = true) => {
+  const k = path.posix.normalize(item); // TODO: only unix path now
+  const k2 = k.split("/"); // TODO: only unix path now
+  // console.log(k2)
+  for (const singlePart of k2) {
+    if (singlePart === "." || singlePart === ".." || singlePart === "") {
+      continue;
+    }
+    if (singlePart[0] === ".") {
+      return true;
+    }
+    if (loose && singlePart[0] === "_") {
+      return true;
+    }
+  }
+  return false;
 };
 
 /**
@@ -31,9 +52,12 @@ export const getFolderLevels = (x: string) => {
 };
 
 export const mkdirpInVault = async (thePath: string, vault: Vault) => {
+  // console.log(thePath);
   const foldersToBuild = getFolderLevels(thePath);
+  // console.log(foldersToBuild);
   for (const folder of foldersToBuild) {
     const r = await vault.adapter.exists(folder);
+    // console.log(r);
     if (!r) {
       console.log(`mkdir ${folder}`);
       await vault.adapter.mkdir(folder);
@@ -46,7 +70,7 @@ export const mkdirpInVault = async (thePath: string, vault: Vault) => {
  * @param b Buffer
  * @returns ArrayBuffer
  */
-export const bufferToArrayBuffer = (b: Buffer) => {
+export const bufferToArrayBuffer = (b: Buffer | Uint8Array) => {
   return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
 };
 
@@ -63,6 +87,27 @@ export const arrayBufferToBase64 = (b: ArrayBuffer) => {
   return arrayBufferToBuffer(b).toString("base64");
 };
 
+export const arrayBufferToHex = (b: ArrayBuffer) => {
+  return arrayBufferToBuffer(b).toString("hex");
+};
+
 export const base64ToArrayBuffer = (b64text: string) => {
   return bufferToArrayBuffer(Buffer.from(b64text, "base64"));
 };
+
+/**
+ * https://stackoverflow.com/questions/43131242
+ * @param hex
+ * @returns
+ */
+export const hexStringToTypedArray = (hex: string) => {
+  return new Uint8Array(
+    hex.match(/[\da-f]{2}/gi).map(function (h) {
+      return parseInt(h, 16);
+    })
+  );
+};
+
+export const base64ToBase32 = (a: string) => {
+  return base32.stringify(Buffer.from(a, "base64"));
+};

src/s3.ts

@@ -109,7 +109,7 @@ export const uploadToRemote = async (
     const localContent = await vault.adapter.readBinary(fileOrFolderPath);
     let remoteContent = localContent;
     if (password !== "") {
-      remoteContent = encryptArrayBuffer(localContent, password);
+      remoteContent = await encryptArrayBuffer(localContent, password);
     }
     const body = arrayBufferToBuffer(remoteContent);
     await s3Client.send(
@@ -214,7 +214,7 @@ export const downloadFromRemote = async (
     );
     let localContent = remoteContent;
     if (password !== "") {
-      localContent = decryptArrayBuffer(remoteContent, password);
+      localContent = await decryptArrayBuffer(remoteContent, password);
     }
     await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
       mtime: mtime,

src/sync.ts

@@ -16,9 +16,18 @@ import {
   deleteFromRemote,
   downloadFromRemote,
 } from "./s3";
-import { mkdirpInVault } from "./misc";
+import { mkdirpInVault, SUPPORTED_SERVICES_TYPE, isHiddenPath } from "./misc";
 import { decryptBase32ToString, encryptStringToBase32 } from "./encrypt";
 
+export type SyncStatusType =
+  | "idle"
+  | "preparing"
+  | "getting_remote_meta"
+  | "getting_local_meta"
+  | "generating_plan"
+  | "syncing"
+  | "finish";
+
 type DecisionType =
   | "undecided"
   | "unknown"
@@ -31,8 +40,6 @@ type DecisionType =
   | "mkdirplocal"
   | "skip";
 
-export type SyncStatusType = "idle" | "preparing" | "syncing";
-
 interface FileOrFolderMixedState {
   key: string;
   exist_local?: boolean;
@@ -48,7 +55,13 @@ interface FileOrFolderMixedState {
   remote_encrypted_key?: string;
 }
 
-export const ensembleMixedStates = async (
+export interface SyncPlanType {
+  ts: number;
+  remoteType: SUPPORTED_SERVICES_TYPE;
+  mixedStates: Record<string, FileOrFolderMixedState>;
+}
+
+const ensembleMixedStates = async (
   remote: S3ObjectType[],
   local: TAbstractFile[],
   deleteHistory: FileFolderHistoryRecord[],
@@ -62,7 +75,7 @@ export const ensembleMixedStates = async (
       const remoteEncryptedKey = entry.Key;
       let key = remoteEncryptedKey;
       if (password !== "") {
-        key = decryptBase32ToString(remoteEncryptedKey, password);
+        key = await decryptBase32ToString(remoteEncryptedKey, password);
       }
       const backwardMapping = await getSyncMetaMappingByRemoteKeyS3(
         db,
@@ -90,6 +103,9 @@ export const ensembleMixedStates = async (
           remote_encrypted_key: remoteEncryptedKey,
         };
       }
+      if (isHiddenPath(key)) {
+        continue;
+      }
       if (results.hasOwnProperty(key)) {
         results[key].key = r.key;
         results[key].exist_remote = r.exist_remote;
@@ -128,6 +144,9 @@ export const ensembleMixedStates = async (
       throw Error(`unexpected ${entry}`);
     }
 
+    if (isHiddenPath(key)) {
+      continue;
+    }
     if (results.hasOwnProperty(key)) {
       results[key].key = r.key;
       results[key].exist_local = r.exist_local;
@@ -155,6 +174,9 @@ export const ensembleMixedStates = async (
       delete_time_local: entry.action_when,
     } as FileOrFolderMixedState;
 
+    if (isHiddenPath(key)) {
+      continue;
+    }
     if (results.hasOwnProperty(key)) {
       results[key].key = r.key;
       results[key].delete_time_local = r.delete_time_local;
@@ -166,9 +188,10 @@ export const ensembleMixedStates = async (
   return results;
 };
 
-export const getOperation = (
+const getOperation = (
   origRecord: FileOrFolderMixedState,
-  inplace: boolean = false
+  inplace: boolean = false,
+  password: string = ""
 ) => {
   let r = origRecord;
   if (!inplace) {
@@ -216,6 +239,7 @@ export const getOperation = (
     r.mtime_remote !== undefined &&
     r.mtime_local !== undefined &&
     r.mtime_remote === r.mtime_local &&
+    password === "" &&
     r.size_local === r.size_remote
   ) {
     r.decision = "skip";
@@ -226,17 +250,31 @@ export const getOperation = (
     r.mtime_remote !== undefined &&
     r.mtime_local !== undefined &&
     r.mtime_remote === r.mtime_local &&
+    password === "" &&
     r.size_local !== r.size_remote
   ) {
     r.decision = "upload_clearhist";
     r.decision_branch = 4;
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote === r.mtime_local &&
+    password !== ""
+  ) {
+    // if we have encryption,
+    // the size is always unequal
+    // only mtime(s) are reliable
+    r.decision = "skip";
+    r.decision_branch = 5;
   } else if (r.exist_remote && r.exist_local && r.mtime_local === undefined) {
     // this must be a folder!
     if (!r.key.endsWith("/")) {
       throw Error(`${r.key} is not a folder but lacks local mtime`);
     }
     r.decision = "skip";
-    r.decision_branch = 5;
+    r.decision_branch = 6;
   } else if (
     r.exist_remote &&
     !r.exist_local &&
@@ -246,7 +284,7 @@ export const getOperation = (
     r.mtime_remote >= r.delete_time_local
   ) {
     r.decision = "download_clearhist";
-    r.decision_branch = 6;
+    r.decision_branch = 7;
   } else if (
     r.exist_remote &&
     !r.exist_local &&
@@ -256,7 +294,7 @@ export const getOperation = (
     r.mtime_remote < r.delete_time_local
   ) {
     r.decision = "delremote_clearhist";
-    r.decision_branch = 7;
+    r.decision_branch = 8;
   } else if (
     r.exist_remote &&
     !r.exist_local &&
@@ -265,10 +303,10 @@ export const getOperation = (
     r.delete_time_local == undefined
   ) {
     r.decision = "download";
-    r.decision_branch = 8;
+    r.decision_branch = 9;
   } else if (!r.exist_remote && r.exist_local && r.mtime_remote === undefined) {
     r.decision = "upload_clearhist";
-    r.decision_branch = 9;
+    r.decision_branch = 10;
   } else if (
     !r.exist_remote &&
     !r.exist_local &&
@@ -276,111 +314,145 @@ export const getOperation = (
     r.mtime_local === undefined
   ) {
     r.decision = "clearhist";
-    r.decision_branch = 10;
+    r.decision_branch = 11;
   }
 
   return r;
 };
 
+export const getSyncPlan = async (
+  remote: S3ObjectType[],
+  local: TAbstractFile[],
+  deleteHistory: FileFolderHistoryRecord[],
+  db: lf.DatabaseConnection,
+  password: string = ""
+) => {
+  const mixedStates = await ensembleMixedStates(
+    remote,
+    local,
+    deleteHistory,
+    db,
+    password
+  );
+  for (const [key, val] of Object.entries(mixedStates)) {
+    getOperation(val, true);
+  }
+  const plan = {
+    ts: Date.now(),
+    remoteType: "s3",
+    mixedStates: mixedStates,
+  } as SyncPlanType;
+  return plan;
+};
+
 export const doActualSync = async (
   s3Client: S3Client,
   s3Config: S3Config,
   db: lf.DatabaseConnection,
   vault: Vault,
-  keyStates: Record<string, FileOrFolderMixedState>,
+  syncPlan: SyncPlanType,
   password: string = ""
 ) => {
-  Object.entries(keyStates)
+  const keyStates = syncPlan.mixedStates;
-    .sort((k, v) => -(k as string).length)
+  await Promise.all(
-    .map(async ([k, v]) => {
+    Object.entries(keyStates)
-      const key = k as string;
+      .sort((k, v) => -(k as string).length)
-      const state = v as FileOrFolderMixedState;
+      .map(async ([k, v]) => {
-      let remoteEncryptedKey = key;
+        const key = k as string;
-      if (password !== "") {
+        const state = v as FileOrFolderMixedState;
-        remoteEncryptedKey = state.remote_encrypted_key;
+        let remoteEncryptedKey = key;
-        if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
+        if (password !== "") {
-          remoteEncryptedKey = encryptStringToBase32(key, password);
+          remoteEncryptedKey = state.remote_encrypted_key;
+          if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
+            remoteEncryptedKey = await encryptStringToBase32(key, password);
+          }
         }
-      }
 
-      if (
+        if (
-        state.decision === undefined ||
+          state.decision === undefined ||
-        state.decision === "unknown" ||
+          state.decision === "unknown" ||
-        state.decision === "undecided"
+          state.decision === "undecided"
-      ) {
+        ) {
-        throw Error(`unknown decision in ${JSON.stringify(state)}`);
+          throw Error(`unknown decision in ${JSON.stringify(state)}`);
-      } else if (state.decision === "skip") {
+        } else if (state.decision === "skip") {
-        // do nothing
+          // do nothing
-      } else if (state.decision === "download_clearhist") {
+        } else if (state.decision === "download_clearhist") {
-        await downloadFromRemote(
+          await downloadFromRemote(
-          s3Client,
+            s3Client,
-          s3Config,
+            s3Config,
-          state.key,
+            state.key,
-          vault,
+            vault,
-          state.mtime_remote,
+            state.mtime_remote,
-          password,
+            password,
-          remoteEncryptedKey
+            remoteEncryptedKey
-        );
+          );
-        await clearDeleteRenameHistoryOfKey(db, state.key);
+          await clearDeleteRenameHistoryOfKey(db, state.key);
-      } else if (state.decision === "upload_clearhist") {
+        } else if (state.decision === "upload_clearhist") {
-        const remoteObjMeta = await uploadToRemote(
+          const remoteObjMeta = await uploadToRemote(
-          s3Client,
+            s3Client,
-          s3Config,
+            s3Config,
-          state.key,
+            state.key,
-          vault,
+            vault,
-          false,
+            false,
-          password,
+            password,
-          remoteEncryptedKey
+            remoteEncryptedKey
-        );
+          );
-        await upsertSyncMetaMappingDataS3(
+          await upsertSyncMetaMappingDataS3(
-          db,
+            db,
-          state.key,
+            state.key,
-          state.mtime_local,
+            state.mtime_local,
-          state.size_local,
+            state.size_local,
-          state.key,
+            state.key,
-          remoteObjMeta.LastModified.valueOf(),
+            remoteObjMeta.LastModified.valueOf(),
-          remoteObjMeta.ContentLength,
+            remoteObjMeta.ContentLength,
-          remoteObjMeta.ETag
+            remoteObjMeta.ETag
-        );
+          );
-        await clearDeleteRenameHistoryOfKey(db, state.key);
+          await clearDeleteRenameHistoryOfKey(db, state.key);
-      } else if (state.decision === "download") {
+        } else if (state.decision === "download") {
-        await mkdirpInVault(state.key, vault);
+          await mkdirpInVault(state.key, vault);
-        await downloadFromRemote(
+          await downloadFromRemote(
-          s3Client,
+            s3Client,
-          s3Config,
+            s3Config,
-          state.key,
+            state.key,
-          vault,
+            vault,
-          state.mtime_remote,
+            state.mtime_remote,
-          password,
+            password,
-          remoteEncryptedKey
+            remoteEncryptedKey
-        );
+          );
-      } else if (state.decision === "delremote_clearhist") {
+        } else if (state.decision === "delremote_clearhist") {
-        await deleteFromRemote(s3Client, s3Config, state.key);
+          await deleteFromRemote(
-        await clearDeleteRenameHistoryOfKey(db, state.key);
+            s3Client,
-      } else if (state.decision === "upload") {
+            s3Config,
-        const remoteObjMeta = await uploadToRemote(
+            state.key,
-          s3Client,
+            password,
-          s3Config,
+            remoteEncryptedKey
-          state.key,
+          );
-          vault,
+          await clearDeleteRenameHistoryOfKey(db, state.key);
-          false,
+        } else if (state.decision === "upload") {
-          password,
+          const remoteObjMeta = await uploadToRemote(
-          remoteEncryptedKey
+            s3Client,
-        );
+            s3Config,
-        await upsertSyncMetaMappingDataS3(
+            state.key,
-          db,
+            vault,
-          state.key,
+            false,
-          state.mtime_local,
+            password,
-          state.size_local,
+            remoteEncryptedKey
-          state.key,
+          );
-          remoteObjMeta.LastModified.valueOf(),
+          await upsertSyncMetaMappingDataS3(
-          remoteObjMeta.ContentLength,
+            db,
-          remoteObjMeta.ETag
+            state.key,
-        );
+            state.mtime_local,
-      } else if (state.decision === "clearhist") {
+            state.size_local,
-        await clearDeleteRenameHistoryOfKey(db, state.key);
+            state.key,
-      } else {
+            remoteObjMeta.LastModified.valueOf(),
-        throw Error("this should never happen!");
+            remoteObjMeta.ContentLength,
-      }
+            remoteObjMeta.ETag
-    });
+          );
+        } else if (state.decision === "clearhist") {
+          await clearDeleteRenameHistoryOfKey(db, state.key);
+        } else {
+          throw Error("this should never happen!");
+        }
+      })
+  );
 };

tests/encrypt.test.ts

@@ -0,0 +1,98 @@
+import * as fs from "fs";
+import * as path from "path";
+import { expect } from "chai";
+import { base64ToBase32, bufferToArrayBuffer } from "../src/misc";
+import {
+  decryptArrayBuffer,
+  decryptBase32ToString,
+  encryptArrayBuffer,
+  encryptStringToBase32,
+} from "../src/encrypt";
+
+describe("Encryption tests", () => {
+  beforeEach(function () {
+    global.window = {
+      crypto: require("crypto").webcrypto,
+    } as any;
+  });
+
+  it("should encrypt string", async () => {
+    const k = "dkjdhkfhdkjgsdklxxd";
+    const password = "hey";
+    expect(await encryptStringToBase32(k, password)).to.not.equal(k);
+  });
+
+  it("should encrypt and decrypt string and get the same result returned", async () => {
+    const k = "jfkkjkjbce7983ycdeknkkjckooAIUHIDIBIE((*BII)njD/d/dd/d/sjxhux";
+    const password = "hfiuibec989###oiu982bj1`";
+    const enc = await encryptStringToBase32(k, password);
+    // console.log(enc);
+    const dec = await decryptBase32ToString(enc, password);
+    // console.log(dec);
+    expect(dec).equal(k);
+  });
+
+  it("should encrypt text file and get the same result as openssl", async () => {
+    const fileContent = (
+      await fs.readFileSync(
+        path.join(__dirname, "static_assets", "sometext.txt")
+      )
+    ).toString("utf-8");
+    const password = "somepassword";
+    const saltHex = "8302F586FAB491EC";
+    const enc = await encryptStringToBase32(
+      fileContent,
+      password,
+      undefined,
+      saltHex
+    );
+
+    // two command returns same result:
+    // cat ./sometext.txt | openssl enc -p -aes-256-cbc -S 8302F586FAB491EC -pbkdf2 -iter 10000 -base64 -pass pass:somepassword
+    // openssl enc -p -aes-256-cbc -S 8302F586FAB491EC -pbkdf2 -iter 10000 -base64 -pass pass:somepassword -in ./sometext.txt
+    const opensslBase64Res =
+      "U2FsdGVkX1+DAvWG+rSR7MSa+yJav1zCE7SSXiBooqwI5Q+LMpIthpk/pXkLj+25";
+    // we output base32, so we need some transformation
+    const opensslBase32Res = base64ToBase32(opensslBase64Res);
+
+    expect(enc).equal(opensslBase32Res);
+  });
+
+  it("should encrypt binary file and get the same result as openssl", async () => {
+    const testFolder = path.join(__dirname, "static_assets", "mona_lisa");
+    const testFileName =
+      "1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg";
+    const fileArrBuf = bufferToArrayBuffer(
+      await fs.readFileSync(path.join(testFolder, testFileName))
+    );
+    const password = "somepassword";
+    const saltHex = "8302F586FAB491EC";
+    const enc = await encryptArrayBuffer(
+      fileArrBuf,
+      password,
+      undefined,
+      saltHex
+    );
+    const opensslArrBuf = bufferToArrayBuffer(
+      await fs.readFileSync(path.join(testFolder, testFileName + ".enc"))
+    );
+
+    expect(Buffer.from(enc).equals(Buffer.from(opensslArrBuf))).to.be.true;
+  });
+
+  it("should descypt binary file and get the same result as openssl", async () => {
+    const testFolder = path.join(__dirname, "static_assets", "mona_lisa");
+    const testFileName =
+      "1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg";
+    const fileArrBuf = bufferToArrayBuffer(
+      await fs.readFileSync(path.join(testFolder, testFileName + ".enc"))
+    );
+    const password = "somepassword";
+    const dec = await decryptArrayBuffer(fileArrBuf, password);
+    const opensslArrBuf = bufferToArrayBuffer(
+      await fs.readFileSync(path.join(testFolder, testFileName))
+    );
+
+    expect(Buffer.from(dec).equals(Buffer.from(opensslArrBuf))).to.be.true;
+  });
+});

tests/misc.test.ts

@@ -0,0 +1,72 @@
+import * as fs from "fs";
+import * as path from "path";
+import { expect } from "chai";
+
+import * as misc from "../src/misc";
+
+describe("Misc: hidden file", () => {
+  it("should find hidden file correctly", () => {
+    let item = "";
+    expect(misc.isHiddenPath(item)).to.be.false;
+
+    item = ".";
+    expect(misc.isHiddenPath(item)).to.be.false;
+
+    item = "..";
+    expect(misc.isHiddenPath(item)).to.be.false;
+
+    item = "/x/y/z/../././../a/b/c";
+    expect(misc.isHiddenPath(item)).to.be.false;
+
+    item = ".hidden";
+    expect(misc.isHiddenPath(item)).to.be.true;
+
+    item = "_hidden_loose";
+    expect(misc.isHiddenPath(item)).to.be.true;
+    expect(misc.isHiddenPath(item, false)).to.be.false;
+
+    item = "/sdd/_hidden_loose";
+    expect(misc.isHiddenPath(item)).to.be.true;
+
+    item = "what/../_hidden_loose/what/what/what";
+    expect(misc.isHiddenPath(item)).to.be.true;
+
+    item = "what/../_hidden_loose/what/what/what";
+    expect(misc.isHiddenPath(item, false)).to.be.false;
+
+    item = "what/../_hidden_loose/../.hidden/what/what/what";
+    expect(misc.isHiddenPath(item, false)).to.be.true;
+  });
+});
+
+describe("Misc: get folder levels", () => {
+  it("should ignore empty path", () => {
+    const item = "";
+    expect(misc.getFolderLevels(item)).to.be.empty;
+  });
+
+  it("should ignore single file", () => {
+    const item = "xxx";
+    expect(misc.getFolderLevels(item)).to.be.empty;
+  });
+
+  it("should detect path ending with /", () => {
+    const item = "xxx/";
+    const res = ["xxx"];
+    expect(misc.getFolderLevels(item)).to.deep.equal(res);
+  });
+
+  it("should correctly split folders and files", () => {
+    const item = "xxx/yyy/zzz.md";
+    const res = ["xxx", "xxx/yyy"];
+    expect(misc.getFolderLevels(item)).to.deep.equal(res);
+
+    const item2 = "xxx/yyy/zzz";
+    const res2 = ["xxx", "xxx/yyy"];
+    expect(misc.getFolderLevels(item2)).to.deep.equal(res2);
+
+    const item3 = "xxx/yyy/zzz/";
+    const res3 = ["xxx", "xxx/yyy", "xxx/yyy/zzz"];
+    expect(misc.getFolderLevels(item3)).to.deep.equal(res3);
+  });
+});

tests/static_assets/mona_lisa/1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.LICENSE.txt

@@ -0,0 +1,3 @@
+The file 1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg is downloaded from Wikimedia Commons: https://commons.wikimedia.org/wiki/File:Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg .
+
+Quoted from the web address: ... This photographic reproduction is therefore also considered to be in the public domain in the United States. In other jurisdictions, re-use of this content may be restricted ...

tests/static_assets/mona_lisa/1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b332c298be98be9968715952822431a051f9d07e67460cdcf91a5a3a305b5df1
+size 1141550

tests/static_assets/mona_lisa/1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg.enc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf67abd791f1e230bce57dd47150f262424247cbefb8be16efcd57d13955576a
+size 1141568

tests/static_assets/mona_lisa/openssl_command.sh

@@ -0,0 +1,3 @@
+# The encryption file is produced by the following command.
+# A salt is explictly provided because we need reproducible output in tests.
+openssl enc -p -aes-256-cbc -S 8302F586FAB491EC -pbkdf2 -iter 10000 -pass pass:somepassword -in '1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg' -out 1374px-Mona_Lisa,_by_Leonardo_da_Vinci,_from_C2RMF_retouched.jpg.enc

tests/static_assets/sometext.txt

@@ -0,0 +1 @@
+A secret text 你好世界

versions.json

@@ -1,3 +1,3 @@
 {
-  "0.0.3": "0.12.15"
+  "0.0.9": "0.12.15"
 }

webpack.config.js

@@ -1,4 +1,5 @@
 const path = require("path");
+const webpack = require("webpack");
 const TerserPlugin = require("terser-webpack-plugin");
 
 module.exports = {
@@ -9,6 +10,16 @@ module.exports = {
     path: __dirname,
     libraryTarget: "commonjs",
   },
+  plugins: [
+    // Work around for Buffer is undefined:
+    // https://github.com/webpack/changelog-v5/issues/10
+    new webpack.ProvidePlugin({
+      Buffer: ["buffer", "Buffer"],
+    }),
+    new webpack.ProvidePlugin({
+      process: "process/browser",
+    }),
+  ],
   module: {
     rules: [
       {
@@ -20,31 +31,31 @@ module.exports = {
   },
   resolve: {
     extensions: [".tsx", ".ts", ".js"],
-    mainFields: ["module", "main"],
+    mainFields: ["browser", "module", "main"],
     fallback: {
-      assert: require.resolve("assert"),
+      // assert: require.resolve("assert"),
-      buffer: require.resolve("buffer/"),
+      // buffer: require.resolve("buffer/"),
-      console: require.resolve("console-browserify"),
+      // console: require.resolve("console-browserify"),
-      constants: require.resolve("constants-browserify"),
+      // constants: require.resolve("constants-browserify"),
-      crypto: require.resolve("crypto-browserify"),
+      // crypto: require.resolve("crypto-browserify"),
-      domain: require.resolve("domain-browser"),
+      // domain: require.resolve("domain-browser"),
-      events: require.resolve("events"),
+      // events: require.resolve("events"),
-      http: require.resolve("stream-http"),
+      // http: require.resolve("stream-http"),
-      https: require.resolve("https-browserify"),
+      // https: require.resolve("https-browserify"),
-      os: require.resolve("os-browserify/browser"),
+      // os: require.resolve("os-browserify/browser"),
       path: require.resolve("path-browserify"),
-      punycode: require.resolve("punycode"),
+      // punycode: require.resolve("punycode"),
       process: require.resolve("process/browser"),
-      querystring: require.resolve("querystring-es3"),
+      // querystring: require.resolve("querystring-es3"),
       stream: require.resolve("stream-browserify"),
-      string_decoder: require.resolve("string_decoder"),
+      // string_decoder: require.resolve("string_decoder"),
-      sys: require.resolve("util"),
+      // sys: require.resolve("util"),
-      timers: require.resolve("timers-browserify"),
+      // timers: require.resolve("timers-browserify"),
-      tty: require.resolve("tty-browserify"),
+      // tty: require.resolve("tty-browserify"),
-      url: require.resolve("url"),
+      // url: require.resolve("url"),
-      util: require.resolve("util"),
+      // util: require.resolve("util"),
-      vm: require.resolve("vm-browserify"),
+      // vm: require.resolve("vm-browserify"),
-      zlib: require.resolve("browserify-zlib"),
+      // zlib: require.resolve("browserify-zlib"),
     },
   },
   externals: {