yes encryption!

merge enc to master

.editorconfig

@@ -0,0 +1,28 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+[*.{js,jsx,mjs,cjs,ts,tsx,json,py}]
+charset = utf-8
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4
+
+# Tab indentation (no size specified)
+[Makefile]
+indent_style = tab
+
+# Indentation override for all JS under lib directory
+[*.{js,jsx,mjs,cjs,ts,tsx,json,py}]
+indent_style = space
+indent_size = 2

.github/workflows/auto-build.yml

@@ -0,0 +1,34 @@
+# This workflow will do a clean install of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: BuildCI
+
+on:
+  push:
+    branches: [master, enc]
+  pull_request:
+    branches: [master, enc]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [16.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: npm install
+      - run: npm run build
+      - uses: actions/upload-artifact@v2
+        with:
+          name: my-dist
+          path: |
+            main.js
+            manifest.json

.prettierignore

@@ -0,0 +1,2 @@
+node_modules/
+main.js

docs/algorithm.md

@@ -0,0 +1,29 @@
+# Algorithm
+
+## Sources
+
+We have three record sources:
+
+1. Local files
+2. Remote files
+3. Local "delete-or-rename" history.
+
+Assuming all sources are reliable.
+
+## Deal with them
+
+We list all combinations mutually exclusive and collectively exhaustive.
+
+| ID  | Remote Files | Local files | Local delete rename history | Extra                                                      | Decision                                                                                          |
+| --- | ------------ | ----------- | --------------------------- | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------- |
+| 1   | exist        | exist       | ignore                      | mtime_remote > mtime_local                                 | download remote file, create local folder if not exists, clear local history if exists            |
+| 2   | exist        | exist       | ignore                      | mtime_remote < mtime_local                                 | upload local file, create remote folder if not exists, clear local history if exists              |
+| 3   | exist        | exist       | ignore                      | mtime_remote === mtime_local && size_remote === size_local | clear local history if exists (the file was synced and no changes after last sync)                |
+| 4   | exist        | exist       | ignore                      | mtime_remote === mtime_local && size_remote !== size_local | upload local file, clear local history if exists (we always prefer local to remote)               |
+| 5   | exist        | exist       | ignore                      | If local is a folder. mtime_local === undefined            | clear local history if exists. TODO: what if a folder and a previous file share the same name?    |
+| 6   | exist        | not exist   | exist                       | mtime_remote >= delete_time_local                          | download remote file, create folder if not exists                                                 |
+| 7   | exist        | not exist   | exist                       | mtime_remote < delete_time_local                           | delete remote file, clear local history                                                           |
+| 8   | exist        | not exist   | not exist                   |                                                            | download remote file, create folder if not exists                                                 |
+| 9   | not exist    | exist       | ignore                      | If local is a single file.                                 | upload local file, create remote folder if not exists, clear local history if exists              |
+| 10  | not exist    | exist       | ignore                      | If local is a folder.                                      | upload local files recursively, create remote folder if not exists, clear local history if exists |
+| 11  | not exist    | not exist   | ignore                      |                                                            | clear local history if exists                                                                     |

docs/code_design.md

@@ -0,0 +1,13 @@
+# Code Design
+
+## Code Organization
+
+1. Every function except `main.ts` should be pure. Pass any stateful information in parameters.
+
+2. `misc.ts` should not depend on any other written code.
+
+3. Each storage code should not depend on `sync.ts`.
+
+## File and Folder Representation
+
+While writing sync codes, folders are always represented by a string ending with `/`.

main.ts

@@ -1,340 +0,0 @@
-import * as path from "path";
-import * as fs from "fs";
-import { Buffer } from "buffer";
-import { Readable } from "stream";
-import * as mime from "mime-types";
-import {
-  App,
-  Modal,
-  Notice,
-  Plugin,
-  PluginSettingTab,
-  Setting,
-  request,
-  Platform,
-} from "obsidian";
-import * as CodeMirror from "codemirror";
-
-import {
-  S3Client,
-  ListObjectsCommand,
-  PutObjectCommand,
-  GetObjectCommand,
-} from "@aws-sdk/client-s3";
-
-interface SaveRemotePluginSettings {
-  s3Endpoint: string;
-  s3Region: string;
-  s3AccessKeyID: string;
-  s3SecretAccessKey: string;
-  s3BucketName: string;
-}
-
-const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
-  s3Endpoint: "",
-  s3Region: "",
-  s3AccessKeyID: "",
-  s3SecretAccessKey: "",
-  s3BucketName: "",
-};
-
-const ignoreHiddenFiles = (item: string) => {
-  const basename = path.basename(item);
-  return basename === "." || basename[0] !== ".";
-};
-
-/**
- * Util func for mkdir -p based on the "path" of original file or folder
- * "a/b/c/" => ["a", "a/b", "a/b/c"]
- * "a/b/c/d/e.txt" => ["a", "a/b", "a/b/c", "a/b/c/d"]
- * @param x string
- * @returns string[] might be empty
- */
-const getFolderLevels = (x: string) => {
-  const res: string[] = [];
-
-  if (x === "" || x === "/") {
-    return res;
-  }
-
-  const y1 = x.split("/");
-  let i = 0;
-  for (let index = 0; index + 1 < y1.length; index++) {
-    res.push(y1.slice(0, index + 1).join("/"));
-  }
-  return res;
-};
-
-/**
- * https://stackoverflow.com/questions/8609289
- * @param b Buffer
- * @returns ArrayBuffer
- */
-const bufferToArrayBuffer = (b: Buffer) => {
-  return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
-};
-
-/**
- * The Body of resp of aws GetObject has mix types
- * and we want to get ArrayBuffer here.
- * See https://github.com/aws/aws-sdk-js-v3/issues/1877
- * @param b The Body of GetObject
- * @returns Promise<ArrayBuffer>
- */
-const getObjectBodyToArrayBuffer = async (
-  b: Readable | ReadableStream | Blob
-) => {
-  if (b instanceof Readable) {
-    const chunks: Uint8Array[] = [];
-    for await (let chunk of b) {
-      chunks.push(chunk);
-    }
-    const buf = Buffer.concat(chunks);
-    return bufferToArrayBuffer(buf);
-  } else if (b instanceof ReadableStream) {
-    return await new Response(b, {}).arrayBuffer();
-  } else if (b instanceof Blob) {
-    return await b.arrayBuffer();
-  } else {
-    throw TypeError(`The type of ${b} is not one of the supported types`);
-  }
-};
-
-export default class SaveRemotePlugin extends Plugin {
-  settings: SaveRemotePluginSettings;
-  cm: CodeMirror.Editor;
-
-  async onload() {
-    console.log("loading plugin obsidian-save-remote");
-
-    await this.loadSettings();
-
-    this.addRibbonIcon("right-arrow-with-tail", "Upload", async () => {
-      // console.log(this.app.vault.getFiles());
-      // console.log(this.app.vault.getAllLoadedFiles());
-      new Notice(`Upload begun.`);
-      const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
-
-      const s3Client = new S3Client({
-        region: this.settings.s3Region,
-        endpoint: this.settings.s3Endpoint,
-        credentials: {
-          accessKeyId: this.settings.s3AccessKeyID,
-          secretAccessKey: this.settings.s3SecretAccessKey,
-        },
-      });
-
-      try {
-        for (const fileOrFolder of allFilesAndFolders) {
-          if (fileOrFolder.path === "/") {
-            console.log('ignore "/"');
-          } else if ("children" in fileOrFolder) {
-            // folder
-            console.log(`folder ${fileOrFolder.path}/`);
-            new Notice(`folder ${fileOrFolder.path}/`);
-
-            const results = await s3Client.send(
-              new PutObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: `${fileOrFolder.path}/`,
-                Body: "",
-              })
-            );
-          } else {
-            // file
-            console.log(`file ${fileOrFolder.path}`);
-            const arrContent = await this.app.vault.adapter.readBinary(
-              fileOrFolder.path
-            );
-            new Notice(`file ${fileOrFolder.path}`);
-            const contentType =
-              mime.contentType(
-                mime.lookup(`${fileOrFolder.path}`) || undefined
-              ) || undefined;
-            // console.log(contentType);
-            const results = await s3Client.send(
-              new PutObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: `${fileOrFolder.path}`,
-                Body: Buffer.from(arrContent),
-                ContentType: contentType,
-              })
-            );
-          }
-        }
-        new Notice("Upload finished!");
-      } catch (err) {
-        console.log("Error", err);
-        new Notice(`${err}`);
-      }
-    });
-
-    this.addRibbonIcon("left-arrow-with-tail", "Download", async () => {
-      const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
-
-      const s3Client = new S3Client({
-        region: this.settings.s3Region,
-        endpoint: this.settings.s3Endpoint,
-        credentials: {
-          accessKeyId: this.settings.s3AccessKeyID,
-          secretAccessKey: this.settings.s3SecretAccessKey,
-        },
-      });
-
-      try {
-        const listObj = await s3Client.send(
-          new ListObjectsCommand({ Bucket: this.settings.s3BucketName })
-        );
-
-        for (const singleContent of listObj.Contents) {
-          const foldersToBuild = getFolderLevels(singleContent.Key);
-          for (const folder of foldersToBuild) {
-            const r = await this.app.vault.adapter.exists(folder);
-            if (!r) {
-              console.log(`mkdir ${folder}`);
-              new Notice(`mkdir ${folder}`);
-              await this.app.vault.adapter.mkdir(folder);
-            }
-          }
-
-          if (singleContent.Key.endsWith("/")) {
-            // kind of a folder
-            // pass
-          } else {
-            // kind of a file
-            // download
-
-            console.log(`download file ${singleContent.Key}`);
-            new Notice(`download file ${singleContent.Key}`);
-
-            const data = await s3Client.send(
-              new GetObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: singleContent.Key,
-              })
-            );
-            const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
-            await this.app.vault.adapter.writeBinary(
-              singleContent.Key,
-              bodyContents
-            );
-          }
-        }
-
-        new Notice("Download finished!");
-      } catch (err) {
-        console.log("Error", err);
-        new Notice(`${err}`);
-      }
-    });
-
-    this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
-
-    this.registerCodeMirror((cm: CodeMirror.Editor) => {
-      this.cm = cm;
-      console.log("codemirror registered.");
-    });
-
-    // this.registerDomEvent(document, "click", (evt: MouseEvent) => {
-    //   console.log("click", evt);
-    // });
-
-    // this.registerInterval(
-    //   window.setInterval(() => console.log("setInterval"), 5 * 60 * 1000)
-    // );
-  }
-
-  onunload() {
-    console.log("unloading plugin obsidian-save-remote");
-  }
-
-  async loadSettings() {
-    this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
-  }
-
-  async saveSettings() {
-    await this.saveData(this.settings);
-  }
-}
-
-class SaveRemoteSettingTab extends PluginSettingTab {
-  plugin: SaveRemotePlugin;
-
-  constructor(app: App, plugin: SaveRemotePlugin) {
-    super(app, plugin);
-    this.plugin = plugin;
-  }
-
-  display(): void {
-    let { containerEl } = this;
-
-    containerEl.empty();
-
-    containerEl.createEl("h2", { text: "Settings for Save Remote" });
-
-    new Setting(containerEl)
-      .setName("s3Endpoint")
-      .setDesc("s3Endpoint")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(this.plugin.settings.s3Endpoint)
-          .onChange(async (value) => {
-            this.plugin.settings.s3Endpoint = value;
-            await this.plugin.saveSettings();
-          })
-      );
-
-    new Setting(containerEl)
-      .setName("s3Region")
-      .setDesc("s3Region")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3Region}`)
-          .onChange(async (value) => {
-            this.plugin.settings.s3Region = value;
-            await this.plugin.saveSettings();
-          })
-      );
-
-    new Setting(containerEl)
-      .setName("s3AccessKeyID")
-      .setDesc("s3AccessKeyID")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3AccessKeyID}`)
-          .onChange(async (value) => {
-            this.plugin.settings.s3AccessKeyID = value;
-            await this.plugin.saveSettings();
-          })
-      );
-
-    new Setting(containerEl)
-      .setName("s3SecretAccessKey")
-      .setDesc("s3SecretAccessKey")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3SecretAccessKey}`)
-          .onChange(async (value) => {
-            this.plugin.settings.s3SecretAccessKey = value;
-            await this.plugin.saveSettings();
-          })
-      );
-
-    new Setting(containerEl)
-      .setName("s3BucketName")
-      .setDesc("s3BucketName")
-      .addText((text) =>
-        text
-          .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3BucketName}`)
-          .onChange(async (value) => {
-            this.plugin.settings.s3BucketName = value;
-            await this.plugin.saveSettings();
-          })
-      );
-  }
-}

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "obsdian-save-remote",
   "name": "Save remote",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "minAppVersion": "0.12.15",
   "description": "This is yet another plugin allowing users to sync notes between local device and the cloud.",
   "author": "fyears",

package.json

@@ -13,8 +13,8 @@
   "author": "",
   "license": "Apache-2.0",
   "devDependencies": {
+    "@types/crypto-js": "^4.0.2",
     "@types/node": "^14.14.37",
-    "parcel": "^2.0.0",
     "prettier": "^2.4.1",
     "terser-webpack-plugin": "^5.2.4",
     "ts-loader": "^9.2.6",
@@ -28,11 +28,37 @@
     "@aws-sdk/client-s3": "^3.37.0",
     "@aws-sdk/signature-v4-crt": "^3.37.0",
     "@types/mime-types": "^2.1.1",
+    "acorn": "^8.5.0",
+    "assert": "^2.0.0",
     "aws-crt": "^1.10.1",
+    "browserify-zlib": "^0.2.0",
+    "buffer": "^6.0.3",
     "codemirror": "^5.63.1",
+    "console-browserify": "^1.2.0",
+    "constants-browserify": "^1.0.0",
+    "crypto-browserify": "^3.12.0",
+    "crypto-js": "^4.1.1",
+    "domain-browser": "^4.22.0",
+    "events": "^3.3.0",
+    "hi-base32": "^0.5.1",
+    "https-browserify": "^1.0.0",
+    "lovefield-ts": "^0.7.0",
     "mime-types": "^2.1.33",
     "obsidian": "^0.12.0",
+    "os-browserify": "^0.3.0",
+    "path-browserify": "^1.0.1",
+    "process": "^0.11.10",
+    "punycode": "^2.1.1",
+    "querystring-es3": "^0.2.1",
     "rimraf": "^3.0.2",
+    "stream-browserify": "^3.0.0",
+    "stream-http": "^3.2.0",
+    "string_decoder": "^1.3.0",
+    "timers-browserify": "^2.0.12",
+    "tty-browserify": "0.0.1",
+    "url": "^0.11.0",
+    "util": "^0.12.4",
+    "vm-browserify": "^1.1.2",
     "webdav": "^4.7.0",
     "webdav-fs": "^4.0.0"
   }

src/encrypt.ts

@@ -0,0 +1,113 @@
+import * as CryptoJS from "crypto-js";
+import * as base32 from "hi-base32";
+import {
+  bufferToArrayBuffer,
+  arrayBufferToBuffer,
+  arrayBufferToBase64,
+  base64ToArrayBuffer,
+} from "./misc";
+
+const DEFAULT_ITER = 10000;
+
+export const encryptWordArray = (
+  wa: CryptoJS.lib.WordArray,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  const prefix = CryptoJS.enc.Utf8.parse("Salted__");
+  const salt = CryptoJS.lib.WordArray.random(8);
+  const derivedKey = CryptoJS.PBKDF2(password, salt, {
+    keySize: 32 + 16,
+    iterations: rounds,
+    hasher: CryptoJS.algo.SHA256,
+  });
+  const key = CryptoJS.lib.WordArray.create(derivedKey.words.slice(0, 32 / 4));
+  const iv = CryptoJS.lib.WordArray.create(
+    derivedKey.words.slice(32 / 4, (32 + 16) / 4)
+  );
+  const encrypted = CryptoJS.AES.encrypt(wa, key, { iv: iv }).ciphertext;
+  const res = CryptoJS.lib.WordArray.create()
+    .concat(prefix)
+    .concat(salt)
+    .concat(encrypted);
+  return res;
+};
+
+export const decryptWordArray = (
+  wa: CryptoJS.lib.WordArray,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  const prefix = CryptoJS.lib.WordArray.create(wa.words.slice(0, 8 / 4));
+
+  const salt = CryptoJS.lib.WordArray.create(
+    wa.words.slice(8 / 4, (8 + 8) / 4)
+  );
+  const derivedKey = CryptoJS.PBKDF2(password, salt, {
+    keySize: 32 + 16,
+    iterations: rounds,
+    hasher: CryptoJS.algo.SHA256,
+  });
+  const key = CryptoJS.lib.WordArray.create(derivedKey.words.slice(0, 32 / 4));
+  const iv = CryptoJS.lib.WordArray.create(
+    derivedKey.words.slice(32 / 4, 32 / 4 + 16 / 4)
+  );
+  const decrypted = CryptoJS.AES.decrypt(
+    CryptoJS.lib.CipherParams.create({
+      ciphertext: CryptoJS.lib.WordArray.create(wa.words.slice((8 + 8) / 4)),
+    }),
+    key,
+    { iv: iv }
+  );
+  return decrypted;
+};
+
+export const encryptArrayBuffer = (
+  arrBuf: ArrayBuffer,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  const b64 = arrayBufferToBase64(arrBuf);
+  const wa = CryptoJS.enc.Base64.parse(b64);
+  const enc = encryptWordArray(wa, password, rounds);
+  const resb64 = CryptoJS.enc.Base64.stringify(enc);
+  const res = base64ToArrayBuffer(resb64);
+  return res;
+};
+
+export const decryptArrayBuffer = (
+  arrBuf: ArrayBuffer,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  const b64 = arrayBufferToBase64(arrBuf);
+  const wa = CryptoJS.enc.Base64.parse(b64);
+  const dec = decryptWordArray(wa, password, rounds);
+  const resb64 = CryptoJS.enc.Base64.stringify(dec);
+  const res = base64ToArrayBuffer(resb64);
+  return res;
+};
+
+export const encryptStringToBase32 = (
+  text: string,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  const wa = CryptoJS.enc.Utf8.parse(text);
+  const enc = encryptWordArray(wa, password, rounds);
+  const enctext = CryptoJS.enc.Base64.stringify(enc);
+  const res = base32.encode(base64ToArrayBuffer(enctext));
+  return res;
+};
+
+export const decryptBase32ToString = (
+  text: string,
+  password: string,
+  rounds: number = DEFAULT_ITER
+) => {
+  const enc = Buffer.from(base32.decode.asBytes(text)).toString("base64");
+  const wa = CryptoJS.enc.Base64.parse(enc);
+  const dec = decryptWordArray(wa, password, rounds);
+  const dectext = CryptoJS.enc.Utf8.stringify(dec);
+  return dectext;
+};

src/localdb.ts

@@ -0,0 +1,260 @@
+import * as lf from "lovefield-ts/dist/es6/lf.js";
+import { TAbstractFile, TFile, TFolder } from "obsidian";
+
+import type { SUPPORTED_SERVICES_TYPE } from "./misc";
+
+export type DatabaseConnection = lf.DatabaseConnection;
+
+export const DEFAULT_DB_NAME = "saveremotedb";
+export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory";
+export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
+
+export interface FileFolderHistoryRecord {
+  key: string;
+  ctime: number;
+  mtime: number;
+  size: number;
+  action_when: number;
+  action_type: "delete" | "rename";
+  key_type: "folder" | "file";
+  rename_to: string;
+}
+
+export interface SyncMetaMappingRecord {
+  local_key: string;
+  remote_key: string;
+  local_size: number;
+  remote_size: number;
+  local_mtime: number;
+  remote_mtime: number;
+  remote_extra_key: string;
+  remote_type: SUPPORTED_SERVICES_TYPE;
+  key_type: "folder" | "file";
+}
+
+export const prepareDBs = async () => {
+  const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1);
+  schemaBuilder
+    .createTable(DEFAULT_TBL_DELETE_HISTORY)
+    .addColumn("id", lf.Type.INTEGER)
+    .addColumn("key", lf.Type.STRING)
+    .addColumn("ctime", lf.Type.INTEGER)
+    .addColumn("mtime", lf.Type.INTEGER)
+    .addColumn("size", lf.Type.INTEGER)
+    .addColumn("action_when", lf.Type.INTEGER)
+    .addColumn("action_type", lf.Type.STRING)
+    .addColumn("key_type", lf.Type.STRING)
+    .addPrimaryKey(["id"], true)
+    .addIndex("idxKey", ["key"]);
+
+  schemaBuilder
+    .createTable(DEFAULT_TBL_SYNC_MAPPING)
+    .addColumn("id", lf.Type.INTEGER)
+    .addColumn("local_key", lf.Type.STRING)
+    .addColumn("remote_key", lf.Type.STRING)
+    .addColumn("local_size", lf.Type.INTEGER)
+    .addColumn("remote_size", lf.Type.INTEGER)
+    .addColumn("local_mtime", lf.Type.INTEGER)
+    .addColumn("remote_mtime", lf.Type.INTEGER)
+    .addColumn("key_type", lf.Type.STRING)
+    .addColumn("remote_extra_key", lf.Type.STRING)
+    .addColumn("remote_type", lf.Type.STRING)
+    .addNullable([
+      "remote_extra_key",
+      "remote_mtime",
+      "remote_size",
+      "local_mtime",
+    ])
+    .addPrimaryKey(["id"], true)
+    .addIndex("idxkey", ["local_key", "remote_key"]);
+
+  const db = await schemaBuilder.connect({
+    storeType: lf.DataStoreType.INDEXED_DB,
+  });
+  console.log("db connected");
+  return db;
+};
+
+export const destroyDBs = async (db: lf.DatabaseConnection) => {
+  db.close();
+  const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME);
+  req.onsuccess = (event) => {
+    console.log("db deleted");
+  };
+  req.onblocked = (event) => {
+    console.warn("trying to delete db but it was blocked");
+  };
+  req.onerror = (event) => {
+    console.error("tried to delete db but something bad!");
+    console.error(event);
+  };
+};
+
+export const loadDeleteRenameHistoryTable = async (
+  db: lf.DatabaseConnection
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+
+  const records = await db
+    .select()
+    .from(schema)
+    .orderBy(schema.col("action_when"), lf.Order.ASC)
+    .exec();
+
+  return records as FileFolderHistoryRecord[];
+};
+
+export const clearDeleteRenameHistoryOfKey = async (
+  db: lf.DatabaseConnection,
+  key: string
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+
+  await db.delete().from(tbl).where(tbl.col("key").eq(key)).exec();
+};
+
+export const insertDeleteRecord = async (
+  db: lf.DatabaseConnection,
+  fileOrFolder: TAbstractFile
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  // console.log(fileOrFolder);
+  let k: FileFolderHistoryRecord;
+  if (fileOrFolder instanceof TFile) {
+    k = {
+      key: fileOrFolder.path,
+      ctime: fileOrFolder.stat.ctime,
+      mtime: fileOrFolder.stat.mtime,
+      size: fileOrFolder.stat.size,
+      action_when: Date.now(),
+      action_type: "delete",
+      key_type: "file",
+      rename_to: "",
+    };
+  } else if (fileOrFolder instanceof TFolder) {
+    // key should endswith "/"
+    const key = fileOrFolder.path.endsWith("/")
+      ? fileOrFolder.path
+      : `${fileOrFolder.path}/`;
+    k = {
+      key: key,
+      ctime: 0,
+      mtime: 0,
+      size: 0,
+      action_when: Date.now(),
+      action_type: "delete",
+      key_type: "folder",
+      rename_to: "",
+    };
+  }
+  const row = tbl.createRow(k);
+  await db.insertOrReplace().into(tbl).values([row]).exec();
+};
+
+export const insertRenameRecord = async (
+  db: lf.DatabaseConnection,
+  fileOrFolder: TAbstractFile,
+  oldPath: string
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  // console.log(fileOrFolder);
+  let k: FileFolderHistoryRecord;
+  if (fileOrFolder instanceof TFile) {
+    k = {
+      key: oldPath,
+      ctime: fileOrFolder.stat.ctime,
+      mtime: fileOrFolder.stat.mtime,
+      size: fileOrFolder.stat.size,
+      action_when: Date.now(),
+      action_type: "rename",
+      key_type: "file",
+      rename_to: fileOrFolder.path,
+    };
+  } else if (fileOrFolder instanceof TFolder) {
+    const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
+    const renameTo = fileOrFolder.path.endsWith("/")
+      ? fileOrFolder.path
+      : `${fileOrFolder.path}/`;
+    k = {
+      key: key,
+      ctime: 0,
+      mtime: 0,
+      size: 0,
+      action_when: Date.now(),
+      action_type: "rename",
+      key_type: "folder",
+      rename_to: renameTo,
+    };
+  }
+  const row = tbl.createRow(k);
+  await db.insertOrReplace().into(tbl).values([row]).exec();
+};
+
+export const getAllDeleteRenameRecords = async (db: lf.DatabaseConnection) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const res1 = await db.select().from(schema).exec();
+  const res2 = res1 as FileFolderHistoryRecord[];
+  return res2;
+};
+
+export const upsertSyncMetaMappingDataS3 = async (
+  db: lf.DatabaseConnection,
+  localKey: string,
+  localMTime: number,
+  localSize: number,
+  remoteKey: string,
+  remoteMTime: number,
+  remoteSize: number,
+  remoteExtraKey: string /* ETag from s3 */
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
+  const aggregratedInfo: SyncMetaMappingRecord = {
+    local_key: localKey,
+    local_mtime: localMTime,
+    local_size: localSize,
+    remote_key: remoteKey,
+    remote_mtime: remoteMTime,
+    remote_size: remoteSize,
+    remote_extra_key: remoteExtraKey,
+    remote_type: "s3",
+    key_type: localKey.endsWith("/") ? "folder" : "file",
+  };
+  const row = schema.createRow(aggregratedInfo);
+  await db.insertOrReplace().into(schema).values([row]).exec();
+};
+
+export const getSyncMetaMappingByRemoteKeyS3 = async (
+  db: lf.DatabaseConnection,
+  remoteKey: string,
+  remoteMTime: number,
+  remoteExtraKey: string
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
+  const tbl = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
+  const res = (await db
+    .select()
+    .from(tbl)
+    .where(
+      lf.op.and(
+        tbl.col("remote_key").eq(remoteKey),
+        tbl.col("remote_mtime").eq(remoteMTime),
+        tbl.col("remote_extra_key").eq(remoteExtraKey),
+        tbl.col("remote_type").eq("s3")
+      )
+    )
+    .exec()) as SyncMetaMappingRecord[];
+
+  if (res.length === 1) {
+    return res[0];
+  }
+
+  if (res.length === 0) {
+    return undefined;
+  }
+
+  throw Error("something bad in sync meta mapping!");
+};

src/main.ts

@@ -0,0 +1,250 @@
+import {
+  App,
+  Modal,
+  Notice,
+  Plugin,
+  PluginSettingTab,
+  Setting,
+  request,
+  Platform,
+  TFile,
+  TFolder,
+} from "obsidian";
+import * as CodeMirror from "codemirror";
+import type { DatabaseConnection } from "./localdb";
+import {
+  prepareDBs,
+  destroyDBs,
+  loadDeleteRenameHistoryTable,
+  insertDeleteRecord,
+  insertRenameRecord,
+  getAllDeleteRenameRecords,
+} from "./localdb";
+
+import type { SyncStatusType } from "./sync";
+import { ensembleMixedStates, getOperation, doActualSync } from "./sync";
+import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
+
+interface SaveRemotePluginSettings {
+  s3?: S3Config;
+  password?: string;
+}
+
+const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
+  s3: DEFAULT_S3_CONFIG,
+  password: "",
+};
+
+export default class SaveRemotePlugin extends Plugin {
+  settings: SaveRemotePluginSettings;
+  cm: CodeMirror.Editor;
+  db: DatabaseConnection;
+  syncStatus: SyncStatusType;
+
+  async onload() {
+    console.log("loading plugin obsidian-save-remote");
+
+    await this.loadSettings();
+
+    await this.prepareDB();
+
+    this.syncStatus = "idle";
+
+    this.registerEvent(
+      this.app.vault.on("delete", async (fileOrFolder) => {
+        await insertDeleteRecord(this.db, fileOrFolder);
+      })
+    );
+
+    this.registerEvent(
+      this.app.vault.on("rename", async (fileOrFolder, oldPath) => {
+        await insertRenameRecord(this.db, fileOrFolder, oldPath);
+      })
+    );
+
+    // this.addRibbonIcon("dice", "Misc", async () => {
+    //   const a = this.app.vault.getAllLoadedFiles();
+    //   console.log(a);
+    //   const h = await getAllRecords(this.db);
+    //   console.log(h);
+    // });
+
+    this.addRibbonIcon("switch", "Save Remote", async () => {
+      if (this.syncStatus !== "idle") {
+        new Notice("Save Remote already running!");
+        return;
+      }
+
+      new Notice("Save Remote Sync Preparing");
+      this.syncStatus = "preparing";
+      const s3Client = getS3Client(this.settings.s3);
+      const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
+      const local = this.app.vault.getAllLoadedFiles();
+      const localHistory = await loadDeleteRenameHistoryTable(this.db);
+      // console.log(remoteRsp);
+      // console.log(local);
+      // console.log(localHistory);
+
+      const mixedStates = await ensembleMixedStates(
+        remoteRsp.Contents,
+        local,
+        localHistory,
+        this.db,
+        this.settings.password
+      );
+
+      for (const [key, val] of Object.entries(mixedStates)) {
+        getOperation(val, true);
+      }
+
+      console.log(mixedStates);
+
+      // The operations above are read only and kind of safe.
+      // The operations below begins to write or delete (!!!) something.
+
+      new Notice("Save Remote Sync data exchanging!");
+
+      await doActualSync(
+        s3Client,
+        this.settings.s3,
+        this.db,
+        this.app.vault,
+        mixedStates,
+        this.settings.password
+      );
+
+      new Notice("Save Remote finish!");
+      this.syncStatus = "idle";
+    });
+
+    this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
+
+    this.registerCodeMirror((cm: CodeMirror.Editor) => {
+      this.cm = cm;
+      console.log("codemirror registered.");
+    });
+
+    // this.registerDomEvent(document, "click", (evt: MouseEvent) => {
+    //   console.log("click", evt);
+    // });
+
+    // this.registerInterval(
+    //   window.setInterval(() => console.log("setInterval"), 5 * 60 * 1000)
+    // );
+  }
+
+  onunload() {
+    console.log("unloading plugin obsidian-save-remote");
+    this.destroyDBs();
+  }
+
+  async loadSettings() {
+    this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
+  }
+
+  async saveSettings() {
+    await this.saveData(this.settings);
+  }
+
+  async prepareDB() {
+    this.db = await prepareDBs();
+  }
+
+  destroyDBs() {
+    destroyDBs(this.db);
+  }
+}
+
+class SaveRemoteSettingTab extends PluginSettingTab {
+  plugin: SaveRemotePlugin;
+
+  constructor(app: App, plugin: SaveRemotePlugin) {
+    super(app, plugin);
+    this.plugin = plugin;
+  }
+
+  display(): void {
+    let { containerEl } = this;
+
+    containerEl.empty();
+
+    containerEl.createEl("h2", { text: "Settings for Save Remote" });
+
+    new Setting(containerEl)
+      .setName("s3Endpoint")
+      .setDesc("s3Endpoint")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(this.plugin.settings.s3.s3Endpoint)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3Endpoint = value;
+            await this.plugin.saveSettings();
+          })
+      );
+
+    new Setting(containerEl)
+      .setName("s3Region")
+      .setDesc("s3Region")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.s3.s3Region}`)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3Region = value;
+            await this.plugin.saveSettings();
+          })
+      );
+
+    new Setting(containerEl)
+      .setName("s3AccessKeyID")
+      .setDesc("s3AccessKeyID")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.s3.s3AccessKeyID}`)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3AccessKeyID = value;
+            await this.plugin.saveSettings();
+          })
+      );
+
+    new Setting(containerEl)
+      .setName("s3SecretAccessKey")
+      .setDesc("s3SecretAccessKey")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3SecretAccessKey = value;
+            await this.plugin.saveSettings();
+          })
+      );
+    new Setting(containerEl)
+      .setName("password")
+      .setDesc("password")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.password}`)
+          .onChange(async (value) => {
+            this.plugin.settings.password = value;
+            await this.plugin.saveSettings();
+          })
+      );
+
+    new Setting(containerEl)
+      .setName("s3BucketName")
+      .setDesc("s3BucketName")
+      .addText((text) =>
+        text
+          .setPlaceholder("")
+          .setValue(`${this.plugin.settings.s3.s3BucketName}`)
+          .onChange(async (value) => {
+            this.plugin.settings.s3.s3BucketName = value;
+            await this.plugin.saveSettings();
+          })
+      );
+  }
+}

src/misc.ts

@@ -0,0 +1,68 @@
+import { Vault } from "obsidian";
+import * as path from "path";
+
+export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "ftp";
+
+export const ignoreHiddenFiles = (item: string) => {
+  const basename = path.basename(item);
+  return basename === "." || basename[0] !== ".";
+};
+
+/**
+ * Util func for mkdir -p based on the "path" of original file or folder
+ * "a/b/c/" => ["a", "a/b", "a/b/c"]
+ * "a/b/c/d/e.txt" => ["a", "a/b", "a/b/c", "a/b/c/d"]
+ * @param x string
+ * @returns string[] might be empty
+ */
+export const getFolderLevels = (x: string) => {
+  const res: string[] = [];
+
+  if (x === "" || x === "/") {
+    return res;
+  }
+
+  const y1 = x.split("/");
+  let i = 0;
+  for (let index = 0; index + 1 < y1.length; index++) {
+    res.push(y1.slice(0, index + 1).join("/"));
+  }
+  return res;
+};
+
+export const mkdirpInVault = async (thePath: string, vault: Vault) => {
+  const foldersToBuild = getFolderLevels(thePath);
+  for (const folder of foldersToBuild) {
+    const r = await vault.adapter.exists(folder);
+    if (!r) {
+      console.log(`mkdir ${folder}`);
+      await vault.adapter.mkdir(folder);
+    }
+  }
+};
+
+/**
+ * https://stackoverflow.com/questions/8609289
+ * @param b Buffer
+ * @returns ArrayBuffer
+ */
+export const bufferToArrayBuffer = (b: Buffer) => {
+  return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
+};
+
+/**
+ * Simple func.
+ * @param b
+ * @returns
+ */
+export const arrayBufferToBuffer = (b: ArrayBuffer) => {
+  return Buffer.from(b);
+};
+
+export const arrayBufferToBase64 = (b: ArrayBuffer) => {
+  return arrayBufferToBuffer(b).toString("base64");
+};
+
+export const base64ToArrayBuffer = (b64text: string) => {
+  return bufferToArrayBuffer(Buffer.from(b64text, "base64"));
+};

src/s3.ts

@@ -0,0 +1,268 @@
+import { Buffer } from "buffer";
+import { Readable } from "stream";
+
+import { Vault } from "obsidian";
+
+import {
+  S3Client,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  GetObjectCommand,
+  DeleteObjectCommand,
+  HeadObjectCommand,
+} from "@aws-sdk/client-s3";
+
+import type { _Object } from "@aws-sdk/client-s3";
+
+import {
+  arrayBufferToBuffer,
+  bufferToArrayBuffer,
+  mkdirpInVault,
+} from "./misc";
+import * as mime from "mime-types";
+import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
+
+export interface S3Config {
+  s3Endpoint: string;
+  s3Region: string;
+  s3AccessKeyID: string;
+  s3SecretAccessKey: string;
+  s3BucketName: string;
+}
+
+export const DEFAULT_S3_CONFIG = {
+  s3Endpoint: "",
+  s3Region: "",
+  s3AccessKeyID: "",
+  s3SecretAccessKey: "",
+  s3BucketName: "",
+};
+
+export type S3ObjectType = _Object;
+
+export const getS3Client = (s3Config: S3Config) => {
+  const s3Client = new S3Client({
+    region: s3Config.s3Region,
+    endpoint: s3Config.s3Endpoint,
+    credentials: {
+      accessKeyId: s3Config.s3AccessKeyID,
+      secretAccessKey: s3Config.s3SecretAccessKey,
+    },
+  });
+  return s3Client;
+};
+
+export const getRemoteMeta = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string
+) => {
+  return await s3Client.send(
+    new HeadObjectCommand({
+      Bucket: s3Config.s3BucketName,
+      Key: fileOrFolderPath,
+    })
+  );
+};
+
+export const uploadToRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string,
+  vault: Vault,
+  isRecursively: boolean = false,
+  password: string = "",
+  remoteEncryptedKey: string = ""
+) => {
+  let uploadFile = fileOrFolderPath;
+  if (password !== "") {
+    uploadFile = remoteEncryptedKey;
+  }
+  const isFolder = fileOrFolderPath.endsWith("/");
+
+  const DEFAULT_CONTENT_TYPE = "application/octet-stream";
+
+  if (isFolder && isRecursively) {
+    throw Error("upload function doesn't implement recursive function yet!");
+  } else if (isFolder && !isRecursively) {
+    // folder
+    const contentType = DEFAULT_CONTENT_TYPE;
+    await s3Client.send(
+      new PutObjectCommand({
+        Bucket: s3Config.s3BucketName,
+        Key: uploadFile,
+        Body: "",
+        ContentType: contentType,
+      })
+    );
+    return await getRemoteMeta(s3Client, s3Config, fileOrFolderPath);
+  } else {
+    // file
+    // we ignore isRecursively parameter here
+    let contentType = DEFAULT_CONTENT_TYPE;
+    if (password === "") {
+      contentType =
+        mime.contentType(
+          mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE
+        ) || DEFAULT_CONTENT_TYPE;
+    }
+    const localContent = await vault.adapter.readBinary(fileOrFolderPath);
+    let remoteContent = localContent;
+    if (password !== "") {
+      remoteContent = encryptArrayBuffer(localContent, password);
+    }
+    const body = arrayBufferToBuffer(remoteContent);
+    await s3Client.send(
+      new PutObjectCommand({
+        Bucket: s3Config.s3BucketName,
+        Key: uploadFile,
+        Body: body,
+        ContentType: contentType,
+      })
+    );
+    return await getRemoteMeta(s3Client, s3Config, uploadFile);
+  }
+};
+
+export const listFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  prefix?: string
+) => {
+  if (prefix !== undefined) {
+    return await s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: s3Config.s3BucketName,
+        Prefix: prefix,
+      })
+    );
+  }
+  return await s3Client.send(
+    new ListObjectsV2Command({ Bucket: s3Config.s3BucketName })
+  );
+};
+
+/**
+ * The Body of resp of aws GetObject has mix types
+ * and we want to get ArrayBuffer here.
+ * See https://github.com/aws/aws-sdk-js-v3/issues/1877
+ * @param b The Body of GetObject
+ * @returns Promise<ArrayBuffer>
+ */
+const getObjectBodyToArrayBuffer = async (
+  b: Readable | ReadableStream | Blob
+) => {
+  if (b instanceof Readable) {
+    const chunks: Uint8Array[] = [];
+    for await (let chunk of b) {
+      chunks.push(chunk);
+    }
+    const buf = Buffer.concat(chunks);
+    return bufferToArrayBuffer(buf);
+  } else if (b instanceof ReadableStream) {
+    return await new Response(b, {}).arrayBuffer();
+  } else if (b instanceof Blob) {
+    return await b.arrayBuffer();
+  } else {
+    throw TypeError(`The type of ${b} is not one of the supported types`);
+  }
+};
+
+export const downloadFromRemoteRaw = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string
+) => {
+  const data = await s3Client.send(
+    new GetObjectCommand({
+      Bucket: s3Config.s3BucketName,
+      Key: fileOrFolderPath,
+    })
+  );
+  const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
+  return bodyContents;
+};
+
+export const downloadFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string,
+  vault: Vault,
+  mtime: number,
+  password: string = "",
+  remoteEncryptedKey: string = ""
+) => {
+  const isFolder = fileOrFolderPath.endsWith("/");
+
+  await mkdirpInVault(fileOrFolderPath, vault);
+
+  // the file is always local file
+  // we need to encrypt it
+
+  if (isFolder) {
+    // mkdirp locally is enough
+    // do nothing here
+  } else {
+    let downloadFile = fileOrFolderPath;
+    if (password !== "") {
+      downloadFile = remoteEncryptedKey;
+    }
+    const remoteContent = await downloadFromRemoteRaw(
+      s3Client,
+      s3Config,
+      downloadFile
+    );
+    let localContent = remoteContent;
+    if (password !== "") {
+      localContent = decryptArrayBuffer(remoteContent, password);
+    }
+    await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
+      mtime: mtime,
+    });
+  }
+};
+
+/**
+ * This function deals with file normally and "folder" recursively.
+ * @param s3Client
+ * @param s3Config
+ * @param fileOrFolderPath
+ * @returns
+ */
+export const deleteFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string,
+  password: string = "",
+  remoteEncryptedKey: string = ""
+) => {
+  if (fileOrFolderPath === "/") {
+    return;
+  }
+  let remoteFileName = fileOrFolderPath;
+  if (password !== "") {
+    remoteFileName = remoteEncryptedKey;
+  }
+  await s3Client.send(
+    new DeleteObjectCommand({
+      Bucket: s3Config.s3BucketName,
+      Key: remoteFileName,
+    })
+  );
+
+  if (fileOrFolderPath.endsWith("/") && password === "") {
+    const x = await listFromRemote(s3Client, s3Config, fileOrFolderPath);
+    x.Contents.forEach(async (element) => {
+      await s3Client.send(
+        new DeleteObjectCommand({
+          Bucket: s3Config.s3BucketName,
+          Key: element.Key,
+        })
+      );
+    });
+  } else if (fileOrFolderPath.endsWith("/") && password !== "") {
+    // TODO
+  } else {
+    // pass
+  }
+};

src/sync.ts

@@ -0,0 +1,386 @@
+import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
+
+import { S3Client } from "@aws-sdk/client-s3";
+import * as lf from "lovefield-ts/dist/es6/lf.js";
+
+import {
+  clearDeleteRenameHistoryOfKey,
+  FileFolderHistoryRecord,
+  upsertSyncMetaMappingDataS3,
+  getSyncMetaMappingByRemoteKeyS3,
+} from "./localdb";
+import {
+  S3Config,
+  S3ObjectType,
+  uploadToRemote,
+  deleteFromRemote,
+  downloadFromRemote,
+} from "./s3";
+import { mkdirpInVault } from "./misc";
+import { decryptBase32ToString, encryptStringToBase32 } from "./encrypt";
+
+type DecisionType =
+  | "undecided"
+  | "unknown"
+  | "upload_clearhist"
+  | "download_clearhist"
+  | "delremote_clearhist"
+  | "download"
+  | "upload"
+  | "clearhist"
+  | "mkdirplocal"
+  | "skip";
+
+export type SyncStatusType = "idle" | "preparing" | "syncing";
+
+interface FileOrFolderMixedState {
+  key: string;
+  exist_local?: boolean;
+  exist_remote?: boolean;
+  mtime_local?: number;
+  mtime_remote?: number;
+  delete_time_local?: number;
+  size_local?: number;
+  size_remote?: number;
+  decision?: DecisionType;
+  syncDone?: "done";
+  decision_branch?: number;
+  remote_encrypted_key?: string;
+}
+
+export const ensembleMixedStates = async (
+  remote: S3ObjectType[],
+  local: TAbstractFile[],
+  deleteHistory: FileFolderHistoryRecord[],
+  db: lf.DatabaseConnection,
+  password: string = ""
+) => {
+  const results = {} as Record<string, FileOrFolderMixedState>;
+
+  if (remote !== undefined) {
+    for (const entry of remote) {
+      const remoteEncryptedKey = entry.Key;
+      let key = remoteEncryptedKey;
+      if (password !== "") {
+        key = decryptBase32ToString(remoteEncryptedKey, password);
+      }
+      const backwardMapping = await getSyncMetaMappingByRemoteKeyS3(
+        db,
+        key,
+        entry.LastModified.valueOf(),
+        entry.ETag
+      );
+
+      let r = {} as FileOrFolderMixedState;
+      if (backwardMapping !== undefined) {
+        key = backwardMapping.local_key;
+        r = {
+          key: key,
+          exist_remote: true,
+          mtime_remote: backwardMapping.local_mtime,
+          size_remote: backwardMapping.local_size,
+          remote_encrypted_key: remoteEncryptedKey,
+        };
+      } else {
+        r = {
+          key: key,
+          exist_remote: true,
+          mtime_remote: entry.LastModified.valueOf(),
+          size_remote: entry.Size,
+          remote_encrypted_key: remoteEncryptedKey,
+        };
+      }
+      if (results.hasOwnProperty(key)) {
+        results[key].key = r.key;
+        results[key].exist_remote = r.exist_remote;
+        results[key].mtime_remote = r.mtime_remote;
+        results[key].size_remote = r.size_remote;
+        results[key].remote_encrypted_key = r.remote_encrypted_key;
+      } else {
+        results[key] = r;
+      }
+    }
+  }
+
+  for (const entry of local) {
+    let r = {} as FileOrFolderMixedState;
+    let key = entry.path;
+
+    if (entry.path === "/") {
+      // ignore
+      continue;
+    } else if (entry instanceof TFile) {
+      r = {
+        key: entry.path,
+        exist_local: true,
+        mtime_local: entry.stat.mtime,
+        size_local: entry.stat.size,
+      };
+    } else if (entry instanceof TFolder) {
+      key = `${entry.path}/`;
+      r = {
+        key: key,
+        exist_local: true,
+        mtime_local: undefined,
+        size_local: 0,
+      };
+    } else {
+      throw Error(`unexpected ${entry}`);
+    }
+
+    if (results.hasOwnProperty(key)) {
+      results[key].key = r.key;
+      results[key].exist_local = r.exist_local;
+      results[key].mtime_local = r.mtime_local;
+      results[key].size_local = r.size_local;
+    } else {
+      results[key] = r;
+    }
+  }
+
+  for (const entry of deleteHistory) {
+    let key = entry.key;
+    if (entry.key_type === "folder") {
+      if (!entry.key.endsWith("/")) {
+        key = `${entry.key}/`;
+      }
+    } else if (entry.key_type === "file") {
+      // pass
+    } else {
+      throw Error(`unexpected ${entry}`);
+    }
+
+    const r = {
+      key: key,
+      delete_time_local: entry.action_when,
+    } as FileOrFolderMixedState;
+
+    if (results.hasOwnProperty(key)) {
+      results[key].key = r.key;
+      results[key].delete_time_local = r.delete_time_local;
+    } else {
+      results[key] = r;
+    }
+  }
+
+  return results;
+};
+
+export const getOperation = (
+  origRecord: FileOrFolderMixedState,
+  inplace: boolean = false
+) => {
+  let r = origRecord;
+  if (!inplace) {
+    r = Object.assign({}, origRecord);
+  }
+
+  if (r.mtime_local === 0) {
+    r.mtime_local = undefined;
+  }
+  if (r.mtime_remote === 0) {
+    r.mtime_remote = undefined;
+  }
+  if (r.delete_time_local === 0) {
+    r.delete_time_local = undefined;
+  }
+  if (r.exist_local === undefined) {
+    r.exist_local = false;
+  }
+  if (r.exist_remote === undefined) {
+    r.exist_remote = false;
+  }
+  r.decision = "unknown";
+
+  if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote > r.mtime_local
+  ) {
+    r.decision = "download_clearhist";
+    r.decision_branch = 1;
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote < r.mtime_local
+  ) {
+    r.decision = "upload_clearhist";
+    r.decision_branch = 2;
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote === r.mtime_local &&
+    r.size_local === r.size_remote
+  ) {
+    r.decision = "skip";
+    r.decision_branch = 3;
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote === r.mtime_local &&
+    r.size_local !== r.size_remote
+  ) {
+    r.decision = "upload_clearhist";
+    r.decision_branch = 4;
+  } else if (r.exist_remote && r.exist_local && r.mtime_local === undefined) {
+    // this must be a folder!
+    if (!r.key.endsWith("/")) {
+      throw Error(`${r.key} is not a folder but lacks local mtime`);
+    }
+    r.decision = "skip";
+    r.decision_branch = 5;
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local !== undefined &&
+    r.mtime_remote >= r.delete_time_local
+  ) {
+    r.decision = "download_clearhist";
+    r.decision_branch = 6;
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local !== undefined &&
+    r.mtime_remote < r.delete_time_local
+  ) {
+    r.decision = "delremote_clearhist";
+    r.decision_branch = 7;
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local == undefined
+  ) {
+    r.decision = "download";
+    r.decision_branch = 8;
+  } else if (!r.exist_remote && r.exist_local && r.mtime_remote === undefined) {
+    r.decision = "upload_clearhist";
+    r.decision_branch = 9;
+  } else if (
+    !r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote === undefined &&
+    r.mtime_local === undefined
+  ) {
+    r.decision = "clearhist";
+    r.decision_branch = 10;
+  }
+
+  return r;
+};
+
+export const doActualSync = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  db: lf.DatabaseConnection,
+  vault: Vault,
+  keyStates: Record<string, FileOrFolderMixedState>,
+  password: string = ""
+) => {
+  Object.entries(keyStates)
+    .sort((k, v) => -(k as string).length)
+    .map(async ([k, v]) => {
+      const key = k as string;
+      const state = v as FileOrFolderMixedState;
+      let remoteEncryptedKey = key;
+      if (password !== "") {
+        remoteEncryptedKey = state.remote_encrypted_key;
+        if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
+          remoteEncryptedKey = encryptStringToBase32(key, password);
+        }
+      }
+
+      if (
+        state.decision === undefined ||
+        state.decision === "unknown" ||
+        state.decision === "undecided"
+      ) {
+        throw Error(`unknown decision in ${JSON.stringify(state)}`);
+      } else if (state.decision === "skip") {
+        // do nothing
+      } else if (state.decision === "download_clearhist") {
+        await downloadFromRemote(
+          s3Client,
+          s3Config,
+          state.key,
+          vault,
+          state.mtime_remote,
+          password,
+          remoteEncryptedKey
+        );
+        await clearDeleteRenameHistoryOfKey(db, state.key);
+      } else if (state.decision === "upload_clearhist") {
+        const remoteObjMeta = await uploadToRemote(
+          s3Client,
+          s3Config,
+          state.key,
+          vault,
+          false,
+          password,
+          remoteEncryptedKey
+        );
+        await upsertSyncMetaMappingDataS3(
+          db,
+          state.key,
+          state.mtime_local,
+          state.size_local,
+          state.key,
+          remoteObjMeta.LastModified.valueOf(),
+          remoteObjMeta.ContentLength,
+          remoteObjMeta.ETag
+        );
+        await clearDeleteRenameHistoryOfKey(db, state.key);
+      } else if (state.decision === "download") {
+        await mkdirpInVault(state.key, vault);
+        await downloadFromRemote(
+          s3Client,
+          s3Config,
+          state.key,
+          vault,
+          state.mtime_remote,
+          password,
+          remoteEncryptedKey
+        );
+      } else if (state.decision === "delremote_clearhist") {
+        await deleteFromRemote(s3Client, s3Config, state.key);
+        await clearDeleteRenameHistoryOfKey(db, state.key);
+      } else if (state.decision === "upload") {
+        const remoteObjMeta = await uploadToRemote(
+          s3Client,
+          s3Config,
+          state.key,
+          vault,
+          false,
+          password,
+          remoteEncryptedKey
+        );
+        await upsertSyncMetaMappingDataS3(
+          db,
+          state.key,
+          state.mtime_local,
+          state.size_local,
+          state.key,
+          remoteObjMeta.LastModified.valueOf(),
+          remoteObjMeta.ContentLength,
+          remoteObjMeta.ETag
+        );
+      } else if (state.decision === "clearhist") {
+        await clearDeleteRenameHistoryOfKey(db, state.key);
+      } else {
+        throw Error("this should never happen!");
+      }
+    });
+};

versions.json

@@ -1,3 +1,3 @@
 {
-  "0.0.1": "0.12.15"
+  "0.0.3": "0.12.15"
 }

webpack.config.js

@@ -2,7 +2,7 @@ const path = require("path");
 const TerserPlugin = require("terser-webpack-plugin");
 
 module.exports = {
-  entry: "./main.ts",
+  entry: "./src/main.ts",
   target: "web",
   output: {
     filename: "main.js",