basically working 2 way sync!

src/localdb.ts

@@ -1,4 +1,5 @@
 import * as lf from "lovefield-ts/dist/es6/lf.js";
+import { TAbstractFile, TFile, TFolder } from "obsidian";
 
 export type DatabaseConnection = lf.DatabaseConnection;
 
@@ -16,7 +17,7 @@ export interface FileFolderHistoryRecord {
   rename_to: string;
 }
 
-export async function prepareDB() {
+export const prepareDB = async () => {
   const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1);
   schemaBuilder
     .createTable(DEFAULT_TBL_DELETE_HISTORY)
@@ -35,9 +36,9 @@ export async function prepareDB() {
   });
   console.log("db connected");
   return db;
-}
+};
 
-export function destroyDB(db: lf.DatabaseConnection) {
+export const destroyDB = async (db: lf.DatabaseConnection) => {
   db.close();
   const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME);
   req.onsuccess = (event) => {
@@ -50,4 +51,113 @@ export function destroyDB(db: lf.DatabaseConnection) {
     console.error("tried to delete db but something bad!");
     console.error(event);
   };
-}
+};
+
+export const loadHistoryTable = async (db: lf.DatabaseConnection) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+
+  const records = await db
+    .select()
+    .from(schema)
+    .orderBy(schema.col("action_when"), lf.Order.ASC)
+    .exec();
+
+  return records as FileFolderHistoryRecord[];
+};
+
+export const clearHistoryOfKey = async (
+  db: lf.DatabaseConnection,
+  key: string
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+
+  await db.delete().from(tbl).where(tbl.col("key").eq(key)).exec();
+};
+
+export const insertDeleteRecord = async (
+  db: lf.DatabaseConnection,
+  fileOrFolder: TAbstractFile
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  // console.log(fileOrFolder);
+  let k: FileFolderHistoryRecord;
+  if (fileOrFolder instanceof TFile) {
+    k = {
+      key: fileOrFolder.path,
+      ctime: fileOrFolder.stat.ctime,
+      mtime: fileOrFolder.stat.mtime,
+      size: fileOrFolder.stat.size,
+      action_when: Date.now(),
+      action_type: "delete",
+      key_type: "file",
+      rename_to: "",
+    };
+  } else if (fileOrFolder instanceof TFolder) {
+    // key should endswith "/"
+    const key = fileOrFolder.path.endsWith("/")
+      ? fileOrFolder.path
+      : `${fileOrFolder.path}/`;
+    k = {
+      key: key,
+      ctime: 0,
+      mtime: 0,
+      size: 0,
+      action_when: Date.now(),
+      action_type: "delete",
+      key_type: "folder",
+      rename_to: "",
+    };
+  }
+  const row = tbl.createRow(k);
+  await db.insertOrReplace().into(tbl).values([row]).exec();
+};
+
+export const insertRenameRecord = async (
+  db: lf.DatabaseConnection,
+  fileOrFolder: TAbstractFile,
+  oldPath: string
+) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  // console.log(fileOrFolder);
+  let k: FileFolderHistoryRecord;
+  if (fileOrFolder instanceof TFile) {
+    k = {
+      key: oldPath,
+      ctime: fileOrFolder.stat.ctime,
+      mtime: fileOrFolder.stat.mtime,
+      size: fileOrFolder.stat.size,
+      action_when: Date.now(),
+      action_type: "rename",
+      key_type: "file",
+      rename_to: fileOrFolder.path,
+    };
+  } else if (fileOrFolder instanceof TFolder) {
+    const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
+    const renameTo = fileOrFolder.path.endsWith("/")
+      ? fileOrFolder.path
+      : `${fileOrFolder.path}/`;
+    k = {
+      key: key,
+      ctime: 0,
+      mtime: 0,
+      size: 0,
+      action_when: Date.now(),
+      action_type: "rename",
+      key_type: "folder",
+      rename_to: renameTo,
+    };
+  }
+  const row = tbl.createRow(k);
+  await db.insertOrReplace().into(tbl).values([row]).exec();
+};
+
+export const getAllRecords = async (db: lf.DatabaseConnection) => {
+  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+  const res1 = await db.select().from(schema).exec();
+  const res2 = res1 as FileFolderHistoryRecord[];
+  return res2;
+};

src/main.ts

@@ -1,6 +1,3 @@
-import { Buffer } from "buffer";
-import { Readable } from "stream";
-import * as mime from "mime-types";
 import {
   App,
   Modal,
@@ -14,47 +11,33 @@ import {
   TFolder,
 } from "obsidian";
 import * as CodeMirror from "codemirror";
-import type { FileFolderHistoryRecord, DatabaseConnection } from "./localdb";
+import type { DatabaseConnection } from "./localdb";
 import {
   prepareDB,
   destroyDB,
-  DEFAULT_DB_NAME,
+  loadHistoryTable,
-  DEFAULT_TBL_DELETE_HISTORY,
+  insertDeleteRecord,
+  insertRenameRecord,
+  getAllRecords,
 } from "./localdb";
 
-import {
+import type { SyncStatusType } from "./sync";
-  getFolderLevels,
+import { ensembleMixedStates, getOperation, doActualSync } from "./sync";
-  bufferToArrayBuffer,
+import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
-  getObjectBodyToArrayBuffer,
-} from "./misc";
-
-import {
-  S3Client,
-  ListObjectsV2Command,
-  PutObjectCommand,
-  GetObjectCommand,
-} from "@aws-sdk/client-s3";
 
 interface SaveRemotePluginSettings {
-  s3Endpoint: string;
+  s3?: S3Config;
-  s3Region: string;
-  s3AccessKeyID: string;
-  s3SecretAccessKey: string;
-  s3BucketName: string;
 }
 
 const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
-  s3Endpoint: "",
+  s3: DEFAULT_S3_CONFIG,
-  s3Region: "",
-  s3AccessKeyID: "",
-  s3SecretAccessKey: "",
-  s3BucketName: "",
 };
 
 export default class SaveRemotePlugin extends Plugin {
   settings: SaveRemotePluginSettings;
   cm: CodeMirror.Editor;
   db: DatabaseConnection;
+  syncStatus: SyncStatusType;
 
   async onload() {
     console.log("loading plugin obsidian-save-remote");
@@ -63,209 +46,70 @@ export default class SaveRemotePlugin extends Plugin {
 
     await this.prepareDB();
 
+    this.syncStatus = "idle";
+
     this.registerEvent(
       this.app.vault.on("delete", async (fileOrFolder) => {
-        const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+        await insertDeleteRecord(this.db, fileOrFolder);
-        const tbl = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
-        // console.log(fileOrFolder);
-        let k: FileFolderHistoryRecord;
-        if (fileOrFolder instanceof TFile) {
-          k = {
-            key: fileOrFolder.path,
-            ctime: fileOrFolder.stat.ctime,
-            mtime: fileOrFolder.stat.mtime,
-            size: fileOrFolder.stat.size,
-            action_when: Date.now(),
-            action_type: "delete",
-            key_type: "file",
-            rename_to: "",
-          };
-        } else if (fileOrFolder instanceof TFolder) {
-          k = {
-            key: fileOrFolder.path,
-            ctime: 0,
-            mtime: 0,
-            size: 0,
-            action_when: Date.now(),
-            action_type: "delete",
-            key_type: "folder",
-            rename_to: "",
-          };
-        }
-        const row = tbl.createRow(k);
-        await this.db.insertOrReplace().into(tbl).values([row]).exec();
       })
     );
 
     this.registerEvent(
       this.app.vault.on("rename", async (fileOrFolder, oldPath) => {
-        const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+        await insertRenameRecord(this.db, fileOrFolder, oldPath);
-        const tbl = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
-        // console.log(fileOrFolder);
-        let k: FileFolderHistoryRecord;
-        if (fileOrFolder instanceof TFile) {
-          k = {
-            key: oldPath,
-            ctime: fileOrFolder.stat.ctime,
-            mtime: fileOrFolder.stat.mtime,
-            size: fileOrFolder.stat.size,
-            action_when: Date.now(),
-            action_type: "rename",
-            key_type: "file",
-            rename_to: fileOrFolder.path,
-          };
-        } else if (fileOrFolder instanceof TFolder) {
-          k = {
-            key: oldPath,
-            ctime: 0,
-            mtime: 0,
-            size: 0,
-            action_when: Date.now(),
-            action_type: "rename",
-            key_type: "folder",
-            rename_to: fileOrFolder.path,
-          };
-        }
-        const row = tbl.createRow(k);
-        await this.db.insertOrReplace().into(tbl).values([row]).exec();
       })
     );
 
-    this.addRibbonIcon("dice", "Misc", async () => {
+    // this.addRibbonIcon("dice", "Misc", async () => {
-      const a = this.app.vault.getAllLoadedFiles();
+    //   const a = this.app.vault.getAllLoadedFiles();
-      console.log(a);
+    //   console.log(a);
+    //   const h = await getAllRecords(this.db);
+    //   console.log(h);
+    // });
 
-      const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+    this.addRibbonIcon("switch", "Save Remote", async () => {
-
+      if (this.syncStatus !== "idle") {
-      const h = await this.db.select().from(schema).exec();
+        new Notice("Save Remote already running!");
-
+        return;
-      console.log(h);
-
-      // console.log(b)
-    });
-
-    this.addRibbonIcon("right-arrow-with-tail", "Upload", async () => {
-      // console.log(this.app.vault.getFiles());
-      // console.log(this.app.vault.getAllLoadedFiles());
-      new Notice(`Upload begun.`);
-      const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
-
-      const s3Client = new S3Client({
-        region: this.settings.s3Region,
-        endpoint: this.settings.s3Endpoint,
-        credentials: {
-          accessKeyId: this.settings.s3AccessKeyID,
-          secretAccessKey: this.settings.s3SecretAccessKey,
-        },
-      });
-
-      try {
-        for (const fileOrFolder of allFilesAndFolders) {
-          if (fileOrFolder.path === "/") {
-            console.log('ignore "/"');
-          } else if ("children" in fileOrFolder) {
-            // folder
-            console.log(`folder ${fileOrFolder.path}/`);
-            new Notice(`folder ${fileOrFolder.path}/`);
-
-            const results = await s3Client.send(
-              new PutObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: `${fileOrFolder.path}/`,
-                Body: "",
-              })
-            );
-          } else {
-            // file
-            console.log(`file ${fileOrFolder.path}`);
-            const arrContent = await this.app.vault.adapter.readBinary(
-              fileOrFolder.path
-            );
-            new Notice(`file ${fileOrFolder.path}`);
-            const contentType =
-              mime.contentType(
-                mime.lookup(`${fileOrFolder.path}`) || undefined
-              ) || undefined;
-            // console.log(contentType);
-            const results = await s3Client.send(
-              new PutObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: `${fileOrFolder.path}`,
-                Body: Buffer.from(arrContent),
-                ContentType: contentType,
-              })
-            );
       }
-        }
-        new Notice("Upload finished!");
-      } catch (err) {
-        console.log("Error", err);
-        new Notice(`${err}`);
-      }
-    });
 
-    this.addRibbonIcon("left-arrow-with-tail", "Download", async () => {
+      new Notice("Save Remote Sync Preparing");
-      const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
+      this.syncStatus = "preparing";
+      const s3Client = getS3Client(this.settings.s3);
+      const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
+      const local = this.app.vault.getAllLoadedFiles();
+      const localHistory = await loadHistoryTable(this.db);
+      // console.log(remoteRsp);
+      // console.log(local);
+      // console.log(localHistory);
 
-      const s3Client = new S3Client({
+      const mixedStates = ensembleMixedStates(
-        region: this.settings.s3Region,
+        remoteRsp.Contents,
-        endpoint: this.settings.s3Endpoint,
+        local,
-        credentials: {
+        localHistory
-          accessKeyId: this.settings.s3AccessKeyID,
-          secretAccessKey: this.settings.s3SecretAccessKey,
-        },
-      });
-
-      try {
-        const listObj = await s3Client.send(
-          new ListObjectsV2Command({ Bucket: this.settings.s3BucketName })
       );
 
-        for (const singleContent of listObj.Contents) {
+      for (const [key, val] of Object.entries(mixedStates)) {
-          const mtimeSec = Math.round(
+        getOperation(val, true);
-            singleContent.LastModified.valueOf() / 1000.0
+      }
+
+      console.log(mixedStates);
+
+      // The operations above are read only and kind of safe.
+      // The operations below begins to write or delete (!!!) something.
+
+      new Notice("Save Remote Sync data exchanging!");
+
+      doActualSync(
+        s3Client,
+        this.settings.s3,
+        this.db,
+        this.app.vault,
+        mixedStates
       );
-          console.log(`key ${singleContent.Key} mtime ${mtimeSec}`);
 
-          const foldersToBuild = getFolderLevels(singleContent.Key);
+      new Notice("Save Remote finish!");
-          for (const folder of foldersToBuild) {
+      this.syncStatus = "idle";
-            const r = await this.app.vault.adapter.exists(folder);
-            if (!r) {
-              console.log(`mkdir ${folder}`);
-              new Notice(`mkdir ${folder}`);
-              await this.app.vault.adapter.mkdir(folder);
-            }
-          }
-
-          if (singleContent.Key.endsWith("/")) {
-            // kind of a folder
-            // pass
-          } else {
-            // kind of a file
-            // download
-
-            console.log(`download file ${singleContent.Key}`);
-            new Notice(`download file ${singleContent.Key}`);
-
-            const data = await s3Client.send(
-              new GetObjectCommand({
-                Bucket: this.settings.s3BucketName,
-                Key: singleContent.Key,
-              })
-            );
-            const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
-            await this.app.vault.adapter.writeBinary(
-              singleContent.Key,
-              bodyContents
-            );
-          }
-        }
-
-        new Notice("Download finished!");
-      } catch (err) {
-        console.log("Error", err);
-        new Notice(`${err}`);
-      }
     });
 
     this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
@@ -327,9 +171,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
       .addText((text) =>
         text
           .setPlaceholder("")
-          .setValue(this.plugin.settings.s3Endpoint)
+          .setValue(this.plugin.settings.s3.s3Endpoint)
           .onChange(async (value) => {
-            this.plugin.settings.s3Endpoint = value;
+            this.plugin.settings.s3.s3Endpoint = value;
             await this.plugin.saveSettings();
           })
       );
@@ -340,9 +184,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
       .addText((text) =>
         text
           .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3Region}`)
+          .setValue(`${this.plugin.settings.s3.s3Region}`)
           .onChange(async (value) => {
-            this.plugin.settings.s3Region = value;
+            this.plugin.settings.s3.s3Region = value;
             await this.plugin.saveSettings();
           })
       );
@@ -353,9 +197,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
       .addText((text) =>
         text
           .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3AccessKeyID}`)
+          .setValue(`${this.plugin.settings.s3.s3AccessKeyID}`)
           .onChange(async (value) => {
-            this.plugin.settings.s3AccessKeyID = value;
+            this.plugin.settings.s3.s3AccessKeyID = value;
             await this.plugin.saveSettings();
           })
       );
@@ -366,9 +210,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
       .addText((text) =>
         text
           .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3SecretAccessKey}`)
+          .setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`)
           .onChange(async (value) => {
-            this.plugin.settings.s3SecretAccessKey = value;
+            this.plugin.settings.s3.s3SecretAccessKey = value;
             await this.plugin.saveSettings();
           })
       );
@@ -379,9 +223,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
       .addText((text) =>
         text
           .setPlaceholder("")
-          .setValue(`${this.plugin.settings.s3BucketName}`)
+          .setValue(`${this.plugin.settings.s3.s3BucketName}`)
           .onChange(async (value) => {
-            this.plugin.settings.s3BucketName = value;
+            this.plugin.settings.s3.s3BucketName = value;
             await this.plugin.saveSettings();
           })
       );

src/misc.ts

@@ -1,7 +1,5 @@
+import { Vault } from "obsidian";
 import * as path from "path";
-import * as fs from "fs";
-import { Buffer } from "buffer";
-import { Readable } from "stream";
 
 export const ignoreHiddenFiles = (item: string) => {
   const basename = path.basename(item);
@@ -30,6 +28,17 @@ export const getFolderLevels = (x: string) => {
   return res;
 };
 
+export const mkdirpInVault = async (thePath: string, vault: Vault) => {
+  const foldersToBuild = getFolderLevels(thePath);
+  for (const folder of foldersToBuild) {
+    const r = await vault.adapter.exists(folder);
+    if (!r) {
+      console.log(`mkdir ${folder}`);
+      await vault.adapter.mkdir(folder);
+    }
+  }
+};
+
 /**
  * https://stackoverflow.com/questions/8609289
  * @param b Buffer
@@ -38,29 +47,3 @@ export const getFolderLevels = (x: string) => {
 export const bufferToArrayBuffer = (b: Buffer) => {
   return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
 };
-
-/**
- * The Body of resp of aws GetObject has mix types
- * and we want to get ArrayBuffer here.
- * See https://github.com/aws/aws-sdk-js-v3/issues/1877
- * @param b The Body of GetObject
- * @returns Promise<ArrayBuffer>
- */
-export const getObjectBodyToArrayBuffer = async (
-  b: Readable | ReadableStream | Blob
-) => {
-  if (b instanceof Readable) {
-    const chunks: Uint8Array[] = [];
-    for await (let chunk of b) {
-      chunks.push(chunk);
-    }
-    const buf = Buffer.concat(chunks);
-    return bufferToArrayBuffer(buf);
-  } else if (b instanceof ReadableStream) {
-    return await new Response(b, {}).arrayBuffer();
-  } else if (b instanceof Blob) {
-    return await b.arrayBuffer();
-  } else {
-    throw TypeError(`The type of ${b} is not one of the supported types`);
-  }
-};

src/s3.ts

@@ -0,0 +1,210 @@
+import { Buffer } from "buffer";
+import { Readable } from "stream";
+
+import { Vault } from "obsidian";
+
+import {
+  S3Client,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  GetObjectCommand,
+  DeleteObjectCommand,
+} from "@aws-sdk/client-s3";
+
+import type { _Object } from "@aws-sdk/client-s3";
+
+import { bufferToArrayBuffer, mkdirpInVault } from "./misc";
+import * as mime from "mime-types";
+
+export interface S3Config {
+  s3Endpoint: string;
+  s3Region: string;
+  s3AccessKeyID: string;
+  s3SecretAccessKey: string;
+  s3BucketName: string;
+}
+
+export const DEFAULT_S3_CONFIG = {
+  s3Endpoint: "",
+  s3Region: "",
+  s3AccessKeyID: "",
+  s3SecretAccessKey: "",
+  s3BucketName: "",
+};
+
+export type S3ObjectType = _Object;
+
+export const getS3Client = (s3Config: S3Config) => {
+  const s3Client = new S3Client({
+    region: s3Config.s3Region,
+    endpoint: s3Config.s3Endpoint,
+    credentials: {
+      accessKeyId: s3Config.s3AccessKeyID,
+      secretAccessKey: s3Config.s3SecretAccessKey,
+    },
+  });
+  return s3Client;
+};
+
+export const uploadToRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string,
+  vault: Vault,
+  isRecursively: boolean = false
+) => {
+  const isFolder = fileOrFolderPath.endsWith("/");
+
+  const DEFAULT_CONTENT_TYPE = "application/octet-stream";
+
+  if (isFolder && isRecursively) {
+    throw Error("upload function doesn't implement recursive function yet!");
+  } else if (isFolder && !isRecursively) {
+    // folder
+    const contentType = DEFAULT_CONTENT_TYPE;
+    return await s3Client.send(
+      new PutObjectCommand({
+        Bucket: s3Config.s3BucketName,
+        Key: fileOrFolderPath,
+        Body: "",
+        ContentType: contentType,
+      })
+    );
+  } else {
+    // file
+    // we ignore isRecursively parameter here
+    const contentType =
+      mime.contentType(mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE) ||
+      DEFAULT_CONTENT_TYPE;
+    const content = await vault.adapter.readBinary(fileOrFolderPath);
+    const body = Buffer.from(content);
+    return await s3Client.send(
+      new PutObjectCommand({
+        Bucket: s3Config.s3BucketName,
+        Key: fileOrFolderPath,
+        Body: body,
+        ContentType: contentType,
+      })
+    );
+  }
+};
+
+export const listFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  prefix?: string
+) => {
+  if (prefix !== undefined) {
+    return await s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: s3Config.s3BucketName,
+        Prefix: prefix,
+      })
+    );
+  }
+  return await s3Client.send(
+    new ListObjectsV2Command({ Bucket: s3Config.s3BucketName })
+  );
+};
+
+/**
+ * The Body of resp of aws GetObject has mix types
+ * and we want to get ArrayBuffer here.
+ * See https://github.com/aws/aws-sdk-js-v3/issues/1877
+ * @param b The Body of GetObject
+ * @returns Promise<ArrayBuffer>
+ */
+const getObjectBodyToArrayBuffer = async (
+  b: Readable | ReadableStream | Blob
+) => {
+  if (b instanceof Readable) {
+    const chunks: Uint8Array[] = [];
+    for await (let chunk of b) {
+      chunks.push(chunk);
+    }
+    const buf = Buffer.concat(chunks);
+    return bufferToArrayBuffer(buf);
+  } else if (b instanceof ReadableStream) {
+    return await new Response(b, {}).arrayBuffer();
+  } else if (b instanceof Blob) {
+    return await b.arrayBuffer();
+  } else {
+    throw TypeError(`The type of ${b} is not one of the supported types`);
+  }
+};
+
+export const downloadFromRemoteRaw = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string
+) => {
+  const data = await s3Client.send(
+    new GetObjectCommand({
+      Bucket: s3Config.s3BucketName,
+      Key: fileOrFolderPath,
+    })
+  );
+  const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
+  return bodyContents;
+};
+
+export const downloadFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string,
+  vault: Vault,
+  mtime: number
+) => {
+  const isFolder = fileOrFolderPath.endsWith("/");
+
+  await mkdirpInVault(fileOrFolderPath, vault);
+
+  if (isFolder) {
+    // mkdirp locally is enough
+    // do nothing here
+  } else {
+    const content = await downloadFromRemoteRaw(
+      s3Client,
+      s3Config,
+      fileOrFolderPath
+    );
+    await vault.adapter.writeBinary(fileOrFolderPath, content, {
+      mtime: mtime,
+    });
+  }
+};
+
+/**
+ * This function deals with file normally and "folder" recursively.
+ * @param s3Client
+ * @param s3Config
+ * @param fileOrFolderPath
+ * @returns
+ */
+export const deleteFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  fileOrFolderPath: string
+) => {
+  if (fileOrFolderPath === "/") {
+    return;
+  }
+  if (fileOrFolderPath.endsWith("/")) {
+    const x = await listFromRemote(s3Client, s3Config, fileOrFolderPath);
+    x.Contents.forEach(async (element) => {
+      await s3Client.send(
+        new DeleteObjectCommand({
+          Bucket: s3Config.s3BucketName,
+          Key: element.Key,
+        })
+      );
+    });
+  } else {
+    await s3Client.send(
+      new DeleteObjectCommand({
+        Bucket: s3Config.s3BucketName,
+        Key: fileOrFolderPath,
+      })
+    );
+  }
+};

src/sync.ts

@@ -0,0 +1,266 @@
+import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
+import { S3Client } from "@aws-sdk/client-s3";
+import * as lf from "lovefield-ts/dist/es6/lf.js";
+
+import { clearHistoryOfKey, FileFolderHistoryRecord } from "./localdb";
+import { S3Config, S3ObjectType, uploadToRemote, deleteFromRemote } from "./s3";
+import { downloadFromRemote } from "./s3";
+import { mkdirpInVault } from "./misc";
+
+type DecisionType =
+  | "undecided"
+  | "unknown"
+  | "upload_clearhist"
+  | "download_clearhist"
+  | "delremote_clearhist"
+  | "download"
+  | "upload"
+  | "clearhist"
+  | "mkdirplocal"
+  | "skip";
+
+export type SyncStatusType = "idle" | "preparing" | "syncing";
+
+interface FileOrFolderMixedState {
+  key: string;
+  exist_local?: boolean;
+  exist_remote?: boolean;
+  mtime_local?: number;
+  mtime_remote?: number;
+  delete_time_local?: number;
+  decision?: DecisionType;
+  syncDone?: "done";
+}
+
+export const ensembleMixedStates = (
+  remote: S3ObjectType[],
+  local: TAbstractFile[],
+  deleteHistory: FileFolderHistoryRecord[]
+) => {
+  const results = {} as Record<string, FileOrFolderMixedState>;
+
+  remote.forEach((entry) => {
+    let r = {} as FileOrFolderMixedState;
+    const key = entry.Key;
+    r = {
+      key: key,
+      exist_remote: true,
+      mtime_remote: entry.LastModified.valueOf(),
+    };
+    if (results.hasOwnProperty(key)) {
+      results[key].key = r.key;
+      results[key].exist_remote = r.exist_remote;
+      results[key].mtime_remote = r.mtime_remote;
+    } else {
+      results[key] = r;
+    }
+  });
+
+  local.forEach((entry) => {
+    let r = {} as FileOrFolderMixedState;
+    let key = entry.path;
+
+    if (entry.path === "/") {
+      // ignore
+      return;
+    } else if (entry instanceof TFile) {
+      r = {
+        key: entry.path,
+        exist_local: true,
+        mtime_local: entry.stat.mtime,
+      };
+    } else if (entry instanceof TFolder) {
+      key = `${entry.path}/`;
+      r = {
+        key: key,
+        exist_local: true,
+        mtime_local: undefined,
+      };
+    } else {
+      throw Error(`unexpected ${entry}`);
+    }
+
+    if (results.hasOwnProperty(key)) {
+      results[key].key = r.key;
+      results[key].exist_local = r.exist_local;
+      results[key].mtime_local = r.mtime_local;
+    } else {
+      results[key] = r;
+    }
+  });
+
+  deleteHistory.forEach((entry) => {
+    let key = entry.key;
+    if (entry.key_type === "folder") {
+      if (!entry.key.endsWith("/")) {
+        key = `${entry.key}/`;
+      }
+    } else if (entry.key_type === "file") {
+      // pass
+    } else {
+      throw Error(`unexpected ${entry}`);
+    }
+
+    const r = {
+      key: key,
+      delete_time_local: entry.action_when,
+    } as FileOrFolderMixedState;
+
+    if (results.hasOwnProperty(key)) {
+      results[key].key = r.key;
+      results[key].delete_time_local = r.delete_time_local;
+    } else {
+      results[key] = r;
+    }
+  });
+
+  return results;
+};
+
+export const getOperation = (
+  origRecord: FileOrFolderMixedState,
+  inplace: boolean = false
+) => {
+  let r = origRecord;
+  if (!inplace) {
+    r = Object.assign({}, origRecord);
+  }
+
+  if (r.mtime_local === 0) {
+    r.mtime_local = undefined;
+  }
+  if (r.mtime_remote === 0) {
+    r.mtime_remote = undefined;
+  }
+  if (r.delete_time_local === 0) {
+    r.delete_time_local = undefined;
+  }
+  if (r.exist_local === undefined) {
+    r.exist_local = false;
+  }
+  if (r.exist_remote === undefined) {
+    r.exist_remote = false;
+  }
+  r.decision = "unknown";
+
+  if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote > r.mtime_local
+  ) {
+    r.decision = "download_clearhist";
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local !== undefined &&
+    r.mtime_remote <= r.mtime_local
+  ) {
+    r.decision = "upload_clearhist";
+  } else if (
+    r.exist_remote &&
+    r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined
+  ) {
+    // this must be a folder!
+    if (!r.key.endsWith("/")) {
+      throw Error(`${r.key} is not a folder but lacks local mtime`);
+    }
+    r.decision = "skip";
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local !== undefined &&
+    r.mtime_remote >= r.delete_time_local
+  ) {
+    r.decision = "download_clearhist";
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local !== undefined &&
+    r.mtime_remote < r.delete_time_local
+  ) {
+    r.decision = "delremote_clearhist";
+  } else if (
+    r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote !== undefined &&
+    r.mtime_local === undefined &&
+    r.delete_time_local == undefined
+  ) {
+    r.decision = "download";
+  } else if (!r.exist_remote && r.exist_local && r.mtime_remote === undefined) {
+    r.decision = "upload_clearhist";
+  } else if (
+    !r.exist_remote &&
+    !r.exist_local &&
+    r.mtime_remote === undefined &&
+    r.mtime_local === undefined
+  ) {
+    r.decision = "clearhist";
+  }
+
+  return r;
+};
+
+export const doActualSync = async (
+  s3Client: S3Client,
+  s3Config: S3Config,
+  db: lf.DatabaseConnection,
+  vault: Vault,
+  keyStates: Record<string, FileOrFolderMixedState>
+) => {
+  Object.entries(keyStates)
+    .sort((k, v) => -(k as string).length)
+    .map(async ([k, v]) => {
+      const key = k as string;
+      const state = v as FileOrFolderMixedState;
+
+      if (
+        state.decision === undefined ||
+        state.decision === "unknown" ||
+        state.decision === "undecided"
+      ) {
+        throw Error(`unknown decision in ${JSON.stringify(state)}`);
+      } else if (state.decision === "skip") {
+        // do nothing
+      } else if (state.decision === "download_clearhist") {
+        await downloadFromRemote(
+          s3Client,
+          s3Config,
+          state.key,
+          vault,
+          state.mtime_remote
+        );
+        await clearHistoryOfKey(db, state.key);
+      } else if (state.decision === "upload_clearhist") {
+        await uploadToRemote(s3Client, s3Config, state.key, vault, false);
+        await clearHistoryOfKey(db, state.key);
+      } else if (state.decision === "download") {
+        await mkdirpInVault(state.key, vault);
+        await downloadFromRemote(
+          s3Client,
+          s3Config,
+          state.key,
+          vault,
+          state.mtime_remote
+        );
+      } else if (state.decision === "delremote_clearhist") {
+        await deleteFromRemote(s3Client, s3Config, state.key);
+        await clearHistoryOfKey(db, state.key);
+      } else if (state.decision === "upload") {
+        await uploadToRemote(s3Client, s3Config, state.key, vault, false);
+      } else if (state.decision === "clearhist") {
+        await clearHistoryOfKey(db, state.key);
+      } else {
+        throw Error("this should never happen!");
+      }
+    });
+};