manually delete indexeddb

.github/workflows/auto-build.yml

@@ -38,3 +38,4 @@ jobs:
           path: |
             main.js
             manifest.json
+            styles.css

manifest.json

@@ -1,7 +1,7 @@
 {
-  "id": "obsdian-save-remote",
+  "id": "obsidian-save-remote",
   "name": "Save remote",
-  "version": "0.0.11",
+  "version": "0.0.15",
   "minAppVersion": "0.12.15",
   "description": "This is yet another plugin allowing users to sync notes between local device and the cloud.",
   "author": "fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "obsidian-save-remote",
-  "version": "0.0.11",
+  "version": "0.0.15",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev": "webpack --mode development --watch",
@@ -15,10 +15,12 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@types/chai": "^4.2.22",
+    "@types/chai-as-promised": "^7.1.4",
     "@types/mime-types": "^2.1.1",
     "@types/mocha": "^9.0.0",
     "@types/node": "^14.14.37",
     "chai": "^4.3.4",
+    "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
     "mocha": "^9.1.3",
     "prettier": "^2.4.1",
@@ -33,12 +35,13 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.37.0",
+    "@aws-sdk/lib-storage": "^3.40.1",
     "@aws-sdk/signature-v4-crt": "^3.37.0",
     "acorn": "^8.5.0",
     "aws-crt": "^1.10.1",
     "buffer": "^6.0.3",
     "codemirror": "^5.63.1",
-    "lovefield-ts": "^0.7.0",
+    "localforage": "^1.10.0",
     "mime-types": "^2.1.33",
     "obsidian": "^0.12.0",
     "path-browserify": "^1.0.1",
@@ -47,6 +50,7 @@
     "rimraf": "^3.0.2",
     "stream-browserify": "^3.0.0",
     "webdav": "^4.7.0",
-    "webdav-fs": "^4.0.0"
+    "webdav-fs": "^4.0.0",
+    "xregexp": "^5.1.0"
   }
 }

src/debugMode.ts

@@ -1,22 +1,14 @@
 import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
 
-import * as lf from "lovefield-ts/dist/es6/lf.js";
-
 import type { SyncPlanType } from "./sync";
-import {
-  insertSyncPlanRecord,
-  clearAllSyncPlanRecords,
-  readAllSyncPlanRecordTexts,
-} from "./localdb";
+import { readAllSyncPlanRecordTexts } from "./localdb";
+import type { InternalDBs } from "./localdb";
 import { mkdirpInVault } from "./misc";
 
 const DEFAULT_DEBUG_FOLDER = "_debug_save_remote/";
 const DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX = "sync_plans_hist_exported_on_";
 
-export const exportSyncPlansToFiles = async (
-  db: lf.DatabaseConnection,
-  vault: Vault
-) => {
+export const exportSyncPlansToFiles = async (db: InternalDBs, vault: Vault) => {
   console.log("exporting");
   await mkdirpInVault(DEFAULT_DEBUG_FOLDER, vault);
   const records = await readAllSyncPlanRecordTexts(db);

src/localdb.ts

@@ -1,12 +1,14 @@
-import * as lf from "lovefield-ts/dist/es6/lf.js";
+import localforage from "localforage";
 import { TAbstractFile, TFile, TFolder } from "obsidian";
 
 import type { SUPPORTED_SERVICES_TYPE } from "./misc";
 import type { SyncPlanType } from "./sync";
 
-export type DatabaseConnection = lf.DatabaseConnection;
+export type LocalForage = typeof localforage;
 
+export const DEFAULT_DB_VERSION_NUMBER: number = 20211114;
 export const DEFAULT_DB_NAME = "saveremotedb";
+export const DEFAULT_TBL_VERSION = "schemaversion";
 export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory";
 export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
 export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
@@ -16,84 +18,75 @@ export interface FileFolderHistoryRecord {
   ctime: number;
   mtime: number;
   size: number;
-  action_when: number;
-  action_type: "delete" | "rename";
-  key_type: "folder" | "file";
-  rename_to: string;
+  actionWhen: number;
+  actionType: "delete" | "rename";
+  keyType: "folder" | "file";
+  renameTo: string;
 }
 
-export interface SyncMetaMappingRecord {
-  local_key: string;
-  remote_key: string;
-  local_size: number;
-  remote_size: number;
-  local_mtime: number;
-  remote_mtime: number;
-  remote_extra_key: string;
-  remote_type: SUPPORTED_SERVICES_TYPE;
-  key_type: "folder" | "file";
+interface SyncMetaMappingRecord {
+  localKey: string;
+  remoteKey: string;
+  localSize: number;
+  remoteSize: number;
+  localMtime: number;
+  remoteMtime: number;
+  remoteExtraKey: string;
+  remoteType: SUPPORTED_SERVICES_TYPE;
+  keyType: "folder" | "file";
 }
 
 interface SyncPlanRecord {
   ts: number;
-  remote_type: string;
-  sync_plan: string;
+  remoteType: string;
+  syncPlan: string;
+}
+
+export interface InternalDBs {
+  versionTbl: LocalForage;
+  deleteHistoryTbl: LocalForage;
+  syncMappingTbl: LocalForage;
+  syncPlansTbl: LocalForage;
 }
 
 export const prepareDBs = async () => {
-  const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1);
-  schemaBuilder
-    .createTable(DEFAULT_TBL_DELETE_HISTORY)
-    .addColumn("id", lf.Type.INTEGER)
-    .addColumn("key", lf.Type.STRING)
-    .addColumn("ctime", lf.Type.INTEGER)
-    .addColumn("mtime", lf.Type.INTEGER)
-    .addColumn("size", lf.Type.INTEGER)
-    .addColumn("action_when", lf.Type.INTEGER)
-    .addColumn("action_type", lf.Type.STRING)
-    .addColumn("key_type", lf.Type.STRING)
-    .addPrimaryKey(["id"], true)
-    .addIndex("idxKey", ["key"]);
+  const db = {
+    versionTbl: localforage.createInstance({
+      name: DEFAULT_DB_NAME,
+      storeName: DEFAULT_TBL_VERSION,
+    }),
+    deleteHistoryTbl: localforage.createInstance({
+      name: DEFAULT_DB_NAME,
+      storeName: DEFAULT_TBL_DELETE_HISTORY,
+    }),
+    syncMappingTbl: localforage.createInstance({
+      name: DEFAULT_DB_NAME,
+      storeName: DEFAULT_TBL_SYNC_MAPPING,
+    }),
+    syncPlansTbl: localforage.createInstance({
+      name: DEFAULT_DB_NAME,
+      storeName: DEFAULT_SYNC_PLANS_HISTORY,
+    }),
+  } as InternalDBs;
 
-  schemaBuilder
-    .createTable(DEFAULT_TBL_SYNC_MAPPING)
-    .addColumn("id", lf.Type.INTEGER)
-    .addColumn("local_key", lf.Type.STRING)
-    .addColumn("remote_key", lf.Type.STRING)
-    .addColumn("local_size", lf.Type.INTEGER)
-    .addColumn("remote_size", lf.Type.INTEGER)
-    .addColumn("local_mtime", lf.Type.INTEGER)
-    .addColumn("remote_mtime", lf.Type.INTEGER)
-    .addColumn("key_type", lf.Type.STRING)
-    .addColumn("remote_extra_key", lf.Type.STRING)
-    .addColumn("remote_type", lf.Type.STRING)
-    .addNullable([
-      "remote_extra_key",
-      "remote_mtime",
-      "remote_size",
-      "local_mtime",
-    ])
-    .addPrimaryKey(["id"], true)
-    .addIndex("idxkey", ["local_key", "remote_key"]);
+  const originalVersion = (await db.versionTbl.getItem("version")) as number;
+  if (originalVersion === null) {
+    await db.versionTbl.setItem("version", DEFAULT_DB_VERSION_NUMBER);
+  } else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) {
+    // do nothing
+  } else {
+    await migrateDBs(db, originalVersion, DEFAULT_DB_VERSION_NUMBER);
+  }
 
-  schemaBuilder
-    .createTable(DEFAULT_SYNC_PLANS_HISTORY)
-    .addColumn("id", lf.Type.INTEGER)
-    .addColumn("ts", lf.Type.INTEGER)
-    .addColumn("remote_type", lf.Type.STRING)
-    .addColumn("sync_plan", lf.Type.STRING)
-    .addPrimaryKey(["id"], true)
-    .addIndex("tskey", ["ts"]);
-
-  const db = await schemaBuilder.connect({
-    storeType: lf.DataStoreType.INDEXED_DB,
-  });
   console.log("db connected");
   return db;
 };
 
-export const destroyDBs = async (db: lf.DatabaseConnection) => {
-  db.close();
+export const destroyDBs = async () => {
+  // await localforage.dropInstance({
+  //   name: DEFAULT_DB_NAME,
+  // });
+  // console.log("db deleted");
   const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME);
   req.onsuccess = (event) => {
     console.log("db deleted");
@@ -107,37 +100,34 @@ export const destroyDBs = async (db: lf.DatabaseConnection) => {
   };
 };
 
-export const loadDeleteRenameHistoryTable = async (
-  db: lf.DatabaseConnection
-) => {
-  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
-  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
+const migrateDBs = async (db: InternalDBs, oldVer: number, newVer: number) => {
+  if (oldVer === newVer) {
+    return;
+  }
+  // not implemented
+  throw Error(`not supported internal db changes from ${oldVer} to ${newVer}`);
+};
 
-  const records = await db
-    .select()
-    .from(schema)
-    .orderBy(schema.col("action_when"), lf.Order.ASC)
-    .exec();
-
-  return records as FileFolderHistoryRecord[];
+export const loadDeleteRenameHistoryTable = async (db: InternalDBs) => {
+  const records = [] as FileFolderHistoryRecord[];
+  await db.deleteHistoryTbl.iterate((value, key, iterationNumber) => {
+    records.push(value as FileFolderHistoryRecord);
+  });
+  records.sort((a, b) => a.actionWhen - b.actionWhen); // ascending
+  return records;
 };
 
 export const clearDeleteRenameHistoryOfKey = async (
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   key: string
 ) => {
-  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
-  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
-
-  await db.delete().from(tbl).where(tbl.col("key").eq(key)).exec();
+  await db.deleteHistoryTbl.removeItem(key);
 };
 
 export const insertDeleteRecord = async (
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   fileOrFolder: TAbstractFile
 ) => {
-  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
-  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
   // console.log(fileOrFolder);
   let k: FileFolderHistoryRecord;
   if (fileOrFolder instanceof TFile) {
@@ -146,10 +136,10 @@ export const insertDeleteRecord = async (
       ctime: fileOrFolder.stat.ctime,
       mtime: fileOrFolder.stat.mtime,
       size: fileOrFolder.stat.size,
-      action_when: Date.now(),
-      action_type: "delete",
-      key_type: "file",
-      rename_to: "",
+      actionWhen: Date.now(),
+      actionType: "delete",
+      keyType: "file",
+      renameTo: "",
     };
   } else if (fileOrFolder instanceof TFolder) {
     // key should endswith "/"
@@ -161,23 +151,20 @@ export const insertDeleteRecord = async (
       ctime: 0,
       mtime: 0,
       size: 0,
-      action_when: Date.now(),
-      action_type: "delete",
-      key_type: "folder",
-      rename_to: "",
+      actionWhen: Date.now(),
+      actionType: "delete",
+      keyType: "folder",
+      renameTo: "",
     };
   }
-  const row = tbl.createRow(k);
-  await db.insertOrReplace().into(tbl).values([row]).exec();
+  await db.deleteHistoryTbl.setItem(k.key, k);
 };
 
 export const insertRenameRecord = async (
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   fileOrFolder: TAbstractFile,
   oldPath: string
 ) => {
-  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
-  const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
   // console.log(fileOrFolder);
   let k: FileFolderHistoryRecord;
   if (fileOrFolder instanceof TFile) {
@@ -186,10 +173,10 @@ export const insertRenameRecord = async (
       ctime: fileOrFolder.stat.ctime,
       mtime: fileOrFolder.stat.mtime,
       size: fileOrFolder.stat.size,
-      action_when: Date.now(),
-      action_type: "rename",
-      key_type: "file",
-      rename_to: fileOrFolder.path,
+      actionWhen: Date.now(),
+      actionType: "rename",
+      keyType: "file",
+      renameTo: fileOrFolder.path,
     };
   } else if (fileOrFolder instanceof TFolder) {
     const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
@@ -201,25 +188,17 @@ export const insertRenameRecord = async (
       ctime: 0,
       mtime: 0,
       size: 0,
-      action_when: Date.now(),
-      action_type: "rename",
-      key_type: "folder",
-      rename_to: renameTo,
+      actionWhen: Date.now(),
+      actionType: "rename",
+      keyType: "folder",
+      renameTo: renameTo,
     };
   }
-  const row = tbl.createRow(k);
-  await db.insertOrReplace().into(tbl).values([row]).exec();
-};
-
-export const getAllDeleteRenameRecords = async (db: lf.DatabaseConnection) => {
-  const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
-  const res1 = await db.select().from(schema).exec();
-  const res2 = res1 as FileFolderHistoryRecord[];
-  return res2;
+  await db.deleteHistoryTbl.setItem(k.key, k);
 };
 
 export const upsertSyncMetaMappingDataS3 = async (
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   localKey: string,
   localMTime: number,
   localSize: number,
@@ -228,89 +207,78 @@ export const upsertSyncMetaMappingDataS3 = async (
   remoteSize: number,
   remoteExtraKey: string /* ETag from s3 */
 ) => {
-  const schema = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
   const aggregratedInfo: SyncMetaMappingRecord = {
-    local_key: localKey,
-    local_mtime: localMTime,
-    local_size: localSize,
-    remote_key: remoteKey,
-    remote_mtime: remoteMTime,
-    remote_size: remoteSize,
-    remote_extra_key: remoteExtraKey,
-    remote_type: "s3",
-    key_type: localKey.endsWith("/") ? "folder" : "file",
+    localKey: localKey,
+    localMtime: localMTime,
+    localSize: localSize,
+    remoteKey: remoteKey,
+    remoteMtime: remoteMTime,
+    remoteSize: remoteSize,
+    remoteExtraKey: remoteExtraKey,
+    remoteType: "s3",
+    keyType: localKey.endsWith("/") ? "folder" : "file",
   };
-  const row = schema.createRow(aggregratedInfo);
-  await db.insertOrReplace().into(schema).values([row]).exec();
+  await db.syncMappingTbl.setItem(remoteKey, aggregratedInfo);
 };
 
 export const getSyncMetaMappingByRemoteKeyS3 = async (
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   remoteKey: string,
   remoteMTime: number,
   remoteExtraKey: string
 ) => {
-  const schema = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
-  const tbl = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
-  const res = (await db
-    .select()
-    .from(tbl)
-    .where(
-      lf.op.and(
-        tbl.col("remote_key").eq(remoteKey),
-        tbl.col("remote_mtime").eq(remoteMTime),
-        tbl.col("remote_extra_key").eq(remoteExtraKey),
-        tbl.col("remote_type").eq("s3")
-      )
-    )
-    .exec()) as SyncMetaMappingRecord[];
+  const potentialItem = (await db.syncMappingTbl.getItem(
+    remoteKey
+  )) as SyncMetaMappingRecord;
 
-  if (res.length === 1) {
-    return res[0];
-  }
-
-  if (res.length === 0) {
+  if (potentialItem === null) {
+    // no result was found
     return undefined;
   }
 
-  throw Error("something bad in sync meta mapping!");
+  if (
+    potentialItem.remoteKey === remoteKey &&
+    potentialItem.remoteMtime === remoteMTime &&
+    potentialItem.remoteExtraKey === remoteExtraKey &&
+    potentialItem.remoteType === "s3"
+  ) {
+    // the result was found
+    return potentialItem;
+  } else {
+    return undefined;
+  }
 };
 
-export const clearAllSyncMetaMapping = async (db: lf.DatabaseConnection) => {
-  const tbl = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
-  await db.delete().from(tbl).exec();
+export const clearAllSyncMetaMapping = async (db: InternalDBs) => {
+  await db.syncMappingTbl.clear();
 };
 
 export const insertSyncPlanRecord = async (
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   syncPlan: SyncPlanType
 ) => {
-  const schema = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
-  const row = schema.createRow({
+  const record = {
     ts: syncPlan.ts,
-    remote_type: syncPlan.remoteType,
-    sync_plan: JSON.stringify(syncPlan, null, 2),
-  } as SyncPlanRecord);
-  await db.insertOrReplace().into(schema).values([row]).exec();
+    remoteType: syncPlan.remoteType,
+    syncPlan: JSON.stringify(syncPlan /* directly stringify */, null, 2),
+  } as SyncPlanRecord;
+  await db.syncPlansTbl.setItem(`${syncPlan.ts}`, record);
 };
 
-export const clearAllSyncPlanRecords = async (db: lf.DatabaseConnection) => {
-  const tbl = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
-  await db.delete().from(tbl).exec();
+export const clearAllSyncPlanRecords = async (db: InternalDBs) => {
+  await db.syncPlansTbl.clear();
 };
 
-export const readAllSyncPlanRecordTexts = async (db: lf.DatabaseConnection) => {
-  const schema = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
-
-  const records = (await db
-    .select()
-    .from(schema)
-    .orderBy(schema.col("ts"), lf.Order.DESC)
-    .exec()) as SyncPlanRecord[];
+export const readAllSyncPlanRecordTexts = async (db: InternalDBs) => {
+  const records = [] as SyncPlanRecord[];
+  await db.syncPlansTbl.iterate((value, key, iterationNumber) => {
+    records.push(value as SyncPlanRecord);
+  });
+  records.sort((a, b) => -(a.ts - b.ts)); // descending
 
   if (records === undefined) {
     return [] as string[];
   } else {
-    return records.map((x) => x.sync_plan);
+    return records.map((x) => x.syncPlan);
   }
 };

src/main.ts

@@ -11,20 +11,17 @@ import {
   TFolder,
 } from "obsidian";
 import * as CodeMirror from "codemirror";
-import {
-  clearAllSyncPlanRecords,
-  clearAllSyncMetaMapping,
-  DatabaseConnection,
-} from "./localdb";
 import {
   prepareDBs,
   destroyDBs,
   loadDeleteRenameHistoryTable,
+  clearAllSyncPlanRecords,
+  clearAllSyncMetaMapping,
   insertDeleteRecord,
   insertRenameRecord,
-  getAllDeleteRenameRecords,
   insertSyncPlanRecord,
 } from "./localdb";
+import type { InternalDBs } from "./localdb";
 
 import type { SyncStatusType, PasswordCheckType } from "./sync";
 import { isPasswordOk, getSyncPlan, doActualSync } from "./sync";
@@ -50,7 +47,7 @@ const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
 export default class SaveRemotePlugin extends Plugin {
   settings: SaveRemotePluginSettings;
   cm: CodeMirror.Editor;
-  db: DatabaseConnection;
+  db: InternalDBs;
   syncStatus: SyncStatusType;
 
   async onload() {
@@ -105,6 +102,7 @@ export default class SaveRemotePlugin extends Plugin {
           this.settings.password
         );
         if (!passwordCheckResult.ok) {
+          new Notice("something goes wrong while checking password");
           throw Error(passwordCheckResult.reason);
         }
 
@@ -200,18 +198,23 @@ export class PasswordModal extends Modal {
     // contentEl.setText("Add Or change password.");
     contentEl.createEl("h2", { text: "Hold on and PLEASE READ ON..." });
     contentEl.createEl("p", {
-      text: "This password allows you encrypt your files before sending to remote services.",
+      text: "If the field is not empty, files are enctrypted using the password locally before sent to remote.",
+    });
+    contentEl.createEl("p", {
+      text: "If the field is empty, then no password is used, and files would be sent without encryption.",
     });
-    contentEl.createEl("p", { text: "Empty means no password." });
 
     contentEl.createEl("p", {
-      text: "Attention 1/3: The password setting itself is stored in PLAIN TEXT LOCALLY (because the plugin needs to use the password to encrypt the files) (and the password would not be sent to remote by this plugin).",
+      text: "Attention 1/4: The password itself is stored in PLAIN TEXT LOCALLY and would not be sent to remote by this plugin.",
     });
     contentEl.createEl("p", {
-      text: "Attention 2/3: If you change the password. You should make sure the remote service (s3/webdav/...) IS EMPTY, or REMOTE FILES WERE ENCRYPTED BY THAT NEW PASSWORD. OTHERWISE SOMETHING BAD WOULD HAPPEN!",
+      text: "Attention 2/4: Non-empty file contents are encrypted using openssl format. File/directory path are also encrypted then applied base32. BUT, some metadata such as file sizes and directory structures are not encrypted or can be easily guessed, and directory path are stored as 0-byte-size object remotely.",
     });
     contentEl.createEl("p", {
-      text: "Attention 3/3: The longer the password, the better.",
+      text: "Attention 3/4: Before changing password, you should make sure the remote store (s3/webdav/...) IS EMPTY, or REMOTE FILES WERE ENCRYPTED BY THAT NEW PASSWORD. OTHERWISE SOMETHING BAD WOULD HAPPEN!",
+    });
+    contentEl.createEl("p", {
+      text: "Attention 4/4: The longer the password, the better.",
     });
 
     new Setting(contentEl)
@@ -226,7 +229,7 @@ export class PasswordModal extends Modal {
         button.setClass("password_second_confirm");
       })
       .addButton((button) => {
-        button.setButtonText("Cancel (password not changed.)");
+        button.setButtonText("Go Back");
         button.onClick(() => {
           this.close();
         });
@@ -368,13 +371,10 @@ class SaveRemoteSettingTab extends PluginSettingTab {
 
     const syncPlanDiv = containerEl.createEl("div");
     syncPlanDiv.createEl("p", {
-      text: "Sync plans are created every time after you trigger sync and before the actual sync.",
-    });
-    syncPlanDiv.createEl("p", {
-      text: "They are useful to know what would actually happen in those sync.",
+      text: "Sync plans are created every time after you trigger sync and before the actual sync. Useful to know what would actually happen in those sync.",
     });
 
-    new Setting(containerEl)
+    new Setting(syncPlanDiv)
       .setName("export sync plans")
       .setDesc("export sync plans")
       .addButton(async (button) => {
@@ -385,7 +385,7 @@ class SaveRemoteSettingTab extends PluginSettingTab {
         });
       });
 
-    new Setting(containerEl)
+    new Setting(syncPlanDiv)
       .setName("delete sync plans history in db")
       .setDesc("delete sync plans history in db")
       .addButton(async (button) => {
@@ -398,14 +398,10 @@ class SaveRemoteSettingTab extends PluginSettingTab {
 
     const syncMappingDiv = containerEl.createEl("div");
     syncMappingDiv.createEl("p", {
-      text: "Sync mappings history stores the actual LOCAL last modified time of the REMOTE objects.",
+      text: "Sync mappings history stores the actual LOCAL last modified time of the REMOTE objects. Clearing it may cause unnecessary data exchanges in next-time sync.",
     });
 
-    syncMappingDiv.createEl("p", {
-      text: "If the sync mappings history are deleted, unnecessary data exchanges may occur in next-time syncing, because whether a remote object and local object with same name are equivalent or not could not be determined correctly by comparing last modified times.",
-    });
-
-    new Setting(containerEl)
+    new Setting(syncMappingDiv)
       .setName("delete sync mappings history in db")
       .setDesc("delete sync mappings history in db")
       .addButton(async (button) => {
@@ -415,5 +411,22 @@ class SaveRemoteSettingTab extends PluginSettingTab {
           new Notice("sync mappings history (in local db) deleted");
         });
       });
+
+    const dbsResetDiv = containerEl.createEl("div");
+    syncMappingDiv.createEl("p", {
+      text: "Reset local internal caches/databases (for debugging purposes). You would want to reload the plugin after resetting this. This option will not empty the {s3, password...} settings.",
+    });
+    new Setting(syncMappingDiv)
+      .setName("reset local internal cache/databases")
+      .setDesc("reset local internal cache/databases")
+      .addButton(async (button) => {
+        button.setButtonText("Reset");
+        button.onClick(async () => {
+          await destroyDBs();
+          new Notice(
+            "Local internal cache/databases deleted. Please manually reload the plugin."
+          );
+        });
+      });
   }
 }

src/misc.ts

@@ -2,6 +2,7 @@ import { Vault } from "obsidian";
 import * as path from "path";
 
 import { base32 } from "rfc4648";
+import XRegExp from "xregexp";
 
 export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "ftp";
 
@@ -111,3 +112,21 @@ export const hexStringToTypedArray = (hex: string) => {
 export const base64ToBase32 = (a: string) => {
   return base32.stringify(Buffer.from(a, "base64"));
 };
+
+/**
+ * iOS Safari could decrypt string with invalid password!
+ * So we need an extra way to test the decrypted result.
+ * One simple way is testing the result are "valid", printable chars or not.
+ *
+ * https://stackoverflow.com/questions/6198986
+ * https://www.regular-expressions.info/unicode.html
+ * Manual test shows that emojis like '🍎' match '\\p{Cs}',
+ * so we need to write the regrex in a form that \p{C} minus \p{Cs}
+ * @param a
+ */
+export const isVaildText = (a: string) => {
+  // If the regex matches, the string is invalid.
+  return !XRegExp("\\p{Cc}|\\p{Cf}|\\p{Co}|\\p{Cn}|\\p{Zl}|\\p{Zp}", "A").test(
+    a
+  );
+};

src/s3.ts

@@ -3,6 +3,7 @@ import { Readable } from "stream";
 
 import { Vault } from "obsidian";
 
+import { Upload } from "@aws-sdk/lib-storage";
 import {
   S3Client,
   ListObjectsV2Command,
@@ -98,7 +99,7 @@ export const uploadToRemote = async (
         ContentType: contentType,
       })
     );
-    return await getRemoteMeta(s3Client, s3Config, fileOrFolderPath);
+    return await getRemoteMeta(s3Client, s3Config, uploadFile);
   } else {
     // file
     // we ignore isRecursively parameter here
@@ -115,14 +116,24 @@ export const uploadToRemote = async (
       remoteContent = await encryptArrayBuffer(localContent, password);
     }
     const body = arrayBufferToBuffer(remoteContent);
-    await s3Client.send(
-      new PutObjectCommand({
+
+    const upload = new Upload({
+      client: s3Client,
+      queueSize: 20, // concurrency
+      partSize: 5242880, // minimal 5MB by default
+      leavePartsOnError: false,
+      params: {
         Bucket: s3Config.s3BucketName,
         Key: uploadFile,
         Body: body,
         ContentType: contentType,
-      })
-    );
+      },
+    });
+    upload.on("httpUploadProgress", (progress) => {
+      // console.log(progress);
+    });
+    await upload.done();
+
     return await getRemoteMeta(s3Client, s3Config, uploadFile);
   }
 };
@@ -149,6 +160,9 @@ export const listFromRemote = async (
     if (rsp.$metadata.httpStatusCode !== 200) {
       throw Error("some thing bad while listing remote!");
     }
+    if (rsp.Contents === undefined) {
+      break;
+    }
     contents.push(...rsp.Contents);
 
     isTruncated = rsp.IsTruncated;

src/sync.ts

@@ -1,14 +1,14 @@
 import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
 
 import { S3Client } from "@aws-sdk/client-s3";
-import * as lf from "lovefield-ts/dist/es6/lf.js";
 
 import {
   clearDeleteRenameHistoryOfKey,
-  FileFolderHistoryRecord,
   upsertSyncMetaMappingDataS3,
   getSyncMetaMappingByRemoteKeyS3,
 } from "./localdb";
+import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
+
 import {
   S3Config,
   S3ObjectType,
@@ -16,7 +16,12 @@ import {
   deleteFromRemote,
   downloadFromRemote,
 } from "./s3";
-import { mkdirpInVault, SUPPORTED_SERVICES_TYPE, isHiddenPath } from "./misc";
+import {
+  mkdirpInVault,
+  SUPPORTED_SERVICES_TYPE,
+  isHiddenPath,
+  isVaildText,
+} from "./misc";
 import {
   decryptBase32ToString,
   encryptStringToBase32,
@@ -74,6 +79,7 @@ export interface PasswordCheckType {
     | "remote_encrypted_local_no_password"
     | "password_matched"
     | "password_not_matched"
+    | "invalid_text_after_decryption"
     | "remote_not_encrypted_local_has_password"
     | "no_password_both_sides";
 }
@@ -101,10 +107,20 @@ export const isPasswordOk = async (
     }
     try {
       const res = await decryptBase32ToString(santyCheckKey, password);
-      return {
-        ok: true,
-        reason: "password_matched",
-      } as PasswordCheckType;
+
+      // additional test
+      // because iOS Safari bypasses decryption with wrong password!
+      if (isVaildText(res)) {
+        return {
+          ok: true,
+          reason: "password_matched",
+        } as PasswordCheckType;
+      } else {
+        return {
+          ok: false,
+          reason: "invalid_text_after_decryption",
+        } as PasswordCheckType;
+      }
     } catch (error) {
       return {
         ok: false,
@@ -130,7 +146,7 @@ const ensembleMixedStates = async (
   remote: S3ObjectType[],
   local: TAbstractFile[],
   deleteHistory: FileFolderHistoryRecord[],
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   password: string = ""
 ) => {
   const results = {} as Record<string, FileOrFolderMixedState>;
@@ -151,12 +167,12 @@ const ensembleMixedStates = async (
 
       let r = {} as FileOrFolderMixedState;
       if (backwardMapping !== undefined) {
-        key = backwardMapping.local_key;
+        key = backwardMapping.localKey;
         r = {
           key: key,
           exist_remote: true,
-          mtime_remote: backwardMapping.local_mtime,
-          size_remote: backwardMapping.local_size,
+          mtime_remote: backwardMapping.localMtime,
+          size_remote: backwardMapping.localSize,
           remote_encrypted_key: remoteEncryptedKey,
         };
       } else {
@@ -224,11 +240,11 @@ const ensembleMixedStates = async (
 
   for (const entry of deleteHistory) {
     let key = entry.key;
-    if (entry.key_type === "folder") {
+    if (entry.keyType === "folder") {
       if (!entry.key.endsWith("/")) {
         key = `${entry.key}/`;
       }
-    } else if (entry.key_type === "file") {
+    } else if (entry.keyType === "file") {
       // pass
     } else {
       throw Error(`unexpected ${entry}`);
@@ -236,7 +252,7 @@ const ensembleMixedStates = async (
 
     const r = {
       key: key,
-      delete_time_local: entry.action_when,
+      delete_time_local: entry.actionWhen,
     } as FileOrFolderMixedState;
 
     if (isHiddenPath(key)) {
@@ -389,7 +405,7 @@ export const getSyncPlan = async (
   remote: S3ObjectType[],
   local: TAbstractFile[],
   deleteHistory: FileFolderHistoryRecord[],
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   password: string = ""
 ) => {
   const mixedStates = await ensembleMixedStates(
@@ -413,7 +429,7 @@ export const getSyncPlan = async (
 export const doActualSync = async (
   s3Client: S3Client,
   s3Config: S3Config,
-  db: lf.DatabaseConnection,
+  db: InternalDBs,
   vault: Vault,
   syncPlan: SyncPlanType,
   password: string = ""

tests/encrypt.test.ts

@@ -1,6 +1,7 @@
 import * as fs from "fs";
 import * as path from "path";
-import { expect } from "chai";
+import * as chai from "chai";
+import chaiAsPromised from "chai-as-promised";
 import { base64ToBase32, bufferToArrayBuffer } from "../src/misc";
 import {
   decryptArrayBuffer,
@@ -9,6 +10,9 @@ import {
   encryptStringToBase32,
 } from "../src/encrypt";
 
+chai.use(chaiAsPromised);
+const expect = chai.expect;
+
 describe("Encryption tests", () => {
   beforeEach(function () {
     global.window = {
@@ -22,6 +26,14 @@ describe("Encryption tests", () => {
     expect(await encryptStringToBase32(k, password)).to.not.equal(k);
   });
 
+  it("should raise error using different password", async () => {
+    const k = "secret text";
+    const password = "hey";
+    const password2 = "hey2";
+    const enc = await encryptStringToBase32(k, password);
+    await expect(decryptBase32ToString(enc, password2)).to.be.rejected;
+  });
+
   it("should encrypt and decrypt string and get the same result returned", async () => {
     const k = "jfkkjkjbce7983ycdeknkkjckooAIUHIDIBIE((*BII)njD/d/dd/d/sjxhux";
     const password = "hfiuibec989###oiu982bj1`";

tests/misc.test.ts

@@ -70,3 +70,22 @@ describe("Misc: get folder levels", () => {
     expect(misc.getFolderLevels(item3)).to.deep.equal(res3);
   });
 });
+
+describe("Misc: vaild file name tests", () => {
+  it("should treat no ascii correctly", async () => {
+    const x = misc.isVaildText("😄🍎 apple 苹果");
+    // console.log(x)
+    expect(x).to.be.true;
+  });
+
+  it("should find not-printable chars correctly", async () => {
+    const x = misc.isVaildText("😄🍎 apple 苹果\u0000");
+    // console.log(x)
+    expect(x).to.be.false;
+  });
+
+  it("should allow spaces/slashes/...", async () => {
+    const x = misc.isVaildText("😄🍎 apple 苹果/-_=/\\*%^&@#$`");
+    expect(x).to.be.true;
+  });
+});

versions.json

@@ -1,3 +1,3 @@
 {
-  "0.0.11": "0.12.15"
+  "0.0.15": "0.12.15"
 }