half way of new sync

docs/sync_algorithm_v3.md

@@ -6,7 +6,7 @@ An absolutely better sync algorithm. Better for tracking deletions and better fo
 
 ## Huge Thanks
 
-Basically a combination of algorithm v2 + [synclone](https://github.com/Jwink3101/syncrclone) + [rsinc](https://github.com/ConorWilliams/rsinc) + (some of rclone [bisync](https://rclone.org/bisync/)). All of the later three are released under MIT License so no worries about the licenses.
+Basically a combination of algorithm v2 + [synclone](https://github.com/Jwink3101/syncrclone/blob/master/docs/algorithm.md) + [rsinc](https://github.com/ConorWilliams/rsinc) + (some of rclone [bisync](https://rclone.org/bisync/)). All of the later three are released under MIT License so no worries about the licenses.
 
 ## Features
 
@@ -27,12 +27,25 @@ Nice to have
 
 ## Description
 
-We have _five_ input sources: local all files, remote all files, _local previous succeeded sync history_, local deletions, remote deletions.
+We have _five_ input sources:
 
-Init run, consuming local deletions and remote deletions :
+1. local all files
+2. remote all files
+3. _local previous succeeded sync history_
+4. local deletions
+5. remote deletions.
+
+Init run, consuming remote deletions :
 
 TBD
 
 Later runs, use the first, second, third sources **only**.
 
-TBD
+Table modified based on synclone and rsinc. The number inside the table cell is the decision branch in the code.
+
+| local\remote    | remote unchanged   | remote modified  | remote deleted     | remote created   |
+| --------------- | ------------------ | ---------------- | ------------------ | ---------------- |
+| local unchanged | (02) do nothing    | (09) pull remote | (07) delete local  | (??) conflict    |
+| local modified  | (10) push local    | (12) conflict    | (08) push local    | (??) conflict    |
+| local deleted   | (04) delete remote | (05) pull        | (01) clean history | (03) pull remote |
+| local created   | (??) conflict      | (??) conflict    | (06) push local    | (11) conflict    |

src/baseTypes.ts

@@ -116,14 +116,6 @@ export interface RemotelySavePluginSettings {
   logToDB?: boolean;
 }
 
-export interface RemoteItem {
-  key: string;
-  lastModified?: number;
-  size: number;
-  remoteType: SUPPORTED_SERVICES_TYPE;
-  etag?: string;
-}
-
 export const COMMAND_URI = "remotely-save";
 export const COMMAND_CALLBACK = "remotely-save-cb";
 export const COMMAND_CALLBACK_ONEDRIVE = "remotely-save-cb-onedrive";
@@ -165,6 +157,68 @@ export type DecisionType =
   | DecisionTypeForFileSize
   | DecisionTypeForFolder;
 
+export type EmptyFolderCleanType = "skip" | "clean_both";
+
+export type ConflictActionType = "keep_newer" | "keep_larger" | "rename_both";
+
+export type DecisionTypeForMixedEntity =
+  | "only_history"
+  | "equal"
+  | "modified_local"
+  | "modified_remote"
+  | "created_local"
+  | "created_remote"
+  | "deleted_local"
+  | "deleted_remote"
+  | "conflict_created_keep_local"
+  | "conflict_created_keep_remote"
+  | "conflict_created_keep_both"
+  | "conflict_modified_keep_local"
+  | "conflict_modified_keep_remote"
+  | "conflict_modified_keep_both"
+  | "folder_existed_both"
+  | "folder_existed_local"
+  | "folder_existed_remote"
+  | "folder_to_be_created"
+  | "folder_to_skip"
+  | "folder_to_be_deleted";
+
+/**
+ * uniform representation
+ * everything should be flat and primitive, so that we can copy.
+ */
+export interface Entity {
+  key: string;
+  keyEnc: string;
+  mtimeCli?: number;
+  mtimeCliFmt?: string;
+  mtimeSvr?: number;
+  mtimeSvrFmt?: string;
+  prevSyncTime?: number;
+  prevSyncTimeFmt?: string;
+  size?: number; // might be unknown or to be filled
+  sizeEnc: number;
+  hash?: string;
+  etag?: string;
+}
+
+/**
+ * A replacement of FileOrFolderMixedState
+ */
+export interface MixedEntity {
+  key: string;
+  local?: Entity;
+  prevSync?: Entity;
+  remote?: Entity;
+
+  decisionBranch?: number;
+  decision?: DecisionTypeForMixedEntity;
+  conflictAction?: ConflictActionType;
+}
+
+/**
+ * @deprecated
+ */
 export interface FileOrFolderMixedState {
   key: string;
   existLocal?: boolean;

src/local.ts

@@ -0,0 +1,63 @@
+import { TFile, TFolder, type Vault } from "obsidian";
+import type { Entity, MixedEntity } from "./baseTypes";
+import { listFilesInObsFolder } from "./obsFolderLister";
+
+export const getLocalEntityList = async (
+  vault: Vault,
+  syncConfigDir: boolean,
+  configDir: string,
+  pluginID: string
+) => {
+  const local: Entity[] = [];
+
+  const localTAbstractFiles = vault.getAllLoadedFiles();
+  for (const entry of localTAbstractFiles) {
+    let r = {} as Entity;
+    let key = entry.path;
+
+    if (entry.path === "/") {
+      // ignore
+      continue;
+    } else if (entry instanceof TFile) {
+      let mtimeLocal: number | undefined = Math.max(
+        entry.stat.mtime ?? 0,
+        entry.stat.ctime
+      );
+      if (mtimeLocal === 0) {
+        mtimeLocal = undefined;
+      }
+      if (mtimeLocal === undefined) {
+        throw Error(
+          `Your file has last modified time 0: ${key}, don't know how to deal with it`
+        );
+      }
+      r = {
+        key: entry.path,
+        keyEnc: entry.path,
+        mtimeCli: mtimeLocal,
+        mtimeSvr: mtimeLocal,
+        size: entry.stat.size,
+        sizeEnc: entry.stat.size,
+      };
+    } else if (entry instanceof TFolder) {
+      key = `${entry.path}/`;
+      r = {
+        key: key,
+        keyEnc: key,
+        size: 0,
+        sizeEnc: 0,
+      };
+    } else {
+      throw Error(`unexpected ${entry}`);
+    }
+  }
+
+  if (syncConfigDir) {
+    const syncFiles = await listFilesInObsFolder(configDir, vault, pluginID);
+    for (const f of syncFiles) {
+      local.push(f);
+    }
+  }
+
+  return local;
+};

src/localdb.ts

@@ -3,35 +3,36 @@ export type LocalForage = typeof localforage;
 import { nanoid } from "nanoid";
 import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian";
 
-import { API_VER_STAT_FOLDER, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
+import { API_VER_STAT_FOLDER } from "./baseTypes";
+import type { Entity, MixedEntity, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
 import type { SyncPlanType } from "./sync";
 import { statFix, toText, unixTimeToStr } from "./misc";
 
 import { log } from "./moreOnLog";
 
-const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326];
+const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220];
-export const DEFAULT_DB_VERSION_NUMBER: number = 20220326;
+export const DEFAULT_DB_VERSION_NUMBER: number = 20240220;
 export const DEFAULT_DB_NAME = "remotelysavedb";
 export const DEFAULT_TBL_VERSION = "schemaversion";
-export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory";
-export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
 export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
 export const DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING = "vaultrandomidmapping";
 export const DEFAULT_TBL_LOGGER_OUTPUT = "loggeroutput";
 export const DEFAULT_TBL_SIMPLE_KV_FOR_MISC = "simplekvformisc";
+export const DEFAULT_TBL_PREV_SYNC_RECORDS = "prevsyncrecords";
 
-export interface FileFolderHistoryRecord {
+/**
-  key: string;
+ * @deprecated
-  ctime: number;
+ */
-  mtime: number;
+export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory";
-  size: number;
+/**
-  actionWhen: number;
+ * @deprecated
-  actionType: "delete" | "rename" | "renameDestination";
+ */
-  keyType: "folder" | "file";
+export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
-  renameTo: string;
-  vaultRandomID: string;
-}
 
+/**
+ * @deprecated
+ * But we cannot remove it. Because we want to migrate the old data.
+ */
 interface SyncMetaMappingRecord {
   localKey: string;
   remoteKey: string;
@@ -54,108 +55,74 @@ interface SyncPlanRecord {
 
 export interface InternalDBs {
   versionTbl: LocalForage;
-  fileHistoryTbl: LocalForage;
-  syncMappingTbl: LocalForage;
   syncPlansTbl: LocalForage;
   vaultRandomIDMappingTbl: LocalForage;
   loggerOutputTbl: LocalForage;
   simpleKVForMiscTbl: LocalForage;
+  prevSyncRecordsTbl: LocalForage;
+
+  /**
+   * @deprecated
+   * But we cannot remove it. Because we want to migrate the old data.
+   */
+  fileHistoryTbl: LocalForage;
+
+  /**
+   * @deprecated
+   * But we cannot remove it. Because we want to migrate the old data.
+   */
+  syncMappingTbl: LocalForage;
 }
 
 /**
- * This migration mainly aims to assign vault name or vault id into all tables.
+ * TODO
- * @param db
+ * @param syncMappings
- * @param vaultRandomID
+ * @returns
  */
-const migrateDBsFrom20211114To20220108 = async (
+const fromSyncMappingsToPrevSyncRecords = (
-  db: InternalDBs,
+  syncMappings: SyncMetaMappingRecord[]
-  vaultRandomID: string
+): Entity[] => {
-) => {
+  return [];
-  const oldVer = 20211114;
-  const newVer = 20220108;
-  log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`);
-
-  const allPromisesToWait: Promise<any>[] = [];
-
-  log.debug("assign vault id to any delete history");
-  const keysInDeleteHistoryTbl = await db.fileHistoryTbl.keys();
-  for (const key of keysInDeleteHistoryTbl) {
-    if (key.startsWith(vaultRandomID)) {
-      continue;
-    }
-    const value = (await db.fileHistoryTbl.getItem(
-      key
-    )) as FileFolderHistoryRecord;
-    if (value === null || value === undefined) {
-      continue;
-    }
-    if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
-      value.vaultRandomID = vaultRandomID;
-    }
-    const newKey = `${vaultRandomID}\t${key}`;
-    allPromisesToWait.push(db.fileHistoryTbl.setItem(newKey, value));
-    allPromisesToWait.push(db.fileHistoryTbl.removeItem(key));
-  }
-
-  log.debug("assign vault id to any sync mapping");
-  const keysInSyncMappingTbl = await db.syncMappingTbl.keys();
-  for (const key of keysInSyncMappingTbl) {
-    if (key.startsWith(vaultRandomID)) {
-      continue;
-    }
-    const value = (await db.syncMappingTbl.getItem(
-      key
-    )) as SyncMetaMappingRecord;
-    if (value === null || value === undefined) {
-      continue;
-    }
-    if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
-      value.vaultRandomID = vaultRandomID;
-    }
-    const newKey = `${vaultRandomID}\t${key}`;
-    allPromisesToWait.push(db.syncMappingTbl.setItem(newKey, value));
-    allPromisesToWait.push(db.syncMappingTbl.removeItem(key));
-  }
-
-  log.debug("assign vault id to any sync plan records");
-  const keysInSyncPlansTbl = await db.syncPlansTbl.keys();
-  for (const key of keysInSyncPlansTbl) {
-    if (key.startsWith(vaultRandomID)) {
-      continue;
-    }
-    const value = (await db.syncPlansTbl.getItem(key)) as SyncPlanRecord;
-    if (value === null || value === undefined) {
-      continue;
-    }
-    if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
-      value.vaultRandomID = vaultRandomID;
-    }
-    const newKey = `${vaultRandomID}\t${key}`;
-    allPromisesToWait.push(db.syncPlansTbl.setItem(newKey, value));
-    allPromisesToWait.push(db.syncPlansTbl.removeItem(key));
-  }
-
-  log.debug("finally update version if everything is ok");
-  await Promise.all(allPromisesToWait);
-  await db.versionTbl.setItem("version", newVer);
-
-  log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
 };
 
 /**
- * no need to do anything except changing version
+ * TODO
- * we just add more file operations in db, and no schema is changed.
  * @param db
  * @param vaultRandomID
+ * @param prevSyncRecord
  */
-const migrateDBsFrom20220108To20220326 = async (
+const setPrevSyncRecordByVault = async (
+  db: InternalDBs,
+  vaultRandomID: string,
+  prevSyncRecord: Entity
+) => {};
+
+/**
+ *
+ * @param db
+ * @param vaultRandomID
+ * Migrate the sync mapping record to sync Entity.
+ */
+const migrateDBsFrom20220326To20240220 = async (
   db: InternalDBs,
   vaultRandomID: string
 ) => {
-  const oldVer = 20220108;
+  const oldVer = 20220326;
-  const newVer = 20220326;
+  const newVer = 20240220;
   log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`);
-  await db.versionTbl.setItem("version", newVer);
+
+  // from sync mapping to prev sync
+  const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID);
+  const prevSyncRecords = fromSyncMappingsToPrevSyncRecords(syncMappings);
+  for (const prevSyncRecord of prevSyncRecords) {
+    await setPrevSyncRecordByVault(db, vaultRandomID, prevSyncRecord);
+  }
+
+  // clear not used data
+  await clearFileHistoryOfEverythingByVault(db, vaultRandomID);
+  await clearAllSyncMetaMappingByVault(db, vaultRandomID);
+
+  await db.versionTbl.setItem(`${vaultRandomID}\tversion`, newVer);
   log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
 };
 
@@ -168,18 +135,19 @@ const migrateDBs = async (
   if (oldVer === newVer) {
     return;
   }
-  if (oldVer === 20211114 && newVer === 20220108) {
+
-    return await migrateDBsFrom20211114To20220108(db, vaultRandomID);
+  // as of 20240220, we assume everyone is using 20220326 already
+  // drop any old code to reduce the verbose
+  if (oldVer < 20220326) {
+    throw Error(
+      "You are using a very old version of Remotely Save. No way to auto update internal DB. Please install and enable 0.3.40 firstly, then install a later version."
+    );
   }
-  if (oldVer === 20220108 && newVer === 20220326) {
+
-    return await migrateDBsFrom20220108To20220326(db, vaultRandomID);
+  if (oldVer === 20220326 && newVer === 20240220) {
-  }
+    return await migrateDBsFrom20220326To20240220(db, vaultRandomID);
-  if (oldVer === 20211114 && newVer === 20220326) {
-    // TODO: more steps with more versions in the future
-    await migrateDBsFrom20211114To20220108(db, vaultRandomID);
-    await migrateDBsFrom20220108To20220326(db, vaultRandomID);
-    return;
   }
+
   if (newVer < oldVer) {
     throw Error(
       "You've installed a new version, but then downgrade to an old version. Stop working!"
@@ -198,14 +166,6 @@ export const prepareDBs = async (
       name: DEFAULT_DB_NAME,
       storeName: DEFAULT_TBL_VERSION,
     }),
-    fileHistoryTbl: localforage.createInstance({
-      name: DEFAULT_DB_NAME,
-      storeName: DEFAULT_TBL_FILE_HISTORY,
-    }),
-    syncMappingTbl: localforage.createInstance({
-      name: DEFAULT_DB_NAME,
-      storeName: DEFAULT_TBL_SYNC_MAPPING,
-    }),
     syncPlansTbl: localforage.createInstance({
       name: DEFAULT_DB_NAME,
       storeName: DEFAULT_SYNC_PLANS_HISTORY,
@@ -222,6 +182,19 @@ export const prepareDBs = async (
       name: DEFAULT_DB_NAME,
       storeName: DEFAULT_TBL_SIMPLE_KV_FOR_MISC,
     }),
+    prevSyncRecordsTbl: localforage.createInstance({
+      name: DEFAULT_DB_NAME,
+      storeName: DEFAULT_TBL_PREV_SYNC_RECORDS,
+    }),
+
+    fileHistoryTbl: localforage.createInstance({
+      name: DEFAULT_DB_NAME,
+      storeName: DEFAULT_TBL_FILE_HISTORY,
+    }),
+    syncMappingTbl: localforage.createInstance({
+      name: DEFAULT_DB_NAME,
+      storeName: DEFAULT_TBL_SYNC_MAPPING,
+    }),
   } as InternalDBs;
 
   // try to get vaultRandomID firstly
@@ -253,12 +226,19 @@ export const prepareDBs = async (
     throw Error("no vaultRandomID found or generated");
   }
 
-  const originalVersion: number | null = await db.versionTbl.getItem("version");
+  // as of 20240220, we set the version per vault, instead of global "version"
+  const originalVersion: number | null =
+    (await db.versionTbl.getItem(`${vaultRandomID}\tversion`)) ??
+    (await db.versionTbl.getItem("version"));
   if (originalVersion === null) {
     log.debug(
       `no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}`
     );
-    await db.versionTbl.setItem("version", DEFAULT_DB_VERSION_NUMBER);
+    // as of 20240220, we set the version per vault, instead of global "version"
+    await db.versionTbl.setItem(
+      `${vaultRandomID}\tversion`,
+      DEFAULT_DB_VERSION_NUMBER
+    );
   } else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) {
     // do nothing
   } else {
@@ -298,272 +278,47 @@ export const destroyDBs = async () => {
   };
 };
 
-export const loadFileHistoryTableByVault = async (
+export const clearFileHistoryOfEverythingByVault = async (
   db: InternalDBs,
   vaultRandomID: string
 ) => {
-  const records = [] as FileFolderHistoryRecord[];
+  const keys = await db.fileHistoryTbl.keys();
-  await db.fileHistoryTbl.iterate((value, key, iterationNumber) => {
+  for (const key of keys) {
     if (key.startsWith(`${vaultRandomID}\t`)) {
-      records.push(value as FileFolderHistoryRecord);
+      await db.fileHistoryTbl.removeItem(key);
     }
-  });
-  records.sort((a, b) => a.actionWhen - b.actionWhen); // ascending
-  return records;
-};
-
-export const clearDeleteRenameHistoryOfKeyAndVault = async (
-  db: InternalDBs,
-  key: string,
-  vaultRandomID: string
-) => {
-  const fullKey = `${vaultRandomID}\t${key}`;
-  const item: FileFolderHistoryRecord | null =
-    await db.fileHistoryTbl.getItem(fullKey);
-  if (
-    item !== null &&
-    (item.actionType === "delete" || item.actionType === "rename")
-  ) {
-    await db.fileHistoryTbl.removeItem(fullKey);
-  }
-};
-
-export const insertDeleteRecordByVault = async (
-  db: InternalDBs,
-  fileOrFolder: TAbstractFile | string,
-  vaultRandomID: string
-) => {
-  // log.info(fileOrFolder);
-  let k: FileFolderHistoryRecord;
-  if (fileOrFolder instanceof TFile) {
-    k = {
-      key: fileOrFolder.path,
-      ctime: fileOrFolder.stat.ctime,
-      mtime: fileOrFolder.stat.mtime,
-      size: fileOrFolder.stat.size,
-      actionWhen: Date.now(),
-      actionType: "delete",
-      keyType: "file",
-      renameTo: "",
-      vaultRandomID: vaultRandomID,
-    };
-    await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
-  } else if (fileOrFolder instanceof TFolder) {
-    // key should endswith "/"
-    const key = fileOrFolder.path.endsWith("/")
-      ? fileOrFolder.path
-      : `${fileOrFolder.path}/`;
-    const ctime = 0; // they are deleted, so no way to get ctime, mtime
-    const mtime = 0; // they are deleted, so no way to get ctime, mtime
-    k = {
-      key: key,
-      ctime: ctime,
-      mtime: mtime,
-      size: 0,
-      actionWhen: Date.now(),
-      actionType: "delete",
-      keyType: "folder",
-      renameTo: "",
-      vaultRandomID: vaultRandomID,
-    };
-    await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
-  } else if (typeof fileOrFolder === "string") {
-    // always the deletions in .obsidian folder
-    // so annoying that the path doesn't exists
-    // and we have to guess whether the path is folder or file
-    k = {
-      key: fileOrFolder,
-      ctime: 0,
-      mtime: 0,
-      size: 0,
-      actionWhen: Date.now(),
-      actionType: "delete",
-      keyType: "file",
-      renameTo: "",
-      vaultRandomID: vaultRandomID,
-    };
-    await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
-    for (const ext of [
-      "json",
-      "js",
-      "mjs",
-      "ts",
-      "md",
-      "txt",
-      "css",
-      "png",
-      "gif",
-      "jpg",
-      "jpeg",
-      "gitignore",
-      "gitkeep",
-    ]) {
-      if (fileOrFolder.endsWith(`.${ext}`)) {
-        // stop here, no more need to insert the folder record later
-        return;
-      }
-    }
-    // also add a deletion record as folder if not ending with special exts
-    k = {
-      key: `${fileOrFolder}/`,
-      ctime: 0,
-      mtime: 0,
-      size: 0,
-      actionWhen: Date.now(),
-      actionType: "delete",
-      keyType: "folder",
-      renameTo: "",
-      vaultRandomID: vaultRandomID,
-    };
-    await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
   }
 };
 
 /**
- * A file/folder is renamed from A to B
+ * @deprecated But we cannot remove it. Because we want to migrate the old data.
- * We insert two records:
- * A with actionType="rename"
- * B with actionType="renameDestination"
  * @param db
- * @param fileOrFolder
- * @param oldPath
  * @param vaultRandomID
+ * @returns
  */
-export const insertRenameRecordByVault = async (
+export const getAllSyncMetaMappingByVault = async (
   db: InternalDBs,
-  fileOrFolder: TAbstractFile,
-  oldPath: string,
   vaultRandomID: string
 ) => {
-  // log.info(fileOrFolder);
+  return await Promise.all(
-  let k1: FileFolderHistoryRecord | undefined;
+    ((await db.syncMappingTbl.keys()) ?? [])
-  let k2: FileFolderHistoryRecord | undefined;
+      .filter((key) => key.startsWith(`${vaultRandomID}\t`))
-  const actionWhen = Date.now();
+      .map(
-  if (fileOrFolder instanceof TFile) {
+        async (key) =>
-    k1 = {
+          (await db.syncMappingTbl.getItem(key)) as SyncMetaMappingRecord
-      key: oldPath,
+      )
-      ctime: fileOrFolder.stat.ctime,
-      mtime: fileOrFolder.stat.mtime,
-      size: fileOrFolder.stat.size,
-      actionWhen: actionWhen,
-      actionType: "rename",
-      keyType: "file",
-      renameTo: fileOrFolder.path,
-      vaultRandomID: vaultRandomID,
-    };
-    k2 = {
-      key: fileOrFolder.path,
-      ctime: fileOrFolder.stat.ctime,
-      mtime: fileOrFolder.stat.mtime,
-      size: fileOrFolder.stat.size,
-      actionWhen: actionWhen,
-      actionType: "renameDestination",
-      keyType: "file",
-      renameTo: "", // itself is the destination, so no need to set this field
-      vaultRandomID: vaultRandomID,
-    };
-  } else if (fileOrFolder instanceof TFolder) {
-    const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
-    const renameTo = fileOrFolder.path.endsWith("/")
-      ? fileOrFolder.path
-      : `${fileOrFolder.path}/`;
-    let ctime = 0;
-    let mtime = 0;
-    if (requireApiVersion(API_VER_STAT_FOLDER)) {
-      // TAbstractFile does not contain these info
-      // but from API_VER_STAT_FOLDER we can manually stat them by path.
-      const s = await statFix(fileOrFolder.vault, fileOrFolder.path);
-      if (s !== undefined && s !== null) {
-        ctime = s.ctime;
-        mtime = s.mtime;
-      }
-    }
-    k1 = {
-      key: key,
-      ctime: ctime,
-      mtime: mtime,
-      size: 0,
-      actionWhen: actionWhen,
-      actionType: "rename",
-      keyType: "folder",
-      renameTo: renameTo,
-      vaultRandomID: vaultRandomID,
-    };
-    k2 = {
-      key: renameTo,
-      ctime: ctime,
-      mtime: mtime,
-      size: 0,
-      actionWhen: actionWhen,
-      actionType: "renameDestination",
-      keyType: "folder",
-      renameTo: "", // itself is the destination, so no need to set this field
-      vaultRandomID: vaultRandomID,
-    };
-  }
-  await Promise.all([
-    db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k1!.key}`, k1),
-    db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k2!.key}`, k2),
-  ]);
-};
-
-export const upsertSyncMetaMappingDataByVault = async (
-  serviceType: SUPPORTED_SERVICES_TYPE,
-  db: InternalDBs,
-  localKey: string,
-  localMTime: number,
-  localSize: number,
-  remoteKey: string,
-  remoteMTime: number,
-  remoteSize: number,
-  remoteExtraKey: string,
-  vaultRandomID: string
-) => {
-  const aggregratedInfo: SyncMetaMappingRecord = {
-    localKey: localKey,
-    localMtime: localMTime,
-    localSize: localSize,
-    remoteKey: remoteKey,
-    remoteMtime: remoteMTime,
-    remoteSize: remoteSize,
-    remoteExtraKey: remoteExtraKey,
-    remoteType: serviceType,
-    keyType: localKey.endsWith("/") ? "folder" : "file",
-    vaultRandomID: vaultRandomID,
-  };
-  await db.syncMappingTbl.setItem(
-    `${vaultRandomID}\t${remoteKey}`,
-    aggregratedInfo
   );
 };
 
-export const getSyncMetaMappingByRemoteKeyAndVault = async (
+export const clearAllSyncMetaMappingByVault = async (
-  serviceType: SUPPORTED_SERVICES_TYPE,
   db: InternalDBs,
-  remoteKey: string,
-  remoteMTime: number,
-  remoteExtraKey: string,
   vaultRandomID: string
 ) => {
-  const potentialItem = (await db.syncMappingTbl.getItem(
+  const keys = await db.syncMappingTbl.keys();
-    `${vaultRandomID}\t${remoteKey}`
+  for (const key of keys) {
-  )) as SyncMetaMappingRecord;
+    if (key.startsWith(`${vaultRandomID}\t`)) {
-
+      await db.syncMappingTbl.removeItem(key);
-  if (potentialItem === null) {
+    }
-    // no result was found
-    return undefined;
-  }
-
-  if (
-    potentialItem.remoteKey === remoteKey &&
-    potentialItem.remoteMtime === remoteMTime &&
-    potentialItem.remoteExtraKey === remoteExtraKey &&
-    potentialItem.remoteType === serviceType
-  ) {
-    // the result was found
-    return potentialItem;
-  } else {
-    return undefined;
   }
 };
 
@@ -651,6 +406,37 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => {
   await Promise.all(ps);
 };
 
+export const upsertPrevSyncRecordByVault = async (
+  db: InternalDBs,
+  vaultRandomID: string,
+  prevSync: Entity
+) => {
+  await db.prevSyncRecordsTbl.setItem(
+    `${vaultRandomID}-${prevSync.key}`,
+    prevSync
+  );
+};
+
+export const clearPrevSyncRecordByVault = async (
+  db: InternalDBs,
+  vaultRandomID: string,
+  key: string
+) => {
+  await db.prevSyncRecordsTbl.removeItem(`${vaultRandomID}-${key}`);
+};
+
+export const clearAllPrevSyncRecordByVault = async (
+  db: InternalDBs,
+  vaultRandomID: string
+) => {
+  const keys = await db.prevSyncRecordsTbl.keys();
+  for (const key of keys) {
+    if (key.startsWith(`${vaultRandomID}\t`)) {
+      await db.prevSyncRecordsTbl.removeItem(key);
+    }
+  }
+};
+
 export const clearAllLoggerOutputRecords = async (db: InternalDBs) => {
   await db.loggerOutputTbl.clear();
   log.debug(`successfully clearAllLoggerOutputRecords`);

src/main.ts

@@ -240,8 +240,8 @@ export default class RemotelySavePlugin extends Plugin {
         this.app.vault.getName(),
         () => self.saveSettings()
       );
-      const remoteRsp = await client.listAllFromRemote();
+      const remoteEntityList = await client.listAllFromRemote();
-      // log.debug(remoteRsp);
+      // log.debug(remoteEntityList);
 
       if (this.settings.currLogLevel === "info") {
         // pass
@@ -250,7 +250,7 @@ export default class RemotelySavePlugin extends Plugin {
       }
       this.syncStatus = "checking_password";
       const passwordCheckResult = await isPasswordOk(
-        remoteRsp.Contents,
+        remoteEntityList,
         this.settings.password
       );
       if (!passwordCheckResult.ok) {
@@ -265,7 +265,7 @@ export default class RemotelySavePlugin extends Plugin {
       }
       this.syncStatus = "getting_remote_extra_meta";
       const { remoteStates, metadataFile } = await parseRemoteItems(
-        remoteRsp.Contents,
+        remoteEntityList,
         this.db,
         this.vaultRandomID,
         client.serviceType,

src/misc.ts

@@ -76,16 +76,17 @@ export const getFolderLevels = (x: string, addEndingSlash: boolean = false) => {
 
 export const mkdirpInVault = async (thePath: string, vault: Vault) => {
   // log.info(thePath);
-  const foldersToBuild = getFolderLevels(thePath);
+
-  // log.info(foldersToBuild);
+  // as of 2020219,
-  for (const folder of foldersToBuild) {
+  // Obsidian can create the folder recursively
-    const r = await vault.adapter.exists(folder);
+  // but the path should not end with '/'
-    // log.info(r);
+  if (thePath === "/" || thePath === "") {
-    if (!r) {
+    return;
-      log.info(`mkdir ${folder}`);
-      await vault.adapter.mkdir(folder);
-    }
   }
+  let thePathNoEnding = thePath.endsWith("/")
+    ? thePath.slice(0, thePath.length - 1)
+    : thePath;
+  await vault.adapter.mkdir(thePathNoEnding);
 };
 
 /**
@@ -435,7 +436,10 @@ export const statFix = async (vault: Vault, path: string) => {
   return s;
 };
 
-export const isFolderToSkip = (x: string, more: string[] | undefined) => {
+export const isSpecialFolderNameToSkip = (
+  x: string,
+  more: string[] | undefined
+) => {
   let specialFolders = [
     ".git",
     ".github",

src/obsFolderLister.ts

@@ -1,17 +1,11 @@
-import { Vault, Stat, ListedFiles } from "obsidian";
+import type { Vault, Stat, ListedFiles } from "obsidian";
+import type { Entity, MixedEntity } from "./baseTypes";
+
 import { Queue } from "@fyears/tsqueue";
 import chunk from "lodash/chunk";
 import flatten from "lodash/flatten";
 import { statFix, isFolderToSkip } from "./misc";
 
-export interface ObsConfigDirFileType {
-  key: string;
-  ctime: number;
-  mtime: number;
-  size: number;
-  type: "folder" | "file";
-}
-
 const isPluginDirItself = (x: string, pluginId: string) => {
   return (
     x === pluginId ||
@@ -48,10 +42,10 @@ export const listFilesInObsFolder = async (
   configDir: string,
   vault: Vault,
   pluginId: string
-) => {
+): Promise<Entity[]> => {
   const q = new Queue([configDir]);
   const CHUNK_SIZE = 10;
-  const contents: ObsConfigDirFileType[] = [];
+  const contents: Entity[] = [];
   while (q.length > 0) {
     const itemsToFetch: string[] = [];
     while (q.length > 0) {
@@ -72,11 +66,26 @@ export const listFilesInObsFolder = async (
           children = await vault.adapter.list(x);
         }
 
+        if (
+          !isFolder &&
+          (statRes.mtime === undefined ||
+            statRes.mtime === null ||
+            statRes.mtime === 0)
+        ) {
+          throw Error(
+            `File in Obsidian ${configDir} has last modified time 0: ${x}, don't know how to deal with it.`
+          );
+        }
+
         return {
           itself: {
             key: isFolder ? `${x}/` : x,
-            ...statRes,
+            keyEnc: isFolder ? `${x}/` : x,
-          } as ObsConfigDirFileType,
+            mtimeCli: statRes.mtime,
+            mtimeSvr: statRes.mtime,
+            size: statRes.size,
+            sizeEnc: statRes.size,
+          },
           children: children,
         };
       });

src/remote.ts

@@ -1,5 +1,6 @@
 import { Vault } from "obsidian";
 import type {
+  Entity,
   DropboxConfig,
   OnedriveConfig,
   S3Config,
@@ -164,7 +165,7 @@ export class RemoteClient {
     }
   };
 
-  listAllFromRemote = async () => {
+  listAllFromRemote = async (): Promise<Entity[]> => {
     if (this.serviceType === "s3") {
       return await s3.listAllFromRemote(
         s3.getS3Client(this.s3Config!),

src/remoteForDropbox.ts

@@ -5,7 +5,7 @@ import { Vault } from "obsidian";
 import * as path from "path";
 import {
   DropboxConfig,
-  RemoteItem,
+  Entity,
   COMMAND_CALLBACK_DROPBOX,
   OAUTH2_FORCE_EXPIRE_MILLISECONDS,
 } from "./baseTypes";
@@ -69,13 +69,13 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
   return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length);
 };
 
-const fromDropboxItemToRemoteItem = (
+const fromDropboxItemToEntity = (
   x:
     | files.FileMetadataReference
     | files.FolderMetadataReference
     | files.DeletedMetadataReference,
   remoteBaseDir: string
-): RemoteItem => {
+): Entity => {
   let key = getNormPath(x.path_display!, remoteBaseDir);
   if (x[".tag"] === "folder" && !key.endsWith("/")) {
     key = `${key}/`;
@@ -84,93 +84,30 @@ const fromDropboxItemToRemoteItem = (
   if (x[".tag"] === "folder") {
     return {
       key: key,
-      lastModified: undefined,
+      keyEnc: key,
       size: 0,
-      remoteType: "dropbox",
+      sizeEnc: 0,
       etag: `${x.id}\t`,
-    } as RemoteItem;
+    } as Entity;
   } else if (x[".tag"] === "file") {
-    let mtime = Date.parse(x.client_modified).valueOf();
+    const mtimeCli = Date.parse(x.client_modified).valueOf();
-    if (mtime === 0) {
+    const mtimeSvr = Date.parse(x.server_modified).valueOf();
-      mtime = Date.parse(x.server_modified).valueOf();
-    }
     return {
       key: key,
-      lastModified: mtime,
+      keyEnc: key,
+      mtimeCli: mtimeCli,
+      mtimeSvr: mtimeSvr,
       size: x.size,
-      remoteType: "dropbox",
+      sizeEnc: x.size,
+      hash: x.content_hash,
       etag: `${x.id}\t${x.content_hash}`,
-    } as RemoteItem;
+    } as Entity;
   } else {
     // x[".tag"] === "deleted"
     throw Error("do not support deleted tag");
   }
 };
 
-/**
- * Dropbox api doesn't return mtime for folders.
- * This is a try to assign mtime by using files in folder.
- * @param allFilesFolders
- * @returns
- */
-const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => {
-  if (allFilesFolders.length === 0) {
-    return;
-  }
-
-  // sort by longer to shorter
-  allFilesFolders.sort((a, b) => b.key.length - a.key.length);
-
-  // a "map" from dir to mtime
-  let potentialMTime = {} as Record<string, number>;
-
-  // first sort pass, from buttom to up
-  for (const item of allFilesFolders) {
-    if (item.key.endsWith("/")) {
-      // itself is a folder, and initially doesn't have mtime
-      if (item.lastModified === undefined && item.key in potentialMTime) {
-        // previously we gathered all sub info of this folder
-        item.lastModified = potentialMTime[item.key];
-      }
-    }
-    const parent = `${path.posix.dirname(item.key)}/`;
-    if (item.lastModified !== undefined) {
-      if (parent in potentialMTime) {
-        potentialMTime[parent] = Math.max(
-          potentialMTime[parent],
-          item.lastModified
-        );
-      } else {
-        potentialMTime[parent] = item.lastModified;
-      }
-    }
-  }
-
-  // second pass, from up to buttom.
-  // fill mtime by parent folder or Date.Now() if still not available.
-  // this is only possible if no any sub-folder-files recursively.
-  // we do not sort the array again, just iterate over it by reverse
-  // using good old for loop.
-  for (let i = allFilesFolders.length - 1; i >= 0; --i) {
-    const item = allFilesFolders[i];
-    if (!item.key.endsWith("/")) {
-      continue; // skip files
-    }
-    if (item.lastModified !== undefined) {
-      continue; // don't need to deal with it
-    }
-    const parent = `${path.posix.dirname(item.key)}/`;
-    if (parent in potentialMTime) {
-      item.lastModified = potentialMTime[parent];
-    } else {
-      item.lastModified = Date.now().valueOf();
-      potentialMTime[item.key] = item.lastModified;
-    }
-  }
-
-  return allFilesFolders;
-};
-
 ////////////////////////////////////////////////////////////////////////////////
 // Dropbox authorization using PKCE
 // see https://dropbox.tech/developers/pkce--what-and-why-
@@ -498,7 +435,7 @@ export const getRemoteMeta = async (
   //     size: 0,
   //     remoteType: "dropbox",
   //     etag: undefined,
-  //   } as RemoteItem;
+  //   } as Entity;
   // }
 
   const rsp = await retryReq(() =>
@@ -512,7 +449,7 @@ export const getRemoteMeta = async (
   if (rsp.status !== 200) {
     throw Error(JSON.stringify(rsp));
   }
-  return fromDropboxItemToRemoteItem(rsp.result, client.remoteBaseDir);
+  return fromDropboxItemToEntity(rsp.result, client.remoteBaseDir);
 };
 
 export const uploadToRemote = async (
@@ -670,7 +607,7 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => {
   const unifiedContents = contents
     .filter((x) => x[".tag"] !== "deleted")
     .filter((x) => x.path_display !== `/${client.remoteBaseDir}`)
-    .map((x) => fromDropboxItemToRemoteItem(x, client.remoteBaseDir));
+    .map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir));
 
   while (res.result.has_more) {
     res = await client.dropbox.filesListFolderContinue({
@@ -684,15 +621,11 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => {
     const unifiedContents2 = contents2
       .filter((x) => x[".tag"] !== "deleted")
       .filter((x) => x.path_display !== `/${client.remoteBaseDir}`)
-      .map((x) => fromDropboxItemToRemoteItem(x, client.remoteBaseDir));
+      .map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir));
     unifiedContents.push(...unifiedContents2);
   }
 
-  fixLastModifiedTimeInplace(unifiedContents);
+  return unifiedContents;
-
-  return {
-    Contents: unifiedContents,
-  };
 };
 
 const downloadFromRemoteRaw = async (

src/remoteForOnedrive.ts

@@ -14,7 +14,7 @@ import {
   DEFAULT_CONTENT_TYPE,
   OAUTH2_FORCE_EXPIRE_MILLISECONDS,
   OnedriveConfig,
-  RemoteItem,
+  Entity,
 } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
 import {
@@ -255,16 +255,13 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
   return fileOrFolderPath.slice(`${prefix}/`.length);
 };
 
-const constructFromDriveItemToRemoteItemError = (x: DriveItem) => {
+const constructFromDriveItemToEntityError = (x: DriveItem) => {
   return `parentPath="${
     x.parentReference?.path ?? "(no parentReference or path)"
   }", selfName="${x.name}"`;
 };
 
-const fromDriveItemToRemoteItem = (
+const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => {
-  x: DriveItem,
-  remoteBaseDir: string
-): RemoteItem => {
   let key = "";
 
   // possible prefix:
@@ -333,14 +330,14 @@ const fromDriveItemToRemoteItem = (
       key = x.name;
     } else {
       throw Error(
-        `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToRemoteItemError(
+        `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToEntityError(
           x
         )}`
       );
     }
   } else {
     throw Error(
-      `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToRemoteItemError(
+      `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToEntityError(
         x
       )}`
     );
@@ -350,11 +347,17 @@ const fromDriveItemToRemoteItem = (
   if (isFolder) {
     key = `${key}/`;
   }
+
+  const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
+  const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
   return {
     key: key,
-    lastModified: Date.parse(x!.fileSystemInfo!.lastModifiedDateTime!),
+    keyEnc: key,
+    mtimeSvr: mtimeSvr,
+    mtimeCli: mtimeCli,
     size: isFolder ? 0 : x.size!,
-    remoteType: "onedrive",
+    sizeEnc: isFolder ? 0 : x.size!,
+    // hash: ?? // TODO
     etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes
   };
 };
@@ -666,14 +669,12 @@ export const listAllFromRemote = async (client: WrappedOnedriveClient) => {
     await client.saveUpdatedConfigFunc();
   }
 
-  // unify everything to RemoteItem
+  // unify everything to Entity
   const unifiedContents = driveItems
-    .map((x) => fromDriveItemToRemoteItem(x, client.remoteBaseDir))
+    .map((x) => fromDriveItemToEntity(x, client.remoteBaseDir))
     .filter((x) => x.key !== "/");
 
-  return {
+  return unifiedContents;
-    Contents: unifiedContents,
-  };
 };
 
 export const getRemoteMeta = async (
@@ -687,7 +688,7 @@ export const getRemoteMeta = async (
   );
   // log.info(rsp);
   const driveItem = rsp as DriveItem;
-  const res = fromDriveItemToRemoteItem(driveItem, client.remoteBaseDir);
+  const res = fromDriveItemToEntity(driveItem, client.remoteBaseDir);
   // log.info(res);
   return res;
 };

src/remoteForS3.ts

@@ -28,7 +28,7 @@ import * as path from "path";
 import AggregateError from "aggregate-error";
 import {
   DEFAULT_CONTENT_TYPE,
-  RemoteItem,
+  Entity,
   S3Config,
   VALID_REQURL,
 } from "./baseTypes";
@@ -220,51 +220,56 @@ const getLocalNoPrefixPath = (
   return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length);
 };
 
-const fromS3ObjectToRemoteItem = (
+const fromS3ObjectToEntity = (
   x: S3ObjectType,
   remotePrefix: string,
   mtimeRecords: Record<string, number>,
   ctimeRecords: Record<string, number>
 ) => {
-  let mtime = x.LastModified!.valueOf();
+  const mtimeSvr = x.LastModified!.valueOf();
+  let mtimeCli = mtimeSvr;
   if (x.Key! in mtimeRecords) {
     const m2 = mtimeRecords[x.Key!];
     if (m2 !== 0) {
-      mtime = m2;
+      mtimeCli = m2;
     }
   }
-  const r: RemoteItem = {
+  const key = getLocalNoPrefixPath(x.Key!, remotePrefix);
-    key: getLocalNoPrefixPath(x.Key!, remotePrefix),
+  const r: Entity = {
-    lastModified: mtime,
+    key: key,
+    keyEnc: key,
+    mtimeSvr: mtimeSvr,
+    mtimeCli: mtimeCli,
     size: x.Size!,
-    remoteType: "s3",
+    sizeEnc: x.Size!,
     etag: x.ETag,
   };
   return r;
 };
 
-const fromS3HeadObjectToRemoteItem = (
+const fromS3HeadObjectToEntity = (
   fileOrFolderPathWithRemotePrefix: string,
   x: HeadObjectCommandOutput,
   remotePrefix: string,
   useAccurateMTime: boolean
 ) => {
-  let mtime = x.LastModified!.valueOf();
+  const mtimeSvr = x.LastModified!.valueOf();
+  let mtimeCli = mtimeSvr;
   if (useAccurateMTime && x.Metadata !== undefined) {
     const m2 = Math.round(
       parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0")
     );
     if (m2 !== 0) {
-      mtime = m2;
+      mtimeCli = m2;
     }
   }
   return {
     key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix),
-    lastModified: mtime,
+    mtimeSvr: mtimeSvr,
+    mtimeCli: mtimeCli,
     size: x.ContentLength,
-    remoteType: "s3",
     etag: x.ETag,
-  } as RemoteItem;
+  } as Entity;
 };
 
 export const getS3Client = (s3Config: S3Config) => {
@@ -330,7 +335,7 @@ export const getRemoteMeta = async (
     })
   );
 
-  return fromS3HeadObjectToRemoteItem(
+  return fromS3HeadObjectToEntity(
     fileOrFolderPathWithRemotePrefix,
     res,
     s3Config.remotePrefix ?? "",
@@ -538,16 +543,14 @@ const listFromRemoteRaw = async (
   // ensemble fake rsp
   // in the end, we need to transform the response list
   // back to the local contents-alike list
-  return {
+  return contents.map((x) =>
-    Contents: contents.map((x) =>
+    fromS3ObjectToEntity(
-      fromS3ObjectToRemoteItem(
+      x,
-        x,
+      s3Config.remotePrefix ?? "",
-        s3Config.remotePrefix ?? "",
+      mtimeRecords,
-        mtimeRecords,
+      ctimeRecords
-        ctimeRecords
+    )
-      )
+  );
-    ),
-  };
 };
 
 export const listAllFromRemote = async (
@@ -692,7 +695,7 @@ export const deleteFromRemote = async (
 
   if (fileOrFolderPath.endsWith("/") && password === "") {
     const x = await listFromRemoteRaw(s3Client, s3Config, remoteFileName);
-    x.Contents.forEach(async (element) => {
+    x.forEach(async (element) => {
       await s3Client.send(
         new DeleteObjectCommand({
           Bucket: s3Config.s3BucketName,

src/remoteForWebdav.ts

@@ -5,7 +5,7 @@ import { Queue } from "@fyears/tsqueue";
 import chunk from "lodash/chunk";
 import flatten from "lodash/flatten";
 import { getReasonPhrase } from "http-status-codes";
-import { RemoteItem, VALID_REQURL, WebdavConfig } from "./baseTypes";
+import { Entity, VALID_REQURL, WebdavConfig } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
 import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
 
@@ -205,18 +205,21 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
   return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length);
 };
 
-const fromWebdavItemToRemoteItem = (x: FileStat, remoteBaseDir: string) => {
+const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string) => {
   let key = getNormPath(x.filename, remoteBaseDir);
   if (x.type === "directory" && !key.endsWith("/")) {
     key = `${key}/`;
   }
+  const mtimeSvr = Date.parse(x.lastmod).valueOf();
   return {
     key: key,
-    lastModified: Date.parse(x.lastmod).valueOf(),
+    keyEnc: key,
+    mtimeSvr: mtimeSvr,
+    mtimeCli: mtimeSvr, // no universal way to set mtime in webdav
     size: x.size,
-    remoteType: "webdav",
+    sizeEnc: x.size,
-    etag: x.etag || undefined,
+    etag: x.etag,
-  } as RemoteItem;
+  } as Entity;
 };
 
 export class WrappedWebdavClient {
@@ -327,7 +330,7 @@ export const getRemoteMeta = async (
     details: false,
   })) as FileStat;
   log.debug(`getRemoteMeta res=${JSON.stringify(res)}`);
-  return fromWebdavItemToRemoteItem(res, client.remoteBaseDir);
+  return fromWebdavItemToEntity(res, client.remoteBaseDir);
 };
 
 export const uploadToRemote = async (
@@ -359,7 +362,7 @@ export const uploadToRemote = async (
     if (password === "") {
       // if not encrypted, mkdir a remote folder
       await client.client.createDirectory(uploadFile, {
-        recursive: false, // the sync algo should guarantee no need to recursive
+        recursive: true,
       });
       const res = await getRemoteMeta(client, uploadFile);
       return res;
@@ -400,7 +403,7 @@ export const uploadToRemote = async (
     // // we need to create folders before uploading
     // const dir = getPathFolder(uploadFile);
     // if (dir !== "/" && dir !== "") {
-    //   await client.client.createDirectory(dir, { recursive: false });
+    //   await client.client.createDirectory(dir, { recursive: true });
     // }
     await client.client.putFileContents(uploadFile, remoteContent, {
       overwrite: true,
@@ -472,11 +475,7 @@ export const listAllFromRemote = async (client: WrappedWebdavClient) => {
       }
     )) as FileStat[];
   }
-  return {
+  return contents.map((x) => fromWebdavItemToEntity(x, client.remoteBaseDir));
-    Contents: contents.map((x) =>
-      fromWebdavItemToRemoteItem(x, client.remoteBaseDir)
-    ),
-  };
 };
 
 const downloadFromRemoteRaw = async (