From 2fbd87eed4dfaae11bbf7f60ab7498f417bcaf65 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 08:21:05 +0800 Subject: [PATCH] half way of new sync --- docs/sync_algorithm_v3.md | 21 +- src/baseTypes.ts | 70 +- src/local.ts | 63 ++ src/localdb.ts | 516 +++-------- src/main.ts | 8 +- src/misc.ts | 24 +- src/obsFolderLister.ts | 35 +- src/remote.ts | 3 +- src/remoteForDropbox.ts | 105 +-- src/remoteForOnedrive.ts | 33 +- src/remoteForS3.ts | 55 +- src/remoteForWebdav.ts | 27 +- src/sync.ts | 1817 +++++++++++++------------------------ 13 files changed, 1042 insertions(+), 1735 deletions(-) create mode 100644 src/local.ts diff --git a/docs/sync_algorithm_v3.md b/docs/sync_algorithm_v3.md index 681ab62..cc50fbc 100644 --- a/docs/sync_algorithm_v3.md +++ b/docs/sync_algorithm_v3.md @@ -6,7 +6,7 @@ An absolutely better sync algorithm. Better for tracking deletions and better fo ## Huge Thanks -Basically a combination of algorithm v2 + [synclone](https://github.com/Jwink3101/syncrclone) + [rsinc](https://github.com/ConorWilliams/rsinc) + (some of rclone [bisync](https://rclone.org/bisync/)). All of the later three are released under MIT License so no worries about the licenses. +Basically a combination of algorithm v2 + [synclone](https://github.com/Jwink3101/syncrclone/blob/master/docs/algorithm.md) + [rsinc](https://github.com/ConorWilliams/rsinc) + (some of rclone [bisync](https://rclone.org/bisync/)). All of the later three are released under MIT License so no worries about the licenses. ## Features @@ -27,12 +27,25 @@ Nice to have ## Description -We have _five_ input sources: local all files, remote all files, _local previous succeeded sync history_, local deletions, remote deletions. +We have _five_ input sources: -Init run, consuming local deletions and remote deletions : +1. local all files +2. remote all files +3. _local previous succeeded sync history_ +4. local deletions +5. remote deletions. + +Init run, consuming remote deletions : TBD Later runs, use the first, second, third sources **only**. -TBD +Table modified based on synclone and rsinc. The number inside the table cell is the decision branch in the code. + +| local\remote | remote unchanged | remote modified | remote deleted | remote created | +| --------------- | ------------------ | ---------------- | ------------------ | ---------------- | +| local unchanged | (02) do nothing | (09) pull remote | (07) delete local | (??) conflict | +| local modified | (10) push local | (12) conflict | (08) push local | (??) conflict | +| local deleted | (04) delete remote | (05) pull | (01) clean history | (03) pull remote | +| local created | (??) conflict | (??) conflict | (06) push local | (11) conflict | diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 6401e20..1a1be04 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -116,14 +116,6 @@ export interface RemotelySavePluginSettings { logToDB?: boolean; } -export interface RemoteItem { - key: string; - lastModified?: number; - size: number; - remoteType: SUPPORTED_SERVICES_TYPE; - etag?: string; -} - export const COMMAND_URI = "remotely-save"; export const COMMAND_CALLBACK = "remotely-save-cb"; export const COMMAND_CALLBACK_ONEDRIVE = "remotely-save-cb-onedrive"; @@ -165,6 +157,68 @@ export type DecisionType = | DecisionTypeForFileSize | DecisionTypeForFolder; +export type EmptyFolderCleanType = "skip" | "clean_both"; + +export type ConflictActionType = "keep_newer" | "keep_larger" | "rename_both"; + +export type DecisionTypeForMixedEntity = + | "only_history" + | "equal" + | "modified_local" + | "modified_remote" + | "created_local" + | "created_remote" + | "deleted_local" + | "deleted_remote" + | "conflict_created_keep_local" + | "conflict_created_keep_remote" + | "conflict_created_keep_both" + | "conflict_modified_keep_local" + | "conflict_modified_keep_remote" + | "conflict_modified_keep_both" + | "folder_existed_both" + | "folder_existed_local" + | "folder_existed_remote" + | "folder_to_be_created" + | "folder_to_skip" + | "folder_to_be_deleted"; + +/** + * uniform representation + * everything should be flat and primitive, so that we can copy. + */ +export interface Entity { + key: string; + keyEnc: string; + mtimeCli?: number; + mtimeCliFmt?: string; + mtimeSvr?: number; + mtimeSvrFmt?: string; + prevSyncTime?: number; + prevSyncTimeFmt?: string; + size?: number; // might be unknown or to be filled + sizeEnc: number; + hash?: string; + etag?: string; +} + +/** + * A replacement of FileOrFolderMixedState + */ +export interface MixedEntity { + key: string; + local?: Entity; + prevSync?: Entity; + remote?: Entity; + + decisionBranch?: number; + decision?: DecisionTypeForMixedEntity; + conflictAction?: ConflictActionType; +} + +/** + * @deprecated + */ export interface FileOrFolderMixedState { key: string; existLocal?: boolean; diff --git a/src/local.ts b/src/local.ts new file mode 100644 index 0000000..cd70e38 --- /dev/null +++ b/src/local.ts @@ -0,0 +1,63 @@ +import { TFile, TFolder, type Vault } from "obsidian"; +import type { Entity, MixedEntity } from "./baseTypes"; +import { listFilesInObsFolder } from "./obsFolderLister"; + +export const getLocalEntityList = async ( + vault: Vault, + syncConfigDir: boolean, + configDir: string, + pluginID: string +) => { + const local: Entity[] = []; + + const localTAbstractFiles = vault.getAllLoadedFiles(); + for (const entry of localTAbstractFiles) { + let r = {} as Entity; + let key = entry.path; + + if (entry.path === "/") { + // ignore + continue; + } else if (entry instanceof TFile) { + let mtimeLocal: number | undefined = Math.max( + entry.stat.mtime ?? 0, + entry.stat.ctime + ); + if (mtimeLocal === 0) { + mtimeLocal = undefined; + } + if (mtimeLocal === undefined) { + throw Error( + `Your file has last modified time 0: ${key}, don't know how to deal with it` + ); + } + r = { + key: entry.path, + keyEnc: entry.path, + mtimeCli: mtimeLocal, + mtimeSvr: mtimeLocal, + size: entry.stat.size, + sizeEnc: entry.stat.size, + }; + } else if (entry instanceof TFolder) { + key = `${entry.path}/`; + r = { + key: key, + keyEnc: key, + size: 0, + sizeEnc: 0, + }; + } else { + throw Error(`unexpected ${entry}`); + } + } + + if (syncConfigDir) { + const syncFiles = await listFilesInObsFolder(configDir, vault, pluginID); + for (const f of syncFiles) { + local.push(f); + } + } + + return local; +}; diff --git a/src/localdb.ts b/src/localdb.ts index 4ed4ffa..ee45c16 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -3,35 +3,36 @@ export type LocalForage = typeof localforage; import { nanoid } from "nanoid"; import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian"; -import { API_VER_STAT_FOLDER, SUPPORTED_SERVICES_TYPE } from "./baseTypes"; +import { API_VER_STAT_FOLDER } from "./baseTypes"; +import type { Entity, MixedEntity, SUPPORTED_SERVICES_TYPE } from "./baseTypes"; import type { SyncPlanType } from "./sync"; import { statFix, toText, unixTimeToStr } from "./misc"; import { log } from "./moreOnLog"; -const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326]; -export const DEFAULT_DB_VERSION_NUMBER: number = 20220326; +const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220]; +export const DEFAULT_DB_VERSION_NUMBER: number = 20240220; export const DEFAULT_DB_NAME = "remotelysavedb"; export const DEFAULT_TBL_VERSION = "schemaversion"; -export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory"; -export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory"; export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory"; export const DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING = "vaultrandomidmapping"; export const DEFAULT_TBL_LOGGER_OUTPUT = "loggeroutput"; export const DEFAULT_TBL_SIMPLE_KV_FOR_MISC = "simplekvformisc"; +export const DEFAULT_TBL_PREV_SYNC_RECORDS = "prevsyncrecords"; -export interface FileFolderHistoryRecord { - key: string; - ctime: number; - mtime: number; - size: number; - actionWhen: number; - actionType: "delete" | "rename" | "renameDestination"; - keyType: "folder" | "file"; - renameTo: string; - vaultRandomID: string; -} +/** + * @deprecated + */ +export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory"; +/** + * @deprecated + */ +export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory"; +/** + * @deprecated + * But we cannot remove it. Because we want to migrate the old data. + */ interface SyncMetaMappingRecord { localKey: string; remoteKey: string; @@ -54,108 +55,74 @@ interface SyncPlanRecord { export interface InternalDBs { versionTbl: LocalForage; - fileHistoryTbl: LocalForage; - syncMappingTbl: LocalForage; syncPlansTbl: LocalForage; vaultRandomIDMappingTbl: LocalForage; loggerOutputTbl: LocalForage; simpleKVForMiscTbl: LocalForage; + prevSyncRecordsTbl: LocalForage; + + /** + * @deprecated + * But we cannot remove it. Because we want to migrate the old data. + */ + fileHistoryTbl: LocalForage; + + /** + * @deprecated + * But we cannot remove it. Because we want to migrate the old data. + */ + syncMappingTbl: LocalForage; } /** - * This migration mainly aims to assign vault name or vault id into all tables. - * @param db - * @param vaultRandomID + * TODO + * @param syncMappings + * @returns */ -const migrateDBsFrom20211114To20220108 = async ( - db: InternalDBs, - vaultRandomID: string -) => { - const oldVer = 20211114; - const newVer = 20220108; - log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`); - - const allPromisesToWait: Promise[] = []; - - log.debug("assign vault id to any delete history"); - const keysInDeleteHistoryTbl = await db.fileHistoryTbl.keys(); - for (const key of keysInDeleteHistoryTbl) { - if (key.startsWith(vaultRandomID)) { - continue; - } - const value = (await db.fileHistoryTbl.getItem( - key - )) as FileFolderHistoryRecord; - if (value === null || value === undefined) { - continue; - } - if (value.vaultRandomID === undefined || value.vaultRandomID === "") { - value.vaultRandomID = vaultRandomID; - } - const newKey = `${vaultRandomID}\t${key}`; - allPromisesToWait.push(db.fileHistoryTbl.setItem(newKey, value)); - allPromisesToWait.push(db.fileHistoryTbl.removeItem(key)); - } - - log.debug("assign vault id to any sync mapping"); - const keysInSyncMappingTbl = await db.syncMappingTbl.keys(); - for (const key of keysInSyncMappingTbl) { - if (key.startsWith(vaultRandomID)) { - continue; - } - const value = (await db.syncMappingTbl.getItem( - key - )) as SyncMetaMappingRecord; - if (value === null || value === undefined) { - continue; - } - if (value.vaultRandomID === undefined || value.vaultRandomID === "") { - value.vaultRandomID = vaultRandomID; - } - const newKey = `${vaultRandomID}\t${key}`; - allPromisesToWait.push(db.syncMappingTbl.setItem(newKey, value)); - allPromisesToWait.push(db.syncMappingTbl.removeItem(key)); - } - - log.debug("assign vault id to any sync plan records"); - const keysInSyncPlansTbl = await db.syncPlansTbl.keys(); - for (const key of keysInSyncPlansTbl) { - if (key.startsWith(vaultRandomID)) { - continue; - } - const value = (await db.syncPlansTbl.getItem(key)) as SyncPlanRecord; - if (value === null || value === undefined) { - continue; - } - if (value.vaultRandomID === undefined || value.vaultRandomID === "") { - value.vaultRandomID = vaultRandomID; - } - const newKey = `${vaultRandomID}\t${key}`; - allPromisesToWait.push(db.syncPlansTbl.setItem(newKey, value)); - allPromisesToWait.push(db.syncPlansTbl.removeItem(key)); - } - - log.debug("finally update version if everything is ok"); - await Promise.all(allPromisesToWait); - await db.versionTbl.setItem("version", newVer); - - log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); +const fromSyncMappingsToPrevSyncRecords = ( + syncMappings: SyncMetaMappingRecord[] +): Entity[] => { + return []; }; /** - * no need to do anything except changing version - * we just add more file operations in db, and no schema is changed. + * TODO * @param db * @param vaultRandomID + * @param prevSyncRecord */ -const migrateDBsFrom20220108To20220326 = async ( +const setPrevSyncRecordByVault = async ( + db: InternalDBs, + vaultRandomID: string, + prevSyncRecord: Entity +) => {}; + +/** + * + * @param db + * @param vaultRandomID + * Migrate the sync mapping record to sync Entity. + */ +const migrateDBsFrom20220326To20240220 = async ( db: InternalDBs, vaultRandomID: string ) => { - const oldVer = 20220108; - const newVer = 20220326; + const oldVer = 20220326; + const newVer = 20240220; log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`); - await db.versionTbl.setItem("version", newVer); + + // from sync mapping to prev sync + const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID); + const prevSyncRecords = fromSyncMappingsToPrevSyncRecords(syncMappings); + for (const prevSyncRecord of prevSyncRecords) { + await setPrevSyncRecordByVault(db, vaultRandomID, prevSyncRecord); + } + + // clear not used data + await clearFileHistoryOfEverythingByVault(db, vaultRandomID); + await clearAllSyncMetaMappingByVault(db, vaultRandomID); + + await db.versionTbl.setItem(`${vaultRandomID}\tversion`, newVer); log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); }; @@ -168,18 +135,19 @@ const migrateDBs = async ( if (oldVer === newVer) { return; } - if (oldVer === 20211114 && newVer === 20220108) { - return await migrateDBsFrom20211114To20220108(db, vaultRandomID); + + // as of 20240220, we assume everyone is using 20220326 already + // drop any old code to reduce the verbose + if (oldVer < 20220326) { + throw Error( + "You are using a very old version of Remotely Save. No way to auto update internal DB. Please install and enable 0.3.40 firstly, then install a later version." + ); } - if (oldVer === 20220108 && newVer === 20220326) { - return await migrateDBsFrom20220108To20220326(db, vaultRandomID); - } - if (oldVer === 20211114 && newVer === 20220326) { - // TODO: more steps with more versions in the future - await migrateDBsFrom20211114To20220108(db, vaultRandomID); - await migrateDBsFrom20220108To20220326(db, vaultRandomID); - return; + + if (oldVer === 20220326 && newVer === 20240220) { + return await migrateDBsFrom20220326To20240220(db, vaultRandomID); } + if (newVer < oldVer) { throw Error( "You've installed a new version, but then downgrade to an old version. Stop working!" @@ -198,14 +166,6 @@ export const prepareDBs = async ( name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_VERSION, }), - fileHistoryTbl: localforage.createInstance({ - name: DEFAULT_DB_NAME, - storeName: DEFAULT_TBL_FILE_HISTORY, - }), - syncMappingTbl: localforage.createInstance({ - name: DEFAULT_DB_NAME, - storeName: DEFAULT_TBL_SYNC_MAPPING, - }), syncPlansTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_SYNC_PLANS_HISTORY, @@ -222,6 +182,19 @@ export const prepareDBs = async ( name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_SIMPLE_KV_FOR_MISC, }), + prevSyncRecordsTbl: localforage.createInstance({ + name: DEFAULT_DB_NAME, + storeName: DEFAULT_TBL_PREV_SYNC_RECORDS, + }), + + fileHistoryTbl: localforage.createInstance({ + name: DEFAULT_DB_NAME, + storeName: DEFAULT_TBL_FILE_HISTORY, + }), + syncMappingTbl: localforage.createInstance({ + name: DEFAULT_DB_NAME, + storeName: DEFAULT_TBL_SYNC_MAPPING, + }), } as InternalDBs; // try to get vaultRandomID firstly @@ -253,12 +226,19 @@ export const prepareDBs = async ( throw Error("no vaultRandomID found or generated"); } - const originalVersion: number | null = await db.versionTbl.getItem("version"); + // as of 20240220, we set the version per vault, instead of global "version" + const originalVersion: number | null = + (await db.versionTbl.getItem(`${vaultRandomID}\tversion`)) ?? + (await db.versionTbl.getItem("version")); if (originalVersion === null) { log.debug( `no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}` ); - await db.versionTbl.setItem("version", DEFAULT_DB_VERSION_NUMBER); + // as of 20240220, we set the version per vault, instead of global "version" + await db.versionTbl.setItem( + `${vaultRandomID}\tversion`, + DEFAULT_DB_VERSION_NUMBER + ); } else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) { // do nothing } else { @@ -298,272 +278,47 @@ export const destroyDBs = async () => { }; }; -export const loadFileHistoryTableByVault = async ( +export const clearFileHistoryOfEverythingByVault = async ( db: InternalDBs, vaultRandomID: string ) => { - const records = [] as FileFolderHistoryRecord[]; - await db.fileHistoryTbl.iterate((value, key, iterationNumber) => { + const keys = await db.fileHistoryTbl.keys(); + for (const key of keys) { if (key.startsWith(`${vaultRandomID}\t`)) { - records.push(value as FileFolderHistoryRecord); + await db.fileHistoryTbl.removeItem(key); } - }); - records.sort((a, b) => a.actionWhen - b.actionWhen); // ascending - return records; -}; - -export const clearDeleteRenameHistoryOfKeyAndVault = async ( - db: InternalDBs, - key: string, - vaultRandomID: string -) => { - const fullKey = `${vaultRandomID}\t${key}`; - const item: FileFolderHistoryRecord | null = - await db.fileHistoryTbl.getItem(fullKey); - if ( - item !== null && - (item.actionType === "delete" || item.actionType === "rename") - ) { - await db.fileHistoryTbl.removeItem(fullKey); - } -}; - -export const insertDeleteRecordByVault = async ( - db: InternalDBs, - fileOrFolder: TAbstractFile | string, - vaultRandomID: string -) => { - // log.info(fileOrFolder); - let k: FileFolderHistoryRecord; - if (fileOrFolder instanceof TFile) { - k = { - key: fileOrFolder.path, - ctime: fileOrFolder.stat.ctime, - mtime: fileOrFolder.stat.mtime, - size: fileOrFolder.stat.size, - actionWhen: Date.now(), - actionType: "delete", - keyType: "file", - renameTo: "", - vaultRandomID: vaultRandomID, - }; - await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); - } else if (fileOrFolder instanceof TFolder) { - // key should endswith "/" - const key = fileOrFolder.path.endsWith("/") - ? fileOrFolder.path - : `${fileOrFolder.path}/`; - const ctime = 0; // they are deleted, so no way to get ctime, mtime - const mtime = 0; // they are deleted, so no way to get ctime, mtime - k = { - key: key, - ctime: ctime, - mtime: mtime, - size: 0, - actionWhen: Date.now(), - actionType: "delete", - keyType: "folder", - renameTo: "", - vaultRandomID: vaultRandomID, - }; - await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); - } else if (typeof fileOrFolder === "string") { - // always the deletions in .obsidian folder - // so annoying that the path doesn't exists - // and we have to guess whether the path is folder or file - k = { - key: fileOrFolder, - ctime: 0, - mtime: 0, - size: 0, - actionWhen: Date.now(), - actionType: "delete", - keyType: "file", - renameTo: "", - vaultRandomID: vaultRandomID, - }; - await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); - for (const ext of [ - "json", - "js", - "mjs", - "ts", - "md", - "txt", - "css", - "png", - "gif", - "jpg", - "jpeg", - "gitignore", - "gitkeep", - ]) { - if (fileOrFolder.endsWith(`.${ext}`)) { - // stop here, no more need to insert the folder record later - return; - } - } - // also add a deletion record as folder if not ending with special exts - k = { - key: `${fileOrFolder}/`, - ctime: 0, - mtime: 0, - size: 0, - actionWhen: Date.now(), - actionType: "delete", - keyType: "folder", - renameTo: "", - vaultRandomID: vaultRandomID, - }; - await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); } }; /** - * A file/folder is renamed from A to B - * We insert two records: - * A with actionType="rename" - * B with actionType="renameDestination" + * @deprecated But we cannot remove it. Because we want to migrate the old data. * @param db - * @param fileOrFolder - * @param oldPath * @param vaultRandomID + * @returns */ -export const insertRenameRecordByVault = async ( +export const getAllSyncMetaMappingByVault = async ( db: InternalDBs, - fileOrFolder: TAbstractFile, - oldPath: string, vaultRandomID: string ) => { - // log.info(fileOrFolder); - let k1: FileFolderHistoryRecord | undefined; - let k2: FileFolderHistoryRecord | undefined; - const actionWhen = Date.now(); - if (fileOrFolder instanceof TFile) { - k1 = { - key: oldPath, - ctime: fileOrFolder.stat.ctime, - mtime: fileOrFolder.stat.mtime, - size: fileOrFolder.stat.size, - actionWhen: actionWhen, - actionType: "rename", - keyType: "file", - renameTo: fileOrFolder.path, - vaultRandomID: vaultRandomID, - }; - k2 = { - key: fileOrFolder.path, - ctime: fileOrFolder.stat.ctime, - mtime: fileOrFolder.stat.mtime, - size: fileOrFolder.stat.size, - actionWhen: actionWhen, - actionType: "renameDestination", - keyType: "file", - renameTo: "", // itself is the destination, so no need to set this field - vaultRandomID: vaultRandomID, - }; - } else if (fileOrFolder instanceof TFolder) { - const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`; - const renameTo = fileOrFolder.path.endsWith("/") - ? fileOrFolder.path - : `${fileOrFolder.path}/`; - let ctime = 0; - let mtime = 0; - if (requireApiVersion(API_VER_STAT_FOLDER)) { - // TAbstractFile does not contain these info - // but from API_VER_STAT_FOLDER we can manually stat them by path. - const s = await statFix(fileOrFolder.vault, fileOrFolder.path); - if (s !== undefined && s !== null) { - ctime = s.ctime; - mtime = s.mtime; - } - } - k1 = { - key: key, - ctime: ctime, - mtime: mtime, - size: 0, - actionWhen: actionWhen, - actionType: "rename", - keyType: "folder", - renameTo: renameTo, - vaultRandomID: vaultRandomID, - }; - k2 = { - key: renameTo, - ctime: ctime, - mtime: mtime, - size: 0, - actionWhen: actionWhen, - actionType: "renameDestination", - keyType: "folder", - renameTo: "", // itself is the destination, so no need to set this field - vaultRandomID: vaultRandomID, - }; - } - await Promise.all([ - db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k1!.key}`, k1), - db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k2!.key}`, k2), - ]); -}; - -export const upsertSyncMetaMappingDataByVault = async ( - serviceType: SUPPORTED_SERVICES_TYPE, - db: InternalDBs, - localKey: string, - localMTime: number, - localSize: number, - remoteKey: string, - remoteMTime: number, - remoteSize: number, - remoteExtraKey: string, - vaultRandomID: string -) => { - const aggregratedInfo: SyncMetaMappingRecord = { - localKey: localKey, - localMtime: localMTime, - localSize: localSize, - remoteKey: remoteKey, - remoteMtime: remoteMTime, - remoteSize: remoteSize, - remoteExtraKey: remoteExtraKey, - remoteType: serviceType, - keyType: localKey.endsWith("/") ? "folder" : "file", - vaultRandomID: vaultRandomID, - }; - await db.syncMappingTbl.setItem( - `${vaultRandomID}\t${remoteKey}`, - aggregratedInfo + return await Promise.all( + ((await db.syncMappingTbl.keys()) ?? []) + .filter((key) => key.startsWith(`${vaultRandomID}\t`)) + .map( + async (key) => + (await db.syncMappingTbl.getItem(key)) as SyncMetaMappingRecord + ) ); }; -export const getSyncMetaMappingByRemoteKeyAndVault = async ( - serviceType: SUPPORTED_SERVICES_TYPE, +export const clearAllSyncMetaMappingByVault = async ( db: InternalDBs, - remoteKey: string, - remoteMTime: number, - remoteExtraKey: string, vaultRandomID: string ) => { - const potentialItem = (await db.syncMappingTbl.getItem( - `${vaultRandomID}\t${remoteKey}` - )) as SyncMetaMappingRecord; - - if (potentialItem === null) { - // no result was found - return undefined; - } - - if ( - potentialItem.remoteKey === remoteKey && - potentialItem.remoteMtime === remoteMTime && - potentialItem.remoteExtraKey === remoteExtraKey && - potentialItem.remoteType === serviceType - ) { - // the result was found - return potentialItem; - } else { - return undefined; + const keys = await db.syncMappingTbl.keys(); + for (const key of keys) { + if (key.startsWith(`${vaultRandomID}\t`)) { + await db.syncMappingTbl.removeItem(key); + } } }; @@ -651,6 +406,37 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => { await Promise.all(ps); }; +export const upsertPrevSyncRecordByVault = async ( + db: InternalDBs, + vaultRandomID: string, + prevSync: Entity +) => { + await db.prevSyncRecordsTbl.setItem( + `${vaultRandomID}-${prevSync.key}`, + prevSync + ); +}; + +export const clearPrevSyncRecordByVault = async ( + db: InternalDBs, + vaultRandomID: string, + key: string +) => { + await db.prevSyncRecordsTbl.removeItem(`${vaultRandomID}-${key}`); +}; + +export const clearAllPrevSyncRecordByVault = async ( + db: InternalDBs, + vaultRandomID: string +) => { + const keys = await db.prevSyncRecordsTbl.keys(); + for (const key of keys) { + if (key.startsWith(`${vaultRandomID}\t`)) { + await db.prevSyncRecordsTbl.removeItem(key); + } + } +}; + export const clearAllLoggerOutputRecords = async (db: InternalDBs) => { await db.loggerOutputTbl.clear(); log.debug(`successfully clearAllLoggerOutputRecords`); diff --git a/src/main.ts b/src/main.ts index ea7a96a..8cda974 100644 --- a/src/main.ts +++ b/src/main.ts @@ -240,8 +240,8 @@ export default class RemotelySavePlugin extends Plugin { this.app.vault.getName(), () => self.saveSettings() ); - const remoteRsp = await client.listAllFromRemote(); - // log.debug(remoteRsp); + const remoteEntityList = await client.listAllFromRemote(); + // log.debug(remoteEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -250,7 +250,7 @@ export default class RemotelySavePlugin extends Plugin { } this.syncStatus = "checking_password"; const passwordCheckResult = await isPasswordOk( - remoteRsp.Contents, + remoteEntityList, this.settings.password ); if (!passwordCheckResult.ok) { @@ -265,7 +265,7 @@ export default class RemotelySavePlugin extends Plugin { } this.syncStatus = "getting_remote_extra_meta"; const { remoteStates, metadataFile } = await parseRemoteItems( - remoteRsp.Contents, + remoteEntityList, this.db, this.vaultRandomID, client.serviceType, diff --git a/src/misc.ts b/src/misc.ts index 5b33ec4..69b0122 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -76,16 +76,17 @@ export const getFolderLevels = (x: string, addEndingSlash: boolean = false) => { export const mkdirpInVault = async (thePath: string, vault: Vault) => { // log.info(thePath); - const foldersToBuild = getFolderLevels(thePath); - // log.info(foldersToBuild); - for (const folder of foldersToBuild) { - const r = await vault.adapter.exists(folder); - // log.info(r); - if (!r) { - log.info(`mkdir ${folder}`); - await vault.adapter.mkdir(folder); - } + + // as of 2020219, + // Obsidian can create the folder recursively + // but the path should not end with '/' + if (thePath === "/" || thePath === "") { + return; } + let thePathNoEnding = thePath.endsWith("/") + ? thePath.slice(0, thePath.length - 1) + : thePath; + await vault.adapter.mkdir(thePathNoEnding); }; /** @@ -435,7 +436,10 @@ export const statFix = async (vault: Vault, path: string) => { return s; }; -export const isFolderToSkip = (x: string, more: string[] | undefined) => { +export const isSpecialFolderNameToSkip = ( + x: string, + more: string[] | undefined +) => { let specialFolders = [ ".git", ".github", diff --git a/src/obsFolderLister.ts b/src/obsFolderLister.ts index d1f9bdf..af88774 100644 --- a/src/obsFolderLister.ts +++ b/src/obsFolderLister.ts @@ -1,17 +1,11 @@ -import { Vault, Stat, ListedFiles } from "obsidian"; +import type { Vault, Stat, ListedFiles } from "obsidian"; +import type { Entity, MixedEntity } from "./baseTypes"; + import { Queue } from "@fyears/tsqueue"; import chunk from "lodash/chunk"; import flatten from "lodash/flatten"; import { statFix, isFolderToSkip } from "./misc"; -export interface ObsConfigDirFileType { - key: string; - ctime: number; - mtime: number; - size: number; - type: "folder" | "file"; -} - const isPluginDirItself = (x: string, pluginId: string) => { return ( x === pluginId || @@ -48,10 +42,10 @@ export const listFilesInObsFolder = async ( configDir: string, vault: Vault, pluginId: string -) => { +): Promise => { const q = new Queue([configDir]); const CHUNK_SIZE = 10; - const contents: ObsConfigDirFileType[] = []; + const contents: Entity[] = []; while (q.length > 0) { const itemsToFetch: string[] = []; while (q.length > 0) { @@ -72,11 +66,26 @@ export const listFilesInObsFolder = async ( children = await vault.adapter.list(x); } + if ( + !isFolder && + (statRes.mtime === undefined || + statRes.mtime === null || + statRes.mtime === 0) + ) { + throw Error( + `File in Obsidian ${configDir} has last modified time 0: ${x}, don't know how to deal with it.` + ); + } + return { itself: { key: isFolder ? `${x}/` : x, - ...statRes, - } as ObsConfigDirFileType, + keyEnc: isFolder ? `${x}/` : x, + mtimeCli: statRes.mtime, + mtimeSvr: statRes.mtime, + size: statRes.size, + sizeEnc: statRes.size, + }, children: children, }; }); diff --git a/src/remote.ts b/src/remote.ts index 6a26fe1..68e5570 100644 --- a/src/remote.ts +++ b/src/remote.ts @@ -1,5 +1,6 @@ import { Vault } from "obsidian"; import type { + Entity, DropboxConfig, OnedriveConfig, S3Config, @@ -164,7 +165,7 @@ export class RemoteClient { } }; - listAllFromRemote = async () => { + listAllFromRemote = async (): Promise => { if (this.serviceType === "s3") { return await s3.listAllFromRemote( s3.getS3Client(this.s3Config!), diff --git a/src/remoteForDropbox.ts b/src/remoteForDropbox.ts index c4ee477..18d4cde 100644 --- a/src/remoteForDropbox.ts +++ b/src/remoteForDropbox.ts @@ -5,7 +5,7 @@ import { Vault } from "obsidian"; import * as path from "path"; import { DropboxConfig, - RemoteItem, + Entity, COMMAND_CALLBACK_DROPBOX, OAUTH2_FORCE_EXPIRE_MILLISECONDS, } from "./baseTypes"; @@ -69,13 +69,13 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => { return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length); }; -const fromDropboxItemToRemoteItem = ( +const fromDropboxItemToEntity = ( x: | files.FileMetadataReference | files.FolderMetadataReference | files.DeletedMetadataReference, remoteBaseDir: string -): RemoteItem => { +): Entity => { let key = getNormPath(x.path_display!, remoteBaseDir); if (x[".tag"] === "folder" && !key.endsWith("/")) { key = `${key}/`; @@ -84,93 +84,30 @@ const fromDropboxItemToRemoteItem = ( if (x[".tag"] === "folder") { return { key: key, - lastModified: undefined, + keyEnc: key, size: 0, - remoteType: "dropbox", + sizeEnc: 0, etag: `${x.id}\t`, - } as RemoteItem; + } as Entity; } else if (x[".tag"] === "file") { - let mtime = Date.parse(x.client_modified).valueOf(); - if (mtime === 0) { - mtime = Date.parse(x.server_modified).valueOf(); - } + const mtimeCli = Date.parse(x.client_modified).valueOf(); + const mtimeSvr = Date.parse(x.server_modified).valueOf(); return { key: key, - lastModified: mtime, + keyEnc: key, + mtimeCli: mtimeCli, + mtimeSvr: mtimeSvr, size: x.size, - remoteType: "dropbox", + sizeEnc: x.size, + hash: x.content_hash, etag: `${x.id}\t${x.content_hash}`, - } as RemoteItem; + } as Entity; } else { // x[".tag"] === "deleted" throw Error("do not support deleted tag"); } }; -/** - * Dropbox api doesn't return mtime for folders. - * This is a try to assign mtime by using files in folder. - * @param allFilesFolders - * @returns - */ -const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => { - if (allFilesFolders.length === 0) { - return; - } - - // sort by longer to shorter - allFilesFolders.sort((a, b) => b.key.length - a.key.length); - - // a "map" from dir to mtime - let potentialMTime = {} as Record; - - // first sort pass, from buttom to up - for (const item of allFilesFolders) { - if (item.key.endsWith("/")) { - // itself is a folder, and initially doesn't have mtime - if (item.lastModified === undefined && item.key in potentialMTime) { - // previously we gathered all sub info of this folder - item.lastModified = potentialMTime[item.key]; - } - } - const parent = `${path.posix.dirname(item.key)}/`; - if (item.lastModified !== undefined) { - if (parent in potentialMTime) { - potentialMTime[parent] = Math.max( - potentialMTime[parent], - item.lastModified - ); - } else { - potentialMTime[parent] = item.lastModified; - } - } - } - - // second pass, from up to buttom. - // fill mtime by parent folder or Date.Now() if still not available. - // this is only possible if no any sub-folder-files recursively. - // we do not sort the array again, just iterate over it by reverse - // using good old for loop. - for (let i = allFilesFolders.length - 1; i >= 0; --i) { - const item = allFilesFolders[i]; - if (!item.key.endsWith("/")) { - continue; // skip files - } - if (item.lastModified !== undefined) { - continue; // don't need to deal with it - } - const parent = `${path.posix.dirname(item.key)}/`; - if (parent in potentialMTime) { - item.lastModified = potentialMTime[parent]; - } else { - item.lastModified = Date.now().valueOf(); - potentialMTime[item.key] = item.lastModified; - } - } - - return allFilesFolders; -}; - //////////////////////////////////////////////////////////////////////////////// // Dropbox authorization using PKCE // see https://dropbox.tech/developers/pkce--what-and-why- @@ -498,7 +435,7 @@ export const getRemoteMeta = async ( // size: 0, // remoteType: "dropbox", // etag: undefined, - // } as RemoteItem; + // } as Entity; // } const rsp = await retryReq(() => @@ -512,7 +449,7 @@ export const getRemoteMeta = async ( if (rsp.status !== 200) { throw Error(JSON.stringify(rsp)); } - return fromDropboxItemToRemoteItem(rsp.result, client.remoteBaseDir); + return fromDropboxItemToEntity(rsp.result, client.remoteBaseDir); }; export const uploadToRemote = async ( @@ -670,7 +607,7 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => { const unifiedContents = contents .filter((x) => x[".tag"] !== "deleted") .filter((x) => x.path_display !== `/${client.remoteBaseDir}`) - .map((x) => fromDropboxItemToRemoteItem(x, client.remoteBaseDir)); + .map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir)); while (res.result.has_more) { res = await client.dropbox.filesListFolderContinue({ @@ -684,15 +621,11 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => { const unifiedContents2 = contents2 .filter((x) => x[".tag"] !== "deleted") .filter((x) => x.path_display !== `/${client.remoteBaseDir}`) - .map((x) => fromDropboxItemToRemoteItem(x, client.remoteBaseDir)); + .map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir)); unifiedContents.push(...unifiedContents2); } - fixLastModifiedTimeInplace(unifiedContents); - - return { - Contents: unifiedContents, - }; + return unifiedContents; }; const downloadFromRemoteRaw = async ( diff --git a/src/remoteForOnedrive.ts b/src/remoteForOnedrive.ts index 9d355bd..ec2759d 100644 --- a/src/remoteForOnedrive.ts +++ b/src/remoteForOnedrive.ts @@ -14,7 +14,7 @@ import { DEFAULT_CONTENT_TYPE, OAUTH2_FORCE_EXPIRE_MILLISECONDS, OnedriveConfig, - RemoteItem, + Entity, } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { @@ -255,16 +255,13 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => { return fileOrFolderPath.slice(`${prefix}/`.length); }; -const constructFromDriveItemToRemoteItemError = (x: DriveItem) => { +const constructFromDriveItemToEntityError = (x: DriveItem) => { return `parentPath="${ x.parentReference?.path ?? "(no parentReference or path)" }", selfName="${x.name}"`; }; -const fromDriveItemToRemoteItem = ( - x: DriveItem, - remoteBaseDir: string -): RemoteItem => { +const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => { let key = ""; // possible prefix: @@ -333,14 +330,14 @@ const fromDriveItemToRemoteItem = ( key = x.name; } else { throw Error( - `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToRemoteItemError( + `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToEntityError( x )}` ); } } else { throw Error( - `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToRemoteItemError( + `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToEntityError( x )}` ); @@ -350,11 +347,17 @@ const fromDriveItemToRemoteItem = ( if (isFolder) { key = `${key}/`; } + + const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!); + const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!); return { key: key, - lastModified: Date.parse(x!.fileSystemInfo!.lastModifiedDateTime!), + keyEnc: key, + mtimeSvr: mtimeSvr, + mtimeCli: mtimeCli, size: isFolder ? 0 : x.size!, - remoteType: "onedrive", + sizeEnc: isFolder ? 0 : x.size!, + // hash: ?? // TODO etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes }; }; @@ -666,14 +669,12 @@ export const listAllFromRemote = async (client: WrappedOnedriveClient) => { await client.saveUpdatedConfigFunc(); } - // unify everything to RemoteItem + // unify everything to Entity const unifiedContents = driveItems - .map((x) => fromDriveItemToRemoteItem(x, client.remoteBaseDir)) + .map((x) => fromDriveItemToEntity(x, client.remoteBaseDir)) .filter((x) => x.key !== "/"); - return { - Contents: unifiedContents, - }; + return unifiedContents; }; export const getRemoteMeta = async ( @@ -687,7 +688,7 @@ export const getRemoteMeta = async ( ); // log.info(rsp); const driveItem = rsp as DriveItem; - const res = fromDriveItemToRemoteItem(driveItem, client.remoteBaseDir); + const res = fromDriveItemToEntity(driveItem, client.remoteBaseDir); // log.info(res); return res; }; diff --git a/src/remoteForS3.ts b/src/remoteForS3.ts index 086d22d..00924da 100644 --- a/src/remoteForS3.ts +++ b/src/remoteForS3.ts @@ -28,7 +28,7 @@ import * as path from "path"; import AggregateError from "aggregate-error"; import { DEFAULT_CONTENT_TYPE, - RemoteItem, + Entity, S3Config, VALID_REQURL, } from "./baseTypes"; @@ -220,51 +220,56 @@ const getLocalNoPrefixPath = ( return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length); }; -const fromS3ObjectToRemoteItem = ( +const fromS3ObjectToEntity = ( x: S3ObjectType, remotePrefix: string, mtimeRecords: Record, ctimeRecords: Record ) => { - let mtime = x.LastModified!.valueOf(); + const mtimeSvr = x.LastModified!.valueOf(); + let mtimeCli = mtimeSvr; if (x.Key! in mtimeRecords) { const m2 = mtimeRecords[x.Key!]; if (m2 !== 0) { - mtime = m2; + mtimeCli = m2; } } - const r: RemoteItem = { - key: getLocalNoPrefixPath(x.Key!, remotePrefix), - lastModified: mtime, + const key = getLocalNoPrefixPath(x.Key!, remotePrefix); + const r: Entity = { + key: key, + keyEnc: key, + mtimeSvr: mtimeSvr, + mtimeCli: mtimeCli, size: x.Size!, - remoteType: "s3", + sizeEnc: x.Size!, etag: x.ETag, }; return r; }; -const fromS3HeadObjectToRemoteItem = ( +const fromS3HeadObjectToEntity = ( fileOrFolderPathWithRemotePrefix: string, x: HeadObjectCommandOutput, remotePrefix: string, useAccurateMTime: boolean ) => { - let mtime = x.LastModified!.valueOf(); + const mtimeSvr = x.LastModified!.valueOf(); + let mtimeCli = mtimeSvr; if (useAccurateMTime && x.Metadata !== undefined) { const m2 = Math.round( parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0") ); if (m2 !== 0) { - mtime = m2; + mtimeCli = m2; } } return { key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix), - lastModified: mtime, + mtimeSvr: mtimeSvr, + mtimeCli: mtimeCli, size: x.ContentLength, - remoteType: "s3", etag: x.ETag, - } as RemoteItem; + } as Entity; }; export const getS3Client = (s3Config: S3Config) => { @@ -330,7 +335,7 @@ export const getRemoteMeta = async ( }) ); - return fromS3HeadObjectToRemoteItem( + return fromS3HeadObjectToEntity( fileOrFolderPathWithRemotePrefix, res, s3Config.remotePrefix ?? "", @@ -538,16 +543,14 @@ const listFromRemoteRaw = async ( // ensemble fake rsp // in the end, we need to transform the response list // back to the local contents-alike list - return { - Contents: contents.map((x) => - fromS3ObjectToRemoteItem( - x, - s3Config.remotePrefix ?? "", - mtimeRecords, - ctimeRecords - ) - ), - }; + return contents.map((x) => + fromS3ObjectToEntity( + x, + s3Config.remotePrefix ?? "", + mtimeRecords, + ctimeRecords + ) + ); }; export const listAllFromRemote = async ( @@ -692,7 +695,7 @@ export const deleteFromRemote = async ( if (fileOrFolderPath.endsWith("/") && password === "") { const x = await listFromRemoteRaw(s3Client, s3Config, remoteFileName); - x.Contents.forEach(async (element) => { + x.forEach(async (element) => { await s3Client.send( new DeleteObjectCommand({ Bucket: s3Config.s3BucketName, diff --git a/src/remoteForWebdav.ts b/src/remoteForWebdav.ts index 27318cd..fe44443 100644 --- a/src/remoteForWebdav.ts +++ b/src/remoteForWebdav.ts @@ -5,7 +5,7 @@ import { Queue } from "@fyears/tsqueue"; import chunk from "lodash/chunk"; import flatten from "lodash/flatten"; import { getReasonPhrase } from "http-status-codes"; -import { RemoteItem, VALID_REQURL, WebdavConfig } from "./baseTypes"; +import { Entity, VALID_REQURL, WebdavConfig } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc"; @@ -205,18 +205,21 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => { return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length); }; -const fromWebdavItemToRemoteItem = (x: FileStat, remoteBaseDir: string) => { +const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string) => { let key = getNormPath(x.filename, remoteBaseDir); if (x.type === "directory" && !key.endsWith("/")) { key = `${key}/`; } + const mtimeSvr = Date.parse(x.lastmod).valueOf(); return { key: key, - lastModified: Date.parse(x.lastmod).valueOf(), + keyEnc: key, + mtimeSvr: mtimeSvr, + mtimeCli: mtimeSvr, // no universal way to set mtime in webdav size: x.size, - remoteType: "webdav", - etag: x.etag || undefined, - } as RemoteItem; + sizeEnc: x.size, + etag: x.etag, + } as Entity; }; export class WrappedWebdavClient { @@ -327,7 +330,7 @@ export const getRemoteMeta = async ( details: false, })) as FileStat; log.debug(`getRemoteMeta res=${JSON.stringify(res)}`); - return fromWebdavItemToRemoteItem(res, client.remoteBaseDir); + return fromWebdavItemToEntity(res, client.remoteBaseDir); }; export const uploadToRemote = async ( @@ -359,7 +362,7 @@ export const uploadToRemote = async ( if (password === "") { // if not encrypted, mkdir a remote folder await client.client.createDirectory(uploadFile, { - recursive: false, // the sync algo should guarantee no need to recursive + recursive: true, }); const res = await getRemoteMeta(client, uploadFile); return res; @@ -400,7 +403,7 @@ export const uploadToRemote = async ( // // we need to create folders before uploading // const dir = getPathFolder(uploadFile); // if (dir !== "/" && dir !== "") { - // await client.client.createDirectory(dir, { recursive: false }); + // await client.client.createDirectory(dir, { recursive: true }); // } await client.client.putFileContents(uploadFile, remoteContent, { overwrite: true, @@ -472,11 +475,7 @@ export const listAllFromRemote = async (client: WrappedWebdavClient) => { } )) as FileStat[]; } - return { - Contents: contents.map((x) => - fromWebdavItemToRemoteItem(x, client.remoteBaseDir) - ), - }; + return contents.map((x) => fromWebdavItemToEntity(x, client.remoteBaseDir)); }; const downloadFromRemoteRaw = async ( diff --git a/src/sync.ts b/src/sync.ts index 1e2f8a1..4f3125a 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -1,79 +1,43 @@ -import { - TAbstractFile, - TFile, - TFolder, - Vault, - requireApiVersion, -} from "obsidian"; -import AggregateError from "aggregate-error"; import PQueue from "p-queue"; import XRegExp from "xregexp"; import type { - RemoteItem, - SyncTriggerSourceType, - DecisionType, - FileOrFolderMixedState, - SUPPORTED_SERVICES_TYPE, + ConflictActionType, + EmptyFolderCleanType, + Entity, + MixedEntity, } from "./baseTypes"; -import { API_VER_STAT_FOLDER } from "./baseTypes"; +import { isInsideObsFolder } from "./obsFolderLister"; import { + isSpecialFolderNameToSkip, + isHiddenPath, + unixTimeToStr, + getParentFolder, + isVaildText, + atWhichLevel, + mkdirpInVault, +} from "./misc"; +import { + DEFAULT_FILE_NAME_FOR_METADATAONREMOTE, + DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2, +} from "./metadataOnRemote"; +import { + MAGIC_ENCRYPTED_PREFIX_BASE32, + MAGIC_ENCRYPTED_PREFIX_BASE64URL, decryptBase32ToString, decryptBase64urlToString, encryptStringToBase64url, getSizeFromOrigToEnc, - MAGIC_ENCRYPTED_PREFIX_BASE32, - MAGIC_ENCRYPTED_PREFIX_BASE64URL, } from "./encrypt"; -import type { FileFolderHistoryRecord, InternalDBs } from "./localdb"; -import { - clearDeleteRenameHistoryOfKeyAndVault, - getSyncMetaMappingByRemoteKeyAndVault, - upsertSyncMetaMappingDataByVault, -} from "./localdb"; -import { - isHiddenPath, - isVaildText, - mkdirpInVault, - getFolderLevels, - getParentFolder, - atWhichLevel, - unixTimeToStr, - statFix, - isFolderToSkip, -} from "./misc"; import { RemoteClient } from "./remote"; -import { - MetadataOnRemote, - DeletionOnRemote, - serializeMetadataOnRemote, - deserializeMetadataOnRemote, - DEFAULT_FILE_NAME_FOR_METADATAONREMOTE, - DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2, - isEqualMetadataOnRemote, -} from "./metadataOnRemote"; -import { isInsideObsFolder, ObsConfigDirFileType } from "./obsFolderLister"; +import { Vault } from "obsidian"; import { log } from "./moreOnLog"; - -export type SyncStatusType = - | "idle" - | "preparing" - | "getting_remote_files_list" - | "getting_remote_extra_meta" - | "getting_local_meta" - | "checking_password" - | "generating_plan" - | "syncing" - | "cleaning" - | "finish"; - -export interface SyncPlanType { - ts: number; - tsFmt?: string; - syncTriggerSource?: SyncTriggerSourceType; - remoteType: SUPPORTED_SERVICES_TYPE; - mixedStates: Record; -} +import AggregateError from "aggregate-error"; +import { + InternalDBs, + clearPrevSyncRecordByVault, + upsertPrevSyncRecordByVault, +} from "./localdb"; export interface PasswordCheckType { ok: boolean; @@ -89,15 +53,15 @@ export interface PasswordCheckType { } export const isPasswordOk = async ( - remote: RemoteItem[], + remote: Entity[], password: string = "" -) => { +): Promise => { if (remote === undefined || remote.length === 0) { // remote empty return { ok: true, reason: "empty_remote", - } as PasswordCheckType; + }; } const santyCheckKey = remote[0].key; if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { @@ -107,7 +71,7 @@ export const isPasswordOk = async ( return { ok: false, reason: "remote_encrypted_local_no_password", - } as PasswordCheckType; + }; } try { const res = await decryptBase32ToString(santyCheckKey, password); @@ -118,18 +82,18 @@ export const isPasswordOk = async ( return { ok: true, reason: "password_matched", - } as PasswordCheckType; + }; } else { return { ok: false, reason: "invalid_text_after_decryption", - } as PasswordCheckType; + }; } } catch (error) { return { ok: false, reason: "password_not_matched", - } as PasswordCheckType; + }; } } if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) { @@ -139,7 +103,7 @@ export const isPasswordOk = async ( return { ok: false, reason: "remote_encrypted_local_no_password", - } as PasswordCheckType; + }; } try { const res = await decryptBase64urlToString(santyCheckKey, password); @@ -150,18 +114,18 @@ export const isPasswordOk = async ( return { ok: true, reason: "password_matched", - } as PasswordCheckType; + }; } else { return { ok: false, reason: "invalid_text_after_decryption", - } as PasswordCheckType; + }; } } catch (error) { return { ok: false, reason: "password_not_matched", - } as PasswordCheckType; + }; } } else { // it is not encrypted! @@ -169,130 +133,16 @@ export const isPasswordOk = async ( return { ok: false, reason: "remote_not_encrypted_local_has_password", - } as PasswordCheckType; + }; } return { ok: true, reason: "no_password_both_sides", - } as PasswordCheckType; - } -}; - -export const parseRemoteItems = async ( - remote: RemoteItem[], - db: InternalDBs, - vaultRandomID: string, - remoteType: SUPPORTED_SERVICES_TYPE, - password: string = "" -) => { - const remoteStates = [] as FileOrFolderMixedState[]; - let metadataFile: FileOrFolderMixedState | undefined = undefined; - if (remote === undefined) { - return { - remoteStates: remoteStates, - metadataFile: metadataFile, }; } - - for (const entry of remote) { - const remoteEncryptedKey = entry.key; - let key = remoteEncryptedKey; - if (password !== "") { - if (remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { - key = await decryptBase32ToString(remoteEncryptedKey, password); - } else if ( - remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL) - ) { - key = await decryptBase64urlToString(remoteEncryptedKey, password); - } else { - throw Error(`unexpected key=${remoteEncryptedKey}`); - } - } - const backwardMapping = await getSyncMetaMappingByRemoteKeyAndVault( - remoteType, - db, - key, - entry.lastModified ?? Date.now(), - entry.etag ?? "", - vaultRandomID - ); - - let r = {} as FileOrFolderMixedState; - if (backwardMapping !== undefined) { - // log.debug(`backwardMapping=${backwardMapping}`); - key = backwardMapping.localKey; - const mtimeRemote = backwardMapping.localMtime || entry.lastModified; - - // the backwardMapping.localSize is the file BEFORE encryption - // we want to split two sizes for comparation later - - r = { - key: key, - existRemote: true, - mtimeRemote: mtimeRemote, - mtimeRemoteFmt: unixTimeToStr(mtimeRemote), - sizeRemote: backwardMapping.localSize, - sizeRemoteEnc: password === "" ? undefined : entry.size, - remoteEncryptedKey: remoteEncryptedKey, - changeRemoteMtimeUsingMapping: true, - }; - } else { - // log.debug(`do not have backwardMapping`); - r = { - key: key, - existRemote: true, - mtimeRemote: entry.lastModified, - mtimeRemoteFmt: unixTimeToStr(entry.lastModified), - sizeRemote: password === "" ? entry.size : undefined, - sizeRemoteEnc: password === "" ? undefined : entry.size, - remoteEncryptedKey: remoteEncryptedKey, - changeRemoteMtimeUsingMapping: false, - }; - } - - if (r.key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE) { - metadataFile = Object.assign({}, r); - } - if (r.key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2) { - throw Error( - `A reserved file name ${r.key} has been found. You may upgrade the plugin to latest version to try to deal with it.` - ); - } - - remoteStates.push(r); - } - return { - remoteStates: remoteStates, - metadataFile: metadataFile, - }; }; -export const fetchMetadataFile = async ( - metadataFile: FileOrFolderMixedState | undefined, - client: RemoteClient, - vault: Vault, - password: string = "" -) => { - if (metadataFile === undefined) { - log.debug("no metadata file, so no fetch"); - return { - deletions: [], - } as MetadataOnRemote; - } - - const buf = await client.downloadFromRemote( - metadataFile.key, - vault, - metadataFile.mtimeRemote ?? Date.now(), - password, - metadataFile.remoteEncryptedKey, - true - ); - const metadata = deserializeMetadataOnRemote(buf); - return metadata; -}; - -const isSkipItem = ( +const isSkipItemByName = ( key: string, syncConfigDir: boolean, syncUnderscoreItems: boolean, @@ -309,7 +159,7 @@ const isSkipItem = ( if (syncConfigDir && isInsideObsFolder(key, configDir)) { return false; } - if (isFolderToSkip(key, [])) { + if (isSpecialFolderNameToSkip(key, [])) { // some special dirs and files are always skipped return true; } @@ -321,72 +171,146 @@ const isSkipItem = ( ); }; -const ensembleMixedStates = async ( - remoteStates: FileOrFolderMixedState[], - local: TAbstractFile[], - localConfigDirContents: ObsConfigDirFileType[] | undefined, - remoteDeleteHistory: DeletionOnRemote[] | undefined, - localFileHistory: FileFolderHistoryRecord[] | undefined, +const copyEntityAndFixTimeFormat = (src: Entity) => { + const result = Object.assign({}, src); + if (result.mtimeCli !== undefined) { + if (result.mtimeCli === 0) { + result.mtimeCli = undefined; + } else { + result.mtimeCliFmt = unixTimeToStr(result.mtimeCli); + } + } + if (result.mtimeSvr !== undefined) { + if (result.mtimeSvr === 0) { + result.mtimeSvr = undefined; + } else { + result.mtimeSvrFmt = unixTimeToStr(result.mtimeSvr); + } + } + if (result.prevSyncTime !== undefined) { + if (result.prevSyncTime === 0) { + result.prevSyncTime = undefined; + } else { + result.prevSyncTimeFmt = unixTimeToStr(result.prevSyncTime); + } + } + + return result; +}; + +/** + * Inplace, no copy again. + * @param remote + * @param password + * @returns + */ +const decryptRemoteEntityInplace = async (remote: Entity, password: string) => { + if (password == undefined || password === "") { + remote.key = remote.keyEnc; + remote.size = remote.sizeEnc; + return remote; + } + + if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { + remote.key = await decryptBase32ToString(remote.keyEnc, password); + } else if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) { + remote.key = await decryptBase64urlToString(remote.keyEnc, password); + } else { + throw Error(`unexpected key to decrypt=${remote.keyEnc}`); + } + + // TODO + // remote.size = getSizeFromEncToOrig(remote.sizeEnc, password); + // but we don't have deterministic way to get a number because the encryption has padding... + + return remote; +}; + +/** + * Directly throw error here. + * We can only defer the checking now, because before decryption we don't know whether it's a file or folder. + * @param remote + */ +const ensureMTimeOfRemoteEntityValid = (remote: Entity) => { + if ( + !remote.key.endsWith("/") && + remote.mtimeCli === undefined && + remote.mtimeSvr === undefined + ) { + if (remote.key === remote.keyEnc) { + throw Error( + `Your remote file ${remote.key} has last modified time 0, don't know how to deal with it.` + ); + } else { + throw Error( + `Your remote file ${remote.key} (encrypted as ${remote.keyEnc}) has last modified time 0, don't know how to deal with it.` + ); + } + } + return remote; +}; + +/** + * Inplace, no copy again. + * @param local + * @param password + * @returns + */ +const encryptLocalEntityInplace = async ( + local: Entity, + password: string, + remoteKeyEnc: string | undefined +) => { + if (password == undefined || password === "") { + return local; + } + if (local.size === local.sizeEnc) { + local.sizeEnc = getSizeFromOrigToEnc(local.size); + } + if (local.key === local.keyEnc) { + if ( + remoteKeyEnc !== undefined && + remoteKeyEnc !== "" && + remoteKeyEnc !== local.key + ) { + // we can reuse remote encrypted key if any + local.keyEnc = remoteKeyEnc; + } else { + // we assign a new encrypted key because of no remote + // the old version uses base32 + // local.keyEnc = await encryptStringToBase32(local.key, password); + // the new version users base64url + local.keyEnc = await encryptStringToBase64url(local.key, password); + } + } + return local; +}; + +const ensembleMixedEnties = async ( + localEntityList: Entity[], + prevSyncEntityList: Entity[], + remoteEntityList: Entity[], + syncConfigDir: boolean, configDir: string, syncUnderscoreItems: boolean, ignorePaths: string[], password: string -) => { - const results = {} as Record; +): Promise> => { + const finalMappings: Record = {}; - for (const r of remoteStates) { - const key = r.key; - - if ( - isSkipItem( - key, - syncConfigDir, - syncUnderscoreItems, - configDir, - ignorePaths + // remote has to be first + for (const remote of remoteEntityList) { + const remoteCopied = ensureMTimeOfRemoteEntityValid( + await decryptRemoteEntityInplace( + copyEntityAndFixTimeFormat(remote), + password ) - ) { - continue; - } - results[key] = r; - results[key].existLocal = false; - } - - for (const entry of local) { - let r = {} as FileOrFolderMixedState; - let key = entry.path; - - if (entry.path === "/") { - // ignore - continue; - } else if (entry instanceof TFile) { - const mtimeLocal = Math.max(entry.stat.mtime ?? 0, entry.stat.ctime ?? 0); - r = { - key: entry.path, - existLocal: true, - mtimeLocal: mtimeLocal, - mtimeLocalFmt: unixTimeToStr(mtimeLocal), - sizeLocal: entry.stat.size, - sizeLocalEnc: - password === "" ? undefined : getSizeFromOrigToEnc(entry.stat.size), - }; - } else if (entry instanceof TFolder) { - key = `${entry.path}/`; - r = { - key: key, - existLocal: true, - mtimeLocal: undefined, - mtimeLocalFmt: undefined, - sizeLocal: 0, - sizeLocalEnc: password === "" ? undefined : getSizeFromOrigToEnc(0), - }; - } else { - throw Error(`unexpected ${entry}`); - } + ); + const key = remoteCopied.key; if ( - isSkipItem( + isSkipItemByName( key, syncConfigDir, syncUnderscoreItems, @@ -397,111 +321,16 @@ const ensembleMixedStates = async ( continue; } - if (results.hasOwnProperty(key)) { - results[key].key = r.key; - results[key].existLocal = r.existLocal; - results[key].mtimeLocal = r.mtimeLocal; - results[key].mtimeLocalFmt = r.mtimeLocalFmt; - results[key].sizeLocal = r.sizeLocal; - results[key].sizeLocalEnc = r.sizeLocalEnc; - } else { - results[key] = r; - results[key].existRemote = false; - } - } - - if (syncConfigDir && localConfigDirContents !== undefined) { - for (const entry of localConfigDirContents) { - const key = entry.key; - let mtimeLocal: number | undefined = Math.max( - entry.mtime ?? 0, - entry.ctime ?? 0 - ); - if (Number.isNaN(mtimeLocal) || mtimeLocal === 0) { - mtimeLocal = undefined; - } - const r: FileOrFolderMixedState = { - key: key, - existLocal: true, - mtimeLocal: mtimeLocal, - mtimeLocalFmt: unixTimeToStr(mtimeLocal), - sizeLocal: entry.size, - sizeLocalEnc: - password === "" ? undefined : getSizeFromOrigToEnc(entry.size), - }; - - if ( - isSkipItem( - key, - syncConfigDir, - syncUnderscoreItems, - configDir, - ignorePaths - ) - ) { - continue; - } - - if (results.hasOwnProperty(key)) { - results[key].key = r.key; - results[key].existLocal = r.existLocal; - results[key].mtimeLocal = r.mtimeLocal; - results[key].mtimeLocalFmt = r.mtimeLocalFmt; - results[key].sizeLocal = r.sizeLocal; - results[key].sizeLocalEnc = r.sizeLocalEnc; - } else { - results[key] = r; - results[key].existRemote = false; - } - } - } - - for (const entry of remoteDeleteHistory ?? []) { - const key = entry.key; - const r = { + finalMappings[key] = { key: key, - deltimeRemote: entry.actionWhen, - deltimeRemoteFmt: unixTimeToStr(entry.actionWhen), - } as FileOrFolderMixedState; - - if ( - isSkipItem( - key, - syncConfigDir, - syncUnderscoreItems, - configDir, - ignorePaths - ) - ) { - continue; - } - - if (results.hasOwnProperty(key)) { - results[key].key = r.key; - results[key].deltimeRemote = r.deltimeRemote; - results[key].deltimeRemoteFmt = r.deltimeRemoteFmt; - } else { - results[key] = r; - - results[key].existLocal = false; - results[key].existRemote = false; - } + remote: remoteCopied, + }; } - for (const entry of localFileHistory ?? []) { - let key = entry.key; - if (entry.keyType === "folder") { - if (!entry.key.endsWith("/")) { - key = `${entry.key}/`; - } - } else if (entry.keyType === "file") { - // pass - } else { - throw Error(`unexpected ${entry}`); - } - + for (const prevSync of prevSyncEntityList) { + const key = prevSync.key; if ( - isSkipItem( + isSkipItemByName( key, syncConfigDir, syncUnderscoreItems, @@ -512,836 +341,382 @@ const ensembleMixedStates = async ( continue; } - if (entry.actionType === "delete" || entry.actionType === "rename") { - const r = { + if (finalMappings.hasOwnProperty(key)) { + const prevSyncCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(prevSync), + password, + finalMappings[key].remote?.keyEnc + ); + finalMappings[key].prevSync = prevSyncCopied; + } else { + const prevSyncCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(prevSync), + password, + undefined + ); + finalMappings[key] = { key: key, - deltimeLocal: entry.actionWhen, - deltimeLocalFmt: unixTimeToStr(entry.actionWhen), - } as FileOrFolderMixedState; - - if (results.hasOwnProperty(key)) { - results[key].deltimeLocal = r.deltimeLocal; - results[key].deltimeLocalFmt = r.deltimeLocalFmt; - } else { - results[key] = r; - results[key].existLocal = false; // we have already checked local - results[key].existRemote = false; // we have already checked remote - } - } else if (entry.actionType === "renameDestination") { - const r = { - key: key, - mtimeLocal: entry.actionWhen, - mtimeLocalFmt: unixTimeToStr(entry.actionWhen), - changeLocalMtimeUsingMapping: true, + prevSync: prevSyncCopied, }; - if (results.hasOwnProperty(key)) { - let mtimeLocal: number | undefined = Math.max( - r.mtimeLocal ?? 0, - results[key].mtimeLocal ?? 0 - ); - if (Number.isNaN(mtimeLocal) || mtimeLocal === 0) { - mtimeLocal = undefined; - } - results[key].mtimeLocal = mtimeLocal; - results[key].mtimeLocalFmt = unixTimeToStr(mtimeLocal); - results[key].changeLocalMtimeUsingMapping = - r.changeLocalMtimeUsingMapping; - } else { - // So, the file doesn't exist, - // except that it existed in the "renamed to" history records. - // Most likely because that the user deleted the file while Obsidian was closed, - // so Obsidian could not track the deletions. - // We are not sure how to deal with this, so do not generate anything here! - // // // The following 3 lines are of old logic, and have been removed: - // // results[key] = r; - // // results[key].existLocal = false; // we have already checked local - // // results[key].existRemote = false; // we have already checked remote - } - } else { - throw Error( - `do not know how to deal with local file history ${entry.key} with ${entry.actionType}` + } + } + + // local has to be last + // because we want to get keyEnc based on the remote + // (we don't consume prevSync here because it gains no benefit) + for (const local of localEntityList) { + const key = local.key; + if ( + isSkipItemByName( + key, + syncConfigDir, + syncUnderscoreItems, + configDir, + ignorePaths + ) + ) { + continue; + } + + if (finalMappings.hasOwnProperty(key)) { + const localCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(local), + password, + finalMappings[key].remote?.keyEnc ); - } - } - - return results; -}; - -const assignOperationToFileInplace = ( - origRecord: FileOrFolderMixedState, - keptFolder: Set, - skipSizeLargerThan: number, - password: string = "" -) => { - let r = origRecord; - - // files and folders are treated differently - // here we only check files - if (r.key.endsWith("/")) { - return r; - } - - // we find the max date from four sources - - // 0. find anything inconsistent - if (r.existLocal && (r.mtimeLocal === undefined || r.mtimeLocal <= 0)) { - throw Error( - `Error: Abnormal last modified time locally: ${JSON.stringify( - r, - null, - 2 - )}` - ); - } - if (r.existRemote && (r.mtimeRemote === undefined || r.mtimeRemote <= 0)) { - throw Error( - `Error: Abnormal last modified time remotely: ${JSON.stringify( - r, - null, - 2 - )}` - ); - } - if (r.deltimeLocal !== undefined && r.deltimeLocal <= 0) { - throw Error( - `Error: Abnormal deletion time locally: ${JSON.stringify(r, null, 2)}` - ); - } - if (r.deltimeRemote !== undefined && r.deltimeRemote <= 0) { - throw Error( - `Error: Abnormal deletion time remotely: ${JSON.stringify(r, null, 2)}` - ); - } - - if ( - (r.existLocal && password !== "" && r.sizeLocalEnc === undefined) || - (r.existRemote && password !== "" && r.sizeRemoteEnc === undefined) - ) { - throw new Error( - `Error: No encryption sizes: ${JSON.stringify(r, null, 2)}` - ); - } - - const sizeLocalComp = password === "" ? r.sizeLocal : r.sizeLocalEnc; - const sizeRemoteComp = password === "" ? r.sizeRemote : r.sizeRemoteEnc; - - // 1. mtimeLocal - if (r.existLocal) { - const mtimeRemote = r.existRemote ? r.mtimeRemote! : -1; - const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; - const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; - if ( - r.mtimeLocal! >= mtimeRemote && - r.mtimeLocal! >= deltimeLocal && - r.mtimeLocal! >= deltimeRemote - ) { - if (sizeLocalComp === undefined) { - throw new Error( - `Error: no local size but has local mtime: ${JSON.stringify( - r, - null, - 2 - )}` - ); - } - if (r.mtimeLocal === r.mtimeRemote) { - // local and remote both exist and mtimes are the same - if (sizeLocalComp === sizeRemoteComp) { - // do not need to consider skipSizeLargerThan in this case - r.decision = "skipUploading"; - r.decisionBranch = 1; - } else { - if (skipSizeLargerThan <= 0) { - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 2; - } else { - // limit the sizes - if (sizeLocalComp <= skipSizeLargerThan) { - if (sizeRemoteComp! <= skipSizeLargerThan) { - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 18; - } else { - r.decision = "errorRemoteTooLargeConflictLocal"; - r.decisionBranch = 19; - } - } else { - if (sizeRemoteComp! <= skipSizeLargerThan) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 20; - } else { - r.decision = "skipUploadingTooLarge"; - r.decisionBranch = 21; - } - } - } - } - } else { - // we have local laregest mtime, - // and the remote not existing or smaller mtime - if (skipSizeLargerThan <= 0) { - // no need to consider sizes - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 4; - } else { - // need to consider sizes - if (sizeLocalComp <= skipSizeLargerThan) { - if (sizeRemoteComp === undefined) { - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 22; - } else if (sizeRemoteComp <= skipSizeLargerThan) { - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 23; - } else { - r.decision = "errorRemoteTooLargeConflictLocal"; - r.decisionBranch = 24; - } - } else { - if (sizeRemoteComp === undefined) { - r.decision = "skipUploadingTooLarge"; - r.decisionBranch = 25; - } else if (sizeRemoteComp <= skipSizeLargerThan) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 26; - } else { - r.decision = "skipUploadingTooLarge"; - r.decisionBranch = 27; - } - } - } - } - keptFolder.add(getParentFolder(r.key)); - return r; - } - } - - // 2. mtimeRemote - if (r.existRemote) { - const mtimeLocal = r.existLocal ? r.mtimeLocal! : -1; - const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; - const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; - if ( - r.mtimeRemote! > mtimeLocal && - r.mtimeRemote! >= deltimeLocal && - r.mtimeRemote! >= deltimeRemote - ) { - // we have remote laregest mtime, - // and the local not existing or smaller mtime - if (sizeRemoteComp === undefined) { - throw new Error( - `Error: no remote size but has remote mtime: ${JSON.stringify( - r, - null, - 2 - )}` - ); - } - - if (skipSizeLargerThan <= 0) { - // no need to consider sizes - r.decision = "downloadRemoteToLocal"; - r.decisionBranch = 5; - } else { - // need to consider sizes - if (sizeRemoteComp <= skipSizeLargerThan) { - if (sizeLocalComp === undefined) { - r.decision = "downloadRemoteToLocal"; - r.decisionBranch = 28; - } else if (sizeLocalComp <= skipSizeLargerThan) { - r.decision = "downloadRemoteToLocal"; - r.decisionBranch = 29; - } else { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 30; - } - } else { - if (sizeLocalComp === undefined) { - r.decision = "skipDownloadingTooLarge"; - r.decisionBranch = 31; - } else if (sizeLocalComp <= skipSizeLargerThan) { - r.decision = "errorRemoteTooLargeConflictLocal"; - r.decisionBranch = 32; - } else { - r.decision = "skipDownloadingTooLarge"; - r.decisionBranch = 33; - } - } - } - - keptFolder.add(getParentFolder(r.key)); - return r; - } - } - - // 3. deltimeLocal - if (r.deltimeLocal !== undefined && r.deltimeLocal !== 0) { - const mtimeLocal = r.existLocal ? r.mtimeLocal! : -1; - const mtimeRemote = r.existRemote ? r.mtimeRemote! : -1; - const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; - if ( - r.deltimeLocal >= mtimeLocal && - r.deltimeLocal >= mtimeRemote && - r.deltimeLocal >= deltimeRemote - ) { - if (skipSizeLargerThan <= 0) { - r.decision = "uploadLocalDelHistToRemote"; - r.decisionBranch = 6; - if (r.existLocal || r.existRemote) { - // actual deletion would happen - } - } else { - const localTooLargeToDelete = - r.existLocal && sizeLocalComp! > skipSizeLargerThan; - const remoteTooLargeToDelete = - r.existRemote && sizeRemoteComp! > skipSizeLargerThan; - if (localTooLargeToDelete) { - if (remoteTooLargeToDelete) { - r.decision = "skipUsingLocalDelTooLarge"; - r.decisionBranch = 34; - } else { - if (r.existRemote) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 35; - } else { - r.decision = "skipUsingLocalDelTooLarge"; - r.decisionBranch = 36; - } - } - } else { - if (remoteTooLargeToDelete) { - if (r.existLocal) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 37; - } else { - r.decision = "skipUsingLocalDelTooLarge"; - r.decisionBranch = 38; - } - } else { - r.decision = "uploadLocalDelHistToRemote"; - r.decisionBranch = 39; - } - } - } - return r; - } - } - - // 4. deltimeRemote - if (r.deltimeRemote !== undefined && r.deltimeRemote !== 0) { - const mtimeLocal = r.existLocal ? r.mtimeLocal! : -1; - const mtimeRemote = r.existRemote ? r.mtimeRemote! : -1; - const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; - if ( - r.deltimeRemote >= mtimeLocal && - r.deltimeRemote >= mtimeRemote && - r.deltimeRemote >= deltimeLocal - ) { - if (skipSizeLargerThan <= 0) { - r.decision = "keepRemoteDelHist"; - r.decisionBranch = 7; - if (r.existLocal || r.existRemote) { - // actual deletion would happen - } - } else { - const localTooLargeToDelete = - r.existLocal && sizeLocalComp! > skipSizeLargerThan; - const remoteTooLargeToDelete = - r.existRemote && sizeRemoteComp! > skipSizeLargerThan; - if (localTooLargeToDelete) { - if (remoteTooLargeToDelete) { - r.decision = "skipUsingRemoteDelTooLarge"; - r.decisionBranch = 40; - } else { - if (r.existRemote) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 41; - } else { - r.decision = "skipUsingRemoteDelTooLarge"; - r.decisionBranch = 42; - } - } - } else { - if (remoteTooLargeToDelete) { - if (r.existLocal) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 43; - } else { - r.decision = "skipUsingRemoteDelTooLarge"; - r.decisionBranch = 44; - } - } else { - r.decision = "keepRemoteDelHist"; - r.decisionBranch = 45; - } - } - } - return r; - } - } - - throw Error(`no decision for ${JSON.stringify(r)}`); -}; - -const assignOperationToFolderInplace = async ( - origRecord: FileOrFolderMixedState, - keptFolder: Set, - vault: Vault, - password: string = "" -) => { - let r = origRecord; - - // files and folders are treated differently - // here we only check folders - if (!r.key.endsWith("/")) { - return r; - } - - if (!keptFolder.has(r.key)) { - // the folder does NOT have any must-be-kept children! - - if (r.deltimeLocal !== undefined || r.deltimeRemote !== undefined) { - // it has some deletion "commands" - - const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; - const deltimeRemote = - r.deltimeRemote !== undefined ? r.deltimeRemote : -1; - - // if it was created after deletion, we should keep it as is - if (requireApiVersion(API_VER_STAT_FOLDER)) { - if (r.existLocal) { - let ctime = 0; - let mtime = 0; - const s = await statFix(vault, r.key); - if (s !== undefined && s !== null) { - ctime = s.ctime; - mtime = s.mtime; - } - const cmtime = Math.max(ctime ?? 0, mtime ?? 0); - if ( - !Number.isNaN(cmtime) && - cmtime > 0 && - cmtime >= deltimeLocal && - cmtime >= deltimeRemote - ) { - keptFolder.add(getParentFolder(r.key)); - if (r.existLocal && r.existRemote) { - r.decision = "skipFolder"; - r.decisionBranch = 14; - } else if (r.existLocal || r.existRemote) { - r.decision = "createFolder"; - r.decisionBranch = 15; - } else { - throw Error( - `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` - ); - } - } - } - } - - // If it was moved to here, after deletion, we should keep it as is. - // The logic not necessarily needs API_VER_STAT_FOLDER. - // The folder needs this logic because it's also determined by file children. - // But the file do not need this logic because the mtimeLocal is checked firstly. - if ( - r.existLocal && - r.changeLocalMtimeUsingMapping && - r.mtimeLocal! > 0 && - r.mtimeLocal! > deltimeLocal && - r.mtimeLocal! > deltimeRemote - ) { - keptFolder.add(getParentFolder(r.key)); - if (r.existLocal && r.existRemote) { - r.decision = "skipFolder"; - r.decisionBranch = 16; - } else if (r.existLocal || r.existRemote) { - r.decision = "createFolder"; - r.decisionBranch = 17; - } else { - throw Error( - `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` - ); - } - } - - if (r.decision === undefined) { - // not yet decided by the above reason - if (deltimeLocal > 0 && deltimeLocal > deltimeRemote) { - r.decision = "uploadLocalDelHistToRemoteFolder"; - r.decisionBranch = 8; - } else { - r.decision = "keepRemoteDelHistFolder"; - r.decisionBranch = 9; - } - } + finalMappings[key].local = localCopied; } else { - // it does not have any deletion commands - // keep it as is, and create it if necessary - keptFolder.add(getParentFolder(r.key)); - if (r.existLocal && r.existRemote) { - r.decision = "skipFolder"; - r.decisionBranch = 10; - } else if (r.existLocal || r.existRemote) { - r.decision = "createFolder"; - r.decisionBranch = 11; - } else { - throw Error( - `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` - ); - } - } - } else { - // the folder has some must be kept children! - // so itself and its parent folder must be kept - keptFolder.add(getParentFolder(r.key)); - if (r.existLocal && r.existRemote) { - r.decision = "skipFolder"; - r.decisionBranch = 12; - } else if (r.existLocal || r.existRemote) { - r.decision = "createFolder"; - r.decisionBranch = 13; - } else { - throw Error( - `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` + const localCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(local), + password, + undefined ); + finalMappings[key] = { + key: key, + local: localCopied, + }; } } - // save the memory, save the world! - // we have dealt with it, so we don't need it any more. - keptFolder.delete(r.key); - return r; + return finalMappings; }; -const DELETION_DECISIONS: Set = new Set([ - "uploadLocalDelHistToRemote", - "keepRemoteDelHist", - "uploadLocalDelHistToRemoteFolder", - "keepRemoteDelHistFolder", -]); -const SIZES_GO_WRONG_DECISIONS: Set = new Set([ - "errorLocalTooLargeConflictRemote", - "errorRemoteTooLargeConflictLocal", -]); - -export const getSyncPlan = async ( - remoteStates: FileOrFolderMixedState[], - local: TAbstractFile[], - localConfigDirContents: ObsConfigDirFileType[] | undefined, - remoteDeleteHistory: DeletionOnRemote[] | undefined, - localFileHistory: FileFolderHistoryRecord[] | undefined, - remoteType: SUPPORTED_SERVICES_TYPE, - triggerSource: SyncTriggerSourceType, - vault: Vault, - syncConfigDir: boolean, - configDir: string, - syncUnderscoreItems: boolean, +/** + * Heavy lifting. + * Basically follow the sync algorithm of https://github.com/Jwink3101/syncrclone + * @param mixedEntityMappings + */ +export const getSyncPlanInplace = async ( + mixedEntityMappings: Record, + howToCleanEmptyFolder: EmptyFolderCleanType, skipSizeLargerThan: number, - ignorePaths: string[], - password: string = "" + conflictAction: ConflictActionType ) => { - const mixedStates = await ensembleMixedStates( - remoteStates, - local, - localConfigDirContents, - remoteDeleteHistory, - localFileHistory, - syncConfigDir, - configDir, - syncUnderscoreItems, - ignorePaths, - password - ); - - const sortedKeys = Object.keys(mixedStates).sort( + // from long(deep) to short(shadow) + const sortedKeys = Object.keys(mixedEntityMappings).sort( (k1, k2) => k2.length - k1.length ); - const sizesGoWrong: FileOrFolderMixedState[] = []; - const deletions: DeletionOnRemote[] = []; - const keptFolder = new Set(); + for (let i = 0; i < sortedKeys.length; ++i) { const key = sortedKeys[i]; - const val = mixedStates[key]; + const mixedEntry = mixedEntityMappings[key]; + const { local, prevSync, remote } = mixedEntry; if (key.endsWith("/")) { - // decide some folders - // because the keys are sorted by length - // so all the children must have been shown up before in the iteration - await assignOperationToFolderInplace(val, keptFolder, vault, password); - } else { - // get all operations of files - // and at the same time get some helper info for folders - assignOperationToFileInplace( - val, - keptFolder, - skipSizeLargerThan, - password - ); - } - - if (SIZES_GO_WRONG_DECISIONS.has(val.decision!)) { - sizesGoWrong.push(val); - } - - if (DELETION_DECISIONS.has(val.decision!)) { - if (val.decision === "uploadLocalDelHistToRemote") { - deletions.push({ - key: key, - actionWhen: val.deltimeLocal!, - }); - } else if (val.decision === "keepRemoteDelHist") { - deletions.push({ - key: key, - actionWhen: val.deltimeRemote!, - }); - } else if (val.decision === "uploadLocalDelHistToRemoteFolder") { - deletions.push({ - key: key, - actionWhen: val.deltimeLocal!, - }); - } else if (val.decision === "keepRemoteDelHistFolder") { - deletions.push({ - key: key, - actionWhen: val.deltimeRemote!, - }); + // folder + // folder doesn't worry about mtime and size, only check their existences + if (keptFolder.has(key)) { + // should fill the missing part + if (local !== undefined && remote !== undefined) { + mixedEntry.decisionBranch = 101; + mixedEntry.decision = "folder_existed_both"; + } else if (local !== undefined && remote === undefined) { + mixedEntry.decisionBranch = 102; + mixedEntry.decision = "folder_existed_local"; + } else if (local === undefined && remote !== undefined) { + mixedEntry.decisionBranch = 103; + mixedEntry.decision = "folder_existed_remote"; + } else { + mixedEntry.decisionBranch = 104; + mixedEntry.decision = "folder_to_be_created"; + } + keptFolder.delete(key); // no need to save it in the Set later } else { - throw Error(`do not know how to delete for decision ${val.decision}`); + if (howToCleanEmptyFolder === "skip") { + mixedEntry.decisionBranch = 105; + mixedEntry.decision = "folder_to_skip"; + } else if (howToCleanEmptyFolder === "clean_both") { + mixedEntry.decisionBranch = 106; + mixedEntry.decision = "folder_to_be_deleted"; + } else { + throw Error( + `do not know how to deal with empty folder ${mixedEntry.key}` + ); + } + } + } else { + // file + + if (local === undefined && remote === undefined) { + // both deleted, only in history + mixedEntry.decisionBranch = 1; + mixedEntry.decision = "only_history"; + } else if (local !== undefined && remote !== undefined) { + if ( + (local.mtimeCli === remote.mtimeCli || + local.mtimeCli === remote.mtimeSvr) && + local.sizeEnc === remote.sizeEnc + ) { + // completely equal / identical + mixedEntry.decisionBranch = 2; + mixedEntry.decision = "equal"; + keptFolder.add(getParentFolder(key)); + } else { + // Both exists, but modified or conflict + // Look for past files of A or B. + + const localEqualPrevSync = + prevSync?.mtimeSvr === local.mtimeCli && + prevSync?.sizeEnc === local.sizeEnc; + const remoteEqualPrevSync = + (prevSync?.mtimeSvr === remote.mtimeCli || + prevSync?.mtimeSvr === remote.mtimeSvr) && + prevSync?.sizeEnc === remote.sizeEnc; + + if (localEqualPrevSync && !remoteEqualPrevSync) { + // If only one compares true (no prev also means it compares False), the other is modified. Backup and sync. + if ( + skipSizeLargerThan <= 0 || + remote.sizeEnc <= skipSizeLargerThan + ) { + mixedEntry.decisionBranch = 9; + mixedEntry.decision = "modified_remote"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `remote is modified (branch 9) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } else if (!localEqualPrevSync && remoteEqualPrevSync) { + // If only one compares true (no prev also means it compares False), the other is modified. Backup and sync. + if ( + skipSizeLargerThan <= 0 || + local.sizeEnc <= skipSizeLargerThan + ) { + mixedEntry.decisionBranch = 10; + mixedEntry.decision = "modified_local"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `local is modified (branch 10) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } else if (!localEqualPrevSync && !remoteEqualPrevSync) { + // If both compare False, (didn't exist means both are new. Both exist but don't compare means both are modified) + if (prevSync === undefined) { + if (conflictAction === "keep_newer") { + if ( + (local.mtimeCli ?? local.mtimeSvr ?? 0) >= + (remote.mtimeCli ?? remote.mtimeSvr ?? 0) + ) { + mixedEntry.decisionBranch = 11; + mixedEntry.decision = "conflict_created_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 12; + mixedEntry.decision = "conflict_created_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else if (conflictAction === "keep_larger") { + if (local.sizeEnc >= remote.sizeEnc) { + mixedEntry.decisionBranch = 13; + mixedEntry.decision = "conflict_created_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 14; + mixedEntry.decision = "conflict_created_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else { + mixedEntry.decisionBranch = 15; + mixedEntry.decision = "conflict_created_keep_both"; + keptFolder.add(getParentFolder(key)); + } + } else { + if (conflictAction === "keep_newer") { + if ( + (local.mtimeCli ?? local.mtimeSvr ?? 0) >= + (remote.mtimeCli ?? remote.mtimeSvr ?? 0) + ) { + mixedEntry.decisionBranch = 16; + mixedEntry.decision = "conflict_modified_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 17; + mixedEntry.decision = "conflict_modified_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else if (conflictAction === "keep_larger") { + if (local.sizeEnc >= remote.sizeEnc) { + mixedEntry.decisionBranch = 18; + mixedEntry.decision = "conflict_modified_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 19; + mixedEntry.decision = "conflict_modified_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else { + mixedEntry.decisionBranch = 20; + mixedEntry.decision = "conflict_modified_keep_both"; + keptFolder.add(getParentFolder(key)); + } + } + } else { + // Both compare true -- This is VERY odd and should not happen + throw Error( + `should not reach branch -2 while getting sync plan: ${JSON.stringify( + mixedEntry + )}` + ); + } + } + } else if (local === undefined && remote !== undefined) { + // A is missing + if (prevSync === undefined) { + // if B is not in the previous list, B is new + if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) { + mixedEntry.decisionBranch = 3; + mixedEntry.decision = "created_remote"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `remote is created (branch 3) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } else if ( + (prevSync.mtimeSvr === remote.mtimeCli || + prevSync.mtimeSvr === remote.mtimeSvr) && + prevSync.sizeEnc === remote.sizeEnc + ) { + // if B is in the previous list and UNMODIFIED, B has been deleted by A + mixedEntry.decisionBranch = 4; + mixedEntry.decision = "deleted_local"; + } else { + // if B is in the previous list and MODIFIED, B has been deleted by A but modified by B + if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) { + mixedEntry.decisionBranch = 5; + mixedEntry.decision = "modified_remote"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `remote is modified (branch 5) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } + } else if (local !== undefined && remote === undefined) { + // B is missing + + if (prevSync === undefined) { + // if A is not in the previous list, A is new + if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) { + mixedEntry.decisionBranch = 6; + mixedEntry.decision = "created_local"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `local is created (branch 6) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } else if ( + prevSync.mtimeSvr === local.mtimeCli && + prevSync.sizeEnc === local.sizeEnc + ) { + // if A is in the previous list and UNMODIFIED, A has been deleted by B + mixedEntry.decisionBranch = 7; + mixedEntry.decision = "deleted_remote"; + } else { + // if A is in the previous list and MODIFIED, A has been deleted by B but modified by A + if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) { + mixedEntry.decisionBranch = 8; + mixedEntry.decision = "modified_local"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `local is modified (branch 8) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } + } else { + throw Error( + `should not reach branch -1 while getting sync plan: ${JSON.stringify( + mixedEntry + )}` + ); + } + + if (mixedEntry.decision === undefined) { + throw Error( + `unexpectedly no decision of file in the end: ${JSON.stringify( + mixedEntry + )}` + ); } } } - const currTs = Date.now(); - const currTsFmt = unixTimeToStr(currTs); - const plan = { - ts: currTs, - tsFmt: currTsFmt, - remoteType: remoteType, - syncTriggerSource: triggerSource, - mixedStates: mixedStates, - } as SyncPlanType; - return { - plan: plan, - sortedKeys: sortedKeys, - deletions: deletions, - sizesGoWrong: sizesGoWrong, - }; + keptFolder.delete("/"); + keptFolder.delete(""); + if (keptFolder.size > 0) { + throw Error(`unexpectedly keptFolder no decisions: ${[...keptFolder]}`); + } + + return mixedEntityMappings; }; -const uploadExtraMeta = async ( - client: RemoteClient, - metadataFile: FileOrFolderMixedState | undefined, - origMetadata: MetadataOnRemote | undefined, - deletions: DeletionOnRemote[], - password: string = "" +const splitThreeStepsOnEntityMappings = ( + mixedEntityMappings: Record ) => { - if (deletions === undefined || deletions.length === 0) { - return; - } + const folderCreationOps: MixedEntity[][] = []; + const deletionOps: MixedEntity[][] = []; + const uploadDownloads: MixedEntity[][] = []; - const key = DEFAULT_FILE_NAME_FOR_METADATAONREMOTE; - let remoteEncryptedKey: string | undefined = key; - - if (password !== "") { - if (metadataFile === undefined) { - remoteEncryptedKey = undefined; - } else { - remoteEncryptedKey = metadataFile.remoteEncryptedKey; - } - if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { - // remoteEncryptedKey = await encryptStringToBase32(key, password); - remoteEncryptedKey = await encryptStringToBase64url(key, password); - } - } - - const newMetadata: MetadataOnRemote = { - deletions: deletions, - }; - - if (isEqualMetadataOnRemote(origMetadata, newMetadata)) { - log.debug( - "metadata are the same, no need to re-generate and re-upload it." - ); - return; - } - - const resultText = serializeMetadataOnRemote(newMetadata); - - await client.uploadToRemote( - key, - undefined, - false, - password, - remoteEncryptedKey, - undefined, - true, - resultText + // from long(deep) to short(shadow) + const sortedKeys = Object.keys(mixedEntityMappings).sort( + (k1, k2) => k2.length - k1.length ); -}; -const dispatchOperationToActual = async ( - key: string, - vaultRandomID: string, - r: FileOrFolderMixedState, - client: RemoteClient, - db: InternalDBs, - vault: Vault, - localDeleteFunc: any, - password: string = "" -) => { - let remoteEncryptedKey: string | undefined = key; - if (password !== "") { - remoteEncryptedKey = r.remoteEncryptedKey; - if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { - // the old version uses base32 - // remoteEncryptedKey = await encryptStringToBase32(key, password); - // the new version users base64url - remoteEncryptedKey = await encryptStringToBase64url(key, password); - } - } - - if (r.decision === undefined) { - throw Error(`unknown decision in ${JSON.stringify(r)}`); - } else if (r.decision === "skipUploading") { - // do nothing! - } else if (r.decision === "uploadLocalDelHistToRemote") { - if (r.existLocal) { - await localDeleteFunc(r.key); - } - if (r.existRemote) { - await client.deleteFromRemote(r.key, password, remoteEncryptedKey); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "keepRemoteDelHist") { - if (r.existLocal) { - await localDeleteFunc(r.key); - } - if (r.existRemote) { - await client.deleteFromRemote(r.key, password, remoteEncryptedKey); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "uploadLocalToRemote") { - if ( - client.serviceType === "onedrive" && - r.sizeLocal === 0 && - password === "" - ) { - // special treatment for empty files for OneDrive - // TODO: it's ugly, any other way? - // special treatment for OneDrive: do nothing, skip empty file without encryption - // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded. - } else { - const remoteObjMeta = await client.uploadToRemote( - r.key, - vault, - false, - password, - remoteEncryptedKey - ); - await upsertSyncMetaMappingDataByVault( - client.serviceType, - db, - r.key, - r.mtimeLocal!, - r.sizeLocal!, - r.key, - remoteObjMeta.lastModified ?? Date.now(), - remoteObjMeta.size, - remoteObjMeta.etag ?? "", - vaultRandomID - ); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "downloadRemoteToLocal") { - await mkdirpInVault(r.key, vault); /* should be unnecessary */ - await client.downloadFromRemote( - r.key, - vault, - r.mtimeRemote!, - password, - remoteEncryptedKey - ); - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "createFolder") { - if (!r.existLocal) { - await mkdirpInVault(r.key, vault); - } - if (!r.existRemote) { - const remoteObjMeta = await client.uploadToRemote( - r.key, - vault, - false, - password, - remoteEncryptedKey - ); - await upsertSyncMetaMappingDataByVault( - client.serviceType, - db, - r.key, - r.mtimeLocal!, - r.sizeLocal!, - r.key, - remoteObjMeta.lastModified ?? Date.now(), - remoteObjMeta.size, - remoteObjMeta.etag ?? "", - vaultRandomID - ); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "uploadLocalDelHistToRemoteFolder") { - if (r.existLocal) { - await localDeleteFunc(r.key); - } - if (r.existRemote) { - await client.deleteFromRemote(r.key, password, remoteEncryptedKey); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "keepRemoteDelHistFolder") { - if (r.existLocal) { - await localDeleteFunc(r.key); - } - if (r.existRemote) { - await client.deleteFromRemote(r.key, password, remoteEncryptedKey); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "skipFolder") { - // do nothing! - } else if (r.decision === "skipUploadingTooLarge") { - // do nothing! - } else if (r.decision === "skipDownloadingTooLarge") { - // do nothing! - } else if (r.decision === "skipUsingLocalDelTooLarge") { - // do nothing! - } else if (r.decision === "skipUsingRemoteDelTooLarge") { - // do nothing! - } else { - throw Error(`unknown decision in ${JSON.stringify(r)}`); - } -}; - -const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => { - const mixedStates = syncPlan.mixedStates; - const totalCount = sortedKeys.length || 0; - - const folderCreationOps: FileOrFolderMixedState[][] = []; - const deletionOps: FileOrFolderMixedState[][] = []; - const uploadDownloads: FileOrFolderMixedState[][] = []; let realTotalCount = 0; for (let i = 0; i < sortedKeys.length; ++i) { const key = sortedKeys[i]; - const val: FileOrFolderMixedState = Object.assign({}, mixedStates[key]); // copy to avoid issue + const val = mixedEntityMappings[key]; if ( - val.decision === "skipFolder" || - val.decision === "skipUploading" || - val.decision === "skipDownloadingTooLarge" || - val.decision === "skipUploadingTooLarge" || - val.decision === "skipUsingLocalDelTooLarge" || - val.decision === "skipUsingRemoteDelTooLarge" + val.decision === "equal" || + val.decision === "folder_existed_both" || + val.decision === "folder_to_skip" ) { // pass - } else if (val.decision === "createFolder") { + } else if ( + val.decision === "folder_existed_local" || + val.decision === "folder_existed_remote" || + val.decision === "folder_to_be_created" + ) { const level = atWhichLevel(key); if (folderCreationOps[level - 1] === undefined) { folderCreationOps[level - 1] = [val]; @@ -1350,10 +725,10 @@ const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => { } realTotalCount += 1; } else if ( - val.decision === "uploadLocalDelHistToRemoteFolder" || - val.decision === "keepRemoteDelHistFolder" || - val.decision === "uploadLocalDelHistToRemote" || - val.decision === "keepRemoteDelHist" + val.decision === "only_history" || + val.decision === "deleted_local" || + val.decision === "deleted_remote" || + val.decision === "folder_to_be_deleted" ) { const level = atWhichLevel(key); if (deletionOps[level - 1] === undefined) { @@ -1363,8 +738,16 @@ const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => { } realTotalCount += 1; } else if ( - val.decision === "uploadLocalToRemote" || - val.decision === "downloadRemoteToLocal" + val.decision === "modified_local" || + val.decision === "modified_remote" || + val.decision === "created_local" || + val.decision === "created_remote" || + val.decision === "conflict_created_keep_local" || + val.decision === "conflict_created_keep_remote" || + val.decision === "conflict_created_keep_both" || + val.decision === "conflict_modified_keep_local" || + val.decision === "conflict_modified_keep_remote" || + val.decision === "conflict_modified_keep_both" ) { if (uploadDownloads.length === 0) { uploadDownloads[0] = [val]; @@ -1390,46 +773,110 @@ const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => { }; }; -export const doActualSync = async ( +const dispatchOperationToActualV3 = async ( + key: string, + vaultRandomID: string, + r: MixedEntity, client: RemoteClient, db: InternalDBs, + vault: Vault, + localDeleteFunc: any, + password: string +) => { + if (r.decision === "only_history") { + clearPrevSyncRecordByVault(db, vaultRandomID, key); + } else if ( + r.decision === "equal" || + r.decision === "folder_to_skip" || + r.decision === "folder_existed_both" + ) { + // pass + } else if ( + r.decision === "modified_local" || + r.decision === "created_local" || + r.decision === "folder_existed_local" || + r.decision === "conflict_created_keep_local" || + r.decision === "conflict_modified_keep_local" + ) { + if ( + client.serviceType === "onedrive" && + r.local!.size === 0 && + password === "" + ) { + // special treatment for empty files for OneDrive + // TODO: it's ugly, any other way? + // special treatment for OneDrive: do nothing, skip empty file without encryption + // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded. + } else { + const remoteObjMeta = await client.uploadToRemote( + r.key, + vault, + false, + password, + r.local!.keyEnc + ); + await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); + } + } else if ( + r.decision === "modified_remote" || + r.decision === "created_remote" || + r.decision === "conflict_created_keep_remote" || + r.decision === "conflict_modified_keep_remote" || + r.decision === "folder_existed_remote" + ) { + await mkdirpInVault(r.key, vault); + await client.downloadFromRemote( + r.key, + vault, + r.remote!.mtimeCli!, + password, + r.remote!.keyEnc + ); + await upsertPrevSyncRecordByVault(db, vaultRandomID, r.remote!); + } else if (r.decision === "deleted_local") { + await localDeleteFunc(r.key); + await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + } else if (r.decision === "deleted_remote") { + await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); + await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + } else if ( + r.decision === "conflict_created_keep_both" || + r.decision === "conflict_modified_keep_both" + ) { + throw Error(`${r.decision} not implemented yet: ${JSON.stringify(r)}`); + } else if (r.decision === "folder_to_be_created") { + await mkdirpInVault(r.key, vault); + const remoteObjMeta = await client.uploadToRemote( + r.key, + vault, + false, + password, + r.local!.keyEnc + ); + await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); + } else if (r.decision === "folder_to_be_deleted") { + await localDeleteFunc(r.key); + await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); + await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + } else { + throw Error(`don't know how to dispatch decision: ${JSON.stringify(r)}`); + } +}; + +export const doActualSyncV3 = async ( + mixedEntityMappings: Record, + client: RemoteClient, vaultRandomID: string, vault: Vault, - syncPlan: SyncPlanType, - sortedKeys: string[], - metadataFile: FileOrFolderMixedState | undefined, - origMetadata: MetadataOnRemote, - sizesGoWrong: FileOrFolderMixedState[], - deletions: DeletionOnRemote[], + password: string, + concurrency: number, localDeleteFunc: any, - password: string = "", - concurrency: number = 1, - callbackSizesGoWrong?: any, - callbackSyncProcess?: any + callbackSyncProcess: any, + db: InternalDBs ) => { - const mixedStates = syncPlan.mixedStates; - const totalCount = sortedKeys.length || 0; - - if (sizesGoWrong.length > 0) { - log.debug(`some sizes are larger than the threshold, abort and show hints`); - callbackSizesGoWrong(sizesGoWrong); - return; - } - - log.debug(`start syncing extra data firstly`); - await uploadExtraMeta( - client, - metadataFile, - origMetadata, - deletions, - password - ); - log.debug(`finish syncing extra data firstly`); - log.debug(`concurrency === ${concurrency}`); - const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } = - splitThreeSteps(syncPlan, sortedKeys); + splitThreeStepsOnEntityMappings(mixedEntityMappings); const nested = [folderCreationOps, deletionOps, uploadDownloads]; const logTexts = [ `1. create all folders from shadowest to deepest, also check undefined decision`, @@ -1438,26 +885,20 @@ export const doActualSync = async ( ]; let realCounter = 0; - for (let i = 0; i < nested.length; ++i) { log.debug(logTexts[i]); - const operations: FileOrFolderMixedState[][] = nested[i]; + const operations = nested[i]; for (let j = 0; j < operations.length; ++j) { - const singleLevelOps: FileOrFolderMixedState[] | undefined = - operations[j]; - - if (singleLevelOps === undefined || singleLevelOps === null) { - continue; - } + const singleLevelOps = operations[j]; const queue = new PQueue({ concurrency: concurrency, autoStart: true }); const potentialErrors: Error[] = []; let tooManyErrors = false; for (let k = 0; k < singleLevelOps.length; ++k) { - const val: FileOrFolderMixedState = singleLevelOps[k]; + const val = singleLevelOps[k]; const key = val.key; const fn = async () => { @@ -1474,7 +915,7 @@ export const doActualSync = async ( realCounter += 1; } - await dispatchOperationToActual( + await dispatchOperationToActualV3( key, vaultRandomID, val,