From 2fbd87eed4dfaae11bbf7f60ab7498f417bcaf65 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 08:21:05 +0800 Subject: [PATCH 01/21] half way of new sync --- docs/sync_algorithm_v3.md | 21 +- src/baseTypes.ts | 70 +- src/local.ts | 63 ++ src/localdb.ts | 516 +++-------- src/main.ts | 8 +- src/misc.ts | 24 +- src/obsFolderLister.ts | 35 +- src/remote.ts | 3 +- src/remoteForDropbox.ts | 105 +-- src/remoteForOnedrive.ts | 33 +- src/remoteForS3.ts | 55 +- src/remoteForWebdav.ts | 27 +- src/sync.ts | 1817 +++++++++++++------------------------ 13 files changed, 1042 insertions(+), 1735 deletions(-) create mode 100644 src/local.ts diff --git a/docs/sync_algorithm_v3.md b/docs/sync_algorithm_v3.md index 681ab62..cc50fbc 100644 --- a/docs/sync_algorithm_v3.md +++ b/docs/sync_algorithm_v3.md @@ -6,7 +6,7 @@ An absolutely better sync algorithm. Better for tracking deletions and better fo ## Huge Thanks -Basically a combination of algorithm v2 + [synclone](https://github.com/Jwink3101/syncrclone) + [rsinc](https://github.com/ConorWilliams/rsinc) + (some of rclone [bisync](https://rclone.org/bisync/)). All of the later three are released under MIT License so no worries about the licenses. +Basically a combination of algorithm v2 + [synclone](https://github.com/Jwink3101/syncrclone/blob/master/docs/algorithm.md) + [rsinc](https://github.com/ConorWilliams/rsinc) + (some of rclone [bisync](https://rclone.org/bisync/)). All of the later three are released under MIT License so no worries about the licenses. ## Features @@ -27,12 +27,25 @@ Nice to have ## Description -We have _five_ input sources: local all files, remote all files, _local previous succeeded sync history_, local deletions, remote deletions. +We have _five_ input sources: -Init run, consuming local deletions and remote deletions : +1. local all files +2. remote all files +3. _local previous succeeded sync history_ +4. local deletions +5. remote deletions. + +Init run, consuming remote deletions : TBD Later runs, use the first, second, third sources **only**. -TBD +Table modified based on synclone and rsinc. The number inside the table cell is the decision branch in the code. + +| local\remote | remote unchanged | remote modified | remote deleted | remote created | +| --------------- | ------------------ | ---------------- | ------------------ | ---------------- | +| local unchanged | (02) do nothing | (09) pull remote | (07) delete local | (??) conflict | +| local modified | (10) push local | (12) conflict | (08) push local | (??) conflict | +| local deleted | (04) delete remote | (05) pull | (01) clean history | (03) pull remote | +| local created | (??) conflict | (??) conflict | (06) push local | (11) conflict | diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 6401e20..1a1be04 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -116,14 +116,6 @@ export interface RemotelySavePluginSettings { logToDB?: boolean; } -export interface RemoteItem { - key: string; - lastModified?: number; - size: number; - remoteType: SUPPORTED_SERVICES_TYPE; - etag?: string; -} - export const COMMAND_URI = "remotely-save"; export const COMMAND_CALLBACK = "remotely-save-cb"; export const COMMAND_CALLBACK_ONEDRIVE = "remotely-save-cb-onedrive"; @@ -165,6 +157,68 @@ export type DecisionType = | DecisionTypeForFileSize | DecisionTypeForFolder; +export type EmptyFolderCleanType = "skip" | "clean_both"; + +export type ConflictActionType = "keep_newer" | "keep_larger" | "rename_both"; + +export type DecisionTypeForMixedEntity = + | "only_history" + | "equal" + | "modified_local" + | "modified_remote" + | "created_local" + | "created_remote" + | "deleted_local" + | "deleted_remote" + | "conflict_created_keep_local" + | "conflict_created_keep_remote" + | "conflict_created_keep_both" + | "conflict_modified_keep_local" + | "conflict_modified_keep_remote" + | "conflict_modified_keep_both" + | "folder_existed_both" + | "folder_existed_local" + | "folder_existed_remote" + | "folder_to_be_created" + | "folder_to_skip" + | "folder_to_be_deleted"; + +/** + * uniform representation + * everything should be flat and primitive, so that we can copy. + */ +export interface Entity { + key: string; + keyEnc: string; + mtimeCli?: number; + mtimeCliFmt?: string; + mtimeSvr?: number; + mtimeSvrFmt?: string; + prevSyncTime?: number; + prevSyncTimeFmt?: string; + size?: number; // might be unknown or to be filled + sizeEnc: number; + hash?: string; + etag?: string; +} + +/** + * A replacement of FileOrFolderMixedState + */ +export interface MixedEntity { + key: string; + local?: Entity; + prevSync?: Entity; + remote?: Entity; + + decisionBranch?: number; + decision?: DecisionTypeForMixedEntity; + conflictAction?: ConflictActionType; +} + +/** + * @deprecated + */ export interface FileOrFolderMixedState { key: string; existLocal?: boolean; diff --git a/src/local.ts b/src/local.ts new file mode 100644 index 0000000..cd70e38 --- /dev/null +++ b/src/local.ts @@ -0,0 +1,63 @@ +import { TFile, TFolder, type Vault } from "obsidian"; +import type { Entity, MixedEntity } from "./baseTypes"; +import { listFilesInObsFolder } from "./obsFolderLister"; + +export const getLocalEntityList = async ( + vault: Vault, + syncConfigDir: boolean, + configDir: string, + pluginID: string +) => { + const local: Entity[] = []; + + const localTAbstractFiles = vault.getAllLoadedFiles(); + for (const entry of localTAbstractFiles) { + let r = {} as Entity; + let key = entry.path; + + if (entry.path === "/") { + // ignore + continue; + } else if (entry instanceof TFile) { + let mtimeLocal: number | undefined = Math.max( + entry.stat.mtime ?? 0, + entry.stat.ctime + ); + if (mtimeLocal === 0) { + mtimeLocal = undefined; + } + if (mtimeLocal === undefined) { + throw Error( + `Your file has last modified time 0: ${key}, don't know how to deal with it` + ); + } + r = { + key: entry.path, + keyEnc: entry.path, + mtimeCli: mtimeLocal, + mtimeSvr: mtimeLocal, + size: entry.stat.size, + sizeEnc: entry.stat.size, + }; + } else if (entry instanceof TFolder) { + key = `${entry.path}/`; + r = { + key: key, + keyEnc: key, + size: 0, + sizeEnc: 0, + }; + } else { + throw Error(`unexpected ${entry}`); + } + } + + if (syncConfigDir) { + const syncFiles = await listFilesInObsFolder(configDir, vault, pluginID); + for (const f of syncFiles) { + local.push(f); + } + } + + return local; +}; diff --git a/src/localdb.ts b/src/localdb.ts index 4ed4ffa..ee45c16 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -3,35 +3,36 @@ export type LocalForage = typeof localforage; import { nanoid } from "nanoid"; import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian"; -import { API_VER_STAT_FOLDER, SUPPORTED_SERVICES_TYPE } from "./baseTypes"; +import { API_VER_STAT_FOLDER } from "./baseTypes"; +import type { Entity, MixedEntity, SUPPORTED_SERVICES_TYPE } from "./baseTypes"; import type { SyncPlanType } from "./sync"; import { statFix, toText, unixTimeToStr } from "./misc"; import { log } from "./moreOnLog"; -const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326]; -export const DEFAULT_DB_VERSION_NUMBER: number = 20220326; +const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220]; +export const DEFAULT_DB_VERSION_NUMBER: number = 20240220; export const DEFAULT_DB_NAME = "remotelysavedb"; export const DEFAULT_TBL_VERSION = "schemaversion"; -export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory"; -export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory"; export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory"; export const DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING = "vaultrandomidmapping"; export const DEFAULT_TBL_LOGGER_OUTPUT = "loggeroutput"; export const DEFAULT_TBL_SIMPLE_KV_FOR_MISC = "simplekvformisc"; +export const DEFAULT_TBL_PREV_SYNC_RECORDS = "prevsyncrecords"; -export interface FileFolderHistoryRecord { - key: string; - ctime: number; - mtime: number; - size: number; - actionWhen: number; - actionType: "delete" | "rename" | "renameDestination"; - keyType: "folder" | "file"; - renameTo: string; - vaultRandomID: string; -} +/** + * @deprecated + */ +export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory"; +/** + * @deprecated + */ +export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory"; +/** + * @deprecated + * But we cannot remove it. Because we want to migrate the old data. + */ interface SyncMetaMappingRecord { localKey: string; remoteKey: string; @@ -54,108 +55,74 @@ interface SyncPlanRecord { export interface InternalDBs { versionTbl: LocalForage; - fileHistoryTbl: LocalForage; - syncMappingTbl: LocalForage; syncPlansTbl: LocalForage; vaultRandomIDMappingTbl: LocalForage; loggerOutputTbl: LocalForage; simpleKVForMiscTbl: LocalForage; + prevSyncRecordsTbl: LocalForage; + + /** + * @deprecated + * But we cannot remove it. Because we want to migrate the old data. + */ + fileHistoryTbl: LocalForage; + + /** + * @deprecated + * But we cannot remove it. Because we want to migrate the old data. + */ + syncMappingTbl: LocalForage; } /** - * This migration mainly aims to assign vault name or vault id into all tables. - * @param db - * @param vaultRandomID + * TODO + * @param syncMappings + * @returns */ -const migrateDBsFrom20211114To20220108 = async ( - db: InternalDBs, - vaultRandomID: string -) => { - const oldVer = 20211114; - const newVer = 20220108; - log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`); - - const allPromisesToWait: Promise[] = []; - - log.debug("assign vault id to any delete history"); - const keysInDeleteHistoryTbl = await db.fileHistoryTbl.keys(); - for (const key of keysInDeleteHistoryTbl) { - if (key.startsWith(vaultRandomID)) { - continue; - } - const value = (await db.fileHistoryTbl.getItem( - key - )) as FileFolderHistoryRecord; - if (value === null || value === undefined) { - continue; - } - if (value.vaultRandomID === undefined || value.vaultRandomID === "") { - value.vaultRandomID = vaultRandomID; - } - const newKey = `${vaultRandomID}\t${key}`; - allPromisesToWait.push(db.fileHistoryTbl.setItem(newKey, value)); - allPromisesToWait.push(db.fileHistoryTbl.removeItem(key)); - } - - log.debug("assign vault id to any sync mapping"); - const keysInSyncMappingTbl = await db.syncMappingTbl.keys(); - for (const key of keysInSyncMappingTbl) { - if (key.startsWith(vaultRandomID)) { - continue; - } - const value = (await db.syncMappingTbl.getItem( - key - )) as SyncMetaMappingRecord; - if (value === null || value === undefined) { - continue; - } - if (value.vaultRandomID === undefined || value.vaultRandomID === "") { - value.vaultRandomID = vaultRandomID; - } - const newKey = `${vaultRandomID}\t${key}`; - allPromisesToWait.push(db.syncMappingTbl.setItem(newKey, value)); - allPromisesToWait.push(db.syncMappingTbl.removeItem(key)); - } - - log.debug("assign vault id to any sync plan records"); - const keysInSyncPlansTbl = await db.syncPlansTbl.keys(); - for (const key of keysInSyncPlansTbl) { - if (key.startsWith(vaultRandomID)) { - continue; - } - const value = (await db.syncPlansTbl.getItem(key)) as SyncPlanRecord; - if (value === null || value === undefined) { - continue; - } - if (value.vaultRandomID === undefined || value.vaultRandomID === "") { - value.vaultRandomID = vaultRandomID; - } - const newKey = `${vaultRandomID}\t${key}`; - allPromisesToWait.push(db.syncPlansTbl.setItem(newKey, value)); - allPromisesToWait.push(db.syncPlansTbl.removeItem(key)); - } - - log.debug("finally update version if everything is ok"); - await Promise.all(allPromisesToWait); - await db.versionTbl.setItem("version", newVer); - - log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); +const fromSyncMappingsToPrevSyncRecords = ( + syncMappings: SyncMetaMappingRecord[] +): Entity[] => { + return []; }; /** - * no need to do anything except changing version - * we just add more file operations in db, and no schema is changed. + * TODO * @param db * @param vaultRandomID + * @param prevSyncRecord */ -const migrateDBsFrom20220108To20220326 = async ( +const setPrevSyncRecordByVault = async ( + db: InternalDBs, + vaultRandomID: string, + prevSyncRecord: Entity +) => {}; + +/** + * + * @param db + * @param vaultRandomID + * Migrate the sync mapping record to sync Entity. + */ +const migrateDBsFrom20220326To20240220 = async ( db: InternalDBs, vaultRandomID: string ) => { - const oldVer = 20220108; - const newVer = 20220326; + const oldVer = 20220326; + const newVer = 20240220; log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`); - await db.versionTbl.setItem("version", newVer); + + // from sync mapping to prev sync + const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID); + const prevSyncRecords = fromSyncMappingsToPrevSyncRecords(syncMappings); + for (const prevSyncRecord of prevSyncRecords) { + await setPrevSyncRecordByVault(db, vaultRandomID, prevSyncRecord); + } + + // clear not used data + await clearFileHistoryOfEverythingByVault(db, vaultRandomID); + await clearAllSyncMetaMappingByVault(db, vaultRandomID); + + await db.versionTbl.setItem(`${vaultRandomID}\tversion`, newVer); log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); }; @@ -168,18 +135,19 @@ const migrateDBs = async ( if (oldVer === newVer) { return; } - if (oldVer === 20211114 && newVer === 20220108) { - return await migrateDBsFrom20211114To20220108(db, vaultRandomID); + + // as of 20240220, we assume everyone is using 20220326 already + // drop any old code to reduce the verbose + if (oldVer < 20220326) { + throw Error( + "You are using a very old version of Remotely Save. No way to auto update internal DB. Please install and enable 0.3.40 firstly, then install a later version." + ); } - if (oldVer === 20220108 && newVer === 20220326) { - return await migrateDBsFrom20220108To20220326(db, vaultRandomID); - } - if (oldVer === 20211114 && newVer === 20220326) { - // TODO: more steps with more versions in the future - await migrateDBsFrom20211114To20220108(db, vaultRandomID); - await migrateDBsFrom20220108To20220326(db, vaultRandomID); - return; + + if (oldVer === 20220326 && newVer === 20240220) { + return await migrateDBsFrom20220326To20240220(db, vaultRandomID); } + if (newVer < oldVer) { throw Error( "You've installed a new version, but then downgrade to an old version. Stop working!" @@ -198,14 +166,6 @@ export const prepareDBs = async ( name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_VERSION, }), - fileHistoryTbl: localforage.createInstance({ - name: DEFAULT_DB_NAME, - storeName: DEFAULT_TBL_FILE_HISTORY, - }), - syncMappingTbl: localforage.createInstance({ - name: DEFAULT_DB_NAME, - storeName: DEFAULT_TBL_SYNC_MAPPING, - }), syncPlansTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_SYNC_PLANS_HISTORY, @@ -222,6 +182,19 @@ export const prepareDBs = async ( name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_SIMPLE_KV_FOR_MISC, }), + prevSyncRecordsTbl: localforage.createInstance({ + name: DEFAULT_DB_NAME, + storeName: DEFAULT_TBL_PREV_SYNC_RECORDS, + }), + + fileHistoryTbl: localforage.createInstance({ + name: DEFAULT_DB_NAME, + storeName: DEFAULT_TBL_FILE_HISTORY, + }), + syncMappingTbl: localforage.createInstance({ + name: DEFAULT_DB_NAME, + storeName: DEFAULT_TBL_SYNC_MAPPING, + }), } as InternalDBs; // try to get vaultRandomID firstly @@ -253,12 +226,19 @@ export const prepareDBs = async ( throw Error("no vaultRandomID found or generated"); } - const originalVersion: number | null = await db.versionTbl.getItem("version"); + // as of 20240220, we set the version per vault, instead of global "version" + const originalVersion: number | null = + (await db.versionTbl.getItem(`${vaultRandomID}\tversion`)) ?? + (await db.versionTbl.getItem("version")); if (originalVersion === null) { log.debug( `no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}` ); - await db.versionTbl.setItem("version", DEFAULT_DB_VERSION_NUMBER); + // as of 20240220, we set the version per vault, instead of global "version" + await db.versionTbl.setItem( + `${vaultRandomID}\tversion`, + DEFAULT_DB_VERSION_NUMBER + ); } else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) { // do nothing } else { @@ -298,272 +278,47 @@ export const destroyDBs = async () => { }; }; -export const loadFileHistoryTableByVault = async ( +export const clearFileHistoryOfEverythingByVault = async ( db: InternalDBs, vaultRandomID: string ) => { - const records = [] as FileFolderHistoryRecord[]; - await db.fileHistoryTbl.iterate((value, key, iterationNumber) => { + const keys = await db.fileHistoryTbl.keys(); + for (const key of keys) { if (key.startsWith(`${vaultRandomID}\t`)) { - records.push(value as FileFolderHistoryRecord); + await db.fileHistoryTbl.removeItem(key); } - }); - records.sort((a, b) => a.actionWhen - b.actionWhen); // ascending - return records; -}; - -export const clearDeleteRenameHistoryOfKeyAndVault = async ( - db: InternalDBs, - key: string, - vaultRandomID: string -) => { - const fullKey = `${vaultRandomID}\t${key}`; - const item: FileFolderHistoryRecord | null = - await db.fileHistoryTbl.getItem(fullKey); - if ( - item !== null && - (item.actionType === "delete" || item.actionType === "rename") - ) { - await db.fileHistoryTbl.removeItem(fullKey); - } -}; - -export const insertDeleteRecordByVault = async ( - db: InternalDBs, - fileOrFolder: TAbstractFile | string, - vaultRandomID: string -) => { - // log.info(fileOrFolder); - let k: FileFolderHistoryRecord; - if (fileOrFolder instanceof TFile) { - k = { - key: fileOrFolder.path, - ctime: fileOrFolder.stat.ctime, - mtime: fileOrFolder.stat.mtime, - size: fileOrFolder.stat.size, - actionWhen: Date.now(), - actionType: "delete", - keyType: "file", - renameTo: "", - vaultRandomID: vaultRandomID, - }; - await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); - } else if (fileOrFolder instanceof TFolder) { - // key should endswith "/" - const key = fileOrFolder.path.endsWith("/") - ? fileOrFolder.path - : `${fileOrFolder.path}/`; - const ctime = 0; // they are deleted, so no way to get ctime, mtime - const mtime = 0; // they are deleted, so no way to get ctime, mtime - k = { - key: key, - ctime: ctime, - mtime: mtime, - size: 0, - actionWhen: Date.now(), - actionType: "delete", - keyType: "folder", - renameTo: "", - vaultRandomID: vaultRandomID, - }; - await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); - } else if (typeof fileOrFolder === "string") { - // always the deletions in .obsidian folder - // so annoying that the path doesn't exists - // and we have to guess whether the path is folder or file - k = { - key: fileOrFolder, - ctime: 0, - mtime: 0, - size: 0, - actionWhen: Date.now(), - actionType: "delete", - keyType: "file", - renameTo: "", - vaultRandomID: vaultRandomID, - }; - await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); - for (const ext of [ - "json", - "js", - "mjs", - "ts", - "md", - "txt", - "css", - "png", - "gif", - "jpg", - "jpeg", - "gitignore", - "gitkeep", - ]) { - if (fileOrFolder.endsWith(`.${ext}`)) { - // stop here, no more need to insert the folder record later - return; - } - } - // also add a deletion record as folder if not ending with special exts - k = { - key: `${fileOrFolder}/`, - ctime: 0, - mtime: 0, - size: 0, - actionWhen: Date.now(), - actionType: "delete", - keyType: "folder", - renameTo: "", - vaultRandomID: vaultRandomID, - }; - await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); } }; /** - * A file/folder is renamed from A to B - * We insert two records: - * A with actionType="rename" - * B with actionType="renameDestination" + * @deprecated But we cannot remove it. Because we want to migrate the old data. * @param db - * @param fileOrFolder - * @param oldPath * @param vaultRandomID + * @returns */ -export const insertRenameRecordByVault = async ( +export const getAllSyncMetaMappingByVault = async ( db: InternalDBs, - fileOrFolder: TAbstractFile, - oldPath: string, vaultRandomID: string ) => { - // log.info(fileOrFolder); - let k1: FileFolderHistoryRecord | undefined; - let k2: FileFolderHistoryRecord | undefined; - const actionWhen = Date.now(); - if (fileOrFolder instanceof TFile) { - k1 = { - key: oldPath, - ctime: fileOrFolder.stat.ctime, - mtime: fileOrFolder.stat.mtime, - size: fileOrFolder.stat.size, - actionWhen: actionWhen, - actionType: "rename", - keyType: "file", - renameTo: fileOrFolder.path, - vaultRandomID: vaultRandomID, - }; - k2 = { - key: fileOrFolder.path, - ctime: fileOrFolder.stat.ctime, - mtime: fileOrFolder.stat.mtime, - size: fileOrFolder.stat.size, - actionWhen: actionWhen, - actionType: "renameDestination", - keyType: "file", - renameTo: "", // itself is the destination, so no need to set this field - vaultRandomID: vaultRandomID, - }; - } else if (fileOrFolder instanceof TFolder) { - const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`; - const renameTo = fileOrFolder.path.endsWith("/") - ? fileOrFolder.path - : `${fileOrFolder.path}/`; - let ctime = 0; - let mtime = 0; - if (requireApiVersion(API_VER_STAT_FOLDER)) { - // TAbstractFile does not contain these info - // but from API_VER_STAT_FOLDER we can manually stat them by path. - const s = await statFix(fileOrFolder.vault, fileOrFolder.path); - if (s !== undefined && s !== null) { - ctime = s.ctime; - mtime = s.mtime; - } - } - k1 = { - key: key, - ctime: ctime, - mtime: mtime, - size: 0, - actionWhen: actionWhen, - actionType: "rename", - keyType: "folder", - renameTo: renameTo, - vaultRandomID: vaultRandomID, - }; - k2 = { - key: renameTo, - ctime: ctime, - mtime: mtime, - size: 0, - actionWhen: actionWhen, - actionType: "renameDestination", - keyType: "folder", - renameTo: "", // itself is the destination, so no need to set this field - vaultRandomID: vaultRandomID, - }; - } - await Promise.all([ - db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k1!.key}`, k1), - db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k2!.key}`, k2), - ]); -}; - -export const upsertSyncMetaMappingDataByVault = async ( - serviceType: SUPPORTED_SERVICES_TYPE, - db: InternalDBs, - localKey: string, - localMTime: number, - localSize: number, - remoteKey: string, - remoteMTime: number, - remoteSize: number, - remoteExtraKey: string, - vaultRandomID: string -) => { - const aggregratedInfo: SyncMetaMappingRecord = { - localKey: localKey, - localMtime: localMTime, - localSize: localSize, - remoteKey: remoteKey, - remoteMtime: remoteMTime, - remoteSize: remoteSize, - remoteExtraKey: remoteExtraKey, - remoteType: serviceType, - keyType: localKey.endsWith("/") ? "folder" : "file", - vaultRandomID: vaultRandomID, - }; - await db.syncMappingTbl.setItem( - `${vaultRandomID}\t${remoteKey}`, - aggregratedInfo + return await Promise.all( + ((await db.syncMappingTbl.keys()) ?? []) + .filter((key) => key.startsWith(`${vaultRandomID}\t`)) + .map( + async (key) => + (await db.syncMappingTbl.getItem(key)) as SyncMetaMappingRecord + ) ); }; -export const getSyncMetaMappingByRemoteKeyAndVault = async ( - serviceType: SUPPORTED_SERVICES_TYPE, +export const clearAllSyncMetaMappingByVault = async ( db: InternalDBs, - remoteKey: string, - remoteMTime: number, - remoteExtraKey: string, vaultRandomID: string ) => { - const potentialItem = (await db.syncMappingTbl.getItem( - `${vaultRandomID}\t${remoteKey}` - )) as SyncMetaMappingRecord; - - if (potentialItem === null) { - // no result was found - return undefined; - } - - if ( - potentialItem.remoteKey === remoteKey && - potentialItem.remoteMtime === remoteMTime && - potentialItem.remoteExtraKey === remoteExtraKey && - potentialItem.remoteType === serviceType - ) { - // the result was found - return potentialItem; - } else { - return undefined; + const keys = await db.syncMappingTbl.keys(); + for (const key of keys) { + if (key.startsWith(`${vaultRandomID}\t`)) { + await db.syncMappingTbl.removeItem(key); + } } }; @@ -651,6 +406,37 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => { await Promise.all(ps); }; +export const upsertPrevSyncRecordByVault = async ( + db: InternalDBs, + vaultRandomID: string, + prevSync: Entity +) => { + await db.prevSyncRecordsTbl.setItem( + `${vaultRandomID}-${prevSync.key}`, + prevSync + ); +}; + +export const clearPrevSyncRecordByVault = async ( + db: InternalDBs, + vaultRandomID: string, + key: string +) => { + await db.prevSyncRecordsTbl.removeItem(`${vaultRandomID}-${key}`); +}; + +export const clearAllPrevSyncRecordByVault = async ( + db: InternalDBs, + vaultRandomID: string +) => { + const keys = await db.prevSyncRecordsTbl.keys(); + for (const key of keys) { + if (key.startsWith(`${vaultRandomID}\t`)) { + await db.prevSyncRecordsTbl.removeItem(key); + } + } +}; + export const clearAllLoggerOutputRecords = async (db: InternalDBs) => { await db.loggerOutputTbl.clear(); log.debug(`successfully clearAllLoggerOutputRecords`); diff --git a/src/main.ts b/src/main.ts index ea7a96a..8cda974 100644 --- a/src/main.ts +++ b/src/main.ts @@ -240,8 +240,8 @@ export default class RemotelySavePlugin extends Plugin { this.app.vault.getName(), () => self.saveSettings() ); - const remoteRsp = await client.listAllFromRemote(); - // log.debug(remoteRsp); + const remoteEntityList = await client.listAllFromRemote(); + // log.debug(remoteEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -250,7 +250,7 @@ export default class RemotelySavePlugin extends Plugin { } this.syncStatus = "checking_password"; const passwordCheckResult = await isPasswordOk( - remoteRsp.Contents, + remoteEntityList, this.settings.password ); if (!passwordCheckResult.ok) { @@ -265,7 +265,7 @@ export default class RemotelySavePlugin extends Plugin { } this.syncStatus = "getting_remote_extra_meta"; const { remoteStates, metadataFile } = await parseRemoteItems( - remoteRsp.Contents, + remoteEntityList, this.db, this.vaultRandomID, client.serviceType, diff --git a/src/misc.ts b/src/misc.ts index 5b33ec4..69b0122 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -76,16 +76,17 @@ export const getFolderLevels = (x: string, addEndingSlash: boolean = false) => { export const mkdirpInVault = async (thePath: string, vault: Vault) => { // log.info(thePath); - const foldersToBuild = getFolderLevels(thePath); - // log.info(foldersToBuild); - for (const folder of foldersToBuild) { - const r = await vault.adapter.exists(folder); - // log.info(r); - if (!r) { - log.info(`mkdir ${folder}`); - await vault.adapter.mkdir(folder); - } + + // as of 2020219, + // Obsidian can create the folder recursively + // but the path should not end with '/' + if (thePath === "/" || thePath === "") { + return; } + let thePathNoEnding = thePath.endsWith("/") + ? thePath.slice(0, thePath.length - 1) + : thePath; + await vault.adapter.mkdir(thePathNoEnding); }; /** @@ -435,7 +436,10 @@ export const statFix = async (vault: Vault, path: string) => { return s; }; -export const isFolderToSkip = (x: string, more: string[] | undefined) => { +export const isSpecialFolderNameToSkip = ( + x: string, + more: string[] | undefined +) => { let specialFolders = [ ".git", ".github", diff --git a/src/obsFolderLister.ts b/src/obsFolderLister.ts index d1f9bdf..af88774 100644 --- a/src/obsFolderLister.ts +++ b/src/obsFolderLister.ts @@ -1,17 +1,11 @@ -import { Vault, Stat, ListedFiles } from "obsidian"; +import type { Vault, Stat, ListedFiles } from "obsidian"; +import type { Entity, MixedEntity } from "./baseTypes"; + import { Queue } from "@fyears/tsqueue"; import chunk from "lodash/chunk"; import flatten from "lodash/flatten"; import { statFix, isFolderToSkip } from "./misc"; -export interface ObsConfigDirFileType { - key: string; - ctime: number; - mtime: number; - size: number; - type: "folder" | "file"; -} - const isPluginDirItself = (x: string, pluginId: string) => { return ( x === pluginId || @@ -48,10 +42,10 @@ export const listFilesInObsFolder = async ( configDir: string, vault: Vault, pluginId: string -) => { +): Promise => { const q = new Queue([configDir]); const CHUNK_SIZE = 10; - const contents: ObsConfigDirFileType[] = []; + const contents: Entity[] = []; while (q.length > 0) { const itemsToFetch: string[] = []; while (q.length > 0) { @@ -72,11 +66,26 @@ export const listFilesInObsFolder = async ( children = await vault.adapter.list(x); } + if ( + !isFolder && + (statRes.mtime === undefined || + statRes.mtime === null || + statRes.mtime === 0) + ) { + throw Error( + `File in Obsidian ${configDir} has last modified time 0: ${x}, don't know how to deal with it.` + ); + } + return { itself: { key: isFolder ? `${x}/` : x, - ...statRes, - } as ObsConfigDirFileType, + keyEnc: isFolder ? `${x}/` : x, + mtimeCli: statRes.mtime, + mtimeSvr: statRes.mtime, + size: statRes.size, + sizeEnc: statRes.size, + }, children: children, }; }); diff --git a/src/remote.ts b/src/remote.ts index 6a26fe1..68e5570 100644 --- a/src/remote.ts +++ b/src/remote.ts @@ -1,5 +1,6 @@ import { Vault } from "obsidian"; import type { + Entity, DropboxConfig, OnedriveConfig, S3Config, @@ -164,7 +165,7 @@ export class RemoteClient { } }; - listAllFromRemote = async () => { + listAllFromRemote = async (): Promise => { if (this.serviceType === "s3") { return await s3.listAllFromRemote( s3.getS3Client(this.s3Config!), diff --git a/src/remoteForDropbox.ts b/src/remoteForDropbox.ts index c4ee477..18d4cde 100644 --- a/src/remoteForDropbox.ts +++ b/src/remoteForDropbox.ts @@ -5,7 +5,7 @@ import { Vault } from "obsidian"; import * as path from "path"; import { DropboxConfig, - RemoteItem, + Entity, COMMAND_CALLBACK_DROPBOX, OAUTH2_FORCE_EXPIRE_MILLISECONDS, } from "./baseTypes"; @@ -69,13 +69,13 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => { return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length); }; -const fromDropboxItemToRemoteItem = ( +const fromDropboxItemToEntity = ( x: | files.FileMetadataReference | files.FolderMetadataReference | files.DeletedMetadataReference, remoteBaseDir: string -): RemoteItem => { +): Entity => { let key = getNormPath(x.path_display!, remoteBaseDir); if (x[".tag"] === "folder" && !key.endsWith("/")) { key = `${key}/`; @@ -84,93 +84,30 @@ const fromDropboxItemToRemoteItem = ( if (x[".tag"] === "folder") { return { key: key, - lastModified: undefined, + keyEnc: key, size: 0, - remoteType: "dropbox", + sizeEnc: 0, etag: `${x.id}\t`, - } as RemoteItem; + } as Entity; } else if (x[".tag"] === "file") { - let mtime = Date.parse(x.client_modified).valueOf(); - if (mtime === 0) { - mtime = Date.parse(x.server_modified).valueOf(); - } + const mtimeCli = Date.parse(x.client_modified).valueOf(); + const mtimeSvr = Date.parse(x.server_modified).valueOf(); return { key: key, - lastModified: mtime, + keyEnc: key, + mtimeCli: mtimeCli, + mtimeSvr: mtimeSvr, size: x.size, - remoteType: "dropbox", + sizeEnc: x.size, + hash: x.content_hash, etag: `${x.id}\t${x.content_hash}`, - } as RemoteItem; + } as Entity; } else { // x[".tag"] === "deleted" throw Error("do not support deleted tag"); } }; -/** - * Dropbox api doesn't return mtime for folders. - * This is a try to assign mtime by using files in folder. - * @param allFilesFolders - * @returns - */ -const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => { - if (allFilesFolders.length === 0) { - return; - } - - // sort by longer to shorter - allFilesFolders.sort((a, b) => b.key.length - a.key.length); - - // a "map" from dir to mtime - let potentialMTime = {} as Record; - - // first sort pass, from buttom to up - for (const item of allFilesFolders) { - if (item.key.endsWith("/")) { - // itself is a folder, and initially doesn't have mtime - if (item.lastModified === undefined && item.key in potentialMTime) { - // previously we gathered all sub info of this folder - item.lastModified = potentialMTime[item.key]; - } - } - const parent = `${path.posix.dirname(item.key)}/`; - if (item.lastModified !== undefined) { - if (parent in potentialMTime) { - potentialMTime[parent] = Math.max( - potentialMTime[parent], - item.lastModified - ); - } else { - potentialMTime[parent] = item.lastModified; - } - } - } - - // second pass, from up to buttom. - // fill mtime by parent folder or Date.Now() if still not available. - // this is only possible if no any sub-folder-files recursively. - // we do not sort the array again, just iterate over it by reverse - // using good old for loop. - for (let i = allFilesFolders.length - 1; i >= 0; --i) { - const item = allFilesFolders[i]; - if (!item.key.endsWith("/")) { - continue; // skip files - } - if (item.lastModified !== undefined) { - continue; // don't need to deal with it - } - const parent = `${path.posix.dirname(item.key)}/`; - if (parent in potentialMTime) { - item.lastModified = potentialMTime[parent]; - } else { - item.lastModified = Date.now().valueOf(); - potentialMTime[item.key] = item.lastModified; - } - } - - return allFilesFolders; -}; - //////////////////////////////////////////////////////////////////////////////// // Dropbox authorization using PKCE // see https://dropbox.tech/developers/pkce--what-and-why- @@ -498,7 +435,7 @@ export const getRemoteMeta = async ( // size: 0, // remoteType: "dropbox", // etag: undefined, - // } as RemoteItem; + // } as Entity; // } const rsp = await retryReq(() => @@ -512,7 +449,7 @@ export const getRemoteMeta = async ( if (rsp.status !== 200) { throw Error(JSON.stringify(rsp)); } - return fromDropboxItemToRemoteItem(rsp.result, client.remoteBaseDir); + return fromDropboxItemToEntity(rsp.result, client.remoteBaseDir); }; export const uploadToRemote = async ( @@ -670,7 +607,7 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => { const unifiedContents = contents .filter((x) => x[".tag"] !== "deleted") .filter((x) => x.path_display !== `/${client.remoteBaseDir}`) - .map((x) => fromDropboxItemToRemoteItem(x, client.remoteBaseDir)); + .map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir)); while (res.result.has_more) { res = await client.dropbox.filesListFolderContinue({ @@ -684,15 +621,11 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => { const unifiedContents2 = contents2 .filter((x) => x[".tag"] !== "deleted") .filter((x) => x.path_display !== `/${client.remoteBaseDir}`) - .map((x) => fromDropboxItemToRemoteItem(x, client.remoteBaseDir)); + .map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir)); unifiedContents.push(...unifiedContents2); } - fixLastModifiedTimeInplace(unifiedContents); - - return { - Contents: unifiedContents, - }; + return unifiedContents; }; const downloadFromRemoteRaw = async ( diff --git a/src/remoteForOnedrive.ts b/src/remoteForOnedrive.ts index 9d355bd..ec2759d 100644 --- a/src/remoteForOnedrive.ts +++ b/src/remoteForOnedrive.ts @@ -14,7 +14,7 @@ import { DEFAULT_CONTENT_TYPE, OAUTH2_FORCE_EXPIRE_MILLISECONDS, OnedriveConfig, - RemoteItem, + Entity, } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { @@ -255,16 +255,13 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => { return fileOrFolderPath.slice(`${prefix}/`.length); }; -const constructFromDriveItemToRemoteItemError = (x: DriveItem) => { +const constructFromDriveItemToEntityError = (x: DriveItem) => { return `parentPath="${ x.parentReference?.path ?? "(no parentReference or path)" }", selfName="${x.name}"`; }; -const fromDriveItemToRemoteItem = ( - x: DriveItem, - remoteBaseDir: string -): RemoteItem => { +const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => { let key = ""; // possible prefix: @@ -333,14 +330,14 @@ const fromDriveItemToRemoteItem = ( key = x.name; } else { throw Error( - `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToRemoteItemError( + `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToEntityError( x )}` ); } } else { throw Error( - `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToRemoteItemError( + `we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToEntityError( x )}` ); @@ -350,11 +347,17 @@ const fromDriveItemToRemoteItem = ( if (isFolder) { key = `${key}/`; } + + const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!); + const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!); return { key: key, - lastModified: Date.parse(x!.fileSystemInfo!.lastModifiedDateTime!), + keyEnc: key, + mtimeSvr: mtimeSvr, + mtimeCli: mtimeCli, size: isFolder ? 0 : x.size!, - remoteType: "onedrive", + sizeEnc: isFolder ? 0 : x.size!, + // hash: ?? // TODO etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes }; }; @@ -666,14 +669,12 @@ export const listAllFromRemote = async (client: WrappedOnedriveClient) => { await client.saveUpdatedConfigFunc(); } - // unify everything to RemoteItem + // unify everything to Entity const unifiedContents = driveItems - .map((x) => fromDriveItemToRemoteItem(x, client.remoteBaseDir)) + .map((x) => fromDriveItemToEntity(x, client.remoteBaseDir)) .filter((x) => x.key !== "/"); - return { - Contents: unifiedContents, - }; + return unifiedContents; }; export const getRemoteMeta = async ( @@ -687,7 +688,7 @@ export const getRemoteMeta = async ( ); // log.info(rsp); const driveItem = rsp as DriveItem; - const res = fromDriveItemToRemoteItem(driveItem, client.remoteBaseDir); + const res = fromDriveItemToEntity(driveItem, client.remoteBaseDir); // log.info(res); return res; }; diff --git a/src/remoteForS3.ts b/src/remoteForS3.ts index 086d22d..00924da 100644 --- a/src/remoteForS3.ts +++ b/src/remoteForS3.ts @@ -28,7 +28,7 @@ import * as path from "path"; import AggregateError from "aggregate-error"; import { DEFAULT_CONTENT_TYPE, - RemoteItem, + Entity, S3Config, VALID_REQURL, } from "./baseTypes"; @@ -220,51 +220,56 @@ const getLocalNoPrefixPath = ( return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length); }; -const fromS3ObjectToRemoteItem = ( +const fromS3ObjectToEntity = ( x: S3ObjectType, remotePrefix: string, mtimeRecords: Record, ctimeRecords: Record ) => { - let mtime = x.LastModified!.valueOf(); + const mtimeSvr = x.LastModified!.valueOf(); + let mtimeCli = mtimeSvr; if (x.Key! in mtimeRecords) { const m2 = mtimeRecords[x.Key!]; if (m2 !== 0) { - mtime = m2; + mtimeCli = m2; } } - const r: RemoteItem = { - key: getLocalNoPrefixPath(x.Key!, remotePrefix), - lastModified: mtime, + const key = getLocalNoPrefixPath(x.Key!, remotePrefix); + const r: Entity = { + key: key, + keyEnc: key, + mtimeSvr: mtimeSvr, + mtimeCli: mtimeCli, size: x.Size!, - remoteType: "s3", + sizeEnc: x.Size!, etag: x.ETag, }; return r; }; -const fromS3HeadObjectToRemoteItem = ( +const fromS3HeadObjectToEntity = ( fileOrFolderPathWithRemotePrefix: string, x: HeadObjectCommandOutput, remotePrefix: string, useAccurateMTime: boolean ) => { - let mtime = x.LastModified!.valueOf(); + const mtimeSvr = x.LastModified!.valueOf(); + let mtimeCli = mtimeSvr; if (useAccurateMTime && x.Metadata !== undefined) { const m2 = Math.round( parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0") ); if (m2 !== 0) { - mtime = m2; + mtimeCli = m2; } } return { key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix), - lastModified: mtime, + mtimeSvr: mtimeSvr, + mtimeCli: mtimeCli, size: x.ContentLength, - remoteType: "s3", etag: x.ETag, - } as RemoteItem; + } as Entity; }; export const getS3Client = (s3Config: S3Config) => { @@ -330,7 +335,7 @@ export const getRemoteMeta = async ( }) ); - return fromS3HeadObjectToRemoteItem( + return fromS3HeadObjectToEntity( fileOrFolderPathWithRemotePrefix, res, s3Config.remotePrefix ?? "", @@ -538,16 +543,14 @@ const listFromRemoteRaw = async ( // ensemble fake rsp // in the end, we need to transform the response list // back to the local contents-alike list - return { - Contents: contents.map((x) => - fromS3ObjectToRemoteItem( - x, - s3Config.remotePrefix ?? "", - mtimeRecords, - ctimeRecords - ) - ), - }; + return contents.map((x) => + fromS3ObjectToEntity( + x, + s3Config.remotePrefix ?? "", + mtimeRecords, + ctimeRecords + ) + ); }; export const listAllFromRemote = async ( @@ -692,7 +695,7 @@ export const deleteFromRemote = async ( if (fileOrFolderPath.endsWith("/") && password === "") { const x = await listFromRemoteRaw(s3Client, s3Config, remoteFileName); - x.Contents.forEach(async (element) => { + x.forEach(async (element) => { await s3Client.send( new DeleteObjectCommand({ Bucket: s3Config.s3BucketName, diff --git a/src/remoteForWebdav.ts b/src/remoteForWebdav.ts index 27318cd..fe44443 100644 --- a/src/remoteForWebdav.ts +++ b/src/remoteForWebdav.ts @@ -5,7 +5,7 @@ import { Queue } from "@fyears/tsqueue"; import chunk from "lodash/chunk"; import flatten from "lodash/flatten"; import { getReasonPhrase } from "http-status-codes"; -import { RemoteItem, VALID_REQURL, WebdavConfig } from "./baseTypes"; +import { Entity, VALID_REQURL, WebdavConfig } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc"; @@ -205,18 +205,21 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => { return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length); }; -const fromWebdavItemToRemoteItem = (x: FileStat, remoteBaseDir: string) => { +const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string) => { let key = getNormPath(x.filename, remoteBaseDir); if (x.type === "directory" && !key.endsWith("/")) { key = `${key}/`; } + const mtimeSvr = Date.parse(x.lastmod).valueOf(); return { key: key, - lastModified: Date.parse(x.lastmod).valueOf(), + keyEnc: key, + mtimeSvr: mtimeSvr, + mtimeCli: mtimeSvr, // no universal way to set mtime in webdav size: x.size, - remoteType: "webdav", - etag: x.etag || undefined, - } as RemoteItem; + sizeEnc: x.size, + etag: x.etag, + } as Entity; }; export class WrappedWebdavClient { @@ -327,7 +330,7 @@ export const getRemoteMeta = async ( details: false, })) as FileStat; log.debug(`getRemoteMeta res=${JSON.stringify(res)}`); - return fromWebdavItemToRemoteItem(res, client.remoteBaseDir); + return fromWebdavItemToEntity(res, client.remoteBaseDir); }; export const uploadToRemote = async ( @@ -359,7 +362,7 @@ export const uploadToRemote = async ( if (password === "") { // if not encrypted, mkdir a remote folder await client.client.createDirectory(uploadFile, { - recursive: false, // the sync algo should guarantee no need to recursive + recursive: true, }); const res = await getRemoteMeta(client, uploadFile); return res; @@ -400,7 +403,7 @@ export const uploadToRemote = async ( // // we need to create folders before uploading // const dir = getPathFolder(uploadFile); // if (dir !== "/" && dir !== "") { - // await client.client.createDirectory(dir, { recursive: false }); + // await client.client.createDirectory(dir, { recursive: true }); // } await client.client.putFileContents(uploadFile, remoteContent, { overwrite: true, @@ -472,11 +475,7 @@ export const listAllFromRemote = async (client: WrappedWebdavClient) => { } )) as FileStat[]; } - return { - Contents: contents.map((x) => - fromWebdavItemToRemoteItem(x, client.remoteBaseDir) - ), - }; + return contents.map((x) => fromWebdavItemToEntity(x, client.remoteBaseDir)); }; const downloadFromRemoteRaw = async ( diff --git a/src/sync.ts b/src/sync.ts index 1e2f8a1..4f3125a 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -1,79 +1,43 @@ -import { - TAbstractFile, - TFile, - TFolder, - Vault, - requireApiVersion, -} from "obsidian"; -import AggregateError from "aggregate-error"; import PQueue from "p-queue"; import XRegExp from "xregexp"; import type { - RemoteItem, - SyncTriggerSourceType, - DecisionType, - FileOrFolderMixedState, - SUPPORTED_SERVICES_TYPE, + ConflictActionType, + EmptyFolderCleanType, + Entity, + MixedEntity, } from "./baseTypes"; -import { API_VER_STAT_FOLDER } from "./baseTypes"; +import { isInsideObsFolder } from "./obsFolderLister"; import { + isSpecialFolderNameToSkip, + isHiddenPath, + unixTimeToStr, + getParentFolder, + isVaildText, + atWhichLevel, + mkdirpInVault, +} from "./misc"; +import { + DEFAULT_FILE_NAME_FOR_METADATAONREMOTE, + DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2, +} from "./metadataOnRemote"; +import { + MAGIC_ENCRYPTED_PREFIX_BASE32, + MAGIC_ENCRYPTED_PREFIX_BASE64URL, decryptBase32ToString, decryptBase64urlToString, encryptStringToBase64url, getSizeFromOrigToEnc, - MAGIC_ENCRYPTED_PREFIX_BASE32, - MAGIC_ENCRYPTED_PREFIX_BASE64URL, } from "./encrypt"; -import type { FileFolderHistoryRecord, InternalDBs } from "./localdb"; -import { - clearDeleteRenameHistoryOfKeyAndVault, - getSyncMetaMappingByRemoteKeyAndVault, - upsertSyncMetaMappingDataByVault, -} from "./localdb"; -import { - isHiddenPath, - isVaildText, - mkdirpInVault, - getFolderLevels, - getParentFolder, - atWhichLevel, - unixTimeToStr, - statFix, - isFolderToSkip, -} from "./misc"; import { RemoteClient } from "./remote"; -import { - MetadataOnRemote, - DeletionOnRemote, - serializeMetadataOnRemote, - deserializeMetadataOnRemote, - DEFAULT_FILE_NAME_FOR_METADATAONREMOTE, - DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2, - isEqualMetadataOnRemote, -} from "./metadataOnRemote"; -import { isInsideObsFolder, ObsConfigDirFileType } from "./obsFolderLister"; +import { Vault } from "obsidian"; import { log } from "./moreOnLog"; - -export type SyncStatusType = - | "idle" - | "preparing" - | "getting_remote_files_list" - | "getting_remote_extra_meta" - | "getting_local_meta" - | "checking_password" - | "generating_plan" - | "syncing" - | "cleaning" - | "finish"; - -export interface SyncPlanType { - ts: number; - tsFmt?: string; - syncTriggerSource?: SyncTriggerSourceType; - remoteType: SUPPORTED_SERVICES_TYPE; - mixedStates: Record; -} +import AggregateError from "aggregate-error"; +import { + InternalDBs, + clearPrevSyncRecordByVault, + upsertPrevSyncRecordByVault, +} from "./localdb"; export interface PasswordCheckType { ok: boolean; @@ -89,15 +53,15 @@ export interface PasswordCheckType { } export const isPasswordOk = async ( - remote: RemoteItem[], + remote: Entity[], password: string = "" -) => { +): Promise => { if (remote === undefined || remote.length === 0) { // remote empty return { ok: true, reason: "empty_remote", - } as PasswordCheckType; + }; } const santyCheckKey = remote[0].key; if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { @@ -107,7 +71,7 @@ export const isPasswordOk = async ( return { ok: false, reason: "remote_encrypted_local_no_password", - } as PasswordCheckType; + }; } try { const res = await decryptBase32ToString(santyCheckKey, password); @@ -118,18 +82,18 @@ export const isPasswordOk = async ( return { ok: true, reason: "password_matched", - } as PasswordCheckType; + }; } else { return { ok: false, reason: "invalid_text_after_decryption", - } as PasswordCheckType; + }; } } catch (error) { return { ok: false, reason: "password_not_matched", - } as PasswordCheckType; + }; } } if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) { @@ -139,7 +103,7 @@ export const isPasswordOk = async ( return { ok: false, reason: "remote_encrypted_local_no_password", - } as PasswordCheckType; + }; } try { const res = await decryptBase64urlToString(santyCheckKey, password); @@ -150,18 +114,18 @@ export const isPasswordOk = async ( return { ok: true, reason: "password_matched", - } as PasswordCheckType; + }; } else { return { ok: false, reason: "invalid_text_after_decryption", - } as PasswordCheckType; + }; } } catch (error) { return { ok: false, reason: "password_not_matched", - } as PasswordCheckType; + }; } } else { // it is not encrypted! @@ -169,130 +133,16 @@ export const isPasswordOk = async ( return { ok: false, reason: "remote_not_encrypted_local_has_password", - } as PasswordCheckType; + }; } return { ok: true, reason: "no_password_both_sides", - } as PasswordCheckType; - } -}; - -export const parseRemoteItems = async ( - remote: RemoteItem[], - db: InternalDBs, - vaultRandomID: string, - remoteType: SUPPORTED_SERVICES_TYPE, - password: string = "" -) => { - const remoteStates = [] as FileOrFolderMixedState[]; - let metadataFile: FileOrFolderMixedState | undefined = undefined; - if (remote === undefined) { - return { - remoteStates: remoteStates, - metadataFile: metadataFile, }; } - - for (const entry of remote) { - const remoteEncryptedKey = entry.key; - let key = remoteEncryptedKey; - if (password !== "") { - if (remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { - key = await decryptBase32ToString(remoteEncryptedKey, password); - } else if ( - remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL) - ) { - key = await decryptBase64urlToString(remoteEncryptedKey, password); - } else { - throw Error(`unexpected key=${remoteEncryptedKey}`); - } - } - const backwardMapping = await getSyncMetaMappingByRemoteKeyAndVault( - remoteType, - db, - key, - entry.lastModified ?? Date.now(), - entry.etag ?? "", - vaultRandomID - ); - - let r = {} as FileOrFolderMixedState; - if (backwardMapping !== undefined) { - // log.debug(`backwardMapping=${backwardMapping}`); - key = backwardMapping.localKey; - const mtimeRemote = backwardMapping.localMtime || entry.lastModified; - - // the backwardMapping.localSize is the file BEFORE encryption - // we want to split two sizes for comparation later - - r = { - key: key, - existRemote: true, - mtimeRemote: mtimeRemote, - mtimeRemoteFmt: unixTimeToStr(mtimeRemote), - sizeRemote: backwardMapping.localSize, - sizeRemoteEnc: password === "" ? undefined : entry.size, - remoteEncryptedKey: remoteEncryptedKey, - changeRemoteMtimeUsingMapping: true, - }; - } else { - // log.debug(`do not have backwardMapping`); - r = { - key: key, - existRemote: true, - mtimeRemote: entry.lastModified, - mtimeRemoteFmt: unixTimeToStr(entry.lastModified), - sizeRemote: password === "" ? entry.size : undefined, - sizeRemoteEnc: password === "" ? undefined : entry.size, - remoteEncryptedKey: remoteEncryptedKey, - changeRemoteMtimeUsingMapping: false, - }; - } - - if (r.key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE) { - metadataFile = Object.assign({}, r); - } - if (r.key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2) { - throw Error( - `A reserved file name ${r.key} has been found. You may upgrade the plugin to latest version to try to deal with it.` - ); - } - - remoteStates.push(r); - } - return { - remoteStates: remoteStates, - metadataFile: metadataFile, - }; }; -export const fetchMetadataFile = async ( - metadataFile: FileOrFolderMixedState | undefined, - client: RemoteClient, - vault: Vault, - password: string = "" -) => { - if (metadataFile === undefined) { - log.debug("no metadata file, so no fetch"); - return { - deletions: [], - } as MetadataOnRemote; - } - - const buf = await client.downloadFromRemote( - metadataFile.key, - vault, - metadataFile.mtimeRemote ?? Date.now(), - password, - metadataFile.remoteEncryptedKey, - true - ); - const metadata = deserializeMetadataOnRemote(buf); - return metadata; -}; - -const isSkipItem = ( +const isSkipItemByName = ( key: string, syncConfigDir: boolean, syncUnderscoreItems: boolean, @@ -309,7 +159,7 @@ const isSkipItem = ( if (syncConfigDir && isInsideObsFolder(key, configDir)) { return false; } - if (isFolderToSkip(key, [])) { + if (isSpecialFolderNameToSkip(key, [])) { // some special dirs and files are always skipped return true; } @@ -321,72 +171,146 @@ const isSkipItem = ( ); }; -const ensembleMixedStates = async ( - remoteStates: FileOrFolderMixedState[], - local: TAbstractFile[], - localConfigDirContents: ObsConfigDirFileType[] | undefined, - remoteDeleteHistory: DeletionOnRemote[] | undefined, - localFileHistory: FileFolderHistoryRecord[] | undefined, +const copyEntityAndFixTimeFormat = (src: Entity) => { + const result = Object.assign({}, src); + if (result.mtimeCli !== undefined) { + if (result.mtimeCli === 0) { + result.mtimeCli = undefined; + } else { + result.mtimeCliFmt = unixTimeToStr(result.mtimeCli); + } + } + if (result.mtimeSvr !== undefined) { + if (result.mtimeSvr === 0) { + result.mtimeSvr = undefined; + } else { + result.mtimeSvrFmt = unixTimeToStr(result.mtimeSvr); + } + } + if (result.prevSyncTime !== undefined) { + if (result.prevSyncTime === 0) { + result.prevSyncTime = undefined; + } else { + result.prevSyncTimeFmt = unixTimeToStr(result.prevSyncTime); + } + } + + return result; +}; + +/** + * Inplace, no copy again. + * @param remote + * @param password + * @returns + */ +const decryptRemoteEntityInplace = async (remote: Entity, password: string) => { + if (password == undefined || password === "") { + remote.key = remote.keyEnc; + remote.size = remote.sizeEnc; + return remote; + } + + if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { + remote.key = await decryptBase32ToString(remote.keyEnc, password); + } else if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) { + remote.key = await decryptBase64urlToString(remote.keyEnc, password); + } else { + throw Error(`unexpected key to decrypt=${remote.keyEnc}`); + } + + // TODO + // remote.size = getSizeFromEncToOrig(remote.sizeEnc, password); + // but we don't have deterministic way to get a number because the encryption has padding... + + return remote; +}; + +/** + * Directly throw error here. + * We can only defer the checking now, because before decryption we don't know whether it's a file or folder. + * @param remote + */ +const ensureMTimeOfRemoteEntityValid = (remote: Entity) => { + if ( + !remote.key.endsWith("/") && + remote.mtimeCli === undefined && + remote.mtimeSvr === undefined + ) { + if (remote.key === remote.keyEnc) { + throw Error( + `Your remote file ${remote.key} has last modified time 0, don't know how to deal with it.` + ); + } else { + throw Error( + `Your remote file ${remote.key} (encrypted as ${remote.keyEnc}) has last modified time 0, don't know how to deal with it.` + ); + } + } + return remote; +}; + +/** + * Inplace, no copy again. + * @param local + * @param password + * @returns + */ +const encryptLocalEntityInplace = async ( + local: Entity, + password: string, + remoteKeyEnc: string | undefined +) => { + if (password == undefined || password === "") { + return local; + } + if (local.size === local.sizeEnc) { + local.sizeEnc = getSizeFromOrigToEnc(local.size); + } + if (local.key === local.keyEnc) { + if ( + remoteKeyEnc !== undefined && + remoteKeyEnc !== "" && + remoteKeyEnc !== local.key + ) { + // we can reuse remote encrypted key if any + local.keyEnc = remoteKeyEnc; + } else { + // we assign a new encrypted key because of no remote + // the old version uses base32 + // local.keyEnc = await encryptStringToBase32(local.key, password); + // the new version users base64url + local.keyEnc = await encryptStringToBase64url(local.key, password); + } + } + return local; +}; + +const ensembleMixedEnties = async ( + localEntityList: Entity[], + prevSyncEntityList: Entity[], + remoteEntityList: Entity[], + syncConfigDir: boolean, configDir: string, syncUnderscoreItems: boolean, ignorePaths: string[], password: string -) => { - const results = {} as Record; +): Promise> => { + const finalMappings: Record = {}; - for (const r of remoteStates) { - const key = r.key; - - if ( - isSkipItem( - key, - syncConfigDir, - syncUnderscoreItems, - configDir, - ignorePaths + // remote has to be first + for (const remote of remoteEntityList) { + const remoteCopied = ensureMTimeOfRemoteEntityValid( + await decryptRemoteEntityInplace( + copyEntityAndFixTimeFormat(remote), + password ) - ) { - continue; - } - results[key] = r; - results[key].existLocal = false; - } - - for (const entry of local) { - let r = {} as FileOrFolderMixedState; - let key = entry.path; - - if (entry.path === "/") { - // ignore - continue; - } else if (entry instanceof TFile) { - const mtimeLocal = Math.max(entry.stat.mtime ?? 0, entry.stat.ctime ?? 0); - r = { - key: entry.path, - existLocal: true, - mtimeLocal: mtimeLocal, - mtimeLocalFmt: unixTimeToStr(mtimeLocal), - sizeLocal: entry.stat.size, - sizeLocalEnc: - password === "" ? undefined : getSizeFromOrigToEnc(entry.stat.size), - }; - } else if (entry instanceof TFolder) { - key = `${entry.path}/`; - r = { - key: key, - existLocal: true, - mtimeLocal: undefined, - mtimeLocalFmt: undefined, - sizeLocal: 0, - sizeLocalEnc: password === "" ? undefined : getSizeFromOrigToEnc(0), - }; - } else { - throw Error(`unexpected ${entry}`); - } + ); + const key = remoteCopied.key; if ( - isSkipItem( + isSkipItemByName( key, syncConfigDir, syncUnderscoreItems, @@ -397,111 +321,16 @@ const ensembleMixedStates = async ( continue; } - if (results.hasOwnProperty(key)) { - results[key].key = r.key; - results[key].existLocal = r.existLocal; - results[key].mtimeLocal = r.mtimeLocal; - results[key].mtimeLocalFmt = r.mtimeLocalFmt; - results[key].sizeLocal = r.sizeLocal; - results[key].sizeLocalEnc = r.sizeLocalEnc; - } else { - results[key] = r; - results[key].existRemote = false; - } - } - - if (syncConfigDir && localConfigDirContents !== undefined) { - for (const entry of localConfigDirContents) { - const key = entry.key; - let mtimeLocal: number | undefined = Math.max( - entry.mtime ?? 0, - entry.ctime ?? 0 - ); - if (Number.isNaN(mtimeLocal) || mtimeLocal === 0) { - mtimeLocal = undefined; - } - const r: FileOrFolderMixedState = { - key: key, - existLocal: true, - mtimeLocal: mtimeLocal, - mtimeLocalFmt: unixTimeToStr(mtimeLocal), - sizeLocal: entry.size, - sizeLocalEnc: - password === "" ? undefined : getSizeFromOrigToEnc(entry.size), - }; - - if ( - isSkipItem( - key, - syncConfigDir, - syncUnderscoreItems, - configDir, - ignorePaths - ) - ) { - continue; - } - - if (results.hasOwnProperty(key)) { - results[key].key = r.key; - results[key].existLocal = r.existLocal; - results[key].mtimeLocal = r.mtimeLocal; - results[key].mtimeLocalFmt = r.mtimeLocalFmt; - results[key].sizeLocal = r.sizeLocal; - results[key].sizeLocalEnc = r.sizeLocalEnc; - } else { - results[key] = r; - results[key].existRemote = false; - } - } - } - - for (const entry of remoteDeleteHistory ?? []) { - const key = entry.key; - const r = { + finalMappings[key] = { key: key, - deltimeRemote: entry.actionWhen, - deltimeRemoteFmt: unixTimeToStr(entry.actionWhen), - } as FileOrFolderMixedState; - - if ( - isSkipItem( - key, - syncConfigDir, - syncUnderscoreItems, - configDir, - ignorePaths - ) - ) { - continue; - } - - if (results.hasOwnProperty(key)) { - results[key].key = r.key; - results[key].deltimeRemote = r.deltimeRemote; - results[key].deltimeRemoteFmt = r.deltimeRemoteFmt; - } else { - results[key] = r; - - results[key].existLocal = false; - results[key].existRemote = false; - } + remote: remoteCopied, + }; } - for (const entry of localFileHistory ?? []) { - let key = entry.key; - if (entry.keyType === "folder") { - if (!entry.key.endsWith("/")) { - key = `${entry.key}/`; - } - } else if (entry.keyType === "file") { - // pass - } else { - throw Error(`unexpected ${entry}`); - } - + for (const prevSync of prevSyncEntityList) { + const key = prevSync.key; if ( - isSkipItem( + isSkipItemByName( key, syncConfigDir, syncUnderscoreItems, @@ -512,836 +341,382 @@ const ensembleMixedStates = async ( continue; } - if (entry.actionType === "delete" || entry.actionType === "rename") { - const r = { + if (finalMappings.hasOwnProperty(key)) { + const prevSyncCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(prevSync), + password, + finalMappings[key].remote?.keyEnc + ); + finalMappings[key].prevSync = prevSyncCopied; + } else { + const prevSyncCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(prevSync), + password, + undefined + ); + finalMappings[key] = { key: key, - deltimeLocal: entry.actionWhen, - deltimeLocalFmt: unixTimeToStr(entry.actionWhen), - } as FileOrFolderMixedState; - - if (results.hasOwnProperty(key)) { - results[key].deltimeLocal = r.deltimeLocal; - results[key].deltimeLocalFmt = r.deltimeLocalFmt; - } else { - results[key] = r; - results[key].existLocal = false; // we have already checked local - results[key].existRemote = false; // we have already checked remote - } - } else if (entry.actionType === "renameDestination") { - const r = { - key: key, - mtimeLocal: entry.actionWhen, - mtimeLocalFmt: unixTimeToStr(entry.actionWhen), - changeLocalMtimeUsingMapping: true, + prevSync: prevSyncCopied, }; - if (results.hasOwnProperty(key)) { - let mtimeLocal: number | undefined = Math.max( - r.mtimeLocal ?? 0, - results[key].mtimeLocal ?? 0 - ); - if (Number.isNaN(mtimeLocal) || mtimeLocal === 0) { - mtimeLocal = undefined; - } - results[key].mtimeLocal = mtimeLocal; - results[key].mtimeLocalFmt = unixTimeToStr(mtimeLocal); - results[key].changeLocalMtimeUsingMapping = - r.changeLocalMtimeUsingMapping; - } else { - // So, the file doesn't exist, - // except that it existed in the "renamed to" history records. - // Most likely because that the user deleted the file while Obsidian was closed, - // so Obsidian could not track the deletions. - // We are not sure how to deal with this, so do not generate anything here! - // // // The following 3 lines are of old logic, and have been removed: - // // results[key] = r; - // // results[key].existLocal = false; // we have already checked local - // // results[key].existRemote = false; // we have already checked remote - } - } else { - throw Error( - `do not know how to deal with local file history ${entry.key} with ${entry.actionType}` + } + } + + // local has to be last + // because we want to get keyEnc based on the remote + // (we don't consume prevSync here because it gains no benefit) + for (const local of localEntityList) { + const key = local.key; + if ( + isSkipItemByName( + key, + syncConfigDir, + syncUnderscoreItems, + configDir, + ignorePaths + ) + ) { + continue; + } + + if (finalMappings.hasOwnProperty(key)) { + const localCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(local), + password, + finalMappings[key].remote?.keyEnc ); - } - } - - return results; -}; - -const assignOperationToFileInplace = ( - origRecord: FileOrFolderMixedState, - keptFolder: Set, - skipSizeLargerThan: number, - password: string = "" -) => { - let r = origRecord; - - // files and folders are treated differently - // here we only check files - if (r.key.endsWith("/")) { - return r; - } - - // we find the max date from four sources - - // 0. find anything inconsistent - if (r.existLocal && (r.mtimeLocal === undefined || r.mtimeLocal <= 0)) { - throw Error( - `Error: Abnormal last modified time locally: ${JSON.stringify( - r, - null, - 2 - )}` - ); - } - if (r.existRemote && (r.mtimeRemote === undefined || r.mtimeRemote <= 0)) { - throw Error( - `Error: Abnormal last modified time remotely: ${JSON.stringify( - r, - null, - 2 - )}` - ); - } - if (r.deltimeLocal !== undefined && r.deltimeLocal <= 0) { - throw Error( - `Error: Abnormal deletion time locally: ${JSON.stringify(r, null, 2)}` - ); - } - if (r.deltimeRemote !== undefined && r.deltimeRemote <= 0) { - throw Error( - `Error: Abnormal deletion time remotely: ${JSON.stringify(r, null, 2)}` - ); - } - - if ( - (r.existLocal && password !== "" && r.sizeLocalEnc === undefined) || - (r.existRemote && password !== "" && r.sizeRemoteEnc === undefined) - ) { - throw new Error( - `Error: No encryption sizes: ${JSON.stringify(r, null, 2)}` - ); - } - - const sizeLocalComp = password === "" ? r.sizeLocal : r.sizeLocalEnc; - const sizeRemoteComp = password === "" ? r.sizeRemote : r.sizeRemoteEnc; - - // 1. mtimeLocal - if (r.existLocal) { - const mtimeRemote = r.existRemote ? r.mtimeRemote! : -1; - const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; - const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; - if ( - r.mtimeLocal! >= mtimeRemote && - r.mtimeLocal! >= deltimeLocal && - r.mtimeLocal! >= deltimeRemote - ) { - if (sizeLocalComp === undefined) { - throw new Error( - `Error: no local size but has local mtime: ${JSON.stringify( - r, - null, - 2 - )}` - ); - } - if (r.mtimeLocal === r.mtimeRemote) { - // local and remote both exist and mtimes are the same - if (sizeLocalComp === sizeRemoteComp) { - // do not need to consider skipSizeLargerThan in this case - r.decision = "skipUploading"; - r.decisionBranch = 1; - } else { - if (skipSizeLargerThan <= 0) { - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 2; - } else { - // limit the sizes - if (sizeLocalComp <= skipSizeLargerThan) { - if (sizeRemoteComp! <= skipSizeLargerThan) { - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 18; - } else { - r.decision = "errorRemoteTooLargeConflictLocal"; - r.decisionBranch = 19; - } - } else { - if (sizeRemoteComp! <= skipSizeLargerThan) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 20; - } else { - r.decision = "skipUploadingTooLarge"; - r.decisionBranch = 21; - } - } - } - } - } else { - // we have local laregest mtime, - // and the remote not existing or smaller mtime - if (skipSizeLargerThan <= 0) { - // no need to consider sizes - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 4; - } else { - // need to consider sizes - if (sizeLocalComp <= skipSizeLargerThan) { - if (sizeRemoteComp === undefined) { - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 22; - } else if (sizeRemoteComp <= skipSizeLargerThan) { - r.decision = "uploadLocalToRemote"; - r.decisionBranch = 23; - } else { - r.decision = "errorRemoteTooLargeConflictLocal"; - r.decisionBranch = 24; - } - } else { - if (sizeRemoteComp === undefined) { - r.decision = "skipUploadingTooLarge"; - r.decisionBranch = 25; - } else if (sizeRemoteComp <= skipSizeLargerThan) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 26; - } else { - r.decision = "skipUploadingTooLarge"; - r.decisionBranch = 27; - } - } - } - } - keptFolder.add(getParentFolder(r.key)); - return r; - } - } - - // 2. mtimeRemote - if (r.existRemote) { - const mtimeLocal = r.existLocal ? r.mtimeLocal! : -1; - const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; - const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; - if ( - r.mtimeRemote! > mtimeLocal && - r.mtimeRemote! >= deltimeLocal && - r.mtimeRemote! >= deltimeRemote - ) { - // we have remote laregest mtime, - // and the local not existing or smaller mtime - if (sizeRemoteComp === undefined) { - throw new Error( - `Error: no remote size but has remote mtime: ${JSON.stringify( - r, - null, - 2 - )}` - ); - } - - if (skipSizeLargerThan <= 0) { - // no need to consider sizes - r.decision = "downloadRemoteToLocal"; - r.decisionBranch = 5; - } else { - // need to consider sizes - if (sizeRemoteComp <= skipSizeLargerThan) { - if (sizeLocalComp === undefined) { - r.decision = "downloadRemoteToLocal"; - r.decisionBranch = 28; - } else if (sizeLocalComp <= skipSizeLargerThan) { - r.decision = "downloadRemoteToLocal"; - r.decisionBranch = 29; - } else { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 30; - } - } else { - if (sizeLocalComp === undefined) { - r.decision = "skipDownloadingTooLarge"; - r.decisionBranch = 31; - } else if (sizeLocalComp <= skipSizeLargerThan) { - r.decision = "errorRemoteTooLargeConflictLocal"; - r.decisionBranch = 32; - } else { - r.decision = "skipDownloadingTooLarge"; - r.decisionBranch = 33; - } - } - } - - keptFolder.add(getParentFolder(r.key)); - return r; - } - } - - // 3. deltimeLocal - if (r.deltimeLocal !== undefined && r.deltimeLocal !== 0) { - const mtimeLocal = r.existLocal ? r.mtimeLocal! : -1; - const mtimeRemote = r.existRemote ? r.mtimeRemote! : -1; - const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; - if ( - r.deltimeLocal >= mtimeLocal && - r.deltimeLocal >= mtimeRemote && - r.deltimeLocal >= deltimeRemote - ) { - if (skipSizeLargerThan <= 0) { - r.decision = "uploadLocalDelHistToRemote"; - r.decisionBranch = 6; - if (r.existLocal || r.existRemote) { - // actual deletion would happen - } - } else { - const localTooLargeToDelete = - r.existLocal && sizeLocalComp! > skipSizeLargerThan; - const remoteTooLargeToDelete = - r.existRemote && sizeRemoteComp! > skipSizeLargerThan; - if (localTooLargeToDelete) { - if (remoteTooLargeToDelete) { - r.decision = "skipUsingLocalDelTooLarge"; - r.decisionBranch = 34; - } else { - if (r.existRemote) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 35; - } else { - r.decision = "skipUsingLocalDelTooLarge"; - r.decisionBranch = 36; - } - } - } else { - if (remoteTooLargeToDelete) { - if (r.existLocal) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 37; - } else { - r.decision = "skipUsingLocalDelTooLarge"; - r.decisionBranch = 38; - } - } else { - r.decision = "uploadLocalDelHistToRemote"; - r.decisionBranch = 39; - } - } - } - return r; - } - } - - // 4. deltimeRemote - if (r.deltimeRemote !== undefined && r.deltimeRemote !== 0) { - const mtimeLocal = r.existLocal ? r.mtimeLocal! : -1; - const mtimeRemote = r.existRemote ? r.mtimeRemote! : -1; - const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; - if ( - r.deltimeRemote >= mtimeLocal && - r.deltimeRemote >= mtimeRemote && - r.deltimeRemote >= deltimeLocal - ) { - if (skipSizeLargerThan <= 0) { - r.decision = "keepRemoteDelHist"; - r.decisionBranch = 7; - if (r.existLocal || r.existRemote) { - // actual deletion would happen - } - } else { - const localTooLargeToDelete = - r.existLocal && sizeLocalComp! > skipSizeLargerThan; - const remoteTooLargeToDelete = - r.existRemote && sizeRemoteComp! > skipSizeLargerThan; - if (localTooLargeToDelete) { - if (remoteTooLargeToDelete) { - r.decision = "skipUsingRemoteDelTooLarge"; - r.decisionBranch = 40; - } else { - if (r.existRemote) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 41; - } else { - r.decision = "skipUsingRemoteDelTooLarge"; - r.decisionBranch = 42; - } - } - } else { - if (remoteTooLargeToDelete) { - if (r.existLocal) { - r.decision = "errorLocalTooLargeConflictRemote"; - r.decisionBranch = 43; - } else { - r.decision = "skipUsingRemoteDelTooLarge"; - r.decisionBranch = 44; - } - } else { - r.decision = "keepRemoteDelHist"; - r.decisionBranch = 45; - } - } - } - return r; - } - } - - throw Error(`no decision for ${JSON.stringify(r)}`); -}; - -const assignOperationToFolderInplace = async ( - origRecord: FileOrFolderMixedState, - keptFolder: Set, - vault: Vault, - password: string = "" -) => { - let r = origRecord; - - // files and folders are treated differently - // here we only check folders - if (!r.key.endsWith("/")) { - return r; - } - - if (!keptFolder.has(r.key)) { - // the folder does NOT have any must-be-kept children! - - if (r.deltimeLocal !== undefined || r.deltimeRemote !== undefined) { - // it has some deletion "commands" - - const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; - const deltimeRemote = - r.deltimeRemote !== undefined ? r.deltimeRemote : -1; - - // if it was created after deletion, we should keep it as is - if (requireApiVersion(API_VER_STAT_FOLDER)) { - if (r.existLocal) { - let ctime = 0; - let mtime = 0; - const s = await statFix(vault, r.key); - if (s !== undefined && s !== null) { - ctime = s.ctime; - mtime = s.mtime; - } - const cmtime = Math.max(ctime ?? 0, mtime ?? 0); - if ( - !Number.isNaN(cmtime) && - cmtime > 0 && - cmtime >= deltimeLocal && - cmtime >= deltimeRemote - ) { - keptFolder.add(getParentFolder(r.key)); - if (r.existLocal && r.existRemote) { - r.decision = "skipFolder"; - r.decisionBranch = 14; - } else if (r.existLocal || r.existRemote) { - r.decision = "createFolder"; - r.decisionBranch = 15; - } else { - throw Error( - `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` - ); - } - } - } - } - - // If it was moved to here, after deletion, we should keep it as is. - // The logic not necessarily needs API_VER_STAT_FOLDER. - // The folder needs this logic because it's also determined by file children. - // But the file do not need this logic because the mtimeLocal is checked firstly. - if ( - r.existLocal && - r.changeLocalMtimeUsingMapping && - r.mtimeLocal! > 0 && - r.mtimeLocal! > deltimeLocal && - r.mtimeLocal! > deltimeRemote - ) { - keptFolder.add(getParentFolder(r.key)); - if (r.existLocal && r.existRemote) { - r.decision = "skipFolder"; - r.decisionBranch = 16; - } else if (r.existLocal || r.existRemote) { - r.decision = "createFolder"; - r.decisionBranch = 17; - } else { - throw Error( - `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` - ); - } - } - - if (r.decision === undefined) { - // not yet decided by the above reason - if (deltimeLocal > 0 && deltimeLocal > deltimeRemote) { - r.decision = "uploadLocalDelHistToRemoteFolder"; - r.decisionBranch = 8; - } else { - r.decision = "keepRemoteDelHistFolder"; - r.decisionBranch = 9; - } - } + finalMappings[key].local = localCopied; } else { - // it does not have any deletion commands - // keep it as is, and create it if necessary - keptFolder.add(getParentFolder(r.key)); - if (r.existLocal && r.existRemote) { - r.decision = "skipFolder"; - r.decisionBranch = 10; - } else if (r.existLocal || r.existRemote) { - r.decision = "createFolder"; - r.decisionBranch = 11; - } else { - throw Error( - `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` - ); - } - } - } else { - // the folder has some must be kept children! - // so itself and its parent folder must be kept - keptFolder.add(getParentFolder(r.key)); - if (r.existLocal && r.existRemote) { - r.decision = "skipFolder"; - r.decisionBranch = 12; - } else if (r.existLocal || r.existRemote) { - r.decision = "createFolder"; - r.decisionBranch = 13; - } else { - throw Error( - `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` + const localCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(local), + password, + undefined ); + finalMappings[key] = { + key: key, + local: localCopied, + }; } } - // save the memory, save the world! - // we have dealt with it, so we don't need it any more. - keptFolder.delete(r.key); - return r; + return finalMappings; }; -const DELETION_DECISIONS: Set = new Set([ - "uploadLocalDelHistToRemote", - "keepRemoteDelHist", - "uploadLocalDelHistToRemoteFolder", - "keepRemoteDelHistFolder", -]); -const SIZES_GO_WRONG_DECISIONS: Set = new Set([ - "errorLocalTooLargeConflictRemote", - "errorRemoteTooLargeConflictLocal", -]); - -export const getSyncPlan = async ( - remoteStates: FileOrFolderMixedState[], - local: TAbstractFile[], - localConfigDirContents: ObsConfigDirFileType[] | undefined, - remoteDeleteHistory: DeletionOnRemote[] | undefined, - localFileHistory: FileFolderHistoryRecord[] | undefined, - remoteType: SUPPORTED_SERVICES_TYPE, - triggerSource: SyncTriggerSourceType, - vault: Vault, - syncConfigDir: boolean, - configDir: string, - syncUnderscoreItems: boolean, +/** + * Heavy lifting. + * Basically follow the sync algorithm of https://github.com/Jwink3101/syncrclone + * @param mixedEntityMappings + */ +export const getSyncPlanInplace = async ( + mixedEntityMappings: Record, + howToCleanEmptyFolder: EmptyFolderCleanType, skipSizeLargerThan: number, - ignorePaths: string[], - password: string = "" + conflictAction: ConflictActionType ) => { - const mixedStates = await ensembleMixedStates( - remoteStates, - local, - localConfigDirContents, - remoteDeleteHistory, - localFileHistory, - syncConfigDir, - configDir, - syncUnderscoreItems, - ignorePaths, - password - ); - - const sortedKeys = Object.keys(mixedStates).sort( + // from long(deep) to short(shadow) + const sortedKeys = Object.keys(mixedEntityMappings).sort( (k1, k2) => k2.length - k1.length ); - const sizesGoWrong: FileOrFolderMixedState[] = []; - const deletions: DeletionOnRemote[] = []; - const keptFolder = new Set(); + for (let i = 0; i < sortedKeys.length; ++i) { const key = sortedKeys[i]; - const val = mixedStates[key]; + const mixedEntry = mixedEntityMappings[key]; + const { local, prevSync, remote } = mixedEntry; if (key.endsWith("/")) { - // decide some folders - // because the keys are sorted by length - // so all the children must have been shown up before in the iteration - await assignOperationToFolderInplace(val, keptFolder, vault, password); - } else { - // get all operations of files - // and at the same time get some helper info for folders - assignOperationToFileInplace( - val, - keptFolder, - skipSizeLargerThan, - password - ); - } - - if (SIZES_GO_WRONG_DECISIONS.has(val.decision!)) { - sizesGoWrong.push(val); - } - - if (DELETION_DECISIONS.has(val.decision!)) { - if (val.decision === "uploadLocalDelHistToRemote") { - deletions.push({ - key: key, - actionWhen: val.deltimeLocal!, - }); - } else if (val.decision === "keepRemoteDelHist") { - deletions.push({ - key: key, - actionWhen: val.deltimeRemote!, - }); - } else if (val.decision === "uploadLocalDelHistToRemoteFolder") { - deletions.push({ - key: key, - actionWhen: val.deltimeLocal!, - }); - } else if (val.decision === "keepRemoteDelHistFolder") { - deletions.push({ - key: key, - actionWhen: val.deltimeRemote!, - }); + // folder + // folder doesn't worry about mtime and size, only check their existences + if (keptFolder.has(key)) { + // should fill the missing part + if (local !== undefined && remote !== undefined) { + mixedEntry.decisionBranch = 101; + mixedEntry.decision = "folder_existed_both"; + } else if (local !== undefined && remote === undefined) { + mixedEntry.decisionBranch = 102; + mixedEntry.decision = "folder_existed_local"; + } else if (local === undefined && remote !== undefined) { + mixedEntry.decisionBranch = 103; + mixedEntry.decision = "folder_existed_remote"; + } else { + mixedEntry.decisionBranch = 104; + mixedEntry.decision = "folder_to_be_created"; + } + keptFolder.delete(key); // no need to save it in the Set later } else { - throw Error(`do not know how to delete for decision ${val.decision}`); + if (howToCleanEmptyFolder === "skip") { + mixedEntry.decisionBranch = 105; + mixedEntry.decision = "folder_to_skip"; + } else if (howToCleanEmptyFolder === "clean_both") { + mixedEntry.decisionBranch = 106; + mixedEntry.decision = "folder_to_be_deleted"; + } else { + throw Error( + `do not know how to deal with empty folder ${mixedEntry.key}` + ); + } + } + } else { + // file + + if (local === undefined && remote === undefined) { + // both deleted, only in history + mixedEntry.decisionBranch = 1; + mixedEntry.decision = "only_history"; + } else if (local !== undefined && remote !== undefined) { + if ( + (local.mtimeCli === remote.mtimeCli || + local.mtimeCli === remote.mtimeSvr) && + local.sizeEnc === remote.sizeEnc + ) { + // completely equal / identical + mixedEntry.decisionBranch = 2; + mixedEntry.decision = "equal"; + keptFolder.add(getParentFolder(key)); + } else { + // Both exists, but modified or conflict + // Look for past files of A or B. + + const localEqualPrevSync = + prevSync?.mtimeSvr === local.mtimeCli && + prevSync?.sizeEnc === local.sizeEnc; + const remoteEqualPrevSync = + (prevSync?.mtimeSvr === remote.mtimeCli || + prevSync?.mtimeSvr === remote.mtimeSvr) && + prevSync?.sizeEnc === remote.sizeEnc; + + if (localEqualPrevSync && !remoteEqualPrevSync) { + // If only one compares true (no prev also means it compares False), the other is modified. Backup and sync. + if ( + skipSizeLargerThan <= 0 || + remote.sizeEnc <= skipSizeLargerThan + ) { + mixedEntry.decisionBranch = 9; + mixedEntry.decision = "modified_remote"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `remote is modified (branch 9) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } else if (!localEqualPrevSync && remoteEqualPrevSync) { + // If only one compares true (no prev also means it compares False), the other is modified. Backup and sync. + if ( + skipSizeLargerThan <= 0 || + local.sizeEnc <= skipSizeLargerThan + ) { + mixedEntry.decisionBranch = 10; + mixedEntry.decision = "modified_local"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `local is modified (branch 10) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } else if (!localEqualPrevSync && !remoteEqualPrevSync) { + // If both compare False, (didn't exist means both are new. Both exist but don't compare means both are modified) + if (prevSync === undefined) { + if (conflictAction === "keep_newer") { + if ( + (local.mtimeCli ?? local.mtimeSvr ?? 0) >= + (remote.mtimeCli ?? remote.mtimeSvr ?? 0) + ) { + mixedEntry.decisionBranch = 11; + mixedEntry.decision = "conflict_created_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 12; + mixedEntry.decision = "conflict_created_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else if (conflictAction === "keep_larger") { + if (local.sizeEnc >= remote.sizeEnc) { + mixedEntry.decisionBranch = 13; + mixedEntry.decision = "conflict_created_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 14; + mixedEntry.decision = "conflict_created_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else { + mixedEntry.decisionBranch = 15; + mixedEntry.decision = "conflict_created_keep_both"; + keptFolder.add(getParentFolder(key)); + } + } else { + if (conflictAction === "keep_newer") { + if ( + (local.mtimeCli ?? local.mtimeSvr ?? 0) >= + (remote.mtimeCli ?? remote.mtimeSvr ?? 0) + ) { + mixedEntry.decisionBranch = 16; + mixedEntry.decision = "conflict_modified_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 17; + mixedEntry.decision = "conflict_modified_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else if (conflictAction === "keep_larger") { + if (local.sizeEnc >= remote.sizeEnc) { + mixedEntry.decisionBranch = 18; + mixedEntry.decision = "conflict_modified_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 19; + mixedEntry.decision = "conflict_modified_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else { + mixedEntry.decisionBranch = 20; + mixedEntry.decision = "conflict_modified_keep_both"; + keptFolder.add(getParentFolder(key)); + } + } + } else { + // Both compare true -- This is VERY odd and should not happen + throw Error( + `should not reach branch -2 while getting sync plan: ${JSON.stringify( + mixedEntry + )}` + ); + } + } + } else if (local === undefined && remote !== undefined) { + // A is missing + if (prevSync === undefined) { + // if B is not in the previous list, B is new + if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) { + mixedEntry.decisionBranch = 3; + mixedEntry.decision = "created_remote"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `remote is created (branch 3) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } else if ( + (prevSync.mtimeSvr === remote.mtimeCli || + prevSync.mtimeSvr === remote.mtimeSvr) && + prevSync.sizeEnc === remote.sizeEnc + ) { + // if B is in the previous list and UNMODIFIED, B has been deleted by A + mixedEntry.decisionBranch = 4; + mixedEntry.decision = "deleted_local"; + } else { + // if B is in the previous list and MODIFIED, B has been deleted by A but modified by B + if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) { + mixedEntry.decisionBranch = 5; + mixedEntry.decision = "modified_remote"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `remote is modified (branch 5) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } + } else if (local !== undefined && remote === undefined) { + // B is missing + + if (prevSync === undefined) { + // if A is not in the previous list, A is new + if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) { + mixedEntry.decisionBranch = 6; + mixedEntry.decision = "created_local"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `local is created (branch 6) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } else if ( + prevSync.mtimeSvr === local.mtimeCli && + prevSync.sizeEnc === local.sizeEnc + ) { + // if A is in the previous list and UNMODIFIED, A has been deleted by B + mixedEntry.decisionBranch = 7; + mixedEntry.decision = "deleted_remote"; + } else { + // if A is in the previous list and MODIFIED, A has been deleted by B but modified by A + if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) { + mixedEntry.decisionBranch = 8; + mixedEntry.decision = "modified_local"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `local is modified (branch 8) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( + mixedEntry + )}` + ); + } + } + } else { + throw Error( + `should not reach branch -1 while getting sync plan: ${JSON.stringify( + mixedEntry + )}` + ); + } + + if (mixedEntry.decision === undefined) { + throw Error( + `unexpectedly no decision of file in the end: ${JSON.stringify( + mixedEntry + )}` + ); } } } - const currTs = Date.now(); - const currTsFmt = unixTimeToStr(currTs); - const plan = { - ts: currTs, - tsFmt: currTsFmt, - remoteType: remoteType, - syncTriggerSource: triggerSource, - mixedStates: mixedStates, - } as SyncPlanType; - return { - plan: plan, - sortedKeys: sortedKeys, - deletions: deletions, - sizesGoWrong: sizesGoWrong, - }; + keptFolder.delete("/"); + keptFolder.delete(""); + if (keptFolder.size > 0) { + throw Error(`unexpectedly keptFolder no decisions: ${[...keptFolder]}`); + } + + return mixedEntityMappings; }; -const uploadExtraMeta = async ( - client: RemoteClient, - metadataFile: FileOrFolderMixedState | undefined, - origMetadata: MetadataOnRemote | undefined, - deletions: DeletionOnRemote[], - password: string = "" +const splitThreeStepsOnEntityMappings = ( + mixedEntityMappings: Record ) => { - if (deletions === undefined || deletions.length === 0) { - return; - } + const folderCreationOps: MixedEntity[][] = []; + const deletionOps: MixedEntity[][] = []; + const uploadDownloads: MixedEntity[][] = []; - const key = DEFAULT_FILE_NAME_FOR_METADATAONREMOTE; - let remoteEncryptedKey: string | undefined = key; - - if (password !== "") { - if (metadataFile === undefined) { - remoteEncryptedKey = undefined; - } else { - remoteEncryptedKey = metadataFile.remoteEncryptedKey; - } - if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { - // remoteEncryptedKey = await encryptStringToBase32(key, password); - remoteEncryptedKey = await encryptStringToBase64url(key, password); - } - } - - const newMetadata: MetadataOnRemote = { - deletions: deletions, - }; - - if (isEqualMetadataOnRemote(origMetadata, newMetadata)) { - log.debug( - "metadata are the same, no need to re-generate and re-upload it." - ); - return; - } - - const resultText = serializeMetadataOnRemote(newMetadata); - - await client.uploadToRemote( - key, - undefined, - false, - password, - remoteEncryptedKey, - undefined, - true, - resultText + // from long(deep) to short(shadow) + const sortedKeys = Object.keys(mixedEntityMappings).sort( + (k1, k2) => k2.length - k1.length ); -}; -const dispatchOperationToActual = async ( - key: string, - vaultRandomID: string, - r: FileOrFolderMixedState, - client: RemoteClient, - db: InternalDBs, - vault: Vault, - localDeleteFunc: any, - password: string = "" -) => { - let remoteEncryptedKey: string | undefined = key; - if (password !== "") { - remoteEncryptedKey = r.remoteEncryptedKey; - if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { - // the old version uses base32 - // remoteEncryptedKey = await encryptStringToBase32(key, password); - // the new version users base64url - remoteEncryptedKey = await encryptStringToBase64url(key, password); - } - } - - if (r.decision === undefined) { - throw Error(`unknown decision in ${JSON.stringify(r)}`); - } else if (r.decision === "skipUploading") { - // do nothing! - } else if (r.decision === "uploadLocalDelHistToRemote") { - if (r.existLocal) { - await localDeleteFunc(r.key); - } - if (r.existRemote) { - await client.deleteFromRemote(r.key, password, remoteEncryptedKey); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "keepRemoteDelHist") { - if (r.existLocal) { - await localDeleteFunc(r.key); - } - if (r.existRemote) { - await client.deleteFromRemote(r.key, password, remoteEncryptedKey); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "uploadLocalToRemote") { - if ( - client.serviceType === "onedrive" && - r.sizeLocal === 0 && - password === "" - ) { - // special treatment for empty files for OneDrive - // TODO: it's ugly, any other way? - // special treatment for OneDrive: do nothing, skip empty file without encryption - // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded. - } else { - const remoteObjMeta = await client.uploadToRemote( - r.key, - vault, - false, - password, - remoteEncryptedKey - ); - await upsertSyncMetaMappingDataByVault( - client.serviceType, - db, - r.key, - r.mtimeLocal!, - r.sizeLocal!, - r.key, - remoteObjMeta.lastModified ?? Date.now(), - remoteObjMeta.size, - remoteObjMeta.etag ?? "", - vaultRandomID - ); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "downloadRemoteToLocal") { - await mkdirpInVault(r.key, vault); /* should be unnecessary */ - await client.downloadFromRemote( - r.key, - vault, - r.mtimeRemote!, - password, - remoteEncryptedKey - ); - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "createFolder") { - if (!r.existLocal) { - await mkdirpInVault(r.key, vault); - } - if (!r.existRemote) { - const remoteObjMeta = await client.uploadToRemote( - r.key, - vault, - false, - password, - remoteEncryptedKey - ); - await upsertSyncMetaMappingDataByVault( - client.serviceType, - db, - r.key, - r.mtimeLocal!, - r.sizeLocal!, - r.key, - remoteObjMeta.lastModified ?? Date.now(), - remoteObjMeta.size, - remoteObjMeta.etag ?? "", - vaultRandomID - ); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "uploadLocalDelHistToRemoteFolder") { - if (r.existLocal) { - await localDeleteFunc(r.key); - } - if (r.existRemote) { - await client.deleteFromRemote(r.key, password, remoteEncryptedKey); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "keepRemoteDelHistFolder") { - if (r.existLocal) { - await localDeleteFunc(r.key); - } - if (r.existRemote) { - await client.deleteFromRemote(r.key, password, remoteEncryptedKey); - } - await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID); - } else if (r.decision === "skipFolder") { - // do nothing! - } else if (r.decision === "skipUploadingTooLarge") { - // do nothing! - } else if (r.decision === "skipDownloadingTooLarge") { - // do nothing! - } else if (r.decision === "skipUsingLocalDelTooLarge") { - // do nothing! - } else if (r.decision === "skipUsingRemoteDelTooLarge") { - // do nothing! - } else { - throw Error(`unknown decision in ${JSON.stringify(r)}`); - } -}; - -const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => { - const mixedStates = syncPlan.mixedStates; - const totalCount = sortedKeys.length || 0; - - const folderCreationOps: FileOrFolderMixedState[][] = []; - const deletionOps: FileOrFolderMixedState[][] = []; - const uploadDownloads: FileOrFolderMixedState[][] = []; let realTotalCount = 0; for (let i = 0; i < sortedKeys.length; ++i) { const key = sortedKeys[i]; - const val: FileOrFolderMixedState = Object.assign({}, mixedStates[key]); // copy to avoid issue + const val = mixedEntityMappings[key]; if ( - val.decision === "skipFolder" || - val.decision === "skipUploading" || - val.decision === "skipDownloadingTooLarge" || - val.decision === "skipUploadingTooLarge" || - val.decision === "skipUsingLocalDelTooLarge" || - val.decision === "skipUsingRemoteDelTooLarge" + val.decision === "equal" || + val.decision === "folder_existed_both" || + val.decision === "folder_to_skip" ) { // pass - } else if (val.decision === "createFolder") { + } else if ( + val.decision === "folder_existed_local" || + val.decision === "folder_existed_remote" || + val.decision === "folder_to_be_created" + ) { const level = atWhichLevel(key); if (folderCreationOps[level - 1] === undefined) { folderCreationOps[level - 1] = [val]; @@ -1350,10 +725,10 @@ const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => { } realTotalCount += 1; } else if ( - val.decision === "uploadLocalDelHistToRemoteFolder" || - val.decision === "keepRemoteDelHistFolder" || - val.decision === "uploadLocalDelHistToRemote" || - val.decision === "keepRemoteDelHist" + val.decision === "only_history" || + val.decision === "deleted_local" || + val.decision === "deleted_remote" || + val.decision === "folder_to_be_deleted" ) { const level = atWhichLevel(key); if (deletionOps[level - 1] === undefined) { @@ -1363,8 +738,16 @@ const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => { } realTotalCount += 1; } else if ( - val.decision === "uploadLocalToRemote" || - val.decision === "downloadRemoteToLocal" + val.decision === "modified_local" || + val.decision === "modified_remote" || + val.decision === "created_local" || + val.decision === "created_remote" || + val.decision === "conflict_created_keep_local" || + val.decision === "conflict_created_keep_remote" || + val.decision === "conflict_created_keep_both" || + val.decision === "conflict_modified_keep_local" || + val.decision === "conflict_modified_keep_remote" || + val.decision === "conflict_modified_keep_both" ) { if (uploadDownloads.length === 0) { uploadDownloads[0] = [val]; @@ -1390,46 +773,110 @@ const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => { }; }; -export const doActualSync = async ( +const dispatchOperationToActualV3 = async ( + key: string, + vaultRandomID: string, + r: MixedEntity, client: RemoteClient, db: InternalDBs, + vault: Vault, + localDeleteFunc: any, + password: string +) => { + if (r.decision === "only_history") { + clearPrevSyncRecordByVault(db, vaultRandomID, key); + } else if ( + r.decision === "equal" || + r.decision === "folder_to_skip" || + r.decision === "folder_existed_both" + ) { + // pass + } else if ( + r.decision === "modified_local" || + r.decision === "created_local" || + r.decision === "folder_existed_local" || + r.decision === "conflict_created_keep_local" || + r.decision === "conflict_modified_keep_local" + ) { + if ( + client.serviceType === "onedrive" && + r.local!.size === 0 && + password === "" + ) { + // special treatment for empty files for OneDrive + // TODO: it's ugly, any other way? + // special treatment for OneDrive: do nothing, skip empty file without encryption + // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded. + } else { + const remoteObjMeta = await client.uploadToRemote( + r.key, + vault, + false, + password, + r.local!.keyEnc + ); + await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); + } + } else if ( + r.decision === "modified_remote" || + r.decision === "created_remote" || + r.decision === "conflict_created_keep_remote" || + r.decision === "conflict_modified_keep_remote" || + r.decision === "folder_existed_remote" + ) { + await mkdirpInVault(r.key, vault); + await client.downloadFromRemote( + r.key, + vault, + r.remote!.mtimeCli!, + password, + r.remote!.keyEnc + ); + await upsertPrevSyncRecordByVault(db, vaultRandomID, r.remote!); + } else if (r.decision === "deleted_local") { + await localDeleteFunc(r.key); + await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + } else if (r.decision === "deleted_remote") { + await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); + await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + } else if ( + r.decision === "conflict_created_keep_both" || + r.decision === "conflict_modified_keep_both" + ) { + throw Error(`${r.decision} not implemented yet: ${JSON.stringify(r)}`); + } else if (r.decision === "folder_to_be_created") { + await mkdirpInVault(r.key, vault); + const remoteObjMeta = await client.uploadToRemote( + r.key, + vault, + false, + password, + r.local!.keyEnc + ); + await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); + } else if (r.decision === "folder_to_be_deleted") { + await localDeleteFunc(r.key); + await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); + await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + } else { + throw Error(`don't know how to dispatch decision: ${JSON.stringify(r)}`); + } +}; + +export const doActualSyncV3 = async ( + mixedEntityMappings: Record, + client: RemoteClient, vaultRandomID: string, vault: Vault, - syncPlan: SyncPlanType, - sortedKeys: string[], - metadataFile: FileOrFolderMixedState | undefined, - origMetadata: MetadataOnRemote, - sizesGoWrong: FileOrFolderMixedState[], - deletions: DeletionOnRemote[], + password: string, + concurrency: number, localDeleteFunc: any, - password: string = "", - concurrency: number = 1, - callbackSizesGoWrong?: any, - callbackSyncProcess?: any + callbackSyncProcess: any, + db: InternalDBs ) => { - const mixedStates = syncPlan.mixedStates; - const totalCount = sortedKeys.length || 0; - - if (sizesGoWrong.length > 0) { - log.debug(`some sizes are larger than the threshold, abort and show hints`); - callbackSizesGoWrong(sizesGoWrong); - return; - } - - log.debug(`start syncing extra data firstly`); - await uploadExtraMeta( - client, - metadataFile, - origMetadata, - deletions, - password - ); - log.debug(`finish syncing extra data firstly`); - log.debug(`concurrency === ${concurrency}`); - const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } = - splitThreeSteps(syncPlan, sortedKeys); + splitThreeStepsOnEntityMappings(mixedEntityMappings); const nested = [folderCreationOps, deletionOps, uploadDownloads]; const logTexts = [ `1. create all folders from shadowest to deepest, also check undefined decision`, @@ -1438,26 +885,20 @@ export const doActualSync = async ( ]; let realCounter = 0; - for (let i = 0; i < nested.length; ++i) { log.debug(logTexts[i]); - const operations: FileOrFolderMixedState[][] = nested[i]; + const operations = nested[i]; for (let j = 0; j < operations.length; ++j) { - const singleLevelOps: FileOrFolderMixedState[] | undefined = - operations[j]; - - if (singleLevelOps === undefined || singleLevelOps === null) { - continue; - } + const singleLevelOps = operations[j]; const queue = new PQueue({ concurrency: concurrency, autoStart: true }); const potentialErrors: Error[] = []; let tooManyErrors = false; for (let k = 0; k < singleLevelOps.length; ++k) { - const val: FileOrFolderMixedState = singleLevelOps[k]; + const val = singleLevelOps[k]; const key = val.key; const fn = async () => { @@ -1474,7 +915,7 @@ export const doActualSync = async ( realCounter += 1; } - await dispatchOperationToActual( + await dispatchOperationToActualV3( key, vaultRandomID, val, From 06b15b21464d37963de0cd3ef6f32b85bac57829 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 08:30:38 +0800 Subject: [PATCH 02/21] never used sync sizes notice --- src/syncSizesConflictNotice.ts | 90 ---------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 src/syncSizesConflictNotice.ts diff --git a/src/syncSizesConflictNotice.ts b/src/syncSizesConflictNotice.ts deleted file mode 100644 index ce88edd..0000000 --- a/src/syncSizesConflictNotice.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { App, Modal, Notice, PluginSettingTab, Setting } from "obsidian"; -import type RemotelySavePlugin from "./main"; // unavoidable -import type { TransItemType } from "./i18n"; -import type { FileOrFolderMixedState } from "./baseTypes"; - -import { log } from "./moreOnLog"; - -export class SizesConflictModal extends Modal { - readonly plugin: RemotelySavePlugin; - readonly skipSizeLargerThan: number; - readonly sizesGoWrong: FileOrFolderMixedState[]; - readonly hasPassword: boolean; - constructor( - app: App, - plugin: RemotelySavePlugin, - skipSizeLargerThan: number, - sizesGoWrong: FileOrFolderMixedState[], - hasPassword: boolean - ) { - super(app); - this.plugin = plugin; - this.skipSizeLargerThan = skipSizeLargerThan; - this.sizesGoWrong = sizesGoWrong; - this.hasPassword = hasPassword; - } - onOpen() { - let { contentEl } = this; - const t = (x: TransItemType, vars?: any) => { - return this.plugin.i18n.t(x, vars); - }; - - contentEl.createEl("h2", { - text: t("modal_sizesconflict_title"), - }); - - t("modal_sizesconflict_desc", { - thresholdMB: `${this.skipSizeLargerThan / 1000 / 1000}`, - thresholdBytes: `${this.skipSizeLargerThan}`, - }) - .split("\n") - .forEach((val) => { - contentEl.createEl("p", { text: val }); - }); - - const info = this.serialize(); - - contentEl.createDiv().createEl( - "button", - { - text: t("modal_sizesconflict_copybutton"), - }, - (el) => { - el.onclick = async () => { - await navigator.clipboard.writeText(info); - new Notice(t("modal_sizesconflict_copynotice")); - }; - } - ); - - contentEl.createEl("pre", { - text: info, - }); - } - - serialize() { - return this.sizesGoWrong - .map((x) => { - return [ - x.key, - this.hasPassword - ? `encrypted name: ${x.remoteEncryptedKey}` - : undefined, - `local ${this.hasPassword ? "encrypted " : ""}bytes: ${ - this.hasPassword ? x.sizeLocalEnc : x.sizeLocal - }`, - `remote ${this.hasPassword ? "encrypted " : ""}bytes: ${ - this.hasPassword ? x.sizeRemoteEnc : x.sizeRemote - }`, - ] - .filter((tmp) => tmp !== undefined) - .join("\n"); - }) - .join("\n\n"); - } - - onClose() { - let { contentEl } = this; - contentEl.empty(); - } -} From 10d22cb9126483a7e165578215d08afcbfb708f5 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 08:30:58 +0800 Subject: [PATCH 03/21] v3 notice --- src/{syncAlgoV2Notice.ts => syncAlgoV3Notice.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{syncAlgoV2Notice.ts => syncAlgoV3Notice.ts} (100%) diff --git a/src/syncAlgoV2Notice.ts b/src/syncAlgoV3Notice.ts similarity index 100% rename from src/syncAlgoV2Notice.ts rename to src/syncAlgoV3Notice.ts From e341c4f7803d23b91dd6a5ef2cef8bccf27091d2 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 08:41:43 +0800 Subject: [PATCH 04/21] sync v3 notice draft --- src/baseTypes.ts | 37 ++++------------- src/langs/en.json | 10 ++--- src/langs/zh_cn.json | 10 ++--- src/langs/zh_tw.json | 9 ++--- src/main.ts | 90 +++++------------------------------------ src/syncAlgoV3Notice.ts | 10 ++--- 6 files changed, 36 insertions(+), 130 deletions(-) diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 1a1be04..9fc09e1 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -94,17 +94,22 @@ export interface RemotelySavePluginSettings { autoRunEveryMilliseconds?: number; initRunAfterMilliseconds?: number; syncOnSaveAfterMilliseconds?: number; - agreeToUploadExtraMetadata?: boolean; + concurrency?: number; syncConfigDir?: boolean; syncUnderscoreItems?: boolean; lang?: LangTypeAndAuto; - + agreeToUseSyncV3?: boolean; skipSizeLargerThan?: number; ignorePaths?: string[]; enableStatusBarInfo?: boolean; deleteToWhere?: "system" | "obsidian"; + /** + * @deprecated + */ + agreeToUploadExtraMetadata?: boolean; + /** * @deprecated */ @@ -131,32 +136,6 @@ export interface UriParams { // 80 days export const OAUTH2_FORCE_EXPIRE_MILLISECONDS = 1000 * 60 * 60 * 24 * 80; -type DecisionTypeForFile = - | "skipUploading" // special, mtimeLocal === mtimeRemote - | "uploadLocalDelHistToRemote" // "delLocalIfExists && delRemoteIfExists && cleanLocalDelHist && uploadLocalDelHistToRemote" - | "keepRemoteDelHist" // "delLocalIfExists && delRemoteIfExists && cleanLocalDelHist && keepRemoteDelHist" - | "uploadLocalToRemote" // "skipLocal && uploadLocalToRemote && cleanLocalDelHist && cleanRemoteDelHist" - | "downloadRemoteToLocal"; // "downloadRemoteToLocal && skipRemote && cleanLocalDelHist && cleanRemoteDelHist" - -type DecisionTypeForFileSize = - | "skipUploadingTooLarge" - | "skipDownloadingTooLarge" - | "skipUsingLocalDelTooLarge" - | "skipUsingRemoteDelTooLarge" - | "errorLocalTooLargeConflictRemote" - | "errorRemoteTooLargeConflictLocal"; - -type DecisionTypeForFolder = - | "createFolder" - | "uploadLocalDelHistToRemoteFolder" - | "keepRemoteDelHistFolder" - | "skipFolder"; - -export type DecisionType = - | DecisionTypeForFile - | DecisionTypeForFileSize - | DecisionTypeForFolder; - export type EmptyFolderCleanType = "skip" | "clean_both"; export type ConflictActionType = "keep_newer" | "keep_larger" | "rename_both"; @@ -233,7 +212,7 @@ export interface FileOrFolderMixedState { sizeRemoteEnc?: number; changeRemoteMtimeUsingMapping?: boolean; changeLocalMtimeUsingMapping?: boolean; - decision?: DecisionType; + decision?: string; // old DecisionType is deleted, fallback to string decisionBranch?: number; syncDone?: "done"; remoteEncryptedKey?: string; diff --git a/src/langs/en.json b/src/langs/en.json index 4e36c3e..cd75af2 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -284,10 +284,8 @@ "settings_resetcache_desc": "Reset local internal caches/databases (for debugging purposes). You would want to reload the plugin after resetting this. This option will not empty the {s3, password...} settings.", "settings_resetcache_button": "Reset", "settings_resetcache_notice": "Local internal cache/databases deleted. Please manually reload the plugin.", - "syncalgov2_title": "Remotely Save has a better sync algorithm", - "syncalgov2_texts": "Welcome to use Remotely Save!\nFrom version 0.3.0, a new algorithm has been developed, but it needs uploading extra meta data files _remotely-save-metadata-on-remote.{json,bin} to YOUR configured cloud destinations, besides your notes.\nSo that, for example, the second device can know that what files/folders have been deleted on the first device by reading those files.\nIf you agree, plase click the button \"Agree\", and enjoy the plugin! AND PLEASE REMEMBER TO BACKUP YOUR VAULT FIRSTLY!\nIf you do not agree, you should stop using the current and later versions of Remotely Save. You could consider manually install the old version 0.2.14 which uses old algorithm and does not upload any extra meta data files. By clicking the \"Do Not Agree\" button, the plugin will unload itself, and you need to manually disable it in Obsidian settings.", - "syncalgov2_button_agree": "Agree", - "syncalgov2_button_disagree": "Do Not Agree", - - "official_notice_2024_first_party": "Plugin Remotely-Save is back to the party and get a HUGE update!🎉🎉🎉 Try it yourself or see the release note on https://github.com/remotely-save/remotely-save/releases." + "syncalgov3_title": "Remotely Save has HUGE update on sync algorithm", + "syncalgov3_texts": "Welcome to use Remotely Save!\nFrom this version, a new algorithm has been developed. If you agree, plase click the button \"Agree\", and enjoy the plugin! AND PLEASE REMEMBER TO BACKUP YOUR VAULT FIRSTLY!\nIf you do not agree, you should stop using the current and later versions of Remotely Save. By clicking the \"Do Not Agree\" button, the plugin will unload itself, and you need to manually disable it in Obsidian settings.", + "syncalgov3_button_agree": "Agree", + "syncalgov3_button_disagree": "Do Not Agree" } diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index a61d17a..cb128af 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -284,10 +284,8 @@ "settings_resetcache_desc": "（出于调试原因）重设本地缓存和数据库。您需要在重设之后重新载入此插件。本重设不会删除 s3，密码……等设定。", "settings_resetcache_button": "重设", "settings_resetcache_notice": "本地同步缓存和数据库已被删除。请手动重新载入此插件。", - "syncalgov2_title": "Remotely Save 的同步算法得到优化", - "syncalgov2_texts": "欢迎使用 Remotely Save!\n从版本 0.3.0 开始，它带来了新的同步算法，但是，除了您的笔记之外，它还需要上传额外的带有元信息的文件 _remotely-save-metadata-on-remote.{json,bin} 到您的云服务目的地上。\n从而，比如说，通过读取这些信息，另一台设备可以知道什么文件或文件夹在第一台设备上被删除了。\n如果您同意此策略，请点击按钮 \"同意\"，然后开始享用此插件！且特别要注意：使用插件之前，请首先备份好您的库（Vault）！\n如果您不同意此策略，您应该停止使用此版本和之后版本的 Remotely Save。您可以考虑手动安装旧版 0.2.14，它使用旧的同步算法，并不上传额外元信息文件。点击 \"不同意\" 之后，插件会自动停止运行（unload），然后您需要 Obsidian 设置里手动停用（disable）此插件。", - "syncalgov2_button_agree": "同意", - "syncalgov2_button_disagree": "不同意", - - "official_notice_2024_first_party": "插件 Remotely-Save 回来了，更新了一大堆功能！🎉🎉🎉请自行使用，或参阅更新文档： https://github.com/remotely-save/remotely-save/releases 。" + "syncalgov3_title": "Remotely Save 的同步算法重大优化", + "syncalgov3_texts": "欢迎使用 Remotely Save!\n从这个版本 0.3.0 开始，它带来了新的同步算法\n如果您同意使用，请点击按钮 \"同意\"，然后开始享用此插件！且特别要注意：使用插件之前，请首先备份好您的库（Vault）！\n如果您不同意此策略，您应该停止使用此版本和之后版本的 Remotely Save。点击 \"不同意\" 之后，插件会自动停止运行（unload），然后您需要 Obsidian 设置里手动停用（disable）此插件。", + "syncalgov3_button_agree": "同意", + "syncalgov3_button_disagree": "不同意" } diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index 25771be..cb81eff 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -284,9 +284,8 @@ "settings_resetcache_desc": "（出於除錯原因）重設本地快取和資料庫。您需要在重設之後重新載入此外掛。本重設不會刪除 s3，密碼……等設定。", "settings_resetcache_button": "重設", "settings_resetcache_notice": "本地同步快取和資料庫已被刪除。請手動重新載入此外掛。", - "syncalgov2_title": "Remotely Save 的同步演算法得到最佳化", - "syncalgov2_texts": "歡迎使用 Remotely Save!\n從版本 0.3.0 開始，它帶來了新的同步演算法，但是，除了您的筆記之外，它還需要上傳額外的帶有元資訊的檔案 _remotely-save-metadata-on-remote.{json,bin} 到您的雲服務目的地上。\n從而，比如說，透過讀取這些資訊，另一臺裝置可以知道什麼檔案或資料夾在第一臺裝置上被刪除了。\n如果您同意此策略，請點選按鈕 \"同意\"，然後開始享用此外掛！且特別要注意：使用外掛之前，請首先備份好您的儲存庫（Vault）！\n如果您不同意此策略，您應該停止使用此版本和之後版本的 Remotely Save。您可以考慮手動安裝舊版 0.2.14，它使用舊的同步演算法，並不上傳額外元資訊檔案。點選 \"不同意\" 之後，外掛會自動停止執行（unload），然後您需要 Obsidian 設定裡手動停用（disable）此外掛。", - "syncalgov2_button_agree": "同意", - "syncalgov2_button_disagree": "不同意", - "official_notice_2024_first_party": "外掛 Remotely-Save 回來了，更新了一大堆功能！🎉🎉🎉請自行使用，或參閱更新文件： https://github.com/remotely-save/remotely-save/releases 。" + "syncalgov3_title": "Remotely Save 的同步演算法重大最佳化", + "syncalgov3_texts": "歡迎使用 Remotely Save!\n從這個版本 0.3.0 開始，它帶來了新的同步演算法\n如果您同意使用，請點選按鈕 \"同意\"，然後開始享用此外掛！且特別要注意：使用外掛之前，請首先備份好您的庫（Vault）！\n如果您不同意此策略，您應該停止使用此版本和之後版本的 Remotely Save。點選 \"不同意\" 之後，外掛會自動停止執行（unload），然後您需要 Obsidian 設定裡手動停用（disable）此外掛。", + "syncalgov3_button_agree": "同意", + "syncalgov3_button_disagree": "不同意" } diff --git a/src/main.ts b/src/main.ts index 8cda974..938cb0b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,7 +15,6 @@ import { import cloneDeep from "lodash/cloneDeep"; import { createElement, RotateCcw, RefreshCcw, FileText } from "lucide"; import type { - FileOrFolderMixedState, RemotelySavePluginSettings, SyncTriggerSourceType, } from "./baseTypes"; @@ -29,10 +28,7 @@ import { } from "./baseTypes"; import { importQrCodeUri } from "./importExport"; import { - insertDeleteRecordByVault, - insertRenameRecordByVault, insertSyncPlanRecordByVault, - loadFileHistoryTableByVault, prepareDBs, InternalDBs, clearExpiredSyncPlanRecords, @@ -57,15 +53,15 @@ import { import { DEFAULT_S3_CONFIG } from "./remoteForS3"; import { DEFAULT_WEBDAV_CONFIG } from "./remoteForWebdav"; import { RemotelySaveSettingTab } from "./settings"; -import { fetchMetadataFile, parseRemoteItems, SyncStatusType } from "./sync"; +import { parseRemoteItems, SyncStatusType } from "./sync"; import { doActualSync, getSyncPlan, isPasswordOk } from "./sync"; import { messyConfigToNormal, normalConfigToMessy } from "./configPersist"; -import { ObsConfigDirFileType, listFilesInObsFolder } from "./obsFolderLister"; +import { getLocalEntityList } from "./local"; import { I18n } from "./i18n"; import type { LangType, LangTypeAndAuto, TransItemType } from "./i18n"; import { DeletionOnRemote, MetadataOnRemote } from "./metadataOnRemote"; -import { SyncAlgoV2Modal } from "./syncAlgoV2Notice"; +import { SyncAlgoV3Modal } from "./syncAlgoV3Notice"; import { applyLogWriterInplace, log } from "./moreOnLog"; import AggregateError from "aggregate-error"; @@ -271,12 +267,6 @@ export default class RemotelySavePlugin extends Plugin { client.serviceType, this.settings.password ); - const origMetadataOnRemote = await fetchMetadataFile( - metadataFile, - client, - this.app.vault, - this.settings.password - ); if (this.settings.currLogLevel === "info") { // pass @@ -285,10 +275,6 @@ export default class RemotelySavePlugin extends Plugin { } this.syncStatus = "getting_local_meta"; const local = this.app.vault.getAllLoadedFiles(); - const localHistory = await loadFileHistoryTableByVault( - this.db, - this.vaultRandomID - ); let localConfigDirContents: ObsConfigDirFileType[] | undefined = undefined; if (this.settings.syncConfigDir) { @@ -345,21 +331,12 @@ export default class RemotelySavePlugin extends Plugin { plan, sortedKeys, metadataFile, - origMetadataOnRemote, sizesGoWrong, deletions, (key: string) => self.trash(key), this.settings.password, this.settings.concurrency, - (ss: FileOrFolderMixedState[]) => { - new SizesConflictModal( - self.app, - self, - this.settings.skipSizeLargerThan ?? -1, - ss, - this.settings.password !== "" - ).open(); - }, + (i: number, totalCount: number, pathName: string, decision: string) => self.setCurrSyncMsg(i, totalCount, pathName, decision) ); @@ -496,52 +473,6 @@ export default class RemotelySavePlugin extends Plugin { this.syncStatus = "idle"; - this.registerEvent( - this.app.vault.on("delete", async (fileOrFolder) => { - await insertDeleteRecordByVault( - this.db, - fileOrFolder, - this.vaultRandomID - ); - }) - ); - - this.registerEvent( - this.app.vault.on("rename", async (fileOrFolder, oldPath) => { - await insertRenameRecordByVault( - this.db, - fileOrFolder, - oldPath, - this.vaultRandomID - ); - }) - ); - - function getMethods(obj: any) { - var result = []; - for (var id in obj) { - try { - if (typeof obj[id] == "function") { - result.push(id + ": " + obj[id].toString()); - } - } catch (err) { - result.push(id + ": inaccessible"); - } - } - return result.join("\n"); - } - this.registerEvent( - this.app.vault.on("raw" as any, async (fileOrFolder) => { - // special track on .obsidian folder - const name = `${fileOrFolder}`; - if (name.startsWith(this.app.vault.configDir)) { - if (!(await this.app.vault.adapter.exists(name))) { - await insertDeleteRecordByVault(this.db, name, this.vaultRandomID); - } - } - }) - ); - this.registerObsidianProtocolHandler(COMMAND_URI, async (inputParams) => { const parsed = importQrCodeUri(inputParams, this.app.vault.getName()); if (parsed.status === "error") { @@ -814,9 +745,9 @@ export default class RemotelySavePlugin extends Plugin { // log.info("click", evt); // }); - if (!this.settings.agreeToUploadExtraMetadata) { - const syncAlgoV2Modal = new SyncAlgoV2Modal(this.app, this); - syncAlgoV2Modal.open(); + if (!this.settings.agreeToUseSyncV3) { + const syncAlgoV3Modal = new SyncAlgoV3Modal(this.app, this); + syncAlgoV3Modal.open(); } else { this.enableAutoSyncIfSet(); this.enableInitSyncIfSet(); @@ -829,9 +760,6 @@ export default class RemotelySavePlugin extends Plugin { this.vaultRandomID, this.manifest.version ); - if (compareVersion(REMOTELY_SAVE_VERSION_2024PREPARE, oldVersion) >= 0) { - new Notice(t("official_notice_2024_first_party"), 10 * 1000); - } } async onunload() { @@ -918,6 +846,10 @@ export default class RemotelySavePlugin extends Plugin { this.settings.s3.bypassCorsLocally = true; // deprecated as of 20240113 } + if (this.settings.agreeToUseSyncV3 === undefined) { + this.settings.agreeToUseSyncV3 = false; + } + await this.saveSettings(); } diff --git a/src/syncAlgoV3Notice.ts b/src/syncAlgoV3Notice.ts index c016762..34b6913 100644 --- a/src/syncAlgoV3Notice.ts +++ b/src/syncAlgoV3Notice.ts @@ -4,7 +4,7 @@ import type { TransItemType } from "./i18n"; import { log } from "./moreOnLog"; -export class SyncAlgoV2Modal extends Modal { +export class SyncAlgoV3Modal extends Modal { agree: boolean; readonly plugin: RemotelySavePlugin; constructor(app: App, plugin: RemotelySavePlugin) { @@ -19,11 +19,11 @@ export class SyncAlgoV2Modal extends Modal { }; contentEl.createEl("h2", { - text: t("syncalgov2_title"), + text: t("syncalgov3_title"), }); const ul = contentEl.createEl("ul"); - t("syncalgov2_texts") + t("syncalgov3_texts") .split("\n") .forEach((val) => { ul.createEl("li", { @@ -33,14 +33,14 @@ export class SyncAlgoV2Modal extends Modal { new Setting(contentEl) .addButton((button) => { - button.setButtonText(t("syncalgov2_button_agree")); + button.setButtonText(t("syncalgov3_button_agree")); button.onClick(async () => { this.agree = true; this.close(); }); }) .addButton((button) => { - button.setButtonText(t("syncalgov2_button_disagree")); + button.setButtonText(t("syncalgov3_button_disagree")); button.onClick(() => { this.close(); }); From 725b12832cf39cd157db8d842ddeb920c78d0389 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 11:31:23 +0800 Subject: [PATCH 05/21] finally buildable --- src/baseTypes.ts | 2 + src/langs/en.json | 4 +- src/langs/zh_cn.json | 4 +- src/langs/zh_tw.json | 4 +- src/localdb.ts | 33 ++++++++--- src/main.ts | 125 ++++++++++++++++++++++------------------- src/obsFolderLister.ts | 10 +++- src/settings.ts | 4 +- src/sync.ts | 22 ++++++-- 9 files changed, 129 insertions(+), 79 deletions(-) diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 9fc09e1..3770f92 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -104,6 +104,8 @@ export interface RemotelySavePluginSettings { ignorePaths?: string[]; enableStatusBarInfo?: boolean; deleteToWhere?: "system" | "obsidian"; + conflictAction?: ConflictActionType; + howToCleanEmptyFolder?: EmptyFolderCleanType; /** * @deprecated diff --git a/src/langs/en.json b/src/langs/en.json index cd75af2..6ee46cd 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -12,8 +12,8 @@ "syncrun_step2": "2/8 Starting to fetch remote meta data.", "syncrun_step3": "3/8 Checking password correct or not.", "syncrun_passworderr": "Something goes wrong while checking password.", - "syncrun_step4": "4/8 Trying to fetch extra meta data from remote.", - "syncrun_step5": "5/8 Starting to fetch local meta data.", + "syncrun_step4": "4/8 Starting to fetch local meta data.", + "syncrun_step5": "5/8 Starting to fetch local prev sync data.", "syncrun_step6": "6/8 Starting to generate sync plan.", "syncrun_step7": "7/8 Remotely Save Sync data exchanging!", "syncrun_step7skip": "7/8 Remotely Save real sync is skipped in dry run mode.", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index cb128af..aae1e2e 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -12,8 +12,8 @@ "syncrun_step2": "2/8 正在获取远端的元数据。", "syncrun_step3": "3/8 正在检查密码正确与否。", "syncrun_passworderr": "检查密码时候出错。", - "syncrun_step4": "4/8 正在获取远端的额外的元数据。", - "syncrun_step5": "5/8 正在获取本地的元数据。", + "syncrun_step4": "4/8 正在获取本地的元数据。", + "syncrun_step5": "5/8 正在获取本地上一次同步的元数据。", "syncrun_step6": "6/8 正在生成同步计划。", "syncrun_step7": "7/8 Remotely Save 开始发生数据交换！", "syncrun_step7skip": "7/8 Remotely Save 在空跑模式，跳过实际数据交换步骤。", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index cb81eff..7057050 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -12,8 +12,8 @@ "syncrun_step2": "2/8 正在獲取遠端的元資料。", "syncrun_step3": "3/8 正在檢查密碼正確與否。", "syncrun_passworderr": "檢查密碼時候出錯。", - "syncrun_step4": "4/8 正在獲取遠端的額外的元資料。", - "syncrun_step5": "5/8 正在獲取本地的元資料。", + "syncrun_step4": "4/8 正在獲取本地的元資料。", + "syncrun_step5": "5/8 正在獲取本地上一次同步的元資料。", "syncrun_step6": "6/8 正在生成同步計劃。", "syncrun_step7": "7/8 Remotely Save 開始發生資料交換！", "syncrun_step7skip": "7/8 Remotely Save 在空跑模式，跳過實際資料交換步驟。", diff --git a/src/localdb.ts b/src/localdb.ts index ee45c16..c992890 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -329,16 +329,18 @@ export const clearAllSyncMetaMapping = async (db: InternalDBs) => { export const insertSyncPlanRecordByVault = async ( db: InternalDBs, syncPlan: SyncPlanType, - vaultRandomID: string + vaultRandomID: string, + remoteType: SUPPORTED_SERVICES_TYPE ) => { + const now = Date.now(); const record = { - ts: syncPlan.ts, - tsFmt: syncPlan.tsFmt, + ts: now, + tsFmt: unixTimeToStr(now), vaultRandomID: vaultRandomID, - remoteType: syncPlan.remoteType, + remoteType: remoteType, syncPlan: JSON.stringify(syncPlan /* directly stringify */, null, 2), } as SyncPlanRecord; - await db.syncPlansTbl.setItem(`${vaultRandomID}\t${syncPlan.ts}`, record); + await db.syncPlansTbl.setItem(`${vaultRandomID}\t${now}`, record); }; export const clearAllSyncPlanRecords = async (db: InternalDBs) => { @@ -406,6 +408,23 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => { await Promise.all(ps); }; +export const getAllPrevSyncRecordsByVault = async ( + db: InternalDBs, + vaultRandomID: string +) => { + const keys = await db.prevSyncRecordsTbl.keys(); + const res: Entity[] = []; + for (const key of keys) { + if (key.startsWith(`${vaultRandomID}\t`)) { + const val: Entity | null = await db.prevSyncRecordsTbl.getItem(key); + if (val !== null) { + res.push(val); + } + } + } + return res; +}; + export const upsertPrevSyncRecordByVault = async ( db: InternalDBs, vaultRandomID: string, @@ -442,7 +461,7 @@ export const clearAllLoggerOutputRecords = async (db: InternalDBs) => { log.debug(`successfully clearAllLoggerOutputRecords`); }; -export const upsertLastSuccessSyncByVault = async ( +export const upsertLastSuccessSyncTimeByVault = async ( db: InternalDBs, vaultRandomID: string, millis: number @@ -453,7 +472,7 @@ export const upsertLastSuccessSyncByVault = async ( ); }; -export const getLastSuccessSyncByVault = async ( +export const getLastSuccessSyncTimeByVault = async ( db: InternalDBs, vaultRandomID: string ) => { diff --git a/src/main.ts b/src/main.ts index 938cb0b..daaed7b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,8 +7,6 @@ import { setIcon, FileSystemAdapter, Platform, - TFile, - TFolder, requestUrl, requireApiVersion, } from "obsidian"; @@ -23,7 +21,6 @@ import { COMMAND_CALLBACK_ONEDRIVE, COMMAND_CALLBACK_DROPBOX, COMMAND_URI, - REMOTELY_SAVE_VERSION_2024PREPARE, API_VER_ENSURE_REQURL_OK, } from "./baseTypes"; import { importQrCodeUri } from "./importExport"; @@ -32,10 +29,11 @@ import { prepareDBs, InternalDBs, clearExpiredSyncPlanRecords, - upsertLastSuccessSyncByVault, - getLastSuccessSyncByVault, upsertPluginVersionByVault, clearAllLoggerOutputRecords, + upsertLastSuccessSyncTimeByVault, + getLastSuccessSyncTimeByVault, + getAllPrevSyncRecordsByVault, } from "./localdb"; import { RemoteClient } from "./remote"; import { @@ -53,20 +51,22 @@ import { import { DEFAULT_S3_CONFIG } from "./remoteForS3"; import { DEFAULT_WEBDAV_CONFIG } from "./remoteForWebdav"; import { RemotelySaveSettingTab } from "./settings"; -import { parseRemoteItems, SyncStatusType } from "./sync"; -import { doActualSync, getSyncPlan, isPasswordOk } from "./sync"; +import { + doActualSync, + ensembleMixedEnties, + getSyncPlanInplace, + isPasswordOk, + SyncStatusType, +} from "./sync"; import { messyConfigToNormal, normalConfigToMessy } from "./configPersist"; import { getLocalEntityList } from "./local"; import { I18n } from "./i18n"; import type { LangType, LangTypeAndAuto, TransItemType } from "./i18n"; - -import { DeletionOnRemote, MetadataOnRemote } from "./metadataOnRemote"; import { SyncAlgoV3Modal } from "./syncAlgoV3Notice"; import { applyLogWriterInplace, log } from "./moreOnLog"; import AggregateError from "aggregate-error"; import { exportVaultSyncPlansToFiles } from "./debugMode"; -import { SizesConflictModal } from "./syncSizesConflictNotice"; import { compareVersion } from "./misc"; const DEFAULT_SETTINGS: RemotelySavePluginSettings = { @@ -91,6 +91,9 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = { ignorePaths: [], enableStatusBarInfo: true, deleteToWhere: "system", + agreeToUseSyncV3: false, + conflictAction: "keep_newer", + howToCleanEmptyFolder: "skip", }; interface OAuth2Info { @@ -259,33 +262,26 @@ export default class RemotelySavePlugin extends Plugin { } else { getNotice(t("syncrun_step4")); } - this.syncStatus = "getting_remote_extra_meta"; - const { remoteStates, metadataFile } = await parseRemoteItems( - remoteEntityList, - this.db, - this.vaultRandomID, - client.serviceType, - this.settings.password + this.syncStatus = "getting_local_meta"; + const localEntityList = await getLocalEntityList( + this.app.vault, + this.settings.syncConfigDir ?? false, + this.app.vault.configDir, + this.manifest.id ); + // log.info(localEntityList); if (this.settings.currLogLevel === "info") { // pass } else { getNotice(t("syncrun_step5")); } - this.syncStatus = "getting_local_meta"; - const local = this.app.vault.getAllLoadedFiles(); - let localConfigDirContents: ObsConfigDirFileType[] | undefined = - undefined; - if (this.settings.syncConfigDir) { - localConfigDirContents = await listFilesInObsFolder( - this.app.vault.configDir, - this.app.vault, - this.manifest.id - ); - } - // log.info(local); - // log.info(localHistory); + this.syncStatus = "getting_local_prev_sync"; + const prevSyncEntityList = await getAllPrevSyncRecordsByVault( + this.db, + this.vaultRandomID + ); + // log.info(prevSyncEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -293,24 +289,29 @@ export default class RemotelySavePlugin extends Plugin { getNotice(t("syncrun_step6")); } this.syncStatus = "generating_plan"; - const { plan, sortedKeys, deletions, sizesGoWrong } = await getSyncPlan( - remoteStates, - local, - localConfigDirContents, - origMetadataOnRemote.deletions, - localHistory, - client.serviceType, - triggerSource, - this.app.vault, + let mixedEntityMappings = await ensembleMixedEnties( + localEntityList, + prevSyncEntityList, + remoteEntityList, this.settings.syncConfigDir ?? false, this.app.vault.configDir, this.settings.syncUnderscoreItems ?? false, - this.settings.skipSizeLargerThan ?? -1, this.settings.ignorePaths ?? [], this.settings.password ); - log.info(plan.mixedStates); // for debugging - await insertSyncPlanRecordByVault(this.db, plan, this.vaultRandomID); + mixedEntityMappings = await getSyncPlanInplace( + mixedEntityMappings, + this.settings.howToCleanEmptyFolder ?? "skip", + this.settings.skipSizeLargerThan ?? -1, + this.settings.conflictAction ?? "keep_newer" + ); + log.info(mixedEntityMappings); // for debugging + await insertSyncPlanRecordByVault( + this.db, + mixedEntityMappings, + this.vaultRandomID, + client.serviceType + ); // The operations above are almost read only and kind of safe. // The operations below begins to write or delete (!!!) something. @@ -322,23 +323,27 @@ export default class RemotelySavePlugin extends Plugin { getNotice(t("syncrun_step7")); } this.syncStatus = "syncing"; - await doActualSync( + mixedEntityMappings, client, - this.db, this.vaultRandomID, this.app.vault, - plan, - sortedKeys, - metadataFile, - sizesGoWrong, - deletions, - (key: string) => self.trash(key), this.settings.password, - this.settings.concurrency, - - (i: number, totalCount: number, pathName: string, decision: string) => - self.setCurrSyncMsg(i, totalCount, pathName, decision) + this.settings.concurrency ?? 5, + (key: string) => self.trash(key), + ( + realCounter: number, + realTotalCount: number, + pathName: string, + decision: string + ) => + self.setCurrSyncMsg( + realCounter, + realTotalCount, + pathName, + decision + ), + this.db ); } else { this.syncStatus = "syncing"; @@ -359,7 +364,7 @@ export default class RemotelySavePlugin extends Plugin { this.syncStatus = "idle"; const lastSuccessSyncMillis = Date.now(); - await upsertLastSuccessSyncByVault( + await upsertLastSuccessSyncTimeByVault( this.db, this.vaultRandomID, lastSuccessSyncMillis @@ -695,13 +700,13 @@ export default class RemotelySavePlugin extends Plugin { this.statusBarElement.setAttribute("data-tooltip-position", "top"); this.updateLastSuccessSyncMsg( - await getLastSuccessSyncByVault(this.db, this.vaultRandomID) + await getLastSuccessSyncTimeByVault(this.db, this.vaultRandomID) ); // update statusbar text every 30 seconds this.registerInterval( window.setInterval(async () => { this.updateLastSuccessSyncMsg( - await getLastSuccessSyncByVault(this.db, this.vaultRandomID) + await getLastSuccessSyncTimeByVault(this.db, this.vaultRandomID) ); }, 1000 * 30) ); @@ -849,6 +854,12 @@ export default class RemotelySavePlugin extends Plugin { if (this.settings.agreeToUseSyncV3 === undefined) { this.settings.agreeToUseSyncV3 = false; } + if (this.settings.conflictAction === undefined) { + this.settings.conflictAction = "keep_newer"; + } + if (this.settings.howToCleanEmptyFolder === undefined) { + this.settings.howToCleanEmptyFolder = "skip"; + } await this.saveSettings(); } diff --git a/src/obsFolderLister.ts b/src/obsFolderLister.ts index af88774..c12fb76 100644 --- a/src/obsFolderLister.ts +++ b/src/obsFolderLister.ts @@ -4,7 +4,7 @@ import type { Entity, MixedEntity } from "./baseTypes"; import { Queue } from "@fyears/tsqueue"; import chunk from "lodash/chunk"; import flatten from "lodash/flatten"; -import { statFix, isFolderToSkip } from "./misc"; +import { statFix, isSpecialFolderNameToSkip } from "./misc"; const isPluginDirItself = (x: string, pluginId: string) => { return ( @@ -96,7 +96,9 @@ export const listFilesInObsFolder = async ( const isInsideSelfPlugin = isPluginDirItself(iter.itself.key, pluginId); if (iter.children !== undefined) { for (const iter2 of iter.children.folders) { - if (isFolderToSkip(iter2, ["workspace", "workspace.json"])) { + if ( + isSpecialFolderNameToSkip(iter2, ["workspace", "workspace.json"]) + ) { continue; } if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) { @@ -106,7 +108,9 @@ export const listFilesInObsFolder = async ( q.push(iter2); } for (const iter2 of iter.children.files) { - if (isFolderToSkip(iter2, ["workspace", "workspace.json"])) { + if ( + isSpecialFolderNameToSkip(iter2, ["workspace", "workspace.json"]) + ) { continue; } if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) { diff --git a/src/settings.ts b/src/settings.ts index 9ca530d..d758f77 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -26,7 +26,7 @@ import { clearAllSyncMetaMapping, clearAllSyncPlanRecords, destroyDBs, - upsertLastSuccessSyncByVault, + upsertLastSuccessSyncTimeByVault, } from "./localdb"; import type RemotelySavePlugin from "./main"; // unavoidable import { RemoteClient } from "./remote"; @@ -1866,7 +1866,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab { button.setButtonText(t("settings_resetstatusbar_button")); button.onClick(async () => { // reset last sync time - await upsertLastSuccessSyncByVault( + await upsertLastSuccessSyncTimeByVault( this.plugin.db, this.plugin.vaultRandomID, -1 diff --git a/src/sync.ts b/src/sync.ts index 4f3125a..7693d14 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -39,6 +39,18 @@ import { upsertPrevSyncRecordByVault, } from "./localdb"; +export type SyncStatusType = + | "idle" + | "preparing" + | "getting_remote_files_list" + | "getting_local_meta" + | "getting_local_prev_sync" + | "checking_password" + | "generating_plan" + | "syncing" + | "cleaning" + | "finish"; + export interface PasswordCheckType { ok: boolean; reason: @@ -286,7 +298,9 @@ const encryptLocalEntityInplace = async ( return local; }; -const ensembleMixedEnties = async ( +export type SyncPlanType = Record; + +export const ensembleMixedEnties = async ( localEntityList: Entity[], prevSyncEntityList: Entity[], remoteEntityList: Entity[], @@ -296,8 +310,8 @@ const ensembleMixedEnties = async ( syncUnderscoreItems: boolean, ignorePaths: string[], password: string -): Promise> => { - const finalMappings: Record = {}; +): Promise => { + const finalMappings: SyncPlanType = {}; // remote has to be first for (const remote of remoteEntityList) { @@ -863,7 +877,7 @@ const dispatchOperationToActualV3 = async ( } }; -export const doActualSyncV3 = async ( +export const doActualSync = async ( mixedEntityMappings: Record, client: RemoteClient, vaultRandomID: string, From e34b120a631a76c336ca1b0db56d27f5866a56b3 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 11:36:58 +0800 Subject: [PATCH 06/21] upgrade modal --- src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.ts b/src/main.ts index daaed7b..bcdc281 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1080,7 +1080,7 @@ export default class RemotelySavePlugin extends Plugin { } async saveAgreeToUseNewSyncAlgorithm() { - this.settings.agreeToUploadExtraMetadata = true; + this.settings.agreeToUseSyncV3 = true; await this.saveSettings(); } From c512330d84391caeecca20048d6d334da07c1bba Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 11:55:35 +0800 Subject: [PATCH 07/21] yes lots of logging and fix --- src/local.ts | 2 ++ src/main.ts | 6 +++--- src/misc.ts | 19 +++++++++---------- src/sync.ts | 9 +++++++++ 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/local.ts b/src/local.ts index cd70e38..5bcc46b 100644 --- a/src/local.ts +++ b/src/local.ts @@ -50,6 +50,8 @@ export const getLocalEntityList = async ( } else { throw Error(`unexpected ${entry}`); } + + local.push(r); } if (syncConfigDir) { diff --git a/src/main.ts b/src/main.ts index bcdc281..30a3fc3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -240,7 +240,7 @@ export default class RemotelySavePlugin extends Plugin { () => self.saveSettings() ); const remoteEntityList = await client.listAllFromRemote(); - // log.debug(remoteEntityList); + log.debug(remoteEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -269,7 +269,7 @@ export default class RemotelySavePlugin extends Plugin { this.app.vault.configDir, this.manifest.id ); - // log.info(localEntityList); + log.debug(localEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -281,7 +281,7 @@ export default class RemotelySavePlugin extends Plugin { this.db, this.vaultRandomID ); - // log.info(prevSyncEntityList); + log.debug(prevSyncEntityList); if (this.settings.currLogLevel === "info") { // pass diff --git a/src/misc.ts b/src/misc.ts index 69b0122..eaa0e88 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -76,17 +76,16 @@ export const getFolderLevels = (x: string, addEndingSlash: boolean = false) => { export const mkdirpInVault = async (thePath: string, vault: Vault) => { // log.info(thePath); - - // as of 2020219, - // Obsidian can create the folder recursively - // but the path should not end with '/' - if (thePath === "/" || thePath === "") { - return; + const foldersToBuild = getFolderLevels(thePath); + // log.info(foldersToBuild); + for (const folder of foldersToBuild) { + const r = await vault.adapter.exists(folder); + // log.info(r); + if (!r) { + log.info(`mkdir ${folder}`); + await vault.adapter.mkdir(folder); + } } - let thePathNoEnding = thePath.endsWith("/") - ? thePath.slice(0, thePath.length - 1) - : thePath; - await vault.adapter.mkdir(thePathNoEnding); }; /** diff --git a/src/sync.ts b/src/sync.ts index 7693d14..cff90a6 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -442,6 +442,8 @@ export const getSyncPlanInplace = async ( // folder // folder doesn't worry about mtime and size, only check their existences if (keptFolder.has(key)) { + // parent should also be kept + keptFolder.add(getParentFolder(key)); // should fill the missing part if (local !== undefined && remote !== undefined) { mixedEntry.decisionBranch = 101; @@ -731,7 +733,9 @@ const splitThreeStepsOnEntityMappings = ( val.decision === "folder_existed_remote" || val.decision === "folder_to_be_created" ) { + log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); const level = atWhichLevel(key); + log.debug(`atWhichLevel: ${level}`); if (folderCreationOps[level - 1] === undefined) { folderCreationOps[level - 1] = [val]; } else { @@ -891,6 +895,11 @@ export const doActualSync = async ( log.debug(`concurrency === ${concurrency}`); const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } = splitThreeStepsOnEntityMappings(mixedEntityMappings); + log.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`); + log.debug(`deletionOps: ${JSON.stringify(deletionOps)}`); + log.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`); + log.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`); + const nested = [folderCreationOps, deletionOps, uploadDownloads]; const logTexts = [ `1. create all folders from shadowest to deepest, also check undefined decision`, From 40020c3e4400efbb6eba51c4d04e5d20db258fda Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 24 Feb 2024 23:25:15 +0800 Subject: [PATCH 08/21] fix many bugs --- docs/sync_algorithm_v3.md | 12 +++---- src/local.ts | 8 ++--- src/localdb.ts | 6 ++-- src/remoteForS3.ts | 26 +++++++++----- src/sync.ts | 71 ++++++++++++++++++++++++++++----------- 5 files changed, 82 insertions(+), 41 deletions(-) diff --git a/docs/sync_algorithm_v3.md b/docs/sync_algorithm_v3.md index cc50fbc..5ad4e27 100644 --- a/docs/sync_algorithm_v3.md +++ b/docs/sync_algorithm_v3.md @@ -43,9 +43,9 @@ Later runs, use the first, second, third sources **only**. Table modified based on synclone and rsinc. The number inside the table cell is the decision branch in the code. -| local\remote | remote unchanged | remote modified | remote deleted | remote created | -| --------------- | ------------------ | ---------------- | ------------------ | ---------------- | -| local unchanged | (02) do nothing | (09) pull remote | (07) delete local | (??) conflict | -| local modified | (10) push local | (12) conflict | (08) push local | (??) conflict | -| local deleted | (04) delete remote | (05) pull | (01) clean history | (03) pull remote | -| local created | (??) conflict | (??) conflict | (06) push local | (11) conflict | +| local\remote | remote unchanged | remote modified | remote deleted | remote created | +| --------------- | ------------------ | ------------------------- | ------------------ | ------------------------- | +| local unchanged | (02/21) do nothing | (09) pull remote | (07) delete local | (??) conflict | +| local modified | (10) push local | (16/17/18/19/20) conflict | (08) push local | (??) conflict | +| local deleted | (04) delete remote | (05) pull | (01) clean history | (03) pull remote | +| local created | (??) conflict | (??) conflict | (06) push local | (11/12/13/14/15) conflict | diff --git a/src/local.ts b/src/local.ts index 5bcc46b..b128dad 100644 --- a/src/local.ts +++ b/src/local.ts @@ -19,10 +19,10 @@ export const getLocalEntityList = async ( // ignore continue; } else if (entry instanceof TFile) { - let mtimeLocal: number | undefined = Math.max( - entry.stat.mtime ?? 0, - entry.stat.ctime - ); + let mtimeLocal: number | undefined = entry.stat.mtime; + if (mtimeLocal <= 0) { + mtimeLocal = entry.stat.ctime; + } if (mtimeLocal === 0) { mtimeLocal = undefined; } diff --git a/src/localdb.ts b/src/localdb.ts index c992890..7bfe87d 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -412,7 +412,9 @@ export const getAllPrevSyncRecordsByVault = async ( db: InternalDBs, vaultRandomID: string ) => { + // log.debug('inside getAllPrevSyncRecordsByVault') const keys = await db.prevSyncRecordsTbl.keys(); + // log.debug(`inside getAllPrevSyncRecordsByVault, keys=${keys}`) const res: Entity[] = []; for (const key of keys) { if (key.startsWith(`${vaultRandomID}\t`)) { @@ -431,7 +433,7 @@ export const upsertPrevSyncRecordByVault = async ( prevSync: Entity ) => { await db.prevSyncRecordsTbl.setItem( - `${vaultRandomID}-${prevSync.key}`, + `${vaultRandomID}\t${prevSync.key}`, prevSync ); }; @@ -441,7 +443,7 @@ export const clearPrevSyncRecordByVault = async ( vaultRandomID: string, key: string ) => { - await db.prevSyncRecordsTbl.removeItem(`${vaultRandomID}-${key}`); + await db.prevSyncRecordsTbl.removeItem(`${vaultRandomID}\t${key}`); }; export const clearAllPrevSyncRecordByVault = async ( diff --git a/src/remoteForS3.ts b/src/remoteForS3.ts index 00924da..fce0783 100644 --- a/src/remoteForS3.ts +++ b/src/remoteForS3.ts @@ -226,7 +226,9 @@ const fromS3ObjectToEntity = ( mtimeRecords: Record, ctimeRecords: Record ) => { - const mtimeSvr = x.LastModified!.valueOf(); + // log.debug(`fromS3ObjectToEntity: ${x.Key!}, ${JSON.stringify(x,null,2)}`); + // S3 officially only supports seconds precision!!!!! + const mtimeSvr = Math.floor(x.LastModified!.valueOf() / 1000.0) * 1000; let mtimeCli = mtimeSvr; if (x.Key! in mtimeRecords) { const m2 = mtimeRecords[x.Key!]; @@ -250,12 +252,13 @@ const fromS3ObjectToEntity = ( const fromS3HeadObjectToEntity = ( fileOrFolderPathWithRemotePrefix: string, x: HeadObjectCommandOutput, - remotePrefix: string, - useAccurateMTime: boolean + remotePrefix: string ) => { - const mtimeSvr = x.LastModified!.valueOf(); + // log.debug(`fromS3HeadObjectToEntity: ${fileOrFolderPathWithRemotePrefix}: ${JSON.stringify(x,null,2)}`); + // S3 officially only supports seconds precision!!!!! + const mtimeSvr = Math.floor(x.LastModified!.valueOf() / 1000.0) * 1000; let mtimeCli = mtimeSvr; - if (useAccurateMTime && x.Metadata !== undefined) { + if (x.Metadata !== undefined) { const m2 = Math.round( parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0") ); @@ -338,8 +341,7 @@ export const getRemoteMeta = async ( return fromS3HeadObjectToEntity( fileOrFolderPathWithRemotePrefix, res, - s3Config.remotePrefix ?? "", - s3Config.useAccurateMTime ?? false + s3Config.remotePrefix ?? "" ); }; @@ -356,6 +358,7 @@ export const uploadToRemote = async ( rawContentMTime: number = 0, rawContentCTime: number = 0 ) => { + log.debug(`uploading ${fileOrFolderPath}`); let uploadFile = fileOrFolderPath; if (password !== "") { uploadFile = remoteEncryptedKey; @@ -390,7 +393,8 @@ export const uploadToRemote = async ( }, }) ); - return await getRemoteMeta(s3Client, s3Config, uploadFile); + const res = await getRemoteMeta(s3Client, s3Config, uploadFile); + return res; } else { // file // we ignore isRecursively parameter here @@ -454,7 +458,11 @@ export const uploadToRemote = async ( }); await upload.done(); - return await getRemoteMeta(s3Client, s3Config, uploadFile); + const res = await getRemoteMeta(s3Client, s3Config, uploadFile); + log.debug( + `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}` + ); + return res; } }; diff --git a/src/sync.ts b/src/sync.ts index cff90a6..7be1c5a 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -274,9 +274,15 @@ const encryptLocalEntityInplace = async ( remoteKeyEnc: string | undefined ) => { if (password == undefined || password === "") { + local.sizeEnc = local.size!; // if no enc, the remote file has the same size + local.keyEnc = local.key; return local; } + + // below is for having password + if (local.size === local.sizeEnc) { + // size not transformed yet, we need to compute sizeEnc local.sizeEnc = getSizeFromOrigToEnc(local.size); } if (local.key === local.keyEnc) { @@ -438,11 +444,14 @@ export const getSyncPlanInplace = async ( const mixedEntry = mixedEntityMappings[key]; const { local, prevSync, remote } = mixedEntry; + // log.debug(`getSyncPlanInplace: key=${key}`) + if (key.endsWith("/")) { // folder // folder doesn't worry about mtime and size, only check their existences if (keptFolder.has(key)) { // parent should also be kept + // log.debug(`${key} in keptFolder`) keptFolder.add(getParentFolder(key)); // should fill the missing part if (local !== undefined && remote !== undefined) { @@ -494,7 +503,7 @@ export const getSyncPlanInplace = async ( // Look for past files of A or B. const localEqualPrevSync = - prevSync?.mtimeSvr === local.mtimeCli && + prevSync?.mtimeCli === local.mtimeCli && prevSync?.sizeEnc === local.sizeEnc; const remoteEqualPrevSync = (prevSync?.mtimeSvr === remote.mtimeCli || @@ -595,12 +604,12 @@ export const getSyncPlanInplace = async ( } } } else { - // Both compare true -- This is VERY odd and should not happen - throw Error( - `should not reach branch -2 while getting sync plan: ${JSON.stringify( - mixedEntry - )}` - ); + // Both compare true. + // This is likely because of the mtimeCli and mtimeSvr tricks. + // The result should be equal!!! + mixedEntry.decisionBranch = 21; + mixedEntry.decision = "equal"; + keptFolder.add(getParentFolder(key)); } } } else if (local === undefined && remote !== undefined) { @@ -657,7 +666,8 @@ export const getSyncPlanInplace = async ( ); } } else if ( - prevSync.mtimeSvr === local.mtimeCli && + (prevSync.mtimeSvr === local.mtimeCli || + prevSync.mtimeCli === local.mtimeCli) && prevSync.sizeEnc === local.sizeEnc ) { // if A is in the previous list and UNMODIFIED, A has been deleted by B @@ -707,9 +717,10 @@ export const getSyncPlanInplace = async ( const splitThreeStepsOnEntityMappings = ( mixedEntityMappings: Record ) => { - const folderCreationOps: MixedEntity[][] = []; - const deletionOps: MixedEntity[][] = []; - const uploadDownloads: MixedEntity[][] = []; + type StepArrayType = MixedEntity[] | undefined | null; + const folderCreationOps: StepArrayType[] = []; + const deletionOps: StepArrayType[] = []; + const uploadDownloads: StepArrayType[] = []; // from long(deep) to short(shadow) const sortedKeys = Object.keys(mixedEntityMappings).sort( @@ -736,10 +747,11 @@ const splitThreeStepsOnEntityMappings = ( log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); const level = atWhichLevel(key); log.debug(`atWhichLevel: ${level}`); - if (folderCreationOps[level - 1] === undefined) { + const k = folderCreationOps[level - 1]; + if (k === undefined || k === null) { folderCreationOps[level - 1] = [val]; } else { - folderCreationOps[level - 1].push(val); + k.push(val); } realTotalCount += 1; } else if ( @@ -749,10 +761,11 @@ const splitThreeStepsOnEntityMappings = ( val.decision === "folder_to_be_deleted" ) { const level = atWhichLevel(key); - if (deletionOps[level - 1] === undefined) { + const k = deletionOps[level - 1]; + if (k === undefined || k === null) { deletionOps[level - 1] = [val]; } else { - deletionOps[level - 1].push(val); + k.push(val); } realTotalCount += 1; } else if ( @@ -767,10 +780,14 @@ const splitThreeStepsOnEntityMappings = ( val.decision === "conflict_modified_keep_remote" || val.decision === "conflict_modified_keep_both" ) { - if (uploadDownloads.length === 0) { + if ( + uploadDownloads.length === 0 || + uploadDownloads[0] === undefined || + uploadDownloads[0] === null + ) { uploadDownloads[0] = [val]; } else { - uploadDownloads[0].push(val); // only one level needed here + uploadDownloads[0].push(val); // only one level is needed here } realTotalCount += 1; } else { @@ -801,6 +818,13 @@ const dispatchOperationToActualV3 = async ( localDeleteFunc: any, password: string ) => { + log.debug( + `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify( + r, + null, + 2 + )}` + ); if (r.decision === "only_history") { clearPrevSyncRecordByVault(db, vaultRandomID, key); } else if ( @@ -852,10 +876,12 @@ const dispatchOperationToActualV3 = async ( ); await upsertPrevSyncRecordByVault(db, vaultRandomID, r.remote!); } else if (r.decision === "deleted_local") { - await localDeleteFunc(r.key); + // local is deleted, we need to delete remote now + await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); } else if (r.decision === "deleted_remote") { - await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); + // remote is deleted, we need to delete local now + await localDeleteFunc(r.key); await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); } else if ( r.decision === "conflict_created_keep_both" || @@ -902,7 +928,7 @@ export const doActualSync = async ( const nested = [folderCreationOps, deletionOps, uploadDownloads]; const logTexts = [ - `1. create all folders from shadowest to deepest, also check undefined decision`, + `1. create all folders from shadowest to deepest`, `2. delete files and folders from deepest to shadowest`, `3. upload or download files in parallel, with the desired concurrency=${concurrency}`, ]; @@ -912,9 +938,14 @@ export const doActualSync = async ( log.debug(logTexts[i]); const operations = nested[i]; + log.debug(`curr operations=${JSON.stringify(operations, null, 2)}`); for (let j = 0; j < operations.length; ++j) { const singleLevelOps = operations[j]; + log.debug(`singleLevelOps=${singleLevelOps}`); + if (singleLevelOps === undefined || singleLevelOps === null) { + continue; + } const queue = new PQueue({ concurrency: concurrency, autoStart: true }); const potentialErrors: Error[] = []; From 1f4737bfb83049bcbba76d0f88dfbabbab71da06 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 25 Feb 2024 15:14:28 +0800 Subject: [PATCH 09/21] fix for encryption --- src/baseTypes.ts | 8 +- src/local.ts | 12 +-- src/main.ts | 4 + src/obsFolderLister.ts | 8 +- src/remoteForDropbox.ts | 17 ++-- src/remoteForOnedrive.ts | 11 ++- src/remoteForS3.ts | 32 +++++-- src/remoteForWebdav.ts | 11 ++- src/sync.ts | 186 ++++++++++++++++++++++++--------------- 9 files changed, 182 insertions(+), 107 deletions(-) diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 3770f92..81fd705 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -169,8 +169,9 @@ export type DecisionTypeForMixedEntity = * everything should be flat and primitive, so that we can copy. */ export interface Entity { - key: string; - keyEnc: string; + key?: string; + keyEnc?: string; + keyRaw: string; mtimeCli?: number; mtimeCliFmt?: string; mtimeSvr?: number; @@ -178,7 +179,8 @@ export interface Entity { prevSyncTime?: number; prevSyncTimeFmt?: string; size?: number; // might be unknown or to be filled - sizeEnc: number; + sizeEnc?: number; + sizeRaw: number; hash?: string; etag?: string; } diff --git a/src/local.ts b/src/local.ts index b128dad..e1ef7ab 100644 --- a/src/local.ts +++ b/src/local.ts @@ -32,20 +32,20 @@ export const getLocalEntityList = async ( ); } r = { - key: entry.path, - keyEnc: entry.path, + key: entry.path, // local always unencrypted + keyRaw: entry.path, mtimeCli: mtimeLocal, mtimeSvr: mtimeLocal, - size: entry.stat.size, - sizeEnc: entry.stat.size, + size: entry.stat.size, // local always unencrypted + sizeRaw: entry.stat.size, }; } else if (entry instanceof TFolder) { key = `${entry.path}/`; r = { key: key, - keyEnc: key, + keyRaw: key, size: 0, - sizeEnc: 0, + sizeRaw: 0, }; } else { throw Error(`unexpected ${entry}`); diff --git a/src/main.ts b/src/main.ts index 30a3fc3..253b922 100644 --- a/src/main.ts +++ b/src/main.ts @@ -240,6 +240,7 @@ export default class RemotelySavePlugin extends Plugin { () => self.saveSettings() ); const remoteEntityList = await client.listAllFromRemote(); + log.debug("remoteEntityList:"); log.debug(remoteEntityList); if (this.settings.currLogLevel === "info") { @@ -269,6 +270,7 @@ export default class RemotelySavePlugin extends Plugin { this.app.vault.configDir, this.manifest.id ); + log.debug("localEntityList:"); log.debug(localEntityList); if (this.settings.currLogLevel === "info") { @@ -281,6 +283,7 @@ export default class RemotelySavePlugin extends Plugin { this.db, this.vaultRandomID ); + log.debug("prevSyncEntityList:"); log.debug(prevSyncEntityList); if (this.settings.currLogLevel === "info") { @@ -305,6 +308,7 @@ export default class RemotelySavePlugin extends Plugin { this.settings.skipSizeLargerThan ?? -1, this.settings.conflictAction ?? "keep_newer" ); + log.info(`mixedEntityMappings:`); log.info(mixedEntityMappings); // for debugging await insertSyncPlanRecordByVault( this.db, diff --git a/src/obsFolderLister.ts b/src/obsFolderLister.ts index c12fb76..0109366 100644 --- a/src/obsFolderLister.ts +++ b/src/obsFolderLister.ts @@ -79,12 +79,12 @@ export const listFilesInObsFolder = async ( return { itself: { - key: isFolder ? `${x}/` : x, - keyEnc: isFolder ? `${x}/` : x, + key: isFolder ? `${x}/` : x, // local always unencrypted + keyRaw: isFolder ? `${x}/` : x, mtimeCli: statRes.mtime, mtimeSvr: statRes.mtime, - size: statRes.size, - sizeEnc: statRes.size, + size: statRes.size, // local always unencrypted + sizeRaw: statRes.size, }, children: children, }; diff --git a/src/remoteForDropbox.ts b/src/remoteForDropbox.ts index 18d4cde..5052049 100644 --- a/src/remoteForDropbox.ts +++ b/src/remoteForDropbox.ts @@ -83,22 +83,18 @@ const fromDropboxItemToEntity = ( if (x[".tag"] === "folder") { return { - key: key, - keyEnc: key, - size: 0, - sizeEnc: 0, + keyRaw: key, + sizeRaw: 0, etag: `${x.id}\t`, } as Entity; } else if (x[".tag"] === "file") { const mtimeCli = Date.parse(x.client_modified).valueOf(); const mtimeSvr = Date.parse(x.server_modified).valueOf(); return { - key: key, - keyEnc: key, + keyRaw: key, mtimeCli: mtimeCli, mtimeSvr: mtimeSvr, - size: x.size, - sizeEnc: x.size, + sizeRaw: x.size, hash: x.content_hash, etag: `${x.id}\t${x.content_hash}`, } as Entity; @@ -469,6 +465,11 @@ export const uploadToRemote = async ( let uploadFile = fileOrFolderPath; if (password !== "") { + if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { + throw Error( + `uploadToRemote(dropbox) you have password but remoteEncryptedKey is empty!` + ); + } uploadFile = remoteEncryptedKey; } uploadFile = getDropboxPath(uploadFile, client.remoteBaseDir); diff --git a/src/remoteForOnedrive.ts b/src/remoteForOnedrive.ts index ec2759d..f2c64bc 100644 --- a/src/remoteForOnedrive.ts +++ b/src/remoteForOnedrive.ts @@ -351,12 +351,10 @@ const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => { const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!); const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!); return { - key: key, - keyEnc: key, + keyRaw: key, mtimeSvr: mtimeSvr, mtimeCli: mtimeCli, - size: isFolder ? 0 : x.size!, - sizeEnc: isFolder ? 0 : x.size!, + sizeRaw: isFolder ? 0 : x.size!, // hash: ?? // TODO etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes }; @@ -708,6 +706,11 @@ export const uploadToRemote = async ( let uploadFile = fileOrFolderPath; if (password !== "") { + if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { + throw Error( + `uploadToRemote(onedrive) you have password but remoteEncryptedKey is empty!` + ); + } uploadFile = remoteEncryptedKey; } uploadFile = getOnedrivePath(uploadFile, client.remoteBaseDir); diff --git a/src/remoteForS3.ts b/src/remoteForS3.ts index fce0783..b367b56 100644 --- a/src/remoteForS3.ts +++ b/src/remoteForS3.ts @@ -238,12 +238,10 @@ const fromS3ObjectToEntity = ( } const key = getLocalNoPrefixPath(x.Key!, remotePrefix); const r: Entity = { - key: key, - keyEnc: key, + keyRaw: key, mtimeSvr: mtimeSvr, mtimeCli: mtimeCli, - size: x.Size!, - sizeEnc: x.Size!, + sizeRaw: x.Size!, etag: x.ETag, }; return r; @@ -266,11 +264,21 @@ const fromS3HeadObjectToEntity = ( mtimeCli = m2; } } + // log.debug( + // `fromS3HeadObjectToEntity, fileOrFolderPathWithRemotePrefix=${fileOrFolderPathWithRemotePrefix}, remotePrefix=${remotePrefix}, x=${JSON.stringify( + // x + // )} ` + // ); + const key = getLocalNoPrefixPath( + fileOrFolderPathWithRemotePrefix, + remotePrefix + ); + // log.debug(`fromS3HeadObjectToEntity, key=${key} after removing prefix`); return { - key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix), + keyRaw: key, mtimeSvr: mtimeSvr, mtimeCli: mtimeCli, - size: x.ContentLength, + sizeRaw: x.ContentLength, etag: x.ETag, } as Entity; }; @@ -361,9 +369,15 @@ export const uploadToRemote = async ( log.debug(`uploading ${fileOrFolderPath}`); let uploadFile = fileOrFolderPath; if (password !== "") { + if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { + throw Error( + `uploadToRemote(s3) you have password but remoteEncryptedKey is empty!` + ); + } uploadFile = remoteEncryptedKey; } uploadFile = getRemoteWithPrefixPath(uploadFile, s3Config.remotePrefix ?? ""); + // log.debug(`actual uploadFile=${uploadFile}`); const isFolder = fileOrFolderPath.endsWith("/"); if (isFolder && isRecursively) { @@ -459,9 +473,9 @@ export const uploadToRemote = async ( await upload.done(); const res = await getRemoteMeta(s3Client, s3Config, uploadFile); - log.debug( - `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}` - ); + // log.debug( + // `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}` + // ); return res; } }; diff --git a/src/remoteForWebdav.ts b/src/remoteForWebdav.ts index fe44443..6ad30cb 100644 --- a/src/remoteForWebdav.ts +++ b/src/remoteForWebdav.ts @@ -212,12 +212,10 @@ const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string) => { } const mtimeSvr = Date.parse(x.lastmod).valueOf(); return { - key: key, - keyEnc: key, + keyRaw: key, mtimeSvr: mtimeSvr, mtimeCli: mtimeSvr, // no universal way to set mtime in webdav - size: x.size, - sizeEnc: x.size, + sizeRaw: x.size, etag: x.etag, } as Entity; }; @@ -346,6 +344,11 @@ export const uploadToRemote = async ( await client.init(); let uploadFile = fileOrFolderPath; if (password !== "") { + if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { + throw Error( + `uploadToRemote(webdav) you have password but remoteEncryptedKey is empty!` + ); + } uploadFile = remoteEncryptedKey; } uploadFile = getWebdavPath(uploadFile, client.remoteBaseDir); diff --git a/src/sync.ts b/src/sync.ts index 7be1c5a..d0a3d2d 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -75,7 +75,7 @@ export const isPasswordOk = async ( reason: "empty_remote", }; } - const santyCheckKey = remote[0].key; + const santyCheckKey = remote[0].keyRaw; if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { // this is encrypted using old base32! // try to decrypt it using the provided password. @@ -161,6 +161,9 @@ const isSkipItemByName = ( configDir: string, ignorePaths: string[] ) => { + if (key === undefined) { + throw Error(`isSkipItemByName meets undefinded key!`); + } if (ignorePaths !== undefined && ignorePaths.length > 0) { for (const r of ignorePaths) { if (XRegExp(r, "A").test(key)) { @@ -218,17 +221,25 @@ const copyEntityAndFixTimeFormat = (src: Entity) => { */ const decryptRemoteEntityInplace = async (remote: Entity, password: string) => { if (password == undefined || password === "") { - remote.key = remote.keyEnc; - remote.size = remote.sizeEnc; + remote.key = remote.keyRaw; + remote.keyEnc = remote.keyRaw; + remote.size = remote.sizeRaw; + remote.sizeEnc = remote.sizeRaw; return remote; } - if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { + if (remote.keyRaw.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { + remote.keyEnc = remote.keyRaw; remote.key = await decryptBase32ToString(remote.keyEnc, password); - } else if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) { + remote.sizeEnc = remote.sizeRaw; + } else if (remote.keyRaw.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) { + remote.keyEnc = remote.keyRaw; remote.key = await decryptBase64urlToString(remote.keyEnc, password); + remote.sizeEnc = remote.sizeRaw; } else { - throw Error(`unexpected key to decrypt=${remote.keyEnc}`); + throw Error( + `unexpected key to decrypt: ${JSON.stringify(remote, null, 2)}` + ); } // TODO @@ -245,7 +256,7 @@ const decryptRemoteEntityInplace = async (remote: Entity, password: string) => { */ const ensureMTimeOfRemoteEntityValid = (remote: Entity) => { if ( - !remote.key.endsWith("/") && + !remote.key!.endsWith("/") && remote.mtimeCli === undefined && remote.mtimeSvr === undefined ) { @@ -273,19 +284,36 @@ const encryptLocalEntityInplace = async ( password: string, remoteKeyEnc: string | undefined ) => { - if (password == undefined || password === "") { - local.sizeEnc = local.size!; // if no enc, the remote file has the same size - local.keyEnc = local.key; + // log.debug( + // `encryptLocalEntityInplace: local=${JSON.stringify( + // local, + // null, + // 2 + // )}, password=${ + // password === undefined || password === "" ? "[empty]" : "[not empty]" + // }, remoteKeyEnc=${remoteKeyEnc}` + // ); + + if (local.key === undefined) { + // local.key should always have value + throw Error(`local ${local.keyRaw} is abnormal without key`); + } + + if (password === undefined || password === "") { + local.sizeEnc = local.sizeRaw; // if no enc, the remote file has the same size + local.keyEnc = local.keyRaw; return local; } // below is for having password - - if (local.size === local.sizeEnc) { - // size not transformed yet, we need to compute sizeEnc + if (local.sizeEnc === undefined && local.size !== undefined) { + // it's not filled yet, we fill it + // local.size is possibly undefined if it's "prevSync" Entity + // but local.key should always have value local.sizeEnc = getSizeFromOrigToEnc(local.size); } - if (local.key === local.keyEnc) { + + if (local.keyEnc === undefined || local.keyEnc === "") { if ( remoteKeyEnc !== undefined && remoteKeyEnc !== "" && @@ -328,7 +356,7 @@ export const ensembleMixedEnties = async ( ) ); - const key = remoteCopied.key; + const key = remoteCopied.key!; if ( isSkipItemByName( key, @@ -347,37 +375,47 @@ export const ensembleMixedEnties = async ( }; } - for (const prevSync of prevSyncEntityList) { - const key = prevSync.key; - if ( - isSkipItemByName( - key, - syncConfigDir, - syncUnderscoreItems, - configDir, - ignorePaths - ) - ) { - continue; - } + if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) { + // Special checking: + // if one side is totally empty, + // usually that's a hard rest. + // So we need to ignore everything of prevSyncEntityList to avoid deletions! + // TODO: acutally erase everything of prevSyncEntityList? + // TODO: local should also go through a isSkipItemByName checking beforehand + } else { + // normally go through the prevSyncEntityList + for (const prevSync of prevSyncEntityList) { + const key = prevSync.key!; + if ( + isSkipItemByName( + key, + syncConfigDir, + syncUnderscoreItems, + configDir, + ignorePaths + ) + ) { + continue; + } - if (finalMappings.hasOwnProperty(key)) { - const prevSyncCopied = await encryptLocalEntityInplace( - copyEntityAndFixTimeFormat(prevSync), - password, - finalMappings[key].remote?.keyEnc - ); - finalMappings[key].prevSync = prevSyncCopied; - } else { - const prevSyncCopied = await encryptLocalEntityInplace( - copyEntityAndFixTimeFormat(prevSync), - password, - undefined - ); - finalMappings[key] = { - key: key, - prevSync: prevSyncCopied, - }; + if (finalMappings.hasOwnProperty(key)) { + const prevSyncCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(prevSync), + password, + finalMappings[key].remote?.keyEnc + ); + finalMappings[key].prevSync = prevSyncCopied; + } else { + const prevSyncCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(prevSync), + password, + undefined + ); + finalMappings[key] = { + key: key, + prevSync: prevSyncCopied, + }; + } } } @@ -385,7 +423,7 @@ export const ensembleMixedEnties = async ( // because we want to get keyEnc based on the remote // (we don't consume prevSync here because it gains no benefit) for (const local of localEntityList) { - const key = local.key; + const key = local.key!; if ( isSkipItemByName( key, @@ -514,7 +552,7 @@ export const getSyncPlanInplace = async ( // If only one compares true (no prev also means it compares False), the other is modified. Backup and sync. if ( skipSizeLargerThan <= 0 || - remote.sizeEnc <= skipSizeLargerThan + remote.sizeEnc! <= skipSizeLargerThan ) { mixedEntry.decisionBranch = 9; mixedEntry.decision = "modified_remote"; @@ -530,7 +568,7 @@ export const getSyncPlanInplace = async ( // If only one compares true (no prev also means it compares False), the other is modified. Backup and sync. if ( skipSizeLargerThan <= 0 || - local.sizeEnc <= skipSizeLargerThan + local.sizeEnc! <= skipSizeLargerThan ) { mixedEntry.decisionBranch = 10; mixedEntry.decision = "modified_local"; @@ -559,7 +597,7 @@ export const getSyncPlanInplace = async ( keptFolder.add(getParentFolder(key)); } } else if (conflictAction === "keep_larger") { - if (local.sizeEnc >= remote.sizeEnc) { + if (local.sizeEnc! >= remote.sizeEnc!) { mixedEntry.decisionBranch = 13; mixedEntry.decision = "conflict_created_keep_local"; keptFolder.add(getParentFolder(key)); @@ -588,7 +626,7 @@ export const getSyncPlanInplace = async ( keptFolder.add(getParentFolder(key)); } } else if (conflictAction === "keep_larger") { - if (local.sizeEnc >= remote.sizeEnc) { + if (local.sizeEnc! >= remote.sizeEnc!) { mixedEntry.decisionBranch = 18; mixedEntry.decision = "conflict_modified_keep_local"; keptFolder.add(getParentFolder(key)); @@ -616,7 +654,10 @@ export const getSyncPlanInplace = async ( // A is missing if (prevSync === undefined) { // if B is not in the previous list, B is new - if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) { + if ( + skipSizeLargerThan <= 0 || + remote.sizeEnc! <= skipSizeLargerThan + ) { mixedEntry.decisionBranch = 3; mixedEntry.decision = "created_remote"; keptFolder.add(getParentFolder(key)); @@ -637,7 +678,10 @@ export const getSyncPlanInplace = async ( mixedEntry.decision = "deleted_local"; } else { // if B is in the previous list and MODIFIED, B has been deleted by A but modified by B - if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) { + if ( + skipSizeLargerThan <= 0 || + remote.sizeEnc! <= skipSizeLargerThan + ) { mixedEntry.decisionBranch = 5; mixedEntry.decision = "modified_remote"; keptFolder.add(getParentFolder(key)); @@ -654,7 +698,7 @@ export const getSyncPlanInplace = async ( if (prevSync === undefined) { // if A is not in the previous list, A is new - if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) { + if (skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan) { mixedEntry.decisionBranch = 6; mixedEntry.decision = "created_local"; keptFolder.add(getParentFolder(key)); @@ -675,7 +719,7 @@ export const getSyncPlanInplace = async ( mixedEntry.decision = "deleted_remote"; } else { // if A is in the previous list and MODIFIED, A has been deleted by B but modified by A - if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) { + if (skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan) { mixedEntry.decisionBranch = 8; mixedEntry.decision = "modified_local"; keptFolder.add(getParentFolder(key)); @@ -744,9 +788,9 @@ const splitThreeStepsOnEntityMappings = ( val.decision === "folder_existed_remote" || val.decision === "folder_to_be_created" ) { - log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); + // log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); const level = atWhichLevel(key); - log.debug(`atWhichLevel: ${level}`); + // log.debug(`atWhichLevel: ${level}`); const k = folderCreationOps[level - 1]; if (k === undefined || k === null) { folderCreationOps[level - 1] = [val]; @@ -818,13 +862,13 @@ const dispatchOperationToActualV3 = async ( localDeleteFunc: any, password: string ) => { - log.debug( - `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify( - r, - null, - 2 - )}` - ); + // log.debug( + // `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify( + // r, + // null, + // 2 + // )}` + // ); if (r.decision === "only_history") { clearPrevSyncRecordByVault(db, vaultRandomID, key); } else if ( @@ -850,6 +894,7 @@ const dispatchOperationToActualV3 = async ( // special treatment for OneDrive: do nothing, skip empty file without encryption // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded. } else { + // log.debug(`before upload in sync, r=${JSON.stringify(r, null, 2)}`); const remoteObjMeta = await client.uploadToRemote( r.key, vault, @@ -857,6 +902,7 @@ const dispatchOperationToActualV3 = async ( password, r.local!.keyEnc ); + await decryptRemoteEntityInplace(remoteObjMeta, password); await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); } } else if ( @@ -897,6 +943,8 @@ const dispatchOperationToActualV3 = async ( password, r.local!.keyEnc ); + // we need to decrypt the key!!! + await decryptRemoteEntityInplace(remoteObjMeta, password); await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); } else if (r.decision === "folder_to_be_deleted") { await localDeleteFunc(r.key); @@ -921,10 +969,10 @@ export const doActualSync = async ( log.debug(`concurrency === ${concurrency}`); const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } = splitThreeStepsOnEntityMappings(mixedEntityMappings); - log.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`); - log.debug(`deletionOps: ${JSON.stringify(deletionOps)}`); - log.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`); - log.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`); + // log.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`); + // log.debug(`deletionOps: ${JSON.stringify(deletionOps)}`); + // log.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`); + // log.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`); const nested = [folderCreationOps, deletionOps, uploadDownloads]; const logTexts = [ @@ -938,11 +986,11 @@ export const doActualSync = async ( log.debug(logTexts[i]); const operations = nested[i]; - log.debug(`curr operations=${JSON.stringify(operations, null, 2)}`); + // log.debug(`curr operations=${JSON.stringify(operations, null, 2)}`); for (let j = 0; j < operations.length; ++j) { const singleLevelOps = operations[j]; - log.debug(`singleLevelOps=${singleLevelOps}`); + log.debug(`singleLevelOps=${JSON.stringify(singleLevelOps, null, 2)}`); if (singleLevelOps === undefined || singleLevelOps === null) { continue; } From 5df06bbed58c62938bcc9876daceaa87ee979de5 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 25 Feb 2024 22:18:14 +0800 Subject: [PATCH 10/21] fix mtime for webdav --- src/baseTypes.ts | 5 +++++ src/remote.ts | 3 ++- src/remoteForDropbox.ts | 18 ++++++++++++++---- src/remoteForOnedrive.ts | 18 ++++++++++++++---- src/remoteForS3.ts | 13 ++++++++++--- src/remoteForWebdav.ts | 21 +++++++++++++++------ src/sync.ts | 34 +++++++++++++++++++++++++++------- 7 files changed, 87 insertions(+), 25 deletions(-) diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 81fd705..38024f4 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -185,6 +185,11 @@ export interface Entity { etag?: string; } +export interface UploadedType { + entity: Entity; + mtimeCli?: number; +} + /** * A replacement of FileOrFolderMixedState */ diff --git a/src/remote.ts b/src/remote.ts index 68e5570..c7467b1 100644 --- a/src/remote.ts +++ b/src/remote.ts @@ -6,6 +6,7 @@ import type { S3Config, SUPPORTED_SERVICES_TYPE, WebdavConfig, + UploadedType, } from "./baseTypes"; import * as dropbox from "./remoteForDropbox"; import * as onedrive from "./remoteForOnedrive"; @@ -112,7 +113,7 @@ export class RemoteClient { foldersCreatedBefore: Set | undefined = undefined, uploadRaw: boolean = false, rawContent: string | ArrayBuffer = "" - ) => { + ): Promise => { if (this.serviceType === "s3") { return await s3.uploadToRemote( s3.getS3Client(this.s3Config!), diff --git a/src/remoteForDropbox.ts b/src/remoteForDropbox.ts index 5052049..a084919 100644 --- a/src/remoteForDropbox.ts +++ b/src/remoteForDropbox.ts @@ -8,6 +8,7 @@ import { Entity, COMMAND_CALLBACK_DROPBOX, OAUTH2_FORCE_EXPIRE_MILLISECONDS, + UploadedType, } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { @@ -460,7 +461,7 @@ export const uploadToRemote = async ( rawContent: string | ArrayBuffer = "", rawContentMTime: number = 0, rawContentCTime: number = 0 -) => { +): Promise => { await client.init(); let uploadFile = fileOrFolderPath; @@ -526,7 +527,10 @@ export const uploadToRemote = async ( } } const res = await getRemoteMeta(client, uploadFile); - return res; + return { + entity: res, + mtimeCli: mtime, + }; } else { // if encrypted, upload a fake file with the encrypted file name await retryReq( @@ -538,7 +542,10 @@ export const uploadToRemote = async ( }), fileOrFolderPath ); - return await getRemoteMeta(client, uploadFile); + return { + entity: await getRemoteMeta(client, uploadFile), + mtimeCli: mtime, + }; } } else { // file @@ -587,7 +594,10 @@ export const uploadToRemote = async ( foldersCreatedBefore?.add(dir); } } - return await getRemoteMeta(client, uploadFile); + return { + entity: await getRemoteMeta(client, uploadFile), + mtimeCli: mtime, + }; } }; diff --git a/src/remoteForOnedrive.ts b/src/remoteForOnedrive.ts index f2c64bc..52cbb4b 100644 --- a/src/remoteForOnedrive.ts +++ b/src/remoteForOnedrive.ts @@ -15,6 +15,7 @@ import { OAUTH2_FORCE_EXPIRE_MILLISECONDS, OnedriveConfig, Entity, + UploadedType, } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { @@ -701,7 +702,7 @@ export const uploadToRemote = async ( foldersCreatedBefore: Set | undefined = undefined, uploadRaw: boolean = false, rawContent: string | ArrayBuffer = "" -) => { +): Promise => { await client.init(); let uploadFile = fileOrFolderPath; @@ -759,7 +760,10 @@ export const uploadToRemote = async ( await client.patchJson(uploadFile, k); } const res = await getRemoteMeta(client, uploadFile); - return res; + return { + entity: res, + mtimeCli: mtime, + }; } else { // if encrypted, // upload a fake, random-size file @@ -790,7 +794,10 @@ export const uploadToRemote = async ( } // log.info(uploadResult) const res = await getRemoteMeta(client, uploadFile); - return res; + return { + entity: res, + mtimeCli: mtime, + }; } } else { // file @@ -889,7 +896,10 @@ export const uploadToRemote = async ( } const res = await getRemoteMeta(client, uploadFile); - return res; + return { + entity: res, + mtimeCli: mtime, + }; } }; diff --git a/src/remoteForS3.ts b/src/remoteForS3.ts index b367b56..6d59406 100644 --- a/src/remoteForS3.ts +++ b/src/remoteForS3.ts @@ -30,6 +30,7 @@ import { DEFAULT_CONTENT_TYPE, Entity, S3Config, + UploadedType, VALID_REQURL, } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; @@ -365,7 +366,7 @@ export const uploadToRemote = async ( rawContent: string | ArrayBuffer = "", rawContentMTime: number = 0, rawContentCTime: number = 0 -) => { +): Promise => { log.debug(`uploading ${fileOrFolderPath}`); let uploadFile = fileOrFolderPath; if (password !== "") { @@ -408,7 +409,10 @@ export const uploadToRemote = async ( }) ); const res = await getRemoteMeta(s3Client, s3Config, uploadFile); - return res; + return { + entity: res, + mtimeCli: mtime, + }; } else { // file // we ignore isRecursively parameter here @@ -476,7 +480,10 @@ export const uploadToRemote = async ( // log.debug( // `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}` // ); - return res; + return { + entity: res, + mtimeCli: mtime, + }; } }; diff --git a/src/remoteForWebdav.ts b/src/remoteForWebdav.ts index 6ad30cb..0630e34 100644 --- a/src/remoteForWebdav.ts +++ b/src/remoteForWebdav.ts @@ -5,7 +5,7 @@ import { Queue } from "@fyears/tsqueue"; import chunk from "lodash/chunk"; import flatten from "lodash/flatten"; import { getReasonPhrase } from "http-status-codes"; -import { Entity, VALID_REQURL, WebdavConfig } from "./baseTypes"; +import { Entity, UploadedType, VALID_REQURL, WebdavConfig } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc"; @@ -340,7 +340,7 @@ export const uploadToRemote = async ( remoteEncryptedKey: string = "", uploadRaw: boolean = false, rawContent: string | ArrayBuffer = "" -) => { +): Promise => { await client.init(); let uploadFile = fileOrFolderPath; if (password !== "") { @@ -368,7 +368,9 @@ export const uploadToRemote = async ( recursive: true, }); const res = await getRemoteMeta(client, uploadFile); - return res; + return { + entity: res, + }; } else { // if encrypted, upload a fake file with the encrypted file name await client.client.putFileContents(uploadFile, "", { @@ -378,12 +380,15 @@ export const uploadToRemote = async ( }, }); - return await getRemoteMeta(client, uploadFile); + return { + entity: await getRemoteMeta(client, uploadFile), + }; } } else { // file // we ignore isRecursively parameter here - let localContent = undefined; + let localContent: ArrayBuffer | undefined = undefined; + let mtimeCli: number | undefined = undefined; if (uploadRaw) { if (typeof rawContent === "string") { localContent = new TextEncoder().encode(rawContent).buffer; @@ -397,6 +402,7 @@ export const uploadToRemote = async ( ); } localContent = await vault.adapter.readBinary(fileOrFolderPath); + mtimeCli = (await vault.adapter.stat(fileOrFolderPath))?.mtime; } let remoteContent = localContent; if (password !== "") { @@ -415,7 +421,10 @@ export const uploadToRemote = async ( }, }); - return await getRemoteMeta(client, uploadFile); + return { + entity: await getRemoteMeta(client, uploadFile), + mtimeCli: mtimeCli, + }; } }; diff --git a/src/sync.ts b/src/sync.ts index d0a3d2d..f09fccc 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -249,6 +249,24 @@ const decryptRemoteEntityInplace = async (remote: Entity, password: string) => { return remote; }; +const fullfillMTimeOfRemoteEntityInplace = ( + remote: Entity, + mtimeCli?: number +) => { + if ( + mtimeCli !== undefined && + mtimeCli > 0 && + (remote.mtimeCli === undefined || + remote.mtimeCli <= 0 || + (remote.mtimeSvr !== undefined && + remote.mtimeSvr > 0 && + remote.mtimeCli >= remote.mtimeSvr)) + ) { + remote.mtimeCli = mtimeCli; + } + return remote; +}; + /** * Directly throw error here. * We can only defer the checking now, because before decryption we don't know whether it's a file or folder. @@ -312,7 +330,7 @@ const encryptLocalEntityInplace = async ( // but local.key should always have value local.sizeEnc = getSizeFromOrigToEnc(local.size); } - + if (local.keyEnc === undefined || local.keyEnc === "") { if ( remoteKeyEnc !== undefined && @@ -895,15 +913,16 @@ const dispatchOperationToActualV3 = async ( // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded. } else { // log.debug(`before upload in sync, r=${JSON.stringify(r, null, 2)}`); - const remoteObjMeta = await client.uploadToRemote( + const { entity, mtimeCli } = await client.uploadToRemote( r.key, vault, false, password, r.local!.keyEnc ); - await decryptRemoteEntityInplace(remoteObjMeta, password); - await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); + await decryptRemoteEntityInplace(entity, password); + await fullfillMTimeOfRemoteEntityInplace(entity, mtimeCli); + await upsertPrevSyncRecordByVault(db, vaultRandomID, entity); } } else if ( r.decision === "modified_remote" || @@ -936,7 +955,7 @@ const dispatchOperationToActualV3 = async ( throw Error(`${r.decision} not implemented yet: ${JSON.stringify(r)}`); } else if (r.decision === "folder_to_be_created") { await mkdirpInVault(r.key, vault); - const remoteObjMeta = await client.uploadToRemote( + const { entity, mtimeCli } = await client.uploadToRemote( r.key, vault, false, @@ -944,8 +963,9 @@ const dispatchOperationToActualV3 = async ( r.local!.keyEnc ); // we need to decrypt the key!!! - await decryptRemoteEntityInplace(remoteObjMeta, password); - await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); + await decryptRemoteEntityInplace(entity, password); + await fullfillMTimeOfRemoteEntityInplace(entity, mtimeCli); + await upsertPrevSyncRecordByVault(db, vaultRandomID, entity); } else if (r.decision === "folder_to_be_deleted") { await localDeleteFunc(r.key); await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); From 5f5b5de50502b4ca553f87d38948c925a5d7d2ad Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 25 Feb 2024 22:44:09 +0800 Subject: [PATCH 11/21] add settings --- src/langs/en.json | 8 ++++++++ src/langs/zh_cn.json | 8 ++++++++ src/langs/zh_tw.json | 8 ++++++++ src/settings.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/src/langs/en.json b/src/langs/en.json index 6ee46cd..2ed5d4f 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -249,6 +249,14 @@ "settings_deletetowhere_desc": "Which trash should the plugin put the files into while deleting?", "settings_deletetowhere_system_trash": "system trash (default)", "settings_deletetowhere_obsidian_trash": "Obsidian .trash folder", + "settings_conflictaction": "Action For Conflict", + "settings_conflictaction_desc": "If a file is created or modified on both side since last update, it's a conflict event. How to deal with it?", + "settings_conflictaction_keep_newer": "newer version survives (default)", + "settings_conflictaction_keep_larger": "larger size version survives", + "settings_cleanemptyfolder": "Action For Empty Folders", + "settings_cleanemptyfolder_desc": "The sync algorithm majorly deals with files, so you need to specify how to deal with empty folders.", + "settings_cleanemptyfolder_skip": "leave them as is (default)", + "settings_cleanemptyfolder_clean_both": "delete local and remote", "settings_importexport": "Import and Export Partial Settings", "settings_export": "Export", "settings_export_desc": "Export not-oauth2 settings by generating a qrcode.", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index aae1e2e..3ba1e1f 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -249,6 +249,14 @@ "settings_deletetowhere_desc": "插件触发删除操作时候，删除到哪里？", "settings_deletetowhere_system_trash": "系统回收站（默认）", "settings_deletetowhere_obsidian_trash": "Obsidian .trash 文件夹", + "settings_conflictaction": "处理冲突", + "settings_conflictaction_desc": "如果一个文件，在本地和服务器都被创建或者修改了，那么这就是一个“冲突”情况。如何处理？", + "settings_conflictaction_keep_newer": "保留最后修改的版本（默认）", + "settings_conflictaction_keep_larger": "保留文件体积较大的版本", + "settings_cleanemptyfolder": "处理空文件夹", + "settings_cleanemptyfolder_desc": "同步算法主要是针对文件处理的，您要要手动指定空文件夹如何处理。", + "settings_cleanemptyfolder_skip": "跳过处理空文件夹（默认）", + "settings_cleanemptyfolder_clean_both": "删除本地和服务器的空文件夹", "settings_importexport": "导入导出部分设置", "settings_export": "导出", "settings_export_desc": "用 QR 码导出非 oauth2 的设置信息。", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index 7057050..ae52d67 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -249,6 +249,14 @@ "settings_deletetowhere_desc": "外掛觸發刪除操作時候，刪除到哪裡？", "settings_deletetowhere_system_trash": "系統回收站（預設）", "settings_deletetowhere_obsidian_trash": "Obsidian .trash 資料夾", + "settings_conflictaction": "處理衝突", + "settings_conflictaction_desc": "如果一個檔案，在本地和伺服器都被建立或者修改了，那麼這就是一個“衝突”情況。如何處理？", + "settings_conflictaction_keep_newer": "保留最後修改的版本（預設）", + "settings_conflictaction_keep_larger": "保留檔案體積較大的版本", + "settings_cleanemptyfolder": "處理空資料夾", + "settings_cleanemptyfolder_desc": "同步演算法主要是針對檔案處理的，您需要手動指定空資料夾如何處理。", + "settings_cleanemptyfolder_skip": "跳過處理空資料夾（預設）", + "settings_cleanemptyfolder_clean_both": "刪除本地和伺服器的空資料夾", "settings_importexport": "匯入匯出部分設定", "settings_export": "匯出", "settings_export_desc": "用 QR 碼匯出非 oauth2 的設定資訊。", diff --git a/src/settings.ts b/src/settings.ts index d758f77..c3a45f0 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -13,7 +13,9 @@ import { createElement, Eye, EyeOff } from "lucide"; import { API_VER_ENSURE_REQURL_OK, API_VER_REQURL, + ConflictActionType, DEFAULT_DEBUG_FOLDER, + EmptyFolderCleanType, SUPPORTED_SERVICES_TYPE, SUPPORTED_SERVICES_TYPE_WITH_REMOTE_BASE_DIR, VALID_REQURL, @@ -1993,6 +1995,44 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); + new Setting(advDiv) + .setName(t("settings_conflictaction")) + .setDesc(t("settings_conflictaction_desc")) + .addDropdown((dropdown) => { + dropdown.addOption( + "keep_newer", + t("settings_conflictaction_keep_newer") + ); + dropdown.addOption( + "keep_larger", + t("settings_conflictaction_keep_larger") + ); + dropdown + .setValue(this.plugin.settings.conflictAction ?? "keep_newer") + .onChange(async (val) => { + this.plugin.settings.conflictAction = val as ConflictActionType; + await this.plugin.saveSettings(); + }); + }); + + new Setting(advDiv) + .setName(t("settings_cleanemptyfolder")) + .setDesc(t("settings_cleanemptyfolder_desc")) + .addDropdown((dropdown) => { + dropdown.addOption("skip", t("settings_cleanemptyfolder_skip")); + dropdown.addOption( + "clean_both", + t("settings_cleanemptyfolder_clean_both") + ); + dropdown + .setValue(this.plugin.settings.howToCleanEmptyFolder ?? "skip") + .onChange(async (val) => { + this.plugin.settings.howToCleanEmptyFolder = + val as EmptyFolderCleanType; + await this.plugin.saveSettings(); + }); + }); + ////////////////////////////////////////////////// // below for import and export functions ////////////////////////////////////////////////// From 768d6636d0b51df385c55baf60de65530ecb3394 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 25 Feb 2024 23:21:42 +0800 Subject: [PATCH 12/21] db migration --- manifest-beta.json | 2 +- manifest.json | 2 +- package.json | 2 +- src/langs/en.json | 8 +++---- src/langs/zh_cn.json | 8 +++---- src/langs/zh_tw.json | 8 +++---- src/localdb.ts | 52 +++++++++++++++++++++++++------------------- src/settings.ts | 15 ++++++++----- 8 files changed, 54 insertions(+), 43 deletions(-) diff --git a/manifest-beta.json b/manifest-beta.json index 661511f..9ef3b0e 100644 --- a/manifest-beta.json +++ b/manifest-beta.json @@ -1,7 +1,7 @@ { "id": "remotely-save", "name": "Remotely Save", - "version": "0.3.40", + "version": "0.4.1", "minAppVersion": "0.13.21", "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.", "author": "fyears", diff --git a/manifest.json b/manifest.json index 661511f..9ef3b0e 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "remotely-save", "name": "Remotely Save", - "version": "0.3.40", + "version": "0.4.1", "minAppVersion": "0.13.21", "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.", "author": "fyears", diff --git a/package.json b/package.json index 629d3b3..168995c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "remotely-save", - "version": "0.3.40", + "version": "0.4.1", "description": "This is yet another sync plugin for Obsidian app.", "scripts": { "dev2": "node esbuild.config.mjs --watch", diff --git a/src/langs/en.json b/src/langs/en.json index 2ed5d4f..49d1115 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -281,10 +281,10 @@ "settings_logtohttpserver": "Log To HTTP(S) Server Temporarily", "settings_logtohttpserver_desc": "It's very dangerous and please use the function with greate cautions!!!!! It will temporarily allow sending console loggings to HTTP(S) server.", "settings_logtohttpserver_reset_notice": "Your input doesn't starts with \"http(s)\". Already removed the setting of logging to HTTP(S) server.", - "settings_delsyncmap": "Delete Sync Mappings History In DB", - "settings_delsyncmap_desc": "Sync mappings history stores the actual LOCAL last modified time of the REMOTE objects. Clearing it may cause unnecessary data exchanges in next-time sync. Click the button to delete sync mappings history in DB.", - "settings_delsyncmap_button": "Delete Sync Mappings", - "settings_delsyncmap_notice": "Sync mappings history (in local DB) deleted", + "settings_delprevsync": "Delete Prev Sync Details In DB", + "settings_delprevsync_desc": "The sync algorithm keeps the previous successful sync information in DB to determine the file changes. If you want to ignore them so that all files are treated newly created, you can delete the prev sync info here.", + "settings_delprevsync_button": "Delete Prev Sync Details", + "settings_delprevsync_notice": "Previous sync history (in local DB) deleted", "settings_outputbasepathvaultid": "Output Vault Base Path And Randomly Assigned ID", "settings_outputbasepathvaultid_desc": "For debugging purposes.", "settings_outputbasepathvaultid_button": "Output", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index 3ba1e1f..6a50e6d 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -281,10 +281,10 @@ "settings_logtohttpserver": "临时设定终端日志实时转发到 HTTP(S) 服务器。", "settings_logtohttpserver_desc": "非常危险，谨慎行动！！！！！临时设定终端日志实时转发到 HTTP(S) 服务器。", "settings_logtohttpserver_reset_notice": "您的输入不是“http(s)”开头的。已移除了终端日志转发到 HTTP(S) 服务器的设定。", - "settings_delsyncmap": "删除数据库里的同步映射历史", - "settings_delsyncmap_desc": "同步映射历史存储了本地真正的最后修改时间和远程文件时间的映射。删除之可能会导致下一次同步时发生不必要的数据交换。点击按钮删除数据库里的同步映射历史。", - "settings_delsyncmap_button": "删除同步映射历史", - "settings_delsyncmap_notice": "（本地数据库里的）同步映射历史已被删除。", + "settings_delprevsync": "删除数据库里的上次同步明细", + "settings_delprevsync_desc": "同步算法需要上次成功同步的信息来决定文件变更，这个信息保存在本地的数据库里。如果您想忽略这些信息从而所有文件都被视为新创建的话，可以在此删除之前的信息。", + "settings_delprevsync_button": "删除上次同步明细", + "settings_delprevsync_notice": "（本地数据库里的）上次同步明细已被删除。", "settings_outputbasepathvaultid": "输出资料库对应的位置和随机分配的 ID", "settings_outputbasepathvaultid_desc": "用于调试。", "settings_outputbasepathvaultid_button": "输出", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index ae52d67..a3d1ad5 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -281,10 +281,10 @@ "settings_logtohttpserver": "臨時設定終端日誌實時轉發到 HTTP(S) 伺服器。", "settings_logtohttpserver_desc": "非常危險，謹慎行動！！！！！臨時設定終端日誌實時轉發到 HTTP(S) 伺服器。", "settings_logtohttpserver_reset_notice": "您的輸入不是“http(s)”開頭的。已移除了終端日誌轉發到 HTTP(S) 伺服器的設定。", - "settings_delsyncmap": "刪除資料庫裡的同步對映歷史", - "settings_delsyncmap_desc": "同步對映歷史儲存了本地真正的最後修改時間和遠端檔案時間的對映。刪除之可能會導致下一次同步時發生不必要的資料交換。點選按鈕刪除資料庫裡的同步對映歷史。", - "settings_delsyncmap_button": "刪除同步對映歷史", - "settings_delsyncmap_notice": "（本地資料庫裡的）同步對映歷史已被刪除。", + "settings_delprevsync": "刪除資料庫裡的上次同步明細", + "settings_delprevsync_desc": "同步演算法需要上次成功同步的資訊來決定檔案變更，這個資訊儲存在本地的資料庫裡。如果您想忽略這些資訊從而所有檔案都被視為新建立的話，可以在此刪除之前的資訊。", + "settings_delprevsync_button": "刪除上次同步明細", + "settings_delprevsync_notice": "（本地資料庫裡的）上次同步明細已被刪除。", "settings_outputbasepathvaultid": "輸出資料庫對應的位置和隨機分配的 ID", "settings_outputbasepathvaultid_desc": "用於除錯。", "settings_outputbasepathvaultid_button": "輸出", diff --git a/src/localdb.ts b/src/localdb.ts index 7bfe87d..e889ee7 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -80,22 +80,32 @@ export interface InternalDBs { * @returns */ const fromSyncMappingsToPrevSyncRecords = ( - syncMappings: SyncMetaMappingRecord[] + oldSyncMappings: SyncMetaMappingRecord[] ): Entity[] => { - return []; -}; + const res: Entity[] = []; + for (const oldMapping of oldSyncMappings) { + const newEntity: Entity = { + key: oldMapping.localKey, + keyEnc: oldMapping.remoteKey, + keyRaw: + oldMapping.remoteKey !== undefined && oldMapping.remoteKey !== "" + ? oldMapping.remoteKey + : oldMapping.localKey, + mtimeCli: oldMapping.localMtime, + mtimeSvr: oldMapping.remoteMtime, + size: oldMapping.localSize, + sizeEnc: oldMapping.remoteSize, + sizeRaw: + oldMapping.remoteKey !== undefined && oldMapping.remoteKey !== "" + ? oldMapping.remoteSize + : oldMapping.localSize, + etag: oldMapping.remoteExtraKey, + }; -/** - * TODO - * @param db - * @param vaultRandomID - * @param prevSyncRecord - */ -const setPrevSyncRecordByVault = async ( - db: InternalDBs, - vaultRandomID: string, - prevSyncRecord: Entity -) => {}; + res.push(newEntity); + } + return res; +}; /** * @@ -115,12 +125,14 @@ const migrateDBsFrom20220326To20240220 = async ( const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID); const prevSyncRecords = fromSyncMappingsToPrevSyncRecords(syncMappings); for (const prevSyncRecord of prevSyncRecords) { - await setPrevSyncRecordByVault(db, vaultRandomID, prevSyncRecord); + await upsertPrevSyncRecordByVault(db, vaultRandomID, prevSyncRecord); } - // clear not used data - await clearFileHistoryOfEverythingByVault(db, vaultRandomID); - await clearAllSyncMetaMappingByVault(db, vaultRandomID); + // // clear not used data + // // as of 20240220, we don't call them, + // // for the opportunity for users to downgrade + // await clearFileHistoryOfEverythingByVault(db, vaultRandomID); + // await clearAllSyncMetaMappingByVault(db, vaultRandomID); await db.versionTbl.setItem(`${vaultRandomID}\tversion`, newVer); log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); @@ -322,10 +334,6 @@ export const clearAllSyncMetaMappingByVault = async ( } }; -export const clearAllSyncMetaMapping = async (db: InternalDBs) => { - await db.syncMappingTbl.clear(); -}; - export const insertSyncPlanRecordByVault = async ( db: InternalDBs, syncPlan: SyncPlanType, diff --git a/src/settings.ts b/src/settings.ts index c3a45f0..cd7c13e 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -25,7 +25,7 @@ import { import { exportVaultSyncPlansToFiles } from "./debugMode"; import { exportQrCodeUri } from "./importExport"; import { - clearAllSyncMetaMapping, + clearAllPrevSyncRecordByVault, clearAllSyncPlanRecords, destroyDBs, upsertLastSuccessSyncTimeByVault, @@ -2166,13 +2166,16 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); new Setting(debugDiv) - .setName(t("settings_delsyncmap")) - .setDesc(t("settings_delsyncmap_desc")) + .setName(t("settings_delprevsync")) + .setDesc(t("settings_delprevsync_desc")) .addButton(async (button) => { - button.setButtonText(t("settings_delsyncmap_button")); + button.setButtonText(t("settings_delprevsync_button")); button.onClick(async () => { - await clearAllSyncMetaMapping(this.plugin.db); - new Notice(t("settings_delsyncmap_notice")); + await clearAllPrevSyncRecordByVault( + this.plugin.db, + this.plugin.vaultRandomID + ); + new Notice(t("settings_delprevsync_notice")); }); }); From 447e0c0aa44324cc8837b22084f0e23f12c2db5f Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 25 Feb 2024 23:39:31 +0800 Subject: [PATCH 13/21] moving sync docs --- docs/{ => sync_algorithm}/sync_ignoring_large_files.md | 0 docs/{sync_algorithm_v1.md => sync_algorithm/v1/README.md} | 0 docs/{sync_algorithm_v2.md => sync_algorithm/v2/README.md} | 0 docs/{sync_algorithm_v3.md => sync_algorithm/v3/design.md} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename docs/{ => sync_algorithm}/sync_ignoring_large_files.md (100%) rename docs/{sync_algorithm_v1.md => sync_algorithm/v1/README.md} (100%) rename docs/{sync_algorithm_v2.md => sync_algorithm/v2/README.md} (100%) rename docs/{sync_algorithm_v3.md => sync_algorithm/v3/design.md} (100%) diff --git a/docs/sync_ignoring_large_files.md b/docs/sync_algorithm/sync_ignoring_large_files.md similarity index 100% rename from docs/sync_ignoring_large_files.md rename to docs/sync_algorithm/sync_ignoring_large_files.md diff --git a/docs/sync_algorithm_v1.md b/docs/sync_algorithm/v1/README.md similarity index 100% rename from docs/sync_algorithm_v1.md rename to docs/sync_algorithm/v1/README.md diff --git a/docs/sync_algorithm_v2.md b/docs/sync_algorithm/v2/README.md similarity index 100% rename from docs/sync_algorithm_v2.md rename to docs/sync_algorithm/v2/README.md diff --git a/docs/sync_algorithm_v3.md b/docs/sync_algorithm/v3/design.md similarity index 100% rename from docs/sync_algorithm_v3.md rename to docs/sync_algorithm/v3/design.md From 9fc46553e1f9aca85ca1b6d4264b192cb810f6fe Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 25 Feb 2024 23:54:54 +0800 Subject: [PATCH 14/21] update doc --- README.md | 5 ++--- docs/minimal_intrusive_design.md | 18 ++++++++++++------ docs/sync_algorithm/README.md | 7 +++++++ docs/sync_algorithm/v3/README.md | 4 ++++ docs/sync_algorithm/v3/intro.md | 12 ++++++++++++ 5 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 docs/sync_algorithm/README.md create mode 100644 docs/sync_algorithm/v3/README.md create mode 100644 docs/sync_algorithm/v3/intro.md diff --git a/README.md b/README.md index 38f0fc7..9aac641 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,11 @@ This is yet another unofficial sync plugin for Obsidian. If you like it or find - **[Minimal Intrusive](./docs/minimal_intrusive_design.md).** - **Skip Large files** and **skip paths** by custom regex conditions! - **Fully open source under [Apache-2.0 License](./LICENSE).** -- **[Sync Algorithm open](./docs/sync_algorithm_v2.md) for discussion.** +- **[Sync Algorithm open](./docs/sync_algorithm/v3/intro.md) for discussion.** +- **[Basic Conflict Detection And Handling](./docs/sync_algorithm/v3/intro.md)** now, more to come! ## Limitations -- **To support deletions sync, extra metadata will also be uploaded.** See [Minimal Intrusive](./docs/minimal_intrusive_design.md). -- **No Conflict resolution. No content-diff-and-patch algorithm.** All files and folders are compared using their local and remote "last modified time" and those with later "last modified time" wins. - **Cloud services cost you money.** Always be aware of the costs and pricing. Specifically, all the operations, including but not limited to downloading, uploading, listing all files, calling any api, storage sizes, may or may not cost you money. - **Some limitations from the browser environment.** More technical details are [in the doc](./docs/browser_env.md). - **You should protect your `data.json` file.** The file contains sensitive information. diff --git a/docs/minimal_intrusive_design.md b/docs/minimal_intrusive_design.md index 68bd86c..d897944 100644 --- a/docs/minimal_intrusive_design.md +++ b/docs/minimal_intrusive_design.md @@ -1,8 +1,10 @@ # Minimal Intrusive Design -Before version 0.3.0, the plugin did not upload additional meta data to the remote. +~~Before version 0.3.0, the plugin did not upload additional meta data to the remote.~~ -From and after version 0.3.0, the plugin just upload minimal extra necessary meta data to the remote. +~~From version 0.3.0 ~ 0.3.40, the plugin just upload minimal extra necessary meta data to the remote.~~ + +From version 0.4.1 and above, the plugin doesn't need uploading meta data due to the sync algorithm upgrade. ## Benefits @@ -12,10 +14,14 @@ For example, it's possbile for a uses to manually upload a file to s3, and next And it's also possible to combine another "sync-to-s3" solution (like, another software) on desktops, and this plugin on mobile devices, together. -## Necessarity Of Uploading Extra Metadata +## ~~Necessarity Of Uploading Extra Metadata from 0.3.0 ~ 0.3.40~~ -The main issue comes from deletions (and renamings which is actually interpreted as "deletion-then-creation"). +~~The main issue comes from deletions (and renamings which is actually interpreted as "deletion-then-creation").~~ -If we don't upload any extra info to the remote, there's usually no way for the second device to know what files / folders have been deleted on the first device. +~~If we don't upload any extra info to the remote, there's usually no way for the second device to know what files / folders have been deleted on the first device.~~ -To overcome this issue, from and after version 0.3.0, the plugin uploads extra metadata files `_remotely-save-metadata-on-remote.{json,bin}` to users' configured cloud services. Those files contain some info about what has been deleted on the first device, so that the second device can read the list to apply the deletions to itself. Some other necessary meta info would also be written into the extra files. +~~To overcome this issue, from and after version 0.3.0, the plugin uploads extra metadata files `_remotely-save-metadata-on-remote.{json,bin}` to users' configured cloud services. Those files contain some info about what has been deleted on the first device, so that the second device can read the list to apply the deletions to itself. Some other necessary meta info would also be written into the extra files.~~ + +## No uploading extra metadata from 0.4.1 + +Some information, including previous successful sync status of each file, is kept locally. diff --git a/docs/sync_algorithm/README.md b/docs/sync_algorithm/README.md new file mode 100644 index 0000000..c2adba5 --- /dev/null +++ b/docs/sync_algorithm/README.md @@ -0,0 +1,7 @@ +# Sync Algorithm + +* [v1](./v1/README.md) +* [v2](./v2/README.md) +* v3 + * [intro doc for end users](./v3/intro.md) + * [design doc](./v3/design.md) diff --git a/docs/sync_algorithm/v3/README.md b/docs/sync_algorithm/v3/README.md new file mode 100644 index 0000000..9e90cb3 --- /dev/null +++ b/docs/sync_algorithm/v3/README.md @@ -0,0 +1,4 @@ +# Sync Algorithm V3 + +* [intro doc for end users](./intro.md) +* [design doc](./design.md) diff --git a/docs/sync_algorithm/v3/intro.md b/docs/sync_algorithm/v3/intro.md new file mode 100644 index 0000000..0ad3a9f --- /dev/null +++ b/docs/sync_algorithm/v3/intro.md @@ -0,0 +1,12 @@ +# Introduction To Sync Algorithm V3 + +* [x] sync conflict: keep newer +* [x] sync conflict: keep larger +* [ ] sync conflict: keep both and rename +* [ ] sync conflict: show warning +* [x] deletion: true deletion status computation +* [x] meta data: no remote meta data any more +* [x] migration: old data auto transfer to new db (hopefully) +* [ ] partial sync: force push +* [ ] partial sync: force pull +* [ ] sync protection: warning based on the threshold From aff5a8b0880f0311a4adfcb50b918bf85c51273e Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Mon, 26 Feb 2024 00:41:40 +0800 Subject: [PATCH 15/21] add intro --- docs/sync_algorithm/README.md | 10 ++--- docs/sync_algorithm/v3/README.md | 4 +- docs/sync_algorithm/v3/intro.md | 20 +++++----- src/langs/en.json | 3 +- src/langs/zh_cn.json | 5 ++- src/langs/zh_tw.json | 5 ++- src/misc.ts | 12 ++++++ src/syncAlgoV3Notice.ts | 65 +++++++++++++++++++++++++------- 8 files changed, 88 insertions(+), 36 deletions(-) diff --git a/docs/sync_algorithm/README.md b/docs/sync_algorithm/README.md index c2adba5..cd84fa4 100644 --- a/docs/sync_algorithm/README.md +++ b/docs/sync_algorithm/README.md @@ -1,7 +1,7 @@ # Sync Algorithm -* [v1](./v1/README.md) -* [v2](./v2/README.md) -* v3 - * [intro doc for end users](./v3/intro.md) - * [design doc](./v3/design.md) +- [v1](./v1/README.md) +- [v2](./v2/README.md) +- v3 + - [intro doc for end users](./v3/intro.md) + - [design doc](./v3/design.md) diff --git a/docs/sync_algorithm/v3/README.md b/docs/sync_algorithm/v3/README.md index 9e90cb3..1f66f0f 100644 --- a/docs/sync_algorithm/v3/README.md +++ b/docs/sync_algorithm/v3/README.md @@ -1,4 +1,4 @@ # Sync Algorithm V3 -* [intro doc for end users](./intro.md) -* [design doc](./design.md) +- [intro doc for end users](./intro.md) +- [design doc](./design.md) diff --git a/docs/sync_algorithm/v3/intro.md b/docs/sync_algorithm/v3/intro.md index 0ad3a9f..0c1a16c 100644 --- a/docs/sync_algorithm/v3/intro.md +++ b/docs/sync_algorithm/v3/intro.md @@ -1,12 +1,12 @@ # Introduction To Sync Algorithm V3 -* [x] sync conflict: keep newer -* [x] sync conflict: keep larger -* [ ] sync conflict: keep both and rename -* [ ] sync conflict: show warning -* [x] deletion: true deletion status computation -* [x] meta data: no remote meta data any more -* [x] migration: old data auto transfer to new db (hopefully) -* [ ] partial sync: force push -* [ ] partial sync: force pull -* [ ] sync protection: warning based on the threshold +- [x] sync conflict: keep newer +- [x] sync conflict: keep larger +- [ ] sync conflict: keep both and rename +- [ ] sync conflict: show warning +- [x] deletion: true deletion status computation +- [x] meta data: no remote meta data any more +- [x] migration: old data auto transfer to new db (hopefully) +- [ ] partial sync: force push +- [ ] partial sync: force pull +- [ ] sync protection: warning based on the threshold diff --git a/src/langs/en.json b/src/langs/en.json index 49d1115..3bb9a39 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -293,7 +293,8 @@ "settings_resetcache_button": "Reset", "settings_resetcache_notice": "Local internal cache/databases deleted. Please manually reload the plugin.", "syncalgov3_title": "Remotely Save has HUGE update on sync algorithm", - "syncalgov3_texts": "Welcome to use Remotely Save!\nFrom this version, a new algorithm has been developed. If you agree, plase click the button \"Agree\", and enjoy the plugin! AND PLEASE REMEMBER TO BACKUP YOUR VAULT FIRSTLY!\nIf you do not agree, you should stop using the current and later versions of Remotely Save. By clicking the \"Do Not Agree\" button, the plugin will unload itself, and you need to manually disable it in Obsidian settings.", + "syncalgov3_texts": "Welcome to use Remotely Save!\nFrom this version, a new algorithm has been developed: More robust deletion sync, basic conflict handling, no more meta data uploaded... Stay tune for more! A full introduction is in the doc website.\nIf you agree to use thew new version and algorithm, plase check \"I WILL BACKUP MY VAULT MANUALLY FIRSTLY.\" then click the \"Agree\" button, and enjoy the plugin!\nIf you do not agree, please click the \"Do Not Agree\" button, the plugin will unload itself, and you need to manually disable it in Obsidian settings.", + "syncalgov3_checkbox_manual_backup": "I WILL BACKUP MY VAULT MANUALLY FIRSTLY.", "syncalgov3_button_agree": "Agree", "syncalgov3_button_disagree": "Do Not Agree" } diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index 6a50e6d..f45503f 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -292,8 +292,9 @@ "settings_resetcache_desc": "（出于调试原因）重设本地缓存和数据库。您需要在重设之后重新载入此插件。本重设不会删除 s3，密码……等设定。", "settings_resetcache_button": "重设", "settings_resetcache_notice": "本地同步缓存和数据库已被删除。请手动重新载入此插件。", - "syncalgov3_title": "Remotely Save 的同步算法重大优化", - "syncalgov3_texts": "欢迎使用 Remotely Save!\n从这个版本 0.3.0 开始，它带来了新的同步算法\n如果您同意使用，请点击按钮 \"同意\"，然后开始享用此插件！且特别要注意：使用插件之前，请首先备份好您的库（Vault）！\n如果您不同意此策略，您应该停止使用此版本和之后版本的 Remotely Save。点击 \"不同意\" 之后，插件会自动停止运行（unload），然后您需要 Obsidian 设置里手动停用（disable）此插件。", + "syncalgov3_title": "Remotely Save 的同步算法有重大更新", + "syncalgov3_texts": "欢迎使用 Remotely Save！\n从这个版本开始，插件更新了同步算法：更稳健的删除同步、引入冲突处理、避免上传元数据…… 敬请期待更多更新！详细介绍请参阅 文档网站。\n如果您同意使用新版本和算法，请勾选“我将会手动备份我的库（Vault）”，然后点击“同意”按钮，开始使用插件吧！\n如果您不同意，请点击“不同意”按钮，插件将自动停止运行（unload），然后您需要 Obsidian 设置里手动停用（disable）此插件。", + "syncalgov3_checkbox_manual_backup": "我将会手动备份我的库（Vault）", "syncalgov3_button_agree": "同意", "syncalgov3_button_disagree": "不同意" } diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index a3d1ad5..f565f91 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -292,8 +292,9 @@ "settings_resetcache_desc": "（出於除錯原因）重設本地快取和資料庫。您需要在重設之後重新載入此外掛。本重設不會刪除 s3，密碼……等設定。", "settings_resetcache_button": "重設", "settings_resetcache_notice": "本地同步快取和資料庫已被刪除。請手動重新載入此外掛。", - "syncalgov3_title": "Remotely Save 的同步演算法重大最佳化", - "syncalgov3_texts": "歡迎使用 Remotely Save!\n從這個版本 0.3.0 開始，它帶來了新的同步演算法\n如果您同意使用，請點選按鈕 \"同意\"，然後開始享用此外掛！且特別要注意：使用外掛之前，請首先備份好您的庫（Vault）！\n如果您不同意此策略，您應該停止使用此版本和之後版本的 Remotely Save。點選 \"不同意\" 之後，外掛會自動停止執行（unload），然後您需要 Obsidian 設定裡手動停用（disable）此外掛。", + "syncalgov3_title": "Remotely Save 的同步演算法有重大更新", + "syncalgov3_texts": "歡迎使用 Remotely Save！\n從這個版本開始，外掛更新了同步演算法：更穩健的刪除同步、引入衝突處理、避免上傳元資料…… 敬請期待更多更新！詳細介紹請參閱 文件網站。\n如果您同意使用新版本和演算法，請勾選“我將會手動備份我的庫（Vault）”，然後點選“同意”按鈕，開始使用外掛吧！\n如果您不同意，請點選“不同意”按鈕，外掛將自動停止執行（unload），然後您需要 Obsidian 設定裡手動停用（disable）此外掛。", + "syncalgov3_checkbox_manual_backup": "我將會手動備份我的庫（Vault）", "syncalgov3_button_agree": "同意", "syncalgov3_button_disagree": "不同意" } diff --git a/src/misc.ts b/src/misc.ts index eaa0e88..83b0145 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -493,3 +493,15 @@ export const compareVersion = (x: string | null, y: string | null) => { } return -1; }; + +/** + * https://stackoverflow.com/questions/19929641/how-to-append-an-html-string-to-a-documentfragment + * To introduce some advanced html fragments. + * @param string + * @returns + */ +export const stringToFragment = (string: string) => { + const wrapper = document.createElement("template"); + wrapper.innerHTML = string; + return wrapper.content; +}; diff --git a/src/syncAlgoV3Notice.ts b/src/syncAlgoV3Notice.ts index 34b6913..a64d9d6 100644 --- a/src/syncAlgoV3Notice.ts +++ b/src/syncAlgoV3Notice.ts @@ -3,14 +3,17 @@ import type RemotelySavePlugin from "./main"; // unavoidable import type { TransItemType } from "./i18n"; import { log } from "./moreOnLog"; +import { stringToFragment } from "./misc"; export class SyncAlgoV3Modal extends Modal { agree: boolean; + manualBackup: boolean; readonly plugin: RemotelySavePlugin; constructor(app: App, plugin: RemotelySavePlugin) { super(app); this.plugin = plugin; this.agree = false; + this.manualBackup = false; } onOpen() { let { contentEl } = this; @@ -27,24 +30,58 @@ export class SyncAlgoV3Modal extends Modal { .split("\n") .forEach((val) => { ul.createEl("li", { - text: val, + text: stringToFragment(val), }); }); - new Setting(contentEl) - .addButton((button) => { - button.setButtonText(t("syncalgov3_button_agree")); - button.onClick(async () => { - this.agree = true; - this.close(); - }); - }) - .addButton((button) => { - button.setButtonText(t("syncalgov3_button_disagree")); - button.onClick(() => { - this.close(); - }); + // code modified partially from BART released under MIT License + contentEl.createDiv("modal-button-container", (buttonContainerEl) => { + let agreeBtn: HTMLButtonElement | undefined = undefined; + buttonContainerEl.createEl( + "label", + { + cls: "mod-checkbox", + }, + (labelEl) => { + const checkboxEl = labelEl.createEl("input", { + attr: { tabindex: -1 }, + type: "checkbox", + }); + checkboxEl.checked = this.manualBackup; + checkboxEl.addEventListener("click", () => { + this.manualBackup = checkboxEl.checked; + if (agreeBtn !== undefined) { + if (checkboxEl.checked) { + agreeBtn.removeAttribute("disabled"); + } else { + agreeBtn.setAttr("disabled", true); + } + } + }); + labelEl.appendText(t("syncalgov3_checkbox_manual_backup")); + } + ); + + agreeBtn = buttonContainerEl.createEl("button", { + attr: { type: "button" }, + cls: "mod-cta", + text: t("syncalgov3_button_agree"), }); + agreeBtn.setAttr("disabled", true); + agreeBtn.addEventListener("click", () => { + this.agree = true; + this.close(); + }); + + buttonContainerEl + .createEl("button", { + attr: { type: "submit" }, + text: t("syncalgov3_button_disagree"), + }) + .addEventListener("click", () => { + this.close(); + }); + }); } onClose() { From 996533328ecd533e0c9d70a1797aedb8d79a6cba Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 17 Mar 2024 15:31:32 +0800 Subject: [PATCH 16/21] fix onedrive filter --- src/remoteForOnedrive.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/remoteForOnedrive.ts b/src/remoteForOnedrive.ts index 52cbb4b..fa7f44d 100644 --- a/src/remoteForOnedrive.ts +++ b/src/remoteForOnedrive.ts @@ -671,7 +671,7 @@ export const listAllFromRemote = async (client: WrappedOnedriveClient) => { // unify everything to Entity const unifiedContents = driveItems .map((x) => fromDriveItemToEntity(x, client.remoteBaseDir)) - .filter((x) => x.key !== "/"); + .filter((x) => x.keyRaw !== "/"); return unifiedContents; }; From d7ff793715cdd60f8fe67045b3a6c8e65bc559c1 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 17 Mar 2024 16:03:40 +0800 Subject: [PATCH 17/21] use raw logging now! --- docs/how_to_debug/README.md | 6 +- .../save_console_output_and_export.md | 25 ---- docs/how_to_debug/use_logstravaganza.md | 14 ++ src/configPersist.ts | 20 ++- src/debugMode.ts | 6 +- src/encrypt.ts | 2 - src/importExport.ts | 4 +- src/langs/en.json | 13 +- src/langs/zh_cn.json | 13 +- src/langs/zh_tw.json | 13 +- src/localdb.ts | 28 ++-- src/main.ts | 67 +++------ src/metadataOnRemote.ts | 1 - src/misc.ts | 12 +- src/moreOnLog.ts | 40 ------ src/remote.ts | 2 - src/remoteForDropbox.ts | 38 +++-- src/remoteForOnedrive.ts | 64 ++++----- src/remoteForS3.ts | 21 ++- src/remoteForWebdav.ts | 50 ++++--- src/settings.ts | 135 ++---------------- src/sync.ts | 39 ++--- src/syncAlgoV3Notice.ts | 5 +- 23 files changed, 196 insertions(+), 422 deletions(-) delete mode 100644 docs/how_to_debug/save_console_output_and_export.md create mode 100644 docs/how_to_debug/use_logstravaganza.md delete mode 100644 src/moreOnLog.ts diff --git a/docs/how_to_debug/README.md b/docs/how_to_debug/README.md index a7c4155..8f46621 100644 --- a/docs/how_to_debug/README.md +++ b/docs/how_to_debug/README.md @@ -12,8 +12,8 @@ See [here](./export_sync_plans.md). See [here](./check_console_output.md). -## Advanced: Save Console Output Then Read Them Later +## Advanced: Use `Logstravaganza` to export logs -This method works for desktop and mobile devices (iOS, Android). +This method works for desktop and mobile devices (iOS, Android), especially useful for iOS. -See [here](./save_console_output_and_export.md). +See [here](./use_logstravaganza.md). diff --git a/docs/how_to_debug/save_console_output_and_export.md b/docs/how_to_debug/save_console_output_and_export.md deleted file mode 100644 index 28b46ed..0000000 --- a/docs/how_to_debug/save_console_output_and_export.md +++ /dev/null @@ -1,25 +0,0 @@ -# Save Console Output And Read Them Later - -## Disable Auto Sync Firstly - -You should disable auto sync to avoid any unexpected running. - -## Set The Output Level To Debug - -Go to the plugin settings, scroll down to the section "Debug" -> "alter console log level", and change it from "info" to "debug". - -## Enable Saving The Output To DB - -Go to the plugin settings, scroll down to the section "Debug" -> "Save Console Logs Into DB", and change it from "disable" to "enable". **This setting has some performance cost, so do NOT always turn this on when not necessary!** - -## Run The Sync - -Trigger the sync manually (by clicking the icon on the ribbon sidebar). Something (hopefully) helpful should show up in the console. The the console logs are also saved into DB now. - -## Export The Output And Read The Logs - -Go to the plugin settings, scroll down to the section "Debug" -> "Export Console Logs From DB", and click the button. A new file `log_hist_exported_on_....md` should be created inside the special folder `_debug_remotely_save/`. You could read it and hopefully find something useful. - -## Disable Saving The Output To DB - -After debugging, go to the plugin settings, scroll down to the section "Debug" -> "Save Console Logs Into DB", and change it from "enable" to "disable". diff --git a/docs/how_to_debug/use_logstravaganza.md b/docs/how_to_debug/use_logstravaganza.md new file mode 100644 index 0000000..17add7f --- /dev/null +++ b/docs/how_to_debug/use_logstravaganza.md @@ -0,0 +1,14 @@ +# Use `Logstravaganza` + +On iOS, it's quite hard to directly check the console logs. + +Luckily, there is a third-party plugin: [`Logstravaganza`](https://obsidian.md/plugins?search=Logstravaganza#), by Carlo Zottmann, that can redirect the output to a note. + +You can just: + +1. Install it. +2. Enable it. +3. Do something, to trigger some console logs. +4. Checkout `LOGGING-NOTE (device name).md` in the root of your vault. + +See more on its site: . diff --git a/src/configPersist.ts b/src/configPersist.ts index c880ec0..91ddb04 100644 --- a/src/configPersist.ts +++ b/src/configPersist.ts @@ -3,8 +3,6 @@ import { reverseString } from "./misc"; import type { RemotelySavePluginSettings } from "./baseTypes"; -import { log } from "./moreOnLog"; - const DEFAULT_README: string = "The file contains sensitive info, so DO NOT take screenshot of, copy, or share it to anyone! It's also generated automatically, so do not edit it manually."; @@ -19,10 +17,10 @@ interface MessyConfigType { export const messyConfigToNormal = ( x: MessyConfigType | RemotelySavePluginSettings | null | undefined ): RemotelySavePluginSettings | null | undefined => { - // log.debug("loading, original config on disk:"); - // log.debug(x); + // console.debug("loading, original config on disk:"); + // console.debug(x); if (x === null || x === undefined) { - log.debug("the messy config is null or undefined, skip"); + console.debug("the messy config is null or undefined, skip"); return x as any; } if ("readme" in x && "d" in x) { @@ -35,12 +33,12 @@ export const messyConfigToNormal = ( }) as Buffer ).toString("utf-8") ); - // log.debug("loading, parsed config is:"); - // log.debug(y); + // console.debug("loading, parsed config is:"); + // console.debug(y); return y; } else { // return as is - // log.debug("loading, parsed config is the same"); + // console.debug("loading, parsed config is the same"); return x; } }; @@ -52,7 +50,7 @@ export const normalConfigToMessy = ( x: RemotelySavePluginSettings | null | undefined ) => { if (x === null || x === undefined) { - log.debug("the normal config is null or undefined, skip"); + console.debug("the normal config is null or undefined, skip"); return x; } const y = { @@ -63,7 +61,7 @@ export const normalConfigToMessy = ( }) ), }; - // log.debug("encoding, encoded config is:"); - // log.debug(y); + // console.debug("encoding, encoded config is:"); + // console.debug(y); return y; }; diff --git a/src/debugMode.ts b/src/debugMode.ts index bc43a5a..d873ca5 100644 --- a/src/debugMode.ts +++ b/src/debugMode.ts @@ -11,8 +11,6 @@ import { FileOrFolderMixedState, } from "./baseTypes"; -import { log } from "./moreOnLog"; - const turnSyncPlanToTable = (record: string) => { const syncPlan: SyncPlanType = JSON.parse(record); const { ts, tsFmt, remoteType, mixedStates } = syncPlan; @@ -77,7 +75,7 @@ export const exportVaultSyncPlansToFiles = async ( vault: Vault, vaultRandomID: string ) => { - log.info("exporting"); + console.info("exporting"); await mkdirpInVault(DEFAULT_DEBUG_FOLDER, vault); const records = await readAllSyncPlanRecordTextsByVault(db, vaultRandomID); let md = ""; @@ -93,5 +91,5 @@ export const exportVaultSyncPlansToFiles = async ( await vault.create(filePath, md, { mtime: ts, }); - log.info("finish exporting"); + console.info("finish exporting"); }; diff --git a/src/encrypt.ts b/src/encrypt.ts index 542d3ad..d275be7 100644 --- a/src/encrypt.ts +++ b/src/encrypt.ts @@ -1,8 +1,6 @@ import { base32, base64url } from "rfc4648"; import { bufferToArrayBuffer, hexStringToTypedArray } from "./misc"; -import { log } from "./moreOnLog"; - const DEFAULT_ITER = 20000; // base32.stringify(Buffer.from('Salted__')) diff --git a/src/importExport.ts b/src/importExport.ts index 74dfc5e..4d9de74 100644 --- a/src/importExport.ts +++ b/src/importExport.ts @@ -7,8 +7,6 @@ import { RemotelySavePluginSettings, } from "./baseTypes"; -import { log } from "./moreOnLog"; - export const exportQrCodeUri = async ( settings: RemotelySavePluginSettings, currentVaultName: string, @@ -22,7 +20,7 @@ export const exportQrCodeUri = async ( const vault = encodeURIComponent(currentVaultName); const version = encodeURIComponent(pluginVersion); const rawUri = `obsidian://${COMMAND_URI}?func=settings&version=${version}&vault=${vault}&data=${data}`; - // log.info(uri) + // console.info(uri) const imgUri = await QRCode.toDataURL(rawUri); return { rawUri, diff --git a/src/langs/en.json b/src/langs/en.json index 3bb9a39..1e59e42 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -106,10 +106,6 @@ "modal_sizesconflict_desc": "You've set skipping files larger than {{thresholdMB}} MB ({{thresholdBytes}} bytes).\nBut the following files have sizes larger than the threshold on one side, and sizes smaller than the threshold on the other side.\nTo avoid unexpected overwriting or deleting, the plugin stops, and you have to manually deal with at least one side of the files.", "modal_sizesconflict_copybutton": "Click to copy all the below sizes conflicts info", "modal_sizesconflict_copynotice": "All the sizes conflicts info have been copied to the clipboard!", - "modal_logtohttpserver_title": "Log To HTTP(S) Server Is DANGEROUS!", - "modal_logtohttpserver_desc": "All your sensitive logging information will be posted to the HTTP(S) server without any authentications!!!!!\nPlease make sure you trust the HTTP(S) server, and it's better to setup a HTTPS one instead of HTTP one.\nIt's for debugging purposes only, especially on mobile.", - "modal_logtohttpserver_secondconfirm": "I know it's dangerous, and insist, and am willing to bear all possible losses.", - "modal_logtohttpserver_notice": "OK.", "settings_basic": "Basic Settings", "settings_password": "Encryption Password", "settings_password_desc": "Password for E2E encryption. Empty for no password. You need to click \"Confirm\". Attention: the password and other info are saved locally.", @@ -264,12 +260,14 @@ "settings_import": "Import", "settings_import_desc": "You should open a camera or scan-qrcode app, to manually scan the QR code.", "settings_debug": "Debug", - "settings_debuglevel": "Alter Console Log Level", - "settings_debuglevel_desc": "By default the log level is \"info\". You can change to \"debug\" to get verbose information in console.", + "settings_debuglevel": "Alter Notice Level", + "settings_debuglevel_desc": "By default the notice level is \"info\". You can change to \"debug\" to get verbose information while syncing.", "settings_outputsettingsconsole": "Output Current Settings From Disk To Console", "settings_outputsettingsconsole_desc": "The settings save on disk in encoded. Click this to see the decoded settings in console.", "settings_outputsettingsconsole_button": "Output", "settings_outputsettingsconsole_notice": "Finished outputing in console.", + "settings_viewconsolelog": "View Console Log", + "settings_viewconsolelog_desc": "On desktop, please press \"ctrl+shift+i\" or \"cmd+shift+i\" to view the log. On mobile, please install the third-party plugin Logstravaganza to export the console log to a note.", "settings_syncplans": "Export Sync Plans", "settings_syncplans_desc": "Sync plans are created every time after you trigger sync and before the actual sync. Useful to know what would actually happen in those sync. Click the button to export sync plans.", "settings_syncplans_button_json": "Export", @@ -278,9 +276,6 @@ "settings_delsyncplans_desc": "Delete sync plans history in DB.", "settings_delsyncplans_button": "Delete Sync Plans History", "settings_delsyncplans_notice": "Sync plans history (in DB) deleted.", - "settings_logtohttpserver": "Log To HTTP(S) Server Temporarily", - "settings_logtohttpserver_desc": "It's very dangerous and please use the function with greate cautions!!!!! It will temporarily allow sending console loggings to HTTP(S) server.", - "settings_logtohttpserver_reset_notice": "Your input doesn't starts with \"http(s)\". Already removed the setting of logging to HTTP(S) server.", "settings_delprevsync": "Delete Prev Sync Details In DB", "settings_delprevsync_desc": "The sync algorithm keeps the previous successful sync information in DB to determine the file changes. If you want to ignore them so that all files are treated newly created, you can delete the prev sync info here.", "settings_delprevsync_button": "Delete Prev Sync Details", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index f45503f..c57bebb 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -106,10 +106,6 @@ "modal_sizesconflict_desc": "您设置了跳过同步大于 {{thresholdMB}} MB（{{thresholdBytes}} bytes）的文件。\n但是以下文件的大小，在一端大于阈值，在另一端则小于阈值。\n为了避免意外的覆盖或删除，插件停止了运作，您需要手动处理至少一端的文件。", "modal_sizesconflict_copybutton": "点击以复制以下所有文件大小冲突信息", "modal_sizesconflict_copynotice": "所有的文件大小冲突信息，已被复制到剪贴板！", - "modal_logtohttpserver_title": "转发终端日志到 HTTP 服务器，此操作很危险！", - "modal_logtohttpserver_desc": "所有您的带敏感信息的终端日志，都会被转发到 HTTP(S) 服务器，没有任何鉴权！！！！！\n请确保您信任对应的服务器，最好设置为 HTTPS 而不是 HTTP。\n仅仅用于 debug 用途，例如手机上的 debug。", - "modal_logtohttpserver_secondconfirm": "我知道很危险，坚持要设置，愿意承担所有可能损失。", - "modal_logtohttpserver_notice": "已设置。", "settings_basic": "基本设置", "settings_password": "密码", "settings_password_desc": "端到端加密的密码。不填写则代表没密码。您需要点击“确认”来修改。注意：密码和其它信息都会在本地保存。", @@ -264,12 +260,14 @@ "settings_import": "导入", "settings_import_desc": "您需要使用系统拍摄 app 或者扫描 QR 码的app，来扫描对应的 QR 码。", "settings_debug": "调试", - "settings_debuglevel": "修改终端输出的 level", - "settings_debuglevel_desc": "默认值为 \"info\"。您可以改为 \"debug\" 从而在终端里获取更多信息。", + "settings_debuglevel": "修改同步提示信息", + "settings_debuglevel_desc": "默认值为 \"info\"。您可以改为 \"debug\" 从而在同步时候里获取更多信息。", "settings_outputsettingsconsole": "读取硬盘上的设置文件输出到终端", "settings_outputsettingsconsole_desc": "硬盘上的设置文件是编码过的，点击这里从而解码并输出到终端。", "settings_outputsettingsconsole_button": "输出", "settings_outputsettingsconsole_notice": "已输出到终端", + "settings_viewconsolelog": "查看终端输出", + "settings_viewconsolelog_desc": "电脑上，输入“ctrl+shift+i”或“cmd+shift+i”来查看终端输出。手机上，安装第三方插件 Logstravaganza 来导出终端输出到一篇笔记上。", "settings_syncplans": "导出同步计划", "settings_syncplans_desc": "每次您启动同步，并在实际上传下载前，插件会生成同步计划。它可以使您知道每次同步发生了什么。点击按钮可以导出同步计划。", "settings_syncplans_button_json": "导出", @@ -278,9 +276,6 @@ "settings_delsyncplans_desc": "删除数据库里的同步计划历史。", "settings_delsyncplans_button": "删除同步计划历史", "settings_delsyncplans_notice": "（数据库里的）同步计划已被删除。", - "settings_logtohttpserver": "临时设定终端日志实时转发到 HTTP(S) 服务器。", - "settings_logtohttpserver_desc": "非常危险，谨慎行动！！！！！临时设定终端日志实时转发到 HTTP(S) 服务器。", - "settings_logtohttpserver_reset_notice": "您的输入不是“http(s)”开头的。已移除了终端日志转发到 HTTP(S) 服务器的设定。", "settings_delprevsync": "删除数据库里的上次同步明细", "settings_delprevsync_desc": "同步算法需要上次成功同步的信息来决定文件变更，这个信息保存在本地的数据库里。如果您想忽略这些信息从而所有文件都被视为新创建的话，可以在此删除之前的信息。", "settings_delprevsync_button": "删除上次同步明细", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index f565f91..f94b42e 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -106,10 +106,6 @@ "modal_sizesconflict_desc": "您設定了跳過同步大於 {{thresholdMB}} MB（{{thresholdBytes}} bytes）的檔案。\n但是以下檔案的大小，在一端大於閾值，在另一端則小於閾值。\n為了避免意外的覆蓋或刪除，外掛停止了運作，您需要手動處理至少一端的檔案。", "modal_sizesconflict_copybutton": "點選以複製以下所有檔案大小衝突資訊", "modal_sizesconflict_copynotice": "所有的檔案大小衝突資訊，已被複制到剪貼簿！", - "modal_logtohttpserver_title": "轉發終端日誌到 HTTP 伺服器，此操作很危險！", - "modal_logtohttpserver_desc": "所有您的帶敏感資訊的終端日誌，都會被轉發到 HTTP(S) 伺服器，沒有任何鑑權！！！！！\n請確保您信任對應的伺服器，最好設定為 HTTPS 而不是 HTTP。\n僅僅用於 debug 用途，例如手機上的 debug。", - "modal_logtohttpserver_secondconfirm": "我知道很危險，堅持要設定，願意承擔所有可能損失。", - "modal_logtohttpserver_notice": "已設定。", "settings_basic": "基本設定", "settings_password": "密碼", "settings_password_desc": "端到端加密的密碼。不填寫則代表沒密碼。您需要點選“確認”來修改。注意：密碼和其它資訊都會在本地儲存。", @@ -264,12 +260,14 @@ "settings_import": "匯入", "settings_import_desc": "您需要使用系統拍攝 app 或者掃描 QR 碼的app，來掃描對應的 QR 碼。", "settings_debug": "除錯", - "settings_debuglevel": "修改終端輸出的 level", - "settings_debuglevel_desc": "預設值為 \"info\"。您可以改為 \"debug\" 從而在終端裡獲取更多資訊。", + "settings_debuglevel": "修改同步提示資訊", + "settings_debuglevel_desc": "預設值為 \"info\"。您可以改為 \"debug\" 從而在同步時候裡獲取更多資訊。", "settings_outputsettingsconsole": "讀取硬碟上的設定檔案輸出到終端", "settings_outputsettingsconsole_desc": "硬碟上的設定檔案是編碼過的，點選這裡從而解碼並輸出到終端。", "settings_outputsettingsconsole_button": "輸出", "settings_outputsettingsconsole_notice": "已輸出到終端", + "settings_viewconsolelog": "檢視終端輸出", + "settings_viewconsolelog_desc": "電腦上，輸入“ctrl+shift+i”或“cmd+shift+i”來檢視終端輸出。手機上，安裝第三方外掛 Logstravaganza 來匯出終端輸出到一篇筆記上。", "settings_syncplans": "匯出同步計劃", "settings_syncplans_desc": "每次您啟動同步，並在實際上傳下載前，外掛會生成同步計劃。它可以使您知道每次同步發生了什麼。點選按鈕可以匯出同步計劃。", "settings_syncplans_button_json": "匯出", @@ -278,9 +276,6 @@ "settings_delsyncplans_desc": "刪除資料庫裡的同步計劃歷史。", "settings_delsyncplans_button": "刪除同步計劃歷史", "settings_delsyncplans_notice": "（資料庫裡的）同步計劃已被刪除。", - "settings_logtohttpserver": "臨時設定終端日誌實時轉發到 HTTP(S) 伺服器。", - "settings_logtohttpserver_desc": "非常危險，謹慎行動！！！！！臨時設定終端日誌實時轉發到 HTTP(S) 伺服器。", - "settings_logtohttpserver_reset_notice": "您的輸入不是“http(s)”開頭的。已移除了終端日誌轉發到 HTTP(S) 伺服器的設定。", "settings_delprevsync": "刪除資料庫裡的上次同步明細", "settings_delprevsync_desc": "同步演算法需要上次成功同步的資訊來決定檔案變更，這個資訊儲存在本地的資料庫裡。如果您想忽略這些資訊從而所有檔案都被視為新建立的話，可以在此刪除之前的資訊。", "settings_delprevsync_button": "刪除上次同步明細", diff --git a/src/localdb.ts b/src/localdb.ts index e889ee7..385b5fe 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -8,8 +8,6 @@ import type { Entity, MixedEntity, SUPPORTED_SERVICES_TYPE } from "./baseTypes"; import type { SyncPlanType } from "./sync"; import { statFix, toText, unixTimeToStr } from "./misc"; -import { log } from "./moreOnLog"; - const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220]; export const DEFAULT_DB_VERSION_NUMBER: number = 20240220; export const DEFAULT_DB_NAME = "remotelysavedb"; @@ -119,7 +117,7 @@ const migrateDBsFrom20220326To20240220 = async ( ) => { const oldVer = 20220326; const newVer = 20240220; - log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`); + console.debug(`start upgrading internal db from ${oldVer} to ${newVer}`); // from sync mapping to prev sync const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID); @@ -135,7 +133,7 @@ const migrateDBsFrom20220326To20240220 = async ( // await clearAllSyncMetaMappingByVault(db, vaultRandomID); await db.versionTbl.setItem(`${vaultRandomID}\tversion`, newVer); - log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); + console.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); }; const migrateDBs = async ( @@ -243,7 +241,7 @@ export const prepareDBs = async ( (await db.versionTbl.getItem(`${vaultRandomID}\tversion`)) ?? (await db.versionTbl.getItem("version")); if (originalVersion === null) { - log.debug( + console.debug( `no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}` ); // as of 20240220, we set the version per vault, instead of global "version" @@ -254,7 +252,7 @@ export const prepareDBs = async ( } else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) { // do nothing } else { - log.debug( + console.debug( `trying to upgrade db version from ${originalVersion} to ${DEFAULT_DB_VERSION_NUMBER}` ); await migrateDBs( @@ -265,7 +263,7 @@ export const prepareDBs = async ( ); } - log.info("db connected"); + console.info("db connected"); return { db: db, vaultRandomID: vaultRandomID, @@ -276,17 +274,17 @@ export const destroyDBs = async () => { // await localforage.dropInstance({ // name: DEFAULT_DB_NAME, // }); - // log.info("db deleted"); + // console.info("db deleted"); const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME); req.onsuccess = (event) => { - log.info("db deleted"); + console.info("db deleted"); }; req.onblocked = (event) => { - log.warn("trying to delete db but it was blocked"); + console.warn("trying to delete db but it was blocked"); }; req.onerror = (event) => { - log.error("tried to delete db but something goes wrong!"); - log.error(event); + console.error("tried to delete db but something goes wrong!"); + console.error(event); }; }; @@ -420,9 +418,9 @@ export const getAllPrevSyncRecordsByVault = async ( db: InternalDBs, vaultRandomID: string ) => { - // log.debug('inside getAllPrevSyncRecordsByVault') + // console.debug('inside getAllPrevSyncRecordsByVault') const keys = await db.prevSyncRecordsTbl.keys(); - // log.debug(`inside getAllPrevSyncRecordsByVault, keys=${keys}`) + // console.debug(`inside getAllPrevSyncRecordsByVault, keys=${keys}`) const res: Entity[] = []; for (const key of keys) { if (key.startsWith(`${vaultRandomID}\t`)) { @@ -468,7 +466,7 @@ export const clearAllPrevSyncRecordByVault = async ( export const clearAllLoggerOutputRecords = async (db: InternalDBs) => { await db.loggerOutputTbl.clear(); - log.debug(`successfully clearAllLoggerOutputRecords`); + console.debug(`successfully clearAllLoggerOutputRecords`); }; export const upsertLastSuccessSyncTimeByVault = async ( diff --git a/src/main.ts b/src/main.ts index 253b922..1d1143d 100644 --- a/src/main.ts +++ b/src/main.ts @@ -64,7 +64,6 @@ import { I18n } from "./i18n"; import type { LangType, LangTypeAndAuto, TransItemType } from "./i18n"; import { SyncAlgoV3Modal } from "./syncAlgoV3Notice"; -import { applyLogWriterInplace, log } from "./moreOnLog"; import AggregateError from "aggregate-error"; import { exportVaultSyncPlansToFiles } from "./debugMode"; import { compareVersion } from "./misc"; @@ -177,7 +176,7 @@ export default class RemotelySavePlugin extends Plugin { } try { - log.info( + console.info( `${ this.manifest.id }-${Date.now()}: start sync, triggerSource=${triggerSource}` @@ -206,7 +205,7 @@ export default class RemotelySavePlugin extends Plugin { if (this.statusBarElement !== undefined) { this.updateLastSuccessSyncMsg(-1); } - //log.info(`huh ${this.settings.password}`) + //console.info(`huh ${this.settings.password}`) if (this.settings.currLogLevel === "info") { getNotice( t("syncrun_shortstep1", { @@ -240,8 +239,8 @@ export default class RemotelySavePlugin extends Plugin { () => self.saveSettings() ); const remoteEntityList = await client.listAllFromRemote(); - log.debug("remoteEntityList:"); - log.debug(remoteEntityList); + console.debug("remoteEntityList:"); + console.debug(remoteEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -270,8 +269,8 @@ export default class RemotelySavePlugin extends Plugin { this.app.vault.configDir, this.manifest.id ); - log.debug("localEntityList:"); - log.debug(localEntityList); + console.debug("localEntityList:"); + console.debug(localEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -283,8 +282,8 @@ export default class RemotelySavePlugin extends Plugin { this.db, this.vaultRandomID ); - log.debug("prevSyncEntityList:"); - log.debug(prevSyncEntityList); + console.debug("prevSyncEntityList:"); + console.debug(prevSyncEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -308,8 +307,8 @@ export default class RemotelySavePlugin extends Plugin { this.settings.skipSizeLargerThan ?? -1, this.settings.conflictAction ?? "keep_newer" ); - log.info(`mixedEntityMappings:`); - log.info(mixedEntityMappings); // for debugging + console.info(`mixedEntityMappings:`); + console.info(mixedEntityMappings); // for debugging await insertSyncPlanRecordByVault( this.db, mixedEntityMappings, @@ -383,7 +382,7 @@ export default class RemotelySavePlugin extends Plugin { this.updateLastSuccessSyncMsg(lastSuccessSyncMillis); } - log.info( + console.info( `${ this.manifest.id }-${Date.now()}: finish sync, triggerSource=${triggerSource}` @@ -395,8 +394,8 @@ export default class RemotelySavePlugin extends Plugin { triggerSource: triggerSource, syncStatus: this.syncStatus, }); - log.error(msg); - log.error(error); + console.error(msg); + console.error(error); getNotice(msg, 10 * 1000); if (error instanceof AggregateError) { for (const e of error.errors) { @@ -414,7 +413,7 @@ export default class RemotelySavePlugin extends Plugin { } async onload() { - log.info(`loading plugin ${this.manifest.id}`); + console.info(`loading plugin ${this.manifest.id}`); const { iconSvgSyncWait, iconSvgSyncRunning, iconSvgLogs } = getIconSvg(); @@ -443,10 +442,6 @@ export default class RemotelySavePlugin extends Plugin { return this.i18n.t(x, vars); }; - if (this.settings.currLogLevel !== undefined) { - log.setLevel(this.settings.currLogLevel as any); - } - await this.checkIfOauthExpires(); // MUST before prepareDB() @@ -474,7 +469,6 @@ export default class RemotelySavePlugin extends Plugin { } // must AFTER preparing DB - this.redirectLoggingOuputBasedOnSetting(); this.enableAutoClearOutputToDBHistIfSet(); // must AFTER preparing DB @@ -751,7 +745,7 @@ export default class RemotelySavePlugin extends Plugin { this.addSettingTab(new RemotelySaveSettingTab(this.app, this)); // this.registerDomEvent(document, "click", (evt: MouseEvent) => { - // log.info("click", evt); + // console.info("click", evt); // }); if (!this.settings.agreeToUseSyncV3) { @@ -772,7 +766,7 @@ export default class RemotelySavePlugin extends Plugin { } async onunload() { - log.info(`unloading plugin ${this.manifest.id}`); + console.info(`unloading plugin ${this.manifest.id}`); this.syncRibbon = undefined; if (this.oauth2Info !== undefined) { this.oauth2Info.helperModal = undefined; @@ -951,7 +945,7 @@ export default class RemotelySavePlugin extends Plugin { // a real string was assigned before vaultRandomID = this.settings.vaultRandomID; } - log.debug("vaultRandomID is no longer saved in data.json"); + console.debug("vaultRandomID is no longer saved in data.json"); delete this.settings.vaultRandomID; await this.saveSettings(); } @@ -1031,7 +1025,7 @@ export default class RemotelySavePlugin extends Plugin { let needToRunAgain = false; const scheduleSyncOnSave = (scheduleTimeFromNow: number) => { - log.info( + console.info( `schedule a run for ${scheduleTimeFromNow} milliseconds later` ); runScheduled = true; @@ -1195,31 +1189,6 @@ export default class RemotelySavePlugin extends Plugin { } } - redirectLoggingOuputBasedOnSetting() { - applyLogWriterInplace((...msg: any[]) => { - if ( - this.debugServerTemp !== undefined && - this.debugServerTemp.trim().startsWith("http") - ) { - try { - requestUrl({ - url: this.debugServerTemp, - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - send_time: Date.now(), - log_text: msg, - }), - }); - } catch (e) { - // pass - } - } - }); - } - enableAutoClearOutputToDBHistIfSet() { const initClearOutputToDBHistAfterMilliseconds = 1000 * 30; diff --git a/src/metadataOnRemote.ts b/src/metadataOnRemote.ts index b1349a4..b47a883 100644 --- a/src/metadataOnRemote.ts +++ b/src/metadataOnRemote.ts @@ -1,7 +1,6 @@ import isEqual from "lodash/isEqual"; import { base64url } from "rfc4648"; import { reverseString } from "./misc"; -import { log } from "./moreOnLog"; const DEFAULT_README_FOR_METADATAONREMOTE = "Do NOT edit or delete the file manually. This file is for the plugin remotely-save to store some necessary meta data on the remote services. Its content is slightly obfuscated."; diff --git a/src/misc.ts b/src/misc.ts index 83b0145..c522c50 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -5,8 +5,6 @@ import { base32, base64url } from "rfc4648"; import XRegExp from "xregexp"; import emojiRegex from "emoji-regex"; -import { log } from "./moreOnLog"; - declare global { interface Window { moment: (...data: any) => any; @@ -30,7 +28,7 @@ export const isHiddenPath = ( } const k = path.posix.normalize(item); // TODO: only unix path now const k2 = k.split("/"); // TODO: only unix path now - // log.info(k2) + // console.info(k2) for (const singlePart of k2) { if (singlePart === "." || singlePart === ".." || singlePart === "") { continue; @@ -75,14 +73,14 @@ export const getFolderLevels = (x: string, addEndingSlash: boolean = false) => { }; export const mkdirpInVault = async (thePath: string, vault: Vault) => { - // log.info(thePath); + // console.info(thePath); const foldersToBuild = getFolderLevels(thePath); - // log.info(foldersToBuild); + // console.info(foldersToBuild); for (const folder of foldersToBuild) { const r = await vault.adapter.exists(folder); - // log.info(r); + // console.info(r); if (!r) { - log.info(`mkdir ${folder}`); + console.info(`mkdir ${folder}`); await vault.adapter.mkdir(folder); } } diff --git a/src/moreOnLog.ts b/src/moreOnLog.ts deleted file mode 100644 index 11d146a..0000000 --- a/src/moreOnLog.ts +++ /dev/null @@ -1,40 +0,0 @@ -// It's very dangerous for this file to depend on other files in the same project. -// We should avoid this situation as much as possible. - -import { TAbstractFile, TFolder, TFile, Vault } from "obsidian"; - -import * as origLog from "loglevel"; -import type { - LogLevelNumbers, - Logger, - LogLevel, - LogLevelDesc, - LogLevelNames, -} from "loglevel"; -const log2 = origLog.getLogger("rs-default"); - -const originalFactory = log2.methodFactory; - -export const applyLogWriterInplace = function (writer: (...msg: any[]) => any) { - log2.methodFactory = function ( - methodName: LogLevelNames, - logLevel: LogLevelNumbers, - loggerName: string | symbol - ) { - const rawMethod = originalFactory(methodName, logLevel, loggerName); - - return function (...msg: any[]) { - rawMethod.apply(undefined, msg); - writer(...msg); - }; - }; - - log2.setLevel(log2.getLevel()); -}; - -export const restoreLogWritterInplace = () => { - log2.methodFactory = originalFactory; - log2.setLevel(log2.getLevel()); -}; - -export const log = log2; diff --git a/src/remote.ts b/src/remote.ts index c7467b1..d361d0f 100644 --- a/src/remote.ts +++ b/src/remote.ts @@ -13,8 +13,6 @@ import * as onedrive from "./remoteForOnedrive"; import * as s3 from "./remoteForS3"; import * as webdav from "./remoteForWebdav"; -import { log } from "./moreOnLog"; - export class RemoteClient { readonly serviceType: SUPPORTED_SERVICES_TYPE; readonly s3Config?: S3Config; diff --git a/src/remoteForDropbox.ts b/src/remoteForDropbox.ts index a084919..a98d5d2 100644 --- a/src/remoteForDropbox.ts +++ b/src/remoteForDropbox.ts @@ -21,8 +21,6 @@ import { export { Dropbox } from "dropbox"; -import { log } from "./moreOnLog"; - export const DEFAULT_DROPBOX_CONFIG: DropboxConfig = { accessToken: "", clientID: process.env.DEFAULT_DROPBOX_APP_KEY ?? "", @@ -43,7 +41,7 @@ export const getDropboxPath = ( // special key = `/${remoteBaseDir}`; } else if (fileOrFolderPath.startsWith("/")) { - log.warn( + console.warn( `why the path ${fileOrFolderPath} starts with '/'? but we just go on.` ); key = `/${remoteBaseDir}${fileOrFolderPath}`; @@ -169,7 +167,7 @@ export const sendAuthReq = async ( const resp2 = (await resp1.json()) as DropboxSuccessAuthRes; return resp2; } catch (e) { - log.error(e); + console.error(e); if (errorCallBack !== undefined) { await errorCallBack(e); } @@ -181,7 +179,7 @@ export const sendRefreshTokenReq = async ( refreshToken: string ) => { try { - log.info("start auto getting refreshed Dropbox access token."); + console.info("start auto getting refreshed Dropbox access token."); const resp1 = await fetch("https://api.dropboxapi.com/oauth2/token", { method: "POST", body: new URLSearchParams({ @@ -191,10 +189,10 @@ export const sendRefreshTokenReq = async ( }), }); const resp2 = (await resp1.json()) as DropboxSuccessAuthRes; - log.info("finish auto getting refreshed Dropbox access token."); + console.info("finish auto getting refreshed Dropbox access token."); return resp2; } catch (e) { - log.error(e); + console.error(e); throw e; } }; @@ -204,7 +202,7 @@ export const setConfigBySuccessfullAuthInplace = async ( authRes: DropboxSuccessAuthRes, saveUpdatedConfigFunc: () => Promise | undefined ) => { - log.info("start updating local info of Dropbox token"); + console.info("start updating local info of Dropbox token"); config.accessToken = authRes.access_token; config.accessTokenExpiresInSeconds = parseInt(authRes.expires_in); @@ -224,7 +222,7 @@ export const setConfigBySuccessfullAuthInplace = async ( await saveUpdatedConfigFunc(); } - log.info("finish updating local info of Dropbox token"); + console.info("finish updating local info of Dropbox token"); }; //////////////////////////////////////////////////////////////////////////////// @@ -245,7 +243,7 @@ async function retryReq( for (let idx = 0; idx < waitSeconds.length; ++idx) { try { if (idx !== 0) { - log.warn( + console.warn( `${extraHint === "" ? "" : extraHint + ": "}The ${ idx + 1 }-th try starts at time ${Date.now()}` @@ -282,7 +280,7 @@ async function retryReq( const fallbackSec = waitSeconds[idx]; const secMin = Math.max(svrSec, fallbackSec); const secMax = Math.max(secMin * 1.8, 2); - log.warn( + console.warn( `${ extraHint === "" ? "" : extraHint + ": " }We have "429 too many requests" error of ${ @@ -355,9 +353,9 @@ export class WrappedDropboxClient { } // check vault folder - // log.info(`checking remote has folder /${this.remoteBaseDir}`); + // console.info(`checking remote has folder /${this.remoteBaseDir}`); if (this.vaultFolderExists) { - // log.info(`already checked, /${this.remoteBaseDir} exist before`) + // console.info(`already checked, /${this.remoteBaseDir} exist before`) } else { const res = await this.dropbox.filesListFolder({ path: "", @@ -370,7 +368,7 @@ export class WrappedDropboxClient { } } if (!this.vaultFolderExists) { - log.info(`remote does not have folder /${this.remoteBaseDir}`); + console.info(`remote does not have folder /${this.remoteBaseDir}`); if (hasEmojiInText(`/${this.remoteBaseDir}`)) { throw new Error( @@ -381,10 +379,10 @@ export class WrappedDropboxClient { await this.dropbox.filesCreateFolderV2({ path: `/${this.remoteBaseDir}`, }); - log.info(`remote folder /${this.remoteBaseDir} created`); + console.info(`remote folder /${this.remoteBaseDir} created`); this.vaultFolderExists = true; } else { - // log.info(`remote folder /${this.remoteBaseDir} exists`); + // console.info(`remote folder /${this.remoteBaseDir} exists`); } } @@ -612,7 +610,7 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => { if (res.status !== 200) { throw Error(JSON.stringify(res)); } - // log.info(res); + // console.info(res); const contents = res.result.entries; const unifiedContents = contents @@ -736,8 +734,8 @@ export const deleteFromRemote = async ( fileOrFolderPath ); } catch (err) { - log.error("some error while deleting"); - log.error(err); + console.error("some error while deleting"); + console.error(err); } }; @@ -753,7 +751,7 @@ export const checkConnectivity = async ( } return true; } catch (err) { - log.debug(err); + console.debug(err); if (callbackFunc !== undefined) { callbackFunc(err); } diff --git a/src/remoteForOnedrive.ts b/src/remoteForOnedrive.ts index fa7f44d..298ccda 100644 --- a/src/remoteForOnedrive.ts +++ b/src/remoteForOnedrive.ts @@ -25,8 +25,6 @@ import { mkdirpInVault, } from "./misc"; -import { log } from "./moreOnLog"; - const SCOPES = ["User.Read", "Files.ReadWrite.AppFolder", "offline_access"]; const REDIRECT_URI = `obsidian://${COMMAND_CALLBACK_ONEDRIVE}`; @@ -117,8 +115,8 @@ export const sendAuthReq = async ( // code: authCode, // codeVerifier: verifier, // PKCE Code Verifier // }); - // log.info('authResponse') - // log.info(authResponse) + // console.info('authResponse') + // console.info(authResponse) // return authResponse; // Because of the CORS problem, @@ -143,7 +141,7 @@ export const sendAuthReq = async ( }); const rsp2 = JSON.parse(rsp1); - // log.info(rsp2); + // console.info(rsp2); if (rsp2.error !== undefined) { return rsp2 as AccessCodeResponseFailedType; @@ -151,7 +149,7 @@ export const sendAuthReq = async ( return rsp2 as AccessCodeResponseSuccessfulType; } } catch (e) { - log.error(e); + console.error(e); await errorCallBack(e); } }; @@ -177,7 +175,7 @@ export const sendRefreshTokenReq = async ( }); const rsp2 = JSON.parse(rsp1); - // log.info(rsp2); + // console.info(rsp2); if (rsp2.error !== undefined) { return rsp2 as AccessCodeResponseFailedType; @@ -185,7 +183,7 @@ export const sendRefreshTokenReq = async ( return rsp2 as AccessCodeResponseSuccessfulType; } } catch (e) { - log.error(e); + console.error(e); throw e; } }; @@ -195,7 +193,7 @@ export const setConfigBySuccessfullAuthInplace = async ( authRes: AccessCodeResponseSuccessfulType, saveUpdatedConfigFunc: () => Promise | undefined ) => { - log.info("start updating local info of OneDrive token"); + console.info("start updating local info of OneDrive token"); config.accessToken = authRes.access_token; config.accessTokenExpiresAtTime = Date.now() + authRes.expires_in - 5 * 60 * 1000; @@ -210,7 +208,7 @@ export const setConfigBySuccessfullAuthInplace = async ( await saveUpdatedConfigFunc(); } - log.info("finish updating local info of Onedrive token"); + console.info("finish updating local info of Onedrive token"); }; //////////////////////////////////////////////////////////////////////////////// @@ -231,7 +229,7 @@ const getOnedrivePath = (fileOrFolderPath: string, remoteBaseDir: string) => { } if (key.startsWith("/")) { - log.warn(`why the path ${key} starts with '/'? but we just go on.`); + console.warn(`why the path ${key} starts with '/'? but we just go on.`); key = `${prefix}${key}`; } else { key = `${prefix}/${key}`; @@ -403,7 +401,7 @@ class MyAuthProvider implements AuthenticationProvider { this.onedriveConfig.accessTokenExpiresAtTime = currentTs + r2.expires_in * 1000 - 60 * 2 * 1000; await this.saveUpdatedConfigFunc(); - log.info("Onedrive accessToken updated"); + console.info("Onedrive accessToken updated"); return this.onedriveConfig.accessToken; } }; @@ -437,26 +435,26 @@ export class WrappedOnedriveClient { } // check vault folder - // log.info(`checking remote has folder /${this.remoteBaseDir}`); + // console.info(`checking remote has folder /${this.remoteBaseDir}`); if (this.vaultFolderExists) { - // log.info(`already checked, /${this.remoteBaseDir} exist before`) + // console.info(`already checked, /${this.remoteBaseDir} exist before`) } else { const k = await this.getJson("/drive/special/approot/children"); - // log.debug(k); + // console.debug(k); this.vaultFolderExists = (k.value as DriveItem[]).filter((x) => x.name === this.remoteBaseDir) .length > 0; if (!this.vaultFolderExists) { - log.info(`remote does not have folder /${this.remoteBaseDir}`); + console.info(`remote does not have folder /${this.remoteBaseDir}`); await this.postJson("/drive/special/approot/children", { name: `${this.remoteBaseDir}`, folder: {}, "@microsoft.graph.conflictBehavior": "replace", }); - log.info(`remote folder /${this.remoteBaseDir} created`); + console.info(`remote folder /${this.remoteBaseDir} created`); this.vaultFolderExists = true; } else { - // log.info(`remote folder /${this.remoteBaseDir} exists`); + // console.info(`remote folder /${this.remoteBaseDir} exists`); } } }; @@ -478,7 +476,7 @@ export class WrappedOnedriveClient { getJson = async (pathFragOrig: string) => { const theUrl = this.buildUrl(pathFragOrig); - log.debug(`getJson, theUrl=${theUrl}`); + console.debug(`getJson, theUrl=${theUrl}`); return JSON.parse( await request({ url: theUrl, @@ -494,7 +492,7 @@ export class WrappedOnedriveClient { postJson = async (pathFragOrig: string, payload: any) => { const theUrl = this.buildUrl(pathFragOrig); - log.debug(`postJson, theUrl=${theUrl}`); + console.debug(`postJson, theUrl=${theUrl}`); return JSON.parse( await request({ url: theUrl, @@ -510,7 +508,7 @@ export class WrappedOnedriveClient { patchJson = async (pathFragOrig: string, payload: any) => { const theUrl = this.buildUrl(pathFragOrig); - log.debug(`patchJson, theUrl=${theUrl}`); + console.debug(`patchJson, theUrl=${theUrl}`); return JSON.parse( await request({ url: theUrl, @@ -526,7 +524,7 @@ export class WrappedOnedriveClient { deleteJson = async (pathFragOrig: string) => { const theUrl = this.buildUrl(pathFragOrig); - log.debug(`deleteJson, theUrl=${theUrl}`); + console.debug(`deleteJson, theUrl=${theUrl}`); if (VALID_REQURL) { await requestUrl({ url: theUrl, @@ -547,7 +545,7 @@ export class WrappedOnedriveClient { putArrayBuffer = async (pathFragOrig: string, payload: ArrayBuffer) => { const theUrl = this.buildUrl(pathFragOrig); - log.debug(`putArrayBuffer, theUrl=${theUrl}`); + console.debug(`putArrayBuffer, theUrl=${theUrl}`); // TODO: // 20220401: On Android, requestUrl has issue that text becomes base64. // Use fetch everywhere instead! @@ -590,7 +588,7 @@ export class WrappedOnedriveClient { size: number ) => { const theUrl = this.buildUrl(pathFragOrig); - log.debug( + console.debug( `putUint8ArrayByRange, theUrl=${theUrl}, range=${rangeStart}-${ rangeEnd - 1 }, len=${rangeEnd - rangeStart}, size=${size}` @@ -655,7 +653,7 @@ export const listAllFromRemote = async (client: WrappedOnedriveClient) => { `/drive/special/approot:/${client.remoteBaseDir}:/delta` ); let driveItems = res.value as DriveItem[]; - // log.debug(driveItems); + // console.debug(driveItems); while (NEXT_LINK_KEY in res) { res = await client.getJson(res[NEXT_LINK_KEY]); @@ -681,14 +679,14 @@ export const getRemoteMeta = async ( remotePath: string ) => { await client.init(); - // log.info(`remotePath=${remotePath}`); + // console.info(`remotePath=${remotePath}`); const rsp = await client.getJson( `${remotePath}?$select=cTag,eTag,fileSystemInfo,folder,file,name,parentReference,size` ); - // log.info(rsp); + // console.info(rsp); const driveItem = rsp as DriveItem; const res = fromDriveItemToEntity(driveItem, client.remoteBaseDir); - // log.info(res); + // console.info(res); return res; }; @@ -715,7 +713,7 @@ export const uploadToRemote = async ( uploadFile = remoteEncryptedKey; } uploadFile = getOnedrivePath(uploadFile, client.remoteBaseDir); - log.debug(`uploadFile=${uploadFile}`); + console.debug(`uploadFile=${uploadFile}`); let mtime = 0; let ctime = 0; @@ -792,7 +790,7 @@ export const uploadToRemote = async ( } as FileSystemInfo, }); } - // log.info(uploadResult) + // console.info(uploadResult) const res = await getRemoteMeta(client, uploadFile); return { entity: res, @@ -874,8 +872,8 @@ export const uploadToRemote = async ( k ); const uploadUrl = s.uploadUrl!; - log.debug("uploadSession = "); - log.debug(s); + console.debug("uploadSession = "); + console.debug(s); // 2. upload by ranges // convert to uint8 @@ -995,7 +993,7 @@ export const checkConnectivity = async ( const k = await getUserDisplayName(client); return k !== ""; } catch (err) { - log.debug(err); + console.debug(err); if (callbackFunc !== undefined) { callbackFunc(err); } diff --git a/src/remoteForS3.ts b/src/remoteForS3.ts index 6d59406..0d2d6de 100644 --- a/src/remoteForS3.ts +++ b/src/remoteForS3.ts @@ -42,7 +42,6 @@ import { export { S3Client } from "@aws-sdk/client-s3"; -import { log } from "./moreOnLog"; import PQueue from "p-queue"; //////////////////////////////////////////////////////////////////////////////// @@ -227,7 +226,7 @@ const fromS3ObjectToEntity = ( mtimeRecords: Record, ctimeRecords: Record ) => { - // log.debug(`fromS3ObjectToEntity: ${x.Key!}, ${JSON.stringify(x,null,2)}`); + // console.debug(`fromS3ObjectToEntity: ${x.Key!}, ${JSON.stringify(x,null,2)}`); // S3 officially only supports seconds precision!!!!! const mtimeSvr = Math.floor(x.LastModified!.valueOf() / 1000.0) * 1000; let mtimeCli = mtimeSvr; @@ -253,7 +252,7 @@ const fromS3HeadObjectToEntity = ( x: HeadObjectCommandOutput, remotePrefix: string ) => { - // log.debug(`fromS3HeadObjectToEntity: ${fileOrFolderPathWithRemotePrefix}: ${JSON.stringify(x,null,2)}`); + // console.debug(`fromS3HeadObjectToEntity: ${fileOrFolderPathWithRemotePrefix}: ${JSON.stringify(x,null,2)}`); // S3 officially only supports seconds precision!!!!! const mtimeSvr = Math.floor(x.LastModified!.valueOf() / 1000.0) * 1000; let mtimeCli = mtimeSvr; @@ -265,7 +264,7 @@ const fromS3HeadObjectToEntity = ( mtimeCli = m2; } } - // log.debug( + // console.debug( // `fromS3HeadObjectToEntity, fileOrFolderPathWithRemotePrefix=${fileOrFolderPathWithRemotePrefix}, remotePrefix=${remotePrefix}, x=${JSON.stringify( // x // )} ` @@ -274,7 +273,7 @@ const fromS3HeadObjectToEntity = ( fileOrFolderPathWithRemotePrefix, remotePrefix ); - // log.debug(`fromS3HeadObjectToEntity, key=${key} after removing prefix`); + // console.debug(`fromS3HeadObjectToEntity, key=${key} after removing prefix`); return { keyRaw: key, mtimeSvr: mtimeSvr, @@ -367,7 +366,7 @@ export const uploadToRemote = async ( rawContentMTime: number = 0, rawContentCTime: number = 0 ): Promise => { - log.debug(`uploading ${fileOrFolderPath}`); + console.debug(`uploading ${fileOrFolderPath}`); let uploadFile = fileOrFolderPath; if (password !== "") { if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { @@ -378,7 +377,7 @@ export const uploadToRemote = async ( uploadFile = remoteEncryptedKey; } uploadFile = getRemoteWithPrefixPath(uploadFile, s3Config.remotePrefix ?? ""); - // log.debug(`actual uploadFile=${uploadFile}`); + // console.debug(`actual uploadFile=${uploadFile}`); const isFolder = fileOrFolderPath.endsWith("/"); if (isFolder && isRecursively) { @@ -472,12 +471,12 @@ export const uploadToRemote = async ( }, }); upload.on("httpUploadProgress", (progress) => { - // log.info(progress); + // console.info(progress); }); await upload.done(); const res = await getRemoteMeta(s3Client, s3Config, uploadFile); - // log.debug( + // console.debug( // `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}` // ); return { @@ -772,7 +771,7 @@ export const checkConnectivity = async ( results.$metadata.httpStatusCode === undefined ) { const err = "results or $metadata or httStatusCode is undefined"; - log.debug(err); + console.debug(err); if (callbackFunc !== undefined) { callbackFunc(err); } @@ -780,7 +779,7 @@ export const checkConnectivity = async ( } return results.$metadata.httpStatusCode === 200; } catch (err: any) { - log.debug(err); + console.debug(err); if (callbackFunc !== undefined) { if (s3Config.s3Endpoint.contains(s3Config.s3BucketName)) { const err2 = new AggregateError([ diff --git a/src/remoteForWebdav.ts b/src/remoteForWebdav.ts index 0630e34..8df877d 100644 --- a/src/remoteForWebdav.ts +++ b/src/remoteForWebdav.ts @@ -9,8 +9,6 @@ import { Entity, UploadedType, VALID_REQURL, WebdavConfig } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc"; -import { log } from "./moreOnLog"; - import type { FileStat, WebDAVClient, @@ -85,9 +83,9 @@ if (VALID_REQURL) { } } } - // log.info(`requesting url=${options.url}`); - // log.info(`contentType=${contentType}`); - // log.info(`rspHeaders=${JSON.stringify(rspHeaders)}`) + // console.info(`requesting url=${options.url}`); + // console.info(`contentType=${contentType}`); + // console.info(`rspHeaders=${JSON.stringify(rspHeaders)}`) // let r2: Response = undefined; // if (contentType.includes("xml")) { @@ -100,9 +98,9 @@ if (VALID_REQURL) { // contentType.includes("json") || // contentType.includes("javascript") // ) { - // log.info('inside json branch'); + // console.info('inside json branch'); // // const j = r.json; - // // log.info(j); + // // console.info(j); // r2 = new Response( // r.text, // yea, here is the text because Response constructor expects a text // { @@ -178,7 +176,7 @@ const getWebdavPath = (fileOrFolderPath: string, remoteBaseDir: string) => { // special key = `/${remoteBaseDir}/`; } else if (fileOrFolderPath.startsWith("/")) { - log.warn( + console.warn( `why the path ${fileOrFolderPath} starts with '/'? but we just go on.` ); key = `/${remoteBaseDir}${fileOrFolderPath}`; @@ -259,7 +257,7 @@ export class WrappedWebdavClient { : AuthType.Password, }); } else { - log.info("no password"); + console.info("no password"); this.client = createClient(this.webdavConfig.address, { headers: headers, }); @@ -271,12 +269,12 @@ export class WrappedWebdavClient { } else { const res = await this.client.exists(`/${this.remoteBaseDir}/`); if (res) { - // log.info("remote vault folder exits!"); + // console.info("remote vault folder exits!"); this.vaultFolderExists = true; } else { - log.info("remote vault folder not exists, creating"); + console.info("remote vault folder not exists, creating"); await this.client.createDirectory(`/${this.remoteBaseDir}/`); - log.info("remote vault folder created!"); + console.info("remote vault folder created!"); this.vaultFolderExists = true; } } @@ -292,7 +290,7 @@ export class WrappedWebdavClient { this.webdavConfig.manualRecursive = true; if (this.saveUpdatedConfigFunc !== undefined) { await this.saveUpdatedConfigFunc(); - log.info( + console.info( `webdav depth="auto_???" is changed to ${this.webdavConfig.depth}` ); } @@ -323,11 +321,11 @@ export const getRemoteMeta = async ( remotePath: string ) => { await client.init(); - log.debug(`getRemoteMeta remotePath = ${remotePath}`); + console.debug(`getRemoteMeta remotePath = ${remotePath}`); const res = (await client.client.stat(remotePath, { details: false, })) as FileStat; - log.debug(`getRemoteMeta res=${JSON.stringify(res)}`); + console.debug(`getRemoteMeta res=${JSON.stringify(res)}`); return fromWebdavItemToEntity(res, client.remoteBaseDir); }; @@ -376,7 +374,7 @@ export const uploadToRemote = async ( await client.client.putFileContents(uploadFile, "", { overwrite: true, onUploadProgress: (progress: any) => { - // log.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`); + // console.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`); }, }); @@ -417,7 +415,7 @@ export const uploadToRemote = async ( await client.client.putFileContents(uploadFile, remoteContent, { overwrite: true, onUploadProgress: (progress: any) => { - log.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`); + console.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`); }, }); @@ -449,7 +447,7 @@ export const listAllFromRemote = async (client: WrappedWebdavClient) => { itemsToFetch.push(q.pop()!); } const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE); - // log.debug(itemsToFetchChunks); + // console.debug(itemsToFetchChunks); const subContents = [] as FileStat[]; for (const singleChunk of itemsToFetchChunks) { const r = singleChunk.map((x) => { @@ -495,7 +493,7 @@ const downloadFromRemoteRaw = async ( remotePath: string ) => { await client.init(); - // log.info(`getWebdavPath=${remotePath}`); + // console.info(`getWebdavPath=${remotePath}`); const buff = (await client.client.getFileContents(remotePath)) as BufferLike; if (buff instanceof ArrayBuffer) { return buff; @@ -535,7 +533,7 @@ export const downloadFromRemote = async ( downloadFile = remoteEncryptedKey; } downloadFile = getWebdavPath(downloadFile, client.remoteBaseDir); - // log.info(`downloadFile=${downloadFile}`); + // console.info(`downloadFile=${downloadFile}`); const remoteContent = await downloadFromRemoteRaw(client, downloadFile); let localContent = remoteContent; if (password !== "") { @@ -568,10 +566,10 @@ export const deleteFromRemote = async ( await client.init(); try { await client.client.deleteFile(remoteFileName); - // log.info(`delete ${remoteFileName} succeeded`); + // console.info(`delete ${remoteFileName} succeeded`); } catch (err) { - log.error("some error while deleting"); - log.error(err); + console.error("some error while deleting"); + console.error(err); } }; @@ -586,7 +584,7 @@ export const checkConnectivity = async ( ) ) { const err = "Error: the url should start with http(s):// but it does not!"; - log.error(err); + console.error(err); if (callbackFunc !== undefined) { callbackFunc(err); } @@ -597,7 +595,7 @@ export const checkConnectivity = async ( const results = await getRemoteMeta(client, `/${client.remoteBaseDir}/`); if (results === undefined) { const err = "results is undefined"; - log.error(err); + console.error(err); if (callbackFunc !== undefined) { callbackFunc(err); } @@ -605,7 +603,7 @@ export const checkConnectivity = async ( } return true; } catch (err) { - log.error(err); + console.error(err); if (callbackFunc !== undefined) { callbackFunc(err); } diff --git a/src/settings.ts b/src/settings.ts index cd7c13e..b61cc67 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -44,13 +44,7 @@ import { } from "./remoteForOnedrive"; import { messyConfigToNormal } from "./configPersist"; import type { TransItemType } from "./i18n"; -import { checkHasSpecialCharForDir } from "./misc"; - -import { - applyLogWriterInplace, - log, - restoreLogWritterInplace, -} from "./moreOnLog"; +import { checkHasSpecialCharForDir, stringToFragment } from "./misc"; import { simpleTransRemotePrefix } from "./remoteForS3"; class PasswordModal extends Modal { @@ -452,7 +446,7 @@ class DropboxAuthModal extends Modal { ); this.close(); } catch (err) { - log.error(err); + console.error(err); new Notice(t("modal_dropboxauth_maualinput_conn_fail")); } }); @@ -588,7 +582,7 @@ export class OnedriveRevokeAuthModal extends Modal { new Notice(t("modal_onedriverevokeauth_clean_notice")); this.close(); } catch (err) { - log.error(err); + console.error(err); new Notice(t("modal_onedriverevokeauth_clean_fail")); } }); @@ -715,65 +709,6 @@ class ExportSettingsQrCodeModal extends Modal { } } -class SetLogToHttpServerModal extends Modal { - plugin: RemotelySavePlugin; - serverAddr: string; - callBack: any; - constructor( - app: App, - plugin: RemotelySavePlugin, - serverAddr: string, - callBack: any - ) { - super(app); - this.plugin = plugin; - this.serverAddr = serverAddr; - this.callBack = callBack; - } - - onOpen() { - let { contentEl } = this; - - const t = (x: TransItemType, vars?: any) => { - return this.plugin.i18n.t(x, vars); - }; - - contentEl.createEl("h2", { text: t("modal_logtohttpserver_title") }); - - const div1 = contentEl.createDiv(); - div1.addClass("logtohttpserver-warning"); - t("modal_logtohttpserver_desc") - .split("\n") - .forEach((val) => { - div1.createEl("p", { - text: val, - }); - }); - - new Setting(contentEl) - .addButton((button) => { - button.setButtonText(t("modal_logtohttpserver_secondconfirm")); - button.setClass("logtohttpserver-warning"); - button.onClick(async () => { - this.callBack(); - new Notice(t("modal_logtohttpserver_notice")); - this.close(); - }); - }) - .addButton((button) => { - button.setButtonText(t("goback")); - button.onClick(() => { - this.close(); - }); - }); - } - - onClose() { - let { contentEl } = this; - contentEl.empty(); - } -} - const getEyesElements = () => { const eyeEl = createElement(Eye); const eyeOffEl = createElement(EyeOff); @@ -1153,7 +1088,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab { ); new Notice(t("settings_dropbox_revoke_notice")); } catch (err) { - log.error(err); + console.error(err); new Notice(t("settings_dropbox_revoke_noticeerr")); } }); @@ -1723,7 +1658,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab { realVal > 0 ) { const intervalID = window.setInterval(() => { - log.info("auto run from settings.ts"); + console.info("auto run from settings.ts"); this.plugin.syncRun("auto"); }, realVal); this.plugin.autoRunIntervalID = intervalID; @@ -1795,7 +1730,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab { // then schedule a run for syncOnSaveAfterMilliseconds after it was modified const lastModified = currentFile.stat.mtime; const currentTime = Date.now(); - // log.debug( + // console.debug( // `Checking if file was modified within last ${ // this.plugin.settings.syncOnSaveAfterMilliseconds / 1000 // } seconds, last modified: ${ @@ -1810,7 +1745,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab { const scheduleTimeFromNow = this.plugin.settings.syncOnSaveAfterMilliseconds! - (currentTime - lastModified); - log.info( + console.info( `schedule a run for ${scheduleTimeFromNow} milliseconds later` ); runScheduled = true; @@ -2074,9 +2009,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab { .setValue(this.plugin.settings.currLogLevel ?? "info") .onChange(async (val: string) => { this.plugin.settings.currLogLevel = val; - log.setLevel(val as any); await this.plugin.saveSettings(); - log.info(`the log level is changed to ${val}`); + console.info(`the log level is changed to ${val}`); }); }); @@ -2087,11 +2021,15 @@ export class RemotelySaveSettingTab extends PluginSettingTab { button.setButtonText(t("settings_outputsettingsconsole_button")); button.onClick(async () => { const c = messyConfigToNormal(await this.plugin.loadData()); - log.info(c); + console.info(c); new Notice(t("settings_outputsettingsconsole_notice")); }); }); + new Setting(debugDiv) + .setName(t("settings_viewconsolelog")) + .setDesc(stringToFragment(t("settings_viewconsolelog_desc"))); + new Setting(debugDiv) .setName(t("settings_syncplans")) .setDesc(t("settings_syncplans_desc")) @@ -2118,53 +2056,6 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); - let logToHttpServer = this.plugin.debugServerTemp || ""; - new Setting(debugDiv) - .setName(t("settings_logtohttpserver")) - .setDesc(t("settings_logtohttpserver_desc")) - .addText(async (text) => { - text.setValue(logToHttpServer).onChange(async (value) => { - logToHttpServer = value.trim(); - }); - }) - .addButton(async (button) => { - button.setButtonText(t("confirm")); - button.onClick(async () => { - if (logToHttpServer === "" || !logToHttpServer.startsWith("http")) { - this.plugin.debugServerTemp = ""; - logToHttpServer = ""; - // restoreLogWritterInplace(); - new Notice(t("settings_logtohttpserver_reset_notice")); - } else { - new SetLogToHttpServerModal( - this.app, - this.plugin, - logToHttpServer, - () => { - this.plugin.debugServerTemp = logToHttpServer; - // applyLogWriterInplace((...msg: any[]) => { - // try { - // requestUrl({ - // url: logToHttpServer, - // method: "POST", - // headers: { - // "Content-Type": "application/json", - // }, - // body: JSON.stringify({ - // send_time: Date.now(), - // log_text: msg, - // }), - // }); - // } catch (e) { - // // pass - // } - // }); - } - ).open(); - } - }); - }); - new Setting(debugDiv) .setName(t("settings_delprevsync")) .setDesc(t("settings_delprevsync_desc")) diff --git a/src/sync.ts b/src/sync.ts index f09fccc..b1d6980 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -31,7 +31,6 @@ import { import { RemoteClient } from "./remote"; import { Vault } from "obsidian"; -import { log } from "./moreOnLog"; import AggregateError from "aggregate-error"; import { InternalDBs, @@ -302,7 +301,7 @@ const encryptLocalEntityInplace = async ( password: string, remoteKeyEnc: string | undefined ) => { - // log.debug( + // console.debug( // `encryptLocalEntityInplace: local=${JSON.stringify( // local, // null, @@ -500,14 +499,14 @@ export const getSyncPlanInplace = async ( const mixedEntry = mixedEntityMappings[key]; const { local, prevSync, remote } = mixedEntry; - // log.debug(`getSyncPlanInplace: key=${key}`) + // console.debug(`getSyncPlanInplace: key=${key}`) if (key.endsWith("/")) { // folder // folder doesn't worry about mtime and size, only check their existences if (keptFolder.has(key)) { // parent should also be kept - // log.debug(`${key} in keptFolder`) + // console.debug(`${key} in keptFolder`) keptFolder.add(getParentFolder(key)); // should fill the missing part if (local !== undefined && remote !== undefined) { @@ -806,9 +805,9 @@ const splitThreeStepsOnEntityMappings = ( val.decision === "folder_existed_remote" || val.decision === "folder_to_be_created" ) { - // log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); + // console.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); const level = atWhichLevel(key); - // log.debug(`atWhichLevel: ${level}`); + // console.debug(`atWhichLevel: ${level}`); const k = folderCreationOps[level - 1]; if (k === undefined || k === null) { folderCreationOps[level - 1] = [val]; @@ -880,7 +879,7 @@ const dispatchOperationToActualV3 = async ( localDeleteFunc: any, password: string ) => { - // log.debug( + // console.debug( // `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify( // r, // null, @@ -912,7 +911,7 @@ const dispatchOperationToActualV3 = async ( // special treatment for OneDrive: do nothing, skip empty file without encryption // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded. } else { - // log.debug(`before upload in sync, r=${JSON.stringify(r, null, 2)}`); + // console.debug(`before upload in sync, r=${JSON.stringify(r, null, 2)}`); const { entity, mtimeCli } = await client.uploadToRemote( r.key, vault, @@ -986,13 +985,13 @@ export const doActualSync = async ( callbackSyncProcess: any, db: InternalDBs ) => { - log.debug(`concurrency === ${concurrency}`); + console.debug(`concurrency === ${concurrency}`); const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } = splitThreeStepsOnEntityMappings(mixedEntityMappings); - // log.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`); - // log.debug(`deletionOps: ${JSON.stringify(deletionOps)}`); - // log.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`); - // log.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`); + // console.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`); + // console.debug(`deletionOps: ${JSON.stringify(deletionOps)}`); + // console.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`); + // console.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`); const nested = [folderCreationOps, deletionOps, uploadDownloads]; const logTexts = [ @@ -1003,14 +1002,16 @@ export const doActualSync = async ( let realCounter = 0; for (let i = 0; i < nested.length; ++i) { - log.debug(logTexts[i]); + console.debug(logTexts[i]); const operations = nested[i]; - // log.debug(`curr operations=${JSON.stringify(operations, null, 2)}`); + // console.debug(`curr operations=${JSON.stringify(operations, null, 2)}`); for (let j = 0; j < operations.length; ++j) { const singleLevelOps = operations[j]; - log.debug(`singleLevelOps=${JSON.stringify(singleLevelOps, null, 2)}`); + console.debug( + `singleLevelOps=${JSON.stringify(singleLevelOps, null, 2)}` + ); if (singleLevelOps === undefined || singleLevelOps === null) { continue; } @@ -1024,7 +1025,9 @@ export const doActualSync = async ( const key = val.key; const fn = async () => { - log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`); + console.debug( + `start syncing "${key}" with plan ${JSON.stringify(val)}` + ); if (callbackSyncProcess !== undefined) { await callbackSyncProcess( @@ -1048,7 +1051,7 @@ export const doActualSync = async ( password ); - log.debug(`finished ${key}`); + console.debug(`finished ${key}`); }; queue.add(fn).catch((e) => { diff --git a/src/syncAlgoV3Notice.ts b/src/syncAlgoV3Notice.ts index a64d9d6..a12ec10 100644 --- a/src/syncAlgoV3Notice.ts +++ b/src/syncAlgoV3Notice.ts @@ -2,7 +2,6 @@ import { App, Modal, Notice, PluginSettingTab, Setting } from "obsidian"; import type RemotelySavePlugin from "./main"; // unavoidable import type { TransItemType } from "./i18n"; -import { log } from "./moreOnLog"; import { stringToFragment } from "./misc"; export class SyncAlgoV3Modal extends Modal { @@ -88,13 +87,13 @@ export class SyncAlgoV3Modal extends Modal { let { contentEl } = this; contentEl.empty(); if (this.agree) { - log.info("agree to use the new algorithm"); + console.info("agree to use the new algorithm"); this.plugin.saveAgreeToUseNewSyncAlgorithm(); this.plugin.enableAutoSyncIfSet(); this.plugin.enableInitSyncIfSet(); this.plugin.enableSyncOnSaveIfSet(); } else { - log.info("do not agree to use the new algorithm"); + console.info("do not agree to use the new algorithm"); this.plugin.unload(); } } From c828a341ba88b54ec7b4df28858f18534ec5d435 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 17 Mar 2024 16:35:02 +0800 Subject: [PATCH 18/21] add profileID --- src/localdb.ts | 44 ++++++++++++++++++++++++++++--------------- src/main.ts | 33 ++++++++++++++++++++++++++------ src/sync.ts | 51 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 98 insertions(+), 30 deletions(-) diff --git a/src/localdb.ts b/src/localdb.ts index 385b5fe..a874096 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -113,7 +113,8 @@ const fromSyncMappingsToPrevSyncRecords = ( */ const migrateDBsFrom20220326To20240220 = async ( db: InternalDBs, - vaultRandomID: string + vaultRandomID: string, + profileID: string ) => { const oldVer = 20220326; const newVer = 20240220; @@ -123,7 +124,12 @@ const migrateDBsFrom20220326To20240220 = async ( const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID); const prevSyncRecords = fromSyncMappingsToPrevSyncRecords(syncMappings); for (const prevSyncRecord of prevSyncRecords) { - await upsertPrevSyncRecordByVault(db, vaultRandomID, prevSyncRecord); + await upsertPrevSyncRecordByVaultAndProfile( + db, + vaultRandomID, + profileID, + prevSyncRecord + ); } // // clear not used data @@ -140,7 +146,8 @@ const migrateDBs = async ( db: InternalDBs, oldVer: number, newVer: number, - vaultRandomID: string + vaultRandomID: string, + profileID: string ) => { if (oldVer === newVer) { return; @@ -155,7 +162,7 @@ const migrateDBs = async ( } if (oldVer === 20220326 && newVer === 20240220) { - return await migrateDBsFrom20220326To20240220(db, vaultRandomID); + return await migrateDBsFrom20220326To20240220(db, vaultRandomID, profileID); } if (newVer < oldVer) { @@ -169,7 +176,8 @@ const migrateDBs = async ( export const prepareDBs = async ( vaultBasePath: string, - vaultRandomIDFromOldConfigFile: string + vaultRandomIDFromOldConfigFile: string, + profileID: string ) => { const db = { versionTbl: localforage.createInstance({ @@ -259,7 +267,8 @@ export const prepareDBs = async ( db, originalVersion, DEFAULT_DB_VERSION_NUMBER, - vaultRandomID + vaultRandomID, + profileID ); } @@ -414,16 +423,17 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => { await Promise.all(ps); }; -export const getAllPrevSyncRecordsByVault = async ( +export const getAllPrevSyncRecordsByVaultAndProfile = async ( db: InternalDBs, - vaultRandomID: string + vaultRandomID: string, + profileID: string ) => { - // console.debug('inside getAllPrevSyncRecordsByVault') + // console.debug('inside getAllPrevSyncRecordsByVaultAndProfile') const keys = await db.prevSyncRecordsTbl.keys(); - // console.debug(`inside getAllPrevSyncRecordsByVault, keys=${keys}`) + // console.debug(`inside getAllPrevSyncRecordsByVaultAndProfile, keys=${keys}`) const res: Entity[] = []; for (const key of keys) { - if (key.startsWith(`${vaultRandomID}\t`)) { + if (key.startsWith(`${vaultRandomID}\t${profileID}\t`)) { const val: Entity | null = await db.prevSyncRecordsTbl.getItem(key); if (val !== null) { res.push(val); @@ -433,23 +443,27 @@ export const getAllPrevSyncRecordsByVault = async ( return res; }; -export const upsertPrevSyncRecordByVault = async ( +export const upsertPrevSyncRecordByVaultAndProfile = async ( db: InternalDBs, vaultRandomID: string, + profileID: string, prevSync: Entity ) => { await db.prevSyncRecordsTbl.setItem( - `${vaultRandomID}\t${prevSync.key}`, + `${vaultRandomID}\t${profileID}\t${prevSync.key}`, prevSync ); }; -export const clearPrevSyncRecordByVault = async ( +export const clearPrevSyncRecordByVaultAndProfile = async ( db: InternalDBs, vaultRandomID: string, + profileID: string, key: string ) => { - await db.prevSyncRecordsTbl.removeItem(`${vaultRandomID}\t${key}`); + await db.prevSyncRecordsTbl.removeItem( + `${vaultRandomID}\t${profileID}\t${key}` + ); }; export const clearAllPrevSyncRecordByVault = async ( diff --git a/src/main.ts b/src/main.ts index 1d1143d..46938ae 100644 --- a/src/main.ts +++ b/src/main.ts @@ -33,7 +33,7 @@ import { clearAllLoggerOutputRecords, upsertLastSuccessSyncTimeByVault, getLastSuccessSyncTimeByVault, - getAllPrevSyncRecordsByVault, + getAllPrevSyncRecordsByVaultAndProfile, } from "./localdb"; import { RemoteClient } from "./remote"; import { @@ -149,6 +149,8 @@ export default class RemotelySavePlugin extends Plugin { return this.i18n.t(x, vars); }; + const profileID = this.getCurrProfileID(); + const getNotice = (x: string, timeout?: number) => { // only show notices in manual mode // no notice in auto mode @@ -278,9 +280,10 @@ export default class RemotelySavePlugin extends Plugin { getNotice(t("syncrun_step5")); } this.syncStatus = "getting_local_prev_sync"; - const prevSyncEntityList = await getAllPrevSyncRecordsByVault( + const prevSyncEntityList = await getAllPrevSyncRecordsByVaultAndProfile( this.db, - this.vaultRandomID + this.vaultRandomID, + profileID ); console.debug("prevSyncEntityList:"); console.debug(prevSyncEntityList); @@ -330,6 +333,7 @@ export default class RemotelySavePlugin extends Plugin { mixedEntityMappings, client, this.vaultRandomID, + profileID, this.app.vault, this.settings.password, this.settings.concurrency ?? 5, @@ -433,6 +437,9 @@ export default class RemotelySavePlugin extends Plugin { await this.loadSettings(); + // MUST after loadSettings and before prepareDB + const profileID: string = this.getCurrProfileID(); + // lang should be load early, but after settings this.i18n = new I18n(this.settings.lang!, async (lang: LangTypeAndAuto) => { this.settings.lang = lang; @@ -458,7 +465,8 @@ export default class RemotelySavePlugin extends Plugin { try { await this.prepareDBAndVaultRandomID( vaultBasePath, - vaultRandomIDFromOldConfigFile + vaultRandomIDFromOldConfigFile, + profileID ); } catch (err: any) { new Notice( @@ -866,6 +874,17 @@ export default class RemotelySavePlugin extends Plugin { await this.saveData(normalConfigToMessy(this.settings)); } + /** + * After 202403 the data should be of profile based. + */ + getCurrProfileID() { + if (this.settings.serviceType !== undefined) { + return `${this.settings.serviceType}-default-1`; + } else { + throw Error("unknown serviceType in the setting!"); + } + } + async checkIfOauthExpires() { let needSave: boolean = false; const current = Date.now(); @@ -975,11 +994,13 @@ export default class RemotelySavePlugin extends Plugin { async prepareDBAndVaultRandomID( vaultBasePath: string, - vaultRandomIDFromOldConfigFile: string + vaultRandomIDFromOldConfigFile: string, + profileID: string ) { const { db, vaultRandomID } = await prepareDBs( vaultBasePath, - vaultRandomIDFromOldConfigFile + vaultRandomIDFromOldConfigFile, + profileID ); this.db = db; this.vaultRandomID = vaultRandomID; diff --git a/src/sync.ts b/src/sync.ts index b1d6980..832daa9 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -34,8 +34,8 @@ import { Vault } from "obsidian"; import AggregateError from "aggregate-error"; import { InternalDBs, - clearPrevSyncRecordByVault, - upsertPrevSyncRecordByVault, + clearPrevSyncRecordByVaultAndProfile, + upsertPrevSyncRecordByVaultAndProfile, } from "./localdb"; export type SyncStatusType = @@ -872,6 +872,7 @@ const splitThreeStepsOnEntityMappings = ( const dispatchOperationToActualV3 = async ( key: string, vaultRandomID: string, + profileID: string, r: MixedEntity, client: RemoteClient, db: InternalDBs, @@ -887,7 +888,7 @@ const dispatchOperationToActualV3 = async ( // )}` // ); if (r.decision === "only_history") { - clearPrevSyncRecordByVault(db, vaultRandomID, key); + clearPrevSyncRecordByVaultAndProfile(db, vaultRandomID, profileID, key); } else if ( r.decision === "equal" || r.decision === "folder_to_skip" || @@ -921,7 +922,12 @@ const dispatchOperationToActualV3 = async ( ); await decryptRemoteEntityInplace(entity, password); await fullfillMTimeOfRemoteEntityInplace(entity, mtimeCli); - await upsertPrevSyncRecordByVault(db, vaultRandomID, entity); + await upsertPrevSyncRecordByVaultAndProfile( + db, + vaultRandomID, + profileID, + entity + ); } } else if ( r.decision === "modified_remote" || @@ -938,15 +944,30 @@ const dispatchOperationToActualV3 = async ( password, r.remote!.keyEnc ); - await upsertPrevSyncRecordByVault(db, vaultRandomID, r.remote!); + await upsertPrevSyncRecordByVaultAndProfile( + db, + vaultRandomID, + profileID, + r.remote! + ); } else if (r.decision === "deleted_local") { // local is deleted, we need to delete remote now await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); - await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + await clearPrevSyncRecordByVaultAndProfile( + db, + vaultRandomID, + profileID, + r.key + ); } else if (r.decision === "deleted_remote") { // remote is deleted, we need to delete local now await localDeleteFunc(r.key); - await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + await clearPrevSyncRecordByVaultAndProfile( + db, + vaultRandomID, + profileID, + r.key + ); } else if ( r.decision === "conflict_created_keep_both" || r.decision === "conflict_modified_keep_both" @@ -964,11 +985,21 @@ const dispatchOperationToActualV3 = async ( // we need to decrypt the key!!! await decryptRemoteEntityInplace(entity, password); await fullfillMTimeOfRemoteEntityInplace(entity, mtimeCli); - await upsertPrevSyncRecordByVault(db, vaultRandomID, entity); + await upsertPrevSyncRecordByVaultAndProfile( + db, + vaultRandomID, + profileID, + entity + ); } else if (r.decision === "folder_to_be_deleted") { await localDeleteFunc(r.key); await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); - await clearPrevSyncRecordByVault(db, vaultRandomID, r.key); + await clearPrevSyncRecordByVaultAndProfile( + db, + vaultRandomID, + profileID, + r.key + ); } else { throw Error(`don't know how to dispatch decision: ${JSON.stringify(r)}`); } @@ -978,6 +1009,7 @@ export const doActualSync = async ( mixedEntityMappings: Record, client: RemoteClient, vaultRandomID: string, + profileID: string, vault: Vault, password: string, concurrency: number, @@ -1043,6 +1075,7 @@ export const doActualSync = async ( await dispatchOperationToActualV3( key, vaultRandomID, + profileID, val, client, db, From 7ec0102c7a931d233a625377cb7b0cfbf348f70e Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 17 Mar 2024 18:01:59 +0800 Subject: [PATCH 19/21] add protection! --- src/baseTypes.ts | 2 ++ src/langs/en.json | 6 +++++ src/langs/zh_cn.json | 6 +++++ src/langs/zh_tw.json | 6 +++++ src/main.ts | 22 +++++++++++++++++ src/settings.ts | 23 ++++++++++++++++++ src/sync.ts | 58 +++++++++++++++++++++++++++++++++++++++++--- 7 files changed, 119 insertions(+), 4 deletions(-) diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 38024f4..0ffe7b9 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -107,6 +107,8 @@ export interface RemotelySavePluginSettings { conflictAction?: ConflictActionType; howToCleanEmptyFolder?: EmptyFolderCleanType; + protectModifyPercentage?: number; + /** * @deprecated */ diff --git a/src/langs/en.json b/src/langs/en.json index 1e59e42..8a18a8b 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -23,6 +23,7 @@ "syncrun_shortstep2skip": "2/2 Remotely Save real sync is skipped in dry run mode.", "syncrun_shortstep2": "2/2 Remotely Save finish!", "syncrun_abort": "{{manifestID}}-{{theDate}}: abort sync, triggerSource={{triggerSource}}, error while {{syncStatus}}", + "syncrun_abort_protectmodifypercentage": "Abort! you set changing files >= {{protectModifyPercentage}}% is not allowed but {{realModifyDeleteCount}}/{{allFilesCount}}={{percent}}% is going to be modified or deleted! If you are sure you want this sync, please adjust the allowed ratio in the settings.", "protocol_saveqr": "New not-oauth2 settings for {{manifestName}} saved. Reopen the plugin Settings to the effect.", "protocol_callbacknotsupported": "Your uri call a callback that's not supported yet: {{params}}", "protocol_dropbox_connecting": "Connecting to Dropbox...\nPlease DO NOT close this modal.", @@ -253,6 +254,11 @@ "settings_cleanemptyfolder_desc": "The sync algorithm majorly deals with files, so you need to specify how to deal with empty folders.", "settings_cleanemptyfolder_skip": "leave them as is (default)", "settings_cleanemptyfolder_clean_both": "delete local and remote", + "settings_protectmodifypercentage": "Abort Sync If Modification Above Percentage", + "settings_protectmodifypercentage_desc": "Abort the sync if more than n% of the files are going to be deleted / modified. Useful to protect users' files from unexpected modifications. You can set to 100 to disable the protection, or set to 0 to always block the sync.", + "settings_protectmodifypercentage_000_desc": "0 (always block)", + "settings_protectmodifypercentage_050_desc": "50 (default)", + "settings_protectmodifypercentage_100_desc": "100 (disable the protection)", "settings_importexport": "Import and Export Partial Settings", "settings_export": "Export", "settings_export_desc": "Export not-oauth2 settings by generating a qrcode.", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index c57bebb..62b1f8d 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -23,6 +23,7 @@ "syncrun_shortstep2skip": "2/2 Remotely Save 在空跑模式，跳过实际数据交换步骤。", "syncrun_shortstep2": "2/2 Remotely Save 已完成同步！", "syncrun_abort": "{{manifestID}}-{{theDate}}：中断同步，同步来源={{triggerSource}}，出错阶段={{syncStatus}}", + "syncrun_abort_protectmodifypercentage": "中断同步！您设置了不允许 >= {{protectModifyPercentage}}% 的变更，但是现在 {{realModifyDeleteCount}}/{{allFilesCount}}={{percent}}% 的文件会被修改或删除！如果您确认这次同步是您想要的，那么请在设置里修改允许比例。", "protocol_saveqr": " {{manifestName}} 新的非 oauth2 设置保存完成。请重启插件设置页使之生效。", "protocol_callbacknotsupported": "您的 uri callback 暂不支持： {{params}}", "protocol_dropbox_connecting": "正在连接 Dropbox……\n请不要关闭此弹窗。", @@ -253,6 +254,11 @@ "settings_cleanemptyfolder_desc": "同步算法主要是针对文件处理的，您要要手动指定空文件夹如何处理。", "settings_cleanemptyfolder_skip": "跳过处理空文件夹（默认）", "settings_cleanemptyfolder_clean_both": "删除本地和服务器的空文件夹", + "settings_protectmodifypercentage": "如果修改超过百分比则中止同步", + "settings_protectmodifypercentage_desc": "如果算法检测到超过 n% 的文件会被修改或删除，则中止同步。从而可以保护用户的文件免受预料之外的修改。您可以设置为 100 而去除此保护，也可以设置为 0 总是强制中止所有同步。", + "settings_protectmodifypercentage_000_desc": "0（总是强制中止）", + "settings_protectmodifypercentage_050_desc": "50（默认值）", + "settings_protectmodifypercentage_100_desc": "100（去除此保护）", "settings_importexport": "导入导出部分设置", "settings_export": "导出", "settings_export_desc": "用 QR 码导出非 oauth2 的设置信息。", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index f94b42e..57e6a72 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -23,6 +23,7 @@ "syncrun_shortstep2skip": "2/2 Remotely Save 在空跑模式，跳過實際資料交換步驟。", "syncrun_shortstep2": "2/2 Remotely Save 已完成同步！", "syncrun_abort": "{{manifestID}}-{{theDate}}：中斷同步，同步來源={{triggerSource}}，出錯階段={{syncStatus}}", + "syncrun_abort_protectmodifypercentage": "中斷同步！您設定了不允許 >= {{protectModifyPercentage}}% 的變更，但是現在 {{realModifyDeleteCount}}/{{allFilesCount}}={{percent}}% 的檔案會被修改或刪除！如果您確認這次同步是您想要的，那麼請在設定裡修改允許比例。", "protocol_saveqr": " {{manifestName}} 新的非 oauth2 設定儲存完成。請重啟外掛設定頁使之生效。", "protocol_callbacknotsupported": "您的 uri callback 暫不支援： {{params}}", "protocol_dropbox_connecting": "正在連線 Dropbox……\n請不要關閉此彈窗。", @@ -253,6 +254,11 @@ "settings_cleanemptyfolder_desc": "同步演算法主要是針對檔案處理的，您需要手動指定空資料夾如何處理。", "settings_cleanemptyfolder_skip": "跳過處理空資料夾（預設）", "settings_cleanemptyfolder_clean_both": "刪除本地和伺服器的空資料夾", + "settings_protectmodifypercentage": "如果修改超過百分比則中止同步", + "settings_protectmodifypercentage_desc": "如果演算法檢測到超過 n% 的檔案會被修改或刪除，則中止同步。從而可以保護使用者的檔案免受預料之外的修改。您可以設定為 100 而去除此保護，也可以設定為 0 總是強制中止所有同步。", + "settings_protectmodifypercentage_000_desc": "0（總是強制中止）", + "settings_protectmodifypercentage_050_desc": "50（預設值）", + "settings_protectmodifypercentage_100_desc": "100（去除此保護）", "settings_importexport": "匯入匯出部分設定", "settings_export": "匯出", "settings_export_desc": "用 QR 碼匯出非 oauth2 的設定資訊。", diff --git a/src/main.ts b/src/main.ts index 46938ae..2d0e5f2 100644 --- a/src/main.ts +++ b/src/main.ts @@ -93,6 +93,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = { agreeToUseSyncV3: false, conflictAction: "keep_newer", howToCleanEmptyFolder: "skip", + protectModifyPercentage: 50, }; interface OAuth2Info { @@ -338,6 +339,24 @@ export default class RemotelySavePlugin extends Plugin { this.settings.password, this.settings.concurrency ?? 5, (key: string) => self.trash(key), + this.settings.protectModifyPercentage ?? 50, + ( + protectModifyPercentage: number, + realModifyDeleteCount: number, + allFilesCount: number + ) => { + const percent = ( + (100 * realModifyDeleteCount) / + allFilesCount + ).toFixed(1); + const res = t("syncrun_abort_protectmodifypercentage", { + protectModifyPercentage, + realModifyDeleteCount, + allFilesCount, + percent, + }); + return res; + }, ( realCounter: number, realTotalCount: number, @@ -866,6 +885,9 @@ export default class RemotelySavePlugin extends Plugin { if (this.settings.howToCleanEmptyFolder === undefined) { this.settings.howToCleanEmptyFolder = "skip"; } + if (this.settings.protectModifyPercentage === undefined) { + this.settings.protectModifyPercentage = 50; + } await this.saveSettings(); } diff --git a/src/settings.ts b/src/settings.ts index b61cc67..4a0062f 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -1968,6 +1968,29 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); + new Setting(advDiv) + .setName(t("settings_protectmodifypercentage")) + .setDesc(t("settings_protectmodifypercentage_desc")) + .addDropdown((dropdown) => { + for (const i of Array.from({ length: 11 }, (x, i) => i * 10)) { + let desc = `${i}`; + if (i === 0) { + desc = t("settings_protectmodifypercentage_000_desc"); + } else if (i === 50) { + desc = t("settings_protectmodifypercentage_050_desc"); + } else if (i === 100) { + desc = t("settings_protectmodifypercentage_100_desc"); + } + dropdown.addOption(`${i}`, desc); + } + dropdown + .setValue(`${this.plugin.settings.protectModifyPercentage ?? 50}`) + .onChange(async (val) => { + this.plugin.settings.protectModifyPercentage = parseInt(val); + await this.plugin.saveSettings(); + }); + }); + ////////////////////////////////////////////////// // below for import and export functions ////////////////////////////////////////////////// diff --git a/src/sync.ts b/src/sync.ts index 832daa9..501eebc 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -788,12 +788,18 @@ const splitThreeStepsOnEntityMappings = ( (k1, k2) => k2.length - k1.length ); - let realTotalCount = 0; + let allFilesCount = 0; // how many files in entities + let realModifyDeleteCount = 0; // how many files to be modified / deleted + let realTotalCount = 0; // how many files to be delt with for (let i = 0; i < sortedKeys.length; ++i) { const key = sortedKeys[i]; const val = mixedEntityMappings[key]; + if (!key.endsWith("/")) { + allFilesCount += 1; + } + if ( val.decision === "equal" || val.decision === "folder_existed_both" || @@ -829,6 +835,10 @@ const splitThreeStepsOnEntityMappings = ( k.push(val); } realTotalCount += 1; + + if (val.decision.startsWith("deleted")) { + realModifyDeleteCount += 1; + } } else if ( val.decision === "modified_local" || val.decision === "modified_remote" || @@ -851,6 +861,13 @@ const splitThreeStepsOnEntityMappings = ( uploadDownloads[0].push(val); // only one level is needed here } realTotalCount += 1; + + if ( + val.decision.startsWith("modified") || + val.decision.startsWith("conflict") + ) { + realModifyDeleteCount += 1; + } } else { throw Error(`unknown decision ${val.decision} for ${key}`); } @@ -865,6 +882,8 @@ const splitThreeStepsOnEntityMappings = ( folderCreationOps: folderCreationOps, deletionOps: deletionOps, uploadDownloads: uploadDownloads, + allFilesCount: allFilesCount, + realModifyDeleteCount: realModifyDeleteCount, realTotalCount: realTotalCount, }; }; @@ -1014,16 +1033,47 @@ export const doActualSync = async ( password: string, concurrency: number, localDeleteFunc: any, + protectModifyPercentage: number, + getProtectModifyPercentageErrorStrFunc: any, callbackSyncProcess: any, db: InternalDBs ) => { console.debug(`concurrency === ${concurrency}`); - const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } = - splitThreeStepsOnEntityMappings(mixedEntityMappings); + const { + folderCreationOps, + deletionOps, + uploadDownloads, + allFilesCount, + realModifyDeleteCount, + realTotalCount, + } = splitThreeStepsOnEntityMappings(mixedEntityMappings); // console.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`); // console.debug(`deletionOps: ${JSON.stringify(deletionOps)}`); // console.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`); - // console.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`); + console.debug(`allFilesCount: ${allFilesCount}`); + console.debug(`realModifyDeleteCount: ${realModifyDeleteCount}`); + console.debug(`realTotalCount: ${realTotalCount}`); + + console.debug(`protectModifyPercentage: ${protectModifyPercentage}`); + + if ( + protectModifyPercentage >= 0 && + realModifyDeleteCount >= 0 && + allFilesCount > 0 + ) { + if ( + realModifyDeleteCount * 100 >= + allFilesCount * protectModifyPercentage + ) { + const errorStr: string = getProtectModifyPercentageErrorStrFunc( + protectModifyPercentage, + realModifyDeleteCount, + allFilesCount + ); + + throw Error(errorStr); + } + } const nested = [folderCreationOps, deletionOps, uploadDownloads]; const logTexts = [ From d51659d3d15060be7127b04d5e1ce8a09d45393f Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 17 Mar 2024 21:05:30 +0800 Subject: [PATCH 20/21] polish welcome screen --- docs/sync_algorithm/v3/intro.md | 2 +- src/langs/en.json | 7 ++++--- src/langs/zh_cn.json | 5 +++-- src/langs/zh_tw.json | 5 +++-- src/syncAlgoV3Notice.ts | 30 +++++++++++++++++++++++++++++- 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/sync_algorithm/v3/intro.md b/docs/sync_algorithm/v3/intro.md index 0c1a16c..1faee0e 100644 --- a/docs/sync_algorithm/v3/intro.md +++ b/docs/sync_algorithm/v3/intro.md @@ -9,4 +9,4 @@ - [x] migration: old data auto transfer to new db (hopefully) - [ ] partial sync: force push - [ ] partial sync: force pull -- [ ] sync protection: warning based on the threshold +- [x] sync protection: warning based on the threshold diff --git a/src/langs/en.json b/src/langs/en.json index 8a18a8b..6d56df5 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -293,9 +293,10 @@ "settings_resetcache_desc": "Reset local internal caches/databases (for debugging purposes). You would want to reload the plugin after resetting this. This option will not empty the {s3, password...} settings.", "settings_resetcache_button": "Reset", "settings_resetcache_notice": "Local internal cache/databases deleted. Please manually reload the plugin.", - "syncalgov3_title": "Remotely Save has HUGE update on sync algorithm", - "syncalgov3_texts": "Welcome to use Remotely Save!\nFrom this version, a new algorithm has been developed: More robust deletion sync, basic conflict handling, no more meta data uploaded... Stay tune for more! A full introduction is in the doc website.\nIf you agree to use thew new version and algorithm, plase check \"I WILL BACKUP MY VAULT MANUALLY FIRSTLY.\" then click the \"Agree\" button, and enjoy the plugin!\nIf you do not agree, please click the \"Do Not Agree\" button, the plugin will unload itself, and you need to manually disable it in Obsidian settings.", - "syncalgov3_checkbox_manual_backup": "I WILL BACKUP MY VAULT MANUALLY FIRSTLY.", + "syncalgov3_title": "Remotely Save has a HUGE update on the sync algorithm", + "syncalgov3_texts": "Welcome to use Remotely Save!\nFrom this version, a new algorithm has been developed: More robust deletion sync, minimal conflict handling, no meta data uploaded any more, deletion / modification protection... Stay tune for more! A full introduction is in the doc website.\nIf you agree to use this, please read and check two checkboxes then click the \"Agree\" button, and enjoy the plugin!\nIf you do not agree, please click the \"Do Not Agree\" button, the plugin will unload itself.\nAlso, please consider visit the GitHub repo and star ⭐ it! Or even buy me a coffee. Your support is very important to me! Thanks!", + "syncalgov3_checkbox_manual_backup": "I will backup my vault manually firstly.", + "syncalgov3_checkbox_requiremultidevupdate": "I understand I need to update the plugin ACROSS ALL DEVICES to make them work properly.", "syncalgov3_button_agree": "Agree", "syncalgov3_button_disagree": "Do Not Agree" } diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index 62b1f8d..9474084 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -294,8 +294,9 @@ "settings_resetcache_button": "重设", "settings_resetcache_notice": "本地同步缓存和数据库已被删除。请手动重新载入此插件。", "syncalgov3_title": "Remotely Save 的同步算法有重大更新", - "syncalgov3_texts": "欢迎使用 Remotely Save！\n从这个版本开始，插件更新了同步算法：更稳健的删除同步、引入冲突处理、避免上传元数据…… 敬请期待更多更新！详细介绍请参阅 文档网站。\n如果您同意使用新版本和算法，请勾选“我将会手动备份我的库（Vault）”，然后点击“同意”按钮，开始使用插件吧！\n如果您不同意，请点击“不同意”按钮，插件将自动停止运行（unload），然后您需要 Obsidian 设置里手动停用（disable）此插件。", - "syncalgov3_checkbox_manual_backup": "我将会手动备份我的库（Vault）", + "syncalgov3_texts": "欢迎使用 Remotely Save！\n从这个版本开始，插件更新了同步算法：更稳健的删除同步、引入冲突处理、避免上传元数据、修改删除保护…… 敬请期待更多更新！详细介绍请参阅文档网站。\n如果您同意使用新版本，请阅读和勾选两个勾选框，然后点击“同意”按钮，开始使用插件吧！\n如果您不同意，请点击“不同意”按钮，插件将自动停止运行（unload）。\n此外，请考虑访问 GitHub 页面然后点赞 ⭐！您的支持对我十分重要！谢谢！", + "syncalgov3_checkbox_manual_backup": "我将会首先手动备份我的库（Vault）。", + "syncalgov3_checkbox_requiremultidevupdate": "我理解，我需要在所有设备上都更新此插件使之正常运行。", "syncalgov3_button_agree": "同意", "syncalgov3_button_disagree": "不同意" } diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index 57e6a72..003370a 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -294,8 +294,9 @@ "settings_resetcache_button": "重設", "settings_resetcache_notice": "本地同步快取和資料庫已被刪除。請手動重新載入此外掛。", "syncalgov3_title": "Remotely Save 的同步演算法有重大更新", - "syncalgov3_texts": "歡迎使用 Remotely Save！\n從這個版本開始，外掛更新了同步演算法：更穩健的刪除同步、引入衝突處理、避免上傳元資料…… 敬請期待更多更新！詳細介紹請參閱 文件網站。\n如果您同意使用新版本和演算法，請勾選“我將會手動備份我的庫（Vault）”，然後點選“同意”按鈕，開始使用外掛吧！\n如果您不同意，請點選“不同意”按鈕，外掛將自動停止執行（unload），然後您需要 Obsidian 設定裡手動停用（disable）此外掛。", - "syncalgov3_checkbox_manual_backup": "我將會手動備份我的庫（Vault）", + "syncalgov3_texts": "歡迎使用 Remotely Save！\n從這個版本開始，外掛更新了同步演算法：更穩健的刪除同步、引入衝突處理、避免上傳元資料、修改刪除保護…… 敬請期待更多更新！詳細介紹請參閱文件網站。\n如果您同意使用新版本，請閱讀和勾選兩個勾選框，然後點選“同意”按鈕，開始使用外掛吧！\n如果您不同意，請點選“不同意”按鈕，外掛將自動停止執行（unload）。\n此外，請考慮訪問 GitHub 頁面然後點贊 ⭐！您的支援對我十分重要！謝謝！", + "syncalgov3_checkbox_manual_backup": "我將會首先手動備份我的庫（Vault）。", + "syncalgov3_checkbox_requiremultidevupdate": "我理解，我需要在所有裝置上都更新此外掛使之正常執行。", "syncalgov3_button_agree": "同意", "syncalgov3_button_disagree": "不同意" } diff --git a/src/syncAlgoV3Notice.ts b/src/syncAlgoV3Notice.ts index a12ec10..22e820b 100644 --- a/src/syncAlgoV3Notice.ts +++ b/src/syncAlgoV3Notice.ts @@ -7,12 +7,14 @@ import { stringToFragment } from "./misc"; export class SyncAlgoV3Modal extends Modal { agree: boolean; manualBackup: boolean; + requireUpdateAllDev: boolean; readonly plugin: RemotelySavePlugin; constructor(app: App, plugin: RemotelySavePlugin) { super(app); this.plugin = plugin; this.agree = false; this.manualBackup = false; + this.requireUpdateAllDev = false; } onOpen() { let { contentEl } = this; @@ -36,6 +38,7 @@ export class SyncAlgoV3Modal extends Modal { // code modified partially from BART released under MIT License contentEl.createDiv("modal-button-container", (buttonContainerEl) => { let agreeBtn: HTMLButtonElement | undefined = undefined; + buttonContainerEl.createEl( "label", { @@ -50,7 +53,7 @@ export class SyncAlgoV3Modal extends Modal { checkboxEl.addEventListener("click", () => { this.manualBackup = checkboxEl.checked; if (agreeBtn !== undefined) { - if (checkboxEl.checked) { + if (this.manualBackup && this.requireUpdateAllDev) { agreeBtn.removeAttribute("disabled"); } else { agreeBtn.setAttr("disabled", true); @@ -61,6 +64,31 @@ export class SyncAlgoV3Modal extends Modal { } ); + buttonContainerEl.createEl( + "label", + { + cls: "mod-checkbox", + }, + (labelEl) => { + const checkboxEl = labelEl.createEl("input", { + attr: { tabindex: -1 }, + type: "checkbox", + }); + checkboxEl.checked = this.requireUpdateAllDev; + checkboxEl.addEventListener("click", () => { + this.requireUpdateAllDev = checkboxEl.checked; + if (agreeBtn !== undefined) { + if (this.manualBackup && this.requireUpdateAllDev) { + agreeBtn.removeAttribute("disabled"); + } else { + agreeBtn.setAttr("disabled", true); + } + } + }); + labelEl.appendText(t("syncalgov3_checkbox_requiremultidevupdate")); + } + ); + agreeBtn = buttonContainerEl.createEl("button", { attr: { type: "button" }, cls: "mod-cta", From 06bf59bf2a6bdfd6629d98a789cd14efdd020a8e Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 17 Mar 2024 23:58:54 +0800 Subject: [PATCH 21/21] incremental push / pull only modes --- docs/sync_algorithm/v3/design.md | 32 +++- docs/sync_algorithm/v3/intro.md | 5 +- src/baseTypes.ts | 37 ++-- src/langs/en.json | 11 +- src/langs/zh_cn.json | 9 +- src/langs/zh_tw.json | 9 +- src/main.ts | 7 +- src/settings.ts | 26 +++ src/sync.ts | 314 ++++++++++++++++++++----------- 9 files changed, 313 insertions(+), 137 deletions(-) diff --git a/docs/sync_algorithm/v3/design.md b/docs/sync_algorithm/v3/design.md index 5ad4e27..8f0a0c0 100644 --- a/docs/sync_algorithm/v3/design.md +++ b/docs/sync_algorithm/v3/design.md @@ -37,15 +37,35 @@ We have _five_ input sources: Init run, consuming remote deletions : -TBD +Change history data into _local previous succeeded sync history_. Later runs, use the first, second, third sources **only**. -Table modified based on synclone and rsinc. The number inside the table cell is the decision branch in the code. +Bidirectional table is modified based on synclone and rsinc. Incremental push / pull only tables is further modified based on the bidirectional table. The number inside the table cell is the decision branch in the code. + +Bidirectional: | local\remote | remote unchanged | remote modified | remote deleted | remote created | | --------------- | ------------------ | ------------------------- | ------------------ | ------------------------- | -| local unchanged | (02/21) do nothing | (09) pull remote | (07) delete local | (??) conflict | -| local modified | (10) push local | (16/17/18/19/20) conflict | (08) push local | (??) conflict | -| local deleted | (04) delete remote | (05) pull | (01) clean history | (03) pull remote | -| local created | (??) conflict | (??) conflict | (06) push local | (11/12/13/14/15) conflict | +| local unchanged | (02/21) do nothing | (09) pull | (07) delete local | (??) conflict | +| local modified | (10) push | (16/17/18/19/20) conflict | (08) push | (??) conflict | +| local deleted | (04) delete remote | (05) pull | (01) clean history | (03) pull | +| local created | (??) conflict | (??) conflict | (06) push | (11/12/13/14/15) conflict | + +Incremental push only: + +| local\remote | remote unchanged | remote modified | remote deleted | remote created | +| --------------- | ---------------------------- | ---------------------------- | ---------------------- | ---------------------------- | +| local unchanged | (02/21) do nothing | **(26) conflict push** | **(32) conflict push** | (??) conflict | +| local modified | (10) push | **(25) conflict push** | (08) push | (??) conflict | +| local deleted | **(29) conflict do nothing** | **(30) conflict do nothing** | (01) clean history | **(28) conflict do nothing** | +| local created | (??) conflict | (??) conflict | (06) push | **(23) conflict push** | + +Incremental pull only: + +| local\remote | remote unchanged | remote modified | remote deleted | remote created | +| --------------- | ---------------------- | ---------------------- | ---------------------------- | ---------------------- | +| local unchanged | (02/21) do nothing | (09) pull | **(33) conflict do nothing** | (??) conflict | +| local modified | **(27) conflict pull** | **(24) conflict pull** | **(34) conflict do nothing** | (??) conflict | +| local deleted | **(35) conflict pull** | (05) pull | (01) clean history | (03) pull | +| local created | (??) conflict | (??) conflict | **(31) conflict do nothing** | **(22) conflict pull** | diff --git a/docs/sync_algorithm/v3/intro.md b/docs/sync_algorithm/v3/intro.md index 1faee0e..4745899 100644 --- a/docs/sync_algorithm/v3/intro.md +++ b/docs/sync_algorithm/v3/intro.md @@ -7,6 +7,7 @@ - [x] deletion: true deletion status computation - [x] meta data: no remote meta data any more - [x] migration: old data auto transfer to new db (hopefully) -- [ ] partial sync: force push -- [ ] partial sync: force pull +- [x] sync direction: incremental push only +- [x] sync direction: incremental pull only - [x] sync protection: warning based on the threshold +- [ ] partial sync: better sync on save diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 0ffe7b9..a2523c8 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -83,6 +83,11 @@ export interface OnedriveConfig { remoteBaseDir?: string; } +export type SyncDirectionType = + | "bidirectional" + | "incremental_pull_only" + | "incremental_push_only"; + export interface RemotelySavePluginSettings { s3: S3Config; webdav: WebdavConfig; @@ -108,6 +113,7 @@ export interface RemotelySavePluginSettings { howToCleanEmptyFolder?: EmptyFolderCleanType; protectModifyPercentage?: number; + syncDirection?: SyncDirectionType; /** * @deprecated @@ -147,21 +153,22 @@ export type ConflictActionType = "keep_newer" | "keep_larger" | "rename_both"; export type DecisionTypeForMixedEntity = | "only_history" | "equal" - | "modified_local" - | "modified_remote" - | "created_local" - | "created_remote" - | "deleted_local" - | "deleted_remote" - | "conflict_created_keep_local" - | "conflict_created_keep_remote" - | "conflict_created_keep_both" - | "conflict_modified_keep_local" - | "conflict_modified_keep_remote" - | "conflict_modified_keep_both" - | "folder_existed_both" - | "folder_existed_local" - | "folder_existed_remote" + | "local_is_modified_then_push" + | "remote_is_modified_then_pull" + | "local_is_created_then_push" + | "remote_is_created_then_pull" + | "local_is_deleted_thus_also_delete_remote" + | "remote_is_deleted_thus_also_delete_local" + | "conflict_created_then_keep_local" + | "conflict_created_then_keep_remote" + | "conflict_created_then_keep_both" + | "conflict_created_then_do_nothing" + | "conflict_modified_then_keep_local" + | "conflict_modified_then_keep_remote" + | "conflict_modified_then_keep_both" + | "folder_existed_both_then_do_nothing" + | "folder_existed_local_then_also_create_remote" + | "folder_existed_remote_then_also_create_local" | "folder_to_be_created" | "folder_to_skip" | "folder_to_be_deleted"; diff --git a/src/langs/en.json b/src/langs/en.json index 6d56df5..143d4b2 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -247,7 +247,7 @@ "settings_deletetowhere_system_trash": "system trash (default)", "settings_deletetowhere_obsidian_trash": "Obsidian .trash folder", "settings_conflictaction": "Action For Conflict", - "settings_conflictaction_desc": "If a file is created or modified on both side since last update, it's a conflict event. How to deal with it?", + "settings_conflictaction_desc": "If a file is created or modified on both side since last update, it's a conflict event. How to deal with it? This only works for bidirectional sync.", "settings_conflictaction_keep_newer": "newer version survives (default)", "settings_conflictaction_keep_larger": "larger size version survives", "settings_cleanemptyfolder": "Action For Empty Folders", @@ -259,6 +259,11 @@ "settings_protectmodifypercentage_000_desc": "0 (always block)", "settings_protectmodifypercentage_050_desc": "50 (default)", "settings_protectmodifypercentage_100_desc": "100 (disable the protection)", + "setting_syncdirection": "Sync Direction", + "setting_syncdirection_desc": "Which direction should the plugin sync to? Please be aware that only CHANGED files (based on time and size) are synced regardless any option.", + "setting_syncdirection_bidirectional_desc": "Bidirectional (default)", + "setting_syncdirection_incremental_push_only_desc": "Incremental Push Only (aka backup mode)", + "setting_syncdirection_incremental_pull_only_desc": "Incremental Pull Only", "settings_importexport": "Import and Export Partial Settings", "settings_export": "Export", "settings_export_desc": "Export not-oauth2 settings by generating a qrcode.", @@ -293,8 +298,8 @@ "settings_resetcache_desc": "Reset local internal caches/databases (for debugging purposes). You would want to reload the plugin after resetting this. This option will not empty the {s3, password...} settings.", "settings_resetcache_button": "Reset", "settings_resetcache_notice": "Local internal cache/databases deleted. Please manually reload the plugin.", - "syncalgov3_title": "Remotely Save has a HUGE update on the sync algorithm", - "syncalgov3_texts": "Welcome to use Remotely Save!\nFrom this version, a new algorithm has been developed: More robust deletion sync, minimal conflict handling, no meta data uploaded any more, deletion / modification protection... Stay tune for more! A full introduction is in the doc website.\nIf you agree to use this, please read and check two checkboxes then click the \"Agree\" button, and enjoy the plugin!\nIf you do not agree, please click the \"Do Not Agree\" button, the plugin will unload itself.\nAlso, please consider visit the GitHub repo and star ⭐ it! Or even buy me a coffee. Your support is very important to me! Thanks!", + "syncalgov3_title": "Remotely Save has HUGE updates on the sync algorithm", + "syncalgov3_texts": "Welcome to use Remotely Save!\nFrom this version, a new algorithm has been developed:\n
  • More robust deletion sync,
  • minimal conflict handling,
  • no meta data uploaded any more,
  • deletion / modification protection,
  • backup mode
  • ...
\nStay tune for more! A full introduction is in the doc website.\nIf you agree to use this, please read and check two checkboxes then click the \"Agree\" button, and enjoy the plugin!\nIf you do not agree, please click the \"Do Not Agree\" button, the plugin will unload itself.\nAlso, please consider visit the GitHub repo and star ⭐ it! Or even buy me a coffee. Your support is very important to me! Thanks!", "syncalgov3_checkbox_manual_backup": "I will backup my vault manually firstly.", "syncalgov3_checkbox_requiremultidevupdate": "I understand I need to update the plugin ACROSS ALL DEVICES to make them work properly.", "syncalgov3_button_agree": "Agree", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index 9474084..09eb696 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -247,7 +247,7 @@ "settings_deletetowhere_system_trash": "系统回收站（默认）", "settings_deletetowhere_obsidian_trash": "Obsidian .trash 文件夹", "settings_conflictaction": "处理冲突", - "settings_conflictaction_desc": "如果一个文件，在本地和服务器都被创建或者修改了，那么这就是一个“冲突”情况。如何处理？", + "settings_conflictaction_desc": "如果一个文件，在本地和服务器都被创建或者修改了，那么这就是一个“冲突”情况。如何处理？这个设置只在双向同步时候生效。", "settings_conflictaction_keep_newer": "保留最后修改的版本（默认）", "settings_conflictaction_keep_larger": "保留文件体积较大的版本", "settings_cleanemptyfolder": "处理空文件夹", @@ -259,6 +259,11 @@ "settings_protectmodifypercentage_000_desc": "0（总是强制中止）", "settings_protectmodifypercentage_050_desc": "50（默认值）", "settings_protectmodifypercentage_100_desc": "100（去除此保护）", + "setting_syncdirection": "同步方向", + "setting_syncdirection_desc": "插件应该向哪里同步？注意每个选项都是只有修改了的文件（基于修改时间和大小判断）才会触发同步动作。", + "setting_syncdirection_bidirectional_desc": "双向同步（默认）", + "setting_syncdirection_incremental_push_only_desc": "只增量推送（也即：备份模式）", + "setting_syncdirection_incremental_pull_only_desc": "只增量拉取", "settings_importexport": "导入导出部分设置", "settings_export": "导出", "settings_export_desc": "用 QR 码导出非 oauth2 的设置信息。", @@ -294,7 +299,7 @@ "settings_resetcache_button": "重设", "settings_resetcache_notice": "本地同步缓存和数据库已被删除。请手动重新载入此插件。", "syncalgov3_title": "Remotely Save 的同步算法有重大更新", - "syncalgov3_texts": "欢迎使用 Remotely Save！\n从这个版本开始，插件更新了同步算法：更稳健的删除同步、引入冲突处理、避免上传元数据、修改删除保护…… 敬请期待更多更新！详细介绍请参阅文档网站。\n如果您同意使用新版本，请阅读和勾选两个勾选框，然后点击“同意”按钮，开始使用插件吧！\n如果您不同意，请点击“不同意”按钮，插件将自动停止运行（unload）。\n此外，请考虑访问 GitHub 页面然后点赞 ⭐！您的支持对我十分重要！谢谢！", + "syncalgov3_texts": "欢迎使用 Remotely Save！\n从这个版本开始，插件更新了同步算法：\n
  • 更稳健的删除同步
  • 引入冲突处理
  • 避免上传元数据
  • 修改删除保护
  • 备份模式
  • ……
\n敬请期待更多更新！详细介绍请参阅文档网站。\n如果您同意使用新版本，请阅读和勾选两个勾选框，然后点击“同意”按钮，开始使用插件吧！\n如果您不同意，请点击“不同意”按钮，插件将自动停止运行（unload）。\n此外，请考虑访问 GitHub 页面然后点赞 ⭐！您的支持对我十分重要！谢谢！", "syncalgov3_checkbox_manual_backup": "我将会首先手动备份我的库（Vault）。", "syncalgov3_checkbox_requiremultidevupdate": "我理解，我需要在所有设备上都更新此插件使之正常运行。", "syncalgov3_button_agree": "同意", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index 003370a..01ba2f0 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -247,7 +247,7 @@ "settings_deletetowhere_system_trash": "系統回收站（預設）", "settings_deletetowhere_obsidian_trash": "Obsidian .trash 資料夾", "settings_conflictaction": "處理衝突", - "settings_conflictaction_desc": "如果一個檔案，在本地和伺服器都被建立或者修改了，那麼這就是一個“衝突”情況。如何處理？", + "settings_conflictaction_desc": "如果一個檔案，在本地和伺服器都被建立或者修改了，那麼這就是一個“衝突”情況。如何處理？這個設定只在雙向同步時候生效。", "settings_conflictaction_keep_newer": "保留最後修改的版本（預設）", "settings_conflictaction_keep_larger": "保留檔案體積較大的版本", "settings_cleanemptyfolder": "處理空資料夾", @@ -259,6 +259,11 @@ "settings_protectmodifypercentage_000_desc": "0（總是強制中止）", "settings_protectmodifypercentage_050_desc": "50（預設值）", "settings_protectmodifypercentage_100_desc": "100（去除此保護）", + "setting_syncdirection": "同步方向", + "setting_syncdirection_desc": "外掛應該向哪裡同步？注意每個選項都是隻有修改了的檔案（基於修改時間和大小判斷）才會觸發同步動作。", + "setting_syncdirection_bidirectional_desc": "雙向同步（預設）", + "setting_syncdirection_incremental_push_only_desc": "只增量推送（也即：備份模式）", + "setting_syncdirection_incremental_pull_only_desc": "只增量拉取", "settings_importexport": "匯入匯出部分設定", "settings_export": "匯出", "settings_export_desc": "用 QR 碼匯出非 oauth2 的設定資訊。", @@ -294,7 +299,7 @@ "settings_resetcache_button": "重設", "settings_resetcache_notice": "本地同步快取和資料庫已被刪除。請手動重新載入此外掛。", "syncalgov3_title": "Remotely Save 的同步演算法有重大更新", - "syncalgov3_texts": "歡迎使用 Remotely Save！\n從這個版本開始，外掛更新了同步演算法：更穩健的刪除同步、引入衝突處理、避免上傳元資料、修改刪除保護…… 敬請期待更多更新！詳細介紹請參閱文件網站。\n如果您同意使用新版本，請閱讀和勾選兩個勾選框，然後點選“同意”按鈕，開始使用外掛吧！\n如果您不同意，請點選“不同意”按鈕，外掛將自動停止執行（unload）。\n此外，請考慮訪問 GitHub 頁面然後點贊 ⭐！您的支援對我十分重要！謝謝！", + "syncalgov3_texts": "歡迎使用 Remotely Save！\n從這個版本開始，外掛更新了同步演算法：\n
  • 更穩健的刪除同步
  • 引入衝突處理
  • 避免上傳元資料
  • 修改刪除保護
  • 備份模式
  • ……
\n敬請期待更多更新！詳細介紹請參閱文件網站。\n如果您同意使用新版本，請閱讀和勾選兩個勾選框，然後點選“同意”按鈕，開始使用外掛吧！\n如果您不同意，請點選“不同意”按鈕，外掛將自動停止執行（unload）。\n此外，請考慮訪問 GitHub 頁面然後點贊 ⭐！您的支援對我十分重要！謝謝！", "syncalgov3_checkbox_manual_backup": "我將會首先手動備份我的庫（Vault）。", "syncalgov3_checkbox_requiremultidevupdate": "我理解，我需要在所有裝置上都更新此外掛使之正常執行。", "syncalgov3_button_agree": "同意", diff --git a/src/main.ts b/src/main.ts index 2d0e5f2..b9d51b3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -94,6 +94,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = { conflictAction: "keep_newer", howToCleanEmptyFolder: "skip", protectModifyPercentage: 50, + syncDirection: "bidirectional", }; interface OAuth2Info { @@ -309,7 +310,8 @@ export default class RemotelySavePlugin extends Plugin { mixedEntityMappings, this.settings.howToCleanEmptyFolder ?? "skip", this.settings.skipSizeLargerThan ?? -1, - this.settings.conflictAction ?? "keep_newer" + this.settings.conflictAction ?? "keep_newer", + this.settings.syncDirection ?? "bidirectional" ); console.info(`mixedEntityMappings:`); console.info(mixedEntityMappings); // for debugging @@ -888,6 +890,9 @@ export default class RemotelySavePlugin extends Plugin { if (this.settings.protectModifyPercentage === undefined) { this.settings.protectModifyPercentage = 50; } + if (this.settings.syncDirection === undefined) { + this.settings.syncDirection = "bidirectional"; + } await this.saveSettings(); } diff --git a/src/settings.ts b/src/settings.ts index 4a0062f..c7f7e55 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -18,6 +18,7 @@ import { EmptyFolderCleanType, SUPPORTED_SERVICES_TYPE, SUPPORTED_SERVICES_TYPE_WITH_REMOTE_BASE_DIR, + SyncDirectionType, VALID_REQURL, WebdavAuthType, WebdavDepthType, @@ -1991,6 +1992,31 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); + new Setting(advDiv) + .setName(t("setting_syncdirection")) + .setDesc(t("setting_syncdirection_desc")) + .addDropdown((dropdown) => { + dropdown.addOption( + "bidirectional", + t("setting_syncdirection_bidirectional_desc") + ); + dropdown.addOption( + "incremental_push_only", + t("setting_syncdirection_incremental_push_only_desc") + ); + dropdown.addOption( + "incremental_pull_only", + t("setting_syncdirection_incremental_pull_only_desc") + ); + + dropdown + .setValue(this.plugin.settings.syncDirection ?? "bidirectional") + .onChange(async (val) => { + this.plugin.settings.syncDirection = val as SyncDirectionType; + await this.plugin.saveSettings(); + }); + }); + ////////////////////////////////////////////////// // below for import and export functions ////////////////////////////////////////////////// diff --git a/src/sync.ts b/src/sync.ts index 501eebc..7eac42a 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -5,6 +5,7 @@ import type { EmptyFolderCleanType, Entity, MixedEntity, + SyncDirectionType, } from "./baseTypes"; import { isInsideObsFolder } from "./obsFolderLister"; import { @@ -479,13 +480,14 @@ export const ensembleMixedEnties = async ( /** * Heavy lifting. * Basically follow the sync algorithm of https://github.com/Jwink3101/syncrclone - * @param mixedEntityMappings + * Also deal with syncDirection which makes it more complicated */ export const getSyncPlanInplace = async ( mixedEntityMappings: Record, howToCleanEmptyFolder: EmptyFolderCleanType, skipSizeLargerThan: number, - conflictAction: ConflictActionType + conflictAction: ConflictActionType, + syncDirection: SyncDirectionType ) => { // from long(deep) to short(shadow) const sortedKeys = Object.keys(mixedEntityMappings).sort( @@ -511,14 +513,27 @@ export const getSyncPlanInplace = async ( // should fill the missing part if (local !== undefined && remote !== undefined) { mixedEntry.decisionBranch = 101; - mixedEntry.decision = "folder_existed_both"; + mixedEntry.decision = "folder_existed_both_then_do_nothing"; } else if (local !== undefined && remote === undefined) { - mixedEntry.decisionBranch = 102; - mixedEntry.decision = "folder_existed_local"; + if (syncDirection === "incremental_pull_only") { + mixedEntry.decisionBranch = 107; + mixedEntry.decision = "folder_to_skip"; + } else { + mixedEntry.decisionBranch = 102; + mixedEntry.decision = + "folder_existed_local_then_also_create_remote"; + } } else if (local === undefined && remote !== undefined) { - mixedEntry.decisionBranch = 103; - mixedEntry.decision = "folder_existed_remote"; + if (syncDirection === "incremental_push_only") { + mixedEntry.decisionBranch = 108; + mixedEntry.decision = "folder_to_skip"; + } else { + mixedEntry.decisionBranch = 103; + mixedEntry.decision = + "folder_existed_remote_then_also_create_local"; + } } else { + // why?? how?? mixedEntry.decisionBranch = 104; mixedEntry.decision = "folder_to_be_created"; } @@ -530,6 +545,7 @@ export const getSyncPlanInplace = async ( } else if (howToCleanEmptyFolder === "clean_both") { mixedEntry.decisionBranch = 106; mixedEntry.decision = "folder_to_be_deleted"; + // TODO: what to do in different sync direction? } else { throw Error( `do not know how to deal with empty folder ${mixedEntry.key}` @@ -571,9 +587,15 @@ export const getSyncPlanInplace = async ( skipSizeLargerThan <= 0 || remote.sizeEnc! <= skipSizeLargerThan ) { - mixedEntry.decisionBranch = 9; - mixedEntry.decision = "modified_remote"; - keptFolder.add(getParentFolder(key)); + if (syncDirection === "incremental_push_only") { + mixedEntry.decisionBranch = 26; + mixedEntry.decision = "conflict_modified_then_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 9; + mixedEntry.decision = "remote_is_modified_then_pull"; + keptFolder.add(getParentFolder(key)); + } } else { throw Error( `remote is modified (branch 9) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( @@ -587,9 +609,15 @@ export const getSyncPlanInplace = async ( skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan ) { - mixedEntry.decisionBranch = 10; - mixedEntry.decision = "modified_local"; - keptFolder.add(getParentFolder(key)); + if (syncDirection === "incremental_pull_only") { + mixedEntry.decisionBranch = 27; + mixedEntry.decision = "conflict_modified_then_keep_remote"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 10; + mixedEntry.decision = "local_is_modified_then_push"; + keptFolder.add(getParentFolder(key)); + } } else { throw Error( `local is modified (branch 10) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( @@ -598,64 +626,94 @@ export const getSyncPlanInplace = async ( ); } } else if (!localEqualPrevSync && !remoteEqualPrevSync) { - // If both compare False, (didn't exist means both are new. Both exist but don't compare means both are modified) + // If both compare False (Didn't exist means both are new. Both exist but don't compare means both are modified) if (prevSync === undefined) { - if (conflictAction === "keep_newer") { - if ( - (local.mtimeCli ?? local.mtimeSvr ?? 0) >= - (remote.mtimeCli ?? remote.mtimeSvr ?? 0) - ) { - mixedEntry.decisionBranch = 11; - mixedEntry.decision = "conflict_created_keep_local"; - keptFolder.add(getParentFolder(key)); + // Didn't exist means both are new + if (syncDirection === "bidirectional") { + if (conflictAction === "keep_newer") { + if ( + (local.mtimeCli ?? local.mtimeSvr ?? 0) >= + (remote.mtimeCli ?? remote.mtimeSvr ?? 0) + ) { + mixedEntry.decisionBranch = 11; + mixedEntry.decision = "conflict_created_then_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 12; + mixedEntry.decision = "conflict_created_then_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else if (conflictAction === "keep_larger") { + if (local.sizeEnc! >= remote.sizeEnc!) { + mixedEntry.decisionBranch = 13; + mixedEntry.decision = "conflict_created_then_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 14; + mixedEntry.decision = "conflict_created_then_keep_remote"; + keptFolder.add(getParentFolder(key)); + } } else { - mixedEntry.decisionBranch = 12; - mixedEntry.decision = "conflict_created_keep_remote"; + mixedEntry.decisionBranch = 15; + mixedEntry.decision = "conflict_created_then_keep_both"; keptFolder.add(getParentFolder(key)); } - } else if (conflictAction === "keep_larger") { - if (local.sizeEnc! >= remote.sizeEnc!) { - mixedEntry.decisionBranch = 13; - mixedEntry.decision = "conflict_created_keep_local"; - keptFolder.add(getParentFolder(key)); - } else { - mixedEntry.decisionBranch = 14; - mixedEntry.decision = "conflict_created_keep_remote"; - keptFolder.add(getParentFolder(key)); - } - } else { - mixedEntry.decisionBranch = 15; - mixedEntry.decision = "conflict_created_keep_both"; + } else if (syncDirection === "incremental_pull_only") { + mixedEntry.decisionBranch = 22; + mixedEntry.decision = "conflict_created_then_keep_remote"; keptFolder.add(getParentFolder(key)); + } else if (syncDirection === "incremental_push_only") { + mixedEntry.decisionBranch = 23; + mixedEntry.decision = "conflict_created_then_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `no idea how to deal with syncDirection=${syncDirection} while conflict created` + ); } } else { - if (conflictAction === "keep_newer") { - if ( - (local.mtimeCli ?? local.mtimeSvr ?? 0) >= - (remote.mtimeCli ?? remote.mtimeSvr ?? 0) - ) { - mixedEntry.decisionBranch = 16; - mixedEntry.decision = "conflict_modified_keep_local"; - keptFolder.add(getParentFolder(key)); + // Both exist but don't compare means both are modified + if (syncDirection === "bidirectional") { + if (conflictAction === "keep_newer") { + if ( + (local.mtimeCli ?? local.mtimeSvr ?? 0) >= + (remote.mtimeCli ?? remote.mtimeSvr ?? 0) + ) { + mixedEntry.decisionBranch = 16; + mixedEntry.decision = "conflict_modified_then_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 17; + mixedEntry.decision = "conflict_modified_then_keep_remote"; + keptFolder.add(getParentFolder(key)); + } + } else if (conflictAction === "keep_larger") { + if (local.sizeEnc! >= remote.sizeEnc!) { + mixedEntry.decisionBranch = 18; + mixedEntry.decision = "conflict_modified_then_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 19; + mixedEntry.decision = "conflict_modified_then_keep_remote"; + keptFolder.add(getParentFolder(key)); + } } else { - mixedEntry.decisionBranch = 17; - mixedEntry.decision = "conflict_modified_keep_remote"; + mixedEntry.decisionBranch = 20; + mixedEntry.decision = "conflict_modified_then_keep_both"; keptFolder.add(getParentFolder(key)); } - } else if (conflictAction === "keep_larger") { - if (local.sizeEnc! >= remote.sizeEnc!) { - mixedEntry.decisionBranch = 18; - mixedEntry.decision = "conflict_modified_keep_local"; - keptFolder.add(getParentFolder(key)); - } else { - mixedEntry.decisionBranch = 19; - mixedEntry.decision = "conflict_modified_keep_remote"; - keptFolder.add(getParentFolder(key)); - } - } else { - mixedEntry.decisionBranch = 20; - mixedEntry.decision = "conflict_modified_keep_both"; + } else if (syncDirection === "incremental_pull_only") { + mixedEntry.decisionBranch = 24; + mixedEntry.decision = "conflict_modified_then_keep_remote"; keptFolder.add(getParentFolder(key)); + } else if (syncDirection === "incremental_push_only") { + mixedEntry.decisionBranch = 25; + mixedEntry.decision = "conflict_modified_then_keep_local"; + keptFolder.add(getParentFolder(key)); + } else { + throw Error( + `no idea how to deal with syncDirection=${syncDirection} while conflict modified` + ); } } } else { @@ -675,9 +733,15 @@ export const getSyncPlanInplace = async ( skipSizeLargerThan <= 0 || remote.sizeEnc! <= skipSizeLargerThan ) { - mixedEntry.decisionBranch = 3; - mixedEntry.decision = "created_remote"; - keptFolder.add(getParentFolder(key)); + if (syncDirection === "incremental_push_only") { + mixedEntry.decisionBranch = 28; + mixedEntry.decision = "conflict_created_then_do_nothing"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 3; + mixedEntry.decision = "remote_is_created_then_pull"; + keptFolder.add(getParentFolder(key)); + } } else { throw Error( `remote is created (branch 3) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( @@ -691,17 +755,33 @@ export const getSyncPlanInplace = async ( prevSync.sizeEnc === remote.sizeEnc ) { // if B is in the previous list and UNMODIFIED, B has been deleted by A - mixedEntry.decisionBranch = 4; - mixedEntry.decision = "deleted_local"; + if (syncDirection === "incremental_push_only") { + mixedEntry.decisionBranch = 29; + mixedEntry.decision = "conflict_created_then_do_nothing"; + keptFolder.add(getParentFolder(key)); + } else if (syncDirection === "incremental_pull_only") { + mixedEntry.decisionBranch = 35; + mixedEntry.decision = "conflict_created_then_keep_remote"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 4; + mixedEntry.decision = "local_is_deleted_thus_also_delete_remote"; + } } else { // if B is in the previous list and MODIFIED, B has been deleted by A but modified by B if ( skipSizeLargerThan <= 0 || remote.sizeEnc! <= skipSizeLargerThan ) { - mixedEntry.decisionBranch = 5; - mixedEntry.decision = "modified_remote"; - keptFolder.add(getParentFolder(key)); + if (syncDirection === "incremental_push_only") { + mixedEntry.decisionBranch = 30; + mixedEntry.decision = "conflict_created_then_do_nothing"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 5; + mixedEntry.decision = "remote_is_modified_then_pull"; + keptFolder.add(getParentFolder(key)); + } } else { throw Error( `remote is modified (branch 5) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( @@ -716,9 +796,15 @@ export const getSyncPlanInplace = async ( if (prevSync === undefined) { // if A is not in the previous list, A is new if (skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan) { - mixedEntry.decisionBranch = 6; - mixedEntry.decision = "created_local"; - keptFolder.add(getParentFolder(key)); + if (syncDirection === "incremental_pull_only") { + mixedEntry.decisionBranch = 31; + mixedEntry.decision = "conflict_created_then_do_nothing"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 6; + mixedEntry.decision = "local_is_created_then_push"; + keptFolder.add(getParentFolder(key)); + } } else { throw Error( `local is created (branch 6) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( @@ -732,14 +818,28 @@ export const getSyncPlanInplace = async ( prevSync.sizeEnc === local.sizeEnc ) { // if A is in the previous list and UNMODIFIED, A has been deleted by B - mixedEntry.decisionBranch = 7; - mixedEntry.decision = "deleted_remote"; + if (syncDirection === "incremental_push_only") { + mixedEntry.decisionBranch = 32; + mixedEntry.decision = "conflict_created_then_keep_local"; + } else if (syncDirection === "incremental_pull_only") { + mixedEntry.decisionBranch = 33; + mixedEntry.decision = "conflict_created_then_do_nothing"; + } else { + mixedEntry.decisionBranch = 7; + mixedEntry.decision = "remote_is_deleted_thus_also_delete_local"; + } } else { // if A is in the previous list and MODIFIED, A has been deleted by B but modified by A if (skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan) { - mixedEntry.decisionBranch = 8; - mixedEntry.decision = "modified_local"; - keptFolder.add(getParentFolder(key)); + if (syncDirection === "incremental_pull_only") { + mixedEntry.decisionBranch = 34; + mixedEntry.decision = "conflict_created_then_do_nothing"; + keptFolder.add(getParentFolder(key)); + } else { + mixedEntry.decisionBranch = 8; + mixedEntry.decision = "local_is_modified_then_push"; + keptFolder.add(getParentFolder(key)); + } } else { throw Error( `local is modified (branch 8) but size larger than ${skipSizeLargerThan}, don't know what to do: ${JSON.stringify( @@ -802,13 +902,14 @@ const splitThreeStepsOnEntityMappings = ( if ( val.decision === "equal" || - val.decision === "folder_existed_both" || + val.decision === "conflict_created_then_do_nothing" || + val.decision === "folder_existed_both_then_do_nothing" || val.decision === "folder_to_skip" ) { // pass } else if ( - val.decision === "folder_existed_local" || - val.decision === "folder_existed_remote" || + val.decision === "folder_existed_local_then_also_create_remote" || + val.decision === "folder_existed_remote_then_also_create_local" || val.decision === "folder_to_be_created" ) { // console.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); @@ -823,8 +924,8 @@ const splitThreeStepsOnEntityMappings = ( realTotalCount += 1; } else if ( val.decision === "only_history" || - val.decision === "deleted_local" || - val.decision === "deleted_remote" || + val.decision === "local_is_deleted_thus_also_delete_remote" || + val.decision === "remote_is_deleted_thus_also_delete_local" || val.decision === "folder_to_be_deleted" ) { const level = atWhichLevel(key); @@ -840,16 +941,16 @@ const splitThreeStepsOnEntityMappings = ( realModifyDeleteCount += 1; } } else if ( - val.decision === "modified_local" || - val.decision === "modified_remote" || - val.decision === "created_local" || - val.decision === "created_remote" || - val.decision === "conflict_created_keep_local" || - val.decision === "conflict_created_keep_remote" || - val.decision === "conflict_created_keep_both" || - val.decision === "conflict_modified_keep_local" || - val.decision === "conflict_modified_keep_remote" || - val.decision === "conflict_modified_keep_both" + val.decision === "local_is_modified_then_push" || + val.decision === "remote_is_modified_then_pull" || + val.decision === "local_is_created_then_push" || + val.decision === "remote_is_created_then_pull" || + val.decision === "conflict_created_then_keep_local" || + val.decision === "conflict_created_then_keep_remote" || + val.decision === "conflict_created_then_keep_both" || + val.decision === "conflict_modified_then_keep_local" || + val.decision === "conflict_modified_then_keep_remote" || + val.decision === "conflict_modified_then_keep_both" ) { if ( uploadDownloads.length === 0 || @@ -910,16 +1011,17 @@ const dispatchOperationToActualV3 = async ( clearPrevSyncRecordByVaultAndProfile(db, vaultRandomID, profileID, key); } else if ( r.decision === "equal" || + r.decision === "conflict_created_then_do_nothing" || r.decision === "folder_to_skip" || - r.decision === "folder_existed_both" + r.decision === "folder_existed_both_then_do_nothing" ) { // pass } else if ( - r.decision === "modified_local" || - r.decision === "created_local" || - r.decision === "folder_existed_local" || - r.decision === "conflict_created_keep_local" || - r.decision === "conflict_modified_keep_local" + r.decision === "local_is_modified_then_push" || + r.decision === "local_is_created_then_push" || + r.decision === "folder_existed_local_then_also_create_remote" || + r.decision === "conflict_created_then_keep_local" || + r.decision === "conflict_modified_then_keep_local" ) { if ( client.serviceType === "onedrive" && @@ -949,11 +1051,11 @@ const dispatchOperationToActualV3 = async ( ); } } else if ( - r.decision === "modified_remote" || - r.decision === "created_remote" || - r.decision === "conflict_created_keep_remote" || - r.decision === "conflict_modified_keep_remote" || - r.decision === "folder_existed_remote" + r.decision === "remote_is_modified_then_pull" || + r.decision === "remote_is_created_then_pull" || + r.decision === "conflict_created_then_keep_remote" || + r.decision === "conflict_modified_then_keep_remote" || + r.decision === "folder_existed_remote_then_also_create_local" ) { await mkdirpInVault(r.key, vault); await client.downloadFromRemote( @@ -969,7 +1071,7 @@ const dispatchOperationToActualV3 = async ( profileID, r.remote! ); - } else if (r.decision === "deleted_local") { + } else if (r.decision === "local_is_deleted_thus_also_delete_remote") { // local is deleted, we need to delete remote now await client.deleteFromRemote(r.key, password, r.remote!.keyEnc); await clearPrevSyncRecordByVaultAndProfile( @@ -978,7 +1080,7 @@ const dispatchOperationToActualV3 = async ( profileID, r.key ); - } else if (r.decision === "deleted_remote") { + } else if (r.decision === "remote_is_deleted_thus_also_delete_local") { // remote is deleted, we need to delete local now await localDeleteFunc(r.key); await clearPrevSyncRecordByVaultAndProfile( @@ -988,8 +1090,8 @@ const dispatchOperationToActualV3 = async ( r.key ); } else if ( - r.decision === "conflict_created_keep_both" || - r.decision === "conflict_modified_keep_both" + r.decision === "conflict_created_then_keep_both" || + r.decision === "conflict_modified_then_keep_both" ) { throw Error(`${r.decision} not implemented yet: ${JSON.stringify(r)}`); } else if (r.decision === "folder_to_be_created") {