diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 9fc09e1..3770f92 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -104,6 +104,8 @@ export interface RemotelySavePluginSettings { ignorePaths?: string[]; enableStatusBarInfo?: boolean; deleteToWhere?: "system" | "obsidian"; + conflictAction?: ConflictActionType; + howToCleanEmptyFolder?: EmptyFolderCleanType; /** * @deprecated diff --git a/src/langs/en.json b/src/langs/en.json index cd75af2..6ee46cd 100644 --- a/src/langs/en.json +++ b/src/langs/en.json @@ -12,8 +12,8 @@ "syncrun_step2": "2/8 Starting to fetch remote meta data.", "syncrun_step3": "3/8 Checking password correct or not.", "syncrun_passworderr": "Something goes wrong while checking password.", - "syncrun_step4": "4/8 Trying to fetch extra meta data from remote.", - "syncrun_step5": "5/8 Starting to fetch local meta data.", + "syncrun_step4": "4/8 Starting to fetch local meta data.", + "syncrun_step5": "5/8 Starting to fetch local prev sync data.", "syncrun_step6": "6/8 Starting to generate sync plan.", "syncrun_step7": "7/8 Remotely Save Sync data exchanging!", "syncrun_step7skip": "7/8 Remotely Save real sync is skipped in dry run mode.", diff --git a/src/langs/zh_cn.json b/src/langs/zh_cn.json index cb128af..aae1e2e 100644 --- a/src/langs/zh_cn.json +++ b/src/langs/zh_cn.json @@ -12,8 +12,8 @@ "syncrun_step2": "2/8 正在获取远端的元数据。", "syncrun_step3": "3/8 正在检查密码正确与否。", "syncrun_passworderr": "检查密码时候出错。", - "syncrun_step4": "4/8 正在获取远端的额外的元数据。", - "syncrun_step5": "5/8 正在获取本地的元数据。", + "syncrun_step4": "4/8 正在获取本地的元数据。", + "syncrun_step5": "5/8 正在获取本地上一次同步的元数据。", "syncrun_step6": "6/8 正在生成同步计划。", "syncrun_step7": "7/8 Remotely Save 开始发生数据交换！", "syncrun_step7skip": "7/8 Remotely Save 在空跑模式，跳过实际数据交换步骤。", diff --git a/src/langs/zh_tw.json b/src/langs/zh_tw.json index cb81eff..7057050 100644 --- a/src/langs/zh_tw.json +++ b/src/langs/zh_tw.json @@ -12,8 +12,8 @@ "syncrun_step2": "2/8 正在獲取遠端的元資料。", "syncrun_step3": "3/8 正在檢查密碼正確與否。", "syncrun_passworderr": "檢查密碼時候出錯。", - "syncrun_step4": "4/8 正在獲取遠端的額外的元資料。", - "syncrun_step5": "5/8 正在獲取本地的元資料。", + "syncrun_step4": "4/8 正在獲取本地的元資料。", + "syncrun_step5": "5/8 正在獲取本地上一次同步的元資料。", "syncrun_step6": "6/8 正在生成同步計劃。", "syncrun_step7": "7/8 Remotely Save 開始發生資料交換！", "syncrun_step7skip": "7/8 Remotely Save 在空跑模式，跳過實際資料交換步驟。", diff --git a/src/localdb.ts b/src/localdb.ts index ee45c16..c992890 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -329,16 +329,18 @@ export const clearAllSyncMetaMapping = async (db: InternalDBs) => { export const insertSyncPlanRecordByVault = async ( db: InternalDBs, syncPlan: SyncPlanType, - vaultRandomID: string + vaultRandomID: string, + remoteType: SUPPORTED_SERVICES_TYPE ) => { + const now = Date.now(); const record = { - ts: syncPlan.ts, - tsFmt: syncPlan.tsFmt, + ts: now, + tsFmt: unixTimeToStr(now), vaultRandomID: vaultRandomID, - remoteType: syncPlan.remoteType, + remoteType: remoteType, syncPlan: JSON.stringify(syncPlan /* directly stringify */, null, 2), } as SyncPlanRecord; - await db.syncPlansTbl.setItem(`${vaultRandomID}\t${syncPlan.ts}`, record); + await db.syncPlansTbl.setItem(`${vaultRandomID}\t${now}`, record); }; export const clearAllSyncPlanRecords = async (db: InternalDBs) => { @@ -406,6 +408,23 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => { await Promise.all(ps); }; +export const getAllPrevSyncRecordsByVault = async ( + db: InternalDBs, + vaultRandomID: string +) => { + const keys = await db.prevSyncRecordsTbl.keys(); + const res: Entity[] = []; + for (const key of keys) { + if (key.startsWith(`${vaultRandomID}\t`)) { + const val: Entity | null = await db.prevSyncRecordsTbl.getItem(key); + if (val !== null) { + res.push(val); + } + } + } + return res; +}; + export const upsertPrevSyncRecordByVault = async ( db: InternalDBs, vaultRandomID: string, @@ -442,7 +461,7 @@ export const clearAllLoggerOutputRecords = async (db: InternalDBs) => { log.debug(`successfully clearAllLoggerOutputRecords`); }; -export const upsertLastSuccessSyncByVault = async ( +export const upsertLastSuccessSyncTimeByVault = async ( db: InternalDBs, vaultRandomID: string, millis: number @@ -453,7 +472,7 @@ export const upsertLastSuccessSyncByVault = async ( ); }; -export const getLastSuccessSyncByVault = async ( +export const getLastSuccessSyncTimeByVault = async ( db: InternalDBs, vaultRandomID: string ) => { diff --git a/src/main.ts b/src/main.ts index 938cb0b..daaed7b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,8 +7,6 @@ import { setIcon, FileSystemAdapter, Platform, - TFile, - TFolder, requestUrl, requireApiVersion, } from "obsidian"; @@ -23,7 +21,6 @@ import { COMMAND_CALLBACK_ONEDRIVE, COMMAND_CALLBACK_DROPBOX, COMMAND_URI, - REMOTELY_SAVE_VERSION_2024PREPARE, API_VER_ENSURE_REQURL_OK, } from "./baseTypes"; import { importQrCodeUri } from "./importExport"; @@ -32,10 +29,11 @@ import { prepareDBs, InternalDBs, clearExpiredSyncPlanRecords, - upsertLastSuccessSyncByVault, - getLastSuccessSyncByVault, upsertPluginVersionByVault, clearAllLoggerOutputRecords, + upsertLastSuccessSyncTimeByVault, + getLastSuccessSyncTimeByVault, + getAllPrevSyncRecordsByVault, } from "./localdb"; import { RemoteClient } from "./remote"; import { @@ -53,20 +51,22 @@ import { import { DEFAULT_S3_CONFIG } from "./remoteForS3"; import { DEFAULT_WEBDAV_CONFIG } from "./remoteForWebdav"; import { RemotelySaveSettingTab } from "./settings"; -import { parseRemoteItems, SyncStatusType } from "./sync"; -import { doActualSync, getSyncPlan, isPasswordOk } from "./sync"; +import { + doActualSync, + ensembleMixedEnties, + getSyncPlanInplace, + isPasswordOk, + SyncStatusType, +} from "./sync"; import { messyConfigToNormal, normalConfigToMessy } from "./configPersist"; import { getLocalEntityList } from "./local"; import { I18n } from "./i18n"; import type { LangType, LangTypeAndAuto, TransItemType } from "./i18n"; - -import { DeletionOnRemote, MetadataOnRemote } from "./metadataOnRemote"; import { SyncAlgoV3Modal } from "./syncAlgoV3Notice"; import { applyLogWriterInplace, log } from "./moreOnLog"; import AggregateError from "aggregate-error"; import { exportVaultSyncPlansToFiles } from "./debugMode"; -import { SizesConflictModal } from "./syncSizesConflictNotice"; import { compareVersion } from "./misc"; const DEFAULT_SETTINGS: RemotelySavePluginSettings = { @@ -91,6 +91,9 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = { ignorePaths: [], enableStatusBarInfo: true, deleteToWhere: "system", + agreeToUseSyncV3: false, + conflictAction: "keep_newer", + howToCleanEmptyFolder: "skip", }; interface OAuth2Info { @@ -259,33 +262,26 @@ export default class RemotelySavePlugin extends Plugin { } else { getNotice(t("syncrun_step4")); } - this.syncStatus = "getting_remote_extra_meta"; - const { remoteStates, metadataFile } = await parseRemoteItems( - remoteEntityList, - this.db, - this.vaultRandomID, - client.serviceType, - this.settings.password + this.syncStatus = "getting_local_meta"; + const localEntityList = await getLocalEntityList( + this.app.vault, + this.settings.syncConfigDir ?? false, + this.app.vault.configDir, + this.manifest.id ); + // log.info(localEntityList); if (this.settings.currLogLevel === "info") { // pass } else { getNotice(t("syncrun_step5")); } - this.syncStatus = "getting_local_meta"; - const local = this.app.vault.getAllLoadedFiles(); - let localConfigDirContents: ObsConfigDirFileType[] | undefined = - undefined; - if (this.settings.syncConfigDir) { - localConfigDirContents = await listFilesInObsFolder( - this.app.vault.configDir, - this.app.vault, - this.manifest.id - ); - } - // log.info(local); - // log.info(localHistory); + this.syncStatus = "getting_local_prev_sync"; + const prevSyncEntityList = await getAllPrevSyncRecordsByVault( + this.db, + this.vaultRandomID + ); + // log.info(prevSyncEntityList); if (this.settings.currLogLevel === "info") { // pass @@ -293,24 +289,29 @@ export default class RemotelySavePlugin extends Plugin { getNotice(t("syncrun_step6")); } this.syncStatus = "generating_plan"; - const { plan, sortedKeys, deletions, sizesGoWrong } = await getSyncPlan( - remoteStates, - local, - localConfigDirContents, - origMetadataOnRemote.deletions, - localHistory, - client.serviceType, - triggerSource, - this.app.vault, + let mixedEntityMappings = await ensembleMixedEnties( + localEntityList, + prevSyncEntityList, + remoteEntityList, this.settings.syncConfigDir ?? false, this.app.vault.configDir, this.settings.syncUnderscoreItems ?? false, - this.settings.skipSizeLargerThan ?? -1, this.settings.ignorePaths ?? [], this.settings.password ); - log.info(plan.mixedStates); // for debugging - await insertSyncPlanRecordByVault(this.db, plan, this.vaultRandomID); + mixedEntityMappings = await getSyncPlanInplace( + mixedEntityMappings, + this.settings.howToCleanEmptyFolder ?? "skip", + this.settings.skipSizeLargerThan ?? -1, + this.settings.conflictAction ?? "keep_newer" + ); + log.info(mixedEntityMappings); // for debugging + await insertSyncPlanRecordByVault( + this.db, + mixedEntityMappings, + this.vaultRandomID, + client.serviceType + ); // The operations above are almost read only and kind of safe. // The operations below begins to write or delete (!!!) something. @@ -322,23 +323,27 @@ export default class RemotelySavePlugin extends Plugin { getNotice(t("syncrun_step7")); } this.syncStatus = "syncing"; - await doActualSync( + mixedEntityMappings, client, - this.db, this.vaultRandomID, this.app.vault, - plan, - sortedKeys, - metadataFile, - sizesGoWrong, - deletions, - (key: string) => self.trash(key), this.settings.password, - this.settings.concurrency, - - (i: number, totalCount: number, pathName: string, decision: string) => - self.setCurrSyncMsg(i, totalCount, pathName, decision) + this.settings.concurrency ?? 5, + (key: string) => self.trash(key), + ( + realCounter: number, + realTotalCount: number, + pathName: string, + decision: string + ) => + self.setCurrSyncMsg( + realCounter, + realTotalCount, + pathName, + decision + ), + this.db ); } else { this.syncStatus = "syncing"; @@ -359,7 +364,7 @@ export default class RemotelySavePlugin extends Plugin { this.syncStatus = "idle"; const lastSuccessSyncMillis = Date.now(); - await upsertLastSuccessSyncByVault( + await upsertLastSuccessSyncTimeByVault( this.db, this.vaultRandomID, lastSuccessSyncMillis @@ -695,13 +700,13 @@ export default class RemotelySavePlugin extends Plugin { this.statusBarElement.setAttribute("data-tooltip-position", "top"); this.updateLastSuccessSyncMsg( - await getLastSuccessSyncByVault(this.db, this.vaultRandomID) + await getLastSuccessSyncTimeByVault(this.db, this.vaultRandomID) ); // update statusbar text every 30 seconds this.registerInterval( window.setInterval(async () => { this.updateLastSuccessSyncMsg( - await getLastSuccessSyncByVault(this.db, this.vaultRandomID) + await getLastSuccessSyncTimeByVault(this.db, this.vaultRandomID) ); }, 1000 * 30) ); @@ -849,6 +854,12 @@ export default class RemotelySavePlugin extends Plugin { if (this.settings.agreeToUseSyncV3 === undefined) { this.settings.agreeToUseSyncV3 = false; } + if (this.settings.conflictAction === undefined) { + this.settings.conflictAction = "keep_newer"; + } + if (this.settings.howToCleanEmptyFolder === undefined) { + this.settings.howToCleanEmptyFolder = "skip"; + } await this.saveSettings(); } diff --git a/src/obsFolderLister.ts b/src/obsFolderLister.ts index af88774..c12fb76 100644 --- a/src/obsFolderLister.ts +++ b/src/obsFolderLister.ts @@ -4,7 +4,7 @@ import type { Entity, MixedEntity } from "./baseTypes"; import { Queue } from "@fyears/tsqueue"; import chunk from "lodash/chunk"; import flatten from "lodash/flatten"; -import { statFix, isFolderToSkip } from "./misc"; +import { statFix, isSpecialFolderNameToSkip } from "./misc"; const isPluginDirItself = (x: string, pluginId: string) => { return ( @@ -96,7 +96,9 @@ export const listFilesInObsFolder = async ( const isInsideSelfPlugin = isPluginDirItself(iter.itself.key, pluginId); if (iter.children !== undefined) { for (const iter2 of iter.children.folders) { - if (isFolderToSkip(iter2, ["workspace", "workspace.json"])) { + if ( + isSpecialFolderNameToSkip(iter2, ["workspace", "workspace.json"]) + ) { continue; } if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) { @@ -106,7 +108,9 @@ export const listFilesInObsFolder = async ( q.push(iter2); } for (const iter2 of iter.children.files) { - if (isFolderToSkip(iter2, ["workspace", "workspace.json"])) { + if ( + isSpecialFolderNameToSkip(iter2, ["workspace", "workspace.json"]) + ) { continue; } if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) { diff --git a/src/settings.ts b/src/settings.ts index 9ca530d..d758f77 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -26,7 +26,7 @@ import { clearAllSyncMetaMapping, clearAllSyncPlanRecords, destroyDBs, - upsertLastSuccessSyncByVault, + upsertLastSuccessSyncTimeByVault, } from "./localdb"; import type RemotelySavePlugin from "./main"; // unavoidable import { RemoteClient } from "./remote"; @@ -1866,7 +1866,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab { button.setButtonText(t("settings_resetstatusbar_button")); button.onClick(async () => { // reset last sync time - await upsertLastSuccessSyncByVault( + await upsertLastSuccessSyncTimeByVault( this.plugin.db, this.plugin.vaultRandomID, -1 diff --git a/src/sync.ts b/src/sync.ts index 4f3125a..7693d14 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -39,6 +39,18 @@ import { upsertPrevSyncRecordByVault, } from "./localdb"; +export type SyncStatusType = + | "idle" + | "preparing" + | "getting_remote_files_list" + | "getting_local_meta" + | "getting_local_prev_sync" + | "checking_password" + | "generating_plan" + | "syncing" + | "cleaning" + | "finish"; + export interface PasswordCheckType { ok: boolean; reason: @@ -286,7 +298,9 @@ const encryptLocalEntityInplace = async ( return local; }; -const ensembleMixedEnties = async ( +export type SyncPlanType = Record; + +export const ensembleMixedEnties = async ( localEntityList: Entity[], prevSyncEntityList: Entity[], remoteEntityList: Entity[], @@ -296,8 +310,8 @@ const ensembleMixedEnties = async ( syncUnderscoreItems: boolean, ignorePaths: string[], password: string -): Promise> => { - const finalMappings: Record = {}; +): Promise => { + const finalMappings: SyncPlanType = {}; // remote has to be first for (const remote of remoteEntityList) { @@ -863,7 +877,7 @@ const dispatchOperationToActualV3 = async ( } }; -export const doActualSyncV3 = async ( +export const doActualSync = async ( mixedEntityMappings: Record, client: RemoteClient, vaultRandomID: string,