diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 7a4d9c9..b33ae50 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -124,7 +124,8 @@ export interface FileOrFolderMixedState { deltimeRemote?: number; sizeLocal?: number; sizeRemote?: number; - changeMtimeUsingMapping?: boolean; + changeRemoteMtimeUsingMapping?: boolean; + changeLocalMtimeUsingMapping?: boolean; decision?: DecisionType; decisionBranch?: number; syncDone?: "done"; diff --git a/src/localdb.ts b/src/localdb.ts index abcc1a0..35c3b74 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -1,7 +1,7 @@ import localforage from "localforage"; -import { TAbstractFile, TFile, TFolder } from "obsidian"; +import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian"; -import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes"; +import { API_VER_STAT_FOLDER, SUPPORTED_SERVICES_TYPE } from "./baseTypes"; import type { SyncPlanType } from "./sync"; export type LocalForage = typeof localforage; @@ -9,11 +9,11 @@ export type LocalForage = typeof localforage; import * as origLog from "loglevel"; const log = origLog.getLogger("rs-default"); -const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108]; -export const DEFAULT_DB_VERSION_NUMBER: number = 20220108; +const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326]; +export const DEFAULT_DB_VERSION_NUMBER: number = 20220326; export const DEFAULT_DB_NAME = "remotelysavedb"; export const DEFAULT_TBL_VERSION = "schemaversion"; -export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory"; +export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory"; export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory"; export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory"; @@ -23,7 +23,7 @@ export interface FileFolderHistoryRecord { mtime: number; size: number; actionWhen: number; - actionType: "delete" | "rename"; + actionType: "delete" | "rename" | "renameDestination"; keyType: "folder" | "file"; renameTo: string; vaultRandomID: string; @@ -51,7 +51,7 @@ interface SyncPlanRecord { export interface InternalDBs { versionTbl: LocalForage; - deleteHistoryTbl: LocalForage; + fileHistoryTbl: LocalForage; syncMappingTbl: LocalForage; syncPlansTbl: LocalForage; } @@ -72,12 +72,12 @@ const migrateDBsFrom20211114To20220108 = async ( const allPromisesToWait: Promise[] = []; log.debug("assign vault id to any delete history"); - const keysInDeleteHistoryTbl = await db.deleteHistoryTbl.keys(); + const keysInDeleteHistoryTbl = await db.fileHistoryTbl.keys(); for (const key of keysInDeleteHistoryTbl) { if (key.startsWith(vaultRandomID)) { continue; } - const value = (await db.deleteHistoryTbl.getItem( + const value = (await db.fileHistoryTbl.getItem( key )) as FileFolderHistoryRecord; if (value === null || value === undefined) { @@ -87,8 +87,8 @@ const migrateDBsFrom20211114To20220108 = async ( value.vaultRandomID = vaultRandomID; } const newKey = `${vaultRandomID}\t${key}`; - allPromisesToWait.push(db.deleteHistoryTbl.setItem(newKey, value)); - allPromisesToWait.push(db.deleteHistoryTbl.removeItem(key)); + allPromisesToWait.push(db.fileHistoryTbl.setItem(newKey, value)); + allPromisesToWait.push(db.fileHistoryTbl.removeItem(key)); } log.debug("assign vault id to any sync mapping"); @@ -136,6 +136,23 @@ const migrateDBsFrom20211114To20220108 = async ( log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); }; +/** + * no need to do anything except changing version + * we just add more file operations in db, and no schema is changed. + * @param db + * @param vaultRandomID + */ +const migrateDBsFrom20220108To20220326 = async ( + db: InternalDBs, + vaultRandomID: string +) => { + const oldVer = 20220108; + const newVer = 20220326; + log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`); + await db.versionTbl.setItem("version", newVer); + log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); +}; + const migrateDBs = async ( db: InternalDBs, oldVer: number, @@ -148,6 +165,20 @@ const migrateDBs = async ( if (oldVer === 20211114 && newVer === 20220108) { return await migrateDBsFrom20211114To20220108(db, vaultRandomID); } + if (oldVer === 20220108 && newVer === 20220326) { + return await migrateDBsFrom20220108To20220326(db, vaultRandomID); + } + if (oldVer === 20211114 && newVer === 20220326) { + // TODO: more steps with more versions in the future + await migrateDBsFrom20211114To20220108(db, vaultRandomID); + await migrateDBsFrom20220108To20220326(db, vaultRandomID); + return; + } + if (newVer < oldVer) { + throw Error( + "You've installed a new version, but then downgrade to an old version. Stop working!" + ); + } // not implemented throw Error(`not supported internal db changes from ${oldVer} to ${newVer}`); }; @@ -158,9 +189,9 @@ export const prepareDBs = async (vaultRandomID: string) => { name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_VERSION, }), - deleteHistoryTbl: localforage.createInstance({ + fileHistoryTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, - storeName: DEFAULT_TBL_DELETE_HISTORY, + storeName: DEFAULT_TBL_FILE_HISTORY, }), syncMappingTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, @@ -172,7 +203,7 @@ export const prepareDBs = async (vaultRandomID: string) => { }), } as InternalDBs; - const originalVersion = (await db.versionTbl.getItem("version")) as number; + const originalVersion: number | null = await db.versionTbl.getItem("version"); if (originalVersion === null) { log.debug( `no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}` @@ -196,26 +227,6 @@ export const prepareDBs = async (vaultRandomID: string) => { return db; }; -export const dropDBs = async (db: InternalDBs) => { - const a1 = localforage.dropInstance({ - name: DEFAULT_DB_NAME, - storeName: DEFAULT_TBL_VERSION, - }); - const a2 = localforage.dropInstance({ - name: DEFAULT_DB_NAME, - storeName: DEFAULT_TBL_DELETE_HISTORY, - }); - const a3 = localforage.dropInstance({ - name: DEFAULT_DB_NAME, - storeName: DEFAULT_TBL_SYNC_MAPPING, - }); - const a4 = localforage.dropInstance({ - name: DEFAULT_DB_NAME, - storeName: DEFAULT_SYNC_PLANS_HISTORY, - }); - await Promise.all([a1, a2, a3, a4]); -}; - export const destroyDBs = async () => { // await localforage.dropInstance({ // name: DEFAULT_DB_NAME, @@ -226,20 +237,20 @@ export const destroyDBs = async () => { log.info("db deleted"); }; req.onblocked = (event) => { - console.warn("trying to delete db but it was blocked"); + log.warn("trying to delete db but it was blocked"); }; req.onerror = (event) => { - console.error("tried to delete db but something bad!"); - console.error(event); + log.error("tried to delete db but something goes wrong!"); + log.error(event); }; }; -export const loadDeleteRenameHistoryTableByVault = async ( +export const loadFileHistoryTableByVault = async ( db: InternalDBs, vaultRandomID: string ) => { const records = [] as FileFolderHistoryRecord[]; - await db.deleteHistoryTbl.iterate((value, key, iterationNumber) => { + await db.fileHistoryTbl.iterate((value, key, iterationNumber) => { if (key.startsWith(`${vaultRandomID}\t`)) { records.push(value as FileFolderHistoryRecord); } @@ -253,7 +264,16 @@ export const clearDeleteRenameHistoryOfKeyAndVault = async ( key: string, vaultRandomID: string ) => { - await db.deleteHistoryTbl.removeItem(`${vaultRandomID}\t${key}`); + const fullKey = `${vaultRandomID}\t${key}`; + const item: FileFolderHistoryRecord | null = await db.fileHistoryTbl.getItem( + fullKey + ); + if ( + item !== null && + (item.actionType === "delete" || item.actionType === "rename") + ) { + await db.fileHistoryTbl.removeItem(fullKey); + } }; export const insertDeleteRecordByVault = async ( @@ -280,10 +300,12 @@ export const insertDeleteRecordByVault = async ( const key = fileOrFolder.path.endsWith("/") ? fileOrFolder.path : `${fileOrFolder.path}/`; + const ctime = 0; // they are deleted, so no way to get ctime, mtime + const mtime = 0; // they are deleted, so no way to get ctime, mtime k = { key: key, - ctime: 0, - mtime: 0, + ctime: ctime, + mtime: mtime, size: 0, actionWhen: Date.now(), actionType: "delete", @@ -292,9 +314,19 @@ export const insertDeleteRecordByVault = async ( vaultRandomID: vaultRandomID, }; } - await db.deleteHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); + await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); }; +/** + * A file/folder is renamed from A to B + * We insert two records: + * A with actionType="rename" + * B with actionType="renameDestination" + * @param db + * @param fileOrFolder + * @param oldPath + * @param vaultRandomID + */ export const insertRenameRecordByVault = async ( db: InternalDBs, fileOrFolder: TAbstractFile, @@ -302,37 +334,73 @@ export const insertRenameRecordByVault = async ( vaultRandomID: string ) => { // log.info(fileOrFolder); - let k: FileFolderHistoryRecord; + let k1: FileFolderHistoryRecord; + let k2: FileFolderHistoryRecord; + const actionWhen = Date.now(); if (fileOrFolder instanceof TFile) { - k = { + k1 = { key: oldPath, ctime: fileOrFolder.stat.ctime, mtime: fileOrFolder.stat.mtime, size: fileOrFolder.stat.size, - actionWhen: Date.now(), + actionWhen: actionWhen, actionType: "rename", keyType: "file", renameTo: fileOrFolder.path, vaultRandomID: vaultRandomID, }; + k2 = { + key: fileOrFolder.path, + ctime: fileOrFolder.stat.ctime, + mtime: fileOrFolder.stat.mtime, + size: fileOrFolder.stat.size, + actionWhen: actionWhen, + actionType: "renameDestination", + keyType: "file", + renameTo: "", // itself is the destination, so no need to set this field + vaultRandomID: vaultRandomID, + }; } else if (fileOrFolder instanceof TFolder) { const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`; const renameTo = fileOrFolder.path.endsWith("/") ? fileOrFolder.path : `${fileOrFolder.path}/`; - k = { + let ctime = 0; + let mtime = 0; + if (requireApiVersion(API_VER_STAT_FOLDER)) { + // TAbstractFile does not contain these info + // but from API_VER_STAT_FOLDER we can manually stat them by path. + const s = await fileOrFolder.vault.adapter.stat(fileOrFolder.path); + ctime = s.ctime; + mtime = s.mtime; + } + k1 = { key: key, - ctime: 0, - mtime: 0, + ctime: ctime, + mtime: mtime, size: 0, - actionWhen: Date.now(), + actionWhen: actionWhen, actionType: "rename", keyType: "folder", renameTo: renameTo, vaultRandomID: vaultRandomID, }; + k2 = { + key: renameTo, + ctime: ctime, + mtime: mtime, + size: 0, + actionWhen: actionWhen, + actionType: "renameDestination", + keyType: "folder", + renameTo: "", // itself is the destination, so no need to set this field + vaultRandomID: vaultRandomID, + }; } - await db.deleteHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k); + await Promise.all([ + db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k1.key}`, k1), + db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k2.key}`, k2), + ]); }; export const upsertSyncMetaMappingDataByVault = async ( diff --git a/src/main.ts b/src/main.ts index c71cac5..c58a06a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -14,9 +14,8 @@ import { insertDeleteRecordByVault, insertRenameRecordByVault, insertSyncPlanRecordByVault, - loadDeleteRenameHistoryTableByVault, + loadFileHistoryTableByVault, prepareDBs, - dropDBs, InternalDBs, } from "./localdb"; import { RemoteClient } from "./remote"; @@ -235,7 +234,7 @@ export default class RemotelySavePlugin extends Plugin { ); this.syncStatus = "getting_local_meta"; const local = this.app.vault.getAllLoadedFiles(); - const localHistory = await loadDeleteRenameHistoryTableByVault( + const localHistory = await loadFileHistoryTableByVault( this.db, this.settings.vaultRandomID ); @@ -390,7 +389,12 @@ export default class RemotelySavePlugin extends Plugin { // no need to await this this.tryToAddIgnoreFile(); - await this.prepareDB(); + try { + await this.prepareDB(); + } catch (err) { + new Notice(err.message, 10 * 1000); + throw err; + } this.syncStatus = "idle"; @@ -643,7 +647,6 @@ export default class RemotelySavePlugin extends Plugin { async onunload() { log.info(`unloading plugin ${this.manifest.id}`); - await dropDBs(this.db); this.syncRibbon = undefined; if (this.oauth2Info !== undefined) { this.oauth2Info.helperModal = undefined; diff --git a/src/sync.ts b/src/sync.ts index 7702bb0..ec424c0 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -218,7 +218,7 @@ export const parseRemoteItems = async ( mtimeRemote: backwardMapping.localMtime || entry.lastModified, sizeRemote: backwardMapping.localSize || entry.size, remoteEncryptedKey: remoteEncryptedKey, - changeMtimeUsingMapping: true, + changeRemoteMtimeUsingMapping: true, }; } else { r = { @@ -227,7 +227,7 @@ export const parseRemoteItems = async ( mtimeRemote: entry.lastModified, sizeRemote: entry.size, remoteEncryptedKey: remoteEncryptedKey, - changeMtimeUsingMapping: false, + changeRemoteMtimeUsingMapping: false, }; } @@ -295,7 +295,7 @@ const ensembleMixedStates = async ( local: TAbstractFile[], localConfigDirContents: ObsConfigDirFileType[] | undefined, remoteDeleteHistory: DeletionOnRemote[], - localDeleteHistory: FileFolderHistoryRecord[], + localFileHistory: FileFolderHistoryRecord[], syncConfigDir: boolean, configDir: string, syncUnderscoreItems: boolean @@ -397,7 +397,7 @@ const ensembleMixedStates = async ( } } - for (const entry of localDeleteHistory) { + for (const entry of localFileHistory) { let key = entry.key; if (entry.keyType === "folder") { if (!entry.key.endsWith("/")) { @@ -409,23 +409,45 @@ const ensembleMixedStates = async ( throw Error(`unexpected ${entry}`); } - const r = { - key: key, - deltimeLocal: entry.actionWhen, - } as FileOrFolderMixedState; - if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) { continue; } - if (results.hasOwnProperty(key)) { - results[key].key = r.key; - results[key].deltimeLocal = r.deltimeLocal; - } else { - results[key] = r; + if (entry.actionType === "delete" || entry.actionType === "rename") { + const r = { + key: key, + deltimeLocal: entry.actionWhen, + } as FileOrFolderMixedState; - results[key].existLocal = false; - results[key].existRemote = false; + if (results.hasOwnProperty(key)) { + results[key].deltimeLocal = r.deltimeLocal; + } else { + results[key] = r; + results[key].existLocal = false; // we have already checked local + results[key].existRemote = false; // we have already checked remote + } + } else if (entry.actionType === "renameDestination") { + const r = { + key: key, + mtimeLocal: entry.actionWhen, + changeLocalMtimeUsingMapping: true, + }; + if (results.hasOwnProperty(key)) { + results[key].mtimeLocal = Math.max( + r.mtimeLocal || 0, + results[key].mtimeLocal || 0 + ); + results[key].changeLocalMtimeUsingMapping = + r.changeLocalMtimeUsingMapping; + } else { + results[key] = r; + results[key].existLocal = false; // we have already checked local + results[key].existRemote = false; // we have already checked remote + } + } else { + throw Error( + `do not know how to deal with local file history ${entry.key} with ${entry.actionType}` + ); } } @@ -480,12 +502,12 @@ const assignOperationToFileInplace = ( // 1. mtimeLocal if (r.existLocal) { const mtimeRemote = r.existRemote ? r.mtimeRemote : -1; - const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; + const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; if ( r.mtimeLocal >= mtimeRemote && r.mtimeLocal >= deltimeLocal && - r.mtimeLocal >= deltime_remote + r.mtimeLocal >= deltimeRemote ) { if (r.mtimeLocal === r.mtimeRemote) { // mtime the same @@ -516,12 +538,12 @@ const assignOperationToFileInplace = ( // 2. mtimeRemote if (r.existRemote) { const mtimeLocal = r.existLocal ? r.mtimeLocal : -1; - const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; + const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1; if ( r.mtimeRemote > mtimeLocal && r.mtimeRemote >= deltimeLocal && - r.mtimeRemote >= deltime_remote + r.mtimeRemote >= deltimeRemote ) { r.decision = "downloadRemoteToLocal"; r.decisionBranch = 5; @@ -534,11 +556,11 @@ const assignOperationToFileInplace = ( if (r.deltimeLocal !== undefined && r.deltimeLocal !== 0) { const mtimeLocal = r.existLocal ? r.mtimeLocal : -1; const mtimeRemote = r.existRemote ? r.mtimeRemote : -1; - const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; + const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1; if ( r.deltimeLocal >= mtimeLocal && r.deltimeLocal >= mtimeRemote && - r.deltimeLocal >= deltime_remote + r.deltimeLocal >= deltimeRemote ) { r.decision = "uploadLocalDelHistToRemote"; r.decisionBranch = 6; @@ -549,7 +571,7 @@ const assignOperationToFileInplace = ( } } - // 4. deltime_remote + // 4. deltimeRemote if (r.deltimeRemote !== undefined && r.deltimeRemote !== 0) { const mtimeLocal = r.existLocal ? r.mtimeLocal : -1; const mtimeRemote = r.existRemote ? r.mtimeRemote : -1; @@ -617,6 +639,31 @@ const assignOperationToFolderInplace = async ( } } + // If it was moved to here, after deletion, we should keep it as is. + // The logic not necessarily needs API_VER_STAT_FOLDER. + // The folder needs this logic because it's also determined by file children. + // But the file do not need this logic because the mtimeLocal is checked firstly. + if ( + r.existLocal && + r.changeLocalMtimeUsingMapping && + r.mtimeLocal > 0 && + r.mtimeLocal > deltimeLocal && + r.mtimeLocal > deltimeRemote + ) { + keptFolder.add(getParentFolder(r.key)); + if (r.existLocal && r.existRemote) { + r.decision = "skipFolder"; + r.decisionBranch = 16; + } else if (r.existLocal || r.existRemote) { + r.decision = "createFolder"; + r.decisionBranch = 17; + } else { + throw Error( + `Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.` + ); + } + } + if (r.decision === undefined) { // not yet decided by the above reason if (deltimeLocal > 0 && deltimeLocal > deltimeRemote) { @@ -678,7 +725,7 @@ export const getSyncPlan = async ( local: TAbstractFile[], localConfigDirContents: ObsConfigDirFileType[] | undefined, remoteDeleteHistory: DeletionOnRemote[], - localDeleteHistory: FileFolderHistoryRecord[], + localFileHistory: FileFolderHistoryRecord[], remoteType: SUPPORTED_SERVICES_TYPE, vault: Vault, syncConfigDir: boolean, @@ -691,7 +738,7 @@ export const getSyncPlan = async ( local, localConfigDirContents, remoteDeleteHistory, - localDeleteHistory, + localFileHistory, syncConfigDir, configDir, syncUnderscoreItems