import localforage from "localforage"; import { extendPrototype as ep1 } from "localforage-getitems"; import { extendPrototype as ep2 } from "localforage-removeitems"; ep1(localforage); ep2(localforage); export type LocalForage = typeof localforage; import { nanoid } from "nanoid"; import type { Entity, SUPPORTED_SERVICES_TYPE } from "./baseTypes"; import { unixTimeToStr } from "./misc"; import type { SyncPlanType } from "./sync"; const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220]; export const DEFAULT_DB_VERSION_NUMBER: number = 20240220; export const DEFAULT_DB_NAME = "remotelysavedb"; export const DEFAULT_TBL_VERSION = "schemaversion"; export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory"; export const DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING = "vaultrandomidmapping"; export const DEFAULT_TBL_LOGGER_OUTPUT = "loggeroutput"; export const DEFAULT_TBL_SIMPLE_KV_FOR_MISC = "simplekvformisc"; export const DEFAULT_TBL_PREV_SYNC_RECORDS = "prevsyncrecords"; export const DEFAULT_TBL_PROFILER_RESULTS = "profilerresults"; /** * @deprecated */ export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory"; /** * @deprecated */ export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory"; /** * @deprecated * But we cannot remove it. Because we want to migrate the old data. */ interface SyncMetaMappingRecord { localKey: string; remoteKey: string; localSize: number; remoteSize: number; localMtime: number; remoteMtime: number; remoteExtraKey: string; remoteType: SUPPORTED_SERVICES_TYPE; keyType: "folder" | "file"; vaultRandomID: string; } interface SyncPlanRecord { ts: number; remoteType: string; syncPlan: string; vaultRandomID: string; } export interface InternalDBs { versionTbl: LocalForage; syncPlansTbl: LocalForage; vaultRandomIDMappingTbl: LocalForage; loggerOutputTbl: LocalForage; simpleKVForMiscTbl: LocalForage; prevSyncRecordsTbl: LocalForage; profilerResultsTbl: LocalForage; /** * @deprecated * But we cannot remove it. Because we want to migrate the old data. */ fileHistoryTbl: LocalForage; /** * @deprecated * But we cannot remove it. Because we want to migrate the old data. */ syncMappingTbl: LocalForage; } /** * TODO * @param syncMappings * @returns */ const fromSyncMappingsToPrevSyncRecords = ( oldSyncMappings: SyncMetaMappingRecord[] ): Entity[] => { const res: Entity[] = []; for (const oldMapping of oldSyncMappings) { const newEntity: Entity = { key: oldMapping.localKey, keyEnc: oldMapping.remoteKey, keyRaw: oldMapping.remoteKey !== undefined && oldMapping.remoteKey !== "" ? oldMapping.remoteKey : oldMapping.localKey, mtimeCli: oldMapping.localMtime, mtimeSvr: oldMapping.remoteMtime, size: oldMapping.localSize, sizeEnc: oldMapping.remoteSize, sizeRaw: oldMapping.remoteKey !== undefined && oldMapping.remoteKey !== "" ? oldMapping.remoteSize : oldMapping.localSize, etag: oldMapping.remoteExtraKey, }; res.push(newEntity); } return res; }; /** * * @param db * @param vaultRandomID * Migrate the sync mapping record to sync Entity. */ const migrateDBsFrom20220326To20240220 = async ( db: InternalDBs, vaultRandomID: string, profileID: string ) => { const oldVer = 20220326; const newVer = 20240220; console.debug(`start upgrading internal db from ${oldVer} to ${newVer}`); // from sync mapping to prev sync const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID); const prevSyncRecords = fromSyncMappingsToPrevSyncRecords(syncMappings); for (const prevSyncRecord of prevSyncRecords) { await upsertPrevSyncRecordByVaultAndProfile( db, vaultRandomID, profileID, prevSyncRecord ); } // // clear not used data // // as of 20240220, we don't call them, // // for the opportunity for users to downgrade // await clearFileHistoryOfEverythingByVault(db, vaultRandomID); // await clearAllSyncMetaMappingByVault(db, vaultRandomID); await db.versionTbl.setItem(`${vaultRandomID}\tversion`, newVer); console.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`); }; const migrateDBs = async ( db: InternalDBs, oldVer: number, newVer: number, vaultRandomID: string, profileID: string ) => { if (oldVer === newVer) { return; } // as of 20240220, we assume everyone is using 20220326 already // drop any old code to reduce the verbose if (oldVer < 20220326) { throw Error( "You are using a very old version of Remotely Save. No way to auto update internal DB. Please install and enable 0.3.40 firstly, then install a later version." ); } if (oldVer === 20220326 && newVer === 20240220) { return await migrateDBsFrom20220326To20240220(db, vaultRandomID, profileID); } if (newVer < oldVer) { throw Error( "You've installed a new version, but then downgrade to an old version. Stop working!" ); } // not implemented throw Error(`not supported internal db changes from ${oldVer} to ${newVer}`); }; export const prepareDBs = async ( vaultBasePath: string, vaultRandomIDFromOldConfigFile: string, profileID: string ) => { const db = { versionTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_VERSION, }), syncPlansTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_SYNC_PLANS_HISTORY, }), vaultRandomIDMappingTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING, }), loggerOutputTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_LOGGER_OUTPUT, }), simpleKVForMiscTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_SIMPLE_KV_FOR_MISC, }), prevSyncRecordsTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_PREV_SYNC_RECORDS, }), profilerResultsTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_PROFILER_RESULTS, }), fileHistoryTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_FILE_HISTORY, }), syncMappingTbl: localforage.createInstance({ name: DEFAULT_DB_NAME, storeName: DEFAULT_TBL_SYNC_MAPPING, }), } as InternalDBs; // try to get vaultRandomID firstly let vaultRandomID = ""; const vaultRandomIDInDB: string | null = await db.vaultRandomIDMappingTbl.getItem(`path2id\t${vaultBasePath}`); if (vaultRandomIDInDB === null) { if (vaultRandomIDFromOldConfigFile !== "") { // reuse the old config id vaultRandomID = vaultRandomIDFromOldConfigFile; } else { // no old config id, we create a random one vaultRandomID = nanoid(); } // save the id back await db.vaultRandomIDMappingTbl.setItem( `path2id\t${vaultBasePath}`, vaultRandomID ); await db.vaultRandomIDMappingTbl.setItem( `id2path\t${vaultRandomID}`, vaultBasePath ); } else { vaultRandomID = vaultRandomIDInDB; } if (vaultRandomID === "") { throw Error("no vaultRandomID found or generated"); } // as of 20240220, we set the version per vault, instead of global "version" const originalVersion: number | null = (await db.versionTbl.getItem(`${vaultRandomID}\tversion`)) ?? (await db.versionTbl.getItem("version")); if (originalVersion === null) { console.debug( `no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}` ); // as of 20240220, we set the version per vault, instead of global "version" await db.versionTbl.setItem( `${vaultRandomID}\tversion`, DEFAULT_DB_VERSION_NUMBER ); } else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) { // do nothing } else { console.debug( `trying to upgrade db version from ${originalVersion} to ${DEFAULT_DB_VERSION_NUMBER}` ); await migrateDBs( db, originalVersion, DEFAULT_DB_VERSION_NUMBER, vaultRandomID, profileID ); } console.info("db connected"); return { db: db, vaultRandomID: vaultRandomID, }; }; export const destroyDBs = async () => { // await localforage.dropInstance({ // name: DEFAULT_DB_NAME, // }); // console.info("db deleted"); const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME); req.onsuccess = (event) => { console.info("db deleted"); }; req.onblocked = (event) => { console.warn("trying to delete db but it was blocked"); }; req.onerror = (event) => { console.error("tried to delete db but something goes wrong!"); console.error(event); }; }; export const clearFileHistoryOfEverythingByVault = async ( db: InternalDBs, vaultRandomID: string ) => { const keys = (await db.fileHistoryTbl.keys()).filter((x) => x.startsWith(`${vaultRandomID}\t`) ); await db.fileHistoryTbl.removeItems(keys); // for (const key of keys) { // if (key.startsWith(`${vaultRandomID}\t`)) { // await db.fileHistoryTbl.removeItem(key); // } // } }; /** * @deprecated But we cannot remove it. Because we want to migrate the old data. * @param db * @param vaultRandomID * @returns */ export const getAllSyncMetaMappingByVault = async ( db: InternalDBs, vaultRandomID: string ) => { return await Promise.all( ((await db.syncMappingTbl.keys()) ?? []) .filter((key) => key.startsWith(`${vaultRandomID}\t`)) .map( async (key) => (await db.syncMappingTbl.getItem(key)) as SyncMetaMappingRecord ) ); }; export const clearAllSyncMetaMappingByVault = async ( db: InternalDBs, vaultRandomID: string ) => { const keys = (await db.syncMappingTbl.keys()).filter((x) => x.startsWith(`${vaultRandomID}\t`) ); await db.syncMappingTbl.removeItems(keys); // for (const key of keys) { // if (key.startsWith(`${vaultRandomID}\t`)) { // await db.syncMappingTbl.removeItem(key); // } // } }; export const insertSyncPlanRecordByVault = async ( db: InternalDBs, syncPlan: SyncPlanType, vaultRandomID: string, remoteType: SUPPORTED_SERVICES_TYPE ) => { const now = Date.now(); const record = { ts: now, tsFmt: unixTimeToStr(now), vaultRandomID: vaultRandomID, remoteType: remoteType, syncPlan: JSON.stringify(syncPlan /* directly stringify */, null, 2), } as SyncPlanRecord; await db.syncPlansTbl.setItem(`${vaultRandomID}\t${now}`, record); }; export const clearAllSyncPlanRecords = async (db: InternalDBs) => { await db.syncPlansTbl.clear(); }; export const readAllSyncPlanRecordTextsByVault = async ( db: InternalDBs, vaultRandomID: string ) => { const records = [] as SyncPlanRecord[]; await db.syncPlansTbl.iterate((value, key, iterationNumber) => { if (key.startsWith(`${vaultRandomID}\t`)) { records.push(value as SyncPlanRecord); } }); records.sort((a, b) => -(a.ts - b.ts)); // descending if (records === undefined) { return [] as string[]; } else { return records.map((x) => x.syncPlan); } }; /** * We remove records that are older than 1 days or 20 records. * It's a heavy operation, so we shall not place it in the start up. * @param db */ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => { const MILLISECONDS_OLD = 1000 * 60 * 60 * 24 * 1; // 1 days const COUNT_TO_MANY = 20; const currTs = Date.now(); const expiredTs = currTs - MILLISECONDS_OLD; let records = (await db.syncPlansTbl.keys()).map((key) => { const ts = Number.parseInt(key.split("\t")[1]); const expired = ts <= expiredTs; return { ts: ts, key: key, expired: expired, }; }); const keysToRemove = new Set( records.filter((x) => x.expired).map((x) => x.key) ); if (records.length - keysToRemove.size > COUNT_TO_MANY) { // we need to find out records beyond 100 records records = records.filter((x) => !x.expired); // shrink the array records.sort((a, b) => -(a.ts - b.ts)); // descending records.slice(COUNT_TO_MANY).forEach((element) => { keysToRemove.add(element.key); }); } // const ps = [] as Promise[]; // keysToRemove.forEach((element) => { // ps.push(db.syncPlansTbl.removeItem(element)); // }); // await Promise.all(ps); await db.syncPlansTbl.removeItems(Array.from(keysToRemove)); }; export const getAllPrevSyncRecordsByVaultAndProfile = async ( db: InternalDBs, vaultRandomID: string, profileID: string ) => { const res: Entity[] = []; const kv: Record = await db.prevSyncRecordsTbl.getItems(); for (const key of Object.getOwnPropertyNames(kv)) { if (key.startsWith(`${vaultRandomID}\t${profileID}\t`)) { const val = kv[key]; if (val !== null) { res.push(val); } } } return res; }; export const upsertPrevSyncRecordByVaultAndProfile = async ( db: InternalDBs, vaultRandomID: string, profileID: string, prevSync: Entity ) => { await db.prevSyncRecordsTbl.setItem( `${vaultRandomID}\t${profileID}\t${prevSync.key}`, prevSync ); }; export const clearPrevSyncRecordByVaultAndProfile = async ( db: InternalDBs, vaultRandomID: string, profileID: string, key: string ) => { await db.prevSyncRecordsTbl.removeItem( `${vaultRandomID}\t${profileID}\t${key}` ); }; export const clearAllPrevSyncRecordByVault = async ( db: InternalDBs, vaultRandomID: string ) => { const keys = (await db.prevSyncRecordsTbl.keys()).filter((x) => x.startsWith(`${vaultRandomID}\t`) ); await db.prevSyncRecordsTbl.removeItems(keys); }; export const clearAllLoggerOutputRecords = async (db: InternalDBs) => { await db.loggerOutputTbl.clear(); console.debug(`successfully clearAllLoggerOutputRecords`); }; export const upsertLastSuccessSyncTimeByVault = async ( db: InternalDBs, vaultRandomID: string, millis: number ) => { await db.simpleKVForMiscTbl.setItem( `${vaultRandomID}-lastSuccessSyncMillis`, millis ); }; export const getLastSuccessSyncTimeByVault = async ( db: InternalDBs, vaultRandomID: string ) => { return (await db.simpleKVForMiscTbl.getItem( `${vaultRandomID}-lastSuccessSyncMillis` )) as number; }; export const upsertPluginVersionByVault = async ( db: InternalDBs, vaultRandomID: string, newVersion: string ) => { let oldVersion: string | null = await db.simpleKVForMiscTbl.getItem( `${vaultRandomID}-pluginversion` ); if (oldVersion === null) { oldVersion = "0.0.0"; } await db.simpleKVForMiscTbl.setItem( `${vaultRandomID}-pluginversion`, newVersion ); return { oldVersion: oldVersion, newVersion: newVersion, }; }; export const insertProfilerResultByVault = async ( db: InternalDBs, profilerStr: string, vaultRandomID: string, remoteType: SUPPORTED_SERVICES_TYPE ) => { const now = Date.now(); await db.profilerResultsTbl.setItem(`${vaultRandomID}\t${now}`, profilerStr); // clear older one while writing const records = (await db.profilerResultsTbl.keys()) .filter((x) => x.startsWith(`${vaultRandomID}\t`)) .map((x) => Number.parseInt(x.split("\t")[1])); records.sort((a, b) => -(a - b)); // descending while (records.length > 5) { const ts = records.pop()!; await db.profilerResultsTbl.removeItem(`${vaultRandomID}\t${ts}`); } }; export const readAllProfilerResultsByVault = async ( db: InternalDBs, vaultRandomID: string ) => { const records = [] as { val: string; ts: number }[]; await db.profilerResultsTbl.iterate((value, key, iterationNumber) => { if (key.startsWith(`${vaultRandomID}\t`)) { records.push({ val: value as string, ts: Number.parseInt(key.split("\t")[1]), }); } }); records.sort((a, b) => -(a.ts - b.ts)); // descending if (records === undefined) { return [] as string[]; } else { return records.map((x) => x.val); } };