bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

half way of new sync

Browse Source
This commit is contained in:
fyears
2024-02-24 08:21:05 +08:00
parent f6ea9938d1
commit 2fbd87eed4
13 changed files with 1042 additions and 1735 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/localdb.ts
+151 -365
View File
@@ -3,35 +3,36 @@ export type LocalForage = typeof localforage;
import { nanoid } from "nanoid";
import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian";
import { API_VER_STAT_FOLDER, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import { API_VER_STAT_FOLDER } from "./baseTypes";
import type { Entity, MixedEntity, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import type { SyncPlanType } from "./sync";
import { statFix, toText, unixTimeToStr } from "./misc";
import { log } from "./moreOnLog";
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326];
export const DEFAULT_DB_VERSION_NUMBER: number = 20220326;
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220];
export const DEFAULT_DB_VERSION_NUMBER: number = 20240220;
export const DEFAULT_DB_NAME = "remotelysavedb";
export const DEFAULT_TBL_VERSION = "schemaversion";
export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory";
export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
export const DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING = "vaultrandomidmapping";
export const DEFAULT_TBL_LOGGER_OUTPUT = "loggeroutput";
export const DEFAULT_TBL_SIMPLE_KV_FOR_MISC = "simplekvformisc";
export const DEFAULT_TBL_PREV_SYNC_RECORDS = "prevsyncrecords";
export interface FileFolderHistoryRecord {
key: string;
ctime: number;
mtime: number;
size: number;
actionWhen: number;
actionType: "delete" | "rename" | "renameDestination";
keyType: "folder" | "file";
renameTo: string;
vaultRandomID: string;
}
/**
* @deprecated
*/
export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory";
/**
* @deprecated
*/
export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
/**
* @deprecated
* But we cannot remove it. Because we want to migrate the old data.
*/
interface SyncMetaMappingRecord {
localKey: string;
remoteKey: string;
@@ -54,108 +55,74 @@ interface SyncPlanRecord {
export interface InternalDBs {
versionTbl: LocalForage;
fileHistoryTbl: LocalForage;
syncMappingTbl: LocalForage;
syncPlansTbl: LocalForage;
vaultRandomIDMappingTbl: LocalForage;
loggerOutputTbl: LocalForage;
simpleKVForMiscTbl: LocalForage;
prevSyncRecordsTbl: LocalForage;
/**
* @deprecated
* But we cannot remove it. Because we want to migrate the old data.
*/
fileHistoryTbl: LocalForage;
/**
* @deprecated
* But we cannot remove it. Because we want to migrate the old data.
*/
syncMappingTbl: LocalForage;
}
/**
* This migration mainly aims to assign vault name or vault id into all tables.
* @param db
* @param vaultRandomID
* TODO
* @param syncMappings
* @returns
*/
const migrateDBsFrom20211114To20220108 = async (
db: InternalDBs,
vaultRandomID: string
) => {
const oldVer = 20211114;
const newVer = 20220108;
log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`);
const allPromisesToWait: Promise<any>[] = [];
log.debug("assign vault id to any delete history");
const keysInDeleteHistoryTbl = await db.fileHistoryTbl.keys();
for (const key of keysInDeleteHistoryTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.fileHistoryTbl.getItem(
key
)) as FileFolderHistoryRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.fileHistoryTbl.setItem(newKey, value));
allPromisesToWait.push(db.fileHistoryTbl.removeItem(key));
}
log.debug("assign vault id to any sync mapping");
const keysInSyncMappingTbl = await db.syncMappingTbl.keys();
for (const key of keysInSyncMappingTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.syncMappingTbl.getItem(
key
)) as SyncMetaMappingRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.syncMappingTbl.setItem(newKey, value));
allPromisesToWait.push(db.syncMappingTbl.removeItem(key));
}
log.debug("assign vault id to any sync plan records");
const keysInSyncPlansTbl = await db.syncPlansTbl.keys();
for (const key of keysInSyncPlansTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.syncPlansTbl.getItem(key)) as SyncPlanRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.syncPlansTbl.setItem(newKey, value));
allPromisesToWait.push(db.syncPlansTbl.removeItem(key));
}
log.debug("finally update version if everything is ok");
await Promise.all(allPromisesToWait);
await db.versionTbl.setItem("version", newVer);
log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
const fromSyncMappingsToPrevSyncRecords = (
syncMappings: SyncMetaMappingRecord[]
): Entity[] => {
return [];
};
/**
* no need to do anything except changing version
* we just add more file operations in db, and no schema is changed.
* TODO
* @param db
* @param vaultRandomID
* @param prevSyncRecord
*/
const migrateDBsFrom20220108To20220326 = async (
const setPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string,
prevSyncRecord: Entity
) => {};
/**
*
* @param db
* @param vaultRandomID
* Migrate the sync mapping record to sync Entity.
*/
const migrateDBsFrom20220326To20240220 = async (
db: InternalDBs,
vaultRandomID: string
) => {
const oldVer = 20220108;
const newVer = 20220326;
const oldVer = 20220326;
const newVer = 20240220;
log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`);
await db.versionTbl.setItem("version", newVer);
// from sync mapping to prev sync
const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID);
const prevSyncRecords = fromSyncMappingsToPrevSyncRecords(syncMappings);
for (const prevSyncRecord of prevSyncRecords) {
await setPrevSyncRecordByVault(db, vaultRandomID, prevSyncRecord);
}
// clear not used data
await clearFileHistoryOfEverythingByVault(db, vaultRandomID);
await clearAllSyncMetaMappingByVault(db, vaultRandomID);
await db.versionTbl.setItem(`${vaultRandomID}\tversion`, newVer);
log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
};
@@ -168,18 +135,19 @@ const migrateDBs = async (
if (oldVer === newVer) {
return;
}
if (oldVer === 20211114 && newVer === 20220108) {
return await migrateDBsFrom20211114To20220108(db, vaultRandomID);
// as of 20240220, we assume everyone is using 20220326 already
// drop any old code to reduce the verbose
if (oldVer < 20220326) {
throw Error(
"You are using a very old version of Remotely Save. No way to auto update internal DB. Please install and enable 0.3.40 firstly, then install a later version."
);
}
if (oldVer === 20220108 && newVer === 20220326) {
return await migrateDBsFrom20220108To20220326(db, vaultRandomID);
}
if (oldVer === 20211114 && newVer === 20220326) {
// TODO: more steps with more versions in the future
await migrateDBsFrom20211114To20220108(db, vaultRandomID);
await migrateDBsFrom20220108To20220326(db, vaultRandomID);
return;
if (oldVer === 20220326 && newVer === 20240220) {
return await migrateDBsFrom20220326To20240220(db, vaultRandomID);
}
if (newVer < oldVer) {
throw Error(
"You've installed a new version, but then downgrade to an old version. Stop working!"
@@ -198,14 +166,6 @@ export const prepareDBs = async (
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_VERSION,
}),
fileHistoryTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_FILE_HISTORY,
}),
syncMappingTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_SYNC_MAPPING,
}),
syncPlansTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_SYNC_PLANS_HISTORY,
@@ -222,6 +182,19 @@ export const prepareDBs = async (
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_SIMPLE_KV_FOR_MISC,
}),
prevSyncRecordsTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_PREV_SYNC_RECORDS,
}),
fileHistoryTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_FILE_HISTORY,
}),
syncMappingTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_SYNC_MAPPING,
}),
} as InternalDBs;
// try to get vaultRandomID firstly
@@ -253,12 +226,19 @@ export const prepareDBs = async (
throw Error("no vaultRandomID found or generated");
}
const originalVersion: number | null = await db.versionTbl.getItem("version");
// as of 20240220, we set the version per vault, instead of global "version"
const originalVersion: number | null =
(await db.versionTbl.getItem(`${vaultRandomID}\tversion`)) ??
(await db.versionTbl.getItem("version"));
if (originalVersion === null) {
log.debug(
`no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}`
);
await db.versionTbl.setItem("version", DEFAULT_DB_VERSION_NUMBER);
// as of 20240220, we set the version per vault, instead of global "version"
await db.versionTbl.setItem(
`${vaultRandomID}\tversion`,
DEFAULT_DB_VERSION_NUMBER
);
} else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) {
// do nothing
} else {
@@ -298,272 +278,47 @@ export const destroyDBs = async () => {
};
};
export const loadFileHistoryTableByVault = async (
export const clearFileHistoryOfEverythingByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const records = [] as FileFolderHistoryRecord[];
await db.fileHistoryTbl.iterate((value, key, iterationNumber) => {
const keys = await db.fileHistoryTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
records.push(value as FileFolderHistoryRecord);
await db.fileHistoryTbl.removeItem(key);
}
});
records.sort((a, b) => a.actionWhen - b.actionWhen); // ascending
return records;
};
export const clearDeleteRenameHistoryOfKeyAndVault = async (
db: InternalDBs,
key: string,
vaultRandomID: string
) => {
const fullKey = `${vaultRandomID}\t${key}`;
const item: FileFolderHistoryRecord | null =
await db.fileHistoryTbl.getItem(fullKey);
if (
item !== null &&
(item.actionType === "delete" || item.actionType === "rename")
) {
await db.fileHistoryTbl.removeItem(fullKey);
}
};
export const insertDeleteRecordByVault = async (
db: InternalDBs,
fileOrFolder: TAbstractFile | string,
vaultRandomID: string
) => {
// log.info(fileOrFolder);
let k: FileFolderHistoryRecord;
if (fileOrFolder instanceof TFile) {
k = {
key: fileOrFolder.path,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
actionWhen: Date.now(),
actionType: "delete",
keyType: "file",
renameTo: "",
vaultRandomID: vaultRandomID,
};
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
} else if (fileOrFolder instanceof TFolder) {
// key should endswith "/"
const key = fileOrFolder.path.endsWith("/")
? fileOrFolder.path
: `${fileOrFolder.path}/`;
const ctime = 0; // they are deleted, so no way to get ctime, mtime
const mtime = 0; // they are deleted, so no way to get ctime, mtime
k = {
key: key,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: Date.now(),
actionType: "delete",
keyType: "folder",
renameTo: "",
vaultRandomID: vaultRandomID,
};
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
} else if (typeof fileOrFolder === "string") {
// always the deletions in .obsidian folder
// so annoying that the path doesn't exists
// and we have to guess whether the path is folder or file
k = {
key: fileOrFolder,
ctime: 0,
mtime: 0,
size: 0,
actionWhen: Date.now(),
actionType: "delete",
keyType: "file",
renameTo: "",
vaultRandomID: vaultRandomID,
};
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
for (const ext of [
"json",
"js",
"mjs",
"ts",
"md",
"txt",
"css",
"png",
"gif",
"jpg",
"jpeg",
"gitignore",
"gitkeep",
]) {
if (fileOrFolder.endsWith(`.${ext}`)) {
// stop here, no more need to insert the folder record later
return;
}
}
// also add a deletion record as folder if not ending with special exts
k = {
key: `${fileOrFolder}/`,
ctime: 0,
mtime: 0,
size: 0,
actionWhen: Date.now(),
actionType: "delete",
keyType: "folder",
renameTo: "",
vaultRandomID: vaultRandomID,
};
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
}
};
/**
* A file/folder is renamed from A to B
* We insert two records:
* A with actionType="rename"
* B with actionType="renameDestination"
* @deprecated But we cannot remove it. Because we want to migrate the old data.
* @param db
* @param fileOrFolder
* @param oldPath
* @param vaultRandomID
* @returns
*/
export const insertRenameRecordByVault = async (
export const getAllSyncMetaMappingByVault = async (
db: InternalDBs,
fileOrFolder: TAbstractFile,
oldPath: string,
vaultRandomID: string
) => {
// log.info(fileOrFolder);
let k1: FileFolderHistoryRecord | undefined;
let k2: FileFolderHistoryRecord | undefined;
const actionWhen = Date.now();
if (fileOrFolder instanceof TFile) {
k1 = {
key: oldPath,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
actionWhen: actionWhen,
actionType: "rename",
keyType: "file",
renameTo: fileOrFolder.path,
vaultRandomID: vaultRandomID,
};
k2 = {
key: fileOrFolder.path,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
actionWhen: actionWhen,
actionType: "renameDestination",
keyType: "file",
renameTo: "", // itself is the destination, so no need to set this field
vaultRandomID: vaultRandomID,
};
} else if (fileOrFolder instanceof TFolder) {
const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
const renameTo = fileOrFolder.path.endsWith("/")
? fileOrFolder.path
: `${fileOrFolder.path}/`;
let ctime = 0;
let mtime = 0;
if (requireApiVersion(API_VER_STAT_FOLDER)) {
// TAbstractFile does not contain these info
// but from API_VER_STAT_FOLDER we can manually stat them by path.
const s = await statFix(fileOrFolder.vault, fileOrFolder.path);
if (s !== undefined && s !== null) {
ctime = s.ctime;
mtime = s.mtime;
}
}
k1 = {
key: key,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: actionWhen,
actionType: "rename",
keyType: "folder",
renameTo: renameTo,
vaultRandomID: vaultRandomID,
};
k2 = {
key: renameTo,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: actionWhen,
actionType: "renameDestination",
keyType: "folder",
renameTo: "", // itself is the destination, so no need to set this field
vaultRandomID: vaultRandomID,
};
}
await Promise.all([
db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k1!.key}`, k1),
db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k2!.key}`, k2),
]);
};
export const upsertSyncMetaMappingDataByVault = async (
serviceType: SUPPORTED_SERVICES_TYPE,
db: InternalDBs,
localKey: string,
localMTime: number,
localSize: number,
remoteKey: string,
remoteMTime: number,
remoteSize: number,
remoteExtraKey: string,
vaultRandomID: string
) => {
const aggregratedInfo: SyncMetaMappingRecord = {
localKey: localKey,
localMtime: localMTime,
localSize: localSize,
remoteKey: remoteKey,
remoteMtime: remoteMTime,
remoteSize: remoteSize,
remoteExtraKey: remoteExtraKey,
remoteType: serviceType,
keyType: localKey.endsWith("/") ? "folder" : "file",
vaultRandomID: vaultRandomID,
};
await db.syncMappingTbl.setItem(
`${vaultRandomID}\t${remoteKey}`,
aggregratedInfo
return await Promise.all(
((await db.syncMappingTbl.keys()) ?? [])
.filter((key) => key.startsWith(`${vaultRandomID}\t`))
.map(
async (key) =>
(await db.syncMappingTbl.getItem(key)) as SyncMetaMappingRecord
)
);
};
export const getSyncMetaMappingByRemoteKeyAndVault = async (
serviceType: SUPPORTED_SERVICES_TYPE,
export const clearAllSyncMetaMappingByVault = async (
db: InternalDBs,
remoteKey: string,
remoteMTime: number,
remoteExtraKey: string,
vaultRandomID: string
) => {
const potentialItem = (await db.syncMappingTbl.getItem(
`${vaultRandomID}\t${remoteKey}`
)) as SyncMetaMappingRecord;
if (potentialItem === null) {
// no result was found
return undefined;
}
if (
potentialItem.remoteKey === remoteKey &&
potentialItem.remoteMtime === remoteMTime &&
potentialItem.remoteExtraKey === remoteExtraKey &&
potentialItem.remoteType === serviceType
) {
// the result was found
return potentialItem;
} else {
return undefined;
const keys = await db.syncMappingTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
await db.syncMappingTbl.removeItem(key);
}
}
};
@@ -651,6 +406,37 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => {
await Promise.all(ps);
};
export const upsertPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string,
prevSync: Entity
) => {
await db.prevSyncRecordsTbl.setItem(
`${vaultRandomID}-${prevSync.key}`,
prevSync
);
};
export const clearPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string,
key: string
) => {
await db.prevSyncRecordsTbl.removeItem(`${vaultRandomID}-${key}`);
};
export const clearAllPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const keys = await db.prevSyncRecordsTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
await db.prevSyncRecordsTbl.removeItem(key);
}
}
};
export const clearAllLoggerOutputRecords = async (db: InternalDBs) => {
await db.loggerOutputTbl.clear();
log.debug(`successfully clearAllLoggerOutputRecords`);