bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add tracking for local movement

Browse Source
This commit is contained in:
fyears 2022-03-26 15:06:34 +08:00
parent dbc9e02ff3
commit 537fae1a2b
4 changed files with 201 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/baseTypes.ts
View File

@ -124,7 +124,8 @@ export interface FileOrFolderMixedState {
deltimeRemote?: number;
sizeLocal?: number;
sizeRemote?: number;
changeMtimeUsingMapping?: boolean;
changeRemoteMtimeUsingMapping?: boolean;
changeLocalMtimeUsingMapping?: boolean;
decision?: DecisionType;
decisionBranch?: number;
syncDone?: "done";

170
src/localdb.ts
View File

@ -1,7 +1,7 @@
import localforage from "localforage";
import { TAbstractFile, TFile, TFolder } from "obsidian";
import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian";
import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import { API_VER_STAT_FOLDER, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import type { SyncPlanType } from "./sync";
export type LocalForage = typeof localforage;
@ -9,11 +9,11 @@ export type LocalForage = typeof localforage;
import * as origLog from "loglevel";
const log = origLog.getLogger("rs-default");
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108];
export const DEFAULT_DB_VERSION_NUMBER: number = 20220108;
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326];
export const DEFAULT_DB_VERSION_NUMBER: number = 20220326;
export const DEFAULT_DB_NAME = "remotelysavedb";
export const DEFAULT_TBL_VERSION = "schemaversion";
export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory";
export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory";
export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
@ -23,7 +23,7 @@ export interface FileFolderHistoryRecord {
mtime: number;
size: number;
actionWhen: number;
actionType: "delete" | "rename";
actionType: "delete" | "rename" | "renameDestination";
keyType: "folder" | "file";
renameTo: string;
vaultRandomID: string;
@ -51,7 +51,7 @@ interface SyncPlanRecord {
export interface InternalDBs {
versionTbl: LocalForage;
deleteHistoryTbl: LocalForage;
fileHistoryTbl: LocalForage;
syncMappingTbl: LocalForage;
syncPlansTbl: LocalForage;
}
@ -72,12 +72,12 @@ const migrateDBsFrom20211114To20220108 = async (
const allPromisesToWait: Promise<any>[] = [];
log.debug("assign vault id to any delete history");
const keysInDeleteHistoryTbl = await db.deleteHistoryTbl.keys();
const keysInDeleteHistoryTbl = await db.fileHistoryTbl.keys();
for (const key of keysInDeleteHistoryTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.deleteHistoryTbl.getItem(
const value = (await db.fileHistoryTbl.getItem(
key
)) as FileFolderHistoryRecord;
if (value === null || value === undefined) {
@ -87,8 +87,8 @@ const migrateDBsFrom20211114To20220108 = async (
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.deleteHistoryTbl.setItem(newKey, value));
allPromisesToWait.push(db.deleteHistoryTbl.removeItem(key));
allPromisesToWait.push(db.fileHistoryTbl.setItem(newKey, value));
allPromisesToWait.push(db.fileHistoryTbl.removeItem(key));
}
log.debug("assign vault id to any sync mapping");
@ -136,6 +136,23 @@ const migrateDBsFrom20211114To20220108 = async (
log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
};
/**
* no need to do anything except changing version
* we just add more file operations in db, and no schema is changed.
* @param db
* @param vaultRandomID
*/
const migrateDBsFrom20220108To20220326 = async (
db: InternalDBs,
vaultRandomID: string
) => {
const oldVer = 20220108;
const newVer = 20220326;
log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`);
await db.versionTbl.setItem("version", newVer);
log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
};
const migrateDBs = async (
db: InternalDBs,
oldVer: number,
@ -148,6 +165,20 @@ const migrateDBs = async (
if (oldVer === 20211114 && newVer === 20220108) {
return await migrateDBsFrom20211114To20220108(db, vaultRandomID);
}
if (oldVer === 20220108 && newVer === 20220326) {
return await migrateDBsFrom20220108To20220326(db, vaultRandomID);
}
if (oldVer === 20211114 && newVer === 20220326) {
// TODO: more steps with more versions in the future
await migrateDBsFrom20211114To20220108(db, vaultRandomID);
await migrateDBsFrom20220108To20220326(db, vaultRandomID);
return;
}
if (newVer < oldVer) {
throw Error(
"You've installed a new version, but then downgrade to an old version. Stop working!"
);
}
// not implemented
throw Error(`not supported internal db changes from ${oldVer} to ${newVer}`);
};
@ -158,9 +189,9 @@ export const prepareDBs = async (vaultRandomID: string) => {
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_VERSION,
}),
deleteHistoryTbl: localforage.createInstance({
fileHistoryTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_DELETE_HISTORY,
storeName: DEFAULT_TBL_FILE_HISTORY,
}),
syncMappingTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
@ -172,7 +203,7 @@ export const prepareDBs = async (vaultRandomID: string) => {
}),
} as InternalDBs;
const originalVersion = (await db.versionTbl.getItem("version")) as number;
const originalVersion: number | null = await db.versionTbl.getItem("version");
if (originalVersion === null) {
log.debug(
`no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}`
@ -196,26 +227,6 @@ export const prepareDBs = async (vaultRandomID: string) => {
return db;
};
export const dropDBs = async (db: InternalDBs) => {
const a1 = localforage.dropInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_VERSION,
});
const a2 = localforage.dropInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_DELETE_HISTORY,
});
const a3 = localforage.dropInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_SYNC_MAPPING,
});
const a4 = localforage.dropInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_SYNC_PLANS_HISTORY,
});
await Promise.all([a1, a2, a3, a4]);
};
export const destroyDBs = async () => {
// await localforage.dropInstance({
// name: DEFAULT_DB_NAME,
@ -226,20 +237,20 @@ export const destroyDBs = async () => {
log.info("db deleted");
};
req.onblocked = (event) => {
console.warn("trying to delete db but it was blocked");
log.warn("trying to delete db but it was blocked");
};
req.onerror = (event) => {
console.error("tried to delete db but something bad!");
console.error(event);
log.error("tried to delete db but something goes wrong!");
log.error(event);
};
};
export const loadDeleteRenameHistoryTableByVault = async (
export const loadFileHistoryTableByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const records = [] as FileFolderHistoryRecord[];
await db.deleteHistoryTbl.iterate((value, key, iterationNumber) => {
await db.fileHistoryTbl.iterate((value, key, iterationNumber) => {
if (key.startsWith(`${vaultRandomID}\t`)) {
records.push(value as FileFolderHistoryRecord);
}
@ -253,7 +264,16 @@ export const clearDeleteRenameHistoryOfKeyAndVault = async (
key: string,
vaultRandomID: string
) => {
await db.deleteHistoryTbl.removeItem(`${vaultRandomID}\t${key}`);
const fullKey = `${vaultRandomID}\t${key}`;
const item: FileFolderHistoryRecord | null = await db.fileHistoryTbl.getItem(
fullKey
);
if (
item !== null &&
(item.actionType === "delete" || item.actionType === "rename")
) {
await db.fileHistoryTbl.removeItem(fullKey);
}
};
export const insertDeleteRecordByVault = async (
@ -280,10 +300,12 @@ export const insertDeleteRecordByVault = async (
const key = fileOrFolder.path.endsWith("/")
? fileOrFolder.path
: `${fileOrFolder.path}/`;
const ctime = 0; // they are deleted, so no way to get ctime, mtime
const mtime = 0; // they are deleted, so no way to get ctime, mtime
k = {
key: key,
ctime: 0,
mtime: 0,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: Date.now(),
actionType: "delete",
@ -292,9 +314,19 @@ export const insertDeleteRecordByVault = async (
vaultRandomID: vaultRandomID,
};
}
await db.deleteHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
};
/**
* A file/folder is renamed from A to B
* We insert two records:
* A with actionType="rename"
* B with actionType="renameDestination"
* @param db
* @param fileOrFolder
* @param oldPath
* @param vaultRandomID
*/
export const insertRenameRecordByVault = async (
db: InternalDBs,
fileOrFolder: TAbstractFile,
@ -302,37 +334,73 @@ export const insertRenameRecordByVault = async (
vaultRandomID: string
) => {
// log.info(fileOrFolder);
let k: FileFolderHistoryRecord;
let k1: FileFolderHistoryRecord;
let k2: FileFolderHistoryRecord;
const actionWhen = Date.now();
if (fileOrFolder instanceof TFile) {
k = {
k1 = {
key: oldPath,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
actionWhen: Date.now(),
actionWhen: actionWhen,
actionType: "rename",
keyType: "file",
renameTo: fileOrFolder.path,
vaultRandomID: vaultRandomID,
};
k2 = {
key: fileOrFolder.path,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
actionWhen: actionWhen,
actionType: "renameDestination",
keyType: "file",
renameTo: "", // itself is the destination, so no need to set this field
vaultRandomID: vaultRandomID,
};
} else if (fileOrFolder instanceof TFolder) {
const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
const renameTo = fileOrFolder.path.endsWith("/")
? fileOrFolder.path
: `${fileOrFolder.path}/`;
k = {
let ctime = 0;
let mtime = 0;
if (requireApiVersion(API_VER_STAT_FOLDER)) {
// TAbstractFile does not contain these info
// but from API_VER_STAT_FOLDER we can manually stat them by path.
const s = await fileOrFolder.vault.adapter.stat(fileOrFolder.path);
ctime = s.ctime;
mtime = s.mtime;
}
k1 = {
key: key,
ctime: 0,
mtime: 0,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: Date.now(),
actionWhen: actionWhen,
actionType: "rename",
keyType: "folder",
renameTo: renameTo,
vaultRandomID: vaultRandomID,
};
k2 = {
key: renameTo,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: actionWhen,
actionType: "renameDestination",
keyType: "folder",
renameTo: "", // itself is the destination, so no need to set this field
vaultRandomID: vaultRandomID,
};
}
await db.deleteHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
await Promise.all([
db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k1.key}`, k1),
db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k2.key}`, k2),
]);
};
export const upsertSyncMetaMappingDataByVault = async (

11
src/main.ts
View File

@ -14,9 +14,8 @@ import {
insertDeleteRecordByVault,
insertRenameRecordByVault,
insertSyncPlanRecordByVault,
loadDeleteRenameHistoryTableByVault,
loadFileHistoryTableByVault,
prepareDBs,
dropDBs,
InternalDBs,
} from "./localdb";
import { RemoteClient } from "./remote";
@ -235,7 +234,7 @@ export default class RemotelySavePlugin extends Plugin {
);
this.syncStatus = "getting_local_meta";
const local = this.app.vault.getAllLoadedFiles();
const localHistory = await loadDeleteRenameHistoryTableByVault(
const localHistory = await loadFileHistoryTableByVault(
this.db,
this.settings.vaultRandomID
);
@ -390,7 +389,12 @@ export default class RemotelySavePlugin extends Plugin {
// no need to await this
this.tryToAddIgnoreFile();
try {
await this.prepareDB();
} catch (err) {
new Notice(err.message, 10 * 1000);
throw err;
}
this.syncStatus = "idle";
@ -643,7 +647,6 @@ export default class RemotelySavePlugin extends Plugin {
async onunload() {
log.info(`unloading plugin ${this.manifest.id}`);
await dropDBs(this.db);
this.syncRibbon = undefined;
if (this.oauth2Info !== undefined) {
this.oauth2Info.helperModal = undefined;

89
src/sync.ts
View File

@ -218,7 +218,7 @@ export const parseRemoteItems = async (
mtimeRemote: backwardMapping.localMtime || entry.lastModified,
sizeRemote: backwardMapping.localSize || entry.size,
remoteEncryptedKey: remoteEncryptedKey,
changeMtimeUsingMapping: true,
changeRemoteMtimeUsingMapping: true,
};
} else {
r = {
@ -227,7 +227,7 @@ export const parseRemoteItems = async (
mtimeRemote: entry.lastModified,
sizeRemote: entry.size,
remoteEncryptedKey: remoteEncryptedKey,
changeMtimeUsingMapping: false,
changeRemoteMtimeUsingMapping: false,
};
}
@ -295,7 +295,7 @@ const ensembleMixedStates = async (
local: TAbstractFile[],
localConfigDirContents: ObsConfigDirFileType[] | undefined,
remoteDeleteHistory: DeletionOnRemote[],
localDeleteHistory: FileFolderHistoryRecord[],
localFileHistory: FileFolderHistoryRecord[],
syncConfigDir: boolean,
configDir: string,
syncUnderscoreItems: boolean
@ -397,7 +397,7 @@ const ensembleMixedStates = async (
}
}
for (const entry of localDeleteHistory) {
for (const entry of localFileHistory) {
let key = entry.key;
if (entry.keyType === "folder") {
if (!entry.key.endsWith("/")) {
@ -409,23 +409,45 @@ const ensembleMixedStates = async (
throw Error(`unexpected ${entry}`);
}
if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
continue;
}
if (entry.actionType === "delete" || entry.actionType === "rename") {
const r = {
key: key,
deltimeLocal: entry.actionWhen,
} as FileOrFolderMixedState;
if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
continue;
}
if (results.hasOwnProperty(key)) {
results[key].key = r.key;
results[key].deltimeLocal = r.deltimeLocal;
} else {
results[key] = r;
results[key].existLocal = false;
results[key].existRemote = false;
results[key].existLocal = false; // we have already checked local
results[key].existRemote = false; // we have already checked remote
}
} else if (entry.actionType === "renameDestination") {
const r = {
key: key,
mtimeLocal: entry.actionWhen,
changeLocalMtimeUsingMapping: true,
};
if (results.hasOwnProperty(key)) {
results[key].mtimeLocal = Math.max(
r.mtimeLocal || 0,
results[key].mtimeLocal || 0
);
results[key].changeLocalMtimeUsingMapping =
r.changeLocalMtimeUsingMapping;
} else {
results[key] = r;
results[key].existLocal = false; // we have already checked local
results[key].existRemote = false; // we have already checked remote
}
} else {
throw Error(
`do not know how to deal with local file history ${entry.key} with ${entry.actionType}`
);
}
}
@ -480,12 +502,12 @@ const assignOperationToFileInplace = (
// 1. mtimeLocal
if (r.existLocal) {
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
if (
r.mtimeLocal >= mtimeRemote &&
r.mtimeLocal >= deltimeLocal &&
r.mtimeLocal >= deltime_remote
r.mtimeLocal >= deltimeRemote
) {
if (r.mtimeLocal === r.mtimeRemote) {
// mtime the same
@ -516,12 +538,12 @@ const assignOperationToFileInplace = (
// 2. mtimeRemote
if (r.existRemote) {
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
if (
r.mtimeRemote > mtimeLocal &&
r.mtimeRemote >= deltimeLocal &&
r.mtimeRemote >= deltime_remote
r.mtimeRemote >= deltimeRemote
) {
r.decision = "downloadRemoteToLocal";
r.decisionBranch = 5;
@ -534,11 +556,11 @@ const assignOperationToFileInplace = (
if (r.deltimeLocal !== undefined && r.deltimeLocal !== 0) {
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
if (
r.deltimeLocal >= mtimeLocal &&
r.deltimeLocal >= mtimeRemote &&
r.deltimeLocal >= deltime_remote
r.deltimeLocal >= deltimeRemote
) {
r.decision = "uploadLocalDelHistToRemote";
r.decisionBranch = 6;
@ -549,7 +571,7 @@ const assignOperationToFileInplace = (
}
}
// 4. deltime_remote
// 4. deltimeRemote
if (r.deltimeRemote !== undefined && r.deltimeRemote !== 0) {
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
@ -617,6 +639,31 @@ const assignOperationToFolderInplace = async (
}
}
// If it was moved to here, after deletion, we should keep it as is.
// The logic not necessarily needs API_VER_STAT_FOLDER.
// The folder needs this logic because it's also determined by file children.
// But the file do not need this logic because the mtimeLocal is checked firstly.
if (
r.existLocal &&
r.changeLocalMtimeUsingMapping &&
r.mtimeLocal > 0 &&
r.mtimeLocal > deltimeLocal &&
r.mtimeLocal > deltimeRemote
) {
keptFolder.add(getParentFolder(r.key));
if (r.existLocal && r.existRemote) {
r.decision = "skipFolder";
r.decisionBranch = 16;
} else if (r.existLocal || r.existRemote) {
r.decision = "createFolder";
r.decisionBranch = 17;
} else {
throw Error(
`Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
);
}
}
if (r.decision === undefined) {
// not yet decided by the above reason
if (deltimeLocal > 0 && deltimeLocal > deltimeRemote) {
@ -678,7 +725,7 @@ export const getSyncPlan = async (
local: TAbstractFile[],
localConfigDirContents: ObsConfigDirFileType[] | undefined,
remoteDeleteHistory: DeletionOnRemote[],
localDeleteHistory: FileFolderHistoryRecord[],
localFileHistory: FileFolderHistoryRecord[],
remoteType: SUPPORTED_SERVICES_TYPE,
vault: Vault,
syncConfigDir: boolean,
@ -691,7 +738,7 @@ export const getSyncPlan = async (
local,
localConfigDirContents,
remoteDeleteHistory,
localDeleteHistory,
localFileHistory,
syncConfigDir,
configDir,
syncUnderscoreItems