bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add vault name into dbs

Browse Source
This commit is contained in:
fyears 2022-01-08 18:41:11 +08:00
parent c637038d95
commit 04f3a0c97c
7 changed files with 258 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -65,6 +65,7 @@
"lodash": "^4.17.21",
"loglevel": "^1.8.0",
"mime-types": "^2.1.33",
"nanoid": "^3.1.30",
"obsidian": "^0.12.0",
"path-browserify": "^1.0.1",
"process": "^0.11.10",

1
src/baseTypes.ts
View File

@ -53,6 +53,7 @@ export interface RemotelySavePluginSettings {
password: string;
serviceType: SUPPORTED_SERVICES_TYPE;
currLogLevel?: string;
vaultRandomID?: string;
}
export interface RemoteItem {

10
src/debugMode.ts
View File

@ -1,7 +1,7 @@
import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
import type { SyncPlanType } from "./sync";
import { readAllSyncPlanRecordTexts } from "./localdb";
import { readAllSyncPlanRecordTextsByVault } from "./localdb";
import type { InternalDBs } from "./localdb";
import { mkdirpInVault } from "./misc";
@ -11,10 +11,14 @@ const log = origLog.getLogger("rs-default");
const DEFAULT_DEBUG_FOLDER = "_debug_remotely_save/";
const DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX = "sync_plans_hist_exported_on_";
export const exportSyncPlansToFiles = async (db: InternalDBs, vault: Vault) => {
export const exportVaultSyncPlansToFiles = async (
db: InternalDBs,
vault: Vault,
vaultRandomID: string
) => {
log.info("exporting");
await mkdirpInVault(DEFAULT_DEBUG_FOLDER, vault);
const records = await readAllSyncPlanRecordTexts(db);
const records = await readAllSyncPlanRecordTextsByVault(db, vaultRandomID);
let md = "";
if (records.length === 0) {
md = "No sync plans history found";

190
src/localdb.ts
View File

@ -9,7 +9,8 @@ export type LocalForage = typeof localforage;
import * as origLog from "loglevel";
const log = origLog.getLogger("rs-default");
export const DEFAULT_DB_VERSION_NUMBER: number = 20211114;
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108];
export const DEFAULT_DB_VERSION_NUMBER: number = 20220108;
export const DEFAULT_DB_NAME = "remotelysavedb";
export const DEFAULT_TBL_VERSION = "schemaversion";
export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory";
@ -25,6 +26,7 @@ export interface FileFolderHistoryRecord {
actionType: "delete" | "rename";
keyType: "folder" | "file";
renameTo: string;
vaultRandomID: string;
}
interface SyncMetaMappingRecord {
@ -37,12 +39,14 @@ interface SyncMetaMappingRecord {
remoteExtraKey: string;
remoteType: SUPPORTED_SERVICES_TYPE;
keyType: "folder" | "file";
vaultRandomID: string;
}
interface SyncPlanRecord {
ts: number;
remoteType: string;
syncPlan: string;
vaultRandomID: string;
}
export interface InternalDBs {
@ -52,7 +56,103 @@ export interface InternalDBs {
syncPlansTbl: LocalForage;
}
export const prepareDBs = async () => {
/**
* This migration mainly aims to assign vault name or vault id into all tables.
* @param db
* @param vaultRandomID
*/
const migrateDBsFrom20211114To20220108 = async (
db: InternalDBs,
vaultRandomID: string
) => {
const oldVer = 20211114;
const newVer = 20220108;
log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`);
const allPromisesToWait: Promise<any>[] = [];
log.debug("assign vault id to any delete history");
const keysInDeleteHistoryTbl = await db.deleteHistoryTbl.keys();
for (const key of keysInDeleteHistoryTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.deleteHistoryTbl.getItem(
key
)) as FileFolderHistoryRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.deleteHistoryTbl.setItem(newKey, value));
allPromisesToWait.push(db.deleteHistoryTbl.removeItem(key));
}
log.debug("assign vault id to any sync mapping");
const keysInSyncMappingTbl = await db.syncMappingTbl.keys();
for (const key of keysInSyncMappingTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.syncMappingTbl.getItem(
key
)) as SyncMetaMappingRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.syncMappingTbl.setItem(newKey, value));
allPromisesToWait.push(db.syncMappingTbl.removeItem(key));
}
log.debug("assign vault id to any sync plan records");
const keysInSyncPlansTbl = await db.syncPlansTbl.keys();
for (const key of keysInSyncPlansTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.syncPlansTbl.getItem(key)) as SyncPlanRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.syncPlansTbl.setItem(newKey, value));
allPromisesToWait.push(db.syncPlansTbl.removeItem(key));
}
log.debug("finally update version if everything is ok");
await Promise.all(allPromisesToWait);
await db.versionTbl.setItem("version", newVer);
log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
};
const migrateDBs = async (
db: InternalDBs,
oldVer: number,
newVer: number,
vaultRandomID: string
) => {
if (oldVer === newVer) {
return;
}
if (oldVer === 20211114 && newVer === 20220108) {
return await migrateDBsFrom20211114To20220108(db, vaultRandomID);
}
// not implemented
throw Error(`not supported internal db changes from ${oldVer} to ${newVer}`);
};
export const prepareDBs = async (vaultRandomID: string) => {
const db = {
versionTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
@ -74,11 +174,22 @@ export const prepareDBs = async () => {
const originalVersion = (await db.versionTbl.getItem("version")) as number;
if (originalVersion === null) {
log.debug(
`no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}`
);
await db.versionTbl.setItem("version", DEFAULT_DB_VERSION_NUMBER);
} else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) {
// do nothing
} else {
await migrateDBs(db, originalVersion, DEFAULT_DB_VERSION_NUMBER);
log.debug(
`trying to upgrade db version from ${originalVersion} to ${DEFAULT_DB_VERSION_NUMBER}`
);
await migrateDBs(
db,
originalVersion,
DEFAULT_DB_VERSION_NUMBER,
vaultRandomID
);
}
log.info("db connected");
@ -103,33 +214,32 @@ export const destroyDBs = async () => {
};
};
const migrateDBs = async (db: InternalDBs, oldVer: number, newVer: number) => {
if (oldVer === newVer) {
return;
}
// not implemented
throw Error(`not supported internal db changes from ${oldVer} to ${newVer}`);
};
export const loadDeleteRenameHistoryTable = async (db: InternalDBs) => {
export const loadDeleteRenameHistoryTableByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const records = [] as FileFolderHistoryRecord[];
await db.deleteHistoryTbl.iterate((value, key, iterationNumber) => {
if (key.startsWith(`${vaultRandomID}\t`)) {
records.push(value as FileFolderHistoryRecord);
}
});
records.sort((a, b) => a.actionWhen - b.actionWhen); // ascending
return records;
};
export const clearDeleteRenameHistoryOfKey = async (
export const clearDeleteRenameHistoryOfKeyAndVault = async (
db: InternalDBs,
key: string
key: string,
vaultRandomID: string
) => {
await db.deleteHistoryTbl.removeItem(key);
await db.deleteHistoryTbl.removeItem(`${vaultRandomID}\t${key}`);
};
export const insertDeleteRecord = async (
export const insertDeleteRecordByVault = async (
db: InternalDBs,
fileOrFolder: TAbstractFile
fileOrFolder: TAbstractFile,
vaultRandomID: string
) => {
// log.info(fileOrFolder);
let k: FileFolderHistoryRecord;
@ -143,6 +253,7 @@ export const insertDeleteRecord = async (
actionType: "delete",
keyType: "file",
renameTo: "",
vaultRandomID: vaultRandomID,
};
} else if (fileOrFolder instanceof TFolder) {
// key should endswith "/"
@ -158,15 +269,17 @@ export const insertDeleteRecord = async (
actionType: "delete",
keyType: "folder",
renameTo: "",
vaultRandomID: vaultRandomID,
};
}
await db.deleteHistoryTbl.setItem(k.key, k);
await db.deleteHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
};
export const insertRenameRecord = async (
export const insertRenameRecordByVault = async (
db: InternalDBs,
fileOrFolder: TAbstractFile,
oldPath: string
oldPath: string,
vaultRandomID: string
) => {
// log.info(fileOrFolder);
let k: FileFolderHistoryRecord;
@ -180,6 +293,7 @@ export const insertRenameRecord = async (
actionType: "rename",
keyType: "file",
renameTo: fileOrFolder.path,
vaultRandomID: vaultRandomID,
};
} else if (fileOrFolder instanceof TFolder) {
const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
@ -195,12 +309,13 @@ export const insertRenameRecord = async (
actionType: "rename",
keyType: "folder",
renameTo: renameTo,
vaultRandomID: vaultRandomID,
};
}
await db.deleteHistoryTbl.setItem(k.key, k);
await db.deleteHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
};
export const upsertSyncMetaMappingData = async (
export const upsertSyncMetaMappingDataByVault = async (
serviceType: SUPPORTED_SERVICES_TYPE,
db: InternalDBs,
localKey: string,
@ -209,7 +324,8 @@ export const upsertSyncMetaMappingData = async (
remoteKey: string,
remoteMTime: number,
remoteSize: number,
remoteExtraKey: string /* ETag from s3 */
remoteExtraKey: string,
vaultRandomID: string
) => {
const aggregratedInfo: SyncMetaMappingRecord = {
localKey: localKey,
@ -221,19 +337,24 @@ export const upsertSyncMetaMappingData = async (
remoteExtraKey: remoteExtraKey,
remoteType: serviceType,
keyType: localKey.endsWith("/") ? "folder" : "file",
vaultRandomID: vaultRandomID,
};
await db.syncMappingTbl.setItem(remoteKey, aggregratedInfo);
await db.syncMappingTbl.setItem(
`${vaultRandomID}\t${remoteKey}`,
aggregratedInfo
);
};
export const getSyncMetaMappingByRemoteKey = async (
export const getSyncMetaMappingByRemoteKeyAndVault = async (
serviceType: SUPPORTED_SERVICES_TYPE,
db: InternalDBs,
remoteKey: string,
remoteMTime: number,
remoteExtraKey: string
remoteExtraKey: string,
vaultRandomID: string
) => {
const potentialItem = (await db.syncMappingTbl.getItem(
remoteKey
`${vaultRandomID}\t${remoteKey}`
)) as SyncMetaMappingRecord;
if (potentialItem === null) {
@ -258,26 +379,33 @@ export const clearAllSyncMetaMapping = async (db: InternalDBs) => {
await db.syncMappingTbl.clear();
};
export const insertSyncPlanRecord = async (
export const insertSyncPlanRecordByVault = async (
db: InternalDBs,
syncPlan: SyncPlanType
syncPlan: SyncPlanType,
vaultRandomID: string
) => {
const record = {
ts: syncPlan.ts,
vaultRandomID: vaultRandomID,
remoteType: syncPlan.remoteType,
syncPlan: JSON.stringify(syncPlan /* directly stringify */, null, 2),
} as SyncPlanRecord;
await db.syncPlansTbl.setItem(`${syncPlan.ts}`, record);
await db.syncPlansTbl.setItem(`${vaultRandomID}\t${syncPlan.ts}`, record);
};
export const clearAllSyncPlanRecords = async (db: InternalDBs) => {
await db.syncPlansTbl.clear();
};
export const readAllSyncPlanRecordTexts = async (db: InternalDBs) => {
export const readAllSyncPlanRecordTextsByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const records = [] as SyncPlanRecord[];
await db.syncPlansTbl.iterate((value, key, iterationNumber) => {
if (key.startsWith(`${vaultRandomID}\t`)) {
records.push(value as SyncPlanRecord);
}
});
records.sort((a, b) => -(a.ts - b.ts)); // descending

51
src/main.ts
View File

@ -1,5 +1,6 @@
import { Modal, Notice, Plugin, Setting } from "obsidian";
import cloneDeep from "lodash/cloneDeep";
import { nanoid } from "nanoid";
import type { RemotelySavePluginSettings } from "./baseTypes";
import {
COMMAND_CALLBACK,
@ -10,10 +11,10 @@ import {
import { importQrCodeUri } from "./importExport";
import type { InternalDBs } from "./localdb";
import {
insertDeleteRecord,
insertRenameRecord,
insertSyncPlanRecord,
loadDeleteRenameHistoryTable,
insertDeleteRecordByVault,
insertRenameRecordByVault,
insertSyncPlanRecordByVault,
loadDeleteRenameHistoryTableByVault,
prepareDBs,
} from "./localdb";
import { RemoteClient } from "./remote";
@ -47,6 +48,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
password: "",
serviceType: "s3",
currLogLevel: "info",
vaultRandomID: "",
};
interface OAuth2Info {
@ -86,6 +88,7 @@ export default class RemotelySavePlugin extends Plugin {
}
await this.checkIfOauthExpires();
await this.checkIfVaultIDAssigned(); // MUST before prepareDB()
await this.prepareDB();
@ -93,13 +96,22 @@ export default class RemotelySavePlugin extends Plugin {
this.registerEvent(
this.app.vault.on("delete", async (fileOrFolder) => {
await insertDeleteRecord(this.db, fileOrFolder);
await insertDeleteRecordByVault(
this.db,
fileOrFolder,
this.settings.vaultRandomID
);
})
);
this.registerEvent(
this.app.vault.on("rename", async (fileOrFolder, oldPath) => {
await insertRenameRecord(this.db, fileOrFolder, oldPath);
await insertRenameRecordByVault(
this.db,
fileOrFolder,
oldPath,
this.settings.vaultRandomID
);
})
);
@ -316,7 +328,10 @@ export default class RemotelySavePlugin extends Plugin {
new Notice("3/7 Starting to fetch local meta data.");
this.syncStatus = "getting_local_meta";
const local = this.app.vault.getAllLoadedFiles();
const localHistory = await loadDeleteRenameHistoryTable(this.db);
const localHistory = await loadDeleteRenameHistoryTableByVault(
this.db,
this.settings.vaultRandomID
);
// log.info(local);
// log.info(localHistory);
@ -338,11 +353,16 @@ export default class RemotelySavePlugin extends Plugin {
local,
localHistory,
this.db,
this.settings.vaultRandomID,
client.serviceType,
this.settings.password
);
log.info(syncPlan.mixedStates); // for debugging
await insertSyncPlanRecord(this.db, syncPlan);
await insertSyncPlanRecordByVault(
this.db,
syncPlan,
this.settings.vaultRandomID
);
// The operations above are read only and kind of safe.
// The operations below begins to write or delete (!!!) something.
@ -353,6 +373,7 @@ export default class RemotelySavePlugin extends Plugin {
await doActualSync(
client,
this.db,
this.settings.vaultRandomID,
this.app.vault,
syncPlan,
this.settings.password,
@ -481,8 +502,18 @@ export default class RemotelySavePlugin extends Plugin {
}
}
async checkIfVaultIDAssigned() {
if (
this.settings.vaultRandomID === undefined ||
this.settings.vaultRandomID === ""
) {
this.settings.vaultRandomID = nanoid();
await this.saveSettings();
}
}
async prepareDB() {
this.db = await prepareDBs();
this.db = await prepareDBs(this.settings.vaultRandomID);
}
destroyDBs() {
@ -495,7 +526,7 @@ export default class RemotelySavePlugin extends Plugin {
pathName: string,
decision: string
) {
const msg = `${i}/${totalCount}, ${decision}, ${pathName}`;
const msg = `syncing progress=${i}/${totalCount},decision=${decision},path=${pathName}`;
this.currSyncMsg = msg;
}
}

27
src/settings.ts
View File

@ -1,6 +1,6 @@
import { App, Modal, Notice, PluginSettingTab, Setting } from "obsidian";
import type { SUPPORTED_SERVICES_TYPE, WebdavAuthType } from "./baseTypes";
import { exportSyncPlansToFiles } from "./debugMode";
import { exportVaultSyncPlansToFiles } from "./debugMode";
import { exportQrCodeUri } from "./importExport";
import {
clearAllSyncMetaMapping,
@ -19,6 +19,7 @@ import {
DEFAULT_ONEDRIVE_CONFIG,
getAuthUrlAndVerifier as getAuthUrlAndVerifierOnedrive,
} from "./remoteForOnedrive";
import { messyConfigToNormal } from "./configPersist";
import * as origLog from "loglevel";
const log = origLog.getLogger("rs-default");
@ -946,6 +947,24 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
log.info(`the log level is changed to ${val}`);
});
});
const outputCurrSettingsDiv = debugDiv.createDiv("div");
new Setting(outputCurrSettingsDiv)
.setName("output current settings from disk to console")
.setDesc(
"The settings save on disk in encoded. Click this to see the decoded settings in console."
)
.addButton(async (button) => {
button.setButtonText("Output");
button.onClick(async () => {
const c = messyConfigToNormal(await this.plugin.loadData());
if (c.currLogLevel === "debug") {
// no need to ouput it again because debug mode already output it
} else {
log.info(c);
}
new Notice("Finished outputing in console.");
});
});
const syncPlanDiv = debugDiv.createEl("div");
new Setting(syncPlanDiv)
.setName("export sync plans")
@ -955,7 +974,11 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
.addButton(async (button) => {
button.setButtonText("Export");
button.onClick(async () => {
await exportSyncPlansToFiles(this.plugin.db, this.app.vault);
await exportVaultSyncPlansToFiles(
this.plugin.db,
this.app.vault,
this.plugin.settings.vaultRandomID
);
new Notice("sync plans history exported");
});
});

35
src/sync.ts
View File

@ -9,9 +9,9 @@ import {
} from "./encrypt";
import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
import {
clearDeleteRenameHistoryOfKey,
getSyncMetaMappingByRemoteKey,
upsertSyncMetaMappingData,
clearDeleteRenameHistoryOfKeyAndVault,
getSyncMetaMappingByRemoteKeyAndVault,
upsertSyncMetaMappingDataByVault,
} from "./localdb";
import { isHiddenPath, isVaildText, mkdirpInVault } from "./misc";
import { RemoteClient } from "./remote";
@ -170,6 +170,7 @@ const ensembleMixedStates = async (
local: TAbstractFile[],
deleteHistory: FileFolderHistoryRecord[],
db: InternalDBs,
vaultRandomID: string,
remoteType: SUPPORTED_SERVICES_TYPE,
password: string = ""
) => {
@ -190,12 +191,13 @@ const ensembleMixedStates = async (
throw Error(`unexpected key=${remoteEncryptedKey}`);
}
}
const backwardMapping = await getSyncMetaMappingByRemoteKey(
const backwardMapping = await getSyncMetaMappingByRemoteKeyAndVault(
remoteType,
db,
key,
entry.lastModified,
entry.etag
entry.etag,
vaultRandomID
);
let r = {} as FileOrFolderMixedState;
@ -443,6 +445,7 @@ export const getSyncPlan = async (
local: TAbstractFile[],
deleteHistory: FileFolderHistoryRecord[],
db: InternalDBs,
vaultRandomID: string,
remoteType: SUPPORTED_SERVICES_TYPE,
password: string = ""
) => {
@ -451,6 +454,7 @@ export const getSyncPlan = async (
local,
deleteHistory,
db,
vaultRandomID,
remoteType,
password
);
@ -467,6 +471,7 @@ export const getSyncPlan = async (
const dispatchOperationToActual = async (
key: string,
vaultRandomID: string,
state: FileOrFolderMixedState,
client: RemoteClient,
db: InternalDBs,
@ -501,7 +506,7 @@ const dispatchOperationToActual = async (
password,
remoteEncryptedKey
);
await clearDeleteRenameHistoryOfKey(db, state.key);
await clearDeleteRenameHistoryOfKeyAndVault(db, state.key, vaultRandomID);
} else if (state.decision === "upload_clearhist") {
const remoteObjMeta = await client.uploadToRemote(
state.key,
@ -511,7 +516,7 @@ const dispatchOperationToActual = async (
remoteEncryptedKey,
foldersCreatedBefore
);
await upsertSyncMetaMappingData(
await upsertSyncMetaMappingDataByVault(
client.serviceType,
db,
state.key,
@ -520,9 +525,10 @@ const dispatchOperationToActual = async (
state.key,
remoteObjMeta.lastModified,
remoteObjMeta.size,
remoteObjMeta.etag
remoteObjMeta.etag,
vaultRandomID
);
await clearDeleteRenameHistoryOfKey(db, state.key);
await clearDeleteRenameHistoryOfKeyAndVault(db, state.key, vaultRandomID);
} else if (state.decision === "download") {
await mkdirpInVault(state.key, vault);
await client.downloadFromRemote(
@ -534,7 +540,7 @@ const dispatchOperationToActual = async (
);
} else if (state.decision === "delremote_clearhist") {
await client.deleteFromRemote(state.key, password, remoteEncryptedKey);
await clearDeleteRenameHistoryOfKey(db, state.key);
await clearDeleteRenameHistoryOfKeyAndVault(db, state.key, vaultRandomID);
} else if (state.decision === "upload") {
const remoteObjMeta = await client.uploadToRemote(
state.key,
@ -544,7 +550,7 @@ const dispatchOperationToActual = async (
remoteEncryptedKey,
foldersCreatedBefore
);
await upsertSyncMetaMappingData(
await upsertSyncMetaMappingDataByVault(
client.serviceType,
db,
state.key,
@ -553,10 +559,11 @@ const dispatchOperationToActual = async (
state.key,
remoteObjMeta.lastModified,
remoteObjMeta.size,
remoteObjMeta.etag
remoteObjMeta.etag,
vaultRandomID
);
} else if (state.decision === "clearhist") {
await clearDeleteRenameHistoryOfKey(db, state.key);
await clearDeleteRenameHistoryOfKeyAndVault(db, state.key, vaultRandomID);
} else {
throw Error("this should never happen!");
}
@ -565,6 +572,7 @@ const dispatchOperationToActual = async (
export const doActualSync = async (
client: RemoteClient,
db: InternalDBs,
vaultRandomID: string,
vault: Vault,
syncPlan: SyncPlanType,
password: string = "",
@ -586,6 +594,7 @@ export const doActualSync = async (
}
await dispatchOperationToActual(
k2,
vaultRandomID,
v2,
client,
db,