bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

half way of new sync

Browse Source
This commit is contained in:
fyears 2024-02-24 08:21:05 +08:00
parent f6ea9938d1
commit 2fbd87eed4
13 changed files with 1042 additions and 1735 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
docs/sync_algorithm_v3.md
View File

@ -6,7 +6,7 @@ An absolutely better sync algorithm. Better for tracking deletions and better fo
## Huge Thanks
Basically a combination of algorithm v2 + [synclone](https://github.com/Jwink3101/syncrclone) + [rsinc](https://github.com/ConorWilliams/rsinc) + (some of rclone [bisync](https://rclone.org/bisync/)). All of the later three are released under MIT License so no worries about the licenses.
Basically a combination of algorithm v2 + [synclone](https://github.com/Jwink3101/syncrclone/blob/master/docs/algorithm.md) + [rsinc](https://github.com/ConorWilliams/rsinc) + (some of rclone [bisync](https://rclone.org/bisync/)). All of the later three are released under MIT License so no worries about the licenses.
## Features
@ -27,12 +27,25 @@ Nice to have
## Description
We have _five_ input sources: local all files, remote all files, _local previous succeeded sync history_, local deletions, remote deletions.
We have _five_ input sources:
Init run, consuming local deletions and remote deletions :
1. local all files
2. remote all files
3. _local previous succeeded sync history_
4. local deletions
5. remote deletions.
Init run, consuming remote deletions :
TBD
Later runs, use the first, second, third sources **only**.
TBD
Table modified based on synclone and rsinc. The number inside the table cell is the decision branch in the code.
| local\remote | remote unchanged | remote modified | remote deleted | remote created |
| --------------- | ------------------ | ---------------- | ------------------ | ---------------- |
| local unchanged | (02) do nothing | (09) pull remote | (07) delete local | (??) conflict |
| local modified | (10) push local | (12) conflict | (08) push local | (??) conflict |
| local deleted | (04) delete remote | (05) pull | (01) clean history | (03) pull remote |
| local created | (??) conflict | (??) conflict | (06) push local | (11) conflict |

70
src/baseTypes.ts
View File

@ -116,14 +116,6 @@ export interface RemotelySavePluginSettings {
logToDB?: boolean;
}
export interface RemoteItem {
key: string;
lastModified?: number;
size: number;
remoteType: SUPPORTED_SERVICES_TYPE;
etag?: string;
}
export const COMMAND_URI = "remotely-save";
export const COMMAND_CALLBACK = "remotely-save-cb";
export const COMMAND_CALLBACK_ONEDRIVE = "remotely-save-cb-onedrive";
@ -165,6 +157,68 @@ export type DecisionType =
| DecisionTypeForFileSize
| DecisionTypeForFolder;
export type EmptyFolderCleanType = "skip" | "clean_both";
export type ConflictActionType = "keep_newer" | "keep_larger" | "rename_both";
export type DecisionTypeForMixedEntity =
| "only_history"
| "equal"
| "modified_local"
| "modified_remote"
| "created_local"
| "created_remote"
| "deleted_local"
| "deleted_remote"
| "conflict_created_keep_local"
| "conflict_created_keep_remote"
| "conflict_created_keep_both"
| "conflict_modified_keep_local"
| "conflict_modified_keep_remote"
| "conflict_modified_keep_both"
| "folder_existed_both"
| "folder_existed_local"
| "folder_existed_remote"
| "folder_to_be_created"
| "folder_to_skip"
| "folder_to_be_deleted";
/**
* uniform representation
* everything should be flat and primitive, so that we can copy.
*/
export interface Entity {
key: string;
keyEnc: string;
mtimeCli?: number;
mtimeCliFmt?: string;
mtimeSvr?: number;
mtimeSvrFmt?: string;
prevSyncTime?: number;
prevSyncTimeFmt?: string;
size?: number; // might be unknown or to be filled
sizeEnc: number;
hash?: string;
etag?: string;
}
/**
* A replacement of FileOrFolderMixedState
*/
export interface MixedEntity {
key: string;
local?: Entity;
prevSync?: Entity;
remote?: Entity;
decisionBranch?: number;
decision?: DecisionTypeForMixedEntity;
conflictAction?: ConflictActionType;
}
/**
* @deprecated
*/
export interface FileOrFolderMixedState {
key: string;
existLocal?: boolean;

63
src/local.ts Normal file
View File

@ -0,0 +1,63 @@
import { TFile, TFolder, type Vault } from "obsidian";
import type { Entity, MixedEntity } from "./baseTypes";
import { listFilesInObsFolder } from "./obsFolderLister";
export const getLocalEntityList = async (
vault: Vault,
syncConfigDir: boolean,
configDir: string,
pluginID: string
) => {
const local: Entity[] = [];
const localTAbstractFiles = vault.getAllLoadedFiles();
for (const entry of localTAbstractFiles) {
let r = {} as Entity;
let key = entry.path;
if (entry.path === "/") {
// ignore
continue;
} else if (entry instanceof TFile) {
let mtimeLocal: number | undefined = Math.max(
entry.stat.mtime ?? 0,
entry.stat.ctime
);
if (mtimeLocal === 0) {
mtimeLocal = undefined;
}
if (mtimeLocal === undefined) {
throw Error(
`Your file has last modified time 0: ${key}, don't know how to deal with it`
);
}
r = {
key: entry.path,
keyEnc: entry.path,
mtimeCli: mtimeLocal,
mtimeSvr: mtimeLocal,
size: entry.stat.size,
sizeEnc: entry.stat.size,
};
} else if (entry instanceof TFolder) {
key = `${entry.path}/`;
r = {
key: key,
keyEnc: key,
size: 0,
sizeEnc: 0,
};
} else {
throw Error(`unexpected ${entry}`);
}
}
if (syncConfigDir) {
const syncFiles = await listFilesInObsFolder(configDir, vault, pluginID);
for (const f of syncFiles) {
local.push(f);
}
}
return local;
};

516
src/localdb.ts
View File

@ -3,35 +3,36 @@ export type LocalForage = typeof localforage;
import { nanoid } from "nanoid";
import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian";
import { API_VER_STAT_FOLDER, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import { API_VER_STAT_FOLDER } from "./baseTypes";
import type { Entity, MixedEntity, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import type { SyncPlanType } from "./sync";
import { statFix, toText, unixTimeToStr } from "./misc";
import { log } from "./moreOnLog";
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326];
export const DEFAULT_DB_VERSION_NUMBER: number = 20220326;
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220];
export const DEFAULT_DB_VERSION_NUMBER: number = 20240220;
export const DEFAULT_DB_NAME = "remotelysavedb";
export const DEFAULT_TBL_VERSION = "schemaversion";
export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory";
export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
export const DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING = "vaultrandomidmapping";
export const DEFAULT_TBL_LOGGER_OUTPUT = "loggeroutput";
export const DEFAULT_TBL_SIMPLE_KV_FOR_MISC = "simplekvformisc";
export const DEFAULT_TBL_PREV_SYNC_RECORDS = "prevsyncrecords";
export interface FileFolderHistoryRecord {
key: string;
ctime: number;
mtime: number;
size: number;
actionWhen: number;
actionType: "delete" | "rename" | "renameDestination";
keyType: "folder" | "file";
renameTo: string;
vaultRandomID: string;
}
/**
* @deprecated
*/
export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory";
/**
* @deprecated
*/
export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
/**
* @deprecated
* But we cannot remove it. Because we want to migrate the old data.
*/
interface SyncMetaMappingRecord {
localKey: string;
remoteKey: string;
@ -54,108 +55,74 @@ interface SyncPlanRecord {
export interface InternalDBs {
versionTbl: LocalForage;
fileHistoryTbl: LocalForage;
syncMappingTbl: LocalForage;
syncPlansTbl: LocalForage;
vaultRandomIDMappingTbl: LocalForage;
loggerOutputTbl: LocalForage;
simpleKVForMiscTbl: LocalForage;
prevSyncRecordsTbl: LocalForage;
/**
* @deprecated
* But we cannot remove it. Because we want to migrate the old data.
*/
fileHistoryTbl: LocalForage;
/**
* @deprecated
* But we cannot remove it. Because we want to migrate the old data.
*/
syncMappingTbl: LocalForage;
}
/**
* This migration mainly aims to assign vault name or vault id into all tables.
* @param db
* @param vaultRandomID
* TODO
* @param syncMappings
* @returns
*/
const migrateDBsFrom20211114To20220108 = async (
db: InternalDBs,
vaultRandomID: string
) => {
const oldVer = 20211114;
const newVer = 20220108;
log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`);
const allPromisesToWait: Promise<any>[] = [];
log.debug("assign vault id to any delete history");
const keysInDeleteHistoryTbl = await db.fileHistoryTbl.keys();
for (const key of keysInDeleteHistoryTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.fileHistoryTbl.getItem(
key
)) as FileFolderHistoryRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.fileHistoryTbl.setItem(newKey, value));
allPromisesToWait.push(db.fileHistoryTbl.removeItem(key));
}
log.debug("assign vault id to any sync mapping");
const keysInSyncMappingTbl = await db.syncMappingTbl.keys();
for (const key of keysInSyncMappingTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.syncMappingTbl.getItem(
key
)) as SyncMetaMappingRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.syncMappingTbl.setItem(newKey, value));
allPromisesToWait.push(db.syncMappingTbl.removeItem(key));
}
log.debug("assign vault id to any sync plan records");
const keysInSyncPlansTbl = await db.syncPlansTbl.keys();
for (const key of keysInSyncPlansTbl) {
if (key.startsWith(vaultRandomID)) {
continue;
}
const value = (await db.syncPlansTbl.getItem(key)) as SyncPlanRecord;
if (value === null || value === undefined) {
continue;
}
if (value.vaultRandomID === undefined || value.vaultRandomID === "") {
value.vaultRandomID = vaultRandomID;
}
const newKey = `${vaultRandomID}\t${key}`;
allPromisesToWait.push(db.syncPlansTbl.setItem(newKey, value));
allPromisesToWait.push(db.syncPlansTbl.removeItem(key));
}
log.debug("finally update version if everything is ok");
await Promise.all(allPromisesToWait);
await db.versionTbl.setItem("version", newVer);
log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
const fromSyncMappingsToPrevSyncRecords = (
syncMappings: SyncMetaMappingRecord[]
): Entity[] => {
return [];
};
/**
* no need to do anything except changing version
* we just add more file operations in db, and no schema is changed.
* TODO
* @param db
* @param vaultRandomID
* @param prevSyncRecord
*/
const migrateDBsFrom20220108To20220326 = async (
const setPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string,
prevSyncRecord: Entity
) => {};
/**
*
* @param db
* @param vaultRandomID
* Migrate the sync mapping record to sync Entity.
*/
const migrateDBsFrom20220326To20240220 = async (
db: InternalDBs,
vaultRandomID: string
) => {
const oldVer = 20220108;
const newVer = 20220326;
const oldVer = 20220326;
const newVer = 20240220;
log.debug(`start upgrading internal db from ${oldVer} to ${newVer}`);
await db.versionTbl.setItem("version", newVer);
// from sync mapping to prev sync
const syncMappings = await getAllSyncMetaMappingByVault(db, vaultRandomID);
const prevSyncRecords = fromSyncMappingsToPrevSyncRecords(syncMappings);
for (const prevSyncRecord of prevSyncRecords) {
await setPrevSyncRecordByVault(db, vaultRandomID, prevSyncRecord);
}
// clear not used data
await clearFileHistoryOfEverythingByVault(db, vaultRandomID);
await clearAllSyncMetaMappingByVault(db, vaultRandomID);
await db.versionTbl.setItem(`${vaultRandomID}\tversion`, newVer);
log.debug(`finish upgrading internal db from ${oldVer} to ${newVer}`);
};
@ -168,18 +135,19 @@ const migrateDBs = async (
if (oldVer === newVer) {
return;
}
if (oldVer === 20211114 && newVer === 20220108) {
return await migrateDBsFrom20211114To20220108(db, vaultRandomID);
// as of 20240220, we assume everyone is using 20220326 already
// drop any old code to reduce the verbose
if (oldVer < 20220326) {
throw Error(
"You are using a very old version of Remotely Save. No way to auto update internal DB. Please install and enable 0.3.40 firstly, then install a later version."
);
}
if (oldVer === 20220108 && newVer === 20220326) {
return await migrateDBsFrom20220108To20220326(db, vaultRandomID);
}
if (oldVer === 20211114 && newVer === 20220326) {
// TODO: more steps with more versions in the future
await migrateDBsFrom20211114To20220108(db, vaultRandomID);
await migrateDBsFrom20220108To20220326(db, vaultRandomID);
return;
if (oldVer === 20220326 && newVer === 20240220) {
return await migrateDBsFrom20220326To20240220(db, vaultRandomID);
}
if (newVer < oldVer) {
throw Error(
"You've installed a new version, but then downgrade to an old version. Stop working!"
@ -198,14 +166,6 @@ export const prepareDBs = async (
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_VERSION,
}),
fileHistoryTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_FILE_HISTORY,
}),
syncMappingTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_SYNC_MAPPING,
}),
syncPlansTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_SYNC_PLANS_HISTORY,
@ -222,6 +182,19 @@ export const prepareDBs = async (
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_SIMPLE_KV_FOR_MISC,
}),
prevSyncRecordsTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_PREV_SYNC_RECORDS,
}),
fileHistoryTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_FILE_HISTORY,
}),
syncMappingTbl: localforage.createInstance({
name: DEFAULT_DB_NAME,
storeName: DEFAULT_TBL_SYNC_MAPPING,
}),
} as InternalDBs;
// try to get vaultRandomID firstly
@ -253,12 +226,19 @@ export const prepareDBs = async (
throw Error("no vaultRandomID found or generated");
}
const originalVersion: number | null = await db.versionTbl.getItem("version");
// as of 20240220, we set the version per vault, instead of global "version"
const originalVersion: number | null =
(await db.versionTbl.getItem(`${vaultRandomID}\tversion`)) ??
(await db.versionTbl.getItem("version"));
if (originalVersion === null) {
log.debug(
`no internal db version, setting it to ${DEFAULT_DB_VERSION_NUMBER}`
);
await db.versionTbl.setItem("version", DEFAULT_DB_VERSION_NUMBER);
// as of 20240220, we set the version per vault, instead of global "version"
await db.versionTbl.setItem(
`${vaultRandomID}\tversion`,
DEFAULT_DB_VERSION_NUMBER
);
} else if (originalVersion === DEFAULT_DB_VERSION_NUMBER) {
// do nothing
} else {
@ -298,272 +278,47 @@ export const destroyDBs = async () => {
};
};
export const loadFileHistoryTableByVault = async (
export const clearFileHistoryOfEverythingByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const records = [] as FileFolderHistoryRecord[];
await db.fileHistoryTbl.iterate((value, key, iterationNumber) => {
const keys = await db.fileHistoryTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
records.push(value as FileFolderHistoryRecord);
await db.fileHistoryTbl.removeItem(key);
}
});
records.sort((a, b) => a.actionWhen - b.actionWhen); // ascending
return records;
};
export const clearDeleteRenameHistoryOfKeyAndVault = async (
db: InternalDBs,
key: string,
vaultRandomID: string
) => {
const fullKey = `${vaultRandomID}\t${key}`;
const item: FileFolderHistoryRecord | null =
await db.fileHistoryTbl.getItem(fullKey);
if (
item !== null &&
(item.actionType === "delete" || item.actionType === "rename")
) {
await db.fileHistoryTbl.removeItem(fullKey);
}
};
export const insertDeleteRecordByVault = async (
db: InternalDBs,
fileOrFolder: TAbstractFile | string,
vaultRandomID: string
) => {
// log.info(fileOrFolder);
let k: FileFolderHistoryRecord;
if (fileOrFolder instanceof TFile) {
k = {
key: fileOrFolder.path,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
actionWhen: Date.now(),
actionType: "delete",
keyType: "file",
renameTo: "",
vaultRandomID: vaultRandomID,
};
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
} else if (fileOrFolder instanceof TFolder) {
// key should endswith "/"
const key = fileOrFolder.path.endsWith("/")
? fileOrFolder.path
: `${fileOrFolder.path}/`;
const ctime = 0; // they are deleted, so no way to get ctime, mtime
const mtime = 0; // they are deleted, so no way to get ctime, mtime
k = {
key: key,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: Date.now(),
actionType: "delete",
keyType: "folder",
renameTo: "",
vaultRandomID: vaultRandomID,
};
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
} else if (typeof fileOrFolder === "string") {
// always the deletions in .obsidian folder
// so annoying that the path doesn't exists
// and we have to guess whether the path is folder or file
k = {
key: fileOrFolder,
ctime: 0,
mtime: 0,
size: 0,
actionWhen: Date.now(),
actionType: "delete",
keyType: "file",
renameTo: "",
vaultRandomID: vaultRandomID,
};
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
for (const ext of [
"json",
"js",
"mjs",
"ts",
"md",
"txt",
"css",
"png",
"gif",
"jpg",
"jpeg",
"gitignore",
"gitkeep",
]) {
if (fileOrFolder.endsWith(`.${ext}`)) {
// stop here, no more need to insert the folder record later
return;
}
}
// also add a deletion record as folder if not ending with special exts
k = {
key: `${fileOrFolder}/`,
ctime: 0,
mtime: 0,
size: 0,
actionWhen: Date.now(),
actionType: "delete",
keyType: "folder",
renameTo: "",
vaultRandomID: vaultRandomID,
};
await db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k.key}`, k);
}
};
/**
* A file/folder is renamed from A to B
* We insert two records:
* A with actionType="rename"
* B with actionType="renameDestination"
* @deprecated But we cannot remove it. Because we want to migrate the old data.
* @param db
* @param fileOrFolder
* @param oldPath
* @param vaultRandomID
* @returns
*/
export const insertRenameRecordByVault = async (
export const getAllSyncMetaMappingByVault = async (
db: InternalDBs,
fileOrFolder: TAbstractFile,
oldPath: string,
vaultRandomID: string
) => {
// log.info(fileOrFolder);
let k1: FileFolderHistoryRecord | undefined;
let k2: FileFolderHistoryRecord | undefined;
const actionWhen = Date.now();
if (fileOrFolder instanceof TFile) {
k1 = {
key: oldPath,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
actionWhen: actionWhen,
actionType: "rename",
keyType: "file",
renameTo: fileOrFolder.path,
vaultRandomID: vaultRandomID,
};
k2 = {
key: fileOrFolder.path,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
actionWhen: actionWhen,
actionType: "renameDestination",
keyType: "file",
renameTo: "", // itself is the destination, so no need to set this field
vaultRandomID: vaultRandomID,
};
} else if (fileOrFolder instanceof TFolder) {
const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
const renameTo = fileOrFolder.path.endsWith("/")
? fileOrFolder.path
: `${fileOrFolder.path}/`;
let ctime = 0;
let mtime = 0;
if (requireApiVersion(API_VER_STAT_FOLDER)) {
// TAbstractFile does not contain these info
// but from API_VER_STAT_FOLDER we can manually stat them by path.
const s = await statFix(fileOrFolder.vault, fileOrFolder.path);
if (s !== undefined && s !== null) {
ctime = s.ctime;
mtime = s.mtime;
}
}
k1 = {
key: key,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: actionWhen,
actionType: "rename",
keyType: "folder",
renameTo: renameTo,
vaultRandomID: vaultRandomID,
};
k2 = {
key: renameTo,
ctime: ctime,
mtime: mtime,
size: 0,
actionWhen: actionWhen,
actionType: "renameDestination",
keyType: "folder",
renameTo: "", // itself is the destination, so no need to set this field
vaultRandomID: vaultRandomID,
};
}
await Promise.all([
db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k1!.key}`, k1),
db.fileHistoryTbl.setItem(`${vaultRandomID}\t${k2!.key}`, k2),
]);
};
export const upsertSyncMetaMappingDataByVault = async (
serviceType: SUPPORTED_SERVICES_TYPE,
db: InternalDBs,
localKey: string,
localMTime: number,
localSize: number,
remoteKey: string,
remoteMTime: number,
remoteSize: number,
remoteExtraKey: string,
vaultRandomID: string
) => {
const aggregratedInfo: SyncMetaMappingRecord = {
localKey: localKey,
localMtime: localMTime,
localSize: localSize,
remoteKey: remoteKey,
remoteMtime: remoteMTime,
remoteSize: remoteSize,
remoteExtraKey: remoteExtraKey,
remoteType: serviceType,
keyType: localKey.endsWith("/") ? "folder" : "file",
vaultRandomID: vaultRandomID,
};
await db.syncMappingTbl.setItem(
`${vaultRandomID}\t${remoteKey}`,
aggregratedInfo
return await Promise.all(
((await db.syncMappingTbl.keys()) ?? [])
.filter((key) => key.startsWith(`${vaultRandomID}\t`))
.map(
async (key) =>
(await db.syncMappingTbl.getItem(key)) as SyncMetaMappingRecord
)
);
};
export const getSyncMetaMappingByRemoteKeyAndVault = async (
serviceType: SUPPORTED_SERVICES_TYPE,
export const clearAllSyncMetaMappingByVault = async (
db: InternalDBs,
remoteKey: string,
remoteMTime: number,
remoteExtraKey: string,
vaultRandomID: string
) => {
const potentialItem = (await db.syncMappingTbl.getItem(
`${vaultRandomID}\t${remoteKey}`
)) as SyncMetaMappingRecord;
if (potentialItem === null) {
// no result was found
return undefined;
}
if (
potentialItem.remoteKey === remoteKey &&
potentialItem.remoteMtime === remoteMTime &&
potentialItem.remoteExtraKey === remoteExtraKey &&
potentialItem.remoteType === serviceType
) {
// the result was found
return potentialItem;
} else {
return undefined;
const keys = await db.syncMappingTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
await db.syncMappingTbl.removeItem(key);
}
}
};
@ -651,6 +406,37 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => {
await Promise.all(ps);
};
export const upsertPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string,
prevSync: Entity
) => {
await db.prevSyncRecordsTbl.setItem(
`${vaultRandomID}-${prevSync.key}`,
prevSync
);
};
export const clearPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string,
key: string
) => {
await db.prevSyncRecordsTbl.removeItem(`${vaultRandomID}-${key}`);
};
export const clearAllPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const keys = await db.prevSyncRecordsTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
await db.prevSyncRecordsTbl.removeItem(key);
}
}
};
export const clearAllLoggerOutputRecords = async (db: InternalDBs) => {
await db.loggerOutputTbl.clear();
log.debug(`successfully clearAllLoggerOutputRecords`);

8
src/main.ts
View File

@ -240,8 +240,8 @@ export default class RemotelySavePlugin extends Plugin {
this.app.vault.getName(),
() => self.saveSettings()
);
const remoteRsp = await client.listAllFromRemote();
// log.debug(remoteRsp);
const remoteEntityList = await client.listAllFromRemote();
// log.debug(remoteEntityList);
if (this.settings.currLogLevel === "info") {
// pass
@ -250,7 +250,7 @@ export default class RemotelySavePlugin extends Plugin {
}
this.syncStatus = "checking_password";
const passwordCheckResult = await isPasswordOk(
remoteRsp.Contents,
remoteEntityList,
this.settings.password
);
if (!passwordCheckResult.ok) {
@ -265,7 +265,7 @@ export default class RemotelySavePlugin extends Plugin {
}
this.syncStatus = "getting_remote_extra_meta";
const { remoteStates, metadataFile } = await parseRemoteItems(
remoteRsp.Contents,
remoteEntityList,
this.db,
this.vaultRandomID,
client.serviceType,

24
src/misc.ts
View File

@ -76,16 +76,17 @@ export const getFolderLevels = (x: string, addEndingSlash: boolean = false) => {
export const mkdirpInVault = async (thePath: string, vault: Vault) => {
// log.info(thePath);
const foldersToBuild = getFolderLevels(thePath);
// log.info(foldersToBuild);
for (const folder of foldersToBuild) {
const r = await vault.adapter.exists(folder);
// log.info(r);
if (!r) {
log.info(`mkdir ${folder}`);
await vault.adapter.mkdir(folder);
}
// as of 2020219,
// Obsidian can create the folder recursively
// but the path should not end with '/'
if (thePath === "/" || thePath === "") {
return;
}
let thePathNoEnding = thePath.endsWith("/")
? thePath.slice(0, thePath.length - 1)
: thePath;
await vault.adapter.mkdir(thePathNoEnding);
};
/**
@ -435,7 +436,10 @@ export const statFix = async (vault: Vault, path: string) => {
return s;
};
export const isFolderToSkip = (x: string, more: string[] | undefined) => {
export const isSpecialFolderNameToSkip = (
x: string,
more: string[] | undefined
) => {
let specialFolders = [
".git",
".github",

35
src/obsFolderLister.ts
View File

@ -1,17 +1,11 @@
import { Vault, Stat, ListedFiles } from "obsidian";
import type { Vault, Stat, ListedFiles } from "obsidian";
import type { Entity, MixedEntity } from "./baseTypes";
import { Queue } from "@fyears/tsqueue";
import chunk from "lodash/chunk";
import flatten from "lodash/flatten";
import { statFix, isFolderToSkip } from "./misc";
export interface ObsConfigDirFileType {
key: string;
ctime: number;
mtime: number;
size: number;
type: "folder" | "file";
}
const isPluginDirItself = (x: string, pluginId: string) => {
return (
x === pluginId ||
@ -48,10 +42,10 @@ export const listFilesInObsFolder = async (
configDir: string,
vault: Vault,
pluginId: string
) => {
): Promise<Entity[]> => {
const q = new Queue([configDir]);
const CHUNK_SIZE = 10;
const contents: ObsConfigDirFileType[] = [];
const contents: Entity[] = [];
while (q.length > 0) {
const itemsToFetch: string[] = [];
while (q.length > 0) {
@ -72,11 +66,26 @@ export const listFilesInObsFolder = async (
children = await vault.adapter.list(x);
}
if (
!isFolder &&
(statRes.mtime === undefined ||
statRes.mtime === null ||
statRes.mtime === 0)
) {
throw Error(
`File in Obsidian ${configDir} has last modified time 0: ${x}, don't know how to deal with it.`
);
}
return {
itself: {
key: isFolder ? `${x}/` : x,
...statRes,
} as ObsConfigDirFileType,
keyEnc: isFolder ? `${x}/` : x,
mtimeCli: statRes.mtime,
mtimeSvr: statRes.mtime,
size: statRes.size,
sizeEnc: statRes.size,
},
children: children,
};
});

3
src/remote.ts
View File

@ -1,5 +1,6 @@
import { Vault } from "obsidian";
import type {
Entity,
DropboxConfig,
OnedriveConfig,
S3Config,
@ -164,7 +165,7 @@ export class RemoteClient {
}
};
listAllFromRemote = async () => {
listAllFromRemote = async (): Promise<Entity[]> => {
if (this.serviceType === "s3") {
return await s3.listAllFromRemote(
s3.getS3Client(this.s3Config!),

105
src/remoteForDropbox.ts
View File

@ -5,7 +5,7 @@ import { Vault } from "obsidian";
import * as path from "path";
import {
DropboxConfig,
RemoteItem,
Entity,
COMMAND_CALLBACK_DROPBOX,
OAUTH2_FORCE_EXPIRE_MILLISECONDS,
} from "./baseTypes";
@ -69,13 +69,13 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length);
};
const fromDropboxItemToRemoteItem = (
const fromDropboxItemToEntity = (
x:
| files.FileMetadataReference
| files.FolderMetadataReference
| files.DeletedMetadataReference,
remoteBaseDir: string
): RemoteItem => {
): Entity => {
let key = getNormPath(x.path_display!, remoteBaseDir);
if (x[".tag"] === "folder" && !key.endsWith("/")) {
key = `${key}/`;
@ -84,93 +84,30 @@ const fromDropboxItemToRemoteItem = (
if (x[".tag"] === "folder") {
return {
key: key,
lastModified: undefined,
keyEnc: key,
size: 0,
remoteType: "dropbox",
sizeEnc: 0,
etag: `${x.id}\t`,
} as RemoteItem;
} as Entity;
} else if (x[".tag"] === "file") {
let mtime = Date.parse(x.client_modified).valueOf();
if (mtime === 0) {
mtime = Date.parse(x.server_modified).valueOf();
}
const mtimeCli = Date.parse(x.client_modified).valueOf();
const mtimeSvr = Date.parse(x.server_modified).valueOf();
return {
key: key,
lastModified: mtime,
keyEnc: key,
mtimeCli: mtimeCli,
mtimeSvr: mtimeSvr,
size: x.size,
remoteType: "dropbox",
sizeEnc: x.size,
hash: x.content_hash,
etag: `${x.id}\t${x.content_hash}`,
} as RemoteItem;
} as Entity;
} else {
// x[".tag"] === "deleted"
throw Error("do not support deleted tag");
}
};
/**
* Dropbox api doesn't return mtime for folders.
* This is a try to assign mtime by using files in folder.
* @param allFilesFolders
* @returns
*/
const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => {
if (allFilesFolders.length === 0) {
return;
}
// sort by longer to shorter
allFilesFolders.sort((a, b) => b.key.length - a.key.length);
// a "map" from dir to mtime
let potentialMTime = {} as Record<string, number>;
// first sort pass, from buttom to up
for (const item of allFilesFolders) {
if (item.key.endsWith("/")) {
// itself is a folder, and initially doesn't have mtime
if (item.lastModified === undefined && item.key in potentialMTime) {
// previously we gathered all sub info of this folder
item.lastModified = potentialMTime[item.key];
}
}
const parent = `${path.posix.dirname(item.key)}/`;
if (item.lastModified !== undefined) {
if (parent in potentialMTime) {
potentialMTime[parent] = Math.max(
potentialMTime[parent],
item.lastModified
);
} else {
potentialMTime[parent] = item.lastModified;
}
}
}
// second pass, from up to buttom.
// fill mtime by parent folder or Date.Now() if still not available.
// this is only possible if no any sub-folder-files recursively.
// we do not sort the array again, just iterate over it by reverse
// using good old for loop.
for (let i = allFilesFolders.length - 1; i >= 0; --i) {
const item = allFilesFolders[i];
if (!item.key.endsWith("/")) {
continue; // skip files
}
if (item.lastModified !== undefined) {
continue; // don't need to deal with it
}
const parent = `${path.posix.dirname(item.key)}/`;
if (parent in potentialMTime) {
item.lastModified = potentialMTime[parent];
} else {
item.lastModified = Date.now().valueOf();
potentialMTime[item.key] = item.lastModified;
}
}
return allFilesFolders;
};
////////////////////////////////////////////////////////////////////////////////
// Dropbox authorization using PKCE
// see https://dropbox.tech/developers/pkce--what-and-why-
@ -498,7 +435,7 @@ export const getRemoteMeta = async (
// size: 0,
// remoteType: "dropbox",
// etag: undefined,
// } as RemoteItem;
// } as Entity;
// }
const rsp = await retryReq(() =>
@ -512,7 +449,7 @@ export const getRemoteMeta = async (
if (rsp.status !== 200) {
throw Error(JSON.stringify(rsp));
}
return fromDropboxItemToRemoteItem(rsp.result, client.remoteBaseDir);
return fromDropboxItemToEntity(rsp.result, client.remoteBaseDir);
};
export const uploadToRemote = async (
@ -670,7 +607,7 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => {
const unifiedContents = contents
.filter((x) => x[".tag"] !== "deleted")
.filter((x) => x.path_display !== `/${client.remoteBaseDir}`)
.map((x) => fromDropboxItemToRemoteItem(x, client.remoteBaseDir));
.map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir));
while (res.result.has_more) {
res = await client.dropbox.filesListFolderContinue({
@ -684,15 +621,11 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => {
const unifiedContents2 = contents2
.filter((x) => x[".tag"] !== "deleted")
.filter((x) => x.path_display !== `/${client.remoteBaseDir}`)
.map((x) => fromDropboxItemToRemoteItem(x, client.remoteBaseDir));
.map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir));
unifiedContents.push(...unifiedContents2);
}
fixLastModifiedTimeInplace(unifiedContents);
return {
Contents: unifiedContents,
};
return unifiedContents;
};
const downloadFromRemoteRaw = async (

33
src/remoteForOnedrive.ts
View File

@ -14,7 +14,7 @@ import {
DEFAULT_CONTENT_TYPE,
OAUTH2_FORCE_EXPIRE_MILLISECONDS,
OnedriveConfig,
RemoteItem,
Entity,
} from "./baseTypes";
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
import {
@ -255,16 +255,13 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
return fileOrFolderPath.slice(`${prefix}/`.length);
};
const constructFromDriveItemToRemoteItemError = (x: DriveItem) => {
const constructFromDriveItemToEntityError = (x: DriveItem) => {
return `parentPath="${
x.parentReference?.path ?? "(no parentReference or path)"
}", selfName="${x.name}"`;
};
const fromDriveItemToRemoteItem = (
x: DriveItem,
remoteBaseDir: string
): RemoteItem => {
const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => {
let key = "";
// possible prefix:
@ -333,14 +330,14 @@ const fromDriveItemToRemoteItem = (
key = x.name;
} else {
throw Error(
`we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToRemoteItemError(
`we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToEntityError(
x
)}`
);
}
} else {
throw Error(
`we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToRemoteItemError(
`we meet file/folder and do not know how to deal with it:\n${constructFromDriveItemToEntityError(
x
)}`
);
@ -350,11 +347,17 @@ const fromDriveItemToRemoteItem = (
if (isFolder) {
key = `${key}/`;
}
const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
return {
key: key,
lastModified: Date.parse(x!.fileSystemInfo!.lastModifiedDateTime!),
keyEnc: key,
mtimeSvr: mtimeSvr,
mtimeCli: mtimeCli,
size: isFolder ? 0 : x.size!,
remoteType: "onedrive",
sizeEnc: isFolder ? 0 : x.size!,
// hash: ?? // TODO
etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes
};
};
@ -666,14 +669,12 @@ export const listAllFromRemote = async (client: WrappedOnedriveClient) => {
await client.saveUpdatedConfigFunc();
}
// unify everything to RemoteItem
// unify everything to Entity
const unifiedContents = driveItems
.map((x) => fromDriveItemToRemoteItem(x, client.remoteBaseDir))
.map((x) => fromDriveItemToEntity(x, client.remoteBaseDir))
.filter((x) => x.key !== "/");
return {
Contents: unifiedContents,
};
return unifiedContents;
};
export const getRemoteMeta = async (
@ -687,7 +688,7 @@ export const getRemoteMeta = async (
);
// log.info(rsp);
const driveItem = rsp as DriveItem;
const res = fromDriveItemToRemoteItem(driveItem, client.remoteBaseDir);
const res = fromDriveItemToEntity(driveItem, client.remoteBaseDir);
// log.info(res);
return res;
};

55
src/remoteForS3.ts
View File

@ -28,7 +28,7 @@ import * as path from "path";
import AggregateError from "aggregate-error";
import {
DEFAULT_CONTENT_TYPE,
RemoteItem,
Entity,
S3Config,
VALID_REQURL,
} from "./baseTypes";
@ -220,51 +220,56 @@ const getLocalNoPrefixPath = (
return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length);
};
const fromS3ObjectToRemoteItem = (
const fromS3ObjectToEntity = (
x: S3ObjectType,
remotePrefix: string,
mtimeRecords: Record<string, number>,
ctimeRecords: Record<string, number>
) => {
let mtime = x.LastModified!.valueOf();
const mtimeSvr = x.LastModified!.valueOf();
let mtimeCli = mtimeSvr;
if (x.Key! in mtimeRecords) {
const m2 = mtimeRecords[x.Key!];
if (m2 !== 0) {
mtime = m2;
mtimeCli = m2;
}
}
const r: RemoteItem = {
key: getLocalNoPrefixPath(x.Key!, remotePrefix),
lastModified: mtime,
const key = getLocalNoPrefixPath(x.Key!, remotePrefix);
const r: Entity = {
key: key,
keyEnc: key,
mtimeSvr: mtimeSvr,
mtimeCli: mtimeCli,
size: x.Size!,
remoteType: "s3",
sizeEnc: x.Size!,
etag: x.ETag,
};
return r;
};
const fromS3HeadObjectToRemoteItem = (
const fromS3HeadObjectToEntity = (
fileOrFolderPathWithRemotePrefix: string,
x: HeadObjectCommandOutput,
remotePrefix: string,
useAccurateMTime: boolean
) => {
let mtime = x.LastModified!.valueOf();
const mtimeSvr = x.LastModified!.valueOf();
let mtimeCli = mtimeSvr;
if (useAccurateMTime && x.Metadata !== undefined) {
const m2 = Math.round(
parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0")
);
if (m2 !== 0) {
mtime = m2;
mtimeCli = m2;
}
}
return {
key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix),
lastModified: mtime,
mtimeSvr: mtimeSvr,
mtimeCli: mtimeCli,
size: x.ContentLength,
remoteType: "s3",
etag: x.ETag,
} as RemoteItem;
} as Entity;
};
export const getS3Client = (s3Config: S3Config) => {
@ -330,7 +335,7 @@ export const getRemoteMeta = async (
})
);
return fromS3HeadObjectToRemoteItem(
return fromS3HeadObjectToEntity(
fileOrFolderPathWithRemotePrefix,
res,
s3Config.remotePrefix ?? "",
@ -538,16 +543,14 @@ const listFromRemoteRaw = async (
// ensemble fake rsp
// in the end, we need to transform the response list
// back to the local contents-alike list
return {
Contents: contents.map((x) =>
fromS3ObjectToRemoteItem(
x,
s3Config.remotePrefix ?? "",
mtimeRecords,
ctimeRecords
)
),
};
return contents.map((x) =>
fromS3ObjectToEntity(
x,
s3Config.remotePrefix ?? "",
mtimeRecords,
ctimeRecords
)
);
};
export const listAllFromRemote = async (
@ -692,7 +695,7 @@ export const deleteFromRemote = async (
if (fileOrFolderPath.endsWith("/") && password === "") {
const x = await listFromRemoteRaw(s3Client, s3Config, remoteFileName);
x.Contents.forEach(async (element) => {
x.forEach(async (element) => {
await s3Client.send(
new DeleteObjectCommand({
Bucket: s3Config.s3BucketName,

27
src/remoteForWebdav.ts
View File

@ -5,7 +5,7 @@ import { Queue } from "@fyears/tsqueue";
import chunk from "lodash/chunk";
import flatten from "lodash/flatten";
import { getReasonPhrase } from "http-status-codes";
import { RemoteItem, VALID_REQURL, WebdavConfig } from "./baseTypes";
import { Entity, VALID_REQURL, WebdavConfig } from "./baseTypes";
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
@ -205,18 +205,21 @@ const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length);
};
const fromWebdavItemToRemoteItem = (x: FileStat, remoteBaseDir: string) => {
const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string) => {
let key = getNormPath(x.filename, remoteBaseDir);
if (x.type === "directory" && !key.endsWith("/")) {
key = `${key}/`;
}
const mtimeSvr = Date.parse(x.lastmod).valueOf();
return {
key: key,
lastModified: Date.parse(x.lastmod).valueOf(),
keyEnc: key,
mtimeSvr: mtimeSvr,
mtimeCli: mtimeSvr, // no universal way to set mtime in webdav
size: x.size,
remoteType: "webdav",
etag: x.etag || undefined,
} as RemoteItem;
sizeEnc: x.size,
etag: x.etag,
} as Entity;
};
export class WrappedWebdavClient {
@ -327,7 +330,7 @@ export const getRemoteMeta = async (
details: false,
})) as FileStat;
log.debug(`getRemoteMeta res=${JSON.stringify(res)}`);
return fromWebdavItemToRemoteItem(res, client.remoteBaseDir);
return fromWebdavItemToEntity(res, client.remoteBaseDir);
};
export const uploadToRemote = async (
@ -359,7 +362,7 @@ export const uploadToRemote = async (
if (password === "") {
// if not encrypted, mkdir a remote folder
await client.client.createDirectory(uploadFile, {
recursive: false, // the sync algo should guarantee no need to recursive
recursive: true,
});
const res = await getRemoteMeta(client, uploadFile);
return res;
@ -400,7 +403,7 @@ export const uploadToRemote = async (
// // we need to create folders before uploading
// const dir = getPathFolder(uploadFile);
// if (dir !== "/" && dir !== "") {
// await client.client.createDirectory(dir, { recursive: false });
// await client.client.createDirectory(dir, { recursive: true });
// }
await client.client.putFileContents(uploadFile, remoteContent, {
overwrite: true,
@ -472,11 +475,7 @@ export const listAllFromRemote = async (client: WrappedWebdavClient) => {
}
)) as FileStat[];
}
return {
Contents: contents.map((x) =>
fromWebdavItemToRemoteItem(x, client.remoteBaseDir)
),
};
return contents.map((x) => fromWebdavItemToEntity(x, client.remoteBaseDir));
};
const downloadFromRemoteRaw = async (

1817
src/sync.ts
View File

File diff suppressed because it is too large Load Diff