new sync algo, squashed commit of the following:
commit 692bb794aea5609b9e9abf5228620f4479e50983 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 27 17:52:43 2022 +0800 bump to 0.3.0 commit 77335412ad2da2b5bd1ed5075061a5af006e3c36 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 27 17:50:57 2022 +0800 change titles for minimal intrusive design commit 2812adebb84344d384749a62acb63fd0c6fd509d Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 27 17:30:53 2022 +0800 remove syncv1 commit 22fc24a76c9851740bbc7c0177d1cb2526e15d8b Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 27 17:30:27 2022 +0800 full notice to any one commit d56ea24a78f6dc1fbea2740011ee1cea9c403d4c Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 23:11:14 2022 +0800 fix test commit 759f82593bbfb9b49079cfd80dbadbbafc0287e5 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 21:59:25 2022 +0800 obfuscate metadata on remote commit 9b6bf05288af0e52d0f02468e5ac8757f4f896df Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 21:33:52 2022 +0800 avoid re-uploading if meta not changed commit 03be1453764e48e99207f44467ee4d5801072ed8 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 00:35:52 2022 +0800 add password condition commit 7f899f7c2572df3e2a35e16cbdaab94db113f366 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 00:22:58 2022 +0800 add decision branch for easier debugging commit cf4071bf3156356ae6ec3a9cb187c2cb541d1b17 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Fri Feb 25 23:40:52 2022 +0800 change folder error commit 964493dd998699a1d53018a201637bda192c4baa Author: fyears <1142836+fyears@users.noreply.github.com> Date: Fri Feb 25 23:09:44 2022 +0800 finnaly remote remove should be working commit 2888e65452f9c0e1dde6005f012c3ee0a6313c3f Author: fyears <1142836+fyears@users.noreply.github.com> Date: Fri Feb 25 01:18:15 2022 +0800 optimize comparation commit 024936951d6180b1296c2a5d56c5bf5d468e9ae7 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Fri Feb 25 01:14:44 2022 +0800 allow uploading extra meta commit 007006701d536da2b4b46389941980579bbc4e67 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Thu Feb 24 01:08:58 2022 +0800 add logic to fetch extra meta commit c9d3a05ca1bf45c06f22233124670e5e45b5f5f1 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Thu Feb 24 00:29:16 2022 +0800 another way to deal with trash commit 82d455f8bf60f7bac8eb4e299a2ca44c331a6d7f Author: fyears <1142836+fyears@users.noreply.github.com> Date: Thu Feb 24 00:28:51 2022 +0800 add return buffer for downloading commit b8e6b79bc078def2854bc73578b7f520cc39ab34 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Feb 23 00:16:06 2022 +0800 half way to actual sync commit 90cceb1411b46af9741f2caa3ab8beafaf69c1b2 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Tue Feb 22 23:36:09 2022 +0800 cleaner way to remember the folder being kept commit c1afb763cc4e0f7905c83e0a8eb20f8ed969a279 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Tue Feb 22 00:03:21 2022 +0800 simplified way to get plans for sync algo v2 commit 5c5ecce39eb3854900f1f5b79c7beb189e78a6a7 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 23:13:58 2022 +0800 archive the old sync algo v1 doc commit 75cdfa1ee9834600f83ded6672b102de2c5f9616 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 23:13:14 2022 +0800 simplify sync algo v2 commit dc9275835d961de07efcbaa81657e4745242e72a Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 22:58:57 2022 +0800 add way to skip recording removing commit f9712ef96021dfed4ae33e6c649f78e940b7e530 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 09:38:09 2022 +0800 fix sync commit 9007dcf22ef317dde36ac4f1dd589d05cc8d5cf6 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 00:54:21 2022 +0800 fix assignment commit 77abee6ad443b62353ed3913e0945ea7d1314f87 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 00:28:43 2022 +0800 draft of sync v2 commit a0c26238bf60692e095cfd8527abf937255b56d4 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 00:23:19 2022 +0800 how to deal with folders commit c10f92a7e8d3c4a4f4c585e39e0abad1a5376c02 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 20 23:39:16 2022 +0800 helper func to get parents commit f903c98b3b7b9d1e785d04b9fc0cfcafdc6e5661 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 20 23:31:21 2022 +0800 add optional ending slash to getFolderLevels commit 2d67c9b2b941e676588fa43ab289fab79f567e5e Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 20 14:44:44 2022 +0800 update sync algo v2 commit 491ed1bb85759df2411706fd02d740acb5598ce6 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 20 14:34:51 2022 +0800 dry run mode commit dfd588dcef512ba7dfe760008bcf97138b97e339 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 19 23:14:32 2022 +0800 prettier commit eae580f882a045ae9df799b816e68c3500704131 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 19 23:12:29 2022 +0800 draft design for sync algo v2
This commit is contained in:
fyears
+116
-50
@@ -34,11 +34,13 @@ import {
|
||||
import { DEFAULT_S3_CONFIG } from "./remoteForS3";
|
||||
import { DEFAULT_WEBDAV_CONFIG } from "./remoteForWebdav";
|
||||
import { RemotelySaveSettingTab } from "./settings";
|
||||
import type { SyncStatusType } from "./sync";
|
||||
import { fetchMetadataFile, parseRemoteItems, SyncStatusType } from "./sync";
|
||||
import { doActualSync, getSyncPlan, isPasswordOk } from "./sync";
|
||||
import { messyConfigToNormal, normalConfigToMessy } from "./configPersist";
|
||||
|
||||
import * as origLog from "loglevel";
|
||||
import { DeletionOnRemote, MetadataOnRemote } from "./metadataOnRemote";
|
||||
import { SyncAlgoV2Modal } from "./syncAlgoV2Notice";
|
||||
const log = origLog.getLogger("rs-default");
|
||||
|
||||
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
||||
@@ -51,6 +53,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
||||
currLogLevel: "info",
|
||||
vaultRandomID: "",
|
||||
autoRunEveryMilliseconds: -1,
|
||||
agreeToUploadExtraMetadata: false,
|
||||
};
|
||||
|
||||
interface OAuth2Info {
|
||||
@@ -61,7 +64,7 @@ interface OAuth2Info {
|
||||
revokeAuthSetting?: Setting;
|
||||
}
|
||||
|
||||
type SyncTriggerSourceType = "manual" | "auto";
|
||||
type SyncTriggerSourceType = "manual" | "auto" | "dry";
|
||||
|
||||
const iconNameSyncWait = `remotely-save-sync-wait`;
|
||||
const iconNameSyncRunning = `remotely-save-sync-running`;
|
||||
@@ -89,7 +92,7 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
const getNotice = (x: string) => {
|
||||
// only show notices in manual mode
|
||||
// no notice in auto mode
|
||||
if (triggerSource === "manual") {
|
||||
if (triggerSource === "manual" || triggerSource === "dry") {
|
||||
new Notice(x);
|
||||
}
|
||||
};
|
||||
@@ -122,14 +125,22 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
);
|
||||
}
|
||||
|
||||
const MAX_STEPS = 8;
|
||||
|
||||
if (triggerSource === "dry") {
|
||||
getNotice(
|
||||
`0/${MAX_STEPS} Remotely Save running in dry mode, not actual file changes would happen.`
|
||||
);
|
||||
}
|
||||
|
||||
//log.info(`huh ${this.settings.password}`)
|
||||
getNotice(
|
||||
`1/7 Remotely Save Sync Preparing (${this.settings.serviceType})`
|
||||
`1/${MAX_STEPS} Remotely Save Sync Preparing (${this.settings.serviceType})`
|
||||
);
|
||||
this.syncStatus = "preparing";
|
||||
|
||||
getNotice("2/7 Starting to fetch remote meta data.");
|
||||
this.syncStatus = "getting_remote_meta";
|
||||
getNotice(`2/${MAX_STEPS} Starting to fetch remote meta data.`);
|
||||
this.syncStatus = "getting_remote_files_list";
|
||||
const self = this;
|
||||
const client = new RemoteClient(
|
||||
this.settings.serviceType,
|
||||
@@ -143,17 +154,7 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
const remoteRsp = await client.listFromRemote();
|
||||
log.info(remoteRsp);
|
||||
|
||||
getNotice("3/7 Starting to fetch local meta data.");
|
||||
this.syncStatus = "getting_local_meta";
|
||||
const local = this.app.vault.getAllLoadedFiles();
|
||||
const localHistory = await loadDeleteRenameHistoryTableByVault(
|
||||
this.db,
|
||||
this.settings.vaultRandomID
|
||||
);
|
||||
// log.info(local);
|
||||
// log.info(localHistory);
|
||||
|
||||
getNotice("4/7 Checking password correct or not.");
|
||||
getNotice(`3/${MAX_STEPS} Checking password correct or not.`);
|
||||
this.syncStatus = "checking_password";
|
||||
const passwordCheckResult = await isPasswordOk(
|
||||
remoteRsp.Contents,
|
||||
@@ -164,43 +165,81 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
throw Error(passwordCheckResult.reason);
|
||||
}
|
||||
|
||||
getNotice("5/7 Starting to generate sync plan.");
|
||||
this.syncStatus = "generating_plan";
|
||||
const syncPlan = await getSyncPlan(
|
||||
getNotice(`4/${MAX_STEPS} Trying to fetch extra meta data from remote.`);
|
||||
this.syncStatus = "getting_remote_extra_meta";
|
||||
const { remoteStates, metadataFile } = await parseRemoteItems(
|
||||
remoteRsp.Contents,
|
||||
local,
|
||||
localHistory,
|
||||
this.db,
|
||||
this.settings.vaultRandomID,
|
||||
client.serviceType,
|
||||
this.settings.password
|
||||
);
|
||||
log.info(syncPlan.mixedStates); // for debugging
|
||||
await insertSyncPlanRecordByVault(
|
||||
const origMetadataOnRemote = await fetchMetadataFile(
|
||||
metadataFile,
|
||||
client,
|
||||
this.app.vault,
|
||||
this.settings.password
|
||||
);
|
||||
|
||||
getNotice(`5/${MAX_STEPS} Starting to fetch local meta data.`);
|
||||
this.syncStatus = "getting_local_meta";
|
||||
const local = this.app.vault.getAllLoadedFiles();
|
||||
const localHistory = await loadDeleteRenameHistoryTableByVault(
|
||||
this.db,
|
||||
syncPlan,
|
||||
this.settings.vaultRandomID
|
||||
);
|
||||
// log.info(local);
|
||||
// log.info(localHistory);
|
||||
|
||||
// The operations above are read only and kind of safe.
|
||||
getNotice(`6/${MAX_STEPS} Starting to generate sync plan.`);
|
||||
this.syncStatus = "generating_plan";
|
||||
const { plan, sortedKeys, deletions } = await getSyncPlan(
|
||||
remoteStates,
|
||||
local,
|
||||
origMetadataOnRemote.deletions,
|
||||
localHistory,
|
||||
client.serviceType,
|
||||
this.settings.password
|
||||
);
|
||||
log.info(plan.mixedStates); // for debugging
|
||||
if (triggerSource !== "dry") {
|
||||
await insertSyncPlanRecordByVault(
|
||||
this.db,
|
||||
plan,
|
||||
this.settings.vaultRandomID
|
||||
);
|
||||
}
|
||||
|
||||
// The operations above are almost read only and kind of safe.
|
||||
// The operations below begins to write or delete (!!!) something.
|
||||
|
||||
getNotice("6/7 Remotely Save Sync data exchanging!");
|
||||
if (triggerSource !== "dry") {
|
||||
getNotice(`7/${MAX_STEPS} Remotely Save Sync data exchanging!`);
|
||||
|
||||
this.syncStatus = "syncing";
|
||||
await doActualSync(
|
||||
client,
|
||||
this.db,
|
||||
this.settings.vaultRandomID,
|
||||
this.app.vault,
|
||||
syncPlan,
|
||||
this.settings.password,
|
||||
(i: number, totalCount: number, pathName: string, decision: string) =>
|
||||
self.setCurrSyncMsg(i, totalCount, pathName, decision)
|
||||
);
|
||||
this.syncStatus = "syncing";
|
||||
await doActualSync(
|
||||
client,
|
||||
this.db,
|
||||
this.settings.vaultRandomID,
|
||||
this.app.vault,
|
||||
plan,
|
||||
sortedKeys,
|
||||
metadataFile,
|
||||
origMetadataOnRemote,
|
||||
deletions,
|
||||
(key: string) => self.trash(key),
|
||||
this.settings.password,
|
||||
(i: number, totalCount: number, pathName: string, decision: string) =>
|
||||
self.setCurrSyncMsg(i, totalCount, pathName, decision)
|
||||
);
|
||||
} else {
|
||||
this.syncStatus = "syncing";
|
||||
getNotice(
|
||||
`7/${MAX_STEPS} Remotely Save real sync is skipped in dry run mode.`
|
||||
);
|
||||
}
|
||||
|
||||
getNotice("7/7 Remotely Save finish!");
|
||||
this.currSyncMsg = "";
|
||||
getNotice(`8/${MAX_STEPS} Remotely Save finish!`);
|
||||
this.syncStatus = "finish";
|
||||
this.syncStatus = "idle";
|
||||
|
||||
@@ -474,22 +513,24 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
},
|
||||
});
|
||||
|
||||
this.addCommand({
|
||||
id: "start-sync-dry-run",
|
||||
name: "start sync (dry run only)",
|
||||
icon: iconNameSyncWait,
|
||||
callback: async () => {
|
||||
this.syncRun("dry");
|
||||
},
|
||||
});
|
||||
|
||||
this.addSettingTab(new RemotelySaveSettingTab(this.app, this));
|
||||
|
||||
// this.registerDomEvent(document, "click", (evt: MouseEvent) => {
|
||||
// log.info("click", evt);
|
||||
// });
|
||||
|
||||
if (
|
||||
this.settings.autoRunEveryMilliseconds !== undefined &&
|
||||
this.settings.autoRunEveryMilliseconds !== null &&
|
||||
this.settings.autoRunEveryMilliseconds > 0
|
||||
) {
|
||||
const intervalID = window.setInterval(() => {
|
||||
this.syncRun("auto");
|
||||
}, this.settings.autoRunEveryMilliseconds);
|
||||
this.autoRunIntervalID = intervalID;
|
||||
this.registerInterval(intervalID);
|
||||
if (!this.settings.agreeToUploadExtraMetadata) {
|
||||
const syncAlgoV2Modal = new SyncAlgoV2Modal(this.app, this);
|
||||
syncAlgoV2Modal.open();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -602,10 +643,35 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
}
|
||||
}
|
||||
|
||||
async trash(x: string) {
|
||||
if (!(await this.app.vault.adapter.trashSystem(x))) {
|
||||
await this.app.vault.adapter.trashLocal(x);
|
||||
}
|
||||
}
|
||||
|
||||
async prepareDB() {
|
||||
this.db = await prepareDBs(this.settings.vaultRandomID);
|
||||
}
|
||||
|
||||
enableAutoSyncIfSet() {
|
||||
if (
|
||||
this.settings.autoRunEveryMilliseconds !== undefined &&
|
||||
this.settings.autoRunEveryMilliseconds !== null &&
|
||||
this.settings.autoRunEveryMilliseconds > 0
|
||||
) {
|
||||
const intervalID = window.setInterval(() => {
|
||||
this.syncRun("auto");
|
||||
}, this.settings.autoRunEveryMilliseconds);
|
||||
this.autoRunIntervalID = intervalID;
|
||||
this.registerInterval(intervalID);
|
||||
}
|
||||
}
|
||||
|
||||
async saveAgreeToUseNewSyncAlgorithm() {
|
||||
this.settings.agreeToUploadExtraMetadata = true;
|
||||
await this.saveSettings();
|
||||
}
|
||||
|
||||
destroyDBs() {
|
||||
/* destroyDBs(this.db); */
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user