bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

finally buildable

Browse Source
This commit is contained in:
fyears 2024-02-24 11:31:23 +08:00
parent e341c4f780
commit 725b12832c
9 changed files with 129 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/baseTypes.ts
View File

@ -104,6 +104,8 @@ export interface RemotelySavePluginSettings {
ignorePaths?: string[];
enableStatusBarInfo?: boolean;
deleteToWhere?: "system" | "obsidian";
conflictAction?: ConflictActionType;
howToCleanEmptyFolder?: EmptyFolderCleanType;
/**
* @deprecated

4
src/langs/en.json
View File

@ -12,8 +12,8 @@
"syncrun_step2": "2/8 Starting to fetch remote meta data.",
"syncrun_step3": "3/8 Checking password correct or not.",
"syncrun_passworderr": "Something goes wrong while checking password.",
"syncrun_step4": "4/8 Trying to fetch extra meta data from remote.",
"syncrun_step5": "5/8 Starting to fetch local meta data.",
"syncrun_step4": "4/8 Starting to fetch local meta data.",
"syncrun_step5": "5/8 Starting to fetch local prev sync data.",
"syncrun_step6": "6/8 Starting to generate sync plan.",
"syncrun_step7": "7/8 Remotely Save Sync data exchanging!",
"syncrun_step7skip": "7/8 Remotely Save real sync is skipped in dry run mode.",

4
src/langs/zh_cn.json
View File

@ -12,8 +12,8 @@
"syncrun_step2": "2/8 正在获取远端的元数据。",
"syncrun_step3": "3/8 正在检查密码正确与否。",
"syncrun_passworderr": "检查密码时候出错。",
"syncrun_step4": "4/8 正在获取远端的额外的元数据。",
"syncrun_step5": "5/8 正在获取本地的元数据。",
"syncrun_step4": "4/8 正在获取本地的元数据。",
"syncrun_step5": "5/8 正在获取本地上一次同步的元数据。",
"syncrun_step6": "6/8 正在生成同步计划。",
"syncrun_step7": "7/8 Remotely Save 开始发生数据交换！",
"syncrun_step7skip": "7/8 Remotely Save 在空跑模式，跳过实际数据交换步骤。",

4
src/langs/zh_tw.json
View File

@ -12,8 +12,8 @@
"syncrun_step2": "2/8 正在獲取遠端的元資料。",
"syncrun_step3": "3/8 正在檢查密碼正確與否。",
"syncrun_passworderr": "檢查密碼時候出錯。",
"syncrun_step4": "4/8 正在獲取遠端的額外的元資料。",
"syncrun_step5": "5/8 正在獲取本地的元資料。",
"syncrun_step4": "4/8 正在獲取本地的元資料。",
"syncrun_step5": "5/8 正在獲取本地上一次同步的元資料。",
"syncrun_step6": "6/8 正在生成同步計劃。",
"syncrun_step7": "7/8 Remotely Save 開始發生資料交換！",
"syncrun_step7skip": "7/8 Remotely Save 在空跑模式，跳過實際資料交換步驟。",

33
src/localdb.ts
View File

@ -329,16 +329,18 @@ export const clearAllSyncMetaMapping = async (db: InternalDBs) => {
export const insertSyncPlanRecordByVault = async (
db: InternalDBs,
syncPlan: SyncPlanType,
vaultRandomID: string
vaultRandomID: string,
remoteType: SUPPORTED_SERVICES_TYPE
) => {
const now = Date.now();
const record = {
ts: syncPlan.ts,
tsFmt: syncPlan.tsFmt,
ts: now,
tsFmt: unixTimeToStr(now),
vaultRandomID: vaultRandomID,
remoteType: syncPlan.remoteType,
remoteType: remoteType,
syncPlan: JSON.stringify(syncPlan /* directly stringify */, null, 2),
} as SyncPlanRecord;
await db.syncPlansTbl.setItem(`${vaultRandomID}\t${syncPlan.ts}`, record);
await db.syncPlansTbl.setItem(`${vaultRandomID}\t${now}`, record);
};
export const clearAllSyncPlanRecords = async (db: InternalDBs) => {
@ -406,6 +408,23 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => {
await Promise.all(ps);
};
export const getAllPrevSyncRecordsByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const keys = await db.prevSyncRecordsTbl.keys();
const res: Entity[] = [];
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
const val: Entity | null = await db.prevSyncRecordsTbl.getItem(key);
if (val !== null) {
res.push(val);
}
}
}
return res;
};
export const upsertPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string,
@ -442,7 +461,7 @@ export const clearAllLoggerOutputRecords = async (db: InternalDBs) => {
log.debug(`successfully clearAllLoggerOutputRecords`);
};
export const upsertLastSuccessSyncByVault = async (
export const upsertLastSuccessSyncTimeByVault = async (
db: InternalDBs,
vaultRandomID: string,
millis: number
@ -453,7 +472,7 @@ export const upsertLastSuccessSyncByVault = async (
);
};
export const getLastSuccessSyncByVault = async (
export const getLastSuccessSyncTimeByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {

123
src/main.ts
View File

@ -7,8 +7,6 @@ import {
setIcon,
FileSystemAdapter,
Platform,
TFile,
TFolder,
requestUrl,
requireApiVersion,
} from "obsidian";
@ -23,7 +21,6 @@ import {
COMMAND_CALLBACK_ONEDRIVE,
COMMAND_CALLBACK_DROPBOX,
COMMAND_URI,
REMOTELY_SAVE_VERSION_2024PREPARE,
API_VER_ENSURE_REQURL_OK,
} from "./baseTypes";
import { importQrCodeUri } from "./importExport";
@ -32,10 +29,11 @@ import {
prepareDBs,
InternalDBs,
clearExpiredSyncPlanRecords,
upsertLastSuccessSyncByVault,
getLastSuccessSyncByVault,
upsertPluginVersionByVault,
clearAllLoggerOutputRecords,
upsertLastSuccessSyncTimeByVault,
getLastSuccessSyncTimeByVault,
getAllPrevSyncRecordsByVault,
} from "./localdb";
import { RemoteClient } from "./remote";
import {
@ -53,20 +51,22 @@ import {
import { DEFAULT_S3_CONFIG } from "./remoteForS3";
import { DEFAULT_WEBDAV_CONFIG } from "./remoteForWebdav";
import { RemotelySaveSettingTab } from "./settings";
import { parseRemoteItems, SyncStatusType } from "./sync";
import { doActualSync, getSyncPlan, isPasswordOk } from "./sync";
import {
doActualSync,
ensembleMixedEnties,
getSyncPlanInplace,
isPasswordOk,
SyncStatusType,
} from "./sync";
import { messyConfigToNormal, normalConfigToMessy } from "./configPersist";
import { getLocalEntityList } from "./local";
import { I18n } from "./i18n";
import type { LangType, LangTypeAndAuto, TransItemType } from "./i18n";
import { DeletionOnRemote, MetadataOnRemote } from "./metadataOnRemote";
import { SyncAlgoV3Modal } from "./syncAlgoV3Notice";
import { applyLogWriterInplace, log } from "./moreOnLog";
import AggregateError from "aggregate-error";
import { exportVaultSyncPlansToFiles } from "./debugMode";
import { SizesConflictModal } from "./syncSizesConflictNotice";
import { compareVersion } from "./misc";
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
@ -91,6 +91,9 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
ignorePaths: [],
enableStatusBarInfo: true,
deleteToWhere: "system",
agreeToUseSyncV3: false,
conflictAction: "keep_newer",
howToCleanEmptyFolder: "skip",
};
interface OAuth2Info {
@ -259,33 +262,26 @@ export default class RemotelySavePlugin extends Plugin {
} else {
getNotice(t("syncrun_step4"));
}
this.syncStatus = "getting_remote_extra_meta";
const { remoteStates, metadataFile } = await parseRemoteItems(
remoteEntityList,
this.db,
this.vaultRandomID,
client.serviceType,
this.settings.password
this.syncStatus = "getting_local_meta";
const localEntityList = await getLocalEntityList(
this.app.vault,
this.settings.syncConfigDir ?? false,
this.app.vault.configDir,
this.manifest.id
);
// log.info(localEntityList);
if (this.settings.currLogLevel === "info") {
// pass
} else {
getNotice(t("syncrun_step5"));
}
this.syncStatus = "getting_local_meta";
const local = this.app.vault.getAllLoadedFiles();
let localConfigDirContents: ObsConfigDirFileType[] | undefined =
undefined;
if (this.settings.syncConfigDir) {
localConfigDirContents = await listFilesInObsFolder(
this.app.vault.configDir,
this.app.vault,
this.manifest.id
this.syncStatus = "getting_local_prev_sync";
const prevSyncEntityList = await getAllPrevSyncRecordsByVault(
this.db,
this.vaultRandomID
);
}
// log.info(local);
// log.info(localHistory);
// log.info(prevSyncEntityList);
if (this.settings.currLogLevel === "info") {
// pass
@ -293,24 +289,29 @@ export default class RemotelySavePlugin extends Plugin {
getNotice(t("syncrun_step6"));
}
this.syncStatus = "generating_plan";
const { plan, sortedKeys, deletions, sizesGoWrong } = await getSyncPlan(
remoteStates,
local,
localConfigDirContents,
origMetadataOnRemote.deletions,
localHistory,
client.serviceType,
triggerSource,
this.app.vault,
let mixedEntityMappings = await ensembleMixedEnties(
localEntityList,
prevSyncEntityList,
remoteEntityList,
this.settings.syncConfigDir ?? false,
this.app.vault.configDir,
this.settings.syncUnderscoreItems ?? false,
this.settings.skipSizeLargerThan ?? -1,
this.settings.ignorePaths ?? [],
this.settings.password
);
log.info(plan.mixedStates); // for debugging
await insertSyncPlanRecordByVault(this.db, plan, this.vaultRandomID);
mixedEntityMappings = await getSyncPlanInplace(
mixedEntityMappings,
this.settings.howToCleanEmptyFolder ?? "skip",
this.settings.skipSizeLargerThan ?? -1,
this.settings.conflictAction ?? "keep_newer"
);
log.info(mixedEntityMappings); // for debugging
await insertSyncPlanRecordByVault(
this.db,
mixedEntityMappings,
this.vaultRandomID,
client.serviceType
);
// The operations above are almost read only and kind of safe.
// The operations below begins to write or delete (!!!) something.
@ -322,23 +323,27 @@ export default class RemotelySavePlugin extends Plugin {
getNotice(t("syncrun_step7"));
}
this.syncStatus = "syncing";
await doActualSync(
mixedEntityMappings,
client,
this.db,
this.vaultRandomID,
this.app.vault,
plan,
sortedKeys,
metadataFile,
sizesGoWrong,
deletions,
(key: string) => self.trash(key),
this.settings.password,
this.settings.concurrency,
(i: number, totalCount: number, pathName: string, decision: string) =>
self.setCurrSyncMsg(i, totalCount, pathName, decision)
this.settings.concurrency ?? 5,
(key: string) => self.trash(key),
(
realCounter: number,
realTotalCount: number,
pathName: string,
decision: string
) =>
self.setCurrSyncMsg(
realCounter,
realTotalCount,
pathName,
decision
),
this.db
);
} else {
this.syncStatus = "syncing";
@ -359,7 +364,7 @@ export default class RemotelySavePlugin extends Plugin {
this.syncStatus = "idle";
const lastSuccessSyncMillis = Date.now();
await upsertLastSuccessSyncByVault(
await upsertLastSuccessSyncTimeByVault(
this.db,
this.vaultRandomID,
lastSuccessSyncMillis
@ -695,13 +700,13 @@ export default class RemotelySavePlugin extends Plugin {
this.statusBarElement.setAttribute("data-tooltip-position", "top");
this.updateLastSuccessSyncMsg(
await getLastSuccessSyncByVault(this.db, this.vaultRandomID)
await getLastSuccessSyncTimeByVault(this.db, this.vaultRandomID)
);
// update statusbar text every 30 seconds
this.registerInterval(
window.setInterval(async () => {
this.updateLastSuccessSyncMsg(
await getLastSuccessSyncByVault(this.db, this.vaultRandomID)
await getLastSuccessSyncTimeByVault(this.db, this.vaultRandomID)
);
}, 1000 * 30)
);
@ -849,6 +854,12 @@ export default class RemotelySavePlugin extends Plugin {
if (this.settings.agreeToUseSyncV3 === undefined) {
this.settings.agreeToUseSyncV3 = false;
}
if (this.settings.conflictAction === undefined) {
this.settings.conflictAction = "keep_newer";
}
if (this.settings.howToCleanEmptyFolder === undefined) {
this.settings.howToCleanEmptyFolder = "skip";
}
await this.saveSettings();
}

10
src/obsFolderLister.ts
View File

@ -4,7 +4,7 @@ import type { Entity, MixedEntity } from "./baseTypes";
import { Queue } from "@fyears/tsqueue";
import chunk from "lodash/chunk";
import flatten from "lodash/flatten";
import { statFix, isFolderToSkip } from "./misc";
import { statFix, isSpecialFolderNameToSkip } from "./misc";
const isPluginDirItself = (x: string, pluginId: string) => {
return (
@ -96,7 +96,9 @@ export const listFilesInObsFolder = async (
const isInsideSelfPlugin = isPluginDirItself(iter.itself.key, pluginId);
if (iter.children !== undefined) {
for (const iter2 of iter.children.folders) {
if (isFolderToSkip(iter2, ["workspace", "workspace.json"])) {
if (
isSpecialFolderNameToSkip(iter2, ["workspace", "workspace.json"])
) {
continue;
}
if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) {
@ -106,7 +108,9 @@ export const listFilesInObsFolder = async (
q.push(iter2);
}
for (const iter2 of iter.children.files) {
if (isFolderToSkip(iter2, ["workspace", "workspace.json"])) {
if (
isSpecialFolderNameToSkip(iter2, ["workspace", "workspace.json"])
) {
continue;
}
if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) {

4
src/settings.ts
View File

@ -26,7 +26,7 @@ import {
clearAllSyncMetaMapping,
clearAllSyncPlanRecords,
destroyDBs,
upsertLastSuccessSyncByVault,
upsertLastSuccessSyncTimeByVault,
} from "./localdb";
import type RemotelySavePlugin from "./main"; // unavoidable
import { RemoteClient } from "./remote";
@ -1866,7 +1866,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
button.setButtonText(t("settings_resetstatusbar_button"));
button.onClick(async () => {
// reset last sync time
await upsertLastSuccessSyncByVault(
await upsertLastSuccessSyncTimeByVault(
this.plugin.db,
this.plugin.vaultRandomID,
-1

22
src/sync.ts
View File

@ -39,6 +39,18 @@ import {
upsertPrevSyncRecordByVault,
} from "./localdb";
export type SyncStatusType =
| "idle"
| "preparing"
| "getting_remote_files_list"
| "getting_local_meta"
| "getting_local_prev_sync"
| "checking_password"
| "generating_plan"
| "syncing"
| "cleaning"
| "finish";
export interface PasswordCheckType {
ok: boolean;
reason:
@ -286,7 +298,9 @@ const encryptLocalEntityInplace = async (
return local;
};
const ensembleMixedEnties = async (
export type SyncPlanType = Record<string, MixedEntity>;
export const ensembleMixedEnties = async (
localEntityList: Entity[],
prevSyncEntityList: Entity[],
remoteEntityList: Entity[],
@ -296,8 +310,8 @@ const ensembleMixedEnties = async (
syncUnderscoreItems: boolean,
ignorePaths: string[],
password: string
): Promise<Record<string, MixedEntity>> => {
const finalMappings: Record<string, MixedEntity> = {};
): Promise<SyncPlanType> => {
const finalMappings: SyncPlanType = {};
// remote has to be first
for (const remote of remoteEntityList) {
@ -863,7 +877,7 @@ const dispatchOperationToActualV3 = async (
}
};
export const doActualSyncV3 = async (
export const doActualSync = async (
mixedEntityMappings: Record<string, MixedEntity>,
client: RemoteClient,
vaultRandomID: string,