bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bypassing large files

Browse Source
This commit is contained in:
fyears
2022-05-01 18:13:06 +08:00
parent af8357ba6e
commit 17135abbd4
8 changed files with 420 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/baseTypes.ts
+15 -1
View File
@@ -85,6 +85,7 @@ export interface RemotelySavePluginSettings {
syncUnderscoreItems?: boolean;
lang?: LangTypeAndAuto;
logToDB?: boolean;
skipSizeLargerThan?: number;
/**
* @deprecated
@@ -122,13 +123,24 @@ type DecisionTypeForFile =
| "uploadLocalToRemote" // "skipLocal && uploadLocalToRemote && cleanLocalDelHist && cleanRemoteDelHist"
| "downloadRemoteToLocal"; // "downloadRemoteToLocal && skipRemote && cleanLocalDelHist && cleanRemoteDelHist"
type DecisionTypeForFileSize =
| "skipUploadingTooLarge"
| "skipDownloadingTooLarge"
| "skipUsingLocalDelTooLarge"
| "skipUsingRemoteDelTooLarge"
| "errorLocalTooLargeConflictRemote"
| "errorRemoteTooLargeConflictLocal";
type DecisionTypeForFolder =
| "createFolder"
| "uploadLocalDelHistToRemoteFolder"
| "keepRemoteDelHistFolder"
| "skipFolder";
export type DecisionType = DecisionTypeForFile | DecisionTypeForFolder;
export type DecisionType =
| DecisionTypeForFile
| DecisionTypeForFileSize
| DecisionTypeForFolder;
export interface FileOrFolderMixedState {
key: string;
@@ -139,7 +151,9 @@ export interface FileOrFolderMixedState {
deltimeLocal?: number;
deltimeRemote?: number;
sizeLocal?: number;
sizeLocalEnc?: number;
sizeRemote?: number;
sizeRemoteEnc?: number;
changeRemoteMtimeUsingMapping?: boolean;
changeLocalMtimeUsingMapping?: boolean;
decision?: DecisionType;
src/debugMode.ts
+2
View File
@@ -26,11 +26,13 @@ const turnSyncPlanToTable = (record: string) => {
"remoteEncryptedKey",
"existLocal",
"sizeLocal",
"sizeLocalEnc",
"mtimeLocal",
"deltimeLocal",
"changeLocalMtimeUsingMapping",
"existRemote",
"sizeRemote",
"sizeRemoteEnc",
"mtimeRemote",
"deltimeRemote",
"changeRemoteMtimeUsingMapping",
src/langs
+1 -1
Submodule src/langs updated: b227202f0e...ad48de9227
src/main.ts
+15 -1
View File
@@ -10,6 +10,7 @@ import {
import cloneDeep from "lodash/cloneDeep";
import { createElement, RotateCcw, RefreshCcw, FileText } from "lucide";
import type {
FileOrFolderMixedState,
RemotelySavePluginSettings,
SyncTriggerSourceType,
} from "./baseTypes";
@@ -64,6 +65,7 @@ import {
exportVaultLoggerOutputToFiles,
exportVaultSyncPlansToFiles,
} from "./debugMode";
import { SizesConflictModal } from "./syncSizesConflictNotice";
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
s3: DEFAULT_S3_CONFIG,
@@ -82,6 +84,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
syncUnderscoreItems: false,
lang: "auto",
logToDB: false,
skipSizeLargerThan: -1,
};
interface OAuth2Info {
@@ -280,7 +283,7 @@ export default class RemotelySavePlugin extends Plugin {
})
);
this.syncStatus = "generating_plan";
const { plan, sortedKeys, deletions } = await getSyncPlan(
const { plan, sortedKeys, deletions, sizesGoWrong } = await getSyncPlan(
remoteStates,
local,
localConfigDirContents,
@@ -292,6 +295,7 @@ export default class RemotelySavePlugin extends Plugin {
this.settings.syncConfigDir,
this.app.vault.configDir,
this.settings.syncUnderscoreItems,
this.settings.skipSizeLargerThan,
this.settings.password
);
log.info(plan.mixedStates); // for debugging
@@ -317,10 +321,20 @@ export default class RemotelySavePlugin extends Plugin {
sortedKeys,
metadataFile,
origMetadataOnRemote,
sizesGoWrong,
deletions,
(key: string) => self.trash(key),
this.settings.password,
this.settings.concurrency,
(ss: FileOrFolderMixedState[]) => {
new SizesConflictModal(
self.app,
self,
this.settings.skipSizeLargerThan,
ss,
this.settings.password !== ""
).open();
},
(i: number, totalCount: number, pathName: string, decision: string) =>
self.setCurrSyncMsg(i, totalCount, pathName, decision)
);
src/settings.ts
+19
View File
@@ -767,6 +767,25 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
});
});
const skipLargeFilesDiv = generalDiv.createEl("div");
new Setting(skipLargeFilesDiv)
.setName(t("settings_skiplargefiles"))
.setDesc(t("settings_skiplargefiles_desc"))
.addDropdown((dropdown) => {
dropdown.addOption("-1", t("settings_skiplargefiles_notset"));
const mbs = [1, 5, 10, 50, 100, 500, 1000];
for (const mb of mbs) {
dropdown.addOption(`${mb * 1000 * 1000}`, `${mb} MB`);
}
dropdown
.setValue(`${this.plugin.settings.skipSizeLargerThan}`)
.onChange(async (val) => {
this.plugin.settings.skipSizeLargerThan = parseInt(val);
await this.plugin.saveSettings();
});
});
//////////////////////////////////////////////////
// below for general chooser (part 1/2)
//////////////////////////////////////////////////
src/sync.ts
+261 -30
View File
@@ -19,6 +19,7 @@ import {
decryptBase32ToString,
decryptBase64urlToString,
encryptStringToBase64url,
getSizeFromOrigToEnc,
MAGIC_ENCRYPTED_PREFIX_BASE32,
MAGIC_ENCRYPTED_PREFIX_BASE64URL,
} from "./encrypt";
@@ -217,22 +218,29 @@ export const parseRemoteItems = async (
if (backwardMapping !== undefined) {
key = backwardMapping.localKey;
const mtimeRemote = backwardMapping.localMtime || entry.lastModified;
// the backwardMapping.localSize is the file BEFORE encryption
// we want to split two sizes for comparation later
r = {
key: key,
existRemote: true,
mtimeRemote: mtimeRemote,
mtimeRemoteFmt: unixTimeToStr(mtimeRemote),
sizeRemote: backwardMapping.localSize || entry.size,
sizeRemote: backwardMapping.localSize,
sizeRemoteEnc: password === "" ? undefined : entry.size,
remoteEncryptedKey: remoteEncryptedKey,
changeRemoteMtimeUsingMapping: true,
};
} else {
// do not have backwardMapping
r = {
key: key,
existRemote: true,
mtimeRemote: entry.lastModified,
mtimeRemoteFmt: unixTimeToStr(entry.lastModified),
sizeRemote: entry.size,
sizeRemote: password === "" ? entry.size : undefined,
sizeRemoteEnc: password === "" ? undefined : entry.size,
remoteEncryptedKey: remoteEncryptedKey,
changeRemoteMtimeUsingMapping: false,
};
@@ -305,7 +313,8 @@ const ensembleMixedStates = async (
localFileHistory: FileFolderHistoryRecord[],
syncConfigDir: boolean,
configDir: string,
syncUnderscoreItems: boolean
syncUnderscoreItems: boolean,
password: string
) => {
const results = {} as Record<string, FileOrFolderMixedState>;
@@ -334,6 +343,8 @@ const ensembleMixedStates = async (
mtimeLocal: mtimeLocal,
mtimeLocalFmt: unixTimeToStr(mtimeLocal),
sizeLocal: entry.stat.size,
sizeLocalEnc:
password === "" ? undefined : getSizeFromOrigToEnc(entry.stat.size),
};
} else if (entry instanceof TFolder) {
key = `${entry.path}/`;
@@ -343,6 +354,7 @@ const ensembleMixedStates = async (
mtimeLocal: undefined,
mtimeLocalFmt: undefined,
sizeLocal: 0,
sizeLocalEnc: password === "" ? undefined : getSizeFromOrigToEnc(0),
};
} else {
throw Error(`unexpected ${entry}`);
@@ -358,6 +370,7 @@ const ensembleMixedStates = async (
results[key].mtimeLocal = r.mtimeLocal;
results[key].mtimeLocalFmt = r.mtimeLocalFmt;
results[key].sizeLocal = r.sizeLocal;
results[key].sizeLocalEnc = r.sizeLocalEnc;
} else {
results[key] = r;
results[key].existRemote = false;
@@ -374,6 +387,8 @@ const ensembleMixedStates = async (
mtimeLocal: mtimeLocal,
mtimeLocalFmt: unixTimeToStr(mtimeLocal),
sizeLocal: entry.size,
sizeLocalEnc:
password === "" ? undefined : getSizeFromOrigToEnc(entry.size),
};
if (results.hasOwnProperty(key)) {
@@ -382,6 +397,7 @@ const ensembleMixedStates = async (
results[key].mtimeLocal = r.mtimeLocal;
results[key].mtimeLocalFmt = r.mtimeLocalFmt;
results[key].sizeLocal = r.sizeLocal;
results[key].sizeLocalEnc = r.sizeLocalEnc;
} else {
results[key] = r;
results[key].existRemote = false;
@@ -484,6 +500,7 @@ const ensembleMixedStates = async (
const assignOperationToFileInplace = (
origRecord: FileOrFolderMixedState,
keptFolder: Set<string>,
skipSizeLargerThan: number,
password: string = ""
) => {
let r = origRecord;
@@ -526,6 +543,18 @@ const assignOperationToFileInplace = (
);
}
if (
(r.existLocal && password !== "" && r.sizeLocalEnc === undefined) ||
(r.existRemote && password !== "" && r.sizeRemoteEnc === undefined)
) {
throw new Error(
`Error: No encryption sizes: ${JSON.stringify(r, null, 2)}`
);
}
const sizeLocalComp = password === "" ? r.sizeLocal : r.sizeLocalEnc;
const sizeRemoteComp = password === "" ? r.sizeRemote : r.sizeRemoteEnc;
// 1. mtimeLocal
if (r.existLocal) {
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
@@ -536,26 +565,79 @@ const assignOperationToFileInplace = (
r.mtimeLocal >= deltimeLocal &&
r.mtimeLocal >= deltimeRemote
) {
if (sizeLocalComp === undefined) {
throw new Error(
`Error: no local size but has local mtime: ${JSON.stringify(
r,
null,
2
)}`
);
}
if (r.mtimeLocal === r.mtimeRemote) {
// mtime the same
if (password === "") {
// no password, we can also compare the sizes!
if (r.sizeLocal === r.sizeRemote) {
r.decision = "skipUploading";
r.decisionBranch = 1;
} else {
// local and remote both exist and mtimes are the same
if (sizeLocalComp === sizeRemoteComp) {
// do not need to consider skipSizeLargerThan in this case
r.decision = "skipUploading";
r.decisionBranch = 1;
} else {
if (skipSizeLargerThan <= 0) {
r.decision = "uploadLocalToRemote";
r.decisionBranch = 2;
} else {
// limit the sizes
if (sizeLocalComp <= skipSizeLargerThan) {
if (sizeRemoteComp <= skipSizeLargerThan) {
r.decision = "uploadLocalToRemote";
r.decisionBranch = 18;
} else {
r.decision = "errorRemoteTooLargeConflictLocal";
r.decisionBranch = 19;
}
} else {
if (sizeRemoteComp <= skipSizeLargerThan) {
r.decision = "errorLocalTooLargeConflictRemote";
r.decisionBranch = 20;
} else {
r.decision = "skipUploadingTooLarge";
r.decisionBranch = 21;
}
}
}
} else {
// we have password, then the sizes are always unequal
// we can only rely on mtime
r.decision = "skipUploading";
r.decisionBranch = 3;
}
} else {
r.decision = "uploadLocalToRemote";
r.decisionBranch = 4;
// we have local laregest mtime,
// and the remote not existing or smaller mtime
if (skipSizeLargerThan <= 0) {
// no need to consider sizes
r.decision = "uploadLocalToRemote";
r.decisionBranch = 4;
} else {
// need to consider sizes
if (sizeLocalComp <= skipSizeLargerThan) {
if (sizeRemoteComp === undefined) {
r.decision = "uploadLocalToRemote";
r.decisionBranch = 22;
} else if (sizeRemoteComp <= skipSizeLargerThan) {
r.decision = "uploadLocalToRemote";
r.decisionBranch = 23;
} else {
r.decision = "errorRemoteTooLargeConflictLocal";
r.decisionBranch = 24;
}
} else {
if (sizeRemoteComp === undefined) {
r.decision = "skipUploadingTooLarge";
r.decisionBranch = 25;
} else if (sizeRemoteComp <= skipSizeLargerThan) {
r.decision = "errorLocalTooLargeConflictRemote";
r.decisionBranch = 26;
} else {
r.decision = "skipUploadingTooLarge";
r.decisionBranch = 27;
}
}
}
}
keptFolder.add(getParentFolder(r.key));
return r;
@@ -572,8 +654,49 @@ const assignOperationToFileInplace = (
r.mtimeRemote >= deltimeLocal &&
r.mtimeRemote >= deltimeRemote
) {
r.decision = "downloadRemoteToLocal";
r.decisionBranch = 5;
// we have remote laregest mtime,
// and the local not existing or smaller mtime
if (sizeRemoteComp === undefined) {
throw new Error(
`Error: no remote size but has remote mtime: ${JSON.stringify(
r,
null,
2
)}`
);
}
if (skipSizeLargerThan <= 0) {
// no need to consider sizes
r.decision = "downloadRemoteToLocal";
r.decisionBranch = 5;
} else {
// need to consider sizes
if (sizeRemoteComp <= skipSizeLargerThan) {
if (sizeLocalComp === undefined) {
r.decision = "downloadRemoteToLocal";
r.decisionBranch = 28;
} else if (sizeLocalComp <= skipSizeLargerThan) {
r.decision = "downloadRemoteToLocal";
r.decisionBranch = 29;
} else {
r.decision = "errorLocalTooLargeConflictRemote";
r.decisionBranch = 30;
}
} else {
if (sizeLocalComp === undefined) {
r.decision = "skipDownloadingTooLarge";
r.decisionBranch = 31;
} else if (sizeLocalComp <= skipSizeLargerThan) {
r.decision = "errorRemoteTooLargeConflictLocal";
r.decisionBranch = 32;
} else {
r.decision = "skipDownloadingTooLarge";
r.decisionBranch = 33;
}
}
}
keptFolder.add(getParentFolder(r.key));
return r;
}
@@ -589,10 +712,44 @@ const assignOperationToFileInplace = (
r.deltimeLocal >= mtimeRemote &&
r.deltimeLocal >= deltimeRemote
) {
r.decision = "uploadLocalDelHistToRemote";
r.decisionBranch = 6;
if (r.existLocal || r.existRemote) {
// actual deletion would happen
if (skipSizeLargerThan <= 0) {
r.decision = "uploadLocalDelHistToRemote";
r.decisionBranch = 6;
if (r.existLocal || r.existRemote) {
// actual deletion would happen
}
} else {
const localTooLargeToDelete =
r.existLocal && sizeLocalComp > skipSizeLargerThan;
const remoteTooLargeToDelete =
r.existRemote && sizeRemoteComp > skipSizeLargerThan;
if (localTooLargeToDelete) {
if (remoteTooLargeToDelete) {
r.decision = "skipUsingLocalDelTooLarge";
r.decisionBranch = 34;
} else {
if (r.existRemote) {
r.decision = "errorLocalTooLargeConflictRemote";
r.decisionBranch = 35;
} else {
r.decision = "skipUsingLocalDelTooLarge";
r.decisionBranch = 36;
}
}
} else {
if (remoteTooLargeToDelete) {
if (r.existLocal) {
r.decision = "errorLocalTooLargeConflictRemote";
r.decisionBranch = 37;
} else {
r.decision = "skipUsingLocalDelTooLarge";
r.decisionBranch = 38;
}
} else {
r.decision = "uploadLocalDelHistToRemote";
r.decisionBranch = 39;
}
}
}
return r;
}
@@ -608,10 +765,44 @@ const assignOperationToFileInplace = (
r.deltimeRemote >= mtimeRemote &&
r.deltimeRemote >= deltimeLocal
) {
r.decision = "keepRemoteDelHist";
r.decisionBranch = 7;
if (r.existLocal || r.existRemote) {
// actual deletion would happen
if (skipSizeLargerThan <= 0) {
r.decision = "keepRemoteDelHist";
r.decisionBranch = 7;
if (r.existLocal || r.existRemote) {
// actual deletion would happen
}
} else {
const localTooLargeToDelete =
r.existLocal && sizeLocalComp > skipSizeLargerThan;
const remoteTooLargeToDelete =
r.existRemote && sizeRemoteComp > skipSizeLargerThan;
if (localTooLargeToDelete) {
if (remoteTooLargeToDelete) {
r.decision = "skipUsingRemoteDelTooLarge";
r.decisionBranch = 40;
} else {
if (r.existRemote) {
r.decision = "errorLocalTooLargeConflictRemote";
r.decisionBranch = 41;
} else {
r.decision = "skipUsingRemoteDelTooLarge";
r.decisionBranch = 42;
}
}
} else {
if (remoteTooLargeToDelete) {
if (r.existLocal) {
r.decision = "errorLocalTooLargeConflictRemote";
r.decisionBranch = 43;
} else {
r.decision = "skipUsingRemoteDelTooLarge";
r.decisionBranch = 44;
}
} else {
r.decision = "keepRemoteDelHist";
r.decisionBranch = 45;
}
}
}
return r;
}
@@ -746,6 +937,10 @@ const DELETION_DECISIONS: Set<DecisionType> = new Set([
"uploadLocalDelHistToRemoteFolder",
"keepRemoteDelHistFolder",
]);
const SIZES_GO_WRONG_DECISIONS: Set<DecisionType> = new Set([
"errorLocalTooLargeConflictRemote",
"errorRemoteTooLargeConflictLocal",
]);
export const getSyncPlan = async (
remoteStates: FileOrFolderMixedState[],
@@ -759,6 +954,7 @@ export const getSyncPlan = async (
syncConfigDir: boolean,
configDir: string,
syncUnderscoreItems: boolean,
skipSizeLargerThan: number,
password: string = ""
) => {
const mixedStates = await ensembleMixedStates(
@@ -769,13 +965,15 @@ export const getSyncPlan = async (
localFileHistory,
syncConfigDir,
configDir,
syncUnderscoreItems
syncUnderscoreItems,
password
);
const sortedKeys = Object.keys(mixedStates).sort(
(k1, k2) => k2.length - k1.length
);
const sizesGoWrong: FileOrFolderMixedState[] = [];
const deletions: DeletionOnRemote[] = [];
const keptFolder = new Set<string>();
@@ -791,7 +989,16 @@ export const getSyncPlan = async (
} else {
// get all operations of files
// and at the same time get some helper info for folders
assignOperationToFileInplace(val, keptFolder, password);
assignOperationToFileInplace(
val,
keptFolder,
skipSizeLargerThan,
password
);
}
if (SIZES_GO_WRONG_DECISIONS.has(val.decision)) {
sizesGoWrong.push(val);
}
if (DELETION_DECISIONS.has(val.decision)) {
@@ -834,6 +1041,7 @@ export const getSyncPlan = async (
plan: plan,
sortedKeys: sortedKeys,
deletions: deletions,
sizesGoWrong: sizesGoWrong,
};
};
@@ -1015,6 +1223,14 @@ const dispatchOperationToActual = async (
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
} else if (r.decision === "skipFolder") {
// do nothing!
} else if (r.decision === "skipUploadingTooLarge") {
// do nothing!
} else if (r.decision === "skipDownloadingTooLarge") {
// do nothing!
} else if (r.decision === "skipUsingLocalDelTooLarge") {
// do nothing!
} else if (r.decision === "skipUsingRemoteDelTooLarge") {
// do nothing!
} else {
throw Error(`unknown decision in ${JSON.stringify(r)}`);
}
@@ -1033,7 +1249,14 @@ const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => {
const key = sortedKeys[i];
const val: FileOrFolderMixedState = Object.assign({}, mixedStates[key]); // copy to avoid issue
if (val.decision === "skipFolder" || val.decision === "skipUploading") {
if (
val.decision === "skipFolder" ||
val.decision === "skipUploading" ||
val.decision === "skipDownloadingTooLarge" ||
val.decision === "skipUploadingTooLarge" ||
val.decision === "skipUsingLocalDelTooLarge" ||
val.decision === "skipUsingRemoteDelTooLarge"
) {
// pass
} else if (val.decision === "createFolder") {
const level = atWhichLevel(key);
@@ -1093,15 +1316,23 @@ export const doActualSync = async (
sortedKeys: string[],
metadataFile: FileOrFolderMixedState,
origMetadata: MetadataOnRemote,
sizesGoWrong: FileOrFolderMixedState[],
deletions: DeletionOnRemote[],
localDeleteFunc: any,
password: string = "",
concurrency: number = 1,
callbackSizesGoWrong?: any,
callbackSyncProcess?: any
) => {
const mixedStates = syncPlan.mixedStates;
const totalCount = sortedKeys.length || 0;
if (sizesGoWrong.length > 0) {
log.debug(`some sizes are larger than the threshold, abort and show hints`);
callbackSizesGoWrong(sizesGoWrong);
return;
}
log.debug(`start syncing extra data firstly`);
await uploadExtraMeta(
client,
src/syncSizesConflictNotice.ts
+90
View File
@@ -0,0 +1,90 @@
import { App, Modal, Notice, PluginSettingTab, Setting } from "obsidian";
import type RemotelySavePlugin from "./main"; // unavoidable
import type { TransItemType } from "./i18n";
import type { FileOrFolderMixedState } from "./baseTypes";
import { log } from "./moreOnLog";
export class SizesConflictModal extends Modal {
readonly plugin: RemotelySavePlugin;
readonly skipSizeLargerThan: number;
readonly sizesGoWrong: FileOrFolderMixedState[];
readonly hasPassword: boolean;
constructor(
app: App,
plugin: RemotelySavePlugin,
skipSizeLargerThan: number,
sizesGoWrong: FileOrFolderMixedState[],
hasPassword: boolean
) {
super(app);
this.plugin = plugin;
this.skipSizeLargerThan = skipSizeLargerThan;
this.sizesGoWrong = sizesGoWrong;
this.hasPassword = hasPassword;
}
onOpen() {
let { contentEl } = this;
const t = (x: TransItemType, vars?: any) => {
return this.plugin.i18n.t(x, vars);
};
contentEl.createEl("h2", {
text: t("modal_sizesconflict_title"),
});
t("modal_sizesconflict_desc", {
thresholdMB: `${this.skipSizeLargerThan / 1000 / 1000}`,
thresholdBytes: `${this.skipSizeLargerThan}`,
})
.split("\n")
.forEach((val) => {
contentEl.createEl("p", { text: val });
});
const info = this.serialize();
contentEl.createDiv().createEl(
"button",
{
text: t("modal_sizesconflict_copybutton"),
},
(el) => {
el.onclick = async () => {
await navigator.clipboard.writeText(info);
new Notice(t("modal_sizesconflict_copynotice"));
};
}
);
contentEl.createEl("pre", {
text: info,
});
}
serialize() {
return this.sizesGoWrong
.map((x) => {
return [
x.key,
this.hasPassword
? `encrypted name: ${x.remoteEncryptedKey}`
: undefined,
`local ${this.hasPassword ? "encrypted " : ""}bytes: ${
this.hasPassword ? x.sizeLocalEnc : x.sizeLocal
}`,
`remote ${this.hasPassword ? "encrypted " : ""}bytes: ${
this.hasPassword ? x.sizeRemoteEnc : x.sizeRemote
}`,
]
.filter((tmp) => tmp !== undefined)
.join("\n");
})
.join("\n\n");
}
onClose() {
let { contentEl } = this;
contentEl.empty();
}
}