bypassing large files
This commit is contained in:
parent
af8357ba6e
commit
17135abbd4
17
docs/sync_ignoring_large_files.md
Normal file
17
docs/sync_ignoring_large_files.md
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
# Sync Ignoring Large Files
|
||||||
|
|
||||||
|
Initially, the plugin does not ignore large files.
|
||||||
|
|
||||||
|
From the new version in May 2022, it can ignore all files with some sizes. But we need some rules to make the function compatible with existing conditions.
|
||||||
|
|
||||||
|
1. If users are using E2E password mode, then the file sizes are compared on the **encrypted sizes**, rather than the original unencripted file sizes. The reasons are: the encrypted ones are in transferations, and the encrypted sizes can be computed from unencrypted sizes but not the reverse.
|
||||||
|
|
||||||
|
2. Assuming the file A, is already synced between local device and remote service before.
|
||||||
|
|
||||||
|
- If the local size and remote size are both below the threshold, then the file can be synced normally.
|
||||||
|
- If the local size and remote size are both above the threshold, then the file will be ignored normally.
|
||||||
|
- If the local size is below the threshold, and the remote size is above the threshold, then the plugin **rejects** the sync, and throws the error to the user.
|
||||||
|
- If the local size is above the threshold, and the remote size is below the threshold, then the plugin **rejects** the sync, and throws the error to the user.
|
||||||
|
- When it somes to deletions, the same rules apply.
|
||||||
|
|
||||||
|
The main point is that, if the file sizes "cross the line", the plugin does not introduce any further trouble and just reject to work for this file.
|
||||||
@ -85,6 +85,7 @@ export interface RemotelySavePluginSettings {
|
|||||||
syncUnderscoreItems?: boolean;
|
syncUnderscoreItems?: boolean;
|
||||||
lang?: LangTypeAndAuto;
|
lang?: LangTypeAndAuto;
|
||||||
logToDB?: boolean;
|
logToDB?: boolean;
|
||||||
|
skipSizeLargerThan?: number;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @deprecated
|
* @deprecated
|
||||||
@ -122,13 +123,24 @@ type DecisionTypeForFile =
|
|||||||
| "uploadLocalToRemote" // "skipLocal && uploadLocalToRemote && cleanLocalDelHist && cleanRemoteDelHist"
|
| "uploadLocalToRemote" // "skipLocal && uploadLocalToRemote && cleanLocalDelHist && cleanRemoteDelHist"
|
||||||
| "downloadRemoteToLocal"; // "downloadRemoteToLocal && skipRemote && cleanLocalDelHist && cleanRemoteDelHist"
|
| "downloadRemoteToLocal"; // "downloadRemoteToLocal && skipRemote && cleanLocalDelHist && cleanRemoteDelHist"
|
||||||
|
|
||||||
|
type DecisionTypeForFileSize =
|
||||||
|
| "skipUploadingTooLarge"
|
||||||
|
| "skipDownloadingTooLarge"
|
||||||
|
| "skipUsingLocalDelTooLarge"
|
||||||
|
| "skipUsingRemoteDelTooLarge"
|
||||||
|
| "errorLocalTooLargeConflictRemote"
|
||||||
|
| "errorRemoteTooLargeConflictLocal";
|
||||||
|
|
||||||
type DecisionTypeForFolder =
|
type DecisionTypeForFolder =
|
||||||
| "createFolder"
|
| "createFolder"
|
||||||
| "uploadLocalDelHistToRemoteFolder"
|
| "uploadLocalDelHistToRemoteFolder"
|
||||||
| "keepRemoteDelHistFolder"
|
| "keepRemoteDelHistFolder"
|
||||||
| "skipFolder";
|
| "skipFolder";
|
||||||
|
|
||||||
export type DecisionType = DecisionTypeForFile | DecisionTypeForFolder;
|
export type DecisionType =
|
||||||
|
| DecisionTypeForFile
|
||||||
|
| DecisionTypeForFileSize
|
||||||
|
| DecisionTypeForFolder;
|
||||||
|
|
||||||
export interface FileOrFolderMixedState {
|
export interface FileOrFolderMixedState {
|
||||||
key: string;
|
key: string;
|
||||||
@ -139,7 +151,9 @@ export interface FileOrFolderMixedState {
|
|||||||
deltimeLocal?: number;
|
deltimeLocal?: number;
|
||||||
deltimeRemote?: number;
|
deltimeRemote?: number;
|
||||||
sizeLocal?: number;
|
sizeLocal?: number;
|
||||||
|
sizeLocalEnc?: number;
|
||||||
sizeRemote?: number;
|
sizeRemote?: number;
|
||||||
|
sizeRemoteEnc?: number;
|
||||||
changeRemoteMtimeUsingMapping?: boolean;
|
changeRemoteMtimeUsingMapping?: boolean;
|
||||||
changeLocalMtimeUsingMapping?: boolean;
|
changeLocalMtimeUsingMapping?: boolean;
|
||||||
decision?: DecisionType;
|
decision?: DecisionType;
|
||||||
|
|||||||
@ -26,11 +26,13 @@ const turnSyncPlanToTable = (record: string) => {
|
|||||||
"remoteEncryptedKey",
|
"remoteEncryptedKey",
|
||||||
"existLocal",
|
"existLocal",
|
||||||
"sizeLocal",
|
"sizeLocal",
|
||||||
|
"sizeLocalEnc",
|
||||||
"mtimeLocal",
|
"mtimeLocal",
|
||||||
"deltimeLocal",
|
"deltimeLocal",
|
||||||
"changeLocalMtimeUsingMapping",
|
"changeLocalMtimeUsingMapping",
|
||||||
"existRemote",
|
"existRemote",
|
||||||
"sizeRemote",
|
"sizeRemote",
|
||||||
|
"sizeRemoteEnc",
|
||||||
"mtimeRemote",
|
"mtimeRemote",
|
||||||
"deltimeRemote",
|
"deltimeRemote",
|
||||||
"changeRemoteMtimeUsingMapping",
|
"changeRemoteMtimeUsingMapping",
|
||||||
|
|||||||
@ -1 +1 @@
|
|||||||
Subproject commit b227202f0e7d012efd93e904f19e145fcc726610
|
Subproject commit ad48de922720d668477583a6b313a5eaaf4a7516
|
||||||
16
src/main.ts
16
src/main.ts
@ -10,6 +10,7 @@ import {
|
|||||||
import cloneDeep from "lodash/cloneDeep";
|
import cloneDeep from "lodash/cloneDeep";
|
||||||
import { createElement, RotateCcw, RefreshCcw, FileText } from "lucide";
|
import { createElement, RotateCcw, RefreshCcw, FileText } from "lucide";
|
||||||
import type {
|
import type {
|
||||||
|
FileOrFolderMixedState,
|
||||||
RemotelySavePluginSettings,
|
RemotelySavePluginSettings,
|
||||||
SyncTriggerSourceType,
|
SyncTriggerSourceType,
|
||||||
} from "./baseTypes";
|
} from "./baseTypes";
|
||||||
@ -64,6 +65,7 @@ import {
|
|||||||
exportVaultLoggerOutputToFiles,
|
exportVaultLoggerOutputToFiles,
|
||||||
exportVaultSyncPlansToFiles,
|
exportVaultSyncPlansToFiles,
|
||||||
} from "./debugMode";
|
} from "./debugMode";
|
||||||
|
import { SizesConflictModal } from "./syncSizesConflictNotice";
|
||||||
|
|
||||||
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
||||||
s3: DEFAULT_S3_CONFIG,
|
s3: DEFAULT_S3_CONFIG,
|
||||||
@ -82,6 +84,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
|||||||
syncUnderscoreItems: false,
|
syncUnderscoreItems: false,
|
||||||
lang: "auto",
|
lang: "auto",
|
||||||
logToDB: false,
|
logToDB: false,
|
||||||
|
skipSizeLargerThan: -1,
|
||||||
};
|
};
|
||||||
|
|
||||||
interface OAuth2Info {
|
interface OAuth2Info {
|
||||||
@ -280,7 +283,7 @@ export default class RemotelySavePlugin extends Plugin {
|
|||||||
})
|
})
|
||||||
);
|
);
|
||||||
this.syncStatus = "generating_plan";
|
this.syncStatus = "generating_plan";
|
||||||
const { plan, sortedKeys, deletions } = await getSyncPlan(
|
const { plan, sortedKeys, deletions, sizesGoWrong } = await getSyncPlan(
|
||||||
remoteStates,
|
remoteStates,
|
||||||
local,
|
local,
|
||||||
localConfigDirContents,
|
localConfigDirContents,
|
||||||
@ -292,6 +295,7 @@ export default class RemotelySavePlugin extends Plugin {
|
|||||||
this.settings.syncConfigDir,
|
this.settings.syncConfigDir,
|
||||||
this.app.vault.configDir,
|
this.app.vault.configDir,
|
||||||
this.settings.syncUnderscoreItems,
|
this.settings.syncUnderscoreItems,
|
||||||
|
this.settings.skipSizeLargerThan,
|
||||||
this.settings.password
|
this.settings.password
|
||||||
);
|
);
|
||||||
log.info(plan.mixedStates); // for debugging
|
log.info(plan.mixedStates); // for debugging
|
||||||
@ -317,10 +321,20 @@ export default class RemotelySavePlugin extends Plugin {
|
|||||||
sortedKeys,
|
sortedKeys,
|
||||||
metadataFile,
|
metadataFile,
|
||||||
origMetadataOnRemote,
|
origMetadataOnRemote,
|
||||||
|
sizesGoWrong,
|
||||||
deletions,
|
deletions,
|
||||||
(key: string) => self.trash(key),
|
(key: string) => self.trash(key),
|
||||||
this.settings.password,
|
this.settings.password,
|
||||||
this.settings.concurrency,
|
this.settings.concurrency,
|
||||||
|
(ss: FileOrFolderMixedState[]) => {
|
||||||
|
new SizesConflictModal(
|
||||||
|
self.app,
|
||||||
|
self,
|
||||||
|
this.settings.skipSizeLargerThan,
|
||||||
|
ss,
|
||||||
|
this.settings.password !== ""
|
||||||
|
).open();
|
||||||
|
},
|
||||||
(i: number, totalCount: number, pathName: string, decision: string) =>
|
(i: number, totalCount: number, pathName: string, decision: string) =>
|
||||||
self.setCurrSyncMsg(i, totalCount, pathName, decision)
|
self.setCurrSyncMsg(i, totalCount, pathName, decision)
|
||||||
);
|
);
|
||||||
|
|||||||
@ -767,6 +767,25 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const skipLargeFilesDiv = generalDiv.createEl("div");
|
||||||
|
new Setting(skipLargeFilesDiv)
|
||||||
|
.setName(t("settings_skiplargefiles"))
|
||||||
|
.setDesc(t("settings_skiplargefiles_desc"))
|
||||||
|
.addDropdown((dropdown) => {
|
||||||
|
dropdown.addOption("-1", t("settings_skiplargefiles_notset"));
|
||||||
|
|
||||||
|
const mbs = [1, 5, 10, 50, 100, 500, 1000];
|
||||||
|
for (const mb of mbs) {
|
||||||
|
dropdown.addOption(`${mb * 1000 * 1000}`, `${mb} MB`);
|
||||||
|
}
|
||||||
|
dropdown
|
||||||
|
.setValue(`${this.plugin.settings.skipSizeLargerThan}`)
|
||||||
|
.onChange(async (val) => {
|
||||||
|
this.plugin.settings.skipSizeLargerThan = parseInt(val);
|
||||||
|
await this.plugin.saveSettings();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
//////////////////////////////////////////////////
|
//////////////////////////////////////////////////
|
||||||
// below for general chooser (part 1/2)
|
// below for general chooser (part 1/2)
|
||||||
//////////////////////////////////////////////////
|
//////////////////////////////////////////////////
|
||||||
|
|||||||
291
src/sync.ts
291
src/sync.ts
@ -19,6 +19,7 @@ import {
|
|||||||
decryptBase32ToString,
|
decryptBase32ToString,
|
||||||
decryptBase64urlToString,
|
decryptBase64urlToString,
|
||||||
encryptStringToBase64url,
|
encryptStringToBase64url,
|
||||||
|
getSizeFromOrigToEnc,
|
||||||
MAGIC_ENCRYPTED_PREFIX_BASE32,
|
MAGIC_ENCRYPTED_PREFIX_BASE32,
|
||||||
MAGIC_ENCRYPTED_PREFIX_BASE64URL,
|
MAGIC_ENCRYPTED_PREFIX_BASE64URL,
|
||||||
} from "./encrypt";
|
} from "./encrypt";
|
||||||
@ -217,22 +218,29 @@ export const parseRemoteItems = async (
|
|||||||
if (backwardMapping !== undefined) {
|
if (backwardMapping !== undefined) {
|
||||||
key = backwardMapping.localKey;
|
key = backwardMapping.localKey;
|
||||||
const mtimeRemote = backwardMapping.localMtime || entry.lastModified;
|
const mtimeRemote = backwardMapping.localMtime || entry.lastModified;
|
||||||
|
|
||||||
|
// the backwardMapping.localSize is the file BEFORE encryption
|
||||||
|
// we want to split two sizes for comparation later
|
||||||
|
|
||||||
r = {
|
r = {
|
||||||
key: key,
|
key: key,
|
||||||
existRemote: true,
|
existRemote: true,
|
||||||
mtimeRemote: mtimeRemote,
|
mtimeRemote: mtimeRemote,
|
||||||
mtimeRemoteFmt: unixTimeToStr(mtimeRemote),
|
mtimeRemoteFmt: unixTimeToStr(mtimeRemote),
|
||||||
sizeRemote: backwardMapping.localSize || entry.size,
|
sizeRemote: backwardMapping.localSize,
|
||||||
|
sizeRemoteEnc: password === "" ? undefined : entry.size,
|
||||||
remoteEncryptedKey: remoteEncryptedKey,
|
remoteEncryptedKey: remoteEncryptedKey,
|
||||||
changeRemoteMtimeUsingMapping: true,
|
changeRemoteMtimeUsingMapping: true,
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
|
// do not have backwardMapping
|
||||||
r = {
|
r = {
|
||||||
key: key,
|
key: key,
|
||||||
existRemote: true,
|
existRemote: true,
|
||||||
mtimeRemote: entry.lastModified,
|
mtimeRemote: entry.lastModified,
|
||||||
mtimeRemoteFmt: unixTimeToStr(entry.lastModified),
|
mtimeRemoteFmt: unixTimeToStr(entry.lastModified),
|
||||||
sizeRemote: entry.size,
|
sizeRemote: password === "" ? entry.size : undefined,
|
||||||
|
sizeRemoteEnc: password === "" ? undefined : entry.size,
|
||||||
remoteEncryptedKey: remoteEncryptedKey,
|
remoteEncryptedKey: remoteEncryptedKey,
|
||||||
changeRemoteMtimeUsingMapping: false,
|
changeRemoteMtimeUsingMapping: false,
|
||||||
};
|
};
|
||||||
@ -305,7 +313,8 @@ const ensembleMixedStates = async (
|
|||||||
localFileHistory: FileFolderHistoryRecord[],
|
localFileHistory: FileFolderHistoryRecord[],
|
||||||
syncConfigDir: boolean,
|
syncConfigDir: boolean,
|
||||||
configDir: string,
|
configDir: string,
|
||||||
syncUnderscoreItems: boolean
|
syncUnderscoreItems: boolean,
|
||||||
|
password: string
|
||||||
) => {
|
) => {
|
||||||
const results = {} as Record<string, FileOrFolderMixedState>;
|
const results = {} as Record<string, FileOrFolderMixedState>;
|
||||||
|
|
||||||
@ -334,6 +343,8 @@ const ensembleMixedStates = async (
|
|||||||
mtimeLocal: mtimeLocal,
|
mtimeLocal: mtimeLocal,
|
||||||
mtimeLocalFmt: unixTimeToStr(mtimeLocal),
|
mtimeLocalFmt: unixTimeToStr(mtimeLocal),
|
||||||
sizeLocal: entry.stat.size,
|
sizeLocal: entry.stat.size,
|
||||||
|
sizeLocalEnc:
|
||||||
|
password === "" ? undefined : getSizeFromOrigToEnc(entry.stat.size),
|
||||||
};
|
};
|
||||||
} else if (entry instanceof TFolder) {
|
} else if (entry instanceof TFolder) {
|
||||||
key = `${entry.path}/`;
|
key = `${entry.path}/`;
|
||||||
@ -343,6 +354,7 @@ const ensembleMixedStates = async (
|
|||||||
mtimeLocal: undefined,
|
mtimeLocal: undefined,
|
||||||
mtimeLocalFmt: undefined,
|
mtimeLocalFmt: undefined,
|
||||||
sizeLocal: 0,
|
sizeLocal: 0,
|
||||||
|
sizeLocalEnc: password === "" ? undefined : getSizeFromOrigToEnc(0),
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
throw Error(`unexpected ${entry}`);
|
throw Error(`unexpected ${entry}`);
|
||||||
@ -358,6 +370,7 @@ const ensembleMixedStates = async (
|
|||||||
results[key].mtimeLocal = r.mtimeLocal;
|
results[key].mtimeLocal = r.mtimeLocal;
|
||||||
results[key].mtimeLocalFmt = r.mtimeLocalFmt;
|
results[key].mtimeLocalFmt = r.mtimeLocalFmt;
|
||||||
results[key].sizeLocal = r.sizeLocal;
|
results[key].sizeLocal = r.sizeLocal;
|
||||||
|
results[key].sizeLocalEnc = r.sizeLocalEnc;
|
||||||
} else {
|
} else {
|
||||||
results[key] = r;
|
results[key] = r;
|
||||||
results[key].existRemote = false;
|
results[key].existRemote = false;
|
||||||
@ -374,6 +387,8 @@ const ensembleMixedStates = async (
|
|||||||
mtimeLocal: mtimeLocal,
|
mtimeLocal: mtimeLocal,
|
||||||
mtimeLocalFmt: unixTimeToStr(mtimeLocal),
|
mtimeLocalFmt: unixTimeToStr(mtimeLocal),
|
||||||
sizeLocal: entry.size,
|
sizeLocal: entry.size,
|
||||||
|
sizeLocalEnc:
|
||||||
|
password === "" ? undefined : getSizeFromOrigToEnc(entry.size),
|
||||||
};
|
};
|
||||||
|
|
||||||
if (results.hasOwnProperty(key)) {
|
if (results.hasOwnProperty(key)) {
|
||||||
@ -382,6 +397,7 @@ const ensembleMixedStates = async (
|
|||||||
results[key].mtimeLocal = r.mtimeLocal;
|
results[key].mtimeLocal = r.mtimeLocal;
|
||||||
results[key].mtimeLocalFmt = r.mtimeLocalFmt;
|
results[key].mtimeLocalFmt = r.mtimeLocalFmt;
|
||||||
results[key].sizeLocal = r.sizeLocal;
|
results[key].sizeLocal = r.sizeLocal;
|
||||||
|
results[key].sizeLocalEnc = r.sizeLocalEnc;
|
||||||
} else {
|
} else {
|
||||||
results[key] = r;
|
results[key] = r;
|
||||||
results[key].existRemote = false;
|
results[key].existRemote = false;
|
||||||
@ -484,6 +500,7 @@ const ensembleMixedStates = async (
|
|||||||
const assignOperationToFileInplace = (
|
const assignOperationToFileInplace = (
|
||||||
origRecord: FileOrFolderMixedState,
|
origRecord: FileOrFolderMixedState,
|
||||||
keptFolder: Set<string>,
|
keptFolder: Set<string>,
|
||||||
|
skipSizeLargerThan: number,
|
||||||
password: string = ""
|
password: string = ""
|
||||||
) => {
|
) => {
|
||||||
let r = origRecord;
|
let r = origRecord;
|
||||||
@ -526,6 +543,18 @@ const assignOperationToFileInplace = (
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
(r.existLocal && password !== "" && r.sizeLocalEnc === undefined) ||
|
||||||
|
(r.existRemote && password !== "" && r.sizeRemoteEnc === undefined)
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
`Error: No encryption sizes: ${JSON.stringify(r, null, 2)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sizeLocalComp = password === "" ? r.sizeLocal : r.sizeLocalEnc;
|
||||||
|
const sizeRemoteComp = password === "" ? r.sizeRemote : r.sizeRemoteEnc;
|
||||||
|
|
||||||
// 1. mtimeLocal
|
// 1. mtimeLocal
|
||||||
if (r.existLocal) {
|
if (r.existLocal) {
|
||||||
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
|
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
|
||||||
@ -536,26 +565,79 @@ const assignOperationToFileInplace = (
|
|||||||
r.mtimeLocal >= deltimeLocal &&
|
r.mtimeLocal >= deltimeLocal &&
|
||||||
r.mtimeLocal >= deltimeRemote
|
r.mtimeLocal >= deltimeRemote
|
||||||
) {
|
) {
|
||||||
|
if (sizeLocalComp === undefined) {
|
||||||
|
throw new Error(
|
||||||
|
`Error: no local size but has local mtime: ${JSON.stringify(
|
||||||
|
r,
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
if (r.mtimeLocal === r.mtimeRemote) {
|
if (r.mtimeLocal === r.mtimeRemote) {
|
||||||
// mtime the same
|
// local and remote both exist and mtimes are the same
|
||||||
if (password === "") {
|
if (sizeLocalComp === sizeRemoteComp) {
|
||||||
// no password, we can also compare the sizes!
|
// do not need to consider skipSizeLargerThan in this case
|
||||||
if (r.sizeLocal === r.sizeRemote) {
|
r.decision = "skipUploading";
|
||||||
r.decision = "skipUploading";
|
r.decisionBranch = 1;
|
||||||
r.decisionBranch = 1;
|
} else {
|
||||||
} else {
|
if (skipSizeLargerThan <= 0) {
|
||||||
r.decision = "uploadLocalToRemote";
|
r.decision = "uploadLocalToRemote";
|
||||||
r.decisionBranch = 2;
|
r.decisionBranch = 2;
|
||||||
|
} else {
|
||||||
|
// limit the sizes
|
||||||
|
if (sizeLocalComp <= skipSizeLargerThan) {
|
||||||
|
if (sizeRemoteComp <= skipSizeLargerThan) {
|
||||||
|
r.decision = "uploadLocalToRemote";
|
||||||
|
r.decisionBranch = 18;
|
||||||
|
} else {
|
||||||
|
r.decision = "errorRemoteTooLargeConflictLocal";
|
||||||
|
r.decisionBranch = 19;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (sizeRemoteComp <= skipSizeLargerThan) {
|
||||||
|
r.decision = "errorLocalTooLargeConflictRemote";
|
||||||
|
r.decisionBranch = 20;
|
||||||
|
} else {
|
||||||
|
r.decision = "skipUploadingTooLarge";
|
||||||
|
r.decisionBranch = 21;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
// we have password, then the sizes are always unequal
|
|
||||||
// we can only rely on mtime
|
|
||||||
r.decision = "skipUploading";
|
|
||||||
r.decisionBranch = 3;
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
r.decision = "uploadLocalToRemote";
|
// we have local laregest mtime,
|
||||||
r.decisionBranch = 4;
|
// and the remote not existing or smaller mtime
|
||||||
|
if (skipSizeLargerThan <= 0) {
|
||||||
|
// no need to consider sizes
|
||||||
|
r.decision = "uploadLocalToRemote";
|
||||||
|
r.decisionBranch = 4;
|
||||||
|
} else {
|
||||||
|
// need to consider sizes
|
||||||
|
if (sizeLocalComp <= skipSizeLargerThan) {
|
||||||
|
if (sizeRemoteComp === undefined) {
|
||||||
|
r.decision = "uploadLocalToRemote";
|
||||||
|
r.decisionBranch = 22;
|
||||||
|
} else if (sizeRemoteComp <= skipSizeLargerThan) {
|
||||||
|
r.decision = "uploadLocalToRemote";
|
||||||
|
r.decisionBranch = 23;
|
||||||
|
} else {
|
||||||
|
r.decision = "errorRemoteTooLargeConflictLocal";
|
||||||
|
r.decisionBranch = 24;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (sizeRemoteComp === undefined) {
|
||||||
|
r.decision = "skipUploadingTooLarge";
|
||||||
|
r.decisionBranch = 25;
|
||||||
|
} else if (sizeRemoteComp <= skipSizeLargerThan) {
|
||||||
|
r.decision = "errorLocalTooLargeConflictRemote";
|
||||||
|
r.decisionBranch = 26;
|
||||||
|
} else {
|
||||||
|
r.decision = "skipUploadingTooLarge";
|
||||||
|
r.decisionBranch = 27;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
keptFolder.add(getParentFolder(r.key));
|
keptFolder.add(getParentFolder(r.key));
|
||||||
return r;
|
return r;
|
||||||
@ -572,8 +654,49 @@ const assignOperationToFileInplace = (
|
|||||||
r.mtimeRemote >= deltimeLocal &&
|
r.mtimeRemote >= deltimeLocal &&
|
||||||
r.mtimeRemote >= deltimeRemote
|
r.mtimeRemote >= deltimeRemote
|
||||||
) {
|
) {
|
||||||
r.decision = "downloadRemoteToLocal";
|
// we have remote laregest mtime,
|
||||||
r.decisionBranch = 5;
|
// and the local not existing or smaller mtime
|
||||||
|
if (sizeRemoteComp === undefined) {
|
||||||
|
throw new Error(
|
||||||
|
`Error: no remote size but has remote mtime: ${JSON.stringify(
|
||||||
|
r,
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (skipSizeLargerThan <= 0) {
|
||||||
|
// no need to consider sizes
|
||||||
|
r.decision = "downloadRemoteToLocal";
|
||||||
|
r.decisionBranch = 5;
|
||||||
|
} else {
|
||||||
|
// need to consider sizes
|
||||||
|
if (sizeRemoteComp <= skipSizeLargerThan) {
|
||||||
|
if (sizeLocalComp === undefined) {
|
||||||
|
r.decision = "downloadRemoteToLocal";
|
||||||
|
r.decisionBranch = 28;
|
||||||
|
} else if (sizeLocalComp <= skipSizeLargerThan) {
|
||||||
|
r.decision = "downloadRemoteToLocal";
|
||||||
|
r.decisionBranch = 29;
|
||||||
|
} else {
|
||||||
|
r.decision = "errorLocalTooLargeConflictRemote";
|
||||||
|
r.decisionBranch = 30;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (sizeLocalComp === undefined) {
|
||||||
|
r.decision = "skipDownloadingTooLarge";
|
||||||
|
r.decisionBranch = 31;
|
||||||
|
} else if (sizeLocalComp <= skipSizeLargerThan) {
|
||||||
|
r.decision = "errorRemoteTooLargeConflictLocal";
|
||||||
|
r.decisionBranch = 32;
|
||||||
|
} else {
|
||||||
|
r.decision = "skipDownloadingTooLarge";
|
||||||
|
r.decisionBranch = 33;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
keptFolder.add(getParentFolder(r.key));
|
keptFolder.add(getParentFolder(r.key));
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
@ -589,10 +712,44 @@ const assignOperationToFileInplace = (
|
|||||||
r.deltimeLocal >= mtimeRemote &&
|
r.deltimeLocal >= mtimeRemote &&
|
||||||
r.deltimeLocal >= deltimeRemote
|
r.deltimeLocal >= deltimeRemote
|
||||||
) {
|
) {
|
||||||
r.decision = "uploadLocalDelHistToRemote";
|
if (skipSizeLargerThan <= 0) {
|
||||||
r.decisionBranch = 6;
|
r.decision = "uploadLocalDelHistToRemote";
|
||||||
if (r.existLocal || r.existRemote) {
|
r.decisionBranch = 6;
|
||||||
// actual deletion would happen
|
if (r.existLocal || r.existRemote) {
|
||||||
|
// actual deletion would happen
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const localTooLargeToDelete =
|
||||||
|
r.existLocal && sizeLocalComp > skipSizeLargerThan;
|
||||||
|
const remoteTooLargeToDelete =
|
||||||
|
r.existRemote && sizeRemoteComp > skipSizeLargerThan;
|
||||||
|
if (localTooLargeToDelete) {
|
||||||
|
if (remoteTooLargeToDelete) {
|
||||||
|
r.decision = "skipUsingLocalDelTooLarge";
|
||||||
|
r.decisionBranch = 34;
|
||||||
|
} else {
|
||||||
|
if (r.existRemote) {
|
||||||
|
r.decision = "errorLocalTooLargeConflictRemote";
|
||||||
|
r.decisionBranch = 35;
|
||||||
|
} else {
|
||||||
|
r.decision = "skipUsingLocalDelTooLarge";
|
||||||
|
r.decisionBranch = 36;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (remoteTooLargeToDelete) {
|
||||||
|
if (r.existLocal) {
|
||||||
|
r.decision = "errorLocalTooLargeConflictRemote";
|
||||||
|
r.decisionBranch = 37;
|
||||||
|
} else {
|
||||||
|
r.decision = "skipUsingLocalDelTooLarge";
|
||||||
|
r.decisionBranch = 38;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
r.decision = "uploadLocalDelHistToRemote";
|
||||||
|
r.decisionBranch = 39;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
@ -608,10 +765,44 @@ const assignOperationToFileInplace = (
|
|||||||
r.deltimeRemote >= mtimeRemote &&
|
r.deltimeRemote >= mtimeRemote &&
|
||||||
r.deltimeRemote >= deltimeLocal
|
r.deltimeRemote >= deltimeLocal
|
||||||
) {
|
) {
|
||||||
r.decision = "keepRemoteDelHist";
|
if (skipSizeLargerThan <= 0) {
|
||||||
r.decisionBranch = 7;
|
r.decision = "keepRemoteDelHist";
|
||||||
if (r.existLocal || r.existRemote) {
|
r.decisionBranch = 7;
|
||||||
// actual deletion would happen
|
if (r.existLocal || r.existRemote) {
|
||||||
|
// actual deletion would happen
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const localTooLargeToDelete =
|
||||||
|
r.existLocal && sizeLocalComp > skipSizeLargerThan;
|
||||||
|
const remoteTooLargeToDelete =
|
||||||
|
r.existRemote && sizeRemoteComp > skipSizeLargerThan;
|
||||||
|
if (localTooLargeToDelete) {
|
||||||
|
if (remoteTooLargeToDelete) {
|
||||||
|
r.decision = "skipUsingRemoteDelTooLarge";
|
||||||
|
r.decisionBranch = 40;
|
||||||
|
} else {
|
||||||
|
if (r.existRemote) {
|
||||||
|
r.decision = "errorLocalTooLargeConflictRemote";
|
||||||
|
r.decisionBranch = 41;
|
||||||
|
} else {
|
||||||
|
r.decision = "skipUsingRemoteDelTooLarge";
|
||||||
|
r.decisionBranch = 42;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (remoteTooLargeToDelete) {
|
||||||
|
if (r.existLocal) {
|
||||||
|
r.decision = "errorLocalTooLargeConflictRemote";
|
||||||
|
r.decisionBranch = 43;
|
||||||
|
} else {
|
||||||
|
r.decision = "skipUsingRemoteDelTooLarge";
|
||||||
|
r.decisionBranch = 44;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
r.decision = "keepRemoteDelHist";
|
||||||
|
r.decisionBranch = 45;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
@ -746,6 +937,10 @@ const DELETION_DECISIONS: Set<DecisionType> = new Set([
|
|||||||
"uploadLocalDelHistToRemoteFolder",
|
"uploadLocalDelHistToRemoteFolder",
|
||||||
"keepRemoteDelHistFolder",
|
"keepRemoteDelHistFolder",
|
||||||
]);
|
]);
|
||||||
|
const SIZES_GO_WRONG_DECISIONS: Set<DecisionType> = new Set([
|
||||||
|
"errorLocalTooLargeConflictRemote",
|
||||||
|
"errorRemoteTooLargeConflictLocal",
|
||||||
|
]);
|
||||||
|
|
||||||
export const getSyncPlan = async (
|
export const getSyncPlan = async (
|
||||||
remoteStates: FileOrFolderMixedState[],
|
remoteStates: FileOrFolderMixedState[],
|
||||||
@ -759,6 +954,7 @@ export const getSyncPlan = async (
|
|||||||
syncConfigDir: boolean,
|
syncConfigDir: boolean,
|
||||||
configDir: string,
|
configDir: string,
|
||||||
syncUnderscoreItems: boolean,
|
syncUnderscoreItems: boolean,
|
||||||
|
skipSizeLargerThan: number,
|
||||||
password: string = ""
|
password: string = ""
|
||||||
) => {
|
) => {
|
||||||
const mixedStates = await ensembleMixedStates(
|
const mixedStates = await ensembleMixedStates(
|
||||||
@ -769,13 +965,15 @@ export const getSyncPlan = async (
|
|||||||
localFileHistory,
|
localFileHistory,
|
||||||
syncConfigDir,
|
syncConfigDir,
|
||||||
configDir,
|
configDir,
|
||||||
syncUnderscoreItems
|
syncUnderscoreItems,
|
||||||
|
password
|
||||||
);
|
);
|
||||||
|
|
||||||
const sortedKeys = Object.keys(mixedStates).sort(
|
const sortedKeys = Object.keys(mixedStates).sort(
|
||||||
(k1, k2) => k2.length - k1.length
|
(k1, k2) => k2.length - k1.length
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const sizesGoWrong: FileOrFolderMixedState[] = [];
|
||||||
const deletions: DeletionOnRemote[] = [];
|
const deletions: DeletionOnRemote[] = [];
|
||||||
|
|
||||||
const keptFolder = new Set<string>();
|
const keptFolder = new Set<string>();
|
||||||
@ -791,7 +989,16 @@ export const getSyncPlan = async (
|
|||||||
} else {
|
} else {
|
||||||
// get all operations of files
|
// get all operations of files
|
||||||
// and at the same time get some helper info for folders
|
// and at the same time get some helper info for folders
|
||||||
assignOperationToFileInplace(val, keptFolder, password);
|
assignOperationToFileInplace(
|
||||||
|
val,
|
||||||
|
keptFolder,
|
||||||
|
skipSizeLargerThan,
|
||||||
|
password
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (SIZES_GO_WRONG_DECISIONS.has(val.decision)) {
|
||||||
|
sizesGoWrong.push(val);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (DELETION_DECISIONS.has(val.decision)) {
|
if (DELETION_DECISIONS.has(val.decision)) {
|
||||||
@ -834,6 +1041,7 @@ export const getSyncPlan = async (
|
|||||||
plan: plan,
|
plan: plan,
|
||||||
sortedKeys: sortedKeys,
|
sortedKeys: sortedKeys,
|
||||||
deletions: deletions,
|
deletions: deletions,
|
||||||
|
sizesGoWrong: sizesGoWrong,
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -1015,6 +1223,14 @@ const dispatchOperationToActual = async (
|
|||||||
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
||||||
} else if (r.decision === "skipFolder") {
|
} else if (r.decision === "skipFolder") {
|
||||||
// do nothing!
|
// do nothing!
|
||||||
|
} else if (r.decision === "skipUploadingTooLarge") {
|
||||||
|
// do nothing!
|
||||||
|
} else if (r.decision === "skipDownloadingTooLarge") {
|
||||||
|
// do nothing!
|
||||||
|
} else if (r.decision === "skipUsingLocalDelTooLarge") {
|
||||||
|
// do nothing!
|
||||||
|
} else if (r.decision === "skipUsingRemoteDelTooLarge") {
|
||||||
|
// do nothing!
|
||||||
} else {
|
} else {
|
||||||
throw Error(`unknown decision in ${JSON.stringify(r)}`);
|
throw Error(`unknown decision in ${JSON.stringify(r)}`);
|
||||||
}
|
}
|
||||||
@ -1033,7 +1249,14 @@ const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => {
|
|||||||
const key = sortedKeys[i];
|
const key = sortedKeys[i];
|
||||||
const val: FileOrFolderMixedState = Object.assign({}, mixedStates[key]); // copy to avoid issue
|
const val: FileOrFolderMixedState = Object.assign({}, mixedStates[key]); // copy to avoid issue
|
||||||
|
|
||||||
if (val.decision === "skipFolder" || val.decision === "skipUploading") {
|
if (
|
||||||
|
val.decision === "skipFolder" ||
|
||||||
|
val.decision === "skipUploading" ||
|
||||||
|
val.decision === "skipDownloadingTooLarge" ||
|
||||||
|
val.decision === "skipUploadingTooLarge" ||
|
||||||
|
val.decision === "skipUsingLocalDelTooLarge" ||
|
||||||
|
val.decision === "skipUsingRemoteDelTooLarge"
|
||||||
|
) {
|
||||||
// pass
|
// pass
|
||||||
} else if (val.decision === "createFolder") {
|
} else if (val.decision === "createFolder") {
|
||||||
const level = atWhichLevel(key);
|
const level = atWhichLevel(key);
|
||||||
@ -1093,15 +1316,23 @@ export const doActualSync = async (
|
|||||||
sortedKeys: string[],
|
sortedKeys: string[],
|
||||||
metadataFile: FileOrFolderMixedState,
|
metadataFile: FileOrFolderMixedState,
|
||||||
origMetadata: MetadataOnRemote,
|
origMetadata: MetadataOnRemote,
|
||||||
|
sizesGoWrong: FileOrFolderMixedState[],
|
||||||
deletions: DeletionOnRemote[],
|
deletions: DeletionOnRemote[],
|
||||||
localDeleteFunc: any,
|
localDeleteFunc: any,
|
||||||
password: string = "",
|
password: string = "",
|
||||||
concurrency: number = 1,
|
concurrency: number = 1,
|
||||||
|
callbackSizesGoWrong?: any,
|
||||||
callbackSyncProcess?: any
|
callbackSyncProcess?: any
|
||||||
) => {
|
) => {
|
||||||
const mixedStates = syncPlan.mixedStates;
|
const mixedStates = syncPlan.mixedStates;
|
||||||
const totalCount = sortedKeys.length || 0;
|
const totalCount = sortedKeys.length || 0;
|
||||||
|
|
||||||
|
if (sizesGoWrong.length > 0) {
|
||||||
|
log.debug(`some sizes are larger than the threshold, abort and show hints`);
|
||||||
|
callbackSizesGoWrong(sizesGoWrong);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
log.debug(`start syncing extra data firstly`);
|
log.debug(`start syncing extra data firstly`);
|
||||||
await uploadExtraMeta(
|
await uploadExtraMeta(
|
||||||
client,
|
client,
|
||||||
|
|||||||
90
src/syncSizesConflictNotice.ts
Normal file
90
src/syncSizesConflictNotice.ts
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
import { App, Modal, Notice, PluginSettingTab, Setting } from "obsidian";
|
||||||
|
import type RemotelySavePlugin from "./main"; // unavoidable
|
||||||
|
import type { TransItemType } from "./i18n";
|
||||||
|
import type { FileOrFolderMixedState } from "./baseTypes";
|
||||||
|
|
||||||
|
import { log } from "./moreOnLog";
|
||||||
|
|
||||||
|
export class SizesConflictModal extends Modal {
|
||||||
|
readonly plugin: RemotelySavePlugin;
|
||||||
|
readonly skipSizeLargerThan: number;
|
||||||
|
readonly sizesGoWrong: FileOrFolderMixedState[];
|
||||||
|
readonly hasPassword: boolean;
|
||||||
|
constructor(
|
||||||
|
app: App,
|
||||||
|
plugin: RemotelySavePlugin,
|
||||||
|
skipSizeLargerThan: number,
|
||||||
|
sizesGoWrong: FileOrFolderMixedState[],
|
||||||
|
hasPassword: boolean
|
||||||
|
) {
|
||||||
|
super(app);
|
||||||
|
this.plugin = plugin;
|
||||||
|
this.skipSizeLargerThan = skipSizeLargerThan;
|
||||||
|
this.sizesGoWrong = sizesGoWrong;
|
||||||
|
this.hasPassword = hasPassword;
|
||||||
|
}
|
||||||
|
onOpen() {
|
||||||
|
let { contentEl } = this;
|
||||||
|
const t = (x: TransItemType, vars?: any) => {
|
||||||
|
return this.plugin.i18n.t(x, vars);
|
||||||
|
};
|
||||||
|
|
||||||
|
contentEl.createEl("h2", {
|
||||||
|
text: t("modal_sizesconflict_title"),
|
||||||
|
});
|
||||||
|
|
||||||
|
t("modal_sizesconflict_desc", {
|
||||||
|
thresholdMB: `${this.skipSizeLargerThan / 1000 / 1000}`,
|
||||||
|
thresholdBytes: `${this.skipSizeLargerThan}`,
|
||||||
|
})
|
||||||
|
.split("\n")
|
||||||
|
.forEach((val) => {
|
||||||
|
contentEl.createEl("p", { text: val });
|
||||||
|
});
|
||||||
|
|
||||||
|
const info = this.serialize();
|
||||||
|
|
||||||
|
contentEl.createDiv().createEl(
|
||||||
|
"button",
|
||||||
|
{
|
||||||
|
text: t("modal_sizesconflict_copybutton"),
|
||||||
|
},
|
||||||
|
(el) => {
|
||||||
|
el.onclick = async () => {
|
||||||
|
await navigator.clipboard.writeText(info);
|
||||||
|
new Notice(t("modal_sizesconflict_copynotice"));
|
||||||
|
};
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
contentEl.createEl("pre", {
|
||||||
|
text: info,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
serialize() {
|
||||||
|
return this.sizesGoWrong
|
||||||
|
.map((x) => {
|
||||||
|
return [
|
||||||
|
x.key,
|
||||||
|
this.hasPassword
|
||||||
|
? `encrypted name: ${x.remoteEncryptedKey}`
|
||||||
|
: undefined,
|
||||||
|
`local ${this.hasPassword ? "encrypted " : ""}bytes: ${
|
||||||
|
this.hasPassword ? x.sizeLocalEnc : x.sizeLocal
|
||||||
|
}`,
|
||||||
|
`remote ${this.hasPassword ? "encrypted " : ""}bytes: ${
|
||||||
|
this.hasPassword ? x.sizeRemoteEnc : x.sizeRemote
|
||||||
|
}`,
|
||||||
|
]
|
||||||
|
.filter((tmp) => tmp !== undefined)
|
||||||
|
.join("\n");
|
||||||
|
})
|
||||||
|
.join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
onClose() {
|
||||||
|
let { contentEl } = this;
|
||||||
|
contentEl.empty();
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user