remotely-save/src/sync.ts
fyears df2d98272c new sync algo, squashed commit of the following:
commit 692bb794aea5609b9e9abf5228620f4479e50983
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Feb 27 17:52:43 2022 +0800

    bump to 0.3.0

commit 77335412ad2da2b5bd1ed5075061a5af006e3c36
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Feb 27 17:50:57 2022 +0800

    change titles for minimal intrusive design

commit 2812adebb84344d384749a62acb63fd0c6fd509d
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Feb 27 17:30:53 2022 +0800

    remove syncv1

commit 22fc24a76c9851740bbc7c0177d1cb2526e15d8b
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Feb 27 17:30:27 2022 +0800

    full notice to any one

commit d56ea24a78f6dc1fbea2740011ee1cea9c403d4c
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sat Feb 26 23:11:14 2022 +0800

    fix test

commit 759f82593bbfb9b49079cfd80dbadbbafc0287e5
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sat Feb 26 21:59:25 2022 +0800

    obfuscate metadata on remote

commit 9b6bf05288af0e52d0f02468e5ac8757f4f896df
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sat Feb 26 21:33:52 2022 +0800

    avoid re-uploading if meta not changed

commit 03be1453764e48e99207f44467ee4d5801072ed8
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sat Feb 26 00:35:52 2022 +0800

    add password condition

commit 7f899f7c2572df3e2a35e16cbdaab94db113f366
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sat Feb 26 00:22:58 2022 +0800

    add decision branch for easier debugging

commit cf4071bf3156356ae6ec3a9cb187c2cb541d1b17
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Fri Feb 25 23:40:52 2022 +0800

    change folder error

commit 964493dd998699a1d53018a201637bda192c4baa
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Fri Feb 25 23:09:44 2022 +0800

    finnaly remote remove should be working

commit 2888e65452f9c0e1dde6005f012c3ee0a6313c3f
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Fri Feb 25 01:18:15 2022 +0800

    optimize comparation

commit 024936951d6180b1296c2a5d56c5bf5d468e9ae7
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Fri Feb 25 01:14:44 2022 +0800

    allow uploading extra meta

commit 007006701d536da2b4b46389941980579bbc4e67
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Thu Feb 24 01:08:58 2022 +0800

    add logic to fetch extra meta

commit c9d3a05ca1bf45c06f22233124670e5e45b5f5f1
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Thu Feb 24 00:29:16 2022 +0800

    another way to deal with trash

commit 82d455f8bf60f7bac8eb4e299a2ca44c331a6d7f
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Thu Feb 24 00:28:51 2022 +0800

    add return buffer for downloading

commit b8e6b79bc078def2854bc73578b7f520cc39ab34
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Wed Feb 23 00:16:06 2022 +0800

    half way to actual sync

commit 90cceb1411b46af9741f2caa3ab8beafaf69c1b2
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Tue Feb 22 23:36:09 2022 +0800

    cleaner way to remember the folder being kept

commit c1afb763cc4e0f7905c83e0a8eb20f8ed969a279
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Tue Feb 22 00:03:21 2022 +0800

    simplified way to get plans for sync algo v2

commit 5c5ecce39eb3854900f1f5b79c7beb189e78a6a7
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Mon Feb 21 23:13:58 2022 +0800

    archive the old sync algo v1 doc

commit 75cdfa1ee9834600f83ded6672b102de2c5f9616
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Mon Feb 21 23:13:14 2022 +0800

    simplify sync algo v2

commit dc9275835d961de07efcbaa81657e4745242e72a
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Mon Feb 21 22:58:57 2022 +0800

    add way to skip recording removing

commit f9712ef96021dfed4ae33e6c649f78e940b7e530
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Mon Feb 21 09:38:09 2022 +0800

    fix sync

commit 9007dcf22ef317dde36ac4f1dd589d05cc8d5cf6
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Mon Feb 21 00:54:21 2022 +0800

    fix assignment

commit 77abee6ad443b62353ed3913e0945ea7d1314f87
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Mon Feb 21 00:28:43 2022 +0800

    draft of sync v2

commit a0c26238bf60692e095cfd8527abf937255b56d4
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Mon Feb 21 00:23:19 2022 +0800

    how to deal with folders

commit c10f92a7e8d3c4a4f4c585e39e0abad1a5376c02
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Feb 20 23:39:16 2022 +0800

    helper func to get parents

commit f903c98b3b7b9d1e785d04b9fc0cfcafdc6e5661
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Feb 20 23:31:21 2022 +0800

    add optional ending slash to getFolderLevels

commit 2d67c9b2b941e676588fa43ab289fab79f567e5e
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Feb 20 14:44:44 2022 +0800

    update sync algo v2

commit 491ed1bb85759df2411706fd02d740acb5598ce6
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Feb 20 14:34:51 2022 +0800

    dry run mode

commit dfd588dcef512ba7dfe760008bcf97138b97e339
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sat Feb 19 23:14:32 2022 +0800

    prettier

commit eae580f882a045ae9df799b816e68c3500704131
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sat Feb 19 23:12:29 2022 +0800

    draft design for sync algo v2
2022-02-27 17:54:31 +08:00

907 lines
24 KiB
TypeScript

import { TAbstractFile, TFile, TFolder, Vault } from "obsidian";
import type {
RemoteItem,
SUPPORTED_SERVICES_TYPE,
DecisionType,
FileOrFolderMixedState,
} from "./baseTypes";
import {
decryptBase32ToString,
decryptBase64urlToString,
encryptStringToBase64url,
MAGIC_ENCRYPTED_PREFIX_BASE32,
MAGIC_ENCRYPTED_PREFIX_BASE64URL,
} from "./encrypt";
import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
import {
clearDeleteRenameHistoryOfKeyAndVault,
getSyncMetaMappingByRemoteKeyAndVault,
upsertSyncMetaMappingDataByVault,
} from "./localdb";
import {
isHiddenPath,
isVaildText,
mkdirpInVault,
getFolderLevels,
getParentFolder,
} from "./misc";
import { RemoteClient } from "./remote";
import {
MetadataOnRemote,
DeletionOnRemote,
serializeMetadataOnRemote,
deserializeMetadataOnRemote,
DEFAULT_FILE_NAME_FOR_METADATAONREMOTE,
DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2,
isEqualMetadataOnRemote,
} from "./metadataOnRemote";
import * as origLog from "loglevel";
import { padEnd } from "lodash";
const log = origLog.getLogger("rs-default");
export type SyncStatusType =
| "idle"
| "preparing"
| "getting_remote_files_list"
| "getting_remote_extra_meta"
| "getting_local_meta"
| "checking_password"
| "generating_plan"
| "syncing"
| "cleaning"
| "finish";
export interface SyncPlanType {
ts: number;
remoteType: SUPPORTED_SERVICES_TYPE;
mixedStates: Record<string, FileOrFolderMixedState>;
}
export interface PasswordCheckType {
ok: boolean;
reason:
| "ok"
| "empty_remote"
| "remote_encrypted_local_no_password"
| "password_matched"
| "password_not_matched"
| "invalid_text_after_decryption"
| "remote_not_encrypted_local_has_password"
| "no_password_both_sides";
}
export const isPasswordOk = async (
remote: RemoteItem[],
password: string = ""
) => {
if (remote === undefined || remote.length === 0) {
// remote empty
return {
ok: true,
reason: "empty_remote",
} as PasswordCheckType;
}
const santyCheckKey = remote[0].key;
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
// this is encrypted using old base32!
// try to decrypt it using the provided password.
if (password === "") {
return {
ok: false,
reason: "remote_encrypted_local_no_password",
} as PasswordCheckType;
}
try {
const res = await decryptBase32ToString(santyCheckKey, password);
// additional test
// because iOS Safari bypasses decryption with wrong password!
if (isVaildText(res)) {
return {
ok: true,
reason: "password_matched",
} as PasswordCheckType;
} else {
return {
ok: false,
reason: "invalid_text_after_decryption",
} as PasswordCheckType;
}
} catch (error) {
return {
ok: false,
reason: "password_not_matched",
} as PasswordCheckType;
}
}
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) {
// this is encrypted using new base64url!
// try to decrypt it using the provided password.
if (password === "") {
return {
ok: false,
reason: "remote_encrypted_local_no_password",
} as PasswordCheckType;
}
try {
const res = await decryptBase64urlToString(santyCheckKey, password);
// additional test
// because iOS Safari bypasses decryption with wrong password!
if (isVaildText(res)) {
return {
ok: true,
reason: "password_matched",
} as PasswordCheckType;
} else {
return {
ok: false,
reason: "invalid_text_after_decryption",
} as PasswordCheckType;
}
} catch (error) {
return {
ok: false,
reason: "password_not_matched",
} as PasswordCheckType;
}
} else {
// it is not encrypted!
if (password !== "") {
return {
ok: false,
reason: "remote_not_encrypted_local_has_password",
} as PasswordCheckType;
}
return {
ok: true,
reason: "no_password_both_sides",
} as PasswordCheckType;
}
};
export const parseRemoteItems = async (
remote: RemoteItem[],
db: InternalDBs,
vaultRandomID: string,
remoteType: SUPPORTED_SERVICES_TYPE,
password: string = ""
) => {
const remoteStates = [] as FileOrFolderMixedState[];
let metadataFile: FileOrFolderMixedState = undefined;
if (remote === undefined) {
return {
remoteStates: remoteStates,
metadataFile: metadataFile,
};
}
for (const entry of remote) {
const remoteEncryptedKey = entry.key;
let key = remoteEncryptedKey;
if (password !== "") {
if (remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
key = await decryptBase32ToString(remoteEncryptedKey, password);
} else if (
remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)
) {
key = await decryptBase64urlToString(remoteEncryptedKey, password);
} else {
throw Error(`unexpected key=${remoteEncryptedKey}`);
}
}
const backwardMapping = await getSyncMetaMappingByRemoteKeyAndVault(
remoteType,
db,
key,
entry.lastModified,
entry.etag,
vaultRandomID
);
let r = {} as FileOrFolderMixedState;
if (backwardMapping !== undefined) {
key = backwardMapping.localKey;
r = {
key: key,
existRemote: true,
mtimeRemote: backwardMapping.localMtime || entry.lastModified,
sizeRemote: backwardMapping.localSize || entry.size,
remoteEncryptedKey: remoteEncryptedKey,
changeMtimeUsingMapping: true,
};
} else {
r = {
key: key,
existRemote: true,
mtimeRemote: entry.lastModified,
sizeRemote: entry.size,
remoteEncryptedKey: remoteEncryptedKey,
changeMtimeUsingMapping: false,
};
}
if (r.key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE) {
metadataFile = Object.assign({}, r);
}
if (r.key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2) {
throw Error(
`A reserved file name ${r.key} has been found. You may upgrade the plugin to latest version to try to deal with it.`
);
}
remoteStates.push(r);
}
return {
remoteStates: remoteStates,
metadataFile: metadataFile,
};
};
export const fetchMetadataFile = async (
metadataFile: FileOrFolderMixedState,
client: RemoteClient,
vault: Vault,
password: string = ""
) => {
if (metadataFile === undefined) {
log.debug("no metadata file, so no fetch");
return {
deletions: [],
} as MetadataOnRemote;
}
const buf = await client.downloadFromRemote(
metadataFile.key,
vault,
metadataFile.mtimeRemote,
password,
metadataFile.remoteEncryptedKey,
true
);
const metadata = deserializeMetadataOnRemote(buf);
return metadata;
};
const ensembleMixedStates = async (
remoteStates: FileOrFolderMixedState[],
local: TAbstractFile[],
remoteDeleteHistory: DeletionOnRemote[],
localDeleteHistory: FileFolderHistoryRecord[]
) => {
const results = {} as Record<string, FileOrFolderMixedState>;
for (const r of remoteStates) {
const key = r.key;
if (isHiddenPath(key)) {
continue;
}
results[key] = r;
results[key].existLocal = false;
}
for (const entry of local) {
let r = {} as FileOrFolderMixedState;
let key = entry.path;
if (entry.path === "/") {
// ignore
continue;
} else if (entry instanceof TFile) {
r = {
key: entry.path,
existLocal: true,
mtimeLocal: entry.stat.mtime,
sizeLocal: entry.stat.size,
};
} else if (entry instanceof TFolder) {
key = `${entry.path}/`;
r = {
key: key,
existLocal: true,
mtimeLocal: undefined,
sizeLocal: 0,
};
} else {
throw Error(`unexpected ${entry}`);
}
if (isHiddenPath(key)) {
continue;
}
if (results.hasOwnProperty(key)) {
results[key].key = r.key;
results[key].existLocal = r.existLocal;
results[key].mtimeLocal = r.mtimeLocal;
results[key].sizeLocal = r.sizeLocal;
} else {
results[key] = r;
results[key].existRemote = false;
}
}
for (const entry of remoteDeleteHistory) {
const key = entry.key;
const r = {
key: key,
deltimeRemote: entry.actionWhen,
} as FileOrFolderMixedState;
if (results.hasOwnProperty(key)) {
results[key].key = r.key;
results[key].deltimeRemote = r.deltimeRemote;
} else {
results[key] = r;
results[key].existLocal = false;
results[key].existRemote = false;
}
}
for (const entry of localDeleteHistory) {
let key = entry.key;
if (entry.keyType === "folder") {
if (!entry.key.endsWith("/")) {
key = `${entry.key}/`;
}
} else if (entry.keyType === "file") {
// pass
} else {
throw Error(`unexpected ${entry}`);
}
const r = {
key: key,
deltimeLocal: entry.actionWhen,
} as FileOrFolderMixedState;
if (isHiddenPath(key)) {
continue;
}
if (results.hasOwnProperty(key)) {
results[key].key = r.key;
results[key].deltimeLocal = r.deltimeLocal;
} else {
results[key] = r;
results[key].existLocal = false;
results[key].existRemote = false;
}
}
return results;
};
const assignOperationToFileInplace = (
origRecord: FileOrFolderMixedState,
keptFolder: Set<string>,
password: string = ""
) => {
let r = origRecord;
// files and folders are treated differently
// here we only check files
if (r.key.endsWith("/")) {
return r;
}
// we find the max date from four sources
// 0. find anything inconsistent
if (r.existLocal && (r.mtimeLocal === undefined || r.mtimeLocal <= 0)) {
throw Error(
`Error: File ${r.key} has a last modified time <=0 or undefined in the local file system. It's abnormal and the plugin stops.`
);
}
if (r.existRemote && (r.mtimeRemote === undefined || r.mtimeRemote <= 0)) {
throw Error(
`Error: File ${r.key} has a last modified time <=0 or undefined on the remote service. It's abnormal and the plugin stops.`
);
}
if (r.deltimeLocal !== undefined && r.deltimeLocal <= 0) {
throw Error(
`Error: File ${r.key} has a local deletion time <=0. It's abnormal and the plugin stops.`
);
}
if (r.deltimeRemote !== undefined && r.deltimeRemote <= 0) {
throw Error(
`Error: File ${r.key} has a remote deletion time <=0. It's abnormal and the plugin stops.`
);
}
// 1. mtimeLocal
if (r.existLocal) {
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
if (
r.mtimeLocal >= mtimeRemote &&
r.mtimeLocal >= deltimeLocal &&
r.mtimeLocal >= deltime_remote
) {
if (r.mtimeLocal === r.mtimeRemote) {
// mtime the same
if (password === "") {
// no password, we can also compare the sizes!
if (r.sizeLocal === r.sizeRemote) {
r.decision = "skipUploading";
r.decisionBranch = 1;
} else {
r.decision = "uploadLocalToRemote";
r.decisionBranch = 2;
}
} else {
// we have password, then the sizes are always unequal
// we can only rely on mtime
r.decision = "skipUploading";
r.decisionBranch = 3;
}
} else {
r.decision = "uploadLocalToRemote";
r.decisionBranch = 4;
}
keptFolder.add(getParentFolder(r.key));
return r;
}
}
// 2. mtimeRemote
if (r.existRemote) {
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
if (
r.mtimeRemote > mtimeLocal &&
r.mtimeRemote >= deltimeLocal &&
r.mtimeRemote >= deltime_remote
) {
r.decision = "downloadRemoteToLocal";
r.decisionBranch = 5;
keptFolder.add(getParentFolder(r.key));
return r;
}
}
// 3. deltimeLocal
if (r.deltimeLocal !== undefined && r.deltimeLocal !== 0) {
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
const deltime_remote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
if (
r.deltimeLocal >= mtimeLocal &&
r.deltimeLocal >= mtimeRemote &&
r.deltimeLocal >= deltime_remote
) {
r.decision = "uploadLocalDelHistToRemote";
r.decisionBranch = 6;
if (r.existLocal || r.existRemote) {
// actual deletion would happen
}
return r;
}
}
// 4. deltime_remote
if (r.deltimeRemote !== undefined && r.deltimeRemote !== 0) {
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
if (
r.deltimeRemote >= mtimeLocal &&
r.deltimeRemote >= mtimeRemote &&
r.deltimeRemote >= deltimeLocal
) {
r.decision = "keepRemoteDelHist";
r.decisionBranch = 7;
if (r.existLocal || r.existRemote) {
// actual deletion would happen
}
return r;
}
}
throw Error(`no decision for ${JSON.stringify(r)}`);
};
const assignOperationToFolderInplace = (
origRecord: FileOrFolderMixedState,
keptFolder: Set<string>,
password: string = ""
) => {
let r = origRecord;
// files and folders are treated differently
// here we only check folders
if (!r.key.endsWith("/")) {
return r;
}
if (!keptFolder.has(r.key)) {
// the folder does NOT have any must-be-kept children!
if (r.deltimeLocal !== undefined || r.deltimeRemote !== undefined) {
// it has some deletion "commands"
if (
r.deltimeLocal !== undefined &&
r.deltimeLocal >= (r.deltimeRemote !== undefined ? r.deltimeRemote : -1)
) {
r.decision = "uploadLocalDelHistToRemoteFolder";
r.decisionBranch = 8;
} else {
r.decision = "keepRemoteDelHistFolder";
r.decisionBranch = 9;
}
} else {
// it does not have any deletion commands
// keep it as is, and create it if necessary
keptFolder.add(getParentFolder(r.key));
if (r.existLocal && r.existRemote) {
r.decision = "skipFolder";
r.decisionBranch = 10;
} else if (r.existLocal || r.existRemote) {
r.decision = "createFolder";
r.decisionBranch = 11;
} else {
throw Error(
`Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
);
}
}
} else {
// the folder has some must be kept children!
// so itself and its parent folder must be kept
keptFolder.add(getParentFolder(r.key));
if (r.existLocal && r.existRemote) {
r.decision = "skipFolder";
r.decisionBranch = 12;
} else if (r.existLocal || r.existRemote) {
r.decision = "createFolder";
r.decisionBranch = 13;
} else {
throw Error(
`Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
);
}
}
// save the memory, save the world!
// we have dealt with it, so we don't need it any more.
keptFolder.delete(r.key);
return r;
};
const DELETION_DECISIONS: Set<DecisionType> = new Set([
"uploadLocalDelHistToRemote",
"keepRemoteDelHist",
"uploadLocalDelHistToRemoteFolder",
"keepRemoteDelHistFolder",
]);
export const getSyncPlan = async (
remoteStates: FileOrFolderMixedState[],
local: TAbstractFile[],
remoteDeleteHistory: DeletionOnRemote[],
localDeleteHistory: FileFolderHistoryRecord[],
remoteType: SUPPORTED_SERVICES_TYPE,
password: string = ""
) => {
const mixedStates = await ensembleMixedStates(
remoteStates,
local,
remoteDeleteHistory,
localDeleteHistory
);
const sortedKeys = Object.keys(mixedStates).sort(
(k1, k2) => k2.length - k1.length
);
const deletions: DeletionOnRemote[] = [];
const keptFolder = new Set<string>();
for (let i = 0; i < sortedKeys.length; ++i) {
const key = sortedKeys[i];
const val = mixedStates[key];
if (key.endsWith("/")) {
// decide some folders
// because the keys are sorted by length
// so all the children must have been shown up before in the iteration
assignOperationToFolderInplace(val, keptFolder, password);
} else {
// get all operations of files
// and at the same time get some helper info for folders
assignOperationToFileInplace(val, keptFolder, password);
}
if (DELETION_DECISIONS.has(val.decision)) {
if (val.decision === "uploadLocalDelHistToRemote") {
deletions.push({
key: key,
actionWhen: val.deltimeLocal,
});
} else if (val.decision === "keepRemoteDelHist") {
deletions.push({
key: key,
actionWhen: val.deltimeRemote,
});
} else if (val.decision === "uploadLocalDelHistToRemoteFolder") {
deletions.push({
key: key,
actionWhen: val.deltimeLocal,
});
} else if (val.decision === "keepRemoteDelHistFolder") {
deletions.push({
key: key,
actionWhen: val.deltimeRemote,
});
} else {
throw Error(`do not know how to delete for decision ${val.decision}`);
}
}
}
const plan = {
ts: Date.now(),
remoteType: remoteType,
mixedStates: mixedStates,
} as SyncPlanType;
return {
plan: plan,
sortedKeys: sortedKeys,
deletions: deletions,
};
};
const uploadExtraMeta = async (
client: RemoteClient,
metadataFile: FileOrFolderMixedState | undefined,
origMetadata: MetadataOnRemote | undefined,
deletions: DeletionOnRemote[],
password: string = ""
) => {
if (deletions === undefined || deletions.length === 0) {
return;
}
const key = DEFAULT_FILE_NAME_FOR_METADATAONREMOTE;
let remoteEncryptedKey = key;
if (password !== "") {
if (metadataFile === undefined) {
remoteEncryptedKey = undefined;
} else {
remoteEncryptedKey = metadataFile.remoteEncryptedKey;
}
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
// remoteEncryptedKey = await encryptStringToBase32(key, password);
remoteEncryptedKey = await encryptStringToBase64url(key, password);
}
}
const newMetadata: MetadataOnRemote = {
deletions: deletions,
};
if (isEqualMetadataOnRemote(origMetadata, newMetadata)) {
log.debug(
"metadata are the same, no need to re-generate and re-upload it."
);
return;
}
const resultText = serializeMetadataOnRemote(newMetadata);
await client.uploadToRemote(
key,
undefined,
false,
password,
remoteEncryptedKey,
undefined,
true,
resultText
);
};
const dispatchOperationToActual = async (
key: string,
vaultRandomID: string,
r: FileOrFolderMixedState,
client: RemoteClient,
db: InternalDBs,
vault: Vault,
localDeleteFunc: any,
password: string = ""
) => {
let remoteEncryptedKey = key;
if (password !== "") {
remoteEncryptedKey = r.remoteEncryptedKey;
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
// the old version uses base32
// remoteEncryptedKey = await encryptStringToBase32(key, password);
// the new version users base64url
remoteEncryptedKey = await encryptStringToBase64url(key, password);
}
}
if (r.decision === undefined) {
throw Error(`unknown decision in ${JSON.stringify(r)}`);
} else if (r.decision === "skipUploading") {
// do nothing!
} else if (r.decision === "uploadLocalDelHistToRemote") {
if (r.existLocal) {
await localDeleteFunc(r.key);
}
if (r.existRemote) {
await client.deleteFromRemote(r.key, password, remoteEncryptedKey);
}
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
} else if (r.decision === "keepRemoteDelHist") {
if (r.existLocal) {
await localDeleteFunc(r.key);
}
if (r.existRemote) {
await client.deleteFromRemote(r.key, password, remoteEncryptedKey);
}
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
} else if (r.decision === "uploadLocalToRemote") {
if (
client.serviceType === "onedrive" &&
r.sizeLocal === 0 &&
password === ""
) {
// special treatment for empty files for OneDrive
// TODO: it's ugly, any other way?
// special treatment for OneDrive: do nothing, skip empty file without encryption
// if it's empty folder, or it's encrypted file/folder, it continues to be uploaded.
} else {
const remoteObjMeta = await client.uploadToRemote(
r.key,
vault,
false,
password,
remoteEncryptedKey
);
await upsertSyncMetaMappingDataByVault(
client.serviceType,
db,
r.key,
r.mtimeLocal,
r.sizeLocal,
r.key,
remoteObjMeta.lastModified,
remoteObjMeta.size,
remoteObjMeta.etag,
vaultRandomID
);
}
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
} else if (r.decision === "downloadRemoteToLocal") {
await mkdirpInVault(r.key, vault); /* should be unnecessary */
await client.downloadFromRemote(
r.key,
vault,
r.mtimeRemote,
password,
remoteEncryptedKey
);
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
} else if (r.decision === "createFolder") {
if (!r.existLocal) {
await mkdirpInVault(r.key, vault);
}
if (!r.existRemote) {
const remoteObjMeta = await client.uploadToRemote(
r.key,
vault,
false,
password,
remoteEncryptedKey
);
await upsertSyncMetaMappingDataByVault(
client.serviceType,
db,
r.key,
r.mtimeLocal,
r.sizeLocal,
r.key,
remoteObjMeta.lastModified,
remoteObjMeta.size,
remoteObjMeta.etag,
vaultRandomID
);
}
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
} else if (r.decision === "uploadLocalDelHistToRemoteFolder") {
if (r.existLocal) {
await localDeleteFunc(r.key);
}
if (r.existRemote) {
await client.deleteFromRemote(r.key, password, remoteEncryptedKey);
}
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
} else if (r.decision === "keepRemoteDelHistFolder") {
if (r.existLocal) {
await localDeleteFunc(r.key);
}
if (r.existRemote) {
await client.deleteFromRemote(r.key, password, remoteEncryptedKey);
}
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
} else if (r.decision === "skipFolder") {
// do nothing!
} else {
throw Error(`unknown decision in ${JSON.stringify(r)}`);
}
};
export const doActualSync = async (
client: RemoteClient,
db: InternalDBs,
vaultRandomID: string,
vault: Vault,
syncPlan: SyncPlanType,
sortedKeys: string[],
metadataFile: FileOrFolderMixedState,
origMetadata: MetadataOnRemote,
deletions: DeletionOnRemote[],
localDeleteFunc: any,
password: string = "",
callbackSyncProcess?: any
) => {
const mixedStates = syncPlan.mixedStates;
let i = 0;
const totalCount = sortedKeys.length || 0;
log.debug(`start syncing extra data firstly`);
await uploadExtraMeta(
client,
metadataFile,
origMetadata,
deletions,
password
);
log.debug(`finish syncing extra data firstly`);
for (let i = 0; i < sortedKeys.length; ++i) {
const key = sortedKeys[i];
const val = mixedStates[key];
log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
if (callbackSyncProcess !== undefined) {
await callbackSyncProcess(i, totalCount, key, val.decision);
}
await dispatchOperationToActual(
key,
vaultRandomID,
val,
client,
db,
vault,
localDeleteFunc,
password
);
log.debug(`finished ${key}`);
// await Promise.all(
// Object.entries(mixedStates).map(async ([k, v]) =>
// dispatchOperationToActual(
// k as string,
// vaultRandomID,
// v as FileOrFolderMixedState,
// client,
// db,
// vault,
// localDeleteFunc,
// password
// )
// )
// );
}
};