diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 3770f92..81fd705 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -169,8 +169,9 @@ export type DecisionTypeForMixedEntity = * everything should be flat and primitive, so that we can copy. */ export interface Entity { - key: string; - keyEnc: string; + key?: string; + keyEnc?: string; + keyRaw: string; mtimeCli?: number; mtimeCliFmt?: string; mtimeSvr?: number; @@ -178,7 +179,8 @@ export interface Entity { prevSyncTime?: number; prevSyncTimeFmt?: string; size?: number; // might be unknown or to be filled - sizeEnc: number; + sizeEnc?: number; + sizeRaw: number; hash?: string; etag?: string; } diff --git a/src/local.ts b/src/local.ts index b128dad..e1ef7ab 100644 --- a/src/local.ts +++ b/src/local.ts @@ -32,20 +32,20 @@ export const getLocalEntityList = async ( ); } r = { - key: entry.path, - keyEnc: entry.path, + key: entry.path, // local always unencrypted + keyRaw: entry.path, mtimeCli: mtimeLocal, mtimeSvr: mtimeLocal, - size: entry.stat.size, - sizeEnc: entry.stat.size, + size: entry.stat.size, // local always unencrypted + sizeRaw: entry.stat.size, }; } else if (entry instanceof TFolder) { key = `${entry.path}/`; r = { key: key, - keyEnc: key, + keyRaw: key, size: 0, - sizeEnc: 0, + sizeRaw: 0, }; } else { throw Error(`unexpected ${entry}`); diff --git a/src/main.ts b/src/main.ts index 30a3fc3..253b922 100644 --- a/src/main.ts +++ b/src/main.ts @@ -240,6 +240,7 @@ export default class RemotelySavePlugin extends Plugin { () => self.saveSettings() ); const remoteEntityList = await client.listAllFromRemote(); + log.debug("remoteEntityList:"); log.debug(remoteEntityList); if (this.settings.currLogLevel === "info") { @@ -269,6 +270,7 @@ export default class RemotelySavePlugin extends Plugin { this.app.vault.configDir, this.manifest.id ); + log.debug("localEntityList:"); log.debug(localEntityList); if (this.settings.currLogLevel === "info") { @@ -281,6 +283,7 @@ export default class RemotelySavePlugin extends Plugin { this.db, this.vaultRandomID ); + log.debug("prevSyncEntityList:"); log.debug(prevSyncEntityList); if (this.settings.currLogLevel === "info") { @@ -305,6 +308,7 @@ export default class RemotelySavePlugin extends Plugin { this.settings.skipSizeLargerThan ?? -1, this.settings.conflictAction ?? "keep_newer" ); + log.info(`mixedEntityMappings:`); log.info(mixedEntityMappings); // for debugging await insertSyncPlanRecordByVault( this.db, diff --git a/src/obsFolderLister.ts b/src/obsFolderLister.ts index c12fb76..0109366 100644 --- a/src/obsFolderLister.ts +++ b/src/obsFolderLister.ts @@ -79,12 +79,12 @@ export const listFilesInObsFolder = async ( return { itself: { - key: isFolder ? `${x}/` : x, - keyEnc: isFolder ? `${x}/` : x, + key: isFolder ? `${x}/` : x, // local always unencrypted + keyRaw: isFolder ? `${x}/` : x, mtimeCli: statRes.mtime, mtimeSvr: statRes.mtime, - size: statRes.size, - sizeEnc: statRes.size, + size: statRes.size, // local always unencrypted + sizeRaw: statRes.size, }, children: children, }; diff --git a/src/remoteForDropbox.ts b/src/remoteForDropbox.ts index 18d4cde..5052049 100644 --- a/src/remoteForDropbox.ts +++ b/src/remoteForDropbox.ts @@ -83,22 +83,18 @@ const fromDropboxItemToEntity = ( if (x[".tag"] === "folder") { return { - key: key, - keyEnc: key, - size: 0, - sizeEnc: 0, + keyRaw: key, + sizeRaw: 0, etag: `${x.id}\t`, } as Entity; } else if (x[".tag"] === "file") { const mtimeCli = Date.parse(x.client_modified).valueOf(); const mtimeSvr = Date.parse(x.server_modified).valueOf(); return { - key: key, - keyEnc: key, + keyRaw: key, mtimeCli: mtimeCli, mtimeSvr: mtimeSvr, - size: x.size, - sizeEnc: x.size, + sizeRaw: x.size, hash: x.content_hash, etag: `${x.id}\t${x.content_hash}`, } as Entity; @@ -469,6 +465,11 @@ export const uploadToRemote = async ( let uploadFile = fileOrFolderPath; if (password !== "") { + if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { + throw Error( + `uploadToRemote(dropbox) you have password but remoteEncryptedKey is empty!` + ); + } uploadFile = remoteEncryptedKey; } uploadFile = getDropboxPath(uploadFile, client.remoteBaseDir); diff --git a/src/remoteForOnedrive.ts b/src/remoteForOnedrive.ts index ec2759d..f2c64bc 100644 --- a/src/remoteForOnedrive.ts +++ b/src/remoteForOnedrive.ts @@ -351,12 +351,10 @@ const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => { const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!); const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!); return { - key: key, - keyEnc: key, + keyRaw: key, mtimeSvr: mtimeSvr, mtimeCli: mtimeCli, - size: isFolder ? 0 : x.size!, - sizeEnc: isFolder ? 0 : x.size!, + sizeRaw: isFolder ? 0 : x.size!, // hash: ?? // TODO etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes }; @@ -708,6 +706,11 @@ export const uploadToRemote = async ( let uploadFile = fileOrFolderPath; if (password !== "") { + if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { + throw Error( + `uploadToRemote(onedrive) you have password but remoteEncryptedKey is empty!` + ); + } uploadFile = remoteEncryptedKey; } uploadFile = getOnedrivePath(uploadFile, client.remoteBaseDir); diff --git a/src/remoteForS3.ts b/src/remoteForS3.ts index fce0783..b367b56 100644 --- a/src/remoteForS3.ts +++ b/src/remoteForS3.ts @@ -238,12 +238,10 @@ const fromS3ObjectToEntity = ( } const key = getLocalNoPrefixPath(x.Key!, remotePrefix); const r: Entity = { - key: key, - keyEnc: key, + keyRaw: key, mtimeSvr: mtimeSvr, mtimeCli: mtimeCli, - size: x.Size!, - sizeEnc: x.Size!, + sizeRaw: x.Size!, etag: x.ETag, }; return r; @@ -266,11 +264,21 @@ const fromS3HeadObjectToEntity = ( mtimeCli = m2; } } + // log.debug( + // `fromS3HeadObjectToEntity, fileOrFolderPathWithRemotePrefix=${fileOrFolderPathWithRemotePrefix}, remotePrefix=${remotePrefix}, x=${JSON.stringify( + // x + // )} ` + // ); + const key = getLocalNoPrefixPath( + fileOrFolderPathWithRemotePrefix, + remotePrefix + ); + // log.debug(`fromS3HeadObjectToEntity, key=${key} after removing prefix`); return { - key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix), + keyRaw: key, mtimeSvr: mtimeSvr, mtimeCli: mtimeCli, - size: x.ContentLength, + sizeRaw: x.ContentLength, etag: x.ETag, } as Entity; }; @@ -361,9 +369,15 @@ export const uploadToRemote = async ( log.debug(`uploading ${fileOrFolderPath}`); let uploadFile = fileOrFolderPath; if (password !== "") { + if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { + throw Error( + `uploadToRemote(s3) you have password but remoteEncryptedKey is empty!` + ); + } uploadFile = remoteEncryptedKey; } uploadFile = getRemoteWithPrefixPath(uploadFile, s3Config.remotePrefix ?? ""); + // log.debug(`actual uploadFile=${uploadFile}`); const isFolder = fileOrFolderPath.endsWith("/"); if (isFolder && isRecursively) { @@ -459,9 +473,9 @@ export const uploadToRemote = async ( await upload.done(); const res = await getRemoteMeta(s3Client, s3Config, uploadFile); - log.debug( - `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}` - ); + // log.debug( + // `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}` + // ); return res; } }; diff --git a/src/remoteForWebdav.ts b/src/remoteForWebdav.ts index fe44443..6ad30cb 100644 --- a/src/remoteForWebdav.ts +++ b/src/remoteForWebdav.ts @@ -212,12 +212,10 @@ const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string) => { } const mtimeSvr = Date.parse(x.lastmod).valueOf(); return { - key: key, - keyEnc: key, + keyRaw: key, mtimeSvr: mtimeSvr, mtimeCli: mtimeSvr, // no universal way to set mtime in webdav - size: x.size, - sizeEnc: x.size, + sizeRaw: x.size, etag: x.etag, } as Entity; }; @@ -346,6 +344,11 @@ export const uploadToRemote = async ( await client.init(); let uploadFile = fileOrFolderPath; if (password !== "") { + if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { + throw Error( + `uploadToRemote(webdav) you have password but remoteEncryptedKey is empty!` + ); + } uploadFile = remoteEncryptedKey; } uploadFile = getWebdavPath(uploadFile, client.remoteBaseDir); diff --git a/src/sync.ts b/src/sync.ts index 7be1c5a..d0a3d2d 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -75,7 +75,7 @@ export const isPasswordOk = async ( reason: "empty_remote", }; } - const santyCheckKey = remote[0].key; + const santyCheckKey = remote[0].keyRaw; if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { // this is encrypted using old base32! // try to decrypt it using the provided password. @@ -161,6 +161,9 @@ const isSkipItemByName = ( configDir: string, ignorePaths: string[] ) => { + if (key === undefined) { + throw Error(`isSkipItemByName meets undefinded key!`); + } if (ignorePaths !== undefined && ignorePaths.length > 0) { for (const r of ignorePaths) { if (XRegExp(r, "A").test(key)) { @@ -218,17 +221,25 @@ const copyEntityAndFixTimeFormat = (src: Entity) => { */ const decryptRemoteEntityInplace = async (remote: Entity, password: string) => { if (password == undefined || password === "") { - remote.key = remote.keyEnc; - remote.size = remote.sizeEnc; + remote.key = remote.keyRaw; + remote.keyEnc = remote.keyRaw; + remote.size = remote.sizeRaw; + remote.sizeEnc = remote.sizeRaw; return remote; } - if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { + if (remote.keyRaw.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { + remote.keyEnc = remote.keyRaw; remote.key = await decryptBase32ToString(remote.keyEnc, password); - } else if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) { + remote.sizeEnc = remote.sizeRaw; + } else if (remote.keyRaw.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) { + remote.keyEnc = remote.keyRaw; remote.key = await decryptBase64urlToString(remote.keyEnc, password); + remote.sizeEnc = remote.sizeRaw; } else { - throw Error(`unexpected key to decrypt=${remote.keyEnc}`); + throw Error( + `unexpected key to decrypt: ${JSON.stringify(remote, null, 2)}` + ); } // TODO @@ -245,7 +256,7 @@ const decryptRemoteEntityInplace = async (remote: Entity, password: string) => { */ const ensureMTimeOfRemoteEntityValid = (remote: Entity) => { if ( - !remote.key.endsWith("/") && + !remote.key!.endsWith("/") && remote.mtimeCli === undefined && remote.mtimeSvr === undefined ) { @@ -273,19 +284,36 @@ const encryptLocalEntityInplace = async ( password: string, remoteKeyEnc: string | undefined ) => { - if (password == undefined || password === "") { - local.sizeEnc = local.size!; // if no enc, the remote file has the same size - local.keyEnc = local.key; + // log.debug( + // `encryptLocalEntityInplace: local=${JSON.stringify( + // local, + // null, + // 2 + // )}, password=${ + // password === undefined || password === "" ? "[empty]" : "[not empty]" + // }, remoteKeyEnc=${remoteKeyEnc}` + // ); + + if (local.key === undefined) { + // local.key should always have value + throw Error(`local ${local.keyRaw} is abnormal without key`); + } + + if (password === undefined || password === "") { + local.sizeEnc = local.sizeRaw; // if no enc, the remote file has the same size + local.keyEnc = local.keyRaw; return local; } // below is for having password - - if (local.size === local.sizeEnc) { - // size not transformed yet, we need to compute sizeEnc + if (local.sizeEnc === undefined && local.size !== undefined) { + // it's not filled yet, we fill it + // local.size is possibly undefined if it's "prevSync" Entity + // but local.key should always have value local.sizeEnc = getSizeFromOrigToEnc(local.size); } - if (local.key === local.keyEnc) { + + if (local.keyEnc === undefined || local.keyEnc === "") { if ( remoteKeyEnc !== undefined && remoteKeyEnc !== "" && @@ -328,7 +356,7 @@ export const ensembleMixedEnties = async ( ) ); - const key = remoteCopied.key; + const key = remoteCopied.key!; if ( isSkipItemByName( key, @@ -347,37 +375,47 @@ export const ensembleMixedEnties = async ( }; } - for (const prevSync of prevSyncEntityList) { - const key = prevSync.key; - if ( - isSkipItemByName( - key, - syncConfigDir, - syncUnderscoreItems, - configDir, - ignorePaths - ) - ) { - continue; - } + if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) { + // Special checking: + // if one side is totally empty, + // usually that's a hard rest. + // So we need to ignore everything of prevSyncEntityList to avoid deletions! + // TODO: acutally erase everything of prevSyncEntityList? + // TODO: local should also go through a isSkipItemByName checking beforehand + } else { + // normally go through the prevSyncEntityList + for (const prevSync of prevSyncEntityList) { + const key = prevSync.key!; + if ( + isSkipItemByName( + key, + syncConfigDir, + syncUnderscoreItems, + configDir, + ignorePaths + ) + ) { + continue; + } - if (finalMappings.hasOwnProperty(key)) { - const prevSyncCopied = await encryptLocalEntityInplace( - copyEntityAndFixTimeFormat(prevSync), - password, - finalMappings[key].remote?.keyEnc - ); - finalMappings[key].prevSync = prevSyncCopied; - } else { - const prevSyncCopied = await encryptLocalEntityInplace( - copyEntityAndFixTimeFormat(prevSync), - password, - undefined - ); - finalMappings[key] = { - key: key, - prevSync: prevSyncCopied, - }; + if (finalMappings.hasOwnProperty(key)) { + const prevSyncCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(prevSync), + password, + finalMappings[key].remote?.keyEnc + ); + finalMappings[key].prevSync = prevSyncCopied; + } else { + const prevSyncCopied = await encryptLocalEntityInplace( + copyEntityAndFixTimeFormat(prevSync), + password, + undefined + ); + finalMappings[key] = { + key: key, + prevSync: prevSyncCopied, + }; + } } } @@ -385,7 +423,7 @@ export const ensembleMixedEnties = async ( // because we want to get keyEnc based on the remote // (we don't consume prevSync here because it gains no benefit) for (const local of localEntityList) { - const key = local.key; + const key = local.key!; if ( isSkipItemByName( key, @@ -514,7 +552,7 @@ export const getSyncPlanInplace = async ( // If only one compares true (no prev also means it compares False), the other is modified. Backup and sync. if ( skipSizeLargerThan <= 0 || - remote.sizeEnc <= skipSizeLargerThan + remote.sizeEnc! <= skipSizeLargerThan ) { mixedEntry.decisionBranch = 9; mixedEntry.decision = "modified_remote"; @@ -530,7 +568,7 @@ export const getSyncPlanInplace = async ( // If only one compares true (no prev also means it compares False), the other is modified. Backup and sync. if ( skipSizeLargerThan <= 0 || - local.sizeEnc <= skipSizeLargerThan + local.sizeEnc! <= skipSizeLargerThan ) { mixedEntry.decisionBranch = 10; mixedEntry.decision = "modified_local"; @@ -559,7 +597,7 @@ export const getSyncPlanInplace = async ( keptFolder.add(getParentFolder(key)); } } else if (conflictAction === "keep_larger") { - if (local.sizeEnc >= remote.sizeEnc) { + if (local.sizeEnc! >= remote.sizeEnc!) { mixedEntry.decisionBranch = 13; mixedEntry.decision = "conflict_created_keep_local"; keptFolder.add(getParentFolder(key)); @@ -588,7 +626,7 @@ export const getSyncPlanInplace = async ( keptFolder.add(getParentFolder(key)); } } else if (conflictAction === "keep_larger") { - if (local.sizeEnc >= remote.sizeEnc) { + if (local.sizeEnc! >= remote.sizeEnc!) { mixedEntry.decisionBranch = 18; mixedEntry.decision = "conflict_modified_keep_local"; keptFolder.add(getParentFolder(key)); @@ -616,7 +654,10 @@ export const getSyncPlanInplace = async ( // A is missing if (prevSync === undefined) { // if B is not in the previous list, B is new - if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) { + if ( + skipSizeLargerThan <= 0 || + remote.sizeEnc! <= skipSizeLargerThan + ) { mixedEntry.decisionBranch = 3; mixedEntry.decision = "created_remote"; keptFolder.add(getParentFolder(key)); @@ -637,7 +678,10 @@ export const getSyncPlanInplace = async ( mixedEntry.decision = "deleted_local"; } else { // if B is in the previous list and MODIFIED, B has been deleted by A but modified by B - if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) { + if ( + skipSizeLargerThan <= 0 || + remote.sizeEnc! <= skipSizeLargerThan + ) { mixedEntry.decisionBranch = 5; mixedEntry.decision = "modified_remote"; keptFolder.add(getParentFolder(key)); @@ -654,7 +698,7 @@ export const getSyncPlanInplace = async ( if (prevSync === undefined) { // if A is not in the previous list, A is new - if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) { + if (skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan) { mixedEntry.decisionBranch = 6; mixedEntry.decision = "created_local"; keptFolder.add(getParentFolder(key)); @@ -675,7 +719,7 @@ export const getSyncPlanInplace = async ( mixedEntry.decision = "deleted_remote"; } else { // if A is in the previous list and MODIFIED, A has been deleted by B but modified by A - if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) { + if (skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan) { mixedEntry.decisionBranch = 8; mixedEntry.decision = "modified_local"; keptFolder.add(getParentFolder(key)); @@ -744,9 +788,9 @@ const splitThreeStepsOnEntityMappings = ( val.decision === "folder_existed_remote" || val.decision === "folder_to_be_created" ) { - log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); + // log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`); const level = atWhichLevel(key); - log.debug(`atWhichLevel: ${level}`); + // log.debug(`atWhichLevel: ${level}`); const k = folderCreationOps[level - 1]; if (k === undefined || k === null) { folderCreationOps[level - 1] = [val]; @@ -818,13 +862,13 @@ const dispatchOperationToActualV3 = async ( localDeleteFunc: any, password: string ) => { - log.debug( - `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify( - r, - null, - 2 - )}` - ); + // log.debug( + // `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify( + // r, + // null, + // 2 + // )}` + // ); if (r.decision === "only_history") { clearPrevSyncRecordByVault(db, vaultRandomID, key); } else if ( @@ -850,6 +894,7 @@ const dispatchOperationToActualV3 = async ( // special treatment for OneDrive: do nothing, skip empty file without encryption // if it's empty folder, or it's encrypted file/folder, it continues to be uploaded. } else { + // log.debug(`before upload in sync, r=${JSON.stringify(r, null, 2)}`); const remoteObjMeta = await client.uploadToRemote( r.key, vault, @@ -857,6 +902,7 @@ const dispatchOperationToActualV3 = async ( password, r.local!.keyEnc ); + await decryptRemoteEntityInplace(remoteObjMeta, password); await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); } } else if ( @@ -897,6 +943,8 @@ const dispatchOperationToActualV3 = async ( password, r.local!.keyEnc ); + // we need to decrypt the key!!! + await decryptRemoteEntityInplace(remoteObjMeta, password); await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta); } else if (r.decision === "folder_to_be_deleted") { await localDeleteFunc(r.key); @@ -921,10 +969,10 @@ export const doActualSync = async ( log.debug(`concurrency === ${concurrency}`); const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } = splitThreeStepsOnEntityMappings(mixedEntityMappings); - log.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`); - log.debug(`deletionOps: ${JSON.stringify(deletionOps)}`); - log.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`); - log.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`); + // log.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`); + // log.debug(`deletionOps: ${JSON.stringify(deletionOps)}`); + // log.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`); + // log.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`); const nested = [folderCreationOps, deletionOps, uploadDownloads]; const logTexts = [ @@ -938,11 +986,11 @@ export const doActualSync = async ( log.debug(logTexts[i]); const operations = nested[i]; - log.debug(`curr operations=${JSON.stringify(operations, null, 2)}`); + // log.debug(`curr operations=${JSON.stringify(operations, null, 2)}`); for (let j = 0; j < operations.length; ++j) { const singleLevelOps = operations[j]; - log.debug(`singleLevelOps=${singleLevelOps}`); + log.debug(`singleLevelOps=${JSON.stringify(singleLevelOps, null, 2)}`); if (singleLevelOps === undefined || singleLevelOps === null) { continue; }