bump to 0.4.12

fix AWS S3 user's policy configuration issue https://github.com/remotely-save/remotely-save/issues/564

docs/remote_services/s3_general/s3_user_policy.md

@@ -33,6 +33,7 @@ Using the principle of least privilege is crucial for security when allowing a t
             "Effect": "Allow",
             "Action": [
                 "s3:HeadObject",
+                "s3:ListBucket",
                 "s3:PutObject",
                 "s3:CopyObject",
                 "s3:UploadPart",
@@ -48,7 +49,10 @@ Using the principle of least privilege is crucial for security when allowing a t
                 "s3:DeleteObject",
                 "s3:DeleteObjects"
             ],
-            "Resource": "arn:aws:s3:::my-bucket/*"
+            "Resource": [
+                "arn:aws:s3:::my-bucket",
+                "arn:aws:s3:::my-bucket/*"
+            ]
         }
     ]
 }

manifest-beta.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.4.11",
+  "version": "0.4.12",
   "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.4.11",
+  "version": "0.4.12",
   "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.4.11",
+  "version": "0.4.12",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs --watch",
@@ -24,7 +24,7 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@microsoft/microsoft-graph-types": "^2.40.0",
-    "@types/chai": "^4.3.11",
+    "@types/chai": "^4.3.14",
     "@types/chai-as-promised": "^7.1.8",
     "@types/jsdom": "^21.1.6",
     "@types/lodash": "^4.14.202",
@@ -34,14 +34,14 @@
     "@types/node": "^20.10.4",
     "@types/qrcode": "^1.5.5",
     "builtin-modules": "^3.3.0",
-    "chai": "^4.3.10",
+    "chai": "^4.4.1",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
     "dotenv": "^16.3.1",
     "esbuild": "^0.19.9",
     "esbuild-plugin-inline-worker": "^0.1.1",
     "jsdom": "^23.0.1",
-    "mocha": "^10.2.0",
+    "mocha": "^10.4.0",
     "npm-check-updates": "^16.14.12",
     "obsidian": "^1.4.11",
     "prettier": "^3.1.1",

src/baseTypes.ts

@@ -204,6 +204,7 @@ export interface Entity {
   sizeRaw: number;
   hash?: string;
   etag?: string;
+  synthesizedFolder?: boolean;
 }
 
 export interface UploadedType {
@@ -223,6 +224,8 @@ export interface MixedEntity {
   decisionBranch?: number;
   decision?: DecisionTypeForMixedEntity;
   conflictAction?: ConflictActionType;
+
+  sideNotes?: any;
 }
 
 /**

src/misc.ts

@@ -534,3 +534,52 @@ export const changeMobileStatusBar = (op: "enable" | "disable") => {
     bar.style.removeProperty("margin-bottom");
   }
 };
+
+/**
+ * https://github.com/remotely-save/remotely-save/issues/567
+ * https://www.dropboxforum.com/t5/Dropbox-API-Support-Feedback/Case-Sensitivity-in-API-2/td-p/191279
+ * @param entities
+ */
+export const fixEntityListCasesInplace = (entities: { keyRaw: string }[]) => {
+  entities.sort((a, b) => a.keyRaw.length - b.keyRaw.length);
+  // console.log(JSON.stringify(entities,null,2));
+
+  const caseMapping: Record<string, string> = { "": "" };
+  for (const e of entities) {
+    // console.log(`looking for: ${JSON.stringify(e, null, 2)}`);
+
+    let parentFolder = getParentFolder(e.keyRaw);
+    if (parentFolder === "/") {
+      parentFolder = "";
+    }
+    const parentFolderLower = parentFolder.toLocaleLowerCase();
+    const segs = e.keyRaw.split("/");
+    if (e.keyRaw.endsWith("/")) {
+      // folder
+      if (caseMapping.hasOwnProperty(parentFolderLower)) {
+        const newKeyRaw = `${caseMapping[parentFolderLower]}${segs
+          .slice(-2)
+          .join("/")}`;
+        caseMapping[newKeyRaw.toLocaleLowerCase()] = newKeyRaw;
+        e.keyRaw = newKeyRaw;
+        // console.log(JSON.stringify(caseMapping,null,2));
+        continue;
+      } else {
+        throw Error(`${parentFolder} doesn't have cases record??`);
+      }
+    } else {
+      // file
+      if (caseMapping.hasOwnProperty(parentFolderLower)) {
+        const newKeyRaw = `${caseMapping[parentFolderLower]}${segs
+          .slice(-1)
+          .join("/")}`;
+        e.keyRaw = newKeyRaw;
+        continue;
+      } else {
+        throw Error(`${parentFolder} doesn't have cases record??`);
+      }
+    }
+  }
+
+  return entities;
+};

src/remote.ts

@@ -239,7 +239,8 @@ export class RemoteClient {
   deleteFromRemote = async (
     fileOrFolderPath: string,
     cipher: Cipher,
-    remoteEncryptedKey: string = ""
+    remoteEncryptedKey: string = "",
+    synthesizedFolder: boolean = false
   ) => {
     if (this.serviceType === "s3") {
       return await s3.deleteFromRemote(
@@ -247,7 +248,8 @@ export class RemoteClient {
         this.s3Config!,
         fileOrFolderPath,
         cipher,
-        remoteEncryptedKey
+        remoteEncryptedKey,
+        synthesizedFolder
       );
     } else if (this.serviceType === "webdav") {
       return await webdav.deleteFromRemote(

src/remoteForDropbox.ts

@@ -12,6 +12,7 @@ import {
 } from "./baseTypes";
 import {
   bufferToArrayBuffer,
+  fixEntityListCasesInplace,
   getFolderLevels,
   hasEmojiInText,
   headersToRecord,
@@ -635,6 +636,8 @@ export const listAllFromRemote = async (client: WrappedDropboxClient) => {
     unifiedContents.push(...unifiedContents2);
   }
 
+  fixEntityListCasesInplace(unifiedContents);
+
   return unifiedContents;
 };
 

src/remoteForOnedrive.ts

@@ -471,6 +471,11 @@ export class WrappedOnedriveClient {
       const pathFrag = encodeURI(pathFragOrig);
       theUrl = `${API_PREFIX}${pathFrag}`;
     }
+    // we want to support file name with hash #
+    // because every url we construct here do not contain the # symbol
+    // thus it should be safe to directly replace the character
+    theUrl = theUrl.replace(/#/g, "%23");
+    // console.debug(`building url: [${pathFragOrig}] => [${theUrl}]`)
     return theUrl;
   };
 

src/remoteForS3.ts

@@ -250,6 +250,7 @@ const fromS3ObjectToEntity = (
     mtimeCli: mtimeCli,
     sizeRaw: x.Size!,
     etag: x.ETag,
+    synthesizedFolder: false,
   };
   return r;
 };
@@ -599,7 +600,10 @@ export const listAllFromRemote = async (
   s3Client: S3Client,
   s3Config: S3Config
 ) => {
-  return await listFromRemoteRaw(s3Client, s3Config, s3Config.remotePrefix);
+  const res = (
+    await listFromRemoteRaw(s3Client, s3Config, s3Config.remotePrefix)
+  ).filter((x) => x.keyRaw !== "" && x.keyRaw !== "/");
+  return res;
 };
 
 /**
@@ -715,11 +719,15 @@ export const deleteFromRemote = async (
   s3Config: S3Config,
   fileOrFolderPath: string,
   cipher: Cipher,
-  remoteEncryptedKey: string = ""
+  remoteEncryptedKey: string = "",
+  synthesizedFolder: boolean = false
 ) => {
   if (fileOrFolderPath === "/") {
     return;
   }
+  if (synthesizedFolder) {
+    return;
+  }
   let remoteFileName = fileOrFolderPath;
   if (!cipher.isPasswordEmpty()) {
     remoteFileName = remoteEncryptedKey;
@@ -771,7 +779,7 @@ export const checkConnectivity = async (
 ) => {
   try {
     // TODO: no universal way now, just check this in connectivity
-    if (Platform.isIosApp && !s3Config.s3Endpoint.startsWith("https")) {
+    if (Platform.isIosApp && s3Config.s3Endpoint.startsWith("http://")) {
       throw Error(
         `Your s3 endpoint could only be https, not http, because of the iOS restriction.`
       );

src/sync.ts

@@ -18,6 +18,7 @@ import {
   isVaildText,
   atWhichLevel,
   mkdirpInVault,
+  getFolderLevels,
 } from "./misc";
 import {
   DEFAULT_FILE_NAME_FOR_METADATAONREMOTE,
@@ -326,7 +327,10 @@ export const ensembleMixedEnties = async (
 ): Promise<SyncPlanType> => {
   const finalMappings: SyncPlanType = {};
 
+  const synthFolders: Record<string, Entity> = {};
+
   // remote has to be first
+  // we also have to synthesize folders here
   for (const remote of remoteEntityList) {
     const remoteCopied = ensureMTimeOfRemoteEntityValid(
       await decryptRemoteEntityInplace(
@@ -352,6 +356,42 @@ export const ensembleMixedEnties = async (
       key: key,
       remote: remoteCopied,
     };
+
+    for (const f of getFolderLevels(key, true)) {
+      if (finalMappings.hasOwnProperty(f)) {
+        delete synthFolders[f];
+        continue;
+      }
+      if (
+        !synthFolders.hasOwnProperty(f) ||
+        remoteCopied.mtimeSvr! >= synthFolders[f].mtimeSvr!
+      ) {
+        synthFolders[f] = {
+          key: f,
+          keyRaw: `<synth: ${f}>`,
+          keyEnc: `<enc synth: ${f}>`,
+          size: 0,
+          sizeRaw: 0,
+          sizeEnc: 0,
+          mtimeSvr: remoteCopied.mtimeSvr,
+          mtimeSvrFmt: remoteCopied.mtimeSvrFmt,
+          mtimeCli: remoteCopied.mtimeCli,
+          mtimeCliFmt: remoteCopied.mtimeCliFmt,
+          synthesizedFolder: true,
+        };
+      }
+    }
+  }
+
+  console.debug(`synthFolders:`);
+  console.debug(synthFolders);
+
+  // special: add synth folders
+  for (const key of Object.keys(synthFolders)) {
+    finalMappings[key] = {
+      key: key,
+      remote: synthFolders[key],
+    };
   }
 
   if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) {
@@ -863,13 +903,27 @@ export const getSyncPlanInplace = async (
     throw Error(`unexpectedly keptFolder no decisions: ${[...keptFolder]}`);
   }
 
+  // finally we want to make our life easier
+  const currTime = Date.now();
+  const currTimeFmt = unixTimeToStr(currTime);
+  // because the path should not as / in the beginning,
+  // we should be safe to add these keys:
+  mixedEntityMappings["/$@meta"] = {
+    key: "/$@meta", // don't mess up with the types
+    sideNotes: {
+      generateTime: currTime,
+      generateTimeFmt: currTimeFmt,
+    },
+  };
+
   return mixedEntityMappings;
 };
 
-const splitThreeStepsOnEntityMappings = (
+const splitFourStepsOnEntityMappings = (
   mixedEntityMappings: Record<string, MixedEntity>
 ) => {
   type StepArrayType = MixedEntity[] | undefined | null;
+  const onlyMarkSyncedOps: StepArrayType[] = [];
   const folderCreationOps: StepArrayType[] = [];
   const deletionOps: StepArrayType[] = [];
   const uploadDownloads: StepArrayType[] = [];
@@ -885,6 +939,11 @@ const splitThreeStepsOnEntityMappings = (
 
   for (let i = 0; i < sortedKeys.length; ++i) {
     const key = sortedKeys[i];
+
+    if (key === "/$@meta") {
+      continue; // special
+    }
+
     const val = mixedEntityMappings[key];
 
     if (!key.endsWith("/")) {
@@ -892,14 +951,27 @@ const splitThreeStepsOnEntityMappings = (
     }
 
     if (
-      val.decision === "equal" ||
-      val.decision === "conflict_created_then_do_nothing" ||
-      val.decision === "folder_existed_both_then_do_nothing" ||
       val.decision === "local_is_created_too_large_then_do_nothing" ||
       val.decision === "remote_is_created_too_large_then_do_nothing" ||
       val.decision === "folder_to_skip"
     ) {
       // pass
+    } else if (
+      val.decision === "equal" ||
+      val.decision === "conflict_created_then_do_nothing" ||
+      val.decision === "folder_existed_both_then_do_nothing"
+    ) {
+      if (
+        onlyMarkSyncedOps.length === 0 ||
+        onlyMarkSyncedOps[0] === undefined ||
+        onlyMarkSyncedOps[0] === null
+      ) {
+        onlyMarkSyncedOps[0] = [val];
+      } else {
+        onlyMarkSyncedOps[0].push(val); // only one level is needed here
+      }
+
+      // don't need to update realTotalCount here
     } else if (
       val.decision === "folder_existed_local_then_also_create_remote" ||
       val.decision === "folder_existed_remote_then_also_create_local" ||
@@ -979,6 +1051,7 @@ const splitThreeStepsOnEntityMappings = (
   deletionOps.reverse(); // inplace reverse
 
   return {
+    onlyMarkSyncedOps: onlyMarkSyncedOps,
     folderCreationOps: folderCreationOps,
     deletionOps: deletionOps,
     uploadDownloads: uploadDownloads,
@@ -1009,14 +1082,36 @@ const dispatchOperationToActualV3 = async (
   if (r.decision === "only_history") {
     clearPrevSyncRecordByVaultAndProfile(db, vaultRandomID, profileID, key);
   } else if (
-    r.decision === "equal" ||
-    r.decision === "conflict_created_then_do_nothing" ||
     r.decision === "local_is_created_too_large_then_do_nothing" ||
     r.decision === "remote_is_created_too_large_then_do_nothing" ||
-    r.decision === "folder_to_skip" ||
+    r.decision === "folder_to_skip"
+  ) {
+    // !! no actual sync being kept happens,
+    // so no sync record here
+    // pass
+  } else if (
+    r.decision === "equal" ||
+    r.decision === "conflict_created_then_do_nothing" ||
     r.decision === "folder_existed_both_then_do_nothing"
   ) {
-    // pass
+    // !! we need to upsert the record,
+    // so that next time we can determine the change delta
+    const entity = r.remote ?? r.local;
+    console.debug(
+      `we are in actual operation of equal, entity=${JSON.stringify(
+        entity,
+        null,
+        2
+      )}`
+    );
+    if (entity !== undefined) {
+      await upsertPrevSyncRecordByVaultAndProfile(
+        db,
+        vaultRandomID,
+        profileID,
+        entity
+      );
+    }
   } else if (
     r.decision === "local_is_modified_then_push" ||
     r.decision === "local_is_created_then_push" ||
@@ -1074,7 +1169,12 @@ const dispatchOperationToActualV3 = async (
     );
   } else if (r.decision === "local_is_deleted_thus_also_delete_remote") {
     // local is deleted, we need to delete remote now
-    await client.deleteFromRemote(r.key, cipher, r.remote!.keyEnc);
+    await client.deleteFromRemote(
+      r.key,
+      cipher,
+      r.remote!.keyEnc,
+      r.remote!.synthesizedFolder
+    );
     await clearPrevSyncRecordByVaultAndProfile(
       db,
       vaultRandomID,
@@ -1128,7 +1228,12 @@ const dispatchOperationToActualV3 = async (
       r.decision === "folder_to_be_deleted_on_both" ||
       r.decision === "folder_to_be_deleted_on_remote"
     ) {
-      await client.deleteFromRemote(r.key, cipher, r.remote!.keyEnc);
+      await client.deleteFromRemote(
+        r.key,
+        cipher,
+        r.remote!.keyEnc,
+        r.remote!.synthesizedFolder
+      );
     }
     await clearPrevSyncRecordByVaultAndProfile(
       db,
@@ -1157,13 +1262,15 @@ export const doActualSync = async (
 ) => {
   console.debug(`concurrency === ${concurrency}`);
   const {
+    onlyMarkSyncedOps,
     folderCreationOps,
     deletionOps,
     uploadDownloads,
     allFilesCount,
     realModifyDeleteCount,
     realTotalCount,
-  } = splitThreeStepsOnEntityMappings(mixedEntityMappings);
+  } = splitFourStepsOnEntityMappings(mixedEntityMappings);
+  // console.debug(`onlyMarkSyncedOps: ${JSON.stringify(onlyMarkSyncedOps)}`);
   // console.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`);
   // console.debug(`deletionOps: ${JSON.stringify(deletionOps)}`);
   // console.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`);
@@ -1198,11 +1305,17 @@ export const doActualSync = async (
     }
   }
 
-  const nested = [folderCreationOps, deletionOps, uploadDownloads];
+  const nested = [
+    onlyMarkSyncedOps,
+    folderCreationOps,
+    deletionOps,
+    uploadDownloads,
+  ];
   const logTexts = [
-    `1. create all folders from shadowest to deepest`,
+    `1. record the items already being synced`,
-    `2. delete files and folders from deepest to shadowest`,
+    `2. create all folders from shadowest to deepest`,
-    `3. upload or download files in parallel, with the desired concurrency=${concurrency}`,
+    `3. delete files and folders from deepest to shadowest`,
+    `4. upload or download files in parallel, with the desired concurrency=${concurrency}`,
   ];
 
   let realCounter = 0;

tests/misc.test.ts

@@ -285,3 +285,56 @@ describe("Misc: special char for dir", () => {
     expect(misc.checkHasSpecialCharForDir("xxx?yyy")).to.be.true;
   });
 });
+
+describe("Misc: Dropbox: should fix the folder name cases", () => {
+  it("should do nothing on empty folders", () => {
+    const input: any[] = [];
+    expect(misc.fixEntityListCasesInplace(input)).to.be.empty;
+  });
+
+  it("should sort folders by length by side effect", () => {
+    const input = [
+      { keyRaw: "aaaa/" },
+      { keyRaw: "bbb/" },
+      { keyRaw: "c/" },
+      { keyRaw: "dd/" },
+    ];
+
+    const output = [
+      { keyRaw: "c/" },
+      { keyRaw: "dd/" },
+      { keyRaw: "bbb/" },
+      { keyRaw: "aaaa/" },
+    ];
+    expect(misc.fixEntityListCasesInplace(input)).to.deep.equal(output);
+  });
+
+  it("should fix folder names", () => {
+    const input = [
+      { keyRaw: "AAA/" },
+      { keyRaw: "aaa/bbb/CCC.md" },
+      { keyRaw: "aaa/BBB/" },
+
+      { keyRaw: "ddd/" },
+      { keyRaw: "DDD/EEE/fff.md" },
+      { keyRaw: "DDD/eee/" },
+
+      { keyRaw: "Ggg/" },
+      { keyRaw: "ggG/hHH你好/Fff世界.md" },
+      { keyRaw: "ggG/Hhh你好/" },
+    ];
+
+    const output = [
+      { keyRaw: "AAA/" },
+      { keyRaw: "ddd/" },
+      { keyRaw: "Ggg/" },
+      { keyRaw: "AAA/BBB/" },
+      { keyRaw: "ddd/eee/" },
+      { keyRaw: "Ggg/Hhh你好/" },
+      { keyRaw: "AAA/BBB/CCC.md" },
+      { keyRaw: "ddd/eee/fff.md" },
+      { keyRaw: "Ggg/Hhh你好/Fff世界.md" },
+    ];
+    expect(misc.fixEntityListCasesInplace(input)).to.deep.equal(output);
+  });
+});