new sync algo, squashed commit of the following:

commit 692bb794aea5609b9e9abf5228620f4479e50983 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 27 17:52:43 2022 +0800 bump to 0.3.0 commit 77335412ad2da2b5bd1ed5075061a5af006e3c36 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 27 17:50:57 2022 +0800 change titles for minimal intrusive design commit 2812adebb84344d384749a62acb63fd0c6fd509d Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 27 17:30:53 2022 +0800 remove syncv1 commit 22fc24a76c9851740bbc7c0177d1cb2526e15d8b Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 27 17:30:27 2022 +0800 full notice to any one commit d56ea24a78f6dc1fbea2740011ee1cea9c403d4c Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 23:11:14 2022 +0800 fix test commit 759f82593bbfb9b49079cfd80dbadbbafc0287e5 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 21:59:25 2022 +0800 obfuscate metadata on remote commit 9b6bf05288af0e52d0f02468e5ac8757f4f896df Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 21:33:52 2022 +0800 avoid re-uploading if meta not changed commit 03be1453764e48e99207f44467ee4d5801072ed8 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 00:35:52 2022 +0800 add password condition commit 7f899f7c2572df3e2a35e16cbdaab94db113f366 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 26 00:22:58 2022 +0800 add decision branch for easier debugging commit cf4071bf3156356ae6ec3a9cb187c2cb541d1b17 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Fri Feb 25 23:40:52 2022 +0800 change folder error commit 964493dd998699a1d53018a201637bda192c4baa Author: fyears <1142836+fyears@users.noreply.github.com> Date: Fri Feb 25 23:09:44 2022 +0800 finnaly remote remove should be working commit 2888e65452f9c0e1dde6005f012c3ee0a6313c3f Author: fyears <1142836+fyears@users.noreply.github.com> Date: Fri Feb 25 01:18:15 2022 +0800 optimize comparation commit 024936951d6180b1296c2a5d56c5bf5d468e9ae7 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Fri Feb 25 01:14:44 2022 +0800 allow uploading extra meta commit 007006701d536da2b4b46389941980579bbc4e67 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Thu Feb 24 01:08:58 2022 +0800 add logic to fetch extra meta commit c9d3a05ca1bf45c06f22233124670e5e45b5f5f1 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Thu Feb 24 00:29:16 2022 +0800 another way to deal with trash commit 82d455f8bf60f7bac8eb4e299a2ca44c331a6d7f Author: fyears <1142836+fyears@users.noreply.github.com> Date: Thu Feb 24 00:28:51 2022 +0800 add return buffer for downloading commit b8e6b79bc078def2854bc73578b7f520cc39ab34 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Feb 23 00:16:06 2022 +0800 half way to actual sync commit 90cceb1411b46af9741f2caa3ab8beafaf69c1b2 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Tue Feb 22 23:36:09 2022 +0800 cleaner way to remember the folder being kept commit c1afb763cc4e0f7905c83e0a8eb20f8ed969a279 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Tue Feb 22 00:03:21 2022 +0800 simplified way to get plans for sync algo v2 commit 5c5ecce39eb3854900f1f5b79c7beb189e78a6a7 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 23:13:58 2022 +0800 archive the old sync algo v1 doc commit 75cdfa1ee9834600f83ded6672b102de2c5f9616 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 23:13:14 2022 +0800 simplify sync algo v2 commit dc9275835d961de07efcbaa81657e4745242e72a Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 22:58:57 2022 +0800 add way to skip recording removing commit f9712ef96021dfed4ae33e6c649f78e940b7e530 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 09:38:09 2022 +0800 fix sync commit 9007dcf22ef317dde36ac4f1dd589d05cc8d5cf6 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 00:54:21 2022 +0800 fix assignment commit 77abee6ad443b62353ed3913e0945ea7d1314f87 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 00:28:43 2022 +0800 draft of sync v2 commit a0c26238bf60692e095cfd8527abf937255b56d4 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 21 00:23:19 2022 +0800 how to deal with folders commit c10f92a7e8d3c4a4f4c585e39e0abad1a5376c02 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 20 23:39:16 2022 +0800 helper func to get parents commit f903c98b3b7b9d1e785d04b9fc0cfcafdc6e5661 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 20 23:31:21 2022 +0800 add optional ending slash to getFolderLevels commit 2d67c9b2b941e676588fa43ab289fab79f567e5e Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 20 14:44:44 2022 +0800 update sync algo v2 commit 491ed1bb85759df2411706fd02d740acb5598ce6 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Feb 20 14:34:51 2022 +0800 dry run mode commit dfd588dcef512ba7dfe760008bcf97138b97e339 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 19 23:14:32 2022 +0800 prettier commit eae580f882a045ae9df799b816e68c3500704131 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sat Feb 19 23:12:29 2022 +0800 draft design for sync algo v2
2022-02-27 17:54:31 +08:00
parent 6045155fbb
commit df2d98272c
20 changed files with 1281 additions and 467 deletions
@@ -0,0 +1,87 @@
+import isEqual from "lodash/isEqual";
+import { base64url } from "rfc4648";
+import { reverseString } from "./misc";
+
+const DEFAULT_README_FOR_METADATAONREMOTE =
+  "Do NOT edit or delete the file manually. This file is for the plugin remotely-save to store some necessary meta data on the remote services. Its content is slightly obfuscated.";
+
+const DEFAULT_VERSION_FOR_METADATAONREMOTE = "20220220";
+
+export const DEFAULT_FILE_NAME_FOR_METADATAONREMOTE =
+  "_remotely-save-metadata-on-remote.json";
+
+export const DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2 =
+  "_remotely-save-metadata-on-remote.bin";
+
+export interface DeletionOnRemote {
+  key: string;
+  actionWhen: number;
+}
+
+export interface MetadataOnRemote {
+  version?: string;
+  generatedWhen?: number;
+  deletions?: DeletionOnRemote[];
+}
+
+export const isEqualMetadataOnRemote = (
+  a: MetadataOnRemote,
+  b: MetadataOnRemote
+) => {
+  const m1 = a === undefined ? { deletions: [] } : a;
+  const m2 = b === undefined ? { deletions: [] } : b;
+
+  // we only need to compare deletions
+  const d1 = m1.deletions === undefined ? [] : m1.deletions;
+  const d2 = m2.deletions === undefined ? [] : m2.deletions;
+  return isEqual(d1, d2);
+};
+
+export const serializeMetadataOnRemote = (x: MetadataOnRemote) => {
+  const y = x;
+
+  if (y["version"] === undefined) {
+    y["version"] === DEFAULT_VERSION_FOR_METADATAONREMOTE;
+  }
+  if (y["generatedWhen"] === undefined) {
+    y["generatedWhen"] = Date.now();
+  }
+  if (y["deletions"] === undefined) {
+    y["deletions"] = [];
+  }
+
+  const z = {
+    readme: DEFAULT_README_FOR_METADATAONREMOTE,
+    d: reverseString(
+      base64url.stringify(Buffer.from(JSON.stringify(x), "utf-8"), {
+        pad: false,
+      })
+    ),
+  };
+
+  return JSON.stringify(z, null, 2);
+};
+
+export const deserializeMetadataOnRemote = (x: string | ArrayBuffer) => {
+  let y1 = "";
+  if (typeof x === "string") {
+    y1 = x;
+  } else {
+    y1 = new TextDecoder().decode(x);
+  }
+  const y2: any = JSON.parse(y1);
+
+  if (!("readme" in y2 && "d" in y2)) {
+    throw Error("invalid remote meta data file!");
+  }
+
+  const y3 = JSON.parse(
+    (
+      base64url.parse(reverseString(y2["d"]), {
+        out: Buffer.allocUnsafe as any,
+        loose: true,
+      }) as Buffer
+    ).toString("utf-8")
+  ) as MetadataOnRemote;
+  return y3;
+};