bump to 0.3.19

manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "remotely-save",
   "name": "Remotely Save",
-  "version": "0.3.18",
+  "version": "0.3.19",
   "minAppVersion": "0.13.21",
   "description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
   "author": "fyears",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "remotely-save",
-  "version": "0.3.18",
+  "version": "0.3.19",
   "description": "This is yet another sync plugin for Obsidian app.",
   "scripts": {
     "dev2": "node esbuild.config.mjs",
@@ -67,7 +67,9 @@
     "aws-crt": "^1.10.1",
     "buffer": "^6.0.3",
     "crypto-browserify": "^3.12.0",
+    "delay": "^5.0.0",
     "dropbox": "^10.22.0",
+    "emoji-regex": "^10.1.0",
     "http-status-codes": "^2.2.0",
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",

src/baseTypes.ts

@@ -154,9 +154,12 @@ export interface FileOrFolderMixedState {
 }
 
 export const API_VER_STAT_FOLDER = "0.13.27";
-export const API_VER_REQURL = "0.13.26";
+export const API_VER_REQURL = "0.13.26"; // desktop ver 0.13.26, iOS ver 1.1.1
+export const API_VER_REQURL_ANDROID = "0.14.6"; // Android ver 1.2.1
+
 export const VALID_REQURL =
-  requireApiVersion(API_VER_REQURL) && !Platform.isAndroidApp;
+  (!Platform.isAndroidApp && requireApiVersion(API_VER_REQURL)) ||
+  (Platform.isAndroidApp && requireApiVersion(API_VER_REQURL_ANDROID));
 
 export const DEFAULT_DEBUG_FOLDER = "_debug_remotely_save/";
 export const DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX =

src/misc.ts

@@ -3,6 +3,7 @@ import * as path from "path";
 
 import { base32, base64url } from "rfc4648";
 import XRegExp from "xregexp";
+import emojiRegex from "emoji-regex";
 
 import { log } from "./moreOnLog";
 
@@ -162,6 +163,34 @@ export const isVaildText = (a: string) => {
   );
 };
 
+/**
+ * Use regex to detect a text contains emoji or not.
+ * @param a
+ * @returns
+ */
+export const hasEmojiInText = (a: string) => {
+  const regex = emojiRegex();
+  return regex.test(a);
+};
+
+/**
+ * Convert the headers to a normal object.
+ * @param h
+ * @param toLower
+ * @returns
+ */
+export const headersToRecord = (h: Headers, toLower: boolean = true) => {
+  const res: Record<string, string> = {};
+  h.forEach((v, k) => {
+    if (toLower) {
+      res[k.toLowerCase()] = v;
+    } else {
+      res[k] = v;
+    }
+  });
+  return res;
+};
+
 /**
  * If input is already a folder, returns it as is;
  * And if input is a file, returns its direname.

src/remoteForDropbox.ts

@@ -1,4 +1,6 @@
-import { Dropbox, DropboxAuth, files } from "dropbox";
+import delay from "delay";
+import { Dropbox, DropboxAuth } from "dropbox";
+import type { files, DropboxResponseError, DropboxResponse } from "dropbox";
 import { Vault } from "obsidian";
 import * as path from "path";
 import {
@@ -8,7 +10,13 @@ import {
   OAUTH2_FORCE_EXPIRE_MILLISECONDS,
 } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
-import { bufferToArrayBuffer, getFolderLevels, mkdirpInVault } from "./misc";
+import {
+  bufferToArrayBuffer,
+  getFolderLevels,
+  hasEmojiInText,
+  headersToRecord,
+  mkdirpInVault,
+} from "./misc";
 
 export { Dropbox } from "dropbox";
 
@@ -270,6 +278,73 @@ export const setConfigBySuccessfullAuthInplace = async (
 // Other usual common methods
 ////////////////////////////////////////////////////////////////////////////////
 
+interface ErrSubType {
+  error: {
+    retry_after: number;
+  };
+}
+
+async function retryReq<T>(
+  reqFunc: () => Promise<DropboxResponse<T>>,
+  extraHint: string = ""
+): Promise<DropboxResponse<T>> {
+  const waitSeconds = [1, 2, 4, 8]; // hard code exponential backoff
+  for (let idx = 0; idx < waitSeconds.length; ++idx) {
+    try {
+      if (idx !== 0) {
+        log.warn(
+          `${extraHint === "" ? "" : extraHint + ": "}The ${
+            idx + 1
+          }-th try starts at time ${Date.now()}`
+        );
+      }
+      return await reqFunc();
+    } catch (e: unknown) {
+      const err = e as DropboxResponseError<ErrSubType>;
+      if (err.status === undefined) {
+        // then the err is not DropboxResponseError
+        throw err;
+      }
+      if (err.status !== 429) {
+        // then the err is not "too many requests", give up
+        throw err;
+      }
+
+      if (idx === waitSeconds.length - 1) {
+        // the last retry also failed, give up
+        throw new Error(
+          `${
+            extraHint === "" ? "" : extraHint + ": "
+          }"429 too many requests", after retrying for ${
+            idx + 1
+          } times still failed.`
+        );
+      }
+
+      const headers = headersToRecord(err.headers);
+      const svrSec =
+        err.error.error.retry_after ||
+        parseInt(headers["retry-after"] || "1") ||
+        1;
+      const fallbackSec = waitSeconds[idx];
+      const secMin = Math.max(svrSec, fallbackSec);
+      const secMax = Math.max(secMin * 1.8, 2);
+      log.warn(
+        `${
+          extraHint === "" ? "" : extraHint + ": "
+        }We have "429 too many requests" error of ${
+          idx + 1
+        }-th try, at time ${Date.now()}, and wait for ${secMin} ~ ${secMax} seconds to retry. Original info: ${JSON.stringify(
+          err.error,
+          null,
+          2
+        )}`
+      );
+      await delay.range(secMin * 1000, secMax * 1000);
+    }
+  }
+}
+
 export class WrappedDropboxClient {
   dropboxConfig: DropboxConfig;
   remoteBaseDir: string;
@@ -338,6 +413,13 @@ export class WrappedDropboxClient {
       }
       if (!this.vaultFolderExists) {
         log.info(`remote does not have folder /${this.remoteBaseDir}`);
+
+        if (hasEmojiInText(`/${this.remoteBaseDir}`)) {
+          throw new Error(
+            `/${this.remoteBaseDir}: Error: Dropbox does not support emoji in folder names.`
+          );
+        }
+
         await this.dropbox.filesCreateFolderV2({
           path: `/${this.remoteBaseDir}`,
         });
@@ -377,10 +459,12 @@ export const getRemoteMeta = async (
     // filesGetMetadata doesn't support root folder
     // we instead try to list files
     // if no error occurs, we ensemble a fake result.
-    const rsp = await client.dropbox.filesListFolder({
+    const rsp = await retryReq(() =>
-      path: `/${client.remoteBaseDir}`,
+      client.dropbox.filesListFolder({
-      recursive: false, // don't need to recursive here
+        path: `/${client.remoteBaseDir}`,
-    });
+        recursive: false, // don't need to recursive here
+      })
+    );
     if (rsp.status !== 200) {
       throw Error(JSON.stringify(rsp));
     }
@@ -395,9 +479,11 @@ export const getRemoteMeta = async (
 
   const key = getDropboxPath(fileOrFolderPath, client.remoteBaseDir);
 
-  const rsp = await client.dropbox.filesGetMetadata({
+  const rsp = await retryReq(() =>
-    path: key,
+    client.dropbox.filesGetMetadata({
-  });
+      path: key,
+    })
+  );
   if (rsp.status !== 200) {
     throw Error(JSON.stringify(rsp));
   }
@@ -423,6 +509,12 @@ export const uploadToRemote = async (
   }
   uploadFile = getDropboxPath(uploadFile, client.remoteBaseDir);
 
+  if (hasEmojiInText(uploadFile)) {
+    throw new Error(
+      `${uploadFile}: Error: Dropbox does not support emoji in file / folder names.`
+    );
+  }
+
   const isFolder = fileOrFolderPath.endsWith("/");
 
   if (isFolder && isRecursively) {
@@ -438,11 +530,19 @@ export const uploadToRemote = async (
         // created, pass
       } else {
         try {
-          await client.dropbox.filesCreateFolderV2({
+          await retryReq(
-            path: uploadFile,
+            () =>
-          });
+              client.dropbox.filesCreateFolderV2({
+                path: uploadFile,
+              }),
+            fileOrFolderPath
+          );
           foldersCreatedBefore?.add(uploadFile);
-        } catch (err) {
+        } catch (e: unknown) {
+          const err = e as DropboxResponseError<files.CreateFolderError>;
+          if (err.status === undefined) {
+            throw err;
+          }
           if (err.status === 409) {
             // pass
             foldersCreatedBefore?.add(uploadFile);
@@ -455,10 +555,14 @@ export const uploadToRemote = async (
       return res;
     } else {
       // if encrypted, upload a fake file with the encrypted file name
-      await client.dropbox.filesUpload({
+      await retryReq(
-        path: uploadFile,
+        () =>
-        contents: "",
+          client.dropbox.filesUpload({
-      });
+            path: uploadFile,
+            contents: "",
+          }),
+        fileOrFolderPath
+      );
       return await getRemoteMeta(client, uploadFile);
     }
   } else {
@@ -480,13 +584,19 @@ export const uploadToRemote = async (
     }
     // in dropbox, we don't need to create folders before uploading! cool!
     // TODO: filesUploadSession for larger files (>=150 MB)
-    await client.dropbox.filesUpload({
+
-      path: uploadFile,
+    await retryReq(
-      contents: remoteContent,
+      () =>
-      mode: {
+        client.dropbox.filesUpload({
-        ".tag": "overwrite",
+          path: uploadFile,
-      },
+          contents: remoteContent,
-    });
+          mode: {
+            ".tag": "overwrite",
+          },
+        }),
+      fileOrFolderPath
+    );
+
     // we want to mark that parent folders are created
     if (foldersCreatedBefore !== undefined) {
       const dirs = getFolderLevels(uploadFile).map((x) =>
@@ -554,9 +664,13 @@ const downloadFromRemoteRaw = async (
 ) => {
   await client.init();
   const key = getDropboxPath(fileOrFolderPath, client.remoteBaseDir);
-  const rsp = await client.dropbox.filesDownload({
+  const rsp = await retryReq(
-    path: key,
+    () =>
-  });
+      client.dropbox.filesDownload({
+        path: key,
+      }),
+    fileOrFolderPath
+  );
   if ((rsp.result as any).fileBlob !== undefined) {
     // we get a Blob
     const content = (rsp.result as any).fileBlob as Blob;
@@ -631,9 +745,13 @@ export const deleteFromRemote = async (
 
   await client.init();
   try {
-    await client.dropbox.filesDeleteV2({
+    await retryReq(
-      path: remoteFileName,
+      () =>
-    });
+        client.dropbox.filesDeleteV2({
+          path: remoteFileName,
+        }),
+      fileOrFolderPath
+    );
   } catch (err) {
     console.error("some error while deleting");
     console.error(err);

src/remoteForOnedrive.ts

@@ -8,7 +8,7 @@ import type {
 import cloneDeep from "lodash/cloneDeep";
 import { request, requestUrl, requireApiVersion, Vault } from "obsidian";
 import {
-  API_VER_REQURL,
+  VALID_REQURL,
   COMMAND_CALLBACK_ONEDRIVE,
   DEFAULT_CONTENT_TYPE,
   OAUTH2_FORCE_EXPIRE_MILLISECONDS,
@@ -484,7 +484,7 @@ export class WrappedOnedriveClient {
   deleteJson = async (pathFragOrig: string) => {
     const theUrl = this.buildUrl(pathFragOrig);
     log.debug(`deleteJson, theUrl=${theUrl}`);
-    if (requireApiVersion(API_VER_REQURL)) {
+    if (VALID_REQURL) {
       await requestUrl({
         url: theUrl,
         method: "DELETE",
@@ -508,7 +508,7 @@ export class WrappedOnedriveClient {
     // TODO:
     // 20220401: On Android, requestUrl has issue that text becomes base64.
     // Use fetch everywhere instead!
-    if (false /*requireApiVersion(API_VER_REQURL)*/) {
+    if (false /*VALID_REQURL*/) {
       await requestUrl({
         url: theUrl,
         method: "PUT",
@@ -556,7 +556,7 @@ export class WrappedOnedriveClient {
     // TODO:
     // 20220401: On Android, requestUrl has issue that text becomes base64.
     // Use fetch everywhere instead!
-    if (false /*requireApiVersion(API_VER_REQURL)*/) {
+    if (false /*VALID_REQURL*/) {
       const res = await requestUrl({
         url: theUrl,
         method: "PUT",
@@ -808,7 +808,7 @@ const downloadFromRemoteRaw = async (
     `${key}?$select=@microsoft.graph.downloadUrl`
   );
   const downloadUrl: string = rsp["@microsoft.graph.downloadUrl"];
-  if (requireApiVersion(API_VER_REQURL)) {
+  if (VALID_REQURL) {
     const content = (await requestUrl({ url: downloadUrl })).arrayBuffer;
     return content;
   } else {

src/settings.ts

@@ -7,6 +7,8 @@ import {
   Platform,
   requireApiVersion,
 } from "obsidian";
+import type { TextComponent } from "obsidian";
+import { createElement, Eye, EyeOff } from "lucide";
 import {
   API_VER_REQURL,
   DEFAULT_DEBUG_FOLDER,
@@ -627,8 +629,33 @@ class ExportSettingsQrCodeModal extends Modal {
   }
 }
 
+const getEyesElements = () => {
+  const eyeEl = createElement(Eye);
+  const eyeOffEl = createElement(EyeOff);
+  return {
+    eye: eyeEl.outerHTML,
+    eyeOff: eyeOffEl.outerHTML,
+  };
+};
+
+const wrapTextWithPasswordHide = (text: TextComponent) => {
+  const { eye, eyeOff } = getEyesElements();
+  const hider = text.inputEl.insertAdjacentElement("afterend", createSpan());
+  // the init type of hider is "hidden" === eyeOff === password
+  hider.innerHTML = eyeOff;
+  hider.addEventListener("click", (e) => {
+    const isText = text.inputEl.getAttribute("type") === "text";
+    text.inputEl.setAttribute("type", isText ? "password" : "text");
+    hider.innerHTML = isText ? eyeOff : eye;
+  });
+
+  // the init type of text el is password
+  text.inputEl.setAttribute("type", "password");
+  return text;
+};
+
 export class RemotelySaveSettingTab extends PluginSettingTab {
-  plugin: RemotelySavePlugin;
+  readonly plugin: RemotelySavePlugin;
 
   constructor(app: App, plugin: RemotelySavePlugin) {
     super(app, plugin);
@@ -658,14 +685,15 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
     new Setting(passwordDiv)
       .setName(t("settings_password"))
       .setDesc(t("settings_password_desc"))
-      .addText((text) =>
+      .addText((text) => {
+        wrapTextWithPasswordHide(text);
         text
           .setPlaceholder("")
           .setValue(`${this.plugin.settings.password}`)
           .onChange(async (value) => {
             newPassword = value.trim();
-          })
+          });
-      )
+      })
       .addButton(async (button) => {
         button.setButtonText(t("confirm"));
         button.onClick(async () => {
@@ -816,29 +844,31 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
 
     new Setting(s3Div)
       .setName(t("settings_s3_accesskeyid"))
-      .setDesc(t("settings_s3_accesskeyid"))
+      .setDesc(t("settings_s3_accesskeyid_desc"))
-      .addText((text) =>
+      .addText((text) => {
+        wrapTextWithPasswordHide(text);
         text
           .setPlaceholder("")
           .setValue(`${this.plugin.settings.s3.s3AccessKeyID}`)
           .onChange(async (value) => {
             this.plugin.settings.s3.s3AccessKeyID = value.trim();
             await this.plugin.saveSettings();
-          })
+          });
-      );
+      });
 
     new Setting(s3Div)
       .setName(t("settings_s3_secretaccesskey"))
-      .setDesc(t("settings_s3_secretaccesskey"))
+      .setDesc(t("settings_s3_secretaccesskey_desc"))
-      .addText((text) =>
+      .addText((text) => {
+        wrapTextWithPasswordHide(text);
         text
           .setPlaceholder("")
           .setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`)
           .onChange(async (value) => {
             this.plugin.settings.s3.s3SecretAccessKey = value.trim();
             await this.plugin.saveSettings();
-          })
+          });
-      );
+      });
 
     new Setting(s3Div)
       .setName(t("settings_s3_bucketname"))
@@ -877,11 +907,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
     if (VALID_REQURL) {
       new Setting(s3Div)
         .setName(t("settings_s3_bypasscorslocally"))
-        .setDesc(
+        .setDesc(t("settings_s3_bypasscorslocally_desc"))
-          t("settings_s3_bypasscorslocally_desc", {
-            ver: API_VER_REQURL,
-          })
-        )
         .addDropdown((dropdown) => {
           dropdown
             .addOption("disable", t("disable"))
@@ -1297,15 +1323,9 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
     });
 
     if (!VALID_REQURL) {
-      if (Platform.isAndroidApp) {
+      webdavDiv.createEl("p", {
-        webdavDiv.createEl("p", {
+        text: t("settings_webdav_cors_os"),
-          text: t("settings_webdav_cors_android"),
+      });
-        });
-      } else {
-        webdavDiv.createEl("p", {
-          text: t("settings_webdav_cors_otheros"),
-        });
-      }
 
       webdavDiv.createEl("p", {
         text: t("settings_webdav_cors"),
@@ -1346,7 +1366,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
     new Setting(webdavDiv)
       .setName(t("settings_webdav_user"))
       .setDesc(t("settings_webdav_user_desc"))
-      .addText((text) =>
+      .addText((text) => {
+        wrapTextWithPasswordHide(text);
         text
           .setPlaceholder("")
           .setValue(this.plugin.settings.webdav.username)
@@ -1359,13 +1380,14 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
               this.plugin.settings.webdav.depth = "auto_unknown";
             }
             await this.plugin.saveSettings();
-          })
+          });
-      );
+      });
 
     new Setting(webdavDiv)
       .setName(t("settings_webdav_password"))
       .setDesc(t("settings_webdav_password_desc"))
-      .addText((text) =>
+      .addText((text) => {
+        wrapTextWithPasswordHide(text);
         text
           .setPlaceholder("")
           .setValue(this.plugin.settings.webdav.password)
@@ -1378,8 +1400,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
               this.plugin.settings.webdav.depth = "auto_unknown";
             }
             await this.plugin.saveSettings();
-          })
+          });
-      );
+      });
 
     new Setting(webdavDiv)
       .setName(t("settings_webdav_auth"))

src/sync.ts

@@ -459,9 +459,15 @@ const ensembleMixedStates = async (
         results[key].changeLocalMtimeUsingMapping =
           r.changeLocalMtimeUsingMapping;
       } else {
-        results[key] = r;
+        // So, the file doesn't exist,
-        results[key].existLocal = false; // we have already checked local
+        // except that it existed in the "renamed to" history records.
-        results[key].existRemote = false; // we have already checked remote
+        // Most likely because that the user deleted the file while Obsidian was closed,
+        // so Obsidian could not track the deletions.
+        // We are not sure how to deal with this, so do not generate anything here!
+        // // // The following 3 lines are of old logic, and have been removed:
+        // // results[key] = r;
+        // // results[key].existLocal = false; // we have already checked local
+        // // results[key].existRemote = false; // we have already checked remote
       }
     } else {
       throw Error(

versions.json

@@ -1,4 +1,4 @@
 {
   "0.3.2": "0.12.15",
-  "0.3.18": "0.13.21"
+  "0.3.19": "0.13.21"
 }