properly deal with 429 too many requests for dropbox

package.json

@@ -67,6 +67,7 @@
     "aws-crt": "^1.10.1",
     "buffer": "^6.0.3",
     "crypto-browserify": "^3.12.0",
+    "delay": "^5.0.0",
     "dropbox": "^10.22.0",
     "emoji-regex": "^10.1.0",
     "http-status-codes": "^2.2.0",

src/remoteForDropbox.ts

@@ -1,4 +1,6 @@
-import { Dropbox, DropboxAuth, files, DropboxResponseError } from "dropbox";
+import delay from "delay";
+import { Dropbox, DropboxAuth } from "dropbox";
+import type { files, DropboxResponseError, DropboxResponse } from "dropbox";
 import { Vault } from "obsidian";
 import * as path from "path";
 import {
@@ -276,6 +278,73 @@ export const setConfigBySuccessfullAuthInplace = async (
 // Other usual common methods
 ////////////////////////////////////////////////////////////////////////////////
 
+interface ErrSubType {
+  error: {
+    retry_after: number;
+  };
+}
+
+async function retryReq<T>(
+  reqFunc: () => Promise<DropboxResponse<T>>,
+  extraHint: string = ""
+): Promise<DropboxResponse<T>> {
+  const waitSeconds = [1, 2, 4, 8]; // hard code exponential backoff
+  for (let idx = 0; idx < waitSeconds.length; ++idx) {
+    try {
+      if (idx !== 0) {
+        log.warn(
+          `${extraHint === "" ? "" : extraHint + ": "}The ${
+            idx + 1
+          }-th try starts at time ${Date.now()}`
+        );
+      }
+      return await reqFunc();
+    } catch (e: unknown) {
+      const err = e as DropboxResponseError<ErrSubType>;
+      if (err.status === undefined) {
+        // then the err is not DropboxResponseError
+        throw err;
+      }
+      if (err.status !== 429) {
+        // then the err is not "too many requests", give up
+        throw err;
+      }
+
+      if (idx === waitSeconds.length - 1) {
+        // the last retry also failed, give up
+        throw new Error(
+          `${
+            extraHint === "" ? "" : extraHint + ": "
+          }"429 too many requests", after retrying for ${
+            idx + 1
+          } times still failed.`
+        );
+      }
+
+      const headers = headersToRecord(err.headers);
+      const svrSec =
+        err.error.error.retry_after ||
+        parseInt(headers["retry-after"] || "1") ||
+        1;
+      const fallbackSec = waitSeconds[idx];
+      const secMin = Math.max(svrSec, fallbackSec);
+      const secMax = Math.max(secMin * 1.8, 2);
+      log.warn(
+        `${
+          extraHint === "" ? "" : extraHint + ": "
+        }We have "429 too many requests" error of ${
+          idx + 1
+        }-th try, at time ${Date.now()}, and wait for ${secMin} ~ ${secMax} seconds to retry. Original info: ${JSON.stringify(
+          err.error,
+          null,
+          2
+        )}`
+      );
+      await delay.range(secMin * 1000, secMax * 1000);
+    }
+  }
+}
+
 export class WrappedDropboxClient {
   dropboxConfig: DropboxConfig;
   remoteBaseDir: string;
@@ -390,10 +459,12 @@ export const getRemoteMeta = async (
     // filesGetMetadata doesn't support root folder
     // we instead try to list files
     // if no error occurs, we ensemble a fake result.
-    const rsp = await client.dropbox.filesListFolder({
+    const rsp = await retryReq(() =>
-      path: `/${client.remoteBaseDir}`,
+      client.dropbox.filesListFolder({
-      recursive: false, // don't need to recursive here
+        path: `/${client.remoteBaseDir}`,
-    });
+        recursive: false, // don't need to recursive here
+      })
+    );
     if (rsp.status !== 200) {
       throw Error(JSON.stringify(rsp));
     }
@@ -408,9 +479,11 @@ export const getRemoteMeta = async (
 
   const key = getDropboxPath(fileOrFolderPath, client.remoteBaseDir);
 
-  const rsp = await client.dropbox.filesGetMetadata({
+  const rsp = await retryReq(() =>
-    path: key,
+    client.dropbox.filesGetMetadata({
-  });
+      path: key,
+    })
+  );
   if (rsp.status !== 200) {
     throw Error(JSON.stringify(rsp));
   }
@@ -457,12 +530,19 @@ export const uploadToRemote = async (
         // created, pass
       } else {
         try {
-          await client.dropbox.filesCreateFolderV2({
+          await retryReq(
-            path: uploadFile,
+            () =>
-          });
+              client.dropbox.filesCreateFolderV2({
+                path: uploadFile,
+              }),
+            fileOrFolderPath
+          );
           foldersCreatedBefore?.add(uploadFile);
         } catch (e: unknown) {
           const err = e as DropboxResponseError<files.CreateFolderError>;
+          if (err.status === undefined) {
+            throw err;
+          }
           if (err.status === 409) {
             // pass
             foldersCreatedBefore?.add(uploadFile);
@@ -475,23 +555,14 @@ export const uploadToRemote = async (
       return res;
     } else {
       // if encrypted, upload a fake file with the encrypted file name
-      try {
+      await retryReq(
-        await client.dropbox.filesUpload({
+        () =>
-          path: uploadFile,
+          client.dropbox.filesUpload({
-          contents: "",
+            path: uploadFile,
-        });
+            contents: "",
-      } catch (e: unknown) {
+          }),
-        const err = e as DropboxResponseError<files.UploadError>;
+        fileOrFolderPath
-        // log.debug(
+      );
-        //   `we are of error: ${JSON.stringify(
-        //     headersToRecord(err.headers),
-        //     null,
-        //     2
-        //   )}, ${err.status}, ${JSON.stringify(err.error, null, 2)}`
-        // );
-        throw err;
-      }
-
       return await getRemoteMeta(client, uploadFile);
     }
   } else {
@@ -513,32 +584,18 @@ export const uploadToRemote = async (
     }
     // in dropbox, we don't need to create folders before uploading! cool!
     // TODO: filesUploadSession for larger files (>=150 MB)
-    try {
+
-      await client.dropbox.filesUpload({
+    await retryReq(
-        path: uploadFile,
+      () =>
-        contents: remoteContent,
+        client.dropbox.filesUpload({
-        mode: {
+          path: uploadFile,
-          ".tag": "overwrite",
+          contents: remoteContent,
-        },
+          mode: {
-      });
+            ".tag": "overwrite",
-    } catch (e: unknown) {
+          },
-      const err = e as DropboxResponseError<files.UploadError>;
+        }),
-      // log.debug(
+      fileOrFolderPath
-      //   `we are of error: ${JSON.stringify(
+    );
-      //     headersToRecord(err.headers),
-      //     null,
-      //     2
-      //   )}, ${err.status}, ${JSON.stringify(err.error, null, 2)}`
-      // );
-      // if (err.status === 429) {
-      //   // too many request
-      // } else if (err.status === 409) {
-      //   // Endpoint Specific Error
-      // } else {
-      //   throw err;
-      // }
-      throw err;
-    }
 
     // we want to mark that parent folders are created
     if (foldersCreatedBefore !== undefined) {
@@ -607,9 +664,13 @@ const downloadFromRemoteRaw = async (
 ) => {
   await client.init();
   const key = getDropboxPath(fileOrFolderPath, client.remoteBaseDir);
-  const rsp = await client.dropbox.filesDownload({
+  const rsp = await retryReq(
-    path: key,
+    () =>
-  });
+      client.dropbox.filesDownload({
+        path: key,
+      }),
+    fileOrFolderPath
+  );
   if ((rsp.result as any).fileBlob !== undefined) {
     // we get a Blob
     const content = (rsp.result as any).fileBlob as Blob;
@@ -684,9 +745,13 @@ export const deleteFromRemote = async (
 
   await client.init();
   try {
-    await client.dropbox.filesDeleteV2({
+    await retryReq(
-      path: remoteFileName,
+      () =>
-    });
+        client.dropbox.filesDeleteV2({
+          path: remoteFileName,
+        }),
+      fileOrFolderPath
+    );
   } catch (err) {
     console.error("some error while deleting");
     console.error(err);