s3 adds support for remote prefix

src/langs/en.json

@@ -70,7 +70,7 @@
   "modal_remoteprefix_title": "You are changing the remote prefix config",
   "modal_remoteprefix_shortdesc": "1. The plugin would NOT automatically move the content from the old directory to the new one directly on the remote. Everything syncs from the beginning again.\n2. If you set the string to the empty, the prefix will be empty and the files will be saved at the root of the bucket.\n3. The remote directory name itself would not be encrypted even you've set an E2E password.\n4. Some special char like '?', '/', '\\' are not allowed. Spaces in the beginning or in the end are also trimmed.",
   "modal_remoteprefix_invaliddirhint": "Your input contains special characters like '?', '/', '\\' which are not allowed.",
-  "modal_remoteprefix_tosave": "The prefix to save is {{{prefix}}}",
+  "modal_remoteprefix_tosave": "The prefix to save is \"{{{prefix}}}\"",
   "modal_remoteprefix_secondconfirm_empty": "The prefix is empty and the files will be saved at the root of the bucket.",
   "modal_remoteprefix_secondconfirm_change": "Confirm To Change",
   "modal_remoteprefix_notice": "New remote prefix config saved!",

src/main.ts

@@ -233,7 +233,7 @@ export default class RemotelySavePlugin extends Plugin {
         this.app.vault.getName(),
         () => self.saveSettings()
       );
-      const remoteRsp = await client.listFromRemote();
+      const remoteRsp = await client.listAllFromRemote();
       // log.debug(remoteRsp);
 
       if (this.settings.currLogLevel === "info") {

src/remote.ts

@@ -164,19 +164,18 @@ export class RemoteClient {
     }
   };
 
-  listFromRemote = async (prefix?: string) => {
+  listAllFromRemote = async () => {
     if (this.serviceType === "s3") {
-      return await s3.listFromRemote(
+      return await s3.listAllFromRemote(
         s3.getS3Client(this.s3Config),
-        this.s3Config,
+        this.s3Config
-        prefix
       );
     } else if (this.serviceType === "webdav") {
-      return await webdav.listFromRemote(this.webdavClient, prefix);
+      return await webdav.listAllFromRemote(this.webdavClient);
     } else if (this.serviceType === "dropbox") {
-      return await dropbox.listFromRemote(this.dropboxClient, prefix);
+      return await dropbox.listAllFromRemote(this.dropboxClient);
     } else if (this.serviceType === "onedrive") {
-      return await onedrive.listFromRemote(this.onedriveClient, prefix);
+      return await onedrive.listAllFromRemote(this.onedriveClient);
     } else {
       throw Error(`not supported service type ${this.serviceType}`);
     }

src/remoteForDropbox.ts

@@ -615,13 +615,7 @@ export const uploadToRemote = async (
   }
 };
 
-export const listFromRemote = async (
+export const listAllFromRemote = async (client: WrappedDropboxClient) => {
-  client: WrappedDropboxClient,
-  prefix?: string
-) => {
-  if (prefix !== undefined) {
-    throw Error("prefix not supported (yet)");
-  }
   await client.init();
   let res = await client.dropbox.filesListFolder({
     path: `/${client.remoteBaseDir}`,

src/remoteForOnedrive.ts

@@ -602,15 +602,8 @@ export const getOnedriveClient = (
  * Use delta api to list all files and folders
  * https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta?view=odsp-graph-online
  * @param client
- * @param prefix
  */
-export const listFromRemote = async (
+export const listAllFromRemote = async (client: WrappedOnedriveClient) => {
-  client: WrappedOnedriveClient,
-  prefix?: string
-) => {
-  if (prefix !== undefined) {
-    throw Error("prefix not supported (yet)");
-  }
   await client.init();
 
   const NEXT_LINK_KEY = "@odata.nextLink";

src/remoteForS3.ts

@@ -174,7 +174,7 @@ export const simpleTransRemotePrefix = (x: string) => {
     return "";
   }
   let y = path.posix.normalize(x.trim());
-  if (y === undefined || y === "" || y === "/") {
+  if (y === undefined || y === "" || y === "/" || y === ".") {
     return "";
   }
   if (y.startsWith("/")) {
@@ -186,9 +186,41 @@ export const simpleTransRemotePrefix = (x: string) => {
   return y;
 };
 
-const fromS3ObjectToRemoteItem = (x: S3ObjectType) => {
+const getRemoteWithPrefixPath = (
+  fileOrFolderPath: string,
+  remotePrefix: string
+) => {
+  let key = fileOrFolderPath;
+  if (fileOrFolderPath === "/" || fileOrFolderPath === "") {
+    // special
+    key = remotePrefix;
+  }
+  if (!fileOrFolderPath.startsWith("/")) {
+    key = `${remotePrefix}${fileOrFolderPath}`;
+  }
+  return key;
+};
+
+const getLocalNoPrefixPath = (
+  fileOrFolderPathWithRemotePrefix: string,
+  remotePrefix: string
+) => {
+  if (
+    !(
+      fileOrFolderPathWithRemotePrefix === `${remotePrefix}` ||
+      fileOrFolderPathWithRemotePrefix.startsWith(`${remotePrefix}`)
+    )
+  ) {
+    throw Error(
+      `"${fileOrFolderPathWithRemotePrefix}" doesn't starts with "${remotePrefix}"`
+    );
+  }
+  return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length);
+};
+
+const fromS3ObjectToRemoteItem = (x: S3ObjectType, remotePrefix: string) => {
   return {
-    key: x.Key,
+    key: getLocalNoPrefixPath(x.Key, remotePrefix),
     lastModified: x.LastModified.valueOf(),
     size: x.Size,
     remoteType: "s3",
@@ -197,11 +229,12 @@ const fromS3ObjectToRemoteItem = (x: S3ObjectType) => {
 };
 
 const fromS3HeadObjectToRemoteItem = (
-  key: string,
+  fileOrFolderPathWithRemotePrefix: string,
-  x: HeadObjectCommandOutput
+  x: HeadObjectCommandOutput,
+  remotePrefix: string
 ) => {
   return {
-    key: key,
+    key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix),
     lastModified: x.LastModified.valueOf(),
     size: x.ContentLength,
     remoteType: "s3",
@@ -256,16 +289,26 @@ export const getS3Client = (s3Config: S3Config) => {
 export const getRemoteMeta = async (
   s3Client: S3Client,
   s3Config: S3Config,
-  fileOrFolderPath: string
+  fileOrFolderPathWithRemotePrefix: string
 ) => {
+  if (
+    s3Config.remotePrefix !== "" &&
+    !fileOrFolderPathWithRemotePrefix.startsWith(s3Config.remotePrefix)
+  ) {
+    throw Error(`s3 getRemoteMeta should only accept prefix-ed path`);
+  }
   const res = await s3Client.send(
     new HeadObjectCommand({
       Bucket: s3Config.s3BucketName,
-      Key: fileOrFolderPath,
+      Key: fileOrFolderPathWithRemotePrefix,
     })
   );
 
-  return fromS3HeadObjectToRemoteItem(fileOrFolderPath, res);
+  return fromS3HeadObjectToRemoteItem(
+    fileOrFolderPathWithRemotePrefix,
+    res,
+    s3Config.remotePrefix
+  );
 };
 
 export const uploadToRemote = async (
@@ -283,6 +326,7 @@ export const uploadToRemote = async (
   if (password !== "") {
     uploadFile = remoteEncryptedKey;
   }
+  uploadFile = getRemoteWithPrefixPath(uploadFile, s3Config.remotePrefix);
   const isFolder = fileOrFolderPath.endsWith("/");
 
   if (isFolder && isRecursively) {
@@ -350,16 +394,16 @@ export const uploadToRemote = async (
   }
 };
 
-export const listFromRemote = async (
+const listFromRemoteRaw = async (
   s3Client: S3Client,
   s3Config: S3Config,
-  prefix?: string
+  prefixOfRawKeys?: string
 ) => {
   const confCmd = {
     Bucket: s3Config.s3BucketName,
   } as ListObjectsV2CommandInput;
-  if (prefix !== undefined) {
+  if (prefixOfRawKeys !== undefined && prefixOfRawKeys !== "") {
-    confCmd.Prefix = prefix;
+    confCmd.Prefix = prefixOfRawKeys;
   }
 
   const contents = [] as _Object[];
@@ -388,11 +432,22 @@ export const listFromRemote = async (
   } while (isTruncated);
 
   // ensemble fake rsp
+  // in the end, we need to transform the response list
+  // back to the local contents-alike list
   return {
-    Contents: contents.map((x) => fromS3ObjectToRemoteItem(x)),
+    Contents: contents.map((x) =>
+      fromS3ObjectToRemoteItem(x, s3Config.remotePrefix)
+    ),
   };
 };
 
+export const listAllFromRemote = async (
+  s3Client: S3Client,
+  s3Config: S3Config
+) => {
+  return await listFromRemoteRaw(s3Client, s3Config, s3Config.remotePrefix);
+};
+
 /**
  * The Body of resp of aws GetObject has mix types
  * and we want to get ArrayBuffer here.
@@ -422,12 +477,18 @@ const getObjectBodyToArrayBuffer = async (
 const downloadFromRemoteRaw = async (
   s3Client: S3Client,
   s3Config: S3Config,
-  fileOrFolderPath: string
+  fileOrFolderPathWithRemotePrefix: string
 ) => {
+  if (
+    s3Config.remotePrefix !== "" &&
+    !fileOrFolderPathWithRemotePrefix.startsWith(s3Config.remotePrefix)
+  ) {
+    throw Error(`downloadFromRemoteRaw should only accept prefix-ed path`);
+  }
   const data = await s3Client.send(
     new GetObjectCommand({
       Bucket: s3Config.s3BucketName,
-      Key: fileOrFolderPath,
+      Key: fileOrFolderPathWithRemotePrefix,
     })
   );
   const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
@@ -462,6 +523,7 @@ export const downloadFromRemote = async (
     if (password !== "") {
       downloadFile = remoteEncryptedKey;
     }
+    downloadFile = getRemoteWithPrefixPath(downloadFile, s3Config.remotePrefix);
     const remoteContent = await downloadFromRemoteRaw(
       s3Client,
       s3Config,
@@ -501,6 +563,10 @@ export const deleteFromRemote = async (
   if (password !== "") {
     remoteFileName = remoteEncryptedKey;
   }
+  remoteFileName = getRemoteWithPrefixPath(
+    remoteFileName,
+    s3Config.remotePrefix
+  );
   await s3Client.send(
     new DeleteObjectCommand({
       Bucket: s3Config.s3BucketName,
@@ -509,7 +575,7 @@ export const deleteFromRemote = async (
   );
 
   if (fileOrFolderPath.endsWith("/") && password === "") {
-    const x = await listFromRemote(s3Client, s3Config, fileOrFolderPath);
+    const x = await listFromRemoteRaw(s3Client, s3Config, remoteFileName);
     x.Contents.forEach(async (element) => {
       await s3Client.send(
         new DeleteObjectCommand({

src/remoteForWebdav.ts

@@ -405,13 +405,7 @@ export const uploadToRemote = async (
   }
 };
 
-export const listFromRemote = async (
+export const listAllFromRemote = async (client: WrappedWebdavClient) => {
-  client: WrappedWebdavClient,
-  prefix?: string
-) => {
-  if (prefix !== undefined) {
-    throw Error("prefix not supported");
-  }
   await client.init();
 
   let contents = [] as FileStat[];

src/sync.ts

@@ -219,6 +219,7 @@ export const parseRemoteItems = async (
 
     let r = {} as FileOrFolderMixedState;
     if (backwardMapping !== undefined) {
+      // log.debug(`backwardMapping=${backwardMapping}`);
       key = backwardMapping.localKey;
       const mtimeRemote = backwardMapping.localMtime || entry.lastModified;
 
@@ -236,7 +237,7 @@ export const parseRemoteItems = async (
         changeRemoteMtimeUsingMapping: true,
       };
     } else {
-      // do not have backwardMapping
+      // log.debug(`do not have backwardMapping`);
       r = {
         key: key,
         existRemote: true,