remove dup files

pro/src/clearDupFiles.ts

@@ -0,0 +1,34 @@
+import type { FakeFsLocal } from "../../src/fsLocal";
+import { getFileRenameForDup } from "./conflictLogic";
+
+export const getDupFiles = async (fsLocal: FakeFsLocal) => {
+  const allFilesAndFolders = await fsLocal.walk();
+
+  allFilesAndFolders.sort((a, b) => -(a.keyRaw.length - b.keyRaw.length)); // descending
+
+  const filenameSet: Set<string> = new Set();
+  const filesToBeRemoved: Set<string> = new Set();
+
+  for (const { keyRaw } of allFilesAndFolders) {
+    if (keyRaw.endsWith("/")) {
+      continue;
+    }
+    if (keyRaw.includes("dup")) {
+      filenameSet.add(keyRaw);
+    }
+
+    const dup = getFileRenameForDup(keyRaw);
+    if (filenameSet.has(dup)) {
+      filesToBeRemoved.add(dup);
+    }
+  }
+
+  return [...filesToBeRemoved];
+};
+
+export const clearDupFiles = async (
+  filesToBeRemoved: string[],
+  fsLocal: FakeFsLocal
+) => {
+  await Promise.all(filesToBeRemoved.map(async (f) => await fsLocal.rm(f)));
+};

pro/src/conflictLogic.ts

@@ -173,7 +173,7 @@ export async function mergeFile(
   };
 }
 
-export function getFileRename(key: string) {
+export function getFileRenameForDup(key: string) {
   if (
     key === "" ||
     key === "." ||
@@ -344,7 +344,7 @@ export async function tryDuplicateFile(
   uploadCallback: (entity: Entity | undefined) => Promise<any>,
   downloadCallback: (entity: Entity | undefined) => Promise<any>
 ) {
-  let key2 = getFileRename(key);
+  let key2 = getFileRenameForDup(key);
   let usable = false;
   do {
     try {
@@ -353,7 +353,7 @@ export async function tryDuplicateFile(
         throw Error(`not exist $${key2}`);
       }
       console.debug(`key2=${key2} exists, cannot use for new file`);
-      key2 = getFileRename(key2);
+      key2 = getFileRenameForDup(key2);
       console.debug(`key2=${key2} is prepared for next try`);
     } catch (e) {
       // not exists, exactly what we want

pro/src/langs/en.json

@@ -129,6 +129,10 @@
   "modal_proauth_maualinput_notice": "Trying to connect, wait...",
   "modal_proauth_maualinput_conn_fail": "Failed to connect",
 
+  "modal_cleardupfiles_warning": "Warning: The plugin just stupidly finds all the file names that looks like duplicated files (.dup in file names). Also, this only deletes local files, so you need to trigger a sync to delete remote files.",
+  "modal_cleardupfiles_warning_confirm": "Confirm to delete",
+  "modal_cleardupfiles_warning_finished": "All the detected duplicated files are removed!",
+
   "settings_onedrivefull": "Remote For Onedrive (for personal) (Full)",
   "settings_chooseservice_onedrivefull": "OneDrive for personal (Full) (PRO)",
   "settings_onedrivefull_disclaimer1": "Disclaimer: This app is NOT an official Microsoft / OneDrive product.",
@@ -278,6 +282,10 @@
   "settings_export_koofr_button": "Export Koofr Part",
   "settings_export_azureblobstorage_button": "Export Azure Blob Storage Part",
 
+  "settings_cleardupfiles": "Clear Duplicated Files By Smart Conflict",
+  "settings_cleardupfiles_desc": "If you have ever used Smart Conflict (PRO) feature, the plugin may generate duplicated files. <strong>IF YOU ARE SURE THE DUPLICATED FILES ARE NO LONGER NEEDED</strong>, you can clear <strong>ALL</strong> of them locally here.",
+  "settings_cleardupfiles_button": "Start Scanning",
+
   "settings_pro": "Account (for PRO features)",
   "settings_pro_tutorial": "<p>Using <stong>basic</strong> features of Remotely Save is <strong>FREE</strong> and do <strong>NOT</strong> need an account.</p><p>However, you will <strong>need</strong> an online account and <strong>PAY</strong> for the <strong>PRO</strong> features such as smart conflict.</p><p>Firstly please click the button to sign up and sign in to the website: <a href=\"https://remotelysave.com\">https://remotelysave.com</a>. Notice: It's different from, and NOT affiliated with Obsidian account.</p><p>Secondly please \"connect\" your local device to your online account.",
   "settings_pro_features": "Features",

pro/src/langs/zh_cn.json

@@ -140,6 +140,10 @@
   "modal_proauth_maualinput_notice": "正在连接，请稍候......",
   "modal_proauth_maualinput_conn_fail": "连接失败",
 
+  "modal_cleardupfiles_warning": "警告：插件只是简单地扫描所有文件名含有 .dup 的作为重复文件。另外，本操作只删除本地文件，因此您需要触发同步来删除远端文件。",
+  "modal_cleardupfiles_warning_confirm": "确认删除",
+  "modal_cleardupfiles_warning_finished": "所有检测到的重复文件已被删除！",
+
   "settings_onedrivefull": "Onedrive（个人版）（Full）设置",
   "settings_chooseservice_onedrivefull": "OneDrive（个人版）（Full）（PRO）",
   "settings_onedrivefull_disclaimer1": "声明：此插件不是微软或 OneDrive 的官方产品。",
@@ -285,6 +289,10 @@
   "settings_export_koofr_button": "导出 Koofr 部分",
   "settings_export_azureblobstorage_button": "导出 Azure Blob Storage 部分",
 
+  "settings_cleardupfiles": "删除智能处理冲突生成的重复文件",
+  "settings_cleardupfiles_desc": "如果用过智能处理冲突 (PRO) 功能，插件可能会生成重复文件。<strong>如果您真的确认了所有重复文件都不需要了，</strong>那么可以删除本地的<strong>所有</strong>这些文件。",
+  "settings_cleardupfiles_button": "开始扫描",
+
   "settings_pro": "账号（PRO 付费功能）",
   "settings_pro_tutorial": "<p>使用 Remotely Save 的<stong>基本</strong>功能是<strong>免费的</strong>，而且<strong>不</strong>需要注册对应账号。</p><p>但是，您<strong>需要</strong>注册账号和对<strong>PRO</strong>功能<strong>付费</strong>使用，如智能处理冲突功能。</p><p>第一步：点击按钮从而注册和登录网站：<a href=\"https://remotelysave.com\">https://remotelysave.com</a>。注意：这和 Obsidian 官方账号无关，是不同的账号。</p><p>第二部：点击“连接”按钮，从而连接本设备和在线账号。",
   "settings_pro_features": "功能",

pro/src/langs/zh_tw.json

@@ -140,6 +140,10 @@
   "modal_proauth_maualinput_notice": "正在連線，請稍候......",
   "modal_proauth_maualinput_conn_fail": "連線失敗",
 
+  "modal_cleardupfiles_warning": "警告：外掛只是簡單地掃描所有檔名含有 .dup 的作為重複檔案。另外，本操作只刪除本地檔案，因此您需要觸發同步來刪除遠端檔案。",
+  "modal_cleardupfiles_warning_confirm": "確認刪除",
+  "modal_cleardupfiles_warning_finished": "所有檢測到的重複檔案已被刪除！",
+
   "settings_onedrivefull": "Onedrive（個人版）（Full）設定",
   "settings_chooseservice_onedrivefull": "OneDrive（個人版）（Full）（PRO）",
   "settings_onedrivefull_disclaimer1": "宣告：此外掛不是微軟或 OneDrive 的官方產品。",
@@ -285,6 +289,10 @@
   "settings_export_koofr_button": "匯出 Koofr 部分",
   "settings_export_azureblobstorage_button": "匯出 Azure Blob Storage 部分",
 
+  "settings_cleardupfiles": "刪除智慧處理衝突生成的重複檔案",
+  "settings_cleardupfiles_desc": "如果用過智慧處理衝突 (PRO) 功能，外掛可能會生成重複檔案。<strong>如果您真的確認了所有重複檔案都不需要了，</strong>那麼可以刪除本地的<strong>所有</strong>這些檔案。",
+  "settings_cleardupfiles_button": "開始掃描",
+
   "settings_pro": "賬號（PRO 付費功能）",
   "settings_pro_tutorial": "<p>使用 Remotely Save 的<stong>基本</strong>功能是<strong>免費的</strong>，而且<strong>不</strong>需要註冊對應賬號。</p><p>但是，您<strong>需要</strong>註冊賬號和對<strong>PRO</strong>功能<strong>付費</strong>使用，如智慧處理衝突功能。</p><p>第一步：點選按鈕從而註冊和登入網站：<a href=\"https://remotelysave.com\">https://remotelysave.com</a>。注意：這和 Obsidian 官方賬號無關，是不同的賬號。</p><p>第二部：點選“連線”按鈕，從而連線本裝置和線上賬號。",
   "settings_pro_features": "功能",

pro/src/settingsClearDupFiles.ts

@@ -0,0 +1,89 @@
+import { type App, Modal, Notice, Setting } from "obsidian";
+import { FakeFsLocal } from "../../src/fsLocal";
+import type { TransItemType } from "../../src/i18n";
+import type RemotelySavePlugin from "../../src/main";
+import { stringToFragment } from "../../src/misc";
+import { clearDupFiles, getDupFiles } from "./clearDupFiles";
+
+class ClearDupFilesModal extends Modal {
+  readonly plugin: RemotelySavePlugin;
+  readonly t: (x: TransItemType, vars?: any) => string;
+  readonly files: string[];
+  readonly fsLocal: FakeFsLocal;
+  constructor(
+    app: App,
+    plugin: RemotelySavePlugin,
+    t: (x: TransItemType, vars?: any) => string,
+    files: string[],
+    fsLocal: FakeFsLocal
+  ) {
+    super(app);
+    this.plugin = plugin;
+    this.t = t;
+    this.files = files;
+    this.fsLocal = fsLocal;
+  }
+
+  async onOpen() {
+    const t = this.t;
+    const { contentEl } = this;
+
+    contentEl.createEl("p", {
+      text: t("modal_cleardupfiles_warning"),
+    });
+
+    contentEl.createEl("pre").createEl("code", {
+      text: this.files.join("\n"),
+    });
+
+    new Setting(contentEl)
+      .addButton((button) => {
+        button.setButtonText(t("modal_cleardupfiles_warning_confirm"));
+        button.onClick(async () => {
+          await clearDupFiles(this.files, this.fsLocal);
+          new Notice(t("modal_cleardupfiles_warning_finished"));
+          this.close();
+        });
+      })
+      .addButton((button) => {
+        button.setButtonText(t("goback"));
+        button.onClick(() => {
+          this.close();
+        });
+      });
+  }
+
+  onClose() {
+    const { contentEl } = this;
+    contentEl.empty();
+  }
+}
+
+export const generateClearDupFilesSettingsPart = (
+  containerEl: HTMLElement,
+  t: (x: TransItemType, vars?: any) => string,
+  app: App,
+  plugin: RemotelySavePlugin
+) => {
+  new Setting(containerEl)
+    .setName(t("settings_cleardupfiles"))
+    .setDesc(stringToFragment(t("settings_cleardupfiles_desc")))
+    .addButton(async (button) => {
+      button.setButtonText(t("settings_cleardupfiles_button"));
+      button.onClick(async () => {
+        const fsLocal = new FakeFsLocal(
+          app.vault,
+          plugin.settings.syncConfigDir ?? false,
+          app.vault.configDir,
+          plugin.manifest.id,
+          undefined,
+          plugin.settings.deleteToWhere ?? "system"
+        );
+
+        const files = await getDupFiles(fsLocal);
+
+        const modal = new ClearDupFilesModal(app, plugin, t, files, fsLocal);
+        modal.open();
+      });
+    });
+};

pro/tests/conflictLogic.test.ts

@@ -1,67 +1,70 @@
 import { deepStrictEqual, rejects, throws } from "assert";
-import { getFileRename } from "../src/conflictLogic";
+import { getFileRenameForDup } from "../src/conflictLogic";
 
 describe("New name is generated", () => {
   it("should throw for empty file", async () => {
     for (const key of ["", "/", ".", ".."]) {
-      throws(() => getFileRename(key));
+      throws(() => getFileRenameForDup(key));
     }
   });
 
   it("should throw for folder", async () => {
     for (const key of ["sss/", "ssss/yyy/"]) {
-      throws(() => getFileRename(key));
+      throws(() => getFileRenameForDup(key));
     }
   });
 
   it("should correctly get no ext files renamed", async () => {
-    deepStrictEqual(getFileRename("abc"), "abc.dup");
+    deepStrictEqual(getFileRenameForDup("abc"), "abc.dup");
 
-    deepStrictEqual(getFileRename("xxxx/yyyy/abc"), "xxxx/yyyy/abc.dup");
+    deepStrictEqual(getFileRenameForDup("xxxx/yyyy/abc"), "xxxx/yyyy/abc.dup");
   });
 
   it("should correctly get dot files renamed", async () => {
-    deepStrictEqual(getFileRename(".abc"), ".abc.dup");
+    deepStrictEqual(getFileRenameForDup(".abc"), ".abc.dup");
 
-    deepStrictEqual(getFileRename("xxxx/yyyy/.efg"), "xxxx/yyyy/.efg.dup");
+    deepStrictEqual(
+      getFileRenameForDup("xxxx/yyyy/.efg"),
+      "xxxx/yyyy/.efg.dup"
+    );
 
-    deepStrictEqual(getFileRename("xxxx/yyyy/hij."), "xxxx/yyyy/hij.dup");
+    deepStrictEqual(getFileRenameForDup("xxxx/yyyy/hij."), "xxxx/yyyy/hij.dup");
   });
 
   it("should correctly get normal files renamed", async () => {
-    deepStrictEqual(getFileRename("abc.efg"), "abc.dup.efg");
+    deepStrictEqual(getFileRenameForDup("abc.efg"), "abc.dup.efg");
 
     deepStrictEqual(
-      getFileRename("xxxx/yyyy/abc.efg"),
+      getFileRenameForDup("xxxx/yyyy/abc.efg"),
       "xxxx/yyyy/abc.dup.efg"
     );
 
     deepStrictEqual(
-      getFileRename("xxxx/yyyy/abc.tar.gz"),
+      getFileRenameForDup("xxxx/yyyy/abc.tar.gz"),
       "xxxx/yyyy/abc.tar.dup.gz"
     );
 
     deepStrictEqual(
-      getFileRename("xxxx/yyyy/.abc.efg"),
+      getFileRenameForDup("xxxx/yyyy/.abc.efg"),
       "xxxx/yyyy/.abc.dup.efg"
     );
   });
 
   it("should correctly get duplicated files renamed again", async () => {
-    deepStrictEqual(getFileRename("abc.dup"), "abc.dup.dup");
+    deepStrictEqual(getFileRenameForDup("abc.dup"), "abc.dup.dup");
 
     deepStrictEqual(
-      getFileRename("xxxx/yyyy/.abc.dup"),
+      getFileRenameForDup("xxxx/yyyy/.abc.dup"),
       "xxxx/yyyy/.abc.dup.dup"
     );
 
     deepStrictEqual(
-      getFileRename("xxxx/yyyy/abc.dup.md"),
+      getFileRenameForDup("xxxx/yyyy/abc.dup.md"),
       "xxxx/yyyy/abc.dup.dup.md"
     );
 
     deepStrictEqual(
-      getFileRename("xxxx/yyyy/.abc.dup.md"),
+      getFileRenameForDup("xxxx/yyyy/.abc.dup.md"),
       "xxxx/yyyy/.abc.dup.dup.md"
     );
   });

src/settings.ts

@@ -23,6 +23,7 @@ import type {
 import cloneDeep from "lodash/cloneDeep";
 import { generateAzureBlobStorageSettingsPart } from "../pro/src/settingsAzureBlobStorage";
 import { generateBoxSettingsPart } from "../pro/src/settingsBox";
+import { generateClearDupFilesSettingsPart } from "../pro/src/settingsClearDupFiles";
 import { generateGoogleDriveSettingsPart } from "../pro/src/settingsGoogleDrive";
 import { generateKoofrSettingsPart } from "../pro/src/settingsKoofr";
 import { generateOnedriveFullSettingsPart } from "../pro/src/settingsOnedriveFull";
@@ -2359,6 +2360,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
         });
     });
 
+    generateClearDupFilesSettingsPart(advDiv, t, this.app, this.plugin);
+
     const percentage1 = new Setting(advDiv)
       .setName(t("settings_protectmodifypercentage"))
       .setDesc(t("settings_protectmodifypercentage_desc"));