From 1012bf3d095224bb4e7c44ffccf7aa46ca4960fd Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 13 Jul 2024 16:31:09 +0800 Subject: [PATCH] remove dup files --- pro/src/clearDupFiles.ts | 34 ++++++++++++ pro/src/conflictLogic.ts | 6 +-- pro/src/langs/en.json | 8 +++ pro/src/langs/zh_cn.json | 8 +++ pro/src/langs/zh_tw.json | 8 +++ pro/src/settingsClearDupFiles.ts | 89 ++++++++++++++++++++++++++++++++ pro/tests/conflictLogic.test.ts | 35 +++++++------ src/settings.ts | 3 ++ 8 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 pro/src/clearDupFiles.ts create mode 100644 pro/src/settingsClearDupFiles.ts diff --git a/pro/src/clearDupFiles.ts b/pro/src/clearDupFiles.ts new file mode 100644 index 0000000..72138bf --- /dev/null +++ b/pro/src/clearDupFiles.ts @@ -0,0 +1,34 @@ +import type { FakeFsLocal } from "../../src/fsLocal"; +import { getFileRenameForDup } from "./conflictLogic"; + +export const getDupFiles = async (fsLocal: FakeFsLocal) => { + const allFilesAndFolders = await fsLocal.walk(); + + allFilesAndFolders.sort((a, b) => -(a.keyRaw.length - b.keyRaw.length)); // descending + + const filenameSet: Set = new Set(); + const filesToBeRemoved: Set = new Set(); + + for (const { keyRaw } of allFilesAndFolders) { + if (keyRaw.endsWith("/")) { + continue; + } + if (keyRaw.includes("dup")) { + filenameSet.add(keyRaw); + } + + const dup = getFileRenameForDup(keyRaw); + if (filenameSet.has(dup)) { + filesToBeRemoved.add(dup); + } + } + + return [...filesToBeRemoved]; +}; + +export const clearDupFiles = async ( + filesToBeRemoved: string[], + fsLocal: FakeFsLocal +) => { + await Promise.all(filesToBeRemoved.map(async (f) => await fsLocal.rm(f))); +}; diff --git a/pro/src/conflictLogic.ts b/pro/src/conflictLogic.ts index 9b98a0c..28b2fb2 100644 --- a/pro/src/conflictLogic.ts +++ b/pro/src/conflictLogic.ts @@ -173,7 +173,7 @@ export async function mergeFile( }; } -export function getFileRename(key: string) { +export function getFileRenameForDup(key: string) { if ( key === "" || key === "." || @@ -344,7 +344,7 @@ export async function tryDuplicateFile( uploadCallback: (entity: Entity | undefined) => Promise, downloadCallback: (entity: Entity | undefined) => Promise ) { - let key2 = getFileRename(key); + let key2 = getFileRenameForDup(key); let usable = false; do { try { @@ -353,7 +353,7 @@ export async function tryDuplicateFile( throw Error(`not exist $${key2}`); } console.debug(`key2=${key2} exists, cannot use for new file`); - key2 = getFileRename(key2); + key2 = getFileRenameForDup(key2); console.debug(`key2=${key2} is prepared for next try`); } catch (e) { // not exists, exactly what we want diff --git a/pro/src/langs/en.json b/pro/src/langs/en.json index d05ba32..6b3161c 100644 --- a/pro/src/langs/en.json +++ b/pro/src/langs/en.json @@ -129,6 +129,10 @@ "modal_proauth_maualinput_notice": "Trying to connect, wait...", "modal_proauth_maualinput_conn_fail": "Failed to connect", + "modal_cleardupfiles_warning": "Warning: The plugin just stupidly finds all the file names that looks like duplicated files (.dup in file names). Also, this only deletes local files, so you need to trigger a sync to delete remote files.", + "modal_cleardupfiles_warning_confirm": "Confirm to delete", + "modal_cleardupfiles_warning_finished": "All the detected duplicated files are removed!", + "settings_onedrivefull": "Remote For Onedrive (for personal) (Full)", "settings_chooseservice_onedrivefull": "OneDrive for personal (Full) (PRO)", "settings_onedrivefull_disclaimer1": "Disclaimer: This app is NOT an official Microsoft / OneDrive product.", @@ -278,6 +282,10 @@ "settings_export_koofr_button": "Export Koofr Part", "settings_export_azureblobstorage_button": "Export Azure Blob Storage Part", + "settings_cleardupfiles": "Clear Duplicated Files By Smart Conflict", + "settings_cleardupfiles_desc": "If you have ever used Smart Conflict (PRO) feature, the plugin may generate duplicated files. IF YOU ARE SURE THE DUPLICATED FILES ARE NO LONGER NEEDED, you can clear ALL of them locally here.", + "settings_cleardupfiles_button": "Start Scanning", + "settings_pro": "Account (for PRO features)", "settings_pro_tutorial": "

Using basic features of Remotely Save is FREE and do NOT need an account.

However, you will need an online account and PAY for the PRO features such as smart conflict.

Firstly please click the button to sign up and sign in to the website: https://remotelysave.com. Notice: It's different from, and NOT affiliated with Obsidian account.

Secondly please \"connect\" your local device to your online account.", "settings_pro_features": "Features", diff --git a/pro/src/langs/zh_cn.json b/pro/src/langs/zh_cn.json index b19f19d..9ebc477 100644 --- a/pro/src/langs/zh_cn.json +++ b/pro/src/langs/zh_cn.json @@ -140,6 +140,10 @@ "modal_proauth_maualinput_notice": "正在连接，请稍候......", "modal_proauth_maualinput_conn_fail": "连接失败", + "modal_cleardupfiles_warning": "警告：插件只是简单地扫描所有文件名含有 .dup 的作为重复文件。另外，本操作只删除本地文件，因此您需要触发同步来删除远端文件。", + "modal_cleardupfiles_warning_confirm": "确认删除", + "modal_cleardupfiles_warning_finished": "所有检测到的重复文件已被删除！", + "settings_onedrivefull": "Onedrive（个人版）（Full）设置", "settings_chooseservice_onedrivefull": "OneDrive（个人版）（Full）（PRO）", "settings_onedrivefull_disclaimer1": "声明：此插件不是微软或 OneDrive 的官方产品。", @@ -285,6 +289,10 @@ "settings_export_koofr_button": "导出 Koofr 部分", "settings_export_azureblobstorage_button": "导出 Azure Blob Storage 部分", + "settings_cleardupfiles": "删除智能处理冲突生成的重复文件", + "settings_cleardupfiles_desc": "如果用过智能处理冲突 (PRO) 功能，插件可能会生成重复文件。如果您真的确认了所有重复文件都不需要了，那么可以删除本地的所有这些文件。", + "settings_cleardupfiles_button": "开始扫描", + "settings_pro": "账号（PRO 付费功能）", "settings_pro_tutorial": "

使用 Remotely Save 的基本功能是免费的，而且不需要注册对应账号。

但是，您需要注册账号和对PRO功能付费使用，如智能处理冲突功能。

第一步：点击按钮从而注册和登录网站：https://remotelysave.com。注意：这和 Obsidian 官方账号无关，是不同的账号。

第二部：点击“连接”按钮，从而连接本设备和在线账号。", "settings_pro_features": "功能", diff --git a/pro/src/langs/zh_tw.json b/pro/src/langs/zh_tw.json index c7b0042..23bb71d 100644 --- a/pro/src/langs/zh_tw.json +++ b/pro/src/langs/zh_tw.json @@ -140,6 +140,10 @@ "modal_proauth_maualinput_notice": "正在連線，請稍候......", "modal_proauth_maualinput_conn_fail": "連線失敗", + "modal_cleardupfiles_warning": "警告：外掛只是簡單地掃描所有檔名含有 .dup 的作為重複檔案。另外，本操作只刪除本地檔案，因此您需要觸發同步來刪除遠端檔案。", + "modal_cleardupfiles_warning_confirm": "確認刪除", + "modal_cleardupfiles_warning_finished": "所有檢測到的重複檔案已被刪除！", + "settings_onedrivefull": "Onedrive（個人版）（Full）設定", "settings_chooseservice_onedrivefull": "OneDrive（個人版）（Full）（PRO）", "settings_onedrivefull_disclaimer1": "宣告：此外掛不是微軟或 OneDrive 的官方產品。", @@ -285,6 +289,10 @@ "settings_export_koofr_button": "匯出 Koofr 部分", "settings_export_azureblobstorage_button": "匯出 Azure Blob Storage 部分", + "settings_cleardupfiles": "刪除智慧處理衝突生成的重複檔案", + "settings_cleardupfiles_desc": "如果用過智慧處理衝突 (PRO) 功能，外掛可能會生成重複檔案。如果您真的確認了所有重複檔案都不需要了，那麼可以刪除本地的所有這些檔案。", + "settings_cleardupfiles_button": "開始掃描", + "settings_pro": "賬號（PRO 付費功能）", "settings_pro_tutorial": "

使用 Remotely Save 的基本功能是免費的，而且不需要註冊對應賬號。

但是，您需要註冊賬號和對PRO功能付費使用，如智慧處理衝突功能。

第一步：點選按鈕從而註冊和登入網站：https://remotelysave.com。注意：這和 Obsidian 官方賬號無關，是不同的賬號。

第二部：點選“連線”按鈕，從而連線本裝置和線上賬號。", "settings_pro_features": "功能", diff --git a/pro/src/settingsClearDupFiles.ts b/pro/src/settingsClearDupFiles.ts new file mode 100644 index 0000000..675445d --- /dev/null +++ b/pro/src/settingsClearDupFiles.ts @@ -0,0 +1,89 @@ +import { type App, Modal, Notice, Setting } from "obsidian"; +import { FakeFsLocal } from "../../src/fsLocal"; +import type { TransItemType } from "../../src/i18n"; +import type RemotelySavePlugin from "../../src/main"; +import { stringToFragment } from "../../src/misc"; +import { clearDupFiles, getDupFiles } from "./clearDupFiles"; + +class ClearDupFilesModal extends Modal { + readonly plugin: RemotelySavePlugin; + readonly t: (x: TransItemType, vars?: any) => string; + readonly files: string[]; + readonly fsLocal: FakeFsLocal; + constructor( + app: App, + plugin: RemotelySavePlugin, + t: (x: TransItemType, vars?: any) => string, + files: string[], + fsLocal: FakeFsLocal + ) { + super(app); + this.plugin = plugin; + this.t = t; + this.files = files; + this.fsLocal = fsLocal; + } + + async onOpen() { + const t = this.t; + const { contentEl } = this; + + contentEl.createEl("p", { + text: t("modal_cleardupfiles_warning"), + }); + + contentEl.createEl("pre").createEl("code", { + text: this.files.join("\n"), + }); + + new Setting(contentEl) + .addButton((button) => { + button.setButtonText(t("modal_cleardupfiles_warning_confirm")); + button.onClick(async () => { + await clearDupFiles(this.files, this.fsLocal); + new Notice(t("modal_cleardupfiles_warning_finished")); + this.close(); + }); + }) + .addButton((button) => { + button.setButtonText(t("goback")); + button.onClick(() => { + this.close(); + }); + }); + } + + onClose() { + const { contentEl } = this; + contentEl.empty(); + } +} + +export const generateClearDupFilesSettingsPart = ( + containerEl: HTMLElement, + t: (x: TransItemType, vars?: any) => string, + app: App, + plugin: RemotelySavePlugin +) => { + new Setting(containerEl) + .setName(t("settings_cleardupfiles")) + .setDesc(stringToFragment(t("settings_cleardupfiles_desc"))) + .addButton(async (button) => { + button.setButtonText(t("settings_cleardupfiles_button")); + button.onClick(async () => { + const fsLocal = new FakeFsLocal( + app.vault, + plugin.settings.syncConfigDir ?? false, + app.vault.configDir, + plugin.manifest.id, + undefined, + plugin.settings.deleteToWhere ?? "system" + ); + + const files = await getDupFiles(fsLocal); + + const modal = new ClearDupFilesModal(app, plugin, t, files, fsLocal); + modal.open(); + }); + }); +}; diff --git a/pro/tests/conflictLogic.test.ts b/pro/tests/conflictLogic.test.ts index ecec402..f9b56e1 100644 --- a/pro/tests/conflictLogic.test.ts +++ b/pro/tests/conflictLogic.test.ts @@ -1,67 +1,70 @@ import { deepStrictEqual, rejects, throws } from "assert"; -import { getFileRename } from "../src/conflictLogic"; +import { getFileRenameForDup } from "../src/conflictLogic"; describe("New name is generated", () => { it("should throw for empty file", async () => { for (const key of ["", "/", ".", ".."]) { - throws(() => getFileRename(key)); + throws(() => getFileRenameForDup(key)); } }); it("should throw for folder", async () => { for (const key of ["sss/", "ssss/yyy/"]) { - throws(() => getFileRename(key)); + throws(() => getFileRenameForDup(key)); } }); it("should correctly get no ext files renamed", async () => { - deepStrictEqual(getFileRename("abc"), "abc.dup"); + deepStrictEqual(getFileRenameForDup("abc"), "abc.dup"); - deepStrictEqual(getFileRename("xxxx/yyyy/abc"), "xxxx/yyyy/abc.dup"); + deepStrictEqual(getFileRenameForDup("xxxx/yyyy/abc"), "xxxx/yyyy/abc.dup"); }); it("should correctly get dot files renamed", async () => { - deepStrictEqual(getFileRename(".abc"), ".abc.dup"); + deepStrictEqual(getFileRenameForDup(".abc"), ".abc.dup"); - deepStrictEqual(getFileRename("xxxx/yyyy/.efg"), "xxxx/yyyy/.efg.dup"); + deepStrictEqual( + getFileRenameForDup("xxxx/yyyy/.efg"), + "xxxx/yyyy/.efg.dup" + ); - deepStrictEqual(getFileRename("xxxx/yyyy/hij."), "xxxx/yyyy/hij.dup"); + deepStrictEqual(getFileRenameForDup("xxxx/yyyy/hij."), "xxxx/yyyy/hij.dup"); }); it("should correctly get normal files renamed", async () => { - deepStrictEqual(getFileRename("abc.efg"), "abc.dup.efg"); + deepStrictEqual(getFileRenameForDup("abc.efg"), "abc.dup.efg"); deepStrictEqual( - getFileRename("xxxx/yyyy/abc.efg"), + getFileRenameForDup("xxxx/yyyy/abc.efg"), "xxxx/yyyy/abc.dup.efg" ); deepStrictEqual( - getFileRename("xxxx/yyyy/abc.tar.gz"), + getFileRenameForDup("xxxx/yyyy/abc.tar.gz"), "xxxx/yyyy/abc.tar.dup.gz" ); deepStrictEqual( - getFileRename("xxxx/yyyy/.abc.efg"), + getFileRenameForDup("xxxx/yyyy/.abc.efg"), "xxxx/yyyy/.abc.dup.efg" ); }); it("should correctly get duplicated files renamed again", async () => { - deepStrictEqual(getFileRename("abc.dup"), "abc.dup.dup"); + deepStrictEqual(getFileRenameForDup("abc.dup"), "abc.dup.dup"); deepStrictEqual( - getFileRename("xxxx/yyyy/.abc.dup"), + getFileRenameForDup("xxxx/yyyy/.abc.dup"), "xxxx/yyyy/.abc.dup.dup" ); deepStrictEqual( - getFileRename("xxxx/yyyy/abc.dup.md"), + getFileRenameForDup("xxxx/yyyy/abc.dup.md"), "xxxx/yyyy/abc.dup.dup.md" ); deepStrictEqual( - getFileRename("xxxx/yyyy/.abc.dup.md"), + getFileRenameForDup("xxxx/yyyy/.abc.dup.md"), "xxxx/yyyy/.abc.dup.dup.md" ); }); diff --git a/src/settings.ts b/src/settings.ts index ecacfad..6eef527 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -23,6 +23,7 @@ import type { import cloneDeep from "lodash/cloneDeep"; import { generateAzureBlobStorageSettingsPart } from "../pro/src/settingsAzureBlobStorage"; import { generateBoxSettingsPart } from "../pro/src/settingsBox"; +import { generateClearDupFilesSettingsPart } from "../pro/src/settingsClearDupFiles"; import { generateGoogleDriveSettingsPart } from "../pro/src/settingsGoogleDrive"; import { generateKoofrSettingsPart } from "../pro/src/settingsKoofr"; import { generateOnedriveFullSettingsPart } from "../pro/src/settingsOnedriveFull"; @@ -2359,6 +2360,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); + generateClearDupFilesSettingsPart(advDiv, t, this.app, this.plugin); + const percentage1 = new Setting(advDiv) .setName(t("settings_protectmodifypercentage")) .setDesc(t("settings_protectmodifypercentage_desc"));