bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove dup files

Browse Source
This commit is contained in:
fyears 2024-07-13 16:31:09 +08:00
parent 57bdc4d0c6
commit 1012bf3d09
8 changed files with 172 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
pro/src/clearDupFiles.ts Normal file
View File

@ -0,0 +1,34 @@
import type { FakeFsLocal } from "../../src/fsLocal";
import { getFileRenameForDup } from "./conflictLogic";
export const getDupFiles = async (fsLocal: FakeFsLocal) => {
const allFilesAndFolders = await fsLocal.walk();
allFilesAndFolders.sort((a, b) => -(a.keyRaw.length - b.keyRaw.length)); // descending
const filenameSet: Set<string> = new Set();
const filesToBeRemoved: Set<string> = new Set();
for (const { keyRaw } of allFilesAndFolders) {
if (keyRaw.endsWith("/")) {
continue;
}
if (keyRaw.includes("dup")) {
filenameSet.add(keyRaw);
}
const dup = getFileRenameForDup(keyRaw);
if (filenameSet.has(dup)) {
filesToBeRemoved.add(dup);
}
}
return [...filesToBeRemoved];
};
export const clearDupFiles = async (
filesToBeRemoved: string[],
fsLocal: FakeFsLocal
) => {
await Promise.all(filesToBeRemoved.map(async (f) => await fsLocal.rm(f)));
};

6
pro/src/conflictLogic.ts
View File

@ -173,7 +173,7 @@ export async function mergeFile(
};
}
export function getFileRename(key: string) {
export function getFileRenameForDup(key: string) {
if (
key === "" ||
key === "." ||
@ -344,7 +344,7 @@ export async function tryDuplicateFile(
uploadCallback: (entity: Entity | undefined) => Promise<any>,
downloadCallback: (entity: Entity | undefined) => Promise<any>
) {
let key2 = getFileRename(key);
let key2 = getFileRenameForDup(key);
let usable = false;
do {
try {
@ -353,7 +353,7 @@ export async function tryDuplicateFile(
throw Error(`not exist $${key2}`);
}
console.debug(`key2=${key2} exists, cannot use for new file`);
key2 = getFileRename(key2);
key2 = getFileRenameForDup(key2);
console.debug(`key2=${key2} is prepared for next try`);
} catch (e) {
// not exists, exactly what we want

8
pro/src/langs/en.json
View File

@ -129,6 +129,10 @@
"modal_proauth_maualinput_notice": "Trying to connect, wait...",
"modal_proauth_maualinput_conn_fail": "Failed to connect",
"modal_cleardupfiles_warning": "Warning: The plugin just stupidly finds all the file names that looks like duplicated files (.dup in file names). Also, this only deletes local files, so you need to trigger a sync to delete remote files.",
"modal_cleardupfiles_warning_confirm": "Confirm to delete",
"modal_cleardupfiles_warning_finished": "All the detected duplicated files are removed!",
"settings_onedrivefull": "Remote For Onedrive (for personal) (Full)",
"settings_chooseservice_onedrivefull": "OneDrive for personal (Full) (PRO)",
"settings_onedrivefull_disclaimer1": "Disclaimer: This app is NOT an official Microsoft / OneDrive product.",
@ -278,6 +282,10 @@
"settings_export_koofr_button": "Export Koofr Part",
"settings_export_azureblobstorage_button": "Export Azure Blob Storage Part",
"settings_cleardupfiles": "Clear Duplicated Files By Smart Conflict",
"settings_cleardupfiles_desc": "If you have ever used Smart Conflict (PRO) feature, the plugin may generate duplicated files. <strong>IF YOU ARE SURE THE DUPLICATED FILES ARE NO LONGER NEEDED</strong>, you can clear <strong>ALL</strong> of them locally here.",
"settings_cleardupfiles_button": "Start Scanning",
"settings_pro": "Account (for PRO features)",
"settings_pro_tutorial": "<p>Using <stong>basic</strong> features of Remotely Save is <strong>FREE</strong> and do <strong>NOT</strong> need an account.</p><p>However, you will <strong>need</strong> an online account and <strong>PAY</strong> for the <strong>PRO</strong> features such as smart conflict.</p><p>Firstly please click the button to sign up and sign in to the website: <a href=\"https://remotelysave.com\">https://remotelysave.com</a>. Notice: It's different from, and NOT affiliated with Obsidian account.</p><p>Secondly please \"connect\" your local device to your online account.",
"settings_pro_features": "Features",

8
pro/src/langs/zh_cn.json
View File

@ -140,6 +140,10 @@
"modal_proauth_maualinput_notice": "正在连接，请稍候......",
"modal_proauth_maualinput_conn_fail": "连接失败",
"modal_cleardupfiles_warning": "警告：插件只是简单地扫描所有文件名含有 .dup 的作为重复文件。另外，本操作只删除本地文件，因此您需要触发同步来删除远端文件。",
"modal_cleardupfiles_warning_confirm": "确认删除",
"modal_cleardupfiles_warning_finished": "所有检测到的重复文件已被删除！",
"settings_onedrivefull": "Onedrive（个人版）（Full）设置",
"settings_chooseservice_onedrivefull": "OneDrive（个人版）（Full）（PRO）",
"settings_onedrivefull_disclaimer1": "声明：此插件不是微软或 OneDrive 的官方产品。",
@ -285,6 +289,10 @@
"settings_export_koofr_button": "导出 Koofr 部分",
"settings_export_azureblobstorage_button": "导出 Azure Blob Storage 部分",
"settings_cleardupfiles": "删除智能处理冲突生成的重复文件",
"settings_cleardupfiles_desc": "如果用过智能处理冲突 (PRO) 功能，插件可能会生成重复文件。<strong>如果您真的确认了所有重复文件都不需要了，</strong>那么可以删除本地的<strong>所有</strong>这些文件。",
"settings_cleardupfiles_button": "开始扫描",
"settings_pro": "账号（PRO 付费功能）",
"settings_pro_tutorial": "<p>使用 Remotely Save 的<stong>基本</strong>功能是<strong>免费的</strong>，而且<strong>不</strong>需要注册对应账号。</p><p>但是，您<strong>需要</strong>注册账号和对<strong>PRO</strong>功能<strong>付费</strong>使用，如智能处理冲突功能。</p><p>第一步：点击按钮从而注册和登录网站：<a href=\"https://remotelysave.com\">https://remotelysave.com</a>。注意：这和 Obsidian 官方账号无关，是不同的账号。</p><p>第二部：点击“连接”按钮，从而连接本设备和在线账号。",
"settings_pro_features": "功能",

8
pro/src/langs/zh_tw.json
View File

@ -140,6 +140,10 @@
"modal_proauth_maualinput_notice": "正在連線，請稍候......",
"modal_proauth_maualinput_conn_fail": "連線失敗",
"modal_cleardupfiles_warning": "警告：外掛只是簡單地掃描所有檔名含有 .dup 的作為重複檔案。另外，本操作只刪除本地檔案，因此您需要觸發同步來刪除遠端檔案。",
"modal_cleardupfiles_warning_confirm": "確認刪除",
"modal_cleardupfiles_warning_finished": "所有檢測到的重複檔案已被刪除！",
"settings_onedrivefull": "Onedrive（個人版）（Full）設定",
"settings_chooseservice_onedrivefull": "OneDrive（個人版）（Full）（PRO）",
"settings_onedrivefull_disclaimer1": "宣告：此外掛不是微軟或 OneDrive 的官方產品。",
@ -285,6 +289,10 @@
"settings_export_koofr_button": "匯出 Koofr 部分",
"settings_export_azureblobstorage_button": "匯出 Azure Blob Storage 部分",
"settings_cleardupfiles": "刪除智慧處理衝突生成的重複檔案",
"settings_cleardupfiles_desc": "如果用過智慧處理衝突 (PRO) 功能，外掛可能會生成重複檔案。<strong>如果您真的確認了所有重複檔案都不需要了，</strong>那麼可以刪除本地的<strong>所有</strong>這些檔案。",
"settings_cleardupfiles_button": "開始掃描",
"settings_pro": "賬號（PRO 付費功能）",
"settings_pro_tutorial": "<p>使用 Remotely Save 的<stong>基本</strong>功能是<strong>免費的</strong>，而且<strong>不</strong>需要註冊對應賬號。</p><p>但是，您<strong>需要</strong>註冊賬號和對<strong>PRO</strong>功能<strong>付費</strong>使用，如智慧處理衝突功能。</p><p>第一步：點選按鈕從而註冊和登入網站：<a href=\"https://remotelysave.com\">https://remotelysave.com</a>。注意：這和 Obsidian 官方賬號無關，是不同的賬號。</p><p>第二部：點選“連線”按鈕，從而連線本裝置和線上賬號。",
"settings_pro_features": "功能",

89
pro/src/settingsClearDupFiles.ts Normal file
View File

@ -0,0 +1,89 @@
import { type App, Modal, Notice, Setting } from "obsidian";
import { FakeFsLocal } from "../../src/fsLocal";
import type { TransItemType } from "../../src/i18n";
import type RemotelySavePlugin from "../../src/main";
import { stringToFragment } from "../../src/misc";
import { clearDupFiles, getDupFiles } from "./clearDupFiles";
class ClearDupFilesModal extends Modal {
readonly plugin: RemotelySavePlugin;
readonly t: (x: TransItemType, vars?: any) => string;
readonly files: string[];
readonly fsLocal: FakeFsLocal;
constructor(
app: App,
plugin: RemotelySavePlugin,
t: (x: TransItemType, vars?: any) => string,
files: string[],
fsLocal: FakeFsLocal
) {
super(app);
this.plugin = plugin;
this.t = t;
this.files = files;
this.fsLocal = fsLocal;
}
async onOpen() {
const t = this.t;
const { contentEl } = this;
contentEl.createEl("p", {
text: t("modal_cleardupfiles_warning"),
});
contentEl.createEl("pre").createEl("code", {
text: this.files.join("\n"),
});
new Setting(contentEl)
.addButton((button) => {
button.setButtonText(t("modal_cleardupfiles_warning_confirm"));
button.onClick(async () => {
await clearDupFiles(this.files, this.fsLocal);
new Notice(t("modal_cleardupfiles_warning_finished"));
this.close();
});
})
.addButton((button) => {
button.setButtonText(t("goback"));
button.onClick(() => {
this.close();
});
});
}
onClose() {
const { contentEl } = this;
contentEl.empty();
}
}
export const generateClearDupFilesSettingsPart = (
containerEl: HTMLElement,
t: (x: TransItemType, vars?: any) => string,
app: App,
plugin: RemotelySavePlugin
) => {
new Setting(containerEl)
.setName(t("settings_cleardupfiles"))
.setDesc(stringToFragment(t("settings_cleardupfiles_desc")))
.addButton(async (button) => {
button.setButtonText(t("settings_cleardupfiles_button"));
button.onClick(async () => {
const fsLocal = new FakeFsLocal(
app.vault,
plugin.settings.syncConfigDir ?? false,
app.vault.configDir,
plugin.manifest.id,
undefined,
plugin.settings.deleteToWhere ?? "system"
);
const files = await getDupFiles(fsLocal);
const modal = new ClearDupFilesModal(app, plugin, t, files, fsLocal);
modal.open();
});
});
};

35
pro/tests/conflictLogic.test.ts
View File

@ -1,67 +1,70 @@
import { deepStrictEqual, rejects, throws } from "assert";
import { getFileRename } from "../src/conflictLogic";
import { getFileRenameForDup } from "../src/conflictLogic";
describe("New name is generated", () => {
it("should throw for empty file", async () => {
for (const key of ["", "/", ".", ".."]) {
throws(() => getFileRename(key));
throws(() => getFileRenameForDup(key));
}
});
it("should throw for folder", async () => {
for (const key of ["sss/", "ssss/yyy/"]) {
throws(() => getFileRename(key));
throws(() => getFileRenameForDup(key));
}
});
it("should correctly get no ext files renamed", async () => {
deepStrictEqual(getFileRename("abc"), "abc.dup");
deepStrictEqual(getFileRenameForDup("abc"), "abc.dup");
deepStrictEqual(getFileRename("xxxx/yyyy/abc"), "xxxx/yyyy/abc.dup");
deepStrictEqual(getFileRenameForDup("xxxx/yyyy/abc"), "xxxx/yyyy/abc.dup");
});
it("should correctly get dot files renamed", async () => {
deepStrictEqual(getFileRename(".abc"), ".abc.dup");
deepStrictEqual(getFileRenameForDup(".abc"), ".abc.dup");
deepStrictEqual(getFileRename("xxxx/yyyy/.efg"), "xxxx/yyyy/.efg.dup");
deepStrictEqual(
getFileRenameForDup("xxxx/yyyy/.efg"),
"xxxx/yyyy/.efg.dup"
);
deepStrictEqual(getFileRename("xxxx/yyyy/hij."), "xxxx/yyyy/hij.dup");
deepStrictEqual(getFileRenameForDup("xxxx/yyyy/hij."), "xxxx/yyyy/hij.dup");
});
it("should correctly get normal files renamed", async () => {
deepStrictEqual(getFileRename("abc.efg"), "abc.dup.efg");
deepStrictEqual(getFileRenameForDup("abc.efg"), "abc.dup.efg");
deepStrictEqual(
getFileRename("xxxx/yyyy/abc.efg"),
getFileRenameForDup("xxxx/yyyy/abc.efg"),
"xxxx/yyyy/abc.dup.efg"
);
deepStrictEqual(
getFileRename("xxxx/yyyy/abc.tar.gz"),
getFileRenameForDup("xxxx/yyyy/abc.tar.gz"),
"xxxx/yyyy/abc.tar.dup.gz"
);
deepStrictEqual(
getFileRename("xxxx/yyyy/.abc.efg"),
getFileRenameForDup("xxxx/yyyy/.abc.efg"),
"xxxx/yyyy/.abc.dup.efg"
);
});
it("should correctly get duplicated files renamed again", async () => {
deepStrictEqual(getFileRename("abc.dup"), "abc.dup.dup");
deepStrictEqual(getFileRenameForDup("abc.dup"), "abc.dup.dup");
deepStrictEqual(
getFileRename("xxxx/yyyy/.abc.dup"),
getFileRenameForDup("xxxx/yyyy/.abc.dup"),
"xxxx/yyyy/.abc.dup.dup"
);
deepStrictEqual(
getFileRename("xxxx/yyyy/abc.dup.md"),
getFileRenameForDup("xxxx/yyyy/abc.dup.md"),
"xxxx/yyyy/abc.dup.dup.md"
);
deepStrictEqual(
getFileRename("xxxx/yyyy/.abc.dup.md"),
getFileRenameForDup("xxxx/yyyy/.abc.dup.md"),
"xxxx/yyyy/.abc.dup.dup.md"
);
});

3
src/settings.ts
View File

@ -23,6 +23,7 @@ import type {
import cloneDeep from "lodash/cloneDeep";
import { generateAzureBlobStorageSettingsPart } from "../pro/src/settingsAzureBlobStorage";
import { generateBoxSettingsPart } from "../pro/src/settingsBox";
import { generateClearDupFilesSettingsPart } from "../pro/src/settingsClearDupFiles";
import { generateGoogleDriveSettingsPart } from "../pro/src/settingsGoogleDrive";
import { generateKoofrSettingsPart } from "../pro/src/settingsKoofr";
import { generateOnedriveFullSettingsPart } from "../pro/src/settingsOnedriveFull";
@ -2359,6 +2360,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
});
});
generateClearDupFilesSettingsPart(advDiv, t, this.app, this.plugin);
const percentage1 = new Setting(advDiv)
.setName(t("settings_protectmodifypercentage"))
.setDesc(t("settings_protectmodifypercentage_desc"));