From af93af90472dfab95b58ade6ecf0df9e2ed056f8 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 13 Mar 2022 22:42:16 +0800 Subject: [PATCH] Squashed commit of sync hidden files: commit 73967756d51d246b3ca203aa3683cdfebf567000 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Mar 13 22:41:28 2022 +0800 fix typo commit 08e16faa9a9ace9bec71acb723e0d70ca361d49f Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Mar 13 22:41:01 2022 +0800 add modal in settings commit 9db7194fa28cb62b1fa4c01ac7f746d5ac3b86a8 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Mar 13 22:03:00 2022 +0800 working sync for .obsidian and _ commit 4be24ba092181c1c3e1aabe5e9d4e9fcff28f987 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Mar 13 16:07:10 2022 +0800 more logic for hidden path --- src/baseTypes.ts | 2 + src/main.ts | 15 ++++ src/misc.ts | 16 +++- src/obsFolderLister.ts | 127 ++++++++++++++++++++++++++++++++ src/remoteForWebdav.ts | 10 ++- src/settings.ts | 163 +++++++++++++++++++++++++++++++++++------ src/sync.ts | 68 +++++++++++++++-- tests/misc.test.ts | 17 ++++- 8 files changed, 380 insertions(+), 38 deletions(-) create mode 100644 src/obsFolderLister.ts diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 287037c..c263ef7 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -67,6 +67,8 @@ export interface RemotelySavePluginSettings { initRunAfterMilliseconds?: number; agreeToUploadExtraMetadata?: boolean; concurrency?: number; + syncConfigDir?: boolean; + syncUnderscoreItems?: boolean; } export interface RemoteItem { diff --git a/src/main.ts b/src/main.ts index 1239af4..20f98e9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -37,6 +37,7 @@ import { RemotelySaveSettingTab } from "./settings"; import { fetchMetadataFile, parseRemoteItems, SyncStatusType } from "./sync"; import { doActualSync, getSyncPlan, isPasswordOk } from "./sync"; import { messyConfigToNormal, normalConfigToMessy } from "./configPersist"; +import { ObsConfigDirFileType, listFilesInObsFolder } from "./obsFolderLister"; import * as origLog from "loglevel"; import { DeletionOnRemote, MetadataOnRemote } from "./metadataOnRemote"; @@ -56,6 +57,8 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = { initRunAfterMilliseconds: -1, agreeToUploadExtraMetadata: false, concurrency: 5, + syncConfigDir: false, + syncUnderscoreItems: false, }; interface OAuth2Info { @@ -190,6 +193,14 @@ export default class RemotelySavePlugin extends Plugin { this.db, this.settings.vaultRandomID ); + let localConfigDirContents: ObsConfigDirFileType[] = undefined; + if (this.settings.syncConfigDir) { + localConfigDirContents = await listFilesInObsFolder( + this.app.vault.configDir, + this.app.vault, + this.manifest.id + ); + } // log.info(local); // log.info(localHistory); @@ -198,10 +209,14 @@ export default class RemotelySavePlugin extends Plugin { const { plan, sortedKeys, deletions } = await getSyncPlan( remoteStates, local, + localConfigDirContents, origMetadataOnRemote.deletions, localHistory, client.serviceType, this.app.vault, + this.settings.syncConfigDir, + this.app.vault.configDir, + this.settings.syncUnderscoreItems, this.settings.password ); log.info(plan.mixedStates); // for debugging diff --git a/src/misc.ts b/src/misc.ts index 712a9f8..0810195 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -10,10 +10,18 @@ const log = origLog.getLogger("rs-default"); /** * If any part of the file starts with '.' or '_' then it's a hidden file. * @param item - * @param loose + * @param dot + * @param underscore * @returns */ -export const isHiddenPath = (item: string, loose: boolean = true) => { +export const isHiddenPath = ( + item: string, + dot: boolean = true, + underscore: boolean = true +) => { + if (!(dot || underscore)) { + throw Error("parameter error for isHiddenPath"); + } const k = path.posix.normalize(item); // TODO: only unix path now const k2 = k.split("/"); // TODO: only unix path now // log.info(k2) @@ -21,10 +29,10 @@ export const isHiddenPath = (item: string, loose: boolean = true) => { if (singlePart === "." || singlePart === ".." || singlePart === "") { continue; } - if (singlePart[0] === ".") { + if (dot && singlePart[0] === ".") { return true; } - if (loose && singlePart[0] === "_") { + if (underscore && singlePart[0] === "_") { return true; } } diff --git a/src/obsFolderLister.ts b/src/obsFolderLister.ts new file mode 100644 index 0000000..806c042 --- /dev/null +++ b/src/obsFolderLister.ts @@ -0,0 +1,127 @@ +import { Vault, Stat, ListedFiles } from "obsidian"; +import { Queue } from "@fyears/tsqueue"; +import chunk from "lodash/chunk"; +import flatten from "lodash/flatten"; + +import * as origLog from "loglevel"; +const log = origLog.getLogger("rs-default"); + +export interface ObsConfigDirFileType { + key: string; + ctime: number; + mtime: number; + size: number; + type: "folder" | "file"; +} + +const isFolderToSkip = (x: string) => { + let specialFolders = [".git", ".svn", "node_modules"]; + for (const iterator of specialFolders) { + if ( + x === iterator || + x === `${iterator}/` || + x.endsWith(`/${iterator}`) || + x.endsWith(`/${iterator}/`) + ) { + return true; + } + } + return false; +}; + +const isPluginDirItself = (x: string, pluginId: string) => { + return ( + x === pluginId || + x === `${pluginId}/` || + x.endsWith(`/${pluginId}`) || + x.endsWith(`/${pluginId}/`) + ); +}; + +const isLikelyPluginSubFiles = (x: string) => { + const reqFiles = [ + "data.json", + "main.js", + "manifest.json", + ".gitignore", + "styles.css", + ]; + for (const iterator of reqFiles) { + if (x === iterator || x.endsWith(`/${iterator}`)) { + return true; + } + } + return false; +}; + +export const isInsideObsFolder = (x: string, configDir: string) => { + if (!configDir.startsWith(".")) { + throw Error(`configDir should starts with . but we get ${configDir}`); + } + return x === configDir || x.startsWith(`${configDir}/`); +}; + +export const listFilesInObsFolder = async ( + configDir: string, + vault: Vault, + pluginId: string +) => { + const q = new Queue([configDir]); + const CHUNK_SIZE = 10; + const contents: ObsConfigDirFileType[] = []; + while (q.length > 0) { + const itemsToFetch = []; + while (q.length > 0) { + itemsToFetch.push(q.pop()); + } + + const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE); + for (const singleChunk of itemsToFetchChunks) { + const r = singleChunk.map(async (x) => { + const statRes = await vault.adapter.stat(x); + const isFolder = statRes.type === "folder"; + let children: ListedFiles = undefined; + if (isFolder) { + children = await vault.adapter.list(x); + } + + return { + itself: { + key: isFolder ? `${x}/` : x, + ...statRes, + } as ObsConfigDirFileType, + children: children, + }; + }); + const r2 = flatten(await Promise.all(r)); + + for (const iter of r2) { + contents.push(iter.itself); + const isInsideSelfPlugin = isPluginDirItself(iter.itself.key, pluginId); + if (iter.children !== undefined) { + for (const iter2 of iter.children.folders) { + if (isFolderToSkip(iter2)) { + continue; + } + if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) { + // special treatment for remotely-save folder + continue; + } + q.push(iter2); + } + for (const iter2 of iter.children.files) { + if (isFolderToSkip(iter2)) { + continue; + } + if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) { + // special treatment for remotely-save folder + continue; + } + q.push(iter2); + } + } + } + } + } + return contents; +}; diff --git a/src/remoteForWebdav.ts b/src/remoteForWebdav.ts index 9a36fa9..9e948df 100644 --- a/src/remoteForWebdav.ts +++ b/src/remoteForWebdav.ts @@ -400,7 +400,10 @@ export const listFromRemote = async ( return client.client.getDirectoryContents(x, { deep: false, details: false /* no need for verbose details here */, - glob: "/**" /* avoid dot files by using glob */, + // TODO: to support .obsidian, + // we need to load all files including dot, + // anyway to reduce the resources? + // glob: "/**" /* avoid dot files by using glob */, }) as Promise; }); const r2 = flatten(await Promise.all(r)); @@ -421,7 +424,10 @@ export const listFromRemote = async ( { deep: true, details: false /* no need for verbose details here */, - glob: "/**" /* avoid dot files by using glob */, + // TODO: to support .obsidian, + // we need to load all files including dot, + // anyway to reduce the resources? + // glob: "/**" /* avoid dot files by using glob */, } )) as FileStat[]; } diff --git a/src/settings.ts b/src/settings.ts index ac2e64a..9cdb097 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -384,6 +384,59 @@ export class OnedriveRevokeAuthModal extends Modal { } } +class SyncConfigDirModal extends Modal { + plugin: RemotelySavePlugin; + saveDropdownFunc: () => void; + constructor( + app: App, + plugin: RemotelySavePlugin, + saveDropdownFunc: () => void + ) { + super(app); + this.plugin = plugin; + this.saveDropdownFunc = saveDropdownFunc; + } + + async onOpen() { + let { contentEl } = this; + + const texts = [ + "Attention 1/3: This only syncs (copies) the whole Obsidian config dir, not other . folders or files. It also doesn't understand the inner structure of the config dir.", + "Attention 2/3: After the config dir is synced, plugins settings might be corrupted, and Obsidian might need to be restarted to load the new settings.", + "Attention 3/3: The deletion (uninstallation) operations of or inside Obsidian config dir cannot be tracked. So if you want to uninstall a plugin, you need to manually uninstall it on all device, before next sync.", + "If you are agreed to take your own risk, please click the following second confirm button.", + ]; + for (const t of texts) { + contentEl.createEl("p", { + text: t, + }); + } + + new Setting(contentEl) + .addButton((button) => { + button.setButtonText("The Second Confirm To Enable."); + button.onClick(async () => { + this.plugin.settings.syncConfigDir = true; + await this.plugin.saveSettings(); + this.saveDropdownFunc(); + new Notice("You've enabled syncing config folder!"); + this.close(); + }); + }) + .addButton((button) => { + button.setButtonText("Go Back"); + button.onClick(() => { + this.close(); + }); + }); + } + + onClose() { + let { contentEl } = this; + contentEl.empty(); + } +} + class ExportSettingsQrCodeModal extends Modal { plugin: RemotelySavePlugin; constructor(app: App, plugin: RemotelySavePlugin) { @@ -552,30 +605,6 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); - const concurrencyDiv = generalDiv.createEl("div"); - new Setting(concurrencyDiv) - .setName("Concurrency") - .setDesc( - "How many files do you want to download or upload in parallel at most? By default it's set to 5. If you meet any problems such as rate limit, you can reduce the concurrency to a lower value." - ) - .addDropdown((dropdown) => { - dropdown.addOption("1", "1"); - dropdown.addOption("2", "2"); - dropdown.addOption("3", "3"); - dropdown.addOption("5", "5 (default)"); - dropdown.addOption("10", "10"); - dropdown.addOption("15", "15"); - dropdown.addOption("20", "20"); - - dropdown - .setValue(`${this.plugin.settings.concurrency}`) - .onChange(async (val) => { - const realVal = parseInt(val); - this.plugin.settings.concurrency = realVal; - await this.plugin.saveSettings(); - }); - }); - ////////////////////////////////////////////////// // below for general chooser (part 1/2) ////////////////////////////////////////////////// @@ -1182,6 +1211,92 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); + ////////////////////////////////////////////////// + // below for advanced settings + ////////////////////////////////////////////////// + const advDiv = containerEl.createEl("div"); + advDiv.createEl("h2", { + text: "Advanced Settings", + }); + + const concurrencyDiv = advDiv.createEl("div"); + new Setting(concurrencyDiv) + .setName("Concurrency") + .setDesc( + "How many files do you want to download or upload in parallel at most? By default it's set to 5. If you meet any problems such as rate limit, you can reduce the concurrency to a lower value." + ) + .addDropdown((dropdown) => { + dropdown.addOption("1", "1"); + dropdown.addOption("2", "2"); + dropdown.addOption("3", "3"); + dropdown.addOption("5", "5 (default)"); + dropdown.addOption("10", "10"); + dropdown.addOption("15", "15"); + dropdown.addOption("20", "20"); + + dropdown + .setValue(`${this.plugin.settings.concurrency}`) + .onChange(async (val) => { + const realVal = parseInt(val); + this.plugin.settings.concurrency = realVal; + await this.plugin.saveSettings(); + }); + }); + + const syncUnderscoreItemsDiv = advDiv.createEl("div"); + new Setting(syncUnderscoreItemsDiv) + .setName("sync _ files or folders") + .setDesc(`Sync files or folders startting with _ ("underscore") or not.`) + .addDropdown((dropdown) => { + dropdown.addOption("disable", "disable"); + dropdown.addOption("enable", "enable"); + dropdown + .setValue( + `${this.plugin.settings.syncUnderscoreItems ? "enable" : "disable"}` + ) + .onChange(async (val) => { + this.plugin.settings.syncUnderscoreItems = val === "enable"; + await this.plugin.saveSettings(); + }); + }); + + const syncConfigDirDiv = advDiv.createEl("div"); + new Setting(syncConfigDirDiv) + .setName("sync config dir (experimental)") + .setDesc( + `Sync config dir ${this.app.vault.configDir} or not. Please be aware that this may impact all your plugins' or Obsidian's settings, and may require you restart Obsidian after sync. Enable this at your own risk.` + ) + .addDropdown((dropdown) => { + dropdown.addOption("disable", "disable"); + dropdown.addOption("enable", "enable"); + + const bridge = { + secondConfirm: false, + }; + dropdown + .setValue( + `${this.plugin.settings.syncConfigDir ? "enable" : "disable"}` + ) + .onChange(async (val) => { + if (val === "enable" && !bridge.secondConfirm) { + dropdown.setValue("disable"); + const modal = new SyncConfigDirModal( + this.app, + this.plugin, + () => { + bridge.secondConfirm = true; + dropdown.setValue("enable"); + } + ); + modal.open(); + } else { + bridge.secondConfirm = false; + this.plugin.settings.syncConfigDir = false; + await this.plugin.saveSettings(); + } + }); + }); + ////////////////////////////////////////////////// // below for import and export functions ////////////////////////////////////////////////// diff --git a/src/sync.ts b/src/sync.ts index 0346e5b..88e0c2a 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -46,6 +46,7 @@ import { import * as origLog from "loglevel"; import { padEnd } from "lodash"; +import { isInsideObsFolder, ObsConfigDirFileType } from "./obsFolderLister"; const log = origLog.getLogger("rs-default"); export type SyncStatusType = @@ -272,18 +273,39 @@ export const fetchMetadataFile = async ( return metadata; }; +const isSkipItem = ( + key: string, + syncConfigDir: boolean, + syncUnderscoreItems: boolean, + configDir: string +) => { + if (syncConfigDir && isInsideObsFolder(key, configDir)) { + return false; + } + return ( + isHiddenPath(key, true, false) || + (!syncUnderscoreItems && isHiddenPath(key, false, true)) || + key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE || + key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2 + ); +}; + const ensembleMixedStates = async ( remoteStates: FileOrFolderMixedState[], local: TAbstractFile[], + localConfigDirContents: ObsConfigDirFileType[] | undefined, remoteDeleteHistory: DeletionOnRemote[], - localDeleteHistory: FileFolderHistoryRecord[] + localDeleteHistory: FileFolderHistoryRecord[], + syncConfigDir: boolean, + configDir: string, + syncUnderscoreItems: boolean ) => { const results = {} as Record; for (const r of remoteStates) { const key = r.key; - if (isHiddenPath(key)) { + if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) { continue; } results[key] = r; @@ -316,9 +338,10 @@ const ensembleMixedStates = async ( throw Error(`unexpected ${entry}`); } - if (isHiddenPath(key)) { + if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) { continue; } + if (results.hasOwnProperty(key)) { results[key].key = r.key; results[key].existLocal = r.existLocal; @@ -330,6 +353,28 @@ const ensembleMixedStates = async ( } } + if (syncConfigDir && localConfigDirContents !== undefined) { + for (const entry of localConfigDirContents) { + const key = entry.key; + const r: FileOrFolderMixedState = { + key: key, + existLocal: true, + mtimeLocal: Math.max(entry.mtime, entry.ctime), + sizeLocal: entry.size, + }; + + if (results.hasOwnProperty(key)) { + results[key].key = r.key; + results[key].existLocal = r.existLocal; + results[key].mtimeLocal = r.mtimeLocal; + results[key].sizeLocal = r.sizeLocal; + } else { + results[key] = r; + results[key].existRemote = false; + } + } + } + for (const entry of remoteDeleteHistory) { const key = entry.key; const r = { @@ -337,6 +382,10 @@ const ensembleMixedStates = async ( deltimeRemote: entry.actionWhen, } as FileOrFolderMixedState; + if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) { + continue; + } + if (results.hasOwnProperty(key)) { results[key].key = r.key; results[key].deltimeRemote = r.deltimeRemote; @@ -365,9 +414,10 @@ const ensembleMixedStates = async ( deltimeLocal: entry.actionWhen, } as FileOrFolderMixedState; - if (isHiddenPath(key)) { + if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) { continue; } + if (results.hasOwnProperty(key)) { results[key].key = r.key; results[key].deltimeLocal = r.deltimeLocal; @@ -618,17 +668,25 @@ const DELETION_DECISIONS: Set = new Set([ export const getSyncPlan = async ( remoteStates: FileOrFolderMixedState[], local: TAbstractFile[], + localConfigDirContents: ObsConfigDirFileType[] | undefined, remoteDeleteHistory: DeletionOnRemote[], localDeleteHistory: FileFolderHistoryRecord[], remoteType: SUPPORTED_SERVICES_TYPE, vault: Vault, + syncConfigDir: boolean, + configDir: string, + syncUnderscoreItems: boolean, password: string = "" ) => { const mixedStates = await ensembleMixedStates( remoteStates, local, + localConfigDirContents, remoteDeleteHistory, - localDeleteHistory + localDeleteHistory, + syncConfigDir, + configDir, + syncUnderscoreItems ); const sortedKeys = Object.keys(mixedStates).sort( diff --git a/tests/misc.test.ts b/tests/misc.test.ts index ce528e6..6ff0010 100644 --- a/tests/misc.test.ts +++ b/tests/misc.test.ts @@ -21,7 +21,7 @@ describe("Misc: hidden file", () => { item = "_hidden_loose"; expect(misc.isHiddenPath(item)).to.be.true; - expect(misc.isHiddenPath(item, false)).to.be.false; + expect(misc.isHiddenPath(item, true, false)).to.be.false; item = "/sdd/_hidden_loose"; expect(misc.isHiddenPath(item)).to.be.true; @@ -30,10 +30,21 @@ describe("Misc: hidden file", () => { expect(misc.isHiddenPath(item)).to.be.true; item = "what/../_hidden_loose/what/what/what"; - expect(misc.isHiddenPath(item, false)).to.be.false; + expect(misc.isHiddenPath(item, true, false)).to.be.false; item = "what/../_hidden_loose/../.hidden/what/what/what"; - expect(misc.isHiddenPath(item, false)).to.be.true; + expect(misc.isHiddenPath(item, true, false)).to.be.true; + + item = "what/../_hidden_loose/../.hidden/what/what/what"; + expect(misc.isHiddenPath(item, false, true)).to.be.false; + + item = "what/_hidden_loose/what/what/what"; + expect(misc.isHiddenPath(item, false, true)).to.be.true; + expect(misc.isHiddenPath(item, true, false)).to.be.false; + + item = "what/.hidden/what/what/what"; + expect(misc.isHiddenPath(item, false, true)).to.be.false; + expect(misc.isHiddenPath(item, true, false)).to.be.true; }); });