diff --git a/package.json b/package.json index 841ef88..3a747ba 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,10 @@ "browser": { "path": "path-browserify", "process": "process/browser", - "stream": "stream-browserify" + "stream": "stream-browserify", + "crypto": "crypto-browserify", + "util": "util/", + "assert": "assert/" }, "source": "main.ts", "keywords": [], @@ -42,9 +45,12 @@ "@aws-sdk/lib-storage": "^3.40.1", "@aws-sdk/signature-v4-crt": "^3.37.0", "acorn": "^8.5.0", + "assert": "^2.0.0", "aws-crt": "^1.10.1", "buffer": "^6.0.3", "codemirror": "^5.63.1", + "crypto-browserify": "^3.12.0", + "dropbox": "^10.22.0", "localforage": "^1.10.0", "mime-types": "^2.1.33", "obsidian": "^0.12.0", @@ -53,6 +59,7 @@ "rfc4648": "^1.5.0", "rimraf": "^3.0.2", "stream-browserify": "^3.0.0", + "util": "^0.12.4", "webdav": "^4.7.0", "webdav-fs": "^4.0.0", "xregexp": "^5.1.0" diff --git a/src/baseTypes.ts b/src/baseTypes.ts index 03feb8e..13e70c7 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -2,7 +2,7 @@ * Only type defs here. */ -export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav"; +export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "dropbox"; export interface RemoteItem { key: string; diff --git a/src/main.ts b/src/main.ts index 39a34f5..cc532bd 100644 --- a/src/main.ts +++ b/src/main.ts @@ -25,8 +25,11 @@ import type { InternalDBs } from "./localdb"; import type { SyncStatusType, PasswordCheckType } from "./sync"; import { isPasswordOk, getSyncPlan, doActualSync } from "./sync"; + import { S3Config, DEFAULT_S3_CONFIG } from "./s3"; import { WebdavConfig, DEFAULT_WEBDAV_CONFIG, WebdavAuthType } from "./webdav"; +import { DropboxConfig, DEFAULT_DROPBOX_CONFIG } from "./remoteForDropbox"; + import { RemoteClient } from "./remote"; import { exportSyncPlansToFiles } from "./debugMode"; import { SUPPORTED_SERVICES_TYPE } from "./baseTypes"; @@ -34,6 +37,7 @@ import { SUPPORTED_SERVICES_TYPE } from "./baseTypes"; interface RemotelySavePluginSettings { s3: S3Config; webdav: WebdavConfig; + dropbox: DropboxConfig; password: string; serviceType: SUPPORTED_SERVICES_TYPE; enableExperimentService: boolean; @@ -42,6 +46,7 @@ interface RemotelySavePluginSettings { const DEFAULT_SETTINGS: RemotelySavePluginSettings = { s3: DEFAULT_S3_CONFIG, webdav: DEFAULT_WEBDAV_CONFIG, + dropbox: DEFAULT_DROPBOX_CONFIG, password: "", serviceType: "s3", enableExperimentService: false, @@ -94,15 +99,16 @@ export default class RemotelySavePlugin extends Plugin { const client = new RemoteClient( this.settings.serviceType, this.settings.s3, - this.settings.webdav + this.settings.webdav, + this.settings.dropbox ); const remoteRsp = await client.listFromRemote(); + // console.log(remoteRsp); new Notice("3/6 Starting to fetch local meta data."); this.syncStatus = "getting_local_meta"; const local = this.app.vault.getAllLoadedFiles(); const localHistory = await loadDeleteRenameHistoryTable(this.db); - // console.log(remoteRsp); // console.log(local); // console.log(localHistory); @@ -453,6 +459,59 @@ class RemotelySaveSettingTab extends PluginSettingTab { }); }); + const dropboxDiv = containerEl.createEl("div", { cls: "dropbox-hide" }); + dropboxDiv.toggleClass( + "dropbox-hide", + this.plugin.settings.serviceType !== "dropbox" + ); + dropboxDiv.createEl("h2", { text: "for Dropbox" }); + dropboxDiv.createEl("p", { + text: "Disclaimer: Sync support for Dropbox are more experimental, and s3 functions are more stable now.", + cls: "dropbox-disclaimer", + }); + dropboxDiv.createEl("p", { + text: "Disclaimer: This app is NOT an official Dropbox product. It just uses Dropbox open api.", + cls: "dropbox-disclaimer", + }); + dropboxDiv.createEl("p", { + text: "We create a folder App/obsidian-remotely-save on your Dropbox. All files/folders sync would happen inside this folder.", + }); + + new Setting(dropboxDiv) + .setName("access token") + .setDesc("access token") + .addText((text) => + text + .setPlaceholder("") + .setValue(this.plugin.settings.dropbox.accessToken) + .onChange(async (value) => { + this.plugin.settings.dropbox.accessToken = value.trim(); + await this.plugin.saveSettings(); + }) + ); + + new Setting(dropboxDiv) + .setName("check connectivity") + .setDesc("check connectivity") + .addButton(async (button) => { + button.setButtonText("Check"); + button.onClick(async () => { + new Notice("Checking..."); + const client = new RemoteClient( + "dropbox", + undefined, + undefined, + this.plugin.settings.dropbox + ); + const res = await client.checkConnectivity(); + if (res) { + new Notice("Great! We can connect to Dropbox!"); + } else { + new Notice("We cannot connect to Dropbox."); + } + }); + }); + const webdavDiv = containerEl.createEl("div", { cls: "webdav-hide" }); webdavDiv.toggleClass( "webdav-hide", @@ -563,6 +622,7 @@ class RemotelySaveSettingTab extends PluginSettingTab { this.plugin.settings.enableExperimentService; dropdown.addOption("s3", "s3 (-compatible)"); + dropdown.addOption("dropbox", "Dropbox"); if (currService === "webdav" || enableExperimentService) { dropdown.addOption("webdav", "webdav (experimental)"); if (!enableExperimentService) { @@ -578,6 +638,10 @@ class RemotelySaveSettingTab extends PluginSettingTab { "s3-hide", this.plugin.settings.serviceType !== "s3" ); + dropboxDiv.toggleClass( + "dropbox-hide", + this.plugin.settings.serviceType !== "dropbox" + ); webdavDiv.toggleClass( "webdav-hide", this.plugin.settings.serviceType !== "webdav" diff --git a/src/remote.ts b/src/remote.ts index 13e3322..617e535 100644 --- a/src/remote.ts +++ b/src/remote.ts @@ -3,6 +3,7 @@ import { Vault } from "obsidian"; import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes"; import * as s3 from "./s3"; import * as webdav from "./webdav"; +import * as dropbox from "./remoteForDropbox"; export class RemoteClient { readonly serviceType: SUPPORTED_SERVICES_TYPE; @@ -10,19 +11,25 @@ export class RemoteClient { readonly s3Config?: s3.S3Config; readonly webdavClient?: webdav.WebDAVClient; readonly webdavConfig?: webdav.WebdavConfig; + readonly dropboxClient?: dropbox.Dropbox; + readonly dropboxConfig?: dropbox.DropboxConfig; constructor( serviceType: SUPPORTED_SERVICES_TYPE, s3Config?: s3.S3Config, - webdavConfig?: webdav.WebdavConfig + webdavConfig?: webdav.WebdavConfig, + dropboxConfig?: dropbox.DropboxConfig ) { this.serviceType = serviceType; if (serviceType === "s3") { - this.s3Config = s3Config; - this.s3Client = s3.getS3Client(s3Config); + this.s3Config = { ...s3Config }; + this.s3Client = s3.getS3Client(this.s3Config); } else if (serviceType === "webdav") { - this.webdavConfig = webdavConfig; - this.webdavClient = webdav.getWebdavClient(webdavConfig); + this.webdavConfig = { ...webdavConfig }; + this.webdavClient = webdav.getWebdavClient(this.webdavConfig); + } else if (serviceType === "dropbox") { + this.dropboxConfig = { ...dropboxConfig }; + this.dropboxClient = dropbox.getDropboxClient(this.dropboxConfig); } else { throw Error(`not supported service type ${this.serviceType}`); } @@ -37,6 +44,8 @@ export class RemoteClient { ); } else if (this.serviceType === "webdav") { return await webdav.getRemoteMeta(this.webdavClient, fileOrFolderPath); + } else if (this.serviceType === "dropbox") { + return await dropbox.getRemoteMeta(this.dropboxClient, fileOrFolderPath); } else { throw Error(`not supported service type ${this.serviceType}`); } @@ -47,7 +56,8 @@ export class RemoteClient { vault: Vault, isRecursively: boolean = false, password: string = "", - remoteEncryptedKey: string = "" + remoteEncryptedKey: string = "", + foldersCreatedBefore: Set | undefined = undefined ) => { if (this.serviceType === "s3") { return await s3.uploadToRemote( @@ -68,6 +78,16 @@ export class RemoteClient { password, remoteEncryptedKey ); + } else if (this.serviceType === "dropbox") { + return await dropbox.uploadToRemote( + this.dropboxClient, + fileOrFolderPath, + vault, + isRecursively, + password, + remoteEncryptedKey, + foldersCreatedBefore + ); } else { throw Error(`not supported service type ${this.serviceType}`); } @@ -78,6 +98,8 @@ export class RemoteClient { return await s3.listFromRemote(this.s3Client, this.s3Config, prefix); } else if (this.serviceType === "webdav") { return await webdav.listFromRemote(this.webdavClient, prefix); + } else if (this.serviceType === "dropbox") { + return await dropbox.listFromRemote(this.dropboxClient, prefix); } else { throw Error(`not supported service type ${this.serviceType}`); } @@ -109,6 +131,15 @@ export class RemoteClient { password, remoteEncryptedKey ); + } else if (this.serviceType === "dropbox") { + return await dropbox.downloadFromRemote( + this.dropboxClient, + fileOrFolderPath, + vault, + mtime, + password, + remoteEncryptedKey + ); } else { throw Error(`not supported service type ${this.serviceType}`); } @@ -134,6 +165,13 @@ export class RemoteClient { password, remoteEncryptedKey ); + } else if (this.serviceType === "dropbox") { + return await dropbox.deleteFromRemote( + this.dropboxClient, + fileOrFolderPath, + password, + remoteEncryptedKey + ); } else { throw Error(`not supported service type ${this.serviceType}`); } @@ -144,6 +182,8 @@ export class RemoteClient { return await s3.checkConnectivity(this.s3Client, this.s3Config); } else if (this.serviceType === "webdav") { return await webdav.checkConnectivity(this.webdavClient); + } else if (this.serviceType === "dropbox") { + return await dropbox.checkConnectivity(this.dropboxClient); } else { throw Error(`not supported service type ${this.serviceType}`); } diff --git a/src/remoteForDropbox.ts b/src/remoteForDropbox.ts new file mode 100644 index 0000000..bd0868c --- /dev/null +++ b/src/remoteForDropbox.ts @@ -0,0 +1,372 @@ +import * as path from "path"; +import { FileStats, Vault } from "obsidian"; + +import { Dropbox, DropboxResponse, files } from "dropbox"; +export { Dropbox } from "dropbox"; +import { RemoteItem } from "./baseTypes"; +import { + arrayBufferToBuffer, + bufferToArrayBuffer, + mkdirpInVault, + getPathFolder, + getFolderLevels, + setToString, +} from "./misc"; +import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; +import { strict as assert } from "assert"; + +export interface DropboxConfig { + accessToken: string; +} + +export const DEFAULT_DROPBOX_CONFIG = { + accessToken: "", +}; + +export const getDropboxPath = (fileOrFolderPath: string) => { + let key = fileOrFolderPath; + if (!fileOrFolderPath.startsWith("/")) { + key = `/${fileOrFolderPath}`; + } + if (key.endsWith("/")) { + key = key.slice(0, key.length - 1); + } + return key; +}; + +const getNormPath = (fileOrFolderPath: string) => { + if (fileOrFolderPath.startsWith("/")) { + return fileOrFolderPath.slice(1); + } + return fileOrFolderPath; +}; + +const fromDropboxItemToRemoteItem = ( + x: + | files.FileMetadataReference + | files.FolderMetadataReference + | files.DeletedMetadataReference +): RemoteItem => { + let key = getNormPath(x.path_display); + if (x[".tag"] === "folder" && !key.endsWith("/")) { + key = `${key}/`; + } + + if (x[".tag"] === "folder") { + return { + key: key, + lastModified: undefined, + size: 0, + remoteType: "dropbox", + etag: `${x.id}\t`, + } as RemoteItem; + } else if (x[".tag"] === "file") { + return { + key: key, + lastModified: Date.parse(x.server_modified).valueOf(), + size: x.size, + remoteType: "dropbox", + etag: `${x.id}\t${x.content_hash}`, + } as RemoteItem; + } else if (x[".tag"] === "deleted") { + throw Error("do not support deleted tag"); + } +}; + +/** + * Dropbox api doesn't return mtime for folders. + * This is a try to assign mtime by using files in folder. + * @param allFilesFolders + * @returns + */ +const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => { + if (allFilesFolders.length === 0) { + return; + } + + // sort by longer to shorter + allFilesFolders.sort((a, b) => b.key.length - a.key.length); + + // a "map" from dir to mtime + let potentialMTime = {} as Record; + + // first sort pass, from buttom to up + for (const item of allFilesFolders) { + if (item.key.endsWith("/")) { + // itself is a folder, and initially doesn't have mtime + if (item.lastModified === undefined && item.key in potentialMTime) { + // previously we gathered all sub info of this folder + item.lastModified = potentialMTime[item.key]; + } + } + const parent = `${path.posix.dirname(item.key)}/`; + if (item.lastModified !== undefined) { + if (parent in potentialMTime) { + potentialMTime[parent] = Math.max( + potentialMTime[parent], + item.lastModified + ); + } else { + potentialMTime[parent] = item.lastModified; + } + } + } + + // second pass, from up to buttom. + // fill mtime by parent folder or Date.Now() if still not available. + // this is only possible if no any sub-folder-files recursively. + // we do not sort the array again, just iterate over it by reverse + // using good old for loop. + for (let i = allFilesFolders.length - 1; i >= 0; --i) { + const item = allFilesFolders[i]; + if (!item.key.endsWith("/")) { + continue; // skip files + } + if (item.lastModified !== undefined) { + continue; // don't need to deal with it + } + assert(!(item.key in potentialMTime)); + const parent = `${path.posix.dirname(item.key)}/`; + if (parent in potentialMTime) { + item.lastModified = potentialMTime[parent]; + } else { + item.lastModified = Date.now().valueOf(); + potentialMTime[item.key] = item.lastModified; + } + } + + return allFilesFolders; +}; + +export const getDropboxClient = ( + accessTokenOrDropboxConfig: string | DropboxConfig +) => { + if (typeof accessTokenOrDropboxConfig === "string") { + return new Dropbox({ accessToken: accessTokenOrDropboxConfig }); + } + return new Dropbox({ + accessToken: accessTokenOrDropboxConfig.accessToken, + }); +}; + +export const getRemoteMeta = async (dbx: Dropbox, fileOrFolderPath: string) => { + if (fileOrFolderPath === "" || fileOrFolderPath === "/") { + // filesGetMetadata doesn't support root folder + // we instead try to list files + // if no error occurs, we ensemble a fake result. + const rsp = await dbx.filesListFolder({ + path: "", + recursive: false, // don't need to recursive here + }); + if (rsp.status !== 200) { + throw Error(JSON.stringify(rsp)); + } + return { + key: fileOrFolderPath, + lastModified: undefined, + size: 0, + remoteType: "dropbox", + etag: undefined, + } as RemoteItem; + } + + const key = getDropboxPath(fileOrFolderPath); + + const rsp = await dbx.filesGetMetadata({ + path: key, + }); + if (rsp.status !== 200) { + throw Error(JSON.stringify(rsp)); + } + return fromDropboxItemToRemoteItem(rsp.result); +}; + +export const uploadToRemote = async ( + dbx: Dropbox, + fileOrFolderPath: string, + vault: Vault, + isRecursively: boolean = false, + password: string = "", + remoteEncryptedKey: string = "", + foldersCreatedBefore: Set | undefined = undefined +) => { + let uploadFile = fileOrFolderPath; + if (password !== "") { + uploadFile = remoteEncryptedKey; + } + uploadFile = getDropboxPath(uploadFile); + + const isFolder = fileOrFolderPath.endsWith("/"); + + if (isFolder && isRecursively) { + throw Error("upload function doesn't implement recursive function yet!"); + } else if (isFolder && !isRecursively) { + // folder + if (password === "") { + // if not encrypted, mkdir a remote folder + if (foldersCreatedBefore?.has(uploadFile)) { + // created, pass + } else { + try { + await dbx.filesCreateFolderV2({ + path: uploadFile, + }); + foldersCreatedBefore?.add(uploadFile); + } catch (err) { + if (err.status === 409) { + // pass + foldersCreatedBefore?.add(uploadFile); + } else { + throw err; + } + } + } + const res = await getRemoteMeta(dbx, uploadFile); + return res; + } else { + // if encrypted, upload a fake file with the encrypted file name + await dbx.filesUpload({ + path: uploadFile, + contents: "", + }); + return await getRemoteMeta(dbx, uploadFile); + } + } else { + // file + // we ignore isRecursively parameter here + const localContent = await vault.adapter.readBinary(fileOrFolderPath); + let remoteContent = localContent; + if (password !== "") { + remoteContent = await encryptArrayBuffer(localContent, password); + } + // in dropbox, we don't need to create folders before uploading! cool! + // TODO: filesUploadSession for larger files (>=150 MB) + await dbx.filesUpload({ + path: uploadFile, + contents: remoteContent, + mode: { + ".tag": "overwrite", + }, + }); + // we want to mark that parent folders are created + if (foldersCreatedBefore !== undefined) { + const dirs = getFolderLevels(uploadFile).map(getDropboxPath); + for (const dir of dirs) { + foldersCreatedBefore?.add(dir); + } + } + return await getRemoteMeta(dbx, uploadFile); + } +}; + +export const listFromRemote = async (dbx: Dropbox, prefix?: string) => { + if (prefix !== undefined) { + throw Error("prefix not supported (yet)"); + } + const res = await dbx.filesListFolder({ path: "", recursive: true }); + if (res.status !== 200) { + throw Error(JSON.stringify(res)); + } + // console.log(res); + const contents = res.result.entries; + const unifiedContents = contents + .filter((x) => x[".tag"] !== "deleted") + .map(fromDropboxItemToRemoteItem); + fixLastModifiedTimeInplace(unifiedContents); + return { + Contents: unifiedContents, + }; +}; + +const downloadFromRemoteRaw = async ( + dbx: Dropbox, + fileOrFolderPath: string +) => { + const key = getDropboxPath(fileOrFolderPath); + const rsp = await dbx.filesDownload({ + path: key, + }); + if ((rsp.result as any).fileBlob !== undefined) { + // we get a Blob + const content = (rsp.result as any).fileBlob as Blob; + return await content.arrayBuffer(); + } else if ((rsp.result as any).fileBinary !== undefined) { + // we get a Buffer + const content = (rsp.result as any).fileBinary as Buffer; + return bufferToArrayBuffer(content); + } else { + throw Error(`unknown rsp from dropbox download: ${rsp}`); + } +}; + +export const downloadFromRemote = async ( + dbx: Dropbox, + fileOrFolderPath: string, + vault: Vault, + mtime: number, + password: string = "", + remoteEncryptedKey: string = "" +) => { + const isFolder = fileOrFolderPath.endsWith("/"); + + await mkdirpInVault(fileOrFolderPath, vault); + + // the file is always local file + // we need to encrypt it + + if (isFolder) { + // mkdirp locally is enough + // do nothing here + } else { + let downloadFile = fileOrFolderPath; + if (password !== "") { + downloadFile = remoteEncryptedKey; + } + downloadFile = getDropboxPath(downloadFile); + const remoteContent = await downloadFromRemoteRaw(dbx, downloadFile); + let localContent = remoteContent; + if (password !== "") { + localContent = await decryptArrayBuffer(remoteContent, password); + } + await vault.adapter.writeBinary(fileOrFolderPath, localContent, { + mtime: mtime, + }); + } +}; + +export const deleteFromRemote = async ( + dbx: Dropbox, + fileOrFolderPath: string, + password: string = "", + remoteEncryptedKey: string = "" +) => { + if (fileOrFolderPath === "/") { + return; + } + let remoteFileName = fileOrFolderPath; + if (password !== "") { + remoteFileName = remoteEncryptedKey; + } + remoteFileName = getDropboxPath(remoteFileName); + + try { + await dbx.filesDeleteV2({ + path: remoteFileName, + }); + } catch (err) { + console.error("some error while deleting"); + console.log(err); + } +}; + +export const checkConnectivity = async (dbx: Dropbox) => { + try { + const results = await getRemoteMeta(dbx, "/"); + if (results === undefined) { + return false; + } + return true; + } catch (err) { + return false; + } +}; diff --git a/src/sync.ts b/src/sync.ts index e38e502..d7eeda6 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -9,7 +9,7 @@ import type { FileFolderHistoryRecord, InternalDBs } from "./localdb"; import { RemoteClient } from "./remote"; import type { SUPPORTED_SERVICES_TYPE, RemoteItem } from "./baseTypes"; -import { mkdirpInVault, isHiddenPath, isVaildText } from "./misc"; +import { mkdirpInVault, isHiddenPath, isVaildText, setToString } from "./misc"; import { decryptBase32ToString, encryptStringToBase32, @@ -389,7 +389,7 @@ const getOperation = ( } if (r.decision === "unknown") { - throw Error(`unknown decision for ${r}`); + throw Error(`unknown decision for ${JSON.stringify(r)}`); } return r; @@ -428,7 +428,8 @@ const dispatchOperationToActual = async ( client: RemoteClient, db: InternalDBs, vault: Vault, - password: string = "" + password: string = "", + foldersCreatedBefore: Set | undefined = undefined ) => { let remoteEncryptedKey = key; if (password !== "") { @@ -461,7 +462,8 @@ const dispatchOperationToActual = async ( vault, false, password, - remoteEncryptedKey + remoteEncryptedKey, + foldersCreatedBefore ); await upsertSyncMetaMappingData( client.serviceType, @@ -493,7 +495,8 @@ const dispatchOperationToActual = async ( vault, false, password, - remoteEncryptedKey + remoteEncryptedKey, + foldersCreatedBefore ); await upsertSyncMetaMappingData( client.serviceType, @@ -521,18 +524,35 @@ export const doActualSync = async ( password: string = "" ) => { const keyStates = syncPlan.mixedStates; - await Promise.all( - Object.entries(keyStates) - .sort((k, v) => -(k as string).length) - .map(async ([k, v]) => - dispatchOperationToActual( - k as string, - v as FileOrFolderMixedState, - client, - db, - vault, - password - ) - ) - ); + const foldersCreatedBefore = new Set(); + for (const [k, v] of Object.entries(keyStates).sort( + ([k1, v1], [k2, v2]) => k2.length - k1.length + )) { + const k2 = k as string; + const v2 = v as FileOrFolderMixedState; + await dispatchOperationToActual( + k as string, + v as FileOrFolderMixedState, + client, + db, + vault, + password, + foldersCreatedBefore + ); + // console.log(`finished ${k}, with ${setToString(foldersCreatedBefore)}`); + } + // await Promise.all( + // Object.entries(keyStates) + // .map(async ([k, v]) => + // dispatchOperationToActual( + // k as string, + // v as FileOrFolderMixedState, + // client, + // db, + // vault, + // password, + // foldersCreatedBefore + // ) + // ) + // ); }; diff --git a/styles.css b/styles.css index b91e9c6..5fcb4d8 100644 --- a/styles.css +++ b/styles.css @@ -11,10 +11,16 @@ display: none; } +.dropbox-disclaimer { + font-weight: bold; +} +.dropbox-hide { + display: none; +} + .webdav-disclaimer { font-weight: bold; } - .webdav-hide { display: none; }