import { Dropbox, DropboxAuth, files } from "dropbox"; import { Vault } from "obsidian"; import * as path from "path"; import { DropboxConfig, RemoteItem, COMMAND_CALLBACK_DROPBOX, OAUTH2_FORCE_EXPIRE_MILLISECONDS, } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { bufferToArrayBuffer, getFolderLevels, mkdirpInVault } from "./misc"; export { Dropbox } from "dropbox"; import * as origLog from "loglevel"; const log = origLog.getLogger("rs-default"); export const DEFAULT_DROPBOX_CONFIG: DropboxConfig = { accessToken: "", clientID: process.env.DEFAULT_DROPBOX_APP_KEY, refreshToken: "", accessTokenExpiresInSeconds: 0, accessTokenExpiresAtTime: 0, accountID: "", username: "", credentialsShouldBeDeletedAtTime: 0, }; export const getDropboxPath = (fileOrFolderPath: string, vaultName: string) => { let key = fileOrFolderPath; if (fileOrFolderPath === "/" || fileOrFolderPath === "") { // special key = `/${vaultName}`; } if (!fileOrFolderPath.startsWith("/")) { // then this is original path in Obsidian key = `/${vaultName}/${fileOrFolderPath}`; } if (key.endsWith("/")) { key = key.slice(0, key.length - 1); } return key; }; const getNormPath = (fileOrFolderPath: string, vaultName: string) => { if ( !( fileOrFolderPath === `/${vaultName}` || fileOrFolderPath.startsWith(`/${vaultName}/`) ) ) { throw Error(`"${fileOrFolderPath}" doesn't starts with "/${vaultName}/"`); } return fileOrFolderPath.slice(`/${vaultName}/`.length); }; const fromDropboxItemToRemoteItem = ( x: | files.FileMetadataReference | files.FolderMetadataReference | files.DeletedMetadataReference, vaultName: string ): RemoteItem => { let key = getNormPath(x.path_display, vaultName); if (x[".tag"] === "folder" && !key.endsWith("/")) { key = `${key}/`; } if (x[".tag"] === "folder") { return { key: key, lastModified: undefined, size: 0, remoteType: "dropbox", etag: `${x.id}\t`, } as RemoteItem; } else if (x[".tag"] === "file") { return { key: key, lastModified: Date.parse(x.server_modified).valueOf(), size: x.size, remoteType: "dropbox", etag: `${x.id}\t${x.content_hash}`, } as RemoteItem; } else if (x[".tag"] === "deleted") { throw Error("do not support deleted tag"); } }; /** * Dropbox api doesn't return mtime for folders. * This is a try to assign mtime by using files in folder. * @param allFilesFolders * @returns */ const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => { if (allFilesFolders.length === 0) { return; } // sort by longer to shorter allFilesFolders.sort((a, b) => b.key.length - a.key.length); // a "map" from dir to mtime let potentialMTime = {} as Record; // first sort pass, from buttom to up for (const item of allFilesFolders) { if (item.key.endsWith("/")) { // itself is a folder, and initially doesn't have mtime if (item.lastModified === undefined && item.key in potentialMTime) { // previously we gathered all sub info of this folder item.lastModified = potentialMTime[item.key]; } } const parent = `${path.posix.dirname(item.key)}/`; if (item.lastModified !== undefined) { if (parent in potentialMTime) { potentialMTime[parent] = Math.max( potentialMTime[parent], item.lastModified ); } else { potentialMTime[parent] = item.lastModified; } } } // second pass, from up to buttom. // fill mtime by parent folder or Date.Now() if still not available. // this is only possible if no any sub-folder-files recursively. // we do not sort the array again, just iterate over it by reverse // using good old for loop. for (let i = allFilesFolders.length - 1; i >= 0; --i) { const item = allFilesFolders[i]; if (!item.key.endsWith("/")) { continue; // skip files } if (item.lastModified !== undefined) { continue; // don't need to deal with it } const parent = `${path.posix.dirname(item.key)}/`; if (parent in potentialMTime) { item.lastModified = potentialMTime[parent]; } else { item.lastModified = Date.now().valueOf(); potentialMTime[item.key] = item.lastModified; } } return allFilesFolders; }; //////////////////////////////////////////////////////////////////////////////// // Dropbox authorization using PKCE // see https://dropbox.tech/developers/pkce--what-and-why- //////////////////////////////////////////////////////////////////////////////// export const getAuthUrlAndVerifier = async (appKey: string) => { const auth = new DropboxAuth({ clientId: appKey, }); const authUrl = ( await auth.getAuthenticationUrl( `obsidian://${COMMAND_CALLBACK_DROPBOX}`, undefined, "code", "offline", undefined, "none", true ) ).toString(); const verifier = auth.getCodeVerifier(); return { authUrl: authUrl, verifier: verifier, }; }; export interface DropboxSuccessAuthRes { access_token: string; token_type: "bearer"; expires_in: string; refresh_token?: string; scope?: string; uid?: string; account_id?: string; } export const sendAuthReq = async ( appKey: string, verifier: string, authCode: string ) => { const resp1 = await fetch("https://api.dropboxapi.com/oauth2/token", { method: "POST", body: new URLSearchParams({ code: authCode, grant_type: "authorization_code", code_verifier: verifier, client_id: appKey, redirect_uri: `obsidian://${COMMAND_CALLBACK_DROPBOX}`, }), }); const resp2 = (await resp1.json()) as DropboxSuccessAuthRes; return resp2; }; export const sendRefreshTokenReq = async ( appKey: string, refreshToken: string ) => { log.info("start auto getting refreshed Dropbox access token."); const resp1 = await fetch("https://api.dropboxapi.com/oauth2/token", { method: "POST", body: new URLSearchParams({ grant_type: "refresh_token", refresh_token: refreshToken, client_id: appKey, }), }); const resp2 = (await resp1.json()) as DropboxSuccessAuthRes; log.info("finish auto getting refreshed Dropbox access token."); return resp2; }; export const setConfigBySuccessfullAuthInplace = async ( config: DropboxConfig, authRes: DropboxSuccessAuthRes, saveUpdatedConfigFunc: () => Promise | undefined ) => { log.info("start updating local info of Dropbox token"); config.accessToken = authRes.access_token; config.accessTokenExpiresInSeconds = parseInt(authRes.expires_in); config.accessTokenExpiresAtTime = Date.now() + parseInt(authRes.expires_in) * 1000 - 10 * 1000; // manually set it expired after 80 days; config.credentialsShouldBeDeletedAtTime = Date.now() + OAUTH2_FORCE_EXPIRE_MILLISECONDS; if (authRes.refresh_token !== undefined) { config.refreshToken = authRes.refresh_token; } if (authRes.refresh_token !== undefined) { config.accountID = authRes.account_id; } if (saveUpdatedConfigFunc !== undefined) { await saveUpdatedConfigFunc(); } log.info("finish updating local info of Dropbox token"); }; //////////////////////////////////////////////////////////////////////////////// // Other usual common methods //////////////////////////////////////////////////////////////////////////////// export class WrappedDropboxClient { dropboxConfig: DropboxConfig; vaultName: string; saveUpdatedConfigFunc: () => Promise; dropbox: Dropbox; vaultFolderExists: boolean; constructor( dropboxConfig: DropboxConfig, vaultName: string, saveUpdatedConfigFunc: () => Promise ) { this.dropboxConfig = dropboxConfig; this.vaultName = vaultName; this.saveUpdatedConfigFunc = saveUpdatedConfigFunc; this.vaultFolderExists = false; } init = async () => { // check token if ( this.dropboxConfig.accessToken === "" || this.dropboxConfig.refreshToken === "" ) { throw Error("The user has not manually auth yet."); } const currentTs = Date.now(); if (this.dropboxConfig.accessTokenExpiresAtTime > currentTs) { this.dropbox = new Dropbox({ accessToken: this.dropboxConfig.accessToken, }); } else { if (this.dropboxConfig.refreshToken === "") { throw Error( "We need to automatically refresh token but none is stored." ); } const resp = await sendRefreshTokenReq( this.dropboxConfig.clientID, this.dropboxConfig.refreshToken ); setConfigBySuccessfullAuthInplace( this.dropboxConfig, resp, this.saveUpdatedConfigFunc ); this.dropbox = new Dropbox({ accessToken: this.dropboxConfig.accessToken, }); } // check vault folder // log.info(`checking remote has folder /${this.vaultName}`); if (this.vaultFolderExists) { // log.info(`already checked, /${this.vaultName} exist before`) } else { const res = await this.dropbox.filesListFolder({ path: "", recursive: false, }); for (const item of res.result.entries) { if (item.path_display === `/${this.vaultName}`) { this.vaultFolderExists = true; break; } } if (!this.vaultFolderExists) { log.info(`remote does not have folder /${this.vaultName}`); await this.dropbox.filesCreateFolderV2({ path: `/${this.vaultName}`, }); log.info(`remote folder /${this.vaultName} created`); this.vaultFolderExists = true; } else { // log.info(`remote folder /${this.vaultName} exists`); } } return this.dropbox; }; } /** * @param dropboxConfig * @returns */ export const getDropboxClient = ( dropboxConfig: DropboxConfig, vaultName: string, saveUpdatedConfigFunc: () => Promise ) => { return new WrappedDropboxClient( dropboxConfig, vaultName, saveUpdatedConfigFunc ); }; export const getRemoteMeta = async ( client: WrappedDropboxClient, fileOrFolderPath: string ) => { await client.init(); if (fileOrFolderPath === "" || fileOrFolderPath === "/") { // filesGetMetadata doesn't support root folder // we instead try to list files // if no error occurs, we ensemble a fake result. const rsp = await client.dropbox.filesListFolder({ path: `/${client.vaultName}`, recursive: false, // don't need to recursive here }); if (rsp.status !== 200) { throw Error(JSON.stringify(rsp)); } return { key: fileOrFolderPath, lastModified: undefined, size: 0, remoteType: "dropbox", etag: undefined, } as RemoteItem; } const key = getDropboxPath(fileOrFolderPath, client.vaultName); const rsp = await client.dropbox.filesGetMetadata({ path: key, }); if (rsp.status !== 200) { throw Error(JSON.stringify(rsp)); } return fromDropboxItemToRemoteItem(rsp.result, client.vaultName); }; export const uploadToRemote = async ( client: WrappedDropboxClient, fileOrFolderPath: string, vault: Vault, isRecursively: boolean = false, password: string = "", remoteEncryptedKey: string = "", foldersCreatedBefore: Set | undefined = undefined, uploadRaw: boolean = false, rawContent: string | ArrayBuffer = "" ) => { await client.init(); let uploadFile = fileOrFolderPath; if (password !== "") { uploadFile = remoteEncryptedKey; } uploadFile = getDropboxPath(uploadFile, client.vaultName); const isFolder = fileOrFolderPath.endsWith("/"); if (isFolder && isRecursively) { throw Error("upload function doesn't implement recursive function yet!"); } else if (isFolder && !isRecursively) { if (uploadRaw) { throw Error(`you specify uploadRaw, but you also provide a folder key!`); } // folder if (password === "") { // if not encrypted, mkdir a remote folder if (foldersCreatedBefore?.has(uploadFile)) { // created, pass } else { try { await client.dropbox.filesCreateFolderV2({ path: uploadFile, }); foldersCreatedBefore?.add(uploadFile); } catch (err) { if (err.status === 409) { // pass foldersCreatedBefore?.add(uploadFile); } else { throw err; } } } const res = await getRemoteMeta(client, uploadFile); return res; } else { // if encrypted, upload a fake file with the encrypted file name await client.dropbox.filesUpload({ path: uploadFile, contents: "", }); return await getRemoteMeta(client, uploadFile); } } else { // file // we ignore isRecursively parameter here let localContent = undefined; if (uploadRaw) { if (typeof rawContent === "string") { localContent = new TextEncoder().encode(rawContent); } else { localContent = rawContent; } } else { localContent = await vault.adapter.readBinary(fileOrFolderPath); } let remoteContent = localContent; if (password !== "") { remoteContent = await encryptArrayBuffer(localContent, password); } // in dropbox, we don't need to create folders before uploading! cool! // TODO: filesUploadSession for larger files (>=150 MB) await client.dropbox.filesUpload({ path: uploadFile, contents: remoteContent, mode: { ".tag": "overwrite", }, }); // we want to mark that parent folders are created if (foldersCreatedBefore !== undefined) { const dirs = getFolderLevels(uploadFile).map((x) => getDropboxPath(x, client.vaultName) ); for (const dir of dirs) { foldersCreatedBefore?.add(dir); } } return await getRemoteMeta(client, uploadFile); } }; export const listFromRemote = async ( client: WrappedDropboxClient, prefix?: string ) => { if (prefix !== undefined) { throw Error("prefix not supported (yet)"); } await client.init(); let res = await client.dropbox.filesListFolder({ path: `/${client.vaultName}`, recursive: true, include_deleted: false, limit: 1000, }); if (res.status !== 200) { throw Error(JSON.stringify(res)); } // log.info(res); const contents = res.result.entries; const unifiedContents = contents .filter((x) => x[".tag"] !== "deleted") .filter((x) => x.path_display !== `/${client.vaultName}`) .map((x) => fromDropboxItemToRemoteItem(x, client.vaultName)); while (res.result.has_more) { res = await client.dropbox.filesListFolderContinue({ cursor: res.result.cursor, }); if (res.status !== 200) { throw Error(JSON.stringify(res)); } const contents2 = res.result.entries; const unifiedContents2 = contents2 .filter((x) => x[".tag"] !== "deleted") .filter((x) => x.path_display !== `/${client.vaultName}`) .map((x) => fromDropboxItemToRemoteItem(x, client.vaultName)); unifiedContents.push(...unifiedContents2); } fixLastModifiedTimeInplace(unifiedContents); return { Contents: unifiedContents, }; }; const downloadFromRemoteRaw = async ( client: WrappedDropboxClient, fileOrFolderPath: string ) => { await client.init(); const key = getDropboxPath(fileOrFolderPath, client.vaultName); const rsp = await client.dropbox.filesDownload({ path: key, }); if ((rsp.result as any).fileBlob !== undefined) { // we get a Blob const content = (rsp.result as any).fileBlob as Blob; return await content.arrayBuffer(); } else if ((rsp.result as any).fileBinary !== undefined) { // we get a Buffer const content = (rsp.result as any).fileBinary as Buffer; return bufferToArrayBuffer(content); } else { throw Error(`unknown rsp from dropbox download: ${rsp}`); } }; export const downloadFromRemote = async ( client: WrappedDropboxClient, fileOrFolderPath: string, vault: Vault, mtime: number, password: string = "", remoteEncryptedKey: string = "", skipSaving: boolean = false ) => { await client.init(); const isFolder = fileOrFolderPath.endsWith("/"); if (!skipSaving) { await mkdirpInVault(fileOrFolderPath, vault); } // the file is always local file // we need to encrypt it if (isFolder) { // mkdirp locally is enough // do nothing here return new ArrayBuffer(0); } else { let downloadFile = fileOrFolderPath; if (password !== "") { downloadFile = remoteEncryptedKey; } downloadFile = getDropboxPath(downloadFile, client.vaultName); const remoteContent = await downloadFromRemoteRaw(client, downloadFile); let localContent = remoteContent; if (password !== "") { localContent = await decryptArrayBuffer(remoteContent, password); } if (!skipSaving) { await vault.adapter.writeBinary(fileOrFolderPath, localContent, { mtime: mtime, }); } return localContent; } }; export const deleteFromRemote = async ( client: WrappedDropboxClient, fileOrFolderPath: string, password: string = "", remoteEncryptedKey: string = "" ) => { if (fileOrFolderPath === "/") { return; } let remoteFileName = fileOrFolderPath; if (password !== "") { remoteFileName = remoteEncryptedKey; } remoteFileName = getDropboxPath(remoteFileName, client.vaultName); await client.init(); try { await client.dropbox.filesDeleteV2({ path: remoteFileName, }); } catch (err) { console.error("some error while deleting"); console.error(err); } }; export const checkConnectivity = async (client: WrappedDropboxClient) => { try { const results = await getRemoteMeta(client, "/"); if (results === undefined) { return false; } return true; } catch (err) { console.error("dropbox connectivity error:"); console.error(err); return false; } }; export const getUserDisplayName = async (client: WrappedDropboxClient) => { await client.init(); const acct = await client.dropbox.usersGetCurrentAccount(); return acct.result.name.display_name; }; export const revokeAuth = async (client: WrappedDropboxClient) => { await client.init(); await client.dropbox.authTokenRevoke(); };