bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

basically working dropbox

Browse Source
This commit is contained in:
fyears 2021-11-28 12:20:38 +08:00
parent 2eb1545f9d
commit 00c840a758
7 changed files with 539 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
package.json
View File

@ -12,7 +12,10 @@
"browser": {
"path": "path-browserify",
"process": "process/browser",
"stream": "stream-browserify"
"stream": "stream-browserify",
"crypto": "crypto-browserify",
"util": "util/",
"assert": "assert/"
},
"source": "main.ts",
"keywords": [],
@ -42,9 +45,12 @@
"@aws-sdk/lib-storage": "^3.40.1",
"@aws-sdk/signature-v4-crt": "^3.37.0",
"acorn": "^8.5.0",
"assert": "^2.0.0",
"aws-crt": "^1.10.1",
"buffer": "^6.0.3",
"codemirror": "^5.63.1",
"crypto-browserify": "^3.12.0",
"dropbox": "^10.22.0",
"localforage": "^1.10.0",
"mime-types": "^2.1.33",
"obsidian": "^0.12.0",
@ -53,6 +59,7 @@
"rfc4648": "^1.5.0",
"rimraf": "^3.0.2",
"stream-browserify": "^3.0.0",
"util": "^0.12.4",
"webdav": "^4.7.0",
"webdav-fs": "^4.0.0",
"xregexp": "^5.1.0"

2
src/baseTypes.ts
View File

@ -2,7 +2,7 @@
* Only type defs here.
*/
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav";
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "dropbox";
export interface RemoteItem {
key: string;

68
src/main.ts
View File

@ -25,8 +25,11 @@ import type { InternalDBs } from "./localdb";
import type { SyncStatusType, PasswordCheckType } from "./sync";
import { isPasswordOk, getSyncPlan, doActualSync } from "./sync";
import { S3Config, DEFAULT_S3_CONFIG } from "./s3";
import { WebdavConfig, DEFAULT_WEBDAV_CONFIG, WebdavAuthType } from "./webdav";
import { DropboxConfig, DEFAULT_DROPBOX_CONFIG } from "./remoteForDropbox";
import { RemoteClient } from "./remote";
import { exportSyncPlansToFiles } from "./debugMode";
import { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
@ -34,6 +37,7 @@ import { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
interface RemotelySavePluginSettings {
s3: S3Config;
webdav: WebdavConfig;
dropbox: DropboxConfig;
password: string;
serviceType: SUPPORTED_SERVICES_TYPE;
enableExperimentService: boolean;
@ -42,6 +46,7 @@ interface RemotelySavePluginSettings {
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
s3: DEFAULT_S3_CONFIG,
webdav: DEFAULT_WEBDAV_CONFIG,
dropbox: DEFAULT_DROPBOX_CONFIG,
password: "",
serviceType: "s3",
enableExperimentService: false,
@ -94,15 +99,16 @@ export default class RemotelySavePlugin extends Plugin {
const client = new RemoteClient(
this.settings.serviceType,
this.settings.s3,
this.settings.webdav
this.settings.webdav,
this.settings.dropbox
);
const remoteRsp = await client.listFromRemote();
// console.log(remoteRsp);
new Notice("3/6 Starting to fetch local meta data.");
this.syncStatus = "getting_local_meta";
const local = this.app.vault.getAllLoadedFiles();
const localHistory = await loadDeleteRenameHistoryTable(this.db);
// console.log(remoteRsp);
// console.log(local);
// console.log(localHistory);
@ -453,6 +459,59 @@ class RemotelySaveSettingTab extends PluginSettingTab {
});
});
const dropboxDiv = containerEl.createEl("div", { cls: "dropbox-hide" });
dropboxDiv.toggleClass(
"dropbox-hide",
this.plugin.settings.serviceType !== "dropbox"
);
dropboxDiv.createEl("h2", { text: "for Dropbox" });
dropboxDiv.createEl("p", {
text: "Disclaimer: Sync support for Dropbox are more experimental, and s3 functions are more stable now.",
cls: "dropbox-disclaimer",
});
dropboxDiv.createEl("p", {
text: "Disclaimer: This app is NOT an official Dropbox product. It just uses Dropbox open api.",
cls: "dropbox-disclaimer",
});
dropboxDiv.createEl("p", {
text: "We create a folder App/obsidian-remotely-save on your Dropbox. All files/folders sync would happen inside this folder.",
});
new Setting(dropboxDiv)
.setName("access token")
.setDesc("access token")
.addText((text) =>
text
.setPlaceholder("")
.setValue(this.plugin.settings.dropbox.accessToken)
.onChange(async (value) => {
this.plugin.settings.dropbox.accessToken = value.trim();
await this.plugin.saveSettings();
})
);
new Setting(dropboxDiv)
.setName("check connectivity")
.setDesc("check connectivity")
.addButton(async (button) => {
button.setButtonText("Check");
button.onClick(async () => {
new Notice("Checking...");
const client = new RemoteClient(
"dropbox",
undefined,
undefined,
this.plugin.settings.dropbox
);
const res = await client.checkConnectivity();
if (res) {
new Notice("Great! We can connect to Dropbox!");
} else {
new Notice("We cannot connect to Dropbox.");
}
});
});
const webdavDiv = containerEl.createEl("div", { cls: "webdav-hide" });
webdavDiv.toggleClass(
"webdav-hide",
@ -563,6 +622,7 @@ class RemotelySaveSettingTab extends PluginSettingTab {
this.plugin.settings.enableExperimentService;
dropdown.addOption("s3", "s3 (-compatible)");
dropdown.addOption("dropbox", "Dropbox");
if (currService === "webdav" || enableExperimentService) {
dropdown.addOption("webdav", "webdav (experimental)");
if (!enableExperimentService) {
@ -578,6 +638,10 @@ class RemotelySaveSettingTab extends PluginSettingTab {
"s3-hide",
this.plugin.settings.serviceType !== "s3"
);
dropboxDiv.toggleClass(
"dropbox-hide",
this.plugin.settings.serviceType !== "dropbox"
);
webdavDiv.toggleClass(
"webdav-hide",
this.plugin.settings.serviceType !== "webdav"

52
src/remote.ts
View File

@ -3,6 +3,7 @@ import { Vault } from "obsidian";
import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import * as s3 from "./s3";
import * as webdav from "./webdav";
import * as dropbox from "./remoteForDropbox";
export class RemoteClient {
readonly serviceType: SUPPORTED_SERVICES_TYPE;
@ -10,19 +11,25 @@ export class RemoteClient {
readonly s3Config?: s3.S3Config;
readonly webdavClient?: webdav.WebDAVClient;
readonly webdavConfig?: webdav.WebdavConfig;
readonly dropboxClient?: dropbox.Dropbox;
readonly dropboxConfig?: dropbox.DropboxConfig;
constructor(
serviceType: SUPPORTED_SERVICES_TYPE,
s3Config?: s3.S3Config,
webdavConfig?: webdav.WebdavConfig
webdavConfig?: webdav.WebdavConfig,
dropboxConfig?: dropbox.DropboxConfig
) {
this.serviceType = serviceType;
if (serviceType === "s3") {
this.s3Config = s3Config;
this.s3Client = s3.getS3Client(s3Config);
this.s3Config = { ...s3Config };
this.s3Client = s3.getS3Client(this.s3Config);
} else if (serviceType === "webdav") {
this.webdavConfig = webdavConfig;
this.webdavClient = webdav.getWebdavClient(webdavConfig);
this.webdavConfig = { ...webdavConfig };
this.webdavClient = webdav.getWebdavClient(this.webdavConfig);
} else if (serviceType === "dropbox") {
this.dropboxConfig = { ...dropboxConfig };
this.dropboxClient = dropbox.getDropboxClient(this.dropboxConfig);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
@ -37,6 +44,8 @@ export class RemoteClient {
);
} else if (this.serviceType === "webdav") {
return await webdav.getRemoteMeta(this.webdavClient, fileOrFolderPath);
} else if (this.serviceType === "dropbox") {
return await dropbox.getRemoteMeta(this.dropboxClient, fileOrFolderPath);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
@ -47,7 +56,8 @@ export class RemoteClient {
vault: Vault,
isRecursively: boolean = false,
password: string = "",
remoteEncryptedKey: string = ""
remoteEncryptedKey: string = "",
foldersCreatedBefore: Set<string> | undefined = undefined
) => {
if (this.serviceType === "s3") {
return await s3.uploadToRemote(
@ -68,6 +78,16 @@ export class RemoteClient {
password,
remoteEncryptedKey
);
} else if (this.serviceType === "dropbox") {
return await dropbox.uploadToRemote(
this.dropboxClient,
fileOrFolderPath,
vault,
isRecursively,
password,
remoteEncryptedKey,
foldersCreatedBefore
);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
@ -78,6 +98,8 @@ export class RemoteClient {
return await s3.listFromRemote(this.s3Client, this.s3Config, prefix);
} else if (this.serviceType === "webdav") {
return await webdav.listFromRemote(this.webdavClient, prefix);
} else if (this.serviceType === "dropbox") {
return await dropbox.listFromRemote(this.dropboxClient, prefix);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
@ -109,6 +131,15 @@ export class RemoteClient {
password,
remoteEncryptedKey
);
} else if (this.serviceType === "dropbox") {
return await dropbox.downloadFromRemote(
this.dropboxClient,
fileOrFolderPath,
vault,
mtime,
password,
remoteEncryptedKey
);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
@ -134,6 +165,13 @@ export class RemoteClient {
password,
remoteEncryptedKey
);
} else if (this.serviceType === "dropbox") {
return await dropbox.deleteFromRemote(
this.dropboxClient,
fileOrFolderPath,
password,
remoteEncryptedKey
);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
@ -144,6 +182,8 @@ export class RemoteClient {
return await s3.checkConnectivity(this.s3Client, this.s3Config);
} else if (this.serviceType === "webdav") {
return await webdav.checkConnectivity(this.webdavClient);
} else if (this.serviceType === "dropbox") {
return await dropbox.checkConnectivity(this.dropboxClient);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}

372
src/remoteForDropbox.ts Normal file
View File

@ -0,0 +1,372 @@
import * as path from "path";
import { FileStats, Vault } from "obsidian";
import { Dropbox, DropboxResponse, files } from "dropbox";
export { Dropbox } from "dropbox";
import { RemoteItem } from "./baseTypes";
import {
arrayBufferToBuffer,
bufferToArrayBuffer,
mkdirpInVault,
getPathFolder,
getFolderLevels,
setToString,
} from "./misc";
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
import { strict as assert } from "assert";
export interface DropboxConfig {
accessToken: string;
}
export const DEFAULT_DROPBOX_CONFIG = {
accessToken: "",
};
export const getDropboxPath = (fileOrFolderPath: string) => {
let key = fileOrFolderPath;
if (!fileOrFolderPath.startsWith("/")) {
key = `/${fileOrFolderPath}`;
}
if (key.endsWith("/")) {
key = key.slice(0, key.length - 1);
}
return key;
};
const getNormPath = (fileOrFolderPath: string) => {
if (fileOrFolderPath.startsWith("/")) {
return fileOrFolderPath.slice(1);
}
return fileOrFolderPath;
};
const fromDropboxItemToRemoteItem = (
x:
| files.FileMetadataReference
| files.FolderMetadataReference
| files.DeletedMetadataReference
): RemoteItem => {
let key = getNormPath(x.path_display);
if (x[".tag"] === "folder" && !key.endsWith("/")) {
key = `${key}/`;
}
if (x[".tag"] === "folder") {
return {
key: key,
lastModified: undefined,
size: 0,
remoteType: "dropbox",
etag: `${x.id}\t`,
} as RemoteItem;
} else if (x[".tag"] === "file") {
return {
key: key,
lastModified: Date.parse(x.server_modified).valueOf(),
size: x.size,
remoteType: "dropbox",
etag: `${x.id}\t${x.content_hash}`,
} as RemoteItem;
} else if (x[".tag"] === "deleted") {
throw Error("do not support deleted tag");
}
};
/**
* Dropbox api doesn't return mtime for folders.
* This is a try to assign mtime by using files in folder.
* @param allFilesFolders
* @returns
*/
const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => {
if (allFilesFolders.length === 0) {
return;
}
// sort by longer to shorter
allFilesFolders.sort((a, b) => b.key.length - a.key.length);
// a "map" from dir to mtime
let potentialMTime = {} as Record<string, number>;
// first sort pass, from buttom to up
for (const item of allFilesFolders) {
if (item.key.endsWith("/")) {
// itself is a folder, and initially doesn't have mtime
if (item.lastModified === undefined && item.key in potentialMTime) {
// previously we gathered all sub info of this folder
item.lastModified = potentialMTime[item.key];
}
}
const parent = `${path.posix.dirname(item.key)}/`;
if (item.lastModified !== undefined) {
if (parent in potentialMTime) {
potentialMTime[parent] = Math.max(
potentialMTime[parent],
item.lastModified
);
} else {
potentialMTime[parent] = item.lastModified;
}
}
}
// second pass, from up to buttom.
// fill mtime by parent folder or Date.Now() if still not available.
// this is only possible if no any sub-folder-files recursively.
// we do not sort the array again, just iterate over it by reverse
// using good old for loop.
for (let i = allFilesFolders.length - 1; i >= 0; --i) {
const item = allFilesFolders[i];
if (!item.key.endsWith("/")) {
continue; // skip files
}
if (item.lastModified !== undefined) {
continue; // don't need to deal with it
}
assert(!(item.key in potentialMTime));
const parent = `${path.posix.dirname(item.key)}/`;
if (parent in potentialMTime) {
item.lastModified = potentialMTime[parent];
} else {
item.lastModified = Date.now().valueOf();
potentialMTime[item.key] = item.lastModified;
}
}
return allFilesFolders;
};
export const getDropboxClient = (
accessTokenOrDropboxConfig: string | DropboxConfig
) => {
if (typeof accessTokenOrDropboxConfig === "string") {
return new Dropbox({ accessToken: accessTokenOrDropboxConfig });
}
return new Dropbox({
accessToken: accessTokenOrDropboxConfig.accessToken,
});
};
export const getRemoteMeta = async (dbx: Dropbox, fileOrFolderPath: string) => {
if (fileOrFolderPath === "" || fileOrFolderPath === "/") {
// filesGetMetadata doesn't support root folder
// we instead try to list files
// if no error occurs, we ensemble a fake result.
const rsp = await dbx.filesListFolder({
path: "",
recursive: false, // don't need to recursive here
});
if (rsp.status !== 200) {
throw Error(JSON.stringify(rsp));
}
return {
key: fileOrFolderPath,
lastModified: undefined,
size: 0,
remoteType: "dropbox",
etag: undefined,
} as RemoteItem;
}
const key = getDropboxPath(fileOrFolderPath);
const rsp = await dbx.filesGetMetadata({
path: key,
});
if (rsp.status !== 200) {
throw Error(JSON.stringify(rsp));
}
return fromDropboxItemToRemoteItem(rsp.result);
};
export const uploadToRemote = async (
dbx: Dropbox,
fileOrFolderPath: string,
vault: Vault,
isRecursively: boolean = false,
password: string = "",
remoteEncryptedKey: string = "",
foldersCreatedBefore: Set<string> | undefined = undefined
) => {
let uploadFile = fileOrFolderPath;
if (password !== "") {
uploadFile = remoteEncryptedKey;
}
uploadFile = getDropboxPath(uploadFile);
const isFolder = fileOrFolderPath.endsWith("/");
if (isFolder && isRecursively) {
throw Error("upload function doesn't implement recursive function yet!");
} else if (isFolder && !isRecursively) {
// folder
if (password === "") {
// if not encrypted, mkdir a remote folder
if (foldersCreatedBefore?.has(uploadFile)) {
// created, pass
} else {
try {
await dbx.filesCreateFolderV2({
path: uploadFile,
});
foldersCreatedBefore?.add(uploadFile);
} catch (err) {
if (err.status === 409) {
// pass
foldersCreatedBefore?.add(uploadFile);
} else {
throw err;
}
}
}
const res = await getRemoteMeta(dbx, uploadFile);
return res;
} else {
// if encrypted, upload a fake file with the encrypted file name
await dbx.filesUpload({
path: uploadFile,
contents: "",
});
return await getRemoteMeta(dbx, uploadFile);
}
} else {
// file
// we ignore isRecursively parameter here
const localContent = await vault.adapter.readBinary(fileOrFolderPath);
let remoteContent = localContent;
if (password !== "") {
remoteContent = await encryptArrayBuffer(localContent, password);
}
// in dropbox, we don't need to create folders before uploading! cool!
// TODO: filesUploadSession for larger files (>=150 MB)
await dbx.filesUpload({
path: uploadFile,
contents: remoteContent,
mode: {
".tag": "overwrite",
},
});
// we want to mark that parent folders are created
if (foldersCreatedBefore !== undefined) {
const dirs = getFolderLevels(uploadFile).map(getDropboxPath);
for (const dir of dirs) {
foldersCreatedBefore?.add(dir);
}
}
return await getRemoteMeta(dbx, uploadFile);
}
};
export const listFromRemote = async (dbx: Dropbox, prefix?: string) => {
if (prefix !== undefined) {
throw Error("prefix not supported (yet)");
}
const res = await dbx.filesListFolder({ path: "", recursive: true });
if (res.status !== 200) {
throw Error(JSON.stringify(res));
}
// console.log(res);
const contents = res.result.entries;
const unifiedContents = contents
.filter((x) => x[".tag"] !== "deleted")
.map(fromDropboxItemToRemoteItem);
fixLastModifiedTimeInplace(unifiedContents);
return {
Contents: unifiedContents,
};
};
const downloadFromRemoteRaw = async (
dbx: Dropbox,
fileOrFolderPath: string
) => {
const key = getDropboxPath(fileOrFolderPath);
const rsp = await dbx.filesDownload({
path: key,
});
if ((rsp.result as any).fileBlob !== undefined) {
// we get a Blob
const content = (rsp.result as any).fileBlob as Blob;
return await content.arrayBuffer();
} else if ((rsp.result as any).fileBinary !== undefined) {
// we get a Buffer
const content = (rsp.result as any).fileBinary as Buffer;
return bufferToArrayBuffer(content);
} else {
throw Error(`unknown rsp from dropbox download: ${rsp}`);
}
};
export const downloadFromRemote = async (
dbx: Dropbox,
fileOrFolderPath: string,
vault: Vault,
mtime: number,
password: string = "",
remoteEncryptedKey: string = ""
) => {
const isFolder = fileOrFolderPath.endsWith("/");
await mkdirpInVault(fileOrFolderPath, vault);
// the file is always local file
// we need to encrypt it
if (isFolder) {
// mkdirp locally is enough
// do nothing here
} else {
let downloadFile = fileOrFolderPath;
if (password !== "") {
downloadFile = remoteEncryptedKey;
}
downloadFile = getDropboxPath(downloadFile);
const remoteContent = await downloadFromRemoteRaw(dbx, downloadFile);
let localContent = remoteContent;
if (password !== "") {
localContent = await decryptArrayBuffer(remoteContent, password);
}
await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
mtime: mtime,
});
}
};
export const deleteFromRemote = async (
dbx: Dropbox,
fileOrFolderPath: string,
password: string = "",
remoteEncryptedKey: string = ""
) => {
if (fileOrFolderPath === "/") {
return;
}
let remoteFileName = fileOrFolderPath;
if (password !== "") {
remoteFileName = remoteEncryptedKey;
}
remoteFileName = getDropboxPath(remoteFileName);
try {
await dbx.filesDeleteV2({
path: remoteFileName,
});
} catch (err) {
console.error("some error while deleting");
console.log(err);
}
};
export const checkConnectivity = async (dbx: Dropbox) => {
try {
const results = await getRemoteMeta(dbx, "/");
if (results === undefined) {
return false;
}
return true;
} catch (err) {
return false;
}
};

46
src/sync.ts
View File

@ -9,7 +9,7 @@ import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
import { RemoteClient } from "./remote";
import type { SUPPORTED_SERVICES_TYPE, RemoteItem } from "./baseTypes";
import { mkdirpInVault, isHiddenPath, isVaildText } from "./misc";
import { mkdirpInVault, isHiddenPath, isVaildText, setToString } from "./misc";
import {
decryptBase32ToString,
encryptStringToBase32,
@ -389,7 +389,7 @@ const getOperation = (
}
if (r.decision === "unknown") {
throw Error(`unknown decision for ${r}`);
throw Error(`unknown decision for ${JSON.stringify(r)}`);
}
return r;
@ -428,7 +428,8 @@ const dispatchOperationToActual = async (
client: RemoteClient,
db: InternalDBs,
vault: Vault,
password: string = ""
password: string = "",
foldersCreatedBefore: Set<string> | undefined = undefined
) => {
let remoteEncryptedKey = key;
if (password !== "") {
@ -461,7 +462,8 @@ const dispatchOperationToActual = async (
vault,
false,
password,
remoteEncryptedKey
remoteEncryptedKey,
foldersCreatedBefore
);
await upsertSyncMetaMappingData(
client.serviceType,
@ -493,7 +495,8 @@ const dispatchOperationToActual = async (
vault,
false,
password,
remoteEncryptedKey
remoteEncryptedKey,
foldersCreatedBefore
);
await upsertSyncMetaMappingData(
client.serviceType,
@ -521,18 +524,35 @@ export const doActualSync = async (
password: string = ""
) => {
const keyStates = syncPlan.mixedStates;
await Promise.all(
Object.entries(keyStates)
.sort((k, v) => -(k as string).length)
.map(async ([k, v]) =>
dispatchOperationToActual(
const foldersCreatedBefore = new Set<string>();
for (const [k, v] of Object.entries(keyStates).sort(
([k1, v1], [k2, v2]) => k2.length - k1.length
)) {
const k2 = k as string;
const v2 = v as FileOrFolderMixedState;
await dispatchOperationToActual(
k as string,
v as FileOrFolderMixedState,
client,
db,
vault,
password
)
)
password,
foldersCreatedBefore
);
// console.log(`finished ${k}, with ${setToString(foldersCreatedBefore)}`);
}
// await Promise.all(
// Object.entries(keyStates)
// .map(async ([k, v]) =>
// dispatchOperationToActual(
// k as string,
// v as FileOrFolderMixedState,
// client,
// db,
// vault,
// password,
// foldersCreatedBefore
// )
// )
// );
};

8
styles.css
View File

@ -11,10 +11,16 @@
display: none;
}
.dropbox-disclaimer {
font-weight: bold;
}
.dropbox-hide {
display: none;
}
.webdav-disclaimer {
font-weight: bold;
}
.webdav-hide {
display: none;
}