basically working dropbox
This commit is contained in:
parent
2eb1545f9d
commit
00c840a758
@ -12,7 +12,10 @@
|
|||||||
"browser": {
|
"browser": {
|
||||||
"path": "path-browserify",
|
"path": "path-browserify",
|
||||||
"process": "process/browser",
|
"process": "process/browser",
|
||||||
"stream": "stream-browserify"
|
"stream": "stream-browserify",
|
||||||
|
"crypto": "crypto-browserify",
|
||||||
|
"util": "util/",
|
||||||
|
"assert": "assert/"
|
||||||
},
|
},
|
||||||
"source": "main.ts",
|
"source": "main.ts",
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
@ -42,9 +45,12 @@
|
|||||||
"@aws-sdk/lib-storage": "^3.40.1",
|
"@aws-sdk/lib-storage": "^3.40.1",
|
||||||
"@aws-sdk/signature-v4-crt": "^3.37.0",
|
"@aws-sdk/signature-v4-crt": "^3.37.0",
|
||||||
"acorn": "^8.5.0",
|
"acorn": "^8.5.0",
|
||||||
|
"assert": "^2.0.0",
|
||||||
"aws-crt": "^1.10.1",
|
"aws-crt": "^1.10.1",
|
||||||
"buffer": "^6.0.3",
|
"buffer": "^6.0.3",
|
||||||
"codemirror": "^5.63.1",
|
"codemirror": "^5.63.1",
|
||||||
|
"crypto-browserify": "^3.12.0",
|
||||||
|
"dropbox": "^10.22.0",
|
||||||
"localforage": "^1.10.0",
|
"localforage": "^1.10.0",
|
||||||
"mime-types": "^2.1.33",
|
"mime-types": "^2.1.33",
|
||||||
"obsidian": "^0.12.0",
|
"obsidian": "^0.12.0",
|
||||||
@ -53,6 +59,7 @@
|
|||||||
"rfc4648": "^1.5.0",
|
"rfc4648": "^1.5.0",
|
||||||
"rimraf": "^3.0.2",
|
"rimraf": "^3.0.2",
|
||||||
"stream-browserify": "^3.0.0",
|
"stream-browserify": "^3.0.0",
|
||||||
|
"util": "^0.12.4",
|
||||||
"webdav": "^4.7.0",
|
"webdav": "^4.7.0",
|
||||||
"webdav-fs": "^4.0.0",
|
"webdav-fs": "^4.0.0",
|
||||||
"xregexp": "^5.1.0"
|
"xregexp": "^5.1.0"
|
||||||
|
|||||||
@ -2,7 +2,7 @@
|
|||||||
* Only type defs here.
|
* Only type defs here.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav";
|
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "dropbox";
|
||||||
|
|
||||||
export interface RemoteItem {
|
export interface RemoteItem {
|
||||||
key: string;
|
key: string;
|
||||||
|
|||||||
68
src/main.ts
68
src/main.ts
@ -25,8 +25,11 @@ import type { InternalDBs } from "./localdb";
|
|||||||
|
|
||||||
import type { SyncStatusType, PasswordCheckType } from "./sync";
|
import type { SyncStatusType, PasswordCheckType } from "./sync";
|
||||||
import { isPasswordOk, getSyncPlan, doActualSync } from "./sync";
|
import { isPasswordOk, getSyncPlan, doActualSync } from "./sync";
|
||||||
|
|
||||||
import { S3Config, DEFAULT_S3_CONFIG } from "./s3";
|
import { S3Config, DEFAULT_S3_CONFIG } from "./s3";
|
||||||
import { WebdavConfig, DEFAULT_WEBDAV_CONFIG, WebdavAuthType } from "./webdav";
|
import { WebdavConfig, DEFAULT_WEBDAV_CONFIG, WebdavAuthType } from "./webdav";
|
||||||
|
import { DropboxConfig, DEFAULT_DROPBOX_CONFIG } from "./remoteForDropbox";
|
||||||
|
|
||||||
import { RemoteClient } from "./remote";
|
import { RemoteClient } from "./remote";
|
||||||
import { exportSyncPlansToFiles } from "./debugMode";
|
import { exportSyncPlansToFiles } from "./debugMode";
|
||||||
import { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
import { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
||||||
@ -34,6 +37,7 @@ import { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
|||||||
interface RemotelySavePluginSettings {
|
interface RemotelySavePluginSettings {
|
||||||
s3: S3Config;
|
s3: S3Config;
|
||||||
webdav: WebdavConfig;
|
webdav: WebdavConfig;
|
||||||
|
dropbox: DropboxConfig;
|
||||||
password: string;
|
password: string;
|
||||||
serviceType: SUPPORTED_SERVICES_TYPE;
|
serviceType: SUPPORTED_SERVICES_TYPE;
|
||||||
enableExperimentService: boolean;
|
enableExperimentService: boolean;
|
||||||
@ -42,6 +46,7 @@ interface RemotelySavePluginSettings {
|
|||||||
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
||||||
s3: DEFAULT_S3_CONFIG,
|
s3: DEFAULT_S3_CONFIG,
|
||||||
webdav: DEFAULT_WEBDAV_CONFIG,
|
webdav: DEFAULT_WEBDAV_CONFIG,
|
||||||
|
dropbox: DEFAULT_DROPBOX_CONFIG,
|
||||||
password: "",
|
password: "",
|
||||||
serviceType: "s3",
|
serviceType: "s3",
|
||||||
enableExperimentService: false,
|
enableExperimentService: false,
|
||||||
@ -94,15 +99,16 @@ export default class RemotelySavePlugin extends Plugin {
|
|||||||
const client = new RemoteClient(
|
const client = new RemoteClient(
|
||||||
this.settings.serviceType,
|
this.settings.serviceType,
|
||||||
this.settings.s3,
|
this.settings.s3,
|
||||||
this.settings.webdav
|
this.settings.webdav,
|
||||||
|
this.settings.dropbox
|
||||||
);
|
);
|
||||||
const remoteRsp = await client.listFromRemote();
|
const remoteRsp = await client.listFromRemote();
|
||||||
|
// console.log(remoteRsp);
|
||||||
|
|
||||||
new Notice("3/6 Starting to fetch local meta data.");
|
new Notice("3/6 Starting to fetch local meta data.");
|
||||||
this.syncStatus = "getting_local_meta";
|
this.syncStatus = "getting_local_meta";
|
||||||
const local = this.app.vault.getAllLoadedFiles();
|
const local = this.app.vault.getAllLoadedFiles();
|
||||||
const localHistory = await loadDeleteRenameHistoryTable(this.db);
|
const localHistory = await loadDeleteRenameHistoryTable(this.db);
|
||||||
// console.log(remoteRsp);
|
|
||||||
// console.log(local);
|
// console.log(local);
|
||||||
// console.log(localHistory);
|
// console.log(localHistory);
|
||||||
|
|
||||||
@ -453,6 +459,59 @@ class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const dropboxDiv = containerEl.createEl("div", { cls: "dropbox-hide" });
|
||||||
|
dropboxDiv.toggleClass(
|
||||||
|
"dropbox-hide",
|
||||||
|
this.plugin.settings.serviceType !== "dropbox"
|
||||||
|
);
|
||||||
|
dropboxDiv.createEl("h2", { text: "for Dropbox" });
|
||||||
|
dropboxDiv.createEl("p", {
|
||||||
|
text: "Disclaimer: Sync support for Dropbox are more experimental, and s3 functions are more stable now.",
|
||||||
|
cls: "dropbox-disclaimer",
|
||||||
|
});
|
||||||
|
dropboxDiv.createEl("p", {
|
||||||
|
text: "Disclaimer: This app is NOT an official Dropbox product. It just uses Dropbox open api.",
|
||||||
|
cls: "dropbox-disclaimer",
|
||||||
|
});
|
||||||
|
dropboxDiv.createEl("p", {
|
||||||
|
text: "We create a folder App/obsidian-remotely-save on your Dropbox. All files/folders sync would happen inside this folder.",
|
||||||
|
});
|
||||||
|
|
||||||
|
new Setting(dropboxDiv)
|
||||||
|
.setName("access token")
|
||||||
|
.setDesc("access token")
|
||||||
|
.addText((text) =>
|
||||||
|
text
|
||||||
|
.setPlaceholder("")
|
||||||
|
.setValue(this.plugin.settings.dropbox.accessToken)
|
||||||
|
.onChange(async (value) => {
|
||||||
|
this.plugin.settings.dropbox.accessToken = value.trim();
|
||||||
|
await this.plugin.saveSettings();
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
new Setting(dropboxDiv)
|
||||||
|
.setName("check connectivity")
|
||||||
|
.setDesc("check connectivity")
|
||||||
|
.addButton(async (button) => {
|
||||||
|
button.setButtonText("Check");
|
||||||
|
button.onClick(async () => {
|
||||||
|
new Notice("Checking...");
|
||||||
|
const client = new RemoteClient(
|
||||||
|
"dropbox",
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
this.plugin.settings.dropbox
|
||||||
|
);
|
||||||
|
const res = await client.checkConnectivity();
|
||||||
|
if (res) {
|
||||||
|
new Notice("Great! We can connect to Dropbox!");
|
||||||
|
} else {
|
||||||
|
new Notice("We cannot connect to Dropbox.");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
const webdavDiv = containerEl.createEl("div", { cls: "webdav-hide" });
|
const webdavDiv = containerEl.createEl("div", { cls: "webdav-hide" });
|
||||||
webdavDiv.toggleClass(
|
webdavDiv.toggleClass(
|
||||||
"webdav-hide",
|
"webdav-hide",
|
||||||
@ -563,6 +622,7 @@ class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
this.plugin.settings.enableExperimentService;
|
this.plugin.settings.enableExperimentService;
|
||||||
|
|
||||||
dropdown.addOption("s3", "s3 (-compatible)");
|
dropdown.addOption("s3", "s3 (-compatible)");
|
||||||
|
dropdown.addOption("dropbox", "Dropbox");
|
||||||
if (currService === "webdav" || enableExperimentService) {
|
if (currService === "webdav" || enableExperimentService) {
|
||||||
dropdown.addOption("webdav", "webdav (experimental)");
|
dropdown.addOption("webdav", "webdav (experimental)");
|
||||||
if (!enableExperimentService) {
|
if (!enableExperimentService) {
|
||||||
@ -578,6 +638,10 @@ class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
"s3-hide",
|
"s3-hide",
|
||||||
this.plugin.settings.serviceType !== "s3"
|
this.plugin.settings.serviceType !== "s3"
|
||||||
);
|
);
|
||||||
|
dropboxDiv.toggleClass(
|
||||||
|
"dropbox-hide",
|
||||||
|
this.plugin.settings.serviceType !== "dropbox"
|
||||||
|
);
|
||||||
webdavDiv.toggleClass(
|
webdavDiv.toggleClass(
|
||||||
"webdav-hide",
|
"webdav-hide",
|
||||||
this.plugin.settings.serviceType !== "webdav"
|
this.plugin.settings.serviceType !== "webdav"
|
||||||
|
|||||||
@ -3,6 +3,7 @@ import { Vault } from "obsidian";
|
|||||||
import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
||||||
import * as s3 from "./s3";
|
import * as s3 from "./s3";
|
||||||
import * as webdav from "./webdav";
|
import * as webdav from "./webdav";
|
||||||
|
import * as dropbox from "./remoteForDropbox";
|
||||||
|
|
||||||
export class RemoteClient {
|
export class RemoteClient {
|
||||||
readonly serviceType: SUPPORTED_SERVICES_TYPE;
|
readonly serviceType: SUPPORTED_SERVICES_TYPE;
|
||||||
@ -10,19 +11,25 @@ export class RemoteClient {
|
|||||||
readonly s3Config?: s3.S3Config;
|
readonly s3Config?: s3.S3Config;
|
||||||
readonly webdavClient?: webdav.WebDAVClient;
|
readonly webdavClient?: webdav.WebDAVClient;
|
||||||
readonly webdavConfig?: webdav.WebdavConfig;
|
readonly webdavConfig?: webdav.WebdavConfig;
|
||||||
|
readonly dropboxClient?: dropbox.Dropbox;
|
||||||
|
readonly dropboxConfig?: dropbox.DropboxConfig;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
serviceType: SUPPORTED_SERVICES_TYPE,
|
serviceType: SUPPORTED_SERVICES_TYPE,
|
||||||
s3Config?: s3.S3Config,
|
s3Config?: s3.S3Config,
|
||||||
webdavConfig?: webdav.WebdavConfig
|
webdavConfig?: webdav.WebdavConfig,
|
||||||
|
dropboxConfig?: dropbox.DropboxConfig
|
||||||
) {
|
) {
|
||||||
this.serviceType = serviceType;
|
this.serviceType = serviceType;
|
||||||
if (serviceType === "s3") {
|
if (serviceType === "s3") {
|
||||||
this.s3Config = s3Config;
|
this.s3Config = { ...s3Config };
|
||||||
this.s3Client = s3.getS3Client(s3Config);
|
this.s3Client = s3.getS3Client(this.s3Config);
|
||||||
} else if (serviceType === "webdav") {
|
} else if (serviceType === "webdav") {
|
||||||
this.webdavConfig = webdavConfig;
|
this.webdavConfig = { ...webdavConfig };
|
||||||
this.webdavClient = webdav.getWebdavClient(webdavConfig);
|
this.webdavClient = webdav.getWebdavClient(this.webdavConfig);
|
||||||
|
} else if (serviceType === "dropbox") {
|
||||||
|
this.dropboxConfig = { ...dropboxConfig };
|
||||||
|
this.dropboxClient = dropbox.getDropboxClient(this.dropboxConfig);
|
||||||
} else {
|
} else {
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
throw Error(`not supported service type ${this.serviceType}`);
|
||||||
}
|
}
|
||||||
@ -37,6 +44,8 @@ export class RemoteClient {
|
|||||||
);
|
);
|
||||||
} else if (this.serviceType === "webdav") {
|
} else if (this.serviceType === "webdav") {
|
||||||
return await webdav.getRemoteMeta(this.webdavClient, fileOrFolderPath);
|
return await webdav.getRemoteMeta(this.webdavClient, fileOrFolderPath);
|
||||||
|
} else if (this.serviceType === "dropbox") {
|
||||||
|
return await dropbox.getRemoteMeta(this.dropboxClient, fileOrFolderPath);
|
||||||
} else {
|
} else {
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
throw Error(`not supported service type ${this.serviceType}`);
|
||||||
}
|
}
|
||||||
@ -47,7 +56,8 @@ export class RemoteClient {
|
|||||||
vault: Vault,
|
vault: Vault,
|
||||||
isRecursively: boolean = false,
|
isRecursively: boolean = false,
|
||||||
password: string = "",
|
password: string = "",
|
||||||
remoteEncryptedKey: string = ""
|
remoteEncryptedKey: string = "",
|
||||||
|
foldersCreatedBefore: Set<string> | undefined = undefined
|
||||||
) => {
|
) => {
|
||||||
if (this.serviceType === "s3") {
|
if (this.serviceType === "s3") {
|
||||||
return await s3.uploadToRemote(
|
return await s3.uploadToRemote(
|
||||||
@ -68,6 +78,16 @@ export class RemoteClient {
|
|||||||
password,
|
password,
|
||||||
remoteEncryptedKey
|
remoteEncryptedKey
|
||||||
);
|
);
|
||||||
|
} else if (this.serviceType === "dropbox") {
|
||||||
|
return await dropbox.uploadToRemote(
|
||||||
|
this.dropboxClient,
|
||||||
|
fileOrFolderPath,
|
||||||
|
vault,
|
||||||
|
isRecursively,
|
||||||
|
password,
|
||||||
|
remoteEncryptedKey,
|
||||||
|
foldersCreatedBefore
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
throw Error(`not supported service type ${this.serviceType}`);
|
||||||
}
|
}
|
||||||
@ -78,6 +98,8 @@ export class RemoteClient {
|
|||||||
return await s3.listFromRemote(this.s3Client, this.s3Config, prefix);
|
return await s3.listFromRemote(this.s3Client, this.s3Config, prefix);
|
||||||
} else if (this.serviceType === "webdav") {
|
} else if (this.serviceType === "webdav") {
|
||||||
return await webdav.listFromRemote(this.webdavClient, prefix);
|
return await webdav.listFromRemote(this.webdavClient, prefix);
|
||||||
|
} else if (this.serviceType === "dropbox") {
|
||||||
|
return await dropbox.listFromRemote(this.dropboxClient, prefix);
|
||||||
} else {
|
} else {
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
throw Error(`not supported service type ${this.serviceType}`);
|
||||||
}
|
}
|
||||||
@ -109,6 +131,15 @@ export class RemoteClient {
|
|||||||
password,
|
password,
|
||||||
remoteEncryptedKey
|
remoteEncryptedKey
|
||||||
);
|
);
|
||||||
|
} else if (this.serviceType === "dropbox") {
|
||||||
|
return await dropbox.downloadFromRemote(
|
||||||
|
this.dropboxClient,
|
||||||
|
fileOrFolderPath,
|
||||||
|
vault,
|
||||||
|
mtime,
|
||||||
|
password,
|
||||||
|
remoteEncryptedKey
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
throw Error(`not supported service type ${this.serviceType}`);
|
||||||
}
|
}
|
||||||
@ -134,6 +165,13 @@ export class RemoteClient {
|
|||||||
password,
|
password,
|
||||||
remoteEncryptedKey
|
remoteEncryptedKey
|
||||||
);
|
);
|
||||||
|
} else if (this.serviceType === "dropbox") {
|
||||||
|
return await dropbox.deleteFromRemote(
|
||||||
|
this.dropboxClient,
|
||||||
|
fileOrFolderPath,
|
||||||
|
password,
|
||||||
|
remoteEncryptedKey
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
throw Error(`not supported service type ${this.serviceType}`);
|
||||||
}
|
}
|
||||||
@ -144,6 +182,8 @@ export class RemoteClient {
|
|||||||
return await s3.checkConnectivity(this.s3Client, this.s3Config);
|
return await s3.checkConnectivity(this.s3Client, this.s3Config);
|
||||||
} else if (this.serviceType === "webdav") {
|
} else if (this.serviceType === "webdav") {
|
||||||
return await webdav.checkConnectivity(this.webdavClient);
|
return await webdav.checkConnectivity(this.webdavClient);
|
||||||
|
} else if (this.serviceType === "dropbox") {
|
||||||
|
return await dropbox.checkConnectivity(this.dropboxClient);
|
||||||
} else {
|
} else {
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
throw Error(`not supported service type ${this.serviceType}`);
|
||||||
}
|
}
|
||||||
|
|||||||
372
src/remoteForDropbox.ts
Normal file
372
src/remoteForDropbox.ts
Normal file
@ -0,0 +1,372 @@
|
|||||||
|
import * as path from "path";
|
||||||
|
import { FileStats, Vault } from "obsidian";
|
||||||
|
|
||||||
|
import { Dropbox, DropboxResponse, files } from "dropbox";
|
||||||
|
export { Dropbox } from "dropbox";
|
||||||
|
import { RemoteItem } from "./baseTypes";
|
||||||
|
import {
|
||||||
|
arrayBufferToBuffer,
|
||||||
|
bufferToArrayBuffer,
|
||||||
|
mkdirpInVault,
|
||||||
|
getPathFolder,
|
||||||
|
getFolderLevels,
|
||||||
|
setToString,
|
||||||
|
} from "./misc";
|
||||||
|
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
|
||||||
|
import { strict as assert } from "assert";
|
||||||
|
|
||||||
|
export interface DropboxConfig {
|
||||||
|
accessToken: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DEFAULT_DROPBOX_CONFIG = {
|
||||||
|
accessToken: "",
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getDropboxPath = (fileOrFolderPath: string) => {
|
||||||
|
let key = fileOrFolderPath;
|
||||||
|
if (!fileOrFolderPath.startsWith("/")) {
|
||||||
|
key = `/${fileOrFolderPath}`;
|
||||||
|
}
|
||||||
|
if (key.endsWith("/")) {
|
||||||
|
key = key.slice(0, key.length - 1);
|
||||||
|
}
|
||||||
|
return key;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getNormPath = (fileOrFolderPath: string) => {
|
||||||
|
if (fileOrFolderPath.startsWith("/")) {
|
||||||
|
return fileOrFolderPath.slice(1);
|
||||||
|
}
|
||||||
|
return fileOrFolderPath;
|
||||||
|
};
|
||||||
|
|
||||||
|
const fromDropboxItemToRemoteItem = (
|
||||||
|
x:
|
||||||
|
| files.FileMetadataReference
|
||||||
|
| files.FolderMetadataReference
|
||||||
|
| files.DeletedMetadataReference
|
||||||
|
): RemoteItem => {
|
||||||
|
let key = getNormPath(x.path_display);
|
||||||
|
if (x[".tag"] === "folder" && !key.endsWith("/")) {
|
||||||
|
key = `${key}/`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (x[".tag"] === "folder") {
|
||||||
|
return {
|
||||||
|
key: key,
|
||||||
|
lastModified: undefined,
|
||||||
|
size: 0,
|
||||||
|
remoteType: "dropbox",
|
||||||
|
etag: `${x.id}\t`,
|
||||||
|
} as RemoteItem;
|
||||||
|
} else if (x[".tag"] === "file") {
|
||||||
|
return {
|
||||||
|
key: key,
|
||||||
|
lastModified: Date.parse(x.server_modified).valueOf(),
|
||||||
|
size: x.size,
|
||||||
|
remoteType: "dropbox",
|
||||||
|
etag: `${x.id}\t${x.content_hash}`,
|
||||||
|
} as RemoteItem;
|
||||||
|
} else if (x[".tag"] === "deleted") {
|
||||||
|
throw Error("do not support deleted tag");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dropbox api doesn't return mtime for folders.
|
||||||
|
* This is a try to assign mtime by using files in folder.
|
||||||
|
* @param allFilesFolders
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
const fixLastModifiedTimeInplace = (allFilesFolders: RemoteItem[]) => {
|
||||||
|
if (allFilesFolders.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// sort by longer to shorter
|
||||||
|
allFilesFolders.sort((a, b) => b.key.length - a.key.length);
|
||||||
|
|
||||||
|
// a "map" from dir to mtime
|
||||||
|
let potentialMTime = {} as Record<string, number>;
|
||||||
|
|
||||||
|
// first sort pass, from buttom to up
|
||||||
|
for (const item of allFilesFolders) {
|
||||||
|
if (item.key.endsWith("/")) {
|
||||||
|
// itself is a folder, and initially doesn't have mtime
|
||||||
|
if (item.lastModified === undefined && item.key in potentialMTime) {
|
||||||
|
// previously we gathered all sub info of this folder
|
||||||
|
item.lastModified = potentialMTime[item.key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const parent = `${path.posix.dirname(item.key)}/`;
|
||||||
|
if (item.lastModified !== undefined) {
|
||||||
|
if (parent in potentialMTime) {
|
||||||
|
potentialMTime[parent] = Math.max(
|
||||||
|
potentialMTime[parent],
|
||||||
|
item.lastModified
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
potentialMTime[parent] = item.lastModified;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// second pass, from up to buttom.
|
||||||
|
// fill mtime by parent folder or Date.Now() if still not available.
|
||||||
|
// this is only possible if no any sub-folder-files recursively.
|
||||||
|
// we do not sort the array again, just iterate over it by reverse
|
||||||
|
// using good old for loop.
|
||||||
|
for (let i = allFilesFolders.length - 1; i >= 0; --i) {
|
||||||
|
const item = allFilesFolders[i];
|
||||||
|
if (!item.key.endsWith("/")) {
|
||||||
|
continue; // skip files
|
||||||
|
}
|
||||||
|
if (item.lastModified !== undefined) {
|
||||||
|
continue; // don't need to deal with it
|
||||||
|
}
|
||||||
|
assert(!(item.key in potentialMTime));
|
||||||
|
const parent = `${path.posix.dirname(item.key)}/`;
|
||||||
|
if (parent in potentialMTime) {
|
||||||
|
item.lastModified = potentialMTime[parent];
|
||||||
|
} else {
|
||||||
|
item.lastModified = Date.now().valueOf();
|
||||||
|
potentialMTime[item.key] = item.lastModified;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return allFilesFolders;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getDropboxClient = (
|
||||||
|
accessTokenOrDropboxConfig: string | DropboxConfig
|
||||||
|
) => {
|
||||||
|
if (typeof accessTokenOrDropboxConfig === "string") {
|
||||||
|
return new Dropbox({ accessToken: accessTokenOrDropboxConfig });
|
||||||
|
}
|
||||||
|
return new Dropbox({
|
||||||
|
accessToken: accessTokenOrDropboxConfig.accessToken,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getRemoteMeta = async (dbx: Dropbox, fileOrFolderPath: string) => {
|
||||||
|
if (fileOrFolderPath === "" || fileOrFolderPath === "/") {
|
||||||
|
// filesGetMetadata doesn't support root folder
|
||||||
|
// we instead try to list files
|
||||||
|
// if no error occurs, we ensemble a fake result.
|
||||||
|
const rsp = await dbx.filesListFolder({
|
||||||
|
path: "",
|
||||||
|
recursive: false, // don't need to recursive here
|
||||||
|
});
|
||||||
|
if (rsp.status !== 200) {
|
||||||
|
throw Error(JSON.stringify(rsp));
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
key: fileOrFolderPath,
|
||||||
|
lastModified: undefined,
|
||||||
|
size: 0,
|
||||||
|
remoteType: "dropbox",
|
||||||
|
etag: undefined,
|
||||||
|
} as RemoteItem;
|
||||||
|
}
|
||||||
|
|
||||||
|
const key = getDropboxPath(fileOrFolderPath);
|
||||||
|
|
||||||
|
const rsp = await dbx.filesGetMetadata({
|
||||||
|
path: key,
|
||||||
|
});
|
||||||
|
if (rsp.status !== 200) {
|
||||||
|
throw Error(JSON.stringify(rsp));
|
||||||
|
}
|
||||||
|
return fromDropboxItemToRemoteItem(rsp.result);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const uploadToRemote = async (
|
||||||
|
dbx: Dropbox,
|
||||||
|
fileOrFolderPath: string,
|
||||||
|
vault: Vault,
|
||||||
|
isRecursively: boolean = false,
|
||||||
|
password: string = "",
|
||||||
|
remoteEncryptedKey: string = "",
|
||||||
|
foldersCreatedBefore: Set<string> | undefined = undefined
|
||||||
|
) => {
|
||||||
|
let uploadFile = fileOrFolderPath;
|
||||||
|
if (password !== "") {
|
||||||
|
uploadFile = remoteEncryptedKey;
|
||||||
|
}
|
||||||
|
uploadFile = getDropboxPath(uploadFile);
|
||||||
|
|
||||||
|
const isFolder = fileOrFolderPath.endsWith("/");
|
||||||
|
|
||||||
|
if (isFolder && isRecursively) {
|
||||||
|
throw Error("upload function doesn't implement recursive function yet!");
|
||||||
|
} else if (isFolder && !isRecursively) {
|
||||||
|
// folder
|
||||||
|
if (password === "") {
|
||||||
|
// if not encrypted, mkdir a remote folder
|
||||||
|
if (foldersCreatedBefore?.has(uploadFile)) {
|
||||||
|
// created, pass
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
await dbx.filesCreateFolderV2({
|
||||||
|
path: uploadFile,
|
||||||
|
});
|
||||||
|
foldersCreatedBefore?.add(uploadFile);
|
||||||
|
} catch (err) {
|
||||||
|
if (err.status === 409) {
|
||||||
|
// pass
|
||||||
|
foldersCreatedBefore?.add(uploadFile);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const res = await getRemoteMeta(dbx, uploadFile);
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
// if encrypted, upload a fake file with the encrypted file name
|
||||||
|
await dbx.filesUpload({
|
||||||
|
path: uploadFile,
|
||||||
|
contents: "",
|
||||||
|
});
|
||||||
|
return await getRemoteMeta(dbx, uploadFile);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// file
|
||||||
|
// we ignore isRecursively parameter here
|
||||||
|
const localContent = await vault.adapter.readBinary(fileOrFolderPath);
|
||||||
|
let remoteContent = localContent;
|
||||||
|
if (password !== "") {
|
||||||
|
remoteContent = await encryptArrayBuffer(localContent, password);
|
||||||
|
}
|
||||||
|
// in dropbox, we don't need to create folders before uploading! cool!
|
||||||
|
// TODO: filesUploadSession for larger files (>=150 MB)
|
||||||
|
await dbx.filesUpload({
|
||||||
|
path: uploadFile,
|
||||||
|
contents: remoteContent,
|
||||||
|
mode: {
|
||||||
|
".tag": "overwrite",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
// we want to mark that parent folders are created
|
||||||
|
if (foldersCreatedBefore !== undefined) {
|
||||||
|
const dirs = getFolderLevels(uploadFile).map(getDropboxPath);
|
||||||
|
for (const dir of dirs) {
|
||||||
|
foldersCreatedBefore?.add(dir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return await getRemoteMeta(dbx, uploadFile);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const listFromRemote = async (dbx: Dropbox, prefix?: string) => {
|
||||||
|
if (prefix !== undefined) {
|
||||||
|
throw Error("prefix not supported (yet)");
|
||||||
|
}
|
||||||
|
const res = await dbx.filesListFolder({ path: "", recursive: true });
|
||||||
|
if (res.status !== 200) {
|
||||||
|
throw Error(JSON.stringify(res));
|
||||||
|
}
|
||||||
|
// console.log(res);
|
||||||
|
const contents = res.result.entries;
|
||||||
|
const unifiedContents = contents
|
||||||
|
.filter((x) => x[".tag"] !== "deleted")
|
||||||
|
.map(fromDropboxItemToRemoteItem);
|
||||||
|
fixLastModifiedTimeInplace(unifiedContents);
|
||||||
|
return {
|
||||||
|
Contents: unifiedContents,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const downloadFromRemoteRaw = async (
|
||||||
|
dbx: Dropbox,
|
||||||
|
fileOrFolderPath: string
|
||||||
|
) => {
|
||||||
|
const key = getDropboxPath(fileOrFolderPath);
|
||||||
|
const rsp = await dbx.filesDownload({
|
||||||
|
path: key,
|
||||||
|
});
|
||||||
|
if ((rsp.result as any).fileBlob !== undefined) {
|
||||||
|
// we get a Blob
|
||||||
|
const content = (rsp.result as any).fileBlob as Blob;
|
||||||
|
return await content.arrayBuffer();
|
||||||
|
} else if ((rsp.result as any).fileBinary !== undefined) {
|
||||||
|
// we get a Buffer
|
||||||
|
const content = (rsp.result as any).fileBinary as Buffer;
|
||||||
|
return bufferToArrayBuffer(content);
|
||||||
|
} else {
|
||||||
|
throw Error(`unknown rsp from dropbox download: ${rsp}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const downloadFromRemote = async (
|
||||||
|
dbx: Dropbox,
|
||||||
|
fileOrFolderPath: string,
|
||||||
|
vault: Vault,
|
||||||
|
mtime: number,
|
||||||
|
password: string = "",
|
||||||
|
remoteEncryptedKey: string = ""
|
||||||
|
) => {
|
||||||
|
const isFolder = fileOrFolderPath.endsWith("/");
|
||||||
|
|
||||||
|
await mkdirpInVault(fileOrFolderPath, vault);
|
||||||
|
|
||||||
|
// the file is always local file
|
||||||
|
// we need to encrypt it
|
||||||
|
|
||||||
|
if (isFolder) {
|
||||||
|
// mkdirp locally is enough
|
||||||
|
// do nothing here
|
||||||
|
} else {
|
||||||
|
let downloadFile = fileOrFolderPath;
|
||||||
|
if (password !== "") {
|
||||||
|
downloadFile = remoteEncryptedKey;
|
||||||
|
}
|
||||||
|
downloadFile = getDropboxPath(downloadFile);
|
||||||
|
const remoteContent = await downloadFromRemoteRaw(dbx, downloadFile);
|
||||||
|
let localContent = remoteContent;
|
||||||
|
if (password !== "") {
|
||||||
|
localContent = await decryptArrayBuffer(remoteContent, password);
|
||||||
|
}
|
||||||
|
await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
|
||||||
|
mtime: mtime,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const deleteFromRemote = async (
|
||||||
|
dbx: Dropbox,
|
||||||
|
fileOrFolderPath: string,
|
||||||
|
password: string = "",
|
||||||
|
remoteEncryptedKey: string = ""
|
||||||
|
) => {
|
||||||
|
if (fileOrFolderPath === "/") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let remoteFileName = fileOrFolderPath;
|
||||||
|
if (password !== "") {
|
||||||
|
remoteFileName = remoteEncryptedKey;
|
||||||
|
}
|
||||||
|
remoteFileName = getDropboxPath(remoteFileName);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await dbx.filesDeleteV2({
|
||||||
|
path: remoteFileName,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
console.error("some error while deleting");
|
||||||
|
console.log(err);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const checkConnectivity = async (dbx: Dropbox) => {
|
||||||
|
try {
|
||||||
|
const results = await getRemoteMeta(dbx, "/");
|
||||||
|
if (results === undefined) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
58
src/sync.ts
58
src/sync.ts
@ -9,7 +9,7 @@ import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
|
|||||||
|
|
||||||
import { RemoteClient } from "./remote";
|
import { RemoteClient } from "./remote";
|
||||||
import type { SUPPORTED_SERVICES_TYPE, RemoteItem } from "./baseTypes";
|
import type { SUPPORTED_SERVICES_TYPE, RemoteItem } from "./baseTypes";
|
||||||
import { mkdirpInVault, isHiddenPath, isVaildText } from "./misc";
|
import { mkdirpInVault, isHiddenPath, isVaildText, setToString } from "./misc";
|
||||||
import {
|
import {
|
||||||
decryptBase32ToString,
|
decryptBase32ToString,
|
||||||
encryptStringToBase32,
|
encryptStringToBase32,
|
||||||
@ -389,7 +389,7 @@ const getOperation = (
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (r.decision === "unknown") {
|
if (r.decision === "unknown") {
|
||||||
throw Error(`unknown decision for ${r}`);
|
throw Error(`unknown decision for ${JSON.stringify(r)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return r;
|
return r;
|
||||||
@ -428,7 +428,8 @@ const dispatchOperationToActual = async (
|
|||||||
client: RemoteClient,
|
client: RemoteClient,
|
||||||
db: InternalDBs,
|
db: InternalDBs,
|
||||||
vault: Vault,
|
vault: Vault,
|
||||||
password: string = ""
|
password: string = "",
|
||||||
|
foldersCreatedBefore: Set<string> | undefined = undefined
|
||||||
) => {
|
) => {
|
||||||
let remoteEncryptedKey = key;
|
let remoteEncryptedKey = key;
|
||||||
if (password !== "") {
|
if (password !== "") {
|
||||||
@ -461,7 +462,8 @@ const dispatchOperationToActual = async (
|
|||||||
vault,
|
vault,
|
||||||
false,
|
false,
|
||||||
password,
|
password,
|
||||||
remoteEncryptedKey
|
remoteEncryptedKey,
|
||||||
|
foldersCreatedBefore
|
||||||
);
|
);
|
||||||
await upsertSyncMetaMappingData(
|
await upsertSyncMetaMappingData(
|
||||||
client.serviceType,
|
client.serviceType,
|
||||||
@ -493,7 +495,8 @@ const dispatchOperationToActual = async (
|
|||||||
vault,
|
vault,
|
||||||
false,
|
false,
|
||||||
password,
|
password,
|
||||||
remoteEncryptedKey
|
remoteEncryptedKey,
|
||||||
|
foldersCreatedBefore
|
||||||
);
|
);
|
||||||
await upsertSyncMetaMappingData(
|
await upsertSyncMetaMappingData(
|
||||||
client.serviceType,
|
client.serviceType,
|
||||||
@ -521,18 +524,35 @@ export const doActualSync = async (
|
|||||||
password: string = ""
|
password: string = ""
|
||||||
) => {
|
) => {
|
||||||
const keyStates = syncPlan.mixedStates;
|
const keyStates = syncPlan.mixedStates;
|
||||||
await Promise.all(
|
const foldersCreatedBefore = new Set<string>();
|
||||||
Object.entries(keyStates)
|
for (const [k, v] of Object.entries(keyStates).sort(
|
||||||
.sort((k, v) => -(k as string).length)
|
([k1, v1], [k2, v2]) => k2.length - k1.length
|
||||||
.map(async ([k, v]) =>
|
)) {
|
||||||
dispatchOperationToActual(
|
const k2 = k as string;
|
||||||
k as string,
|
const v2 = v as FileOrFolderMixedState;
|
||||||
v as FileOrFolderMixedState,
|
await dispatchOperationToActual(
|
||||||
client,
|
k as string,
|
||||||
db,
|
v as FileOrFolderMixedState,
|
||||||
vault,
|
client,
|
||||||
password
|
db,
|
||||||
)
|
vault,
|
||||||
)
|
password,
|
||||||
);
|
foldersCreatedBefore
|
||||||
|
);
|
||||||
|
// console.log(`finished ${k}, with ${setToString(foldersCreatedBefore)}`);
|
||||||
|
}
|
||||||
|
// await Promise.all(
|
||||||
|
// Object.entries(keyStates)
|
||||||
|
// .map(async ([k, v]) =>
|
||||||
|
// dispatchOperationToActual(
|
||||||
|
// k as string,
|
||||||
|
// v as FileOrFolderMixedState,
|
||||||
|
// client,
|
||||||
|
// db,
|
||||||
|
// vault,
|
||||||
|
// password,
|
||||||
|
// foldersCreatedBefore
|
||||||
|
// )
|
||||||
|
// )
|
||||||
|
// );
|
||||||
};
|
};
|
||||||
|
|||||||
@ -11,10 +11,16 @@
|
|||||||
display: none;
|
display: none;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dropbox-disclaimer {
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
|
.dropbox-hide {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
.webdav-disclaimer {
|
.webdav-disclaimer {
|
||||||
font-weight: bold;
|
font-weight: bold;
|
||||||
}
|
}
|
||||||
|
|
||||||
.webdav-hide {
|
.webdav-hide {
|
||||||
display: none;
|
display: none;
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user