basically workable webdav

This commit is contained in:
fyears 2021-11-21 15:31:20 +08:00
parent ce0cc232c8
commit d1839706af
9 changed files with 521 additions and 192 deletions

13
src/baseTypes.ts Normal file
View File

@ -0,0 +1,13 @@
/**
* Only type defs here.
*/
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav";
export interface RemoteItem {
key: string;
lastModified: number;
size: number;
remoteType: SUPPORTED_SERVICES_TYPE;
etag?: string;
}

View File

@ -1,7 +1,7 @@
import localforage from "localforage"; import localforage from "localforage";
import { TAbstractFile, TFile, TFolder } from "obsidian"; import { TAbstractFile, TFile, TFolder } from "obsidian";
import type { SUPPORTED_SERVICES_TYPE } from "./misc"; import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import type { SyncPlanType } from "./sync"; import type { SyncPlanType } from "./sync";
export type LocalForage = typeof localforage; export type LocalForage = typeof localforage;
@ -197,7 +197,8 @@ export const insertRenameRecord = async (
await db.deleteHistoryTbl.setItem(k.key, k); await db.deleteHistoryTbl.setItem(k.key, k);
}; };
export const upsertSyncMetaMappingDataS3 = async ( export const upsertSyncMetaMappingData = async (
serviceType: SUPPORTED_SERVICES_TYPE,
db: InternalDBs, db: InternalDBs,
localKey: string, localKey: string,
localMTime: number, localMTime: number,
@ -215,13 +216,14 @@ export const upsertSyncMetaMappingDataS3 = async (
remoteMtime: remoteMTime, remoteMtime: remoteMTime,
remoteSize: remoteSize, remoteSize: remoteSize,
remoteExtraKey: remoteExtraKey, remoteExtraKey: remoteExtraKey,
remoteType: "s3", remoteType: serviceType,
keyType: localKey.endsWith("/") ? "folder" : "file", keyType: localKey.endsWith("/") ? "folder" : "file",
}; };
await db.syncMappingTbl.setItem(remoteKey, aggregratedInfo); await db.syncMappingTbl.setItem(remoteKey, aggregratedInfo);
}; };
export const getSyncMetaMappingByRemoteKeyS3 = async ( export const getSyncMetaMappingByRemoteKey = async (
serviceType: SUPPORTED_SERVICES_TYPE,
db: InternalDBs, db: InternalDBs,
remoteKey: string, remoteKey: string,
remoteMTime: number, remoteMTime: number,
@ -240,7 +242,7 @@ export const getSyncMetaMappingByRemoteKeyS3 = async (
potentialItem.remoteKey === remoteKey && potentialItem.remoteKey === remoteKey &&
potentialItem.remoteMtime === remoteMTime && potentialItem.remoteMtime === remoteMTime &&
potentialItem.remoteExtraKey === remoteExtraKey && potentialItem.remoteExtraKey === remoteExtraKey &&
potentialItem.remoteType === "s3" potentialItem.remoteType === serviceType
) { ) {
// the result was found // the result was found
return potentialItem; return potentialItem;

View File

@ -25,22 +25,20 @@ import type { InternalDBs } from "./localdb";
import type { SyncStatusType, PasswordCheckType } from "./sync"; import type { SyncStatusType, PasswordCheckType } from "./sync";
import { isPasswordOk, getSyncPlan, doActualSync } from "./sync"; import { isPasswordOk, getSyncPlan, doActualSync } from "./sync";
import { import { S3Config, DEFAULT_S3_CONFIG } from "./s3";
DEFAULT_S3_CONFIG, import { WebdavConfig, DEFAULT_WEBDAV_CONFIG } from "./webdav";
getS3Client, import { RemoteClient } from "./remote";
listFromRemote,
S3Config,
checkS3Connectivity,
} from "./s3";
import { exportSyncPlansToFiles } from "./debugMode"; import { exportSyncPlansToFiles } from "./debugMode";
interface RemotelySavePluginSettings { interface RemotelySavePluginSettings {
s3?: S3Config; s3: S3Config;
password?: string; webdav: WebdavConfig;
password: string;
} }
const DEFAULT_SETTINGS: RemotelySavePluginSettings = { const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
s3: DEFAULT_S3_CONFIG, s3: DEFAULT_S3_CONFIG,
webdav: DEFAULT_WEBDAV_CONFIG,
password: "", password: "",
}; };
@ -71,6 +69,16 @@ export default class RemotelySavePlugin extends Plugin {
}) })
); );
this.addRibbonIcon("dice", "Remotely Save", async () => {
const client = new RemoteClient(
"webdav",
undefined,
this.settings.webdav
);
const xx = await client.listFromRemote()
console.log(xx);
});
this.addRibbonIcon("switch", "Remotely Save", async () => { this.addRibbonIcon("switch", "Remotely Save", async () => {
if (this.syncStatus !== "idle") { if (this.syncStatus !== "idle") {
new Notice( new Notice(
@ -86,8 +94,13 @@ export default class RemotelySavePlugin extends Plugin {
new Notice("2/6 Starting to fetch remote meta data."); new Notice("2/6 Starting to fetch remote meta data.");
this.syncStatus = "getting_remote_meta"; this.syncStatus = "getting_remote_meta";
const s3Client = getS3Client(this.settings.s3); // const client = new RemoteClient('s3', this.settings.s3, undefined);
const remoteRsp = await listFromRemote(s3Client, this.settings.s3); const client = new RemoteClient(
"webdav",
undefined,
this.settings.webdav
);
const remoteRsp = await client.listFromRemote();
new Notice("3/6 Starting to fetch local meta data."); new Notice("3/6 Starting to fetch local meta data.");
this.syncStatus = "getting_local_meta"; this.syncStatus = "getting_local_meta";
@ -115,6 +128,7 @@ export default class RemotelySavePlugin extends Plugin {
local, local,
localHistory, localHistory,
this.db, this.db,
client.serviceType,
this.settings.password this.settings.password
); );
console.log(syncPlan.mixedStates); // for debugging console.log(syncPlan.mixedStates); // for debugging
@ -127,8 +141,7 @@ export default class RemotelySavePlugin extends Plugin {
this.syncStatus = "syncing"; this.syncStatus = "syncing";
await doActualSync( await doActualSync(
s3Client, client,
this.settings.s3,
this.db, this.db,
this.app.vault, this.app.vault,
syncPlan, syncPlan,
@ -259,6 +272,86 @@ class RemotelySaveSettingTab extends PluginSettingTab {
containerEl.createEl("h1", { text: "Remotely Save" }); containerEl.createEl("h1", { text: "Remotely Save" });
const webdavDiv = containerEl.createEl("div");
webdavDiv.createEl("h2", { text: "Webdav Service" });
new Setting(webdavDiv)
.setName("server address")
.setDesc("server address")
.addText((text) =>
text
.setPlaceholder("")
.setValue(this.plugin.settings.webdav.address)
.onChange(async (value) => {
this.plugin.settings.webdav.address = value.trim();
await this.plugin.saveSettings();
})
);
new Setting(webdavDiv)
.setName("server username")
.setDesc("server username")
.addText((text) =>
text
.setPlaceholder("")
.setValue(this.plugin.settings.webdav.username)
.onChange(async (value) => {
this.plugin.settings.webdav.username = value.trim();
await this.plugin.saveSettings();
})
);
new Setting(webdavDiv)
.setName("server password")
.setDesc("server password")
.addText((text) =>
text
.setPlaceholder("")
.setValue(this.plugin.settings.webdav.password)
.onChange(async (value) => {
this.plugin.settings.webdav.password = value.trim();
await this.plugin.saveSettings();
})
);
new Setting(webdavDiv)
.setName("server auth type")
.setDesc("server auth type")
.addText((text) =>
text
.setPlaceholder("")
.setValue(this.plugin.settings.webdav.authType)
.onChange(async (value) => {
if (value.trim() === "digest") {
this.plugin.settings.webdav.authType = "digest";
} else {
this.plugin.settings.webdav.authType = "basic";
}
await this.plugin.saveSettings();
})
);
new Setting(webdavDiv)
.setName("check connectivity")
.setDesc("check connectivity")
.addButton(async (button) => {
button.setButtonText("Check");
button.onClick(async () => {
new Notice("Checking...");
const client = new RemoteClient(
"webdav",
undefined,
this.plugin.settings.webdav
);
const res = await client.checkConnectivity();
if (res) {
new Notice("Great! The webdav server can be accessed.");
} else {
new Notice("The webdav server cannot be reached.");
}
});
});
const s3Div = containerEl.createEl("div"); const s3Div = containerEl.createEl("div");
s3Div.createEl("h2", { text: "S3 (-compatible) Service" }); s3Div.createEl("h2", { text: "S3 (-compatible) Service" });
@ -368,11 +461,12 @@ class RemotelySaveSettingTab extends PluginSettingTab {
button.setButtonText("Check"); button.setButtonText("Check");
button.onClick(async () => { button.onClick(async () => {
new Notice("Checking..."); new Notice("Checking...");
const s3Client = getS3Client(this.plugin.settings.s3); const client = new RemoteClient(
const res = await checkS3Connectivity( "s3",
s3Client, this.plugin.settings.s3,
this.plugin.settings.s3 undefined
); );
const res = await client.checkConnectivity();
if (res) { if (res) {
new Notice("Great! The bucket can be accessed."); new Notice("Great! The bucket can be accessed.");
} else { } else {

View File

@ -4,8 +4,6 @@ import * as path from "path";
import { base32 } from "rfc4648"; import { base32 } from "rfc4648";
import XRegExp from "xregexp"; import XRegExp from "xregexp";
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "ftp";
/** /**
* If any part of the file starts with '.' or '_' then it's a hidden file. * If any part of the file starts with '.' or '_' then it's a hidden file.
* @param item * @param item
@ -130,3 +128,17 @@ export const isVaildText = (a: string) => {
a a
); );
}; };
/**
* If input is already a folder, returns it as is;
* And if input is a file, returns its direname.
* @param a
* @returns
*/
export const getPathFolder = (a: string) => {
if (a.endsWith("/")) {
return a;
}
const b = path.posix.dirname(a);
return b.endsWith("/") ? b : `${b}/`;
};

151
src/remote.ts Normal file
View File

@ -0,0 +1,151 @@
import { Vault } from "obsidian";
import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import * as s3 from "./s3";
import * as webdav from "./webdav";
export class RemoteClient {
readonly serviceType: SUPPORTED_SERVICES_TYPE;
readonly s3Client?: s3.S3Client;
readonly s3Config?: s3.S3Config;
readonly webdavClient?: webdav.WebDAVClient;
readonly webdavConfig?: webdav.WebdavConfig;
constructor(
serviceType: SUPPORTED_SERVICES_TYPE,
s3Config?: s3.S3Config,
webdavConfig?: webdav.WebdavConfig
) {
this.serviceType = serviceType;
if (serviceType === "s3") {
this.s3Config = s3Config;
this.s3Client = s3.getS3Client(s3Config);
} else if (serviceType === "webdav") {
this.webdavConfig = webdavConfig;
this.webdavClient = webdav.getWebdavClient(webdavConfig);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
}
getRemoteMeta = async (fileOrFolderPath: string) => {
if (this.serviceType === "s3") {
return await s3.getRemoteMeta(
this.s3Client,
this.s3Config,
fileOrFolderPath
);
} else if (this.serviceType === "webdav") {
return await webdav.getRemoteMeta(this.webdavClient, fileOrFolderPath);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
};
uploadToRemote = async (
fileOrFolderPath: string,
vault: Vault,
isRecursively: boolean = false,
password: string = "",
remoteEncryptedKey: string = ""
) => {
if (this.serviceType === "s3") {
return await s3.uploadToRemote(
this.s3Client,
this.s3Config,
fileOrFolderPath,
vault,
isRecursively,
password,
remoteEncryptedKey
);
} else if (this.serviceType === "webdav") {
return await webdav.uploadToRemote(
this.webdavClient,
fileOrFolderPath,
vault,
isRecursively,
password,
remoteEncryptedKey
);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
};
listFromRemote = async (prefix?: string) => {
if (this.serviceType === "s3") {
return await s3.listFromRemote(this.s3Client, this.s3Config, prefix);
} else if (this.serviceType === "webdav") {
return await webdav.listFromRemote(this.webdavClient, prefix);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
};
downloadFromRemote = async (
fileOrFolderPath: string,
vault: Vault,
mtime: number,
password: string = "",
remoteEncryptedKey: string = ""
) => {
if (this.serviceType === "s3") {
return await s3.downloadFromRemote(
this.s3Client,
this.s3Config,
fileOrFolderPath,
vault,
mtime,
password,
remoteEncryptedKey
);
} else if (this.serviceType === "webdav") {
return await webdav.downloadFromRemote(
this.webdavClient,
fileOrFolderPath,
vault,
mtime,
password,
remoteEncryptedKey
);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
};
deleteFromRemote = async (
fileOrFolderPath: string,
password: string = "",
remoteEncryptedKey: string = ""
) => {
if (this.serviceType === "s3") {
return await s3.deleteFromRemote(
this.s3Client,
this.s3Config,
fileOrFolderPath,
password,
remoteEncryptedKey
);
} else if (this.serviceType === "webdav") {
return await webdav.deleteFromRemote(
this.webdavClient,
fileOrFolderPath,
password,
remoteEncryptedKey
);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
};
checkConnectivity = async () => {
if (this.serviceType === "s3") {
return await s3.checkConnectivity(this.s3Client, this.s3Config);
} else if (this.serviceType === "webdav") {
return await webdav.checkConnectivity(this.webdavClient);
} else {
throw Error(`not supported service type ${this.serviceType}`);
}
};
}

View File

@ -14,7 +14,9 @@ import {
HeadBucketCommand, HeadBucketCommand,
ListObjectsV2CommandInput, ListObjectsV2CommandInput,
ListObjectsV2CommandOutput, ListObjectsV2CommandOutput,
HeadObjectCommandOutput,
} from "@aws-sdk/client-s3"; } from "@aws-sdk/client-s3";
export { S3Client } from "@aws-sdk/client-s3";
import type { _Object } from "@aws-sdk/client-s3"; import type { _Object } from "@aws-sdk/client-s3";
@ -24,6 +26,8 @@ import {
mkdirpInVault, mkdirpInVault,
} from "./misc"; } from "./misc";
import * as mime from "mime-types"; import * as mime from "mime-types";
import { RemoteItem } from "./baseTypes";
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
export interface S3Config { export interface S3Config {
@ -44,6 +48,29 @@ export const DEFAULT_S3_CONFIG = {
export type S3ObjectType = _Object; export type S3ObjectType = _Object;
const fromS3ObjectToRemoteItem = (x: S3ObjectType) => {
return {
key: x.Key,
lastModified: x.LastModified.valueOf(),
size: x.Size,
remoteType: "s3",
etag: x.ETag,
} as RemoteItem;
};
const fromS3HeadObjectToRemoteItem = (
key: string,
x: HeadObjectCommandOutput
) => {
return {
key: key,
lastModified: x.LastModified.valueOf(),
size: x.ContentLength,
remoteType: "s3",
etag: x.ETag,
} as RemoteItem;
};
export const getS3Client = (s3Config: S3Config) => { export const getS3Client = (s3Config: S3Config) => {
let endpoint = s3Config.s3Endpoint; let endpoint = s3Config.s3Endpoint;
if (!(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) { if (!(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {
@ -65,12 +92,14 @@ export const getRemoteMeta = async (
s3Config: S3Config, s3Config: S3Config,
fileOrFolderPath: string fileOrFolderPath: string
) => { ) => {
return await s3Client.send( const res = await s3Client.send(
new HeadObjectCommand({ new HeadObjectCommand({
Bucket: s3Config.s3BucketName, Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath, Key: fileOrFolderPath,
}) })
); );
return fromS3HeadObjectToRemoteItem(fileOrFolderPath, res);
}; };
export const uploadToRemote = async ( export const uploadToRemote = async (
@ -181,10 +210,7 @@ export const listFromRemote = async (
// ensemble fake rsp // ensemble fake rsp
return { return {
"$.metadata": { Contents: contents.map((x) => fromS3ObjectToRemoteItem(x)),
httpStatusCode: 200,
},
Contents: contents,
}; };
}; };
@ -214,7 +240,7 @@ const getObjectBodyToArrayBuffer = async (
} }
}; };
export const downloadFromRemoteRaw = async ( const downloadFromRemoteRaw = async (
s3Client: S3Client, s3Client: S3Client,
s3Config: S3Config, s3Config: S3Config,
fileOrFolderPath: string fileOrFolderPath: string
@ -302,7 +328,7 @@ export const deleteFromRemote = async (
await s3Client.send( await s3Client.send(
new DeleteObjectCommand({ new DeleteObjectCommand({
Bucket: s3Config.s3BucketName, Bucket: s3Config.s3BucketName,
Key: element.Key, Key: element.key,
}) })
); );
}); });
@ -320,7 +346,7 @@ export const deleteFromRemote = async (
* @param s3Config * @param s3Config
* @returns * @returns
*/ */
export const checkS3Connectivity = async ( export const checkConnectivity = async (
s3Client: S3Client, s3Client: S3Client,
s3Config: S3Config s3Config: S3Config
) => { ) => {

View File

@ -1,27 +1,15 @@
import { TAbstractFile, TFolder, TFile, Vault } from "obsidian"; import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
import { S3Client } from "@aws-sdk/client-s3";
import { import {
clearDeleteRenameHistoryOfKey, clearDeleteRenameHistoryOfKey,
upsertSyncMetaMappingDataS3, upsertSyncMetaMappingData,
getSyncMetaMappingByRemoteKeyS3, getSyncMetaMappingByRemoteKey,
} from "./localdb"; } from "./localdb";
import type { FileFolderHistoryRecord, InternalDBs } from "./localdb"; import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
import { import { RemoteClient } from "./remote";
S3Config, import type { SUPPORTED_SERVICES_TYPE, RemoteItem } from "./baseTypes";
S3ObjectType, import { mkdirpInVault, isHiddenPath, isVaildText } from "./misc";
uploadToRemote,
deleteFromRemote,
downloadFromRemote,
} from "./s3";
import {
mkdirpInVault,
SUPPORTED_SERVICES_TYPE,
isHiddenPath,
isVaildText,
} from "./misc";
import { import {
decryptBase32ToString, decryptBase32ToString,
encryptStringToBase32, encryptStringToBase32,
@ -85,7 +73,7 @@ export interface PasswordCheckType {
} }
export const isPasswordOk = async ( export const isPasswordOk = async (
remote: S3ObjectType[], remote: RemoteItem[],
password: string = "" password: string = ""
) => { ) => {
if (remote === undefined || remote.length === 0) { if (remote === undefined || remote.length === 0) {
@ -95,7 +83,7 @@ export const isPasswordOk = async (
reason: "empty_remote", reason: "empty_remote",
} as PasswordCheckType; } as PasswordCheckType;
} }
const santyCheckKey = remote[0].Key; const santyCheckKey = remote[0].key;
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) { if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
// this is encrypted! // this is encrypted!
// try to decrypt it using the provided password. // try to decrypt it using the provided password.
@ -143,26 +131,28 @@ export const isPasswordOk = async (
}; };
const ensembleMixedStates = async ( const ensembleMixedStates = async (
remote: S3ObjectType[], remote: RemoteItem[],
local: TAbstractFile[], local: TAbstractFile[],
deleteHistory: FileFolderHistoryRecord[], deleteHistory: FileFolderHistoryRecord[],
db: InternalDBs, db: InternalDBs,
remoteType: SUPPORTED_SERVICES_TYPE,
password: string = "" password: string = ""
) => { ) => {
const results = {} as Record<string, FileOrFolderMixedState>; const results = {} as Record<string, FileOrFolderMixedState>;
if (remote !== undefined) { if (remote !== undefined) {
for (const entry of remote) { for (const entry of remote) {
const remoteEncryptedKey = entry.Key; const remoteEncryptedKey = entry.key;
let key = remoteEncryptedKey; let key = remoteEncryptedKey;
if (password !== "") { if (password !== "") {
key = await decryptBase32ToString(remoteEncryptedKey, password); key = await decryptBase32ToString(remoteEncryptedKey, password);
} }
const backwardMapping = await getSyncMetaMappingByRemoteKeyS3( const backwardMapping = await getSyncMetaMappingByRemoteKey(
remoteType,
db, db,
key, key,
entry.LastModified.valueOf(), entry.lastModified,
entry.ETag entry.etag
); );
let r = {} as FileOrFolderMixedState; let r = {} as FileOrFolderMixedState;
@ -179,8 +169,8 @@ const ensembleMixedStates = async (
r = { r = {
key: key, key: key,
exist_remote: true, exist_remote: true,
mtime_remote: entry.LastModified.valueOf(), mtime_remote: entry.lastModified,
size_remote: entry.Size, size_remote: entry.size,
remote_encrypted_key: remoteEncryptedKey, remote_encrypted_key: remoteEncryptedKey,
}; };
} }
@ -402,10 +392,11 @@ const getOperation = (
}; };
export const getSyncPlan = async ( export const getSyncPlan = async (
remote: S3ObjectType[], remote: RemoteItem[],
local: TAbstractFile[], local: TAbstractFile[],
deleteHistory: FileFolderHistoryRecord[], deleteHistory: FileFolderHistoryRecord[],
db: InternalDBs, db: InternalDBs,
remoteType: SUPPORTED_SERVICES_TYPE,
password: string = "" password: string = ""
) => { ) => {
const mixedStates = await ensembleMixedStates( const mixedStates = await ensembleMixedStates(
@ -413,6 +404,7 @@ export const getSyncPlan = async (
local, local,
deleteHistory, deleteHistory,
db, db,
remoteType,
password password
); );
for (const [key, val] of Object.entries(mixedStates)) { for (const [key, val] of Object.entries(mixedStates)) {
@ -420,15 +412,105 @@ export const getSyncPlan = async (
} }
const plan = { const plan = {
ts: Date.now(), ts: Date.now(),
remoteType: "s3", remoteType: remoteType,
mixedStates: mixedStates, mixedStates: mixedStates,
} as SyncPlanType; } as SyncPlanType;
return plan; return plan;
}; };
const dispatchOperationToActual = async (
key: string,
state: FileOrFolderMixedState,
client: RemoteClient,
db: InternalDBs,
vault: Vault,
password: string = ""
) => {
let remoteEncryptedKey = key;
if (password !== "") {
remoteEncryptedKey = state.remote_encrypted_key;
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
remoteEncryptedKey = await encryptStringToBase32(key, password);
}
}
if (
state.decision === undefined ||
state.decision === "unknown" ||
state.decision === "undecided"
) {
throw Error(`unknown decision in ${JSON.stringify(state)}`);
} else if (state.decision === "skip") {
// do nothing
} else if (state.decision === "download_clearhist") {
await client.downloadFromRemote(
state.key,
vault,
state.mtime_remote,
password,
remoteEncryptedKey
);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "upload_clearhist") {
const remoteObjMeta = await client.uploadToRemote(
state.key,
vault,
false,
password,
remoteEncryptedKey
);
await upsertSyncMetaMappingData(
client.serviceType,
db,
state.key,
state.mtime_local,
state.size_local,
state.key,
remoteObjMeta.lastModified,
remoteObjMeta.size,
remoteObjMeta.etag
);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "download") {
await mkdirpInVault(state.key, vault);
await client.downloadFromRemote(
state.key,
vault,
state.mtime_remote,
password,
remoteEncryptedKey
);
} else if (state.decision === "delremote_clearhist") {
await client.deleteFromRemote(state.key, password, remoteEncryptedKey);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "upload") {
const remoteObjMeta = await client.uploadToRemote(
state.key,
vault,
false,
password,
remoteEncryptedKey
);
await upsertSyncMetaMappingData(
client.serviceType,
db,
state.key,
state.mtime_local,
state.size_local,
state.key,
remoteObjMeta.lastModified,
remoteObjMeta.size,
remoteObjMeta.etag
);
} else if (state.decision === "clearhist") {
await clearDeleteRenameHistoryOfKey(db, state.key);
} else {
throw Error("this should never happen!");
}
};
export const doActualSync = async ( export const doActualSync = async (
s3Client: S3Client, client: RemoteClient,
s3Config: S3Config,
db: InternalDBs, db: InternalDBs,
vault: Vault, vault: Vault,
syncPlan: SyncPlanType, syncPlan: SyncPlanType,
@ -438,102 +520,15 @@ export const doActualSync = async (
await Promise.all( await Promise.all(
Object.entries(keyStates) Object.entries(keyStates)
.sort((k, v) => -(k as string).length) .sort((k, v) => -(k as string).length)
.map(async ([k, v]) => { .map(async ([k, v]) =>
const key = k as string; dispatchOperationToActual(
const state = v as FileOrFolderMixedState; k as string,
let remoteEncryptedKey = key; v as FileOrFolderMixedState,
if (password !== "") { client,
remoteEncryptedKey = state.remote_encrypted_key; db,
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") { vault,
remoteEncryptedKey = await encryptStringToBase32(key, password); password
} )
} )
if (
state.decision === undefined ||
state.decision === "unknown" ||
state.decision === "undecided"
) {
throw Error(`unknown decision in ${JSON.stringify(state)}`);
} else if (state.decision === "skip") {
// do nothing
} else if (state.decision === "download_clearhist") {
await downloadFromRemote(
s3Client,
s3Config,
state.key,
vault,
state.mtime_remote,
password,
remoteEncryptedKey
);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "upload_clearhist") {
const remoteObjMeta = await uploadToRemote(
s3Client,
s3Config,
state.key,
vault,
false,
password,
remoteEncryptedKey
);
await upsertSyncMetaMappingDataS3(
db,
state.key,
state.mtime_local,
state.size_local,
state.key,
remoteObjMeta.LastModified.valueOf(),
remoteObjMeta.ContentLength,
remoteObjMeta.ETag
);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "download") {
await mkdirpInVault(state.key, vault);
await downloadFromRemote(
s3Client,
s3Config,
state.key,
vault,
state.mtime_remote,
password,
remoteEncryptedKey
);
} else if (state.decision === "delremote_clearhist") {
await deleteFromRemote(
s3Client,
s3Config,
state.key,
password,
remoteEncryptedKey
);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "upload") {
const remoteObjMeta = await uploadToRemote(
s3Client,
s3Config,
state.key,
vault,
false,
password,
remoteEncryptedKey
);
await upsertSyncMetaMappingDataS3(
db,
state.key,
state.mtime_local,
state.size_local,
state.key,
remoteObjMeta.LastModified.valueOf(),
remoteObjMeta.ContentLength,
remoteObjMeta.ETag
);
} else if (state.decision === "clearhist") {
await clearDeleteRenameHistoryOfKey(db, state.key);
} else {
throw Error("this should never happen!");
}
})
); );
}; };

View File

@ -2,14 +2,17 @@ import { Buffer } from "buffer";
import { FileStats, Vault } from "obsidian"; import { FileStats, Vault } from "obsidian";
import { AuthType, BufferLike, createClient } from "webdav/web"; import { AuthType, BufferLike, createClient } from "webdav/web";
import type { WebDAVClient, ResponseDataDetailed, FileStat } from "webdav/web"; import type { WebDAVClient, ResponseDataDetailed, FileStat } from "webdav/web";
export type { WebDAVClient } from "webdav/web";
import type { RemoteItem } from "./baseTypes";
import { import {
arrayBufferToBuffer, arrayBufferToBuffer,
bufferToArrayBuffer, bufferToArrayBuffer,
mkdirpInVault, mkdirpInVault,
getPathFolder,
} from "./misc"; } from "./misc";
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
import { fileURLToPath } from "url";
export interface WebdavConfig { export interface WebdavConfig {
address: string; address: string;
@ -25,6 +28,34 @@ export const DEFAULT_WEBDAV_CONFIG = {
authType: "basic", authType: "basic",
} as WebdavConfig; } as WebdavConfig;
const getWebdavPath = (fileOrFolderPath: string) => {
if (!fileOrFolderPath.startsWith("/")) {
return `/${fileOrFolderPath}`;
}
return fileOrFolderPath;
};
const getNormPath = (fileOrFolderPath: string) => {
if (fileOrFolderPath.startsWith("/")) {
return fileOrFolderPath.slice(1);
}
return fileOrFolderPath;
};
const fromWebdavItemToRemoteItem = (x: FileStat) => {
let key = getNormPath(x.filename);
if (x.type === "directory" && !key.endsWith("/")) {
key = `${key}/`;
}
return {
key: key,
lastModified: Date.parse(x.lastmod).valueOf(),
size: x.size,
remoteType: "webdav",
etag: x.etag || undefined,
} as RemoteItem;
};
export const getWebdavClient = (webdavConfig: WebdavConfig) => { export const getWebdavClient = (webdavConfig: WebdavConfig) => {
if (webdavConfig.username !== "" && webdavConfig.password !== "") { if (webdavConfig.username !== "" && webdavConfig.password !== "") {
return createClient(webdavConfig.address, { return createClient(webdavConfig.address, {
@ -41,29 +72,14 @@ export const getWebdavClient = (webdavConfig: WebdavConfig) => {
} }
}; };
const getWebdavPath = (fileOrFolderPath: string) => {
if (!fileOrFolderPath.startsWith("/")) {
return `/${fileOrFolderPath}`;
}
return fileOrFolderPath;
};
const getNormPath = (fileOrFolderPath: string) => {
if (fileOrFolderPath.startsWith("/")) {
return fileOrFolderPath.slice(1);
}
return fileOrFolderPath;
};
export const getRemoteMeta = async ( export const getRemoteMeta = async (
client: WebDAVClient, client: WebDAVClient,
fileOrFolderPath: string fileOrFolderPath: string
) => { ) => {
const res = (await client.stat(getWebdavPath(fileOrFolderPath), { const res = (await client.stat(getWebdavPath(fileOrFolderPath), {
details: true, details: false,
})) as ResponseDataDetailed<FileStat>; })) as FileStat;
res.data.filename = getNormPath(res.data.filename); return fromWebdavItemToRemoteItem(res);
return res;
}; };
export const uploadToRemote = async ( export const uploadToRemote = async (
@ -88,10 +104,11 @@ export const uploadToRemote = async (
// folder // folder
if (password === "") { if (password === "") {
// if not encrypted, mkdir a remote folder // if not encrypted, mkdir a remote folder
client.createDirectory(uploadFile, { await client.createDirectory(uploadFile, {
recursive: true, recursive: true,
}); });
return await getRemoteMeta(client, uploadFile); const res = await getRemoteMeta(client, uploadFile);
return res;
} else { } else {
// if encrypted, upload a fake file with the encrypted file name // if encrypted, upload a fake file with the encrypted file name
await client.putFileContents(uploadFile, "", { await client.putFileContents(uploadFile, "", {
@ -111,6 +128,11 @@ export const uploadToRemote = async (
if (password !== "") { if (password !== "") {
remoteContent = await encryptArrayBuffer(localContent, password); remoteContent = await encryptArrayBuffer(localContent, password);
} }
// we need to create folders before uploading
const dir = getPathFolder(uploadFile);
if (dir !== "/" && dir !== "") {
await client.createDirectory(dir, { recursive: true });
}
await client.putFileContents(uploadFile, remoteContent, { await client.putFileContents(uploadFile, remoteContent, {
overwrite: true, overwrite: true,
onUploadProgress: (progress) => { onUploadProgress: (progress) => {
@ -131,15 +153,12 @@ export const listFromRemote = async (client: WebDAVClient, prefix?: string) => {
details: false /* no need for verbose details here */, details: false /* no need for verbose details here */,
glob: "/**" /* avoid dot files by using glob */, glob: "/**" /* avoid dot files by using glob */,
})) as FileStat[]; })) as FileStat[];
for (const singleItem of contents) {
singleItem.filename = getNormPath(singleItem.filename);
}
return { return {
Contents: contents, Contents: contents.map((x) => fromWebdavItemToRemoteItem(x)),
}; };
}; };
export const downloadFromRemoteRaw = async ( const downloadFromRemoteRaw = async (
client: WebDAVClient, client: WebDAVClient,
fileOrFolderPath: string fileOrFolderPath: string
) => { ) => {
@ -212,15 +231,10 @@ export const deleteFromRemote = async (
} }
}; };
export const checkWebdavConnectivity = async (client: WebDAVClient) => { export const checkConnectivity = async (client: WebDAVClient) => {
try { try {
const results = await getRemoteMeta(client, "/"); const results = await getRemoteMeta(client, "/");
if ( if (results === undefined) {
results === undefined ||
results.data === undefined ||
results.data.type === undefined ||
results.data.type !== "directory"
) {
return false; return false;
} }
return true; return true;

View File

@ -89,3 +89,25 @@ describe("Misc: vaild file name tests", () => {
expect(x).to.be.true; expect(x).to.be.true;
}); });
}); });
describe("Misc: get dirname", () => {
it("should return itself for folder", async () => {
const x = misc.getPathFolder("ssss/");
// console.log(x)
expect(x).to.equal("ssss/");
});
it("should return folder for file", async () => {
const x = misc.getPathFolder("sss/yyy");
// console.log(x)
expect(x).to.equal("sss/");
});
it("should treat / specially", async () => {
const x = misc.getPathFolder("/");
expect(x).to.equal("/");
const y = misc.getPathFolder("/abc");
expect(y).to.equal("/");
});
});