Compare commits

...
15 Commits
Author SHA1 Message Date
fyears d9ccd2c7eb deal with empty remote 2021-10-25 01:01:04 +08:00
fyears f58a71c1c4 bump version 2021-10-25 00:46:03 +08:00
fyears 69bc1f0e03 add mapping of local and remote 2021-10-25 00:41:13 +08:00
fyears 5ef032523a skip same file locally 2021-10-24 22:12:32 +08:00
fyears 5a56526de1 avoid rewriting dist main.js 2021-10-24 20:40:30 +08:00
fyears a26158055d basically working 2 way sync! 2021-10-24 20:38:04 +08:00
fyears 13e5af0c34 prettier docs 2021-10-24 20:36:40 +08:00
fyears 29145604be add code design 2021-10-24 13:12:45 +08:00
fyears 21f8789b27 more situations 2021-10-24 13:02:51 +08:00
fyears 0b26898c99 split func to misc 2021-10-23 16:51:46 +08:00
fyears fa11b3fe7c polish algo 2021-10-23 16:44:04 +08:00
fyears 8800c16b91 force editors 2021-10-23 15:48:42 +08:00
fyears a5155b06d8 note down algorithm 2021-10-23 12:34:45 +08:00
fyears e43a23a93d no parcel 2021-10-23 12:12:24 +08:00
fyears c75962aad5 track local history 2021-10-23 12:02:03 +08:00
14 changed files with 1207 additions and 344 deletions
+28
View File
@@ -0,0 +1,28 @@
# EditorConfig is awesome: https://EditorConfig.org
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
[*]
end_of_line = lf
insert_final_newline = true
# Matches multiple files with brace expansion notation
# Set default charset
[*.{js,jsx,mjs,cjs,ts,tsx,json,py}]
charset = utf-8
# 4 space indentation
[*.py]
indent_style = space
indent_size = 4
# Tab indentation (no size specified)
[Makefile]
indent_style = tab
# Indentation override for all JS under lib directory
[*.{js,jsx,mjs,cjs,ts,tsx,json,py}]
indent_style = space
indent_size = 2
+2
View File
@@ -0,0 +1,2 @@
node_modules/
main.js
+29
View File
@@ -0,0 +1,29 @@
# Algorithm
## Sources
We have three record sources:
1. Local files
2. Remote files
3. Local "delete-or-rename" history.
Assuming all sources are reliable.
## Deal with them
We list all combinations mutually exclusive and collectively exhaustive.
| ID | Remote Files | Local files | Local delete rename history | Extra | Decision |
| --- | ------------ | ----------- | --------------------------- | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------- |
| 1 | exist | exist | ignore | mtime_remote > mtime_local | download remote file, create local folder if not exists, clear local history if exists |
| 2 | exist | exist | ignore | mtime_remote < mtime_local | upload local file, create remote folder if not exists, clear local history if exists |
| 3 | exist | exist | ignore | mtime_remote === mtime_local && size_remote === size_local | clear local history if exists (the file was synced and no changes after last sync) |
| 4 | exist | exist | ignore | mtime_remote === mtime_local && size_remote !== size_local | upload local file, clear local history if exists (we always prefer local to remote) |
| 5 | exist | exist | ignore | If local is a folder. mtime_local === undefined | clear local history if exists. TODO: what if a folder and a previous file share the same name? |
| 6 | exist | not exist | exist | mtime_remote >= delete_time_local | download remote file, create folder if not exists |
| 7 | exist | not exist | exist | mtime_remote < delete_time_local | delete remote file, clear local history |
| 8 | exist | not exist | not exist | | download remote file, create folder if not exists |
| 9 | not exist | exist | ignore | If local is a single file. | upload local file, create remote folder if not exists, clear local history if exists |
| 10 | not exist | exist | ignore | If local is a folder. | upload local files recursively, create remote folder if not exists, clear local history if exists |
| 11 | not exist | not exist | ignore | | clear local history if exists |
+13
View File
@@ -0,0 +1,13 @@
# Code Design
## Code Organization
1. Every function except `main.ts` should be pure. Pass any stateful information in parameters.
2. `misc.ts` should not depend on any other written code.
3. Each storage code should not depend on `sync.ts`.
## File and Folder Representation
While writing sync codes, folders are always represented by a string ending with `/`.
-340
View File
@@ -1,340 +0,0 @@
import * as path from "path";
import * as fs from "fs";
import { Buffer } from "buffer";
import { Readable } from "stream";
import * as mime from "mime-types";
import {
App,
Modal,
Notice,
Plugin,
PluginSettingTab,
Setting,
request,
Platform,
} from "obsidian";
import * as CodeMirror from "codemirror";
import {
S3Client,
ListObjectsCommand,
PutObjectCommand,
GetObjectCommand,
} from "@aws-sdk/client-s3";
interface SaveRemotePluginSettings {
s3Endpoint: string;
s3Region: string;
s3AccessKeyID: string;
s3SecretAccessKey: string;
s3BucketName: string;
}
const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
s3Endpoint: "",
s3Region: "",
s3AccessKeyID: "",
s3SecretAccessKey: "",
s3BucketName: "",
};
const ignoreHiddenFiles = (item: string) => {
const basename = path.basename(item);
return basename === "." || basename[0] !== ".";
};
/**
* Util func for mkdir -p based on the "path" of original file or folder
* "a/b/c/" => ["a", "a/b", "a/b/c"]
* "a/b/c/d/e.txt" => ["a", "a/b", "a/b/c", "a/b/c/d"]
* @param x string
* @returns string[] might be empty
*/
const getFolderLevels = (x: string) => {
const res: string[] = [];
if (x === "" || x === "/") {
return res;
}
const y1 = x.split("/");
let i = 0;
for (let index = 0; index + 1 < y1.length; index++) {
res.push(y1.slice(0, index + 1).join("/"));
}
return res;
};
/**
* https://stackoverflow.com/questions/8609289
* @param b Buffer
* @returns ArrayBuffer
*/
const bufferToArrayBuffer = (b: Buffer) => {
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
};
/**
* The Body of resp of aws GetObject has mix types
* and we want to get ArrayBuffer here.
* See https://github.com/aws/aws-sdk-js-v3/issues/1877
* @param b The Body of GetObject
* @returns Promise<ArrayBuffer>
*/
const getObjectBodyToArrayBuffer = async (
b: Readable | ReadableStream | Blob
) => {
if (b instanceof Readable) {
const chunks: Uint8Array[] = [];
for await (let chunk of b) {
chunks.push(chunk);
}
const buf = Buffer.concat(chunks);
return bufferToArrayBuffer(buf);
} else if (b instanceof ReadableStream) {
return await new Response(b, {}).arrayBuffer();
} else if (b instanceof Blob) {
return await b.arrayBuffer();
} else {
throw TypeError(`The type of ${b} is not one of the supported types`);
}
};
export default class SaveRemotePlugin extends Plugin {
settings: SaveRemotePluginSettings;
cm: CodeMirror.Editor;
async onload() {
console.log("loading plugin obsidian-save-remote");
await this.loadSettings();
this.addRibbonIcon("right-arrow-with-tail", "Upload", async () => {
// console.log(this.app.vault.getFiles());
// console.log(this.app.vault.getAllLoadedFiles());
new Notice(`Upload begun.`);
const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
const s3Client = new S3Client({
region: this.settings.s3Region,
endpoint: this.settings.s3Endpoint,
credentials: {
accessKeyId: this.settings.s3AccessKeyID,
secretAccessKey: this.settings.s3SecretAccessKey,
},
});
try {
for (const fileOrFolder of allFilesAndFolders) {
if (fileOrFolder.path === "/") {
console.log('ignore "/"');
} else if ("children" in fileOrFolder) {
// folder
console.log(`folder ${fileOrFolder.path}/`);
new Notice(`folder ${fileOrFolder.path}/`);
const results = await s3Client.send(
new PutObjectCommand({
Bucket: this.settings.s3BucketName,
Key: `${fileOrFolder.path}/`,
Body: "",
})
);
} else {
// file
console.log(`file ${fileOrFolder.path}`);
const arrContent = await this.app.vault.adapter.readBinary(
fileOrFolder.path
);
new Notice(`file ${fileOrFolder.path}`);
const contentType =
mime.contentType(
mime.lookup(`${fileOrFolder.path}`) || undefined
) || undefined;
// console.log(contentType);
const results = await s3Client.send(
new PutObjectCommand({
Bucket: this.settings.s3BucketName,
Key: `${fileOrFolder.path}`,
Body: Buffer.from(arrContent),
ContentType: contentType,
})
);
}
}
new Notice("Upload finished!");
} catch (err) {
console.log("Error", err);
new Notice(`${err}`);
}
});
this.addRibbonIcon("left-arrow-with-tail", "Download", async () => {
const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
const s3Client = new S3Client({
region: this.settings.s3Region,
endpoint: this.settings.s3Endpoint,
credentials: {
accessKeyId: this.settings.s3AccessKeyID,
secretAccessKey: this.settings.s3SecretAccessKey,
},
});
try {
const listObj = await s3Client.send(
new ListObjectsCommand({ Bucket: this.settings.s3BucketName })
);
for (const singleContent of listObj.Contents) {
const foldersToBuild = getFolderLevels(singleContent.Key);
for (const folder of foldersToBuild) {
const r = await this.app.vault.adapter.exists(folder);
if (!r) {
console.log(`mkdir ${folder}`);
new Notice(`mkdir ${folder}`);
await this.app.vault.adapter.mkdir(folder);
}
}
if (singleContent.Key.endsWith("/")) {
// kind of a folder
// pass
} else {
// kind of a file
// download
console.log(`download file ${singleContent.Key}`);
new Notice(`download file ${singleContent.Key}`);
const data = await s3Client.send(
new GetObjectCommand({
Bucket: this.settings.s3BucketName,
Key: singleContent.Key,
})
);
const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
await this.app.vault.adapter.writeBinary(
singleContent.Key,
bodyContents
);
}
}
new Notice("Download finished!");
} catch (err) {
console.log("Error", err);
new Notice(`${err}`);
}
});
this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
this.registerCodeMirror((cm: CodeMirror.Editor) => {
this.cm = cm;
console.log("codemirror registered.");
});
// this.registerDomEvent(document, "click", (evt: MouseEvent) => {
// console.log("click", evt);
// });
// this.registerInterval(
// window.setInterval(() => console.log("setInterval"), 5 * 60 * 1000)
// );
}
onunload() {
console.log("unloading plugin obsidian-save-remote");
}
async loadSettings() {
this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
}
async saveSettings() {
await this.saveData(this.settings);
}
}
class SaveRemoteSettingTab extends PluginSettingTab {
plugin: SaveRemotePlugin;
constructor(app: App, plugin: SaveRemotePlugin) {
super(app, plugin);
this.plugin = plugin;
}
display(): void {
let { containerEl } = this;
containerEl.empty();
containerEl.createEl("h2", { text: "Settings for Save Remote" });
new Setting(containerEl)
.setName("s3Endpoint")
.setDesc("s3Endpoint")
.addText((text) =>
text
.setPlaceholder("")
.setValue(this.plugin.settings.s3Endpoint)
.onChange(async (value) => {
this.plugin.settings.s3Endpoint = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3Region")
.setDesc("s3Region")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3Region}`)
.onChange(async (value) => {
this.plugin.settings.s3Region = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3AccessKeyID")
.setDesc("s3AccessKeyID")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3AccessKeyID}`)
.onChange(async (value) => {
this.plugin.settings.s3AccessKeyID = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3SecretAccessKey")
.setDesc("s3SecretAccessKey")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3SecretAccessKey}`)
.onChange(async (value) => {
this.plugin.settings.s3SecretAccessKey = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3BucketName")
.setDesc("s3BucketName")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3BucketName}`)
.onChange(async (value) => {
this.plugin.settings.s3BucketName = value;
await this.plugin.saveSettings();
})
);
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"id": "obsdian-save-remote", "id": "obsdian-save-remote",
"name": "Save remote", "name": "Save remote",
"version": "0.0.1", "version": "0.0.2",
"minAppVersion": "0.12.15", "minAppVersion": "0.12.15",
"description": "This is yet another plugin allowing users to sync notes between local device and the cloud.", "description": "This is yet another plugin allowing users to sync notes between local device and the cloud.",
"author": "fyears", "author": "fyears",
+1 -1
View File
@@ -14,7 +14,6 @@
"license": "Apache-2.0", "license": "Apache-2.0",
"devDependencies": { "devDependencies": {
"@types/node": "^14.14.37", "@types/node": "^14.14.37",
"parcel": "^2.0.0",
"prettier": "^2.4.1", "prettier": "^2.4.1",
"terser-webpack-plugin": "^5.2.4", "terser-webpack-plugin": "^5.2.4",
"ts-loader": "^9.2.6", "ts-loader": "^9.2.6",
@@ -30,6 +29,7 @@
"@types/mime-types": "^2.1.1", "@types/mime-types": "^2.1.1",
"aws-crt": "^1.10.1", "aws-crt": "^1.10.1",
"codemirror": "^5.63.1", "codemirror": "^5.63.1",
"lovefield-ts": "^0.7.0",
"mime-types": "^2.1.33", "mime-types": "^2.1.33",
"obsidian": "^0.12.0", "obsidian": "^0.12.0",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
+260
View File
@@ -0,0 +1,260 @@
import * as lf from "lovefield-ts/dist/es6/lf.js";
import { TAbstractFile, TFile, TFolder } from "obsidian";
import type { SUPPORTED_SERVICES_TYPE } from "./misc";
export type DatabaseConnection = lf.DatabaseConnection;
export const DEFAULT_DB_NAME = "saveremotedb";
export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory";
export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
export interface FileFolderHistoryRecord {
key: string;
ctime: number;
mtime: number;
size: number;
action_when: number;
action_type: "delete" | "rename";
key_type: "folder" | "file";
rename_to: string;
}
export interface SyncMetaMappingRecord {
local_key: string;
remote_key: string;
local_size: number;
remote_size: number;
local_mtime: number;
remote_mtime: number;
remote_extra_key: string;
remote_type: SUPPORTED_SERVICES_TYPE;
key_type: "folder" | "file";
}
export const prepareDBs = async () => {
const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1);
schemaBuilder
.createTable(DEFAULT_TBL_DELETE_HISTORY)
.addColumn("id", lf.Type.INTEGER)
.addColumn("key", lf.Type.STRING)
.addColumn("ctime", lf.Type.INTEGER)
.addColumn("mtime", lf.Type.INTEGER)
.addColumn("size", lf.Type.INTEGER)
.addColumn("action_when", lf.Type.INTEGER)
.addColumn("action_type", lf.Type.STRING)
.addColumn("key_type", lf.Type.STRING)
.addPrimaryKey(["id"], true)
.addIndex("idxKey", ["key"]);
schemaBuilder
.createTable(DEFAULT_TBL_SYNC_MAPPING)
.addColumn("id", lf.Type.INTEGER)
.addColumn("local_key", lf.Type.STRING)
.addColumn("remote_key", lf.Type.STRING)
.addColumn("local_size", lf.Type.INTEGER)
.addColumn("remote_size", lf.Type.INTEGER)
.addColumn("local_mtime", lf.Type.INTEGER)
.addColumn("remote_mtime", lf.Type.INTEGER)
.addColumn("key_type", lf.Type.STRING)
.addColumn("remote_extra_key", lf.Type.STRING)
.addColumn("remote_type", lf.Type.STRING)
.addNullable([
"remote_extra_key",
"remote_mtime",
"remote_size",
"local_mtime",
])
.addPrimaryKey(["id"], true)
.addIndex("idxkey", ["local_key", "remote_key"]);
const db = await schemaBuilder.connect({
storeType: lf.DataStoreType.INDEXED_DB,
});
console.log("db connected");
return db;
};
export const destroyDBs = async (db: lf.DatabaseConnection) => {
db.close();
const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME);
req.onsuccess = (event) => {
console.log("db deleted");
};
req.onblocked = (event) => {
console.warn("trying to delete db but it was blocked");
};
req.onerror = (event) => {
console.error("tried to delete db but something bad!");
console.error(event);
};
};
export const loadDeleteRenameHistoryTable = async (
db: lf.DatabaseConnection
) => {
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const records = await db
.select()
.from(schema)
.orderBy(schema.col("action_when"), lf.Order.ASC)
.exec();
return records as FileFolderHistoryRecord[];
};
export const clearDeleteRenameHistoryOfKey = async (
db: lf.DatabaseConnection,
key: string
) => {
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
await db.delete().from(tbl).where(tbl.col("key").eq(key)).exec();
};
export const insertDeleteRecord = async (
db: lf.DatabaseConnection,
fileOrFolder: TAbstractFile
) => {
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
// console.log(fileOrFolder);
let k: FileFolderHistoryRecord;
if (fileOrFolder instanceof TFile) {
k = {
key: fileOrFolder.path,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
action_when: Date.now(),
action_type: "delete",
key_type: "file",
rename_to: "",
};
} else if (fileOrFolder instanceof TFolder) {
// key should endswith "/"
const key = fileOrFolder.path.endsWith("/")
? fileOrFolder.path
: `${fileOrFolder.path}/`;
k = {
key: key,
ctime: 0,
mtime: 0,
size: 0,
action_when: Date.now(),
action_type: "delete",
key_type: "folder",
rename_to: "",
};
}
const row = tbl.createRow(k);
await db.insertOrReplace().into(tbl).values([row]).exec();
};
export const insertRenameRecord = async (
db: lf.DatabaseConnection,
fileOrFolder: TAbstractFile,
oldPath: string
) => {
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
// console.log(fileOrFolder);
let k: FileFolderHistoryRecord;
if (fileOrFolder instanceof TFile) {
k = {
key: oldPath,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
action_when: Date.now(),
action_type: "rename",
key_type: "file",
rename_to: fileOrFolder.path,
};
} else if (fileOrFolder instanceof TFolder) {
const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
const renameTo = fileOrFolder.path.endsWith("/")
? fileOrFolder.path
: `${fileOrFolder.path}/`;
k = {
key: key,
ctime: 0,
mtime: 0,
size: 0,
action_when: Date.now(),
action_type: "rename",
key_type: "folder",
rename_to: renameTo,
};
}
const row = tbl.createRow(k);
await db.insertOrReplace().into(tbl).values([row]).exec();
};
export const getAllDeleteRenameRecords = async (db: lf.DatabaseConnection) => {
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const res1 = await db.select().from(schema).exec();
const res2 = res1 as FileFolderHistoryRecord[];
return res2;
};
export const upsertSyncMetaMappingDataS3 = async (
db: lf.DatabaseConnection,
localKey: string,
localMTime: number,
localSize: number,
remoteKey: string,
remoteMTime: number,
remoteSize: number,
remoteExtraKey: string /* ETag from s3 */
) => {
const schema = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
const aggregratedInfo: SyncMetaMappingRecord = {
local_key: localKey,
local_mtime: localMTime,
local_size: localSize,
remote_key: remoteKey,
remote_mtime: remoteMTime,
remote_size: remoteSize,
remote_extra_key: remoteExtraKey,
remote_type: "s3",
key_type: localKey.endsWith("/") ? "folder" : "file",
};
const row = schema.createRow(aggregratedInfo);
await db.insertOrReplace().into(schema).values([row]).exec();
};
export const getSyncMetaMappingByRemoteKeyS3 = async (
db: lf.DatabaseConnection,
remoteKey: string,
remoteMTime: number,
remoteExtraKey: string
) => {
const schema = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
const tbl = db.getSchema().table(DEFAULT_TBL_SYNC_MAPPING);
const res = (await db
.select()
.from(tbl)
.where(
lf.op.and(
tbl.col("remote_key").eq(remoteKey),
tbl.col("remote_mtime").eq(remoteMTime),
tbl.col("remote_extra_key").eq(remoteExtraKey),
tbl.col("remote_type").eq("s3")
)
)
.exec()) as SyncMetaMappingRecord[];
if (res.length === 1) {
return res[0];
}
if (res.length === 0) {
return undefined;
}
throw Error("something bad in sync meta mapping!");
};
+234
View File
@@ -0,0 +1,234 @@
import {
App,
Modal,
Notice,
Plugin,
PluginSettingTab,
Setting,
request,
Platform,
TFile,
TFolder,
} from "obsidian";
import * as CodeMirror from "codemirror";
import type { DatabaseConnection } from "./localdb";
import {
prepareDBs,
destroyDBs,
loadDeleteRenameHistoryTable,
insertDeleteRecord,
insertRenameRecord,
getAllDeleteRenameRecords,
} from "./localdb";
import type { SyncStatusType } from "./sync";
import { ensembleMixedStates, getOperation, doActualSync } from "./sync";
import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
interface SaveRemotePluginSettings {
s3?: S3Config;
}
const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
s3: DEFAULT_S3_CONFIG,
};
export default class SaveRemotePlugin extends Plugin {
settings: SaveRemotePluginSettings;
cm: CodeMirror.Editor;
db: DatabaseConnection;
syncStatus: SyncStatusType;
async onload() {
console.log("loading plugin obsidian-save-remote");
await this.loadSettings();
await this.prepareDB();
this.syncStatus = "idle";
this.registerEvent(
this.app.vault.on("delete", async (fileOrFolder) => {
await insertDeleteRecord(this.db, fileOrFolder);
})
);
this.registerEvent(
this.app.vault.on("rename", async (fileOrFolder, oldPath) => {
await insertRenameRecord(this.db, fileOrFolder, oldPath);
})
);
// this.addRibbonIcon("dice", "Misc", async () => {
// const a = this.app.vault.getAllLoadedFiles();
// console.log(a);
// const h = await getAllRecords(this.db);
// console.log(h);
// });
this.addRibbonIcon("switch", "Save Remote", async () => {
if (this.syncStatus !== "idle") {
new Notice("Save Remote already running!");
return;
}
new Notice("Save Remote Sync Preparing");
this.syncStatus = "preparing";
const s3Client = getS3Client(this.settings.s3);
const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
const local = this.app.vault.getAllLoadedFiles();
const localHistory = await loadDeleteRenameHistoryTable(this.db);
// console.log(remoteRsp);
// console.log(local);
// console.log(localHistory);
const mixedStates = await ensembleMixedStates(
remoteRsp.Contents,
local,
localHistory,
this.db
);
for (const [key, val] of Object.entries(mixedStates)) {
getOperation(val, true);
}
console.log(mixedStates);
// The operations above are read only and kind of safe.
// The operations below begins to write or delete (!!!) something.
new Notice("Save Remote Sync data exchanging!");
await doActualSync(
s3Client,
this.settings.s3,
this.db,
this.app.vault,
mixedStates
);
new Notice("Save Remote finish!");
this.syncStatus = "idle";
});
this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
this.registerCodeMirror((cm: CodeMirror.Editor) => {
this.cm = cm;
console.log("codemirror registered.");
});
// this.registerDomEvent(document, "click", (evt: MouseEvent) => {
// console.log("click", evt);
// });
// this.registerInterval(
// window.setInterval(() => console.log("setInterval"), 5 * 60 * 1000)
// );
}
onunload() {
console.log("unloading plugin obsidian-save-remote");
this.destroyDBs();
}
async loadSettings() {
this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
}
async saveSettings() {
await this.saveData(this.settings);
}
async prepareDB() {
this.db = await prepareDBs();
}
destroyDBs() {
destroyDBs(this.db);
}
}
class SaveRemoteSettingTab extends PluginSettingTab {
plugin: SaveRemotePlugin;
constructor(app: App, plugin: SaveRemotePlugin) {
super(app, plugin);
this.plugin = plugin;
}
display(): void {
let { containerEl } = this;
containerEl.empty();
containerEl.createEl("h2", { text: "Settings for Save Remote" });
new Setting(containerEl)
.setName("s3Endpoint")
.setDesc("s3Endpoint")
.addText((text) =>
text
.setPlaceholder("")
.setValue(this.plugin.settings.s3.s3Endpoint)
.onChange(async (value) => {
this.plugin.settings.s3.s3Endpoint = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3Region")
.setDesc("s3Region")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3.s3Region}`)
.onChange(async (value) => {
this.plugin.settings.s3.s3Region = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3AccessKeyID")
.setDesc("s3AccessKeyID")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3.s3AccessKeyID}`)
.onChange(async (value) => {
this.plugin.settings.s3.s3AccessKeyID = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3SecretAccessKey")
.setDesc("s3SecretAccessKey")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`)
.onChange(async (value) => {
this.plugin.settings.s3.s3SecretAccessKey = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3BucketName")
.setDesc("s3BucketName")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3.s3BucketName}`)
.onChange(async (value) => {
this.plugin.settings.s3.s3BucketName = value;
await this.plugin.saveSettings();
})
);
}
}
+51
View File
@@ -0,0 +1,51 @@
import { Vault } from "obsidian";
import * as path from "path";
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "ftp";
export const ignoreHiddenFiles = (item: string) => {
const basename = path.basename(item);
return basename === "." || basename[0] !== ".";
};
/**
* Util func for mkdir -p based on the "path" of original file or folder
* "a/b/c/" => ["a", "a/b", "a/b/c"]
* "a/b/c/d/e.txt" => ["a", "a/b", "a/b/c", "a/b/c/d"]
* @param x string
* @returns string[] might be empty
*/
export const getFolderLevels = (x: string) => {
const res: string[] = [];
if (x === "" || x === "/") {
return res;
}
const y1 = x.split("/");
let i = 0;
for (let index = 0; index + 1 < y1.length; index++) {
res.push(y1.slice(0, index + 1).join("/"));
}
return res;
};
export const mkdirpInVault = async (thePath: string, vault: Vault) => {
const foldersToBuild = getFolderLevels(thePath);
for (const folder of foldersToBuild) {
const r = await vault.adapter.exists(folder);
if (!r) {
console.log(`mkdir ${folder}`);
await vault.adapter.mkdir(folder);
}
}
};
/**
* https://stackoverflow.com/questions/8609289
* @param b Buffer
* @returns ArrayBuffer
*/
export const bufferToArrayBuffer = (b: Buffer) => {
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
};
+226
View File
@@ -0,0 +1,226 @@
import { Buffer } from "buffer";
import { Readable } from "stream";
import { Vault } from "obsidian";
import {
S3Client,
ListObjectsV2Command,
PutObjectCommand,
GetObjectCommand,
DeleteObjectCommand,
HeadObjectCommand,
} from "@aws-sdk/client-s3";
import type { _Object } from "@aws-sdk/client-s3";
import { bufferToArrayBuffer, mkdirpInVault } from "./misc";
import * as mime from "mime-types";
export interface S3Config {
s3Endpoint: string;
s3Region: string;
s3AccessKeyID: string;
s3SecretAccessKey: string;
s3BucketName: string;
}
export const DEFAULT_S3_CONFIG = {
s3Endpoint: "",
s3Region: "",
s3AccessKeyID: "",
s3SecretAccessKey: "",
s3BucketName: "",
};
export type S3ObjectType = _Object;
export const getS3Client = (s3Config: S3Config) => {
const s3Client = new S3Client({
region: s3Config.s3Region,
endpoint: s3Config.s3Endpoint,
credentials: {
accessKeyId: s3Config.s3AccessKeyID,
secretAccessKey: s3Config.s3SecretAccessKey,
},
});
return s3Client;
};
export const getRemoteMeta = async (
s3Client: S3Client,
s3Config: S3Config,
fileOrFolderPath: string
) => {
return await s3Client.send(
new HeadObjectCommand({
Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath,
})
);
};
export const uploadToRemote = async (
s3Client: S3Client,
s3Config: S3Config,
fileOrFolderPath: string,
vault: Vault,
isRecursively: boolean = false
) => {
const isFolder = fileOrFolderPath.endsWith("/");
const DEFAULT_CONTENT_TYPE = "application/octet-stream";
if (isFolder && isRecursively) {
throw Error("upload function doesn't implement recursive function yet!");
} else if (isFolder && !isRecursively) {
// folder
const contentType = DEFAULT_CONTENT_TYPE;
await s3Client.send(
new PutObjectCommand({
Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath,
Body: "",
ContentType: contentType,
})
);
return await getRemoteMeta(s3Client, s3Config, fileOrFolderPath);
} else {
// file
// we ignore isRecursively parameter here
const contentType =
mime.contentType(mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE) ||
DEFAULT_CONTENT_TYPE;
const content = await vault.adapter.readBinary(fileOrFolderPath);
const body = Buffer.from(content);
await s3Client.send(
new PutObjectCommand({
Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath,
Body: body,
ContentType: contentType,
})
);
return await getRemoteMeta(s3Client, s3Config, fileOrFolderPath);
}
};
export const listFromRemote = async (
s3Client: S3Client,
s3Config: S3Config,
prefix?: string
) => {
if (prefix !== undefined) {
return await s3Client.send(
new ListObjectsV2Command({
Bucket: s3Config.s3BucketName,
Prefix: prefix,
})
);
}
return await s3Client.send(
new ListObjectsV2Command({ Bucket: s3Config.s3BucketName })
);
};
/**
* The Body of resp of aws GetObject has mix types
* and we want to get ArrayBuffer here.
* See https://github.com/aws/aws-sdk-js-v3/issues/1877
* @param b The Body of GetObject
* @returns Promise<ArrayBuffer>
*/
const getObjectBodyToArrayBuffer = async (
b: Readable | ReadableStream | Blob
) => {
if (b instanceof Readable) {
const chunks: Uint8Array[] = [];
for await (let chunk of b) {
chunks.push(chunk);
}
const buf = Buffer.concat(chunks);
return bufferToArrayBuffer(buf);
} else if (b instanceof ReadableStream) {
return await new Response(b, {}).arrayBuffer();
} else if (b instanceof Blob) {
return await b.arrayBuffer();
} else {
throw TypeError(`The type of ${b} is not one of the supported types`);
}
};
export const downloadFromRemoteRaw = async (
s3Client: S3Client,
s3Config: S3Config,
fileOrFolderPath: string
) => {
const data = await s3Client.send(
new GetObjectCommand({
Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath,
})
);
const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
return bodyContents;
};
export const downloadFromRemote = async (
s3Client: S3Client,
s3Config: S3Config,
fileOrFolderPath: string,
vault: Vault,
mtime: number
) => {
const isFolder = fileOrFolderPath.endsWith("/");
await mkdirpInVault(fileOrFolderPath, vault);
if (isFolder) {
// mkdirp locally is enough
// do nothing here
} else {
const content = await downloadFromRemoteRaw(
s3Client,
s3Config,
fileOrFolderPath
);
await vault.adapter.writeBinary(fileOrFolderPath, content, {
mtime: mtime,
});
}
};
/**
* This function deals with file normally and "folder" recursively.
* @param s3Client
* @param s3Config
* @param fileOrFolderPath
* @returns
*/
export const deleteFromRemote = async (
s3Client: S3Client,
s3Config: S3Config,
fileOrFolderPath: string
) => {
if (fileOrFolderPath === "/") {
return;
}
if (fileOrFolderPath.endsWith("/")) {
const x = await listFromRemote(s3Client, s3Config, fileOrFolderPath);
x.Contents.forEach(async (element) => {
await s3Client.send(
new DeleteObjectCommand({
Bucket: s3Config.s3BucketName,
Key: element.Key,
})
);
});
} else {
await s3Client.send(
new DeleteObjectCommand({
Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath,
})
);
}
};
+360
View File
@@ -0,0 +1,360 @@
import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
import { S3Client } from "@aws-sdk/client-s3";
import * as lf from "lovefield-ts/dist/es6/lf.js";
import {
clearDeleteRenameHistoryOfKey,
FileFolderHistoryRecord,
upsertSyncMetaMappingDataS3,
getSyncMetaMappingByRemoteKeyS3,
} from "./localdb";
import {
S3Config,
S3ObjectType,
uploadToRemote,
deleteFromRemote,
downloadFromRemote,
} from "./s3";
import { mkdirpInVault } from "./misc";
type DecisionType =
| "undecided"
| "unknown"
| "upload_clearhist"
| "download_clearhist"
| "delremote_clearhist"
| "download"
| "upload"
| "clearhist"
| "mkdirplocal"
| "skip";
export type SyncStatusType = "idle" | "preparing" | "syncing";
interface FileOrFolderMixedState {
key: string;
exist_local?: boolean;
exist_remote?: boolean;
mtime_local?: number;
mtime_remote?: number;
delete_time_local?: number;
size_local?: number;
size_remote?: number;
decision?: DecisionType;
syncDone?: "done";
decision_branch?: number;
}
export const ensembleMixedStates = async (
remote: S3ObjectType[],
local: TAbstractFile[],
deleteHistory: FileFolderHistoryRecord[],
db: lf.DatabaseConnection
) => {
const results = {} as Record<string, FileOrFolderMixedState>;
if (remote !== undefined) {
for (const entry of remote) {
const backwardMapping = await getSyncMetaMappingByRemoteKeyS3(
db,
entry.Key,
entry.LastModified.valueOf(),
entry.ETag
);
let key = entry.Key;
let r = {} as FileOrFolderMixedState;
if (backwardMapping !== undefined) {
key = backwardMapping.local_key;
r = {
key: key,
exist_remote: true,
mtime_remote: backwardMapping.local_mtime,
size_remote: backwardMapping.local_size,
};
} else {
r = {
key: key,
exist_remote: true,
mtime_remote: entry.LastModified.valueOf(),
size_remote: entry.Size,
};
}
if (results.hasOwnProperty(key)) {
results[key].key = r.key;
results[key].exist_remote = r.exist_remote;
results[key].mtime_remote = r.mtime_remote;
results[key].size_remote = r.size_remote;
} else {
results[key] = r;
}
}
}
for (const entry of local) {
let r = {} as FileOrFolderMixedState;
let key = entry.path;
if (entry.path === "/") {
// ignore
continue;
} else if (entry instanceof TFile) {
r = {
key: entry.path,
exist_local: true,
mtime_local: entry.stat.mtime,
size_local: entry.stat.size,
};
} else if (entry instanceof TFolder) {
key = `${entry.path}/`;
r = {
key: key,
exist_local: true,
mtime_local: undefined,
size_local: 0,
};
} else {
throw Error(`unexpected ${entry}`);
}
if (results.hasOwnProperty(key)) {
results[key].key = r.key;
results[key].exist_local = r.exist_local;
results[key].mtime_local = r.mtime_local;
results[key].size_local = r.size_local;
} else {
results[key] = r;
}
}
for (const entry of deleteHistory) {
let key = entry.key;
if (entry.key_type === "folder") {
if (!entry.key.endsWith("/")) {
key = `${entry.key}/`;
}
} else if (entry.key_type === "file") {
// pass
} else {
throw Error(`unexpected ${entry}`);
}
const r = {
key: key,
delete_time_local: entry.action_when,
} as FileOrFolderMixedState;
if (results.hasOwnProperty(key)) {
results[key].key = r.key;
results[key].delete_time_local = r.delete_time_local;
} else {
results[key] = r;
}
}
return results;
};
export const getOperation = (
origRecord: FileOrFolderMixedState,
inplace: boolean = false
) => {
let r = origRecord;
if (!inplace) {
r = Object.assign({}, origRecord);
}
if (r.mtime_local === 0) {
r.mtime_local = undefined;
}
if (r.mtime_remote === 0) {
r.mtime_remote = undefined;
}
if (r.delete_time_local === 0) {
r.delete_time_local = undefined;
}
if (r.exist_local === undefined) {
r.exist_local = false;
}
if (r.exist_remote === undefined) {
r.exist_remote = false;
}
r.decision = "unknown";
if (
r.exist_remote &&
r.exist_local &&
r.mtime_remote !== undefined &&
r.mtime_local !== undefined &&
r.mtime_remote > r.mtime_local
) {
r.decision = "download_clearhist";
r.decision_branch = 1;
} else if (
r.exist_remote &&
r.exist_local &&
r.mtime_remote !== undefined &&
r.mtime_local !== undefined &&
r.mtime_remote < r.mtime_local
) {
r.decision = "upload_clearhist";
r.decision_branch = 2;
} else if (
r.exist_remote &&
r.exist_local &&
r.mtime_remote !== undefined &&
r.mtime_local !== undefined &&
r.mtime_remote === r.mtime_local &&
r.size_local === r.size_remote
) {
r.decision = "skip";
r.decision_branch = 3;
} else if (
r.exist_remote &&
r.exist_local &&
r.mtime_remote !== undefined &&
r.mtime_local !== undefined &&
r.mtime_remote === r.mtime_local &&
r.size_local !== r.size_remote
) {
r.decision = "upload_clearhist";
r.decision_branch = 4;
} else if (r.exist_remote && r.exist_local && r.mtime_local === undefined) {
// this must be a folder!
if (!r.key.endsWith("/")) {
throw Error(`${r.key} is not a folder but lacks local mtime`);
}
r.decision = "skip";
r.decision_branch = 5;
} else if (
r.exist_remote &&
!r.exist_local &&
r.mtime_remote !== undefined &&
r.mtime_local === undefined &&
r.delete_time_local !== undefined &&
r.mtime_remote >= r.delete_time_local
) {
r.decision = "download_clearhist";
r.decision_branch = 6;
} else if (
r.exist_remote &&
!r.exist_local &&
r.mtime_remote !== undefined &&
r.mtime_local === undefined &&
r.delete_time_local !== undefined &&
r.mtime_remote < r.delete_time_local
) {
r.decision = "delremote_clearhist";
r.decision_branch = 7;
} else if (
r.exist_remote &&
!r.exist_local &&
r.mtime_remote !== undefined &&
r.mtime_local === undefined &&
r.delete_time_local == undefined
) {
r.decision = "download";
r.decision_branch = 8;
} else if (!r.exist_remote && r.exist_local && r.mtime_remote === undefined) {
r.decision = "upload_clearhist";
r.decision_branch = 9;
} else if (
!r.exist_remote &&
!r.exist_local &&
r.mtime_remote === undefined &&
r.mtime_local === undefined
) {
r.decision = "clearhist";
r.decision_branch = 10;
}
return r;
};
export const doActualSync = async (
s3Client: S3Client,
s3Config: S3Config,
db: lf.DatabaseConnection,
vault: Vault,
keyStates: Record<string, FileOrFolderMixedState>
) => {
Object.entries(keyStates)
.sort((k, v) => -(k as string).length)
.map(async ([k, v]) => {
const key = k as string;
const state = v as FileOrFolderMixedState;
if (
state.decision === undefined ||
state.decision === "unknown" ||
state.decision === "undecided"
) {
throw Error(`unknown decision in ${JSON.stringify(state)}`);
} else if (state.decision === "skip") {
// do nothing
} else if (state.decision === "download_clearhist") {
await downloadFromRemote(
s3Client,
s3Config,
state.key,
vault,
state.mtime_remote
);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "upload_clearhist") {
const remoteObjMeta = await uploadToRemote(
s3Client,
s3Config,
state.key,
vault,
false
);
await upsertSyncMetaMappingDataS3(
db,
state.key,
state.mtime_local,
state.size_local,
state.key,
remoteObjMeta.LastModified.valueOf(),
remoteObjMeta.ContentLength,
remoteObjMeta.ETag
);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "download") {
await mkdirpInVault(state.key, vault);
await downloadFromRemote(
s3Client,
s3Config,
state.key,
vault,
state.mtime_remote
);
} else if (state.decision === "delremote_clearhist") {
await deleteFromRemote(s3Client, s3Config, state.key);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "upload") {
const remoteObjMeta = await uploadToRemote(
s3Client,
s3Config,
state.key,
vault,
false
);
await upsertSyncMetaMappingDataS3(
db,
state.key,
state.mtime_local,
state.size_local,
state.key,
remoteObjMeta.LastModified.valueOf(),
remoteObjMeta.ContentLength,
remoteObjMeta.ETag
);
} else if (state.decision === "clearhist") {
await clearDeleteRenameHistoryOfKey(db, state.key);
} else {
throw Error("this should never happen!");
}
});
};
+1 -1
View File
@@ -1,3 +1,3 @@
{ {
"0.0.1": "0.12.15" "0.0.2": "0.12.15"
} }
+1 -1
View File
@@ -2,7 +2,7 @@ const path = require("path");
const TerserPlugin = require("terser-webpack-plugin"); const TerserPlugin = require("terser-webpack-plugin");
module.exports = { module.exports = {
entry: "./main.ts", entry: "./src/main.ts",
target: "web", target: "web",
output: { output: {
filename: "main.js", filename: "main.js",