basically working 2 way sync!
This commit is contained in:
fyears
parent
13e5af0c34
commit
a26158055d
118
src/localdb.ts
118
src/localdb.ts
@ -1,4 +1,5 @@
|
||||
import * as lf from "lovefield-ts/dist/es6/lf.js";
|
||||
import { TAbstractFile, TFile, TFolder } from "obsidian";
|
||||
|
||||
export type DatabaseConnection = lf.DatabaseConnection;
|
||||
|
||||
@ -16,7 +17,7 @@ export interface FileFolderHistoryRecord {
|
||||
rename_to: string;
|
||||
}
|
||||
|
||||
export async function prepareDB() {
|
||||
export const prepareDB = async () => {
|
||||
const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1);
|
||||
schemaBuilder
|
||||
.createTable(DEFAULT_TBL_DELETE_HISTORY)
|
||||
@ -35,9 +36,9 @@ export async function prepareDB() {
|
||||
});
|
||||
console.log("db connected");
|
||||
return db;
|
||||
}
|
||||
};
|
||||
|
||||
export function destroyDB(db: lf.DatabaseConnection) {
|
||||
export const destroyDB = async (db: lf.DatabaseConnection) => {
|
||||
db.close();
|
||||
const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME);
|
||||
req.onsuccess = (event) => {
|
||||
@ -50,4 +51,113 @@ export function destroyDB(db: lf.DatabaseConnection) {
|
||||
console.error("tried to delete db but something bad!");
|
||||
console.error(event);
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const loadHistoryTable = async (db: lf.DatabaseConnection) => {
|
||||
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
|
||||
const records = await db
|
||||
.select()
|
||||
.from(schema)
|
||||
.orderBy(schema.col("action_when"), lf.Order.ASC)
|
||||
.exec();
|
||||
|
||||
return records as FileFolderHistoryRecord[];
|
||||
};
|
||||
|
||||
export const clearHistoryOfKey = async (
|
||||
db: lf.DatabaseConnection,
|
||||
key: string
|
||||
) => {
|
||||
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
|
||||
await db.delete().from(tbl).where(tbl.col("key").eq(key)).exec();
|
||||
};
|
||||
|
||||
export const insertDeleteRecord = async (
|
||||
db: lf.DatabaseConnection,
|
||||
fileOrFolder: TAbstractFile
|
||||
) => {
|
||||
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
// console.log(fileOrFolder);
|
||||
let k: FileFolderHistoryRecord;
|
||||
if (fileOrFolder instanceof TFile) {
|
||||
k = {
|
||||
key: fileOrFolder.path,
|
||||
ctime: fileOrFolder.stat.ctime,
|
||||
mtime: fileOrFolder.stat.mtime,
|
||||
size: fileOrFolder.stat.size,
|
||||
action_when: Date.now(),
|
||||
action_type: "delete",
|
||||
key_type: "file",
|
||||
rename_to: "",
|
||||
};
|
||||
} else if (fileOrFolder instanceof TFolder) {
|
||||
// key should endswith "/"
|
||||
const key = fileOrFolder.path.endsWith("/")
|
||||
? fileOrFolder.path
|
||||
: `${fileOrFolder.path}/`;
|
||||
k = {
|
||||
key: key,
|
||||
ctime: 0,
|
||||
mtime: 0,
|
||||
size: 0,
|
||||
action_when: Date.now(),
|
||||
action_type: "delete",
|
||||
key_type: "folder",
|
||||
rename_to: "",
|
||||
};
|
||||
}
|
||||
const row = tbl.createRow(k);
|
||||
await db.insertOrReplace().into(tbl).values([row]).exec();
|
||||
};
|
||||
|
||||
export const insertRenameRecord = async (
|
||||
db: lf.DatabaseConnection,
|
||||
fileOrFolder: TAbstractFile,
|
||||
oldPath: string
|
||||
) => {
|
||||
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
// console.log(fileOrFolder);
|
||||
let k: FileFolderHistoryRecord;
|
||||
if (fileOrFolder instanceof TFile) {
|
||||
k = {
|
||||
key: oldPath,
|
||||
ctime: fileOrFolder.stat.ctime,
|
||||
mtime: fileOrFolder.stat.mtime,
|
||||
size: fileOrFolder.stat.size,
|
||||
action_when: Date.now(),
|
||||
action_type: "rename",
|
||||
key_type: "file",
|
||||
rename_to: fileOrFolder.path,
|
||||
};
|
||||
} else if (fileOrFolder instanceof TFolder) {
|
||||
const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`;
|
||||
const renameTo = fileOrFolder.path.endsWith("/")
|
||||
? fileOrFolder.path
|
||||
: `${fileOrFolder.path}/`;
|
||||
k = {
|
||||
key: key,
|
||||
ctime: 0,
|
||||
mtime: 0,
|
||||
size: 0,
|
||||
action_when: Date.now(),
|
||||
action_type: "rename",
|
||||
key_type: "folder",
|
||||
rename_to: renameTo,
|
||||
};
|
||||
}
|
||||
const row = tbl.createRow(k);
|
||||
await db.insertOrReplace().into(tbl).values([row]).exec();
|
||||
};
|
||||
|
||||
export const getAllRecords = async (db: lf.DatabaseConnection) => {
|
||||
const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
const res1 = await db.select().from(schema).exec();
|
||||
const res2 = res1 as FileFolderHistoryRecord[];
|
||||
return res2;
|
||||
};
|
||||
|
||||
290
src/main.ts
290
src/main.ts
@ -1,6 +1,3 @@
|
||||
import { Buffer } from "buffer";
|
||||
import { Readable } from "stream";
|
||||
import * as mime from "mime-types";
|
||||
import {
|
||||
App,
|
||||
Modal,
|
||||
@ -14,47 +11,33 @@ import {
|
||||
TFolder,
|
||||
} from "obsidian";
|
||||
import * as CodeMirror from "codemirror";
|
||||
import type { FileFolderHistoryRecord, DatabaseConnection } from "./localdb";
|
||||
import type { DatabaseConnection } from "./localdb";
|
||||
import {
|
||||
prepareDB,
|
||||
destroyDB,
|
||||
DEFAULT_DB_NAME,
|
||||
DEFAULT_TBL_DELETE_HISTORY,
|
||||
loadHistoryTable,
|
||||
insertDeleteRecord,
|
||||
insertRenameRecord,
|
||||
getAllRecords,
|
||||
} from "./localdb";
|
||||
|
||||
import {
|
||||
getFolderLevels,
|
||||
bufferToArrayBuffer,
|
||||
getObjectBodyToArrayBuffer,
|
||||
} from "./misc";
|
||||
|
||||
import {
|
||||
S3Client,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import type { SyncStatusType } from "./sync";
|
||||
import { ensembleMixedStates, getOperation, doActualSync } from "./sync";
|
||||
import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
|
||||
|
||||
interface SaveRemotePluginSettings {
|
||||
s3Endpoint: string;
|
||||
s3Region: string;
|
||||
s3AccessKeyID: string;
|
||||
s3SecretAccessKey: string;
|
||||
s3BucketName: string;
|
||||
s3?: S3Config;
|
||||
}
|
||||
|
||||
const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
|
||||
s3Endpoint: "",
|
||||
s3Region: "",
|
||||
s3AccessKeyID: "",
|
||||
s3SecretAccessKey: "",
|
||||
s3BucketName: "",
|
||||
s3: DEFAULT_S3_CONFIG,
|
||||
};
|
||||
|
||||
export default class SaveRemotePlugin extends Plugin {
|
||||
settings: SaveRemotePluginSettings;
|
||||
cm: CodeMirror.Editor;
|
||||
db: DatabaseConnection;
|
||||
syncStatus: SyncStatusType;
|
||||
|
||||
async onload() {
|
||||
console.log("loading plugin obsidian-save-remote");
|
||||
@ -63,209 +46,70 @@ export default class SaveRemotePlugin extends Plugin {
|
||||
|
||||
await this.prepareDB();
|
||||
|
||||
this.syncStatus = "idle";
|
||||
|
||||
this.registerEvent(
|
||||
this.app.vault.on("delete", async (fileOrFolder) => {
|
||||
const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
const tbl = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
// console.log(fileOrFolder);
|
||||
let k: FileFolderHistoryRecord;
|
||||
if (fileOrFolder instanceof TFile) {
|
||||
k = {
|
||||
key: fileOrFolder.path,
|
||||
ctime: fileOrFolder.stat.ctime,
|
||||
mtime: fileOrFolder.stat.mtime,
|
||||
size: fileOrFolder.stat.size,
|
||||
action_when: Date.now(),
|
||||
action_type: "delete",
|
||||
key_type: "file",
|
||||
rename_to: "",
|
||||
};
|
||||
} else if (fileOrFolder instanceof TFolder) {
|
||||
k = {
|
||||
key: fileOrFolder.path,
|
||||
ctime: 0,
|
||||
mtime: 0,
|
||||
size: 0,
|
||||
action_when: Date.now(),
|
||||
action_type: "delete",
|
||||
key_type: "folder",
|
||||
rename_to: "",
|
||||
};
|
||||
}
|
||||
const row = tbl.createRow(k);
|
||||
await this.db.insertOrReplace().into(tbl).values([row]).exec();
|
||||
await insertDeleteRecord(this.db, fileOrFolder);
|
||||
})
|
||||
);
|
||||
|
||||
this.registerEvent(
|
||||
this.app.vault.on("rename", async (fileOrFolder, oldPath) => {
|
||||
const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
const tbl = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
// console.log(fileOrFolder);
|
||||
let k: FileFolderHistoryRecord;
|
||||
if (fileOrFolder instanceof TFile) {
|
||||
k = {
|
||||
key: oldPath,
|
||||
ctime: fileOrFolder.stat.ctime,
|
||||
mtime: fileOrFolder.stat.mtime,
|
||||
size: fileOrFolder.stat.size,
|
||||
action_when: Date.now(),
|
||||
action_type: "rename",
|
||||
key_type: "file",
|
||||
rename_to: fileOrFolder.path,
|
||||
};
|
||||
} else if (fileOrFolder instanceof TFolder) {
|
||||
k = {
|
||||
key: oldPath,
|
||||
ctime: 0,
|
||||
mtime: 0,
|
||||
size: 0,
|
||||
action_when: Date.now(),
|
||||
action_type: "rename",
|
||||
key_type: "folder",
|
||||
rename_to: fileOrFolder.path,
|
||||
};
|
||||
}
|
||||
const row = tbl.createRow(k);
|
||||
await this.db.insertOrReplace().into(tbl).values([row]).exec();
|
||||
await insertRenameRecord(this.db, fileOrFolder, oldPath);
|
||||
})
|
||||
);
|
||||
|
||||
this.addRibbonIcon("dice", "Misc", async () => {
|
||||
const a = this.app.vault.getAllLoadedFiles();
|
||||
console.log(a);
|
||||
// this.addRibbonIcon("dice", "Misc", async () => {
|
||||
// const a = this.app.vault.getAllLoadedFiles();
|
||||
// console.log(a);
|
||||
// const h = await getAllRecords(this.db);
|
||||
// console.log(h);
|
||||
// });
|
||||
|
||||
const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
|
||||
|
||||
const h = await this.db.select().from(schema).exec();
|
||||
|
||||
console.log(h);
|
||||
|
||||
// console.log(b)
|
||||
});
|
||||
|
||||
this.addRibbonIcon("right-arrow-with-tail", "Upload", async () => {
|
||||
// console.log(this.app.vault.getFiles());
|
||||
// console.log(this.app.vault.getAllLoadedFiles());
|
||||
new Notice(`Upload begun.`);
|
||||
const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
|
||||
|
||||
const s3Client = new S3Client({
|
||||
region: this.settings.s3Region,
|
||||
endpoint: this.settings.s3Endpoint,
|
||||
credentials: {
|
||||
accessKeyId: this.settings.s3AccessKeyID,
|
||||
secretAccessKey: this.settings.s3SecretAccessKey,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
for (const fileOrFolder of allFilesAndFolders) {
|
||||
if (fileOrFolder.path === "/") {
|
||||
console.log('ignore "/"');
|
||||
} else if ("children" in fileOrFolder) {
|
||||
// folder
|
||||
console.log(`folder ${fileOrFolder.path}/`);
|
||||
new Notice(`folder ${fileOrFolder.path}/`);
|
||||
|
||||
const results = await s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.settings.s3BucketName,
|
||||
Key: `${fileOrFolder.path}/`,
|
||||
Body: "",
|
||||
})
|
||||
);
|
||||
} else {
|
||||
// file
|
||||
console.log(`file ${fileOrFolder.path}`);
|
||||
const arrContent = await this.app.vault.adapter.readBinary(
|
||||
fileOrFolder.path
|
||||
);
|
||||
new Notice(`file ${fileOrFolder.path}`);
|
||||
const contentType =
|
||||
mime.contentType(
|
||||
mime.lookup(`${fileOrFolder.path}`) || undefined
|
||||
) || undefined;
|
||||
// console.log(contentType);
|
||||
const results = await s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.settings.s3BucketName,
|
||||
Key: `${fileOrFolder.path}`,
|
||||
Body: Buffer.from(arrContent),
|
||||
ContentType: contentType,
|
||||
})
|
||||
);
|
||||
this.addRibbonIcon("switch", "Save Remote", async () => {
|
||||
if (this.syncStatus !== "idle") {
|
||||
new Notice("Save Remote already running!");
|
||||
return;
|
||||
}
|
||||
}
|
||||
new Notice("Upload finished!");
|
||||
} catch (err) {
|
||||
console.log("Error", err);
|
||||
new Notice(`${err}`);
|
||||
}
|
||||
});
|
||||
|
||||
this.addRibbonIcon("left-arrow-with-tail", "Download", async () => {
|
||||
const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
|
||||
new Notice("Save Remote Sync Preparing");
|
||||
this.syncStatus = "preparing";
|
||||
const s3Client = getS3Client(this.settings.s3);
|
||||
const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
|
||||
const local = this.app.vault.getAllLoadedFiles();
|
||||
const localHistory = await loadHistoryTable(this.db);
|
||||
// console.log(remoteRsp);
|
||||
// console.log(local);
|
||||
// console.log(localHistory);
|
||||
|
||||
const s3Client = new S3Client({
|
||||
region: this.settings.s3Region,
|
||||
endpoint: this.settings.s3Endpoint,
|
||||
credentials: {
|
||||
accessKeyId: this.settings.s3AccessKeyID,
|
||||
secretAccessKey: this.settings.s3SecretAccessKey,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const listObj = await s3Client.send(
|
||||
new ListObjectsV2Command({ Bucket: this.settings.s3BucketName })
|
||||
const mixedStates = ensembleMixedStates(
|
||||
remoteRsp.Contents,
|
||||
local,
|
||||
localHistory
|
||||
);
|
||||
|
||||
for (const singleContent of listObj.Contents) {
|
||||
const mtimeSec = Math.round(
|
||||
singleContent.LastModified.valueOf() / 1000.0
|
||||
for (const [key, val] of Object.entries(mixedStates)) {
|
||||
getOperation(val, true);
|
||||
}
|
||||
|
||||
console.log(mixedStates);
|
||||
|
||||
// The operations above are read only and kind of safe.
|
||||
// The operations below begins to write or delete (!!!) something.
|
||||
|
||||
new Notice("Save Remote Sync data exchanging!");
|
||||
|
||||
doActualSync(
|
||||
s3Client,
|
||||
this.settings.s3,
|
||||
this.db,
|
||||
this.app.vault,
|
||||
mixedStates
|
||||
);
|
||||
console.log(`key ${singleContent.Key} mtime ${mtimeSec}`);
|
||||
|
||||
const foldersToBuild = getFolderLevels(singleContent.Key);
|
||||
for (const folder of foldersToBuild) {
|
||||
const r = await this.app.vault.adapter.exists(folder);
|
||||
if (!r) {
|
||||
console.log(`mkdir ${folder}`);
|
||||
new Notice(`mkdir ${folder}`);
|
||||
await this.app.vault.adapter.mkdir(folder);
|
||||
}
|
||||
}
|
||||
|
||||
if (singleContent.Key.endsWith("/")) {
|
||||
// kind of a folder
|
||||
// pass
|
||||
} else {
|
||||
// kind of a file
|
||||
// download
|
||||
|
||||
console.log(`download file ${singleContent.Key}`);
|
||||
new Notice(`download file ${singleContent.Key}`);
|
||||
|
||||
const data = await s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.settings.s3BucketName,
|
||||
Key: singleContent.Key,
|
||||
})
|
||||
);
|
||||
const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
|
||||
await this.app.vault.adapter.writeBinary(
|
||||
singleContent.Key,
|
||||
bodyContents
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
new Notice("Download finished!");
|
||||
} catch (err) {
|
||||
console.log("Error", err);
|
||||
new Notice(`${err}`);
|
||||
}
|
||||
new Notice("Save Remote finish!");
|
||||
this.syncStatus = "idle";
|
||||
});
|
||||
|
||||
this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
|
||||
@ -327,9 +171,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(this.plugin.settings.s3Endpoint)
|
||||
.setValue(this.plugin.settings.s3.s3Endpoint)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.s3Endpoint = value;
|
||||
this.plugin.settings.s3.s3Endpoint = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
@ -340,9 +184,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(`${this.plugin.settings.s3Region}`)
|
||||
.setValue(`${this.plugin.settings.s3.s3Region}`)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.s3Region = value;
|
||||
this.plugin.settings.s3.s3Region = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
@ -353,9 +197,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(`${this.plugin.settings.s3AccessKeyID}`)
|
||||
.setValue(`${this.plugin.settings.s3.s3AccessKeyID}`)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.s3AccessKeyID = value;
|
||||
this.plugin.settings.s3.s3AccessKeyID = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
@ -366,9 +210,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(`${this.plugin.settings.s3SecretAccessKey}`)
|
||||
.setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.s3SecretAccessKey = value;
|
||||
this.plugin.settings.s3.s3SecretAccessKey = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
@ -379,9 +223,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(`${this.plugin.settings.s3BucketName}`)
|
||||
.setValue(`${this.plugin.settings.s3.s3BucketName}`)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.s3BucketName = value;
|
||||
this.plugin.settings.s3.s3BucketName = value;
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
41
src/misc.ts
41
src/misc.ts
@ -1,7 +1,5 @@
|
||||
import { Vault } from "obsidian";
|
||||
import * as path from "path";
|
||||
import * as fs from "fs";
|
||||
import { Buffer } from "buffer";
|
||||
import { Readable } from "stream";
|
||||
|
||||
export const ignoreHiddenFiles = (item: string) => {
|
||||
const basename = path.basename(item);
|
||||
@ -30,6 +28,17 @@ export const getFolderLevels = (x: string) => {
|
||||
return res;
|
||||
};
|
||||
|
||||
export const mkdirpInVault = async (thePath: string, vault: Vault) => {
|
||||
const foldersToBuild = getFolderLevels(thePath);
|
||||
for (const folder of foldersToBuild) {
|
||||
const r = await vault.adapter.exists(folder);
|
||||
if (!r) {
|
||||
console.log(`mkdir ${folder}`);
|
||||
await vault.adapter.mkdir(folder);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* https://stackoverflow.com/questions/8609289
|
||||
* @param b Buffer
|
||||
@ -38,29 +47,3 @@ export const getFolderLevels = (x: string) => {
|
||||
export const bufferToArrayBuffer = (b: Buffer) => {
|
||||
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
|
||||
};
|
||||
|
||||
/**
|
||||
* The Body of resp of aws GetObject has mix types
|
||||
* and we want to get ArrayBuffer here.
|
||||
* See https://github.com/aws/aws-sdk-js-v3/issues/1877
|
||||
* @param b The Body of GetObject
|
||||
* @returns Promise<ArrayBuffer>
|
||||
*/
|
||||
export const getObjectBodyToArrayBuffer = async (
|
||||
b: Readable | ReadableStream | Blob
|
||||
) => {
|
||||
if (b instanceof Readable) {
|
||||
const chunks: Uint8Array[] = [];
|
||||
for await (let chunk of b) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const buf = Buffer.concat(chunks);
|
||||
return bufferToArrayBuffer(buf);
|
||||
} else if (b instanceof ReadableStream) {
|
||||
return await new Response(b, {}).arrayBuffer();
|
||||
} else if (b instanceof Blob) {
|
||||
return await b.arrayBuffer();
|
||||
} else {
|
||||
throw TypeError(`The type of ${b} is not one of the supported types`);
|
||||
}
|
||||
};
|
||||
|
||||
210
src/s3.ts
Normal file
210
src/s3.ts
Normal file
@ -0,0 +1,210 @@
|
||||
import { Buffer } from "buffer";
|
||||
import { Readable } from "stream";
|
||||
|
||||
import { Vault } from "obsidian";
|
||||
|
||||
import {
|
||||
S3Client,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
|
||||
import type { _Object } from "@aws-sdk/client-s3";
|
||||
|
||||
import { bufferToArrayBuffer, mkdirpInVault } from "./misc";
|
||||
import * as mime from "mime-types";
|
||||
|
||||
export interface S3Config {
|
||||
s3Endpoint: string;
|
||||
s3Region: string;
|
||||
s3AccessKeyID: string;
|
||||
s3SecretAccessKey: string;
|
||||
s3BucketName: string;
|
||||
}
|
||||
|
||||
export const DEFAULT_S3_CONFIG = {
|
||||
s3Endpoint: "",
|
||||
s3Region: "",
|
||||
s3AccessKeyID: "",
|
||||
s3SecretAccessKey: "",
|
||||
s3BucketName: "",
|
||||
};
|
||||
|
||||
export type S3ObjectType = _Object;
|
||||
|
||||
export const getS3Client = (s3Config: S3Config) => {
|
||||
const s3Client = new S3Client({
|
||||
region: s3Config.s3Region,
|
||||
endpoint: s3Config.s3Endpoint,
|
||||
credentials: {
|
||||
accessKeyId: s3Config.s3AccessKeyID,
|
||||
secretAccessKey: s3Config.s3SecretAccessKey,
|
||||
},
|
||||
});
|
||||
return s3Client;
|
||||
};
|
||||
|
||||
export const uploadToRemote = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config,
|
||||
fileOrFolderPath: string,
|
||||
vault: Vault,
|
||||
isRecursively: boolean = false
|
||||
) => {
|
||||
const isFolder = fileOrFolderPath.endsWith("/");
|
||||
|
||||
const DEFAULT_CONTENT_TYPE = "application/octet-stream";
|
||||
|
||||
if (isFolder && isRecursively) {
|
||||
throw Error("upload function doesn't implement recursive function yet!");
|
||||
} else if (isFolder && !isRecursively) {
|
||||
// folder
|
||||
const contentType = DEFAULT_CONTENT_TYPE;
|
||||
return await s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: s3Config.s3BucketName,
|
||||
Key: fileOrFolderPath,
|
||||
Body: "",
|
||||
ContentType: contentType,
|
||||
})
|
||||
);
|
||||
} else {
|
||||
// file
|
||||
// we ignore isRecursively parameter here
|
||||
const contentType =
|
||||
mime.contentType(mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE) ||
|
||||
DEFAULT_CONTENT_TYPE;
|
||||
const content = await vault.adapter.readBinary(fileOrFolderPath);
|
||||
const body = Buffer.from(content);
|
||||
return await s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: s3Config.s3BucketName,
|
||||
Key: fileOrFolderPath,
|
||||
Body: body,
|
||||
ContentType: contentType,
|
||||
})
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const listFromRemote = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config,
|
||||
prefix?: string
|
||||
) => {
|
||||
if (prefix !== undefined) {
|
||||
return await s3Client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: s3Config.s3BucketName,
|
||||
Prefix: prefix,
|
||||
})
|
||||
);
|
||||
}
|
||||
return await s3Client.send(
|
||||
new ListObjectsV2Command({ Bucket: s3Config.s3BucketName })
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* The Body of resp of aws GetObject has mix types
|
||||
* and we want to get ArrayBuffer here.
|
||||
* See https://github.com/aws/aws-sdk-js-v3/issues/1877
|
||||
* @param b The Body of GetObject
|
||||
* @returns Promise<ArrayBuffer>
|
||||
*/
|
||||
const getObjectBodyToArrayBuffer = async (
|
||||
b: Readable | ReadableStream | Blob
|
||||
) => {
|
||||
if (b instanceof Readable) {
|
||||
const chunks: Uint8Array[] = [];
|
||||
for await (let chunk of b) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const buf = Buffer.concat(chunks);
|
||||
return bufferToArrayBuffer(buf);
|
||||
} else if (b instanceof ReadableStream) {
|
||||
return await new Response(b, {}).arrayBuffer();
|
||||
} else if (b instanceof Blob) {
|
||||
return await b.arrayBuffer();
|
||||
} else {
|
||||
throw TypeError(`The type of ${b} is not one of the supported types`);
|
||||
}
|
||||
};
|
||||
|
||||
export const downloadFromRemoteRaw = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config,
|
||||
fileOrFolderPath: string
|
||||
) => {
|
||||
const data = await s3Client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: s3Config.s3BucketName,
|
||||
Key: fileOrFolderPath,
|
||||
})
|
||||
);
|
||||
const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
|
||||
return bodyContents;
|
||||
};
|
||||
|
||||
export const downloadFromRemote = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config,
|
||||
fileOrFolderPath: string,
|
||||
vault: Vault,
|
||||
mtime: number
|
||||
) => {
|
||||
const isFolder = fileOrFolderPath.endsWith("/");
|
||||
|
||||
await mkdirpInVault(fileOrFolderPath, vault);
|
||||
|
||||
if (isFolder) {
|
||||
// mkdirp locally is enough
|
||||
// do nothing here
|
||||
} else {
|
||||
const content = await downloadFromRemoteRaw(
|
||||
s3Client,
|
||||
s3Config,
|
||||
fileOrFolderPath
|
||||
);
|
||||
await vault.adapter.writeBinary(fileOrFolderPath, content, {
|
||||
mtime: mtime,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* This function deals with file normally and "folder" recursively.
|
||||
* @param s3Client
|
||||
* @param s3Config
|
||||
* @param fileOrFolderPath
|
||||
* @returns
|
||||
*/
|
||||
export const deleteFromRemote = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config,
|
||||
fileOrFolderPath: string
|
||||
) => {
|
||||
if (fileOrFolderPath === "/") {
|
||||
return;
|
||||
}
|
||||
if (fileOrFolderPath.endsWith("/")) {
|
||||
const x = await listFromRemote(s3Client, s3Config, fileOrFolderPath);
|
||||
x.Contents.forEach(async (element) => {
|
||||
await s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: s3Config.s3BucketName,
|
||||
Key: element.Key,
|
||||
})
|
||||
);
|
||||
});
|
||||
} else {
|
||||
await s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: s3Config.s3BucketName,
|
||||
Key: fileOrFolderPath,
|
||||
})
|
||||
);
|
||||
}
|
||||
};
|
||||
266
src/sync.ts
Normal file
266
src/sync.ts
Normal file
@ -0,0 +1,266 @@
|
||||
import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import * as lf from "lovefield-ts/dist/es6/lf.js";
|
||||
|
||||
import { clearHistoryOfKey, FileFolderHistoryRecord } from "./localdb";
|
||||
import { S3Config, S3ObjectType, uploadToRemote, deleteFromRemote } from "./s3";
|
||||
import { downloadFromRemote } from "./s3";
|
||||
import { mkdirpInVault } from "./misc";
|
||||
|
||||
type DecisionType =
|
||||
| "undecided"
|
||||
| "unknown"
|
||||
| "upload_clearhist"
|
||||
| "download_clearhist"
|
||||
| "delremote_clearhist"
|
||||
| "download"
|
||||
| "upload"
|
||||
| "clearhist"
|
||||
| "mkdirplocal"
|
||||
| "skip";
|
||||
|
||||
export type SyncStatusType = "idle" | "preparing" | "syncing";
|
||||
|
||||
interface FileOrFolderMixedState {
|
||||
key: string;
|
||||
exist_local?: boolean;
|
||||
exist_remote?: boolean;
|
||||
mtime_local?: number;
|
||||
mtime_remote?: number;
|
||||
delete_time_local?: number;
|
||||
decision?: DecisionType;
|
||||
syncDone?: "done";
|
||||
}
|
||||
|
||||
export const ensembleMixedStates = (
|
||||
remote: S3ObjectType[],
|
||||
local: TAbstractFile[],
|
||||
deleteHistory: FileFolderHistoryRecord[]
|
||||
) => {
|
||||
const results = {} as Record<string, FileOrFolderMixedState>;
|
||||
|
||||
remote.forEach((entry) => {
|
||||
let r = {} as FileOrFolderMixedState;
|
||||
const key = entry.Key;
|
||||
r = {
|
||||
key: key,
|
||||
exist_remote: true,
|
||||
mtime_remote: entry.LastModified.valueOf(),
|
||||
};
|
||||
if (results.hasOwnProperty(key)) {
|
||||
results[key].key = r.key;
|
||||
results[key].exist_remote = r.exist_remote;
|
||||
results[key].mtime_remote = r.mtime_remote;
|
||||
} else {
|
||||
results[key] = r;
|
||||
}
|
||||
});
|
||||
|
||||
local.forEach((entry) => {
|
||||
let r = {} as FileOrFolderMixedState;
|
||||
let key = entry.path;
|
||||
|
||||
if (entry.path === "/") {
|
||||
// ignore
|
||||
return;
|
||||
} else if (entry instanceof TFile) {
|
||||
r = {
|
||||
key: entry.path,
|
||||
exist_local: true,
|
||||
mtime_local: entry.stat.mtime,
|
||||
};
|
||||
} else if (entry instanceof TFolder) {
|
||||
key = `${entry.path}/`;
|
||||
r = {
|
||||
key: key,
|
||||
exist_local: true,
|
||||
mtime_local: undefined,
|
||||
};
|
||||
} else {
|
||||
throw Error(`unexpected ${entry}`);
|
||||
}
|
||||
|
||||
if (results.hasOwnProperty(key)) {
|
||||
results[key].key = r.key;
|
||||
results[key].exist_local = r.exist_local;
|
||||
results[key].mtime_local = r.mtime_local;
|
||||
} else {
|
||||
results[key] = r;
|
||||
}
|
||||
});
|
||||
|
||||
deleteHistory.forEach((entry) => {
|
||||
let key = entry.key;
|
||||
if (entry.key_type === "folder") {
|
||||
if (!entry.key.endsWith("/")) {
|
||||
key = `${entry.key}/`;
|
||||
}
|
||||
} else if (entry.key_type === "file") {
|
||||
// pass
|
||||
} else {
|
||||
throw Error(`unexpected ${entry}`);
|
||||
}
|
||||
|
||||
const r = {
|
||||
key: key,
|
||||
delete_time_local: entry.action_when,
|
||||
} as FileOrFolderMixedState;
|
||||
|
||||
if (results.hasOwnProperty(key)) {
|
||||
results[key].key = r.key;
|
||||
results[key].delete_time_local = r.delete_time_local;
|
||||
} else {
|
||||
results[key] = r;
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
};
|
||||
|
||||
export const getOperation = (
|
||||
origRecord: FileOrFolderMixedState,
|
||||
inplace: boolean = false
|
||||
) => {
|
||||
let r = origRecord;
|
||||
if (!inplace) {
|
||||
r = Object.assign({}, origRecord);
|
||||
}
|
||||
|
||||
if (r.mtime_local === 0) {
|
||||
r.mtime_local = undefined;
|
||||
}
|
||||
if (r.mtime_remote === 0) {
|
||||
r.mtime_remote = undefined;
|
||||
}
|
||||
if (r.delete_time_local === 0) {
|
||||
r.delete_time_local = undefined;
|
||||
}
|
||||
if (r.exist_local === undefined) {
|
||||
r.exist_local = false;
|
||||
}
|
||||
if (r.exist_remote === undefined) {
|
||||
r.exist_remote = false;
|
||||
}
|
||||
r.decision = "unknown";
|
||||
|
||||
if (
|
||||
r.exist_remote &&
|
||||
r.exist_local &&
|
||||
r.mtime_remote !== undefined &&
|
||||
r.mtime_local !== undefined &&
|
||||
r.mtime_remote > r.mtime_local
|
||||
) {
|
||||
r.decision = "download_clearhist";
|
||||
} else if (
|
||||
r.exist_remote &&
|
||||
r.exist_local &&
|
||||
r.mtime_remote !== undefined &&
|
||||
r.mtime_local !== undefined &&
|
||||
r.mtime_remote <= r.mtime_local
|
||||
) {
|
||||
r.decision = "upload_clearhist";
|
||||
} else if (
|
||||
r.exist_remote &&
|
||||
r.exist_local &&
|
||||
r.mtime_remote !== undefined &&
|
||||
r.mtime_local === undefined
|
||||
) {
|
||||
// this must be a folder!
|
||||
if (!r.key.endsWith("/")) {
|
||||
throw Error(`${r.key} is not a folder but lacks local mtime`);
|
||||
}
|
||||
r.decision = "skip";
|
||||
} else if (
|
||||
r.exist_remote &&
|
||||
!r.exist_local &&
|
||||
r.mtime_remote !== undefined &&
|
||||
r.mtime_local === undefined &&
|
||||
r.delete_time_local !== undefined &&
|
||||
r.mtime_remote >= r.delete_time_local
|
||||
) {
|
||||
r.decision = "download_clearhist";
|
||||
} else if (
|
||||
r.exist_remote &&
|
||||
!r.exist_local &&
|
||||
r.mtime_remote !== undefined &&
|
||||
r.mtime_local === undefined &&
|
||||
r.delete_time_local !== undefined &&
|
||||
r.mtime_remote < r.delete_time_local
|
||||
) {
|
||||
r.decision = "delremote_clearhist";
|
||||
} else if (
|
||||
r.exist_remote &&
|
||||
!r.exist_local &&
|
||||
r.mtime_remote !== undefined &&
|
||||
r.mtime_local === undefined &&
|
||||
r.delete_time_local == undefined
|
||||
) {
|
||||
r.decision = "download";
|
||||
} else if (!r.exist_remote && r.exist_local && r.mtime_remote === undefined) {
|
||||
r.decision = "upload_clearhist";
|
||||
} else if (
|
||||
!r.exist_remote &&
|
||||
!r.exist_local &&
|
||||
r.mtime_remote === undefined &&
|
||||
r.mtime_local === undefined
|
||||
) {
|
||||
r.decision = "clearhist";
|
||||
}
|
||||
|
||||
return r;
|
||||
};
|
||||
|
||||
export const doActualSync = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config,
|
||||
db: lf.DatabaseConnection,
|
||||
vault: Vault,
|
||||
keyStates: Record<string, FileOrFolderMixedState>
|
||||
) => {
|
||||
Object.entries(keyStates)
|
||||
.sort((k, v) => -(k as string).length)
|
||||
.map(async ([k, v]) => {
|
||||
const key = k as string;
|
||||
const state = v as FileOrFolderMixedState;
|
||||
|
||||
if (
|
||||
state.decision === undefined ||
|
||||
state.decision === "unknown" ||
|
||||
state.decision === "undecided"
|
||||
) {
|
||||
throw Error(`unknown decision in ${JSON.stringify(state)}`);
|
||||
} else if (state.decision === "skip") {
|
||||
// do nothing
|
||||
} else if (state.decision === "download_clearhist") {
|
||||
await downloadFromRemote(
|
||||
s3Client,
|
||||
s3Config,
|
||||
state.key,
|
||||
vault,
|
||||
state.mtime_remote
|
||||
);
|
||||
await clearHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "upload_clearhist") {
|
||||
await uploadToRemote(s3Client, s3Config, state.key, vault, false);
|
||||
await clearHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "download") {
|
||||
await mkdirpInVault(state.key, vault);
|
||||
await downloadFromRemote(
|
||||
s3Client,
|
||||
s3Config,
|
||||
state.key,
|
||||
vault,
|
||||
state.mtime_remote
|
||||
);
|
||||
} else if (state.decision === "delremote_clearhist") {
|
||||
await deleteFromRemote(s3Client, s3Config, state.key);
|
||||
await clearHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "upload") {
|
||||
await uploadToRemote(s3Client, s3Config, state.key, vault, false);
|
||||
} else if (state.decision === "clearhist") {
|
||||
await clearHistoryOfKey(db, state.key);
|
||||
} else {
|
||||
throw Error("this should never happen!");
|
||||
}
|
||||
});
|
||||
};
|
||||
Loading…
Reference in New Issue
Block a user