bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

basically working 2 way sync!

Browse Source
This commit is contained in:
fyears
2021-10-24 20:38:04 +08:00
parent 13e5af0c34
commit a26158055d
5 changed files with 671 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.ts
+69 -225
View File
@@ -1,6 +1,3 @@
import { Buffer } from "buffer";
import { Readable } from "stream";
import * as mime from "mime-types";
import {
App,
Modal,
@@ -14,47 +11,33 @@ import {
TFolder,
} from "obsidian";
import * as CodeMirror from "codemirror";
import type { FileFolderHistoryRecord, DatabaseConnection } from "./localdb";
import type { DatabaseConnection } from "./localdb";
import {
prepareDB,
destroyDB,
DEFAULT_DB_NAME,
DEFAULT_TBL_DELETE_HISTORY,
loadHistoryTable,
insertDeleteRecord,
insertRenameRecord,
getAllRecords,
} from "./localdb";
import {
getFolderLevels,
bufferToArrayBuffer,
getObjectBodyToArrayBuffer,
} from "./misc";
import {
S3Client,
ListObjectsV2Command,
PutObjectCommand,
GetObjectCommand,
} from "@aws-sdk/client-s3";
import type { SyncStatusType } from "./sync";
import { ensembleMixedStates, getOperation, doActualSync } from "./sync";
import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
interface SaveRemotePluginSettings {
s3Endpoint: string;
s3Region: string;
s3AccessKeyID: string;
s3SecretAccessKey: string;
s3BucketName: string;
s3?: S3Config;
}
const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
s3Endpoint: "",
s3Region: "",
s3AccessKeyID: "",
s3SecretAccessKey: "",
s3BucketName: "",
s3: DEFAULT_S3_CONFIG,
};
export default class SaveRemotePlugin extends Plugin {
settings: SaveRemotePluginSettings;
cm: CodeMirror.Editor;
db: DatabaseConnection;
syncStatus: SyncStatusType;
async onload() {
console.log("loading plugin obsidian-save-remote");
@@ -63,209 +46,70 @@ export default class SaveRemotePlugin extends Plugin {
await this.prepareDB();
this.syncStatus = "idle";
this.registerEvent(
this.app.vault.on("delete", async (fileOrFolder) => {
const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const tbl = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
// console.log(fileOrFolder);
let k: FileFolderHistoryRecord;
if (fileOrFolder instanceof TFile) {
k = {
key: fileOrFolder.path,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
action_when: Date.now(),
action_type: "delete",
key_type: "file",
rename_to: "",
};
} else if (fileOrFolder instanceof TFolder) {
k = {
key: fileOrFolder.path,
ctime: 0,
mtime: 0,
size: 0,
action_when: Date.now(),
action_type: "delete",
key_type: "folder",
rename_to: "",
};
}
const row = tbl.createRow(k);
await this.db.insertOrReplace().into(tbl).values([row]).exec();
await insertDeleteRecord(this.db, fileOrFolder);
})
);
this.registerEvent(
this.app.vault.on("rename", async (fileOrFolder, oldPath) => {
const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const tbl = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
// console.log(fileOrFolder);
let k: FileFolderHistoryRecord;
if (fileOrFolder instanceof TFile) {
k = {
key: oldPath,
ctime: fileOrFolder.stat.ctime,
mtime: fileOrFolder.stat.mtime,
size: fileOrFolder.stat.size,
action_when: Date.now(),
action_type: "rename",
key_type: "file",
rename_to: fileOrFolder.path,
};
} else if (fileOrFolder instanceof TFolder) {
k = {
key: oldPath,
ctime: 0,
mtime: 0,
size: 0,
action_when: Date.now(),
action_type: "rename",
key_type: "folder",
rename_to: fileOrFolder.path,
};
}
const row = tbl.createRow(k);
await this.db.insertOrReplace().into(tbl).values([row]).exec();
await insertRenameRecord(this.db, fileOrFolder, oldPath);
})
);
this.addRibbonIcon("dice", "Misc", async () => {
const a = this.app.vault.getAllLoadedFiles();
console.log(a);
// this.addRibbonIcon("dice", "Misc", async () => {
// const a = this.app.vault.getAllLoadedFiles();
// console.log(a);
// const h = await getAllRecords(this.db);
// console.log(h);
// });
const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY);
const h = await this.db.select().from(schema).exec();
console.log(h);
// console.log(b)
});
this.addRibbonIcon("right-arrow-with-tail", "Upload", async () => {
// console.log(this.app.vault.getFiles());
// console.log(this.app.vault.getAllLoadedFiles());
new Notice(`Upload begun.`);
const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
const s3Client = new S3Client({
region: this.settings.s3Region,
endpoint: this.settings.s3Endpoint,
credentials: {
accessKeyId: this.settings.s3AccessKeyID,
secretAccessKey: this.settings.s3SecretAccessKey,
},
});
try {
for (const fileOrFolder of allFilesAndFolders) {
if (fileOrFolder.path === "/") {
console.log('ignore "/"');
} else if ("children" in fileOrFolder) {
// folder
console.log(`folder ${fileOrFolder.path}/`);
new Notice(`folder ${fileOrFolder.path}/`);
const results = await s3Client.send(
new PutObjectCommand({
Bucket: this.settings.s3BucketName,
Key: `${fileOrFolder.path}/`,
Body: "",
})
);
} else {
// file
console.log(`file ${fileOrFolder.path}`);
const arrContent = await this.app.vault.adapter.readBinary(
fileOrFolder.path
);
new Notice(`file ${fileOrFolder.path}`);
const contentType =
mime.contentType(
mime.lookup(`${fileOrFolder.path}`) || undefined
) || undefined;
// console.log(contentType);
const results = await s3Client.send(
new PutObjectCommand({
Bucket: this.settings.s3BucketName,
Key: `${fileOrFolder.path}`,
Body: Buffer.from(arrContent),
ContentType: contentType,
})
);
}
}
new Notice("Upload finished!");
} catch (err) {
console.log("Error", err);
new Notice(`${err}`);
this.addRibbonIcon("switch", "Save Remote", async () => {
if (this.syncStatus !== "idle") {
new Notice("Save Remote already running!");
return;
}
});
this.addRibbonIcon("left-arrow-with-tail", "Download", async () => {
const allFilesAndFolders = this.app.vault.getAllLoadedFiles();
new Notice("Save Remote Sync Preparing");
this.syncStatus = "preparing";
const s3Client = getS3Client(this.settings.s3);
const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
const local = this.app.vault.getAllLoadedFiles();
const localHistory = await loadHistoryTable(this.db);
// console.log(remoteRsp);
// console.log(local);
// console.log(localHistory);
const s3Client = new S3Client({
region: this.settings.s3Region,
endpoint: this.settings.s3Endpoint,
credentials: {
accessKeyId: this.settings.s3AccessKeyID,
secretAccessKey: this.settings.s3SecretAccessKey,
},
});
const mixedStates = ensembleMixedStates(
remoteRsp.Contents,
local,
localHistory
);
try {
const listObj = await s3Client.send(
new ListObjectsV2Command({ Bucket: this.settings.s3BucketName })
);
for (const singleContent of listObj.Contents) {
const mtimeSec = Math.round(
singleContent.LastModified.valueOf() / 1000.0
);
console.log(`key ${singleContent.Key} mtime ${mtimeSec}`);
const foldersToBuild = getFolderLevels(singleContent.Key);
for (const folder of foldersToBuild) {
const r = await this.app.vault.adapter.exists(folder);
if (!r) {
console.log(`mkdir ${folder}`);
new Notice(`mkdir ${folder}`);
await this.app.vault.adapter.mkdir(folder);
}
}
if (singleContent.Key.endsWith("/")) {
// kind of a folder
// pass
} else {
// kind of a file
// download
console.log(`download file ${singleContent.Key}`);
new Notice(`download file ${singleContent.Key}`);
const data = await s3Client.send(
new GetObjectCommand({
Bucket: this.settings.s3BucketName,
Key: singleContent.Key,
})
);
const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
await this.app.vault.adapter.writeBinary(
singleContent.Key,
bodyContents
);
}
}
new Notice("Download finished!");
} catch (err) {
console.log("Error", err);
new Notice(`${err}`);
for (const [key, val] of Object.entries(mixedStates)) {
getOperation(val, true);
}
console.log(mixedStates);
// The operations above are read only and kind of safe.
// The operations below begins to write or delete (!!!) something.
new Notice("Save Remote Sync data exchanging!");
doActualSync(
s3Client,
this.settings.s3,
this.db,
this.app.vault,
mixedStates
);
new Notice("Save Remote finish!");
this.syncStatus = "idle";
});
this.addSettingTab(new SaveRemoteSettingTab(this.app, this));
@@ -327,9 +171,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
.addText((text) =>
text
.setPlaceholder("")
.setValue(this.plugin.settings.s3Endpoint)
.setValue(this.plugin.settings.s3.s3Endpoint)
.onChange(async (value) => {
this.plugin.settings.s3Endpoint = value;
this.plugin.settings.s3.s3Endpoint = value;
await this.plugin.saveSettings();
})
);
@@ -340,9 +184,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3Region}`)
.setValue(`${this.plugin.settings.s3.s3Region}`)
.onChange(async (value) => {
this.plugin.settings.s3Region = value;
this.plugin.settings.s3.s3Region = value;
await this.plugin.saveSettings();
})
);
@@ -353,9 +197,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3AccessKeyID}`)
.setValue(`${this.plugin.settings.s3.s3AccessKeyID}`)
.onChange(async (value) => {
this.plugin.settings.s3AccessKeyID = value;
this.plugin.settings.s3.s3AccessKeyID = value;
await this.plugin.saveSettings();
})
);
@@ -366,9 +210,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3SecretAccessKey}`)
.setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`)
.onChange(async (value) => {
this.plugin.settings.s3SecretAccessKey = value;
this.plugin.settings.s3.s3SecretAccessKey = value;
await this.plugin.saveSettings();
})
);
@@ -379,9 +223,9 @@ class SaveRemoteSettingTab extends PluginSettingTab {
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.s3BucketName}`)
.setValue(`${this.plugin.settings.s3.s3BucketName}`)
.onChange(async (value) => {
this.plugin.settings.s3BucketName = value;
this.plugin.settings.s3.s3BucketName = value;
await this.plugin.saveSettings();
})
);