From a26158055de47e16d4f3ad4c49f6c343a92d48ba Mon Sep 17 00:00:00 2001 From: fyears Date: Sun, 24 Oct 2021 20:38:04 +0800 Subject: [PATCH] basically working 2 way sync! --- src/localdb.ts | 118 +++++++++++++++++++- src/main.ts | 294 ++++++++++++------------------------------------- src/misc.ts | 41 ++----- src/s3.ts | 210 +++++++++++++++++++++++++++++++++++ src/sync.ts | 266 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 671 insertions(+), 258 deletions(-) create mode 100644 src/s3.ts create mode 100644 src/sync.ts diff --git a/src/localdb.ts b/src/localdb.ts index ba8cbe2..806eb8a 100644 --- a/src/localdb.ts +++ b/src/localdb.ts @@ -1,4 +1,5 @@ import * as lf from "lovefield-ts/dist/es6/lf.js"; +import { TAbstractFile, TFile, TFolder } from "obsidian"; export type DatabaseConnection = lf.DatabaseConnection; @@ -16,7 +17,7 @@ export interface FileFolderHistoryRecord { rename_to: string; } -export async function prepareDB() { +export const prepareDB = async () => { const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1); schemaBuilder .createTable(DEFAULT_TBL_DELETE_HISTORY) @@ -35,9 +36,9 @@ export async function prepareDB() { }); console.log("db connected"); return db; -} +}; -export function destroyDB(db: lf.DatabaseConnection) { +export const destroyDB = async (db: lf.DatabaseConnection) => { db.close(); const req = indexedDB.deleteDatabase(DEFAULT_DB_NAME); req.onsuccess = (event) => { @@ -50,4 +51,113 @@ export function destroyDB(db: lf.DatabaseConnection) { console.error("tried to delete db but something bad!"); console.error(event); }; -} +}; + +export const loadHistoryTable = async (db: lf.DatabaseConnection) => { + const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + + const records = await db + .select() + .from(schema) + .orderBy(schema.col("action_when"), lf.Order.ASC) + .exec(); + + return records as FileFolderHistoryRecord[]; +}; + +export const clearHistoryOfKey = async ( + db: lf.DatabaseConnection, + key: string +) => { + const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + + await db.delete().from(tbl).where(tbl.col("key").eq(key)).exec(); +}; + +export const insertDeleteRecord = async ( + db: lf.DatabaseConnection, + fileOrFolder: TAbstractFile +) => { + const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + // console.log(fileOrFolder); + let k: FileFolderHistoryRecord; + if (fileOrFolder instanceof TFile) { + k = { + key: fileOrFolder.path, + ctime: fileOrFolder.stat.ctime, + mtime: fileOrFolder.stat.mtime, + size: fileOrFolder.stat.size, + action_when: Date.now(), + action_type: "delete", + key_type: "file", + rename_to: "", + }; + } else if (fileOrFolder instanceof TFolder) { + // key should endswith "/" + const key = fileOrFolder.path.endsWith("/") + ? fileOrFolder.path + : `${fileOrFolder.path}/`; + k = { + key: key, + ctime: 0, + mtime: 0, + size: 0, + action_when: Date.now(), + action_type: "delete", + key_type: "folder", + rename_to: "", + }; + } + const row = tbl.createRow(k); + await db.insertOrReplace().into(tbl).values([row]).exec(); +}; + +export const insertRenameRecord = async ( + db: lf.DatabaseConnection, + fileOrFolder: TAbstractFile, + oldPath: string +) => { + const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + const tbl = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + // console.log(fileOrFolder); + let k: FileFolderHistoryRecord; + if (fileOrFolder instanceof TFile) { + k = { + key: oldPath, + ctime: fileOrFolder.stat.ctime, + mtime: fileOrFolder.stat.mtime, + size: fileOrFolder.stat.size, + action_when: Date.now(), + action_type: "rename", + key_type: "file", + rename_to: fileOrFolder.path, + }; + } else if (fileOrFolder instanceof TFolder) { + const key = oldPath.endsWith("/") ? oldPath : `${oldPath}/`; + const renameTo = fileOrFolder.path.endsWith("/") + ? fileOrFolder.path + : `${fileOrFolder.path}/`; + k = { + key: key, + ctime: 0, + mtime: 0, + size: 0, + action_when: Date.now(), + action_type: "rename", + key_type: "folder", + rename_to: renameTo, + }; + } + const row = tbl.createRow(k); + await db.insertOrReplace().into(tbl).values([row]).exec(); +}; + +export const getAllRecords = async (db: lf.DatabaseConnection) => { + const schema = db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); + const res1 = await db.select().from(schema).exec(); + const res2 = res1 as FileFolderHistoryRecord[]; + return res2; +}; diff --git a/src/main.ts b/src/main.ts index 6379ee2..2014425 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,6 +1,3 @@ -import { Buffer } from "buffer"; -import { Readable } from "stream"; -import * as mime from "mime-types"; import { App, Modal, @@ -14,47 +11,33 @@ import { TFolder, } from "obsidian"; import * as CodeMirror from "codemirror"; -import type { FileFolderHistoryRecord, DatabaseConnection } from "./localdb"; +import type { DatabaseConnection } from "./localdb"; import { prepareDB, destroyDB, - DEFAULT_DB_NAME, - DEFAULT_TBL_DELETE_HISTORY, + loadHistoryTable, + insertDeleteRecord, + insertRenameRecord, + getAllRecords, } from "./localdb"; -import { - getFolderLevels, - bufferToArrayBuffer, - getObjectBodyToArrayBuffer, -} from "./misc"; - -import { - S3Client, - ListObjectsV2Command, - PutObjectCommand, - GetObjectCommand, -} from "@aws-sdk/client-s3"; +import type { SyncStatusType } from "./sync"; +import { ensembleMixedStates, getOperation, doActualSync } from "./sync"; +import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3"; interface SaveRemotePluginSettings { - s3Endpoint: string; - s3Region: string; - s3AccessKeyID: string; - s3SecretAccessKey: string; - s3BucketName: string; + s3?: S3Config; } const DEFAULT_SETTINGS: SaveRemotePluginSettings = { - s3Endpoint: "", - s3Region: "", - s3AccessKeyID: "", - s3SecretAccessKey: "", - s3BucketName: "", + s3: DEFAULT_S3_CONFIG, }; export default class SaveRemotePlugin extends Plugin { settings: SaveRemotePluginSettings; cm: CodeMirror.Editor; db: DatabaseConnection; + syncStatus: SyncStatusType; async onload() { console.log("loading plugin obsidian-save-remote"); @@ -63,209 +46,70 @@ export default class SaveRemotePlugin extends Plugin { await this.prepareDB(); + this.syncStatus = "idle"; + this.registerEvent( this.app.vault.on("delete", async (fileOrFolder) => { - const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); - const tbl = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); - // console.log(fileOrFolder); - let k: FileFolderHistoryRecord; - if (fileOrFolder instanceof TFile) { - k = { - key: fileOrFolder.path, - ctime: fileOrFolder.stat.ctime, - mtime: fileOrFolder.stat.mtime, - size: fileOrFolder.stat.size, - action_when: Date.now(), - action_type: "delete", - key_type: "file", - rename_to: "", - }; - } else if (fileOrFolder instanceof TFolder) { - k = { - key: fileOrFolder.path, - ctime: 0, - mtime: 0, - size: 0, - action_when: Date.now(), - action_type: "delete", - key_type: "folder", - rename_to: "", - }; - } - const row = tbl.createRow(k); - await this.db.insertOrReplace().into(tbl).values([row]).exec(); + await insertDeleteRecord(this.db, fileOrFolder); }) ); this.registerEvent( this.app.vault.on("rename", async (fileOrFolder, oldPath) => { - const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); - const tbl = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); - // console.log(fileOrFolder); - let k: FileFolderHistoryRecord; - if (fileOrFolder instanceof TFile) { - k = { - key: oldPath, - ctime: fileOrFolder.stat.ctime, - mtime: fileOrFolder.stat.mtime, - size: fileOrFolder.stat.size, - action_when: Date.now(), - action_type: "rename", - key_type: "file", - rename_to: fileOrFolder.path, - }; - } else if (fileOrFolder instanceof TFolder) { - k = { - key: oldPath, - ctime: 0, - mtime: 0, - size: 0, - action_when: Date.now(), - action_type: "rename", - key_type: "folder", - rename_to: fileOrFolder.path, - }; - } - const row = tbl.createRow(k); - await this.db.insertOrReplace().into(tbl).values([row]).exec(); + await insertRenameRecord(this.db, fileOrFolder, oldPath); }) ); - this.addRibbonIcon("dice", "Misc", async () => { - const a = this.app.vault.getAllLoadedFiles(); - console.log(a); + // this.addRibbonIcon("dice", "Misc", async () => { + // const a = this.app.vault.getAllLoadedFiles(); + // console.log(a); + // const h = await getAllRecords(this.db); + // console.log(h); + // }); - const schema = this.db.getSchema().table(DEFAULT_TBL_DELETE_HISTORY); - - const h = await this.db.select().from(schema).exec(); - - console.log(h); - - // console.log(b) - }); - - this.addRibbonIcon("right-arrow-with-tail", "Upload", async () => { - // console.log(this.app.vault.getFiles()); - // console.log(this.app.vault.getAllLoadedFiles()); - new Notice(`Upload begun.`); - const allFilesAndFolders = this.app.vault.getAllLoadedFiles(); - - const s3Client = new S3Client({ - region: this.settings.s3Region, - endpoint: this.settings.s3Endpoint, - credentials: { - accessKeyId: this.settings.s3AccessKeyID, - secretAccessKey: this.settings.s3SecretAccessKey, - }, - }); - - try { - for (const fileOrFolder of allFilesAndFolders) { - if (fileOrFolder.path === "/") { - console.log('ignore "/"'); - } else if ("children" in fileOrFolder) { - // folder - console.log(`folder ${fileOrFolder.path}/`); - new Notice(`folder ${fileOrFolder.path}/`); - - const results = await s3Client.send( - new PutObjectCommand({ - Bucket: this.settings.s3BucketName, - Key: `${fileOrFolder.path}/`, - Body: "", - }) - ); - } else { - // file - console.log(`file ${fileOrFolder.path}`); - const arrContent = await this.app.vault.adapter.readBinary( - fileOrFolder.path - ); - new Notice(`file ${fileOrFolder.path}`); - const contentType = - mime.contentType( - mime.lookup(`${fileOrFolder.path}`) || undefined - ) || undefined; - // console.log(contentType); - const results = await s3Client.send( - new PutObjectCommand({ - Bucket: this.settings.s3BucketName, - Key: `${fileOrFolder.path}`, - Body: Buffer.from(arrContent), - ContentType: contentType, - }) - ); - } - } - new Notice("Upload finished!"); - } catch (err) { - console.log("Error", err); - new Notice(`${err}`); + this.addRibbonIcon("switch", "Save Remote", async () => { + if (this.syncStatus !== "idle") { + new Notice("Save Remote already running!"); + return; } - }); - this.addRibbonIcon("left-arrow-with-tail", "Download", async () => { - const allFilesAndFolders = this.app.vault.getAllLoadedFiles(); + new Notice("Save Remote Sync Preparing"); + this.syncStatus = "preparing"; + const s3Client = getS3Client(this.settings.s3); + const remoteRsp = await listFromRemote(s3Client, this.settings.s3); + const local = this.app.vault.getAllLoadedFiles(); + const localHistory = await loadHistoryTable(this.db); + // console.log(remoteRsp); + // console.log(local); + // console.log(localHistory); - const s3Client = new S3Client({ - region: this.settings.s3Region, - endpoint: this.settings.s3Endpoint, - credentials: { - accessKeyId: this.settings.s3AccessKeyID, - secretAccessKey: this.settings.s3SecretAccessKey, - }, - }); + const mixedStates = ensembleMixedStates( + remoteRsp.Contents, + local, + localHistory + ); - try { - const listObj = await s3Client.send( - new ListObjectsV2Command({ Bucket: this.settings.s3BucketName }) - ); - - for (const singleContent of listObj.Contents) { - const mtimeSec = Math.round( - singleContent.LastModified.valueOf() / 1000.0 - ); - console.log(`key ${singleContent.Key} mtime ${mtimeSec}`); - - const foldersToBuild = getFolderLevels(singleContent.Key); - for (const folder of foldersToBuild) { - const r = await this.app.vault.adapter.exists(folder); - if (!r) { - console.log(`mkdir ${folder}`); - new Notice(`mkdir ${folder}`); - await this.app.vault.adapter.mkdir(folder); - } - } - - if (singleContent.Key.endsWith("/")) { - // kind of a folder - // pass - } else { - // kind of a file - // download - - console.log(`download file ${singleContent.Key}`); - new Notice(`download file ${singleContent.Key}`); - - const data = await s3Client.send( - new GetObjectCommand({ - Bucket: this.settings.s3BucketName, - Key: singleContent.Key, - }) - ); - const bodyContents = await getObjectBodyToArrayBuffer(data.Body); - await this.app.vault.adapter.writeBinary( - singleContent.Key, - bodyContents - ); - } - } - - new Notice("Download finished!"); - } catch (err) { - console.log("Error", err); - new Notice(`${err}`); + for (const [key, val] of Object.entries(mixedStates)) { + getOperation(val, true); } + + console.log(mixedStates); + + // The operations above are read only and kind of safe. + // The operations below begins to write or delete (!!!) something. + + new Notice("Save Remote Sync data exchanging!"); + + doActualSync( + s3Client, + this.settings.s3, + this.db, + this.app.vault, + mixedStates + ); + + new Notice("Save Remote finish!"); + this.syncStatus = "idle"; }); this.addSettingTab(new SaveRemoteSettingTab(this.app, this)); @@ -327,9 +171,9 @@ class SaveRemoteSettingTab extends PluginSettingTab { .addText((text) => text .setPlaceholder("") - .setValue(this.plugin.settings.s3Endpoint) + .setValue(this.plugin.settings.s3.s3Endpoint) .onChange(async (value) => { - this.plugin.settings.s3Endpoint = value; + this.plugin.settings.s3.s3Endpoint = value; await this.plugin.saveSettings(); }) ); @@ -340,9 +184,9 @@ class SaveRemoteSettingTab extends PluginSettingTab { .addText((text) => text .setPlaceholder("") - .setValue(`${this.plugin.settings.s3Region}`) + .setValue(`${this.plugin.settings.s3.s3Region}`) .onChange(async (value) => { - this.plugin.settings.s3Region = value; + this.plugin.settings.s3.s3Region = value; await this.plugin.saveSettings(); }) ); @@ -353,9 +197,9 @@ class SaveRemoteSettingTab extends PluginSettingTab { .addText((text) => text .setPlaceholder("") - .setValue(`${this.plugin.settings.s3AccessKeyID}`) + .setValue(`${this.plugin.settings.s3.s3AccessKeyID}`) .onChange(async (value) => { - this.plugin.settings.s3AccessKeyID = value; + this.plugin.settings.s3.s3AccessKeyID = value; await this.plugin.saveSettings(); }) ); @@ -366,9 +210,9 @@ class SaveRemoteSettingTab extends PluginSettingTab { .addText((text) => text .setPlaceholder("") - .setValue(`${this.plugin.settings.s3SecretAccessKey}`) + .setValue(`${this.plugin.settings.s3.s3SecretAccessKey}`) .onChange(async (value) => { - this.plugin.settings.s3SecretAccessKey = value; + this.plugin.settings.s3.s3SecretAccessKey = value; await this.plugin.saveSettings(); }) ); @@ -379,9 +223,9 @@ class SaveRemoteSettingTab extends PluginSettingTab { .addText((text) => text .setPlaceholder("") - .setValue(`${this.plugin.settings.s3BucketName}`) + .setValue(`${this.plugin.settings.s3.s3BucketName}`) .onChange(async (value) => { - this.plugin.settings.s3BucketName = value; + this.plugin.settings.s3.s3BucketName = value; await this.plugin.saveSettings(); }) ); diff --git a/src/misc.ts b/src/misc.ts index f4215ca..1aa11b1 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -1,7 +1,5 @@ +import { Vault } from "obsidian"; import * as path from "path"; -import * as fs from "fs"; -import { Buffer } from "buffer"; -import { Readable } from "stream"; export const ignoreHiddenFiles = (item: string) => { const basename = path.basename(item); @@ -30,6 +28,17 @@ export const getFolderLevels = (x: string) => { return res; }; +export const mkdirpInVault = async (thePath: string, vault: Vault) => { + const foldersToBuild = getFolderLevels(thePath); + for (const folder of foldersToBuild) { + const r = await vault.adapter.exists(folder); + if (!r) { + console.log(`mkdir ${folder}`); + await vault.adapter.mkdir(folder); + } + } +}; + /** * https://stackoverflow.com/questions/8609289 * @param b Buffer @@ -38,29 +47,3 @@ export const getFolderLevels = (x: string) => { export const bufferToArrayBuffer = (b: Buffer) => { return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength); }; - -/** - * The Body of resp of aws GetObject has mix types - * and we want to get ArrayBuffer here. - * See https://github.com/aws/aws-sdk-js-v3/issues/1877 - * @param b The Body of GetObject - * @returns Promise - */ -export const getObjectBodyToArrayBuffer = async ( - b: Readable | ReadableStream | Blob -) => { - if (b instanceof Readable) { - const chunks: Uint8Array[] = []; - for await (let chunk of b) { - chunks.push(chunk); - } - const buf = Buffer.concat(chunks); - return bufferToArrayBuffer(buf); - } else if (b instanceof ReadableStream) { - return await new Response(b, {}).arrayBuffer(); - } else if (b instanceof Blob) { - return await b.arrayBuffer(); - } else { - throw TypeError(`The type of ${b} is not one of the supported types`); - } -}; diff --git a/src/s3.ts b/src/s3.ts new file mode 100644 index 0000000..7e3997d --- /dev/null +++ b/src/s3.ts @@ -0,0 +1,210 @@ +import { Buffer } from "buffer"; +import { Readable } from "stream"; + +import { Vault } from "obsidian"; + +import { + S3Client, + ListObjectsV2Command, + PutObjectCommand, + GetObjectCommand, + DeleteObjectCommand, +} from "@aws-sdk/client-s3"; + +import type { _Object } from "@aws-sdk/client-s3"; + +import { bufferToArrayBuffer, mkdirpInVault } from "./misc"; +import * as mime from "mime-types"; + +export interface S3Config { + s3Endpoint: string; + s3Region: string; + s3AccessKeyID: string; + s3SecretAccessKey: string; + s3BucketName: string; +} + +export const DEFAULT_S3_CONFIG = { + s3Endpoint: "", + s3Region: "", + s3AccessKeyID: "", + s3SecretAccessKey: "", + s3BucketName: "", +}; + +export type S3ObjectType = _Object; + +export const getS3Client = (s3Config: S3Config) => { + const s3Client = new S3Client({ + region: s3Config.s3Region, + endpoint: s3Config.s3Endpoint, + credentials: { + accessKeyId: s3Config.s3AccessKeyID, + secretAccessKey: s3Config.s3SecretAccessKey, + }, + }); + return s3Client; +}; + +export const uploadToRemote = async ( + s3Client: S3Client, + s3Config: S3Config, + fileOrFolderPath: string, + vault: Vault, + isRecursively: boolean = false +) => { + const isFolder = fileOrFolderPath.endsWith("/"); + + const DEFAULT_CONTENT_TYPE = "application/octet-stream"; + + if (isFolder && isRecursively) { + throw Error("upload function doesn't implement recursive function yet!"); + } else if (isFolder && !isRecursively) { + // folder + const contentType = DEFAULT_CONTENT_TYPE; + return await s3Client.send( + new PutObjectCommand({ + Bucket: s3Config.s3BucketName, + Key: fileOrFolderPath, + Body: "", + ContentType: contentType, + }) + ); + } else { + // file + // we ignore isRecursively parameter here + const contentType = + mime.contentType(mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE) || + DEFAULT_CONTENT_TYPE; + const content = await vault.adapter.readBinary(fileOrFolderPath); + const body = Buffer.from(content); + return await s3Client.send( + new PutObjectCommand({ + Bucket: s3Config.s3BucketName, + Key: fileOrFolderPath, + Body: body, + ContentType: contentType, + }) + ); + } +}; + +export const listFromRemote = async ( + s3Client: S3Client, + s3Config: S3Config, + prefix?: string +) => { + if (prefix !== undefined) { + return await s3Client.send( + new ListObjectsV2Command({ + Bucket: s3Config.s3BucketName, + Prefix: prefix, + }) + ); + } + return await s3Client.send( + new ListObjectsV2Command({ Bucket: s3Config.s3BucketName }) + ); +}; + +/** + * The Body of resp of aws GetObject has mix types + * and we want to get ArrayBuffer here. + * See https://github.com/aws/aws-sdk-js-v3/issues/1877 + * @param b The Body of GetObject + * @returns Promise + */ +const getObjectBodyToArrayBuffer = async ( + b: Readable | ReadableStream | Blob +) => { + if (b instanceof Readable) { + const chunks: Uint8Array[] = []; + for await (let chunk of b) { + chunks.push(chunk); + } + const buf = Buffer.concat(chunks); + return bufferToArrayBuffer(buf); + } else if (b instanceof ReadableStream) { + return await new Response(b, {}).arrayBuffer(); + } else if (b instanceof Blob) { + return await b.arrayBuffer(); + } else { + throw TypeError(`The type of ${b} is not one of the supported types`); + } +}; + +export const downloadFromRemoteRaw = async ( + s3Client: S3Client, + s3Config: S3Config, + fileOrFolderPath: string +) => { + const data = await s3Client.send( + new GetObjectCommand({ + Bucket: s3Config.s3BucketName, + Key: fileOrFolderPath, + }) + ); + const bodyContents = await getObjectBodyToArrayBuffer(data.Body); + return bodyContents; +}; + +export const downloadFromRemote = async ( + s3Client: S3Client, + s3Config: S3Config, + fileOrFolderPath: string, + vault: Vault, + mtime: number +) => { + const isFolder = fileOrFolderPath.endsWith("/"); + + await mkdirpInVault(fileOrFolderPath, vault); + + if (isFolder) { + // mkdirp locally is enough + // do nothing here + } else { + const content = await downloadFromRemoteRaw( + s3Client, + s3Config, + fileOrFolderPath + ); + await vault.adapter.writeBinary(fileOrFolderPath, content, { + mtime: mtime, + }); + } +}; + +/** + * This function deals with file normally and "folder" recursively. + * @param s3Client + * @param s3Config + * @param fileOrFolderPath + * @returns + */ +export const deleteFromRemote = async ( + s3Client: S3Client, + s3Config: S3Config, + fileOrFolderPath: string +) => { + if (fileOrFolderPath === "/") { + return; + } + if (fileOrFolderPath.endsWith("/")) { + const x = await listFromRemote(s3Client, s3Config, fileOrFolderPath); + x.Contents.forEach(async (element) => { + await s3Client.send( + new DeleteObjectCommand({ + Bucket: s3Config.s3BucketName, + Key: element.Key, + }) + ); + }); + } else { + await s3Client.send( + new DeleteObjectCommand({ + Bucket: s3Config.s3BucketName, + Key: fileOrFolderPath, + }) + ); + } +}; diff --git a/src/sync.ts b/src/sync.ts new file mode 100644 index 0000000..0eaed27 --- /dev/null +++ b/src/sync.ts @@ -0,0 +1,266 @@ +import { TAbstractFile, TFolder, TFile, Vault } from "obsidian"; +import { S3Client } from "@aws-sdk/client-s3"; +import * as lf from "lovefield-ts/dist/es6/lf.js"; + +import { clearHistoryOfKey, FileFolderHistoryRecord } from "./localdb"; +import { S3Config, S3ObjectType, uploadToRemote, deleteFromRemote } from "./s3"; +import { downloadFromRemote } from "./s3"; +import { mkdirpInVault } from "./misc"; + +type DecisionType = + | "undecided" + | "unknown" + | "upload_clearhist" + | "download_clearhist" + | "delremote_clearhist" + | "download" + | "upload" + | "clearhist" + | "mkdirplocal" + | "skip"; + +export type SyncStatusType = "idle" | "preparing" | "syncing"; + +interface FileOrFolderMixedState { + key: string; + exist_local?: boolean; + exist_remote?: boolean; + mtime_local?: number; + mtime_remote?: number; + delete_time_local?: number; + decision?: DecisionType; + syncDone?: "done"; +} + +export const ensembleMixedStates = ( + remote: S3ObjectType[], + local: TAbstractFile[], + deleteHistory: FileFolderHistoryRecord[] +) => { + const results = {} as Record; + + remote.forEach((entry) => { + let r = {} as FileOrFolderMixedState; + const key = entry.Key; + r = { + key: key, + exist_remote: true, + mtime_remote: entry.LastModified.valueOf(), + }; + if (results.hasOwnProperty(key)) { + results[key].key = r.key; + results[key].exist_remote = r.exist_remote; + results[key].mtime_remote = r.mtime_remote; + } else { + results[key] = r; + } + }); + + local.forEach((entry) => { + let r = {} as FileOrFolderMixedState; + let key = entry.path; + + if (entry.path === "/") { + // ignore + return; + } else if (entry instanceof TFile) { + r = { + key: entry.path, + exist_local: true, + mtime_local: entry.stat.mtime, + }; + } else if (entry instanceof TFolder) { + key = `${entry.path}/`; + r = { + key: key, + exist_local: true, + mtime_local: undefined, + }; + } else { + throw Error(`unexpected ${entry}`); + } + + if (results.hasOwnProperty(key)) { + results[key].key = r.key; + results[key].exist_local = r.exist_local; + results[key].mtime_local = r.mtime_local; + } else { + results[key] = r; + } + }); + + deleteHistory.forEach((entry) => { + let key = entry.key; + if (entry.key_type === "folder") { + if (!entry.key.endsWith("/")) { + key = `${entry.key}/`; + } + } else if (entry.key_type === "file") { + // pass + } else { + throw Error(`unexpected ${entry}`); + } + + const r = { + key: key, + delete_time_local: entry.action_when, + } as FileOrFolderMixedState; + + if (results.hasOwnProperty(key)) { + results[key].key = r.key; + results[key].delete_time_local = r.delete_time_local; + } else { + results[key] = r; + } + }); + + return results; +}; + +export const getOperation = ( + origRecord: FileOrFolderMixedState, + inplace: boolean = false +) => { + let r = origRecord; + if (!inplace) { + r = Object.assign({}, origRecord); + } + + if (r.mtime_local === 0) { + r.mtime_local = undefined; + } + if (r.mtime_remote === 0) { + r.mtime_remote = undefined; + } + if (r.delete_time_local === 0) { + r.delete_time_local = undefined; + } + if (r.exist_local === undefined) { + r.exist_local = false; + } + if (r.exist_remote === undefined) { + r.exist_remote = false; + } + r.decision = "unknown"; + + if ( + r.exist_remote && + r.exist_local && + r.mtime_remote !== undefined && + r.mtime_local !== undefined && + r.mtime_remote > r.mtime_local + ) { + r.decision = "download_clearhist"; + } else if ( + r.exist_remote && + r.exist_local && + r.mtime_remote !== undefined && + r.mtime_local !== undefined && + r.mtime_remote <= r.mtime_local + ) { + r.decision = "upload_clearhist"; + } else if ( + r.exist_remote && + r.exist_local && + r.mtime_remote !== undefined && + r.mtime_local === undefined + ) { + // this must be a folder! + if (!r.key.endsWith("/")) { + throw Error(`${r.key} is not a folder but lacks local mtime`); + } + r.decision = "skip"; + } else if ( + r.exist_remote && + !r.exist_local && + r.mtime_remote !== undefined && + r.mtime_local === undefined && + r.delete_time_local !== undefined && + r.mtime_remote >= r.delete_time_local + ) { + r.decision = "download_clearhist"; + } else if ( + r.exist_remote && + !r.exist_local && + r.mtime_remote !== undefined && + r.mtime_local === undefined && + r.delete_time_local !== undefined && + r.mtime_remote < r.delete_time_local + ) { + r.decision = "delremote_clearhist"; + } else if ( + r.exist_remote && + !r.exist_local && + r.mtime_remote !== undefined && + r.mtime_local === undefined && + r.delete_time_local == undefined + ) { + r.decision = "download"; + } else if (!r.exist_remote && r.exist_local && r.mtime_remote === undefined) { + r.decision = "upload_clearhist"; + } else if ( + !r.exist_remote && + !r.exist_local && + r.mtime_remote === undefined && + r.mtime_local === undefined + ) { + r.decision = "clearhist"; + } + + return r; +}; + +export const doActualSync = async ( + s3Client: S3Client, + s3Config: S3Config, + db: lf.DatabaseConnection, + vault: Vault, + keyStates: Record +) => { + Object.entries(keyStates) + .sort((k, v) => -(k as string).length) + .map(async ([k, v]) => { + const key = k as string; + const state = v as FileOrFolderMixedState; + + if ( + state.decision === undefined || + state.decision === "unknown" || + state.decision === "undecided" + ) { + throw Error(`unknown decision in ${JSON.stringify(state)}`); + } else if (state.decision === "skip") { + // do nothing + } else if (state.decision === "download_clearhist") { + await downloadFromRemote( + s3Client, + s3Config, + state.key, + vault, + state.mtime_remote + ); + await clearHistoryOfKey(db, state.key); + } else if (state.decision === "upload_clearhist") { + await uploadToRemote(s3Client, s3Config, state.key, vault, false); + await clearHistoryOfKey(db, state.key); + } else if (state.decision === "download") { + await mkdirpInVault(state.key, vault); + await downloadFromRemote( + s3Client, + s3Config, + state.key, + vault, + state.mtime_remote + ); + } else if (state.decision === "delremote_clearhist") { + await deleteFromRemote(s3Client, s3Config, state.key); + await clearHistoryOfKey(db, state.key); + } else if (state.decision === "upload") { + await uploadToRemote(s3Client, s3Config, state.key, vault, false); + } else if (state.decision === "clearhist") { + await clearHistoryOfKey(db, state.key); + } else { + throw Error("this should never happen!"); + } + }); +};