debug mode: export sync plans

src/debugMode.ts

@@ -0,0 +1,37 @@
+import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
+
+import * as lf from "lovefield-ts/dist/es6/lf.js";
+
+import type { SyncPlanType } from "./sync";
+import {
+  insertSyncPlanRecord,
+  clearAllSyncPlanRecords,
+  readAllSyncPlanRecordTexts,
+} from "./localdb";
+import { mkdirpInVault } from "./misc";
+
+const DEFAULT_DEBUG_FOLDER = "_debug_save_remote/";
+const DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX = "sync_plans_hist_exported_on_";
+
+export const exportSyncPlansToFiles = async (
+  db: lf.DatabaseConnection,
+  vault: Vault
+) => {
+  console.log("exporting");
+  await mkdirpInVault(DEFAULT_DEBUG_FOLDER, vault);
+  const records = await readAllSyncPlanRecordTexts(db);
+  let md = "";
+  if (records.length === 0) {
+    md = "No sync plans history found";
+  } else {
+    md =
+      "Sync plans found:\n\n" +
+      records.map((x) => "```json\n" + x + "\n```\n").join("\n");
+  }
+  const ts = Date.now();
+  const filePath = `${DEFAULT_DEBUG_FOLDER}${DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX}${ts}.md`;
+  await vault.create(filePath, md, {
+    mtime: ts,
+  });
+  console.log("finish exporting");
+};

src/localdb.ts

@@ -2,12 +2,14 @@ import * as lf from "lovefield-ts/dist/es6/lf.js";
 import { TAbstractFile, TFile, TFolder } from "obsidian";
 
 import type { SUPPORTED_SERVICES_TYPE } from "./misc";
+import type { SyncPlanType } from "./sync";
 
 export type DatabaseConnection = lf.DatabaseConnection;
 
 export const DEFAULT_DB_NAME = "saveremotedb";
 export const DEFAULT_TBL_DELETE_HISTORY = "filefolderoperationhistory";
 export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
+export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
 
 export interface FileFolderHistoryRecord {
   key: string;
@@ -32,6 +34,12 @@ export interface SyncMetaMappingRecord {
   key_type: "folder" | "file";
 }
 
+interface SyncPlanRecord {
+  ts: number;
+  remote_type: string;
+  sync_plan: string;
+}
+
 export const prepareDBs = async () => {
   const schemaBuilder = lf.schema.create(DEFAULT_DB_NAME, 1);
   schemaBuilder
@@ -68,6 +76,15 @@ export const prepareDBs = async () => {
     .addPrimaryKey(["id"], true)
     .addIndex("idxkey", ["local_key", "remote_key"]);
 
+  schemaBuilder
+    .createTable(DEFAULT_SYNC_PLANS_HISTORY)
+    .addColumn("id", lf.Type.INTEGER)
+    .addColumn("ts", lf.Type.INTEGER)
+    .addColumn("remote_type", lf.Type.STRING)
+    .addColumn("sync_plan", lf.Type.STRING)
+    .addPrimaryKey(["id"], true)
+    .addIndex("tskey", ["ts"]);
+
   const db = await schemaBuilder.connect({
     storeType: lf.DataStoreType.INDEXED_DB,
   });
@@ -258,3 +275,37 @@ export const getSyncMetaMappingByRemoteKeyS3 = async (
 
   throw Error("something bad in sync meta mapping!");
 };
+
+export const insertSyncPlanRecord = async (
+  db: lf.DatabaseConnection,
+  syncPlan: SyncPlanType
+) => {
+  const schema = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+  const row = schema.createRow({
+    ts: syncPlan.ts,
+    remote_type: syncPlan.remoteType,
+    sync_plan: JSON.stringify(syncPlan, null, 2),
+  } as SyncPlanRecord);
+  await db.insertOrReplace().into(schema).values([row]).exec();
+};
+
+export const clearAllSyncPlanRecords = async (db: lf.DatabaseConnection) => {
+  const tbl = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+  await db.delete().from(tbl).exec();
+};
+
+export const readAllSyncPlanRecordTexts = async (db: lf.DatabaseConnection) => {
+  const schema = db.getSchema().table(DEFAULT_SYNC_PLANS_HISTORY);
+
+  const records = (await db
+    .select()
+    .from(schema)
+    .orderBy(schema.col("ts"), lf.Order.DESC)
+    .exec()) as SyncPlanRecord[];
+
+  if (records === undefined) {
+    return [] as string[];
+  } else {
+    return records.map((x) => x.sync_plan);
+  }
+};

src/main.ts

@@ -11,7 +11,7 @@ import {
   TFolder,
 } from "obsidian";
 import * as CodeMirror from "codemirror";
-import type { DatabaseConnection } from "./localdb";
+import { clearAllSyncPlanRecords, DatabaseConnection } from "./localdb";
 import {
   prepareDBs,
   destroyDBs,
@@ -19,11 +19,13 @@ import {
   insertDeleteRecord,
   insertRenameRecord,
   getAllDeleteRenameRecords,
+  insertSyncPlanRecord,
 } from "./localdb";
 
 import type { SyncStatusType } from "./sync";
 import { getSyncPlan, doActualSync } from "./sync";
 import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
+import { exportSyncPlansToFiles } from "./debugMode";
 
 interface SaveRemotePluginSettings {
   s3?: S3Config;
@@ -95,6 +97,7 @@ export default class SaveRemotePlugin extends Plugin {
           this.settings.password
         );
         console.log(syncPlan.mixedStates); // for debugging
+        await insertSyncPlanRecord(this.db, syncPlan);
 
         // The operations above are read only and kind of safe.
         // The operations below begins to write or delete (!!!) something.
@@ -259,5 +262,27 @@ class SaveRemoteSettingTab extends PluginSettingTab {
       );
 
     containerEl.createEl("h2", { text: "Debug" });
+
+    new Setting(containerEl)
+      .setName("export sync plans")
+      .setDesc("export sync plans")
+      .addButton(async (button) => {
+        button.setButtonText("Export");
+        button.onClick(async () => {
+          await exportSyncPlansToFiles(this.plugin.db, this.app.vault);
+          new Notice("sync plans history exported");
+        });
+      });
+
+    new Setting(containerEl)
+      .setName("delete sync plans history in db")
+      .setDesc("delete sync plans history in db")
+      .addButton(async (button) => {
+        button.setButtonText("Delete History");
+        button.onClick(async () => {
+          await clearAllSyncPlanRecords(this.plugin.db);
+          new Notice("sync plans history (in db) deleted");
+        });
+      });
   }
 }

src/misc.ts

@@ -52,9 +52,12 @@ export const getFolderLevels = (x: string) => {
 };
 
 export const mkdirpInVault = async (thePath: string, vault: Vault) => {
+  console.log(thePath);
   const foldersToBuild = getFolderLevels(thePath);
+  console.log(foldersToBuild);
   for (const folder of foldersToBuild) {
     const r = await vault.adapter.exists(folder);
+    console.log(r);
     if (!r) {
       console.log(`mkdir ${folder}`);
       await vault.adapter.mkdir(folder);

src/sync.ts

@@ -16,7 +16,7 @@ import {
   deleteFromRemote,
   downloadFromRemote,
 } from "./s3";
-import { mkdirpInVault, SUPPORTED_SERVICES_TYPE } from "./misc";
+import { mkdirpInVault, SUPPORTED_SERVICES_TYPE, isHiddenPath } from "./misc";
 import { decryptBase32ToString, encryptStringToBase32 } from "./encrypt";
 
 export type SyncStatusType =
@@ -55,7 +55,7 @@ interface FileOrFolderMixedState {
   remote_encrypted_key?: string;
 }
 
-interface SyncPlanType {
+export interface SyncPlanType {
   ts: number;
   remoteType: SUPPORTED_SERVICES_TYPE;
   mixedStates: Record<string, FileOrFolderMixedState>;
@@ -103,6 +103,9 @@ const ensembleMixedStates = async (
           remote_encrypted_key: remoteEncryptedKey,
         };
       }
+      if (isHiddenPath(key)) {
+        continue;
+      }
       if (results.hasOwnProperty(key)) {
         results[key].key = r.key;
         results[key].exist_remote = r.exist_remote;
@@ -141,6 +144,9 @@ const ensembleMixedStates = async (
       throw Error(`unexpected ${entry}`);
     }
 
+    if (isHiddenPath(key)) {
+      continue;
+    }
     if (results.hasOwnProperty(key)) {
       results[key].key = r.key;
       results[key].exist_local = r.exist_local;
@@ -168,6 +174,9 @@ const ensembleMixedStates = async (
       delete_time_local: entry.action_when,
     } as FileOrFolderMixedState;
 
+    if (isHiddenPath(key)) {
+      continue;
+    }
     if (results.hasOwnProperty(key)) {
       results[key].key = r.key;
       results[key].delete_time_local = r.delete_time_local;

tests/misc.test.ts

@@ -2,39 +2,71 @@ import * as fs from "fs";
 import * as path from "path";
 import { expect } from "chai";
 
-import * as misc from '../src/misc'
+import * as misc from "../src/misc";
 
 describe("Misc: hidden file", () => {
   it("should find hidden file correctly", () => {
-    let item = '';
+    let item = "";
     expect(misc.isHiddenPath(item)).to.be.false;
 
-    item = '.'
+    item = ".";
     expect(misc.isHiddenPath(item)).to.be.false;
 
-    item = '..'
+    item = "..";
     expect(misc.isHiddenPath(item)).to.be.false;
 
-    item = '/x/y/z/../././../a/b/c'
+    item = "/x/y/z/../././../a/b/c";
     expect(misc.isHiddenPath(item)).to.be.false;
 
-    item = '.hidden'
+    item = ".hidden";
     expect(misc.isHiddenPath(item)).to.be.true;
 
-    item = '_hidden_loose'
+    item = "_hidden_loose";
     expect(misc.isHiddenPath(item)).to.be.true;
     expect(misc.isHiddenPath(item, false)).to.be.false;
 
-    item = '/sdd/_hidden_loose'
+    item = "/sdd/_hidden_loose";
     expect(misc.isHiddenPath(item)).to.be.true;
 
-    item = 'what/../_hidden_loose/what/what/what'
+    item = "what/../_hidden_loose/what/what/what";
     expect(misc.isHiddenPath(item)).to.be.true;
 
-    item = 'what/../_hidden_loose/what/what/what'
+    item = "what/../_hidden_loose/what/what/what";
     expect(misc.isHiddenPath(item, false)).to.be.false;
 
-    item = 'what/../_hidden_loose/../.hidden/what/what/what'
+    item = "what/../_hidden_loose/../.hidden/what/what/what";
     expect(misc.isHiddenPath(item, false)).to.be.true;
   });
 });
+
+describe("Misc: get folder levels", () => {
+  it("should ignore empty path", () => {
+    const item = "";
+    expect(misc.getFolderLevels(item)).to.be.empty;
+  });
+
+  it("should ignore single file", () => {
+    const item = "xxx";
+    expect(misc.getFolderLevels(item)).to.be.empty;
+  });
+
+  it("should detect path ending with /", () => {
+    const item = "xxx/";
+    const res = ["xxx"];
+    expect(misc.getFolderLevels(item)).to.deep.equal(res);
+  });
+
+  it("should correctly split folders and files", () => {
+    const item = "xxx/yyy/zzz.md";
+    const res = ["xxx", "xxx/yyy"];
+    expect(misc.getFolderLevels(item)).to.deep.equal(res);
+
+    const item2 = "xxx/yyy/zzz";
+    const res2 = ["xxx", "xxx/yyy"];
+    expect(misc.getFolderLevels(item2)).to.deep.equal(res2);
+
+    const item3 = "xxx/yyy/zzz/";
+    const res3 = ["xxx", "xxx/yyy", "xxx/yyy/zzz"];
+    expect(misc.getFolderLevels(item3)).to.deep.equal(res3);
+  });
+});