logtodb

src/baseTypes.ts

@@ -84,6 +84,7 @@ export interface RemotelySavePluginSettings {
   syncConfigDir?: boolean;
   syncUnderscoreItems?: boolean;
   lang?: LangTypeAndAuto;
+  logToDB?: boolean;
 
   /**
    * @deprecated
@@ -156,3 +157,8 @@ export const API_VER_STAT_FOLDER = "0.13.27";
 export const API_VER_REQURL = "0.13.26";
 export const VALID_REQURL =
   requireApiVersion(API_VER_REQURL) && !Platform.isAndroidApp;
+
+export const DEFAULT_DEBUG_FOLDER = "_debug_remotely_save/";
+export const DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX =
+  "sync_plans_hist_exported_on_";
+export const DEFAULT_LOG_HISTORY_FILE_PREFIX = "log_hist_exported_on_";

src/configPersist.ts

@@ -3,8 +3,7 @@ import { reverseString } from "./misc";
 
 import type { RemotelySavePluginSettings } from "./baseTypes";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 const DEFAULT_README: string =
   "The file contains sensitive info, so DO NOT take screenshot of, copy, or share it to anyone! It's also generated automatically, so do not edit it manually.";

src/debugMode.ts

@@ -1,15 +1,19 @@
 import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
 
 import type { SyncPlanType } from "./sync";
-import { readAllSyncPlanRecordTextsByVault } from "./localdb";
+import {
+  readAllSyncPlanRecordTextsByVault,
+  readAllLogRecordTextsByVault,
+} from "./localdb";
 import type { InternalDBs } from "./localdb";
 import { mkdirpInVault } from "./misc";
+import {
+  DEFAULT_DEBUG_FOLDER,
+  DEFAULT_LOG_HISTORY_FILE_PREFIX,
+  DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX,
+} from "./baseTypes";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
-
-const DEFAULT_DEBUG_FOLDER = "_debug_remotely_save/";
-const DEFAULT_SYNC_PLANS_HISTORY_FILE_PREFIX = "sync_plans_hist_exported_on_";
+import { log } from "./moreOnLog";
 
 export const exportVaultSyncPlansToFiles = async (
   db: InternalDBs,
@@ -34,3 +38,27 @@ export const exportVaultSyncPlansToFiles = async (
   });
   log.info("finish exporting");
 };
+
+export const exportVaultLoggerOutputToFiles = async (
+  db: InternalDBs,
+  vault: Vault,
+  vaultRandomID: string
+) => {
+  await mkdirpInVault(DEFAULT_DEBUG_FOLDER, vault);
+  const records = await readAllLogRecordTextsByVault(db, vaultRandomID);
+  let md = "";
+  if (records.length === 0) {
+    md = "No logger history found.";
+  } else {
+    md =
+      "Logger history found:\n\n" +
+      "```text\n" +
+      records.join("\n") +
+      "\n```\n";
+  }
+  const ts = Date.now();
+  const filePath = `${DEFAULT_DEBUG_FOLDER}${DEFAULT_LOG_HISTORY_FILE_PREFIX}${ts}.md`;
+  await vault.create(filePath, md, {
+    mtime: ts,
+  });
+};

src/encrypt.ts

@@ -1,8 +1,7 @@
 import { base32, base64url } from "rfc4648";
 import { bufferToArrayBuffer, hexStringToTypedArray } from "./misc";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 const DEFAULT_ITER = 20000;
 

src/importExport.ts

@@ -7,8 +7,7 @@ import {
   RemotelySavePluginSettings,
 } from "./baseTypes";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 export const exportQrCodeUri = async (
   settings: RemotelySavePluginSettings,

src/langs

@@ -1 +1 @@
-Subproject commit 75a3b2a830c5e3286fa26a0c13178505f7ff5cac
+Subproject commit 2afc10b080356dc8d25e77105838fcfc91f072c6

src/localdb.ts

@@ -1,14 +1,13 @@
 import localforage from "localforage";
+export type LocalForage = typeof localforage;
+import { nanoid } from "nanoid";
 import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian";
 
 import { API_VER_STAT_FOLDER, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
 import type { SyncPlanType } from "./sync";
+import { toText, unixTimeToStr } from "./misc";
 
-export type LocalForage = typeof localforage;
-
-import * as origLog from "loglevel";
-import { nanoid } from "nanoid";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326];
 export const DEFAULT_DB_VERSION_NUMBER: number = 20220326;
@@ -18,6 +17,7 @@ export const DEFAULT_TBL_FILE_HISTORY = "filefolderoperationhistory";
 export const DEFAULT_TBL_SYNC_MAPPING = "syncmetadatahistory";
 export const DEFAULT_SYNC_PLANS_HISTORY = "syncplanshistory";
 export const DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING = "vaultrandomidmapping";
+export const DEFAULT_TBL_LOGGER_OUTPUT = "loggeroutput";
 
 export interface FileFolderHistoryRecord {
   key: string;
@@ -57,6 +57,7 @@ export interface InternalDBs {
   syncMappingTbl: LocalForage;
   syncPlansTbl: LocalForage;
   vaultRandomIDMappingTbl: LocalForage;
+  loggerOutputTbl: LocalForage;
 }
 
 /**
@@ -211,6 +212,10 @@ export const prepareDBs = async (
       name: DEFAULT_DB_NAME,
       storeName: DEFAULT_TBL_VAULT_RANDOM_ID_MAPPING,
     }),
+    loggerOutputTbl: localforage.createInstance({
+      name: DEFAULT_DB_NAME,
+      storeName: DEFAULT_TBL_LOGGER_OUTPUT,
+    }),
   } as InternalDBs;
 
   // try to get vaultRandomID firstly
@@ -546,3 +551,91 @@ export const readAllSyncPlanRecordTextsByVault = async (
     return records.map((x) => x.syncPlan);
   }
 };
+
+export const readAllLogRecordTextsByVault = async (
+  db: InternalDBs,
+  vaultRandomID: string
+) => {
+  const records = [] as { ts: number; r: string }[];
+  await db.loggerOutputTbl.iterate((value, key, iterationNumber) => {
+    if (key.startsWith(`${vaultRandomID}\t`)) {
+      const item = {
+        ts: parseInt(key.split("\t")[1]),
+        r: value as string,
+      };
+      records.push(item);
+    }
+  });
+
+  // while reading the logs, we want it to be ascending
+  records.sort((a, b) => a.ts - b.ts);
+
+  if (records === undefined) {
+    return [] as string[];
+  } else {
+    return records.map((x) => x.r);
+  }
+};
+
+export const insertLoggerOutputByVault = async (
+  db: InternalDBs,
+  vaultRandomID: string,
+  ...msg: any[]
+) => {
+  const ts = Date.now();
+  const tsFmt = unixTimeToStr(ts);
+  const key = `${vaultRandomID}\t${ts}`;
+
+  try {
+    const val = [`[${tsFmt}]`, ...msg.map((x) => toText(x))].join(" ");
+    db.loggerOutputTbl.setItem(key, val);
+  } catch (err) {
+    // give up, and let it pass
+  }
+};
+
+export const clearAllLoggerOutputRecords = async (db: InternalDBs) => {
+  await db.loggerOutputTbl.clear();
+};
+
+/**
+ * We remove records that are older than 7 days or 10000 records.
+ * It's a heavy operation, so we shall not place it in the start up.
+ * @param db
+ */
+export const clearExpiredLoggerOutputRecords = async (db: InternalDBs) => {
+  const MILLISECONDS_OLD = 1000 * 60 * 60 * 24 * 7; // 7 days
+  const COUNT_TO_MANY = 10000;
+
+  const currTs = Date.now();
+  const expiredTs = currTs - MILLISECONDS_OLD;
+
+  let records = (await db.loggerOutputTbl.keys()).map((key) => {
+    const ts = parseInt(key.split("\t")[1]);
+    const expired = ts <= expiredTs;
+    return {
+      ts: ts,
+      key: key,
+      expired: expired,
+    };
+  });
+
+  const keysToRemove = new Set(
+    records.filter((x) => x.expired).map((x) => x.key)
+  );
+
+  if (records.length - keysToRemove.size > COUNT_TO_MANY) {
+    // we need to find out records beyond 10000 records
+    records = records.filter((x) => !x.expired); // shrink the array
+    records.sort((a, b) => -(a.ts - b.ts)); // descending
+    records.slice(COUNT_TO_MANY).forEach((element) => {
+      keysToRemove.add(element.key);
+    });
+  }
+
+  const ps = [] as Promise<void>[];
+  keysToRemove.forEach((element) => {
+    ps.push(db.loggerOutputTbl.removeItem(element));
+  });
+  await Promise.all(ps);
+};

src/main.ts

@@ -24,6 +24,8 @@ import {
   loadFileHistoryTableByVault,
   prepareDBs,
   InternalDBs,
+  insertLoggerOutputByVault,
+  clearExpiredLoggerOutputRecords,
 } from "./localdb";
 import { RemoteClient } from "./remote";
 import {
@@ -48,11 +50,11 @@ import { ObsConfigDirFileType, listFilesInObsFolder } from "./obsFolderLister";
 import { I18n } from "./i18n";
 import type { LangType, LangTypeAndAuto, TransItemType } from "./i18n";
 
-import * as origLog from "loglevel";
 import { DeletionOnRemote, MetadataOnRemote } from "./metadataOnRemote";
 import { SyncAlgoV2Modal } from "./syncAlgoV2Notice";
 import { applyPresetRulesInplace } from "./presetRules";
-const log = origLog.getLogger("rs-default");
+
+import { applyLogWriterInplace, log } from "./moreOnLog";
 
 const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
   s3: DEFAULT_S3_CONFIG,
@@ -70,6 +72,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
   syncConfigDir: false,
   syncUnderscoreItems: false,
   lang: "auto",
+  logToDB: false,
 };
 
 interface OAuth2Info {
@@ -343,8 +346,8 @@ export default class RemotelySavePlugin extends Plugin {
         triggerSource: triggerSource,
         syncStatus: this.syncStatus,
       });
-      log.info(msg);
-      log.info(error);
+      log.error(msg);
+      log.error(error);
       getNotice(msg, 10 * 1000);
       getNotice(error.message, 10 * 1000);
       this.syncStatus = "idle";
@@ -412,6 +415,10 @@ export default class RemotelySavePlugin extends Plugin {
       throw err;
     }
 
+    // must AFTER preparing DB
+    this.addOutputToDBIfSet();
+    this.enableAutoClearOutputToDBHistIfSet();
+
     this.syncStatus = "idle";
 
     this.registerEvent(
@@ -911,4 +918,34 @@ export default class RemotelySavePlugin extends Plugin {
       // just skip
     }
   }
+
+  addOutputToDBIfSet() {
+    if (this.settings.logToDB) {
+      applyLogWriterInplace((...msg: any[]) => {
+        insertLoggerOutputByVault(this.db, this.vaultRandomID, ...msg);
+      });
+    }
+  }
+
+  enableAutoClearOutputToDBHistIfSet() {
+    const initClearOutputToDBHistAfterMilliseconds = 1000 * 45;
+    const autoClearOutputToDBHistAfterMilliseconds = 1000 * 60 * 5;
+
+    this.app.workspace.onLayoutReady(() => {
+      // init run
+      window.setTimeout(() => {
+        if (this.settings.logToDB) {
+          clearExpiredLoggerOutputRecords(this.db);
+        }
+      }, initClearOutputToDBHistAfterMilliseconds);
+
+      // scheduled run
+      const intervalID = window.setInterval(() => {
+        if (this.settings.logToDB) {
+          clearExpiredLoggerOutputRecords(this.db);
+        }
+      }, autoClearOutputToDBHistAfterMilliseconds);
+      this.registerInterval(intervalID);
+    });
+  }
 }

src/misc.ts

@@ -4,8 +4,7 @@ import * as path from "path";
 import { base32, base64url } from "rfc4648";
 import XRegExp from "xregexp";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 declare global {
   interface Window {
@@ -319,3 +318,64 @@ export const unixTimeToStr = (x: number | undefined | null) => {
   }
   return window.moment(x).format() as string;
 };
+
+/**
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#examples
+ * @returns
+ */
+const getCircularReplacer = () => {
+  const seen = new WeakSet();
+  return (key: any, value: any) => {
+    if (typeof value === "object" && value !== null) {
+      if (seen.has(value)) {
+        return;
+      }
+      seen.add(value);
+    }
+    return value;
+  };
+};
+
+/**
+ * Convert "any" value to string.
+ * @param x
+ * @returns
+ */
+export const toText = (x: any) => {
+  if (x === undefined || x === null) {
+    return `${x}`;
+  }
+  if (typeof x === "string") {
+    return x;
+  }
+  if (
+    x instanceof String ||
+    x instanceof Date ||
+    typeof x === "number" ||
+    typeof x === "bigint" ||
+    typeof x === "boolean"
+  ) {
+    return `${x}`;
+  }
+
+  if (
+    x instanceof Error ||
+    (x &&
+      x.stack &&
+      x.message &&
+      typeof x.stack === "string" &&
+      typeof x.message === "string")
+  ) {
+    return (x.stack || x.message) as string;
+  }
+
+  try {
+    const y = JSON.stringify(x, getCircularReplacer(), 2);
+    if (y !== undefined) {
+      return y;
+    }
+    throw new Error("not jsonable");
+  } catch {
+    return `${x}`;
+  }
+};

src/moreOnLog.ts

@@ -0,0 +1,34 @@
+// It's very dangerous for this file to depend on other files in the same project.
+// We should avoid this situation as much as possible.
+
+import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
+
+import * as origLog from "loglevel";
+import type { LogLevelNumbers, Logger, LogLevel, LogLevelDesc } from "loglevel";
+const log2 = origLog.getLogger("rs-default");
+
+const originalFactory = log2.methodFactory;
+
+export const applyLogWriterInplace = function (writer: (...msg: any[]) => any) {
+  log2.methodFactory = function (
+    methodName: string,
+    logLevel: LogLevelNumbers,
+    loggerName: string | symbol
+  ) {
+    const rawMethod = originalFactory(methodName, logLevel, loggerName);
+
+    return function (...msg: any[]) {
+      rawMethod.apply(undefined, msg);
+      writer(...msg);
+    };
+  };
+
+  log2.setLevel(log2.getLevel());
+};
+
+export const restoreLogWritterInplace = () => {
+  log2.methodFactory = originalFactory;
+  log2.setLevel(log2.getLevel());
+};
+
+export const log = log2;

src/obsFolderLister.ts

@@ -3,9 +3,6 @@ import { Queue } from "@fyears/tsqueue";
 import chunk from "lodash/chunk";
 import flatten from "lodash/flatten";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
-
 export interface ObsConfigDirFileType {
   key: string;
   ctime: number;

src/remote.ts

@@ -11,8 +11,7 @@ import * as onedrive from "./remoteForOnedrive";
 import * as s3 from "./remoteForS3";
 import * as webdav from "./remoteForWebdav";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 export class RemoteClient {
   readonly serviceType: SUPPORTED_SERVICES_TYPE;

src/remoteForDropbox.ts

@@ -12,8 +12,7 @@ import { bufferToArrayBuffer, getFolderLevels, mkdirpInVault } from "./misc";
 
 export { Dropbox } from "dropbox";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 export const DEFAULT_DROPBOX_CONFIG: DropboxConfig = {
   accessToken: "",

src/remoteForOnedrive.ts

@@ -6,7 +6,6 @@ import type {
   User,
 } from "@microsoft/microsoft-graph-types";
 import cloneDeep from "lodash/cloneDeep";
-import * as origLog from "loglevel";
 import { request, requestUrl, requireApiVersion, Vault } from "obsidian";
 import {
   API_VER_REQURL,
@@ -24,7 +23,7 @@ import {
   mkdirpInVault,
 } from "./misc";
 
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 const SCOPES = ["User.Read", "Files.ReadWrite.AppFolder", "offline_access"];
 const REDIRECT_URI = `obsidian://${COMMAND_CALLBACK_ONEDRIVE}`;

src/remoteForS3.ts

@@ -39,8 +39,7 @@ import {
 
 export { S3Client } from "@aws-sdk/client-s3";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 ////////////////////////////////////////////////////////////////////////////////
 // special handler using Obsidian requestUrl

src/remoteForWebdav.ts

@@ -9,8 +9,7 @@ import { RemoteItem, VALID_REQURL, WebdavConfig } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
 import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 import type {
   FileStat,

src/settings.ts

@@ -9,18 +9,25 @@ import {
 } from "obsidian";
 import {
   API_VER_REQURL,
+  DEFAULT_DEBUG_FOLDER,
   SUPPORTED_SERVICES_TYPE,
   SUPPORTED_SERVICES_TYPE_WITH_REMOTE_BASE_DIR,
   VALID_REQURL,
   WebdavAuthType,
   WebdavDepthType,
 } from "./baseTypes";
-import { exportVaultSyncPlansToFiles } from "./debugMode";
+import {
+  exportVaultSyncPlansToFiles,
+  exportVaultLoggerOutputToFiles,
+} from "./debugMode";
 import { exportQrCodeUri } from "./importExport";
 import {
   clearAllSyncMetaMapping,
   clearAllSyncPlanRecords,
   destroyDBs,
+  clearAllLoggerOutputRecords,
+  insertLoggerOutputByVault,
+  clearExpiredLoggerOutputRecords,
 } from "./localdb";
 import type RemotelySavePlugin from "./main"; // unavoidable
 import { RemoteClient } from "./remote";
@@ -36,11 +43,14 @@ import {
 } from "./remoteForOnedrive";
 import { messyConfigToNormal } from "./configPersist";
 import type { TransItemType } from "./i18n";
-
-import * as origLog from "loglevel";
 import { checkHasSpecialCharForDir } from "./misc";
 import { applyWebdavPresetRulesInplace } from "./presetRules";
-const log = origLog.getLogger("rs-default");
+
+import {
+  applyLogWriterInplace,
+  log,
+  restoreLogWritterInplace,
+} from "./moreOnLog";
 
 class PasswordModal extends Modal {
   plugin: RemotelySavePlugin;
@@ -1702,6 +1712,64 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
         });
       });
 
+    const logToDBDiv = debugDiv.createEl("div");
+    new Setting(logToDBDiv)
+      .setName(t("settings_logtodb"))
+      .setDesc(t("settings_logtodb_desc"))
+      .addDropdown(async (dropdown) => {
+        dropdown.addOption("enable", t("enable"));
+        dropdown.addOption("disable", t("disable"));
+        dropdown
+          .setValue(this.plugin.settings.logToDB ? "enable" : "disable")
+          .onChange(async (val: string) => {
+            const logToDB = val === "enable";
+            if (logToDB) {
+              applyLogWriterInplace((...msg: any[]) => {
+                insertLoggerOutputByVault(
+                  this.plugin.db,
+                  this.plugin.vaultRandomID,
+                  ...msg
+                );
+              });
+            } else {
+              restoreLogWritterInplace();
+            }
+            clearExpiredLoggerOutputRecords(this.plugin.db);
+            this.plugin.settings.logToDB = logToDB;
+            await this.plugin.saveSettings();
+          });
+      });
+
+    new Setting(logToDBDiv)
+      .setName(t("settings_logtodbexport"))
+      .setDesc(
+        t("settings_logtodbexport_desc", {
+          debugFolder: DEFAULT_DEBUG_FOLDER,
+        })
+      )
+      .addButton(async (button) => {
+        button.setButtonText(t("settings_logtodbexport_button"));
+        button.onClick(async () => {
+          await exportVaultLoggerOutputToFiles(
+            this.plugin.db,
+            this.app.vault,
+            this.plugin.vaultRandomID
+          );
+          new Notice(t("settings_logtodbexport_notice"));
+        });
+      });
+
+    new Setting(logToDBDiv)
+      .setName(t("settings_logtodbclear"))
+      .setDesc(t("settings_logtodbclear_desc"))
+      .addButton(async (button) => {
+        button.setButtonText(t("settings_logtodbclear_button"));
+        button.onClick(async () => {
+          await clearAllLoggerOutputRecords(this.plugin.db);
+          new Notice(t("settings_logtodbclear_notice"));
+        });
+      });
+
     const syncMappingDiv = debugDiv.createEl("div");
     new Setting(syncMappingDiv)
       .setName(t("settings_delsyncmap"))

src/sync.ts

@@ -47,8 +47,7 @@ import {
 } from "./metadataOnRemote";
 import { isInsideObsFolder, ObsConfigDirFileType } from "./obsFolderLister";
 
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+import { log } from "./moreOnLog";
 
 export type SyncStatusType =
   | "idle"

src/syncAlgoV2Notice.ts

@@ -1,8 +1,8 @@
 import { App, Modal, Notice, PluginSettingTab, Setting } from "obsidian";
 import type RemotelySavePlugin from "./main"; // unavoidable
 import type { TransItemType } from "./i18n";
-import * as origLog from "loglevel";
-const log = origLog.getLogger("rs-default");
+
+import { log } from "./moreOnLog";
 
 export class SyncAlgoV2Modal extends Modal {
   agree: boolean;