Compare commits

..
34 Commits
Author SHA1 Message Date
fyears 0802767726 bump to 0.4.25
Release A New Version / build (16.x) (push) Failing after 46s
2024-05-25 15:58:52 +08:00
fyears be4a2d3271 nextcloud address 2024-05-25 15:57:48 +08:00
fyears 7ca2d19255 more debug info 2024-05-25 15:53:38 +08:00
fyears ff765d5ae7 change back to local time 2024-05-25 15:53:27 +08:00
fyears bdbf0b1484 nextcloud is good 2024-05-25 15:39:29 +08:00
fyears 7497b5fae7 split export 2024-05-25 15:22:19 +08:00
fyears 408acb6230 add custom protection 2024-05-25 14:37:37 +08:00
fyears d9cab7b1ff fix kind
Release A New Version / build (16.x) (push) Failing after 43s
2024-05-24 23:13:40 +08:00
fyears 5e53967e01 bump to 0.4.24 2024-05-24 23:11:37 +08:00
fyears fb9f4a67b4 fix size bug 2024-05-24 23:11:01 +08:00
fyears 26a426dda8 bump to 0.4.23
Release A New Version / build (16.x) (push) Failing after 42s
2024-05-24 22:42:21 +08:00
fyears de64c3c53f fix condition for partial update 2024-05-24 22:41:43 +08:00
fyears 0cefafa491 special treatment for jianguoyun 2024-05-24 22:08:02 +08:00
fyears b769becb97 bump to 0.4.22
Release A New Version / build (16.x) (push) Failing after 40s
2024-05-21 01:04:22 +08:00
fyears 7b3600a46f correct way for nextcloud 2024-05-20 09:56:09 +08:00
fyears 69e72eae1d correctly set range update 2024-05-19 21:41:01 +08:00
fyears b0acde0ba6 remove recursive to fix digest 2024-05-19 20:36:34 +08:00
fyears 11b7fee80b remove unnecessary get 2024-05-19 19:59:16 +08:00
fyears c4c39f6b79 make it more robust 2024-05-19 19:13:56 +08:00
fyears 45578a01dd chunk webdav upload to 5 mb 2024-05-19 18:10:32 +08:00
fyears 0391c42999 wrap settings 2024-05-19 18:09:41 +08:00
fyears 807eec928e optimize sync plan export 2024-05-19 17:51:44 +08:00
fyears 3d7c4d2a4a optimize webdav 2024-05-19 17:03:14 +08:00
fyears cb779fc7bf add upload by chunks for webdav 2024-05-19 15:58:49 +08:00
fyears 9d8e2af7b9 allow skipping empty file in onedrive 2024-05-18 12:03:53 +08:00
Jason a48440e60b Refine wording and grammar in some descriptions of settings. (#655)
* Refine wording and grammar in some descriptions of settings.

* Remove extra space in settings_encryptionmethod_desc.

* Add required ending comma syntax to settings_webdav_auth_desc.
2024-05-18 02:32:07 +08:00
fyears 2a84fae368 fix typo 2024-05-18 02:31:28 +08:00
fyears 3c3426a842 add more size range 2024-05-18 02:30:50 +08:00
fyears 74de7da89a fix timestamp to date using native js 2024-05-18 01:41:54 +08:00
fyears 1f33ac5d7a remove verbose webdav output 2024-05-18 01:34:46 +08:00
fyears 63c54d1956 check password using walkPartial instead of cache 2024-05-18 01:33:20 +08:00
fyears b584f89a95 profiler itself might impact performance 2024-05-18 00:03:21 +08:00
fyears fa17ea074b format 2024-05-17 22:59:44 +08:00
fyears cb98bae79a buck? localforage 2024-05-17 22:59:34 +08:00
28 changed files with 1092 additions and 236 deletions
+2
View File
@@ -100,6 +100,8 @@ Additionally, the plugin author may occasionally visit Obsidian official forum a
### webdav
- Tutorials / Examples:
- [Nextcloud](./docs/remote_services/webdav_nextcloud/README.md)
- [The Good Cloud](./docs/remote_services/webdav_thegoodcloud/README.md)
- [ownCloud](./docs/remote_services/webdav_owncloud/README.md)
- [InfiniCloud](./docs/remote_services/webdav_infinicloud_teracloud/README.md)
- [Synology webdav server](./docs/remote_services/webdav_synology_webdav_server/README.md) | [群晖 webdav server](./docs/remote_services/webdav_synology_webdav_server/README.zh-cn.md)
@@ -0,0 +1,13 @@
# Nextcloud
## Link
<https://nextcloud.com/>
## Steps
1. Install, or find a hosted version.
* The docker version <https://github.com/nextcloud/docker> for internal network, and [Caddy as reverse proxy](https://caddyserver.com/docs/quick-starts/reverse-proxy) (for https), are personally recommended.
* If you find installing Nextcloud by yourselves is difficult, you can find some "Nextcloud's trusted, certified providers" on [Nextcloud Sign up page](https://nextcloud.com/sign-up/); For example, [The Good Cloud](https://thegood.cloud/) there generously provides 2 GB free stoarage space.
* Remotely Save is tested to be working with the docker version and The Good Cloud.
2. Go to Nextcloud's settings. Find the webdav url (something like `https://cloud.example.com/remote.php/dav/files/USERNAME`). Use this (without tailing slash), and your account and your password, in Remotely Save.
@@ -0,0 +1,9 @@
# The Good Cloud
## Link
<https://thegood.cloud/>
## Steps
It's a hosted version of Nextcloud providing 2GB free spaces. See [NextCloud](../webdav_nextcloud/README.md) for more instructions.
+1 -1
View File
@@ -1,7 +1,7 @@
{
"id": "remotely-save",
"name": "Remotely Save",
"version": "0.4.21",
"version": "0.4.25",
"minAppVersion": "0.13.21",
"description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
"author": "fyears",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"id": "remotely-save",
"name": "Remotely Save",
"version": "0.4.21",
"version": "0.4.25",
"minAppVersion": "0.13.21",
"description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
"author": "fyears",
+2 -1
View File
@@ -1,6 +1,6 @@
{
"name": "remotely-save",
"version": "0.4.21",
"version": "0.4.25",
"description": "This is yet another sync plugin for Obsidian app.",
"scripts": {
"dev2": "node esbuild.config.mjs --watch",
@@ -77,6 +77,7 @@
"http-status-codes": "^2.3.0",
"localforage": "^1.10.0",
"localforage-getitems": "^1.4.2",
"localforage-removeitems": "^1.4.0",
"lodash": "^4.17.21",
"lucide": "^0.376.1",
"mime-types": "^2.1.35",
+11 -1
View File
@@ -89,6 +89,7 @@ export interface OnedriveConfig {
username: string;
credentialsShouldBeDeletedAtTime?: number;
remoteBaseDir?: string;
emptyFile: "skip" | "error";
}
export interface WebdisConfig {
@@ -105,7 +106,13 @@ export type SyncDirectionType =
export type CipherMethodType = "rclone-base64" | "openssl-base64" | "unknown";
export type QRExportType = "all_but_oauth2" | "dropbox" | "onedrive";
export type QRExportType =
| "basic_and_advanced"
| "s3"
| "dropbox"
| "onedrive"
| "webdav"
| "webdis";
export interface ProfilerConfig {
enablePrinting?: boolean;
@@ -230,6 +237,7 @@ export interface Entity {
hash?: string;
etag?: string;
synthesizedFolder?: boolean;
synthesizedFile?: boolean;
}
export interface UploadedType {
@@ -250,6 +258,8 @@ export interface MixedEntity {
decision?: DecisionTypeForMixedEntity;
conflictAction?: ConflictActionType;
change?: boolean;
sideNotes?: any;
}
+27 -3
View File
@@ -11,12 +11,30 @@ import {
} from "./localdb";
import type { InternalDBs } from "./localdb";
import { mkdirpInVault } from "./misc";
import type { SyncPlanType } from "./sync";
const getSubsetOfSyncPlan = (x: string, onlyChange: boolean) => {
if (!onlyChange) {
return x;
}
const y: SyncPlanType = JSON.parse(x);
const z: SyncPlanType = Object.fromEntries(
Object.entries(y).filter(([key, val]) => {
if (key === "/$@meta") {
return true;
}
return val.change === undefined || val.change === true;
})
);
return JSON.stringify(z, null, 2);
};
export const exportVaultSyncPlansToFiles = async (
db: InternalDBs,
vault: Vault,
vaultRandomID: string,
howMany: number
howMany: number,
onlyChange: boolean
) => {
console.info("exporting sync plans");
await mkdirpInVault(DEFAULT_DEBUG_FOLDER, vault);
@@ -28,12 +46,18 @@ export const exportVaultSyncPlansToFiles = async (
if (howMany <= 0) {
md =
"Sync plans found:\n\n" +
records.map((x) => "```json\n" + x + "\n```\n").join("\n");
records
.map(
(x) => "```json\n" + getSubsetOfSyncPlan(x, onlyChange) + "\n```\n"
)
.join("\n");
} else {
md =
"Sync plans found:\n\n" +
records
.map((x) => "```json\n" + x + "\n```\n")
.map(
(x) => "```json\n" + getSubsetOfSyncPlan(x, onlyChange) + "\n```\n"
)
.slice(0, howMany)
.join("\n");
}
+1
View File
@@ -3,6 +3,7 @@ import type { Entity } from "./baseTypes";
export abstract class FakeFs {
abstract kind: string;
abstract walk(): Promise<Entity[]>;
abstract walkPartial(): Promise<Entity[]>;
abstract stat(key: string): Promise<Entity>;
abstract mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity>;
abstract writeFile(
+25 -15
View File
@@ -452,13 +452,21 @@ export class FakeFsDropbox extends FakeFs {
}
async walk(): Promise<Entity[]> {
return await this._walk(false);
}
async walkPartial(): Promise<Entity[]> {
return await this._walk(true);
}
async _walk(partial: boolean): Promise<Entity[]> {
await this._init();
let res = await this.dropbox.filesListFolder({
path: `/${this.remoteBaseDir}`,
recursive: true,
recursive: !partial,
include_deleted: false,
limit: 1000,
limit: partial ? 10 : 1000,
});
if (res.status !== 200) {
throw Error(JSON.stringify(res));
@@ -471,20 +479,22 @@ export class FakeFsDropbox extends FakeFs {
.filter((x) => x.path_display !== `/${this.remoteBaseDir}`)
.map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir));
while (res.result.has_more) {
res = await this.dropbox.filesListFolderContinue({
cursor: res.result.cursor,
});
if (res.status !== 200) {
throw Error(JSON.stringify(res));
}
if (!partial) {
while (res.result.has_more) {
res = await this.dropbox.filesListFolderContinue({
cursor: res.result.cursor,
});
if (res.status !== 200) {
throw Error(JSON.stringify(res));
}
const contents2 = res.result.entries;
const unifiedContents2 = contents2
.filter((x) => x[".tag"] !== "deleted")
.filter((x) => x.path_display !== `/${this.remoteBaseDir}`)
.map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir));
unifiedContents.push(...unifiedContents2);
const contents2 = res.result.entries;
const unifiedContents2 = contents2
.filter((x) => x[".tag"] !== "deleted")
.filter((x) => x.path_display !== `/${this.remoteBaseDir}`)
.map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir));
unifiedContents.push(...unifiedContents2);
}
}
fixEntityListCasesInplace(unifiedContents);
+13 -23
View File
@@ -78,8 +78,6 @@ export class FakeFsEncrypt extends FakeFs {
cacheMapOrigToEnc: Record<string, string>;
hasCacheMap: boolean;
kind: string;
innerWalkResultCache?: Entity[];
innerWalkResultCacheTime?: number;
constructor(innerFs: FakeFs, password: string, method: CipherMethodType) {
super();
@@ -89,7 +87,9 @@ export class FakeFsEncrypt extends FakeFs {
this.cacheMapOrigToEnc = {};
this.hasCacheMap = false;
this.kind = `encrypt(${this.innerFs.kind},${method})`;
this.kind = `encrypt(${this.innerFs.kind},${
this.password !== "" ? method : "no password"
})`;
if (method === "rclone-base64") {
this.cipherRClone = new rclone.CipherRclone(password, 5);
@@ -110,26 +110,8 @@ export class FakeFsEncrypt extends FakeFs {
throw Error(`no idea about isFolderAware for method=${this.method}`);
}
/**
* we want a little caching here.
*/
async _getInnerWalkResult(): Promise<Entity[]> {
let innerWalkResult: Entity[] | undefined = undefined;
if (
this.innerWalkResultCacheTime !== undefined &&
this.innerWalkResultCacheTime >= Date.now() - 1000
) {
innerWalkResult = this.innerWalkResultCache!;
} else {
innerWalkResult = await this.innerFs.walk();
this.innerWalkResultCache = innerWalkResult;
this.innerWalkResultCacheTime = Date.now();
}
return innerWalkResult;
}
async isPasswordOk(): Promise<PasswordCheckType> {
const innerWalkResult = await this._getInnerWalkResult();
const innerWalkResult = await this.walkPartial();
if (innerWalkResult === undefined || innerWalkResult.length === 0) {
// remote empty
@@ -186,8 +168,16 @@ export class FakeFsEncrypt extends FakeFs {
}
async walk(): Promise<Entity[]> {
const innerWalkResult = await this._getInnerWalkResult();
const innerWalkResult = await this.innerFs.walk();
return await this._dealWithWalk(innerWalkResult);
}
async walkPartial(): Promise<Entity[]> {
const innerWalkResult = await this.innerFs.walkPartial();
return await this._dealWithWalk(innerWalkResult);
}
async _dealWithWalk(innerWalkResult: Entity[]): Promise<Entity[]> {
const res: Entity[] = [];
if (this.isPasswordEmpty()) {
+4
View File
@@ -109,6 +109,10 @@ export class FakeFsLocal extends FakeFs {
return local;
}
async walkPartial(): Promise<Entity[]> {
return await this.walk();
}
async stat(key: string): Promise<Entity> {
const statRes = await statFix(this.vault, key);
if (statRes === undefined || statRes === null) {
+4
View File
@@ -13,6 +13,10 @@ export class FakeFsMock extends FakeFs {
throw new Error("Method not implemented.");
}
async walkPartial(): Promise<Entity[]> {
return await this.walk();
}
async stat(key: string): Promise<Entity> {
throw new Error("Method not implemented.");
}
+47 -5
View File
@@ -32,6 +32,7 @@ export const DEFAULT_ONEDRIVE_CONFIG: OnedriveConfig = {
deltaLink: "",
username: "",
credentialsShouldBeDeletedAtTime: 0,
emptyFile: "skip",
};
////////////////////////////////////////////////////////////////////////////////
@@ -344,6 +345,7 @@ const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => {
mtimeCli: mtimeCli,
size: isFolder ? 0 : x.size!,
sizeRaw: isFolder ? 0 : x.size!,
synthesizedFile: false,
// hash: ?? // TODO
};
};
@@ -682,6 +684,31 @@ export class FakeFsOnedrive extends FakeFs {
return unifiedContents;
}
async walkPartial(): Promise<Entity[]> {
await this._init();
const DELTA_LINK_KEY = "@odata.deltaLink";
const res = await this._getJson(
`/drive/special/approot:/${this.remoteBaseDir}:/delta`
);
const driveItems = res.value as DriveItem[];
// console.debug(driveItems);
// lastly we should have delta link?
if (DELTA_LINK_KEY in res) {
this.onedriveConfig.deltaLink = res[DELTA_LINK_KEY];
await this.saveUpdatedConfigFunc();
}
// unify everything to Entity
const unifiedContents = driveItems
.map((x) => fromDriveItemToEntity(x, this.remoteBaseDir))
.filter((x) => x.key !== "/");
return unifiedContents;
}
async stat(key: string): Promise<Entity> {
await this._init();
return await this._statFromRoot(getOnedrivePath(key, this.remoteBaseDir));
@@ -760,7 +787,8 @@ export class FakeFsOnedrive extends FakeFs {
content,
mtime,
ctime,
key
key,
this.onedriveConfig.emptyFile
);
}
@@ -769,12 +797,26 @@ export class FakeFsOnedrive extends FakeFs {
content: ArrayBuffer,
mtime: number,
ctime: number,
origKey: string
origKey: string,
emptyFile: "skip" | "error"
): Promise<Entity> {
if (content.byteLength === 0) {
throw Error(
`${origKey}: Empty file is not allowed in OneDrive, and please write something in it.`
);
if (emptyFile === "error") {
throw Error(
`${origKey}: Empty file is not allowed in OneDrive, and please write something in it.`
);
} else {
return {
key: origKey,
keyRaw: origKey,
mtimeSvr: mtime,
mtimeCli: mtime,
size: 0,
sizeRaw: 0,
synthesizedFile: true,
// hash: ?? // TODO
};
}
}
const ctimeStr = new Date(ctime).toISOString();
+30 -13
View File
@@ -399,9 +399,16 @@ export class FakeFsS3 extends FakeFs {
}
async walk(): Promise<Entity[]> {
const res = (await this._walkFromRoot(this.s3Config.remotePrefix)).filter(
(x) => x.key !== "" && x.key !== "/"
);
const res = (
await this._walkFromRoot(this.s3Config.remotePrefix, false)
).filter((x) => x.key !== "" && x.key !== "/");
return res;
}
async walkPartial(): Promise<Entity[]> {
const res = (
await this._walkFromRoot(this.s3Config.remotePrefix, true)
).filter((x) => x.key !== "" && x.key !== "/");
return res;
}
@@ -409,19 +416,23 @@ export class FakeFsS3 extends FakeFs {
* the input key contains basedir (prefix),
* but the result doesn't contain it.
*/
async _walkFromRoot(prefixOfRawKeys: string | undefined) {
async _walkFromRoot(prefixOfRawKeys: string | undefined, partial: boolean) {
const confCmd = {
Bucket: this.s3Config.s3BucketName,
} as ListObjectsV2CommandInput;
if (prefixOfRawKeys !== undefined && prefixOfRawKeys !== "") {
confCmd.Prefix = prefixOfRawKeys;
}
if (partial) {
confCmd.MaxKeys = 10; // no need to list more!
}
const contents = [] as _Object[];
const mtimeRecords: Record<string, number> = {};
const ctimeRecords: Record<string, number> = {};
const partsConcurrency = partial ? 1 : this.s3Config.partsConcurrency;
const queueHead = new PQueue({
concurrency: this.s3Config.partsConcurrency,
concurrency: partsConcurrency,
autoStart: true,
});
queueHead.on("error", (error) => {
@@ -473,14 +484,20 @@ export class FakeFsS3 extends FakeFs {
}
}
isTruncated = rsp.IsTruncated ?? false;
confCmd.ContinuationToken = rsp.NextContinuationToken;
if (
isTruncated &&
(confCmd.ContinuationToken === undefined ||
confCmd.ContinuationToken === "")
) {
throw Error("isTruncated is true but no continuationToken provided");
if (partial) {
// do not loop over
isTruncated = false;
} else {
// loop over
isTruncated = rsp.IsTruncated ?? false;
confCmd.ContinuationToken = rsp.NextContinuationToken;
if (
isTruncated &&
(confCmd.ContinuationToken === undefined ||
confCmd.ContinuationToken === "")
) {
throw Error("isTruncated is true but no continuationToken provided");
}
}
} while (isTruncated);
+387 -28
View File
@@ -4,7 +4,9 @@ import { getReasonPhrase } from "http-status-codes/build/cjs/utils-functions";
import chunk from "lodash/chunk";
import cloneDeep from "lodash/cloneDeep";
import flatten from "lodash/flatten";
import { Platform, requestUrl } from "obsidian";
import isString from "lodash/isString";
import { nanoid } from "nanoid";
import { Platform, type RequestUrlParam, requestUrl } from "obsidian";
import type {
FileStat,
RequestOptionsWithState,
@@ -15,7 +17,7 @@ import type {
import type { Entity, WebdavConfig } from "./baseTypes";
import { VALID_REQURL } from "./baseTypesObs";
import { FakeFs } from "./fsAll";
import { bufferToArrayBuffer } from "./misc";
import { bufferToArrayBuffer, delay, splitFileSizeToChunkRanges } from "./misc";
/**
* https://stackoverflow.com/questions/32850898/how-to-check-if-a-string-has-any-non-iso-8859-1-characters-with-javascript
@@ -55,20 +57,35 @@ if (VALID_REQURL) {
retractedHeaders["authorization"] = "<retracted>";
}
console.debug(`before request:`);
console.debug(`url: ${options.url}`);
console.debug(`method: ${options.method}`);
console.debug(`headers: ${JSON.stringify(retractedHeaders, null, 2)}`);
console.debug(`reqContentType: ${reqContentType}`);
// console.debug(`before request:`);
// console.debug(`url: ${options.url}`);
// console.debug(`method: ${options.method}`);
// console.debug(`headers: ${JSON.stringify(retractedHeaders, null, 2)}`);
// console.debug(`reqContentType: ${reqContentType}`);
let r = await requestUrl({
const p: RequestUrlParam = {
url: options.url,
method: options.method,
body: options.data as string | ArrayBuffer,
// body: options.data as string | ArrayBuffer,
headers: transformedHeaders,
contentType: reqContentType,
throw: false,
});
};
if (
options.data === undefined ||
options.data === null ||
isString(options.data)
) {
p.body = options.data;
} else {
if (typeof (options.data as any).transfer === "function") {
p.body = (options.data as any).transfer();
} else {
p.body = options.data as ArrayBuffer;
}
}
let r = await requestUrl(p);
if (
r.status === 401 &&
@@ -82,19 +99,13 @@ if (VALID_REQURL) {
// if a folder doesn't exist without slash, the servers return 401 instead of 404
// here is a dirty hack that works
console.debug(`so we have 401, try appending request url with slash`);
r = await requestUrl({
url: `${options.url}/`,
method: options.method,
body: options.data as string | ArrayBuffer,
headers: transformedHeaders,
contentType: reqContentType,
throw: false,
});
p.url = `${options.url}/`;
r = await requestUrl(p);
}
console.debug(`after request:`);
// console.debug(`after request:`);
const rspHeaders = objKeyToLower({ ...r.headers });
console.debug(`rspHeaders: ${JSON.stringify(rspHeaders, null, 2)}`);
// console.debug(`rspHeaders: ${JSON.stringify(rspHeaders, null, 2)}`);
for (const key in rspHeaders) {
if (rspHeaders.hasOwnProperty(key)) {
// avoid the error:
@@ -111,7 +122,7 @@ if (VALID_REQURL) {
// }
// }
if (!onlyAscii(rspHeaders[key])) {
console.debug(`rspHeaders[key] needs encode: ${key}`);
// console.debug(`rspHeaders[key] needs encode: ${key}`);
rspHeaders[key] = encodeURIComponent(rspHeaders[key]);
}
}
@@ -119,7 +130,7 @@ if (VALID_REQURL) {
let r2: Response | undefined = undefined;
const statusText = getReasonPhrase(r.status);
console.debug(`statusText: ${statusText}`);
// console.debug(`statusText: ${statusText}`);
if ([101, 103, 204, 205, 304].includes(r.status)) {
// A null body status is a status that is 101, 103, 204, 205, or 304.
// https://fetch.spec.whatwg.org/#statuses
@@ -203,6 +214,14 @@ const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string): Entity => {
};
};
const tryEncodeURI = (x: string) => {
if (x.includes("%")) {
// likely encoded before!
return x;
}
return encodeURI(x);
};
export class FakeFsWebdav extends FakeFs {
kind: "webdav";
@@ -212,6 +231,10 @@ export class FakeFsWebdav extends FakeFs {
vaultFolderExists: boolean;
saveUpdatedConfigFunc: () => Promise<any>;
supportNativePartial: boolean;
isNextcloud: boolean;
nextcloudUploadServerAddress: string;
constructor(
webdavConfig: WebdavConfig,
vaultName: string,
@@ -220,10 +243,14 @@ export class FakeFsWebdav extends FakeFs {
super();
this.kind = "webdav";
this.webdavConfig = cloneDeep(webdavConfig);
this.webdavConfig.address = encodeURI(this.webdavConfig.address);
this.webdavConfig.address = tryEncodeURI(this.webdavConfig.address);
this.remoteBaseDir = this.webdavConfig.remoteBaseDir || vaultName || "";
this.vaultFolderExists = false;
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
this.supportNativePartial = false;
this.isNextcloud = false;
this.nextcloudUploadServerAddress = "";
}
async _init() {
@@ -293,6 +320,81 @@ export class FakeFsWebdav extends FakeFs {
);
}
}
await this._checkPartialSupport();
}
/**
* <server>/remote.php/dav/files/<userid>
* => <server>/remote.php/dav/uploads/<userid>
*/
_getnextcloudUploadServerAddress = () => {
let k = this.webdavConfig.address;
if (k.endsWith('/')) {
// no tailing slash
k = k.substring(0, k.length-1);
}
const s = k.split("/");
if (
s.length > 3 &&
s[s.length - 3] === "dav" &&
s[s.length - 2] === "files" &&
s[s.length - 1] !== ""
) {
s[s.length - 2] = "uploads";
return s.join("/");
}
throw Error(`cannot construct upload address for ${s}`);
};
async _checkPartialSupport() {
const compliance = await this.client.getDAVCompliance(
`/${this.remoteBaseDir}/`
);
for (const c of compliance.compliance) {
// nextcloud AND with an account
if (
c.toLocaleLowerCase().includes("nextcloud") &&
this.webdavConfig.username !== "" &&
this.webdavConfig.password !== ""
) {
// the address is parsable
const s = this.webdavConfig.address.split("/");
if (
s.length > 3 &&
s[s.length - 3] === "dav" &&
s[s.length - 2] === "files" &&
s[s.length - 1] !== ""
) {
this.isNextcloud = true;
this.nextcloudUploadServerAddress =
this._getnextcloudUploadServerAddress();
console.debug(
`isNextcloud=${this.isNextcloud}, uploadFolder=${this.nextcloudUploadServerAddress}`
);
return true;
} else {
return false;
}
}
}
// taken from https://github.com/perry-mitchell/webdav-client/blob/master/source/operations/partialUpdateFileContents.ts
// which is under MIT license
if (
(compliance.server.includes("Apache") &&
compliance.compliance.includes(
"<http://apache.org/dav/propset/fs/1>"
)) ||
compliance.compliance.includes("sabredav-partialupdate")
) {
this.supportNativePartial = true;
console.debug(`supportNativePartial=true`);
return true;
}
return false;
}
async walk(): Promise<Entity[]> {
@@ -329,7 +431,20 @@ export class FakeFsWebdav extends FakeFs {
// glob: "/**" /* avoid dot files by using glob */,
}) as Promise<FileStat[]>;
});
const r2 = flatten(await Promise.all(r));
const r3 = await Promise.all(r);
for (const r4 of r3) {
if (
this.webdavConfig.address.includes("jianguoyun.com") &&
r4.length >= 749
) {
// https://help.jianguoyun.com/?p=2064
// no more than 750 per request
throw Error(
`出错:坚果云 api 有限制,文件列表加载不全。终止同步!`
);
}
}
const r2 = flatten(r3);
subContents.push(...r2);
}
for (let i = 0; i < subContents.length; ++i) {
@@ -357,6 +472,19 @@ export class FakeFsWebdav extends FakeFs {
return contents.map((x) => fromWebdavItemToEntity(x, this.remoteBaseDir));
}
async walkPartial(): Promise<Entity[]> {
await this._init();
const contents = (await this.client.getDirectoryContents(
`/${this.remoteBaseDir}`,
{
deep: false, // partial, no need to recursive here
details: false /* no need for verbose details here */,
}
)) as FileStat[];
return contents.map((x) => fromWebdavItemToEntity(x, this.remoteBaseDir));
}
async stat(key: string): Promise<Entity> {
await this._init();
const fullPath = getWebdavPath(key, this.remoteBaseDir);
@@ -384,8 +512,10 @@ export class FakeFsWebdav extends FakeFs {
mtime?: number,
ctime?: number
): Promise<Entity> {
// the sync algorithm should do recursive manually already.
// if we set recursive: true here, Digest auth will return some error inside the PROPFIND
await this.client.createDirectory(key, {
recursive: true,
recursive: false,
});
return await this._statFromRoot(key);
}
@@ -401,22 +531,251 @@ export class FakeFsWebdav extends FakeFs {
}
await this._init();
const uploadFile = getWebdavPath(key, this.remoteBaseDir);
return await this._writeFileFromRoot(uploadFile, content, mtime, ctime);
return await this._writeFileFromRoot(
uploadFile,
content,
mtime,
ctime,
key
);
}
async _writeFileFromRoot(
key: string,
content: ArrayBuffer,
mtime: number,
ctime: number
ctime: number,
origKey: string
): Promise<Entity> {
// less than 10 MB
if (content.byteLength <= 10 * 1024 * 1024) {
return await this._writeFileFromRootFull(
key,
content,
mtime,
ctime,
origKey
);
}
// larger than 10 MB
if (!this.isNextcloud && !this.supportNativePartial) {
// give up and upload by whole, and directly return
return await this._writeFileFromRootFull(
key,
content,
mtime,
ctime,
origKey
);
}
// try to upload by chunks
try {
if (this.isNextcloud) {
return await this._writeFileFromRootNextcloud(
key,
content,
mtime,
ctime,
origKey
);
} else if (this.supportNativePartial) {
return await this._writeFileFromRootNativePartial(
key,
content,
mtime,
ctime,
origKey
);
}
throw Error(`Error: partial upload / update method is not implemented??`);
} catch (e) {
console.error(
`we fail to write file partially for nextcloud or apache or sabre/dav, stop!`
);
console.error(e);
throw e;
// this.isNextcloud = false;
// this.supportNativePartial = false;
// return await this._writeFileFromRootFull(
// key,
// content,
// mtime,
// ctime,
// origKey
// );
}
}
async _writeFileFromRootFull(
key: string,
content: ArrayBuffer,
mtime: number,
ctime: number,
origKey: string
): Promise<Entity> {
// console.debug(`start _writeFileFromRootFull`);
await this.client.putFileContents(key, content, {
overwrite: true,
onUploadProgress: (progress: any) => {
console.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
},
});
return await this._statFromRoot(key);
const k = await this._statFromRoot(key);
// console.debug(`end _writeFileFromRootFull`);
return k;
}
/**
* https://docs.nextcloud.com/server/latest/developer_manual/client_apis/WebDAV/chunking.html
* @param key
* @param content
* @param mtime
* @param ctime
* @returns
*/
async _writeFileFromRootNextcloud(
key: string,
content: ArrayBuffer,
mtime: number,
ctime: number,
origKey: string
): Promise<Entity> {
if (key.endsWith("/")) {
throw Error(
`key=${key} should not have tailing slash in _writeFileFromRootNextcloud`
);
}
const destUrl = `${this.webdavConfig.address}/${encodeURI(key)}`;
console.debug(`destUrl=${destUrl}`);
const getTmpFolder = (x: string) => {
if (x.endsWith("/")) {
throw Error(`file to upload by chunk should not ends with /`);
}
const y = x.split("/");
const z = encodeURI(`${y[y.length - 1]}`);
return z;
};
const uploadServerAddress = this.nextcloudUploadServerAddress;
console.debug(`uploadServerAddress=${uploadServerAddress}`);
const tmpFolderName = getTmpFolder(key);
console.debug(`tmpFolderName=${tmpFolderName}`);
const clientForUpload = createClient(uploadServerAddress, {
username: this.webdavConfig.username,
password: this.webdavConfig.password,
headers: {
"Cache-Control": "no-cache",
},
authType:
this.webdavConfig.authType === "digest"
? AuthType.Digest
: AuthType.Password,
});
// create folder
await clientForUpload.createDirectory(tmpFolderName, {
method: "MKCOL",
headers: {
Destination: destUrl,
},
});
console.debug(`finish creating folder`);
// upload by chunks
const sizePerChunk = 5 * 1024 * 1024; // 5 mb
const chunkRanges = splitFileSizeToChunkRanges(
content.byteLength,
sizePerChunk
);
for (let i = 0; i < chunkRanges.length; ++i) {
const { start, end } = chunkRanges[i];
const tmpFileName = `${i + 1}`.padStart(5, "0");
const tmpFileNameWithFolder = `${tmpFolderName}/${tmpFileName}`;
console.debug(
`start to upload chunk ${
i + 1
} to ${tmpFileNameWithFolder} with startInclusive=${start}, endInclusive=${end}`
);
await clientForUpload.putFileContents(
tmpFileNameWithFolder,
content.slice(start, end + 1),
{
headers: {
Destination: destUrl,
"OC-Total-Length": `${content.byteLength}`,
},
}
);
}
console.debug(`finish upload all chunks`);
// move to assemble
const fakeFileToMoveUrl = `${tmpFolderName}/.file`;
console.debug(`fakeFileToMoveUrl=${fakeFileToMoveUrl}`);
await clientForUpload.customRequest(fakeFileToMoveUrl, {
method: "MOVE",
headers: {
Destination: destUrl,
"OC-Total-Length": `${content.byteLength}`,
},
});
console.debug(`finish moving file`);
// TODO: setting X-OC-Mtime
// stat
console.debug(`before stat origKey=${origKey}`);
const k = await this.stat(origKey);
console.debug(`after stat`);
if (k.sizeRaw !== content.byteLength) {
// we failed!
this.isNextcloud = false; // give up next time!
const err = `unable to upload file ${key} by chunks to nextcloud`;
console.error(err);
throw Error(err);
}
console.debug(`after stat, k=${JSON.stringify(k, null, 2)}`);
return k;
}
async _writeFileFromRootNativePartial(
key: string,
content: ArrayBuffer,
mtime: number,
ctime: number,
origKey: string
): Promise<Entity> {
// firstly upload a 0-byte data
await this._writeFileFromRootFull(
key,
new ArrayBuffer(0),
mtime,
ctime,
origKey
);
// then "update" by chunks
const sizePerChunk = 5 * 1024 * 1024; // 5 mb
const chunkRanges = splitFileSizeToChunkRanges(
content.byteLength,
sizePerChunk
);
for (let i = 0; i < chunkRanges.length; ++i) {
const { start, end } = chunkRanges[i];
await this.client.partialUpdateFileContents(
key,
start,
end,
content.slice(start, end + 1)
);
}
// lastly return
return await this.stat(origKey);
}
async readFile(key: string): Promise<ArrayBuffer> {
+19
View File
@@ -133,6 +133,25 @@ export class FakeFsWebdis extends FakeFs {
return res;
}
async walkPartial(): Promise<Entity[]> {
let cursor = "0";
const res: Entity[] = [];
const command = `SCAN/${cursor}/MATCH/rs:fs:v1:*:meta/COUNT/10`; // fewer keys
const rsp = (await (await this._fetchCommand("GET", command)).json())[
"SCAN"
];
// console.debug(rsp);
cursor = rsp[0];
for (const fullKeyWithMeta of rsp[1]) {
const realKey = getOrigPath(fullKeyWithMeta, this.remoteBaseDir);
res.push(await this.stat(realKey));
}
// no need to loop over cursor
// console.debug(`walk res:`);
// console.debug(res);
return res;
}
async stat(key: string): Promise<Entity> {
const fullKey = getWebdisPath(key, this.remoteBaseDir);
return await this._statFromRaw(fullKey);
+10 -1
View File
@@ -17,14 +17,23 @@ export const exportQrCodeUri = async (
) => {
let settings2: Partial<RemotelySavePluginSettings> = {};
if (exportFields === "all_but_oauth2") {
if (exportFields === "basic_and_advanced") {
settings2 = cloneDeep(settings);
delete settings2.s3;
delete settings2.dropbox;
delete settings2.onedrive;
delete settings2.webdav;
delete settings2.webdis;
} else if (exportFields === "s3") {
settings2 = { s3: cloneDeep(settings.s3) };
} else if (exportFields === "dropbox") {
settings2 = { dropbox: cloneDeep(settings.dropbox) };
} else if (exportFields === "onedrive") {
settings2 = { onedrive: getShrinkedSettings(settings.onedrive) };
} else if (exportFields === "webdav") {
settings2 = { webdav: cloneDeep(settings.webdav) };
} else if (exportFields === "webdis") {
settings2 = { webdis: cloneDeep(settings.webdis) };
}
delete settings2.vaultRandomID;
+20 -8
View File
@@ -11,7 +11,7 @@
"syncrun_step1": "1/8 Remotely Save is preparing ({{serviceType}})",
"syncrun_step2": "2/8 Starting to fetch remote meta data and check password.",
"syncrun_passworderr": "Something goes wrong while checking password.",
"syncrun_step3": "3/8 Startting to process remote data.",
"syncrun_step3": "3/8 Starting to process remote data.",
"syncrun_step4": "4/8 Starting to fetch local meta data.",
"syncrun_step5": "5/8 Starting to fetch local prev sync data.",
"syncrun_step6": "6/8 Starting to generate sync plan.",
@@ -38,6 +38,7 @@
"protocol_onedrive_connect_unknown": "Do not know how to deal with the callback: {{params}}",
"command_startsync": "start sync",
"command_drynrun": "start sync (dry run only)",
"command_exportsyncplans_1_only_change": "export sync plans (latest 1) (change part)",
"command_exportsyncplans_1": "export sync plans (latest 1)",
"command_exportsyncplans_5": "export sync plans (latest 5)",
"command_exportsyncplans_all": "export sync plans (all)",
@@ -115,9 +116,9 @@
"modal_sizesconflict_copynotice": "All the sizes conflicts info have been copied to the clipboard!",
"settings_basic": "Basic Settings",
"settings_password": "Encryption Password",
"settings_password_desc": "Password for E2E encryption. Empty for no password. You need to click \"Confirm\". Attention: The password and other info are saved locally. After changing the password, you need to manually delete every original files in the remote, and re-sync (so that upload) the encrypted files again.",
"settings_password_desc": "Password for E2E encryption. Empty for no password. You need to click \"Confirm\". Attention: The password and other info are saved locally. After changing the password, you need to manually delete all files from the remote location and re-sync to upload the encrypted files again.",
"settings_encryptionmethod": "Encryption Method",
"settings_encryptionmethod_desc": "Encryption method for E2E encryption. RClone Crypt format is recommended but it doesn't encrypt path structure. OpenSSL enc is the legacy format of this plugin. <b>Both are not affliated with official RClone and OpenSSL product or community.</b> Attention: After switching the method, you need to manually delete every original files in the remote and re-sync (so that upload) the encrypted files again. More info in the <a href='https://github.com/remotely-save/remotely-save/tree/master/docs/encryption'>online doc</a>.",
"settings_encryptionmethod_desc": "Encryption method for E2E encryption. The RClone Crypt format is recommended, although it does not encrypt the path structure. OpenSSL enc is the legacy format of this plugin. <b>Both are not affliated with the official RClone and OpenSSL products or communities.</b> Attention: After switching encryption methods, you need to manually delete all original files from the remote location and re-sync to upload the encrypted files again. More info is available in the <a href='https://github.com/remotely-save/remotely-save/tree/master/docs/encryption'>online doc</a>.",
"settings_encryptionmethod_rclone": "RClone Crypt (recommended)",
"settings_encryptionmethod_openssl": "OpenSSL enc (legacy)",
@@ -129,7 +130,7 @@
"settings_autorun_10min": "every 10 minutes",
"settings_autorun_30min": "every 30 minutes",
"settings_runoncestartup": "Run Once On Start Up Automatically",
"settings_runoncestartup_desc": "This settings allows setting running ONCE on start up automatically. This will take effect on NEXT start up after changing. This setting, is different from \"schedule for auto run\" which starts syncing after EVERY interval.",
"settings_runoncestartup_desc": "This setting causes the sync to run once automatically at startup. Changes will take effect on the NEXT startup. This setting is different from the \"schedule for auto run\" setting, which initiates syncing after every specified interval.",
"settings_runoncestartup_notset": "(not set)",
"settings_runoncestartup_1sec": "sync once after 1 second of start up",
"settings_runoncestartup_10sec": "sync once after 10 seconds of start up",
@@ -219,8 +220,12 @@
"settings_onedrive_auth": "Auth",
"settings_onedrive_auth_desc": "Auth.",
"settings_onedrive_auth_button": "Auth",
"settings_onedrive_connect_succ": "Great! We can connect to Onedrive!",
"settings_onedrive_connect_fail": "We cannot connect to Onedrive.",
"settings_onedrive_connect_succ": "Great! We can connect to OneDrive!",
"settings_onedrive_connect_fail": "We cannot connect to OneDrive.",
"settings_onedrive_emptyfile": "Empty File Handling",
"settings_onedrive_emptyfile_desc": "OneDrive doesn't allow uploading empty file (even in its official website). Do you want to show up errors or silently skip the empty files?",
"settings_onedrive_emptyfile_skip": "Skip",
"settings_onedrive_emptyfile_error": "Error and abort",
"settings_webdav": "Remote For Webdav",
"settings_webdav_disclaimer1": "Disclaimer: The information is stored locally. Other malicious/harmful/faulty plugins may read the info. If you see any unintentional access to your webdav server, please immediately change the username and password.",
"settings_webdav_cors_os": "Obsidian desktop>=0.13.25 or iOS>=1.1.1 or Android>=1.2.1 supports bypassing CORS locally. But you are using an old version, and you're suggested to upgrade Obsidian.",
@@ -233,7 +238,7 @@
"settings_webdav_password": "Password",
"settings_webdav_password_desc": "Password. Attention: the password and other info are saved locally.",
"settings_webdav_auth": "Auth Type",
"settings_webdav_auth_desc": "If no password, this option would be ignored.",
"settings_webdav_auth_desc": "If no password is provided, this option will be ignored.",
"settings_webdav_depth": "Depth Header Sent To Servers",
"settings_webdav_depth_desc": "Webdav servers should be configured to allow requests with header Depth being '1' or 'Infinity'. If you are not sure what's this, choose \"depth='1'\". If you are sure your server supports depth='infinity', please choose that to get way better performance.",
"settings_webdav_depth_1": "only supports depth='1'",
@@ -284,6 +289,9 @@
"settings_protectmodifypercentage_000_desc": "0 (always block)",
"settings_protectmodifypercentage_050_desc": "50 (default)",
"settings_protectmodifypercentage_100_desc": "100 (disable the protection)",
"settings_protectmodifypercentage_custom_desc": "custom",
"settings_protectmodifypercentage_customfield": "Custom Abort Sync If Modification Above Percentage",
"settings_protectmodifypercentage_customfield_desc": "You need to enter a number between 0 (inclusive) and 100 (inclusive). Float number is also allowed.",
"setting_syncdirection": "Sync Direction",
"setting_syncdirection_desc": "Which direction should the plugin sync to? Please be aware that only CHANGED files (based on time and size) are synced regardless any option.",
"setting_syncdirection_bidirectional_desc": "Bidirectional (default)",
@@ -294,9 +302,12 @@
"settings_importexport": "Import and Export Partial Settings",
"settings_export": "Export",
"settings_export_desc": "Export settings by generating a QR code or URI.",
"settings_export_all_but_oauth2_button": "Export Non-Oauth2 Part",
"settings_export_basic_and_advanced_button": "Export Basic And Advanced Part",
"settings_export_s3_button": "Export S3 Part",
"settings_export_dropbox_button": "Export Dropbox Part",
"settings_export_onedrive_button": "Export OneDrive Part",
"settings_export_webdav_button": "Export Webdav Part",
"settings_export_webdis_button": "Export Webdis Part",
"settings_import": "Import",
"settings_import_desc": "Paste the exported URI into here and click \"Import\". Or, you can open a camera or scan-qrcode app to scan the QR code.",
"settings_import_button": "Import",
@@ -314,6 +325,7 @@
"settings_viewconsolelog_desc": "On desktop, please press \"ctrl+shift+i\" or \"cmd+shift+i\" to view the log. On mobile, please install the third-party plugin <a href='https://obsidian.md/plugins?search=Logstravaganza'>Logstravaganza</a> to export the console log to a note.",
"settings_syncplans": "Export Sync Plans",
"settings_syncplans_desc": "Sync plans are created every time after you trigger sync and before the actual sync. Useful to know what would actually happen in those sync. Click the button to export sync plans.",
"settings_syncplans_button_1_only_change": "Export latest 1 (change part)",
"settings_syncplans_button_1": "Export latest 1",
"settings_syncplans_button_5": "Export latest 5",
"settings_syncplans_button_all": "Export All",
+13 -1
View File
@@ -39,6 +39,7 @@
"command_startsync": "开始同步(start sync",
"command_drynrun": "开始同步(空跑模式)(start sync (dry run only)",
"command_exportsyncplans_json": "导出同步计划为 json 格式(export sync plans in json format",
"command_exportsyncplans_1_only_change": "导出同步计划(最近 1 次)(仅修改部分)(export sync plans (latest 1) (change part)",
"command_exportsyncplans_1": "导出同步计划(最近 1 次)(export sync plans (latest 1)",
"command_exportsyncplans_5": "导出同步计划(最近 5 次)(export sync plans (latest 5)",
"command_exportsyncplans_all": "导出同步计划(所有)(export sync plans (all)",
@@ -220,6 +221,10 @@
"settings_onedrive_auth_button": "鉴权",
"settings_onedrive_connect_succ": "很好!我们可连接上 OneDrive",
"settings_onedrive_connect_fail": "我们未能连接上 OneDrive。",
"settings_onedrive_emptyfile": "空文件处理",
"settings_onedrive_emptyfile_desc": "OneDrive 不允许上传空文件(即使官网也是不允许的)。那么你想跳过空文件还是返回错误?",
"settings_onedrive_emptyfile_skip": "跳过",
"settings_onedrive_emptyfile_error": "返回错误和中断",
"settings_webdav": "Webdav 设置",
"settings_webdav_disclaimer1": "声明:您所输入的信息存储于本地。其它有害的或者出错的插件,是有可能读取到这些信息的。如果您发现了 Webdav 服务器有不符合预期的访问,请立刻修改用户名和密码。",
"settings_webdav_cors_os": "Obsidian 桌面版>=0.13.25 或 iOS>=1.1.1 或 Android>=1.2.1 支持跳过 CORS 设置。但您正在使用旧版,建议升级。",
@@ -283,6 +288,9 @@
"settings_protectmodifypercentage_000_desc": "0(总是强制中止)",
"settings_protectmodifypercentage_050_desc": "50(默认值)",
"settings_protectmodifypercentage_100_desc": "100(去除此保护)",
"settings_protectmodifypercentage_custom_desc": "自定义",
"settings_protectmodifypercentage_customfield": "如果修改超过自定义百分比则中止同步",
"settings_protectmodifypercentage_customfield_desc": "您需要输入 0(含)~ 100(含)的数字。小数也是可以的。",
"setting_syncdirection": "同步方向",
"setting_syncdirection_desc": "插件应该向哪里同步?注意每个选项都是只有修改了的文件(基于修改时间和大小判断)才会触发同步动作。",
"setting_syncdirection_bidirectional_desc": "双向同步(默认)",
@@ -293,9 +301,12 @@
"settings_importexport": "导入导出部分设置",
"settings_export": "导出",
"settings_export_desc": "用 QR 码或 URI 导出设置信息。",
"settings_export_all_but_oauth2_button": "导出非 Oauth2 部分",
"settings_export_basic_and_advanced_button": "导出基本或进阶设置",
"settings_export_s3_button": "导出 S3 部分",
"settings_export_dropbox_button": "导出 Dropbox 部分",
"settings_export_onedrive_button": "导出 OneDrive 部分",
"settings_export_webdav_button": "导出 Webdav 部分",
"settings_export_webdis_button": "导出 Webdis 部分",
"settings_import": "导入",
"settings_import_desc": "粘贴之前导出的 URI 到这里然后点击“导入”。或,使用拍摄 app 或者扫描 QR 码的 app,来扫描对应的 QR 码。",
"settings_import_button": "导入",
@@ -313,6 +324,7 @@
"settings_viewconsolelog_desc": "电脑上,输入“ctrl+shift+i”或“cmd+shift+i”来查看终端输出。手机上,安装第三方插件 <a href='https://obsidian.md/plugins?search=Logstravaganza'>Logstravaganza</a> 来导出终端输出到一篇笔记上。",
"settings_syncplans": "导出同步计划",
"settings_syncplans_desc": "每次您启动同步,并在实际上传下载前,插件会生成同步计划。它可以使您知道每次同步发生了什么。点击按钮可以导出同步计划。",
"settings_syncplans_button_1_only_change": "导出最近 1 次(仅修改部分)",
"settings_syncplans_button_1": "导出最近 1 次",
"settings_syncplans_button_5": "导出最近 5 次",
"settings_syncplans_button_all": "导出所有",
+13 -1
View File
@@ -38,6 +38,7 @@
"protocol_onedrive_connect_unknown": "不知道如何處理此 callback{{params}}",
"command_startsync": "開始同步(start sync",
"command_drynrun": "開始同步(空跑模式)(start sync (dry run only)",
"command_exportsyncplans_1_only_change": "匯出同步計劃(最近 1 次)(僅修改部分)(export sync plans (latest 1) (change part)",
"command_exportsyncplans_1": "匯出同步計劃(最近 1 次)(export sync plans (latest 1)",
"command_exportsyncplans_5": "匯出同步計劃(最近 5 次)(export sync plans (latest 5)",
"command_exportsyncplans_all": "匯出同步計劃(所有)(export sync plans (all)",
@@ -219,6 +220,10 @@
"settings_onedrive_auth_button": "鑑權",
"settings_onedrive_connect_succ": "很好!我們可連線上 OneDrive",
"settings_onedrive_connect_fail": "我們未能連線上 OneDrive。",
"settings_onedrive_emptyfile": "空檔案處理",
"settings_onedrive_emptyfile_desc": "OneDrive 不允許上傳空檔案(即使官網也是不允許的)。那麼你想跳過空檔案還是返回錯誤?",
"settings_onedrive_emptyfile_skip": "跳過",
"settings_onedrive_emptyfile_error": "返回錯誤和中斷",
"settings_webdav": "Webdav 設定",
"settings_webdav_disclaimer1": "宣告:您所輸入的資訊儲存於本地。其它有害的或者出錯的外掛,是有可能讀取到這些資訊的。如果您發現了 Webdav 伺服器有不符合預期的訪問,請立刻修改使用者名稱和密碼。",
"settings_webdav_cors_os": "Obsidian 桌面版>=0.13.25 或 iOS>=1.1.1 或 Android>=1.1.1 支援跳過 CORS 設定。但您正在使用舊版,建議升級。",
@@ -282,6 +287,9 @@
"settings_protectmodifypercentage_000_desc": "0(總是強制中止)",
"settings_protectmodifypercentage_050_desc": "50(預設值)",
"settings_protectmodifypercentage_100_desc": "100(去除此保護)",
"settings_protectmodifypercentage_custom_desc": "自定義",
"settings_protectmodifypercentage_customfield": "如果修改超過自定義百分比則中止同步",
"settings_protectmodifypercentage_customfield_desc": "您需要輸入 0(含)~ 100(含)的數字。小數也是可以的。",
"setting_syncdirection": "同步方向",
"setting_syncdirection_desc": "外掛應該向哪裡同步?注意每個選項都是隻有修改了的檔案(基於修改時間和大小判斷)才會觸發同步動作。",
"setting_syncdirection_bidirectional_desc": "雙向同步(預設)",
@@ -292,9 +300,12 @@
"settings_importexport": "匯入匯出部分設定",
"settings_export": "匯出",
"settings_export_desc": "用 QR 碼或 URI 匯出設定資訊。",
"settings_export_all_but_oauth2_button": "匯出非 Oauth2 部分",
"settings_export_basic_and_advanced_button": "匯出基本或進階設定",
"settings_export_s3_button": "匯出 S3 部分",
"settings_export_dropbox_button": "匯出 Dropbox 部分",
"settings_export_onedrive_button": "匯出 OneDrive 部分",
"settings_export_webdav_button": "匯出 Webdav 部分",
"settings_export_webdis_button": "匯出 Webdis 部分",
"settings_import": "匯入",
"settings_import_desc": "貼上之前匯出的 URI 到這裡然後點選“匯入”。或,使用拍攝 app 或者掃描 QR 碼的 app,來掃描對應的 QR 碼。",
"settings_import_button": "匯入",
@@ -312,6 +323,7 @@
"settings_viewconsolelog_desc": "電腦上,輸入“ctrl+shift+i”或“cmd+shift+i”來檢視終端輸出。手機上,安裝第三方外掛 <a href='https://obsidian.md/plugins?search=Logstravaganza'>Logstravaganza</a> 來匯出終端輸出到一篇筆記上。",
"settings_syncplans": "匯出同步計劃",
"settings_syncplans_desc": "每次您啟動同步,並在實際上傳下載前,外掛會生成同步計劃。它可以使您知道每次同步發生了什麼。點選按鈕可以匯出同步計劃。",
"settings_syncplans_button_1_only_change": "匯出最近 1 次(僅修改部分)",
"settings_syncplans_button_1": "匯出最近 1 次",
"settings_syncplans_button_5": "匯出最近 5 次",
"settings_syncplans_button_all": "匯出所有",
+32 -25
View File
@@ -1,6 +1,8 @@
import localforage from "localforage";
import { extendPrototype } from "localforage-getitems";
extendPrototype(localforage);
import { extendPrototype as ep1 } from "localforage-getitems";
import { extendPrototype as ep2 } from "localforage-removeitems";
ep1(localforage);
ep2(localforage);
export type LocalForage = typeof localforage;
import { nanoid } from "nanoid";
@@ -307,12 +309,15 @@ export const clearFileHistoryOfEverythingByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const keys = await db.fileHistoryTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
await db.fileHistoryTbl.removeItem(key);
}
}
const keys = (await db.fileHistoryTbl.keys()).filter((x) =>
x.startsWith(`${vaultRandomID}\t`)
);
await db.fileHistoryTbl.removeItems(keys);
// for (const key of keys) {
// if (key.startsWith(`${vaultRandomID}\t`)) {
// await db.fileHistoryTbl.removeItem(key);
// }
// }
};
/**
@@ -339,12 +344,15 @@ export const clearAllSyncMetaMappingByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const keys = await db.syncMappingTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
await db.syncMappingTbl.removeItem(key);
}
}
const keys = (await db.syncMappingTbl.keys()).filter((x) =>
x.startsWith(`${vaultRandomID}\t`)
);
await db.syncMappingTbl.removeItems(keys);
// for (const key of keys) {
// if (key.startsWith(`${vaultRandomID}\t`)) {
// await db.syncMappingTbl.removeItem(key);
// }
// }
};
export const insertSyncPlanRecordByVault = async (
@@ -422,11 +430,12 @@ export const clearExpiredSyncPlanRecords = async (db: InternalDBs) => {
});
}
const ps = [] as Promise<void>[];
keysToRemove.forEach((element) => {
ps.push(db.syncPlansTbl.removeItem(element));
});
await Promise.all(ps);
// const ps = [] as Promise<void>[];
// keysToRemove.forEach((element) => {
// ps.push(db.syncPlansTbl.removeItem(element));
// });
// await Promise.all(ps);
await db.syncPlansTbl.removeItems(Array.from(keysToRemove));
};
export const getAllPrevSyncRecordsByVaultAndProfile = async (
@@ -475,12 +484,10 @@ export const clearAllPrevSyncRecordByVault = async (
db: InternalDBs,
vaultRandomID: string
) => {
const keys = await db.prevSyncRecordsTbl.keys();
for (const key of keys) {
if (key.startsWith(`${vaultRandomID}\t`)) {
await db.prevSyncRecordsTbl.removeItem(key);
}
}
const keys = (await db.prevSyncRecordsTbl.keys()).filter((x) =>
x.startsWith(`${vaultRandomID}\t`)
);
await db.prevSyncRecordsTbl.removeItems(keys);
};
export const clearAllLoggerOutputRecords = async (db: InternalDBs) => {
+33 -10
View File
@@ -62,7 +62,7 @@ import { getClient } from "./fsGetter";
import { FakeFsLocal } from "./fsLocal";
import { DEFAULT_WEBDIS_CONFIG } from "./fsWebdis";
import { changeMobileStatusBar } from "./misc";
import { DEFAULT_PROFILER_CONFIG, Profiler } from "./profiler";
import { DEFAULT_PROFILER_CONFIG, type Profiler } from "./profiler";
import { syncer } from "./sync";
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
@@ -152,11 +152,12 @@ export default class RemotelySavePlugin extends Plugin {
appContainerObserver?: MutationObserver;
async syncRun(triggerSource: SyncTriggerSourceType = "manual") {
const profiler = new Profiler(
undefined,
this.settings.profiler?.enablePrinting ?? false,
this.settings.profiler?.recordSize ?? false
);
// const profiler = new Profiler(
// undefined,
// this.settings.profiler?.enablePrinting ?? false,
// this.settings.profiler?.recordSize ?? false
// );
const profiler: Profiler | undefined = undefined;
const fsLocal = new FakeFsLocal(
this.app.vault,
this.settings.syncConfigDir ?? false,
@@ -419,7 +420,7 @@ export default class RemotelySavePlugin extends Plugin {
);
fsEncrypt.closeResources();
profiler.clear();
(profiler as Profiler | undefined)?.clear();
this.syncEvent?.trigger("SYNC_DONE");
}
@@ -744,6 +745,22 @@ export default class RemotelySavePlugin extends Plugin {
},
});
this.addCommand({
id: "export-sync-plans-1-only-change",
name: t("command_exportsyncplans_1_only_change"),
icon: iconNameLogs,
callback: async () => {
await exportVaultSyncPlansToFiles(
this.db,
this.app.vault,
this.vaultRandomID,
1,
true
);
new Notice(t("settings_syncplans_notice"));
},
});
this.addCommand({
id: "export-sync-plans-1",
name: t("command_exportsyncplans_1"),
@@ -753,7 +770,8 @@ export default class RemotelySavePlugin extends Plugin {
this.db,
this.app.vault,
this.vaultRandomID,
1
1,
false
);
new Notice(t("settings_syncplans_notice"));
},
@@ -768,7 +786,8 @@ export default class RemotelySavePlugin extends Plugin {
this.db,
this.app.vault,
this.vaultRandomID,
5
5,
false
);
new Notice(t("settings_syncplans_notice"));
},
@@ -783,7 +802,8 @@ export default class RemotelySavePlugin extends Plugin {
this.db,
this.app.vault,
this.vaultRandomID,
-1
-1,
false
);
new Notice(t("settings_syncplans_notice"));
},
@@ -854,6 +874,9 @@ export default class RemotelySavePlugin extends Plugin {
if (this.settings.onedrive.remoteBaseDir === undefined) {
this.settings.onedrive.remoteBaseDir = "";
}
if (this.settings.onedrive.emptyFile === undefined) {
this.settings.onedrive.emptyFile = "skip";
}
if (this.settings.webdav.manualRecursive === undefined) {
this.settings.webdav.manualRecursive = true;
}
+31
View File
@@ -679,3 +679,34 @@ export const roughSizeOfObject = (object: any) => {
}
return bytes;
};
export const splitFileSizeToChunkRanges = (
totalSize: number,
chunkSize: number
) => {
if (totalSize < 0) {
throw Error(`totalSize should not be negative`);
}
if (chunkSize <= 0) {
throw Error(`chunkSize should not be negative or zero`);
}
if (totalSize === 0) {
return [];
}
if (totalSize <= chunkSize) {
return [{ start: 0, end: totalSize - 1 }];
}
const res: { start: number; end: number }[] = [];
const blocksCount = Math.ceil((totalSize * 1.0) / chunkSize);
for (let i = 0; i < blocksCount; ++i) {
res.push({
start: i * chunkSize,
end: Math.min((i + 1) * chunkSize - 1, totalSize - 1),
});
}
return res;
};
+130 -32
View File
@@ -1418,6 +1418,20 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
});
});
new Setting(onedriveDiv)
.setName(t("settings_onedrive_emptyfile"))
.setDesc(t("settings_onedrive_emptyfile_desc"))
.addDropdown(async (dropdown) => {
dropdown
.addOption("skip", t("settings_onedrive_emptyfile_skip"))
.addOption("error", t("settings_onedrive_emptyfile_error"))
.setValue(this.plugin.settings.onedrive.emptyFile)
.onChange(async (val) => {
this.plugin.settings.onedrive.emptyFile = val as any;
await this.plugin.saveSettings();
});
});
new Setting(onedriveDiv)
.setName(t("settings_checkonnectivity"))
.setDesc(t("settings_checkonnectivity_desc"))
@@ -1955,7 +1969,7 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
.addDropdown((dropdown) => {
dropdown.addOption("-1", t("settings_skiplargefiles_notset"));
const mbs = [1, 5, 10, 50, 100, 500, 1000];
const mbs = [1, 5, 10, 20, 50, 100, 200, 500, 1000];
for (const mb of mbs) {
dropdown.addOption(`${mb * 1000 * 1000}`, `${mb} MB`);
}
@@ -2154,28 +2168,77 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
});
});
new Setting(advDiv)
const percentage1 = new Setting(advDiv)
.setName(t("settings_protectmodifypercentage"))
.setDesc(t("settings_protectmodifypercentage_desc"))
.addDropdown((dropdown) => {
for (const i of Array.from({ length: 11 }, (x, i) => i * 10)) {
let desc = `${i}`;
if (i === 0) {
desc = t("settings_protectmodifypercentage_000_desc");
} else if (i === 50) {
desc = t("settings_protectmodifypercentage_050_desc");
} else if (i === 100) {
desc = t("settings_protectmodifypercentage_100_desc");
}
dropdown.addOption(`${i}`, desc);
}
dropdown
.setValue(`${this.plugin.settings.protectModifyPercentage ?? 50}`)
.onChange(async (val) => {
this.plugin.settings.protectModifyPercentage = Number.parseInt(val);
.setDesc(t("settings_protectmodifypercentage_desc"));
const percentage2 = new Setting(advDiv)
.setName(t("settings_protectmodifypercentage_customfield"))
.setDesc(t("settings_protectmodifypercentage_customfield_desc"));
if ((this.plugin.settings.protectModifyPercentage ?? 50) % 10 === 0) {
percentage2.settingEl.addClass("settings-percentage-custom-hide");
}
let percentage2Text: TextComponent | undefined = undefined;
percentage2.addText((text) => {
text.inputEl.type = "number";
percentage2Text = text;
text
.setPlaceholder("0 ~ 100")
.setValue(`${this.plugin.settings.protectModifyPercentage ?? 50}`)
.onChange(async (val) => {
let k = Number.parseFloat(val.trim());
if (Number.isNaN(k)) {
// do nothing!
} else {
if (k < 0) {
k = 0;
} else if (k > 100) {
k = 100;
}
this.plugin.settings.protectModifyPercentage = k;
await this.plugin.saveSettings();
});
}
});
});
percentage1.addDropdown((dropdown) => {
for (const i of Array.from({ length: 11 }, (x, i) => i * 10)) {
let desc = `${i}`;
if (i === 0) {
desc = t("settings_protectmodifypercentage_000_desc");
} else if (i === 50) {
desc = t("settings_protectmodifypercentage_050_desc");
} else if (i === 100) {
desc = t("settings_protectmodifypercentage_100_desc");
}
dropdown.addOption(`${i}`, desc);
}
dropdown.addOption(
"custom",
t("settings_protectmodifypercentage_custom_desc")
);
const p = this.plugin.settings.protectModifyPercentage ?? 50;
let initVal = "custom";
if (p % 10 === 0) {
initVal = `${p}`;
} else {
// show custom
percentage2.settingEl.removeClass("settings-percentage-custom");
}
dropdown.setValue(initVal).onChange(async (val) => {
const k = Number.parseInt(val);
if (val === "custom" || Number.isNaN(k)) {
// do nothing until user changes something in custom field
percentage2.settingEl.removeClass("settings-percentage-custom-hide");
} else {
this.plugin.settings.protectModifyPercentage = k;
percentage2.settingEl.addClass("settings-percentage-custom-hide");
percentage2Text?.setValue(`${k}`);
await this.plugin.saveSettings();
}
});
});
new Setting(advDiv)
.setName(t("setting_syncdirection"))
@@ -2247,23 +2310,28 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
importExportDiv.createEl("h2", {
text: t("settings_importexport"),
});
if (Platform.isMobile) {
importExportDiv.addClass("setting-need-wrapping-mobile");
}
new Setting(importExportDiv)
const importExportDivSetting1 = new Setting(importExportDiv)
.setName(t("settings_export"))
.setDesc(t("settings_export_desc"))
.setDesc(t("settings_export_desc"));
importExportDivSetting1.settingEl.addClass("setting-need-wrapping");
importExportDivSetting1
.addButton(async (button) => {
button.setButtonText(t("settings_export_all_but_oauth2_button"));
button.setButtonText(t("settings_export_basic_and_advanced_button"));
button.onClick(async () => {
new ExportSettingsQrCodeModal(
this.app,
this.plugin,
"all_but_oauth2"
"basic_and_advanced"
).open();
});
})
.addButton(async (button) => {
button.setButtonText(t("settings_export_s3_button"));
button.onClick(async () => {
new ExportSettingsQrCodeModal(this.app, this.plugin, "s3").open();
});
})
.addButton(async (button) => {
button.setButtonText(t("settings_export_dropbox_button"));
button.onClick(async () => {
@@ -2283,6 +2351,18 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
"onedrive"
).open();
});
})
.addButton(async (button) => {
button.setButtonText(t("settings_export_webdav_button"));
button.onClick(async () => {
new ExportSettingsQrCodeModal(this.app, this.plugin, "webdav").open();
});
})
.addButton(async (button) => {
button.setButtonText(t("settings_export_webdis_button"));
button.onClick(async () => {
new ExportSettingsQrCodeModal(this.app, this.plugin, "webdis").open();
});
});
let importSettingVal = "";
@@ -2399,9 +2479,24 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
.setName(t("settings_viewconsolelog"))
.setDesc(stringToFragment(t("settings_viewconsolelog_desc")));
new Setting(debugDiv)
const debugDivExportSyncPlans = new Setting(debugDiv)
.setName(t("settings_syncplans"))
.setDesc(t("settings_syncplans_desc"))
.setDesc(t("settings_syncplans_desc"));
debugDivExportSyncPlans.settingEl.addClass("setting-need-wrapping");
debugDivExportSyncPlans
.addButton(async (button) => {
button.setButtonText(t("settings_syncplans_button_1_only_change"));
button.onClick(async () => {
await exportVaultSyncPlansToFiles(
this.plugin.db,
this.app.vault,
this.plugin.vaultRandomID,
1,
true
);
new Notice(t("settings_syncplans_notice"));
});
})
.addButton(async (button) => {
button.setButtonText(t("settings_syncplans_button_1"));
button.onClick(async () => {
@@ -2409,7 +2504,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
this.plugin.db,
this.app.vault,
this.plugin.vaultRandomID,
1
1,
false
);
new Notice(t("settings_syncplans_notice"));
});
@@ -2421,7 +2517,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
this.plugin.db,
this.app.vault,
this.plugin.vaultRandomID,
5
5,
false
);
new Notice(t("settings_syncplans_notice"));
});
@@ -2433,7 +2530,8 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
this.plugin.db,
this.app.vault,
this.plugin.vaultRandomID,
-1
-1,
false
);
new Notice(t("settings_syncplans_notice"));
});
+130 -65
View File
@@ -155,13 +155,13 @@ const ensembleMixedEnties = async (
fsEncrypt: FakeFsEncrypt,
serviceType: SUPPORTED_SERVICES_TYPE,
profiler: Profiler
profiler: Profiler | undefined
): Promise<SyncPlanType> => {
profiler.addIndent();
profiler.insert("ensembleMixedEnties: enter");
profiler.insertSize("sizeof localEntityList", localEntityList);
profiler.insertSize("sizeof prevSyncEntityList", prevSyncEntityList);
profiler.insertSize("sizeof remoteEntityList", remoteEntityList);
profiler?.addIndent();
profiler?.insert("ensembleMixedEnties: enter");
profiler?.insertSize("sizeof localEntityList", localEntityList);
profiler?.insertSize("sizeof prevSyncEntityList", prevSyncEntityList);
profiler?.insertSize("sizeof remoteEntityList", remoteEntityList);
const finalMappings: SyncPlanType = {};
@@ -190,8 +190,8 @@ const ensembleMixedEnties = async (
};
}
profiler.insert("ensembleMixedEnties: finish remote");
profiler.insertSize("sizeof finalMappings", finalMappings);
profiler?.insert("ensembleMixedEnties: finish remote");
profiler?.insertSize("sizeof finalMappings", finalMappings);
if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) {
// Special checking:
@@ -231,8 +231,8 @@ const ensembleMixedEnties = async (
}
}
profiler.insert("ensembleMixedEnties: finish prevSync");
profiler.insertSize("sizeof finalMappings", finalMappings);
profiler?.insert("ensembleMixedEnties: finish prevSync");
profiler?.insertSize("sizeof finalMappings", finalMappings);
// local has to be last
// because we want to get keyEnc based on the remote
@@ -265,14 +265,14 @@ const ensembleMixedEnties = async (
}
}
profiler.insert("ensembleMixedEnties: finish local");
profiler.insertSize("sizeof finalMappings", finalMappings);
profiler?.insert("ensembleMixedEnties: finish local");
profiler?.insertSize("sizeof finalMappings", finalMappings);
// console.debug("in the end of ensembleMixedEnties, finalMappings is:");
// console.debug(finalMappings);
profiler.insert("ensembleMixedEnties: exit");
profiler.removeIndent();
profiler?.insert("ensembleMixedEnties: exit");
profiler?.removeIndent();
return finalMappings;
};
@@ -287,24 +287,24 @@ const getSyncPlanInplace = async (
skipSizeLargerThan: number,
conflictAction: ConflictActionType,
syncDirection: SyncDirectionType,
profiler: Profiler,
profiler: Profiler | undefined,
settings: RemotelySavePluginSettings,
triggerSource: SyncTriggerSourceType
) => {
profiler.addIndent();
profiler.insert("getSyncPlanInplace: enter");
profiler?.addIndent();
profiler?.insert("getSyncPlanInplace: enter");
// from long(deep) to short(shadow)
const sortedKeys = Object.keys(mixedEntityMappings).sort(
(k1, k2) => k2.length - k1.length
);
profiler.insert("getSyncPlanInplace: finish sorting");
profiler.insertSize("sizeof sortedKeys", sortedKeys);
profiler?.insert("getSyncPlanInplace: finish sorting");
profiler?.insertSize("sizeof sortedKeys", sortedKeys);
const keptFolder = new Set<string>();
for (let i = 0; i < sortedKeys.length; ++i) {
if (i % 100 === 0) {
profiler.insertSize(
profiler?.insertSize(
`sizeof sortedKeys in the beginning of i=${i}`,
mixedEntityMappings
);
@@ -326,69 +326,83 @@ const getSyncPlanInplace = async (
if (local !== undefined && remote !== undefined) {
mixedEntry.decisionBranch = 101;
mixedEntry.decision = "folder_existed_both_then_do_nothing";
mixedEntry.change = false;
} else if (local !== undefined && remote === undefined) {
if (syncDirection === "incremental_pull_only") {
mixedEntry.decisionBranch = 107;
mixedEntry.decision = "folder_to_skip";
mixedEntry.change = false;
} else {
mixedEntry.decisionBranch = 102;
mixedEntry.decision =
"folder_existed_local_then_also_create_remote";
mixedEntry.change = true;
}
} else if (local === undefined && remote !== undefined) {
if (syncDirection === "incremental_push_only") {
mixedEntry.decisionBranch = 108;
mixedEntry.decision = "folder_to_skip";
mixedEntry.change = false;
} else {
mixedEntry.decisionBranch = 103;
mixedEntry.decision =
"folder_existed_remote_then_also_create_local";
mixedEntry.change = true;
}
} else {
// why?? how??
mixedEntry.decisionBranch = 104;
mixedEntry.decision = "folder_to_be_created";
mixedEntry.change = true;
}
keptFolder.delete(key); // no need to save it in the Set later
} else {
if (howToCleanEmptyFolder === "skip") {
mixedEntry.decisionBranch = 105;
mixedEntry.decision = "folder_to_skip";
mixedEntry.change = false;
} else if (howToCleanEmptyFolder === "clean_both") {
if (local !== undefined && remote !== undefined) {
if (syncDirection === "bidirectional") {
mixedEntry.decisionBranch = 106;
mixedEntry.decision = "folder_to_be_deleted_on_both";
mixedEntry.change = true;
} else {
// right now it does nothing because of "incremental"
// TODO: should we delete??
mixedEntry.decisionBranch = 109;
mixedEntry.decision = "folder_to_skip";
mixedEntry.change = false;
}
} else if (local !== undefined && remote === undefined) {
if (syncDirection === "bidirectional") {
mixedEntry.decisionBranch = 110;
mixedEntry.decision = "folder_to_be_deleted_on_local";
mixedEntry.change = true;
} else {
// right now it does nothing because of "incremental"
// TODO: should we delete??
mixedEntry.decisionBranch = 111;
mixedEntry.decision = "folder_to_skip";
mixedEntry.change = false;
}
} else if (local === undefined && remote !== undefined) {
if (syncDirection === "bidirectional") {
mixedEntry.decisionBranch = 112;
mixedEntry.decision = "folder_to_be_deleted_on_remote";
mixedEntry.change = true;
} else {
// right now it does nothing because of "incremental"
// TODO: should we delete??
mixedEntry.decisionBranch = 113;
mixedEntry.decision = "folder_to_skip";
mixedEntry.change = false;
}
} else {
// no folder to delete, do nothing
mixedEntry.decisionBranch = 114;
mixedEntry.decision = "folder_to_skip";
mixedEntry.change = false;
}
} else {
throw Error(
@@ -403,6 +417,7 @@ const getSyncPlanInplace = async (
// both deleted, only in history
mixedEntry.decisionBranch = 1;
mixedEntry.decision = "only_history";
mixedEntry.change = false;
} else if (local !== undefined && remote !== undefined) {
if (
(local.mtimeCli === remote.mtimeCli ||
@@ -412,6 +427,7 @@ const getSyncPlanInplace = async (
// completely equal / identical
mixedEntry.decisionBranch = 2;
mixedEntry.decision = "equal";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
} else {
// Both exists, but modified or conflict
@@ -434,10 +450,12 @@ const getSyncPlanInplace = async (
if (syncDirection === "incremental_push_only") {
mixedEntry.decisionBranch = 26;
mixedEntry.decision = "conflict_modified_then_keep_local";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 9;
mixedEntry.decision = "remote_is_modified_then_pull";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else {
@@ -456,10 +474,12 @@ const getSyncPlanInplace = async (
if (syncDirection === "incremental_pull_only") {
mixedEntry.decisionBranch = 27;
mixedEntry.decision = "conflict_modified_then_keep_remote";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 10;
mixedEntry.decision = "local_is_modified_then_push";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else {
@@ -481,34 +501,41 @@ const getSyncPlanInplace = async (
) {
mixedEntry.decisionBranch = 11;
mixedEntry.decision = "conflict_created_then_keep_local";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 12;
mixedEntry.decision = "conflict_created_then_keep_remote";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else if (conflictAction === "keep_larger") {
if (local.sizeEnc! >= remote.sizeEnc!) {
mixedEntry.decisionBranch = 13;
mixedEntry.decision = "conflict_created_then_keep_local";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 14;
mixedEntry.decision = "conflict_created_then_keep_remote";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else {
mixedEntry.decisionBranch = 15;
mixedEntry.decision = "conflict_created_then_keep_both";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else if (syncDirection === "incremental_pull_only") {
mixedEntry.decisionBranch = 22;
mixedEntry.decision = "conflict_created_then_keep_remote";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else if (syncDirection === "incremental_push_only") {
mixedEntry.decisionBranch = 23;
mixedEntry.decision = "conflict_created_then_keep_local";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
throw Error(
@@ -525,34 +552,41 @@ const getSyncPlanInplace = async (
) {
mixedEntry.decisionBranch = 16;
mixedEntry.decision = "conflict_modified_then_keep_local";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 17;
mixedEntry.decision = "conflict_modified_then_keep_remote";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else if (conflictAction === "keep_larger") {
if (local.sizeEnc! >= remote.sizeEnc!) {
mixedEntry.decisionBranch = 18;
mixedEntry.decision = "conflict_modified_then_keep_local";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 19;
mixedEntry.decision = "conflict_modified_then_keep_remote";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else {
mixedEntry.decisionBranch = 20;
mixedEntry.decision = "conflict_modified_then_keep_both";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else if (syncDirection === "incremental_pull_only") {
mixedEntry.decisionBranch = 24;
mixedEntry.decision = "conflict_modified_then_keep_remote";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else if (syncDirection === "incremental_push_only") {
mixedEntry.decisionBranch = 25;
mixedEntry.decision = "conflict_modified_then_keep_local";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
throw Error(
@@ -566,6 +600,7 @@ const getSyncPlanInplace = async (
// The result should be equal!!!
mixedEntry.decisionBranch = 21;
mixedEntry.decision = "equal";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
}
}
@@ -580,15 +615,18 @@ const getSyncPlanInplace = async (
if (syncDirection === "incremental_push_only") {
mixedEntry.decisionBranch = 28;
mixedEntry.decision = "conflict_created_then_do_nothing";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 3;
mixedEntry.decision = "remote_is_created_then_pull";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else {
mixedEntry.decisionBranch = 36;
mixedEntry.decision = "remote_is_created_too_large_then_do_nothing";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
}
} else if (
@@ -600,14 +638,17 @@ const getSyncPlanInplace = async (
if (syncDirection === "incremental_push_only") {
mixedEntry.decisionBranch = 29;
mixedEntry.decision = "conflict_created_then_do_nothing";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
} else if (syncDirection === "incremental_pull_only") {
mixedEntry.decisionBranch = 35;
mixedEntry.decision = "conflict_created_then_keep_remote";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 4;
mixedEntry.decision = "local_is_deleted_thus_also_delete_remote";
mixedEntry.change = true;
}
} else {
// if B is in the previous list and MODIFIED, B has been deleted by A but modified by B
@@ -618,10 +659,12 @@ const getSyncPlanInplace = async (
if (syncDirection === "incremental_push_only") {
mixedEntry.decisionBranch = 30;
mixedEntry.decision = "conflict_created_then_do_nothing";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 5;
mixedEntry.decision = "remote_is_modified_then_pull";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else {
@@ -641,15 +684,18 @@ const getSyncPlanInplace = async (
if (syncDirection === "incremental_pull_only") {
mixedEntry.decisionBranch = 31;
mixedEntry.decision = "conflict_created_then_do_nothing";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 6;
mixedEntry.decision = "local_is_created_then_push";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else {
mixedEntry.decisionBranch = 37;
mixedEntry.decision = "local_is_created_too_large_then_do_nothing";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
}
} else if (
@@ -661,12 +707,15 @@ const getSyncPlanInplace = async (
if (syncDirection === "incremental_push_only") {
mixedEntry.decisionBranch = 32;
mixedEntry.decision = "conflict_created_then_keep_local";
mixedEntry.change = true;
} else if (syncDirection === "incremental_pull_only") {
mixedEntry.decisionBranch = 33;
mixedEntry.decision = "conflict_created_then_do_nothing";
mixedEntry.change = false;
} else {
mixedEntry.decisionBranch = 7;
mixedEntry.decision = "remote_is_deleted_thus_also_delete_local";
mixedEntry.change = true;
}
} else {
// if A is in the previous list and MODIFIED, A has been deleted by B but modified by A
@@ -674,10 +723,12 @@ const getSyncPlanInplace = async (
if (syncDirection === "incremental_pull_only") {
mixedEntry.decisionBranch = 34;
mixedEntry.decision = "conflict_created_then_do_nothing";
mixedEntry.change = false;
keptFolder.add(getParentFolder(key));
} else {
mixedEntry.decisionBranch = 8;
mixedEntry.decision = "local_is_modified_then_push";
mixedEntry.change = true;
keptFolder.add(getParentFolder(key));
}
} else {
@@ -706,7 +757,7 @@ const getSyncPlanInplace = async (
}
}
profiler.insert("getSyncPlanInplace: finish looping");
profiler?.insert("getSyncPlanInplace: finish looping");
keptFolder.delete("/");
keptFolder.delete("");
@@ -723,25 +774,29 @@ const getSyncPlanInplace = async (
mixedEntityMappings["/$@meta"] = {
key: "/$@meta", // don't mess up with the types
sideNotes: {
version: "20240508 fs version",
version: "20240525 fs version",
generateTime: currTime,
generateTimeFmt: currTimeFmt,
service: settings.serviceType,
concurrency: settings.concurrency,
hasPassword: settings.password !== "",
syncConfigDir: settings.syncConfigDir,
syncUnderscoreItems: settings.syncUnderscoreItems,
skipSizeLargerThan: settings.skipSizeLargerThan,
protectModifyPercentage: settings.protectModifyPercentage,
conflictAction: conflictAction,
syncDirection: syncDirection,
triggerSource: triggerSource,
sizeof: sizeofmixedEntityMappings
sizeof: sizeofmixedEntityMappings,
},
};
profiler.insert("getSyncPlanInplace: exit");
profiler.insertSize(
profiler?.insert("getSyncPlanInplace: exit");
profiler?.insertSize(
"sizeof mixedEntityMappings in the end of getSyncPlanInplace",
mixedEntityMappings
);
profiler.removeIndent();
profiler?.removeIndent();
return mixedEntityMappings;
};
@@ -935,7 +990,8 @@ async function copyFile(
const statsLeft = await left.stat(key);
const content = await left.readFile(key);
if (statsLeft.size === undefined) {
if (statsLeft.size === undefined || statsLeft.size === 0) {
// some weird bugs on android not returning size. just ignore them
statsLeft.size = content.byteLength;
} else {
if (statsLeft.size !== content.byteLength) {
@@ -950,12 +1006,21 @@ async function copyFile(
}
// console.debug(`copyFile: about to start right.writeFile`);
return await right.writeFile(
key,
content,
statsLeft.mtimeCli,
statsLeft.mtimeCli /* TODO */
);
if (typeof (content as any).transfer === "function") {
return await right.writeFile(
key,
(content as any).transfer(),
statsLeft.mtimeCli,
statsLeft.mtimeCli /* TODO */
);
} else {
return await right.writeFile(
key,
content,
statsLeft.mtimeCli,
statsLeft.mtimeCli /* TODO */
);
}
}
async function copyFileOrFolder(
@@ -1130,11 +1195,11 @@ export const doActualSync = async (
protectModifyPercentage: number,
getProtectModifyPercentageErrorStrFunc: any,
db: InternalDBs,
profiler: Profiler,
profiler: Profiler | undefined,
callbackSyncProcess?: any
) => {
profiler.addIndent();
profiler.insert("doActualSync: enter");
profiler?.addIndent();
profiler?.insert("doActualSync: enter");
console.debug(`concurrency === ${concurrency}`);
const {
onlyMarkSyncedOps,
@@ -1152,17 +1217,17 @@ export const doActualSync = async (
console.debug(`allFilesCount: ${allFilesCount}`);
console.debug(`realModifyDeleteCount: ${realModifyDeleteCount}`);
console.debug(`realTotalCount: ${realTotalCount}`);
profiler.insert("doActualSync: finish splitting steps");
profiler?.insert("doActualSync: finish splitting steps");
profiler.insertSize(
profiler?.insertSize(
"doActualSync: sizeof onlyMarkSyncedOps",
onlyMarkSyncedOps
);
profiler.insertSize(
profiler?.insertSize(
"doActualSync: sizeof folderCreationOps",
folderCreationOps
);
profiler.insertSize("doActualSync: sizeof realTotalCount", deletionOps);
profiler?.insertSize("doActualSync: sizeof realTotalCount", deletionOps);
console.debug(`protectModifyPercentage: ${protectModifyPercentage}`);
@@ -1187,8 +1252,8 @@ export const doActualSync = async (
allFilesCount
);
profiler.insert("doActualSync: error branch");
profiler.removeIndent();
profiler?.insert("doActualSync: error branch");
profiler?.removeIndent();
throw Error(errorStr);
}
}
@@ -1208,8 +1273,8 @@ export const doActualSync = async (
let realCounter = 0;
for (let i = 0; i < nested.length; ++i) {
profiler.addIndent();
profiler.insert(`doActualSync: step ${i} start`);
profiler?.addIndent();
profiler?.insert(`doActualSync: step ${i} start`);
console.debug(logTexts[i]);
const operations = nested[i];
@@ -1282,12 +1347,12 @@ export const doActualSync = async (
}
}
profiler.insert(`doActualSync: step ${i} end`);
profiler.removeIndent();
profiler?.insert(`doActualSync: step ${i} end`);
profiler?.removeIndent();
}
profiler.insert(`doActualSync: exit`);
profiler.removeIndent();
profiler?.insert(`doActualSync: exit`);
profiler?.removeIndent();
};
export type SyncStatusType =
@@ -1309,7 +1374,7 @@ export async function syncer(
fsLocal: FakeFs,
fsRemote: FakeFs,
fsEncrypt: FakeFsEncrypt,
profiler: Profiler,
profiler: Profiler | undefined,
db: InternalDBs,
triggerSource: SyncTriggerSourceType,
profileID: string,
@@ -1340,7 +1405,7 @@ export async function syncer(
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
profiler.insert("start big sync func");
profiler?.insert("start big sync func");
try {
step = 2;
@@ -1354,7 +1419,7 @@ export async function syncer(
if (!passwordCheckResult.ok) {
throw Error(passwordCheckResult.reason);
}
profiler.insert(
profiler?.insert(
`finish step${step} (list partial remote and check password)`
);
@@ -1365,7 +1430,7 @@ export async function syncer(
const remoteEntityList = await fsEncrypt.walk();
// console.debug(`remoteEntityList:`);
// console.debug(remoteEntityList);
profiler.insert(`finish step${step} (list remote)`);
profiler?.insert(`finish step${step} (list remote)`);
step = 4;
await notifyFunc?.(triggerSource, step);
@@ -1374,7 +1439,7 @@ export async function syncer(
const localEntityList = await fsLocal.walk();
// console.debug(`localEntityList:`);
// console.debug(localEntityList);
profiler.insert(`finish step${step} (list local)`);
profiler?.insert(`finish step${step} (list local)`);
step = 5;
await notifyFunc?.(triggerSource, step);
@@ -1387,7 +1452,7 @@ export async function syncer(
);
// console.debug(`prevSyncEntityList:`);
// console.debug(prevSyncEntityList);
profiler.insert(`finish step${step} (prev sync)`);
profiler?.insert(`finish step${step} (prev sync)`);
step = 6;
await notifyFunc?.(triggerSource, step);
@@ -1405,7 +1470,7 @@ export async function syncer(
settings.serviceType,
profiler
);
profiler.insert(`finish step${step} (build partial mixedEntity)`);
profiler?.insert(`finish step${step} (build partial mixedEntity)`);
mixedEntityMappings = await getSyncPlanInplace(
mixedEntityMappings,
@@ -1419,7 +1484,7 @@ export async function syncer(
);
console.debug(`mixedEntityMappings:`);
console.debug(mixedEntityMappings); // for debugging
profiler.insert("finish building full sync plan");
profiler?.insert("finish building full sync plan");
await insertSyncPlanRecordByVault(
db,
@@ -1427,8 +1492,8 @@ export async function syncer(
vaultRandomID,
settings.serviceType
);
profiler.insert("finish writing sync plan");
profiler.insert(`finish step${step} (make plan)`);
profiler?.insert("finish writing sync plan");
profiler?.insert(`finish step${step} (make plan)`);
// The operations above are almost read only and kind of safe.
// The operations below begins to write or delete (!!!) something.
@@ -1451,27 +1516,27 @@ export async function syncer(
profiler,
callbackSyncProcess
);
profiler.insert(`finish step${step} (actual sync)`);
profiler?.insert(`finish step${step} (actual sync)`);
} else {
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
profiler.insert(
profiler?.insert(
`finish step${step} (skip actual sync because of dry run)`
);
}
} catch (error: any) {
profiler.insert("start error branch");
profiler?.insert("start error branch");
everythingOk = false;
await errNotifyFunc?.(triggerSource, error as Error);
profiler.insert("finish error branch");
profiler?.insert("finish error branch");
} finally {
}
profiler.insert("finish syncRun");
// console.debug(profiler.toString());
await profiler.save(db, vaultRandomID, settings.serviceType);
profiler?.insert("finish syncRun");
// console.debug(profiler?.toString());
await profiler?.save(db, vaultRandomID, settings.serviceType);
step = 8;
await notifyFunc?.(triggerSource, step);
+7 -2
View File
@@ -17,6 +17,10 @@
padding-top: 18px;
}
.settings-percentage-custom-hide {
display: none;
}
.s3-disclaimer {
font-weight: bold;
}
@@ -82,6 +86,7 @@
font-weight: bolder;
}
.setting-need-wrapping-mobile .setting-item-control {
flex-wrap: wrap;
.setting-need-wrapping .setting-item-control {
/* flex-wrap: wrap; */
display: grid;
}
+77
View File
@@ -286,6 +286,83 @@ describe("Misc: special char for dir", () => {
});
});
describe("Misc: split chunk ranges", () => {
it("should fail on negative numner", () => {
assert.throws(() => misc.splitFileSizeToChunkRanges(-1, 2));
assert.throws(() => misc.splitFileSizeToChunkRanges(1, -1));
assert.throws(() => misc.splitFileSizeToChunkRanges(1, 0));
});
it("should return nothing for 0 input", () => {
let input: [number, number] = [0, 1];
let output: any = [];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
input = [0, 100];
output = [];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
});
it("should return single item for 1 input", () => {
let input: [number, number] = [1, 1];
let output = [{ start: 0, end: 0 }];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
input = [1, 100];
output = [{ start: 0, end: 0 }];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
});
it("should return single item for larger or equal input", () => {
let input: [number, number] = [10, 10];
let output = [{ start: 0, end: 9 }];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
input = [10, 21];
output = [{ start: 0, end: 9 }];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
});
it("should return correct items for normal input", () => {
let input: [number, number] = [10, 9];
let output = [
{ start: 0, end: 8 },
{ start: 9, end: 9 },
];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
input = [10, 5];
output = [
{ start: 0, end: 4 },
{ start: 5, end: 9 },
];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
input = [3, 1];
output = [
{ start: 0, end: 0 },
{ start: 1, end: 1 },
{ start: 2, end: 2 },
];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
input = [15, 5];
output = [
{ start: 0, end: 4 },
{ start: 5, end: 9 },
{ start: 10, end: 14 },
];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
input = [1024, 578];
output = [
{ start: 0, end: 577 },
{ start: 578, end: 1023 },
];
assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
});
});
describe("Misc: Dropbox: should fix the folder name cases", () => {
it("should do nothing on empty folders", () => {
const input: any[] = [];