A big commit Squashed commit of CORS:
commit 8cffa38ebae2a46b7c8e855c7b21a124e35adc89
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Thu Mar 10 23:52:56 2022 +0800
bypass more cors for onedrive
commit 1b59ac1e58032099068aab55d22ef96c6396f203
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 23:58:28 2022 +0800
change wordings for webdav cors
commit 73142eb18b59fff20839680e866f51cfcb0a6226
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 23:38:58 2022 +0800
remove cors hint for webdav
commit 7dbb0b49d50e529b2b72e55ea2c8503ba7fa9268
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 23:31:54 2022 +0800
fix webdav
commit c28c4e19720a56230d483acf306463d42e619fa4
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 23:31:35 2022 +0800
remove more headers
commit 4eeae7043fa68d669a5c23c5549c14c1260ce638
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 23:18:32 2022 +0800
polish cors hints for s3
commit d9e55a91a1c413e9419cd6b30a3a6e3b403d483b
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 22:40:37 2022 +0800
fix format
commit b780a3eb4e37b05b8e8b92d6a2f9283b3459d738
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 22:37:02 2022 +0800
finally correctly inject requestUrl into s3
commit 6a55a1a43d7653d65579ab88aa816e5d54cd276a
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 22:33:18 2022 +0800
to arraybuffer from view
commit 2f2607b4f0a3d9db5943528ced57cb2fdb419607
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Wed Mar 9 13:31:22 2022 +0800
add split ranges
commit ea24da24dea83fdb770e7e391cf8a2e4fea78d0d
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Sun Mar 6 22:57:50 2022 +0800
add settings of bypassing for s3
commit 2f099dc8ca1e66ea137b28dd329be50968734ba6
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Sun Mar 6 22:38:07 2022 +0800
use api ver var
commit 74c7ce2449a88cbe7c7f50cbb687b36ff3732c04
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Sun Mar 6 22:37:25 2022 +0800
correct way to inject s3
commit f29945d73132d21b2c44472ec2cafc06b9d71e8f
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Tue Mar 1 00:09:57 2022 +0800
add new http handler
commit d55104cb08e168cbcc243cf901cbd7f46f2e324b
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Mon Feb 28 22:59:55 2022 +0800
add types for patch
commit 50b79ade7188ee7dfab9c1d03119585db358ba6f
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Mon Feb 28 08:25:19 2022 +0800
remove verbose
commit 83f0e71aa15aa7586f6d4e105cd77c25c2e113ce
Author: fyears <1142836+fyears@users.noreply.github.com>
Date: Mon Feb 28 08:25:04 2022 +0800
patch webdav!
This commit is contained in:
parent
5402baed0a
commit
c04876e0c5
@ -52,8 +52,12 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.37.0",
|
"@aws-sdk/client-s3": "^3.37.0",
|
||||||
"@aws-sdk/lib-storage": "^3.40.1",
|
"@aws-sdk/fetch-http-handler": "^3.53.0",
|
||||||
|
"@aws-sdk/lib-storage": "^3.53.1",
|
||||||
|
"@aws-sdk/protocol-http": "^3.53.0",
|
||||||
|
"@aws-sdk/querystring-builder": "^3.53.0",
|
||||||
"@aws-sdk/signature-v4-crt": "^3.37.0",
|
"@aws-sdk/signature-v4-crt": "^3.37.0",
|
||||||
|
"@aws-sdk/types": "^3.53.0",
|
||||||
"@azure/msal-node": "^1.6.0",
|
"@azure/msal-node": "^1.6.0",
|
||||||
"@fyears/tsqueue": "^1.0.1",
|
"@fyears/tsqueue": "^1.0.1",
|
||||||
"@microsoft/microsoft-graph-client": "^3.0.1",
|
"@microsoft/microsoft-graph-client": "^3.0.1",
|
||||||
@ -65,6 +69,7 @@
|
|||||||
"crypto-browserify": "^3.12.0",
|
"crypto-browserify": "^3.12.0",
|
||||||
"dropbox": "^10.22.0",
|
"dropbox": "^10.22.0",
|
||||||
"feather-icons": "^4.28.0",
|
"feather-icons": "^4.28.0",
|
||||||
|
"http-status-codes": "^2.2.0",
|
||||||
"localforage": "^1.10.0",
|
"localforage": "^1.10.0",
|
||||||
"lodash": "^4.17.21",
|
"lodash": "^4.17.21",
|
||||||
"loglevel": "^1.8.0",
|
"loglevel": "^1.8.0",
|
||||||
|
|||||||
@ -11,6 +11,7 @@ export interface S3Config {
|
|||||||
s3AccessKeyID: string;
|
s3AccessKeyID: string;
|
||||||
s3SecretAccessKey: string;
|
s3SecretAccessKey: string;
|
||||||
s3BucketName: string;
|
s3BucketName: string;
|
||||||
|
bypassCorsLocally?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DropboxConfig {
|
export interface DropboxConfig {
|
||||||
@ -117,3 +118,4 @@ export interface FileOrFolderMixedState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const API_VER_STAT_FOLDER = "0.13.27";
|
export const API_VER_STAT_FOLDER = "0.13.27";
|
||||||
|
export const API_VER_REQURL = "0.13.26";
|
||||||
|
|||||||
@ -154,7 +154,7 @@ export default class RemotelySavePlugin extends Plugin {
|
|||||||
() => self.saveSettings()
|
() => self.saveSettings()
|
||||||
);
|
);
|
||||||
const remoteRsp = await client.listFromRemote();
|
const remoteRsp = await client.listFromRemote();
|
||||||
log.info(remoteRsp);
|
log.debug(remoteRsp);
|
||||||
|
|
||||||
getNotice(`3/${MAX_STEPS} Checking password correct or not.`);
|
getNotice(`3/${MAX_STEPS} Checking password correct or not.`);
|
||||||
this.syncStatus = "checking_password";
|
this.syncStatus = "checking_password";
|
||||||
|
|||||||
32
src/misc.ts
32
src/misc.ts
@ -79,7 +79,9 @@ export const mkdirpInVault = async (thePath: string, vault: Vault) => {
|
|||||||
* @param b Buffer
|
* @param b Buffer
|
||||||
* @returns ArrayBuffer
|
* @returns ArrayBuffer
|
||||||
*/
|
*/
|
||||||
export const bufferToArrayBuffer = (b: Buffer | Uint8Array) => {
|
export const bufferToArrayBuffer = (
|
||||||
|
b: Buffer | Uint8Array | ArrayBufferView
|
||||||
|
) => {
|
||||||
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
|
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -230,3 +232,31 @@ export const getRandomArrayBuffer = (byteLength: number) => {
|
|||||||
export const reverseString = (x: string) => {
|
export const reverseString = (x: string) => {
|
||||||
return [...x].reverse().join("");
|
return [...x].reverse().join("");
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export interface SplitRange {
|
||||||
|
partNum: number; // startting from 1
|
||||||
|
start: number;
|
||||||
|
end: number; // exclusive
|
||||||
|
}
|
||||||
|
export const getSplitRanges = (bytesTotal: number, bytesEachPart: number) => {
|
||||||
|
const res: SplitRange[] = [];
|
||||||
|
if (bytesEachPart >= bytesTotal) {
|
||||||
|
res.push({
|
||||||
|
partNum: 1,
|
||||||
|
start: 0,
|
||||||
|
end: bytesTotal,
|
||||||
|
});
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
const remainder = bytesTotal % bytesEachPart;
|
||||||
|
const howMany =
|
||||||
|
Math.floor(bytesTotal / bytesEachPart) + (remainder === 0 ? 0 : 1);
|
||||||
|
for (let i = 0; i < howMany; ++i) {
|
||||||
|
res.push({
|
||||||
|
partNum: i + 1,
|
||||||
|
start: bytesEachPart * i,
|
||||||
|
end: Math.min(bytesEachPart * (i + 1), bytesTotal),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|||||||
@ -7,8 +7,9 @@ import type {
|
|||||||
} from "@microsoft/microsoft-graph-types";
|
} from "@microsoft/microsoft-graph-types";
|
||||||
import cloneDeep from "lodash/cloneDeep";
|
import cloneDeep from "lodash/cloneDeep";
|
||||||
import * as origLog from "loglevel";
|
import * as origLog from "loglevel";
|
||||||
import { request, Vault } from "obsidian";
|
import { request, requestUrl, requireApiVersion, Vault } from "obsidian";
|
||||||
import {
|
import {
|
||||||
|
API_VER_REQURL,
|
||||||
COMMAND_CALLBACK_ONEDRIVE,
|
COMMAND_CALLBACK_ONEDRIVE,
|
||||||
OAUTH2_FORCE_EXPIRE_MILLISECONDS,
|
OAUTH2_FORCE_EXPIRE_MILLISECONDS,
|
||||||
OnedriveConfig,
|
OnedriveConfig,
|
||||||
@ -482,21 +483,37 @@ export class WrappedOnedriveClient {
|
|||||||
deleteJson = async (pathFragOrig: string) => {
|
deleteJson = async (pathFragOrig: string) => {
|
||||||
const theUrl = this.buildUrl(pathFragOrig);
|
const theUrl = this.buildUrl(pathFragOrig);
|
||||||
log.debug(`deleteJson, theUrl=${theUrl}`);
|
log.debug(`deleteJson, theUrl=${theUrl}`);
|
||||||
// TODO: delete does not have response, so Obsidian request may have error
|
if (requireApiVersion(API_VER_REQURL)) {
|
||||||
// currently downgraded to fetch()!
|
await requestUrl({
|
||||||
|
url: theUrl,
|
||||||
|
method: "DELETE",
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} else {
|
||||||
await fetch(theUrl, {
|
await fetch(theUrl, {
|
||||||
method: "DELETE",
|
method: "DELETE",
|
||||||
headers: {
|
headers: {
|
||||||
Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
|
Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
putArrayBuffer = async (pathFragOrig: string, payload: ArrayBuffer) => {
|
putArrayBuffer = async (pathFragOrig: string, payload: ArrayBuffer) => {
|
||||||
const theUrl = this.buildUrl(pathFragOrig);
|
const theUrl = this.buildUrl(pathFragOrig);
|
||||||
log.debug(`putArrayBuffer, theUrl=${theUrl}`);
|
log.debug(`putArrayBuffer, theUrl=${theUrl}`);
|
||||||
// TODO: Obsidian doesn't support ArrayBuffer
|
if (requireApiVersion(API_VER_REQURL)) {
|
||||||
// currently downgraded to fetch()!
|
await requestUrl({
|
||||||
|
url: theUrl,
|
||||||
|
method: "PUT",
|
||||||
|
body: payload,
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} else {
|
||||||
await fetch(theUrl, {
|
await fetch(theUrl, {
|
||||||
method: "PUT",
|
method: "PUT",
|
||||||
body: payload,
|
body: payload,
|
||||||
@ -504,6 +521,7 @@ export class WrappedOnedriveClient {
|
|||||||
Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
|
Authorization: `Bearer ${await this.authGetter.getAccessToken()}`,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -527,7 +545,7 @@ export class WrappedOnedriveClient {
|
|||||||
rangeEnd - 1
|
rangeEnd - 1
|
||||||
}, len=${rangeEnd - rangeStart}, size=${size}`
|
}, len=${rangeEnd - rangeStart}, size=${size}`
|
||||||
);
|
);
|
||||||
// TODO: Obsidian doesn't support ArrayBuffer
|
// obsidian requestUrl doesn't support setting Content-Length
|
||||||
// currently downgraded to fetch()!
|
// currently downgraded to fetch()!
|
||||||
// AND, NO AUTH HEADER here!
|
// AND, NO AUTH HEADER here!
|
||||||
const res = await fetch(theUrl, {
|
const res = await fetch(theUrl, {
|
||||||
@ -539,8 +557,7 @@ export class WrappedOnedriveClient {
|
|||||||
"Content-Type": "application/octet-stream",
|
"Content-Type": "application/octet-stream",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
return (await res.json()) as DriveItem | UploadSession;
|
||||||
return res.json() as DriveItem | UploadSession;
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -704,6 +721,19 @@ export const uploadToRemote = async (
|
|||||||
|
|
||||||
// no need to create parent folders firstly, cool!
|
// no need to create parent folders firstly, cool!
|
||||||
|
|
||||||
|
// hard code range size
|
||||||
|
const MIN_UNIT = 327680; // bytes in msft doc, about 0.32768 MB
|
||||||
|
const RANGE_SIZE = MIN_UNIT * 20; // about 6.5536 MB
|
||||||
|
|
||||||
|
if (remoteContent.byteLength <= RANGE_SIZE) {
|
||||||
|
// directly using put!
|
||||||
|
await client.putArrayBuffer(
|
||||||
|
`${uploadFile}:/content?${new URLSearchParams({
|
||||||
|
"@microsoft.graph.conflictBehavior": "replace",
|
||||||
|
})}`,
|
||||||
|
remoteContent
|
||||||
|
);
|
||||||
|
} else {
|
||||||
// upload large files!
|
// upload large files!
|
||||||
// ref: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession?view=odsp-graph-online
|
// ref: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession?view=odsp-graph-online
|
||||||
|
|
||||||
@ -724,9 +754,7 @@ export const uploadToRemote = async (
|
|||||||
// 2. upload by ranges
|
// 2. upload by ranges
|
||||||
// convert to uint8
|
// convert to uint8
|
||||||
const uint8 = new Uint8Array(remoteContent);
|
const uint8 = new Uint8Array(remoteContent);
|
||||||
// hard code range size
|
|
||||||
const MIN_UNIT = 327680; // bytes in msft doc, about 0.32768 MB
|
|
||||||
const RANGE_SIZE = MIN_UNIT * 20; // about 6.5536 MB
|
|
||||||
// upload the ranges one by one
|
// upload the ranges one by one
|
||||||
let rangeStart = 0;
|
let rangeStart = 0;
|
||||||
while (rangeStart < uint8.byteLength) {
|
while (rangeStart < uint8.byteLength) {
|
||||||
@ -739,6 +767,7 @@ export const uploadToRemote = async (
|
|||||||
);
|
);
|
||||||
rangeStart += RANGE_SIZE;
|
rangeStart += RANGE_SIZE;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const res = await getRemoteMeta(client, uploadFile);
|
const res = await getRemoteMeta(client, uploadFile);
|
||||||
return res;
|
return res;
|
||||||
@ -755,8 +784,13 @@ const downloadFromRemoteRaw = async (
|
|||||||
`${key}?$select=@microsoft.graph.downloadUrl`
|
`${key}?$select=@microsoft.graph.downloadUrl`
|
||||||
);
|
);
|
||||||
const downloadUrl: string = rsp["@microsoft.graph.downloadUrl"];
|
const downloadUrl: string = rsp["@microsoft.graph.downloadUrl"];
|
||||||
|
if (requireApiVersion(API_VER_REQURL)) {
|
||||||
|
const content = (await requestUrl({ url: downloadUrl })).arrayBuffer;
|
||||||
|
return content;
|
||||||
|
} else {
|
||||||
const content = await (await fetch(downloadUrl)).arrayBuffer();
|
const content = await (await fetch(downloadUrl)).arrayBuffer();
|
||||||
return content;
|
return content;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const downloadFromRemote = async (
|
export const downloadFromRemote = async (
|
||||||
|
|||||||
@ -11,11 +11,26 @@ import {
|
|||||||
S3Client,
|
S3Client,
|
||||||
} from "@aws-sdk/client-s3";
|
} from "@aws-sdk/client-s3";
|
||||||
import { Upload } from "@aws-sdk/lib-storage";
|
import { Upload } from "@aws-sdk/lib-storage";
|
||||||
|
import { HttpHandler, HttpRequest, HttpResponse } from "@aws-sdk/protocol-http";
|
||||||
|
import {
|
||||||
|
FetchHttpHandler,
|
||||||
|
FetchHttpHandlerOptions,
|
||||||
|
} from "@aws-sdk/fetch-http-handler";
|
||||||
|
// @ts-ignore
|
||||||
|
import { requestTimeout } from "@aws-sdk/fetch-http-handler/dist-es/request-timeout";
|
||||||
|
import { buildQueryString } from "@aws-sdk/querystring-builder";
|
||||||
|
import { HeaderBag, HttpHandlerOptions, Provider } from "@aws-sdk/types";
|
||||||
import { Buffer } from "buffer";
|
import { Buffer } from "buffer";
|
||||||
import * as mime from "mime-types";
|
import * as mime from "mime-types";
|
||||||
import { Vault } from "obsidian";
|
import {
|
||||||
|
Vault,
|
||||||
|
requestUrl,
|
||||||
|
RequestUrlParam,
|
||||||
|
RequestUrlResponse,
|
||||||
|
requireApiVersion,
|
||||||
|
} from "obsidian";
|
||||||
import { Readable } from "stream";
|
import { Readable } from "stream";
|
||||||
import { RemoteItem, S3Config } from "./baseTypes";
|
import { API_VER_REQURL, RemoteItem, S3Config } from "./baseTypes";
|
||||||
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
|
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
|
||||||
import {
|
import {
|
||||||
arrayBufferToBuffer,
|
arrayBufferToBuffer,
|
||||||
@ -28,12 +43,117 @@ export { S3Client } from "@aws-sdk/client-s3";
|
|||||||
import * as origLog from "loglevel";
|
import * as origLog from "loglevel";
|
||||||
const log = origLog.getLogger("rs-default");
|
const log = origLog.getLogger("rs-default");
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// special handler using Obsidian requestUrl
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is close to origin implementation of FetchHttpHandler
|
||||||
|
* https://github.com/aws/aws-sdk-js-v3/blob/main/packages/fetch-http-handler/src/fetch-http-handler.ts
|
||||||
|
* that is released under Apache 2 License.
|
||||||
|
* But this uses Obsidian requestUrl instead.
|
||||||
|
*/
|
||||||
|
class ObsHttpHandler extends FetchHttpHandler {
|
||||||
|
requestTimeoutInMs: number;
|
||||||
|
constructor(options?: FetchHttpHandlerOptions) {
|
||||||
|
super(options);
|
||||||
|
this.requestTimeoutInMs =
|
||||||
|
options === undefined ? undefined : options.requestTimeout;
|
||||||
|
}
|
||||||
|
async handle(
|
||||||
|
request: HttpRequest,
|
||||||
|
{ abortSignal }: HttpHandlerOptions = {}
|
||||||
|
): Promise<{ response: HttpResponse }> {
|
||||||
|
if (abortSignal?.aborted) {
|
||||||
|
const abortError = new Error("Request aborted");
|
||||||
|
abortError.name = "AbortError";
|
||||||
|
return Promise.reject(abortError);
|
||||||
|
}
|
||||||
|
|
||||||
|
let path = request.path;
|
||||||
|
if (request.query) {
|
||||||
|
const queryString = buildQueryString(request.query);
|
||||||
|
if (queryString) {
|
||||||
|
path += `?${queryString}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const { port, method } = request;
|
||||||
|
const url = `${request.protocol}//${request.hostname}${
|
||||||
|
port ? `:${port}` : ""
|
||||||
|
}${path}`;
|
||||||
|
const body =
|
||||||
|
method === "GET" || method === "HEAD" ? undefined : request.body;
|
||||||
|
|
||||||
|
const transformedHeaders = { ...request.headers };
|
||||||
|
delete transformedHeaders["host"];
|
||||||
|
delete transformedHeaders["Host"];
|
||||||
|
delete transformedHeaders["content-length"];
|
||||||
|
delete transformedHeaders["Content-Length"];
|
||||||
|
|
||||||
|
let contentType: string = undefined;
|
||||||
|
if (transformedHeaders["content-type"] !== undefined) {
|
||||||
|
contentType = transformedHeaders["content-type"];
|
||||||
|
}
|
||||||
|
|
||||||
|
let transformedBody: any = body;
|
||||||
|
if (ArrayBuffer.isView(body)) {
|
||||||
|
transformedBody = bufferToArrayBuffer(body);
|
||||||
|
}
|
||||||
|
|
||||||
|
const param: RequestUrlParam = {
|
||||||
|
body: transformedBody,
|
||||||
|
headers: transformedHeaders,
|
||||||
|
method: method,
|
||||||
|
url: url,
|
||||||
|
contentType: contentType,
|
||||||
|
};
|
||||||
|
|
||||||
|
const raceOfPromises = [
|
||||||
|
requestUrl(param).then((rsp) => {
|
||||||
|
const stream = new ReadableStream<Uint8Array>({
|
||||||
|
start(controller) {
|
||||||
|
controller.enqueue(new Uint8Array(rsp.arrayBuffer));
|
||||||
|
controller.close();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
response: new HttpResponse({
|
||||||
|
headers: rsp.headers,
|
||||||
|
statusCode: rsp.status,
|
||||||
|
body: stream,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
requestTimeout(this.requestTimeoutInMs),
|
||||||
|
];
|
||||||
|
|
||||||
|
if (abortSignal) {
|
||||||
|
raceOfPromises.push(
|
||||||
|
new Promise<never>((resolve, reject) => {
|
||||||
|
abortSignal.onabort = () => {
|
||||||
|
const abortError = new Error("Request aborted");
|
||||||
|
abortError.name = "AbortError";
|
||||||
|
reject(abortError);
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Promise.race(raceOfPromises);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// other stuffs
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
export const DEFAULT_S3_CONFIG = {
|
export const DEFAULT_S3_CONFIG = {
|
||||||
s3Endpoint: "",
|
s3Endpoint: "",
|
||||||
s3Region: "",
|
s3Region: "",
|
||||||
s3AccessKeyID: "",
|
s3AccessKeyID: "",
|
||||||
s3SecretAccessKey: "",
|
s3SecretAccessKey: "",
|
||||||
s3BucketName: "",
|
s3BucketName: "",
|
||||||
|
bypassCorsLocally: true,
|
||||||
};
|
};
|
||||||
|
|
||||||
export type S3ObjectType = _Object;
|
export type S3ObjectType = _Object;
|
||||||
@ -66,6 +186,19 @@ export const getS3Client = (s3Config: S3Config) => {
|
|||||||
if (!(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {
|
if (!(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {
|
||||||
endpoint = `https://${endpoint}`;
|
endpoint = `https://${endpoint}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (requireApiVersion(API_VER_REQURL) && s3Config.bypassCorsLocally) {
|
||||||
|
const s3Client = new S3Client({
|
||||||
|
region: s3Config.s3Region,
|
||||||
|
endpoint: endpoint,
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: s3Config.s3AccessKeyID,
|
||||||
|
secretAccessKey: s3Config.s3SecretAccessKey,
|
||||||
|
},
|
||||||
|
requestHandler: new ObsHttpHandler(),
|
||||||
|
});
|
||||||
|
return s3Client;
|
||||||
|
} else {
|
||||||
const s3Client = new S3Client({
|
const s3Client = new S3Client({
|
||||||
region: s3Config.s3Region,
|
region: s3Config.s3Region,
|
||||||
endpoint: endpoint,
|
endpoint: endpoint,
|
||||||
@ -75,6 +208,7 @@ export const getS3Client = (s3Config: S3Config) => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
return s3Client;
|
return s3Client;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getRemoteMeta = async (
|
export const getRemoteMeta = async (
|
||||||
@ -152,12 +286,13 @@ export const uploadToRemote = async (
|
|||||||
if (password !== "") {
|
if (password !== "") {
|
||||||
remoteContent = await encryptArrayBuffer(localContent, password);
|
remoteContent = await encryptArrayBuffer(localContent, password);
|
||||||
}
|
}
|
||||||
const body = arrayBufferToBuffer(remoteContent);
|
|
||||||
|
|
||||||
|
const bytesIn5MB = 5242880;
|
||||||
|
const body = new Uint8Array(remoteContent);
|
||||||
const upload = new Upload({
|
const upload = new Upload({
|
||||||
client: s3Client,
|
client: s3Client,
|
||||||
queueSize: 20, // concurrency
|
queueSize: 20, // concurrency
|
||||||
partSize: 5242880, // minimal 5MB by default
|
partSize: bytesIn5MB, // minimal 5MB by default
|
||||||
leavePartsOnError: false,
|
leavePartsOnError: false,
|
||||||
params: {
|
params: {
|
||||||
Bucket: s3Config.s3BucketName,
|
Bucket: s3Config.s3BucketName,
|
||||||
|
|||||||
@ -1,18 +1,117 @@
|
|||||||
import { Buffer } from "buffer";
|
import { Buffer } from "buffer";
|
||||||
import { Vault } from "obsidian";
|
import { Vault, request, requestUrl, requireApiVersion } from "obsidian";
|
||||||
import type { FileStat, WebDAVClient } from "webdav/web";
|
|
||||||
import { AuthType, BufferLike, createClient } from "webdav/web";
|
|
||||||
import { Queue } from "@fyears/tsqueue";
|
import { Queue } from "@fyears/tsqueue";
|
||||||
import chunk from "lodash/chunk";
|
import chunk from "lodash/chunk";
|
||||||
import flatten from "lodash/flatten";
|
import flatten from "lodash/flatten";
|
||||||
import type { RemoteItem, WebdavConfig } from "./baseTypes";
|
import { getReasonPhrase } from "http-status-codes";
|
||||||
|
import { API_VER_REQURL, RemoteItem, WebdavConfig } from "./baseTypes";
|
||||||
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
|
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
|
||||||
import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
|
import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
|
||||||
export type { WebDAVClient } from "webdav/web";
|
|
||||||
|
|
||||||
import * as origLog from "loglevel";
|
import * as origLog from "loglevel";
|
||||||
const log = origLog.getLogger("rs-default");
|
const log = origLog.getLogger("rs-default");
|
||||||
|
|
||||||
|
import type {
|
||||||
|
FileStat,
|
||||||
|
WebDAVClient,
|
||||||
|
RequestOptionsWithState,
|
||||||
|
Response,
|
||||||
|
ResponseDataDetailed,
|
||||||
|
} from "webdav/web";
|
||||||
|
import { getPatcher } from "webdav/web";
|
||||||
|
if (requireApiVersion(API_VER_REQURL)) {
|
||||||
|
getPatcher().patch(
|
||||||
|
"request",
|
||||||
|
async (
|
||||||
|
options: RequestOptionsWithState
|
||||||
|
): Promise<Response | ResponseDataDetailed<any>> => {
|
||||||
|
const transformedHeaders = { ...options.headers };
|
||||||
|
delete transformedHeaders["host"];
|
||||||
|
delete transformedHeaders["Host"];
|
||||||
|
delete transformedHeaders["content-length"];
|
||||||
|
delete transformedHeaders["Content-Length"];
|
||||||
|
const r = await requestUrl({
|
||||||
|
url: options.url,
|
||||||
|
method: options.method,
|
||||||
|
body: options.data as string | ArrayBuffer,
|
||||||
|
headers: transformedHeaders,
|
||||||
|
});
|
||||||
|
|
||||||
|
let r2: Response | ResponseDataDetailed<any> = undefined;
|
||||||
|
if (options.responseType === undefined) {
|
||||||
|
r2 = {
|
||||||
|
data: undefined,
|
||||||
|
status: r.status,
|
||||||
|
statusText: getReasonPhrase(r.status),
|
||||||
|
headers: r.headers,
|
||||||
|
};
|
||||||
|
} else if (options.responseType === "json") {
|
||||||
|
r2 = {
|
||||||
|
data: r.json,
|
||||||
|
status: r.status,
|
||||||
|
statusText: getReasonPhrase(r.status),
|
||||||
|
headers: r.headers,
|
||||||
|
};
|
||||||
|
} else if (options.responseType === "text") {
|
||||||
|
r2 = {
|
||||||
|
data: r.text,
|
||||||
|
status: r.status,
|
||||||
|
statusText: getReasonPhrase(r.status),
|
||||||
|
headers: r.headers,
|
||||||
|
};
|
||||||
|
} else if (options.responseType === "arraybuffer") {
|
||||||
|
r2 = {
|
||||||
|
data: r.arrayBuffer,
|
||||||
|
status: r.status,
|
||||||
|
statusText: getReasonPhrase(r.status),
|
||||||
|
headers: r.headers,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
throw Error(
|
||||||
|
`do not know how to deal with responseType = ${options.responseType}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return r2;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// getPatcher().patch("request", (options: any) => {
|
||||||
|
// // console.log("using fetch");
|
||||||
|
// const r = fetch(options.url, {
|
||||||
|
// method: options.method,
|
||||||
|
// body: options.data as any,
|
||||||
|
// headers: options.headers,
|
||||||
|
// signal: options.signal,
|
||||||
|
// })
|
||||||
|
// .then((rsp) => {
|
||||||
|
// if (options.responseType === undefined) {
|
||||||
|
// return Promise.all([undefined, rsp]);
|
||||||
|
// }
|
||||||
|
// if (options.responseType === "json") {
|
||||||
|
// return Promise.all([rsp.json(), rsp]);
|
||||||
|
// }
|
||||||
|
// if (options.responseType === "text") {
|
||||||
|
// return Promise.all([rsp.text(), rsp]);
|
||||||
|
// }
|
||||||
|
// if (options.responseType === "arraybuffer") {
|
||||||
|
// return Promise.all([rsp.arrayBuffer(), rsp]);
|
||||||
|
// }
|
||||||
|
// })
|
||||||
|
// .then(([d, r]) => {
|
||||||
|
// return {
|
||||||
|
// data: d,
|
||||||
|
// status: r.status,
|
||||||
|
// statusText: r.statusText,
|
||||||
|
// headers: r.headers,
|
||||||
|
// };
|
||||||
|
// });
|
||||||
|
// // console.log("using fetch");
|
||||||
|
// return r;
|
||||||
|
// });
|
||||||
|
import { AuthType, BufferLike, createClient } from "webdav/web";
|
||||||
|
export type { WebDAVClient } from "webdav/web";
|
||||||
|
|
||||||
export const DEFAULT_WEBDAV_CONFIG = {
|
export const DEFAULT_WEBDAV_CONFIG = {
|
||||||
address: "",
|
address: "",
|
||||||
username: "",
|
username: "",
|
||||||
@ -169,7 +268,7 @@ export const uploadToRemote = async (
|
|||||||
// if encrypted, upload a fake file with the encrypted file name
|
// if encrypted, upload a fake file with the encrypted file name
|
||||||
await client.client.putFileContents(uploadFile, "", {
|
await client.client.putFileContents(uploadFile, "", {
|
||||||
overwrite: true,
|
overwrite: true,
|
||||||
onUploadProgress: (progress) => {
|
onUploadProgress: (progress: any) => {
|
||||||
// log.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
|
// log.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@ -200,7 +299,7 @@ export const uploadToRemote = async (
|
|||||||
}
|
}
|
||||||
await client.client.putFileContents(uploadFile, remoteContent, {
|
await client.client.putFileContents(uploadFile, remoteContent, {
|
||||||
overwrite: true,
|
overwrite: true,
|
||||||
onUploadProgress: (progress) => {
|
onUploadProgress: (progress: any) => {
|
||||||
log.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
|
log.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@ -5,8 +5,13 @@ import {
|
|||||||
PluginSettingTab,
|
PluginSettingTab,
|
||||||
Setting,
|
Setting,
|
||||||
Platform,
|
Platform,
|
||||||
|
requireApiVersion,
|
||||||
} from "obsidian";
|
} from "obsidian";
|
||||||
import type { SUPPORTED_SERVICES_TYPE, WebdavAuthType } from "./baseTypes";
|
import {
|
||||||
|
API_VER_REQURL,
|
||||||
|
SUPPORTED_SERVICES_TYPE,
|
||||||
|
WebdavAuthType,
|
||||||
|
} from "./baseTypes";
|
||||||
import { exportVaultSyncPlansToFiles } from "./debugMode";
|
import { exportVaultSyncPlansToFiles } from "./debugMode";
|
||||||
import { exportQrCodeUri } from "./importExport";
|
import { exportQrCodeUri } from "./importExport";
|
||||||
import {
|
import {
|
||||||
@ -595,9 +600,11 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
cls: "s3-disclaimer",
|
cls: "s3-disclaimer",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (!requireApiVersion(API_VER_REQURL)) {
|
||||||
s3Div.createEl("p", {
|
s3Div.createEl("p", {
|
||||||
text: "You need to configure CORS to allow requests from origin app://obsidian.md and capacitor://localhost and http://localhost",
|
text: "You need to configure CORS to allow requests from origin app://obsidian.md and capacitor://localhost and http://localhost",
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
s3Div.createEl("p", {
|
s3Div.createEl("p", {
|
||||||
text: "Some Amazon S3 official docs for references:",
|
text: "Some Amazon S3 official docs for references:",
|
||||||
@ -615,10 +622,12 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
text: "Access key ID and Secret access key info",
|
text: "Access key ID and Secret access key info",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (!requireApiVersion(API_VER_REQURL)) {
|
||||||
s3LinksUl.createEl("li").createEl("a", {
|
s3LinksUl.createEl("li").createEl("a", {
|
||||||
href: "https://docs.aws.amazon.com/AmazonS3/latest/userguide/enabling-cors-examples.html",
|
href: "https://docs.aws.amazon.com/AmazonS3/latest/userguide/enabling-cors-examples.html",
|
||||||
text: "Configuring CORS",
|
text: "Configuring CORS",
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
new Setting(s3Div)
|
new Setting(s3Div)
|
||||||
.setName("s3Endpoint")
|
.setName("s3Endpoint")
|
||||||
@ -687,6 +696,34 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
|
if (requireApiVersion(API_VER_REQURL)) {
|
||||||
|
new Setting(s3Div)
|
||||||
|
.setName("bypass CORS issue locally")
|
||||||
|
.setDesc(
|
||||||
|
`The plugin allows skipping server CORS config in new version (Obsidian>=${API_VER_REQURL}). If you encounter any issues, please disable this setting and config CORS (app://obsidian.md and capacitor://localhost and http://localhost) on server.`
|
||||||
|
)
|
||||||
|
.addDropdown((dropdown) => {
|
||||||
|
dropdown
|
||||||
|
.addOption("disable", "disable")
|
||||||
|
.addOption("enable", "enable");
|
||||||
|
|
||||||
|
dropdown
|
||||||
|
.setValue(
|
||||||
|
`${
|
||||||
|
this.plugin.settings.s3.bypassCorsLocally ? "enable" : "disable"
|
||||||
|
}`
|
||||||
|
)
|
||||||
|
.onChange(async (value) => {
|
||||||
|
if (value === "enable") {
|
||||||
|
this.plugin.settings.s3.bypassCorsLocally = true;
|
||||||
|
} else {
|
||||||
|
this.plugin.settings.s3.bypassCorsLocally = false;
|
||||||
|
}
|
||||||
|
await this.plugin.saveSettings();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
new Setting(s3Div)
|
new Setting(s3Div)
|
||||||
.setName("check connectivity")
|
.setName("check connectivity")
|
||||||
.setDesc("check connectivity")
|
.setDesc("check connectivity")
|
||||||
@ -960,9 +997,11 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
cls: "webdav-disclaimer",
|
cls: "webdav-disclaimer",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (!requireApiVersion(API_VER_REQURL)) {
|
||||||
webdavDiv.createEl("p", {
|
webdavDiv.createEl("p", {
|
||||||
text: "You need to configure CORS to allow requests from origin app://obsidian.md and capacitor://localhost and http://localhost",
|
text: "You need to configure CORS to allow requests from origin app://obsidian.md and capacitor://localhost and http://localhost",
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
webdavDiv.createEl("p", {
|
webdavDiv.createEl("p", {
|
||||||
text: `We will create and sync inside the folder /${this.app.vault.getName()} on your server.`,
|
text: `We will create and sync inside the folder /${this.app.vault.getName()} on your server.`,
|
||||||
@ -1010,9 +1049,20 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
new Setting(webdavDiv)
|
new Setting(webdavDiv)
|
||||||
.setName("server auth type")
|
.setName("server auth type")
|
||||||
.setDesc("If no password, this option would be ignored.")
|
.setDesc("If no password, this option would be ignored.")
|
||||||
.addDropdown((dropdown) => {
|
.addDropdown(async (dropdown) => {
|
||||||
dropdown.addOption("basic", "basic");
|
dropdown.addOption("basic", "basic");
|
||||||
// dropdown.addOption("digest", "digest");
|
if (requireApiVersion(API_VER_REQURL)) {
|
||||||
|
dropdown.addOption("digest", "digest");
|
||||||
|
}
|
||||||
|
|
||||||
|
// new version config, copied to old version, we need to reset it
|
||||||
|
if (
|
||||||
|
!requireApiVersion(API_VER_REQURL) &&
|
||||||
|
this.plugin.settings.webdav.authType !== "basic"
|
||||||
|
) {
|
||||||
|
this.plugin.settings.webdav.authType = "basic";
|
||||||
|
await this.plugin.saveSettings();
|
||||||
|
}
|
||||||
|
|
||||||
dropdown
|
dropdown
|
||||||
.setValue(this.plugin.settings.webdav.authType)
|
.setValue(this.plugin.settings.webdav.authType)
|
||||||
@ -1067,8 +1117,13 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
|||||||
if (res) {
|
if (res) {
|
||||||
new Notice("Great! The webdav server can be accessed.");
|
new Notice("Great! The webdav server can be accessed.");
|
||||||
} else {
|
} else {
|
||||||
|
let corsErrMsg = "/CORS";
|
||||||
|
if (requireApiVersion(API_VER_REQURL)) {
|
||||||
|
corsErrMsg = "";
|
||||||
|
}
|
||||||
|
|
||||||
new Notice(
|
new Notice(
|
||||||
"The webdav server cannot be reached (possible to be any of address/username/password/authtype/CORS errors)."
|
`The webdav server cannot be reached (possible to be any of address/username/password/authtype${corsErrMsg} errors).`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@ -180,3 +180,56 @@ describe("Misc: extract svg", () => {
|
|||||||
expect(y).to.equal("<rect/><g/>");
|
expect(y).to.equal("<rect/><g/>");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("Misc: get split ranges", () => {
|
||||||
|
it("should deal with big parts", () => {
|
||||||
|
const k = misc.getSplitRanges(10, 20);
|
||||||
|
const k2: misc.SplitRange[] = [
|
||||||
|
{
|
||||||
|
partNum: 1,
|
||||||
|
start: 0,
|
||||||
|
end: 10,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
expect(k).to.deep.equal(k2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should deal with 0 remainder", () => {
|
||||||
|
const k = misc.getSplitRanges(20, 10);
|
||||||
|
const k2: misc.SplitRange[] = [
|
||||||
|
{
|
||||||
|
partNum: 1,
|
||||||
|
start: 0,
|
||||||
|
end: 10,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
partNum: 2,
|
||||||
|
start: 10,
|
||||||
|
end: 20,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
expect(k).to.deep.equal(k2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should deal with not-0 remainder", () => {
|
||||||
|
const k = misc.getSplitRanges(25, 10);
|
||||||
|
const k2: misc.SplitRange[] = [
|
||||||
|
{
|
||||||
|
partNum: 1,
|
||||||
|
start: 0,
|
||||||
|
end: 10,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
partNum: 2,
|
||||||
|
start: 10,
|
||||||
|
end: 20,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
partNum: 3,
|
||||||
|
start: 20,
|
||||||
|
end: 25,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
expect(k).to.deep.equal(k2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user