A big commit Squashed commit of CORS:
commit 8cffa38ebae2a46b7c8e855c7b21a124e35adc89 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Thu Mar 10 23:52:56 2022 +0800 bypass more cors for onedrive commit 1b59ac1e58032099068aab55d22ef96c6396f203 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 23:58:28 2022 +0800 change wordings for webdav cors commit 73142eb18b59fff20839680e866f51cfcb0a6226 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 23:38:58 2022 +0800 remove cors hint for webdav commit 7dbb0b49d50e529b2b72e55ea2c8503ba7fa9268 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 23:31:54 2022 +0800 fix webdav commit c28c4e19720a56230d483acf306463d42e619fa4 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 23:31:35 2022 +0800 remove more headers commit 4eeae7043fa68d669a5c23c5549c14c1260ce638 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 23:18:32 2022 +0800 polish cors hints for s3 commit d9e55a91a1c413e9419cd6b30a3a6e3b403d483b Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 22:40:37 2022 +0800 fix format commit b780a3eb4e37b05b8e8b92d6a2f9283b3459d738 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 22:37:02 2022 +0800 finally correctly inject requestUrl into s3 commit 6a55a1a43d7653d65579ab88aa816e5d54cd276a Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 22:33:18 2022 +0800 to arraybuffer from view commit 2f2607b4f0a3d9db5943528ced57cb2fdb419607 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Wed Mar 9 13:31:22 2022 +0800 add split ranges commit ea24da24dea83fdb770e7e391cf8a2e4fea78d0d Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Mar 6 22:57:50 2022 +0800 add settings of bypassing for s3 commit 2f099dc8ca1e66ea137b28dd329be50968734ba6 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Mar 6 22:38:07 2022 +0800 use api ver var commit 74c7ce2449a88cbe7c7f50cbb687b36ff3732c04 Author: fyears <1142836+fyears@users.noreply.github.com> Date: Sun Mar 6 22:37:25 2022 +0800 correct way to inject s3 commit f29945d73132d21b2c44472ec2cafc06b9d71e8f Author: fyears <1142836+fyears@users.noreply.github.com> Date: Tue Mar 1 00:09:57 2022 +0800 add new http handler commit d55104cb08e168cbcc243cf901cbd7f46f2e324b Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 28 22:59:55 2022 +0800 add types for patch commit 50b79ade7188ee7dfab9c1d03119585db358ba6f Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 28 08:25:19 2022 +0800 remove verbose commit 83f0e71aa15aa7586f6d4e105cd77c25c2e113ce Author: fyears <1142836+fyears@users.noreply.github.com> Date: Mon Feb 28 08:25:04 2022 +0800 patch webdav!
This commit is contained in:
fyears
+148
-13
@@ -11,11 +11,26 @@ import {
|
||||
S3Client,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
import { HttpHandler, HttpRequest, HttpResponse } from "@aws-sdk/protocol-http";
|
||||
import {
|
||||
FetchHttpHandler,
|
||||
FetchHttpHandlerOptions,
|
||||
} from "@aws-sdk/fetch-http-handler";
|
||||
// @ts-ignore
|
||||
import { requestTimeout } from "@aws-sdk/fetch-http-handler/dist-es/request-timeout";
|
||||
import { buildQueryString } from "@aws-sdk/querystring-builder";
|
||||
import { HeaderBag, HttpHandlerOptions, Provider } from "@aws-sdk/types";
|
||||
import { Buffer } from "buffer";
|
||||
import * as mime from "mime-types";
|
||||
import { Vault } from "obsidian";
|
||||
import {
|
||||
Vault,
|
||||
requestUrl,
|
||||
RequestUrlParam,
|
||||
RequestUrlResponse,
|
||||
requireApiVersion,
|
||||
} from "obsidian";
|
||||
import { Readable } from "stream";
|
||||
import { RemoteItem, S3Config } from "./baseTypes";
|
||||
import { API_VER_REQURL, RemoteItem, S3Config } from "./baseTypes";
|
||||
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
|
||||
import {
|
||||
arrayBufferToBuffer,
|
||||
@@ -28,12 +43,117 @@ export { S3Client } from "@aws-sdk/client-s3";
|
||||
import * as origLog from "loglevel";
|
||||
const log = origLog.getLogger("rs-default");
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// special handler using Obsidian requestUrl
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/**
|
||||
* This is close to origin implementation of FetchHttpHandler
|
||||
* https://github.com/aws/aws-sdk-js-v3/blob/main/packages/fetch-http-handler/src/fetch-http-handler.ts
|
||||
* that is released under Apache 2 License.
|
||||
* But this uses Obsidian requestUrl instead.
|
||||
*/
|
||||
class ObsHttpHandler extends FetchHttpHandler {
|
||||
requestTimeoutInMs: number;
|
||||
constructor(options?: FetchHttpHandlerOptions) {
|
||||
super(options);
|
||||
this.requestTimeoutInMs =
|
||||
options === undefined ? undefined : options.requestTimeout;
|
||||
}
|
||||
async handle(
|
||||
request: HttpRequest,
|
||||
{ abortSignal }: HttpHandlerOptions = {}
|
||||
): Promise<{ response: HttpResponse }> {
|
||||
if (abortSignal?.aborted) {
|
||||
const abortError = new Error("Request aborted");
|
||||
abortError.name = "AbortError";
|
||||
return Promise.reject(abortError);
|
||||
}
|
||||
|
||||
let path = request.path;
|
||||
if (request.query) {
|
||||
const queryString = buildQueryString(request.query);
|
||||
if (queryString) {
|
||||
path += `?${queryString}`;
|
||||
}
|
||||
}
|
||||
|
||||
const { port, method } = request;
|
||||
const url = `${request.protocol}//${request.hostname}${
|
||||
port ? `:${port}` : ""
|
||||
}${path}`;
|
||||
const body =
|
||||
method === "GET" || method === "HEAD" ? undefined : request.body;
|
||||
|
||||
const transformedHeaders = { ...request.headers };
|
||||
delete transformedHeaders["host"];
|
||||
delete transformedHeaders["Host"];
|
||||
delete transformedHeaders["content-length"];
|
||||
delete transformedHeaders["Content-Length"];
|
||||
|
||||
let contentType: string = undefined;
|
||||
if (transformedHeaders["content-type"] !== undefined) {
|
||||
contentType = transformedHeaders["content-type"];
|
||||
}
|
||||
|
||||
let transformedBody: any = body;
|
||||
if (ArrayBuffer.isView(body)) {
|
||||
transformedBody = bufferToArrayBuffer(body);
|
||||
}
|
||||
|
||||
const param: RequestUrlParam = {
|
||||
body: transformedBody,
|
||||
headers: transformedHeaders,
|
||||
method: method,
|
||||
url: url,
|
||||
contentType: contentType,
|
||||
};
|
||||
|
||||
const raceOfPromises = [
|
||||
requestUrl(param).then((rsp) => {
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(rsp.arrayBuffer));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
return {
|
||||
response: new HttpResponse({
|
||||
headers: rsp.headers,
|
||||
statusCode: rsp.status,
|
||||
body: stream,
|
||||
}),
|
||||
};
|
||||
}),
|
||||
requestTimeout(this.requestTimeoutInMs),
|
||||
];
|
||||
|
||||
if (abortSignal) {
|
||||
raceOfPromises.push(
|
||||
new Promise<never>((resolve, reject) => {
|
||||
abortSignal.onabort = () => {
|
||||
const abortError = new Error("Request aborted");
|
||||
abortError.name = "AbortError";
|
||||
reject(abortError);
|
||||
};
|
||||
})
|
||||
);
|
||||
}
|
||||
return Promise.race(raceOfPromises);
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// other stuffs
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
export const DEFAULT_S3_CONFIG = {
|
||||
s3Endpoint: "",
|
||||
s3Region: "",
|
||||
s3AccessKeyID: "",
|
||||
s3SecretAccessKey: "",
|
||||
s3BucketName: "",
|
||||
bypassCorsLocally: true,
|
||||
};
|
||||
|
||||
export type S3ObjectType = _Object;
|
||||
@@ -66,15 +186,29 @@ export const getS3Client = (s3Config: S3Config) => {
|
||||
if (!(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {
|
||||
endpoint = `https://${endpoint}`;
|
||||
}
|
||||
const s3Client = new S3Client({
|
||||
region: s3Config.s3Region,
|
||||
endpoint: endpoint,
|
||||
credentials: {
|
||||
accessKeyId: s3Config.s3AccessKeyID,
|
||||
secretAccessKey: s3Config.s3SecretAccessKey,
|
||||
},
|
||||
});
|
||||
return s3Client;
|
||||
|
||||
if (requireApiVersion(API_VER_REQURL) && s3Config.bypassCorsLocally) {
|
||||
const s3Client = new S3Client({
|
||||
region: s3Config.s3Region,
|
||||
endpoint: endpoint,
|
||||
credentials: {
|
||||
accessKeyId: s3Config.s3AccessKeyID,
|
||||
secretAccessKey: s3Config.s3SecretAccessKey,
|
||||
},
|
||||
requestHandler: new ObsHttpHandler(),
|
||||
});
|
||||
return s3Client;
|
||||
} else {
|
||||
const s3Client = new S3Client({
|
||||
region: s3Config.s3Region,
|
||||
endpoint: endpoint,
|
||||
credentials: {
|
||||
accessKeyId: s3Config.s3AccessKeyID,
|
||||
secretAccessKey: s3Config.s3SecretAccessKey,
|
||||
},
|
||||
});
|
||||
return s3Client;
|
||||
}
|
||||
};
|
||||
|
||||
export const getRemoteMeta = async (
|
||||
@@ -152,12 +286,13 @@ export const uploadToRemote = async (
|
||||
if (password !== "") {
|
||||
remoteContent = await encryptArrayBuffer(localContent, password);
|
||||
}
|
||||
const body = arrayBufferToBuffer(remoteContent);
|
||||
|
||||
const bytesIn5MB = 5242880;
|
||||
const body = new Uint8Array(remoteContent);
|
||||
const upload = new Upload({
|
||||
client: s3Client,
|
||||
queueSize: 20, // concurrency
|
||||
partSize: 5242880, // minimal 5MB by default
|
||||
partSize: bytesIn5MB, // minimal 5MB by default
|
||||
leavePartsOnError: false,
|
||||
params: {
|
||||
Bucket: s3Config.s3BucketName,
|
||||
|
||||
Reference in New Issue
Block a user