From 69e72eae1d10caf571569e93defc5ef6aa4e4273 Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sun, 19 May 2024 21:41:01 +0800 Subject: [PATCH] correctly set range update --- src/fsWebdav.ts | 53 +++++++++++++++---------------- src/misc.ts | 31 +++++++++++++++++++ tests/misc.test.ts | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 26 deletions(-) diff --git a/src/fsWebdav.ts b/src/fsWebdav.ts index 486d3fd..cadc7c2 100644 --- a/src/fsWebdav.ts +++ b/src/fsWebdav.ts @@ -17,7 +17,7 @@ import type { import type { Entity, WebdavConfig } from "./baseTypes"; import { VALID_REQURL } from "./baseTypesObs"; import { FakeFs } from "./fsAll"; -import { bufferToArrayBuffer, delay } from "./misc"; +import { bufferToArrayBuffer, delay, splitFileSizeToChunkRanges } from "./misc"; /** * https://stackoverflow.com/questions/32850898/how-to-check-if-a-string-has-any-non-iso-8859-1-characters-with-javascript @@ -550,7 +550,7 @@ export class FakeFsWebdav extends FakeFs { ctime: number, origKey: string ): Promise { - console.debug(`start _writeFileFromRootFull`); + // console.debug(`start _writeFileFromRootFull`); await this.client.putFileContents(key, content, { overwrite: true, onUploadProgress: (progress: any) => { @@ -558,7 +558,7 @@ export class FakeFsWebdav extends FakeFs { }, }); const k = await this._statFromRoot(key); - console.debug(`end _writeFileFromRootFull`); + // console.debug(`end _writeFileFromRootFull`); return k; } @@ -623,19 +623,23 @@ export class FakeFsWebdav extends FakeFs { console.debug(`finish creating folder`); // upload by chunks - const size_5mb = 5 * 1024 * 1024; - let tmpFileIdx = 1; // a number between 1 and 10000 - let startInclusive = 0; - let endInclusive = Math.min(size_5mb, content.byteLength); - do { - const tmpFileName = `${tmpFileIdx}`.padStart(5, "0"); + const sizePerChunk = 5 * 1024 * 1024; // 5 mb + const chunkRanges = splitFileSizeToChunkRanges( + content.byteLength, + sizePerChunk + ); + for (let i = 0; i < chunkRanges.length; ++i) { + const { start, end } = chunkRanges[i]; + const tmpFileName = `${i + 1}`.padStart(5, "0"); const tmpFileNameWithFolder = `${tmpFolder}/${tmpFileName}`; console.debug( - `start to upload chunk ${tmpFileIdx} to ${tmpFileNameWithFolder} with startInclusive=${startInclusive}, endInclusive=${endInclusive}` + `start to upload chunk ${ + i + 1 + } to ${tmpFileNameWithFolder} with startInclusive=${start}, endInclusive=${end}` ); await this.client.putFileContents( tmpFileNameWithFolder, - content.slice(startInclusive, endInclusive - 1), + content.slice(start, end + 1), { headers: { Destination: destUrl, @@ -643,10 +647,7 @@ export class FakeFsWebdav extends FakeFs { }, } ); - tmpFileIdx += 1; - startInclusive = Math.min(startInclusive + size_5mb, content.byteLength); - endInclusive = Math.min(endInclusive + size_5mb, content.byteLength); - } while (startInclusive < content.byteLength); + } console.debug(`finish upload all chunks`); // move to assemble @@ -710,20 +711,20 @@ export class FakeFsWebdav extends FakeFs { ); // then "update" by chunks - const size_5mb = 5 * 1024 * 1024; - let startInclusive = 0; - let endInclusive = Math.min(size_5mb, content.byteLength); - do { + const sizePerChunk = 5 * 1024 * 1024; // 5 mb + const chunkRanges = splitFileSizeToChunkRanges( + content.byteLength, + sizePerChunk + ); + for (let i = 0; i < chunkRanges.length; ++i) { + const { start, end } = chunkRanges[i]; await this.client.partialUpdateFileContents( key, - startInclusive, - endInclusive, - content.slice(startInclusive, endInclusive - 1) + start, + end, + content.slice(start, end + 1) ); - - startInclusive = Math.min(startInclusive + size_5mb, content.byteLength); - endInclusive = Math.min(endInclusive + size_5mb, content.byteLength); - } while (startInclusive < content.byteLength); + } // lastly return return await this.stat(origKey); diff --git a/src/misc.ts b/src/misc.ts index d90d60b..e24b794 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -679,3 +679,34 @@ export const roughSizeOfObject = (object: any) => { } return bytes; }; + +export const splitFileSizeToChunkRanges = ( + totalSize: number, + chunkSize: number +) => { + if (totalSize < 0) { + throw Error(`totalSize should not be negative`); + } + if (chunkSize <= 0) { + throw Error(`chunkSize should not be negative or zero`); + } + + if (totalSize === 0) { + return []; + } + if (totalSize <= chunkSize) { + return [{ start: 0, end: totalSize - 1 }]; + } + + const res: { start: number; end: number }[] = []; + + const blocksCount = Math.ceil((totalSize * 1.0) / chunkSize); + + for (let i = 0; i < blocksCount; ++i) { + res.push({ + start: i * chunkSize, + end: Math.min((i + 1) * chunkSize - 1, totalSize - 1), + }); + } + return res; +}; diff --git a/tests/misc.test.ts b/tests/misc.test.ts index b71ac93..1379e0c 100644 --- a/tests/misc.test.ts +++ b/tests/misc.test.ts @@ -286,6 +286,83 @@ describe("Misc: special char for dir", () => { }); }); +describe("Misc: split chunk ranges", () => { + it("should fail on negative numner", () => { + assert.throws(() => misc.splitFileSizeToChunkRanges(-1, 2)); + assert.throws(() => misc.splitFileSizeToChunkRanges(1, -1)); + assert.throws(() => misc.splitFileSizeToChunkRanges(1, 0)); + }); + + it("should return nothing for 0 input", () => { + let input: [number, number] = [0, 1]; + let output: any = []; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + + input = [0, 100]; + output = []; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + }); + + it("should return single item for 1 input", () => { + let input: [number, number] = [1, 1]; + let output = [{ start: 0, end: 0 }]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + + input = [1, 100]; + output = [{ start: 0, end: 0 }]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + }); + + it("should return single item for larger or equal input", () => { + let input: [number, number] = [10, 10]; + let output = [{ start: 0, end: 9 }]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + + input = [10, 21]; + output = [{ start: 0, end: 9 }]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + }); + + it("should return correct items for normal input", () => { + let input: [number, number] = [10, 9]; + let output = [ + { start: 0, end: 8 }, + { start: 9, end: 9 }, + ]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + + input = [10, 5]; + output = [ + { start: 0, end: 4 }, + { start: 5, end: 9 }, + ]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + + input = [3, 1]; + output = [ + { start: 0, end: 0 }, + { start: 1, end: 1 }, + { start: 2, end: 2 }, + ]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + + input = [15, 5]; + output = [ + { start: 0, end: 4 }, + { start: 5, end: 9 }, + { start: 10, end: 14 }, + ]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + + input = [1024, 578]; + output = [ + { start: 0, end: 577 }, + { start: 578, end: 1023 }, + ]; + assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input)); + }); +}); + describe("Misc: Dropbox: should fix the folder name cases", () => { it("should do nothing on empty folders", () => { const input: any[] = [];