correctly set range update

src/fsWebdav.ts

@@ -17,7 +17,7 @@ import type {
 import type { Entity, WebdavConfig } from "./baseTypes";
 import { VALID_REQURL } from "./baseTypesObs";
 import { FakeFs } from "./fsAll";
-import { bufferToArrayBuffer, delay } from "./misc";
+import { bufferToArrayBuffer, delay, splitFileSizeToChunkRanges } from "./misc";
 
 /**
  * https://stackoverflow.com/questions/32850898/how-to-check-if-a-string-has-any-non-iso-8859-1-characters-with-javascript
@@ -550,7 +550,7 @@ export class FakeFsWebdav extends FakeFs {
     ctime: number,
     origKey: string
   ): Promise<Entity> {
-    console.debug(`start _writeFileFromRootFull`);
+    // console.debug(`start _writeFileFromRootFull`);
     await this.client.putFileContents(key, content, {
       overwrite: true,
       onUploadProgress: (progress: any) => {
@@ -558,7 +558,7 @@ export class FakeFsWebdav extends FakeFs {
       },
     });
     const k = await this._statFromRoot(key);
-    console.debug(`end _writeFileFromRootFull`);
+    // console.debug(`end _writeFileFromRootFull`);
     return k;
   }
 
@@ -623,19 +623,23 @@ export class FakeFsWebdav extends FakeFs {
     console.debug(`finish creating folder`);
 
     // upload by chunks
-    const size_5mb = 5 * 1024 * 1024;
+    const sizePerChunk = 5 * 1024 * 1024; // 5 mb
-    let tmpFileIdx = 1; // a number between 1 and 10000
+    const chunkRanges = splitFileSizeToChunkRanges(
-    let startInclusive = 0;
+      content.byteLength,
-    let endInclusive = Math.min(size_5mb, content.byteLength);
+      sizePerChunk
-    do {
+    );
-      const tmpFileName = `${tmpFileIdx}`.padStart(5, "0");
+    for (let i = 0; i < chunkRanges.length; ++i) {
+      const { start, end } = chunkRanges[i];
+      const tmpFileName = `${i + 1}`.padStart(5, "0");
       const tmpFileNameWithFolder = `${tmpFolder}/${tmpFileName}`;
       console.debug(
-        `start to upload chunk ${tmpFileIdx} to ${tmpFileNameWithFolder} with startInclusive=${startInclusive}, endInclusive=${endInclusive}`
+        `start to upload chunk ${
+          i + 1
+        } to ${tmpFileNameWithFolder} with startInclusive=${start}, endInclusive=${end}`
       );
       await this.client.putFileContents(
         tmpFileNameWithFolder,
-        content.slice(startInclusive, endInclusive - 1),
+        content.slice(start, end + 1),
         {
           headers: {
             Destination: destUrl,
@@ -643,10 +647,7 @@ export class FakeFsWebdav extends FakeFs {
           },
         }
       );
-      tmpFileIdx += 1;
+    }
-      startInclusive = Math.min(startInclusive + size_5mb, content.byteLength);
-      endInclusive = Math.min(endInclusive + size_5mb, content.byteLength);
-    } while (startInclusive < content.byteLength);
     console.debug(`finish upload all chunks`);
 
     // move to assemble
@@ -710,20 +711,20 @@ export class FakeFsWebdav extends FakeFs {
     );
 
     // then "update" by chunks
-    const size_5mb = 5 * 1024 * 1024;
+    const sizePerChunk = 5 * 1024 * 1024; // 5 mb
-    let startInclusive = 0;
+    const chunkRanges = splitFileSizeToChunkRanges(
-    let endInclusive = Math.min(size_5mb, content.byteLength);
+      content.byteLength,
-    do {
+      sizePerChunk
+    );
+    for (let i = 0; i < chunkRanges.length; ++i) {
+      const { start, end } = chunkRanges[i];
       await this.client.partialUpdateFileContents(
         key,
-        startInclusive,
+        start,
-        endInclusive,
+        end,
-        content.slice(startInclusive, endInclusive - 1)
+        content.slice(start, end + 1)
       );
-
+    }
-      startInclusive = Math.min(startInclusive + size_5mb, content.byteLength);
-      endInclusive = Math.min(endInclusive + size_5mb, content.byteLength);
-    } while (startInclusive < content.byteLength);
 
     // lastly return
     return await this.stat(origKey);

src/misc.ts

@@ -679,3 +679,34 @@ export const roughSizeOfObject = (object: any) => {
   }
   return bytes;
 };
+
+export const splitFileSizeToChunkRanges = (
+  totalSize: number,
+  chunkSize: number
+) => {
+  if (totalSize < 0) {
+    throw Error(`totalSize should not be negative`);
+  } 
+  if (chunkSize <= 0) {
+    throw Error(`chunkSize should not be negative or zero`);
+  }
+
+  if (totalSize === 0) {
+    return [];
+  }
+  if (totalSize <= chunkSize) {
+    return [{ start: 0, end: totalSize - 1 }];
+  }
+
+  const res: { start: number; end: number }[] = [];
+
+  const blocksCount = Math.ceil((totalSize * 1.0) / chunkSize);
+
+  for (let i = 0; i < blocksCount; ++i) {
+    res.push({
+      start: i * chunkSize,
+      end: Math.min((i + 1) * chunkSize - 1, totalSize - 1),
+    });
+  }
+  return res;
+};

tests/misc.test.ts

@@ -286,6 +286,83 @@ describe("Misc: special char for dir", () => {
   });
 });
 
+describe("Misc: split chunk ranges", () => {
+  it("should fail on negative numner", () => {
+    assert.throws(() => misc.splitFileSizeToChunkRanges(-1, 2));
+    assert.throws(() => misc.splitFileSizeToChunkRanges(1, -1));
+    assert.throws(() => misc.splitFileSizeToChunkRanges(1, 0));
+  });
+
+  it("should return nothing for 0 input", () => {
+    let input: [number, number] = [0, 1];
+    let output: any = [];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+
+    input = [0, 100];
+    output = [];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+  });
+
+  it("should return single item for 1 input", () => {
+    let input: [number, number] = [1, 1];
+    let output = [{ start: 0, end: 0 }];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+
+    input = [1, 100];
+    output = [{ start: 0, end: 0 }];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+  });
+
+  it("should return single item for larger or equal input", () => {
+    let input: [number, number] = [10, 10];
+    let output = [{ start: 0, end: 9 }];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+
+    input = [10, 21];
+    output = [{ start: 0, end: 9 }];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+  });
+
+  it("should return correct items for normal input", () => {
+    let input: [number, number] = [10, 9];
+    let output = [
+      { start: 0, end: 8 },
+      { start: 9, end: 9 },
+    ];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+
+    input = [10, 5];
+    output = [
+      { start: 0, end: 4 },
+      { start: 5, end: 9 },
+    ];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+
+    input = [3, 1];
+    output = [
+      { start: 0, end: 0 },
+      { start: 1, end: 1 },
+      { start: 2, end: 2 },
+    ];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+
+    input = [15, 5];
+    output = [
+      { start: 0, end: 4 },
+      { start: 5, end: 9 },
+      { start: 10, end: 14 },
+    ];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+
+    input = [1024, 578];
+    output = [
+      { start: 0, end: 577 },
+      { start: 578, end: 1023 },
+    ];
+    assert.deepStrictEqual(output, misc.splitFileSizeToChunkRanges(...input));
+  });
+});
+
 describe("Misc: Dropbox: should fix the folder name cases", () => {
   it("should do nothing on empty folders", () => {
     const input: any[] = [];