make it more robust

This commit is contained in:
fyears 2024-05-19 19:13:56 +08:00
parent 45578a01dd
commit c4c39f6b79

View File

@ -476,18 +476,31 @@ export class FakeFsWebdav extends FakeFs {
} }
await this._init(); await this._init();
const uploadFile = getWebdavPath(key, this.remoteBaseDir); const uploadFile = getWebdavPath(key, this.remoteBaseDir);
return await this._writeFileFromRoot(uploadFile, content, mtime, ctime); return await this._writeFileFromRoot(
uploadFile,
content,
mtime,
ctime,
key
);
} }
async _writeFileFromRoot( async _writeFileFromRoot(
key: string, key: string,
content: ArrayBuffer, content: ArrayBuffer,
mtime: number, mtime: number,
ctime: number ctime: number,
origKey: string
): Promise<Entity> { ): Promise<Entity> {
// less than 10 MB // less than 10 MB
if (content.byteLength <= 10 * 1024 * 1024) { if (content.byteLength <= 10 * 1024 * 1024) {
return await this._writeFileFromRootFull(key, content, mtime, ctime); return await this._writeFileFromRootFull(
key,
content,
mtime,
ctime,
origKey
);
} }
// larger than 10 MB, try to upload by chunks // larger than 10 MB, try to upload by chunks
@ -497,19 +510,28 @@ export class FakeFsWebdav extends FakeFs {
key, key,
content, content,
mtime, mtime,
ctime ctime,
origKey
); );
} else if (this.supportNativePartial) { } else if (this.supportNativePartial) {
return await this._writeFileFromRootNativePartial( return await this._writeFileFromRootNativePartial(
key, key,
content, content,
mtime, mtime,
ctime ctime,
origKey
); );
} }
throw Error(`no partial upload / update`); throw Error(`no partial upload / update`);
} catch (e) { } catch (e) {
return await this._writeFileFromRootFull(key, content, mtime, ctime); console.error(
`we fail to write file partially, so downgrade to full and ignore the error:`
);
console.error(e);
// throw e;
this.isNextcloud = false;
this.supportNativePartial = false;
return await this._writeFileFromRootFull(key, content, mtime, ctime, origKey);
} }
} }
@ -517,15 +539,19 @@ export class FakeFsWebdav extends FakeFs {
key: string, key: string,
content: ArrayBuffer, content: ArrayBuffer,
mtime: number, mtime: number,
ctime: number ctime: number,
origKey: string
): Promise<Entity> { ): Promise<Entity> {
console.debug(`start _writeFileFromRootFull`);
await this.client.putFileContents(key, content, { await this.client.putFileContents(key, content, {
overwrite: true, overwrite: true,
onUploadProgress: (progress: any) => { onUploadProgress: (progress: any) => {
console.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`); console.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
}, },
}); });
return await this._statFromRoot(key); const k = await this._statFromRoot(key);
console.debug(`end _writeFileFromRootFull`);
return k;
} }
/** /**
@ -540,7 +566,8 @@ export class FakeFsWebdav extends FakeFs {
key: string, key: string,
content: ArrayBuffer, content: ArrayBuffer,
mtime: number, mtime: number,
ctime: number ctime: number,
origKey: string
): Promise<Entity> { ): Promise<Entity> {
if (key.endsWith("/")) { if (key.endsWith("/")) {
throw Error( throw Error(
@ -549,7 +576,21 @@ export class FakeFsWebdav extends FakeFs {
} }
const destUrl = `${this.webdavConfig.address}/${encodeURI(key)}`; const destUrl = `${this.webdavConfig.address}/${encodeURI(key)}`;
console.debug(`destUrl=${destUrl}`); console.debug(`destUrl=${destUrl}`);
const tmpFolder = `${key}-${nanoid()}`;
const getTmpFolder = (x: string) => {
const y = x.split("/");
y[y.length - 1] = `${y[y.length - 1]}-${nanoid()}`;
const nodot = y.join("/");
y[y.length - 1] = `.${y[y.length - 1]}`;
const withdot = y.join("/");
return {
withdot: withdot,
nodot: nodot,
};
};
const tmpFolders = getTmpFolder(key);
const tmpFolder = tmpFolders.withdot;
const tmpFolderNoDot = tmpFolders.nodot;
console.debug(`tmpFolder=${tmpFolder}`); console.debug(`tmpFolder=${tmpFolder}`);
const tmpFolderUrl = `${this.webdavConfig.address}/${encodeURI(tmpFolder)}`; const tmpFolderUrl = `${this.webdavConfig.address}/${encodeURI(tmpFolder)}`;
console.debug(`tmpFolderUrl=${tmpFolderUrl}`); console.debug(`tmpFolderUrl=${tmpFolderUrl}`);
@ -560,6 +601,17 @@ export class FakeFsWebdav extends FakeFs {
Destination: destUrl, Destination: destUrl,
}, },
}); });
try {
const tmpFolderResult = await this.client.stat(tmpFolder);
} catch (e) {
// not exists??
try {
// try to clean no dot folder
await this.client.deleteFile(tmpFolderNoDot);
} catch (e2) {}
this.isNextcloud = false;
throw Error(`cannot create hidden file into nextcloud: ${tmpFolder}`);
}
console.debug(`finish creating folder`); console.debug(`finish creating folder`);
// upload by chunks // upload by chunks
@ -590,28 +642,16 @@ export class FakeFsWebdav extends FakeFs {
console.debug(`finish upload all chunks`); console.debug(`finish upload all chunks`);
// move to assemble // move to assemble
try { const fakeFileToMoveUrl = `${tmpFolderUrl}/.file`;
const fakeFileToMoveUrl = `${tmpFolderUrl}/.file`; console.debug(`fakeFileToMoveUrl=${fakeFileToMoveUrl}`);
console.debug(`fakeFileToMoveUrl=${fakeFileToMoveUrl}`); await this.client.customRequest(fakeFileToMoveUrl, {
await this.client.customRequest(fakeFileToMoveUrl, { method: "MOVE",
method: "MOVE", headers: {
headers: { Destination: destUrl,
Destination: destUrl, "OC-Total-Length": `${content.byteLength}`,
"OC-Total-Length": `${content.byteLength}`, },
}, });
}); console.debug(`finish moving file`);
console.debug(`finish moving file`);
} catch (e) {
// sometimes the server returns 404 but actually it works,
// we ignore the error.
console.error(
`while assembling chunks of nextcloud, some errors occur but we ignore them:`
);
console.error(e);
// wait for a few time!
await delay(1000);
}
// TODO: setting X-OC-Mtime // TODO: setting X-OC-Mtime
// wait for anything broken?? // wait for anything broken??
@ -620,14 +660,6 @@ export class FakeFsWebdav extends FakeFs {
// clean up! // clean up!
console.debug(`try to clean up`); console.debug(`try to clean up`);
try { try {
// tmpFileIdx -= 1;
// do {
// const tmpFileName = `${tmpFileIdx}`.padStart(5, "0");
// const tmpFileNameWithFolder = `${tmpFolder}/${tmpFileName}`;
// await this.client.deleteFile(tmpFileNameWithFolder);
// tmpFileIdx -= 1;
// } while (tmpFileIdx > 0);
await this.client.deleteFile(tmpFolder); await this.client.deleteFile(tmpFolder);
console.debug(`finish clean up`); console.debug(`finish clean up`);
} catch (e) { } catch (e) {
@ -638,8 +670,8 @@ export class FakeFsWebdav extends FakeFs {
} }
// stat // stat
console.debug(`before stat key=${key}`); console.debug(`before stat origKey=${origKey}`);
const k = await this._statFromRoot(key); const k = await this.stat(origKey);
console.debug(`after stat`); console.debug(`after stat`);
if (k.sizeRaw !== content.byteLength) { if (k.sizeRaw !== content.byteLength) {
// we failed! // we failed!
@ -657,10 +689,17 @@ export class FakeFsWebdav extends FakeFs {
key: string, key: string,
content: ArrayBuffer, content: ArrayBuffer,
mtime: number, mtime: number,
ctime: number ctime: number,
origKey: string
): Promise<Entity> { ): Promise<Entity> {
// firstly upload a 0-byte data // firstly upload a 0-byte data
await this._writeFileFromRootFull(key, new ArrayBuffer(0), mtime, ctime); await this._writeFileFromRootFull(
key,
new ArrayBuffer(0),
mtime,
ctime,
origKey
);
// then "update" by chunks // then "update" by chunks
const size_5mb = 5 * 1024 * 1024; const size_5mb = 5 * 1024 * 1024;
@ -679,7 +718,7 @@ export class FakeFsWebdav extends FakeFs {
} while (startInclusive < content.byteLength); } while (startInclusive < content.byteLength);
// lastly return // lastly return
return await this._statFromRoot(key); return await this.stat(origKey);
} }
async readFile(key: string): Promise<ArrayBuffer> { async readFile(key: string): Promise<ArrayBuffer> {