bruno/remotely-save
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2 from fyears/enc

Browse Source 
merge enc to master
This commit is contained in:
fyears 2021-10-28 01:54:03 +08:00 committed by GitHub
commit 9fc67e37f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 243 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
manifest.json
View File

@ -1,7 +1,7 @@
{
"id": "obsdian-save-remote",
"name": "Save remote",
"version": "0.0.2",
"version": "0.0.3",
"minAppVersion": "0.12.15",
"description": "This is yet another plugin allowing users to sync notes between local device and the cloud.",
"author": "fyears",

24
package.json
View File

@ -27,12 +27,36 @@
"@aws-sdk/client-s3": "^3.37.0",
"@aws-sdk/signature-v4-crt": "^3.37.0",
"@types/mime-types": "^2.1.1",
"acorn": "^8.5.0",
"assert": "^2.0.0",
"aws-crt": "^1.10.1",
"browserify-zlib": "^0.2.0",
"buffer": "^6.0.3",
"codemirror": "^5.63.1",
"console-browserify": "^1.2.0",
"constants-browserify": "^1.0.0",
"crypto-browserify": "^3.12.0",
"domain-browser": "^4.22.0",
"events": "^3.3.0",
"hi-base32": "^0.5.1",
"https-browserify": "^1.0.0",
"lovefield-ts": "^0.7.0",
"mime-types": "^2.1.33",
"obsidian": "^0.12.0",
"os-browserify": "^0.3.0",
"path-browserify": "^1.0.1",
"process": "^0.11.10",
"punycode": "^2.1.1",
"querystring-es3": "^0.2.1",
"rimraf": "^3.0.2",
"stream-browserify": "^3.0.0",
"stream-http": "^3.2.0",
"string_decoder": "^1.3.0",
"timers-browserify": "^2.0.12",
"tty-browserify": "0.0.1",
"url": "^0.11.0",
"util": "^0.12.4",
"vm-browserify": "^1.1.2",
"webdav": "^4.7.0",
"webdav-fs": "^4.0.0"
}

92
src/encrypt.ts Normal file
View File

@ -0,0 +1,92 @@
import * as crypto from "crypto";
import * as base32 from "hi-base32";
import { bufferToArrayBuffer, arrayBufferToBuffer } from "./misc";
const DEFAULT_ITER = 10000;
export const encryptBuffer = (
buf: Buffer,
password: string,
rounds: number = DEFAULT_ITER
) => {
const salt = crypto.randomBytes(8);
const derivedKey = crypto.pbkdf2Sync(
password,
salt,
rounds,
32 + 16,
"sha256"
);
const key = derivedKey.slice(0, 32);
const iv = derivedKey.slice(32, 32 + 16);
const cipher = crypto.createCipheriv("aes-256-cbc", key, iv);
cipher.write(buf);
cipher.end();
const encrypted = cipher.read();
const res = Buffer.concat([Buffer.from("Salted__"), salt, encrypted]);
return res;
};
export const decryptBuffer = (
buf: Buffer,
password: string,
rounds: number = DEFAULT_ITER
) => {
const prefix = buf.slice(0, 8);
const salt = buf.slice(8, 16);
const derivedKey = crypto.pbkdf2Sync(
password,
salt,
rounds,
32 + 16,
"sha256"
);
const key = derivedKey.slice(0, 32);
const iv = derivedKey.slice(32, 32 + 16);
const decipher = crypto.createDecipheriv("aes-256-cbc", key, iv);
decipher.write(buf.slice(16));
decipher.end();
const decrypted = decipher.read();
return decrypted as Buffer;
};
export const encryptArrayBuffer = (
arrBuf: ArrayBuffer,
password: string,
rounds: number = DEFAULT_ITER
) => {
return bufferToArrayBuffer(
encryptBuffer(arrayBufferToBuffer(arrBuf), password, rounds)
);
};
export const decryptArrayBuffer = (
arrBuf: ArrayBuffer,
password: string,
rounds: number = DEFAULT_ITER
) => {
return bufferToArrayBuffer(
decryptBuffer(arrayBufferToBuffer(arrBuf), password, rounds)
);
};
export const encryptStringToBase32 = (
text: string,
password: string,
rounds: number = DEFAULT_ITER
) => {
return base32.encode(encryptBuffer(Buffer.from(text), password, rounds));
};
export const decryptBase32ToString = (
text: string,
password: string,
rounds: number = DEFAULT_ITER
) => {
return decryptBuffer(
Buffer.from(base32.decode.asBytes(text)),
password,
rounds
).toString();
};

20
src/main.ts
View File

@ -27,10 +27,12 @@ import { DEFAULT_S3_CONFIG, getS3Client, listFromRemote, S3Config } from "./s3";
interface SaveRemotePluginSettings {
s3?: S3Config;
password?: string;
}
const DEFAULT_SETTINGS: SaveRemotePluginSettings = {
s3: DEFAULT_S3_CONFIG,
password: "",
};
export default class SaveRemotePlugin extends Plugin {
@ -87,7 +89,8 @@ export default class SaveRemotePlugin extends Plugin {
remoteRsp.Contents,
local,
localHistory,
this.db
this.db,
this.settings.password
);
for (const [key, val] of Object.entries(mixedStates)) {
@ -106,7 +109,8 @@ export default class SaveRemotePlugin extends Plugin {
this.settings.s3,
this.db,
this.app.vault,
mixedStates
mixedStates,
this.settings.password
);
new Notice("Save Remote finish!");
@ -217,6 +221,18 @@ class SaveRemoteSettingTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("password")
.setDesc("password")
.addText((text) =>
text
.setPlaceholder("")
.setValue(`${this.plugin.settings.password}`)
.onChange(async (value) => {
this.plugin.settings.password = value;
await this.plugin.saveSettings();
})
);
new Setting(containerEl)
.setName("s3BucketName")

9
src/misc.ts
View File

@ -49,3 +49,12 @@ export const mkdirpInVault = async (thePath: string, vault: Vault) => {
export const bufferToArrayBuffer = (b: Buffer) => {
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
};
/**
* Simple func.
* @param b
* @returns
*/
export const arrayBufferToBuffer = (b: ArrayBuffer) => {
return Buffer.from(b);
};

86
src/s3.ts
View File

@ -14,8 +14,13 @@ import {
import type { _Object } from "@aws-sdk/client-s3";
import { bufferToArrayBuffer, mkdirpInVault } from "./misc";
import {
arrayBufferToBuffer,
bufferToArrayBuffer,
mkdirpInVault,
} from "./misc";
import * as mime from "mime-types";
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
export interface S3Config {
s3Endpoint: string;
@ -65,8 +70,14 @@ export const uploadToRemote = async (
s3Config: S3Config,
fileOrFolderPath: string,
vault: Vault,
isRecursively: boolean = false
isRecursively: boolean = false,
password: string = "",
remoteEncryptedKey: string = ""
) => {
let uploadFile = fileOrFolderPath;
if (password !== "") {
uploadFile = remoteEncryptedKey;
}
const isFolder = fileOrFolderPath.endsWith("/");
const DEFAULT_CONTENT_TYPE = "application/octet-stream";
@ -79,7 +90,7 @@ export const uploadToRemote = async (
await s3Client.send(
new PutObjectCommand({
Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath,
Key: uploadFile,
Body: "",
ContentType: contentType,
})
@ -88,20 +99,28 @@ export const uploadToRemote = async (
} else {
// file
// we ignore isRecursively parameter here
const contentType =
mime.contentType(mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE) ||
DEFAULT_CONTENT_TYPE;
const content = await vault.adapter.readBinary(fileOrFolderPath);
const body = Buffer.from(content);
let contentType = DEFAULT_CONTENT_TYPE;
if (password === "") {
contentType =
mime.contentType(
mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE
) || DEFAULT_CONTENT_TYPE;
}
const localContent = await vault.adapter.readBinary(fileOrFolderPath);
let remoteContent = localContent;
if (password !== "") {
remoteContent = encryptArrayBuffer(localContent, password);
}
const body = arrayBufferToBuffer(remoteContent);
await s3Client.send(
new PutObjectCommand({
Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath,
Key: uploadFile,
Body: body,
ContentType: contentType,
})
);
return await getRemoteMeta(s3Client, s3Config, fileOrFolderPath);
return await getRemoteMeta(s3Client, s3Config, uploadFile);
}
};
@ -169,22 +188,35 @@ export const downloadFromRemote = async (
s3Config: S3Config,
fileOrFolderPath: string,
vault: Vault,
mtime: number
mtime: number,
password: string = "",
remoteEncryptedKey: string = ""
) => {
const isFolder = fileOrFolderPath.endsWith("/");
await mkdirpInVault(fileOrFolderPath, vault);
// the file is always local file
// we need to encrypt it
if (isFolder) {
// mkdirp locally is enough
// do nothing here
} else {
const content = await downloadFromRemoteRaw(
let downloadFile = fileOrFolderPath;
if (password !== "") {
downloadFile = remoteEncryptedKey;
}
const remoteContent = await downloadFromRemoteRaw(
s3Client,
s3Config,
fileOrFolderPath
downloadFile
);
await vault.adapter.writeBinary(fileOrFolderPath, content, {
let localContent = remoteContent;
if (password !== "") {
localContent = decryptArrayBuffer(remoteContent, password);
}
await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
mtime: mtime,
});
}
@ -200,12 +232,25 @@ export const downloadFromRemote = async (
export const deleteFromRemote = async (
s3Client: S3Client,
s3Config: S3Config,
fileOrFolderPath: string
fileOrFolderPath: string,
password: string = "",
remoteEncryptedKey: string = ""
) => {
if (fileOrFolderPath === "/") {
return;
}
if (fileOrFolderPath.endsWith("/")) {
let remoteFileName = fileOrFolderPath;
if (password !== "") {
remoteFileName = remoteEncryptedKey;
}
await s3Client.send(
new DeleteObjectCommand({
Bucket: s3Config.s3BucketName,
Key: remoteFileName,
})
);
if (fileOrFolderPath.endsWith("/") && password === "") {
const x = await listFromRemote(s3Client, s3Config, fileOrFolderPath);
x.Contents.forEach(async (element) => {
await s3Client.send(
@ -215,12 +260,9 @@ export const deleteFromRemote = async (
})
);
});
} else if (fileOrFolderPath.endsWith("/") && password !== "") {
// TODO
} else {
await s3Client.send(
new DeleteObjectCommand({
Bucket: s3Config.s3BucketName,
Key: fileOrFolderPath,
})
);
// pass
}
};

42
src/sync.ts
View File

@ -17,6 +17,7 @@ import {
downloadFromRemote,
} from "./s3";
import { mkdirpInVault } from "./misc";
import { decryptBase32ToString, encryptStringToBase32 } from "./encrypt";
type DecisionType =
| "undecided"
@ -44,26 +45,32 @@ interface FileOrFolderMixedState {
decision?: DecisionType;
syncDone?: "done";
decision_branch?: number;
remote_encrypted_key?: string;
}
export const ensembleMixedStates = async (
remote: S3ObjectType[],
local: TAbstractFile[],
deleteHistory: FileFolderHistoryRecord[],
db: lf.DatabaseConnection
db: lf.DatabaseConnection,
password: string = ""
) => {
const results = {} as Record<string, FileOrFolderMixedState>;
if (remote !== undefined) {
for (const entry of remote) {
const remoteEncryptedKey = entry.Key;
let key = remoteEncryptedKey;
if (password !== "") {
key = decryptBase32ToString(remoteEncryptedKey, password);
}
const backwardMapping = await getSyncMetaMappingByRemoteKeyS3(
db,
entry.Key,
key,
entry.LastModified.valueOf(),
entry.ETag
);
let key = entry.Key;
let r = {} as FileOrFolderMixedState;
if (backwardMapping !== undefined) {
key = backwardMapping.local_key;
@ -72,6 +79,7 @@ export const ensembleMixedStates = async (
exist_remote: true,
mtime_remote: backwardMapping.local_mtime,
size_remote: backwardMapping.local_size,
remote_encrypted_key: remoteEncryptedKey,
};
} else {
r = {
@ -79,6 +87,7 @@ export const ensembleMixedStates = async (
exist_remote: true,
mtime_remote: entry.LastModified.valueOf(),
size_remote: entry.Size,
remote_encrypted_key: remoteEncryptedKey,
};
}
if (results.hasOwnProperty(key)) {
@ -86,6 +95,7 @@ export const ensembleMixedStates = async (
results[key].exist_remote = r.exist_remote;
results[key].mtime_remote = r.mtime_remote;
results[key].size_remote = r.size_remote;
results[key].remote_encrypted_key = r.remote_encrypted_key;
} else {
results[key] = r;
}
@ -277,13 +287,21 @@ export const doActualSync = async (
s3Config: S3Config,
db: lf.DatabaseConnection,
vault: Vault,
keyStates: Record<string, FileOrFolderMixedState>
keyStates: Record<string, FileOrFolderMixedState>,
password: string = ""
) => {
Object.entries(keyStates)
.sort((k, v) => -(k as string).length)
.map(async ([k, v]) => {
const key = k as string;
const state = v as FileOrFolderMixedState;
let remoteEncryptedKey = key;
if (password !== "") {
remoteEncryptedKey = state.remote_encrypted_key;
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
remoteEncryptedKey = encryptStringToBase32(key, password);
}
}
if (
state.decision === undefined ||
@ -299,7 +317,9 @@ export const doActualSync = async (
s3Config,
state.key,
vault,
state.mtime_remote
state.mtime_remote,
password,
remoteEncryptedKey
);
await clearDeleteRenameHistoryOfKey(db, state.key);
} else if (state.decision === "upload_clearhist") {
@ -308,7 +328,9 @@ export const doActualSync = async (
s3Config,
state.key,
vault,
false
false,
password,
remoteEncryptedKey
);
await upsertSyncMetaMappingDataS3(
db,
@ -328,7 +350,9 @@ export const doActualSync = async (
s3Config,
state.key,
vault,
state.mtime_remote
state.mtime_remote,
password,
remoteEncryptedKey
);
} else if (state.decision === "delremote_clearhist") {
await deleteFromRemote(s3Client, s3Config, state.key);
@ -339,7 +363,9 @@ export const doActualSync = async (
s3Config,
state.key,
vault,
false
false,
password,
remoteEncryptedKey
);
await upsertSyncMetaMappingDataS3(
db,

2
versions.json
View File

@ -1,3 +1,3 @@
{
"0.0.2": "0.12.15"
"0.0.3": "0.12.15"
}