basic of gdrive

This commit is contained in:
fyears 2024-06-01 11:09:51 +08:00
parent 2ace90155c
commit f6ba4631e1
17 changed files with 1177 additions and 7 deletions

View File

@ -3,3 +3,5 @@ ONEDRIVE_CLIENT_ID=
ONEDRIVE_AUTHORITY=https://
REMOTELYSAVE_WEBSITE=http://127.0.0.1:46683
REMOTELYSAVE_CLIENT_ID=cli-xxx
GOOGLEDRIVE_CLIENT_ID=xxx.apps.googleusercontent.com
GOOGLEDRIVE_CLIENT_SECRET=GOCSPX-sss

View File

@ -21,6 +21,8 @@ jobs:
ONEDRIVE_AUTHORITY: ${{secrets.ONEDRIVE_AUTHORITY}}
REMOTELYSAVE_WEBSITE: ${{secrets.REMOTELYSAVE_WEBSITE}}
REMOTELYSAVE_CLIENT_ID: ${{secrets.REMOTELYSAVE_CLIENT_ID}}
GOOGLEDRIVE_CLIENT_ID: ${{secrets.GOOGLEDRIVE_CLIENT_ID}}
GOOGLEDRIVE_CLIENT_SECRET: ${{secrets.GOOGLEDRIVE_CLIENT_SECRET}}
strategy:
matrix:

View File

@ -25,6 +25,8 @@ jobs:
ONEDRIVE_AUTHORITY: ${{secrets.ONEDRIVE_AUTHORITY}}
REMOTELYSAVE_WEBSITE: ${{secrets.REMOTELYSAVE_WEBSITE}}
REMOTELYSAVE_CLIENT_ID: ${{secrets.REMOTELYSAVE_CLIENT_ID}}
GOOGLEDRIVE_CLIENT_ID: ${{secrets.GOOGLEDRIVE_CLIENT_ID}}
GOOGLEDRIVE_CLIENT_SECRET: ${{secrets.GOOGLEDRIVE_CLIENT_SECRET}}
strategy:
matrix:

View File

@ -19,6 +19,9 @@ const DEFAULT_ONEDRIVE_CLIENT_ID = process.env.ONEDRIVE_CLIENT_ID || "";
const DEFAULT_ONEDRIVE_AUTHORITY = process.env.ONEDRIVE_AUTHORITY || "";
const DEFAULT_REMOTELYSAVE_WEBSITE = process.env.REMOTELYSAVE_WEBSITE || "";
const DEFAULT_REMOTELYSAVE_CLIENT_ID = process.env.REMOTELYSAVE_CLIENT_ID || "";
const DEFAULT_GOOGLEDRIVE_CLIENT_ID = process.env.GOOGLEDRIVE_CLIENT_ID || "";
const DEFAULT_GOOGLEDRIVE_CLIENT_SECRET =
process.env.GOOGLEDRIVE_CLIENT_SECRET || "";
esbuild
.context({
@ -56,6 +59,8 @@ esbuild
"process.env.DEFAULT_ONEDRIVE_AUTHORITY": `"${DEFAULT_ONEDRIVE_AUTHORITY}"`,
"process.env.DEFAULT_REMOTELYSAVE_WEBSITE": `"${DEFAULT_REMOTELYSAVE_WEBSITE}"`,
"process.env.DEFAULT_REMOTELYSAVE_CLIENT_ID": `"${DEFAULT_REMOTELYSAVE_CLIENT_ID}"`,
"process.env.DEFAULT_GOOGLEDRIVE_CLIENT_ID": `"${DEFAULT_GOOGLEDRIVE_CLIENT_ID}"`,
"process.env.DEFAULT_GOOGLEDRIVE_CLIENT_SECRET": `"${DEFAULT_GOOGLEDRIVE_CLIENT_SECRET}"`,
global: "window",
"process.env.NODE_DEBUG": `undefined`, // ugly fix
"process.env.DEBUG": `undefined`, // ugly fix

View File

@ -23,3 +23,18 @@ export interface ProConfig {
enabledProFeatures: FeatureInfo[];
credentialsShouldBeDeletedAtTimeMs?: number;
}
export interface GoogleDriveConfig {
accessToken: string;
accessTokenExpiresInMs: number;
accessTokenExpiresAtTimeMs: number;
refreshToken: string;
remoteBaseDir?: string;
credentialsShouldBeDeletedAtTimeMs?: number;
scope: "https://www.googleapis.com/auth/drive.file";
}
export const DEFAULT_GOOGLEDRIVE_CLIENT_ID =
process.env.DEFAULT_GOOGLEDRIVE_CLIENT_ID;
export const DEFAULT_GOOGLEDRIVE_CLIENT_SECRET =
process.env.DEFAULT_GOOGLEDRIVE_CLIENT_SECRET;

765
pro/src/fsGoogleDrive.ts Normal file
View File

@ -0,0 +1,765 @@
// https://developers.google.com/identity/protocols/oauth2/native-app
// https://developers.google.com/identity/protocols/oauth2/javascript-implicit-flow
// https://developers.google.com/identity/protocols/oauth2/web-server
import { entries } from "lodash";
import * as mime from "mime-types";
import { requestUrl } from "obsidian";
import PQueue from "p-queue";
import { DEFAULT_CONTENT_TYPE, type Entity } from "../../src/baseTypes";
import { FakeFs } from "../../src/fsAll";
import {
getFolderLevels,
splitFileSizeToChunkRanges,
unixTimeToStr,
} from "../../src/misc";
import {
DEFAULT_GOOGLEDRIVE_CLIENT_ID,
DEFAULT_GOOGLEDRIVE_CLIENT_SECRET,
type GoogleDriveConfig,
} from "./baseTypesPro";
export const DEFAULT_GOOGLEDRIVE_CONFIG: GoogleDriveConfig = {
accessToken: "",
refreshToken: "",
accessTokenExpiresInMs: 0,
accessTokenExpiresAtTimeMs: 0,
credentialsShouldBeDeletedAtTimeMs: 0,
scope: "https://www.googleapis.com/auth/drive.file",
};
const FOLDER_MIME_TYPE = "application/vnd.google-apps.folder";
/**
* A simplified version of the type
*
*/
interface File {
kind?: string;
driveId?: string;
fileExtension?: string;
copyRequiresWriterPermission?: boolean;
md5Checksum?: string;
writersCanShare?: boolean;
viewedByMe?: boolean;
mimeType?: string;
parents?: string[];
thumbnailLink?: string;
iconLink?: string;
shared?: boolean;
headRevisionId?: string;
webViewLink?: string;
webContentLink?: string;
size?: string;
viewersCanCopyContent?: boolean;
hasThumbnail?: boolean;
spaces?: string[];
folderColorRgb?: string;
id?: string;
name?: string;
description?: string;
starred?: boolean;
trashed?: boolean;
explicitlyTrashed?: boolean;
createdTime?: string;
modifiedTime?: string;
modifiedByMeTime?: string;
viewedByMeTime?: string;
sharedWithMeTime?: string;
quotaBytesUsed?: string;
version?: string;
originalFilename?: string;
ownedByMe?: boolean;
fullFileExtension?: string;
isAppAuthorized?: boolean;
teamDriveId?: string;
hasAugmentedPermissions?: boolean;
thumbnailVersion?: string;
trashedTime?: string;
modifiedByMe?: boolean;
permissionIds?: string[];
resourceKey?: string;
sha1Checksum?: string;
sha256Checksum?: string;
}
interface GDEntity extends Entity {
id: string;
parentID: string | undefined;
parentIDPath: string | undefined;
isFolder: boolean;
}
/**
* https://developers.google.com/identity/protocols/oauth2/web-server#httprest_7
* @param refreshToken
*/
export const sendRefreshTokenReq = async (refreshToken: string) => {
console.debug(`refreshing token`);
const x = await fetch("https://oauth2.googleapis.com/token", {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
body: new URLSearchParams({
client_id: DEFAULT_GOOGLEDRIVE_CLIENT_ID ?? "",
client_secret: DEFAULT_GOOGLEDRIVE_CLIENT_SECRET ?? "",
grant_type: "refresh_token",
refresh_token: refreshToken,
}).toString(),
});
if (x.status === 200) {
const y = await x.json();
console.debug(`new token obtained`);
return y;
} else {
throw Error(`cannot refresh an access token`);
}
// {
// "access_token": "1/fFAGRNJru1FTz70BzhT3Zg",
// "expires_in": 3920,
// "scope": "https://www.googleapis.com/auth/drive.file",
// "token_type": "Bearer"
// }
};
const fromFileToGDEntity = (
file: File,
parentID: string,
parentFolderPath: string | undefined /* for bfs */
) => {
if (parentID === undefined || parentID === "" || parentID === "root") {
throw Error(`parentID=${parentID} should not be in fromFileToGDEntity`);
}
let keyRaw = file.name!;
if (
parentFolderPath !== undefined &&
parentFolderPath !== "" &&
parentFolderPath !== "/"
) {
if (!parentFolderPath.endsWith("/")) {
throw Error(
`parentFolderPath=${parentFolderPath} should not be in fromFileToGDEntity`
);
}
keyRaw = `${parentFolderPath}${file.name}`;
}
const isFolder = file.mimeType === FOLDER_MIME_TYPE;
if (isFolder) {
keyRaw = `${keyRaw}/`;
}
return {
key: keyRaw,
keyRaw: keyRaw,
mtimeCli: Date.parse(file.modifiedTime!),
mtimeSvr: Date.parse(file.modifiedTime!),
size: isFolder ? 0 : Number.parseInt(file.size!),
sizeRaw: isFolder ? 0 : Number.parseInt(file.size!),
hash: isFolder ? undefined : file.md5Checksum!,
id: file.id!,
parentID: parentID,
isFolder: isFolder,
} as GDEntity;
};
export class FakeFsGoogleDrive extends FakeFs {
kind: string;
googleDriveConfig: GoogleDriveConfig;
remoteBaseDir: string;
vaultFolderExists: boolean;
saveUpdatedConfigFunc: () => Promise<any>;
keyToGDEntity: Record<string, GDEntity>;
baseDirID: string;
constructor(
googleDriveConfig: GoogleDriveConfig,
vaultName: string,
saveUpdatedConfigFunc: () => Promise<any>
) {
super();
this.kind = "googledrive";
this.googleDriveConfig = googleDriveConfig;
this.remoteBaseDir =
this.googleDriveConfig.remoteBaseDir || vaultName || "";
this.vaultFolderExists = false;
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
this.keyToGDEntity = {};
this.baseDirID = "";
}
async _init() {
// get accessToken
await this._getAccessToken();
// check vault folder exists
if (this.vaultFolderExists) {
// pass
} else {
const q = encodeURIComponent(
`name='${this.remoteBaseDir}' and mimeType='application/vnd.google-apps.folder' and trashed=false`
);
const url: string = `https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=1000&fields=kind,nextPageToken,files(kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum)`;
const k = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
},
});
const k1: { files: File[] } = await k.json();
console.debug(k1);
if (k1.files.length > 0) {
// yeah we find it
this.baseDirID = k1.files[0].id!;
this.vaultFolderExists = true;
} else {
// wait, we need to create the folder!
console.debug(`we need to create the base dir ${this.remoteBaseDir}`);
const meta: any = {
mimeType: FOLDER_MIME_TYPE,
name: this.remoteBaseDir,
};
const res = await fetch("https://www.googleapis.com/drive/v3/files", {
method: "POST",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
"Content-Type": "application/json",
},
body: JSON.stringify(meta),
});
const res2: File = await res.json();
if (res.status === 200) {
console.debug(`succeed to create the base dir ${this.remoteBaseDir}`);
this.baseDirID = res2.id!;
this.vaultFolderExists = true;
} else {
throw Error(
`cannot create base dir ${this.remoteBaseDir} in init func.`
);
}
}
}
}
async _getAccessToken() {
if (
this.googleDriveConfig.accessToken === "" ||
this.googleDriveConfig.refreshToken === ""
) {
throw Error("The user has not manually auth yet.");
}
const ts = Date.now();
if (this.googleDriveConfig.accessTokenExpiresAtTimeMs > ts) {
return this.googleDriveConfig.accessToken;
}
// refresh
const k = await sendRefreshTokenReq(this.googleDriveConfig.refreshToken);
this.googleDriveConfig.accessToken = k.access_token;
this.googleDriveConfig.accessTokenExpiresInMs = k.expires_in * 1000;
this.googleDriveConfig.accessTokenExpiresAtTimeMs =
ts + k.expires_in * 1000 - 60 * 2 * 1000;
await this.saveUpdatedConfigFunc();
console.info("Google Drive accessToken updated");
return this.googleDriveConfig.accessToken;
}
/**
* https://developers.google.com/drive/api/reference/rest/v3/files/list
*/
async walk(): Promise<Entity[]> {
await this._init();
const allFiles: GDEntity[] = [];
// bfs
const queue = new PQueue({
concurrency: 5, // TODO: make it configurable?
autoStart: true,
});
queue.on("error", (error) => {
queue.pause();
queue.clear();
throw error;
});
let parents = [
{
id: this.baseDirID, // special init, from already created root folder ID
folderPath: "",
},
];
while (parents.length !== 0) {
const children: typeof parents = [];
for (const { id, folderPath } of parents) {
queue.add(async () => {
const filesUnderFolder = await this._walkFolder(id, folderPath);
for (const f of filesUnderFolder) {
allFiles.push(f);
if (f.isFolder) {
// keyRaw itself already has a tailing slash, no more slash here
// keyRaw itself also already has full path
const child = {
id: f.id,
folderPath: f.keyRaw,
};
console.debug(
`looping result of _walkFolder(${id},${folderPath}), adding child=${JSON.stringify(
child
)}`
);
children.push(child);
}
}
});
}
await queue.onIdle();
parents = children;
}
console.debug(`in the end of walk:`);
console.debug(allFiles);
console.debug(this.keyToGDEntity);
return allFiles;
}
async _walkFolder(parentID: string, parentFolderPath: string) {
console.debug(
`input of single level: parentID=${parentID}, parentFolderPath=${parentFolderPath}`
);
const filesOneLevel: GDEntity[] = [];
let nextPageToken: string | undefined = undefined;
if (parentID === undefined || parentID === "" || parentID === "root") {
// we should never start from root
// because we encapsulate the vault inside a folder
throw Error(`something goes wrong walking folder`);
}
do {
const q = encodeURIComponent(
`'${parentID}' in parents and trashed=false`
);
const pageToken =
nextPageToken !== undefined ? `&pageToken=${nextPageToken}` : "";
const url: string = `https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=1000&fields=kind,nextPageToken,files(kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum)${pageToken}`;
const k = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
},
});
if (k.status !== 200) {
throw Error(`cannot walk for parentID=${parentID}`);
}
const k1 = await k.json();
console.debug(k1);
for (const file of k1.files as File[]) {
const entity = fromFileToGDEntity(file, parentID, parentFolderPath);
this.keyToGDEntity[entity.keyRaw] = entity; // build cache
filesOneLevel.push(entity);
}
nextPageToken = k1.nextPageToken;
} while (nextPageToken !== undefined);
// console.debug(filesOneLevel);
return filesOneLevel;
}
async walkPartial(): Promise<Entity[]> {
await this._init();
const filesInLevel = await this._walkFolder(this.baseDirID, "");
return filesInLevel;
}
/**
* https://developers.google.com/drive/api/reference/rest/v3/files/get
* https://developers.google.com/drive/api/guides/fields-parameter
*/
async stat(key: string): Promise<Entity> {
await this._init();
// TODO: we already have a cache, should we call again?
const cachedEntity = this.keyToGDEntity[key];
const fileID = cachedEntity?.id;
if (cachedEntity === undefined || fileID === undefined) {
throw Error(`no fileID found for key=${key}`);
}
const url: string = `https://www.googleapis.com/drive/v3/files/${fileID}?fields=kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum`;
const k = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
},
});
if (k.status !== 200) {
throw Error(`cannot get file meta fileID=${fileID}, key=${key}`);
}
const k1: File = await k.json();
const entity = fromFileToGDEntity(
k1,
cachedEntity.parentID!,
cachedEntity.parentIDPath!
);
// insert back to cache?? to update it??
this.keyToGDEntity[key] = entity;
return entity;
}
/**
* https://developers.google.com/drive/api/guides/folder
*/
async mkdir(
key: string,
mtime: number | undefined,
ctime: number | undefined
): Promise<Entity> {
if (!key.endsWith("/")) {
throw Error(`you should not mkdir on key=${key}`);
}
await this._init();
// xxx/ => ["xxx"]
// xxx/yyy/zzz/ => ["xxx", "xxx/yyy", "xxx/yyy/zzz"]
const folderLevels = getFolderLevels(key);
let parentFolderPath: string | undefined = undefined;
let parentID: string | undefined = undefined;
if (folderLevels.length === 0) {
throw Error(`cannot getFolderLevels of ${key}`);
} else if (folderLevels.length === 1) {
parentID = this.baseDirID;
parentFolderPath = ""; // ignore base dir
} else {
// length > 1
parentFolderPath = `${folderLevels[folderLevels.length - 2]}/`;
if (!(parentFolderPath in this.keyToGDEntity)) {
throw Error(
`parent of ${key}: ${parentFolderPath} is not created before??`
);
}
parentID = this.keyToGDEntity[parentFolderPath].id;
}
// xxx/yyy/zzz/ => ["xxx", "xxx/yyy", "xxx/yyy/zzz"] => "xxx/yyy/zzz" => "zzz"
let folderItselfWithoutSlash = folderLevels[folderLevels.length - 1];
folderItselfWithoutSlash = folderItselfWithoutSlash.split("/").pop()!;
const meta: any = {
mimeType: FOLDER_MIME_TYPE,
modifiedTime: unixTimeToStr(mtime, true),
createdTime: unixTimeToStr(ctime, true),
name: folderItselfWithoutSlash,
parents: [parentID],
};
const res = await fetch("https://www.googleapis.com/drive/v3/files", {
method: "POST",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
"Content-Type": "application/json",
},
body: JSON.stringify(meta),
});
if (res.status !== 200 && res.status !== 201) {
throw Error(`create folder ${key} failed! meta=${JSON.stringify(meta)}`);
}
const res2: File = await res.json();
// console.debug(res2);
const entity = fromFileToGDEntity(res2, parentID, parentFolderPath);
// insert into cache
this.keyToGDEntity[key] = entity;
return entity;
}
/**
* https://developers.google.com/drive/api/guides/manage-uploads
* https://stackoverflow.com/questions/65181932/how-i-can-upload-file-to-google-drive-with-google-drive-api
*/
async writeFile(
key: string,
content: ArrayBuffer,
mtime: number,
ctime: number
): Promise<Entity> {
if (key.endsWith("/")) {
throw Error(`should not call writeFile on ${key}`);
}
await this._init();
const contentType =
mime.contentType(mime.lookup(key) || DEFAULT_CONTENT_TYPE) ||
DEFAULT_CONTENT_TYPE;
let parentID: string | undefined = undefined;
let parentFolderPath: string | undefined = undefined;
// "xxx" => []
// "xxx/yyy/zzz.md" => ["xxx", "xxx/yyy"]
const folderLevels = getFolderLevels(key);
if (folderLevels.length === 0) {
// root
parentID = this.baseDirID;
parentFolderPath = "";
} else {
parentFolderPath = `${folderLevels[folderLevels.length - 1]}/`;
if (!(parentFolderPath in this.keyToGDEntity)) {
throw Error(
`parent of ${key}: ${parentFolderPath} is not created before??`
);
}
parentID = this.keyToGDEntity[parentFolderPath].id;
}
const fileItself = key.split("/").pop()!;
if (content.byteLength <= 5 * 1024 * 1024) {
const formData = new FormData();
const meta: any = {
name: fileItself,
modifiedTime: unixTimeToStr(mtime, true),
createdTime: unixTimeToStr(ctime, true),
parents: [parentID],
};
formData.append(
"metadata",
new Blob([JSON.stringify(meta)], {
type: "application/json; charset=UTF-8",
})
);
formData.append("media", new Blob([content], { type: contentType }));
const res = await fetch(
"https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&fields=kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum",
{
method: "POST",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
},
body: formData,
}
);
if (res.status !== 200 && res.status !== 201) {
throw Error(`create file ${key} failed! meta=${JSON.stringify(meta)}`);
}
const res2: File = await res.json();
console.debug(
`upload ${key} with ${JSON.stringify(meta)}, res2=${JSON.stringify(
res2
)}`
);
const entity = fromFileToGDEntity(res2, parentID, parentFolderPath);
// insert into cache
this.keyToGDEntity[key] = entity;
return entity;
} else {
const meta: any = {
name: fileItself,
modifiedTime: unixTimeToStr(mtime, true),
createdTime: unixTimeToStr(ctime, true),
parents: [parentID],
};
const bodyStr = JSON.stringify(meta);
const headers: HeadersInit = {
Authorization: `Bearer ${await this._getAccessToken()}`,
"Content-Type": "application/json",
"Content-Length": `${bodyStr.length}`,
"X-Upload-Content-Type": contentType,
"X-Upload-Content-Length": `${content.byteLength}`,
};
const res = await fetch(
"https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&fields=kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum",
{
method: "POST",
headers: headers,
body: bodyStr,
}
);
if (res.status !== 200) {
throw Error(
`create resumable file ${key} failed! meta=${JSON.stringify(
meta
)}, header=${JSON.stringify(headers)}`
);
}
const uploadLocation = res.headers.get("Location");
if (uploadLocation === null || !uploadLocation.startsWith("http")) {
throw Error(
`create resumable file ${key} failed! meta=${JSON.stringify(
meta
)}, header=${JSON.stringify(headers)}`
);
}
console.debug(`key=${key}, uploadLocaltion=${uploadLocation}`);
// multiples of 256 KB (256 x 1024 bytes) in size
const sizePerChunk = 5 * 4 * 256 * 1024; // 5.24 mb
const chunkRanges = splitFileSizeToChunkRanges(
content.byteLength,
sizePerChunk
);
let entity: GDEntity | undefined = undefined;
// TODO: deal with "Resume an interrupted upload"
// currently (202405) only assume everything goes well...
// TODO: parallel
for (const { start, end } of chunkRanges) {
console.debug(
`key=${key}, start upload chunk ${start}-${end}/${content.byteLength}`
);
const res = await fetch(uploadLocation, {
method: "PUT",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
"Content-Length": `${end - start + 1}`, // the number of bytes in the current chunk
"Content-Range": `bytes ${start}-${end}/${content.byteLength}`,
},
body: content.slice(start, end + 1), // TODO: slice() is a copy, may be we can optimize it
});
if (res.status >= 400 && res.status <= 599) {
throw Error(
`create resumable file ${key} failed! meta=${JSON.stringify(
meta
)}, header=${JSON.stringify(headers)}`
);
}
if (res.status === 200 || res.status === 201) {
const res2: File = await res.json();
console.debug(
`upload ${key} with ${JSON.stringify(meta)}, res2=${JSON.stringify(
res2
)}`
);
if (res2.id === undefined || res2.id === null || res2.id === "") {
// TODO: what's this??
} else {
entity = fromFileToGDEntity(res2, parentID, parentFolderPath);
// insert into cache
this.keyToGDEntity[key] = entity;
}
}
}
if (entity === undefined) {
throw Error(`something goes wrong while uploading large file ${key}`);
}
return entity;
}
}
/**
* https://developers.google.com/drive/api/reference/rest/v3/files/get
*/
async readFile(key: string): Promise<ArrayBuffer> {
if (key.endsWith("/")) {
throw Error(`you should not call readFile on ${key}`);
}
await this._init();
const fileID = this.keyToGDEntity[key]?.id;
if (fileID === undefined) {
throw Error(`no fileID found for key=${key}`);
}
const res1 = await fetch(
`https://www.googleapis.com/drive/v3/files/${fileID}?alt=media`,
{
method: "GET",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
},
}
);
if (res1.status !== 200) {
throw Error(`cannot download ${key} using fileID=${fileID}`);
}
const res2 = await res1.arrayBuffer();
return res2;
}
async rename(key1: string, key2: string): Promise<void> {
throw new Error("Method not implemented.");
}
/**
* https://developers.google.com/drive/api/guides/delete
* https://developers.google.com/drive/api/reference/rest/v3/files/update
*/
async rm(key: string): Promise<void> {
await this._init();
const fileID = this.keyToGDEntity[key]?.id;
if (fileID === undefined) {
throw Error(`no fileID found for key=${key}`);
}
const res1 = await fetch(
`https://www.googleapis.com/drive/v3/files/${fileID}`,
{
method: "PATCH",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
},
body: JSON.stringify({
trashed: true,
}),
}
);
if (res1.status !== 200) {
throw Error(`cannot rm ${key} using fileID=${fileID}`);
}
}
async checkConnect(callbackFunc?: any): Promise<boolean> {
// if we can init, we can connect
try {
await this._init();
return true;
} catch (err) {
console.debug(err);
callbackFunc?.(err);
return false;
}
}
async getUserDisplayName(): Promise<string> {
throw new Error("Method not implemented.");
}
/**
* https://developers.google.com/identity/protocols/oauth2/web-server#tokenrevoke
*/
async revokeAuth(): Promise<any> {
const x = await fetch(
`https://oauth2.googleapis.com/revoke?token=${this._getAccessToken()}`,
{
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
}
);
if (x.status === 200) {
return true;
} else {
throw Error(`cannot revoke`);
}
}
allowEmptyFile(): boolean {
return true;
}
}

View File

@ -7,6 +7,16 @@
"protocol_pro_connect_fail": "Something went wrong from response from Remotely Save official website. Maybe the network connection is not good. Maybe you rejected the auth?",
"protocol_pro_connect_succ_revoke": "You've connected as user {{email}}. If you want to disconnect, click this button.",
"modal_googledriveauth_copybutton": "Click to copy the auth url",
"modal_googledriveauth_copynotice": "The auth url is copied to the clipboard!",
"modal_googledriverevokeauth_step1": "Step 1: Go to the following address, you can remove the connection there.",
"modal_googledriverevokeauth_step2": "Step 2: Click the button below, to clean the locally-saved login credentials.",
"modal_googledriverevokeauth_clean": "Clean Locally-Saved Login Credentials",
"modal_googledriverevokeauth_clean_desc": "You need to click the button.",
"modal_googledriverevokeauth_clean_button": "Clean",
"modal_googledriverevokeauth_clean_notice": "Cleaned!",
"modal_googledriverevokeauth_clean_fail": "Something goes wrong while revoking.",
"modal_prorevokeauth": "Revoke auth by clicking here and follow the steps.",
"modal_prorevokeauth_clean": "Clean",
"modal_prorevokeauth_clean_desc": "Clean local auth record",
@ -20,6 +30,22 @@
"modal_proauth_maualinput_notice": "Trying to connect, wait...",
"modal_proauth_maualinput_conn_fail": "Failed to connect",
"settings_googledrive": "Google Drive (PRO)",
"settings_chooseservice_googledrive": "Google Drive (PRO)",
"settings_googledrive_disclaimer1": "Disclaimer: This app is NOT an official Google product.",
"settings_googledrive_disclaimer2": "Disclaimer: The information is stored locally. Other malicious/harmful/faulty plugins could read the info. If you see any unintentional access to your Google Drive, please immediately disconnect this app on https://myaccount.google.com/permissions .",
"settings_googledrive_folder": "We will create and sync inside the folder {{remoteBaseDir}} on your Google Drive. DO NOT create this folder by yourself manually.",
"settings_googledrive_revoke": "Revoke Auth",
"settings_googledrive_revoke_desc": "You've connected. If you want to disconnect, click this button.",
"settings_googledrive_revoke_button": "Revoke Auth",
"settings_googledrive_auth": "Auth",
"settings_googledrive_auth_desc": "Auth.",
"settings_googledrive_auth_button": "Auth",
"settings_googledrive_connect_succ": "Great! We can connect to Google Drive!",
"settings_googledrive_connect_fail": "We cannot connect to Google Drive.",
"settings_export_googledrive_button": "Export Google Drive Part",
"settings_pro": "Account (for PRO features)",
"settings_pro_tutorial": "<p>Using <stong>basic</strong> features of Remotely Save is <strong>FREE</strong> and do <strong>NOT</strong> need an account.</p><p>However, you will <strong>need</strong> an online account and <strong>PAY</strong> for the <strong>PRO</strong> features such as smart conflict.</p><p>Firstly please click the button to sign up and sign in to the website: <a href=\"https://remotelysave.com\">https://remotelysave.com</a>. Notice: It's different from, and NOT affiliated with Obsidian account.</p><p>Secondly please \"connect\" your local device to your online account.",
"settings_pro_features": "Features",

View File

@ -0,0 +1,273 @@
import cloneDeep from "lodash/cloneDeep";
import { type App, Modal, Notice, Setting } from "obsidian";
import { getClient } from "../../src/fsGetter";
import type { TransItemType } from "../../src/i18n";
import type RemotelySavePlugin from "../../src/main";
import { ChangeRemoteBaseDirModal } from "../../src/settings";
import { DEFAULT_GOOGLEDRIVE_CONFIG } from "./fsGoogleDrive";
class GoogleDriveAuthModal extends Modal {
readonly plugin: RemotelySavePlugin;
readonly authDiv: HTMLDivElement;
readonly revokeAuthDiv: HTMLDivElement;
readonly revokeAuthSetting: Setting;
readonly t: (x: TransItemType, vars?: any) => string;
constructor(
app: App,
plugin: RemotelySavePlugin,
authDiv: HTMLDivElement,
revokeAuthDiv: HTMLDivElement,
revokeAuthSetting: Setting,
t: (x: TransItemType, vars?: any) => string
) {
super(app);
this.plugin = plugin;
this.authDiv = authDiv;
this.revokeAuthDiv = revokeAuthDiv;
this.revokeAuthSetting = revokeAuthSetting;
this.t = t;
}
async onOpen() {
const { contentEl } = this;
const t = this.t;
const authUrl = "https://remotelysave.com/auth/googledrive/start";
const div2 = contentEl.createDiv();
div2.createEl(
"button",
{
text: t("modal_googledriveauth_copybutton"),
},
(el) => {
el.onclick = async () => {
await navigator.clipboard.writeText(authUrl);
new Notice(t("modal_googledriveauth_copynotice"));
};
}
);
contentEl.createEl("p").createEl("a", {
href: authUrl,
text: authUrl,
});
}
onClose() {
const { contentEl } = this;
contentEl.empty();
}
}
class GoogleDriveRevokeAuthModal extends Modal {
readonly plugin: RemotelySavePlugin;
readonly authDiv: HTMLDivElement;
readonly revokeAuthDiv: HTMLDivElement;
readonly t: (x: TransItemType, vars?: any) => string;
constructor(
app: App,
plugin: RemotelySavePlugin,
authDiv: HTMLDivElement,
revokeAuthDiv: HTMLDivElement,
t: (x: TransItemType, vars?: any) => string
) {
super(app);
this.plugin = plugin;
this.authDiv = authDiv;
this.revokeAuthDiv = revokeAuthDiv;
this.t = t;
}
async onOpen() {
const t = this.t;
const { contentEl } = this;
contentEl.createEl("p", {
text: t("modal_googledriverevokeauth_step1"),
});
const consentUrl = "https://myaccount.google.com/permissions";
contentEl.createEl("p").createEl("a", {
href: consentUrl,
text: consentUrl,
});
contentEl.createEl("p", {
text: t("modal_googledriverevokeauth_step2"),
});
new Setting(contentEl)
.setName(t("modal_googledriverevokeauth_clean"))
.setDesc(t("modal_googledriverevokeauth_clean_desc"))
.addButton(async (button) => {
button.setButtonText(t("modal_googledriverevokeauth_clean_button"));
button.onClick(async () => {
try {
this.plugin.settings.googledrive = cloneDeep(
DEFAULT_GOOGLEDRIVE_CONFIG
);
await this.plugin.saveSettings();
this.authDiv.toggleClass(
"onedrive-auth-button-hide",
this.plugin.settings.onedrive.username !== ""
);
this.revokeAuthDiv.toggleClass(
"onedrive-revoke-auth-button-hide",
this.plugin.settings.onedrive.username === ""
);
new Notice(t("modal_googledriverevokeauth_clean_notice"));
this.close();
} catch (err) {
console.error(err);
new Notice(t("modal_googledriverevokeauth_clean_fail"));
}
});
});
}
onClose() {
const { contentEl } = this;
contentEl.empty();
}
}
export const generateGoogleDriveSettingsPart = (
containerEl: HTMLElement,
t: (x: TransItemType, vars?: any) => string,
app: App,
plugin: RemotelySavePlugin,
saveUpdatedConfigFunc: () => Promise<any> | undefined
) => {
const googleDriveDiv = containerEl.createEl("div", {
cls: "googledrive-hide",
});
googleDriveDiv.toggleClass(
"googledrive-hide",
plugin.settings.serviceType !== "googledrive"
);
googleDriveDiv.createEl("h2", { text: t("settings_googledrive") });
const googleDriveLongDescDiv = googleDriveDiv.createEl("div", {
cls: "settings-long-desc",
});
for (const c of [
t("settings_googledrive_disclaimer1"),
t("settings_googledrive_disclaimer2"),
]) {
googleDriveLongDescDiv.createEl("p", {
text: c,
cls: "googledrive-disclaimer",
});
}
googleDriveLongDescDiv.createEl("p", {
text: t("settings_googledrive_folder", {
remoteBaseDir:
plugin.settings.googledrive.remoteBaseDir || app.vault.getName(),
}),
});
const googleDriveSelectAuthDiv = googleDriveDiv.createDiv();
const googleDriveAuthDiv = googleDriveSelectAuthDiv.createDiv({
cls: "googledrive-auth-button-hide settings-auth-related",
});
const googleDriveRevokeAuthDiv = googleDriveSelectAuthDiv.createDiv({
cls: "googledrive-revoke-auth-button-hide settings-auth-related",
});
const googleDriveRevokeAuthSetting = new Setting(googleDriveRevokeAuthDiv)
.setName(t("settings_googledrive_revoke"))
.setDesc(t("settings_googledrive_revoke_desc"))
.addButton(async (button) => {
button.setButtonText(t("settings_googledrive_revoke_button"));
button.onClick(async () => {
new GoogleDriveRevokeAuthModal(
app,
plugin,
googleDriveAuthDiv,
googleDriveRevokeAuthDiv,
t
).open();
});
});
new Setting(googleDriveAuthDiv)
.setName(t("settings_googledrive_auth"))
.setDesc(t("settings_googledrive_auth_desc"))
.addButton(async (button) => {
button.setButtonText(t("settings_googledrive_auth_button"));
button.onClick(async () => {
const modal = new GoogleDriveAuthModal(
app,
plugin,
googleDriveAuthDiv,
googleDriveRevokeAuthDiv,
googleDriveRevokeAuthSetting,
t
);
plugin.oauth2Info.helperModal = modal;
plugin.oauth2Info.authDiv = googleDriveAuthDiv;
plugin.oauth2Info.revokeDiv = googleDriveRevokeAuthDiv;
plugin.oauth2Info.revokeAuthSetting = googleDriveRevokeAuthSetting;
modal.open();
});
});
googleDriveAuthDiv.toggleClass(
"googledrive-auth-button-hide",
plugin.settings.googledrive.refreshToken !== ""
);
googleDriveRevokeAuthDiv.toggleClass(
"googledrive-revoke-auth-button-hide",
plugin.settings.googledrive.refreshToken === ""
);
let newgoogleDriveRemoteBaseDir =
plugin.settings.googledrive.remoteBaseDir || "";
new Setting(googleDriveDiv)
.setName(t("settings_remotebasedir"))
.setDesc(t("settings_remotebasedir_desc"))
.addText((text) =>
text
.setPlaceholder(app.vault.getName())
.setValue(newgoogleDriveRemoteBaseDir)
.onChange((value) => {
newgoogleDriveRemoteBaseDir = value.trim();
})
)
.addButton((button) => {
button.setButtonText(t("confirm"));
button.onClick(() => {
new ChangeRemoteBaseDirModal(
app,
plugin,
newgoogleDriveRemoteBaseDir,
"googledrive"
).open();
});
});
new Setting(googleDriveDiv)
.setName(t("settings_checkonnectivity"))
.setDesc(t("settings_checkonnectivity_desc"))
.addButton(async (button) => {
button.setButtonText(t("settings_checkonnectivity_button"));
button.onClick(async () => {
new Notice(t("settings_checkonnectivity_checking"));
const client = getClient(plugin.settings, app.vault.getName(), () =>
plugin.saveSettings()
);
const errors = { msg: "" };
const res = await client.checkConnect((err: any) => {
errors.msg = `${err}`;
});
if (res) {
new Notice(t("settings_googledrive_connect_succ"));
} else {
new Notice(t("settings_googledrive_connect_fail"));
new Notice(errors.msg);
}
});
});
return googleDriveDiv;
};

View File

@ -3,7 +3,7 @@
* To avoid circular dependency.
*/
import type { ProConfig } from "../pro/src/baseTypesPro";
import type { GoogleDriveConfig, ProConfig } from "../pro/src/baseTypesPro";
import type { LangTypeAndAuto } from "./i18n";
export const DEFAULT_CONTENT_TYPE = "application/octet-stream";
@ -13,13 +13,15 @@ export type SUPPORTED_SERVICES_TYPE =
| "webdav"
| "dropbox"
| "onedrive"
| "webdis";
| "webdis"
| "googledrive";
export type SUPPORTED_SERVICES_TYPE_WITH_REMOTE_BASE_DIR =
| "webdav"
| "dropbox"
| "onedrive"
| "webdis";
| "webdis"
| "googledrive";
export interface S3Config {
s3Endpoint: string;
@ -113,7 +115,8 @@ export type QRExportType =
| "dropbox"
| "onedrive"
| "webdav"
| "webdis";
| "webdis"
| "googledrive";
export interface ProfilerConfig {
enablePrinting?: boolean;
@ -126,6 +129,7 @@ export interface RemotelySavePluginSettings {
dropbox: DropboxConfig;
onedrive: OnedriveConfig;
webdis: WebdisConfig;
googledrive: GoogleDriveConfig;
password: string;
serviceType: SUPPORTED_SERVICES_TYPE;
currLogLevel?: string;

View File

@ -1,3 +1,4 @@
import { FakeFsGoogleDrive } from "../pro/src/fsGoogleDrive";
import type { RemotelySavePluginSettings } from "./baseTypes";
import type { FakeFs } from "./fsAll";
import { FakeFsDropbox } from "./fsDropbox";
@ -41,6 +42,12 @@ export function getClient(
vaultName,
saveUpdatedConfigFunc
);
case "googledrive":
return new FakeFsGoogleDrive(
settings.googledrive,
vaultName,
saveUpdatedConfigFunc
);
default:
throw Error(`cannot init client for serviceType=${settings.serviceType}`);
}

View File

@ -24,6 +24,7 @@ export const exportQrCodeUri = async (
delete settings2.onedrive;
delete settings2.webdav;
delete settings2.webdis;
delete settings2.googledrive;
delete settings2.pro;
} else if (exportFields === "s3") {
settings2 = { s3: cloneDeep(settings.s3) };
@ -35,6 +36,8 @@ export const exportQrCodeUri = async (
settings2 = { webdav: cloneDeep(settings.webdav) };
} else if (exportFields === "webdis") {
settings2 = { webdis: cloneDeep(settings.webdis) };
} else if (exportFields === "googledrive") {
settings2 = { googledrive: cloneDeep(settings.googledrive) };
}
delete settings2.vaultRandomID;

View File

@ -64,6 +64,7 @@ import { SyncAlgoV3Modal } from "./syncAlgoV3Notice";
import AggregateError from "aggregate-error";
import throttle from "lodash/throttle";
import { COMMAND_CALLBACK_PRO } from "../pro/src/baseTypesPro";
import { DEFAULT_GOOGLEDRIVE_CONFIG } from "../pro/src/fsGoogleDrive";
import { exportVaultSyncPlansToFiles } from "./debugMode";
import { FakeFsEncrypt } from "./fsEncrypt";
import { getClient } from "./fsGetter";
@ -79,6 +80,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
dropbox: DEFAULT_DROPBOX_CONFIG,
onedrive: DEFAULT_ONEDRIVE_CONFIG,
webdis: DEFAULT_WEBDIS_CONFIG,
googledrive: DEFAULT_GOOGLEDRIVE_CONFIG,
password: "",
serviceType: "s3",
currLogLevel: "info",
@ -1062,6 +1064,10 @@ export default class RemotelySavePlugin extends Plugin {
this.settings.profiler.recordSize = false;
}
if (this.settings.googledrive === undefined) {
this.settings.googledrive = DEFAULT_GOOGLEDRIVE_CONFIG;
}
await this.saveSettings();
}

View File

@ -341,11 +341,17 @@ export const checkHasSpecialCharForDir = (x: string) => {
return /[?/\\]/.test(x);
};
export const unixTimeToStr = (x: number | undefined | null) => {
export const unixTimeToStr = (x: number | undefined | null, hasMs = false) => {
if (x === undefined || x === null || Number.isNaN(x)) {
return undefined;
}
return window.moment(x).format() as string;
if (hasMs) {
// 1716712162574 => '2024-05-26T16:29:22.574+08:00'
return window.moment(x).toISOString(true);
} else {
// 1716712162574 => '2024-05-26T16:29:22+08:00'
return window.moment(x).format() as string;
}
};
/**

View File

@ -21,6 +21,7 @@ import type {
} from "./baseTypes";
import cloneDeep from "lodash/cloneDeep";
import { generateGoogleDriveSettingsPart } from "../pro/src/settingsGoogleDrive";
import { generateProSettingsPart } from "../pro/src/settingsPro";
import { API_VER_ENSURE_REQURL_OK, VALID_REQURL } from "./baseTypesObs";
import { messyConfigToNormal } from "./configPersist";
@ -169,7 +170,7 @@ class EncryptionMethodModal extends Modal {
}
}
class ChangeRemoteBaseDirModal extends Modal {
export class ChangeRemoteBaseDirModal extends Modal {
readonly plugin: RemotelySavePlugin;
readonly newRemoteBaseDir: string;
readonly service: SUPPORTED_SERVICES_TYPE_WITH_REMOTE_BASE_DIR;
@ -1791,6 +1792,18 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
});
});
//////////////////////////////////////////////////
// below for googledrive
//////////////////////////////////////////////////
const googleDriveDiv = generateGoogleDriveSettingsPart(
containerEl,
t,
this.app,
this.plugin,
() => this.plugin.saveSettings()
);
//////////////////////////////////////////////////
// below for general chooser (part 2/2)
//////////////////////////////////////////////////
@ -1806,6 +1819,10 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
dropdown.addOption("webdav", t("settings_chooseservice_webdav"));
dropdown.addOption("onedrive", t("settings_chooseservice_onedrive"));
dropdown.addOption("webdis", t("settings_chooseservice_webdis"));
dropdown.addOption(
"googledrive",
t("settings_chooseservice_googledrive")
);
dropdown
.setValue(this.plugin.settings.serviceType)
@ -1831,6 +1848,10 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
"webdis-hide",
this.plugin.settings.serviceType !== "webdis"
);
googleDriveDiv.toggleClass(
"googledrive-hide",
this.plugin.settings.serviceType !== "googledrive"
);
await this.plugin.saveSettings();
});
});
@ -2383,6 +2404,16 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
button.onClick(async () => {
new ExportSettingsQrCodeModal(this.app, this.plugin, "webdis").open();
});
})
.addButton(async (button) => {
button.setButtonText(t("settings_export_googledrive_button"));
button.onClick(async () => {
new ExportSettingsQrCodeModal(
this.app,
this.plugin,
"googledrive"
).open();
});
});
let importSettingVal = "";

View File

@ -72,6 +72,21 @@
display: none;
}
.googledrive-disclaimer {
font-weight: bold;
}
.googledrive-hide {
display: none;
}
.googledrive-auth-button-hide {
display: none;
}
.googledrive-revoke-auth-button-hide {
display: none;
}
.qrcode-img {
width: 350px;
height: 350px;

View File

@ -19,6 +19,9 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
webdis: {
address: "addr",
} as any,
googledrive: {
refreshToken: "xxx",
} as any,
password: "password",
serviceType: "s3",
currLogLevel: "info",

View File

@ -8,6 +8,9 @@ const DEFAULT_ONEDRIVE_CLIENT_ID = process.env.ONEDRIVE_CLIENT_ID || "";
const DEFAULT_ONEDRIVE_AUTHORITY = process.env.ONEDRIVE_AUTHORITY || "";
const DEFAULT_REMOTELYSAVE_WEBSITE = process.env.REMOTELYSAVE_WEBSITE || "";
const DEFAULT_REMOTELYSAVE_CLIENT_ID = process.env.REMOTELYSAVE_CLIENT_ID || "";
const DEFAULT_GOOGLEDRIVE_CLIENT_ID = process.env.GOOGLEDRIVE_CLIENT_ID || "";
const DEFAULT_GOOGLEDRIVE_CLIENT_SECRET =
process.env.GOOGLEDRIVE_CLIENT_SECRET || "";
module.exports = {
entry: "./src/main.ts",
@ -24,6 +27,8 @@ module.exports = {
"process.env.DEFAULT_ONEDRIVE_AUTHORITY": `"${DEFAULT_ONEDRIVE_AUTHORITY}"`,
"process.env.DEFAULT_REMOTELYSAVE_WEBSITE": `"${DEFAULT_REMOTELYSAVE_WEBSITE}"`,
"process.env.DEFAULT_REMOTELYSAVE_CLIENT_ID": `"${DEFAULT_REMOTELYSAVE_CLIENT_ID}"`,
"process.env.DEFAULT_GOOGLEDRIVE_CLIENT_ID": `"${DEFAULT_GOOGLEDRIVE_CLIENT_ID}"`,
"process.env.DEFAULT_GOOGLEDRIVE_CLIENT_SECRET": `"${DEFAULT_GOOGLEDRIVE_CLIENT_SECRET}"`,
}),
// Work around for Buffer is undefined:
// https://github.com/webpack/changelog-v5/issues/10