Compare commits

...

10 Commits

Author SHA1 Message Date
Bruno Miiller
39711d117c Add CLAUDE.md for AI pair programmer context
Some checks failed
CI / Lint (push) Failing after 44s
CI / Test (push) Has been skipped
CI / Security (push) Has been skipped
- Stack and pin policy (Node 24, npm 11, 30-day dep aging)
- Fork-specific convention: English commits (vs upstream-aligned)
- Coverage ratchet at 8% (only goes up)
- CI structure: Lint → Test → Security, build moved to release.yml
- Pointers to docs/ARCHITECTURE, DEVELOPMENT, CONTRIBUTING
- Sensitive areas flagged: fake-license shortcut in pro/account.ts,
  large untested files (main.ts, settings.ts, pro/sync.ts)
2026-05-19 23:15:06 -03:00
Bruno Miiller
60bad0fcd3 Keep CI test stage focused: drop build step
The Lint → Test → Security pattern keeps each stage to a single concern.
`npm run build` belonged to test only to publish a PR-preview artifact;
release.yml already handles production builds, and developers can run
`npm run build` locally. Removing it from the test stage:

- aligns with code-standards/ci/pipeline-structure.md
- shaves ~70s off PR CI time
- removes the "dist" artifact upload (coverage upload stays)
2026-05-19 23:09:27 -03:00
Bruno Miiller
46f86c0091 Tighten CI: coverage ratchet at 8% and ignore build artifacts
- package.json: test:coverage adds --check-coverage --lines=8 (current
  baseline is 8.32%; floor only goes up)
- biome.json: ignore *.main.js webpack chunks, coverage/, node_modules/
- docs/CONTRIBUTING.md: document ratchet policy and long-term 70% target
2026-05-19 23:07:36 -03:00
Bruno Miiller
af79356e58 Apply biome format and lint fixes
- pro/src/account.ts: cast parens (}) as T; biome-ignore on intentional
  unreachable code after fake-license shortcut
- pro/src/fsGoogleDrive.ts: let → const; missing semicolon; == → ===
  (mimeType comparison and parent lookup)
- src/misc.ts: line break on long b.buffer.slice() chain
2026-05-19 23:07:36 -03:00
Bruno Miiller
6a6f192942 Make build pass on Node 24 and reduce npm audit surface
Build fixes:
- tsconfig.json: moduleResolution "node" → "bundler" (resolves node-diff3
  v3.2.0 exports-only package); add "es2021" to lib for AggregateError
- webpack.config.js: NormalModuleReplacementPlugin to strip "node:" URI
  prefix so resolve.fallback browserify shims apply
- Remove "aggregate-error" import in src/main.ts, src/fsS3.ts,
  pro/src/sync.ts — use native global AggregateError (Node 15+, Electron
  98+, both satisfied by Obsidian runtime)
- .gitignore: ignore *.main.js webpack chunks

Audit reduction (22 → 4 low):
- Remove npm-check-updates devDep (use `npx ncu` ad-hoc) — kills 15 vulns
  from transitive cacache/sigstore/tar/pacote chain
- Pin to versions ≥ 30 days old (supply-chain hygiene): @types/node 24.12.2,
  c8 11.0.0, mocha 11.7.5, esbuild 0.28.0, crypto-browserify 3.12.1
- package.json overrides: elliptic@6.6.1, diff@9.0.0,
  serialize-javascript@7.0.5 to push transitive fixes
- Remaining 4 lows are all elliptic (advisory marks all versions
  vulnerable; no upstream fix available)
2026-05-19 22:57:29 -03:00
A
87059c37a0 replace bigint with number 2026-05-19 22:12:40 -03:00
A
c96de5ccbc fix esbuild and typing issues 2026-05-19 22:12:40 -03:00
Bruno Miiller
f92bcd630d Add full CI pipeline (Lint → Test → Security)
- ci.yml: 3-stage GitHub Actions following code-standards/ci pattern
  - Lint (3min): biome ci
  - Test (10min): npm test with c8 coverage + webpack build + artifacts
  - Security (5min): npm audit (high) + biome lint
- Replace auto-build.yml (single-job install+test+build)
- Add c8 devDep + test:coverage script (text/lcov/html)
- .c8rc.json: include src/ and pro/src/, exclude tests/langs
- Track package-lock.json (required by npm ci in CI)
- Clean up .gitignore: remove .* allowlist antipattern, list specific ignores
2026-05-19 22:12:33 -03:00
Bruno Miiller
48b4f7e19c Add docs/ARCHITECTURE, CONTRIBUTING and DEVELOPMENT
- ARCHITECTURE.md: high-level view of FakeFs abstraction, sync flow, state
- DEVELOPMENT.md: mise setup + npm commands + local install
- CONTRIBUTING.md: workflow, English commit conventions, upstream PR cherry-pick
2026-05-19 22:10:31 -03:00
Bruno Miiller
9dde1bcdc8 Add mise.toml and pin Node LTS
- mise.toml with node = "24.15.0" (current LTS)
- package.json: packageManager = "npm@11.12.1", engines.node >= 24.15.0
2026-05-19 22:10:31 -03:00
25 changed files with 11987 additions and 135 deletions

6
.c8rc.json Normal file
View File

@ -0,0 +1,6 @@
{
"all": true,
"include": ["src/**/*.ts", "pro/src/**/*.ts"],
"exclude": ["tests/**", "pro/tests/**", "**/*.d.ts", "**/langs/**"],
"reports-dir": "coverage"
}

View File

@ -1,7 +1,4 @@
# This workflow will do a clean install of node dependencies, cache/restore them, build the source code and run tests across different versions of node
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
name: BuildCI
name: CI
on:
push:
@ -9,12 +6,27 @@ on:
pull_request:
branches: [master]
permissions:
contents: read
jobs:
build:
lint:
name: Lint
runs-on: ubuntu-latest
timeout-minutes: 3
steps:
- uses: actions/checkout@v4
- uses: jdx/mise-action@v2
- run: npm ci
- name: Biome check
run: npx @biomejs/biome ci .
test:
name: Test
needs: lint
runs-on: ubuntu-latest
timeout-minutes: 10
environment: env-for-buildci
env:
DROPBOX_APP_KEY: ${{secrets.DROPBOX_APP_KEY}}
ONEDRIVE_CLIENT_ID: ${{secrets.ONEDRIVE_CLIENT_ID}}
@ -31,15 +43,8 @@ jobs:
YANDEXDISK_CLIENT_SECRET: ${{secrets.YANDEXDISK_CLIENT_SECRET}}
KOOFR_CLIENT_ID: ${{secrets.KOOFR_CLIENT_ID}}
KOOFR_CLIENT_SECRET: ${{secrets.KOOFR_CLIENT_SECRET}}
strategy:
matrix:
node-version: [20.x]
# See supported Node.js release schedule at https://nodejs.org/en/about/releases/
steps:
- name: Checkout codes
uses: actions/checkout@v4
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Checkout LFS file list
@ -53,17 +58,26 @@ jobs:
${{ runner.os }}-lfs-
- name: Git LFS Pull
run: git lfs pull
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- run: npm install
- run: npm test
- run: npm run build
- uses: jdx/mise-action@v2
- run: npm ci
- name: Testes com cobertura
run: npm run test:coverage
- uses: actions/upload-artifact@v4
if: always()
with:
name: my-dist
path: |
main.js
manifest.json
styles.css
name: coverage
path: coverage/
security:
name: Security
needs: test
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: jdx/mise-action@v2
- run: npm ci
- name: npm audit
run: npm audit --audit-level=high
- name: Biome lint (regras de seguranca)
run: npx @biomejs/biome lint .

18
.gitignore vendored
View File

@ -1,14 +1,14 @@
# Intellij
*.iml
.idea
.idea/
# npm
node_modules
package-lock.json
node_modules/
pnpm-lock.yaml
# build
main.js
*.main.js
*.js.map
# obsidian
@ -17,5 +17,13 @@ data.json
# debug
logs.txt
# hidden files
.*
# coverage
coverage/
# env / secrets
.env
.env.local
# OS
.DS_Store
Thumbs.db

84
CLAUDE.md Normal file
View File

@ -0,0 +1,84 @@
# CLAUDE.md
Guia para Claude Code (claude.ai/code) trabalhar neste fork.
## Sobre o projeto
Fork de [`remotely-save/remotely-save`](https://github.com/remotely-save/remotely-save), plugin Obsidian que sincroniza vaults com WebDAV/S3/Dropbox/OneDrive/etc. Upstream com main branch parada desde 2024-11. Este fork retoma manutenção.
Detalhes técnicos completos em:
- `docs/ARCHITECTURE.md` — abstração `FakeFs`, fluxo de sync, state em IndexedDB
- `docs/DEVELOPMENT.md` — setup mise, comandos npm
- `docs/CONTRIBUTING.md` — workflow, convenções, política de cobertura
## Stack
- TypeScript 5.5 + Node 24.15.0 (pin via `mise.toml`)
- npm 11 (pin via `packageManager`)
- Webpack (build principal) + esbuild (alternativo)
- Biome (lint + format)
- Mocha + chai-as-promised + tsx (testes)
- c8 (cobertura)
## Convenções específicas deste fork
### Commits
**Inglês**, não pt-BR (alinhar com upstream). Imperativo, primeira letra maiúscula, ~72 caracteres na primeira linha. Corpo opcional com bullets `-`. Citar PR upstream quando aplicável (`based on upstream PR #1094`).
### Dependências
- **Só versões com ≥ 30 dias** publicadas no npm registry — checar `time` antes de subir
- Pinar **exato** (sem `^`/`~`) quando você mexer numa versão
- Usar `overrides` para forçar transitivas seguras quando o upstream não atualizou
### Cobertura
Floor atual: **8%** (ratchet — só sobe). Configurado em `package.json:scripts.test:coverage` com `c8 --check-coverage --lines=N`. Ao subir cobertura, atualizar `N` no script.
### Lint
Pode usar `// biome-ignore <rule>: <razão>` para suprimir regra com justificativa explícita. Sem suprimir silenciosamente.
### Estilo de código
- **SRP**: uma coisa por função, uma responsabilidade por módulo.
- **Nomes**: específicos e únicos. Evitar `data`, `handler`, `Manager`. Preferir nomes que retornem menos de 5 hits num grep do codebase.
- **Tipos**: explícitos. Sem `any`, sem `Dict`, sem funções sem tipo.
- **Funções**: ≤ 50 linhas. Dividir se passar.
- **Controle de fluxo**: early returns, no máximo 2 níveis de aninhamento.
- **Exceções**: incluir o valor ofensor e o formato esperado na mensagem.
## CI (`.github/workflows/ci.yml`)
Padrão `Lint → Test → Security`, cada um com responsabilidade única:
| Stage | Comando | Timeout |
|---|---|---|
| Lint | `npx @biomejs/biome ci .` | 3min |
| Test | `npm run test:coverage` | 10min |
| Security | `npm audit --audit-level=high` + `npx @biomejs/biome lint .` | 5min |
Build de produção fica em `release.yml` (não no CI de PR). Local: `npm run build`.
## Cherry-pick de PRs do upstream
```bash
git fetch upstream pull/<N>/head:pr-<N>
git cherry-pick <commits>
```
Validar local (`npm run format && npm test && npm run build`) antes de commit. Mencionar o PR no commit message.
## Cuidados ao mexer
- `pro/src/account.ts:200-201` tem código intencional para liberar pro features sem servidor. Não remover.
- `src/main.ts` (~2k LOC) e `src/settings.ts` (~3k LOC) são funções gigantes — refactors em passos pequenos.
- `pro/src/sync.ts:doActualSync` é o coração. **Sem testes** atualmente. Qualquer mudança aqui precisa teste novo.
- Backends `fs*.ts` seguem `FakeFs`. Não criar novo sem implementar a interface completa.
## Filosofia
- AI como pair programmer: narrar decisões técnicas (especialmente em tsconfig/build/CI). Perguntar em vez de assumir. Explicar trade-off.
- Mudança incremental: commits pequenos e bisect-friendly.
- Cada feature ou bugfix inclui teste.

View File

@ -4,7 +4,7 @@
"enabled": true
},
"files": {
"ignore": ["main.js"]
"ignore": ["main.js", "*.main.js", "coverage/**", "node_modules/**"]
},
"formatter": {
"enabled": true,

71
docs/ARCHITECTURE.md Normal file
View File

@ -0,0 +1,71 @@
# Arquitetura
Plugin Obsidian (TypeScript) que sincroniza vaults com serviços de armazenamento remoto (WebDAV, S3, Dropbox, OneDrive, etc).
## Estrutura
```
src/ código público (MIT)
├── main.ts entry point, estende Plugin do Obsidian (~2k linhas)
├── settings.ts UI de settings (~3k linhas)
├── fsAll.ts abstração FakeFs — interface comum dos backends
├── fsGetter.ts roteamento backend → instância (switch por tipo)
├── fs<Backend>.ts implementações: S3, WebDAV, Dropbox, OneDrive, Webdis, Local, Encrypt
├── localdb.ts IndexedDB via LocalForage — state local
├── configPersist.ts serialização da config no disco do plugin
├── i18n.ts Mustache + moment, idiomas em src/langs/
└── ...
pro/ código tier pago (PolyForm Strict License)
├── src/
│ ├── sync.ts motor de sincronização (~2k linhas, doActualSync + dispatchDecision)
│ ├── fs<Backend>.ts GoogleDrive, Box, pCloud, YandexDisk, Koofr, Azure, OnedriveFull
│ └── ...
└── tests/
tests/ testes Mocha (estrutura básica, sem cobertura do core)
docs/ documentação
.github/workflows/ CI
```
## Abstração FakeFs
Todo backend implementa a classe abstrata `FakeFs` (`src/fsAll.ts`):
```typescript
abstract class FakeFs {
walk() / walkPartial() / stat() / mkdir() / writeFile()
readFile() / rename() / rm()
checkConnect() / getUserDisplayName() / revokeAuth() / allowEmptyFile()
}
```
`fsGetter.ts` faz o roteamento via switch — ao adicionar backend novo, registrar nele.
## Fluxo de sincronização
1. `main.ts` dispara sync (manual, auto-sync, evento de save)
2. `pro/src/sync.ts:doActualSync()` carrega árvore local + remota
3. Diff 3-way usando snapshot anterior (`prevSyncRecordsTbl` em IndexedDB)
4. `dispatchDecision()` aplica decisão por entry (copy, merge, delete, skip)
5. Conflitos: 3-way merge via `node-diff3` (apenas Markdown < 1MB, tier pro)
6. Erros agregados em `AggregateError`; falha até 3x antes de abortar
## State
LocalForage (IndexedDB) com 8 tabelas em `src/localdb.ts`:
- `prevSyncRecordsTbl` — snapshot da última sync
- `fileContentHistoryTbl` — histórico de conteúdo (smart conflict)
- `syncPlansTbl` — log das operações planejadas
- `versionTbl`, `loggerOutputTbl`, `profilerResultsTbl` — metadados
Config persiste via API do Obsidian em `<vault>/.obsidian/plugins/remotely-save/data.json`,
com obfuscação base64 reverse (decorativa, não criptográfica).
## Pontos de atenção
- **mtime no WebDAV**: padrão WebDAV não define mtime — Nextcloud expõe via header custom (`OC-LastModified`). Inconsistência pode causar falsa detecção de modificação.
- **OAuth2 duplicado**: cada backend implementa fluxo próprio (~100-300 linhas duplicadas). Sem extração comum.
- **Sem retry HTTP**: 429/503 quebram sync. PR upstream #1034 propõe retry para WebDAV.
- **141 `: any` residuais**: concentrados em callbacks de error e payloads OAuth2.

67
docs/CONTRIBUTING.md Normal file
View File

@ -0,0 +1,67 @@
# Contribuindo
## Workflow
1. Branch a partir de `master`: `git checkout -b <tipo>/<descrição>`
- `feat/` — feature nova
- `fix/` — bugfix
- `refactor/` — refactor sem mudança de comportamento
- `docs/` — só documentação
- `chore/` — config, build, CI
2. Commits pequenos, foco único
3. Antes de abrir PR: `npm run format && npm test && npm run build`
4. Abrir PR descrevendo o porquê (não o quê — o diff mostra o quê)
## Mensagens de commit
- Inglês, imperativo (alinhado com o upstream)
- Primeira linha curta (~72 caracteres), inicia com letra maiúscula
- Corpo opcional em lista com `- ` detalhando o porquê
Exemplo:
```
Add WebDAV retry on 429/503
- new retryWithBackoff helper in src/fsWebdav.ts
- apply to walk/stat/writeFile methods
- based on upstream PR #1034
```
## Cherry-pick de PRs do upstream
Upstream: https://github.com/remotely-save/remotely-save
```bash
git remote add upstream https://github.com/remotely-save/remotely-save.git
git fetch upstream pull/<N>/head:pr-<N>
git cherry-pick <commits-de-pr-N>
```
Citar o PR upstream na mensagem de commit (`baseado no PR upstream #<N>`).
## Estilo de código
- Biome formata e linta (`npm run format`)
- TypeScript strict — evitar `any`
- Funções idealmente 420 linhas; refatorar se passar
- Nomes específicos, não `data`/`handler`/`Manager`
- Sem comentários óbvios — explicar só o porquê quando não-trivial
- Early returns (no máximo 2 níveis de aninhamento)
## Testes
- Toda feature ou bugfix inclui teste
- Localização: `tests/` para src/, `pro/tests/` para pro/
- Comando: `npm run test:coverage`
### Política de cobertura (ratchet)
Cobertura mínima de linhas é um **piso que só sobe**. Atualmente em `8%`
(via `c8 --check-coverage --lines=8` no script `test:coverage`).
Quando você adicionar testes que elevam a cobertura geral, **suba o piso**
no script `test:coverage` para o novo valor (truncado). Nunca abaixar.
Meta de longo prazo: 70% (padrão code-standards). Áreas zero-cobertura
prioritárias: `src/main.ts`, `src/settings.ts`, backends `fs*.ts`.

64
docs/DEVELOPMENT.md Normal file
View File

@ -0,0 +1,64 @@
# Desenvolvimento
## Setup
Pré-requisito: [mise](https://mise.jdx.dev/) instalado.
```bash
mise install # instala Node conforme mise.toml
npm ci # instala dependências do lockfile
```
## Comandos
```bash
npm run dev # webpack watch (dev)
npm run build # build de produção (webpack)
npm run dev2 # esbuild watch (alternativo)
npm run build2 # build de produção (esbuild + tsc check)
npm test # mocha (tests/ + pro/tests/)
npm run format # biome check --write
npm run clean # remove main.js
```
## Instalar build local no Obsidian
Após `npm run build`, copiar para o vault:
```bash
cp main.js manifest.json styles.css \
/path/to/vault/.obsidian/plugins/remotely-save/
```
Reload do Obsidian (Ctrl+R) para carregar a nova versão.
## Testes
Framework: Mocha + chai-as-promised + tsx (sem transpile separado).
Estrutura:
```
tests/ testes do código público (src/)
pro/tests/ testes do código pro/
```
Comando: `npm test`.
## Lint/format
Biome (`biome.json` na raiz):
```bash
npm run format # corrige automaticamente
npx @biomejs/biome check # só verifica
```
## Release
Disparado via tag git. Workflow `.github/workflows/release.yml` constrói e publica artefatos:
```bash
git tag x.y.z
git push origin x.y.z
```

View File

@ -57,7 +57,7 @@ esbuild
inject: ["./esbuild.injecthelper.mjs"],
format: "cjs",
// watch: !prod, // no longer valid in esbuild 0.17
target: "es2016",
target: "es2020",
logLevel: "info",
sourcemap: prod ? false : "inline",
treeShaking: true,

2
mise.toml Normal file
View File

@ -0,0 +1,2 @@
[tools]
node = "24.15.0"

11423
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -2,6 +2,10 @@
"name": "remotely-save",
"version": "0.5.25",
"description": "This is yet another sync plugin for Obsidian app.",
"packageManager": "npm@11.12.1",
"engines": {
"node": ">=24.15.0"
},
"scripts": {
"dev2": "node esbuild.config.mjs --watch",
"build2": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
@ -9,7 +13,8 @@
"dev": "webpack --mode development --watch",
"format": "npx @biomejs/biome check --write .",
"clean": "npx rimraf main.js",
"test": "mocha --import=tsx 'tests/**/*.ts' 'pro/tests/**/*.ts'"
"test": "mocha --import=tsx 'tests/**/*.ts' 'pro/tests/**/*.ts'",
"test:coverage": "c8 --check-coverage --lines=8 --reporter=text --reporter=lcov --reporter=html npm test"
},
"browser": {
"path": "path-browserify",
@ -21,11 +26,17 @@
"vm": false
},
"source": "main.ts",
"overrides": {
"elliptic": "6.6.1",
"diff": "9.0.0",
"serialize-javascript": "7.0.5"
},
"keywords": [],
"author": "",
"license": "SEE LICENSE IN LICENSE",
"devDependencies": {
"@biomejs/biome": "1.8.3",
"c8": "11.0.0",
"@microsoft/microsoft-graph-types": "^2.40.0",
"@types/chai": "^4.3.16",
"@types/chai-as-promised": "^7.1.8",
@ -34,16 +45,15 @@
"@types/mime-types": "^2.1.4",
"@types/mocha": "^10.0.7",
"@types/mustache": "^4.2.5",
"@types/node": "^20.14.12",
"@types/node": "24.12.2",
"@types/qrcode": "^1.5.5",
"builtin-modules": "^4.0.0",
"cross-env": "^7.0.3",
"dotenv": "^16.4.5",
"esbuild": "^0.23.0",
"esbuild": "0.28.0",
"esbuild-plugin-inline-worker": "^0.1.1",
"jsdom": "^24.1.1",
"mocha": "^10.7.0",
"npm-check-updates": "^16.14.20",
"mocha": "11.7.5",
"obsidian": "^1.5.7",
"openapi-typescript": "^7.1.0",
"ts-loader": "^9.5.1",
@ -70,12 +80,11 @@
"@smithy/protocol-http": "^4.1.0",
"@smithy/querystring-builder": "^3.0.3",
"acorn": "^8.12.1",
"aggregate-error": "^5.0.0",
"assert": "^2.1.0",
"aws-crt": "^1.21.3",
"box-typescript-sdk-gen": "^1.3.0",
"buffer": "^6.0.3",
"crypto-browserify": "^3.12.0",
"crypto-browserify": "3.12.1",
"dropbox": "^10.34.0",
"emoji-regex": "^10.3.0",
"http-status-codes": "^2.3.0",

View File

@ -173,6 +173,32 @@ export const getAndSaveProFeatures = async (
pluginVersion: string,
saveUpdatedConfigFunc: () => Promise<any> | undefined
) => {
const features = [
"feature-smart_conflict",
"feature-onedrive_full",
"feature-google_drive",
"feature-box",
"feature-pcloud",
"feature-yandex_disk",
"feature-koofr",
"feature-azure_blob_storage",
];
const res = {
proFeatures: features.map(
(i) =>
({
featureName: i,
enableAtTimeMs: 1e12,
expireAtTimeMs: 3e12,
}) as FeatureInfo
),
};
config.enabledProFeatures = res.proFeatures;
await saveUpdatedConfigFunc?.();
return res;
// biome-ignore lint/correctness/noUnreachable: original upstream flow kept for reference (the fake-license shortcut above is intentional)
const access = await getAccessToken(config, saveUpdatedConfigFunc);
const resp1 = await fetch(`${site}/api/v1/pro/list`, {

View File

@ -37,8 +37,8 @@ export type PRO_FEATURE_TYPE =
export interface FeatureInfo {
featureName: PRO_FEATURE_TYPE;
enableAtTimeMs: bigint;
expireAtTimeMs: bigint;
enableAtTimeMs: number;
expireAtTimeMs: number;
}
export interface ProConfig {

View File

@ -78,7 +78,7 @@ const fromBlobPropsToEntity = (
let hash: undefined | string = undefined;
if (props.contentMD5 !== undefined) {
hash = arrayBufferToHex(props.contentMD5.buffer);
hash = arrayBufferToHex(props.contentMD5.buffer as ArrayBuffer);
}
const entity: Entity = {

View File

@ -2,7 +2,6 @@
// https://developers.google.com/identity/protocols/oauth2/javascript-implicit-flow
// https://developers.google.com/identity/protocols/oauth2/web-server
import { entries } from "lodash";
import * as mime from "mime-types";
import { requestUrl } from "obsidian";
import PQueue from "p-queue";
@ -177,6 +176,7 @@ export class FakeFsGoogleDrive extends FakeFs {
keyToGDEntity: Record<string, GDEntity>;
baseDirID: string;
ready = false;
constructor(
googleDriveConfig: GoogleDriveConfig,
@ -199,13 +199,17 @@ export class FakeFsGoogleDrive extends FakeFs {
await this._getAccessToken();
// check vault folder exists
if (this.vaultFolderExists) {
// pass
} else {
const q = encodeURIComponent(
`name='${this.remoteBaseDir}' and mimeType='application/vnd.google-apps.folder' and trashed=false`
if (!this.vaultFolderExists) {
const q = `name='${this.remoteBaseDir}' and mimeType='application/vnd.google-apps.folder' and trashed=false`;
const url = new URL("https://www.googleapis.com/drive/v3/files");
url.searchParams.set("q", q);
url.searchParams.set("pageSize", "1000");
url.searchParams.set(
"fields",
"kind,nextPageToken," +
"files(kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum)"
);
const url: string = `https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=1000&fields=kind,nextPageToken,files(kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum)`;
url.searchParams.set("orderBy", "modifiedTime desc");
const k = await fetch(url, {
method: "GET",
headers: {
@ -274,8 +278,16 @@ export class FakeFsGoogleDrive extends FakeFs {
/**
* https://developers.google.com/drive/api/reference/rest/v3/files/list
*/
async walk(): Promise<Entity[]> {
async walk(): Promise<GDEntity[]> {
await this._init();
// const allFiles = await this._listAllFiles();
// this.keyToGDEntity = allFiles.reduce((p, c) => {
// p[c.keyRaw] = c;
// return p;
// }, {} as Record<string, GDEntity>); // rebuild cache
// return allFiles;
const allFiles: GDEntity[] = [];
// bfs
@ -289,39 +301,23 @@ export class FakeFsGoogleDrive extends FakeFs {
throw error;
});
let parents = [
{
id: this.baseDirID, // special init, from already created root folder ID
folderPath: "",
},
];
while (parents.length !== 0) {
const children: typeof parents = [];
for (const { id, folderPath } of parents) {
queue.add(async () => {
const filesUnderFolder = await this._walkFolder(id, folderPath);
const newWalkTask = (id: string, folderPath: string) => {
return async () => {
const filesUnderFolder = await this._listFolder(id, folderPath);
for (const f of filesUnderFolder) {
allFiles.push(f);
if (f.isFolder) {
// keyRaw itself already has a tailing slash, no more slash here
// keyRaw itself also already has full path
const child = {
id: f.id,
folderPath: f.keyRaw,
queue.add(newWalkTask(f.id, f.keyRaw));
}
}
};
// console.debug(
// `looping result of _walkFolder(${id},${folderPath}), adding child=${JSON.stringify(
// child
// )}`
// );
children.push(child);
}
}
});
}
};
queue.add(newWalkTask(this.baseDirID, "")); // special init, from already created root folder ID
await queue.onIdle();
parents = children;
}
// console.debug(`in the end of walk:`);
// console.debug(allFiles);
@ -329,25 +325,97 @@ export class FakeFsGoogleDrive extends FakeFs {
return allFiles;
}
async _walkFolder(parentID: string, parentFolderPath: string) {
async _listAllFiles(): Promise<GDEntity[]> {
const allFileRes: File[] = [];
let nextPageToken = "";
do {
const q = `'${this.baseDirID}' in parents and trashed=false`;
const url = new URL("https://www.googleapis.com/drive/v3/files");
url.searchParams.set("q", q);
url.searchParams.set("pageSize", "1000");
url.searchParams.set(
"fields",
"kind,nextPageToken,files(kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum)"
);
url.searchParams.set("orderBy", "modifiedTime");
url.searchParams.set("pageToken", nextPageToken);
const res = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
},
});
if (res.status !== 200) {
throw Error(`Error on list all files`);
}
const fileRes = await res.json();
(fileRes.files as File[]).forEach((i) => allFileRes.push(i));
nextPageToken = fileRes.nextPageToken;
} while (nextPageToken !== undefined);
const allFolderRes = allFileRes.filter(
(i) => i.mimeType === FOLDER_MIME_TYPE
);
const allFolders = [
{
id: this.baseDirID, // special init, from already created root folder ID
folderPath: "",
},
];
for (const targetFolder of allFolders) {
allFolderRes
.filter((i) => i.parents?.includes(targetFolder.id))
.forEach((i) => {
allFolders.push({
id: i.id!,
folderPath: `${targetFolder}${i.name!}/`,
});
});
}
const allFiles: GDEntity[] = [];
for (const file of allFileRes) {
if (!file.parents) continue;
file.parents.forEach((parent) => {
const folder = allFolders.find((folder) => folder.id === parent);
if (!folder) return;
const entity = fromFileToGDEntity(file, folder.id, folder.folderPath);
allFiles.push(entity);
});
}
return allFiles;
}
async _listFolder(parentID: string, parentFolderPath: string) {
// console.debug(
// `input of single level: parentID=${parentID}, parentFolderPath=${parentFolderPath}`
// );
const filesOneLevel: GDEntity[] = [];
let nextPageToken: string | undefined = undefined;
let nextPageToken = "";
if (parentID === undefined || parentID === "" || parentID === "root") {
// we should never start from root
// because we encapsulate the vault inside a folder
throw Error(`something goes wrong walking folder`);
}
do {
const q = encodeURIComponent(
`'${parentID}' in parents and trashed=false`
const q = `'${parentID}' in parents and trashed=false`;
const url = new URL("https://www.googleapis.com/drive/v3/files");
url.searchParams.set("q", q);
url.searchParams.set("pageSize", "1000");
url.searchParams.set(
"fields",
"kind,nextPageToken,files(kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum)"
);
const pageToken =
nextPageToken !== undefined ? `&pageToken=${nextPageToken}` : "";
const url: string = `https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=1000&fields=kind,nextPageToken,files(kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum)${pageToken}`;
url.searchParams.set("orderBy", "modifiedTime");
url.searchParams.set("pageToken", nextPageToken);
const k = await fetch(url, {
method: "GET",
@ -377,7 +445,7 @@ export class FakeFsGoogleDrive extends FakeFs {
async walkPartial(): Promise<Entity[]> {
await this._init();
const filesInLevel = await this._walkFolder(this.baseDirID, "");
const filesInLevel = await this._listFolder(this.baseDirID, "");
return filesInLevel;
}
@ -508,6 +576,8 @@ export class FakeFsGoogleDrive extends FakeFs {
// "xxx" => []
// "xxx/yyy/zzz.md" => ["xxx", "xxx/yyy"]
const folderLevels = getFolderLevels(key);
console.log(key);
console.log(folderLevels);
if (folderLevels.length === 0) {
// root
parentID = this.baseDirID;
@ -522,34 +592,42 @@ export class FakeFsGoogleDrive extends FakeFs {
parentID = this.keyToGDEntity[parentFolderPath].id;
}
const fileItself = key.split("/").pop()!;
const targetFileId = this.keyToGDEntity[key]?.id;
if (content.byteLength <= 5 * 1024 * 1024) {
const formData = new FormData();
const fileItself = key.split("/").pop()!;
const meta: any = {
name: fileItself,
modifiedTime: unixTimeToStr(mtime, true),
createdTime: unixTimeToStr(ctime, true),
parents: [parentID],
};
if (!targetFileId) meta.parents = [parentID];
if (content.byteLength <= 5 * 1024 * 1024) {
const formData = new FormData();
formData.append(
"metadata",
new Blob([JSON.stringify(meta)], {
new Blob(targetFileId ? [] : [JSON.stringify(meta)], {
type: "application/json; charset=UTF-8",
})
);
formData.append("media", new Blob([content], { type: contentType }));
const res = await fetch(
"https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&fields=kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum",
{
method: "POST",
const url = new URL("https://www.googleapis.com/upload/drive/v3/files");
if (targetFileId) url.pathname += `/${targetFileId}`;
url.searchParams.set("uploadType", "multipart");
url.searchParams.set(
"fields",
"kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum"
);
const res = await fetch(url, {
method: targetFileId ? "PATCH" : "POST",
headers: {
Authorization: `Bearer ${await this._getAccessToken()}`,
},
body: formData,
}
);
});
if (res.status !== 200 && res.status !== 201) {
throw Error(`create file ${key} failed! meta=${JSON.stringify(meta)}`);
}
@ -564,13 +642,7 @@ export class FakeFsGoogleDrive extends FakeFs {
this.keyToGDEntity[key] = entity;
return entity;
} else {
const meta: any = {
name: fileItself,
modifiedTime: unixTimeToStr(mtime, true),
createdTime: unixTimeToStr(ctime, true),
parents: [parentID],
};
const bodyStr = JSON.stringify(meta);
const bodyStr = targetFileId ? "" : JSON.stringify(meta);
const headers: HeadersInit = {
Authorization: `Bearer ${await this._getAccessToken()}`,
"Content-Type": "application/json",
@ -578,14 +650,18 @@ export class FakeFsGoogleDrive extends FakeFs {
"X-Upload-Content-Type": contentType,
"X-Upload-Content-Length": `${content.byteLength}`,
};
const res = await fetch(
"https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&fields=kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum",
{
method: "POST",
const url = new URL("https://www.googleapis.com/upload/drive/v3/files");
if (targetFileId) url.pathname += `/${targetFileId}`;
url.searchParams.set("uploadType", "resumable");
url.searchParams.set(
"fields",
"kind,fileExtension,md5Checksum,mimeType,parents,size,spaces,id,name,trashed,createdTime,modifiedTime,quotaBytesUsed,originalFilename,fullFileExtension,sha1Checksum,sha256Checksum"
);
const res = await fetch(url, {
method: targetFileId ? "PATCH" : "POST",
headers: headers,
body: bodyStr,
}
);
});
if (res.status !== 200) {
throw Error(
`create resumable file ${key} failed! meta=${JSON.stringify(

View File

@ -575,7 +575,7 @@ export class FakeFsOnedriveFull extends FakeFs {
*/
async _putUint8ArrayByRange(
pathFragOrig: string,
payload: Uint8Array,
payload: Uint8Array<ArrayBuffer>,
rangeStart: number,
rangeEnd: number,
size: number

View File

@ -1,5 +1,3 @@
// biome-ignore lint/suspicious/noShadowRestrictedNames: <explanation>
import AggregateError from "aggregate-error";
import PQueue from "p-queue";
import XRegExp from "xregexp";
import type {

View File

@ -24,7 +24,7 @@ const getKeyIVFromPassword = async (
const k2 = await window.crypto.subtle.deriveBits(
{
name: "PBKDF2",
salt: salt,
salt: salt as Uint8Array<ArrayBuffer>,
iterations: rounds,
hash: "SHA-256",
},

View File

@ -729,7 +729,7 @@ export class FakeFsOnedrive extends FakeFs {
*/
async _putUint8ArrayByRange(
pathFragOrig: string,
payload: Uint8Array,
payload: Uint8Array<ArrayBuffer>,
rangeStart: number,
rangeEnd: number,
size: number

View File

@ -22,8 +22,6 @@ import {
import { requestTimeout } from "@smithy/fetch-http-handler/dist-es/request-timeout";
import { type HttpRequest, HttpResponse } from "@smithy/protocol-http";
import { buildQueryString } from "@smithy/querystring-builder";
// biome-ignore lint/suspicious/noShadowRestrictedNames: <explanation>
import AggregateError from "aggregate-error";
import * as mime from "mime-types";
import { Platform, type RequestUrlParam, requestUrl } from "obsidian";
import PQueue from "p-queue";

View File

@ -1,5 +1,3 @@
// biome-ignore lint/suspicious/noShadowRestrictedNames: <explanation>
import AggregateError from "aggregate-error";
import cloneDeep from "lodash/cloneDeep";
import throttle from "lodash/throttle";
import { FileText, RefreshCcw, RotateCcw, createElement } from "lucide";

View File

@ -88,9 +88,12 @@ export const mkdirpInVault = async (thePath: string, vault: Vault) => {
* @returns ArrayBuffer
*/
export const bufferToArrayBuffer = (
b: Buffer | Uint8Array | ArrayBufferView
b: Buffer | Uint8Array<ArrayBuffer> | ArrayBufferView
) => {
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength);
return b.buffer.slice(
b.byteOffset,
b.byteOffset + b.byteLength
) as ArrayBuffer;
};
/**

View File

@ -8,13 +8,13 @@
"strict": true,
"allowJs": true,
"noImplicitAny": true,
"moduleResolution": "node",
"moduleResolution": "bundler",
// "allowSyntheticDefaultImports": true,
"resolveJsonModule": true,
"esModuleInterop": true,
"importHelpers": true,
"isolatedModules": true,
"lib": ["dom", "es5", "scripthost", "es2015", "webworker"]
"lib": ["dom", "es5", "scripthost", "es2015", "es2021", "webworker"]
},
"include": ["src/global.d.ts", "**/*.ts"]
}

View File

@ -65,6 +65,11 @@ module.exports = {
new webpack.ProvidePlugin({
process: "process/browser",
}),
// Strip `node:` URI prefix so resolve.fallback (browserify shims) applies.
// Required because some deps (AWS smithy, glob, ...) use `node:url` etc.
new webpack.NormalModuleReplacementPlugin(/^node:/, (resource) => {
resource.request = resource.request.replace(/^node:/, "");
}),
],
module: {
rules: [