Compare commits
31 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b77c3f8cf5 | |||
| 1754ea9316 | |||
| 6a127de950 | |||
| 9d08e6af23 | |||
| 941a9f16e1 | |||
| c8205e474e | |||
| 81ee02f11a | |||
| eb80a830f5 | |||
| 988c6c31a9 | |||
| 2af279e8a9 | |||
| faeb76a7e7 | |||
| a022c45499 | |||
| 6f82d1350e | |||
| 7a039b23f4 | |||
| 8445eef3cf | |||
| 1d4cfd58c5 | |||
| 763d7881fb | |||
| 2cada618bf | |||
| 7914b0140b | |||
| 178fa8ddce | |||
| 9c8f20a618 | |||
| 9c46e281c2 | |||
| 8b2dfed239 | |||
| 2da375451a | |||
| e4de2dcec3 | |||
| 00c1e8b1e7 | |||
| 9a1b21ec39 | |||
| b1ce715ec7 | |||
| 064d41595f | |||
| fa2456b99e | |||
| 0cf5d44062 |
@@ -39,3 +39,9 @@ HOUSE_WORKER_TOKEN=
|
|||||||
HOUSE_DISCORD_WEBHOOK_URL=
|
HOUSE_DISCORD_WEBHOOK_URL=
|
||||||
HOUSE_NTFY_URL=
|
HOUSE_NTFY_URL=
|
||||||
HOUSE_NTFY_TOKEN=
|
HOUSE_NTFY_TOKEN=
|
||||||
|
|
||||||
|
# Money tracker bank sync (scripts/bank-link.ts, scripts/bank-sync.ts; not read by the web app).
|
||||||
|
# Keep these in ~/.config/money-sync.env (mode 600), never in git.
|
||||||
|
ENABLEBANKING_APP_ID=
|
||||||
|
ENABLEBANKING_KEY_PATH=
|
||||||
|
ENABLEBANKING_REDIRECT_URL=https://stanrunge.dev/projects/money-tracker/bank-callback
|
||||||
|
|||||||
@@ -19,3 +19,9 @@ data/cod-challenges.json
|
|||||||
data/game-images.json
|
data/game-images.json
|
||||||
data/cod-achievement-art.json
|
data/cod-achievement-art.json
|
||||||
src/lib/house-tracker/fixtures/
|
src/lib/house-tracker/fixtures/
|
||||||
|
|
||||||
|
# Vendored third-party code
|
||||||
|
static/vendor/
|
||||||
|
|
||||||
|
# Generated drizzle snapshots
|
||||||
|
drizzle/*/snapshot.json
|
||||||
|
|||||||
28
README.md
28
README.md
@@ -236,9 +236,33 @@ local database and migrations, and `db/reorg/README.md` for the one-time move of
|
|||||||
old flat schema to this layout. Catalog seeds (`npm run db:seed`, `db:seed:cod`, …)
|
old flat schema to this layout. Catalog seeds (`npm run db:seed`, `db:seed:cod`, …)
|
||||||
upsert reference data and run on every production deploy.
|
upsert reference data and run on every production deploy.
|
||||||
|
|
||||||
## Money tracker
|
## Book tracker
|
||||||
|
|
||||||
`/projects/money-tracker` is restricted to administrator accounts. Its tables
|
`/trackers/books` is the public catalogue (covers, status, progress, stats). `/trackers/books/<id>`
|
||||||
|
shows a book; the owner also sees files, notes, highlights and reading sessions there and can
|
||||||
|
edit status, rating and progress. `/trackers/books/<id>/read` is the owner-only reader:
|
||||||
|
EPUB, MOBI/AZW3 and FB2 open in [foliate-js](https://github.com/johnfactotum/foliate-js)
|
||||||
|
(vendored in `static/vendor/foliate-js`, no conversion), PDFs in pdf.js. The reader remembers the
|
||||||
|
position per file, autosaves progress and sessions, and stores highlights with notes.
|
||||||
|
|
||||||
|
Files are never public. They are streamed (with Range support) by
|
||||||
|
`/trackers/books/files/<id>` to the owner only; everyone else gets a 404. The reader, progress
|
||||||
|
and notes endpoints are owner-only too (`src/lib/server/owner.ts`, backstopped in
|
||||||
|
`hooks.server.ts`).
|
||||||
|
|
||||||
|
Files are read from `BOOKS_ROOT` (default `/books` when it exists, otherwise `~/books`):
|
||||||
|
|
||||||
|
- `BOOKS_ROOT/library/<books.files.path>` for book files (written by `scripts/books-ingest.py`)
|
||||||
|
- `BOOKS_ROOT/covers/<books.works.cover_path>` for downloaded covers
|
||||||
|
|
||||||
|
In production, `scripts/deploy-remote.sh` writes `~deploy/compose.books.yml` (the deploy account may only sudo-tee `/opt/personal-website/.env`), which
|
||||||
|
bind-mounts the host's `BOOKS_HOST_DIR` (default `/home/stan/books`) read-only at `/books` and sets
|
||||||
|
`BOOKS_ROOT=/books`. Without a `library` directory there the deploy still succeeds, but files can't
|
||||||
|
be opened.
|
||||||
|
|
||||||
|
## Money
|
||||||
|
|
||||||
|
`/trackers/money` (formerly `/projects/money-tracker`, which now redirects) is restricted to administrator accounts. Its tables
|
||||||
are created by `npm run db:migrate:money-tracker`; apply that migration to each
|
are created by `npm run db:migrate:money-tracker`; apply that migration to each
|
||||||
environment before deploying the new route. The deployment preflight checks for
|
environment before deploying the new route. The deployment preflight checks for
|
||||||
all nine tracker tables and the projection columns.
|
all nine tracker tables and the projection columns.
|
||||||
|
|||||||
28
bun.lock
28
bun.lock
@@ -14,9 +14,11 @@
|
|||||||
"drizzle-orm": "^1.0.0-rc.4",
|
"drizzle-orm": "^1.0.0-rc.4",
|
||||||
"echarts": "^6.1.0",
|
"echarts": "^6.1.0",
|
||||||
"highlight.js": "^11.12.0",
|
"highlight.js": "^11.12.0",
|
||||||
|
"jose": "^6.2.10",
|
||||||
"marked": "^18.0.11",
|
"marked": "^18.0.11",
|
||||||
"marked-highlight": "^2.2.4",
|
"marked-highlight": "^2.2.4",
|
||||||
"mode-watcher": "^1.1.0",
|
"mode-watcher": "^1.1.0",
|
||||||
|
"pdfjs-dist": "^6.4.299",
|
||||||
"postgres": "^3.4.9",
|
"postgres": "^3.4.9",
|
||||||
"topojson-client": "^3.1.0",
|
"topojson-client": "^3.1.0",
|
||||||
"world-atlas": "^2.0.2",
|
"world-atlas": "^2.0.2",
|
||||||
@@ -325,6 +327,30 @@
|
|||||||
|
|
||||||
"@melt-ui/svelte": ["@melt-ui/svelte@0.76.2", "", { "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", "@internationalized/date": "^3.5.0", "dequal": "^2.0.3", "focus-trap": "^7.5.2", "nanoid": "^5.0.4" }, "peerDependencies": { "svelte": ">=3 <5" } }, "sha512-7SbOa11tXUS95T3fReL+dwDs5FyJtCEqrqG3inRziDws346SYLsxOQ6HmX+4BkIsQh1R8U3XNa+EMmdMt38lMA=="],
|
"@melt-ui/svelte": ["@melt-ui/svelte@0.76.2", "", { "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", "@internationalized/date": "^3.5.0", "dequal": "^2.0.3", "focus-trap": "^7.5.2", "nanoid": "^5.0.4" }, "peerDependencies": { "svelte": ">=3 <5" } }, "sha512-7SbOa11tXUS95T3fReL+dwDs5FyJtCEqrqG3inRziDws346SYLsxOQ6HmX+4BkIsQh1R8U3XNa+EMmdMt38lMA=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas": ["@napi-rs/canvas@1.0.13", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.13", "@napi-rs/canvas-darwin-arm64": "1.0.13", "@napi-rs/canvas-darwin-x64": "1.0.13", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.13", "@napi-rs/canvas-linux-arm64-gnu": "1.0.13", "@napi-rs/canvas-linux-arm64-musl": "1.0.13", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.13", "@napi-rs/canvas-linux-x64-gnu": "1.0.13", "@napi-rs/canvas-linux-x64-musl": "1.0.13", "@napi-rs/canvas-win32-arm64-msvc": "1.0.13", "@napi-rs/canvas-win32-x64-msvc": "1.0.13" } }, "sha512-b87ugZpQ5ViwqVBkjervxpwa0GmJPsKHLBNoZN6GpjWeHzTbDUGryriq+2DYW2XbB+xCqSIINQysC0/I87nPbw=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.13", "", { "os": "android", "cpu": "arm64" }, "sha512-trn7AaDzQ+yeeA0ezhkC7cB1ZDPtp4UYk0EvB5F8qtYL9jEVWtVrU30WOpI52DszoTxaTsCgf5PP4ZVaOlOIHQ=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.13", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6Y2djEetL8copxfNK6hg7oig/xqTwOchv19hBOm+zBNdKc/02Z3kjZN5wrLzv/l5O04/bAQS6I/AlQpwBlXgPg=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.13", "", { "os": "darwin", "cpu": "x64" }, "sha512-OZoPwwu4TMMg8GTCFBrg8UFExwQTU/55i3uTC/izDat6PHE+ro9dimyQoOboFf91+NY7Pnn70ScR8f8I8+eZXw=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.13", "", { "os": "linux", "cpu": "arm" }, "sha512-dZSYaKROuswU2eUWG5aBKndNVe68elo1YX93xtRax7q658tj8rYE+HwduovWJgs6qwv1r1/CLR60nDeBbyjvyg=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.13", "", { "os": "linux", "cpu": "arm64" }, "sha512-W6FzYE8ADHI93In1vi80KBL6btDtAMuj50Ch5ptkWS8xuntyzbR4h5JDUmXoakxZn2yMInksudTEqVSWoFqFGg=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.13", "", { "os": "linux", "cpu": "arm64" }, "sha512-+6fQ+rR49hkAa4voh2stLIwQ+4t+gmoseM8Bc53/DlMXb73IKymDPWK9A/NtWeqTW4ijWppYuyuxS2hPI8tzNA=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.13", "", { "os": "linux", "cpu": "none" }, "sha512-17XEMO3t9RW/TtHtSFf2AkaSWuCWdzbPeidz7zrOFawHxoLSJCnpTI1hN5aUvrZNnkNNmt/vZVJCh0ryUaik3A=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.13", "", { "os": "linux", "cpu": "x64" }, "sha512-lc48IwknVHBsNzXn9Xs5OkUsRX38Ml3Z1aOybQ51mMEzZckm8+4A0qROlyYHEoBPzr1kL3IU8ZUAqDb7wtdX8w=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.13", "", { "os": "linux", "cpu": "x64" }, "sha512-bGY1r26S2bs8UNFJcPWcCSNdCFbKep5PATsvtgtC9A5u6SRgr2/jJE/RlhHL5m9ZpixNDhVkLY/5h16Kx5V9YA=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.13", "", { "os": "win32", "cpu": "arm64" }, "sha512-2vRPBZoaIvJrUkDhZTE2Hn0YrnpJRCWVM3EJACoBKjuWI6WgPxGtUoRMsxRviCsyks8bKTSFdyw14CellnG2qA=="],
|
||||||
|
|
||||||
|
"@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.13", "", { "os": "win32", "cpu": "x64" }, "sha512-DUlW3lZPz63kYsf8mxgyyZuevhlH0T3/BGP9l+NazhB0F1+gL6D4hqZdnhZpw24DlWmTpATDcyYSZlJOi+DdfQ=="],
|
||||||
|
|
||||||
"@noble/ciphers": ["@noble/ciphers@2.4.0", "", {}, "sha512-AnjFn0Jv92laAkvMrghlFZq4qQCIN/4DxFV/eooqtC2YTjB7kBeLMS2T9KJX4Dn+ZVXLOwK0lSgqDtx9gvxtiw=="],
|
"@noble/ciphers": ["@noble/ciphers@2.4.0", "", {}, "sha512-AnjFn0Jv92laAkvMrghlFZq4qQCIN/4DxFV/eooqtC2YTjB7kBeLMS2T9KJX4Dn+ZVXLOwK0lSgqDtx9gvxtiw=="],
|
||||||
|
|
||||||
"@noble/hashes": ["@noble/hashes@2.4.0", "", {}, "sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA=="],
|
"@noble/hashes": ["@noble/hashes@2.4.0", "", {}, "sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA=="],
|
||||||
@@ -837,6 +863,8 @@
|
|||||||
|
|
||||||
"pbf": ["pbf@4.0.2", "", { "dependencies": { "resolve-protobuf-schema": "^2.1.0" }, "bin": "bin/pbf" }, "sha512-J0ajxARhZfpUEebxYs1vhMGMuLSXtBe1e+fFPDrf2uA2hgo+UshKfNUWOz92HJNz6/NFEXseQPddnHkTreWRqg=="],
|
"pbf": ["pbf@4.0.2", "", { "dependencies": { "resolve-protobuf-schema": "^2.1.0" }, "bin": "bin/pbf" }, "sha512-J0ajxARhZfpUEebxYs1vhMGMuLSXtBe1e+fFPDrf2uA2hgo+UshKfNUWOz92HJNz6/NFEXseQPddnHkTreWRqg=="],
|
||||||
|
|
||||||
|
"pdfjs-dist": ["pdfjs-dist@6.4.299", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.10" } }, "sha512-AVl138zALtfaAPvADulE0PZThbYzCBS79nL4pOSL/6Sm/4AH5A21BD9VHt97OlCuzJuCpmeZtAtkinisF4Vb1g=="],
|
||||||
|
|
||||||
"pg-int8": ["pg-int8@1.0.1", "", {}, "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="],
|
"pg-int8": ["pg-int8@1.0.1", "", {}, "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="],
|
||||||
|
|
||||||
"pg-protocol": ["pg-protocol@1.16.0", "", {}, "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg=="],
|
"pg-protocol": ["pg-protocol@1.16.0", "", {}, "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg=="],
|
||||||
|
|||||||
643
data/tasks/daisy-import.json
Normal file
643
data/tasks/daisy-import.json
Normal file
@@ -0,0 +1,643 @@
|
|||||||
|
{
|
||||||
|
"source": "daisy-import",
|
||||||
|
"areas": [
|
||||||
|
{
|
||||||
|
"slug": "deadlines",
|
||||||
|
"name": "Deadlines and dates",
|
||||||
|
"sortOrder": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "house",
|
||||||
|
"name": "House and moving",
|
||||||
|
"description": "Jan–Feb 2027 with Juhee",
|
||||||
|
"sortOrder": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "kibi",
|
||||||
|
"name": "Kibi",
|
||||||
|
"sortOrder": 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "admin",
|
||||||
|
"name": "Phone, subscriptions and admin",
|
||||||
|
"sortOrder": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "clients",
|
||||||
|
"name": "Clients",
|
||||||
|
"sortOrder": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "website",
|
||||||
|
"name": "Website (stanrunge.dev)",
|
||||||
|
"sortOrder": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "setup",
|
||||||
|
"name": "Setup and security",
|
||||||
|
"description": "Perma evolvable setup, before 2027",
|
||||||
|
"sortOrder": 70
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "home",
|
||||||
|
"name": "Home",
|
||||||
|
"sortOrder": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "health",
|
||||||
|
"name": "Health and self-care",
|
||||||
|
"sortOrder": 90
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "reading",
|
||||||
|
"name": "Reading",
|
||||||
|
"sortOrder": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slug": "planning",
|
||||||
|
"name": "Planning and goals",
|
||||||
|
"sortOrder": 110
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"goals": [
|
||||||
|
{
|
||||||
|
"title": "€10k net/month into own Revolut by 2030",
|
||||||
|
"area": "planning",
|
||||||
|
"targetDate": "2030-01-01",
|
||||||
|
"metric": "Net income per month",
|
||||||
|
"metricUnit": "EUR",
|
||||||
|
"metricTarget": 10000
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Own a really nice Ford Mustang GT"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "PC overhaul",
|
||||||
|
"notes": "Top AMD X3D CPU, RTX 5090.",
|
||||||
|
"metric": "Budget",
|
||||||
|
"metricUnit": "EUR",
|
||||||
|
"metricTarget": 10000
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Organise goals into a plan",
|
||||||
|
"area": "planning",
|
||||||
|
"notes": "Social media, Grok Bot, IT company."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"area": "setup",
|
||||||
|
"targetDate": "2026-12-31"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Get in shape long term",
|
||||||
|
"area": "health"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tasks": [
|
||||||
|
{
|
||||||
|
"area": "deadlines",
|
||||||
|
"title": "Jan Loman printer visit",
|
||||||
|
"status": "done",
|
||||||
|
"notes": "Settled.",
|
||||||
|
"due": "2026-10-24",
|
||||||
|
"dueTime": "14:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "deadlines",
|
||||||
|
"title": "Call BackHome about Kibi's chip",
|
||||||
|
"status": "todo",
|
||||||
|
"due": "2026-10-26",
|
||||||
|
"dueTime": "10:00",
|
||||||
|
"tags": ["kibi", "call"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "deadlines",
|
||||||
|
"title": "Cargill collective health insurance (OHRA, code 114)",
|
||||||
|
"status": "todo",
|
||||||
|
"notes": "Needs current insurer + premium to compare.",
|
||||||
|
"due": "2026-12-31",
|
||||||
|
"tags": ["insurance"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "deadlines",
|
||||||
|
"title": "Reply to mom about insurance info",
|
||||||
|
"status": "todo",
|
||||||
|
"notes": "Waiting since Oct 9.",
|
||||||
|
"tags": ["insurance"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "deadlines",
|
||||||
|
"title": "Stop zorgtoeslag from Jan 2027",
|
||||||
|
"status": "todo",
|
||||||
|
"due": "2027-01-01",
|
||||||
|
"reminderAt": "2026-12-07T09:00:00+01:00",
|
||||||
|
"tags": ["admin"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "deadlines",
|
||||||
|
"title": "Supermarket product list",
|
||||||
|
"status": "todo",
|
||||||
|
"reminderAt": "2026-10-24T11:50:00+02:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "deadlines",
|
||||||
|
"title": "Book haircut",
|
||||||
|
"status": "todo",
|
||||||
|
"notes": "Around Nov 14.",
|
||||||
|
"due": "2026-11-14"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "house",
|
||||||
|
"title": "House overview",
|
||||||
|
"status": "in_progress",
|
||||||
|
"flagged": true,
|
||||||
|
"notes": "Applied only to Graaf Hendrikstraat 16 so far."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "house",
|
||||||
|
"title": "React to house options",
|
||||||
|
"status": "todo",
|
||||||
|
"notes": "Raam €1,475; Burg. van Dijkesingel 214-B; Groenhovenweg 161; Spoorstraat €795; Noothoven van Goorstraat 1-206."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "house",
|
||||||
|
"title": "Tell Daisy budget, move date, must-haves, Funda/Pararius applications",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "house",
|
||||||
|
"title": "Mijndak Midden-Holland sign-up",
|
||||||
|
"status": "todo",
|
||||||
|
"flagged": true,
|
||||||
|
"notes": "€23 option."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "house",
|
||||||
|
"title": "Sign up: socialehuurwoning.nl, Vesteda Goudshof, VBT, Viramah, agency Jesse's mom uses",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "house",
|
||||||
|
"title": "House tracker/scraper on the site",
|
||||||
|
"status": "todo",
|
||||||
|
"tags": ["website"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "house",
|
||||||
|
"title": "Update address everywhere",
|
||||||
|
"status": "todo",
|
||||||
|
"notes": "Checklist ready."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "house",
|
||||||
|
"title": "UniFi network for the new place",
|
||||||
|
"status": "todo",
|
||||||
|
"tags": ["network"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "kibi",
|
||||||
|
"title": "Food ordered",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "kibi",
|
||||||
|
"title": "Chip scanner ordered",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "kibi",
|
||||||
|
"title": "Chip number found",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "kibi",
|
||||||
|
"title": "Chip owner change at BackHome",
|
||||||
|
"status": "todo",
|
||||||
|
"notes": "Still 'Kwiebus' under E Zwart."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "kibi",
|
||||||
|
"title": "Call vet about Kibi's patient card",
|
||||||
|
"status": "todo",
|
||||||
|
"tags": ["call"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "kibi",
|
||||||
|
"title": "GPS tracker",
|
||||||
|
"status": "todo",
|
||||||
|
"notes": "Dad may have done it already."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "kibi",
|
||||||
|
"title": "Dash KIBI",
|
||||||
|
"status": "todo",
|
||||||
|
"flagged": true,
|
||||||
|
"notes": "Meaning unclear, ask Stan."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "admin",
|
||||||
|
"title": "Basic-Fit cancelled",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "admin",
|
||||||
|
"title": "Ohra inboedelverzekering cancelled",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "admin",
|
||||||
|
"title": "Zorgtoeslag income updated for 2026",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "admin",
|
||||||
|
"title": "Move number to Revolut eSIM, then iPad eSIM, cancel Odido",
|
||||||
|
"status": "in_progress"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "admin",
|
||||||
|
"title": "Move SIM + WhatsApp from iPhone to Pixel 11 Pro XL",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "admin",
|
||||||
|
"title": "Check Oasen and Delta Energie fully cancelled",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "admin",
|
||||||
|
"title": "Cancel other unused subscriptions",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "admin",
|
||||||
|
"title": "Merge banks into Revolut",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "clients",
|
||||||
|
"title": "Quote dack €450 for vet clinic site fix",
|
||||||
|
"status": "done",
|
||||||
|
"tags": ["dack"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "clients",
|
||||||
|
"title": "Reply to dack",
|
||||||
|
"status": "done",
|
||||||
|
"tags": ["dack"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "clients",
|
||||||
|
"title": "Cloudflare DNS/SSL for dack",
|
||||||
|
"status": "in_progress",
|
||||||
|
"tags": ["dack"],
|
||||||
|
"notes": "Grind-week goal."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "clients",
|
||||||
|
"title": "Send dack's invoice via Moneybird",
|
||||||
|
"status": "todo",
|
||||||
|
"tags": ["dack"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "clients",
|
||||||
|
"title": "Cheap shared WordPress hosting for dack's clients on own VMs",
|
||||||
|
"status": "todo",
|
||||||
|
"tags": ["dack"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "clients",
|
||||||
|
"title": "Karolina's Wix site, Master Purpose (NL/PL)",
|
||||||
|
"status": "todo",
|
||||||
|
"tags": ["karolina"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "clients",
|
||||||
|
"title": "Wix Studio + Velo/CLI so an AI can edit code",
|
||||||
|
"status": "todo",
|
||||||
|
"tags": ["karolina"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "DB reorganised into schemas, staging retired",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Energy tracker (NED data) live",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Health tracker (private) + 15 RIVM vaccinations",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Book tracker, reader, 362 books live",
|
||||||
|
"status": "done"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Reader zoom, 2-page view, scroll/paging",
|
||||||
|
"status": "in_progress"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Money tracker with MoneyMan",
|
||||||
|
"status": "in_progress"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "NED backfill",
|
||||||
|
"status": "in_progress"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Music listening tracker + now-playing",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Task/goal tracker to replace TickTick",
|
||||||
|
"status": "in_progress"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Hardware DB (current PC parts first)",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "List of every organisation holding my data and what they know",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Move to SvelteKit 3",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "website",
|
||||||
|
"title": "Juhee website data",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "AWS cleanup ($0/month)",
|
||||||
|
"status": "done",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Google Cloud cleanup",
|
||||||
|
"status": "done",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "stan-vm hardened (ufw, Tailscale-only SSH)",
|
||||||
|
"status": "done",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Nightly restic backups to B2 + Storage Box",
|
||||||
|
"status": "done",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "WhatsApp bridge on stan-vm",
|
||||||
|
"status": "done",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "gcloud + moneybird-cli on stan-vm",
|
||||||
|
"status": "done",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Domains Squarespace to Cloudflare",
|
||||||
|
"status": "in_progress",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"notes": "Both transfers pending.",
|
||||||
|
"due": "2026-10-29"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Syncthing: Introducer + Auto Accept on Arch and Windows",
|
||||||
|
"status": "in_progress",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Backup follow-up: restic passwords into 1Password",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"due": "2026-10-24",
|
||||||
|
"dueTime": "11:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Delete empty B2 bucket",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"due": "2026-10-25"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Rotate weak admin secret + DB password",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"tags": ["security"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "1Password overhaul",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"notes": "Imports, 2FA, passkeys, dedupe, delete unused accounts.",
|
||||||
|
"tags": ["security"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Moneybird API token, then clean up Moneybird",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Authentik SSO on stan-vm",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "vash-prod memory warnings + finish security audit",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"tags": ["security"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Remove old grok-bot devices from Tailscale",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Close empty GCP billing account; decide on AWS 'Vash Software'",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"notes": "AWS part optional."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Raise WSL RAM",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"notes": "Sees 8 of 16 GB."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Home server rebuild with ZFS",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Analytics, monitoring, alerts across devices",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Mouse sensitivity cm/360 across devices",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Buy Logitech G840 XL mousepad",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"tags": ["buy"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Router and networking setup",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"tags": ["network"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "J5 Hexagon research, then move wachtboden",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "Agent-deck + AI/Linux dev setup",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "setup",
|
||||||
|
"title": "MITM/certificate lab",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||||
|
"notes": "Idea.",
|
||||||
|
"tags": ["idea"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "home",
|
||||||
|
"title": "Unpack boxes; organise batteries and tools; buy thermometer",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "home",
|
||||||
|
"title": "Clean desk, laundry, clothes",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "home",
|
||||||
|
"title": "Connect vinyl player, printer, monitor, switch, osu! pad, tablet, fridge, microwave",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "home",
|
||||||
|
"title": "Hue lights + Home Assistant + Apple Home",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "home",
|
||||||
|
"title": "Fan controller",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "home",
|
||||||
|
"title": "Opa's stuff (WhatsApp, sensor install)",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "health",
|
||||||
|
"title": "Getting in shape with Doc",
|
||||||
|
"status": "in_progress",
|
||||||
|
"notes": "Lesson 1 done; ~2,300–2,400 kcal and 160–190 g protein a day.",
|
||||||
|
"goal": "Get in shape long term"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "health",
|
||||||
|
"title": "Join the gym my parents use",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Get in shape long term"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "health",
|
||||||
|
"title": "Brush teeth twice a day, shower, hair care",
|
||||||
|
"status": "todo",
|
||||||
|
"tags": ["habit"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "health",
|
||||||
|
"title": "Dentist checkup",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "health",
|
||||||
|
"title": "Pick hardware for Withings/Garmin syncs",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "health",
|
||||||
|
"title": "Curaçao vaccines",
|
||||||
|
"status": "todo",
|
||||||
|
"notes": "Via Doc."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "reading",
|
||||||
|
"title": "Read 'De meeste mensen deugen' (Bregman)",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "reading",
|
||||||
|
"title": "Re-download 'Haves and Have-Nots'",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "reading",
|
||||||
|
"title": "Book questions: duplicates, 6 unknown files, Nick's repo 404",
|
||||||
|
"status": "todo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"area": "planning",
|
||||||
|
"title": "Schedule sync (paper agenda, TickTick, Daisy)",
|
||||||
|
"status": "todo",
|
||||||
|
"goal": "Organise goals into a plan"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
7
data/tasks/ticktick-merges.json
Normal file
7
data/tasks/ticktick-merges.json
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
[
|
||||||
|
{ "ticktickId": "6ad52191ebcd7400000000ce", "title": "Jan Loman printer visit" },
|
||||||
|
{ "ticktickId": "690d2fd98c824268f297a676", "title": "Router and networking setup" },
|
||||||
|
{ "ticktickId": "6844769df471d103461fa9f1", "title": "Home server rebuild with ZFS" },
|
||||||
|
{ "ticktickId": "6ab6f32759b6090811e91a00", "title": "Dentist checkup" },
|
||||||
|
{ "ticktickId": "68225a8dbfe81104eac49a7a", "title": "Merge banks into Revolut" }
|
||||||
|
]
|
||||||
13
deploy/systemd/money-sync.service
Normal file
13
deploy/systemd/money-sync.service
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Money tracker bank sync (Enable Banking -> money.transactions)
|
||||||
|
Wants=network-online.target
|
||||||
|
After=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
# A checkout of stan/website (main once merged). Not the live dev tree.
|
||||||
|
WorkingDirectory=%h/dev/website-money
|
||||||
|
# DB_URL (app role), ENABLEBANKING_APP_ID, ENABLEBANKING_KEY_PATH; mode 600, never committed.
|
||||||
|
EnvironmentFile=%h/.config/money-sync.env
|
||||||
|
ExecStart=/usr/bin/bun scripts/bank-sync.ts
|
||||||
|
TimeoutStartSec=600
|
||||||
10
deploy/systemd/money-sync.timer
Normal file
10
deploy/systemd/money-sync.timer
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Money tracker bank sync, 4x per day (PSD2 allows ~4 unattended fetches/day)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 06,11,16,21:20:00 Europe/Amsterdam
|
||||||
|
RandomizedDelaySec=300
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -20,7 +20,9 @@ export default defineConfig({
|
|||||||
'animaldex',
|
'animaldex',
|
||||||
'site',
|
'site',
|
||||||
'meta',
|
'meta',
|
||||||
'health'
|
'health',
|
||||||
|
'books',
|
||||||
|
'tasks'
|
||||||
],
|
],
|
||||||
migrations: { schema: 'drizzle', table: '__drizzle_migrations' },
|
migrations: { schema: 'drizzle', table: '__drizzle_migrations' },
|
||||||
verbose: true,
|
verbose: true,
|
||||||
|
|||||||
65
drizzle/20261024003809_money_bank_sync/migration.sql
Normal file
65
drizzle/20261024003809_money_bank_sync/migration.sql
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
CREATE TABLE "money"."bank_sessions" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"user_id" text NOT NULL,
|
||||||
|
"provider" text DEFAULT 'enablebanking' NOT NULL,
|
||||||
|
"session_id" text NOT NULL CONSTRAINT "bank_sessions_session_id_key" UNIQUE,
|
||||||
|
"aspsp_name" text NOT NULL,
|
||||||
|
"aspsp_country" text NOT NULL,
|
||||||
|
"psu_type" text,
|
||||||
|
"status" text DEFAULT 'authorized' NOT NULL,
|
||||||
|
"valid_until" timestamp with time zone,
|
||||||
|
"accounts" jsonb DEFAULT '[]' NOT NULL,
|
||||||
|
"last_synced_at" timestamp with time zone,
|
||||||
|
"last_error" text,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "bank_sessions_status_check" CHECK (status IN ('authorized','expired','revoked','error'))
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "scope" text DEFAULT 'personal' NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "owner" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "iban" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "external_uid" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "provider" text DEFAULT 'manual' NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD COLUMN "external_id" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD COLUMN "source" text DEFAULT 'manual' NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD COLUMN "booking_status" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD COLUMN "counterparty" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD COLUMN "category" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD COLUMN "raw" jsonb;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD COLUMN "value_date" date;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD CONSTRAINT "accounts_external_uid_key" UNIQUE("external_uid");--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD CONSTRAINT "transactions_account_external_id_key" UNIQUE("account_id","external_id");--> statement-breakpoint
|
||||||
|
CREATE INDEX "bank_sessions_user_id_index" ON "money"."bank_sessions" ("user_id");--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."bank_sessions" ADD CONSTRAINT "bank_sessions_user_id_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "auth"."user"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD CONSTRAINT "accounts_scope_check" CHECK (scope IN ('personal','joint'));--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD CONSTRAINT "accounts_provider_check" CHECK (provider IN ('manual','enablebanking'));--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."transactions" ADD CONSTRAINT "transactions_source_check" CHECK (source IN ('manual','csv','enablebanking'));--> statement-breakpoint
|
||||||
|
-- Hand-written below this line (not generated by drizzle-kit).
|
||||||
|
-- Backfill: the shared account with Juhee is the only joint one; everything else stays personal.
|
||||||
|
UPDATE "money"."accounts" SET "scope" = 'joint', "owner" = 'Stan & Juhee' WHERE lower("name") = 'joint';
|
||||||
|
--> statement-breakpoint
|
||||||
|
UPDATE "money"."accounts" SET "owner" = 'Stan' WHERE "scope" = 'personal' AND "owner" IS NULL;
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- updated_at trigger, same as every other core table (07_triggers_meta_grants.sql).
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
|
||||||
|
CREATE TRIGGER touch_updated_at BEFORE UPDATE ON money.bank_sessions
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at();
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- Grants only run on prod (migrations run as the schema owner `personal`); the dev DB
|
||||||
|
-- (owned by personal_dev) carries no prod grants, matching db-refresh-dev.sh. Default
|
||||||
|
-- privileges on `money` already cover new tables; this is explicit for clarity.
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON money.bank_sessions TO personal_app;
|
||||||
|
GRANT USAGE, SELECT ON SEQUENCE money.bank_sessions_id_seq TO personal_app;
|
||||||
|
END IF;
|
||||||
|
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||||
|
GRANT SELECT ON money.bank_sessions TO personal_ro;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
11543
drizzle/20261024003809_money_bank_sync/snapshot.json
Normal file
11543
drizzle/20261024003809_money_bank_sync/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
3
drizzle/20261024011919_money_bank_balance/migration.sql
Normal file
3
drizzle/20261024011919_money_bank_balance/migration.sql
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_cents" integer;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_type" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_at" timestamp with time zone;
|
||||||
12076
drizzle/20261024011919_money_bank_balance/snapshot.json
Normal file
12076
drizzle/20261024011919_money_bank_balance/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
194
drizzle/20261024013208_books/migration.sql
Normal file
194
drizzle/20261024013208_books/migration.sql
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
CREATE SCHEMA "books";
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TYPE "books"."metadata_status" AS ENUM('pending', 'matched', 'low_confidence', 'not_found', 'manual');--> statement-breakpoint
|
||||||
|
CREATE TYPE "books"."reading_status" AS ENUM('want_to_read', 'reading', 'finished', 'abandoned');--> statement-breakpoint
|
||||||
|
CREATE TABLE "books"."files" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"work_id" integer NOT NULL,
|
||||||
|
"path" text NOT NULL,
|
||||||
|
"format" text NOT NULL,
|
||||||
|
"size_bytes" bigint NOT NULL,
|
||||||
|
"sha256" text NOT NULL,
|
||||||
|
"md5" text,
|
||||||
|
"source" text,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "books"."notes" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"work_id" integer NOT NULL,
|
||||||
|
"file_id" integer,
|
||||||
|
"locator" text,
|
||||||
|
"page" integer,
|
||||||
|
"highlight" text,
|
||||||
|
"note" text,
|
||||||
|
"color" text,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "books"."progress" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"work_id" integer NOT NULL,
|
||||||
|
"file_id" integer,
|
||||||
|
"locator" text,
|
||||||
|
"page" integer,
|
||||||
|
"percent" real DEFAULT 0 NOT NULL,
|
||||||
|
"started_at" timestamp with time zone,
|
||||||
|
"finished_at" timestamp with time zone,
|
||||||
|
"status" "books"."reading_status" DEFAULT 'want_to_read'::"books"."reading_status" NOT NULL,
|
||||||
|
"rating" smallint,
|
||||||
|
"last_read_at" timestamp with time zone,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "progress_percent_check" CHECK ("percent" between 0 and 100),
|
||||||
|
CONSTRAINT "progress_rating_check" CHECK ("rating" is null or "rating" between 1 and 5)
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "books"."sessions" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"work_id" integer NOT NULL,
|
||||||
|
"file_id" integer,
|
||||||
|
"started_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"ended_at" timestamp with time zone,
|
||||||
|
"start_percent" real,
|
||||||
|
"end_percent" real,
|
||||||
|
"start_page" integer,
|
||||||
|
"end_page" integer,
|
||||||
|
"pages_read" integer,
|
||||||
|
"source" text DEFAULT 'reader' NOT NULL,
|
||||||
|
"note" text,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "sessions_percent_check" CHECK (("start_percent" is null or "start_percent" between 0 and 100) and ("end_percent" is null or "end_percent" between 0 and 100)),
|
||||||
|
CONSTRAINT "sessions_end_check" CHECK ("ended_at" is null or "ended_at" >= "started_at")
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "books"."shelf_works" (
|
||||||
|
"shelf_id" integer,
|
||||||
|
"work_id" integer,
|
||||||
|
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||||
|
"added_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "shelf_works_pkey" PRIMARY KEY("shelf_id","work_id")
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "books"."shelves" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"slug" text NOT NULL,
|
||||||
|
"name" text NOT NULL,
|
||||||
|
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "books"."works" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"title" text NOT NULL,
|
||||||
|
"subtitle" text,
|
||||||
|
"authors" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||||
|
"isbn10" text,
|
||||||
|
"isbn13" text,
|
||||||
|
"publisher" text,
|
||||||
|
"year" integer,
|
||||||
|
"language" text,
|
||||||
|
"page_count" integer,
|
||||||
|
"description" text,
|
||||||
|
"tags" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||||
|
"cover_url" text,
|
||||||
|
"cover_path" text,
|
||||||
|
"openlibrary_work_id" text,
|
||||||
|
"openlibrary_edition_id" text,
|
||||||
|
"google_books_id" text,
|
||||||
|
"annas_md5" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||||
|
"source_refs" jsonb DEFAULT '{}' NOT NULL,
|
||||||
|
"metadata_status" "books"."metadata_status" DEFAULT 'pending'::"books"."metadata_status" NOT NULL,
|
||||||
|
"metadata_source" text,
|
||||||
|
"metadata_fetched_at" timestamp with time zone,
|
||||||
|
"notes" text,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "works_year_check" CHECK ("year" is null or "year" between 0 and 2100)
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX "files_path_key" ON "books"."files" ("path");--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX "files_sha256_key" ON "books"."files" ("sha256");--> statement-breakpoint
|
||||||
|
CREATE INDEX "files_work_id_idx" ON "books"."files" ("work_id");--> statement-breakpoint
|
||||||
|
CREATE INDEX "notes_work_id_idx" ON "books"."notes" ("work_id");--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX "progress_work_id_key" ON "books"."progress" ("work_id");--> statement-breakpoint
|
||||||
|
CREATE INDEX "progress_status_idx" ON "books"."progress" ("status");--> statement-breakpoint
|
||||||
|
CREATE INDEX "sessions_work_id_idx" ON "books"."sessions" ("work_id");--> statement-breakpoint
|
||||||
|
CREATE INDEX "sessions_started_at_idx" ON "books"."sessions" ("started_at");--> statement-breakpoint
|
||||||
|
CREATE INDEX "shelf_works_work_id_idx" ON "books"."shelf_works" ("work_id");--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX "shelves_slug_key" ON "books"."shelves" ("slug");--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX "works_isbn13_key" ON "books"."works" ("isbn13") WHERE "isbn13" is not null;--> statement-breakpoint
|
||||||
|
CREATE INDEX "works_title_idx" ON "books"."works" (lower("title"));--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."files" ADD CONSTRAINT "files_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."notes" ADD CONSTRAINT "notes_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."notes" ADD CONSTRAINT "notes_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."shelf_works" ADD CONSTRAINT "shelf_works_shelf_id_shelves_id_fkey" FOREIGN KEY ("shelf_id") REFERENCES "books"."shelves"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||||
|
ALTER TABLE "books"."shelf_works" ADD CONSTRAINT "shelf_works_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- Hand-written: updated_at triggers, grants, default shelves, GUI registration.
|
||||||
|
DO $$
|
||||||
|
DECLARE t text;
|
||||||
|
BEGIN
|
||||||
|
FOREACH t IN ARRAY ARRAY['works','files','shelves','progress','notes'] LOOP
|
||||||
|
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON books.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||||
|
END LOOP;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||||
|
GRANT USAGE ON SCHEMA books TO personal_app;
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA books TO personal_app;
|
||||||
|
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA books TO personal_app;
|
||||||
|
ALTER DEFAULT PRIVILEGES IN SCHEMA books GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||||
|
ALTER DEFAULT PRIVILEGES IN SCHEMA books GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||||
|
END IF;
|
||||||
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||||
|
GRANT USAGE ON SCHEMA books TO personal_ro;
|
||||||
|
GRANT SELECT ON ALL TABLES IN SCHEMA books TO personal_ro;
|
||||||
|
ALTER DEFAULT PRIVILEGES IN SCHEMA books GRANT SELECT ON TABLES TO personal_ro;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
INSERT INTO books.shelves (slug, name, sort_order) VALUES
|
||||||
|
('to-read', 'To read', 10), ('reading', 'Reading', 20), ('done', 'Done', 30), ('reference', 'Reference', 40)
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||||
|
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||||
|
VALUES ('books','works','books-works','Books','core',true,false,1000),
|
||||||
|
('books','shelves','books-shelves','Book shelves','core',true,false,1000),
|
||||||
|
('books','files','books-files','Book files','core',false,false,1000)
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
-- meta.sync() is executable by personal_app/personal_ddl, not by the migration role
|
||||||
|
-- (personal) in prod; deploy-remote.sh runs it as the app role after migrating.
|
||||||
|
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||||
|
PERFORM meta.sync();
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- Hand-written: updated_at trigger and explicit grants (default privileges from the books migration cover new tables too).
|
||||||
|
CREATE TRIGGER touch_updated_at BEFORE UPDATE ON books.sessions FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at();
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON books.sessions TO personal_app;
|
||||||
|
GRANT USAGE, SELECT ON SEQUENCE books.sessions_id_seq TO personal_app;
|
||||||
|
END IF;
|
||||||
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||||
|
GRANT SELECT ON books.sessions TO personal_ro;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
13722
drizzle/20261024013208_books/snapshot.json
Normal file
13722
drizzle/20261024013208_books/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE "money"."accounts" ADD COLUMN "include_in_total" boolean DEFAULT true NOT NULL;
|
||||||
13735
drizzle/20261024021157_money_include_in_total/snapshot.json
Normal file
13735
drizzle/20261024021157_money_include_in_total/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
142
drizzle/20261024022836_tasks/migration.sql
Normal file
142
drizzle/20261024022836_tasks/migration.sql
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
CREATE SCHEMA "tasks";
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "tasks"."activity" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"task_id" integer,
|
||||||
|
"goal_id" integer,
|
||||||
|
"kind" text NOT NULL,
|
||||||
|
"from_status" text,
|
||||||
|
"to_status" text,
|
||||||
|
"note" text,
|
||||||
|
"value" numeric(14,2),
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "activity_target_check" CHECK ("task_id" IS NOT NULL OR "goal_id" IS NOT NULL),
|
||||||
|
CONSTRAINT "activity_kind_check" CHECK ("kind" IN ('created','status','edit','note','progress'))
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "tasks"."areas" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"slug" text NOT NULL,
|
||||||
|
"name" text NOT NULL,
|
||||||
|
"description" text,
|
||||||
|
"color" text,
|
||||||
|
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||||
|
"archived_at" timestamp with time zone,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "areas_slug_check" CHECK ("slug" ~ '^[a-z0-9][a-z0-9-]{0,62}$'),
|
||||||
|
CONSTRAINT "areas_name_check" CHECK (length(btrim("name")) > 0)
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "tasks"."goals" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"title" text NOT NULL,
|
||||||
|
"notes" text,
|
||||||
|
"status" text DEFAULT 'active' NOT NULL,
|
||||||
|
"area_id" integer,
|
||||||
|
"target_date" date,
|
||||||
|
"metric" text,
|
||||||
|
"metric_unit" text,
|
||||||
|
"metric_target" numeric(14,2),
|
||||||
|
"metric_current" numeric(14,2),
|
||||||
|
"flagged" boolean DEFAULT false NOT NULL,
|
||||||
|
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||||
|
"source" text,
|
||||||
|
"achieved_at" timestamp with time zone,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "goals_title_check" CHECK (length(btrim("title")) > 0),
|
||||||
|
CONSTRAINT "goals_status_check" CHECK ("status" IN ('active','achieved','dropped'))
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE "tasks"."tasks" (
|
||||||
|
"id" serial PRIMARY KEY,
|
||||||
|
"title" text NOT NULL,
|
||||||
|
"notes" text,
|
||||||
|
"status" text DEFAULT 'todo' NOT NULL,
|
||||||
|
"priority" smallint DEFAULT 0 NOT NULL,
|
||||||
|
"flagged" boolean DEFAULT false NOT NULL,
|
||||||
|
"due_date" date,
|
||||||
|
"due_time" time,
|
||||||
|
"reminder_at" timestamp with time zone,
|
||||||
|
"recurrence" text,
|
||||||
|
"area_id" integer,
|
||||||
|
"goal_id" integer,
|
||||||
|
"parent_id" integer,
|
||||||
|
"tags" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||||
|
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||||
|
"source" text,
|
||||||
|
"external_id" text,
|
||||||
|
"started_at" timestamp with time zone,
|
||||||
|
"completed_at" timestamp with time zone,
|
||||||
|
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
CONSTRAINT "tasks_title_check" CHECK (length(btrim("title")) > 0),
|
||||||
|
CONSTRAINT "tasks_status_check" CHECK ("status" IN ('todo','in_progress','done','dropped')),
|
||||||
|
CONSTRAINT "tasks_priority_check" CHECK ("priority" BETWEEN 0 AND 3),
|
||||||
|
CONSTRAINT "tasks_parent_check" CHECK ("parent_id" IS NULL OR "parent_id" <> "id"),
|
||||||
|
CONSTRAINT "tasks_time_check" CHECK ("due_time" IS NULL OR "due_date" IS NOT NULL)
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX "activity_task_idx" ON "tasks"."activity" ("task_id","created_at");--> statement-breakpoint
|
||||||
|
CREATE INDEX "activity_goal_idx" ON "tasks"."activity" ("goal_id","created_at");--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX "areas_slug_key" ON "tasks"."areas" ("slug");--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX "goals_title_key" ON "tasks"."goals" (lower("title"));--> statement-breakpoint
|
||||||
|
CREATE INDEX "tasks_area_idx" ON "tasks"."tasks" ("area_id");--> statement-breakpoint
|
||||||
|
CREATE INDEX "tasks_goal_idx" ON "tasks"."tasks" ("goal_id");--> statement-breakpoint
|
||||||
|
CREATE INDEX "tasks_parent_idx" ON "tasks"."tasks" ("parent_id");--> statement-breakpoint
|
||||||
|
CREATE INDEX "tasks_status_due_idx" ON "tasks"."tasks" ("status","due_date");--> statement-breakpoint
|
||||||
|
CREATE INDEX "tasks_tags_idx" ON "tasks"."tasks" USING gin ("tags");--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX "tasks_source_external_key" ON "tasks"."tasks" ("source","external_id") WHERE "external_id" IS NOT NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "tasks"."activity" ADD CONSTRAINT "activity_task_id_tasks_id_fkey" FOREIGN KEY ("task_id") REFERENCES "tasks"."tasks"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||||
|
ALTER TABLE "tasks"."activity" ADD CONSTRAINT "activity_goal_id_goals_id_fkey" FOREIGN KEY ("goal_id") REFERENCES "tasks"."goals"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||||
|
ALTER TABLE "tasks"."goals" ADD CONSTRAINT "goals_area_id_areas_id_fkey" FOREIGN KEY ("area_id") REFERENCES "tasks"."areas"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_area_id_areas_id_fkey" FOREIGN KEY ("area_id") REFERENCES "tasks"."areas"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_goal_id_goals_id_fkey" FOREIGN KEY ("goal_id") REFERENCES "tasks"."goals"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||||
|
ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_parent_id_tasks_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "tasks"."tasks"("id") ON DELETE CASCADE;
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- Hand-written below this line (not generated by drizzle-kit).
|
||||||
|
-- updated_at triggers, same as every other core table.
|
||||||
|
DO $$
|
||||||
|
DECLARE t text;
|
||||||
|
BEGIN
|
||||||
|
IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
|
||||||
|
FOREACH t IN ARRAY ARRAY['areas','goals','tasks'] LOOP
|
||||||
|
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON tasks.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||||
|
END LOOP;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- Privacy: like health, the task tracker is readable/writable by the app role only
|
||||||
|
-- (no personal_ro grant). Grants only run on prod, where migrations run as `personal`.
|
||||||
|
REVOKE ALL ON SCHEMA tasks FROM PUBLIC;
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||||
|
GRANT USAGE ON SCHEMA tasks TO personal_app;
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA tasks TO personal_app;
|
||||||
|
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA tasks TO personal_app;
|
||||||
|
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA tasks
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||||
|
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA tasks
|
||||||
|
GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- Registered in meta.tables (structure only; the lists GUI only shows kind = 'list').
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||||
|
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||||
|
VALUES ('tasks','tasks','tasks-tasks','Tasks','core',true,true,1000),
|
||||||
|
('tasks','goals','tasks-goals','Goals','core',true,false,1000),
|
||||||
|
('tasks','areas','tasks-areas','Task areas','core',true,false,1000),
|
||||||
|
('tasks','activity','tasks-activity','Task activity','core',false,false,1000)
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
-- deploy-remote.sh runs meta.sync() as the app role after migrating.
|
||||||
|
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||||
|
PERFORM meta.sync();
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
14935
drizzle/20261024022836_tasks/snapshot.json
Normal file
14935
drizzle/20261024022836_tasks/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
29
drizzle/20261024024249_meta_edit_log/migration.sql
Normal file
29
drizzle/20261024024249_meta_edit_log/migration.sql
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
CREATE TABLE "meta"."edit_log" (
|
||||||
|
"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "meta"."edit_log_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
|
||||||
|
"at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||||
|
"user_id" text,
|
||||||
|
"user_email" text,
|
||||||
|
"schema_name" text NOT NULL,
|
||||||
|
"table_name" text NOT NULL,
|
||||||
|
"action" text NOT NULL,
|
||||||
|
"pk" jsonb,
|
||||||
|
"before" jsonb,
|
||||||
|
"after" jsonb,
|
||||||
|
CONSTRAINT "edit_log_action_check" CHECK (action IN ('insert','update','delete','import','add_column'))
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE INDEX "edit_log_table_at_index" ON "meta"."edit_log" ("schema_name","table_name","at");--> statement-breakpoint
|
||||||
|
-- Hand-written below this line (not generated by drizzle-kit).
|
||||||
|
-- The app role appends to and reads the log; nobody updates or deletes through the app.
|
||||||
|
-- personal_ro gets no access: before/after values can hold private (health) data.
|
||||||
|
-- Grants only run on prod (as `personal`); the dev DB is owned by personal_dev.
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||||
|
GRANT SELECT, INSERT ON meta.edit_log TO personal_app;
|
||||||
|
GRANT USAGE ON SEQUENCE meta.edit_log_id_seq TO personal_app;
|
||||||
|
END IF;
|
||||||
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||||
|
REVOKE ALL ON meta.edit_log FROM personal_ro;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
15132
drizzle/20261024024249_meta_edit_log/snapshot.json
Normal file
15132
drizzle/20261024024249_meta_edit_log/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
2
drizzle/20261024030132_tasks_ticktick/migration.sql
Normal file
2
drizzle/20261024030132_tasks_ticktick/migration.sql
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE "tasks"."tasks" ADD COLUMN "start_at" timestamp with time zone;--> statement-breakpoint
|
||||||
|
ALTER TABLE "tasks"."tasks" ADD COLUMN "time_zone" text;
|
||||||
15158
drizzle/20261024030132_tasks_ticktick/snapshot.json
Normal file
15158
drizzle/20261024030132_tasks_ticktick/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -34,6 +34,6 @@ export default [
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
ignores: ['build/', '.svelte-kit/', 'dist/']
|
ignores: ['build/', '.svelte-kit/', 'dist/', 'static/vendor/']
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -92,9 +92,11 @@
|
|||||||
"drizzle-orm": "^1.0.0-rc.4",
|
"drizzle-orm": "^1.0.0-rc.4",
|
||||||
"echarts": "^6.1.0",
|
"echarts": "^6.1.0",
|
||||||
"highlight.js": "^11.12.0",
|
"highlight.js": "^11.12.0",
|
||||||
|
"jose": "^6.2.10",
|
||||||
"marked": "^18.0.11",
|
"marked": "^18.0.11",
|
||||||
"marked-highlight": "^2.2.4",
|
"marked-highlight": "^2.2.4",
|
||||||
"mode-watcher": "^1.1.0",
|
"mode-watcher": "^1.1.0",
|
||||||
|
"pdfjs-dist": "^6.4.299",
|
||||||
"postgres": "^3.4.9",
|
"postgres": "^3.4.9",
|
||||||
"topojson-client": "^3.1.0",
|
"topojson-client": "^3.1.0",
|
||||||
"world-atlas": "^2.0.2",
|
"world-atlas": "^2.0.2",
|
||||||
|
|||||||
161
scripts/bank-link.ts
Normal file
161
scripts/bank-link.ts
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
// Link a bank to the money tracker through Enable Banking. Each bank gets its own session;
|
||||||
|
// linking one never touches another bank's session.
|
||||||
|
//
|
||||||
|
// bun --env-file=$HOME/.config/money-sync.env scripts/bank-link.ts Revolut (personal + joint)
|
||||||
|
// ... scripts/bank-link.ts --bank abn ABN AMRO -> "ABN Amro"
|
||||||
|
// ... scripts/bank-link.ts --bank knab Knab -> "Knab"
|
||||||
|
// ... scripts/bank-link.ts --check verify key + app id only
|
||||||
|
// ... scripts/bank-link.ts --code '<redirect URL or code>' [--yes] finish a started consent
|
||||||
|
// ... scripts/bank-link.ts --map <uid|IBAN>=<Tracker account> [--map ...] override the mapping
|
||||||
|
// ... scripts/bank-link.ts --aspsp '<ASPSP name>' --account <Tracker account> --country NL --user <id>
|
||||||
|
//
|
||||||
|
// Then sync within the hour: the first sync of an account asks for the longest history the
|
||||||
|
// bank serves, and most banks only serve ~90 days once consent is older than an hour.
|
||||||
|
//
|
||||||
|
// Needs ENABLEBANKING_APP_ID, ENABLEBANKING_KEY_PATH, DB_URL (and optionally
|
||||||
|
// ENABLEBANKING_REDIRECT_URL). Never prints the private key.
|
||||||
|
import { createInterface } from 'node:readline/promises';
|
||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
import postgres from 'postgres';
|
||||||
|
import {
|
||||||
|
configFromEnv,
|
||||||
|
createEnableBankingClient,
|
||||||
|
parseAuthCode
|
||||||
|
} from '../src/lib/server/enable-banking';
|
||||||
|
import { planAccountMapping, saveSession, trackerUserId } from '../src/lib/server/bank-sync';
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const flag = (name: string) => args.includes(name);
|
||||||
|
const value = (name: string) => {
|
||||||
|
const i = args.indexOf(name);
|
||||||
|
return i >= 0 ? args[i + 1] : undefined;
|
||||||
|
};
|
||||||
|
const overrides: Record<string, string> = {};
|
||||||
|
args.forEach((arg, i) => {
|
||||||
|
if (arg !== '--map') return;
|
||||||
|
const [key, ...rest] = (args[i + 1] ?? '').split('=');
|
||||||
|
if (!key) return;
|
||||||
|
overrides[key] = rest.join('=');
|
||||||
|
overrides[key.replace(/\s/g, '').toUpperCase()] = rest.join('=');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Enable Banking ASPSP names (GET /aspsps?country=NL) and the tracker account they feed.
|
||||||
|
const BANKS: Record<string, { aspsp: string; account: string; hint: string }> = {
|
||||||
|
revolut: {
|
||||||
|
aspsp: 'Revolut',
|
||||||
|
account: 'Revolut',
|
||||||
|
hint: 'Tick BOTH your personal account and the joint account with Juhee.'
|
||||||
|
},
|
||||||
|
abn: {
|
||||||
|
aspsp: 'ABN AMRO',
|
||||||
|
account: 'ABN Amro',
|
||||||
|
hint: 'Log in with the ABN AMRO app or identifier and approve your current account.'
|
||||||
|
},
|
||||||
|
knab: {
|
||||||
|
aspsp: 'Knab',
|
||||||
|
account: 'Knab',
|
||||||
|
hint: 'Approve your Knab current account (savings accounts are not needed).'
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const bankKey = (value('--bank') ?? 'revolut').toLowerCase().replace(/[\s-]|amro/g, '');
|
||||||
|
const preset = value('--aspsp') && !value('--bank') ? undefined : BANKS[bankKey];
|
||||||
|
if (!preset && !value('--aspsp'))
|
||||||
|
throw new Error(`Unknown --bank ${value('--bank')}; use ${Object.keys(BANKS).join(', ')}`);
|
||||||
|
const client = createEnableBankingClient(configFromEnv());
|
||||||
|
const aspsp = {
|
||||||
|
name: value('--aspsp') ?? preset!.aspsp,
|
||||||
|
country: value('--country') ?? 'NL'
|
||||||
|
};
|
||||||
|
const defaultAccount = value('--account') ?? preset?.account ?? aspsp.name;
|
||||||
|
const redirectUrl =
|
||||||
|
process.env.ENABLEBANKING_REDIRECT_URL ??
|
||||||
|
'https://stanrunge.dev/projects/money-tracker/bank-callback'; // registered with Enable Banking; 308s to /trackers/money/bank-callback with the query kept
|
||||||
|
|
||||||
|
const app = await client.application();
|
||||||
|
console.log(
|
||||||
|
`Enable Banking app "${app.name}" (${app.environment}, ${app.active ? 'active' : 'INACTIVE'}), services ${app.services.join(',')}`
|
||||||
|
);
|
||||||
|
const { aspsps } = await client.aspsps(aspsp.country);
|
||||||
|
const bank = aspsps.find((a) => a.name.toLowerCase() === aspsp.name.toLowerCase());
|
||||||
|
if (!bank) throw new Error(`${aspsp.name} (${aspsp.country}) is not available for this app`);
|
||||||
|
console.log(
|
||||||
|
`${bank.name} ${bank.country}${bank.bic ? ' ' + bank.bic : ''}: max consent ` +
|
||||||
|
`${Math.floor(bank.maximum_consent_validity / 86400)} days${bank.beta ? ' (beta)' : ''}` +
|
||||||
|
(bank.required_psu_headers?.length
|
||||||
|
? `; requires PSU headers ${bank.required_psu_headers.join(', ')} (background syncs ` +
|
||||||
|
'run as "PSU not present", which PSD2 limits to ~4 calls a day per account)'
|
||||||
|
: '')
|
||||||
|
);
|
||||||
|
console.log(
|
||||||
|
`Linked accounts go to tracker account "${defaultAccount}" unless --map says otherwise.`
|
||||||
|
);
|
||||||
|
if (flag('--check')) process.exit(0);
|
||||||
|
|
||||||
|
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
||||||
|
let pasted = value('--code');
|
||||||
|
if (!pasted) {
|
||||||
|
// Stay one hour under the bank's maximum so the API never rejects valid_until.
|
||||||
|
const validUntil = new Date(Date.now() + (bank.maximum_consent_validity - 3600) * 1000);
|
||||||
|
const auth = await client.startAuth({
|
||||||
|
aspsp: { name: bank.name, country: bank.country },
|
||||||
|
redirectUrl,
|
||||||
|
validUntil,
|
||||||
|
psuType: 'personal',
|
||||||
|
state: randomUUID()
|
||||||
|
});
|
||||||
|
console.log(`\n1. Open this link and approve access at ${bank.name}.`);
|
||||||
|
console.log(` ${preset?.hint ?? 'Approve the account(s) you want to track.'}\n`);
|
||||||
|
console.log(' ' + auth.url + '\n');
|
||||||
|
console.log(`2. You'll land on ${redirectUrl}?code=...`);
|
||||||
|
console.log(
|
||||||
|
' Copy the full address from the browser bar (or just the code) and paste it here.'
|
||||||
|
);
|
||||||
|
console.log(' The code expires quickly, so paste it within a few minutes.\n');
|
||||||
|
pasted = await rl.question('Redirect URL or code: ');
|
||||||
|
}
|
||||||
|
const parsed = parseAuthCode(pasted ?? '');
|
||||||
|
if (parsed.error) throw new Error(`Consent failed: ${parsed.error}`);
|
||||||
|
if (!parsed.code) throw new Error('No authorisation code found in what was pasted');
|
||||||
|
|
||||||
|
const session = await client.createSession(parsed.code);
|
||||||
|
console.log(
|
||||||
|
`\nSession authorised until ${session.access.valid_until}; ${session.accounts.length} account(s):`
|
||||||
|
);
|
||||||
|
|
||||||
|
const sql = postgres(process.env.DB_URL!, { max: 1, onnotice: () => {} });
|
||||||
|
try {
|
||||||
|
const userId = await trackerUserId(sql, value('--user'));
|
||||||
|
const tracker = await sql<
|
||||||
|
{ id: number; name: string; scope: string; iban: string | null; external_uid: string | null }[]
|
||||||
|
>`select id, name, scope, iban, external_uid from money.accounts where user_id = ${userId} order by id`;
|
||||||
|
const plan = planAccountMapping(session.accounts, tracker, overrides, {
|
||||||
|
defaultAccount,
|
||||||
|
joint: bank.name.toLowerCase() === 'revolut'
|
||||||
|
});
|
||||||
|
for (const { bank: account, tracker: target, reason } of plan) {
|
||||||
|
const iban = account.account_id?.iban ?? '(no IBAN)';
|
||||||
|
console.log(
|
||||||
|
` ${account.uid} ${iban} ${account.currency ?? ''} ${account.name ?? ''}${account.details ? ' / ' + account.details : ''}` +
|
||||||
|
`\n -> ${target ? target.name : 'not linked'} (${reason})`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const answer = flag('--yes') ? 'y' : await rl.question('\nSave this session and mapping? [y/N] ');
|
||||||
|
if (!/^y(es)?$/i.test(answer.trim())) {
|
||||||
|
console.log('Nothing saved. Re-run with --code is not possible (codes are single-use);');
|
||||||
|
console.log('start again and use --map <uid>=<Tracker account> to fix the mapping.');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
await saveSession(sql, userId, session, plan);
|
||||||
|
const linked = plan.filter((p) => p.tracker).map((p) => p.tracker!.name);
|
||||||
|
const only = linked.length === 1 ? ` --account '${linked[0]}'` : '';
|
||||||
|
console.log('Saved. Within the next hour (full history is only served right after consent):');
|
||||||
|
console.log(
|
||||||
|
` bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile --dry-run${only}`
|
||||||
|
);
|
||||||
|
console.log(
|
||||||
|
` bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile${only}`
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
rl.close();
|
||||||
|
await sql.end();
|
||||||
|
}
|
||||||
53
scripts/bank-sync.ts
Normal file
53
scripts/bank-sync.ts
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
// Pull booked bank transactions into money.transactions (Enable Banking).
|
||||||
|
//
|
||||||
|
// bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts [options]
|
||||||
|
// --dry-run fetch and report, write nothing
|
||||||
|
// --reconcile match against ALL hand-entered rows (account + amount + date ±1 day)
|
||||||
|
// and attach the bank id to them instead of inserting duplicates;
|
||||||
|
// run once after linking (with --dry-run first)
|
||||||
|
// --from YYYY-MM-DD override the start date (default: last synced date - 7 days, or the
|
||||||
|
// account's first transaction on the first run)
|
||||||
|
// --full ask for the longest history the bank serves (automatic on an
|
||||||
|
// account's first sync; only works within an hour of consent at most banks)
|
||||||
|
// --account <name> only this tracker account
|
||||||
|
//
|
||||||
|
// Dedupe: unique (account_id, external_id); re-running is always safe.
|
||||||
|
import postgres from 'postgres';
|
||||||
|
import { configFromEnv, createEnableBankingClient } from '../src/lib/server/enable-banking';
|
||||||
|
import { syncBankAccounts } from '../src/lib/server/bank-sync';
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const value = (name: string) => {
|
||||||
|
const i = args.indexOf(name);
|
||||||
|
return i >= 0 ? args[i + 1] : undefined;
|
||||||
|
};
|
||||||
|
const from = value('--from');
|
||||||
|
if (from && !/^\d{4}-\d{2}-\d{2}$/.test(from)) throw new Error('--from must be YYYY-MM-DD');
|
||||||
|
if (!process.env.DB_URL) throw new Error('DB_URL is not set');
|
||||||
|
|
||||||
|
const client = createEnableBankingClient(configFromEnv());
|
||||||
|
const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
||||||
|
const dryRun = args.includes('--dry-run');
|
||||||
|
try {
|
||||||
|
const results = await syncBankAccounts(sql, client, {
|
||||||
|
dryRun,
|
||||||
|
reconcile: args.includes('--reconcile'),
|
||||||
|
from,
|
||||||
|
full: args.includes('--full'),
|
||||||
|
accountName: value('--account')
|
||||||
|
});
|
||||||
|
for (const r of results) {
|
||||||
|
console.log(
|
||||||
|
`${r.account}: fetched ${r.fetched}, booked ${r.booked}, new ${r.inserted}, ` +
|
||||||
|
`matched to existing ${r.matched}, replaced sum entries ${r.absorbed}, ` +
|
||||||
|
`already synced ${r.alreadyKnown}` +
|
||||||
|
(r.historyFrom ? `, history from ${r.historyFrom}` : '') +
|
||||||
|
(r.balance ? `, bank balance ${r.balance}` : '') +
|
||||||
|
(r.error ? `\n ERROR ${r.error}` : '')
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (dryRun) console.log('(dry run: nothing written)');
|
||||||
|
if (results.some((r) => r.error)) process.exitCode = 1;
|
||||||
|
} finally {
|
||||||
|
await sql.end();
|
||||||
|
}
|
||||||
952
scripts/books-ingest.py
Normal file
952
scripts/books-ingest.py
Normal file
@@ -0,0 +1,952 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Books ingest + metadata enrichment.
|
||||||
|
|
||||||
|
Builds books.works / books.files from:
|
||||||
|
* lists.books rows (the old GUI list) -> placeholders
|
||||||
|
* Obsidian "Book links.md" md5s resolved via the libgen JSON API -> placeholders
|
||||||
|
* ebook files dropped in $BOOKS_ROOT/inbox (sha256-deduped) -> works + files
|
||||||
|
|
||||||
|
Metadata: embedded (EPUB OPF / PDF info + ISBN scan) -> Open Library -> Google Books
|
||||||
|
(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/<sha1(cover_url)[:16]>.jpg (stable across dev/prod).
|
||||||
|
Files are moved to $BOOKS_ROOT/library/<Author>/<Title>/.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
DB_URL=postgresql://... BOOKS_ROOT=~/books \
|
||||||
|
python scripts/books-ingest.py --obsidian-json resolved.json [--reset] [--dry-run]
|
||||||
|
|
||||||
|
Network calls are cached in $BOOKS_ROOT/.cache/http so reruns are cheap and polite.
|
||||||
|
Refuses to run against the prod database name ('personal') unless --allow-prod.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse, difflib, hashlib, html, json, os, re, shutil, sys, time, unicodedata, urllib.parse, urllib.request, zipfile
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from xml.etree import ElementTree as ET
|
||||||
|
|
||||||
|
def log(*a):
|
||||||
|
print(*a, flush=True)
|
||||||
|
|
||||||
|
UA = "stan-books-ingest/0.1 (personal library catalogue; low volume)"
|
||||||
|
LANG = {"eng": "en", "dut": "nl", "nld": "nl", "ger": "de", "deu": "de", "fre": "fr", "fra": "fr", "spa": "es", "ita": "it",
|
||||||
|
"por": "pt", "rus": "ru", "jpn": "ja", "kor": "ko", "chi": "zh", "zho": "zh", "lat": "la", "gre": "el", "swe": "sv"}
|
||||||
|
BOOK_EXT = {".pdf", ".epub", ".mobi", ".azw3", ".azw", ".lit", ".djvu", ".docx", ".fb2", ".cbz"}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- helpers
|
||||||
|
|
||||||
|
def strip_accents(s: str) -> str:
|
||||||
|
s = (s or "").replace("'", "").replace("’", "")
|
||||||
|
return "".join(c for c in unicodedata.normalize("NFKD", s) if not unicodedata.combining(c))
|
||||||
|
|
||||||
|
def norm_title(t: str) -> str:
|
||||||
|
t = strip_accents(html.unescape(t or "")).lower().replace("&", " and ")
|
||||||
|
t = re.split(r"\s*[:(\[]\s*|\s+[-–—]\s+|_\s", t, maxsplit=1)[0] # main title only
|
||||||
|
t = re.sub(r"[^a-z0-9]+", " ", t).strip()
|
||||||
|
t = re.sub(r"^(the|a|an|de|het|een)\s+", "", t)
|
||||||
|
t = re.sub(r"\s+(\d+(st|nd|rd|th)?\s+edition|ed)$", "", t)
|
||||||
|
return t
|
||||||
|
|
||||||
|
def full_norm(t: str) -> str:
|
||||||
|
return re.sub(r"[^a-z0-9]+", " ", strip_accents(t or "").lower()).strip()
|
||||||
|
|
||||||
|
def sim(a: str, b: str) -> float:
|
||||||
|
na, nb = norm_title(a), norm_title(b)
|
||||||
|
a, b = full_norm(a), full_norm(b)
|
||||||
|
if not a or not b:
|
||||||
|
return 0.0
|
||||||
|
if a == b:
|
||||||
|
return 1.0
|
||||||
|
# volume / part / edition markers must agree ("Finance I" vs "Finance II", "Part 1" vs "Part 2")
|
||||||
|
marks = lambda x: {t for t in x.split() if re.fullmatch(r"\d{1,2}|[ivx]{1,4}", t)}
|
||||||
|
if marks(a) != marks(b) and marks(a) and marks(b):
|
||||||
|
return 0.5
|
||||||
|
r = difflib.SequenceMatcher(None, a, b).ratio()
|
||||||
|
if na and na == nb and len(na) >= 15:
|
||||||
|
r = max(r, 0.93)
|
||||||
|
return r
|
||||||
|
|
||||||
|
def surnames(authors: list[str]) -> set[str]:
|
||||||
|
out = set()
|
||||||
|
for a in authors or []:
|
||||||
|
a = strip_accents(a).lower()
|
||||||
|
if "," in a: # "King, Michael"
|
||||||
|
a = a.split(",")[0]
|
||||||
|
parts = re.findall(r"[a-z]{3,}", a)
|
||||||
|
if parts:
|
||||||
|
out.add(parts[-1] if "," not in a else parts[0])
|
||||||
|
out.update(parts)
|
||||||
|
return out
|
||||||
|
|
||||||
|
JUNK_AUTHORS = {"admin", "administrator", "user", "owner", "unknown", "oreily", "oreilly", "o'reilly", "author", "john", "mr2"}
|
||||||
|
|
||||||
|
def clean_authors(names) -> list[str]:
|
||||||
|
out = []
|
||||||
|
for n in names or []:
|
||||||
|
n = re.sub(r"\s+", " ", (n or "").strip(" ,;"))
|
||||||
|
low = n.lower()
|
||||||
|
if not n or low in JUNK_AUTHORS or re.search(r"\d|@|\.(com|org|net)\b", n) or " " not in n and "," not in n and "." not in n:
|
||||||
|
continue
|
||||||
|
if n.isupper():
|
||||||
|
n = n.title()
|
||||||
|
out.append(n)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def authors_compatible(a: list[str], b: list[str]) -> bool:
|
||||||
|
if not a or not b:
|
||||||
|
return True
|
||||||
|
return bool(surnames(a) & surnames(b))
|
||||||
|
|
||||||
|
def isbn_ok(s: str) -> str | None:
|
||||||
|
s = re.sub(r"[^0-9Xx]", "", s or "").upper()
|
||||||
|
if len(s) == 13 and s.isdigit() and s[:3] in ("978", "979"):
|
||||||
|
tot = sum(int(c) * (1 if i % 2 == 0 else 3) for i, c in enumerate(s[:12]))
|
||||||
|
return s if (10 - tot % 10) % 10 == int(s[12]) else None
|
||||||
|
if len(s) == 10 and s[:9].isdigit():
|
||||||
|
tot = sum((10 - i) * int(c) for i, c in enumerate(s[:9]))
|
||||||
|
chk = (11 - tot % 11) % 11
|
||||||
|
return s if (("X" if chk == 10 else str(chk)) == s[9]) else None
|
||||||
|
return None
|
||||||
|
|
||||||
|
def to13(s: str) -> str | None:
|
||||||
|
s = isbn_ok(s)
|
||||||
|
if not s:
|
||||||
|
return None
|
||||||
|
if len(s) == 13:
|
||||||
|
return s
|
||||||
|
core = "978" + s[:9]
|
||||||
|
tot = sum(int(c) * (1 if i % 2 == 0 else 3) for i, c in enumerate(core))
|
||||||
|
return core + str((10 - tot % 10) % 10)
|
||||||
|
|
||||||
|
def to10(s13: str) -> str | None:
|
||||||
|
if not s13 or not s13.startswith("978"):
|
||||||
|
return None
|
||||||
|
core = s13[3:12]
|
||||||
|
tot = sum((10 - i) * int(c) for i, c in enumerate(core))
|
||||||
|
chk = (11 - tot % 11) % 11
|
||||||
|
return core + ("X" if chk == 10 else str(chk))
|
||||||
|
|
||||||
|
ISBN_RE = re.compile(r"ISBN(?:-1[03])?[:\s]*((?:97[89][\s-]?)?(?:\d[\s-]?){9}[\dXx])", re.I)
|
||||||
|
|
||||||
|
def find_isbns(text: str) -> list[str]:
|
||||||
|
out = []
|
||||||
|
for m in ISBN_RE.finditer(text or ""):
|
||||||
|
i = to13(m.group(1))
|
||||||
|
if i and i not in out:
|
||||||
|
out.append(i)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def safe_name(s: str, n: int = 80) -> str:
|
||||||
|
s = strip_accents(s or "").replace("/", "-").replace("\\", "-")
|
||||||
|
s = re.sub(r'[<>:"|?*\x00-\x1f]', "", s)
|
||||||
|
s = re.sub(r"\s+", " ", s).strip(" .")
|
||||||
|
return (s[:n].rstrip(" .") or "Untitled")
|
||||||
|
|
||||||
|
def year_of(v) -> int | None:
|
||||||
|
m = re.search(r"(1[5-9]\d\d|20\d\d)", str(v or ""))
|
||||||
|
return int(m.group(1)) if m else None
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- http with cache
|
||||||
|
|
||||||
|
class Http:
|
||||||
|
def __init__(self, cache: Path, delay: float = 1.0):
|
||||||
|
self.cache, self.delay, self.last = cache, delay, 0.0
|
||||||
|
cache.mkdir(parents=True, exist_ok=True)
|
||||||
|
self.calls = 0
|
||||||
|
self.dead_hosts: set[str] = set()
|
||||||
|
|
||||||
|
def _get(self, url: str, binary=False):
|
||||||
|
host = urllib.parse.urlparse(url).netloc
|
||||||
|
if host in self.dead_hosts: # daily quota hit (Google Books): stop hammering it this run
|
||||||
|
return 429, b""
|
||||||
|
wait = self.delay - (time.time() - self.last)
|
||||||
|
if wait > 0:
|
||||||
|
time.sleep(wait)
|
||||||
|
self.last = time.time()
|
||||||
|
self.calls += 1
|
||||||
|
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "*/*"})
|
||||||
|
for attempt in range(3):
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||||||
|
return r.status, r.read()
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code == 429 and "googleapis" in host:
|
||||||
|
self.dead_hosts.add(host)
|
||||||
|
log(f" {host}: quota exhausted (429), skipping it for the rest of this run")
|
||||||
|
return 429, b""
|
||||||
|
if e.code in (429, 503) and attempt < 2:
|
||||||
|
time.sleep(5 * (attempt + 1))
|
||||||
|
continue
|
||||||
|
return e.code, b""
|
||||||
|
except Exception:
|
||||||
|
if attempt < 2:
|
||||||
|
time.sleep(3)
|
||||||
|
continue
|
||||||
|
return 0, b""
|
||||||
|
return 0, b""
|
||||||
|
|
||||||
|
def json(self, url: str):
|
||||||
|
key = self.cache / (hashlib.sha1(url.encode()).hexdigest() + ".json")
|
||||||
|
if key.exists():
|
||||||
|
return json.loads(key.read_text())
|
||||||
|
status, body = self._get(url)
|
||||||
|
data = None
|
||||||
|
if status == 200:
|
||||||
|
try:
|
||||||
|
data = json.loads(body)
|
||||||
|
except Exception:
|
||||||
|
data = None
|
||||||
|
if status in (200, 404): # don't cache transient errors
|
||||||
|
key.write_text(json.dumps(data))
|
||||||
|
return data
|
||||||
|
|
||||||
|
def text(self, url: str):
|
||||||
|
key = self.cache / (hashlib.sha1(url.encode()).hexdigest() + ".txt")
|
||||||
|
if key.exists():
|
||||||
|
return key.read_text()
|
||||||
|
status, body = self._get(url)
|
||||||
|
if status == 200:
|
||||||
|
key.write_text(body.decode("utf-8", "replace"))
|
||||||
|
return body.decode("utf-8", "replace")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def download(self, url: str, dest: Path) -> bool:
|
||||||
|
status, body = self._get(url, binary=True)
|
||||||
|
if status == 200 and len(body) > 1500: # skip 1x1 placeholders
|
||||||
|
dest.write_bytes(body)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- model
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Work:
|
||||||
|
title: str
|
||||||
|
authors: list[str] = field(default_factory=list)
|
||||||
|
subtitle: str | None = None
|
||||||
|
isbns: list[str] = field(default_factory=list) # isbn13s, best first
|
||||||
|
publisher: str | None = None
|
||||||
|
year: int | None = None
|
||||||
|
language: str | None = None
|
||||||
|
page_count: int | None = None
|
||||||
|
description: str | None = None
|
||||||
|
tags: list[str] = field(default_factory=list)
|
||||||
|
cover_url: str | None = None
|
||||||
|
cover_path: str | None = None
|
||||||
|
ol_work: str | None = None
|
||||||
|
ol_edition: str | None = None
|
||||||
|
gb_id: str | None = None
|
||||||
|
md5s: list[str] = field(default_factory=list)
|
||||||
|
refs: dict = field(default_factory=dict)
|
||||||
|
status: str = "pending"
|
||||||
|
source: str | None = None
|
||||||
|
notes: str | None = None
|
||||||
|
files: list = field(default_factory=list)
|
||||||
|
shelves: set = field(default_factory=set)
|
||||||
|
origins: set = field(default_factory=set)
|
||||||
|
db_id: int | None = None
|
||||||
|
|
||||||
|
def add_ref(self, k, v):
|
||||||
|
self.refs.setdefault(k, [])
|
||||||
|
if v not in self.refs[k]:
|
||||||
|
self.refs[k].append(v)
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FileRec:
|
||||||
|
path: Path
|
||||||
|
rel_inbox: str
|
||||||
|
sha256: str
|
||||||
|
md5: str
|
||||||
|
size: int
|
||||||
|
fmt: str
|
||||||
|
title: str | None = None
|
||||||
|
authors: list[str] = field(default_factory=list)
|
||||||
|
isbns: list[str] = field(default_factory=list)
|
||||||
|
year: int | None = None
|
||||||
|
publisher: str | None = None
|
||||||
|
language: str | None = None
|
||||||
|
pages: int | None = None
|
||||||
|
is_paper: bool = False
|
||||||
|
broken: str | None = None
|
||||||
|
dest_rel: str | None = None
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- embedded metadata
|
||||||
|
|
||||||
|
def epub_meta(p: Path) -> dict:
|
||||||
|
out: dict = {}
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(p) as z:
|
||||||
|
cont = ET.fromstring(z.read("META-INF/container.xml"))
|
||||||
|
opf_path = next(e.get("full-path") for e in cont.iter() if e.tag.endswith("rootfile"))
|
||||||
|
opf = ET.fromstring(z.read(opf_path))
|
||||||
|
md = next((e for e in opf.iter() if e.tag.endswith("metadata")), None)
|
||||||
|
if md is not None:
|
||||||
|
for e in md:
|
||||||
|
tag = e.tag.split("}")[-1]
|
||||||
|
txt = (e.text or "").strip()
|
||||||
|
if not txt:
|
||||||
|
continue
|
||||||
|
if tag == "title" and "title" not in out:
|
||||||
|
out["title"] = txt
|
||||||
|
elif tag == "creator":
|
||||||
|
out.setdefault("authors", []).append(txt)
|
||||||
|
elif tag == "identifier":
|
||||||
|
i = to13(txt.replace("urn:isbn:", ""))
|
||||||
|
if i:
|
||||||
|
out.setdefault("isbns", []).append(i)
|
||||||
|
elif tag == "publisher":
|
||||||
|
out.setdefault("publisher", txt)
|
||||||
|
elif tag == "date":
|
||||||
|
out.setdefault("year", year_of(txt))
|
||||||
|
elif tag == "language":
|
||||||
|
out.setdefault("language", txt[:2].lower())
|
||||||
|
# scan the first few content docs (copyright page) for ISBNs
|
||||||
|
names = [n for n in z.namelist() if n.lower().endswith((".xhtml", ".html", ".htm"))][:12]
|
||||||
|
for n in names:
|
||||||
|
for i in find_isbns(re.sub(r"<[^>]+>", " ", z.read(n).decode("utf-8", "ignore"))):
|
||||||
|
out.setdefault("isbns", [])
|
||||||
|
if i not in out["isbns"]:
|
||||||
|
out["isbns"].append(i)
|
||||||
|
except Exception as e: # noqa
|
||||||
|
out["error"] = f"epub: {e.__class__.__name__}"
|
||||||
|
return out
|
||||||
|
|
||||||
|
def pdf_meta(p: Path) -> dict:
|
||||||
|
out: dict = {}
|
||||||
|
try:
|
||||||
|
from pypdf import PdfReader
|
||||||
|
import logging
|
||||||
|
logging.getLogger("pypdf").setLevel(logging.ERROR)
|
||||||
|
r = PdfReader(str(p), strict=False)
|
||||||
|
out["pages"] = len(r.pages)
|
||||||
|
info = r.metadata or {}
|
||||||
|
t = (info.get("/Title") or "").strip() if info else ""
|
||||||
|
a = (info.get("/Author") or "").strip() if info else ""
|
||||||
|
if t:
|
||||||
|
out["title"] = t
|
||||||
|
if a:
|
||||||
|
out["authors"] = clean_authors(re.split(r";|\band\b|&", a))
|
||||||
|
text = ""
|
||||||
|
for i in range(min(len(r.pages), 10)):
|
||||||
|
try:
|
||||||
|
text += (r.pages[i].extract_text() or "") + "\n"
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
out["isbns"] = find_isbns(text)
|
||||||
|
if re.search(r"arXiv:\d{4}\.\d{4,5}", text):
|
||||||
|
out["arxiv"] = re.search(r"arXiv:(\d{4}\.\d{4,5})", text).group(1)
|
||||||
|
except Exception as e: # noqa
|
||||||
|
out["error"] = f"pdf: {e.__class__.__name__}"
|
||||||
|
return out
|
||||||
|
|
||||||
|
ANNAS_RE = re.compile(r"^(?P<title>.+?) -- (?P<author>.+?) -- (?P<rest>.+?) -- (?P<md5>[0-9a-f]{32}) -- Anna’s Archive$")
|
||||||
|
|
||||||
|
def filename_meta(stem: str) -> dict:
|
||||||
|
out: dict = {}
|
||||||
|
m = ANNAS_RE.match(stem)
|
||||||
|
if m:
|
||||||
|
out["title"] = m["title"].replace("_ ", ": ").replace("_", " ")
|
||||||
|
out["authors"] = [a.strip().replace("_", ".") for a in m["author"].split(",") if a.strip()]
|
||||||
|
out["md5"] = m["md5"]
|
||||||
|
for part in m["rest"].split(" -- "):
|
||||||
|
i = to13(part)
|
||||||
|
if i:
|
||||||
|
out.setdefault("isbns", []).append(i)
|
||||||
|
elif year_of(part) and "year" not in out:
|
||||||
|
out["year"] = year_of(part)
|
||||||
|
return out
|
||||||
|
s = re.sub(r"\s*-\s*libgen\.(li|rs|is)$", "", stem)
|
||||||
|
s = re.sub(r"\s*\((z-library|z-lib\.org|1lib\.\w+)\)", "", s, flags=re.I)
|
||||||
|
s = re.sub(r"\s+(penguin (modern )?classics)$", "", s, flags=re.I)
|
||||||
|
s = re.sub(r"\s+0$", "", s)
|
||||||
|
mm = re.match(r"^([^-]+?) : ([^-]+?) - (.+)$", s) # "A : B - Title" (libgen multi-author)
|
||||||
|
if mm:
|
||||||
|
out["authors"] = [mm[1].strip(), mm[2].strip()]
|
||||||
|
s = mm[3]
|
||||||
|
pm = re.search(r"\s*\((\d{4}),\s*([^)]+)\)\s*$", s)
|
||||||
|
if pm:
|
||||||
|
out["year"], out["publisher"] = int(pm[1]), pm[2].strip()
|
||||||
|
s = s[: pm.start()]
|
||||||
|
# "Author - Title" (only when the left part looks like a person name)
|
||||||
|
mm = re.match(r"^([A-Z][a-z]+(?: [A-Z][a-z]+){1,2}) - (.+)$", s)
|
||||||
|
if mm:
|
||||||
|
out["authors"] = [mm[1]]
|
||||||
|
s = mm[2]
|
||||||
|
mm = re.match(r"^(.+?) by ([A-Z][\w.]+(?: [A-Z][\w.]+){1,2})$", s)
|
||||||
|
if mm:
|
||||||
|
s, out["authors"] = mm[1], [mm[2]]
|
||||||
|
if s.count("-") >= 4 and " " not in s: # The-Lean-Startup_-How-...-Eric-Ries-2011-Crown-Business-030788791X-
|
||||||
|
for tok in s.split("-"):
|
||||||
|
i = to13(tok)
|
||||||
|
if i:
|
||||||
|
out.setdefault("isbns", []).append(i)
|
||||||
|
s = s.split("_")[0].replace("-", " ")
|
||||||
|
s = s.replace("_ ", ": ").replace("_", " ")
|
||||||
|
if s.count(".") >= 3 and s.count(" ") <= 1: # Sipser_Introduction.to.the.Theory.of.Computation.3E
|
||||||
|
s = s.replace(".", " ")
|
||||||
|
s = re.sub(r"\s+", " ", s).strip(" -")
|
||||||
|
out["title"] = s
|
||||||
|
return out
|
||||||
|
|
||||||
|
def good_embedded_title(t: str | None, fname_title: str) -> bool:
|
||||||
|
if not t or len(t) < 4:
|
||||||
|
return False
|
||||||
|
bad = ("untitled", "microsoft word", ".doc", ".pdf", ".indd", "layout 1", "unknown", "title", "cover")
|
||||||
|
tl = t.lower()
|
||||||
|
if any(b in tl for b in bad) or re.fullmatch(r"[\w-]+\.\w+", t) or re.fullmatch(r"[0-9a-f\-_ ]+", tl):
|
||||||
|
return False
|
||||||
|
return sim(t, fname_title) > 0.55
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- enrichment
|
||||||
|
|
||||||
|
class Enricher:
|
||||||
|
def __init__(self, http: Http):
|
||||||
|
self.h = http
|
||||||
|
|
||||||
|
def ol_isbn(self, isbn: str):
|
||||||
|
ed = self.h.json(f"https://openlibrary.org/isbn/{isbn}.json")
|
||||||
|
if not ed or "title" not in ed:
|
||||||
|
return None
|
||||||
|
rec = {"title": ed.get("title"), "subtitle": ed.get("subtitle"), "publisher": (ed.get("publishers") or [None])[0],
|
||||||
|
"year": year_of(ed.get("publish_date")), "pages": ed.get("number_of_pages"),
|
||||||
|
"ol_edition": ed.get("key", "").split("/")[-1] or None, "isbn": isbn,
|
||||||
|
"cover": f"https://covers.openlibrary.org/b/id/{ed['covers'][0]}-L.jpg" if ed.get("covers") and ed["covers"][0] > 0 else None,
|
||||||
|
"lang": LANG.get(((ed.get("languages") or [{}])[0].get("key") or "").split("/")[-1])}
|
||||||
|
wk = (ed.get("works") or [{}])[0].get("key")
|
||||||
|
if wk:
|
||||||
|
w = self.h.json(f"https://openlibrary.org{wk}.json") or {}
|
||||||
|
rec["ol_work"] = wk.split("/")[-1]
|
||||||
|
d = w.get("description")
|
||||||
|
rec["description"] = d.get("value") if isinstance(d, dict) else d
|
||||||
|
rec["subjects"] = (w.get("subjects") or [])[:8]
|
||||||
|
names = []
|
||||||
|
for a in (w.get("authors") or [])[:4]:
|
||||||
|
k = (a.get("author") or {}).get("key")
|
||||||
|
if k:
|
||||||
|
aj = self.h.json(f"https://openlibrary.org{k}.json") or {}
|
||||||
|
if aj.get("name"):
|
||||||
|
names.append(aj["name"])
|
||||||
|
rec["authors"] = names
|
||||||
|
if not rec["cover"] and w.get("covers") and w["covers"][0] > 0:
|
||||||
|
rec["cover"] = f"https://covers.openlibrary.org/b/id/{w['covers'][0]}-L.jpg"
|
||||||
|
return rec
|
||||||
|
|
||||||
|
def ol_search(self, title: str, authors: list[str]):
|
||||||
|
q = {"title": norm_title(title) if len(norm_title(title)) > 3 else title, "limit": 8,
|
||||||
|
"fields": "key,title,subtitle,author_name,first_publish_year,isbn,cover_i,cover_edition_key,language,number_of_pages_median,publisher,subject"}
|
||||||
|
if authors:
|
||||||
|
q["author"] = re.sub(r"[^\w\s.-]", " ", authors[0])
|
||||||
|
d = self.h.json("https://openlibrary.org/search.json?" + urllib.parse.urlencode(q))
|
||||||
|
docs = (d or {}).get("docs") or []
|
||||||
|
if not docs and authors:
|
||||||
|
q.pop("author")
|
||||||
|
d = self.h.json("https://openlibrary.org/search.json?" + urllib.parse.urlencode(q))
|
||||||
|
docs = (d or {}).get("docs") or []
|
||||||
|
best, bs = None, 0.0
|
||||||
|
for doc in docs:
|
||||||
|
t = doc.get("title", "")
|
||||||
|
s = max(sim(title, t), sim(title, f"{t}: {doc.get('subtitle', '')}"))
|
||||||
|
if norm_title(title) and norm_title(title) == norm_title(t) and (
|
||||||
|
authors and doc.get("author_name") and authors_compatible(authors, doc["author_name"]) or len(norm_title(t)) >= 15):
|
||||||
|
s = max(s, 0.92)
|
||||||
|
if authors and doc.get("author_name"):
|
||||||
|
s += 0.05 if authors_compatible(authors, doc["author_name"]) else -0.25
|
||||||
|
if s > bs:
|
||||||
|
best, bs = doc, s
|
||||||
|
if not best:
|
||||||
|
return None, 0.0
|
||||||
|
isbns = [to13(i) for i in (best.get("isbn") or [])]
|
||||||
|
isbns = [i for i in isbns if i]
|
||||||
|
rec = {"title": best.get("title"), "subtitle": best.get("subtitle"), "authors": best.get("author_name") or [],
|
||||||
|
"year": best.get("first_publish_year"), "ol_work": best.get("key", "").split("/")[-1] or None,
|
||||||
|
"ol_edition": best.get("cover_edition_key"), "pages": best.get("number_of_pages_median"),
|
||||||
|
"publisher": (best.get("publisher") or [None])[0],
|
||||||
|
"cover": f"https://covers.openlibrary.org/b/id/{best['cover_i']}-L.jpg" if best.get("cover_i") else None,
|
||||||
|
"lang": LANG.get((best.get("language") or [""])[0]), "subjects": (best.get("subject") or [])[:8],
|
||||||
|
"isbn": next((i for i in isbns if i.startswith("978")), isbns[0] if isbns else None)}
|
||||||
|
if rec["ol_work"]:
|
||||||
|
w = self.h.json(f"https://openlibrary.org/works/{rec['ol_work']}.json") or {}
|
||||||
|
dsc = w.get("description")
|
||||||
|
rec["description"] = dsc.get("value") if isinstance(dsc, dict) else dsc
|
||||||
|
return rec, min(bs, 1.0)
|
||||||
|
|
||||||
|
def gb(self, title: str | None, authors: list[str], isbn: str | None = None):
|
||||||
|
if isbn:
|
||||||
|
q = f"isbn:{isbn}"
|
||||||
|
else:
|
||||||
|
q = f'intitle:"{norm_title(title) or title}"' + (f" inauthor:{surnames(authors[:1]).pop()}" if authors and surnames(authors[:1]) else "")
|
||||||
|
d = self.h.json("https://www.googleapis.com/books/v1/volumes?" + urllib.parse.urlencode({"q": q, "maxResults": 8, "printType": "books"}))
|
||||||
|
items = (d or {}).get("items") or []
|
||||||
|
best, bs = None, 0.0
|
||||||
|
for it in items:
|
||||||
|
v = it.get("volumeInfo", {})
|
||||||
|
if isbn:
|
||||||
|
s = 1.0
|
||||||
|
else:
|
||||||
|
s = max(sim(title, v.get("title", "")), sim(title, f"{v.get('title', '')}: {v.get('subtitle', '')}"))
|
||||||
|
if norm_title(title) and norm_title(title) == norm_title(v.get("title", "")) and (
|
||||||
|
authors and v.get("authors") and authors_compatible(authors, v["authors"]) or len(norm_title(title)) >= 15):
|
||||||
|
s = max(s, 0.92)
|
||||||
|
if authors and v.get("authors"):
|
||||||
|
s += 0.05 if authors_compatible(authors, v["authors"]) else -0.25
|
||||||
|
if s > bs:
|
||||||
|
best, bs = it, s
|
||||||
|
if isbn:
|
||||||
|
break
|
||||||
|
if not best:
|
||||||
|
return None, 0.0
|
||||||
|
v = best["volumeInfo"]
|
||||||
|
ids = {x.get("type"): x.get("identifier") for x in v.get("industryIdentifiers", [])}
|
||||||
|
img = (v.get("imageLinks") or {})
|
||||||
|
cover = img.get("extraLarge") or img.get("large") or img.get("medium") or img.get("thumbnail")
|
||||||
|
if cover:
|
||||||
|
cover = cover.replace("http://", "https://").replace("&edge=curl", "")
|
||||||
|
return {"title": v.get("title"), "subtitle": v.get("subtitle"), "authors": v.get("authors") or [],
|
||||||
|
"publisher": v.get("publisher"), "year": year_of(v.get("publishedDate")), "pages": v.get("pageCount"),
|
||||||
|
"description": re.sub(r"<[^>]+>", "", v.get("description") or "") or None,
|
||||||
|
"lang": (v.get("language") or "")[:2] or None, "gb_id": best.get("id"), "cover": cover,
|
||||||
|
"isbn": to13(ids.get("ISBN_13") or ids.get("ISBN_10") or ""), "subjects": v.get("categories") or []}, min(bs, 1.0)
|
||||||
|
|
||||||
|
def arxiv(self, title: str, arxiv_id: str | None = None):
|
||||||
|
if arxiv_id:
|
||||||
|
url = f"https://export.arxiv.org/api/query?id_list={arxiv_id}"
|
||||||
|
else:
|
||||||
|
t = re.sub(r"[^\w\s]", " ", title)
|
||||||
|
url = "https://export.arxiv.org/api/query?" + urllib.parse.urlencode({"search_query": f'ti:"{t}"', "max_results": 5})
|
||||||
|
x = self.h.text(url)
|
||||||
|
if not x:
|
||||||
|
return None, 0.0
|
||||||
|
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||||
|
try:
|
||||||
|
root = ET.fromstring(x)
|
||||||
|
except ET.ParseError:
|
||||||
|
return None, 0.0
|
||||||
|
best, bs = None, 0.0
|
||||||
|
for e in root.findall("a:entry", ns):
|
||||||
|
t = re.sub(r"\s+", " ", e.findtext("a:title", "", ns)).strip()
|
||||||
|
if not t or t == "Error":
|
||||||
|
continue
|
||||||
|
s = 1.0 if arxiv_id else sim(title, t)
|
||||||
|
if s > bs:
|
||||||
|
best, bs = (e, t), s
|
||||||
|
if not best:
|
||||||
|
return None, 0.0
|
||||||
|
e, t = best
|
||||||
|
return {"title": t, "authors": [a.findtext("a:name", "", ns) for a in e.findall("a:author", ns)],
|
||||||
|
"year": year_of(e.findtext("a:published", "", ns)),
|
||||||
|
"description": re.sub(r"\s+", " ", e.findtext("a:summary", "", ns)).strip(),
|
||||||
|
"arxiv": e.findtext("a:id", "", ns).split("/abs/")[-1]}, bs
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- main
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--obsidian-json", help="md5 -> libgen record map from the resolver")
|
||||||
|
ap.add_argument("--extra", action="append", default=[], help='extra placeholder "Title|Author|isbn13|origin"')
|
||||||
|
ap.add_argument("--reset", action="store_true", help="truncate books.works/files first (scratch DBs only)")
|
||||||
|
ap.add_argument("--dry-run", action="store_true", help="no DB writes, no file moves; write plan json")
|
||||||
|
ap.add_argument("--no-move", action="store_true", help="write DB rows (and covers) but leave files in the inbox; for a dev rehearsal before the prod run")
|
||||||
|
ap.add_argument("--no-network", action="store_true")
|
||||||
|
ap.add_argument("--allow-prod", action="store_true")
|
||||||
|
ap.add_argument("--source", default="gdrive")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
root = Path(os.path.expanduser(os.environ.get("BOOKS_ROOT", "/srv/books")))
|
||||||
|
inbox, library, covers = root / "inbox", root / "library", root / "covers"
|
||||||
|
for d in (inbox, library, covers):
|
||||||
|
d.mkdir(parents=True, exist_ok=True)
|
||||||
|
db_url = os.environ.get("DB_URL")
|
||||||
|
if not db_url:
|
||||||
|
sys.exit("DB_URL required")
|
||||||
|
dbname = urllib.parse.urlparse(db_url).path.lstrip("/")
|
||||||
|
if dbname == "personal" and not args.allow_prod:
|
||||||
|
sys.exit("refusing to run against prod DB 'personal' (use --allow-prod)")
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
conn = psycopg.connect(db_url)
|
||||||
|
http = Http(root / ".cache" / "http", delay=0.0 if args.no_network else 1.0)
|
||||||
|
en = Enricher(http)
|
||||||
|
works: list[Work] = []
|
||||||
|
|
||||||
|
|
||||||
|
# ---- 1. placeholders: lists.books
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("select to_regclass('lists.books')")
|
||||||
|
has_lists = cur.fetchone()[0] is not None
|
||||||
|
rows = []
|
||||||
|
if has_lists:
|
||||||
|
cur.execute("select id, name, author, notes, done_at from lists.books order by sort_order nulls last, id")
|
||||||
|
rows = cur.fetchall()
|
||||||
|
for bid, name, author, notes, done_at in rows:
|
||||||
|
w = Work(title=name.strip(), authors=[author.strip()] if author else [], notes=notes or None)
|
||||||
|
w.add_ref("lists_books_id", bid)
|
||||||
|
w.origins.add("lists")
|
||||||
|
w.shelves.add("done" if done_at else "to-read")
|
||||||
|
if done_at:
|
||||||
|
w.refs["done_at"] = done_at.isoformat()
|
||||||
|
works.append(w)
|
||||||
|
n_lists = len(rows)
|
||||||
|
|
||||||
|
def find_work(title, authors, isbns, md5=None):
|
||||||
|
for w in works:
|
||||||
|
if md5 and md5 in w.md5s:
|
||||||
|
return w
|
||||||
|
if isbns and set(isbns) & set(w.isbns) and (not title or not w.title or sim(title, w.title) >= 0.85):
|
||||||
|
return w
|
||||||
|
best, bs = None, 0.0
|
||||||
|
for w in works:
|
||||||
|
s = sim(title, w.title)
|
||||||
|
if s >= 0.9 and authors_compatible(authors, w.authors) and s > bs:
|
||||||
|
best, bs = w, s
|
||||||
|
return best
|
||||||
|
|
||||||
|
def merge(w: Work, title, authors, isbns, **kw):
|
||||||
|
if authors and not w.authors:
|
||||||
|
w.authors = authors
|
||||||
|
for i in isbns or []:
|
||||||
|
if i not in w.isbns:
|
||||||
|
w.isbns.append(i)
|
||||||
|
for k in ("year", "publisher", "language"):
|
||||||
|
if kw.get(k) and not getattr(w, k):
|
||||||
|
setattr(w, k, kw[k])
|
||||||
|
# prefer the longer, properly-cased title when the list name is a lowercase shorthand
|
||||||
|
if title and (w.title.islower() or len(norm_title(w.title)) < len(norm_title(title)) and sim(w.title, title) > 0.9 and w.title == w.title.lower()):
|
||||||
|
w.title = title
|
||||||
|
|
||||||
|
# ---- 2. placeholders: Obsidian md5s
|
||||||
|
obs_total = obs_resolved = obs_overlap = 0
|
||||||
|
if args.obsidian_json:
|
||||||
|
data = json.load(open(args.obsidian_json))
|
||||||
|
for md5, rec in data.items():
|
||||||
|
obs_total += 1
|
||||||
|
eds = (rec or {}).get("editions") or []
|
||||||
|
if not eds:
|
||||||
|
continue
|
||||||
|
obs_resolved += 1
|
||||||
|
ed = next((e for e in eds if e.get("isbns")), eds[0])
|
||||||
|
isbns = []
|
||||||
|
for e in eds:
|
||||||
|
for i in e.get("isbns") or []:
|
||||||
|
i13 = to13(i)
|
||||||
|
if i13 and i13 not in isbns:
|
||||||
|
isbns.append(i13)
|
||||||
|
title = (ed.get("title") or rec.get("locator") or md5).strip()
|
||||||
|
authors = [a.strip() for a in re.split(r";|, (?=[A-Z][a-z]+ [A-Z])", ed.get("author") or "") if a.strip()]
|
||||||
|
lang = LANG.get((ed.get("lang") or [""])[0], None) if isinstance(ed.get("lang"), list) else None
|
||||||
|
w = find_work(title, authors, isbns, md5)
|
||||||
|
if w:
|
||||||
|
if "obsidian" not in w.origins:
|
||||||
|
obs_overlap += 1
|
||||||
|
merge(w, title, authors, isbns, year=year_of(ed.get("year")), publisher=ed.get("publisher") or None, language=lang)
|
||||||
|
else:
|
||||||
|
w = Work(title=title, authors=authors, isbns=isbns, year=year_of(ed.get("year")), publisher=ed.get("publisher") or None, language=lang)
|
||||||
|
works.append(w)
|
||||||
|
w.shelves.add("to-read")
|
||||||
|
w.origins.add("obsidian")
|
||||||
|
if md5 not in w.md5s:
|
||||||
|
w.md5s.append(md5)
|
||||||
|
w.add_ref("obsidian", {"md5": md5, "ext": rec.get("ext"), "filesize": rec.get("filesize")})
|
||||||
|
for ex in args.extra:
|
||||||
|
t, a, i, origin = (ex.split("|") + ["", "", "", ""])[:4]
|
||||||
|
isbns = [to13(i)] if to13(i) else []
|
||||||
|
authors = [a] if a else []
|
||||||
|
w = find_work(t, authors, isbns)
|
||||||
|
if w:
|
||||||
|
merge(w, t, authors, isbns)
|
||||||
|
else:
|
||||||
|
w = Work(title=t, authors=authors, isbns=isbns)
|
||||||
|
works.append(w)
|
||||||
|
w.shelves.add("to-read")
|
||||||
|
w.origins.add("obsidian")
|
||||||
|
w.add_ref("obsidian_link", origin)
|
||||||
|
n_placeholders = len(works)
|
||||||
|
|
||||||
|
# ---- 3. files in inbox
|
||||||
|
files: list[FileRec] = []
|
||||||
|
for p in sorted(inbox.rglob("*")):
|
||||||
|
if not p.is_file() or p.name.startswith("."):
|
||||||
|
continue
|
||||||
|
h256, h5 = hashlib.sha256(), hashlib.md5()
|
||||||
|
with open(p, "rb") as f:
|
||||||
|
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||||
|
h256.update(chunk)
|
||||||
|
h5.update(chunk)
|
||||||
|
rel = str(p.relative_to(inbox))
|
||||||
|
ext = p.suffix.lower()
|
||||||
|
with open(p, "rb") as hf:
|
||||||
|
head = hf.read(8)
|
||||||
|
fmt = ext.lstrip(".") or "bin"
|
||||||
|
if head.startswith(b"%PDF"):
|
||||||
|
fmt = "pdf"
|
||||||
|
elif head.startswith(b"PK") and ext in (".epub", ""):
|
||||||
|
fmt = "epub"
|
||||||
|
fr = FileRec(path=p, rel_inbox=rel, sha256=h256.hexdigest(), md5=h5.hexdigest(), size=p.stat().st_size, fmt=fmt)
|
||||||
|
fr.is_paper = rel.split("/")[0].lower() == "papers"
|
||||||
|
if ext == ".crdownload":
|
||||||
|
fr.broken = "partial Chrome download (.crdownload)"
|
||||||
|
fr.fmt = "pdf" if head.startswith(b"%PDF") else "partial"
|
||||||
|
files.append(fr)
|
||||||
|
by_sha: dict[str, FileRec] = {}
|
||||||
|
dups: list[tuple[FileRec, FileRec]] = []
|
||||||
|
uniq: list[FileRec] = []
|
||||||
|
for fr in files:
|
||||||
|
if fr.sha256 in by_sha:
|
||||||
|
dups.append((fr, by_sha[fr.sha256]))
|
||||||
|
else:
|
||||||
|
by_sha[fr.sha256] = fr
|
||||||
|
uniq.append(fr)
|
||||||
|
log(f"files: {len(files)} total, {len(uniq)} unique, {len(dups)} sha256 duplicates")
|
||||||
|
|
||||||
|
# already-ingested shas (rerun without --reset)
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("select sha256 from books.files")
|
||||||
|
known = {r[0] for r in cur.fetchall()} if not args.reset else set()
|
||||||
|
|
||||||
|
matched_existing = 0
|
||||||
|
for fr in uniq:
|
||||||
|
if fr.sha256 in known:
|
||||||
|
continue
|
||||||
|
stem = Path(fr.rel_inbox).name
|
||||||
|
stem = stem[: -len(Path(stem).suffix)] if Path(stem).suffix.lower() in BOOK_EXT | {".crdownload"} else stem
|
||||||
|
fm = filename_meta(stem)
|
||||||
|
em = epub_meta(fr.path) if fr.fmt == "epub" else pdf_meta(fr.path) if fr.fmt == "pdf" and not fr.broken else {}
|
||||||
|
fr.title = fm.get("title") or stem
|
||||||
|
if fr.fmt == "epub" and good_embedded_title(em.get("title"), fr.title):
|
||||||
|
fr.title = em["title"]
|
||||||
|
fr.authors = fm.get("authors") or [a for a in em.get("authors", []) if len(a) > 2 and "unknown" not in a.lower()][:4]
|
||||||
|
fr.isbns = list(dict.fromkeys((fm.get("isbns") or []) + (em.get("isbns") or [])))[:6]
|
||||||
|
fr.year = fm.get("year") or em.get("year")
|
||||||
|
fr.publisher = fm.get("publisher") or em.get("publisher")
|
||||||
|
fr.language = em.get("language")
|
||||||
|
fr.pages = em.get("pages")
|
||||||
|
md5 = fm.get("md5") or fr.md5
|
||||||
|
w = find_work(fr.title, fr.authors, fr.isbns[:2], md5) or (find_work(fr.title, fr.authors, [], fr.md5))
|
||||||
|
if w and w.files == [] and w.origins & {"lists", "obsidian"} and "drive" not in w.origins:
|
||||||
|
matched_existing += 1
|
||||||
|
if w:
|
||||||
|
merge(w, fr.title, fr.authors, fr.isbns[:2], year=fr.year, publisher=fr.publisher, language=fr.language)
|
||||||
|
else:
|
||||||
|
w = Work(title=fr.title, authors=fr.authors, isbns=fr.isbns[:2], year=fr.year, publisher=fr.publisher, language=fr.language, page_count=fr.pages)
|
||||||
|
works.append(w)
|
||||||
|
if fr.is_paper and "paper" not in w.tags:
|
||||||
|
w.tags.append("paper")
|
||||||
|
if fr.md5 not in w.md5s and fm.get("md5"):
|
||||||
|
w.md5s.append(fm["md5"])
|
||||||
|
w.origins.add("drive")
|
||||||
|
w.add_ref("drive", fr.rel_inbox)
|
||||||
|
if em.get("arxiv"):
|
||||||
|
w.refs["arxiv"] = em["arxiv"]
|
||||||
|
if fr.broken:
|
||||||
|
w.notes = ((w.notes + "\n") if w.notes else "") + f"Drive copy is broken: {fr.broken}; needs a fresh copy."
|
||||||
|
w.add_ref("broken_files", fr.rel_inbox)
|
||||||
|
else:
|
||||||
|
w.files.append(fr)
|
||||||
|
for d, orig in dups:
|
||||||
|
for w in works:
|
||||||
|
if orig in w.files:
|
||||||
|
w.add_ref("drive_duplicates", d.rel_inbox)
|
||||||
|
log(f"works: {len(works)} ({n_placeholders} placeholders before files; {matched_existing} placeholders got files)")
|
||||||
|
|
||||||
|
# ---- 4. enrichment
|
||||||
|
stats = {"matched": 0, "low_confidence": 0, "not_found": 0}
|
||||||
|
for n, w in enumerate(works, 1):
|
||||||
|
if args.no_network:
|
||||||
|
break
|
||||||
|
rec, score, src = None, 0.0, None
|
||||||
|
if "paper" in w.tags:
|
||||||
|
rec, score = en.arxiv(w.title, w.refs.get("arxiv"))
|
||||||
|
src = "arxiv"
|
||||||
|
else:
|
||||||
|
for isbn in w.isbns[:3]:
|
||||||
|
rec = en.ol_isbn(isbn)
|
||||||
|
if rec and (sim(w.title, rec["title"]) > 0.6 or sim(w.title, f"{rec['title']}: {rec.get('subtitle') or ''}") > 0.6 or "drive" not in w.origins):
|
||||||
|
score, src = 1.0, "openlibrary"
|
||||||
|
break
|
||||||
|
rec = None
|
||||||
|
if not rec:
|
||||||
|
rec, score = en.ol_search(w.title, w.authors)
|
||||||
|
src = "openlibrary"
|
||||||
|
if score < 0.9:
|
||||||
|
g, gs = en.gb(None, [], w.isbns[0]) if w.isbns else (None, 0.0)
|
||||||
|
if g and sim(w.title, g["title"] or "") < 0.6 and sim(w.title, f"{g['title']}: {g.get('subtitle') or ''}") < 0.6:
|
||||||
|
g, gs = None, 0.0
|
||||||
|
if not g:
|
||||||
|
g, gs = en.gb(w.title, w.authors)
|
||||||
|
if g and gs > score:
|
||||||
|
rec, score, src = g, gs, "googlebooks"
|
||||||
|
if rec and score >= 0.9:
|
||||||
|
w.status = "matched"
|
||||||
|
elif rec and score >= 0.75:
|
||||||
|
w.status = "low_confidence"
|
||||||
|
else:
|
||||||
|
w.status = "not_found"
|
||||||
|
stats[w.status] += 1
|
||||||
|
if rec:
|
||||||
|
w.refs["match"] = {"source": src, "score": round(score, 3), "title": rec.get("title"), "authors": rec.get("authors")}
|
||||||
|
if w.status == "matched":
|
||||||
|
w.source = src
|
||||||
|
if src == "openlibrary" or src == "googlebooks":
|
||||||
|
# keep Stan's/the file's title wording unless it was a lowercase list shorthand or a filename
|
||||||
|
# keep Stan's own list wording; otherwise take the catalogue's properly-cased title
|
||||||
|
if rec.get("title") and ("lists" not in w.origins or w.title.islower()):
|
||||||
|
w.title = rec["title"]
|
||||||
|
w.subtitle = w.subtitle or rec.get("subtitle")
|
||||||
|
elif src == "arxiv":
|
||||||
|
w.title = rec["title"]
|
||||||
|
if rec.get("authors"):
|
||||||
|
w.authors = rec["authors"][:6]
|
||||||
|
w.year = w.year or rec.get("year")
|
||||||
|
if src == "openlibrary" and rec.get("year") and w.year and rec["year"] < w.year:
|
||||||
|
w.year = w.year # edition year stays; first-publish year recorded in refs
|
||||||
|
w.refs["first_publish_year"] = rec["year"]
|
||||||
|
w.publisher = w.publisher or rec.get("publisher")
|
||||||
|
w.page_count = w.page_count or rec.get("pages")
|
||||||
|
w.language = w.language or rec.get("lang")
|
||||||
|
w.description = w.description or rec.get("description")
|
||||||
|
if rec.get("isbn") and rec["isbn"] not in w.isbns:
|
||||||
|
w.isbns.insert(0 if not w.isbns else len(w.isbns), rec["isbn"])
|
||||||
|
w.ol_work = w.ol_work or rec.get("ol_work")
|
||||||
|
w.ol_edition = w.ol_edition or rec.get("ol_edition")
|
||||||
|
w.gb_id = w.gb_id or rec.get("gb_id")
|
||||||
|
w.cover_url = w.cover_url or rec.get("cover")
|
||||||
|
if rec.get("arxiv"):
|
||||||
|
w.refs["arxiv"] = rec["arxiv"]
|
||||||
|
for s in (rec.get("subjects") or [])[:5]:
|
||||||
|
s = s.lower().strip()
|
||||||
|
if 2 < len(s) < 40 and s not in w.tags:
|
||||||
|
w.tags.append(s)
|
||||||
|
if n % 25 == 0:
|
||||||
|
log(f" enriched {n}/{len(works)} ({http.calls} http calls)")
|
||||||
|
if args.no_network:
|
||||||
|
stats = {"skipped": len(works)}
|
||||||
|
log("metadata:", stats)
|
||||||
|
|
||||||
|
# cover fallback for matched works lacking one: Google Books by isbn
|
||||||
|
if not args.no_network:
|
||||||
|
for w in works:
|
||||||
|
if not w.cover_url and w.status == "matched" and "paper" not in w.tags and w.isbns:
|
||||||
|
g, _ = en.gb(None, [], w.isbns[0])
|
||||||
|
if g and g.get("cover"):
|
||||||
|
w.cover_url = g["cover"]
|
||||||
|
w.gb_id = w.gb_id or g.get("gb_id")
|
||||||
|
|
||||||
|
# ---- 5. plan destinations
|
||||||
|
used: set[str] = set()
|
||||||
|
for w in works:
|
||||||
|
author_dir = safe_name(w.authors[0] if w.authors else "Unknown", 60)
|
||||||
|
title_dir = safe_name(w.title, 90)
|
||||||
|
for fr in w.files:
|
||||||
|
base = f"{author_dir}/{title_dir}/{title_dir}"
|
||||||
|
rel = f"{base}.{fr.fmt}"
|
||||||
|
k = 2
|
||||||
|
while rel in used or (library / rel).exists() and not args.reset:
|
||||||
|
rel = f"{base} ({k}).{fr.fmt}"
|
||||||
|
k += 1
|
||||||
|
used.add(rel)
|
||||||
|
fr.dest_rel = rel
|
||||||
|
|
||||||
|
plan = [{"title": w.title, "authors": w.authors, "isbns": w.isbns, "status": w.status, "origins": sorted(w.origins),
|
||||||
|
"files": [{"from": f.rel_inbox, "to": f.dest_rel, "sha256": f.sha256} for f in w.files], "refs": w.refs,
|
||||||
|
"cover": w.cover_url} for w in works]
|
||||||
|
(root / ".cache").mkdir(exist_ok=True)
|
||||||
|
(root / ".cache" / "ingest-plan.json").write_text(json.dumps(plan, indent=1, default=str))
|
||||||
|
if args.dry_run:
|
||||||
|
log("dry run: plan written to", root / ".cache" / "ingest-plan.json")
|
||||||
|
return
|
||||||
|
|
||||||
|
# ---- 6. write DB
|
||||||
|
def isbn13(w):
|
||||||
|
return next((i for i in w.isbns if i.startswith("978") or i.startswith("979")), None)
|
||||||
|
seen13 = set()
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
with conn.transaction():
|
||||||
|
cur = conn.cursor()
|
||||||
|
if args.reset:
|
||||||
|
cur.execute("truncate books.notes, books.progress, books.shelf_works, books.files, books.works restart identity cascade")
|
||||||
|
cur.execute("select slug, id from books.shelves")
|
||||||
|
shelves = dict(cur.fetchall())
|
||||||
|
for w in works:
|
||||||
|
i13 = isbn13(w)
|
||||||
|
if i13 in seen13:
|
||||||
|
w.refs["isbn13_shared_with_other_work"] = i13
|
||||||
|
i13 = None
|
||||||
|
if i13:
|
||||||
|
seen13.add(i13)
|
||||||
|
refs = dict(w.refs)
|
||||||
|
refs["origins"] = sorted(w.origins)
|
||||||
|
if len(w.isbns) > 1:
|
||||||
|
refs["all_isbns"] = w.isbns
|
||||||
|
cur.execute(
|
||||||
|
"""insert into books.works (title, subtitle, authors, isbn10, isbn13, publisher, year, language, page_count,
|
||||||
|
description, tags, cover_url, openlibrary_work_id, openlibrary_edition_id, google_books_id, annas_md5,
|
||||||
|
source_refs, metadata_status, metadata_source, metadata_fetched_at, notes)
|
||||||
|
values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) returning id""",
|
||||||
|
(w.title, w.subtitle, w.authors, to10(i13) if i13 else None, i13, w.publisher,
|
||||||
|
w.year if w.year and 1400 < w.year < 2100 else None, w.language, w.page_count, w.description, w.tags,
|
||||||
|
w.cover_url, w.ol_work, w.ol_edition, w.gb_id, w.md5s, json.dumps(refs, default=str), w.status,
|
||||||
|
w.source, now if w.status != "pending" else None, w.notes))
|
||||||
|
w.db_id = cur.fetchone()[0]
|
||||||
|
for fr in w.files:
|
||||||
|
cur.execute("insert into books.files (work_id, path, format, size_bytes, sha256, md5, source) values (%s,%s,%s,%s,%s,%s,%s)",
|
||||||
|
(w.db_id, fr.dest_rel, fr.fmt, fr.size, fr.sha256, fr.md5, args.source))
|
||||||
|
for s in sorted(w.shelves):
|
||||||
|
if s in shelves:
|
||||||
|
cur.execute("insert into books.shelf_works (shelf_id, work_id) values (%s,%s) on conflict do nothing", (shelves[s], w.db_id))
|
||||||
|
if "done" in w.shelves and w.refs.get("done_at"):
|
||||||
|
cur.execute("insert into books.progress (work_id, percent, finished_at, status) values (%s, 100, %s, 'finished') on conflict do nothing", (w.db_id, w.refs["done_at"]))
|
||||||
|
|
||||||
|
# ---- 7. covers
|
||||||
|
got = 0
|
||||||
|
if not args.no_network:
|
||||||
|
for w in works:
|
||||||
|
if not w.cover_url:
|
||||||
|
continue
|
||||||
|
dest = covers / (hashlib.sha1(w.cover_url.encode()).hexdigest()[:16] + ".jpg")
|
||||||
|
if dest.exists() or http.download(w.cover_url, dest):
|
||||||
|
got += 1
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute("update books.works set cover_path=%s where id=%s", (dest.name, w.db_id))
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
# ---- 8. move files (inbox -> library); write undo manifest
|
||||||
|
if args.no_move:
|
||||||
|
log("--no-move: DB rows written, files left in inbox")
|
||||||
|
return
|
||||||
|
manifest = root / ".cache" / f"moves-{time.strftime('%Y%m%d-%H%M%S')}.tsv"
|
||||||
|
dupdir = root / "duplicates"
|
||||||
|
qdir = root / "quarantine"
|
||||||
|
with open(manifest, "w") as mf:
|
||||||
|
for w in works:
|
||||||
|
for fr in w.files:
|
||||||
|
dest = library / fr.dest_rel
|
||||||
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.move(str(fr.path), dest)
|
||||||
|
mf.write(f"{fr.path}\t{dest}\n")
|
||||||
|
for d, _ in dups:
|
||||||
|
dest = dupdir / d.rel_inbox
|
||||||
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.move(str(d.path), dest)
|
||||||
|
mf.write(f"{d.path}\t{dest}\n")
|
||||||
|
for fr in uniq:
|
||||||
|
if fr.broken and fr.path.exists():
|
||||||
|
dest = qdir / fr.rel_inbox
|
||||||
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.move(str(fr.path), dest)
|
||||||
|
mf.write(f"{fr.path}\t{dest}\n")
|
||||||
|
for d in sorted(inbox.rglob("*"), reverse=True):
|
||||||
|
if d.is_dir() and not any(d.iterdir()):
|
||||||
|
d.rmdir()
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"lists_rows": n_lists, "obsidian_md5s": obs_total, "obsidian_resolved": obs_resolved,
|
||||||
|
"obsidian_overlapping_lists": obs_overlap, "placeholders": n_placeholders,
|
||||||
|
"files_total": len(files), "files_unique": len(uniq), "sha256_duplicates": len(dups),
|
||||||
|
"placeholders_with_files": matched_existing, "works_total": len(works),
|
||||||
|
"works_with_files": sum(1 for w in works if w.files), "metadata": stats, "covers": got,
|
||||||
|
"papers": sum(1 for w in works if "paper" in w.tags), "http_calls": http.calls, "manifest": str(manifest),
|
||||||
|
}
|
||||||
|
(root / ".cache" / "ingest-summary.json").write_text(json.dumps(summary, indent=1))
|
||||||
|
print(json.dumps(summary, indent=1))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
19
scripts/db-meta-sync.ts
Normal file
19
scripts/db-meta-sync.ts
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
// Refreshes meta.columns from the live catalog for every table in meta.tables
|
||||||
|
// (meta.sync() is SECURITY DEFINER and granted to the app role). Idempotent.
|
||||||
|
// bun scripts/db-meta-sync.ts uses DB_URL (personal_app)
|
||||||
|
import postgres from 'postgres';
|
||||||
|
|
||||||
|
const url = process.env.DB_URL;
|
||||||
|
if (!url) throw new Error('Set DB_URL (app role)');
|
||||||
|
const client = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
||||||
|
try {
|
||||||
|
const [registry] = await client`select to_regprocedure('meta.sync(integer)') is not null as ok`;
|
||||||
|
if (!registry.ok) {
|
||||||
|
console.log('meta.sync() not present; skipping');
|
||||||
|
} else {
|
||||||
|
const [row] = await client`select meta.sync() as added`;
|
||||||
|
console.log(`meta.sync(): ${row.added} column(s) added`);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await client.end();
|
||||||
|
}
|
||||||
@@ -107,6 +107,61 @@ try {
|
|||||||
}
|
}
|
||||||
if (schema.app_has_ddl)
|
if (schema.app_has_ddl)
|
||||||
throw new Error('DB_URL must use the app role (personal_app), not the schema owner');
|
throw new Error('DB_URL must use the app role (personal_app), not the schema owner');
|
||||||
|
// Private health schema: the app role reads and writes it, nobody else (not even personal_ro).
|
||||||
|
const [health] = await client`
|
||||||
|
select
|
||||||
|
count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
|
||||||
|
count(*) filter (
|
||||||
|
where exists (
|
||||||
|
select 1 from aclexplode(c.relacl) a join pg_roles r on r.oid = a.grantee
|
||||||
|
where r.rolname = 'personal_ro'
|
||||||
|
)
|
||||||
|
) as ro_read,
|
||||||
|
count(*) as tables,
|
||||||
|
has_schema_privilege('public', 'health', 'usage') as public_usage
|
||||||
|
from pg_class c join pg_namespace n on n.oid = c.relnamespace
|
||||||
|
where n.nspname = 'health' and c.relkind = 'r'
|
||||||
|
`;
|
||||||
|
if (
|
||||||
|
Number(health.tables) !== 7 ||
|
||||||
|
Number(health.app_rw) !== 7 ||
|
||||||
|
Number(health.ro_read) !== 0 ||
|
||||||
|
health.public_usage
|
||||||
|
)
|
||||||
|
throw new Error(`health schema grants are wrong: ${JSON.stringify(health)}`);
|
||||||
|
// Private task tracker: same rules as health (app role read/write, personal_ro none).
|
||||||
|
const [tasks] = await client`
|
||||||
|
select
|
||||||
|
count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
|
||||||
|
count(*) filter (
|
||||||
|
where exists (
|
||||||
|
select 1 from aclexplode(c.relacl) a join pg_roles r on r.oid = a.grantee
|
||||||
|
where r.rolname = 'personal_ro'
|
||||||
|
)
|
||||||
|
) as ro_read,
|
||||||
|
count(*) as tables,
|
||||||
|
has_schema_privilege('public', 'tasks', 'usage') as public_usage
|
||||||
|
from pg_class c join pg_namespace n on n.oid = c.relnamespace
|
||||||
|
where n.nspname = 'tasks' and c.relkind = 'r'
|
||||||
|
`;
|
||||||
|
if (
|
||||||
|
Number(tasks.tables) !== 4 ||
|
||||||
|
Number(tasks.app_rw) !== 4 ||
|
||||||
|
Number(tasks.ro_read) !== 0 ||
|
||||||
|
tasks.public_usage
|
||||||
|
)
|
||||||
|
throw new Error(`tasks schema grants are wrong: ${JSON.stringify(tasks)}`);
|
||||||
|
// Energy tracker reads the ned-collector tables (SELECT only).
|
||||||
|
const [energy] = await client`
|
||||||
|
select count(*) as hours, max(valid_from) as latest from energy.ned_utilization
|
||||||
|
where granularity = 'Hour' and classification = 'Current'
|
||||||
|
and valid_from > now() - interval '7 days'
|
||||||
|
`;
|
||||||
|
if (!Number(energy.hours))
|
||||||
|
console.warn('Warning: energy.ned_utilization has no hourly data from the last 7 days');
|
||||||
|
console.log(
|
||||||
|
`health: ${health.tables} tables, tasks: ${tasks.tables} tables, app read/write, personal_ro none; energy: ${energy.hours} hourly rows, latest ${energy.latest ? new Date(energy.latest).toISOString() : 'none'}`
|
||||||
|
);
|
||||||
const [catalog] = await client`
|
const [catalog] = await client`
|
||||||
select
|
select
|
||||||
(select count(*) from ref.pokemon) as pokemon,
|
(select count(*) from ref.pokemon) as pokemon,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ environment_name=production
|
|||||||
|
|
||||||
image="git.stanrunge.dev/stan/website:$sha"
|
image="git.stanrunge.dev/stan/website:$sha"
|
||||||
compose="$directory/compose.prod.yml"
|
compose="$directory/compose.prod.yml"
|
||||||
|
compose_files=(-f "$compose")
|
||||||
# Images built before the repo rename (stan/personal-website -> stan/website) keep
|
# Images built before the repo rename (stan/personal-website -> stan/website) keep
|
||||||
# their old local name; retag so rollbacks to those SHAs still work.
|
# their old local name; retag so rollbacks to those SHAs still work.
|
||||||
legacy_image="git.stanrunge.dev/stan/personal-website:$sha"
|
legacy_image="git.stanrunge.dev/stan/personal-website:$sha"
|
||||||
@@ -28,13 +29,42 @@ fi
|
|||||||
if ! sudo docker image inspect "$image" >/dev/null 2>&1; then
|
if ! sudo docker image inspect "$image" >/dev/null 2>&1; then
|
||||||
echo "Image $image is not on this host" >&2; exit 1
|
echo "Image $image is not on this host" >&2; exit 1
|
||||||
fi
|
fi
|
||||||
|
# Book files (ebooks + covers) live on the host and are bind-mounted read-only at /books;
|
||||||
|
# the app streams them only to the owner via /trackers/books/files/<id> (no static path).
|
||||||
|
# Layout: $books_host_dir/library/<files.path> and $books_host_dir/covers/<works.cover_path>.
|
||||||
|
# compose.prod.yml is host-managed, so the mount goes in a generated override file (~deploy).
|
||||||
|
books_host_dir=${BOOKS_HOST_DIR:-/home/stan/books}
|
||||||
|
case "$books_host_dir" in /*) ;; *) echo 'BOOKS_HOST_DIR must be absolute' >&2; exit 2 ;; esac
|
||||||
|
# The deploy account can't traverse /home/stan (0750); check from a root container instead.
|
||||||
|
if sudo docker run --rm --entrypoint sh \
|
||||||
|
--mount "type=bind,source=$books_host_dir,target=/books,readonly" "$image" \
|
||||||
|
-c 'test -d /books/library' 2>/dev/null; then
|
||||||
|
# The deploy account may only sudo-tee $directory/.env, so the override lives in its home;
|
||||||
|
# the compose CLI runs as root and can read it from there.
|
||||||
|
books_override="$HOME/compose.books.yml"
|
||||||
|
rm -f "$books_override"
|
||||||
|
cat >"$books_override" <<YAML
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
environment:
|
||||||
|
BOOKS_ROOT: /books
|
||||||
|
volumes:
|
||||||
|
- type: bind
|
||||||
|
source: $books_host_dir
|
||||||
|
target: /books
|
||||||
|
read_only: true
|
||||||
|
YAML
|
||||||
|
compose_files+=(-f "$books_override")
|
||||||
|
else
|
||||||
|
echo "No $books_host_dir/library on this host; book files will not be readable" >&2
|
||||||
|
fi
|
||||||
previous_tag=$(sed -n 's/^IMAGE_TAG=//p' "$directory/.env" | head -n 1)
|
previous_tag=$(sed -n 's/^IMAGE_TAG=//p' "$directory/.env" | head -n 1)
|
||||||
switched=0
|
switched=0
|
||||||
rollback() {
|
rollback() {
|
||||||
if [ "$switched" -eq 1 ] && [ -n "$previous_tag" ]; then
|
if [ "$switched" -eq 1 ] && [ -n "$previous_tag" ]; then
|
||||||
echo "Health check failed; restoring prior image tag $previous_tag" >&2
|
echo "Health check failed; restoring prior image tag $previous_tag" >&2
|
||||||
printf 'IMAGE_TAG=%s\n' "$previous_tag" | sudo tee "$directory/.env" >/dev/null
|
printf 'IMAGE_TAG=%s\n' "$previous_tag" | sudo tee "$directory/.env" >/dev/null
|
||||||
sudo docker compose --env-file "$directory/.env" -f "$compose" up -d
|
sudo docker compose --env-file "$directory/.env" "${compose_files[@]}" up -d
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
trap rollback ERR
|
trap rollback ERR
|
||||||
@@ -80,6 +110,12 @@ run_database_script() {
|
|||||||
if [ "$mode" = deploy ]; then
|
if [ "$mode" = deploy ]; then
|
||||||
# Schema changes: Drizzle migrations from ./drizzle, as the schema owner.
|
# Schema changes: Drizzle migrations from ./drizzle, as the schema owner.
|
||||||
run_database_script scripts/db-migrate.ts
|
run_database_script scripts/db-migrate.ts
|
||||||
|
# Refresh the generic table GUI's column registry (meta.columns) for tables that migrations
|
||||||
|
# registered in meta.tables. meta.sync() is SECURITY DEFINER and granted to the app role, not
|
||||||
|
# to the schema owner, so run it as DB_URL (personal_app).
|
||||||
|
echo "Running meta.sync()"
|
||||||
|
sudo docker run --rm --network backend --env-file "$environment" "$image" \
|
||||||
|
bun scripts/db-meta-sync.ts
|
||||||
run_database_script scripts/seed-catalog.ts
|
run_database_script scripts/seed-catalog.ts
|
||||||
run_database_script scripts/seed-cod-catalog.ts
|
run_database_script scripts/seed-cod-catalog.ts
|
||||||
run_database_script scripts/seed-pokemon-research.ts
|
run_database_script scripts/seed-pokemon-research.ts
|
||||||
@@ -92,9 +128,9 @@ sudo docker run --rm --network backend --env-file "$environment" "$image" \
|
|||||||
|
|
||||||
printf 'IMAGE_TAG=%s\n' "$sha" | sudo tee "$directory/.env" >/dev/null
|
printf 'IMAGE_TAG=%s\n' "$sha" | sudo tee "$directory/.env" >/dev/null
|
||||||
switched=1
|
switched=1
|
||||||
sudo docker compose --env-file "$directory/.env" -f "$compose" up -d
|
sudo docker compose --env-file "$directory/.env" "${compose_files[@]}" up -d
|
||||||
|
|
||||||
container=$(sudo docker compose --env-file "$directory/.env" -f "$compose" ps -q app)
|
container=$(sudo docker compose --env-file "$directory/.env" "${compose_files[@]}" ps -q app)
|
||||||
expected_image=$(sudo docker image inspect -f '{{.Id}}' "$image")
|
expected_image=$(sudo docker image inspect -f '{{.Id}}' "$image")
|
||||||
actual_image=$(sudo docker inspect -f '{{.Image}}' "$container")
|
actual_image=$(sudo docker inspect -f '{{.Image}}' "$container")
|
||||||
if [ "$actual_image" != "$expected_image" ]; then
|
if [ "$actual_image" != "$expected_image" ]; then
|
||||||
|
|||||||
46
scripts/tasks-import.ts
Normal file
46
scripts/tasks-import.ts
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
/**
|
||||||
|
* Import tasks, goals and areas into the private `tasks` schema (idempotent).
|
||||||
|
*
|
||||||
|
* bun scripts/tasks-import.ts data/tasks/daisy-import.json # insert what is missing
|
||||||
|
* bun scripts/tasks-import.ts --dry-run data/tasks/daisy-import.json # validate only
|
||||||
|
* bun scripts/tasks-import.ts --update data/tasks/daisy-import.json # also overwrite existing rows
|
||||||
|
*
|
||||||
|
* Dedupe: areas on slug, goals on title, tasks on area + title (case-insensitive).
|
||||||
|
* Uses DB_URL (the app role on prod, personal_dev on dev).
|
||||||
|
*/
|
||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import { drizzle } from 'drizzle-orm/postgres-js';
|
||||||
|
import postgres from 'postgres';
|
||||||
|
import { importSeed, seedFile } from '../src/lib/server/tasks';
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const dryRun = args.includes('--dry-run');
|
||||||
|
const update = args.includes('--update');
|
||||||
|
const files = args.filter((a) => !a.startsWith('--'));
|
||||||
|
if (files.length !== 1) {
|
||||||
|
console.error('usage: bun scripts/tasks-import.ts [--dry-run] [--update] <seed.json>');
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
const parsed = seedFile.safeParse(JSON.parse(await readFile(files[0], 'utf8')));
|
||||||
|
if (!parsed.success) {
|
||||||
|
for (const issue of parsed.error.issues.slice(0, 50))
|
||||||
|
console.error(`${issue.path.join('.')}: ${issue.message}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
const seed = parsed.data;
|
||||||
|
console.log(
|
||||||
|
`Valid: ${seed.areas.length} areas, ${seed.goals.length} goals, ${seed.tasks.length} tasks`
|
||||||
|
);
|
||||||
|
if (dryRun) process.exit(0);
|
||||||
|
if (!process.env.DB_URL) throw new Error('DB_URL is required');
|
||||||
|
const client = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
|
||||||
|
try {
|
||||||
|
const r = await importSeed(drizzle({ client }), seed, { update });
|
||||||
|
console.log(
|
||||||
|
`areas: ${r.areas.inserted} inserted, ${r.areas.existing} existing; ` +
|
||||||
|
`goals: ${r.goals.inserted} inserted, ${r.goals.existing} existing, ${r.goals.updated} updated; ` +
|
||||||
|
`tasks: ${r.tasks.inserted} inserted, ${r.tasks.existing} existing, ${r.tasks.updated} updated`
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await client.end();
|
||||||
|
}
|
||||||
79
scripts/ticktick-export.ts
Normal file
79
scripts/ticktick-export.ts
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
/**
|
||||||
|
* Export every TickTick task reachable through the official CLI (Open API v1):
|
||||||
|
* open, won't-do and completed tasks of the inbox and every list, plus completed
|
||||||
|
* history month by month (windows are split further when they hit the API's
|
||||||
|
* 200-result cap). Needs `ticktick auth login` done once; never reads the token.
|
||||||
|
*
|
||||||
|
* bun scripts/ticktick-export.ts ~/.cache/ticktick-export/all.json
|
||||||
|
*
|
||||||
|
* Then: bun scripts/ticktick-import.ts ~/.cache/ticktick-export/all.json
|
||||||
|
*/
|
||||||
|
import { $ } from 'bun';
|
||||||
|
|
||||||
|
const out = process.argv[2];
|
||||||
|
if (!out) {
|
||||||
|
console.error('usage: bun scripts/ticktick-export.ts <out.json> [--since YYYY]');
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
const sinceArg = process.argv.indexOf('--since');
|
||||||
|
const since = sinceArg > 0 ? Number(process.argv[sinceArg + 1]) : 2015;
|
||||||
|
const CAP = 200;
|
||||||
|
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
type Json = any;
|
||||||
|
async function tt(...args: string[]): Promise<Json> {
|
||||||
|
for (let attempt = 0; ; attempt++) {
|
||||||
|
const r = await $`ticktick ${args} --json`.quiet().nothrow();
|
||||||
|
if (r.exitCode === 0) return JSON.parse(r.stdout.toString() || '[]');
|
||||||
|
if (attempt >= 3)
|
||||||
|
throw new Error(
|
||||||
|
`ticktick ${args[0]} ${args[1]} failed: ${r.stderr.toString().slice(0, 300)}`
|
||||||
|
);
|
||||||
|
await Bun.sleep(2000 * (attempt + 1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const fmt = (d: Date) => d.toISOString().replace(/\.\d{3}Z$/, '+0000');
|
||||||
|
const byId = new Map<string, Json>();
|
||||||
|
const add = (list: Json[]) => list.filter((t) => !byId.has(t.id) && byId.set(t.id, t)).length;
|
||||||
|
|
||||||
|
const projects: Json[] = await tt('project', 'list');
|
||||||
|
// The inbox is not a project; its id shows up on tasks (inbox<userId>).
|
||||||
|
const probe: Json[] = await tt('task', 'filter', '--status=0,2,-1');
|
||||||
|
const inboxId = probe.map((t) => t.projectId).find((p: string) => p?.startsWith('inbox'));
|
||||||
|
add(probe);
|
||||||
|
const ids = [...(inboxId ? [inboxId] : []), ...projects.map((p) => p.id)];
|
||||||
|
let capped = 0;
|
||||||
|
for (const status of ['0', '-1', '2'])
|
||||||
|
for (const p of ids) {
|
||||||
|
const r = await tt('task', 'filter', '--projects', p, `--status=${status}`);
|
||||||
|
if (r.length >= CAP) capped++;
|
||||||
|
add(r);
|
||||||
|
}
|
||||||
|
for (const p of projects) add((await tt('project', 'data', p.id)).tasks ?? []);
|
||||||
|
|
||||||
|
async function completed(start: Date, end: Date): Promise<void> {
|
||||||
|
const r = await tt('task', 'completed', '--start-date', fmt(start), '--end-date', fmt(end));
|
||||||
|
if (r.length >= CAP && end.getTime() - start.getTime() > 3_600_000) {
|
||||||
|
const mid = new Date(Math.floor((start.getTime() + end.getTime()) / 2000) * 1000);
|
||||||
|
await completed(start, mid);
|
||||||
|
await completed(new Date(mid.getTime() + 1000), end);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
add(r);
|
||||||
|
}
|
||||||
|
const now = new Date();
|
||||||
|
for (let y = since; y <= now.getUTCFullYear(); y++)
|
||||||
|
for (let m = 0; m < 12; m++) {
|
||||||
|
const start = new Date(Date.UTC(y, m, 1));
|
||||||
|
if (start > now) break;
|
||||||
|
await completed(start, new Date(Date.UTC(y, m + 1, 1) - 1000));
|
||||||
|
}
|
||||||
|
|
||||||
|
const tasks = [...byId.values()];
|
||||||
|
await Bun.write(out, JSON.stringify({ exportedAt: now.toISOString(), inboxId, projects, tasks }));
|
||||||
|
const count = (s: number) => tasks.filter((t) => t.status === s).length;
|
||||||
|
console.log(
|
||||||
|
`${tasks.length} tasks (${count(0)} open, ${count(2)} completed, ${count(-1)} won't do) from ` +
|
||||||
|
`${projects.length} lists + inbox` +
|
||||||
|
(capped ? `; ${capped} status/list queries hit the ${CAP} cap (older ones may be missing)` : '')
|
||||||
|
);
|
||||||
65
scripts/ticktick-import.ts
Normal file
65
scripts/ticktick-import.ts
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
/**
|
||||||
|
* Sync TickTick into the private `tasks` schema. Idempotent; rerun after a new export.
|
||||||
|
*
|
||||||
|
* bun scripts/ticktick-import.ts [--dry-run] [--csv backup.csv] ~/.cache/ticktick-export/all.json
|
||||||
|
*
|
||||||
|
* all.json comes from scripts/ticktick-export.ts (Open API). --csv adds the tasks from a
|
||||||
|
* TickTick backup (Settings → Backup & Import) that the API no longer returns (closed lists,
|
||||||
|
* old won't-do tasks); rows the API also has are matched and skipped (see mergeCsv).
|
||||||
|
* Links listed in data/tasks/ticktick-merges.json (plus exact title matches, see
|
||||||
|
* syncTickTick) attach a TickTick task to an existing task instead of duplicating it.
|
||||||
|
* Uses DB_URL (the app role on prod, personal_dev on dev).
|
||||||
|
*/
|
||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import { drizzle } from 'drizzle-orm/postgres-js';
|
||||||
|
import postgres from 'postgres';
|
||||||
|
import {
|
||||||
|
mergeCsv,
|
||||||
|
parseTickTickCsv,
|
||||||
|
syncTickTick,
|
||||||
|
type TtExport,
|
||||||
|
type TtMerge
|
||||||
|
} from '../src/lib/server/ticktick';
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const dryRun = args.includes('--dry-run');
|
||||||
|
const csvAt = args.indexOf('--csv');
|
||||||
|
const csvPath = csvAt >= 0 ? args[csvAt + 1] : undefined;
|
||||||
|
const files = args.filter((a, i) => !a.startsWith('--') && (csvAt < 0 || i !== csvAt + 1));
|
||||||
|
if (files.length !== 1 || (csvAt >= 0 && !csvPath)) {
|
||||||
|
console.error(
|
||||||
|
'usage: bun scripts/ticktick-import.ts [--dry-run] [--csv backup.csv] <export.json>'
|
||||||
|
);
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
let data = JSON.parse(await readFile(files[0], 'utf8')) as TtExport;
|
||||||
|
if (csvPath) {
|
||||||
|
const merged = mergeCsv(data, parseTickTickCsv(await readFile(csvPath, 'utf8')));
|
||||||
|
data = merged.data;
|
||||||
|
console.log(JSON.stringify({ csv: merged.report }));
|
||||||
|
}
|
||||||
|
const merges = JSON.parse(
|
||||||
|
await readFile(new URL('../data/tasks/ticktick-merges.json', import.meta.url), 'utf8')
|
||||||
|
) as TtMerge[];
|
||||||
|
if (!process.env.DB_URL) throw new Error('DB_URL is required');
|
||||||
|
const client = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
|
||||||
|
class DryRun extends Error {}
|
||||||
|
try {
|
||||||
|
const db = drizzle({ client });
|
||||||
|
try {
|
||||||
|
const report = await db.transaction(async (tx) => {
|
||||||
|
const r = await syncTickTick(tx, data, { merges });
|
||||||
|
if (dryRun) {
|
||||||
|
console.log(JSON.stringify(r, null, 2));
|
||||||
|
throw new DryRun();
|
||||||
|
}
|
||||||
|
return r;
|
||||||
|
});
|
||||||
|
console.log(JSON.stringify(report, null, 2));
|
||||||
|
} catch (e) {
|
||||||
|
if (!(e instanceof DryRun)) throw e;
|
||||||
|
console.log('(dry run, rolled back)');
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await client.end();
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import type { Handle } from '@sveltejs/kit/hooks';
|
import type { Handle } from '@sveltejs/kit/hooks';
|
||||||
import { building } from '$app/env';
|
import { building } from '$app/env';
|
||||||
import { auth } from '#lib/server/auth.js';
|
import { auth } from '#lib/server/auth.js';
|
||||||
import { isOwner, isOwnerOnlyPath } from '#lib/server/owner.js';
|
import { isOwner, isOwnerOnlyPath, isOwnerWritePath } from '#lib/server/owner.js';
|
||||||
import { error } from '@sveltejs/kit';
|
import { error } from '@sveltejs/kit';
|
||||||
|
|
||||||
export const handle: Handle = async ({ event, resolve }) => {
|
export const handle: Handle = async ({ event, resolve }) => {
|
||||||
@@ -11,17 +11,15 @@ export const handle: Handle = async ({ event, resolve }) => {
|
|||||||
const session = building ? null : await auth.api.getSession({ headers: event.request.headers });
|
const session = building ? null : await auth.api.getSession({ headers: event.request.headers });
|
||||||
event.locals.user = session?.user ?? null;
|
event.locals.user = session?.user ?? null;
|
||||||
event.locals.session = session?.session ?? null;
|
event.locals.session = session?.session ?? null;
|
||||||
// Owner-only areas (health tracker). Page loads, __data.json and the import endpoint
|
// Owner-only areas (health tracker, book files, reader and book APIs). Page loads,
|
||||||
// 404 for non-owners in the route itself (layout load / endpoint), which renders the
|
// __data.json and endpoints 404 for non-owners in the route itself (layout load /
|
||||||
// same 404 page as an unknown URL. As a backstop, any other method (form actions,
|
// endpoint), which renders the same 404 page as an unknown URL. As a backstop, any other
|
||||||
// which skip layout loads) is refused here before a route runs.
|
// method (form actions, which skip layout loads) is refused here before a route runs,
|
||||||
|
// including writes to the otherwise public /trackers/books pages.
|
||||||
const ownerOnly = isOwnerOnlyPath(event.url.pathname);
|
const ownerOnly = isOwnerOnlyPath(event.url.pathname);
|
||||||
if (
|
const owner = isOwner(event);
|
||||||
ownerOnly &&
|
const read = event.request.method === 'GET' || event.request.method === 'HEAD';
|
||||||
!isOwner(event) &&
|
if (!owner && ((ownerOnly && !read) || (!read && isOwnerWritePath(event.url.pathname))))
|
||||||
event.request.method !== 'GET' &&
|
|
||||||
event.request.method !== 'HEAD'
|
|
||||||
)
|
|
||||||
error(404, 'Not Found');
|
error(404, 'Not Found');
|
||||||
const response = await resolve(event);
|
const response = await resolve(event);
|
||||||
if (ownerOnly) {
|
if (ownerOnly) {
|
||||||
|
|||||||
133
src/lib/books/BookCover.svelte
Normal file
133
src/lib/books/BookCover.svelte
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
// A book cover: the real image when there is one, otherwise a generated cover in a
|
||||||
|
// colour derived from the title (also used when the image fails to load).
|
||||||
|
import { authorLine, coverHue } from './shared.js';
|
||||||
|
|
||||||
|
let {
|
||||||
|
src,
|
||||||
|
title,
|
||||||
|
authors = [],
|
||||||
|
placeholder = false,
|
||||||
|
size = 'md',
|
||||||
|
class: className = ''
|
||||||
|
}: {
|
||||||
|
src: string | null;
|
||||||
|
title: string;
|
||||||
|
authors?: readonly string[];
|
||||||
|
placeholder?: boolean;
|
||||||
|
size?: 'sm' | 'md' | 'lg';
|
||||||
|
class?: string;
|
||||||
|
} = $props();
|
||||||
|
|
||||||
|
let failed = $state<string | null>(null);
|
||||||
|
const showImage = $derived(!!src && failed !== src);
|
||||||
|
const hue = $derived(coverHue(title));
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div
|
||||||
|
class={['cover', `cover-${size}`, placeholder && 'is-placeholder', className]}
|
||||||
|
style:--hue={hue}
|
||||||
|
>
|
||||||
|
{#if showImage}
|
||||||
|
<img
|
||||||
|
{src}
|
||||||
|
alt=""
|
||||||
|
loading="lazy"
|
||||||
|
decoding="async"
|
||||||
|
referrerpolicy="no-referrer"
|
||||||
|
onerror={() => (failed = src)}
|
||||||
|
/>
|
||||||
|
{:else}
|
||||||
|
<div class="generated" aria-hidden="true">
|
||||||
|
<span class="g-title">{title}</span>
|
||||||
|
{#if authors.length}<span class="g-author">{authorLine(authors)}</span>{/if}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.cover {
|
||||||
|
position: relative;
|
||||||
|
aspect-ratio: 2 / 3;
|
||||||
|
width: 100%;
|
||||||
|
overflow: hidden;
|
||||||
|
container-type: inline-size;
|
||||||
|
border-radius: 0.375rem 0.625rem 0.625rem 0.375rem;
|
||||||
|
background: var(--site-hover);
|
||||||
|
box-shadow:
|
||||||
|
0 1px 2px rgb(0 0 0 / 0.25),
|
||||||
|
0 8px 24px -12px rgb(0 0 0 / 0.5);
|
||||||
|
}
|
||||||
|
/* Spine highlight, like a real paperback. */
|
||||||
|
.cover::after {
|
||||||
|
content: '';
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
pointer-events: none;
|
||||||
|
background: linear-gradient(
|
||||||
|
90deg,
|
||||||
|
rgb(0 0 0 / 0.28) 0,
|
||||||
|
rgb(255 255 255 / 0.12) 2.5%,
|
||||||
|
rgb(0 0 0 / 0.08) 5%,
|
||||||
|
transparent 9%
|
||||||
|
);
|
||||||
|
border-radius: inherit;
|
||||||
|
}
|
||||||
|
img {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
object-fit: cover;
|
||||||
|
}
|
||||||
|
.generated {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
justify-content: space-between;
|
||||||
|
padding: 14% 10% 10% 14%;
|
||||||
|
color: hsl(var(--hue) 30% 96%);
|
||||||
|
background:
|
||||||
|
radial-gradient(
|
||||||
|
120% 80% at 100% 0%,
|
||||||
|
hsl(calc(var(--hue) + 40) 60% 45% / 0.55),
|
||||||
|
transparent 60%
|
||||||
|
),
|
||||||
|
linear-gradient(160deg, hsl(var(--hue) 45% 32%), hsl(calc(var(--hue) + 20) 50% 16%));
|
||||||
|
}
|
||||||
|
.g-title {
|
||||||
|
font-family: Georgia, 'Times New Roman', serif;
|
||||||
|
font-weight: 600;
|
||||||
|
line-height: 1.15;
|
||||||
|
/* Scale with the cover's width so thumbnails don't break words mid-way. */
|
||||||
|
font-size: clamp(0.55rem, 12cqi, 1.6rem);
|
||||||
|
display: -webkit-box;
|
||||||
|
-webkit-line-clamp: 6;
|
||||||
|
line-clamp: 6;
|
||||||
|
-webkit-box-orient: vertical;
|
||||||
|
overflow: hidden;
|
||||||
|
overflow-wrap: break-word;
|
||||||
|
hyphens: auto;
|
||||||
|
}
|
||||||
|
.g-author {
|
||||||
|
font-size: clamp(0.5rem, 6.5cqi, 0.85rem);
|
||||||
|
letter-spacing: 0.04em;
|
||||||
|
text-transform: uppercase;
|
||||||
|
opacity: 0.8;
|
||||||
|
}
|
||||||
|
@container (max-width: 90px) {
|
||||||
|
.g-author {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.is-placeholder {
|
||||||
|
box-shadow: none;
|
||||||
|
outline: 1.5px dashed color-mix(in srgb, var(--site-muted) 60%, transparent);
|
||||||
|
outline-offset: 2px;
|
||||||
|
}
|
||||||
|
.is-placeholder img,
|
||||||
|
.is-placeholder .generated {
|
||||||
|
filter: grayscale(0.85) opacity(0.55);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
68
src/lib/books/Stars.svelte
Normal file
68
src/lib/books/Stars.svelte
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
// 1-5 star rating. Read-only by default; with `name` it renders radio buttons for a form.
|
||||||
|
import Star from 'lucide-svelte/icons/star';
|
||||||
|
|
||||||
|
let {
|
||||||
|
value,
|
||||||
|
size = 14,
|
||||||
|
name,
|
||||||
|
onchange
|
||||||
|
}: {
|
||||||
|
value: number | null;
|
||||||
|
size?: number;
|
||||||
|
name?: string;
|
||||||
|
onchange?: (value: number | null) => void;
|
||||||
|
} = $props();
|
||||||
|
|
||||||
|
let hover = $state<number | null>(null);
|
||||||
|
const shown = $derived(hover ?? value ?? 0);
|
||||||
|
</script>
|
||||||
|
|
||||||
|
{#if name || onchange}
|
||||||
|
<span class="stars interactive" role="radiogroup" aria-label="Rating">
|
||||||
|
{#each [1, 2, 3, 4, 5] as n (n)}
|
||||||
|
<button
|
||||||
|
type={name ? 'submit' : 'button'}
|
||||||
|
{name}
|
||||||
|
value={value === n ? '' : String(n)}
|
||||||
|
role="radio"
|
||||||
|
aria-checked={value === n}
|
||||||
|
aria-label={`${n} star${n > 1 ? 's' : ''}${value === n ? ' (click to clear)' : ''}`}
|
||||||
|
onpointerenter={() => (hover = n)}
|
||||||
|
onpointerleave={() => (hover = null)}
|
||||||
|
onclick={() => onchange?.(value === n ? null : n)}
|
||||||
|
>
|
||||||
|
<Star {size} class={n <= shown ? 'on' : 'off'} />
|
||||||
|
</button>
|
||||||
|
{/each}
|
||||||
|
</span>
|
||||||
|
{:else if value}
|
||||||
|
<span class="stars" aria-label={`${value} out of 5 stars`} title={`${value}/5`}>
|
||||||
|
{#each [1, 2, 3, 4, 5] as n (n)}
|
||||||
|
<Star {size} class={n <= value ? 'on' : 'off'} aria-hidden="true" />
|
||||||
|
{/each}
|
||||||
|
</span>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.stars {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 1px;
|
||||||
|
}
|
||||||
|
.stars :global(.on) {
|
||||||
|
fill: #f5b301;
|
||||||
|
color: #f5b301;
|
||||||
|
}
|
||||||
|
.stars :global(.off) {
|
||||||
|
color: color-mix(in srgb, var(--site-muted) 50%, transparent);
|
||||||
|
}
|
||||||
|
.interactive button {
|
||||||
|
padding: 2px;
|
||||||
|
border-radius: 4px;
|
||||||
|
line-height: 0;
|
||||||
|
}
|
||||||
|
.interactive button:hover {
|
||||||
|
transform: scale(1.12);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
56
src/lib/books/progress.test.ts
Normal file
56
src/lib/books/progress.test.ts
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { pagesRead, parseProgress } from './progress';
|
||||||
|
import { parsePdfLocator, pdfLocator } from './shared';
|
||||||
|
|
||||||
|
describe('progress payload', () => {
|
||||||
|
it('accepts a reader save and rounds the percent', () => {
|
||||||
|
expect(
|
||||||
|
parseProgress({
|
||||||
|
fileId: 3,
|
||||||
|
locator: 'epubcfi(/6/4!/4/2)',
|
||||||
|
percent: 12.34567,
|
||||||
|
sessionId: null
|
||||||
|
})
|
||||||
|
).toMatchObject({ fileId: 3, locator: 'epubcfi(/6/4!/4/2)', percent: 12.35 });
|
||||||
|
});
|
||||||
|
it.each([
|
||||||
|
null,
|
||||||
|
'nope',
|
||||||
|
{},
|
||||||
|
{ percent: -1 },
|
||||||
|
{ percent: 101 },
|
||||||
|
{ percent: 'ten' },
|
||||||
|
{ percent: 10, page: 1.5 },
|
||||||
|
{ percent: 10, fileId: -2 },
|
||||||
|
{ percent: 10, locator: 'x'.repeat(2001) }
|
||||||
|
])('rejects %j', (body) => {
|
||||||
|
expect(parseProgress(body)).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('pages read', () => {
|
||||||
|
it('uses the page delta when pages are known', () => {
|
||||||
|
expect(pagesRead({ startPage: 10, endPage: 34 })).toBe(24);
|
||||||
|
});
|
||||||
|
it('scales the percent delta by the page count', () => {
|
||||||
|
expect(pagesRead({ startPercent: 10, endPercent: 20, pageCount: 300 })).toBe(30);
|
||||||
|
});
|
||||||
|
it('never goes negative and needs enough data', () => {
|
||||||
|
expect(pagesRead({ startPage: 50, endPage: 20 })).toBe(0);
|
||||||
|
expect(pagesRead({ startPercent: 40, endPercent: 30, pageCount: 100 })).toBe(0);
|
||||||
|
expect(pagesRead({ startPercent: 10, endPercent: 20 })).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('pdf locators', () => {
|
||||||
|
it('round-trips pages and rects', () => {
|
||||||
|
const loc = pdfLocator(12, [{ x: 0.1, y: 0.2, w: 0.3333333, h: 0.02 }]);
|
||||||
|
expect(loc).toBe('page:12#r=0.1,0.2,0.3333,0.02');
|
||||||
|
expect(parsePdfLocator(loc)).toEqual({
|
||||||
|
page: 12,
|
||||||
|
rects: [{ x: 0.1, y: 0.2, w: 0.3333, h: 0.02 }]
|
||||||
|
});
|
||||||
|
expect(parsePdfLocator('page:3')).toEqual({ page: 3, rects: [] });
|
||||||
|
expect(parsePdfLocator('epubcfi(/6/2)')).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
41
src/lib/books/progress.ts
Normal file
41
src/lib/books/progress.ts
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
// Validation and maths for the reader's progress API (pure, unit tested).
|
||||||
|
import * as z from 'zod';
|
||||||
|
|
||||||
|
export const progressPayload = z.object({
|
||||||
|
fileId: z.number().int().positive().nullish(),
|
||||||
|
locator: z.string().max(2000).nullish(),
|
||||||
|
page: z.number().int().min(0).max(100000).nullish(),
|
||||||
|
percent: z.number().min(0).max(100),
|
||||||
|
sessionId: z.number().int().positive().nullish(),
|
||||||
|
/** Total pages in the opened file (PDF), used for pages-read on sessions. */
|
||||||
|
pageCount: z.number().int().positive().max(100000).nullish()
|
||||||
|
});
|
||||||
|
export type ProgressPayload = z.infer<typeof progressPayload>;
|
||||||
|
|
||||||
|
export function parseProgress(body: unknown): ProgressPayload | null {
|
||||||
|
const result = progressPayload.safeParse(body);
|
||||||
|
if (!result.success) return null;
|
||||||
|
const value = result.data;
|
||||||
|
return { ...value, percent: Math.round(value.percent * 100) / 100 };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pages read in a session: the page delta when both pages are known (PDF), otherwise the
|
||||||
|
* percent delta scaled by the book's page count. Never negative (going back isn't reading).
|
||||||
|
*/
|
||||||
|
export function pagesRead(input: {
|
||||||
|
startPage?: number | null;
|
||||||
|
endPage?: number | null;
|
||||||
|
startPercent?: number | null;
|
||||||
|
endPercent?: number | null;
|
||||||
|
pageCount?: number | null;
|
||||||
|
}): number | null {
|
||||||
|
const { startPage, endPage, startPercent, endPercent, pageCount } = input;
|
||||||
|
if (startPage != null && endPage != null) return Math.max(0, endPage - startPage);
|
||||||
|
if (startPercent != null && endPercent != null && pageCount)
|
||||||
|
return Math.max(0, Math.round(((endPercent - startPercent) / 100) * pageCount));
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A reader session continues if the last save was less than this long ago. */
|
||||||
|
export const SESSION_GAP_MS = 30 * 60 * 1000;
|
||||||
255
src/lib/books/reader/EpubReader.svelte
Normal file
255
src/lib/books/reader/EpubReader.svelte
Normal file
@@ -0,0 +1,255 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
// Reflowable books (EPUB, MOBI/AZW3, FB2, CBZ) via the vendored foliate-js.
|
||||||
|
import { onMount, untrack } from 'svelte';
|
||||||
|
import { highlightCss, type Highlight, type HighlightColor } from '#lib/books/shared.js';
|
||||||
|
import { wheelZoom, type ViewLayout } from './layout.js';
|
||||||
|
import {
|
||||||
|
fetchWithProgress,
|
||||||
|
themeColors,
|
||||||
|
type EngineProps,
|
||||||
|
type ReaderSettings,
|
||||||
|
type ReaderTheme,
|
||||||
|
type TocItem
|
||||||
|
} from './types.js';
|
||||||
|
|
||||||
|
type FoliateToc = { label?: string; href?: string; subitems?: FoliateToc[] | null };
|
||||||
|
type Annotation = { value: string; color?: string };
|
||||||
|
type Renderer = HTMLElement & {
|
||||||
|
setStyles(css: string): void;
|
||||||
|
getContents(): { doc: Document; index: number }[];
|
||||||
|
};
|
||||||
|
interface FoliateView extends HTMLElement {
|
||||||
|
book: { toc?: FoliateToc[] };
|
||||||
|
renderer: Renderer;
|
||||||
|
open(file: File): Promise<void>;
|
||||||
|
init(options: { lastLocation?: unknown; showTextStart?: boolean }): Promise<void>;
|
||||||
|
close(): void;
|
||||||
|
goTo(target: string | number): Promise<unknown>;
|
||||||
|
goToFraction(fraction: number): Promise<void>;
|
||||||
|
goLeft(): Promise<void>;
|
||||||
|
goRight(): Promise<void>;
|
||||||
|
addAnnotation(annotation: Annotation): Promise<unknown>;
|
||||||
|
deleteAnnotation(annotation: Annotation): Promise<unknown>;
|
||||||
|
getCFI(index: number, range: Range): string;
|
||||||
|
}
|
||||||
|
type DrawFn = (rects: unknown, options: Record<string, unknown>) => SVGElement;
|
||||||
|
|
||||||
|
let {
|
||||||
|
url,
|
||||||
|
name,
|
||||||
|
start,
|
||||||
|
startPercent,
|
||||||
|
highlights,
|
||||||
|
settings,
|
||||||
|
layout,
|
||||||
|
theme,
|
||||||
|
onzoom,
|
||||||
|
onready,
|
||||||
|
onrelocate,
|
||||||
|
onselection,
|
||||||
|
onhighlightclick,
|
||||||
|
onkey,
|
||||||
|
onloading,
|
||||||
|
onerror
|
||||||
|
}: EngineProps = $props();
|
||||||
|
|
||||||
|
let host = $state<FoliateView>();
|
||||||
|
let view = $state<FoliateView>();
|
||||||
|
let highlightDraw: DrawFn | undefined;
|
||||||
|
// Zoom re-lays out the whole section, so follow fast changes (Ctrl+wheel, pinch) lazily.
|
||||||
|
let zoom = $state(untrack(() => layout.zoom));
|
||||||
|
$effect(() => {
|
||||||
|
const z = layout.zoom;
|
||||||
|
const t = setTimeout(() => (zoom = z), 120);
|
||||||
|
return () => clearTimeout(t);
|
||||||
|
});
|
||||||
|
|
||||||
|
const fonts = {
|
||||||
|
serif: 'Charter, "Iowan Old Style", Georgia, "Palatino Linotype", serif',
|
||||||
|
sans: 'system-ui, -apple-system, "Segoe UI", Roboto, sans-serif'
|
||||||
|
};
|
||||||
|
|
||||||
|
function css(s: ReaderSettings, t: ReaderTheme, z: number) {
|
||||||
|
const c = themeColors[t];
|
||||||
|
const family = s.font === 'publisher' ? null : fonts[s.font];
|
||||||
|
return `
|
||||||
|
@namespace epub "http://www.idpf.org/2007/ops";
|
||||||
|
html { color-scheme: ${t === 'dark' ? 'dark' : 'light'}; background: ${c.bg} !important; color: ${c.fg} !important; font-size: ${Math.round(z * 100)}% !important; }
|
||||||
|
body { background: transparent !important; color: inherit !important; ${family ? `font-family: ${family} !important;` : ''} }
|
||||||
|
${family ? 'body :not(pre, code, kbd, samp, tt, pre *, code *) { font-family: inherit !important; }' : ''}
|
||||||
|
p, li, blockquote, dd, dt { line-height: ${s.lineHeight} !important; }
|
||||||
|
p { hyphens: auto; -webkit-hyphens: auto; widows: 2; orphans: 2; }
|
||||||
|
a:any-link { color: ${c.link} !important; }
|
||||||
|
${t === 'light' ? '' : 'body *:not(a, a *) { color: inherit !important; background-color: transparent !important; border-color: currentColor; }'}
|
||||||
|
img, svg, video { max-width: 100%; height: auto; }
|
||||||
|
aside[epub|type~="footnote"], aside[epub|type~="endnote"] { display: none; }
|
||||||
|
::selection { background: rgba(250, 204, 21, 0.45); }`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fixed-layout books (comics, FXL EPUB) use foliate's own page fitting and spreads.
|
||||||
|
const isFixed = (v: FoliateView) => v.renderer?.tagName === 'FOLIATE-FXL';
|
||||||
|
|
||||||
|
function apply(v: FoliateView, l: Pick<ViewLayout, 'mode' | 'spread'>, z: number) {
|
||||||
|
const r = v.renderer;
|
||||||
|
if (!isFixed(v)) {
|
||||||
|
r.setAttribute('flow', l.mode === 'scroll' ? 'scrolled' : 'paginated');
|
||||||
|
r.setAttribute('margin', '48px');
|
||||||
|
r.setAttribute('gap', '6%');
|
||||||
|
// Wider columns as the text grows, so a zoomed spread falls back to one column
|
||||||
|
// instead of squeezing a few words per line.
|
||||||
|
r.setAttribute('max-inline-size', `${Math.round(720 * Math.max(1, z))}px`);
|
||||||
|
r.setAttribute('max-column-count', String(l.spread));
|
||||||
|
}
|
||||||
|
r.setStyles?.(css(settings, theme, z));
|
||||||
|
v.style.background = themeColors[theme].bg;
|
||||||
|
}
|
||||||
|
|
||||||
|
$effect(() => {
|
||||||
|
if (view) apply(view, { mode: layout.mode, spread: layout.spread }, zoom);
|
||||||
|
});
|
||||||
|
|
||||||
|
const color = (h: Highlight) =>
|
||||||
|
highlightCss[(h.color as HighlightColor) ?? 'yellow'] ?? highlightCss.yellow;
|
||||||
|
const toToc = (items: FoliateToc[] = []): TocItem[] =>
|
||||||
|
items
|
||||||
|
.filter((i) => i.href)
|
||||||
|
.map((i) => ({
|
||||||
|
label: (i.label ?? '').trim() || 'Untitled',
|
||||||
|
target: i.href!,
|
||||||
|
children: toToc(i.subitems ?? [])
|
||||||
|
}));
|
||||||
|
|
||||||
|
export async function goTo(target: string | number) {
|
||||||
|
await view?.goTo(target);
|
||||||
|
}
|
||||||
|
export async function goToPercent(percent: number) {
|
||||||
|
await view?.goToFraction(Math.min(0.9999, Math.max(0, percent / 100)));
|
||||||
|
}
|
||||||
|
export function next() {
|
||||||
|
void view?.goRight();
|
||||||
|
}
|
||||||
|
export function prev() {
|
||||||
|
void view?.goLeft();
|
||||||
|
}
|
||||||
|
export function addHighlight(h: Highlight) {
|
||||||
|
if (h.locator) void view?.addAnnotation({ value: h.locator, color: color(h) }).catch(() => {});
|
||||||
|
}
|
||||||
|
export function removeHighlight(h: Highlight) {
|
||||||
|
if (h.locator) void view?.deleteAnnotation({ value: h.locator }).catch(() => {});
|
||||||
|
}
|
||||||
|
export function clearSelection() {
|
||||||
|
for (const { doc } of view?.renderer.getContents() ?? []) doc.getSelection()?.removeAllRanges();
|
||||||
|
}
|
||||||
|
|
||||||
|
onMount(() => {
|
||||||
|
const abort = new AbortController();
|
||||||
|
let disposed = false;
|
||||||
|
let el: FoliateView | undefined;
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
try {
|
||||||
|
const base = '/vendor/foliate-js';
|
||||||
|
const [, overlayer] = await Promise.all([
|
||||||
|
import(/* @vite-ignore */ `${base}/view.js`),
|
||||||
|
import(/* @vite-ignore */ `${base}/overlayer.js`)
|
||||||
|
]);
|
||||||
|
highlightDraw = overlayer.Overlayer.highlight;
|
||||||
|
onloading(0);
|
||||||
|
const blob = await fetchWithProgress(url, onloading, abort.signal);
|
||||||
|
if (disposed) return;
|
||||||
|
onloading(null);
|
||||||
|
|
||||||
|
el = host;
|
||||||
|
if (!el) return;
|
||||||
|
|
||||||
|
el.addEventListener('relocate', (e) => {
|
||||||
|
const d = (e as CustomEvent).detail;
|
||||||
|
onrelocate({
|
||||||
|
percent: Math.max(0, Math.min(100, (d.fraction ?? 0) * 100)),
|
||||||
|
locator: d.cfi,
|
||||||
|
page: null,
|
||||||
|
pageCount: null,
|
||||||
|
label: d.tocItem?.label?.trim() || null
|
||||||
|
});
|
||||||
|
});
|
||||||
|
el.addEventListener('load', (e) => {
|
||||||
|
const { doc, index } = (e as CustomEvent).detail as { doc: Document; index: number };
|
||||||
|
doc.addEventListener('keydown', onkey);
|
||||||
|
// Ctrl+wheel / trackpad pinch inside the book text zooms instead of the page.
|
||||||
|
doc.addEventListener(
|
||||||
|
'wheel',
|
||||||
|
(we) => {
|
||||||
|
if (!we.ctrlKey && !we.metaKey) return;
|
||||||
|
we.preventDefault();
|
||||||
|
onzoom(wheelZoom(layout.zoom, we.deltaY, 'foliate'));
|
||||||
|
},
|
||||||
|
{ passive: false }
|
||||||
|
);
|
||||||
|
let timer: ReturnType<typeof setTimeout>;
|
||||||
|
doc.addEventListener('selectionchange', () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
timer = setTimeout(() => {
|
||||||
|
const sel = doc.getSelection();
|
||||||
|
const text = sel?.toString().trim();
|
||||||
|
if (!sel || sel.isCollapsed || !sel.rangeCount || !text) return onselection(null);
|
||||||
|
try {
|
||||||
|
onselection({ text, locator: el!.getCFI(index, sel.getRangeAt(0)), page: null });
|
||||||
|
} catch {
|
||||||
|
onselection(null);
|
||||||
|
}
|
||||||
|
}, 250);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
el.addEventListener('create-overlay', () => {
|
||||||
|
for (const h of highlights) addHighlight(h);
|
||||||
|
});
|
||||||
|
el.addEventListener('draw-annotation', (e) => {
|
||||||
|
const { draw, annotation } = (e as CustomEvent).detail as {
|
||||||
|
draw: (fn: DrawFn, opts: Record<string, unknown>) => void;
|
||||||
|
annotation: Annotation;
|
||||||
|
};
|
||||||
|
if (highlightDraw)
|
||||||
|
draw(highlightDraw, { color: annotation.color ?? highlightCss.yellow });
|
||||||
|
});
|
||||||
|
el.addEventListener('show-annotation', (e) => {
|
||||||
|
const value = (e as CustomEvent).detail.value as string;
|
||||||
|
const h = highlights.find((x) => x.locator === value);
|
||||||
|
if (h) onhighlightclick(h.id);
|
||||||
|
});
|
||||||
|
|
||||||
|
await el.open(new File([blob], name, { type: blob.type }));
|
||||||
|
if (disposed) return;
|
||||||
|
apply(el, layout, layout.zoom);
|
||||||
|
view = el;
|
||||||
|
const lastLocation = start ?? (startPercent > 0 ? { fraction: startPercent / 100 } : null);
|
||||||
|
await el.init({ lastLocation, showTextStart: !lastLocation });
|
||||||
|
onready(toToc(el.book.toc), { fixedLayout: isFixed(el) });
|
||||||
|
} catch (e) {
|
||||||
|
if (!disposed && !(e instanceof DOMException && e.name === 'AbortError'))
|
||||||
|
onerror(e instanceof Error ? e.message : 'Could not open this book');
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
disposed = true;
|
||||||
|
abort.abort();
|
||||||
|
el?.close?.();
|
||||||
|
};
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class="host">
|
||||||
|
<foliate-view bind:this={host}></foliate-view>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.host {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
}
|
||||||
|
.host :global(foliate-view) {
|
||||||
|
display: block;
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
687
src/lib/books/reader/PdfReader.svelte
Normal file
687
src/lib/books/reader/PdfReader.svelte
Normal file
@@ -0,0 +1,687 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
// PDFs via pdf.js: continuous scroll or one page/spread at a time, 1 or 2 pages per row,
|
||||||
|
// pages rendered lazily near the viewport, a text layer for selection, and highlights stored
|
||||||
|
// as page-relative rectangles.
|
||||||
|
import { onMount, tick, untrack } from 'svelte';
|
||||||
|
import workerUrl from 'pdfjs-dist/build/pdf.worker.min.mjs?url';
|
||||||
|
import type { PDFDocumentProxy, PDFPageProxy, RenderTask } from 'pdfjs-dist';
|
||||||
|
import {
|
||||||
|
highlightCss,
|
||||||
|
parsePdfLocator,
|
||||||
|
pdfLocator,
|
||||||
|
type Highlight,
|
||||||
|
type HighlightColor,
|
||||||
|
type Rect
|
||||||
|
} from '#lib/books/shared.js';
|
||||||
|
import { clampZoom, pdfRows, rowIndexOf, type Spread } from './layout.js';
|
||||||
|
import { closePdf } from './pdf.js';
|
||||||
|
import { type EngineProps, type TocItem } from './types.js';
|
||||||
|
|
||||||
|
let {
|
||||||
|
url,
|
||||||
|
start,
|
||||||
|
startPercent,
|
||||||
|
highlights,
|
||||||
|
layout,
|
||||||
|
theme,
|
||||||
|
onzoom,
|
||||||
|
onready,
|
||||||
|
onrelocate,
|
||||||
|
onselection,
|
||||||
|
onhighlightclick,
|
||||||
|
onloading,
|
||||||
|
onerror
|
||||||
|
}: EngineProps = $props();
|
||||||
|
|
||||||
|
type PageInfo = { num: number; width: number; height: number }; // at scale 1
|
||||||
|
type Live = { task?: RenderTask; el: HTMLElement; layer: HTMLElement; scale: number };
|
||||||
|
const GAP = 12;
|
||||||
|
|
||||||
|
let scroller: HTMLDivElement;
|
||||||
|
let pdf: PDFDocumentProxy | undefined;
|
||||||
|
let pdfjs: typeof import('pdfjs-dist') | undefined;
|
||||||
|
let pages = $state<PageInfo[]>([]);
|
||||||
|
let width = $state(0);
|
||||||
|
let height = $state(0);
|
||||||
|
let current = 1;
|
||||||
|
let currentRow = $state(0); // paged mode: the row (page or spread) on screen
|
||||||
|
// eslint-disable-next-line svelte/prefer-svelte-reactivity -- plain cache, not reactive state
|
||||||
|
const live = new Map<number, Live>();
|
||||||
|
// eslint-disable-next-line svelte/prefer-svelte-reactivity -- plain cache, not reactive state
|
||||||
|
const pageEls = new Map<number, HTMLElement>();
|
||||||
|
let observer: IntersectionObserver | undefined;
|
||||||
|
|
||||||
|
const paged = $derived(layout.mode === 'paged');
|
||||||
|
// Two pages side by side only when there's room for them.
|
||||||
|
const spread = $derived<Spread>(layout.spread === 2 && width >= 640 ? 2 : 1);
|
||||||
|
const rows = $derived(pdfRows(pages.length, spread));
|
||||||
|
const shownRows = $derived(paged ? rows.slice(currentRow, currentRow + 1) : rows);
|
||||||
|
|
||||||
|
// Zoom 1 = fit the row's width (scroll) or the whole page/spread (paged).
|
||||||
|
const scale = $derived.by(() => {
|
||||||
|
const first = pages[0];
|
||||||
|
if (!first || !width) return 1;
|
||||||
|
const avail = width - 2 * GAP - (spread - 1) * GAP;
|
||||||
|
let fit = Math.min(avail, 920 * spread) / (first.width * spread);
|
||||||
|
if (paged && height > 2 * GAP) fit = Math.min(fit, (height - 2 * GAP) / first.height);
|
||||||
|
return Math.max(0.2, fit * layout.zoom);
|
||||||
|
});
|
||||||
|
|
||||||
|
const byPage = $derived.by(() => {
|
||||||
|
// eslint-disable-next-line svelte/prefer-svelte-reactivity -- plain cache, not reactive state
|
||||||
|
const map = new Map<number, { h: Highlight; rects: Rect[] }[]>();
|
||||||
|
for (const h of highlights) {
|
||||||
|
const loc = parsePdfLocator(h.locator);
|
||||||
|
if (!loc || !loc.rects.length) continue;
|
||||||
|
const list = map.get(loc.page) ?? [];
|
||||||
|
list.push({ h, rects: loc.rects });
|
||||||
|
map.set(loc.page, list);
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
});
|
||||||
|
const color = (h: Highlight) =>
|
||||||
|
highlightCss[(h.color as HighlightColor) ?? 'yellow'] ?? highlightCss.yellow;
|
||||||
|
|
||||||
|
async function render(num: number) {
|
||||||
|
const el = pageEls.get(num);
|
||||||
|
if (!pdf || !pdfjs || !el || live.has(num)) return;
|
||||||
|
const layer = el.querySelector<HTMLElement>('.layer')!;
|
||||||
|
const entry: Live = { el, layer, scale };
|
||||||
|
live.set(num, entry);
|
||||||
|
const page: PDFPageProxy = await pdf.getPage(num);
|
||||||
|
if (live.get(num) !== entry) return;
|
||||||
|
const s = scale;
|
||||||
|
entry.scale = s;
|
||||||
|
const viewport = page.getViewport({ scale: s });
|
||||||
|
const dpr = Math.min(window.devicePixelRatio || 1, 2);
|
||||||
|
const canvas = document.createElement('canvas');
|
||||||
|
canvas.width = Math.floor(viewport.width * dpr);
|
||||||
|
canvas.height = Math.floor(viewport.height * dpr);
|
||||||
|
const text = document.createElement('div');
|
||||||
|
text.className = 'textLayer';
|
||||||
|
layer.replaceChildren(canvas, text);
|
||||||
|
entry.task = page.render({
|
||||||
|
canvas,
|
||||||
|
viewport,
|
||||||
|
transform: dpr !== 1 ? [dpr, 0, 0, dpr, 0, 0] : undefined
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await entry.task.promise;
|
||||||
|
if (live.get(num) !== entry) return;
|
||||||
|
await new pdfjs.TextLayer({
|
||||||
|
textContentSource: page.streamTextContent(),
|
||||||
|
container: text,
|
||||||
|
viewport
|
||||||
|
}).render();
|
||||||
|
} catch {
|
||||||
|
// cancelled (scrolled away or re-zoomed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function unrender(num: number) {
|
||||||
|
const entry = live.get(num);
|
||||||
|
if (!entry) return;
|
||||||
|
entry.task?.cancel();
|
||||||
|
entry.layer.replaceChildren();
|
||||||
|
live.delete(num);
|
||||||
|
}
|
||||||
|
|
||||||
|
function page(node: HTMLElement, num: number) {
|
||||||
|
pageEls.set(num, node);
|
||||||
|
observer?.observe(node);
|
||||||
|
return {
|
||||||
|
destroy() {
|
||||||
|
observer?.unobserve(node);
|
||||||
|
pageEls.delete(num);
|
||||||
|
unrender(num);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// When the zoom, width, spread or mode changes: keep the reading position, resize the page
|
||||||
|
// boxes right away (old canvases stretch meanwhile) and re-render sharp once it settles.
|
||||||
|
const layoutKey = $derived(`${layout.mode}|${spread}|${scale.toFixed(4)}`);
|
||||||
|
let lastKey = '';
|
||||||
|
let keep: { page: number; fraction: number } | null = null;
|
||||||
|
$effect.pre(() => {
|
||||||
|
const key = layoutKey;
|
||||||
|
untrack(() => {
|
||||||
|
// The DOM still has the old layout here, so this reads the position the reader sees.
|
||||||
|
if (lastKey && key !== lastKey && pages.length) keep ??= position();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
$effect(() => {
|
||||||
|
const key = layoutKey;
|
||||||
|
untrack(() => {
|
||||||
|
if (!pages.length || !width) return void (keep = null);
|
||||||
|
// The first layout with pages is the baseline; the initial jump to the saved position
|
||||||
|
// happens in onMount and must not be undone by a "keep position" here.
|
||||||
|
if (!lastKey || key === lastKey) return void ((lastKey = key), (keep = null));
|
||||||
|
lastKey = key;
|
||||||
|
const k = keep;
|
||||||
|
keep = null;
|
||||||
|
if (k) {
|
||||||
|
if (paged) currentRow = rowIndexOf(rows, k.page);
|
||||||
|
void tick().then(() => {
|
||||||
|
place(k.page, k.fraction);
|
||||||
|
onscroll();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
scheduleRerender();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
let rerenderTimer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
function scheduleRerender() {
|
||||||
|
clearTimeout(rerenderTimer);
|
||||||
|
rerenderTimer = setTimeout(() => {
|
||||||
|
for (const [num, entry] of [...live])
|
||||||
|
if (Math.abs(entry.scale - scale) > 0.001) unrender(num);
|
||||||
|
if (!scroller) return;
|
||||||
|
const v = scroller.getBoundingClientRect();
|
||||||
|
for (const [num, node] of pageEls) {
|
||||||
|
const r = node.getBoundingClientRect();
|
||||||
|
if (r.bottom > v.top - v.height && r.top < v.bottom + v.height) void render(num);
|
||||||
|
}
|
||||||
|
}, 160);
|
||||||
|
}
|
||||||
|
|
||||||
|
function position() {
|
||||||
|
if (!scroller) return { page: 1, fraction: 0 };
|
||||||
|
const anchor = scroller.scrollTop + scroller.clientHeight * 0.3;
|
||||||
|
for (const p of pages) {
|
||||||
|
const node = pageEls.get(p.num);
|
||||||
|
if (!node) continue;
|
||||||
|
const top = node.offsetTop;
|
||||||
|
const bottom = top + node.offsetHeight;
|
||||||
|
if (anchor < bottom + 8)
|
||||||
|
return {
|
||||||
|
page: p.num,
|
||||||
|
fraction: Math.max(0, Math.min(1, (anchor - top) / node.offsetHeight))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { page: pages.length, fraction: 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scroll so `num` (at `fraction` down the page) is in view; it must be in the DOM. */
|
||||||
|
function place(num: number, fraction = 0) {
|
||||||
|
const node = pageEls.get(num);
|
||||||
|
if (!node || !scroller) return;
|
||||||
|
scroller.scrollTop =
|
||||||
|
node.offsetTop +
|
||||||
|
node.offsetHeight * fraction -
|
||||||
|
(fraction ? scroller.clientHeight * 0.3 : GAP);
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrollToPosition(num: number, fraction = 0) {
|
||||||
|
if (!pages.length) return;
|
||||||
|
num = Math.max(1, Math.min(pages.length, num));
|
||||||
|
if (paged) {
|
||||||
|
const r = rowIndexOf(rows, num);
|
||||||
|
if (r !== currentRow) {
|
||||||
|
currentRow = r;
|
||||||
|
void tick().then(() => {
|
||||||
|
place(num, fraction);
|
||||||
|
onscroll();
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
place(num, fraction);
|
||||||
|
onscroll(); // the scroll position may not change between rows
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
place(num, fraction);
|
||||||
|
}
|
||||||
|
|
||||||
|
function showRow(i: number) {
|
||||||
|
const row = rows[Math.max(0, Math.min(rows.length - 1, i))];
|
||||||
|
if (row) scrollToPosition(row[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
let frame = 0;
|
||||||
|
function onscroll() {
|
||||||
|
cancelAnimationFrame(frame);
|
||||||
|
frame = requestAnimationFrame(() => {
|
||||||
|
if (!pages.length) return;
|
||||||
|
const { page: num, fraction } = position();
|
||||||
|
current = num;
|
||||||
|
const row = rows[rowIndexOf(rows, num)];
|
||||||
|
onrelocate({
|
||||||
|
percent: Math.min(100, ((num - 1 + fraction) / pages.length) * 100),
|
||||||
|
locator: pdfLocator(num),
|
||||||
|
page: num,
|
||||||
|
pageEnd: row && row.length > 1 ? row[row.length - 1] : null,
|
||||||
|
pageCount: pages.length,
|
||||||
|
label: labelFor(num)
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let flatToc: { page: number; label: string }[] = [];
|
||||||
|
const labelFor = (num: number) => {
|
||||||
|
let label: string | null = null;
|
||||||
|
for (const item of flatToc) if (item.page <= num) label = item.label;
|
||||||
|
return label;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function goTo(target: string | number) {
|
||||||
|
if (typeof target === 'number') return scrollToPosition(target);
|
||||||
|
const loc = parsePdfLocator(target);
|
||||||
|
if (!loc) return;
|
||||||
|
const y = loc.rects[0]?.y;
|
||||||
|
if (y != null) scrollToPosition(loc.page, Math.max(0, y - 0.05) || 0.0001);
|
||||||
|
else scrollToPosition(loc.page);
|
||||||
|
}
|
||||||
|
export async function goToPercent(percent: number) {
|
||||||
|
const p = (percent / 100) * pages.length;
|
||||||
|
scrollToPosition(Math.floor(p) + 1, p % 1 || 0);
|
||||||
|
}
|
||||||
|
// Next/prev move a whole row: one page, or two in spread mode.
|
||||||
|
export function next() {
|
||||||
|
if (paged) return showRow(currentRow + 1);
|
||||||
|
showRow(rowIndexOf(rows, current) + 1);
|
||||||
|
}
|
||||||
|
export function prev() {
|
||||||
|
if (paged) return showRow(currentRow - 1);
|
||||||
|
// Scrolled into the current row: back to its top first, else the previous row.
|
||||||
|
const r = rowIndexOf(rows, current);
|
||||||
|
const node = pageEls.get(rows[r]?.[0] ?? 1);
|
||||||
|
const into = node && scroller ? scroller.scrollTop - (node.offsetTop - GAP) : 0;
|
||||||
|
showRow(into > 24 ? r : r - 1);
|
||||||
|
}
|
||||||
|
export function addHighlight() {
|
||||||
|
// Highlights are derived from props; nothing to draw imperatively.
|
||||||
|
}
|
||||||
|
export function removeHighlight() {}
|
||||||
|
export function clearSelection() {
|
||||||
|
document.getSelection()?.removeAllRanges();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Selection rects as fractions of the page, merged per line. */
|
||||||
|
function selectionRects(range: Range, pageEl: HTMLElement): Rect[] {
|
||||||
|
const box = pageEl.getBoundingClientRect();
|
||||||
|
const rects = [...range.getClientRects()]
|
||||||
|
.filter((r) => r.width > 1 && r.height > 1)
|
||||||
|
.map((r) => ({
|
||||||
|
x: (r.left - box.left) / box.width,
|
||||||
|
y: (r.top - box.top) / box.height,
|
||||||
|
w: r.width / box.width,
|
||||||
|
h: r.height / box.height
|
||||||
|
}))
|
||||||
|
.filter((r) => r.x >= -0.01 && r.y >= -0.01 && r.x < 1 && r.y < 1)
|
||||||
|
.sort((a, b) => a.y - b.y || a.x - b.x);
|
||||||
|
const merged: Rect[] = [];
|
||||||
|
for (const r of rects) {
|
||||||
|
const last = merged.at(-1);
|
||||||
|
if (last && Math.abs(last.y - r.y) < last.h * 0.5 && r.x <= last.x + last.w + 0.02) {
|
||||||
|
const right = Math.max(last.x + last.w, r.x + r.w);
|
||||||
|
const bottom = Math.max(last.y + last.h, r.y + r.h);
|
||||||
|
last.x = Math.min(last.x, r.x);
|
||||||
|
last.y = Math.min(last.y, r.y);
|
||||||
|
last.w = right - last.x;
|
||||||
|
last.h = bottom - last.y;
|
||||||
|
} else merged.push({ ...r });
|
||||||
|
}
|
||||||
|
return merged.slice(0, 60);
|
||||||
|
}
|
||||||
|
|
||||||
|
function hitHighlight(e: MouseEvent, num: number) {
|
||||||
|
if (!document.getSelection()?.isCollapsed) return;
|
||||||
|
const box = (e.currentTarget as HTMLElement).getBoundingClientRect();
|
||||||
|
const x = (e.clientX - box.left) / box.width;
|
||||||
|
const y = (e.clientY - box.top) / box.height;
|
||||||
|
const hit = byPage
|
||||||
|
.get(num)
|
||||||
|
?.find(({ rects }) =>
|
||||||
|
rects.some((r) => x >= r.x && x <= r.x + r.w && y >= r.y && y <= r.y + r.h)
|
||||||
|
);
|
||||||
|
if (hit) onhighlightclick(hit.h.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveToc(doc: PDFDocumentProxy): Promise<TocItem[]> {
|
||||||
|
const outline = await doc.getOutline().catch(() => null);
|
||||||
|
if (!outline) return [];
|
||||||
|
let budget = 600;
|
||||||
|
const walk = async (items: typeof outline): Promise<TocItem[]> => {
|
||||||
|
const out: TocItem[] = [];
|
||||||
|
for (const item of items) {
|
||||||
|
if (budget-- <= 0) break;
|
||||||
|
let num: number | null = null;
|
||||||
|
try {
|
||||||
|
const dest =
|
||||||
|
typeof item.dest === 'string' ? await doc.getDestination(item.dest) : item.dest;
|
||||||
|
const ref = dest?.[0];
|
||||||
|
if (ref && typeof ref === 'object') num = (await doc.getPageIndex(ref)) + 1;
|
||||||
|
else if (typeof ref === 'number') num = ref + 1;
|
||||||
|
} catch {
|
||||||
|
num = null;
|
||||||
|
}
|
||||||
|
const children = item.items?.length ? await walk(item.items) : [];
|
||||||
|
if (num) {
|
||||||
|
out.push({ label: item.title.trim() || `Page ${num}`, target: num, children });
|
||||||
|
flatToc.push({ page: num, label: item.title.trim() });
|
||||||
|
} else out.push(...children);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
};
|
||||||
|
const toc = await walk(outline);
|
||||||
|
flatToc = flatToc.sort((a, b) => a.page - b.page);
|
||||||
|
return toc;
|
||||||
|
}
|
||||||
|
|
||||||
|
onMount(() => {
|
||||||
|
let disposed = false;
|
||||||
|
let loadingTask: ReturnType<typeof import('pdfjs-dist').getDocument> | undefined;
|
||||||
|
const measure = () => {
|
||||||
|
width = scroller.clientWidth;
|
||||||
|
height = scroller.clientHeight;
|
||||||
|
};
|
||||||
|
const resize = new ResizeObserver(measure);
|
||||||
|
resize.observe(scroller);
|
||||||
|
measure();
|
||||||
|
|
||||||
|
// Ctrl+wheel / trackpad pinch is handled by the shell; here: wheel turns pages in paged
|
||||||
|
// mode when the page fits, and two-finger pinch zooms on touch screens.
|
||||||
|
let wheelAcc = 0;
|
||||||
|
let wheelLock = 0;
|
||||||
|
const onWheel = (e: WheelEvent) => {
|
||||||
|
if (e.ctrlKey || e.metaKey || !paged) return;
|
||||||
|
if (scroller.scrollHeight > scroller.clientHeight + 1) return;
|
||||||
|
e.preventDefault();
|
||||||
|
if (Date.now() < wheelLock) return;
|
||||||
|
wheelAcc += e.deltaY;
|
||||||
|
if (Math.abs(wheelAcc) < 60) return;
|
||||||
|
if (wheelAcc > 0) next();
|
||||||
|
else prev();
|
||||||
|
wheelAcc = 0;
|
||||||
|
wheelLock = Date.now() + 350;
|
||||||
|
};
|
||||||
|
let pinch: { d: number; z: number } | null = null;
|
||||||
|
const dist = (t: TouchList) =>
|
||||||
|
Math.hypot(t[0].clientX - t[1].clientX, t[0].clientY - t[1].clientY) || 1;
|
||||||
|
const onTouchStart = (e: TouchEvent) => {
|
||||||
|
pinch = e.touches.length === 2 ? { d: dist(e.touches), z: layout.zoom } : null;
|
||||||
|
};
|
||||||
|
const onTouchMove = (e: TouchEvent) => {
|
||||||
|
if (!pinch || e.touches.length !== 2) return;
|
||||||
|
e.preventDefault();
|
||||||
|
const z = clampZoom((pinch.z * dist(e.touches)) / pinch.d, 'pdf');
|
||||||
|
if (Math.abs(z - layout.zoom) > 0.01) onzoom(z);
|
||||||
|
};
|
||||||
|
const onTouchEnd = (e: TouchEvent) => {
|
||||||
|
if (e.touches.length < 2) pinch = null;
|
||||||
|
};
|
||||||
|
scroller.addEventListener('wheel', onWheel, { passive: false });
|
||||||
|
scroller.addEventListener('touchstart', onTouchStart, { passive: true });
|
||||||
|
scroller.addEventListener('touchmove', onTouchMove, { passive: false });
|
||||||
|
scroller.addEventListener('touchend', onTouchEnd);
|
||||||
|
scroller.addEventListener('touchcancel', onTouchEnd);
|
||||||
|
observer = new IntersectionObserver(
|
||||||
|
(entries) => {
|
||||||
|
for (const entry of entries) {
|
||||||
|
const num = Number((entry.target as HTMLElement).dataset.page);
|
||||||
|
if (entry.isIntersecting) void render(num);
|
||||||
|
else unrender(num);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{ root: scroller, rootMargin: '150% 0px' }
|
||||||
|
);
|
||||||
|
|
||||||
|
let selTimer: ReturnType<typeof setTimeout>;
|
||||||
|
const onSelectionChange = () => {
|
||||||
|
clearTimeout(selTimer);
|
||||||
|
selTimer = setTimeout(() => {
|
||||||
|
const sel = document.getSelection();
|
||||||
|
const text = sel?.toString().trim();
|
||||||
|
const node = sel?.anchorNode;
|
||||||
|
const pageEl = (node instanceof Element ? node : node?.parentElement)?.closest<HTMLElement>(
|
||||||
|
'.pdf-page'
|
||||||
|
);
|
||||||
|
if (!sel || sel.isCollapsed || !text || !pageEl || !scroller.contains(pageEl))
|
||||||
|
return onselection(null);
|
||||||
|
const num = Number(pageEl.dataset.page);
|
||||||
|
const rects = selectionRects(sel.getRangeAt(0), pageEl);
|
||||||
|
if (!rects.length) return onselection(null);
|
||||||
|
onselection({ text, locator: pdfLocator(num, rects), page: num });
|
||||||
|
}, 250);
|
||||||
|
};
|
||||||
|
document.addEventListener('selectionchange', onSelectionChange);
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
try {
|
||||||
|
pdfjs = await import('pdfjs-dist');
|
||||||
|
pdfjs.GlobalWorkerOptions.workerSrc = workerUrl;
|
||||||
|
onloading(0);
|
||||||
|
const task = (loadingTask = pdfjs.getDocument({
|
||||||
|
url,
|
||||||
|
withCredentials: true,
|
||||||
|
rangeChunkSize: 1 << 20
|
||||||
|
}));
|
||||||
|
task.onProgress = ({ loaded, total }: { loaded: number; total: number }) =>
|
||||||
|
onloading(total ? Math.min(1, loaded / total) : null);
|
||||||
|
const doc = await task.promise;
|
||||||
|
if (disposed) return void closePdf(task);
|
||||||
|
pdf = doc;
|
||||||
|
onloading(null);
|
||||||
|
// Use the first page's size for every page up front; real sizes fill in quickly.
|
||||||
|
const first = (await doc.getPage(1)).getViewport({ scale: 1 });
|
||||||
|
pages = Array.from({ length: doc.numPages }, (_, i) => ({
|
||||||
|
num: i + 1,
|
||||||
|
width: first.width,
|
||||||
|
height: first.height
|
||||||
|
}));
|
||||||
|
void (async () => {
|
||||||
|
for (let i = 2; i <= Math.min(doc.numPages, 2000) && !disposed; i++) {
|
||||||
|
const vp = (await doc.getPage(i)).getViewport({ scale: 1 });
|
||||||
|
if (vp.width !== first.width || vp.height !== first.height)
|
||||||
|
pages[i - 1] = { num: i, width: vp.width, height: vp.height };
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
requestAnimationFrame(async () => {
|
||||||
|
const loc = parsePdfLocator(start);
|
||||||
|
if (loc) scrollToPosition(loc.page, loc.rects[0]?.y ?? 0);
|
||||||
|
else if (startPercent > 0) await goToPercent(startPercent);
|
||||||
|
onscroll();
|
||||||
|
onready(await resolveToc(doc));
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
if (!disposed) onerror(e instanceof Error ? e.message : 'Could not open this PDF');
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
disposed = true;
|
||||||
|
document.removeEventListener('selectionchange', onSelectionChange);
|
||||||
|
scroller.removeEventListener('wheel', onWheel);
|
||||||
|
scroller.removeEventListener('touchstart', onTouchStart);
|
||||||
|
scroller.removeEventListener('touchmove', onTouchMove);
|
||||||
|
scroller.removeEventListener('touchend', onTouchEnd);
|
||||||
|
scroller.removeEventListener('touchcancel', onTouchEnd);
|
||||||
|
clearTimeout(rerenderTimer);
|
||||||
|
resize.disconnect();
|
||||||
|
observer?.disconnect();
|
||||||
|
for (const num of [...live.keys()]) unrender(num);
|
||||||
|
// pdf.js 6 has no PDFDocumentProxy.destroy(); calling it threw on every unmount.
|
||||||
|
void closePdf(loadingTask);
|
||||||
|
};
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div
|
||||||
|
class={['scroller', `t-${theme}`]}
|
||||||
|
style:background={theme === 'light' ? '#e9e9e7' : theme === 'sepia' ? '#e6dcc4' : '#0b0b0c'}
|
||||||
|
bind:this={scroller}
|
||||||
|
{onscroll}
|
||||||
|
>
|
||||||
|
<div class={['pages', paged && 'paged', spread === 2 && 'spread']}>
|
||||||
|
{#each shownRows as row (row[0])}
|
||||||
|
<div class="row">
|
||||||
|
{#each row as num (num)}
|
||||||
|
{@const p = pages[num - 1]}
|
||||||
|
<!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_static_element_interactions -->
|
||||||
|
<div
|
||||||
|
class="pdf-page"
|
||||||
|
data-page={p.num}
|
||||||
|
style:width={`${Math.floor(p.width * scale)}px`}
|
||||||
|
style:height={`${Math.floor(p.height * scale)}px`}
|
||||||
|
style:--scale-factor={scale}
|
||||||
|
style:--total-scale-factor={scale}
|
||||||
|
use:page={p.num}
|
||||||
|
onclick={(e) => hitHighlight(e, p.num)}
|
||||||
|
>
|
||||||
|
<div class="layer"></div>
|
||||||
|
{#each byPage.get(p.num) ?? [] as { h, rects } (h.id)}
|
||||||
|
{#each rects as r, i (i)}
|
||||||
|
<span
|
||||||
|
class="hl"
|
||||||
|
style:left={`${r.x * 100}%`}
|
||||||
|
style:top={`${r.y * 100}%`}
|
||||||
|
style:width={`${r.w * 100}%`}
|
||||||
|
style:height={`${r.h * 100}%`}
|
||||||
|
style:background={color(h)}
|
||||||
|
></span>
|
||||||
|
{/each}
|
||||||
|
{/each}
|
||||||
|
<span class="num">{p.num}</span>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.scroller {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
overflow: auto;
|
||||||
|
overscroll-behavior: contain;
|
||||||
|
}
|
||||||
|
.pages {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
gap: 12px;
|
||||||
|
padding: 12px 12px 96px;
|
||||||
|
min-width: fit-content;
|
||||||
|
}
|
||||||
|
.pages.paged {
|
||||||
|
min-height: 100%;
|
||||||
|
justify-content: center;
|
||||||
|
padding: 12px;
|
||||||
|
}
|
||||||
|
/* rows are not positioned, so page offsetTop stays relative to the scroller */
|
||||||
|
.row {
|
||||||
|
display: flex;
|
||||||
|
align-items: flex-start;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
.spread .num {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
.pdf-page {
|
||||||
|
position: relative;
|
||||||
|
background: white;
|
||||||
|
box-shadow:
|
||||||
|
0 1px 3px rgb(0 0 0 / 0.12),
|
||||||
|
0 8px 24px rgb(0 0 0 / 0.08);
|
||||||
|
flex: none;
|
||||||
|
}
|
||||||
|
.t-sepia .pdf-page :global(canvas) {
|
||||||
|
filter: sepia(0.35) brightness(0.97);
|
||||||
|
}
|
||||||
|
.t-dark .pdf-page {
|
||||||
|
background: #1a1a1c;
|
||||||
|
}
|
||||||
|
.t-dark .pdf-page :global(canvas) {
|
||||||
|
filter: invert(0.9) hue-rotate(180deg);
|
||||||
|
}
|
||||||
|
.layer,
|
||||||
|
.layer :global(canvas) {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
.hl {
|
||||||
|
position: absolute;
|
||||||
|
mix-blend-mode: multiply;
|
||||||
|
opacity: 0.45;
|
||||||
|
border-radius: 2px;
|
||||||
|
pointer-events: none;
|
||||||
|
z-index: 1;
|
||||||
|
}
|
||||||
|
.t-dark .hl {
|
||||||
|
mix-blend-mode: screen;
|
||||||
|
opacity: 0.35;
|
||||||
|
}
|
||||||
|
.num {
|
||||||
|
position: absolute;
|
||||||
|
right: -2.25rem;
|
||||||
|
bottom: 0;
|
||||||
|
font-size: 0.7rem;
|
||||||
|
color: rgb(128 128 128 / 0.8);
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
@media (max-width: 1100px) {
|
||||||
|
.num {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* pdf.js text layer (from pdfjs-dist/web/pdf_viewer.css, trimmed) */
|
||||||
|
.pdf-page :global(.textLayer) {
|
||||||
|
position: absolute;
|
||||||
|
text-align: initial;
|
||||||
|
inset: 0;
|
||||||
|
overflow: clip;
|
||||||
|
opacity: 1;
|
||||||
|
line-height: 1;
|
||||||
|
text-size-adjust: none;
|
||||||
|
forced-color-adjust: none;
|
||||||
|
transform-origin: 0 0;
|
||||||
|
caret-color: CanvasText;
|
||||||
|
z-index: 2;
|
||||||
|
--min-font-size: 1;
|
||||||
|
--text-scale-factor: calc(var(--total-scale-factor) * var(--min-font-size));
|
||||||
|
--min-font-size-inv: calc(1 / var(--min-font-size));
|
||||||
|
--scale-round-x: 1px;
|
||||||
|
--scale-round-y: 1px;
|
||||||
|
}
|
||||||
|
.pdf-page :global(.textLayer :is(span, br)) {
|
||||||
|
color: transparent;
|
||||||
|
position: absolute;
|
||||||
|
white-space: pre;
|
||||||
|
cursor: text;
|
||||||
|
transform-origin: 0% 0%;
|
||||||
|
user-select: text;
|
||||||
|
}
|
||||||
|
.pdf-page :global(.textLayer > :not(.markedContent)),
|
||||||
|
.pdf-page :global(.textLayer .markedContent span:not(.markedContent)) {
|
||||||
|
z-index: 1;
|
||||||
|
--font-height: 0;
|
||||||
|
font-size: calc(var(--text-scale-factor) * var(--font-height));
|
||||||
|
--scale-x: 1;
|
||||||
|
--rotate: 0deg;
|
||||||
|
transform: rotate(var(--rotate)) scaleX(var(--scale-x)) scale(var(--min-font-size-inv));
|
||||||
|
}
|
||||||
|
.pdf-page :global(.textLayer .markedContent) {
|
||||||
|
display: contents;
|
||||||
|
}
|
||||||
|
.pdf-page :global(.textLayer ::selection) {
|
||||||
|
background: rgb(250 204 21 / 0.4);
|
||||||
|
color: transparent;
|
||||||
|
}
|
||||||
|
.pdf-page :global(.textLayer br::selection) {
|
||||||
|
background: transparent;
|
||||||
|
}
|
||||||
|
.pdf-page :global(.textLayer .endOfContent) {
|
||||||
|
display: block;
|
||||||
|
position: absolute;
|
||||||
|
inset: 100% 0 0;
|
||||||
|
z-index: 0;
|
||||||
|
cursor: default;
|
||||||
|
user-select: none;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
111
src/lib/books/reader/layout.test.ts
Normal file
111
src/lib/books/reader/layout.test.ts
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import {
|
||||||
|
clampZoom,
|
||||||
|
defaultLayouts,
|
||||||
|
loadLayout,
|
||||||
|
normalizeLayout,
|
||||||
|
pdfRows,
|
||||||
|
rowIndexOf,
|
||||||
|
saveLayout,
|
||||||
|
stepZoom,
|
||||||
|
wheelZoom
|
||||||
|
} from './layout.js';
|
||||||
|
|
||||||
|
function memoryStore(init: Record<string, string> = {}) {
|
||||||
|
const data = new Map(Object.entries(init));
|
||||||
|
return {
|
||||||
|
data,
|
||||||
|
getItem: (k: string) => data.get(k) ?? null,
|
||||||
|
setItem: (k: string, v: string) => void data.set(k, v)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('zoom helpers', () => {
|
||||||
|
it('clamps per format and repairs junk', () => {
|
||||||
|
expect(clampZoom(10, 'pdf')).toBe(4);
|
||||||
|
expect(clampZoom(0.1, 'pdf')).toBe(0.5);
|
||||||
|
expect(clampZoom(10, 'foliate')).toBe(2.5);
|
||||||
|
expect(clampZoom(Number.NaN, 'pdf')).toBe(1);
|
||||||
|
expect(clampZoom(-2, 'foliate')).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('steps through presets from any value', () => {
|
||||||
|
expect(stepZoom(1, 1, 'pdf')).toBe(1.1);
|
||||||
|
expect(stepZoom(1, -1, 'pdf')).toBe(0.9);
|
||||||
|
expect(stepZoom(1.03, 1, 'pdf')).toBe(1.1);
|
||||||
|
expect(stepZoom(1.03, -1, 'pdf')).toBe(1);
|
||||||
|
expect(stepZoom(4, 1, 'pdf')).toBe(4);
|
||||||
|
expect(stepZoom(0.5, -1, 'pdf')).toBe(0.5);
|
||||||
|
expect(stepZoom(2.5, 1, 'foliate')).toBe(2.5);
|
||||||
|
expect(stepZoom(0.6, -1, 'foliate')).toBe(0.6);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('wheel zooms smoothly in the right direction', () => {
|
||||||
|
expect(wheelZoom(1, -100, 'pdf')).toBeGreaterThan(1);
|
||||||
|
expect(wheelZoom(1, 100, 'pdf')).toBeLessThan(1);
|
||||||
|
// small trackpad deltas still move
|
||||||
|
expect(wheelZoom(1, -2, 'pdf')).not.toBe(1);
|
||||||
|
expect(wheelZoom(4, -100, 'pdf')).toBe(4);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('pdf rows', () => {
|
||||||
|
it('single pages', () => {
|
||||||
|
expect(pdfRows(3, 1)).toEqual([[1], [2], [3]]);
|
||||||
|
expect(pdfRows(0, 2)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('spreads keep the cover alone', () => {
|
||||||
|
expect(pdfRows(1, 2)).toEqual([[1]]);
|
||||||
|
expect(pdfRows(4, 2)).toEqual([[1], [2, 3], [4]]);
|
||||||
|
expect(pdfRows(5, 2)).toEqual([[1], [2, 3], [4, 5]]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('finds the row of a page', () => {
|
||||||
|
const rows = pdfRows(5, 2);
|
||||||
|
expect(rowIndexOf(rows, 1)).toBe(0);
|
||||||
|
expect(rowIndexOf(rows, 3)).toBe(1);
|
||||||
|
expect(rowIndexOf(rows, 4)).toBe(2);
|
||||||
|
expect(rowIndexOf(rows, 99)).toBe(2);
|
||||||
|
expect(rowIndexOf(rows, -1)).toBe(0);
|
||||||
|
expect(rowIndexOf([], 3)).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('layout persistence', () => {
|
||||||
|
it('defaults per format', () => {
|
||||||
|
const s = memoryStore();
|
||||||
|
expect(loadLayout(s, 'pdf', 1)).toEqual(defaultLayouts.pdf);
|
||||||
|
expect(loadLayout(s, 'foliate', 1)).toEqual(defaultLayouts.foliate);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('remembers layout per format and zoom per book', () => {
|
||||||
|
const s = memoryStore();
|
||||||
|
saveLayout(s, 'pdf', 7, { mode: 'paged', spread: 2, zoom: 1.5 });
|
||||||
|
expect(loadLayout(s, 'pdf', 7)).toEqual({ mode: 'paged', spread: 2, zoom: 1.5 });
|
||||||
|
// another PDF inherits the format's last layout
|
||||||
|
expect(loadLayout(s, 'pdf', 8)).toEqual({ mode: 'paged', spread: 2, zoom: 1.5 });
|
||||||
|
saveLayout(s, 'pdf', 8, { mode: 'paged', spread: 2, zoom: 0.75 });
|
||||||
|
expect(loadLayout(s, 'pdf', 7).zoom).toBe(1.5);
|
||||||
|
// formats are independent
|
||||||
|
expect(loadLayout(s, 'foliate', 7)).toEqual(defaultLayouts.foliate);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('migrates the old settings once', () => {
|
||||||
|
const s = memoryStore({
|
||||||
|
'books-reader-settings': JSON.stringify({ flow: 'scrolled', fontSize: 120, pdfZoom: 1.3 })
|
||||||
|
});
|
||||||
|
expect(loadLayout(s, 'foliate', 1)).toEqual({ mode: 'scroll', spread: 2, zoom: 1.2 });
|
||||||
|
expect(loadLayout(s, 'pdf', 1)).toEqual({ mode: 'scroll', spread: 1, zoom: 1.3 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('survives broken storage', () => {
|
||||||
|
const s = memoryStore({ 'books-reader-layout:pdf': '{nope', 'books-reader-zoom:pdf:1': '"x"' });
|
||||||
|
expect(loadLayout(s, 'pdf', 1)).toEqual(defaultLayouts.pdf);
|
||||||
|
expect(normalizeLayout({ mode: 'x', spread: 3, zoom: 99 }, 'foliate')).toEqual({
|
||||||
|
mode: 'paged',
|
||||||
|
spread: 2,
|
||||||
|
zoom: 2.5
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
124
src/lib/books/reader/layout.ts
Normal file
124
src/lib/books/reader/layout.ts
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
// View layout for the reader: zoom, 1/2-page spreads and scroll vs paged reading.
|
||||||
|
// Pure helpers (unit tested) plus localStorage persistence: mode/spread/zoom are remembered
|
||||||
|
// per format (pdf vs reflowable), and the zoom is also remembered per book.
|
||||||
|
|
||||||
|
export type EngineKind = 'pdf' | 'foliate';
|
||||||
|
export type ViewMode = 'scroll' | 'paged';
|
||||||
|
export type Spread = 1 | 2;
|
||||||
|
export type ViewLayout = { mode: ViewMode; spread: Spread; zoom: number };
|
||||||
|
|
||||||
|
export const defaultLayouts: Record<EngineKind, ViewLayout> = {
|
||||||
|
pdf: { mode: 'scroll', spread: 1, zoom: 1 },
|
||||||
|
foliate: { mode: 'paged', spread: 2, zoom: 1 }
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Zoom limits: PDF zoom scales the page render (1 = fit), foliate zoom scales the text. */
|
||||||
|
export const zoomRange: Record<EngineKind, [min: number, max: number]> = {
|
||||||
|
pdf: [0.5, 4],
|
||||||
|
foliate: [0.6, 2.5]
|
||||||
|
};
|
||||||
|
const ZOOM_STEPS = [0.5, 0.6, 0.67, 0.75, 0.8, 0.9, 1, 1.1, 1.25, 1.5, 1.75, 2, 2.5, 3, 4];
|
||||||
|
|
||||||
|
export function clampZoom(zoom: number, kind: EngineKind): number {
|
||||||
|
const [min, max] = zoomRange[kind];
|
||||||
|
const z = Number.isFinite(zoom) && zoom > 0 ? zoom : 1;
|
||||||
|
return Math.round(Math.min(max, Math.max(min, z)) * 1000) / 1000;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Next preset zoom level up (dir 1) or down (dir -1) from wherever the zoom is now. */
|
||||||
|
export function stepZoom(zoom: number, dir: 1 | -1, kind: EngineKind): number {
|
||||||
|
const [min, max] = zoomRange[kind];
|
||||||
|
const z = clampZoom(zoom, kind);
|
||||||
|
const next =
|
||||||
|
dir > 0
|
||||||
|
? (ZOOM_STEPS.find((s) => s > z + 0.001) ?? max)
|
||||||
|
: (ZOOM_STEPS.findLast((s) => s < z - 0.001) ?? min);
|
||||||
|
return clampZoom(next, kind);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Smooth zoom for Ctrl+wheel and trackpad pinch (which browsers report as Ctrl+wheel). */
|
||||||
|
export function wheelZoom(zoom: number, deltaY: number, kind: EngineKind): number {
|
||||||
|
// A mouse notch (~100) is one ~10% step; pinch deltas are small and smooth.
|
||||||
|
const d = Math.max(-50, Math.min(50, deltaY));
|
||||||
|
return clampZoom(zoom * Math.exp(-d * 0.002), kind);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** PDF rows: one page per row, or book spreads with the cover on its own ([1], [2,3], [4,5]…). */
|
||||||
|
export function pdfRows(count: number, spread: Spread): number[][] {
|
||||||
|
const rows: number[][] = [];
|
||||||
|
if (count <= 0) return rows;
|
||||||
|
if (spread === 1) {
|
||||||
|
for (let n = 1; n <= count; n++) rows.push([n]);
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
rows.push([1]);
|
||||||
|
for (let n = 2; n <= count; n += 2) rows.push(n + 1 <= count ? [n, n + 1] : [n]);
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Index of the row containing `page` (clamped into range). */
|
||||||
|
export function rowIndexOf(rows: number[][], page: number): number {
|
||||||
|
if (!rows.length) return 0;
|
||||||
|
const idx = rows.findIndex((r) => r.includes(page));
|
||||||
|
if (idx >= 0) return idx;
|
||||||
|
return page < rows[0][0] ? 0 : rows.length - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
type Store = Pick<Storage, 'getItem' | 'setItem'>;
|
||||||
|
const formatKey = (kind: EngineKind) => `books-reader-layout:${kind}`;
|
||||||
|
const bookKey = (kind: EngineKind, bookId: number) => `books-reader-zoom:${kind}:${bookId}`;
|
||||||
|
const LEGACY_KEY = 'books-reader-settings';
|
||||||
|
|
||||||
|
function read(store: Store, key: string): unknown {
|
||||||
|
try {
|
||||||
|
return JSON.parse(store.getItem(key) ?? 'null');
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Sanitises anything (old/broken storage) into a valid layout. */
|
||||||
|
export function normalizeLayout(raw: unknown, kind: EngineKind): ViewLayout {
|
||||||
|
const base = defaultLayouts[kind];
|
||||||
|
const o = raw && typeof raw === 'object' ? (raw as Record<string, unknown>) : {};
|
||||||
|
return {
|
||||||
|
mode: o.mode === 'scroll' || o.mode === 'paged' ? o.mode : base.mode,
|
||||||
|
spread: o.spread === 1 || o.spread === 2 ? o.spread : base.spread,
|
||||||
|
zoom: typeof o.zoom === 'number' ? clampZoom(o.zoom, kind) : base.zoom
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function loadLayout(store: Store, kind: EngineKind, bookId: number): ViewLayout {
|
||||||
|
let saved = read(store, formatKey(kind));
|
||||||
|
if (!saved) {
|
||||||
|
// Carry over the pre-layout settings (flow, fontSize, pdfZoom) once.
|
||||||
|
const legacy = read(store, LEGACY_KEY) as Record<string, unknown> | null;
|
||||||
|
if (legacy && typeof legacy === 'object') {
|
||||||
|
saved =
|
||||||
|
kind === 'pdf'
|
||||||
|
? { zoom: legacy.pdfZoom }
|
||||||
|
: {
|
||||||
|
mode:
|
||||||
|
legacy.flow === 'scrolled'
|
||||||
|
? 'scroll'
|
||||||
|
: legacy.flow === 'paginated'
|
||||||
|
? 'paged'
|
||||||
|
: null,
|
||||||
|
zoom: typeof legacy.fontSize === 'number' ? legacy.fontSize / 100 : null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const layout = normalizeLayout(saved, kind);
|
||||||
|
const bookZoom = read(store, bookKey(kind, bookId));
|
||||||
|
if (typeof bookZoom === 'number') layout.zoom = clampZoom(bookZoom, kind);
|
||||||
|
return layout;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function saveLayout(store: Store, kind: EngineKind, bookId: number, layout: ViewLayout) {
|
||||||
|
try {
|
||||||
|
store.setItem(formatKey(kind), JSON.stringify(layout));
|
||||||
|
store.setItem(bookKey(kind, bookId), JSON.stringify(layout.zoom));
|
||||||
|
} catch {
|
||||||
|
// storage full or disabled: the layout still applies for this visit
|
||||||
|
}
|
||||||
|
}
|
||||||
39
src/lib/books/reader/pdf.test.ts
Normal file
39
src/lib/books/reader/pdf.test.ts
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { closePdf } from './pdf.js';
|
||||||
|
|
||||||
|
// A tiny one-page PDF (pdf.js tolerates the missing xref table).
|
||||||
|
const PDF =
|
||||||
|
'%PDF-1.4\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n' +
|
||||||
|
'2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj\n' +
|
||||||
|
'3 0 obj<</Type/Page/Parent 2 0 R/MediaBox[0 0 200 300]>>endobj\ntrailer<</Root 1 0 R>>\n%%EOF';
|
||||||
|
|
||||||
|
const open = async () => {
|
||||||
|
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
||||||
|
return pdfjs.getDocument({ data: new TextEncoder().encode(PDF) });
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('closePdf', () => {
|
||||||
|
it('the document itself has no destroy() in pdf.js 6 (the old teardown threw)', async () => {
|
||||||
|
const task = await open();
|
||||||
|
const doc = await task.promise;
|
||||||
|
expect(doc.numPages).toBe(1);
|
||||||
|
expect((doc as unknown as { destroy?: unknown }).destroy).toBeUndefined();
|
||||||
|
await closePdf(task);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('destroys a loaded document via its loading task, idempotently', async () => {
|
||||||
|
const task = await open();
|
||||||
|
await task.promise;
|
||||||
|
await closePdf(task);
|
||||||
|
expect(task.destroyed).toBe(true);
|
||||||
|
await expect(closePdf(task)).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('can close while still loading, or with nothing open', async () => {
|
||||||
|
const task = await open();
|
||||||
|
task.promise.catch(() => {}); // rejected by the destroy
|
||||||
|
await closePdf(task);
|
||||||
|
expect(task.destroyed).toBe(true);
|
||||||
|
await expect(closePdf(undefined)).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
18
src/lib/books/reader/pdf.ts
Normal file
18
src/lib/books/reader/pdf.ts
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
// pdf.js lifecycle helpers for the reader.
|
||||||
|
|
||||||
|
type LoadingTask = { destroyed: boolean; destroy(): Promise<void> };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Closes a PDF opened with pdfjs.getDocument(). pdf.js 6 removed PDFDocumentProxy.destroy();
|
||||||
|
* the loading task owns the document, its network requests and the worker, so tearing it
|
||||||
|
* down works both mid-load and after. Never throws: it runs while the reader unmounts
|
||||||
|
* (e.g. the back button), where an exception would break the navigation.
|
||||||
|
*/
|
||||||
|
export async function closePdf(task: LoadingTask | null | undefined): Promise<void> {
|
||||||
|
if (!task || task.destroyed) return;
|
||||||
|
try {
|
||||||
|
await task.destroy();
|
||||||
|
} catch {
|
||||||
|
// already torn down
|
||||||
|
}
|
||||||
|
}
|
||||||
89
src/lib/books/reader/types.ts
Normal file
89
src/lib/books/reader/types.ts
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
// Contract between the reader shell (the /read page) and the EPUB/PDF engines.
|
||||||
|
import type { Highlight } from '#lib/books/shared.js';
|
||||||
|
import type { ViewLayout } from './layout.js';
|
||||||
|
|
||||||
|
export type TocItem = { label: string; target: string | number; children: TocItem[] };
|
||||||
|
|
||||||
|
export type Relocation = {
|
||||||
|
percent: number; // 0-100
|
||||||
|
locator: string; // EPUB CFI or `page:N`
|
||||||
|
page: number | null; // PDF page (1-based), or null for reflowable books
|
||||||
|
pageEnd?: number | null; // last page shown when a PDF spread shows two
|
||||||
|
pageCount: number | null;
|
||||||
|
label: string | null; // current chapter
|
||||||
|
};
|
||||||
|
|
||||||
|
export type TextSelection = { text: string; locator: string; page: number | null };
|
||||||
|
|
||||||
|
export type ReaderTheme = 'light' | 'sepia' | 'dark';
|
||||||
|
export type ReaderSettings = {
|
||||||
|
theme: 'auto' | ReaderTheme;
|
||||||
|
lineHeight: number;
|
||||||
|
font: 'publisher' | 'serif' | 'sans';
|
||||||
|
};
|
||||||
|
export const defaultSettings: ReaderSettings = {
|
||||||
|
theme: 'auto',
|
||||||
|
lineHeight: 1.5,
|
||||||
|
font: 'publisher'
|
||||||
|
};
|
||||||
|
|
||||||
|
export const themeColors: Record<ReaderTheme, { bg: string; fg: string; link: string }> = {
|
||||||
|
light: { bg: '#fdfdfb', fg: '#1c1c1c', link: '#2563eb' },
|
||||||
|
sepia: { bg: '#f4ecd8', fg: '#433422', link: '#8a4b14' },
|
||||||
|
dark: { bg: '#121214', fg: '#d8d8d8', link: '#7aa8ff' }
|
||||||
|
};
|
||||||
|
|
||||||
|
/** What both engines expose to the shell through bind:this. */
|
||||||
|
export interface ReaderEngine {
|
||||||
|
goTo(target: string | number): Promise<void>;
|
||||||
|
goToPercent(percent: number): Promise<void>;
|
||||||
|
next(): void;
|
||||||
|
prev(): void;
|
||||||
|
addHighlight(h: Highlight): void;
|
||||||
|
removeHighlight(h: Highlight): void;
|
||||||
|
clearSelection(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type EngineProps = {
|
||||||
|
url: string;
|
||||||
|
name: string;
|
||||||
|
start: string | null;
|
||||||
|
startPercent: number;
|
||||||
|
highlights: Highlight[];
|
||||||
|
settings: ReaderSettings;
|
||||||
|
/** Zoom, 1/2-page spread and scroll vs paged (see layout.ts). */
|
||||||
|
layout: ViewLayout;
|
||||||
|
theme: ReaderTheme;
|
||||||
|
/** Ctrl+wheel / pinch inside the engine asks the shell for a new zoom. */
|
||||||
|
onzoom: (zoom: number) => void;
|
||||||
|
onready: (toc: TocItem[], info?: { fixedLayout?: boolean }) => void;
|
||||||
|
onrelocate: (r: Relocation) => void;
|
||||||
|
onselection: (s: TextSelection | null) => void;
|
||||||
|
onhighlightclick: (id: number) => void;
|
||||||
|
onkey: (e: KeyboardEvent) => void;
|
||||||
|
onloading: (fraction: number | null) => void;
|
||||||
|
onerror: (message: string) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Downloads a file with progress, so big books show a loading bar instead of a blank page. */
|
||||||
|
export async function fetchWithProgress(
|
||||||
|
url: string,
|
||||||
|
onprogress: (fraction: number | null) => void,
|
||||||
|
signal?: AbortSignal
|
||||||
|
): Promise<Blob> {
|
||||||
|
const res = await fetch(url, { signal, credentials: 'same-origin' });
|
||||||
|
if (!res.ok) throw new Error(res.status === 404 ? 'File not found' : `HTTP ${res.status}`);
|
||||||
|
const total = Number(res.headers.get('content-length')) || 0;
|
||||||
|
if (!res.body || !total) return res.blob();
|
||||||
|
const reader = res.body.getReader();
|
||||||
|
const chunks: Uint8Array<ArrayBuffer>[] = [];
|
||||||
|
let loaded = 0;
|
||||||
|
for (;;) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
chunks.push(value);
|
||||||
|
loaded += value.byteLength;
|
||||||
|
onprogress(Math.min(1, loaded / total));
|
||||||
|
}
|
||||||
|
return new Blob(chunks, { type: res.headers.get('content-type') ?? '' });
|
||||||
|
}
|
||||||
104
src/lib/books/shared.ts
Normal file
104
src/lib/books/shared.ts
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
// Shared (client + server) types and helpers for the book tracker.
|
||||||
|
export const readingStatuses = ['want_to_read', 'reading', 'finished', 'abandoned'] as const;
|
||||||
|
export type ReadingStatus = (typeof readingStatuses)[number];
|
||||||
|
|
||||||
|
export const statusLabels: Record<ReadingStatus, string> = {
|
||||||
|
want_to_read: 'want to read',
|
||||||
|
reading: 'reading',
|
||||||
|
finished: 'finished',
|
||||||
|
abandoned: 'abandoned'
|
||||||
|
};
|
||||||
|
|
||||||
|
export function isReadingStatus(value: unknown): value is ReadingStatus {
|
||||||
|
return typeof value === 'string' && (readingStatuses as readonly string[]).includes(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Formats the in-browser reader can open. MOBI/AZW3/FB2/CBZ go through foliate-js too. */
|
||||||
|
export const readableFormats = ['epub', 'pdf', 'mobi', 'azw3', 'azw', 'fb2', 'cbz'] as const;
|
||||||
|
export function isReadable(format: string): boolean {
|
||||||
|
return (readableFormats as readonly string[]).includes(format.toLowerCase());
|
||||||
|
}
|
||||||
|
/** Preferred file to open when a work has several formats. */
|
||||||
|
export const formatPreference = ['epub', 'azw3', 'mobi', 'azw', 'fb2', 'pdf', 'cbz'];
|
||||||
|
|
||||||
|
/** One card in the library. Public fields only, unless noted. */
|
||||||
|
export type BookCard = {
|
||||||
|
id: number;
|
||||||
|
title: string;
|
||||||
|
subtitle: string | null;
|
||||||
|
authors: string[];
|
||||||
|
year: number | null;
|
||||||
|
tags: string[];
|
||||||
|
pageCount: number | null;
|
||||||
|
cover: string | null; // URL (our cover endpoint or a remote cover), or null for a generated one
|
||||||
|
status: ReadingStatus | null; // null: never touched
|
||||||
|
percent: number;
|
||||||
|
rating: number | null;
|
||||||
|
lastReadAt: string | null;
|
||||||
|
finishedAt: string | null;
|
||||||
|
addedAt: string;
|
||||||
|
// Owner only (empty / false for everyone else):
|
||||||
|
formats: string[];
|
||||||
|
hasFile: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type LibraryStats = {
|
||||||
|
total: number;
|
||||||
|
finishedThisYear: number;
|
||||||
|
pagesThisYear: number;
|
||||||
|
reading: number;
|
||||||
|
year: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type Highlight = {
|
||||||
|
id: number;
|
||||||
|
fileId: number | null;
|
||||||
|
locator: string | null;
|
||||||
|
page: number | null;
|
||||||
|
highlight: string | null;
|
||||||
|
note: string | null;
|
||||||
|
color: string | null;
|
||||||
|
createdAt: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const highlightColors = ['yellow', 'green', 'blue', 'pink', 'purple'] as const;
|
||||||
|
export type HighlightColor = (typeof highlightColors)[number];
|
||||||
|
export const highlightCss: Record<HighlightColor, string> = {
|
||||||
|
yellow: '#facc15',
|
||||||
|
green: '#4ade80',
|
||||||
|
blue: '#60a5fa',
|
||||||
|
pink: '#f472b6',
|
||||||
|
purple: '#c084fc'
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Deterministic hue for generated covers, so a book always gets the same colour. */
|
||||||
|
export function coverHue(title: string): number {
|
||||||
|
let hash = 0;
|
||||||
|
for (const char of title) hash = (hash * 31 + char.codePointAt(0)!) >>> 0;
|
||||||
|
return hash % 360;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function authorLine(authors: readonly string[]): string {
|
||||||
|
if (authors.length <= 2) return authors.join(' & ');
|
||||||
|
return `${authors[0]} et al.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** PDF highlight locators: `page:12` or `page:12#r=x,y,w,h;x,y,w,h` (rects as page fractions). */
|
||||||
|
export type Rect = { x: number; y: number; w: number; h: number };
|
||||||
|
export function pdfLocator(page: number, rects: Rect[] = []): string {
|
||||||
|
const r = rects
|
||||||
|
.map((rect) => [rect.x, rect.y, rect.w, rect.h].map((n) => +n.toFixed(4)).join(','))
|
||||||
|
.join(';');
|
||||||
|
return r ? `page:${page}#r=${r}` : `page:${page}`;
|
||||||
|
}
|
||||||
|
export function parsePdfLocator(locator: string | null): { page: number; rects: Rect[] } | null {
|
||||||
|
const match = /^page:(\d+)(?:#r=([\d.,;-]+))?$/.exec(locator ?? '');
|
||||||
|
if (!match) return null;
|
||||||
|
const rects = (match[2] ?? '')
|
||||||
|
.split(';')
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((part) => part.split(',').map(Number))
|
||||||
|
.filter((n) => n.length === 4 && n.every(Number.isFinite))
|
||||||
|
.map(([x, y, w, h]) => ({ x, y, w, h }));
|
||||||
|
return { page: Number(match[1]), rects };
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
|
import BookOpen from 'lucide-svelte/icons/book-open';
|
||||||
import CandlestickChart from 'lucide-svelte/icons/candlestick-chart';
|
import CandlestickChart from 'lucide-svelte/icons/candlestick-chart';
|
||||||
import ChartNoAxesCombined from 'lucide-svelte/icons/chart-no-axes-combined';
|
import ChartNoAxesCombined from 'lucide-svelte/icons/chart-no-axes-combined';
|
||||||
import FolderKanban from 'lucide-svelte/icons/folder-kanban';
|
import FolderKanban from 'lucide-svelte/icons/folder-kanban';
|
||||||
@@ -32,11 +33,12 @@
|
|||||||
'derby lens': ChartNoAxesCombined,
|
'derby lens': ChartNoAxesCombined,
|
||||||
'world opinions': Globe2,
|
'world opinions': Globe2,
|
||||||
'language learner': Languages,
|
'language learner': Languages,
|
||||||
'money tracker': Wallet,
|
money: Wallet,
|
||||||
'cs prices': CandlestickChart,
|
'cs prices': CandlestickChart,
|
||||||
'house tracker': House,
|
'house tracker': House,
|
||||||
lists: ListChecks,
|
lists: ListChecks,
|
||||||
health: HeartPulse
|
health: HeartPulse,
|
||||||
|
books: BookOpen
|
||||||
};
|
};
|
||||||
|
|
||||||
const Icon = $derived(icons[name as keyof typeof icons] ?? FolderKanban);
|
const Icon = $derived(icons[name as keyof typeof icons] ?? FolderKanban);
|
||||||
|
|||||||
155
src/lib/components/data/FieldInput.svelte
Normal file
155
src/lib/components/data/FieldInput.svelte
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import type { Col } from './data-format.js';
|
||||||
|
|
||||||
|
let {
|
||||||
|
column,
|
||||||
|
value = $bindable(null),
|
||||||
|
lookupUrl,
|
||||||
|
autofocus = false,
|
||||||
|
compact = false,
|
||||||
|
onsave,
|
||||||
|
oncancel
|
||||||
|
}: {
|
||||||
|
column: Col;
|
||||||
|
value: string | null;
|
||||||
|
lookupUrl?: string;
|
||||||
|
autofocus?: boolean;
|
||||||
|
compact?: boolean;
|
||||||
|
onsave?: (move?: 'next' | 'prev') => void;
|
||||||
|
oncancel?: () => void;
|
||||||
|
} = $props();
|
||||||
|
|
||||||
|
const kind = $derived(column.kind);
|
||||||
|
const long = $derived(
|
||||||
|
kind === 'json' ||
|
||||||
|
column.uiType === 'longtext' ||
|
||||||
|
(kind === 'text' && (value?.length ?? 0) > 80)
|
||||||
|
);
|
||||||
|
const listId = `fk-${Math.random().toString(36).slice(2)}`;
|
||||||
|
let options = $state.raw<{ value: string; label: string }[]>([]);
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
|
||||||
|
async function lookup(q: string) {
|
||||||
|
if (!column.fk || !lookupUrl) return;
|
||||||
|
const url = `${lookupUrl}?column=${encodeURIComponent(column.name)}&q=${encodeURIComponent(q)}`;
|
||||||
|
const res = await fetch(url).catch(() => null);
|
||||||
|
if (res?.ok) options = (await res.json()).options;
|
||||||
|
}
|
||||||
|
function queueLookup(q: string) {
|
||||||
|
clearTimeout(timer);
|
||||||
|
timer = setTimeout(() => lookup(q), 200);
|
||||||
|
}
|
||||||
|
|
||||||
|
function focusOnMount(node: HTMLElement) {
|
||||||
|
if (!autofocus) return;
|
||||||
|
node.focus();
|
||||||
|
if (node instanceof HTMLInputElement && node.type === 'text') node.select();
|
||||||
|
if (column.fk) lookup('');
|
||||||
|
}
|
||||||
|
|
||||||
|
function keydown(e: KeyboardEvent) {
|
||||||
|
if (e.key === 'Escape') {
|
||||||
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
|
oncancel?.();
|
||||||
|
} else if (e.key === 'Enter' && (!long || e.ctrlKey || e.metaKey)) {
|
||||||
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
|
onsave?.();
|
||||||
|
} else if (e.key === 'Tab' && onsave) {
|
||||||
|
e.preventDefault();
|
||||||
|
e.stopPropagation();
|
||||||
|
onsave(e.shiftKey ? 'prev' : 'next');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const base =
|
||||||
|
'w-full rounded border border-[var(--site-border)] bg-[var(--site-bg)] px-2 py-1 text-sm focus:outline-2 focus:outline-[var(--site-text)]';
|
||||||
|
const placeholder = $derived(
|
||||||
|
value === null
|
||||||
|
? column.hasDefault
|
||||||
|
? `default${column.defaultExpr ? `: ${column.defaultExpr}` : ''}`
|
||||||
|
: 'null'
|
||||||
|
: ''
|
||||||
|
);
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class={['flex items-start gap-1', compact ? '' : 'w-full']}>
|
||||||
|
{#if kind === 'bool'}
|
||||||
|
<select
|
||||||
|
class={base}
|
||||||
|
value={value ?? ''}
|
||||||
|
onchange={(e) => (value = e.currentTarget.value === '' ? null : e.currentTarget.value)}
|
||||||
|
onkeydown={keydown}
|
||||||
|
{@attach focusOnMount}
|
||||||
|
aria-label={column.label}
|
||||||
|
>
|
||||||
|
{#if column.nullable || column.hasDefault}<option value=""
|
||||||
|
>{column.hasDefault ? '(default)' : 'null'}</option
|
||||||
|
>{/if}
|
||||||
|
<option value="true">true</option>
|
||||||
|
<option value="false">false</option>
|
||||||
|
</select>
|
||||||
|
{:else if kind === 'enum' || (column.uiType === 'select' && column.enumValues.length)}
|
||||||
|
<select
|
||||||
|
class={base}
|
||||||
|
value={value ?? ''}
|
||||||
|
onchange={(e) => (value = e.currentTarget.value === '' ? null : e.currentTarget.value)}
|
||||||
|
onkeydown={keydown}
|
||||||
|
{@attach focusOnMount}
|
||||||
|
aria-label={column.label}
|
||||||
|
>
|
||||||
|
{#if column.nullable || column.hasDefault}<option value=""
|
||||||
|
>{column.hasDefault ? '(default)' : 'null'}</option
|
||||||
|
>{/if}
|
||||||
|
{#each column.enumValues as v (v)}<option value={v}>{v}</option>{/each}
|
||||||
|
</select>
|
||||||
|
{:else if long}
|
||||||
|
<textarea
|
||||||
|
class={[base, 'min-h-24 font-mono']}
|
||||||
|
rows={kind === 'json' ? 6 : 3}
|
||||||
|
value={value ?? ''}
|
||||||
|
{placeholder}
|
||||||
|
oninput={(e) => (value = e.currentTarget.value)}
|
||||||
|
onkeydown={keydown}
|
||||||
|
{@attach focusOnMount}
|
||||||
|
aria-label={column.label}></textarea>
|
||||||
|
{:else}
|
||||||
|
<input
|
||||||
|
class={base}
|
||||||
|
type={kind === 'date'
|
||||||
|
? 'date'
|
||||||
|
: kind === 'datetime'
|
||||||
|
? 'datetime-local'
|
||||||
|
: kind === 'time'
|
||||||
|
? 'time'
|
||||||
|
: kind === 'number'
|
||||||
|
? 'number'
|
||||||
|
: 'text'}
|
||||||
|
step={kind === 'number' ? 'any' : kind === 'datetime' || kind === 'time' ? 1 : undefined}
|
||||||
|
list={column.fk ? listId : undefined}
|
||||||
|
value={value ?? ''}
|
||||||
|
{placeholder}
|
||||||
|
oninput={(e) => {
|
||||||
|
value = e.currentTarget.value;
|
||||||
|
if (column.fk) queueLookup(e.currentTarget.value);
|
||||||
|
}}
|
||||||
|
onkeydown={keydown}
|
||||||
|
{@attach focusOnMount}
|
||||||
|
aria-label={column.label}
|
||||||
|
/>
|
||||||
|
{#if column.fk}
|
||||||
|
<datalist id={listId}>
|
||||||
|
{#each options as o (o.value)}<option value={o.value}>{o.label}</option>{/each}
|
||||||
|
</datalist>
|
||||||
|
{/if}
|
||||||
|
{/if}
|
||||||
|
{#if column.nullable && value !== null}
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="shrink-0 rounded px-1.5 py-1 text-xs text-[var(--site-muted)] hover:bg-[var(--site-hover)]"
|
||||||
|
title="Set to NULL"
|
||||||
|
onclick={() => (value = null)}>null</button
|
||||||
|
>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
72
src/lib/components/data/data-format.ts
Normal file
72
src/lib/components/data/data-format.ts
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
// Client-side value helpers for the /lists/db grid (shared by the grid and FieldInput).
|
||||||
|
export type Col = {
|
||||||
|
name: string;
|
||||||
|
label: string;
|
||||||
|
type: string;
|
||||||
|
kind: string;
|
||||||
|
uiType: string | null;
|
||||||
|
nullable: boolean;
|
||||||
|
hasDefault: boolean;
|
||||||
|
defaultExpr: string | null;
|
||||||
|
readOnly: boolean;
|
||||||
|
enumValues: string[];
|
||||||
|
pk: boolean;
|
||||||
|
fk: { schema: string; table: string; column: string } | null;
|
||||||
|
help: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const pad = (n: number) => String(n).padStart(2, '0');
|
||||||
|
const withTz = (c: Col) => c.type.includes('with time zone');
|
||||||
|
|
||||||
|
/** Raw JSON value from the server -> editor string (null stays null). */
|
||||||
|
export function toEditor(c: Col, v: unknown): string | null {
|
||||||
|
if (v === null || v === undefined) return null;
|
||||||
|
if (c.kind === 'json') return JSON.stringify(v, null, 2);
|
||||||
|
if (c.kind === 'array') return JSON.stringify(v);
|
||||||
|
if (c.kind === 'datetime' && typeof v === 'string') {
|
||||||
|
if (!withTz(c)) return v.slice(0, 19);
|
||||||
|
const d = new Date(v);
|
||||||
|
if (Number.isNaN(d.getTime())) return v;
|
||||||
|
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}T${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
|
||||||
|
}
|
||||||
|
if (c.kind === 'date' && typeof v === 'string') return v.slice(0, 10);
|
||||||
|
return String(v);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Editor string -> value sent to the server. */
|
||||||
|
export function fromEditor(c: Col, s: string | null): string | null {
|
||||||
|
if (s === null) return null;
|
||||||
|
if (c.kind === 'datetime' && withTz(c) && s.trim()) {
|
||||||
|
const d = new Date(s);
|
||||||
|
return Number.isNaN(d.getTime()) ? s : d.toISOString();
|
||||||
|
}
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Short display text for a grid cell. */
|
||||||
|
export function display(c: Col, v: unknown): string {
|
||||||
|
if (v === null || v === undefined) return '';
|
||||||
|
if (c.kind === 'bool') return v ? '✓' : '✗';
|
||||||
|
if (c.uiType === 'cents' && typeof v === 'number') return `€${(v / 100).toFixed(2)}`;
|
||||||
|
if (c.kind === 'datetime' && typeof v === 'string') {
|
||||||
|
const d = new Date(v);
|
||||||
|
if (Number.isNaN(d.getTime())) return v;
|
||||||
|
return withTz(c)
|
||||||
|
? d.toLocaleString('en-GB', { dateStyle: 'medium', timeStyle: 'short' })
|
||||||
|
: v.replace('T', ' ').slice(0, 16);
|
||||||
|
}
|
||||||
|
if (typeof v === 'object') {
|
||||||
|
const s = JSON.stringify(v);
|
||||||
|
return s.length > 120 ? `${s.slice(0, 117)}…` : s;
|
||||||
|
}
|
||||||
|
const s = String(v);
|
||||||
|
return s.length > 160 ? `${s.slice(0, 157)}…` : s;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shortType(t: string): string {
|
||||||
|
return t
|
||||||
|
.replace('timestamp with time zone', 'timestamptz')
|
||||||
|
.replace('timestamp without time zone', 'timestamp')
|
||||||
|
.replace('character varying', 'varchar')
|
||||||
|
.replace('double precision', 'float8');
|
||||||
|
}
|
||||||
104
src/lib/energy/EnergyChart.svelte
Normal file
104
src/lib/energy/EnergyChart.svelte
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
<script lang="ts" module>
|
||||||
|
export type ChartTheme = {
|
||||||
|
text: string;
|
||||||
|
muted: string;
|
||||||
|
border: string;
|
||||||
|
surface: string;
|
||||||
|
dark: boolean;
|
||||||
|
};
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<script lang="ts">
|
||||||
|
import { onMount } from 'svelte';
|
||||||
|
import type { ECharts, EChartsCoreOption } from 'echarts/core';
|
||||||
|
|
||||||
|
// Generic ECharts wrapper for the energy tracker: same lazy-loading and theme handling as
|
||||||
|
// the CS price chart, but the caller builds the option from the current theme colours.
|
||||||
|
let {
|
||||||
|
option,
|
||||||
|
label,
|
||||||
|
height = 'min(52vh, 440px)'
|
||||||
|
}: {
|
||||||
|
option: (theme: ChartTheme) => EChartsCoreOption;
|
||||||
|
label: string;
|
||||||
|
height?: string;
|
||||||
|
} = $props();
|
||||||
|
let element: HTMLDivElement;
|
||||||
|
let chart: ECharts | undefined;
|
||||||
|
let ready = $state(false);
|
||||||
|
|
||||||
|
function update() {
|
||||||
|
if (!chart) return;
|
||||||
|
const style = getComputedStyle(document.documentElement);
|
||||||
|
const read = (name: string) => style.getPropertyValue(name).trim();
|
||||||
|
chart.setOption(
|
||||||
|
option({
|
||||||
|
text: read('--site-text'),
|
||||||
|
muted: read('--site-muted'),
|
||||||
|
border: read('--site-border'),
|
||||||
|
surface: read('--site-surface'),
|
||||||
|
dark: read('--site-bg') === '#000'
|
||||||
|
}),
|
||||||
|
true
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
onMount(() => {
|
||||||
|
let disposed = false;
|
||||||
|
let resize: ResizeObserver | undefined;
|
||||||
|
let theme: MutationObserver | undefined;
|
||||||
|
import('echarts/core').then(async (echarts) => {
|
||||||
|
const [
|
||||||
|
{ LineChart, BarChart, PieChart },
|
||||||
|
{ GridComponent, TooltipComponent, LegendComponent, DataZoomComponent, MarkAreaComponent },
|
||||||
|
{ CanvasRenderer }
|
||||||
|
] = await Promise.all([
|
||||||
|
import('echarts/charts'),
|
||||||
|
import('echarts/components'),
|
||||||
|
import('echarts/renderers')
|
||||||
|
]);
|
||||||
|
if (disposed) return;
|
||||||
|
echarts.use([
|
||||||
|
LineChart,
|
||||||
|
BarChart,
|
||||||
|
PieChart,
|
||||||
|
GridComponent,
|
||||||
|
TooltipComponent,
|
||||||
|
LegendComponent,
|
||||||
|
DataZoomComponent,
|
||||||
|
MarkAreaComponent,
|
||||||
|
CanvasRenderer
|
||||||
|
]);
|
||||||
|
chart = echarts.init(element, undefined, { renderer: 'canvas' });
|
||||||
|
ready = true;
|
||||||
|
update();
|
||||||
|
resize = new ResizeObserver(() => chart?.resize());
|
||||||
|
resize.observe(element);
|
||||||
|
theme = new MutationObserver(update);
|
||||||
|
theme.observe(document.documentElement, {
|
||||||
|
attributes: true,
|
||||||
|
attributeFilter: ['data-theme']
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
disposed = true;
|
||||||
|
resize?.disconnect();
|
||||||
|
theme?.disconnect();
|
||||||
|
chart?.dispose();
|
||||||
|
};
|
||||||
|
});
|
||||||
|
$effect(() => {
|
||||||
|
// Re-render when the option builder (and therefore its inputs) changes.
|
||||||
|
void option;
|
||||||
|
if (ready) update();
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class="chart" style:height bind:this={element} role="img" aria-label={label}></div>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.chart {
|
||||||
|
width: 100%;
|
||||||
|
min-height: 260px;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
42
src/lib/energy/data.test.ts
Normal file
42
src/lib/energy/data.test.ts
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { fillGaps, gaps, latestSnapshot, seasonalTime, toBuckets, type EnergyRow } from './data';
|
||||||
|
|
||||||
|
const hour = 3_600_000;
|
||||||
|
const rows: EnergyRow[] = [
|
||||||
|
{ t: 0, type: 'ElectricityMix', value: 100, ef: 0.2 },
|
||||||
|
{ t: 0, type: 'ElectricityLoad', value: 90 },
|
||||||
|
{ t: 0, type: 'Wind', value: 60, pct: 0.5 },
|
||||||
|
{ t: 0, type: 'FossilGasPower', value: 30, pct: 0.1 },
|
||||||
|
{ t: 0, type: 'Nuclear', value: 10, pct: 1 },
|
||||||
|
{ t: hour, type: 'ElectricityMix', value: 100 },
|
||||||
|
{ t: hour, type: 'Wind', value: 20 }
|
||||||
|
];
|
||||||
|
|
||||||
|
describe('energy buckets', () => {
|
||||||
|
it('flags buckets whose sources do not add up to the total', () => {
|
||||||
|
const buckets = toBuckets(rows);
|
||||||
|
expect(buckets.map((bucket) => bucket.incomplete)).toEqual([false, true]);
|
||||||
|
expect(buckets[0].sum).toBe(100);
|
||||||
|
expect(buckets[0].load).toBe(90);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('summarises the latest complete bucket', () => {
|
||||||
|
const snapshot = latestSnapshot(toBuckets(rows))!;
|
||||||
|
expect(snapshot.t).toBe(0);
|
||||||
|
expect(snapshot.renewableShare).toBeCloseTo(0.6);
|
||||||
|
expect(snapshot.lowCarbonShare).toBeCloseTo(0.7);
|
||||||
|
expect(snapshot.fossilShare).toBeCloseTo(0.3);
|
||||||
|
expect(snapshot.sources[0]).toMatchObject({ type: 'Wind', share: 0.6, utilization: 0.5 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('finds and fills missing hours', () => {
|
||||||
|
expect(gaps([0, hour, 4 * hour])).toEqual([{ from: 2 * hour, to: 4 * hour, hours: 2 }]);
|
||||||
|
const filled = fillGaps(toBuckets([...rows, { t: 3 * hour, type: 'Wind', value: 1 }]), hour);
|
||||||
|
expect(filled.map((bucket) => 'missing' in bucket)).toEqual([false, false, true, false]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('maps dates onto one seasonal year', () => {
|
||||||
|
expect(seasonalTime('2023-02-28')).toBe(Date.UTC(2000, 1, 28));
|
||||||
|
expect(seasonalTime('2024-12-31')).toBe(Date.UTC(2000, 11, 31));
|
||||||
|
});
|
||||||
|
});
|
||||||
195
src/lib/energy/data.ts
Normal file
195
src/lib/energy/data.ts
Normal file
@@ -0,0 +1,195 @@
|
|||||||
|
// Pure helpers for the energy tracker. Inputs come from the ned-collector tables in the
|
||||||
|
// `energy` schema (NED.nl electricity data and AGSI/GIE gas storage).
|
||||||
|
|
||||||
|
export type SourceType =
|
||||||
|
| 'Nuclear'
|
||||||
|
| 'WindOffshoreC'
|
||||||
|
| 'Wind'
|
||||||
|
| 'Solar'
|
||||||
|
| 'BiomassPower'
|
||||||
|
| 'WastePower'
|
||||||
|
| 'WKK Total'
|
||||||
|
| 'FossilGasPower'
|
||||||
|
| 'FossilHardCoal'
|
||||||
|
| 'OtherPower';
|
||||||
|
|
||||||
|
export type Source = {
|
||||||
|
type: SourceType;
|
||||||
|
label: string;
|
||||||
|
color: string;
|
||||||
|
renewable: boolean;
|
||||||
|
fossil: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Stacking order, bottom to top: steady baseload first, weather-driven next, fossil on top.
|
||||||
|
export const SOURCES: readonly Source[] = [
|
||||||
|
{ type: 'Nuclear', label: 'Nuclear', color: '#a78bfa', renewable: false, fossil: false },
|
||||||
|
{
|
||||||
|
type: 'WindOffshoreC',
|
||||||
|
label: 'Wind offshore',
|
||||||
|
color: '#0284c7',
|
||||||
|
renewable: true,
|
||||||
|
fossil: false
|
||||||
|
},
|
||||||
|
{ type: 'Wind', label: 'Wind onshore', color: '#38bdf8', renewable: true, fossil: false },
|
||||||
|
{ type: 'Solar', label: 'Solar', color: '#facc15', renewable: true, fossil: false },
|
||||||
|
{ type: 'BiomassPower', label: 'Biomass', color: '#65a30d', renewable: true, fossil: false },
|
||||||
|
{ type: 'WastePower', label: 'Waste', color: '#a3a3a3', renewable: false, fossil: false },
|
||||||
|
{ type: 'WKK Total', label: 'CHP (WKK)', color: '#fdba74', renewable: false, fossil: true },
|
||||||
|
{ type: 'FossilGasPower', label: 'Fossil gas', color: '#f97316', renewable: false, fossil: true },
|
||||||
|
{ type: 'FossilHardCoal', label: 'Hard coal', color: '#57534e', renewable: false, fossil: true },
|
||||||
|
{ type: 'OtherPower', label: 'Other', color: '#94a3b8', renewable: false, fossil: false }
|
||||||
|
];
|
||||||
|
|
||||||
|
export const MIX = 'ElectricityMix';
|
||||||
|
export const LOAD = 'ElectricityLoad';
|
||||||
|
const sourceTypes = new Set<string>(SOURCES.map((source) => source.type));
|
||||||
|
|
||||||
|
/** One row per (time bucket, type). `value` is MWh for hours/days and GWh for months/years. */
|
||||||
|
export type EnergyRow = {
|
||||||
|
t: number;
|
||||||
|
type: string;
|
||||||
|
value: number;
|
||||||
|
pct?: number | null;
|
||||||
|
ef?: number | null;
|
||||||
|
hours?: number;
|
||||||
|
expected?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type Bucket = {
|
||||||
|
t: number;
|
||||||
|
mix: number | null;
|
||||||
|
load: number | null;
|
||||||
|
ef: number | null;
|
||||||
|
sources: Partial<Record<SourceType, { value: number; pct: number | null }>>;
|
||||||
|
sum: number;
|
||||||
|
hours?: number;
|
||||||
|
/** Hours in the period (23 or 25 on DST change days). */
|
||||||
|
expected?: number;
|
||||||
|
/** The source rows don't add up to the NED total, so the bucket is still being filled in. */
|
||||||
|
incomplete: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Group rows into buckets ordered by time. A bucket is incomplete when its sources sum to
|
||||||
|
* less than 98% (or more than 102%) of the published total. */
|
||||||
|
export function toBuckets(rows: readonly EnergyRow[]): Bucket[] {
|
||||||
|
const buckets = new Map<number, Bucket>();
|
||||||
|
for (const row of rows) {
|
||||||
|
let bucket = buckets.get(row.t);
|
||||||
|
if (!bucket) {
|
||||||
|
bucket = {
|
||||||
|
t: row.t,
|
||||||
|
mix: null,
|
||||||
|
load: null,
|
||||||
|
ef: null,
|
||||||
|
sources: {},
|
||||||
|
sum: 0,
|
||||||
|
incomplete: false
|
||||||
|
};
|
||||||
|
buckets.set(row.t, bucket);
|
||||||
|
}
|
||||||
|
if (row.hours !== undefined) bucket.hours = Math.max(bucket.hours ?? 0, row.hours);
|
||||||
|
if (row.expected !== undefined) bucket.expected = row.expected;
|
||||||
|
if (row.type === MIX) {
|
||||||
|
bucket.mix = row.value;
|
||||||
|
bucket.ef = row.ef ?? null;
|
||||||
|
} else if (row.type === LOAD) bucket.load = row.value;
|
||||||
|
else if (sourceTypes.has(row.type)) {
|
||||||
|
bucket.sources[row.type as SourceType] = { value: row.value, pct: row.pct ?? null };
|
||||||
|
bucket.sum += row.value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const list = [...buckets.values()].sort((a, b) => a.t - b.t);
|
||||||
|
for (const bucket of list) {
|
||||||
|
bucket.incomplete =
|
||||||
|
bucket.mix === null ? bucket.sum === 0 : Math.abs(bucket.sum / bucket.mix - 1) > 0.02;
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Snapshot = {
|
||||||
|
t: number;
|
||||||
|
total: number;
|
||||||
|
load: number | null;
|
||||||
|
ef: number | null;
|
||||||
|
renewableShare: number;
|
||||||
|
lowCarbonShare: number;
|
||||||
|
fossilShare: number;
|
||||||
|
sources: {
|
||||||
|
type: SourceType;
|
||||||
|
label: string;
|
||||||
|
color: string;
|
||||||
|
value: number;
|
||||||
|
share: number;
|
||||||
|
utilization: number | null;
|
||||||
|
}[];
|
||||||
|
};
|
||||||
|
|
||||||
|
/** The latest complete bucket, broken down by source. */
|
||||||
|
export function latestSnapshot(buckets: readonly Bucket[]): Snapshot | null {
|
||||||
|
const bucket = [...buckets]
|
||||||
|
.reverse()
|
||||||
|
.find((candidate) => !candidate.incomplete && candidate.sum > 0);
|
||||||
|
if (!bucket) return null;
|
||||||
|
const total = bucket.sum;
|
||||||
|
const share = (filter: (source: Source) => boolean) =>
|
||||||
|
SOURCES.filter(filter).reduce(
|
||||||
|
(sum, source) => sum + (bucket.sources[source.type]?.value ?? 0),
|
||||||
|
0
|
||||||
|
) / total;
|
||||||
|
return {
|
||||||
|
t: bucket.t,
|
||||||
|
total,
|
||||||
|
load: bucket.load,
|
||||||
|
ef: bucket.ef,
|
||||||
|
renewableShare: share((source) => source.renewable),
|
||||||
|
lowCarbonShare: share((source) => source.renewable || source.type === 'Nuclear'),
|
||||||
|
fossilShare: share((source) => source.fossil),
|
||||||
|
sources: SOURCES.map((source) => {
|
||||||
|
const entry = bucket.sources[source.type];
|
||||||
|
return {
|
||||||
|
type: source.type,
|
||||||
|
label: source.label,
|
||||||
|
color: source.color,
|
||||||
|
value: entry?.value ?? 0,
|
||||||
|
share: (entry?.value ?? 0) / total,
|
||||||
|
utilization: entry?.pct ?? null
|
||||||
|
};
|
||||||
|
}).sort((a, b) => b.value - a.value)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Insert empty buckets so charts show gaps where the collector missed data. */
|
||||||
|
export function fillGaps(
|
||||||
|
buckets: readonly Bucket[],
|
||||||
|
step: number
|
||||||
|
): (Bucket | { t: number; missing: true })[] {
|
||||||
|
const out: (Bucket | { t: number; missing: true })[] = [];
|
||||||
|
for (const bucket of buckets) {
|
||||||
|
const previous = out.at(-1);
|
||||||
|
if (previous)
|
||||||
|
for (let t = previous.t + step; t < bucket.t; t += step) out.push({ t, missing: true });
|
||||||
|
out.push(bucket);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Hours missing between consecutive buckets, as [from, to) ranges in epoch ms. */
|
||||||
|
export function gaps(
|
||||||
|
times: readonly number[],
|
||||||
|
step = 3_600_000
|
||||||
|
): { from: number; to: number; hours: number }[] {
|
||||||
|
const result = [];
|
||||||
|
for (let index = 1; index < times.length; index++) {
|
||||||
|
const missing = Math.round((times[index] - times[index - 1]) / step) - 1;
|
||||||
|
if (missing > 0)
|
||||||
|
result.push({ from: times[index - 1] + step, to: times[index], hours: missing });
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Map a date onto leap year 2000 so different years can share one seasonal x axis. */
|
||||||
|
export function seasonalTime(day: string) {
|
||||||
|
const [, month, date] = day.split('-').map(Number);
|
||||||
|
return Date.UTC(2000, month - 1, date);
|
||||||
|
}
|
||||||
@@ -1,16 +1,19 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import type { ChartPoint, ChartSeries } from './charts';
|
import { timeGrid, yearLabels, type ChartPoint, type ChartSeries, type GridMode } from './charts';
|
||||||
|
|
||||||
let {
|
let {
|
||||||
series,
|
series,
|
||||||
title,
|
title,
|
||||||
unit = '€',
|
unit = '€',
|
||||||
marker
|
marker,
|
||||||
|
grid
|
||||||
}: {
|
}: {
|
||||||
series: ChartSeries[];
|
series: ChartSeries[];
|
||||||
title: string;
|
title: string;
|
||||||
unit?: string;
|
unit?: string;
|
||||||
marker?: { date: string; label: string };
|
marker?: { date: string; label: string };
|
||||||
|
/** Vertical gridlines: every month (1 January drawn stronger) or every year. */
|
||||||
|
grid?: GridMode;
|
||||||
} = $props();
|
} = $props();
|
||||||
let clientWidth = $state(800);
|
let clientWidth = $state(800);
|
||||||
let width = $derived(Math.max(280, clientWidth));
|
let width = $derived(Math.max(280, clientWidth));
|
||||||
@@ -26,6 +29,14 @@
|
|||||||
let from = $derived(Math.min(...times));
|
let from = $derived(Math.min(...times));
|
||||||
let until = $derived(Math.max(...times));
|
let until = $derived(Math.max(...times));
|
||||||
let dateSpan = $derived(until - from || 1);
|
let dateSpan = $derived(until - from || 1);
|
||||||
|
let gridLines = $derived(
|
||||||
|
grid && values.length ? timeGrid(isoDate(from), isoDate(until), grid) : []
|
||||||
|
);
|
||||||
|
let axisTicks = $derived(
|
||||||
|
grid === 'years'
|
||||||
|
? yearLabels(gridLines, Math.max(2, Math.floor((width - plot.left - plot.right) / 44)))
|
||||||
|
: ticks()
|
||||||
|
);
|
||||||
let snapDates = $derived(
|
let snapDates = $derived(
|
||||||
[...new Set(series.flatMap((item) => item.points.map((point) => point.date)))].sort()
|
[...new Set(series.flatMap((item) => item.points.map((point) => point.date)))].sort()
|
||||||
);
|
);
|
||||||
@@ -59,6 +70,9 @@
|
|||||||
return { min, max, ticks };
|
return { min, max, ticks };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isoDate(ms: number) {
|
||||||
|
return new Date(ms).toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
function time(date: string) {
|
function time(date: string) {
|
||||||
return Date.parse(`${date}T12:00:00Z`);
|
return Date.parse(`${date}T12:00:00Z`);
|
||||||
}
|
}
|
||||||
@@ -108,6 +122,9 @@
|
|||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
function tickLabel(date: string) {
|
||||||
|
return grid === 'years' ? date.slice(0, 4) : dateLabel(date);
|
||||||
|
}
|
||||||
function dateLabel(date: string, withDay = false) {
|
function dateLabel(date: string, withDay = false) {
|
||||||
return new Date(time(date)).toLocaleDateString('en-NL', {
|
return new Date(time(date)).toLocaleDateString('en-NL', {
|
||||||
day: withDay ? 'numeric' : undefined,
|
day: withDay ? 'numeric' : undefined,
|
||||||
@@ -145,6 +162,16 @@
|
|||||||
onpointerdown={hover}
|
onpointerdown={hover}
|
||||||
onpointerleave={() => (hoverDate = null)}
|
onpointerleave={() => (hoverDate = null)}
|
||||||
>
|
>
|
||||||
|
{#each gridLines as line (line.date)}
|
||||||
|
<line
|
||||||
|
class="grid-x"
|
||||||
|
class:year={line.year && grid === 'months'}
|
||||||
|
x1={x(line.date)}
|
||||||
|
x2={x(line.date)}
|
||||||
|
y1={plot.top}
|
||||||
|
y2={height - plot.bottom}
|
||||||
|
/>
|
||||||
|
{/each}
|
||||||
{#each scale.ticks as value (value)}
|
{#each scale.ticks as value (value)}
|
||||||
<line
|
<line
|
||||||
x1={plot.left}
|
x1={plot.left}
|
||||||
@@ -162,13 +189,13 @@
|
|||||||
font-size="11">{format(value)}</text
|
font-size="11">{format(value)}</text
|
||||||
>
|
>
|
||||||
{/each}
|
{/each}
|
||||||
{#each ticks() as tick (tick)}
|
{#each axisTicks as tick (tick)}
|
||||||
<text
|
<text
|
||||||
x={x(tick)}
|
x={x(tick)}
|
||||||
y={height - 8}
|
y={height - 8}
|
||||||
text-anchor="middle"
|
text-anchor="middle"
|
||||||
fill="var(--site-muted)"
|
fill="var(--site-muted)"
|
||||||
font-size="11">{dateLabel(tick)}</text
|
font-size="11">{tickLabel(tick)}</text
|
||||||
>
|
>
|
||||||
{/each}
|
{/each}
|
||||||
{#if marker && time(marker.date) >= from && time(marker.date) <= until}
|
{#if marker && time(marker.date) >= from && time(marker.date) <= until}
|
||||||
@@ -294,6 +321,18 @@
|
|||||||
overflow: visible;
|
overflow: visible;
|
||||||
user-select: none;
|
user-select: none;
|
||||||
}
|
}
|
||||||
|
/* Vertical time gridlines: quieter than the value gridlines so the data stays in front. */
|
||||||
|
.grid-x {
|
||||||
|
stroke: var(--site-border);
|
||||||
|
stroke-width: 1;
|
||||||
|
stroke-opacity: 0.55;
|
||||||
|
shape-rendering: crispEdges;
|
||||||
|
}
|
||||||
|
.grid-x.year {
|
||||||
|
stroke: var(--site-muted);
|
||||||
|
stroke-width: 1.5;
|
||||||
|
stroke-opacity: 0.4;
|
||||||
|
}
|
||||||
.tooltip {
|
.tooltip {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
top: 4px;
|
top: 4px;
|
||||||
|
|||||||
269
src/lib/money-tracker/QuickAdd.svelte
Normal file
269
src/lib/money-tracker/QuickAdd.svelte
Normal file
@@ -0,0 +1,269 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { enhance } from '$app/forms';
|
||||||
|
import { tick } from 'svelte';
|
||||||
|
import { Button } from '#lib/components/ui/button/index.js';
|
||||||
|
import { Input } from '#lib/components/ui/input/index.js';
|
||||||
|
import { NativeSelect } from '#lib/components/ui/native-select/index.js';
|
||||||
|
import Plus from 'lucide-svelte/icons/plus';
|
||||||
|
import { descriptionSuggestions, type SuggestionSource } from './quick-add.js';
|
||||||
|
|
||||||
|
let {
|
||||||
|
accounts,
|
||||||
|
transactions,
|
||||||
|
today
|
||||||
|
}: {
|
||||||
|
accounts: { id: number; name: string }[];
|
||||||
|
transactions: SuggestionSource[];
|
||||||
|
today: string;
|
||||||
|
} = $props();
|
||||||
|
|
||||||
|
const storageKey = 'money-tracker:last-account';
|
||||||
|
const euro = new Intl.NumberFormat('en-NL', { style: 'currency', currency: 'EUR' });
|
||||||
|
let kind = $state<'expense' | 'income'>('expense');
|
||||||
|
let description = $state('');
|
||||||
|
let amount = $state('');
|
||||||
|
let accountId = $state('');
|
||||||
|
let date = $state('');
|
||||||
|
let listOpen = $state(false);
|
||||||
|
let highlighted = $state(-1);
|
||||||
|
let message = $state('');
|
||||||
|
let failed = $state(false);
|
||||||
|
let saving = $state(false);
|
||||||
|
let descriptionInput = $state<HTMLInputElement | null>(null);
|
||||||
|
let amountInput = $state<HTMLInputElement | null>(null);
|
||||||
|
|
||||||
|
let suggestions = $derived(descriptionSuggestions(transactions, today));
|
||||||
|
let matches = $derived.by(() => {
|
||||||
|
const query = description.trim().toLowerCase();
|
||||||
|
if (!query) return [];
|
||||||
|
return suggestions
|
||||||
|
.filter((item) => item.key.includes(query) && item.key !== query)
|
||||||
|
.sort((a, b) => Number(b.key.startsWith(query)) - Number(a.key.startsWith(query)))
|
||||||
|
.slice(0, 6);
|
||||||
|
});
|
||||||
|
let signedAmount = $derived.by(() => {
|
||||||
|
const raw = amount.trim().replace(',', '.').replace(/^[-+]/, '');
|
||||||
|
return raw ? (kind === 'expense' ? '-' : '') + raw : '';
|
||||||
|
});
|
||||||
|
|
||||||
|
$effect.pre(() => {
|
||||||
|
if (!date) date = today;
|
||||||
|
if (!accountId && accounts.length) {
|
||||||
|
const remembered =
|
||||||
|
typeof localStorage === 'undefined' ? null : localStorage.getItem(storageKey);
|
||||||
|
accountId = String(
|
||||||
|
accounts.find((account) => String(account.id) === remembered)?.id ?? accounts[0].id
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
async function choose(index: number) {
|
||||||
|
const item = matches[index];
|
||||||
|
if (!item) return;
|
||||||
|
description = item.description;
|
||||||
|
amount = (Math.abs(item.amountCents) / 100).toFixed(2);
|
||||||
|
kind = item.amountCents < 0 ? 'expense' : 'income';
|
||||||
|
if (item.accountId && accounts.some((account) => account.id === item.accountId))
|
||||||
|
accountId = String(item.accountId);
|
||||||
|
listOpen = false;
|
||||||
|
highlighted = -1;
|
||||||
|
await tick();
|
||||||
|
amountInput?.select();
|
||||||
|
}
|
||||||
|
|
||||||
|
function keydown(event: KeyboardEvent) {
|
||||||
|
if (!matches.length) return;
|
||||||
|
if (event.key === 'ArrowDown' || event.key === 'ArrowUp') {
|
||||||
|
event.preventDefault();
|
||||||
|
listOpen = true;
|
||||||
|
const step = event.key === 'ArrowDown' ? 1 : -1;
|
||||||
|
highlighted = (highlighted + step + matches.length) % matches.length;
|
||||||
|
} else if (event.key === 'Enter' && listOpen && highlighted >= 0) {
|
||||||
|
event.preventDefault();
|
||||||
|
choose(highlighted);
|
||||||
|
} else if (event.key === 'Escape') {
|
||||||
|
listOpen = false;
|
||||||
|
highlighted = -1;
|
||||||
|
} else if (event.key === 'Tab' && listOpen && highlighted >= 0) choose(highlighted);
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<form
|
||||||
|
class="quick-add"
|
||||||
|
method="POST"
|
||||||
|
action="?/saveTransaction"
|
||||||
|
aria-label="Quick add transaction"
|
||||||
|
use:enhance={() => {
|
||||||
|
saving = true;
|
||||||
|
const saved = { description, signedAmount };
|
||||||
|
return async ({ result, update }) => {
|
||||||
|
saving = false;
|
||||||
|
if (result.type === 'success') {
|
||||||
|
localStorage.setItem(storageKey, accountId);
|
||||||
|
failed = false;
|
||||||
|
message = `Saved ${saved.description} · ${euro.format(Number(saved.signedAmount))}`;
|
||||||
|
description = '';
|
||||||
|
amount = '';
|
||||||
|
await update({ reset: false });
|
||||||
|
descriptionInput?.focus();
|
||||||
|
} else {
|
||||||
|
failed = true;
|
||||||
|
message =
|
||||||
|
result.type === 'failure' && typeof result.data?.message === 'string'
|
||||||
|
? result.data.message
|
||||||
|
: 'Could not save this transaction.';
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div class="quick-kind" role="group" aria-label="Transaction type">
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant={kind === 'expense' ? 'default' : 'ghost'}
|
||||||
|
aria-pressed={kind === 'expense'}
|
||||||
|
onclick={() => (kind = 'expense')}>Expense</Button
|
||||||
|
><Button
|
||||||
|
type="button"
|
||||||
|
variant={kind === 'income' ? 'default' : 'ghost'}
|
||||||
|
aria-pressed={kind === 'income'}
|
||||||
|
onclick={() => (kind = 'income')}>Income</Button
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
<div class="quick-description">
|
||||||
|
<label class="sr-only" for="quick-description">Description</label>
|
||||||
|
<Input
|
||||||
|
id="quick-description"
|
||||||
|
name="description"
|
||||||
|
autocomplete="off"
|
||||||
|
placeholder="What was it? e.g. Albert Heijn"
|
||||||
|
required
|
||||||
|
maxlength={500}
|
||||||
|
role="combobox"
|
||||||
|
aria-expanded={listOpen && matches.length > 0}
|
||||||
|
aria-controls="quick-suggestions"
|
||||||
|
aria-autocomplete="list"
|
||||||
|
aria-activedescendant={highlighted >= 0 ? `quick-suggestion-${highlighted}` : undefined}
|
||||||
|
bind:ref={descriptionInput}
|
||||||
|
bind:value={description}
|
||||||
|
oninput={() => {
|
||||||
|
listOpen = true;
|
||||||
|
highlighted = -1;
|
||||||
|
}}
|
||||||
|
onkeydown={keydown}
|
||||||
|
onfocus={() => (listOpen = true)}
|
||||||
|
onblur={() => setTimeout(() => (listOpen = false), 120)}
|
||||||
|
/>
|
||||||
|
{#if listOpen && matches.length}
|
||||||
|
<ul id="quick-suggestions" role="listbox" class="quick-suggestions">
|
||||||
|
{#each matches as item, index (item.key)}
|
||||||
|
<li
|
||||||
|
id={`quick-suggestion-${index}`}
|
||||||
|
role="option"
|
||||||
|
aria-selected={index === highlighted}
|
||||||
|
class:highlighted={index === highlighted}
|
||||||
|
onmousedown={(event) => {
|
||||||
|
event.preventDefault();
|
||||||
|
choose(index);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span>{item.description}</span>
|
||||||
|
<span class="muted"
|
||||||
|
>{euro.format(item.amountCents / 100)} · {accounts.find(
|
||||||
|
(a) => a.id === item.accountId
|
||||||
|
)?.name ?? ''}</span
|
||||||
|
>
|
||||||
|
</li>
|
||||||
|
{/each}
|
||||||
|
</ul>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
<label class="sr-only" for="quick-amount">Amount in euros</label>
|
||||||
|
<Input
|
||||||
|
id="quick-amount"
|
||||||
|
class="quick-amount"
|
||||||
|
inputmode="decimal"
|
||||||
|
placeholder="0.00"
|
||||||
|
required
|
||||||
|
pattern="[-+]?[0-9]+([.,][0-9]{(1, 2)})?"
|
||||||
|
bind:ref={amountInput}
|
||||||
|
bind:value={amount}
|
||||||
|
/>
|
||||||
|
<input type="hidden" name="amount" value={signedAmount} />
|
||||||
|
<label class="sr-only" for="quick-account">Account</label>
|
||||||
|
<NativeSelect id="quick-account" name="accountId" bind:value={accountId} required
|
||||||
|
>{#each accounts as account (account.id)}<option value={String(account.id)}
|
||||||
|
>{account.name}</option
|
||||||
|
>{/each}</NativeSelect
|
||||||
|
>
|
||||||
|
<label class="sr-only" for="quick-date">Date</label>
|
||||||
|
<Input id="quick-date" class="quick-date" type="date" name="date" bind:value={date} required />
|
||||||
|
<Button type="submit" disabled={saving || !accounts.length}><Plus size={16} /> Add</Button>
|
||||||
|
{#if message}<p class="quick-message" class:form-error={failed} role="status">{message}</p>{/if}
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
.quick-add {
|
||||||
|
position: relative;
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
margin: 0 0 20px;
|
||||||
|
padding: 12px;
|
||||||
|
border: 1px solid var(--site-border);
|
||||||
|
border-radius: 10px;
|
||||||
|
}
|
||||||
|
.quick-kind {
|
||||||
|
display: inline-flex;
|
||||||
|
gap: 2px;
|
||||||
|
padding: 2px;
|
||||||
|
border: 1px solid var(--site-border);
|
||||||
|
border-radius: 8px;
|
||||||
|
}
|
||||||
|
.quick-description {
|
||||||
|
position: relative;
|
||||||
|
flex: 1 1 220px;
|
||||||
|
}
|
||||||
|
.quick-add :global(.quick-amount) {
|
||||||
|
width: 110px;
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
.quick-add :global(.quick-date) {
|
||||||
|
width: 150px;
|
||||||
|
}
|
||||||
|
.quick-suggestions {
|
||||||
|
position: absolute;
|
||||||
|
z-index: 30;
|
||||||
|
top: calc(100% + 4px);
|
||||||
|
left: 0;
|
||||||
|
right: 0;
|
||||||
|
margin: 0;
|
||||||
|
padding: 4px;
|
||||||
|
list-style: none;
|
||||||
|
background: var(--site-bg);
|
||||||
|
border: 1px solid var(--site-border);
|
||||||
|
border-radius: 8px;
|
||||||
|
box-shadow: 0 8px 24px rgb(0 0 0 / 0.12);
|
||||||
|
}
|
||||||
|
.quick-suggestions li {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 12px;
|
||||||
|
padding: 6px 8px;
|
||||||
|
border-radius: 6px;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
.quick-suggestions li.highlighted,
|
||||||
|
.quick-suggestions li:hover {
|
||||||
|
background: var(--site-border);
|
||||||
|
}
|
||||||
|
.quick-message {
|
||||||
|
flex-basis: 100%;
|
||||||
|
margin: 0;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
.muted {
|
||||||
|
opacity: 0.65;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -4,6 +4,9 @@ import {
|
|||||||
accountColor,
|
accountColor,
|
||||||
balanceSeries,
|
balanceSeries,
|
||||||
forecastSeries,
|
forecastSeries,
|
||||||
|
parseHistoryRange,
|
||||||
|
timeGrid,
|
||||||
|
yearLabels,
|
||||||
utilityEstimate,
|
utilityEstimate,
|
||||||
wealthProjection,
|
wealthProjection,
|
||||||
wealthTimeline
|
wealthTimeline
|
||||||
@@ -257,3 +260,93 @@ describe('money tracker calculations', () => {
|
|||||||
).toBe(30);
|
).toBe(30);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('history range', () => {
|
||||||
|
it('starts lines at the range start with the balance they had then', async () => {
|
||||||
|
const { clipSeries, historyStart } = await import('./charts');
|
||||||
|
expect(historyStart('1y', '2026-10-24')).toBe('2025-10-24');
|
||||||
|
expect(historyStart('all', '2026-10-24')).toBeNull();
|
||||||
|
const series = [
|
||||||
|
{
|
||||||
|
label: 'Knab',
|
||||||
|
color: 'x',
|
||||||
|
points: [
|
||||||
|
{ date: '2021-08-24', value: 100 },
|
||||||
|
{ date: '2025-01-01', value: 50 },
|
||||||
|
{ date: '2026-01-01', value: 70 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
];
|
||||||
|
expect(clipSeries(series, '2025-10-24')[0].points).toEqual([
|
||||||
|
{ date: '2025-10-24', value: 50 },
|
||||||
|
{ date: '2026-01-01', value: 70 }
|
||||||
|
]);
|
||||||
|
expect(clipSeries(series, null)).toBe(series);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('leaves accounts with includeInTotal false out of the totals but keeps their own line', () => {
|
||||||
|
const accounts = [
|
||||||
|
{ id: 1, name: 'Bank', kind: 'spendable', openingBalanceCents: 1000 },
|
||||||
|
{ id: 2, name: 'Parents', kind: 'spendable', openingBalanceCents: 0, includeInTotal: false }
|
||||||
|
];
|
||||||
|
const actual = [
|
||||||
|
{ accountId: 2, date: '2019-03-01', amountCents: 1_000_000 },
|
||||||
|
{ accountId: 2, date: '2019-04-01', amountCents: -1_000_000 },
|
||||||
|
{ accountId: 1, date: '2019-04-01', amountCents: 500 }
|
||||||
|
];
|
||||||
|
const series = balanceSeries(accounts, actual);
|
||||||
|
expect(series[1].points.map((point) => point.value)).toEqual([1_000_000, 0]);
|
||||||
|
expect(series.at(-1)!.points.map((point) => point.value)).toEqual([1000, 1500]);
|
||||||
|
const balances = accountBalances(accounts, actual);
|
||||||
|
const future = [
|
||||||
|
{ accountId: 2, date: '2026-10-01', amountCents: 9999 },
|
||||||
|
{ accountId: 1, date: '2026-10-01', amountCents: 100 }
|
||||||
|
];
|
||||||
|
const projection = wealthProjection(accounts, balances, future, [], [], null, '2026-09-23');
|
||||||
|
expect(projection[0].points[0].value).toBe(1500);
|
||||||
|
expect(projection[0].points.at(-1)!.value).toBe(1600);
|
||||||
|
const [, spendable] = wealthTimeline(accounts, actual, projection, '2026-09-23', 'split');
|
||||||
|
expect(spendable.points.slice(0, 2).map((point) => point.value)).toEqual([1000, 1500]);
|
||||||
|
const byAccount = wealthTimeline(accounts, actual, projection, '2026-09-23', 'accounts');
|
||||||
|
expect(byAccount.map((line) => line.label)).toEqual(['Bank', 'Parents', 'Total']);
|
||||||
|
expect(byAccount.at(-1)!.points[0].value).toBe(1000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reads the history range from the URL, defaulting to one year', () => {
|
||||||
|
expect(parseHistoryRange('3y')).toBe('3y');
|
||||||
|
expect(parseHistoryRange('all')).toBe('all');
|
||||||
|
expect(parseHistoryRange('1y')).toBe('1y');
|
||||||
|
expect(parseHistoryRange('5y')).toBe('1y');
|
||||||
|
expect(parseHistoryRange(null)).toBe('1y');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('puts a gridline on every month start and marks 1 January as a year line', () => {
|
||||||
|
const lines = timeGrid('2025-10-24', '2026-03-01', 'months');
|
||||||
|
expect(lines).toEqual([
|
||||||
|
{ date: '2025-11-01', year: false },
|
||||||
|
{ date: '2025-12-01', year: false },
|
||||||
|
{ date: '2026-01-01', year: true },
|
||||||
|
{ date: '2026-02-01', year: false },
|
||||||
|
{ date: '2026-03-01', year: false }
|
||||||
|
]);
|
||||||
|
expect(timeGrid('2025-01-01', '2025-02-15', 'months').map((line) => line.date)).toEqual([
|
||||||
|
'2025-01-01',
|
||||||
|
'2025-02-01'
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('puts only year lines on the all-time grid and thins crowded year labels', () => {
|
||||||
|
const lines = timeGrid('2007-02-01', '2027-10-24', 'years');
|
||||||
|
expect(lines).toHaveLength(20);
|
||||||
|
expect(lines.every((line) => line.year && line.date.endsWith('-01-01'))).toBe(true);
|
||||||
|
expect(lines[0].date).toBe('2008-01-01');
|
||||||
|
expect(yearLabels(lines, 30)).toHaveLength(20);
|
||||||
|
expect(yearLabels(lines, 6).map((date) => date.slice(0, 4))).toEqual([
|
||||||
|
'2010',
|
||||||
|
'2015',
|
||||||
|
'2020',
|
||||||
|
'2025'
|
||||||
|
]);
|
||||||
|
expect(timeGrid('2026-01-02', '2026-01-01', 'months')).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -15,10 +15,24 @@ export function accountColor(account: { color?: string | null }, index: number):
|
|||||||
return account.color || palette[index % palette.length];
|
return account.color || palette[index % palette.length];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Accounts kept out of net worth and total wealth (`includeInTotal: false`). */
|
||||||
|
export function excludedIds(accounts: { id: number; includeInTotal?: boolean }[]): Set<number> {
|
||||||
|
return new Set(
|
||||||
|
accounts.filter((account) => account.includeInTotal === false).map((account) => account.id)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function balanceSeries(
|
export function balanceSeries(
|
||||||
accounts: { id: number; name: string; openingBalanceCents: number; color?: string | null }[],
|
accounts: {
|
||||||
|
id: number;
|
||||||
|
name: string;
|
||||||
|
openingBalanceCents: number;
|
||||||
|
color?: string | null;
|
||||||
|
includeInTotal?: boolean;
|
||||||
|
}[],
|
||||||
transactions: { accountId: number; date: string; amountCents: number }[]
|
transactions: { accountId: number; date: string; amountCents: number }[]
|
||||||
): ChartSeries[] {
|
): ChartSeries[] {
|
||||||
|
const excluded = excludedIds(accounts);
|
||||||
const ordered = [...transactions].sort((a, b) => a.date.localeCompare(b.date));
|
const ordered = [...transactions].sort((a, b) => a.date.localeCompare(b.date));
|
||||||
const balances = new Map(accounts.map((account) => [account.id, account.openingBalanceCents]));
|
const balances = new Map(accounts.map((account) => [account.id, account.openingBalanceCents]));
|
||||||
const accountPoints = new Map(accounts.map((account) => [account.id, [] as ChartPoint[]]));
|
const accountPoints = new Map(accounts.map((account) => [account.id, [] as ChartPoint[]]));
|
||||||
@@ -36,7 +50,10 @@ export function balanceSeries(
|
|||||||
accountPoints.get(account.id)!.push({ date, value: balances.get(account.id)! });
|
accountPoints.get(account.id)!.push({ date, value: balances.get(account.id)! });
|
||||||
combinedPoints.push({
|
combinedPoints.push({
|
||||||
date,
|
date,
|
||||||
value: [...balances.values()].reduce((sum, balance) => sum + balance, 0)
|
value: [...balances].reduce(
|
||||||
|
(sum, [id, balance]) => (excluded.has(id) ? sum : sum + balance),
|
||||||
|
0
|
||||||
|
)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return [
|
return [
|
||||||
@@ -86,7 +103,12 @@ export function forecastSeries(
|
|||||||
return [{ label: 'Projected total', color: '#0070f3', points }];
|
return [{ label: 'Projected total', color: '#0070f3', points }];
|
||||||
}
|
}
|
||||||
|
|
||||||
type ProjectionAccount = { id: number; kind: string; openingBalanceCents: number };
|
type ProjectionAccount = {
|
||||||
|
id: number;
|
||||||
|
kind: string;
|
||||||
|
openingBalanceCents: number;
|
||||||
|
includeInTotal?: boolean;
|
||||||
|
};
|
||||||
type ProjectionMovement = { date: string; amountCents: number; accountId: number | null };
|
type ProjectionMovement = { date: string; amountCents: number; accountId: number | null };
|
||||||
type ProjectionRecurring = {
|
type ProjectionRecurring = {
|
||||||
amountCents: number;
|
amountCents: number;
|
||||||
@@ -114,6 +136,8 @@ export function wealthProjection(
|
|||||||
settings: ProjectionSettings | null,
|
settings: ProjectionSettings | null,
|
||||||
today: string
|
today: string
|
||||||
): ChartSeries[] {
|
): ChartSeries[] {
|
||||||
|
const excluded = excludedIds(accounts);
|
||||||
|
const counted = (accountId: number | null) => accountId === null || !excluded.has(accountId);
|
||||||
const accountKind = new Map(accounts.map((account) => [account.id, account.kind]));
|
const accountKind = new Map(accounts.map((account) => [account.id, account.kind]));
|
||||||
const investmentId = settings?.investmentAccountId ?? null;
|
const investmentId = settings?.investmentAccountId ?? null;
|
||||||
const validInvestmentId =
|
const validInvestmentId =
|
||||||
@@ -121,6 +145,7 @@ export function wealthProjection(
|
|||||||
let spendable = 0;
|
let spendable = 0;
|
||||||
let invested = 0;
|
let invested = 0;
|
||||||
for (const account of accounts) {
|
for (const account of accounts) {
|
||||||
|
if (excluded.has(account.id)) continue;
|
||||||
const value = balances.get(account.id) ?? 0;
|
const value = balances.get(account.id) ?? 0;
|
||||||
if (account.kind === 'investment') invested += value;
|
if (account.kind === 'investment') invested += value;
|
||||||
else spendable += value;
|
else spendable += value;
|
||||||
@@ -129,7 +154,9 @@ export function wealthProjection(
|
|||||||
const totalPoints: ChartPoint[] = [{ date: today, value: spendable + invested }];
|
const totalPoints: ChartPoint[] = [{ date: today, value: spendable + invested }];
|
||||||
const spendablePoints: ChartPoint[] = [{ date: today, value: spendable }];
|
const spendablePoints: ChartPoint[] = [{ date: today, value: spendable }];
|
||||||
const investedPoints: ChartPoint[] = [{ date: today, value: invested }];
|
const investedPoints: ChartPoint[] = [{ date: today, value: invested }];
|
||||||
const movements = [...futureTransactions, ...forecasts].filter((entry) => entry.date > today);
|
const movements = [...futureTransactions, ...forecasts].filter(
|
||||||
|
(entry) => entry.date > today && counted(entry.accountId)
|
||||||
|
);
|
||||||
const latestMovement = movements.reduce(
|
const latestMovement = movements.reduce(
|
||||||
(latest, entry) => (entry.date > latest ? entry.date : latest),
|
(latest, entry) => (entry.date > latest ? entry.date : latest),
|
||||||
''
|
''
|
||||||
@@ -162,6 +189,7 @@ export function wealthProjection(
|
|||||||
}
|
}
|
||||||
if (settings?.includeRecurring !== false && index > 0) {
|
if (settings?.includeRecurring !== false && index > 0) {
|
||||||
for (const item of recurring) {
|
for (const item of recurring) {
|
||||||
|
if (!counted(item.accountId)) continue;
|
||||||
if ((item.startsOn && item.startsOn > date) || (item.endsOn && item.endsOn < first))
|
if ((item.startsOn && item.startsOn > date) || (item.endsOn && item.endsOn < first))
|
||||||
continue;
|
continue;
|
||||||
const value = Math.round(monthlyCents(item.amountCents, item.cadence));
|
const value = Math.round(monthlyCents(item.amountCents, item.cadence));
|
||||||
@@ -215,6 +243,7 @@ export function wealthTimeline(
|
|||||||
kind: string;
|
kind: string;
|
||||||
openingBalanceCents: number;
|
openingBalanceCents: number;
|
||||||
color?: string | null;
|
color?: string | null;
|
||||||
|
includeInTotal?: boolean;
|
||||||
}[],
|
}[],
|
||||||
actualTransactions: { accountId: number; date: string; amountCents: number }[],
|
actualTransactions: { accountId: number; date: string; amountCents: number }[],
|
||||||
projection: ChartSeries[],
|
projection: ChartSeries[],
|
||||||
@@ -229,7 +258,7 @@ export function wealthTimeline(
|
|||||||
date,
|
date,
|
||||||
value: accounts.reduce(
|
value: accounts.reduce(
|
||||||
(sum, account, accountIndex) =>
|
(sum, account, accountIndex) =>
|
||||||
(account.kind === 'investment') === investment
|
account.includeInTotal !== false && (account.kind === 'investment') === investment
|
||||||
? sum + accountLines[accountIndex].points[index].value
|
? sum + accountLines[accountIndex].points[index].value
|
||||||
: sum,
|
: sum,
|
||||||
0
|
0
|
||||||
@@ -306,3 +335,70 @@ export function utilityEstimate(
|
|||||||
differenceCents: advancesCents - estimatedCents
|
differenceCents: advancesCents - estimatedCents
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type HistoryRange = '1y' | '3y' | 'all';
|
||||||
|
|
||||||
|
/** First date shown for a history range (null = everything). */
|
||||||
|
export function historyStart(range: HistoryRange, today: string): string | null {
|
||||||
|
if (range === 'all') return null;
|
||||||
|
const years = range === '1y' ? 1 : 3;
|
||||||
|
return `${Number(today.slice(0, 4)) - years}${today.slice(4)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hides history before `from`, starting each line at `from` with the balance it had then,
|
||||||
|
* so a long bank history doesn't squash the recent months.
|
||||||
|
*/
|
||||||
|
export function clipSeries(series: ChartSeries[], from: string | null): ChartSeries[] {
|
||||||
|
if (!from) return series;
|
||||||
|
return series.map((line) => {
|
||||||
|
const before = line.points.filter((point) => point.date < from);
|
||||||
|
const after = line.points.filter((point) => point.date >= from);
|
||||||
|
const carried = before.at(-1);
|
||||||
|
const points =
|
||||||
|
carried && after[0]?.date !== from ? [{ date: from, value: carried.value }, ...after] : after;
|
||||||
|
return { ...line, points };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reads a `?range=` value, falling back to one year for anything unknown. */
|
||||||
|
export function parseHistoryRange(value: string | null | undefined): HistoryRange {
|
||||||
|
return value === '3y' || value === 'all' ? value : '1y';
|
||||||
|
}
|
||||||
|
|
||||||
|
export type GridMode = 'months' | 'years';
|
||||||
|
export type GridLine = { date: string; year: boolean };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Vertical gridlines between two dates (inclusive): the first of every month (`months`),
|
||||||
|
* or only 1 January (`years`). Lines on 1 January are flagged so they can be drawn stronger.
|
||||||
|
*/
|
||||||
|
export function timeGrid(from: string, until: string, mode: GridMode): GridLine[] {
|
||||||
|
const lines: GridLine[] = [];
|
||||||
|
if (!from || !until || from > until) return lines;
|
||||||
|
let year = Number(from.slice(0, 4));
|
||||||
|
let month = Number(from.slice(5, 7)) - 1;
|
||||||
|
if (from.slice(8) !== '01') month++;
|
||||||
|
for (let guard = 0; guard < 1200; guard++) {
|
||||||
|
if (month > 11) {
|
||||||
|
month = 0;
|
||||||
|
year++;
|
||||||
|
}
|
||||||
|
if (mode === 'years' && month !== 0) {
|
||||||
|
month = 12;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const date = `${year}-${String(month + 1).padStart(2, '0')}-01`;
|
||||||
|
if (date > until) break;
|
||||||
|
lines.push({ date, year: month === 0 });
|
||||||
|
month++;
|
||||||
|
}
|
||||||
|
return lines;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Year labels for the `years` grid, thinned to every 2nd/5th/10th year when crowded. */
|
||||||
|
export function yearLabels(lines: GridLine[], maxLabels: number): string[] {
|
||||||
|
const years = lines.filter((line) => line.year).map((line) => line.date);
|
||||||
|
const step = [1, 2, 5, 10, 20].find((size) => Math.ceil(years.length / size) <= maxLabels) ?? 50;
|
||||||
|
return years.filter((date) => Number(date.slice(0, 4)) % step === 0);
|
||||||
|
}
|
||||||
|
|||||||
24
src/lib/money-tracker/quick-add.test.ts
Normal file
24
src/lib/money-tracker/quick-add.test.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { descriptionSuggestions } from './quick-add';
|
||||||
|
|
||||||
|
describe('quick-add suggestions', () => {
|
||||||
|
it('ranks by use and fills the usual amount and latest account', () => {
|
||||||
|
const rows = [
|
||||||
|
{ description: 'PLUS', amountCents: -1200, accountId: 1, date: '2026-10-01' },
|
||||||
|
{ description: 'plus', amountCents: -1200, accountId: 1, date: '2026-10-03' },
|
||||||
|
{ description: 'Plus', amountCents: -800, accountId: 4, date: '2026-10-05' },
|
||||||
|
{ description: 'From Juhee', amountCents: 5000, accountId: 4, date: '2026-10-02' },
|
||||||
|
{ description: 'Future', amountCents: -1, accountId: 1, date: '2026-12-01' }
|
||||||
|
];
|
||||||
|
const [first, second, ...rest] = descriptionSuggestions(rows, '2026-10-24');
|
||||||
|
expect(first).toMatchObject({
|
||||||
|
key: 'plus',
|
||||||
|
description: 'Plus',
|
||||||
|
amountCents: -1200,
|
||||||
|
accountId: 4,
|
||||||
|
count: 3
|
||||||
|
});
|
||||||
|
expect(second).toMatchObject({ description: 'From Juhee', amountCents: 5000 });
|
||||||
|
expect(rest).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
73
src/lib/money-tracker/quick-add.ts
Normal file
73
src/lib/money-tracker/quick-add.ts
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
export type SuggestionSource = {
|
||||||
|
description: string;
|
||||||
|
amountCents: number;
|
||||||
|
accountId: number | null;
|
||||||
|
date: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type Suggestion = {
|
||||||
|
key: string;
|
||||||
|
description: string;
|
||||||
|
/** Most common amount for this description (latest wins ties). */
|
||||||
|
amountCents: number;
|
||||||
|
/** Account used most recently with this description. */
|
||||||
|
accountId: number | null;
|
||||||
|
count: number;
|
||||||
|
lastDate: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Past descriptions ranked by use, each with its usual amount and latest account. */
|
||||||
|
export function descriptionSuggestions(rows: SuggestionSource[], today: string): Suggestion[] {
|
||||||
|
const groups = new Map<
|
||||||
|
string,
|
||||||
|
{
|
||||||
|
description: string;
|
||||||
|
amounts: Map<number, number>;
|
||||||
|
accountId: number | null;
|
||||||
|
count: number;
|
||||||
|
lastDate: string;
|
||||||
|
lastAmount: number;
|
||||||
|
}
|
||||||
|
>();
|
||||||
|
for (const row of rows) {
|
||||||
|
if (row.date > today) continue;
|
||||||
|
const key = row.description.trim().toLowerCase();
|
||||||
|
if (!key) continue;
|
||||||
|
const group = groups.get(key) ?? {
|
||||||
|
description: row.description.trim(),
|
||||||
|
amounts: new Map(),
|
||||||
|
accountId: row.accountId,
|
||||||
|
count: 0,
|
||||||
|
lastDate: '',
|
||||||
|
lastAmount: row.amountCents
|
||||||
|
};
|
||||||
|
group.count++;
|
||||||
|
group.amounts.set(row.amountCents, (group.amounts.get(row.amountCents) ?? 0) + 1);
|
||||||
|
if (row.date >= group.lastDate) {
|
||||||
|
group.lastDate = row.date;
|
||||||
|
group.description = row.description.trim();
|
||||||
|
group.accountId = row.accountId;
|
||||||
|
group.lastAmount = row.amountCents;
|
||||||
|
}
|
||||||
|
groups.set(key, group);
|
||||||
|
}
|
||||||
|
return [...groups.entries()]
|
||||||
|
.map(([key, group]) => {
|
||||||
|
let amountCents = group.lastAmount;
|
||||||
|
let best = group.amounts.get(amountCents) ?? 0;
|
||||||
|
for (const [value, uses] of group.amounts)
|
||||||
|
if (uses > best) {
|
||||||
|
best = uses;
|
||||||
|
amountCents = value;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
description: group.description,
|
||||||
|
amountCents,
|
||||||
|
accountId: group.accountId,
|
||||||
|
count: group.count,
|
||||||
|
lastDate: group.lastDate
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.sort((a, b) => b.count - a.count || b.lastDate.localeCompare(a.lastDate));
|
||||||
|
}
|
||||||
26
src/lib/money-tracker/wishlist.test.ts
Normal file
26
src/lib/money-tracker/wishlist.test.ts
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { sortWishlist, tierClass } from './wishlist';
|
||||||
|
|
||||||
|
describe('wish list order', () => {
|
||||||
|
it('sorts by tier descending, then price descending, unpriced and untiered last', () => {
|
||||||
|
const items = [
|
||||||
|
{ id: 1, priority: 3, estimatedCents: 5000 },
|
||||||
|
{ id: 2, priority: 5, estimatedCents: 1000 },
|
||||||
|
{ id: 3, priority: 5, estimatedCents: null },
|
||||||
|
{ id: 4, priority: 5, estimatedCents: 90000 },
|
||||||
|
{ id: 5, priority: null, estimatedCents: 99999 },
|
||||||
|
{ id: 6, priority: 1, estimatedCents: 200 },
|
||||||
|
{ id: 7, priority: 3, estimatedCents: 25000 },
|
||||||
|
{ id: 8, priority: 3, estimatedCents: 5000 }
|
||||||
|
];
|
||||||
|
expect(sortWishlist(items).map((item) => item.id)).toEqual([4, 2, 3, 7, 1, 8, 6, 5]);
|
||||||
|
expect(items[0].id).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('maps tiers to colour classes', () => {
|
||||||
|
expect(tierClass(5)).toBe('tier-5');
|
||||||
|
expect(tierClass(1)).toBe('tier-1');
|
||||||
|
expect(tierClass(null)).toBe('tier-none');
|
||||||
|
expect(tierClass(9)).toBe('tier-none');
|
||||||
|
});
|
||||||
|
});
|
||||||
25
src/lib/money-tracker/wishlist.ts
Normal file
25
src/lib/money-tracker/wishlist.ts
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
export type WishSortable = {
|
||||||
|
id?: number;
|
||||||
|
priority: number | null;
|
||||||
|
estimatedCents: number | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wish list order: tier 5 first down to tier 1, then the most expensive first within a tier.
|
||||||
|
* Items without a tier sink below every tier; unpriced items sink to the end of their tier.
|
||||||
|
*/
|
||||||
|
export function sortWishlist<T extends WishSortable>(items: readonly T[]): T[] {
|
||||||
|
const rank = (value: number | null) => (value == null ? -Infinity : value);
|
||||||
|
return [...items].sort(
|
||||||
|
(a, b) =>
|
||||||
|
rank(b.priority) - rank(a.priority) ||
|
||||||
|
(a.estimatedCents == null ? 1 : 0) - (b.estimatedCents == null ? 1 : 0) ||
|
||||||
|
(b.estimatedCents ?? 0) - (a.estimatedCents ?? 0) ||
|
||||||
|
(a.id ?? 0) - (b.id ?? 0)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** CSS class for a tier colour (tier-1 … tier-5), or "tier-none". */
|
||||||
|
export function tierClass(priority: number | null): string {
|
||||||
|
return priority != null && priority >= 1 && priority <= 5 ? `tier-${priority}` : 'tier-none';
|
||||||
|
}
|
||||||
21
src/lib/projects.test.ts
Normal file
21
src/lib/projects.test.ts
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { visibleTrackers } from './projects';
|
||||||
|
|
||||||
|
describe('trackers', () => {
|
||||||
|
it('lists tasks, money, books first for the owner', () => {
|
||||||
|
expect(
|
||||||
|
visibleTrackers(true)
|
||||||
|
.map((t) => t.name)
|
||||||
|
.slice(0, 4)
|
||||||
|
).toEqual(['tasks', 'money', 'books', 'health']);
|
||||||
|
});
|
||||||
|
it('hides private trackers when logged out', () => {
|
||||||
|
const names = visibleTrackers(false).map((t) => t.name);
|
||||||
|
expect(names).not.toContain('tasks');
|
||||||
|
expect(names).not.toContain('health');
|
||||||
|
expect(names.slice(0, 2)).toEqual(['money', 'books']);
|
||||||
|
});
|
||||||
|
it('uses short single-word names', () => {
|
||||||
|
for (const t of visibleTrackers(true)) expect(t.name).toMatch(/^[a-z]+$/);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -32,58 +32,96 @@ export const projects = [
|
|||||||
}
|
}
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
// Trackers keep their existing /projects/... URLs (no moves, so no redirects needed).
|
// Trackers, in display order. Names are short single nouns. Most keep their existing
|
||||||
export const trackers = [
|
// /projects/... URLs (no moves, so no redirects needed). `private` trackers are shown in the
|
||||||
|
// header, home page and /trackers only to the logged-in owner (layout data `isOwner`), never
|
||||||
|
// in the sitemap; their routes 404 for everyone else.
|
||||||
|
const allTrackers = [
|
||||||
{
|
{
|
||||||
name: 'game tracker',
|
name: 'tasks',
|
||||||
|
subtitle: 'private: to-dos, goals and my TickTick history',
|
||||||
|
href: '/trackers/tasks',
|
||||||
|
external: false,
|
||||||
|
private: true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'money',
|
||||||
|
subtitle: 'a manual ledger for accounts, bills and utilities',
|
||||||
|
href: '/trackers/money',
|
||||||
|
external: false,
|
||||||
|
private: false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'books',
|
||||||
|
subtitle: 'my library: what I am reading, have read and want to read',
|
||||||
|
href: '/trackers/books',
|
||||||
|
external: false,
|
||||||
|
private: false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'health',
|
||||||
|
subtitle: 'private: body, workouts, sleep, meds and vaccinations',
|
||||||
|
href: '/trackers/health',
|
||||||
|
external: false,
|
||||||
|
private: true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'energy',
|
||||||
|
subtitle: 'Dutch electricity mix, utilization and gas storage',
|
||||||
|
href: '/projects/energy',
|
||||||
|
external: false,
|
||||||
|
private: false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'games',
|
||||||
subtitle: 'every pokemon in every game, and call of duty camo mastery',
|
subtitle: 'every pokemon in every game, and call of duty camo mastery',
|
||||||
href: '/projects/game-tracker',
|
href: '/projects/game-tracker',
|
||||||
external: false
|
external: false,
|
||||||
|
private: false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'money tracker',
|
name: 'houses',
|
||||||
subtitle: 'a manual ledger for accounts, bills and utilities',
|
|
||||||
href: '/projects/money-tracker',
|
|
||||||
external: false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'house tracker',
|
|
||||||
subtitle: 'Dutch rental and sale listings, with alerts',
|
subtitle: 'Dutch rental and sale listings, with alerts',
|
||||||
href: '/projects/house-tracker',
|
href: '/projects/house-tracker',
|
||||||
external: false
|
external: false,
|
||||||
|
private: false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'cs prices',
|
name: 'skins',
|
||||||
subtitle: 'Counter-Strike market prices across time',
|
subtitle: 'Counter-Strike skin market prices across time',
|
||||||
href: '/projects/cs-prices',
|
href: '/projects/cs-prices',
|
||||||
external: false
|
external: false,
|
||||||
|
private: false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'animaldex',
|
name: 'pets',
|
||||||
subtitle: 'add neighborhood pets to the world map',
|
subtitle: 'animaldex: add neighborhood pets to the world map',
|
||||||
href: '/projects/animaldex',
|
href: '/projects/animaldex',
|
||||||
external: false
|
external: false,
|
||||||
|
private: false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'lists',
|
name: 'lists',
|
||||||
subtitle: 'books, games, places and other things worth keeping',
|
subtitle: 'books, games, places and other things worth keeping',
|
||||||
href: '/lists',
|
href: '/lists',
|
||||||
external: false
|
external: false,
|
||||||
}
|
private: false
|
||||||
] as const;
|
|
||||||
|
|
||||||
// Owner-only trackers: shown in the header, home page and /trackers only to the logged-in
|
|
||||||
// owner (layout data `isOwner`), never in the sitemap. Their routes 404 for everyone else.
|
|
||||||
export const privateTrackers = [
|
|
||||||
{
|
|
||||||
name: 'health',
|
|
||||||
subtitle: 'private: body, workouts, sleep, meds and vaccinations',
|
|
||||||
href: '/trackers/health',
|
|
||||||
external: false
|
|
||||||
}
|
}
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
type Tracker = { name: string; subtitle: string; href: string; external: boolean };
|
type Tracker = { name: string; subtitle: string; href: string; external: boolean };
|
||||||
|
const strip = ({ name, subtitle, href, external }: Tracker): Tracker => ({
|
||||||
|
name,
|
||||||
|
subtitle,
|
||||||
|
href,
|
||||||
|
external
|
||||||
|
});
|
||||||
|
/** Public trackers (sitemap, logged-out navigation). */
|
||||||
|
export const trackers: readonly Tracker[] = allTrackers.filter((t) => !t.private).map(strip);
|
||||||
|
/** Owner-only trackers. */
|
||||||
|
export const privateTrackers: readonly Tracker[] = allTrackers.filter((t) => t.private).map(strip);
|
||||||
|
const ownerTrackers: readonly Tracker[] = allTrackers.map(strip);
|
||||||
|
|
||||||
export function visibleTrackers(isOwner: boolean): readonly Tracker[] {
|
export function visibleTrackers(isOwner: boolean): readonly Tracker[] {
|
||||||
return isOwner ? [...trackers, ...privateTrackers] : trackers;
|
return isOwner ? ownerTrackers : trackers;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,7 +38,11 @@ const publicPages: Record<string, { title: string; description: string }> = {
|
|||||||
'/trackers': {
|
'/trackers': {
|
||||||
title: 'Trackers',
|
title: 'Trackers',
|
||||||
description:
|
description:
|
||||||
'Stan Runge’s trackers: games, money, houses, Counter-Strike prices, Animaldex and lists.'
|
'Stan Runge’s trackers: games, money, houses, books, Counter-Strike prices, Dutch energy, Animaldex and lists.'
|
||||||
|
},
|
||||||
|
'/trackers/books': {
|
||||||
|
title: 'Books',
|
||||||
|
description: 'Stan Runge’s library: books being read, finished and on the to-read pile.'
|
||||||
},
|
},
|
||||||
'/lists': { title: 'Lists', description: 'Lists collected and maintained by Stan Runge.' },
|
'/lists': { title: 'Lists', description: 'Lists collected and maintained by Stan Runge.' },
|
||||||
'/projects/game-tracker': {
|
'/projects/game-tracker': {
|
||||||
@@ -69,6 +73,11 @@ const publicPages: Record<string, { title: string; description: string }> = {
|
|||||||
description:
|
description:
|
||||||
'Compare historical Counter-Strike market prices by listing, weapon, skin, wear, and other item dimensions.'
|
'Compare historical Counter-Strike market prices by listing, weapon, skin, wear, and other item dimensions.'
|
||||||
},
|
},
|
||||||
|
'/projects/energy': {
|
||||||
|
title: 'Energy',
|
||||||
|
description:
|
||||||
|
'Dutch electricity generation mix, source utilization, consumption trends, CO₂ intensity and gas storage, from NED.nl data.'
|
||||||
|
},
|
||||||
'/projects/derby': {
|
'/projects/derby': {
|
||||||
title: 'Derby Lens',
|
title: 'Derby Lens',
|
||||||
description: 'Estimate horse racing odds and compare them with market prices.'
|
description: 'Estimate horse racing odds and compare them with market prices.'
|
||||||
|
|||||||
208
src/lib/server/bank-sync.test.ts
Normal file
208
src/lib/server/bank-sync.test.ts
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { generateKeyPairSync } from 'node:crypto';
|
||||||
|
import { decodeProtectedHeader, jwtVerify } from 'jose';
|
||||||
|
import {
|
||||||
|
amountToCents,
|
||||||
|
externalIdFor,
|
||||||
|
findMatch,
|
||||||
|
mapBankTransaction,
|
||||||
|
pickBalance,
|
||||||
|
planAccountMapping
|
||||||
|
} from './bank-sync';
|
||||||
|
import { createEnableBankingClient, parseAuthCode, type BankTransaction } from './enable-banking';
|
||||||
|
|
||||||
|
const tx = (overrides: Partial<BankTransaction> = {}): BankTransaction => ({
|
||||||
|
entry_reference: 'ref-1',
|
||||||
|
transaction_amount: { currency: 'EUR', amount: '12.34' },
|
||||||
|
credit_debit_indicator: 'DBIT',
|
||||||
|
status: 'BOOK',
|
||||||
|
booking_date: '2026-10-20',
|
||||||
|
value_date: '2026-10-19',
|
||||||
|
creditor: { name: 'Albert Heijn' },
|
||||||
|
remittance_information: ['AH 1234 Gouda'],
|
||||||
|
...overrides
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('bank transaction mapping', () => {
|
||||||
|
it('parses decimal amounts exactly', () => {
|
||||||
|
expect(amountToCents('12.34')).toBe(1234);
|
||||||
|
expect(amountToCents('-0.5')).toBe(-50);
|
||||||
|
expect(amountToCents('100')).toBe(10000);
|
||||||
|
expect(amountToCents('1.005')).toBe(101);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('signs debits negative and uses the counterparty as description', () => {
|
||||||
|
const row = mapBankTransaction(tx())!;
|
||||||
|
expect(row).toMatchObject({
|
||||||
|
externalId: 'ref-1',
|
||||||
|
date: '2026-10-20',
|
||||||
|
valueDate: '2026-10-19',
|
||||||
|
amountCents: -1234,
|
||||||
|
description: 'Albert Heijn',
|
||||||
|
counterparty: 'Albert Heijn',
|
||||||
|
bookingStatus: 'BOOK'
|
||||||
|
});
|
||||||
|
const credit = mapBankTransaction(
|
||||||
|
tx({ credit_debit_indicator: 'CRDT', creditor: null, debtor: { name: 'Juhee' } })
|
||||||
|
)!;
|
||||||
|
expect(credit.amountCents).toBe(1234);
|
||||||
|
expect(credit.description).toBe('Juhee');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips pending transactions', () => {
|
||||||
|
expect(mapBankTransaction(tx({ status: 'PDNG' }))).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to remittance info and to a stable content hash', () => {
|
||||||
|
const anonymous = tx({ entry_reference: null, creditor: null });
|
||||||
|
const row = mapBankTransaction(anonymous)!;
|
||||||
|
expect(row.description).toBe('AH 1234 Gouda');
|
||||||
|
expect(row.externalId).toMatch(/^h:[0-9a-f]{32}$/);
|
||||||
|
expect(externalIdFor(anonymous)).toBe(row.externalId);
|
||||||
|
expect(
|
||||||
|
externalIdFor({ ...anonymous, transaction_amount: { currency: 'EUR', amount: '1.00' } })
|
||||||
|
).not.toBe(row.externalId);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('reconcile matching', () => {
|
||||||
|
const rows = [
|
||||||
|
{ id: 1, date: '2026-10-18', amountCents: -1234, description: 'Groceries' },
|
||||||
|
{ id: 2, date: '2026-10-21', amountCents: -1234, description: 'Albert Heijn' },
|
||||||
|
{ id: 3, date: '2026-10-20', amountCents: -999, description: 'Albert Heijn' }
|
||||||
|
];
|
||||||
|
it('matches same amount within one day, preferring description overlap', () => {
|
||||||
|
const m = findMatch(
|
||||||
|
{ date: '2026-10-20', amountCents: -1234, description: 'Albert Heijn' },
|
||||||
|
rows,
|
||||||
|
new Set()
|
||||||
|
);
|
||||||
|
expect(m?.id).toBe(2);
|
||||||
|
});
|
||||||
|
it('never reuses a row and ignores rows more than a day away', () => {
|
||||||
|
expect(
|
||||||
|
findMatch({ date: '2026-10-20', amountCents: -1234, description: 'x' }, rows, new Set([2]))
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('account mapping', () => {
|
||||||
|
const tracker = [
|
||||||
|
{ id: 1, name: 'Revolut', scope: 'personal', iban: null, external_uid: null },
|
||||||
|
{ id: 4, name: 'Joint', scope: 'joint', iban: null, external_uid: null }
|
||||||
|
];
|
||||||
|
it('maps the joint account and the personal EUR account', () => {
|
||||||
|
const plan = planAccountMapping(
|
||||||
|
[
|
||||||
|
{ uid: 'a', account_id: { iban: 'LT1' }, name: 'Stan Runge', currency: 'EUR' },
|
||||||
|
{ uid: 'b', account_id: { iban: 'LT2' }, name: 'Stan Runge & Juhee', currency: 'EUR' },
|
||||||
|
{ uid: 'c', account_id: { iban: 'LT3' }, name: 'Stan Runge', currency: 'USD' }
|
||||||
|
],
|
||||||
|
tracker
|
||||||
|
);
|
||||||
|
expect(plan.map((p) => [p.bank.uid, p.tracker?.name ?? null])).toEqual([
|
||||||
|
['b', 'Joint'],
|
||||||
|
['a', 'Revolut'],
|
||||||
|
['c', null]
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
it('honours overrides and existing IBANs', () => {
|
||||||
|
const plan = planAccountMapping(
|
||||||
|
[
|
||||||
|
{ uid: 'a', account_id: { iban: 'LT1' }, name: 'X', currency: 'EUR' },
|
||||||
|
{ uid: 'b', account_id: { iban: 'LT2' }, name: 'Y', currency: 'EUR' }
|
||||||
|
],
|
||||||
|
[{ ...tracker[0], iban: 'LT2' }, tracker[1]],
|
||||||
|
{ a: 'Joint' }
|
||||||
|
);
|
||||||
|
expect(plan.map((p) => [p.bank.uid, p.tracker?.name])).toEqual([
|
||||||
|
['a', 'Joint'],
|
||||||
|
['b', 'Revolut']
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
it('maps another bank to its own tracker account without the joint heuristic', () => {
|
||||||
|
const plan = planAccountMapping(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
uid: 'n',
|
||||||
|
account_id: { iban: 'NL01ABNA0123456789' },
|
||||||
|
name: 'S Runge & J Kim',
|
||||||
|
currency: 'EUR'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
[...tracker, { id: 7, name: 'ABN Amro', scope: 'personal', iban: null, external_uid: null }],
|
||||||
|
{},
|
||||||
|
{ defaultAccount: 'ABN Amro', joint: false }
|
||||||
|
);
|
||||||
|
expect(plan.map((p) => [p.bank.uid, p.tracker?.name ?? null])).toEqual([['n', 'ABN Amro']]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('enable banking client', () => {
|
||||||
|
it('parses codes from redirect URLs, query strings and bare codes', () => {
|
||||||
|
expect(
|
||||||
|
parseAuthCode(
|
||||||
|
'https://stanrunge.dev/projects/money-tracker/bank-callback?state=s&code=abc-123'
|
||||||
|
)
|
||||||
|
).toEqual({
|
||||||
|
code: 'abc-123',
|
||||||
|
state: 's',
|
||||||
|
error: undefined
|
||||||
|
});
|
||||||
|
expect(parseAuthCode('abc-123').code).toBe('abc-123');
|
||||||
|
expect(parseAuthCode('?error=access_denied&error_description=Cancelled').error).toBe(
|
||||||
|
'access_denied: Cancelled'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('signs RS256 JWTs with the app id as kid and pages through transactions', async () => {
|
||||||
|
const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 });
|
||||||
|
const calls: string[] = [];
|
||||||
|
const fakeFetch = (async (url: string, init: RequestInit) => {
|
||||||
|
calls.push(url);
|
||||||
|
const auth = String((init.headers as Record<string, string>).Authorization).slice(7);
|
||||||
|
expect(decodeProtectedHeader(auth)).toMatchObject({
|
||||||
|
alg: 'RS256',
|
||||||
|
kid: 'app-id',
|
||||||
|
typ: 'JWT'
|
||||||
|
});
|
||||||
|
const { payload } = await jwtVerify(auth, publicKey, {
|
||||||
|
issuer: 'enablebanking.com',
|
||||||
|
audience: 'api.enablebanking.com'
|
||||||
|
});
|
||||||
|
expect(payload.exp! - payload.iat!).toBeLessThanOrEqual(86400);
|
||||||
|
const page = new URL(url).searchParams.get('continuation_key');
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify(
|
||||||
|
page
|
||||||
|
? { transactions: [tx({ entry_reference: 'r2' })] }
|
||||||
|
: { transactions: [tx()], continuation_key: 'next' }
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}) as unknown as typeof fetch;
|
||||||
|
const client = createEnableBankingClient({
|
||||||
|
appId: 'app-id',
|
||||||
|
privateKeyPem: privateKey.export({ type: 'pkcs8', format: 'pem' }).toString(),
|
||||||
|
fetch: fakeFetch
|
||||||
|
});
|
||||||
|
const all = await client.transactions('uid-1', { dateFrom: '2026-01-01' });
|
||||||
|
expect(all.map((t) => t.entry_reference)).toEqual(['ref-1', 'r2']);
|
||||||
|
expect(calls[1]).toContain('continuation_key=next');
|
||||||
|
expect(calls[0]).toContain('date_from=2026-01-01');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('bank balance', () => {
|
||||||
|
const b = (balance_type: string, amount: string) => ({
|
||||||
|
balance_type,
|
||||||
|
balance_amount: { amount, currency: 'EUR' }
|
||||||
|
});
|
||||||
|
it('prefers the available balance the bank app shows', () => {
|
||||||
|
expect(pickBalance([b('CLBD', '70.00'), b('ITAV', '69.88')])?.balance_amount.amount).toBe(
|
||||||
|
'69.88'
|
||||||
|
);
|
||||||
|
expect(pickBalance([b('XPCD', '1.00'), b('CLBD', '2.00')])?.balance_type).toBe('CLBD');
|
||||||
|
expect(pickBalance([b('OTHR', '3.00')])?.balance_type).toBe('OTHR');
|
||||||
|
expect(pickBalance([])).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
537
src/lib/server/bank-sync.ts
Normal file
537
src/lib/server/bank-sync.ts
Normal file
@@ -0,0 +1,537 @@
|
|||||||
|
// Bank -> money.transactions sync (Enable Banking). Pure mapping/matching helpers are
|
||||||
|
// exported for unit tests; the DB functions take a postgres.js `Sql` so they run the
|
||||||
|
// same from the bun scripts (scripts/bank-link.ts, scripts/bank-sync.ts).
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import type { Sql } from 'postgres';
|
||||||
|
import type {
|
||||||
|
Balance,
|
||||||
|
BankAccount,
|
||||||
|
BankTransaction,
|
||||||
|
EnableBankingClient,
|
||||||
|
SessionResponse
|
||||||
|
} from './enable-banking';
|
||||||
|
import { EnableBankingError } from './enable-banking';
|
||||||
|
import { findSumMatch } from './reconcile';
|
||||||
|
|
||||||
|
export type MappedTransaction = {
|
||||||
|
externalId: string;
|
||||||
|
date: string;
|
||||||
|
valueDate: string | null;
|
||||||
|
description: string;
|
||||||
|
counterparty: string | null;
|
||||||
|
amountCents: number;
|
||||||
|
bookingStatus: string;
|
||||||
|
raw: BankTransaction;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function amountToCents(amount: string): number {
|
||||||
|
const raw = amount.trim();
|
||||||
|
if (!/^-?\d+(?:\.\d+)?$/.test(raw)) throw new Error(`Invalid amount ${amount}`);
|
||||||
|
const negative = raw.startsWith('-');
|
||||||
|
const [whole, fraction = ''] = raw.replace('-', '').split('.');
|
||||||
|
const cents = Number(whole) * 100 + Number((fraction + '00').slice(0, 2));
|
||||||
|
const rounded = Number(fraction.slice(2, 3) || '0') >= 5 ? cents + 1 : cents;
|
||||||
|
return negative ? -rounded : rounded;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stable id for a bank transaction: the bank's own reference, else a content hash. */
|
||||||
|
export function externalIdFor(tx: BankTransaction): string {
|
||||||
|
const ref = tx.entry_reference?.trim() || tx.transaction_id?.trim();
|
||||||
|
if (ref) return ref;
|
||||||
|
const fingerprint = [
|
||||||
|
tx.booking_date ?? tx.transaction_date ?? tx.value_date ?? '',
|
||||||
|
tx.credit_debit_indicator,
|
||||||
|
tx.transaction_amount.amount,
|
||||||
|
tx.transaction_amount.currency,
|
||||||
|
tx.creditor?.name ?? '',
|
||||||
|
tx.debtor?.name ?? '',
|
||||||
|
(tx.remittance_information ?? []).join('|')
|
||||||
|
].join('\u001f');
|
||||||
|
return 'h:' + createHash('sha256').update(fingerprint).digest('hex').slice(0, 32);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isBooked(tx: BankTransaction) {
|
||||||
|
return tx.status ? tx.status === 'BOOK' : Boolean(tx.booking_date);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Maps one booked bank transaction to a tracker row, or null for pending ones. */
|
||||||
|
export function mapBankTransaction(tx: BankTransaction): MappedTransaction | null {
|
||||||
|
if (!isBooked(tx)) return null;
|
||||||
|
const date = tx.booking_date ?? tx.transaction_date ?? tx.value_date;
|
||||||
|
if (!date) return null;
|
||||||
|
const magnitude = Math.abs(amountToCents(tx.transaction_amount.amount));
|
||||||
|
const amountCents = tx.credit_debit_indicator === 'DBIT' ? -magnitude : magnitude;
|
||||||
|
const counterparty =
|
||||||
|
(tx.credit_debit_indicator === 'DBIT' ? tx.creditor?.name : tx.debtor?.name)?.trim() || null;
|
||||||
|
const remittance = (tx.remittance_information ?? []).map((line) => line.trim()).filter(Boolean);
|
||||||
|
const description = (
|
||||||
|
counterparty ||
|
||||||
|
remittance.join(' ') ||
|
||||||
|
tx.bank_transaction_code?.description ||
|
||||||
|
tx.note ||
|
||||||
|
(amountCents < 0 ? 'Card payment' : 'Incoming transfer')
|
||||||
|
).slice(0, 500);
|
||||||
|
return {
|
||||||
|
externalId: externalIdFor(tx),
|
||||||
|
date: date.slice(0, 10),
|
||||||
|
valueDate: tx.value_date?.slice(0, 10) ?? null,
|
||||||
|
description,
|
||||||
|
counterparty,
|
||||||
|
amountCents,
|
||||||
|
bookingStatus: tx.status ?? 'BOOK',
|
||||||
|
raw: tx
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function dayDiff(a: string, b: string) {
|
||||||
|
return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const isOpeningBalance = (description: string) =>
|
||||||
|
/^(start|opening) balance/i.test(description.trim());
|
||||||
|
|
||||||
|
export type ExistingRow = { id: number; date: string; amountCents: number; description: string };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Finds the hand-entered row a bank transaction corresponds to: same amount, date within
|
||||||
|
* ±1 day, not matched yet. Prefers the closest date, then a description overlap.
|
||||||
|
*/
|
||||||
|
export function findMatch(
|
||||||
|
tx: Pick<MappedTransaction, 'date' | 'amountCents' | 'description'>,
|
||||||
|
candidates: ExistingRow[],
|
||||||
|
used: Set<number>
|
||||||
|
): ExistingRow | null {
|
||||||
|
const words = new Set(
|
||||||
|
tx.description
|
||||||
|
.toLowerCase()
|
||||||
|
.split(/\W+/)
|
||||||
|
.filter((w) => w.length > 2)
|
||||||
|
);
|
||||||
|
let best: { row: ExistingRow; score: number } | null = null;
|
||||||
|
for (const row of candidates) {
|
||||||
|
if (used.has(row.id) || row.amountCents !== tx.amountCents) continue;
|
||||||
|
const diff = dayDiff(row.date, tx.date);
|
||||||
|
if (diff > 1) continue;
|
||||||
|
const overlap = row.description
|
||||||
|
.toLowerCase()
|
||||||
|
.split(/\W+/)
|
||||||
|
.some((w) => words.has(w));
|
||||||
|
const score = diff * 10 - (overlap ? 1 : 0) + row.id / 1e9;
|
||||||
|
if (!best || score < best.score) best = { row, score };
|
||||||
|
}
|
||||||
|
return best?.row ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isoDate(date: Date) {
|
||||||
|
return date.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function addDays(date: string, days: number) {
|
||||||
|
return isoDate(new Date(Date.parse(date + 'T00:00:00Z') + days * 86_400_000));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Heuristic: is this bank account the joint one shared with Juhee? */
|
||||||
|
export function looksJoint(account: BankAccount) {
|
||||||
|
const text = [account.name, account.details, account.product].filter(Boolean).join(' ');
|
||||||
|
return /joint|gezamenlijk|shared|\bjuhee\b|\s&\s|\sand\s|,/i.test(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
type TrackerAccount = {
|
||||||
|
id: number;
|
||||||
|
name: string;
|
||||||
|
scope: string;
|
||||||
|
iban: string | null;
|
||||||
|
external_uid: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decides which tracker account each bank account feeds. Order: explicit override
|
||||||
|
* (uid/IBAN -> tracker account name), same IBAN, joint heuristic -> the joint-scoped
|
||||||
|
* account (Revolut only), first remaining EUR current account -> `defaultAccount`
|
||||||
|
* ("Revolut" unless given). Others are skipped.
|
||||||
|
*/
|
||||||
|
export function planAccountMapping(
|
||||||
|
bankAccounts: BankAccount[],
|
||||||
|
tracker: TrackerAccount[],
|
||||||
|
overrides: Record<string, string> = {},
|
||||||
|
options: {
|
||||||
|
/** Tracker account for the first personal EUR current account (default "Revolut"). */
|
||||||
|
defaultAccount?: string;
|
||||||
|
/** Route joint-looking accounts to the joint-scoped tracker account (Revolut only). */
|
||||||
|
joint?: boolean;
|
||||||
|
} = {}
|
||||||
|
) {
|
||||||
|
const defaultAccount = (options.defaultAccount ?? 'Revolut').toLowerCase();
|
||||||
|
const joint = options.joint ?? defaultAccount === 'revolut';
|
||||||
|
const byName = new Map(tracker.map((a) => [a.name.toLowerCase(), a]));
|
||||||
|
const used = new Set<number>();
|
||||||
|
const plan: { bank: BankAccount; tracker: TrackerAccount | null; reason: string }[] = [];
|
||||||
|
const take = (bank: BankAccount, account: TrackerAccount | undefined | null, reason: string) => {
|
||||||
|
if (account && !used.has(account.id)) {
|
||||||
|
used.add(account.id);
|
||||||
|
plan.push({ bank, tracker: account, reason });
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
const pending: BankAccount[] = [];
|
||||||
|
for (const bank of bankAccounts) {
|
||||||
|
const iban = bank.account_id?.iban?.replace(/\s/g, '').toUpperCase();
|
||||||
|
const override = overrides[bank.uid ?? ''] ?? (iban ? overrides[iban] : undefined);
|
||||||
|
if (override !== undefined) {
|
||||||
|
if (override === '' || override === '-')
|
||||||
|
plan.push({ bank, tracker: null, reason: 'skipped (override)' });
|
||||||
|
else if (!take(bank, byName.get(override.toLowerCase()), 'override'))
|
||||||
|
plan.push({ bank, tracker: null, reason: `override target "${override}" not found/used` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
iban &&
|
||||||
|
take(
|
||||||
|
bank,
|
||||||
|
tracker.find((a) => a.iban?.replace(/\s/g, '').toUpperCase() === iban),
|
||||||
|
'same IBAN'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue;
|
||||||
|
pending.push(bank);
|
||||||
|
}
|
||||||
|
const eur = (bank: BankAccount) => !bank.currency || bank.currency === 'EUR';
|
||||||
|
// Joint first, so a personal-looking account listed earlier can't take "Revolut" from it.
|
||||||
|
const rest = pending.filter(
|
||||||
|
(bank) =>
|
||||||
|
!(
|
||||||
|
joint &&
|
||||||
|
eur(bank) &&
|
||||||
|
looksJoint(bank) &&
|
||||||
|
take(
|
||||||
|
bank,
|
||||||
|
tracker.find((a) => a.scope === 'joint'),
|
||||||
|
'joint account'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
for (const bank of rest) {
|
||||||
|
if (
|
||||||
|
eur(bank) &&
|
||||||
|
bank.cash_account_type !== 'SVGS' &&
|
||||||
|
take(bank, byName.get(defaultAccount), 'personal EUR account')
|
||||||
|
)
|
||||||
|
continue;
|
||||||
|
plan.push({ bank, tracker: null, reason: 'no match (use --map uid=Account)' });
|
||||||
|
}
|
||||||
|
return plan;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function trackerUserId(sql: Sql, explicit?: string) {
|
||||||
|
if (explicit) return explicit;
|
||||||
|
const rows = await sql<{ user_id: string }[]>`select distinct user_id from money.accounts`;
|
||||||
|
if (rows.length !== 1)
|
||||||
|
throw new Error(`Expected one money-tracker user, found ${rows.length}; pass --user <id>`);
|
||||||
|
return rows[0].user_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stores a freshly authorised session and links bank accounts to tracker accounts. */
|
||||||
|
export async function saveSession(
|
||||||
|
sql: Sql,
|
||||||
|
userId: string,
|
||||||
|
session: SessionResponse,
|
||||||
|
plan: ReturnType<typeof planAccountMapping>
|
||||||
|
) {
|
||||||
|
const accounts = session.accounts.map((a) => ({
|
||||||
|
uid: a.uid ?? null,
|
||||||
|
iban: a.account_id?.iban ?? null,
|
||||||
|
name: a.name ?? null,
|
||||||
|
details: a.details ?? null,
|
||||||
|
product: a.product ?? null,
|
||||||
|
currency: a.currency ?? null,
|
||||||
|
cash_account_type: a.cash_account_type ?? null,
|
||||||
|
identification_hash: a.identification_hash ?? null
|
||||||
|
}));
|
||||||
|
await sql.begin(async (tx) => {
|
||||||
|
await tx`
|
||||||
|
insert into money.bank_sessions
|
||||||
|
(user_id, provider, session_id, aspsp_name, aspsp_country, psu_type, status, valid_until, accounts)
|
||||||
|
values (${userId}, 'enablebanking', ${session.session_id}, ${session.aspsp.name},
|
||||||
|
${session.aspsp.country}, ${session.psu_type}, 'authorized', ${session.access.valid_until},
|
||||||
|
${tx.json(accounts)})
|
||||||
|
on conflict (session_id) do update set status = 'authorized', valid_until = excluded.valid_until,
|
||||||
|
accounts = excluded.accounts, last_error = null`;
|
||||||
|
for (const { bank, tracker } of plan) {
|
||||||
|
if (!tracker || !bank.uid) continue;
|
||||||
|
// Account uids are per session: free the uid from any older mapping first.
|
||||||
|
await tx`update money.accounts set external_uid = null where external_uid = ${bank.uid} and id <> ${tracker.id}`;
|
||||||
|
await tx`
|
||||||
|
update money.accounts
|
||||||
|
set external_uid = ${bank.uid}, provider = 'enablebanking',
|
||||||
|
iban = coalesce(${bank.account_id?.iban ?? null}, iban)
|
||||||
|
where id = ${tracker.id} and user_id = ${userId}`;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The balance Stan sees in the bank app: interim available, else closing booked. */
|
||||||
|
export function pickBalance(balances: Balance[]): Balance | undefined {
|
||||||
|
const order = ['ITAV', 'CLAV', 'ITBD', 'CLBD', 'XPCD'];
|
||||||
|
for (const type of order) {
|
||||||
|
const found = balances.find((b) => b.balance_type === type);
|
||||||
|
if (found) return found;
|
||||||
|
}
|
||||||
|
return balances[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Synthetic row that makes the tracker's sum equal the bank balance. */
|
||||||
|
export const BALANCE_ADJUSTMENT_ID = 'balance-adjustment';
|
||||||
|
|
||||||
|
export type SyncOptions = {
|
||||||
|
dryRun?: boolean;
|
||||||
|
/** Match against the whole history of hand-entered rows (one-time), not just the window. */
|
||||||
|
reconcile?: boolean;
|
||||||
|
from?: string;
|
||||||
|
/** Fetch the longest history the bank serves (automatic on an account's first sync). */
|
||||||
|
full?: boolean;
|
||||||
|
accountName?: string;
|
||||||
|
log?: (line: string) => void;
|
||||||
|
today?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type AccountSyncResult = {
|
||||||
|
account: string;
|
||||||
|
fetched: number;
|
||||||
|
booked: number;
|
||||||
|
inserted: number;
|
||||||
|
matched: number;
|
||||||
|
/** Hand-entered rows replaced because they equal the sum of 2-3 bank payments. */
|
||||||
|
absorbed: number;
|
||||||
|
alreadyKnown: number;
|
||||||
|
/** Oldest booked date the bank returned. */
|
||||||
|
historyFrom?: string;
|
||||||
|
balance?: string;
|
||||||
|
/** Bank balance minus the tracker's own rows, absorbed by the adjustment row. */
|
||||||
|
adjustmentCents?: number;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function syncBankAccounts(
|
||||||
|
sql: Sql,
|
||||||
|
client: EnableBankingClient,
|
||||||
|
options: SyncOptions = {}
|
||||||
|
) {
|
||||||
|
const log = options.log ?? console.log;
|
||||||
|
const today = options.today ?? isoDate(new Date());
|
||||||
|
const accounts = await sql<
|
||||||
|
{ id: number; user_id: string; name: string; external_uid: string }[]
|
||||||
|
>`select id, user_id, name, external_uid from money.accounts
|
||||||
|
where provider = 'enablebanking' and external_uid is not null order by id`;
|
||||||
|
const selected = options.accountName
|
||||||
|
? accounts.filter((a) => a.name.toLowerCase() === options.accountName!.toLowerCase())
|
||||||
|
: accounts;
|
||||||
|
if (!selected.length) {
|
||||||
|
log('No linked bank accounts. Run scripts/bank-link.ts first.');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const results: AccountSyncResult[] = [];
|
||||||
|
for (const account of selected) {
|
||||||
|
const result: AccountSyncResult = {
|
||||||
|
account: account.name,
|
||||||
|
fetched: 0,
|
||||||
|
booked: 0,
|
||||||
|
inserted: 0,
|
||||||
|
matched: 0,
|
||||||
|
absorbed: 0,
|
||||||
|
alreadyKnown: 0
|
||||||
|
};
|
||||||
|
results.push(result);
|
||||||
|
try {
|
||||||
|
const [{ last }] = await sql<{ last: string | null }[]>`
|
||||||
|
select max(date)::text as last from money.transactions
|
||||||
|
where account_id = ${account.id} and source = 'enablebanking'`;
|
||||||
|
// First sync of an account (or --full): ask for the longest history the bank serves.
|
||||||
|
const longest = options.full || (!last && !options.from);
|
||||||
|
const dateFrom = options.from ?? (last ? addDays(last, -7) : undefined);
|
||||||
|
let raw: BankTransaction[];
|
||||||
|
try {
|
||||||
|
raw = await client.transactions(
|
||||||
|
account.external_uid,
|
||||||
|
longest ? { strategy: 'longest', dateFrom: options.from } : { dateFrom }
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
// Many banks only serve ~90 days of history after the first hour of consent.
|
||||||
|
if (error instanceof EnableBankingError && error.code === 'WRONG_TRANSACTIONS_PERIOD') {
|
||||||
|
log(
|
||||||
|
`${account.name}: ${dateFrom ?? 'full history'} is not available; retrying from 89 days ago`
|
||||||
|
);
|
||||||
|
raw = await client.transactions(account.external_uid, { dateFrom: addDays(today, -89) });
|
||||||
|
} else throw error;
|
||||||
|
}
|
||||||
|
result.fetched = raw.length;
|
||||||
|
const mapped = raw.map(mapBankTransaction).filter((t): t is MappedTransaction => t !== null);
|
||||||
|
result.booked = mapped.length;
|
||||||
|
result.historyFrom = mapped.reduce<string | undefined>(
|
||||||
|
(min, t) => (!min || t.date < min ? t.date : min),
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
if (!mapped.length) {
|
||||||
|
await syncBalance(sql, client, account, result, options, today, log);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ids = mapped.map((t) => t.externalId);
|
||||||
|
const known = new Set(
|
||||||
|
(
|
||||||
|
await sql<{ external_id: string }[]>`
|
||||||
|
select external_id from money.transactions
|
||||||
|
where account_id = ${account.id} and external_id = any(${ids})`
|
||||||
|
).map((r) => r.external_id)
|
||||||
|
);
|
||||||
|
const minDate = mapped.reduce((m, t) => (t.date < m ? t.date : m), mapped[0].date);
|
||||||
|
// Opening-balance rows are not payments; never match or replace them.
|
||||||
|
const candidates = (
|
||||||
|
await sql<ExistingRow[]>`
|
||||||
|
select id, date::text as date, amount_cents as "amountCents", description
|
||||||
|
from money.transactions
|
||||||
|
where account_id = ${account.id} and external_id is null
|
||||||
|
and date >= ${options.reconcile ? '0001-01-01' : addDays(minDate, -1)}`
|
||||||
|
).filter((row) => !isOpeningBalance(row.description));
|
||||||
|
const used = new Set<number>();
|
||||||
|
const inserts: MappedTransaction[] = [];
|
||||||
|
const updates: { id: number; tx: MappedTransaction }[] = [];
|
||||||
|
// Oldest first so matches are claimed in chronological order.
|
||||||
|
for (const tx of [...mapped].sort((a, b) => a.date.localeCompare(b.date))) {
|
||||||
|
if (known.has(tx.externalId)) {
|
||||||
|
result.alreadyKnown++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
known.add(tx.externalId);
|
||||||
|
const match = findMatch(tx, candidates, used);
|
||||||
|
if (match) {
|
||||||
|
used.add(match.id);
|
||||||
|
updates.push({ id: match.id, tx });
|
||||||
|
} else inserts.push(tx);
|
||||||
|
}
|
||||||
|
// A hand entry can stand for several bank payments ("To Joint 209" = 200 + 3 + 6):
|
||||||
|
// replace it with the real payments instead of keeping both.
|
||||||
|
const absorbed: { row: ExistingRow; parts: MappedTransaction[] }[] = [];
|
||||||
|
const takenInserts = new Set<number>();
|
||||||
|
for (const row of candidates) {
|
||||||
|
if (used.has(row.id)) continue;
|
||||||
|
const parts = findSumMatch(row, inserts, takenInserts);
|
||||||
|
if (!parts) continue;
|
||||||
|
parts.forEach((index) => takenInserts.add(index));
|
||||||
|
used.add(row.id);
|
||||||
|
absorbed.push({ row, parts: parts.map((index) => inserts[index]) });
|
||||||
|
}
|
||||||
|
for (const { row, parts } of absorbed)
|
||||||
|
log(
|
||||||
|
`${account.name}: hand entry #${row.id} ${row.date} "${row.description}" ${row.amountCents} = ` +
|
||||||
|
parts.map((p) => `${p.date} ${p.description} ${p.amountCents}`).join(' + ') +
|
||||||
|
(options.dryRun ? ' (would replace)' : ' (replaced)')
|
||||||
|
);
|
||||||
|
result.matched = updates.length;
|
||||||
|
result.absorbed = absorbed.length;
|
||||||
|
result.inserted = inserts.length;
|
||||||
|
if (!options.dryRun) {
|
||||||
|
await sql.begin(async (q) => {
|
||||||
|
for (const { row } of absorbed)
|
||||||
|
await q`delete from money.transactions where id = ${row.id} and external_id is null`;
|
||||||
|
for (const { id, tx } of updates) {
|
||||||
|
// Keep Stan's own description/category; attach the bank identity.
|
||||||
|
await q`
|
||||||
|
update money.transactions set external_id = ${tx.externalId},
|
||||||
|
counterparty = coalesce(counterparty, ${tx.counterparty}),
|
||||||
|
booking_status = ${tx.bookingStatus}, value_date = ${tx.valueDate},
|
||||||
|
raw = ${q.json(tx.raw as never)}
|
||||||
|
where id = ${id} and external_id is null`;
|
||||||
|
}
|
||||||
|
for (const tx of inserts) {
|
||||||
|
await q`
|
||||||
|
insert into money.transactions
|
||||||
|
(user_id, account_id, date, description, amount_cents, external_id, source,
|
||||||
|
booking_status, counterparty, raw, value_date)
|
||||||
|
values (${account.user_id}, ${account.id}, ${tx.date}, ${tx.description},
|
||||||
|
${tx.amountCents}, ${tx.externalId}, 'enablebanking', ${tx.bookingStatus},
|
||||||
|
${tx.counterparty}, ${q.json(tx.raw as never)}, ${tx.valueDate})
|
||||||
|
on conflict (account_id, external_id) do nothing`;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
await syncBalance(sql, client, account, result, options, today, log);
|
||||||
|
} catch (error) {
|
||||||
|
result.error = error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!options.dryRun) {
|
||||||
|
// Each bank has its own session; only touch the sessions whose accounts were synced.
|
||||||
|
const sessions = await sql<{ id: number; accounts: { uid: string | null }[] }[]>`
|
||||||
|
select id, accounts from money.bank_sessions where status = 'authorized'`;
|
||||||
|
for (const session of sessions) {
|
||||||
|
const uids = new Set(session.accounts.map((a) => a.uid).filter(Boolean));
|
||||||
|
const mine = results.filter((_, index) => uids.has(selected[index].external_uid));
|
||||||
|
if (!mine.length) continue;
|
||||||
|
const failed = mine.filter((r) => r.error).map((r) => `${r.account}: ${r.error}`);
|
||||||
|
const expired = failed.some((f) => /EXPIRED_SESSION|REVOKED_SESSION|CLOSED_SESSION/.test(f));
|
||||||
|
await sql`
|
||||||
|
update money.bank_sessions set last_synced_at = now(),
|
||||||
|
last_error = ${failed.length ? failed.join('\n').slice(0, 2000) : null},
|
||||||
|
status = case when ${expired} then 'expired'
|
||||||
|
when valid_until is not null and valid_until < now() then 'expired' else status end
|
||||||
|
where id = ${session.id}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncBalance(
|
||||||
|
sql: Sql,
|
||||||
|
client: EnableBankingClient,
|
||||||
|
account: { id: number; user_id: string; name: string; external_uid: string },
|
||||||
|
result: AccountSyncResult,
|
||||||
|
options: SyncOptions,
|
||||||
|
today: string,
|
||||||
|
log: (line: string) => void
|
||||||
|
) {
|
||||||
|
let balances: Balance[];
|
||||||
|
try {
|
||||||
|
({ balances } = await client.balances(account.external_uid));
|
||||||
|
} catch (error) {
|
||||||
|
log(`${account.name}: balance unavailable (${error instanceof Error ? error.message : error})`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const chosen = pickBalance(balances);
|
||||||
|
if (!chosen) return;
|
||||||
|
const bankCents = amountToCents(chosen.balance_amount.amount);
|
||||||
|
result.balance = `${chosen.balance_amount.amount} ${chosen.balance_amount.currency} (${chosen.balance_type})`;
|
||||||
|
await sql.begin(async (q) => {
|
||||||
|
const [row] = await q<{ tracked: number; adjustment: number | null; first: string | null }[]>`
|
||||||
|
select (a.opening_balance_cents + coalesce(sum(t.amount_cents) filter (
|
||||||
|
where t.date <= ${today} and t.external_id is distinct from ${BALANCE_ADJUSTMENT_ID}), 0))::int as tracked,
|
||||||
|
(select amount_cents from money.transactions
|
||||||
|
where account_id = a.id and external_id = ${BALANCE_ADJUSTMENT_ID}) as adjustment,
|
||||||
|
min(t.date)::text as first
|
||||||
|
from money.accounts a left join money.transactions t on t.account_id = a.id
|
||||||
|
where a.id = ${account.id} group by a.id`;
|
||||||
|
const needed = bankCents - row.tracked;
|
||||||
|
result.adjustmentCents = needed;
|
||||||
|
if (row.adjustment !== needed)
|
||||||
|
log(
|
||||||
|
`${account.name}: tracker rows sum differs from bank by ${(needed / 100).toFixed(2)}; adjustment ${row.adjustment ?? 'none'} -> ${needed}`
|
||||||
|
);
|
||||||
|
if (options.dryRun) return;
|
||||||
|
await q`
|
||||||
|
update money.accounts set bank_balance_cents = ${bankCents},
|
||||||
|
bank_balance_type = ${chosen.balance_type}, bank_balance_at = now()
|
||||||
|
where id = ${account.id}`;
|
||||||
|
if (row.adjustment === null) {
|
||||||
|
if (needed !== 0)
|
||||||
|
await q`
|
||||||
|
insert into money.transactions
|
||||||
|
(user_id, account_id, date, description, amount_cents, external_id, source, booking_status)
|
||||||
|
values (${account.user_id}, ${account.id}, ${row.first ?? today}, 'Bank balance adjustment',
|
||||||
|
${needed}, ${BALANCE_ADJUSTMENT_ID}, 'enablebanking', 'ADJ')`;
|
||||||
|
} else if (row.adjustment !== needed) {
|
||||||
|
await q`
|
||||||
|
update money.transactions set amount_cents = ${needed}
|
||||||
|
where account_id = ${account.id} and external_id = ${BALANCE_ADJUSTMENT_ID}`;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
77
src/lib/server/books/files.test.ts
Normal file
77
src/lib/server/books/files.test.ts
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||||
|
import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { parseRange, safeResolve, streamFile } from './files';
|
||||||
|
|
||||||
|
let root: string;
|
||||||
|
beforeAll(async () => {
|
||||||
|
root = await mkdtemp(path.join(tmpdir(), 'books-'));
|
||||||
|
await mkdir(path.join(root, 'library', 'a'), { recursive: true });
|
||||||
|
await writeFile(path.join(root, 'library', 'a', 'book.epub'), '0123456789');
|
||||||
|
await writeFile(path.join(root, 'secret.txt'), 'nope');
|
||||||
|
await symlink(path.join(root, 'secret.txt'), path.join(root, 'library', 'escape.epub'));
|
||||||
|
});
|
||||||
|
afterAll(() => rm(root, { recursive: true, force: true }));
|
||||||
|
|
||||||
|
describe('safeResolve', () => {
|
||||||
|
it('finds files inside the library', async () => {
|
||||||
|
expect(await safeResolve('library', 'a/book.epub', root)).toBe(
|
||||||
|
path.join(root, 'library', 'a', 'book.epub')
|
||||||
|
);
|
||||||
|
});
|
||||||
|
it.each([
|
||||||
|
'../secret.txt',
|
||||||
|
'a/../../secret.txt',
|
||||||
|
'/etc/passwd',
|
||||||
|
'escape.epub',
|
||||||
|
'a',
|
||||||
|
'',
|
||||||
|
'missing.pdf'
|
||||||
|
])('refuses %s', async (relative) => {
|
||||||
|
expect(await safeResolve('library', relative, root)).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('parseRange', () => {
|
||||||
|
it('parses single ranges', () => {
|
||||||
|
expect(parseRange('bytes=0-3', 10)).toEqual({ start: 0, end: 3 });
|
||||||
|
expect(parseRange('bytes=5-', 10)).toEqual({ start: 5, end: 9 });
|
||||||
|
expect(parseRange('bytes=-4', 10)).toEqual({ start: 6, end: 9 });
|
||||||
|
expect(parseRange('bytes=8-100', 10)).toEqual({ start: 8, end: 9 });
|
||||||
|
});
|
||||||
|
it('ignores or rejects bad ranges', () => {
|
||||||
|
expect(parseRange(null, 10)).toBeNull();
|
||||||
|
expect(parseRange('bytes=0-1,4-5', 10)).toBeNull();
|
||||||
|
expect(parseRange('bytes=10-', 10)).toBe('invalid');
|
||||||
|
expect(parseRange('bytes=5-2', 10)).toBe('invalid');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('streamFile', () => {
|
||||||
|
const file = () => path.join(root, 'library', 'a', 'book.epub');
|
||||||
|
it('serves the whole file', async () => {
|
||||||
|
const res = await streamFile(file(), new Request('http://x/'), {});
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(res.headers.get('content-type')).toBe('application/epub+zip');
|
||||||
|
expect(await res.text()).toBe('0123456789');
|
||||||
|
});
|
||||||
|
it('serves partial content', async () => {
|
||||||
|
const res = await streamFile(
|
||||||
|
file(),
|
||||||
|
new Request('http://x/', { headers: { range: 'bytes=2-4' } }),
|
||||||
|
{}
|
||||||
|
);
|
||||||
|
expect(res.status).toBe(206);
|
||||||
|
expect(res.headers.get('content-range')).toBe('bytes 2-4/10');
|
||||||
|
expect(await res.text()).toBe('234');
|
||||||
|
});
|
||||||
|
it('answers 416 for unsatisfiable ranges', async () => {
|
||||||
|
const res = await streamFile(
|
||||||
|
file(),
|
||||||
|
new Request('http://x/', { headers: { range: 'bytes=20-' } }),
|
||||||
|
{}
|
||||||
|
);
|
||||||
|
expect(res.status).toBe(416);
|
||||||
|
});
|
||||||
|
});
|
||||||
115
src/lib/server/books/files.ts
Normal file
115
src/lib/server/books/files.ts
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
// Book files and covers on disk. Files are owner-only and only ever streamed through an
|
||||||
|
// authenticated endpoint; nothing here is under static/.
|
||||||
|
//
|
||||||
|
// BOOKS_ROOT holds library/ (book files, books.files.path is relative to it) and covers/
|
||||||
|
// (books.works.cover_path is relative to it). In prod the container gets the host's books
|
||||||
|
// directory as a read-only bind mount at /books (see scripts/deploy-remote.sh).
|
||||||
|
import { createReadStream, existsSync } from 'node:fs';
|
||||||
|
import { realpath, stat } from 'node:fs/promises';
|
||||||
|
import { homedir } from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { Readable } from 'node:stream';
|
||||||
|
|
||||||
|
export function booksRoot(): string {
|
||||||
|
if (process.env.BOOKS_ROOT) return process.env.BOOKS_ROOT;
|
||||||
|
return existsSync('/books/library') ? '/books' : path.join(homedir(), 'books');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolves `relative` inside `<root>/<area>`, following symlinks, and refuses anything that
|
||||||
|
* escapes it (`..`, absolute paths, symlinks out). Returns null when missing or unsafe.
|
||||||
|
*/
|
||||||
|
export async function safeResolve(
|
||||||
|
area: 'library' | 'covers',
|
||||||
|
relative: string,
|
||||||
|
root = booksRoot()
|
||||||
|
): Promise<string | null> {
|
||||||
|
if (!relative || relative.includes('\0') || path.isAbsolute(relative)) return null;
|
||||||
|
const base = path.join(root, area);
|
||||||
|
try {
|
||||||
|
const realBase = await realpath(base);
|
||||||
|
const real = await realpath(path.join(base, relative));
|
||||||
|
if (real !== realBase && !real.startsWith(realBase + path.sep)) return null;
|
||||||
|
return (await stat(real)).isFile() ? real : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentTypes: Record<string, string> = {
|
||||||
|
epub: 'application/epub+zip',
|
||||||
|
pdf: 'application/pdf',
|
||||||
|
mobi: 'application/x-mobipocket-ebook',
|
||||||
|
azw: 'application/vnd.amazon.ebook',
|
||||||
|
azw3: 'application/vnd.amazon.ebook',
|
||||||
|
fb2: 'application/x-fictionbook+xml',
|
||||||
|
cbz: 'application/vnd.comicbook+zip',
|
||||||
|
djvu: 'image/vnd.djvu',
|
||||||
|
jpg: 'image/jpeg',
|
||||||
|
jpeg: 'image/jpeg',
|
||||||
|
png: 'image/png',
|
||||||
|
webp: 'image/webp',
|
||||||
|
gif: 'image/gif',
|
||||||
|
avif: 'image/avif'
|
||||||
|
};
|
||||||
|
export function contentType(file: string): string {
|
||||||
|
return contentTypes[path.extname(file).slice(1).toLowerCase()] ?? 'application/octet-stream';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parses a single `bytes=a-b` range. Multi-range requests get the whole file. */
|
||||||
|
export function parseRange(
|
||||||
|
header: string | null,
|
||||||
|
size: number
|
||||||
|
): { start: number; end: number } | 'invalid' | null {
|
||||||
|
if (!header) return null;
|
||||||
|
const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim());
|
||||||
|
if (!match || (!match[1] && !match[2])) return null;
|
||||||
|
let start: number;
|
||||||
|
let end: number;
|
||||||
|
if (!match[1]) {
|
||||||
|
const suffix = Number(match[2]);
|
||||||
|
if (suffix === 0) return 'invalid';
|
||||||
|
start = Math.max(0, size - suffix);
|
||||||
|
end = size - 1;
|
||||||
|
} else {
|
||||||
|
start = Number(match[1]);
|
||||||
|
end = match[2] ? Math.min(Number(match[2]), size - 1) : size - 1;
|
||||||
|
}
|
||||||
|
if (start > end || start >= size) return 'invalid';
|
||||||
|
return { start, end };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Streams a file with Range support (pdf.js fetches large PDFs in chunks). */
|
||||||
|
export async function streamFile(
|
||||||
|
file: string,
|
||||||
|
request: Request,
|
||||||
|
headers: Record<string, string>
|
||||||
|
): Promise<Response> {
|
||||||
|
const { size, mtime } = await stat(file);
|
||||||
|
const base = {
|
||||||
|
'content-type': contentType(file),
|
||||||
|
'accept-ranges': 'bytes',
|
||||||
|
'last-modified': mtime.toUTCString(),
|
||||||
|
...headers
|
||||||
|
};
|
||||||
|
const range = parseRange(request.headers.get('range'), size);
|
||||||
|
if (range === 'invalid')
|
||||||
|
return new Response(null, {
|
||||||
|
status: 416,
|
||||||
|
headers: { ...base, 'content-range': `bytes */${size}` }
|
||||||
|
});
|
||||||
|
const { start, end } = range ?? { start: 0, end: size - 1 };
|
||||||
|
const length = size === 0 ? 0 : end - start + 1;
|
||||||
|
const body =
|
||||||
|
request.method === 'HEAD' || size === 0
|
||||||
|
? null
|
||||||
|
: (Readable.toWeb(createReadStream(file, { start, end })) as unknown as ReadableStream);
|
||||||
|
return new Response(body, {
|
||||||
|
status: range ? 206 : 200,
|
||||||
|
headers: {
|
||||||
|
...base,
|
||||||
|
'content-length': String(length),
|
||||||
|
...(range ? { 'content-range': `bytes ${start}-${end}/${size}` } : {})
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
373
src/lib/server/books/library.ts
Normal file
373
src/lib/server/books/library.ts
Normal file
@@ -0,0 +1,373 @@
|
|||||||
|
// Book tracker queries. Public callers get catalogue fields only; owner-only data (files,
|
||||||
|
// notes, highlights, sessions) is loaded separately and only after requireOwner().
|
||||||
|
import { db } from '#lib/server/db/index.js';
|
||||||
|
import {
|
||||||
|
bookFiles,
|
||||||
|
bookNotes,
|
||||||
|
bookProgress,
|
||||||
|
bookSessions,
|
||||||
|
bookWorks
|
||||||
|
} from '#lib/server/db/schema/index.js';
|
||||||
|
import { and, asc, desc, eq, gte, sql } from 'drizzle-orm';
|
||||||
|
import {
|
||||||
|
formatPreference,
|
||||||
|
isReadable,
|
||||||
|
type BookCard,
|
||||||
|
type Highlight,
|
||||||
|
type LibraryStats,
|
||||||
|
type ReadingStatus
|
||||||
|
} from '#lib/books/shared.js';
|
||||||
|
import { pagesRead, SESSION_GAP_MS, type ProgressPayload } from '#lib/books/progress.js';
|
||||||
|
|
||||||
|
const TIME_ZONE = 'Europe/Amsterdam';
|
||||||
|
export const currentYear = () =>
|
||||||
|
Number(
|
||||||
|
new Intl.DateTimeFormat('en-CA', { timeZone: TIME_ZONE, year: 'numeric' }).format(new Date())
|
||||||
|
);
|
||||||
|
|
||||||
|
export function coverSrc(work: {
|
||||||
|
id: number;
|
||||||
|
coverPath: string | null;
|
||||||
|
coverUrl: string | null;
|
||||||
|
updatedAt?: Date | null;
|
||||||
|
}) {
|
||||||
|
if (work.coverPath)
|
||||||
|
return `/trackers/books/covers/${work.id}${work.updatedAt ? `?v=${work.updatedAt.getTime().toString(36)}` : ''}`;
|
||||||
|
if (work.coverUrl?.startsWith('https://')) return work.coverUrl;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const iso = (d: Date | null | undefined) => (d ? d.toISOString() : null);
|
||||||
|
|
||||||
|
export async function libraryCards(owner: boolean): Promise<BookCard[]> {
|
||||||
|
const formats = db
|
||||||
|
.select({
|
||||||
|
workId: bookFiles.workId,
|
||||||
|
formats: sql<string[]>`array_agg(distinct lower(${bookFiles.format}))`.as('formats')
|
||||||
|
})
|
||||||
|
.from(bookFiles)
|
||||||
|
.groupBy(bookFiles.workId)
|
||||||
|
.as('f');
|
||||||
|
const rows = await db
|
||||||
|
.select({
|
||||||
|
id: bookWorks.id,
|
||||||
|
title: bookWorks.title,
|
||||||
|
subtitle: bookWorks.subtitle,
|
||||||
|
authors: bookWorks.authors,
|
||||||
|
year: bookWorks.year,
|
||||||
|
tags: bookWorks.tags,
|
||||||
|
pageCount: bookWorks.pageCount,
|
||||||
|
coverPath: bookWorks.coverPath,
|
||||||
|
coverUrl: bookWorks.coverUrl,
|
||||||
|
createdAt: bookWorks.createdAt,
|
||||||
|
updatedAt: bookWorks.updatedAt,
|
||||||
|
status: bookProgress.status,
|
||||||
|
percent: bookProgress.percent,
|
||||||
|
rating: bookProgress.rating,
|
||||||
|
lastReadAt: bookProgress.lastReadAt,
|
||||||
|
finishedAt: bookProgress.finishedAt,
|
||||||
|
formats: formats.formats
|
||||||
|
})
|
||||||
|
.from(bookWorks)
|
||||||
|
.leftJoin(bookProgress, eq(bookProgress.workId, bookWorks.id))
|
||||||
|
.leftJoin(formats, eq(formats.workId, bookWorks.id))
|
||||||
|
.orderBy(asc(bookWorks.title));
|
||||||
|
return rows.map((r) => ({
|
||||||
|
id: r.id,
|
||||||
|
title: r.title,
|
||||||
|
subtitle: r.subtitle,
|
||||||
|
authors: r.authors,
|
||||||
|
year: r.year,
|
||||||
|
tags: r.tags,
|
||||||
|
pageCount: r.pageCount,
|
||||||
|
cover: coverSrc(r),
|
||||||
|
status: r.status ?? null,
|
||||||
|
percent: r.percent ?? 0,
|
||||||
|
rating: r.rating ?? null,
|
||||||
|
lastReadAt: iso(r.lastReadAt),
|
||||||
|
finishedAt: iso(r.finishedAt),
|
||||||
|
addedAt: r.createdAt.toISOString(),
|
||||||
|
formats: owner ? (r.formats ?? []) : [],
|
||||||
|
hasFile: owner ? (r.formats?.length ?? 0) > 0 : false
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function libraryStats(): Promise<LibraryStats> {
|
||||||
|
const year = currentYear();
|
||||||
|
const yearStart = sql`make_timestamptz(${year}, 1, 1, 0, 0, 0, ${TIME_ZONE})`;
|
||||||
|
const [[counts], [pages]] = await Promise.all([
|
||||||
|
db
|
||||||
|
.select({
|
||||||
|
total: sql<number>`(select count(*)::int from ${bookWorks})`,
|
||||||
|
finished: sql<number>`count(*) filter (where ${bookProgress.status} = 'finished' and ${bookProgress.finishedAt} >= ${yearStart})::int`,
|
||||||
|
reading: sql<number>`count(*) filter (where ${bookProgress.status} = 'reading')::int`
|
||||||
|
})
|
||||||
|
.from(bookProgress),
|
||||||
|
db
|
||||||
|
.select({ pages: sql<number>`coalesce(sum(${bookSessions.pagesRead}), 0)::int` })
|
||||||
|
.from(bookSessions)
|
||||||
|
.where(gte(bookSessions.startedAt, yearStart))
|
||||||
|
]);
|
||||||
|
return {
|
||||||
|
total: counts?.total ?? 0,
|
||||||
|
finishedThisYear: counts?.finished ?? 0,
|
||||||
|
pagesThisYear: pages?.pages ?? 0,
|
||||||
|
reading: counts?.reading ?? 0,
|
||||||
|
year
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getWork(id: number) {
|
||||||
|
const [work] = await db.select().from(bookWorks).where(eq(bookWorks.id, id)).limit(1);
|
||||||
|
if (!work) return null;
|
||||||
|
const [progress] = await db
|
||||||
|
.select()
|
||||||
|
.from(bookProgress)
|
||||||
|
.where(eq(bookProgress.workId, id))
|
||||||
|
.limit(1);
|
||||||
|
return { work, progress: progress ?? null };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function workFiles(workId: number) {
|
||||||
|
const files = await db
|
||||||
|
.select({
|
||||||
|
id: bookFiles.id,
|
||||||
|
path: bookFiles.path,
|
||||||
|
format: bookFiles.format,
|
||||||
|
sizeBytes: bookFiles.sizeBytes
|
||||||
|
})
|
||||||
|
.from(bookFiles)
|
||||||
|
.where(eq(bookFiles.workId, workId));
|
||||||
|
const rank = (f: string) => {
|
||||||
|
const i = formatPreference.indexOf(f.toLowerCase());
|
||||||
|
return i === -1 ? 99 : i;
|
||||||
|
};
|
||||||
|
return files
|
||||||
|
.map((f) => ({ ...f, format: f.format.toLowerCase(), readable: isReadable(f.format) }))
|
||||||
|
.sort((a, b) => rank(a.format) - rank(b.format));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function workNotes(workId: number): Promise<Highlight[]> {
|
||||||
|
const rows = await db
|
||||||
|
.select()
|
||||||
|
.from(bookNotes)
|
||||||
|
.where(eq(bookNotes.workId, workId))
|
||||||
|
.orderBy(desc(bookNotes.createdAt));
|
||||||
|
return rows.map((n) => ({
|
||||||
|
id: n.id,
|
||||||
|
fileId: n.fileId,
|
||||||
|
locator: n.locator,
|
||||||
|
page: n.page,
|
||||||
|
highlight: n.highlight,
|
||||||
|
note: n.note,
|
||||||
|
color: n.color,
|
||||||
|
createdAt: n.createdAt.toISOString()
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function workSessions(workId: number) {
|
||||||
|
const rows = await db
|
||||||
|
.select()
|
||||||
|
.from(bookSessions)
|
||||||
|
.where(eq(bookSessions.workId, workId))
|
||||||
|
.orderBy(desc(bookSessions.startedAt))
|
||||||
|
.limit(200);
|
||||||
|
return rows.map((s) => ({
|
||||||
|
id: s.id,
|
||||||
|
startedAt: s.startedAt.toISOString(),
|
||||||
|
endedAt: iso(s.endedAt),
|
||||||
|
startPercent: s.startPercent,
|
||||||
|
endPercent: s.endPercent,
|
||||||
|
startPage: s.startPage,
|
||||||
|
endPage: s.endPage,
|
||||||
|
pagesRead: s.pagesRead,
|
||||||
|
source: s.source,
|
||||||
|
note: s.note
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Sets the reading status, keeping the start/finish dates and percent consistent. */
|
||||||
|
export async function setStatus(workId: number, status: ReadingStatus) {
|
||||||
|
const now = new Date();
|
||||||
|
await db
|
||||||
|
.insert(bookProgress)
|
||||||
|
.values({
|
||||||
|
workId,
|
||||||
|
status,
|
||||||
|
startedAt: status === 'want_to_read' ? null : now,
|
||||||
|
finishedAt: status === 'finished' ? now : null,
|
||||||
|
percent: status === 'finished' ? 100 : 0
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: bookProgress.workId,
|
||||||
|
set: {
|
||||||
|
status,
|
||||||
|
startedAt:
|
||||||
|
status === 'want_to_read'
|
||||||
|
? sql`${bookProgress.startedAt}`
|
||||||
|
: sql`coalesce(${bookProgress.startedAt}, ${now.toISOString()}::timestamptz)`,
|
||||||
|
finishedAt:
|
||||||
|
status === 'finished'
|
||||||
|
? sql`coalesce(${bookProgress.finishedAt}, ${now.toISOString()}::timestamptz)`
|
||||||
|
: null,
|
||||||
|
percent: status === 'finished' ? 100 : sql`${bookProgress.percent}`
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function setRating(workId: number, rating: number | null) {
|
||||||
|
await db
|
||||||
|
.insert(bookProgress)
|
||||||
|
.values({ workId, rating })
|
||||||
|
.onConflictDoUpdate({ target: bookProgress.workId, set: { rating } });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Manual progress from the detail page (percent and/or page). */
|
||||||
|
export async function setManualProgress(workId: number, percent: number, page: number | null) {
|
||||||
|
await db
|
||||||
|
.insert(bookProgress)
|
||||||
|
.values({
|
||||||
|
workId,
|
||||||
|
percent,
|
||||||
|
page,
|
||||||
|
status: percent > 0 ? 'reading' : 'want_to_read',
|
||||||
|
startedAt: percent > 0 ? new Date() : null
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: bookProgress.workId,
|
||||||
|
set: {
|
||||||
|
percent,
|
||||||
|
page,
|
||||||
|
status: sql`case when ${bookProgress.status} = 'want_to_read' and ${percent} > 0 then 'reading'::books.reading_status else ${bookProgress.status} end`,
|
||||||
|
startedAt: sql`coalesce(${bookProgress.startedAt}, case when ${percent} > 0 then now() end)`
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the reader position and keeps a reading session going. Returns the session id the
|
||||||
|
* reader should send back with its next save. A save more than SESSION_GAP_MS after the
|
||||||
|
* last one starts a new session.
|
||||||
|
*/
|
||||||
|
export async function saveProgress(workId: number, input: ProgressPayload) {
|
||||||
|
return db.transaction(async (tx) => {
|
||||||
|
const [work] = await tx
|
||||||
|
.select({ pageCount: bookWorks.pageCount })
|
||||||
|
.from(bookWorks)
|
||||||
|
.where(eq(bookWorks.id, workId))
|
||||||
|
.limit(1);
|
||||||
|
if (!work) return null;
|
||||||
|
if (input.fileId) {
|
||||||
|
const [file] = await tx
|
||||||
|
.select({ id: bookFiles.id })
|
||||||
|
.from(bookFiles)
|
||||||
|
.where(and(eq(bookFiles.id, input.fileId), eq(bookFiles.workId, workId)))
|
||||||
|
.limit(1);
|
||||||
|
if (!file) return null;
|
||||||
|
}
|
||||||
|
const [before] = await tx
|
||||||
|
.select()
|
||||||
|
.from(bookProgress)
|
||||||
|
.where(eq(bookProgress.workId, workId))
|
||||||
|
.limit(1);
|
||||||
|
const now = new Date();
|
||||||
|
await tx
|
||||||
|
.insert(bookProgress)
|
||||||
|
.values({
|
||||||
|
workId,
|
||||||
|
fileId: input.fileId ?? null,
|
||||||
|
locator: input.locator ?? null,
|
||||||
|
page: input.page ?? null,
|
||||||
|
percent: input.percent,
|
||||||
|
status: 'reading',
|
||||||
|
startedAt: now,
|
||||||
|
lastReadAt: now
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: bookProgress.workId,
|
||||||
|
set: {
|
||||||
|
fileId: input.fileId ?? null,
|
||||||
|
locator: input.locator ?? null,
|
||||||
|
page: input.page ?? null,
|
||||||
|
percent: input.percent,
|
||||||
|
lastReadAt: now,
|
||||||
|
status: sql`case when ${bookProgress.status} = 'want_to_read' then 'reading'::books.reading_status else ${bookProgress.status} end`,
|
||||||
|
startedAt: sql`coalesce(${bookProgress.startedAt}, now())`
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const pageCount = input.pageCount ?? work.pageCount;
|
||||||
|
let session = input.sessionId
|
||||||
|
? (
|
||||||
|
await tx
|
||||||
|
.select()
|
||||||
|
.from(bookSessions)
|
||||||
|
.where(and(eq(bookSessions.id, input.sessionId), eq(bookSessions.workId, workId)))
|
||||||
|
.limit(1)
|
||||||
|
)[0]
|
||||||
|
: undefined;
|
||||||
|
const last = session?.endedAt ?? session?.startedAt;
|
||||||
|
if (session && last && now.getTime() - last.getTime() > SESSION_GAP_MS) session = undefined;
|
||||||
|
if (!session) {
|
||||||
|
[session] = await tx
|
||||||
|
.insert(bookSessions)
|
||||||
|
.values({
|
||||||
|
workId,
|
||||||
|
fileId: input.fileId ?? null,
|
||||||
|
startedAt: now,
|
||||||
|
endedAt: now,
|
||||||
|
startPercent: before?.percent ?? input.percent,
|
||||||
|
endPercent: input.percent,
|
||||||
|
startPage: before?.page ?? input.page ?? null,
|
||||||
|
endPage: input.page ?? null,
|
||||||
|
pagesRead: 0,
|
||||||
|
source: 'reader'
|
||||||
|
})
|
||||||
|
.returning();
|
||||||
|
} else {
|
||||||
|
const startPage = session.startPage ?? input.page ?? null;
|
||||||
|
await tx
|
||||||
|
.update(bookSessions)
|
||||||
|
.set({
|
||||||
|
endedAt: now,
|
||||||
|
endPercent: input.percent,
|
||||||
|
endPage: input.page ?? null,
|
||||||
|
startPage,
|
||||||
|
pagesRead: pagesRead({
|
||||||
|
startPage: input.page != null ? startPage : null,
|
||||||
|
endPage: input.page,
|
||||||
|
startPercent: session.startPercent,
|
||||||
|
endPercent: input.percent,
|
||||||
|
pageCount
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.where(eq(bookSessions.id, session.id));
|
||||||
|
}
|
||||||
|
return { sessionId: session.id };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function logManualSession(
|
||||||
|
workId: number,
|
||||||
|
input: {
|
||||||
|
startedAt: Date;
|
||||||
|
endedAt: Date | null;
|
||||||
|
startPage: number | null;
|
||||||
|
endPage: number | null;
|
||||||
|
startPercent: number | null;
|
||||||
|
endPercent: number | null;
|
||||||
|
note: string | null;
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
const [work] = await db
|
||||||
|
.select({ pageCount: bookWorks.pageCount })
|
||||||
|
.from(bookWorks)
|
||||||
|
.where(eq(bookWorks.id, workId))
|
||||||
|
.limit(1);
|
||||||
|
await db.insert(bookSessions).values({
|
||||||
|
workId,
|
||||||
|
...input,
|
||||||
|
pagesRead: pagesRead({ ...input, pageCount: work?.pageCount }),
|
||||||
|
source: 'manual'
|
||||||
|
});
|
||||||
|
}
|
||||||
84
src/lib/server/books/notes.ts
Normal file
84
src/lib/server/books/notes.ts
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
// Highlights and notes (books.notes). A row with `highlight` is a highlight from the reader
|
||||||
|
// (optionally with a note); a row with only `note` is a free-form note on the book.
|
||||||
|
import * as z from 'zod';
|
||||||
|
import { and, eq } from 'drizzle-orm';
|
||||||
|
import { db } from '#lib/server/db/index.js';
|
||||||
|
import { bookFiles, bookNotes, bookWorks } from '#lib/server/db/schema/index.js';
|
||||||
|
import { highlightColors, type Highlight } from '#lib/books/shared.js';
|
||||||
|
|
||||||
|
export const notePayload = z
|
||||||
|
.object({
|
||||||
|
fileId: z.number().int().positive().nullish(),
|
||||||
|
locator: z.string().max(4000).nullish(),
|
||||||
|
page: z.number().int().min(0).max(100000).nullish(),
|
||||||
|
highlight: z.string().trim().max(20000).nullish(),
|
||||||
|
note: z.string().trim().max(20000).nullish(),
|
||||||
|
color: z.enum(highlightColors).nullish()
|
||||||
|
})
|
||||||
|
.refine((n) => !!(n.highlight || n.note), { message: 'Empty note' });
|
||||||
|
export const noteUpdate = z.object({
|
||||||
|
note: z.string().trim().max(20000).nullable().optional(),
|
||||||
|
color: z.enum(highlightColors).nullable().optional()
|
||||||
|
});
|
||||||
|
|
||||||
|
const toHighlight = (n: typeof bookNotes.$inferSelect): Highlight => ({
|
||||||
|
id: n.id,
|
||||||
|
fileId: n.fileId,
|
||||||
|
locator: n.locator,
|
||||||
|
page: n.page,
|
||||||
|
highlight: n.highlight,
|
||||||
|
note: n.note,
|
||||||
|
color: n.color,
|
||||||
|
createdAt: n.createdAt.toISOString()
|
||||||
|
});
|
||||||
|
|
||||||
|
export async function createNote(workId: number, input: z.infer<typeof notePayload>) {
|
||||||
|
const [work] = await db
|
||||||
|
.select({ id: bookWorks.id })
|
||||||
|
.from(bookWorks)
|
||||||
|
.where(eq(bookWorks.id, workId))
|
||||||
|
.limit(1);
|
||||||
|
if (!work) return null;
|
||||||
|
if (input.fileId) {
|
||||||
|
const [file] = await db
|
||||||
|
.select({ id: bookFiles.id })
|
||||||
|
.from(bookFiles)
|
||||||
|
.where(and(eq(bookFiles.id, input.fileId), eq(bookFiles.workId, workId)))
|
||||||
|
.limit(1);
|
||||||
|
if (!file) return null;
|
||||||
|
}
|
||||||
|
const [row] = await db
|
||||||
|
.insert(bookNotes)
|
||||||
|
.values({
|
||||||
|
workId,
|
||||||
|
fileId: input.fileId ?? null,
|
||||||
|
locator: input.locator ?? null,
|
||||||
|
page: input.page ?? null,
|
||||||
|
highlight: input.highlight || null,
|
||||||
|
note: input.note || null,
|
||||||
|
color: input.highlight ? (input.color ?? 'yellow') : null
|
||||||
|
})
|
||||||
|
.returning();
|
||||||
|
return toHighlight(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateNote(workId: number, id: number, input: z.infer<typeof noteUpdate>) {
|
||||||
|
const set: Partial<typeof bookNotes.$inferInsert> = {};
|
||||||
|
if (input.note !== undefined) set.note = input.note || null;
|
||||||
|
if (input.color !== undefined) set.color = input.color;
|
||||||
|
if (!Object.keys(set).length) return null;
|
||||||
|
const [row] = await db
|
||||||
|
.update(bookNotes)
|
||||||
|
.set(set)
|
||||||
|
.where(and(eq(bookNotes.id, id), eq(bookNotes.workId, workId)))
|
||||||
|
.returning();
|
||||||
|
return row ? toHighlight(row) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteNote(workId: number, id: number) {
|
||||||
|
const rows = await db
|
||||||
|
.delete(bookNotes)
|
||||||
|
.where(and(eq(bookNotes.id, id), eq(bookNotes.workId, workId)))
|
||||||
|
.returning({ id: bookNotes.id });
|
||||||
|
return rows.length > 0;
|
||||||
|
}
|
||||||
758
src/lib/server/data-editor.ts
Normal file
758
src/lib/server/data-editor.ts
Normal file
@@ -0,0 +1,758 @@
|
|||||||
|
// Owner-only database browser/editor behind /lists/db. Every table and column name used
|
||||||
|
// in SQL comes from pg_catalog (looked up with bound parameters) and is quoted with
|
||||||
|
// sql.identifier; every value is a bound parameter. Access rules live in data-policy.ts.
|
||||||
|
import { sql, type SQL } from 'drizzle-orm';
|
||||||
|
import { db } from '#lib/server/db/index.js';
|
||||||
|
import {
|
||||||
|
coerce,
|
||||||
|
hiddenColumn,
|
||||||
|
plausibleIdent,
|
||||||
|
safeTypeName,
|
||||||
|
tableAccess,
|
||||||
|
typeKind,
|
||||||
|
writeAllowed,
|
||||||
|
type Access,
|
||||||
|
type Filter,
|
||||||
|
type Kind
|
||||||
|
} from '#lib/server/data-policy.js';
|
||||||
|
|
||||||
|
export type TableSummary = {
|
||||||
|
schema: string;
|
||||||
|
name: string;
|
||||||
|
displayName: string;
|
||||||
|
description: string | null;
|
||||||
|
relkind: string;
|
||||||
|
access: Access;
|
||||||
|
rows: number | null;
|
||||||
|
approximate: boolean;
|
||||||
|
listSlug: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ColumnInfo = {
|
||||||
|
name: string;
|
||||||
|
label: string;
|
||||||
|
type: string;
|
||||||
|
kind: Kind;
|
||||||
|
uiType: string | null;
|
||||||
|
nullable: boolean;
|
||||||
|
hasDefault: boolean;
|
||||||
|
defaultExpr: string | null;
|
||||||
|
/** identity ALWAYS / generated columns cannot be written */
|
||||||
|
readOnly: boolean;
|
||||||
|
enumValues: string[];
|
||||||
|
pk: boolean;
|
||||||
|
fk: { schema: string; table: string; column: string } | null;
|
||||||
|
help: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type TableInfo = TableSummary & {
|
||||||
|
oid: number;
|
||||||
|
columns: ColumnInfo[];
|
||||||
|
primaryKey: string[];
|
||||||
|
canInsert: boolean;
|
||||||
|
canUpdate: boolean;
|
||||||
|
canDelete: boolean;
|
||||||
|
isList: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type CatalogRow = {
|
||||||
|
oid: number;
|
||||||
|
schema: string;
|
||||||
|
name: string;
|
||||||
|
relkind: string;
|
||||||
|
estimate: number;
|
||||||
|
can_insert: boolean;
|
||||||
|
can_update: boolean;
|
||||||
|
can_delete: boolean;
|
||||||
|
has_pk: boolean;
|
||||||
|
comment: string | null;
|
||||||
|
display_name: string | null;
|
||||||
|
description: string | null;
|
||||||
|
meta_kind: string | null;
|
||||||
|
slug: string | null;
|
||||||
|
archived: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
const MAX_EXACT = 200_000;
|
||||||
|
|
||||||
|
function catalogQuery(where: SQL) {
|
||||||
|
return sql`
|
||||||
|
SELECT c.oid::int AS oid, n.nspname AS schema, c.relname AS name, c.relkind::text AS relkind,
|
||||||
|
greatest(c.reltuples, 0)::bigint::float8 AS estimate,
|
||||||
|
has_table_privilege(c.oid, 'INSERT') AS can_insert,
|
||||||
|
has_table_privilege(c.oid, 'UPDATE') AS can_update,
|
||||||
|
has_table_privilege(c.oid, 'DELETE') AS can_delete,
|
||||||
|
EXISTS (SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary) AS has_pk,
|
||||||
|
obj_description(c.oid, 'pg_class') AS comment,
|
||||||
|
mt.display_name, mt.description, mt.kind AS meta_kind, mt.slug,
|
||||||
|
(mt.archived_at IS NOT NULL) AS archived
|
||||||
|
FROM pg_class c
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
LEFT JOIN meta.tables mt ON mt.schema_name = n.nspname AND mt.table_name = c.relname
|
||||||
|
WHERE c.relkind IN ('r', 'p', 'v', 'm') AND NOT c.relispartition
|
||||||
|
AND n.nspname <> 'information_schema' AND n.nspname NOT LIKE 'pg\\_%'
|
||||||
|
AND has_schema_privilege(n.oid, 'USAGE') AND has_table_privilege(c.oid, 'SELECT')
|
||||||
|
AND ${where}
|
||||||
|
ORDER BY n.nspname, c.relname`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function summarise(r: CatalogRow): Omit<TableSummary, 'rows' | 'approximate'> {
|
||||||
|
const access = tableAccess({
|
||||||
|
schema: r.schema,
|
||||||
|
name: r.name,
|
||||||
|
relkind: r.relkind,
|
||||||
|
hasPrimaryKey: r.has_pk,
|
||||||
|
canInsert: r.can_insert,
|
||||||
|
canUpdate: r.can_update,
|
||||||
|
canDelete: r.can_delete
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
schema: r.schema,
|
||||||
|
name: r.name,
|
||||||
|
displayName: r.display_name ?? r.name.replace(/_/g, ' '),
|
||||||
|
description: r.description ?? r.comment,
|
||||||
|
relkind: r.relkind,
|
||||||
|
access: r.archived ? 'locked' : access,
|
||||||
|
listSlug: r.meta_kind === 'list' && !r.archived ? r.slug : null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const rel = (schema: string, name: string) =>
|
||||||
|
sql`${sql.identifier(schema)}.${sql.identifier(name)}`;
|
||||||
|
|
||||||
|
/** Every table/view the app role can read, minus hidden ones, with row counts. */
|
||||||
|
export async function listTables(): Promise<TableSummary[]> {
|
||||||
|
const rows = (await db.execute<CatalogRow>(catalogQuery(sql`true`))) as unknown as CatalogRow[];
|
||||||
|
const visible = rows
|
||||||
|
.map((r) => ({ r, s: summarise(r) }))
|
||||||
|
.filter(({ s }) => s.access !== 'hidden');
|
||||||
|
// exact counts for real tables up to MAX_EXACT rows; estimates for huge ones; views are
|
||||||
|
// counted only when opened (some aggregate big tables).
|
||||||
|
const exact = visible.filter(
|
||||||
|
({ r }) => (r.relkind === 'r' || r.relkind === 'p') && r.estimate < MAX_EXACT
|
||||||
|
);
|
||||||
|
const counts = new Map<string, number>();
|
||||||
|
if (exact.length) {
|
||||||
|
const res = await db.transaction(async (tx) => {
|
||||||
|
await tx.execute(sql`SET LOCAL statement_timeout = '10s'`);
|
||||||
|
return tx.execute<{ k: string; n: number }>(
|
||||||
|
sql.join(
|
||||||
|
exact.map(
|
||||||
|
({ r }) =>
|
||||||
|
sql`SELECT ${`${r.schema}.${r.name}`}::text AS k, count(*)::int AS n FROM ${rel(r.schema, r.name)}`
|
||||||
|
),
|
||||||
|
sql` UNION ALL `
|
||||||
|
)
|
||||||
|
);
|
||||||
|
});
|
||||||
|
for (const c of res as unknown as { k: string; n: number }[]) counts.set(c.k, Number(c.n));
|
||||||
|
}
|
||||||
|
return visible.map(({ r, s }) => {
|
||||||
|
const n = counts.get(`${r.schema}.${r.name}`);
|
||||||
|
return n !== undefined
|
||||||
|
? { ...s, rows: n, approximate: false }
|
||||||
|
: r.relkind === 'r' || r.relkind === 'p'
|
||||||
|
? { ...s, rows: r.estimate, approximate: true }
|
||||||
|
: { ...s, rows: null, approximate: false };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
type ColumnRow = {
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
notnull: boolean;
|
||||||
|
default_expr: string | null;
|
||||||
|
identity: string;
|
||||||
|
generated: string;
|
||||||
|
is_enum: boolean;
|
||||||
|
enum_values: string[] | null;
|
||||||
|
comment: string | null;
|
||||||
|
pk_pos: number | null;
|
||||||
|
fk_schema: string | null;
|
||||||
|
fk_table: string | null;
|
||||||
|
fk_column: string | null;
|
||||||
|
ui_type: string | null;
|
||||||
|
label: string | null;
|
||||||
|
help_text: string | null;
|
||||||
|
meta_hidden: boolean | null;
|
||||||
|
select_values: string[] | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** One table by schema + name (bound parameters), or null when unknown or hidden. */
|
||||||
|
export async function getTable(schema: string, name: string): Promise<TableInfo | null> {
|
||||||
|
if (!plausibleIdent(schema) || !plausibleIdent(name)) return null;
|
||||||
|
const [r] = (await db.execute<CatalogRow>(
|
||||||
|
catalogQuery(sql`n.nspname = ${schema} AND c.relname = ${name}`)
|
||||||
|
)) as unknown as CatalogRow[];
|
||||||
|
if (!r) return null;
|
||||||
|
const s = summarise(r);
|
||||||
|
if (s.access === 'hidden') return null;
|
||||||
|
const cols = (await db.execute<ColumnRow>(sql`
|
||||||
|
SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS type, a.attnotnull AS notnull,
|
||||||
|
pg_get_expr(d.adbin, d.adrelid) AS default_expr, a.attidentity::text AS identity,
|
||||||
|
a.attgenerated::text AS generated,
|
||||||
|
(et.typtype = 'e') AS is_enum,
|
||||||
|
(SELECT array_agg(e.enumlabel::text ORDER BY e.enumsortorder) FROM pg_enum e WHERE e.enumtypid = et.oid) AS enum_values,
|
||||||
|
col_description(a.attrelid, a.attnum) AS comment,
|
||||||
|
(SELECT array_position(i.indkey::int2[], a.attnum) FROM pg_index i
|
||||||
|
WHERE i.indrelid = a.attrelid AND i.indisprimary) AS pk_pos,
|
||||||
|
fk.fk_schema, fk.fk_table, fk.fk_column,
|
||||||
|
mc.ui_type, mc.label, mc.help_text, mc.hidden AS meta_hidden,
|
||||||
|
CASE WHEN jsonb_typeof(mc.options->'values') = 'array'
|
||||||
|
THEN ARRAY(SELECT jsonb_array_elements_text(mc.options->'values')) END AS select_values
|
||||||
|
FROM pg_attribute a
|
||||||
|
JOIN pg_type t ON t.oid = a.atttypid
|
||||||
|
LEFT JOIN pg_type et ON et.oid = CASE WHEN t.typelem <> 0 AND t.typcategory = 'A' THEN t.typelem ELSE t.oid END
|
||||||
|
LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
|
||||||
|
LEFT JOIN LATERAL (
|
||||||
|
SELECT fn.nspname AS fk_schema, fc.relname AS fk_table, fa.attname AS fk_column
|
||||||
|
FROM pg_constraint k
|
||||||
|
JOIN pg_class fc ON fc.oid = k.confrelid
|
||||||
|
JOIN pg_namespace fn ON fn.oid = fc.relnamespace
|
||||||
|
JOIN pg_attribute fa ON fa.attrelid = k.confrelid AND fa.attnum = k.confkey[1]
|
||||||
|
WHERE k.conrelid = a.attrelid AND k.contype = 'f' AND cardinality(k.conkey) = 1
|
||||||
|
AND k.conkey[1] = a.attnum
|
||||||
|
LIMIT 1) fk ON true
|
||||||
|
LEFT JOIN meta.tables mt ON mt.schema_name = ${schema} AND mt.table_name = ${name}
|
||||||
|
LEFT JOIN meta.columns mc ON mc.table_id = mt.id AND mc.column_name = a.attname
|
||||||
|
WHERE a.attrelid = ${r.oid} AND a.attnum > 0 AND NOT a.attisdropped
|
||||||
|
ORDER BY a.attnum`)) as unknown as ColumnRow[];
|
||||||
|
const columns: ColumnInfo[] = cols
|
||||||
|
.filter((c) => !hiddenColumn(c.name) && safeTypeName(c.type))
|
||||||
|
.map((c) => ({
|
||||||
|
name: c.name,
|
||||||
|
label: c.label ?? c.name.replace(/_/g, ' '),
|
||||||
|
type: c.type,
|
||||||
|
kind: typeKind(c.type, Boolean(c.is_enum)),
|
||||||
|
uiType: c.ui_type,
|
||||||
|
nullable: !c.notnull,
|
||||||
|
hasDefault: c.default_expr !== null || c.identity !== '' || c.generated !== '',
|
||||||
|
defaultExpr: c.default_expr,
|
||||||
|
readOnly: c.identity === 'a' || c.generated !== '',
|
||||||
|
enumValues: c.enum_values ?? (c.ui_type === 'select' ? (c.select_values ?? []) : []),
|
||||||
|
pk: c.pk_pos !== null,
|
||||||
|
fk: c.fk_schema ? { schema: c.fk_schema, table: c.fk_table!, column: c.fk_column! } : null,
|
||||||
|
help: c.help_text ?? c.comment
|
||||||
|
}));
|
||||||
|
const primaryKey = cols
|
||||||
|
.filter((c) => c.pk_pos !== null)
|
||||||
|
.sort((a, b) => a.pk_pos! - b.pk_pos!)
|
||||||
|
.map((c) => c.name);
|
||||||
|
// a key column that is hidden or has an odd type makes rows unaddressable: read-only
|
||||||
|
const keyUsable =
|
||||||
|
primaryKey.length > 0 && primaryKey.every((k) => columns.some((c) => c.name === k));
|
||||||
|
return {
|
||||||
|
...s,
|
||||||
|
access: s.access !== 'locked' && !keyUsable ? 'locked' : s.access,
|
||||||
|
rows: null,
|
||||||
|
approximate: false,
|
||||||
|
oid: r.oid,
|
||||||
|
columns,
|
||||||
|
primaryKey,
|
||||||
|
canInsert: r.can_insert,
|
||||||
|
canUpdate: r.can_update,
|
||||||
|
canDelete: r.can_delete,
|
||||||
|
isList: s.listSlug !== null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- reading ----------
|
||||||
|
|
||||||
|
export type RowQuery = {
|
||||||
|
q?: string;
|
||||||
|
filters?: Filter[];
|
||||||
|
sort?: string;
|
||||||
|
dir?: 'asc' | 'desc';
|
||||||
|
limit?: number;
|
||||||
|
offset?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
function colRef(t: TableInfo, name: string) {
|
||||||
|
const c = t.columns.find((x) => x.name === name);
|
||||||
|
if (!c) throw new DataError(400, `Unknown column ${name}.`);
|
||||||
|
return { c, ref: sql`t.${sql.identifier(c.name)}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
function cast(value: string | null, c: ColumnInfo) {
|
||||||
|
return sql`CAST(${value} AS ${sql.raw(c.type)})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function whereClause(t: TableInfo, q: RowQuery): SQL {
|
||||||
|
const parts: SQL[] = [];
|
||||||
|
const search = q.q?.trim();
|
||||||
|
if (search) {
|
||||||
|
const texts = t.columns.map((c) => sql`t.${sql.identifier(c.name)}::text`);
|
||||||
|
parts.push(
|
||||||
|
sql`concat_ws(' ', ${sql.join(texts, sql`, `)}) ILIKE ${`%${search.replace(/[\\%_]/g, '\\$&')}%`}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const f of q.filters ?? []) {
|
||||||
|
const { c, ref } = colRef(t, f.column);
|
||||||
|
if (f.op === 'null') parts.push(sql`${ref} IS NULL`);
|
||||||
|
else if (f.op === 'notnull') parts.push(sql`${ref} IS NOT NULL`);
|
||||||
|
else if (f.op === 'contains')
|
||||||
|
parts.push(sql`${ref}::text ILIKE ${`%${f.value.replace(/[\\%_]/g, '\\$&')}%`}`);
|
||||||
|
else {
|
||||||
|
const v = coerce(c.kind, f.value, c.label);
|
||||||
|
if (!v.ok) throw new DataError(400, v.error);
|
||||||
|
const val = c.kind === 'json' ? sql`${v.value}::jsonb` : cast(v.value, c);
|
||||||
|
const lhs = c.kind === 'json' ? sql`${ref}::jsonb` : ref;
|
||||||
|
const op = { eq: '=', ne: 'IS DISTINCT FROM', gt: '>', gte: '>=', lt: '<', lte: '<=' }[f.op];
|
||||||
|
parts.push(sql`${lhs} ${sql.raw(op)} ${val}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return parts.length ? sql`WHERE ${sql.join(parts, sql` AND `)}` : sql``;
|
||||||
|
}
|
||||||
|
|
||||||
|
function orderClause(t: TableInfo, q: RowQuery): SQL {
|
||||||
|
const keys: SQL[] = [];
|
||||||
|
if (q.sort) {
|
||||||
|
const { ref } = colRef(t, q.sort);
|
||||||
|
keys.push(sql`${ref} ${sql.raw(q.dir === 'desc' ? 'DESC NULLS LAST' : 'ASC NULLS FIRST')}`);
|
||||||
|
}
|
||||||
|
for (const k of t.primaryKey.length ? t.primaryKey : [])
|
||||||
|
if (k !== q.sort)
|
||||||
|
keys.push(
|
||||||
|
sql`t.${sql.identifier(k)} ${sql.raw(q.dir === 'desc' && !q.sort ? 'DESC' : 'ASC')}`
|
||||||
|
);
|
||||||
|
return keys.length ? sql`ORDER BY ${sql.join(keys, sql`, `)}` : sql``;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** to_jsonb of the visible columns only (hidden credential columns are never selected). */
|
||||||
|
function rowJson(t: TableInfo) {
|
||||||
|
// jsonb_build_object takes at most 100 arguments: build it in chunks of 50 columns
|
||||||
|
const chunks: SQL[] = [];
|
||||||
|
for (let i = 0; i < t.columns.length; i += 50)
|
||||||
|
chunks.push(
|
||||||
|
sql`jsonb_build_object(${sql.join(
|
||||||
|
t.columns.slice(i, i + 50).map((c) => sql`${c.name}::text, t.${sql.identifier(c.name)}`),
|
||||||
|
sql`, `
|
||||||
|
)})`
|
||||||
|
);
|
||||||
|
return chunks.length ? sql.join(chunks, sql` || `) : sql`'{}'::jsonb`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type RowsResult = {
|
||||||
|
rows: Record<string, unknown>[];
|
||||||
|
total: number | null;
|
||||||
|
approximate: boolean;
|
||||||
|
fkLabels: Record<string, Record<string, string>>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function readRows(t: TableInfo, q: RowQuery): Promise<RowsResult> {
|
||||||
|
const limit = Math.min(Math.max(q.limit ?? 50, 1), 500);
|
||||||
|
const offset = Math.max(q.offset ?? 0, 0);
|
||||||
|
const where = whereClause(t, q);
|
||||||
|
const filtered = Boolean(q.q?.trim() || q.filters?.length);
|
||||||
|
return db.transaction(async (tx) => {
|
||||||
|
await tx.execute(sql`SET TRANSACTION READ ONLY`);
|
||||||
|
await tx.execute(sql`SET LOCAL statement_timeout = '15s'`);
|
||||||
|
const select = rowJson(t);
|
||||||
|
const rows = (await tx.execute<{ r: Record<string, unknown> }>(
|
||||||
|
sql`SELECT ${select} AS r FROM ${rel(t.schema, t.name)} t ${where} ${orderClause(t, q)} LIMIT ${limit} OFFSET ${offset}`
|
||||||
|
)) as unknown as { r: Record<string, unknown> }[];
|
||||||
|
let total: number | null = null;
|
||||||
|
let approximate = false;
|
||||||
|
const [est] = (await tx.execute<{ e: number }>(
|
||||||
|
sql`SELECT greatest(reltuples, 0)::float8 AS e FROM pg_class WHERE oid = ${t.oid}`
|
||||||
|
)) as unknown as { e: number }[];
|
||||||
|
if (!filtered && (t.relkind === 'r' || t.relkind === 'p') && est && est.e > MAX_EXACT) {
|
||||||
|
total = Math.round(est.e);
|
||||||
|
approximate = true;
|
||||||
|
} else {
|
||||||
|
await tx.execute(sql`SAVEPOINT c`);
|
||||||
|
try {
|
||||||
|
await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
|
||||||
|
const [c] = (await tx.execute<{ n: number }>(
|
||||||
|
sql`SELECT count(*)::float8 AS n FROM ${rel(t.schema, t.name)} t ${where}`
|
||||||
|
)) as unknown as { n: number }[];
|
||||||
|
total = Number(c.n);
|
||||||
|
} catch {
|
||||||
|
await tx.execute(sql`ROLLBACK TO SAVEPOINT c`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const visible = rows.map(({ r }) => {
|
||||||
|
const out: Record<string, unknown> = {};
|
||||||
|
for (const c of t.columns) out[c.name] = r[c.name] ?? null;
|
||||||
|
return out;
|
||||||
|
});
|
||||||
|
const fkLabels: RowsResult['fkLabels'] = {};
|
||||||
|
for (const c of t.columns) {
|
||||||
|
if (!c.fk) continue;
|
||||||
|
const values = [...new Set(visible.map((r) => r[c.name]).filter((v) => v !== null))].map(
|
||||||
|
String
|
||||||
|
);
|
||||||
|
if (!values.length) continue;
|
||||||
|
const labels = await fkOptions(c, { values }, tx).catch(() => []);
|
||||||
|
if (labels.length)
|
||||||
|
fkLabels[c.name] = Object.fromEntries(labels.map((o) => [o.value, o.label]));
|
||||||
|
}
|
||||||
|
return { rows: visible, total, approximate, fkLabels };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
type Tx = Parameters<Parameters<typeof db.transaction>[0]>[0];
|
||||||
|
type Exec = Pick<Tx, 'execute'>;
|
||||||
|
|
||||||
|
/** Rows of the table a foreign key points at, as {value, label} for a dropdown. */
|
||||||
|
export async function fkOptions(
|
||||||
|
c: ColumnInfo,
|
||||||
|
opts: { q?: string; values?: string[] },
|
||||||
|
runner: Exec = db
|
||||||
|
): Promise<{ value: string; label: string }[]> {
|
||||||
|
if (!c.fk) return [];
|
||||||
|
const target = await getTable(c.fk.schema, c.fk.table);
|
||||||
|
if (!target) return [];
|
||||||
|
const key = target.columns.find((x) => x.name === c.fk!.column);
|
||||||
|
if (!key) return [];
|
||||||
|
const labelCol =
|
||||||
|
['name', 'display_name', 'title', 'label', 'slug', 'email', 'code']
|
||||||
|
.map((n) => target.columns.find((x) => x.name === n))
|
||||||
|
.find(Boolean) ?? key;
|
||||||
|
const k = sql`t.${sql.identifier(key.name)}`;
|
||||||
|
const l = sql`t.${sql.identifier(labelCol.name)}`;
|
||||||
|
const conds: SQL[] = [];
|
||||||
|
if (opts.values?.length)
|
||||||
|
conds.push(
|
||||||
|
sql`${k}::text IN (${sql.join(
|
||||||
|
opts.values.slice(0, 500).map((v) => sql`${v}`),
|
||||||
|
sql`, `
|
||||||
|
)})`
|
||||||
|
);
|
||||||
|
if (opts.q?.trim()) {
|
||||||
|
const like = `%${opts.q.trim().replace(/[\\%_]/g, '\\$&')}%`;
|
||||||
|
conds.push(sql`(${l}::text ILIKE ${like} OR ${k}::text ILIKE ${like})`);
|
||||||
|
}
|
||||||
|
const rows = (await runner.execute<{ value: string; label: string }>(sql`
|
||||||
|
SELECT ${k}::text AS value, ${l}::text AS label FROM ${rel(target.schema, target.name)} t
|
||||||
|
${conds.length ? sql`WHERE ${sql.join(conds, sql` AND `)}` : sql``}
|
||||||
|
ORDER BY 2 NULLS LAST LIMIT ${opts.values?.length ? 500 : 50}`)) as unknown as {
|
||||||
|
value: string;
|
||||||
|
label: string;
|
||||||
|
}[];
|
||||||
|
return rows.map((r) => ({
|
||||||
|
value: r.value,
|
||||||
|
label: labelCol === key ? r.value : `${r.label ?? '—'} (${r.value})`
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** All matching rows for CSV export (capped). */
|
||||||
|
export async function exportRows(t: TableInfo, q: RowQuery, cap = 200_000) {
|
||||||
|
return db.transaction(async (tx) => {
|
||||||
|
await tx.execute(sql`SET TRANSACTION READ ONLY`);
|
||||||
|
await tx.execute(sql`SET LOCAL statement_timeout = '60s'`);
|
||||||
|
const cols = t.columns.map((c) =>
|
||||||
|
c.kind === 'json' || c.kind === 'array'
|
||||||
|
? sql`to_json(t.${sql.identifier(c.name)})::text`
|
||||||
|
: sql`t.${sql.identifier(c.name)}::text`
|
||||||
|
);
|
||||||
|
const rows = (await tx.execute<{ v: (string | null)[] }>(
|
||||||
|
sql`SELECT to_jsonb(ARRAY[${sql.join(cols, sql`, `)}]::text[]) AS v FROM ${rel(t.schema, t.name)} t ${whereClause(t, q)} ${orderClause(t, q)} LIMIT ${cap}`
|
||||||
|
)) as unknown as { v: (string | null)[] }[];
|
||||||
|
return rows.map((r) => r.v);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- writing ----------
|
||||||
|
|
||||||
|
export class DataError extends Error {
|
||||||
|
constructor(
|
||||||
|
public status: number,
|
||||||
|
message: string
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Actor = { id: string; email: string };
|
||||||
|
export type Pk = Record<string, unknown>;
|
||||||
|
|
||||||
|
function assertWritable(t: TableInfo, unlocked: boolean, need: 'insert' | 'update' | 'delete') {
|
||||||
|
if (t.access === 'locked') throw new DataError(403, 'This table is read-only here.');
|
||||||
|
if (!writeAllowed(t.access, unlocked))
|
||||||
|
throw new DataError(423, 'This is an app table: unlock editing first.');
|
||||||
|
const ok = need === 'insert' ? t.canInsert : need === 'update' ? t.canUpdate : t.canDelete;
|
||||||
|
if (!ok) throw new DataError(403, `The app has no ${need} permission on this table.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function assignments(t: TableInfo, values: Record<string, unknown>) {
|
||||||
|
const out: { c: ColumnInfo; v: string | null }[] = [];
|
||||||
|
for (const [name, raw] of Object.entries(values)) {
|
||||||
|
const c = t.columns.find((x) => x.name === name);
|
||||||
|
if (!c) throw new DataError(400, `Unknown column ${name}.`);
|
||||||
|
if (c.readOnly) throw new DataError(400, `${c.label} is generated and cannot be set.`);
|
||||||
|
const v = coerce(c.kind, raw, c.label);
|
||||||
|
if (!v.ok) throw new DataError(400, v.error);
|
||||||
|
if (v.value === null && !c.nullable && !(c.hasDefault && raw === undefined))
|
||||||
|
throw new DataError(400, `${c.label} is required.`);
|
||||||
|
if (
|
||||||
|
c.kind === 'enum' &&
|
||||||
|
v.value !== null &&
|
||||||
|
c.enumValues.length &&
|
||||||
|
!c.enumValues.includes(v.value)
|
||||||
|
)
|
||||||
|
throw new DataError(400, `${c.label} must be one of ${c.enumValues.join(', ')}.`);
|
||||||
|
out.push({ c, v: v.value });
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function pkWhere(t: TableInfo, pk: Pk) {
|
||||||
|
if (!t.primaryKey.length) throw new DataError(400, 'This table has no primary key.');
|
||||||
|
return sql.join(
|
||||||
|
t.primaryKey.map((k) => {
|
||||||
|
const c = t.columns.find((x) => x.name === k)!;
|
||||||
|
const raw = pk[k];
|
||||||
|
if (raw === undefined || raw === null) throw new DataError(400, `Missing key ${k}.`);
|
||||||
|
const v = coerce(
|
||||||
|
c.kind,
|
||||||
|
typeof raw === 'object' ? JSON.stringify(raw) : String(raw),
|
||||||
|
c.label
|
||||||
|
);
|
||||||
|
if (!v.ok || v.value === null) throw new DataError(400, `Invalid key ${k}.`);
|
||||||
|
return sql`t.${sql.identifier(k)} = ${cast(v.value, c)}`;
|
||||||
|
}),
|
||||||
|
sql` AND `
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let auditAvailable: boolean | null = null;
|
||||||
|
async function audit(
|
||||||
|
tx: Exec,
|
||||||
|
actor: Actor,
|
||||||
|
t: TableInfo,
|
||||||
|
action: 'insert' | 'update' | 'delete' | 'import' | 'add_column',
|
||||||
|
pk: unknown,
|
||||||
|
before: unknown,
|
||||||
|
after: unknown
|
||||||
|
) {
|
||||||
|
if (auditAvailable === null) {
|
||||||
|
const [r] = (await tx.execute<{ ok: boolean }>(
|
||||||
|
sql`SELECT to_regclass('meta.edit_log') IS NOT NULL AND has_table_privilege('meta.edit_log', 'INSERT') AS ok`
|
||||||
|
)) as unknown as { ok: boolean }[];
|
||||||
|
auditAvailable = Boolean(r?.ok);
|
||||||
|
}
|
||||||
|
if (!auditAvailable) return;
|
||||||
|
await tx.execute(sql`
|
||||||
|
INSERT INTO meta.edit_log (user_id, user_email, schema_name, table_name, action, pk, before, after)
|
||||||
|
VALUES (${actor.id}, ${actor.email}, ${t.schema}, ${t.name}, ${action},
|
||||||
|
${pk === null ? null : JSON.stringify(pk)}::jsonb,
|
||||||
|
${before === null ? null : JSON.stringify(before)}::jsonb,
|
||||||
|
${after === null ? null : JSON.stringify(after)}::jsonb)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function pgMessage(e: unknown): DataError {
|
||||||
|
if (e instanceof DataError) return e;
|
||||||
|
const err = (e as { cause?: unknown })?.cause ?? e;
|
||||||
|
const code = (err as { code?: string })?.code ?? '';
|
||||||
|
const msg = (err as { message?: string })?.message ?? 'Database error';
|
||||||
|
// constraint / data errors are the user's input; show Postgres' explanation
|
||||||
|
if (/^(22|23)/.test(code)) return new DataError(400, msg);
|
||||||
|
if (code === '42501') return new DataError(403, 'The app has no permission for this change.');
|
||||||
|
console.error('data editor write failed', e);
|
||||||
|
return new DataError(500, 'The database refused the change.');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function write<T>(fn: (tx: Tx) => Promise<T>): Promise<T> {
|
||||||
|
try {
|
||||||
|
return await db.transaction(async (tx) => {
|
||||||
|
await tx.execute(sql`SET LOCAL statement_timeout = '30s'`);
|
||||||
|
return fn(tx);
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
throw pgMessage(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const one = <T>(rows: unknown) => (rows as T[])[0];
|
||||||
|
|
||||||
|
export async function insertRow(
|
||||||
|
t: TableInfo,
|
||||||
|
values: Record<string, unknown>,
|
||||||
|
unlocked: boolean,
|
||||||
|
actor: Actor
|
||||||
|
) {
|
||||||
|
assertWritable(t, unlocked, 'insert');
|
||||||
|
const set = assignments(t, values);
|
||||||
|
return write(async (tx) => {
|
||||||
|
const row = one<{ r: Record<string, unknown> }>(
|
||||||
|
await tx.execute(
|
||||||
|
set.length
|
||||||
|
? sql`INSERT INTO ${rel(t.schema, t.name)} AS t (${sql.join(
|
||||||
|
set.map((s) => sql.identifier(s.c.name)),
|
||||||
|
sql`, `
|
||||||
|
)})
|
||||||
|
VALUES (${sql.join(
|
||||||
|
set.map((s) => cast(s.v, s.c)),
|
||||||
|
sql`, `
|
||||||
|
)}) RETURNING ${rowJson(t)} AS r`
|
||||||
|
: sql`INSERT INTO ${rel(t.schema, t.name)} AS t DEFAULT VALUES RETURNING ${rowJson(t)} AS r`
|
||||||
|
)
|
||||||
|
).r;
|
||||||
|
const pk = Object.fromEntries(t.primaryKey.map((k) => [k, row[k]]));
|
||||||
|
await audit(tx, actor, t, 'insert', pk, null, row);
|
||||||
|
return row;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateRow(
|
||||||
|
t: TableInfo,
|
||||||
|
pk: Pk,
|
||||||
|
values: Record<string, unknown>,
|
||||||
|
unlocked: boolean,
|
||||||
|
actor: Actor
|
||||||
|
) {
|
||||||
|
assertWritable(t, unlocked, 'update');
|
||||||
|
const set = assignments(t, values);
|
||||||
|
if (!set.length) throw new DataError(400, 'Nothing to change.');
|
||||||
|
const where = pkWhere(t, pk);
|
||||||
|
return write(async (tx) => {
|
||||||
|
const before = one<{ r: Record<string, unknown> }>(
|
||||||
|
await tx.execute(
|
||||||
|
sql`SELECT ${rowJson(t)} AS r FROM ${rel(t.schema, t.name)} t WHERE ${where} FOR UPDATE`
|
||||||
|
)
|
||||||
|
)?.r;
|
||||||
|
if (!before) throw new DataError(404, 'That row no longer exists.');
|
||||||
|
const after = one<{ r: Record<string, unknown> }>(
|
||||||
|
await tx.execute(sql`UPDATE ${rel(t.schema, t.name)} AS t
|
||||||
|
SET ${sql.join(
|
||||||
|
set.map((s) => sql`${sql.identifier(s.c.name)} = ${cast(s.v, s.c)}`),
|
||||||
|
sql`, `
|
||||||
|
)}
|
||||||
|
WHERE ${where} RETURNING ${rowJson(t)} AS r`)
|
||||||
|
).r;
|
||||||
|
const changed = Object.fromEntries(set.map((s) => [s.c.name, before[s.c.name]]));
|
||||||
|
await audit(
|
||||||
|
tx,
|
||||||
|
actor,
|
||||||
|
t,
|
||||||
|
'update',
|
||||||
|
pk,
|
||||||
|
changed,
|
||||||
|
Object.fromEntries(set.map((s) => [s.c.name, after[s.c.name]]))
|
||||||
|
);
|
||||||
|
return after;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteRows(t: TableInfo, pks: Pk[], unlocked: boolean, actor: Actor) {
|
||||||
|
assertWritable(t, unlocked, 'delete');
|
||||||
|
if (!pks.length || pks.length > 500) throw new DataError(400, 'Select 1–500 rows to delete.');
|
||||||
|
const wheres = pks.map((pk) => pkWhere(t, pk));
|
||||||
|
return write(async (tx) => {
|
||||||
|
let n = 0;
|
||||||
|
for (let i = 0; i < wheres.length; i++) {
|
||||||
|
const before = one<{ r: Record<string, unknown> }>(
|
||||||
|
await tx.execute(
|
||||||
|
sql`DELETE FROM ${rel(t.schema, t.name)} AS t WHERE ${wheres[i]} RETURNING ${rowJson(t)} AS r`
|
||||||
|
)
|
||||||
|
)?.r;
|
||||||
|
if (!before) continue;
|
||||||
|
n++;
|
||||||
|
await audit(tx, actor, t, 'delete', pks[i], before, null);
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Append CSV rows (header = column names) in one transaction: all or nothing. */
|
||||||
|
export async function importRows(
|
||||||
|
t: TableInfo,
|
||||||
|
table: string[][],
|
||||||
|
unlocked: boolean,
|
||||||
|
actor: Actor
|
||||||
|
): Promise<number> {
|
||||||
|
assertWritable(t, unlocked, 'insert');
|
||||||
|
const [header, ...body] = table;
|
||||||
|
if (!header?.length) throw new DataError(400, 'The CSV is empty.');
|
||||||
|
if (body.length > 20_000) throw new DataError(400, 'Import at most 20,000 rows at a time.');
|
||||||
|
const names = header.map((h) => h.trim());
|
||||||
|
const unknown = names.filter((n) => n && !t.columns.some((c) => c.name === n));
|
||||||
|
if (unknown.length) throw new DataError(400, `Unknown column(s): ${unknown.join(', ')}.`);
|
||||||
|
if (new Set(names).size !== names.length)
|
||||||
|
throw new DataError(400, 'Duplicate column in the header.');
|
||||||
|
const prepared = body.map((cells, i) => {
|
||||||
|
if (cells.length !== names.length)
|
||||||
|
throw new DataError(400, `Row ${i + 2} has ${cells.length} cells, expected ${names.length}.`);
|
||||||
|
const values: Record<string, unknown> = {};
|
||||||
|
names.forEach((n, j) => {
|
||||||
|
if (!n) return;
|
||||||
|
const c = t.columns.find((x) => x.name === n)!;
|
||||||
|
if (c.readOnly) return;
|
||||||
|
// empty cell: column default when it has one, NULL otherwise (text stays '')
|
||||||
|
if (cells[j] === '' && (c.hasDefault || c.kind !== 'text')) return;
|
||||||
|
values[n] = cells[j];
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
return assignments(t, values);
|
||||||
|
} catch (e) {
|
||||||
|
throw new DataError(400, `Row ${i + 2}: ${(e as Error).message}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return write(async (tx) => {
|
||||||
|
for (const set of prepared)
|
||||||
|
await tx.execute(
|
||||||
|
set.length
|
||||||
|
? sql`INSERT INTO ${rel(t.schema, t.name)} (${sql.join(
|
||||||
|
set.map((s) => sql.identifier(s.c.name)),
|
||||||
|
sql`, `
|
||||||
|
)})
|
||||||
|
VALUES (${sql.join(
|
||||||
|
set.map((s) => cast(s.v, s.c)),
|
||||||
|
sql`, `
|
||||||
|
)})`
|
||||||
|
: sql`INSERT INTO ${rel(t.schema, t.name)} DEFAULT VALUES`
|
||||||
|
);
|
||||||
|
await audit(tx, actor, t, 'import', null, null, { rows: prepared.length, columns: names });
|
||||||
|
return prepared.length;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export const LIST_UI_TYPES = [
|
||||||
|
'text',
|
||||||
|
'longtext',
|
||||||
|
'url',
|
||||||
|
'image',
|
||||||
|
'number',
|
||||||
|
'cents',
|
||||||
|
'date',
|
||||||
|
'datetime',
|
||||||
|
'bool',
|
||||||
|
'select',
|
||||||
|
'multiselect',
|
||||||
|
'json'
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
/** Add a typed column to a runtime list (lists.* only) through meta.add_column(). */
|
||||||
|
export async function addListColumn(
|
||||||
|
t: TableInfo,
|
||||||
|
input: { name: string; uiType: string; label?: string; options?: string[] },
|
||||||
|
actor: Actor
|
||||||
|
) {
|
||||||
|
if (!t.isList || !t.listSlug || t.access !== 'open')
|
||||||
|
throw new DataError(403, 'Columns can only be added to lists.');
|
||||||
|
if (!/^[a-z][a-z0-9_]{0,62}$/.test(input.name))
|
||||||
|
throw new DataError(
|
||||||
|
400,
|
||||||
|
'Column names use lowercase letters, digits and _ and start with a letter.'
|
||||||
|
);
|
||||||
|
if (!(LIST_UI_TYPES as readonly string[]).includes(input.uiType))
|
||||||
|
throw new DataError(400, 'Unknown column type.');
|
||||||
|
if (t.columns.some((c) => c.name === input.name))
|
||||||
|
throw new DataError(409, 'That column already exists.');
|
||||||
|
const col: Record<string, unknown> = { name: input.name, ui_type: input.uiType };
|
||||||
|
if (input.label?.trim()) col.label = input.label.trim().slice(0, 100);
|
||||||
|
if (input.uiType === 'select' || input.uiType === 'multiselect') {
|
||||||
|
const values = (input.options ?? [])
|
||||||
|
.map((o) => o.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.slice(0, 100);
|
||||||
|
if (input.uiType === 'select' && !values.length)
|
||||||
|
throw new DataError(400, 'Give the select at least one option.');
|
||||||
|
col.options = { values };
|
||||||
|
}
|
||||||
|
await write(async (tx) => {
|
||||||
|
await tx.execute(sql`SELECT meta.add_column(${t.listSlug}, ${JSON.stringify(col)}::jsonb)`);
|
||||||
|
await audit(tx, actor, t, 'add_column', null, null, col);
|
||||||
|
});
|
||||||
|
}
|
||||||
146
src/lib/server/data-policy.test.ts
Normal file
146
src/lib/server/data-policy.test.ts
Normal file
@@ -0,0 +1,146 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import {
|
||||||
|
coerce,
|
||||||
|
hiddenColumn,
|
||||||
|
parseCsv,
|
||||||
|
parseFilter,
|
||||||
|
pgArrayLiteral,
|
||||||
|
plausibleIdent,
|
||||||
|
safeTypeName,
|
||||||
|
tableAccess,
|
||||||
|
toCsv,
|
||||||
|
typeKind,
|
||||||
|
writeAllowed,
|
||||||
|
type TableFacts
|
||||||
|
} from './data-policy';
|
||||||
|
|
||||||
|
const t = (schema: string, name: string, extra: Partial<TableFacts> = {}): TableFacts => ({
|
||||||
|
schema,
|
||||||
|
name,
|
||||||
|
relkind: 'r',
|
||||||
|
hasPrimaryKey: true,
|
||||||
|
canInsert: true,
|
||||||
|
canUpdate: true,
|
||||||
|
canDelete: true,
|
||||||
|
...extra
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('tableAccess', () => {
|
||||||
|
it('hides auth secrets, migrations and credential tables', () => {
|
||||||
|
for (const name of ['session', 'account', 'verification', 'jwks'])
|
||||||
|
expect(tableAccess(t('auth', name))).toBe('hidden');
|
||||||
|
expect(tableAccess(t('drizzle', '__drizzle_migrations'))).toBe('hidden');
|
||||||
|
expect(tableAccess(t('money', 'bank_sessions'))).toBe('hidden');
|
||||||
|
expect(tableAccess(t('money', 'api_keys'))).toBe('hidden');
|
||||||
|
expect(tableAccess(t('public', 'anything'))).toBe('hidden');
|
||||||
|
});
|
||||||
|
it('keeps auth.user, meta and legacy read-only', () => {
|
||||||
|
expect(tableAccess(t('auth', 'user'))).toBe('locked');
|
||||||
|
expect(tableAccess(t('meta', 'tables'))).toBe('locked');
|
||||||
|
expect(tableAccess(t('legacy', 'lists'))).toBe('locked');
|
||||||
|
});
|
||||||
|
it('locks views, tables without a key or without write grants', () => {
|
||||||
|
expect(tableAccess(t('energy', 'gas_storage_latest', { relkind: 'v' }))).toBe('locked');
|
||||||
|
expect(tableAccess(t('money', 'x', { hasPrimaryKey: false }))).toBe('locked');
|
||||||
|
expect(
|
||||||
|
tableAccess(t('ref', 'pokemon', { canInsert: false, canUpdate: false, canDelete: false }))
|
||||||
|
).toBe('locked');
|
||||||
|
});
|
||||||
|
it('lists are open, app tables are core (unlock first)', () => {
|
||||||
|
expect(tableAccess(t('lists', 'books'))).toBe('open');
|
||||||
|
expect(tableAccess(t('money', 'transactions'))).toBe('core');
|
||||||
|
expect(tableAccess(t('books', 'sessions'))).toBe('core');
|
||||||
|
expect(writeAllowed('open', false)).toBe(true);
|
||||||
|
expect(writeAllowed('core', false)).toBe(false);
|
||||||
|
expect(writeAllowed('core', true)).toBe(true);
|
||||||
|
expect(writeAllowed('locked', true)).toBe(false);
|
||||||
|
expect(writeAllowed('hidden', true)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('columns and identifiers', () => {
|
||||||
|
it('hides credential-like columns only', () => {
|
||||||
|
for (const c of [
|
||||||
|
'password',
|
||||||
|
'access_token',
|
||||||
|
'refresh_token_expires_at',
|
||||||
|
'api_key',
|
||||||
|
'client_secret'
|
||||||
|
])
|
||||||
|
expect(hiddenColumn(c)).toBe(true);
|
||||||
|
for (const c of ['key', 'session_id', 'iban', 'author', 'market_hash_name', 'r2_key'])
|
||||||
|
expect(hiddenColumn(c)).toBe(false);
|
||||||
|
});
|
||||||
|
it('validates identifiers and type names', () => {
|
||||||
|
expect(plausibleIdent('done_at')).toBe(true);
|
||||||
|
expect(plausibleIdent('a"; drop table x; --')).toBe(false);
|
||||||
|
expect(plausibleIdent('')).toBe(false);
|
||||||
|
expect(safeTypeName('timestamp with time zone')).toBe(true);
|
||||||
|
expect(safeTypeName('numeric(12,2)')).toBe(true);
|
||||||
|
expect(safeTypeName('"books"."metadata_status"')).toBe(true);
|
||||||
|
expect(safeTypeName('text); drop table x; --')).toBe(false);
|
||||||
|
});
|
||||||
|
it('maps types to editor kinds', () => {
|
||||||
|
expect(typeKind('integer', false)).toBe('number');
|
||||||
|
expect(typeKind('numeric(12,2)', false)).toBe('number');
|
||||||
|
expect(typeKind('text[]', false)).toBe('array');
|
||||||
|
expect(typeKind('books.metadata_status', true)).toBe('enum');
|
||||||
|
expect(typeKind('timestamp with time zone', false)).toBe('datetime');
|
||||||
|
expect(typeKind('jsonb', false)).toBe('json');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('coerce', () => {
|
||||||
|
it('validates per kind', () => {
|
||||||
|
expect(coerce('number', '12.5', 'n')).toEqual({ ok: true, value: '12.5' });
|
||||||
|
expect(coerce('number', 'abc', 'n').ok).toBe(false);
|
||||||
|
expect(coerce('number', '', 'n')).toEqual({ ok: true, value: null });
|
||||||
|
expect(coerce('bool', 'yes', 'b')).toEqual({ ok: true, value: 'true' });
|
||||||
|
expect(coerce('bool', 'maybe', 'b').ok).toBe(false);
|
||||||
|
expect(coerce('date', '2026-10-24', 'd').ok).toBe(true);
|
||||||
|
expect(coerce('date', '24/10', 'd').ok).toBe(false);
|
||||||
|
expect(coerce('json', '{"a":1}', 'j').ok).toBe(true);
|
||||||
|
expect(coerce('json', '{a:1}', 'j').ok).toBe(false);
|
||||||
|
expect(coerce('text', '', 't')).toEqual({ ok: true, value: '' });
|
||||||
|
expect(coerce('text', null, 't')).toEqual({ ok: true, value: null });
|
||||||
|
});
|
||||||
|
it('builds array literals', () => {
|
||||||
|
expect(pgArrayLiteral(['a', 'b c', 'x"y'])).toBe('{"a","b c","x\\"y"}');
|
||||||
|
expect(coerce('array', '["a","b"]', 'a')).toEqual({ ok: true, value: '{"a","b"}' });
|
||||||
|
expect(coerce('array', 'a, b', 'a')).toEqual({ ok: true, value: '{"a","b"}' });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('filters and csv', () => {
|
||||||
|
it('parses filters', () => {
|
||||||
|
expect(parseFilter('name:contains:foo:bar')).toEqual({
|
||||||
|
column: 'name',
|
||||||
|
op: 'contains',
|
||||||
|
value: 'foo:bar'
|
||||||
|
});
|
||||||
|
expect(parseFilter('done_at:null')).toEqual({ column: 'done_at', op: 'null', value: '' });
|
||||||
|
expect(parseFilter('x;y:eq:1')).toBeNull();
|
||||||
|
expect(parseFilter('name:like:1')).toBeNull();
|
||||||
|
});
|
||||||
|
it('round-trips CSV', () => {
|
||||||
|
const csv = toCsv(
|
||||||
|
['a', 'b'],
|
||||||
|
[
|
||||||
|
['x, y', 'he said "hi"'],
|
||||||
|
[null, { k: 1 }],
|
||||||
|
['line\nbreak', 3]
|
||||||
|
]
|
||||||
|
);
|
||||||
|
expect(parseCsv(csv)).toEqual([
|
||||||
|
['a', 'b'],
|
||||||
|
['x, y', 'he said "hi"'],
|
||||||
|
['', '{"k":1}'],
|
||||||
|
['line\nbreak', '3']
|
||||||
|
]);
|
||||||
|
expect(parseCsv('\ufeffa,b\r\n1,2')).toEqual([
|
||||||
|
['a', 'b'],
|
||||||
|
['1', '2']
|
||||||
|
]);
|
||||||
|
expect(() => parseCsv('a\n"open')).toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
244
src/lib/server/data-policy.ts
Normal file
244
src/lib/server/data-policy.ts
Normal file
@@ -0,0 +1,244 @@
|
|||||||
|
// Pure rules for the owner-only database browser at /lists/db: which tables and columns
|
||||||
|
// are visible or editable, identifier checks, value coercion and CSV. No DB access here,
|
||||||
|
// so it is unit tested directly (data-policy.test.ts).
|
||||||
|
|
||||||
|
/** hidden: never listed or served. locked: always read-only. core: read-only until the
|
||||||
|
* owner unlocks editing for the table. open: freely editable (runtime lists). */
|
||||||
|
export type Access = 'hidden' | 'locked' | 'core' | 'open';
|
||||||
|
|
||||||
|
export type TableFacts = {
|
||||||
|
schema: string;
|
||||||
|
name: string;
|
||||||
|
/** pg_class.relkind: r table, p partitioned, v view, m materialized view */
|
||||||
|
relkind: string;
|
||||||
|
hasPrimaryKey: boolean;
|
||||||
|
canInsert: boolean;
|
||||||
|
canUpdate: boolean;
|
||||||
|
canDelete: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Schemas that are never shown: migration bookkeeping and Postgres internals. */
|
||||||
|
const HIDDEN_SCHEMAS = new Set(['drizzle', 'information_schema', 'public']);
|
||||||
|
/** Better Auth: only the user table is visible (read-only); sessions, accounts (OAuth
|
||||||
|
* tokens, password hashes) and verification codes are hidden. */
|
||||||
|
const AUTH_VISIBLE = new Set(['user']);
|
||||||
|
/** Individually hidden tables holding third-party credentials. */
|
||||||
|
const HIDDEN_TABLES = new Set(['money.bank_sessions']);
|
||||||
|
/** Schemas whose tables are always read-only here (registry, rollback copies, auth). */
|
||||||
|
const LOCKED_SCHEMAS = new Set(['meta', 'legacy', 'auth']);
|
||||||
|
const CREDENTIAL_NAME =
|
||||||
|
/(^|_)(passwords?|passwd|secrets?|tokens?|api_?keys?|private_?keys?|credentials?|salt|jwks|otp)($|_)/i;
|
||||||
|
|
||||||
|
export function tableAccess(t: TableFacts): Access {
|
||||||
|
if (HIDDEN_SCHEMAS.has(t.schema) || t.schema.startsWith('pg_')) return 'hidden';
|
||||||
|
if (t.schema === 'auth' && !AUTH_VISIBLE.has(t.name)) return 'hidden';
|
||||||
|
if (HIDDEN_TABLES.has(`${t.schema}.${t.name}`) || CREDENTIAL_NAME.test(t.name)) return 'hidden';
|
||||||
|
if (LOCKED_SCHEMAS.has(t.schema)) return 'locked';
|
||||||
|
if (t.relkind !== 'r' && t.relkind !== 'p') return 'locked';
|
||||||
|
if (!t.hasPrimaryKey) return 'locked';
|
||||||
|
if (!t.canInsert && !t.canUpdate && !t.canDelete) return 'locked';
|
||||||
|
return t.schema === 'lists' ? 'open' : 'core';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Columns that are never sent to the browser (tokens, password hashes, ...). */
|
||||||
|
export function hiddenColumn(name: string): boolean {
|
||||||
|
return CREDENTIAL_NAME.test(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a write may go ahead: open tables always, core tables only when unlocked. */
|
||||||
|
export function writeAllowed(access: Access, unlocked: boolean): boolean {
|
||||||
|
return access === 'open' || (access === 'core' && unlocked);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Identifiers coming from the URL/body are looked up in the catalog; this is a cheap
|
||||||
|
* pre-filter so obviously bad input never reaches a query at all. */
|
||||||
|
export function plausibleIdent(s: unknown): s is string {
|
||||||
|
return (
|
||||||
|
typeof s === 'string' && s.length > 0 && s.length <= 63 && /^[A-Za-z_][A-Za-z0-9_$]*$/.test(s)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** format_type() output that is safe to splice into a CAST (it comes from pg_catalog). */
|
||||||
|
export function safeTypeName(type: string): boolean {
|
||||||
|
return /^[A-Za-z0-9_ ."(),[\]]{1,120}$/.test(type) && !type.includes('--') && !type.includes(';');
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Kind =
|
||||||
|
| 'text'
|
||||||
|
| 'number'
|
||||||
|
| 'bool'
|
||||||
|
| 'date'
|
||||||
|
| 'datetime'
|
||||||
|
| 'time'
|
||||||
|
| 'json'
|
||||||
|
| 'enum'
|
||||||
|
| 'array'
|
||||||
|
| 'uuid'
|
||||||
|
| 'other';
|
||||||
|
|
||||||
|
export function typeKind(type: string, isEnum: boolean): Kind {
|
||||||
|
if (type.endsWith('[]')) return 'array';
|
||||||
|
if (isEnum) return 'enum';
|
||||||
|
if (/^(smallint|integer|bigint|real|double precision|numeric|money)/.test(type)) return 'number';
|
||||||
|
if (type === 'boolean') return 'bool';
|
||||||
|
if (type === 'date') return 'date';
|
||||||
|
if (type.startsWith('timestamp')) return 'datetime';
|
||||||
|
if (type.startsWith('time')) return 'time';
|
||||||
|
if (type === 'json' || type === 'jsonb') return 'json';
|
||||||
|
if (type === 'uuid') return 'uuid';
|
||||||
|
if (/^(text|character|"char"|citext|name)/.test(type)) return 'text';
|
||||||
|
return 'other';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Postgres array literal for a list of strings, e.g. ['a', 'b c'] -> {"a","b c"}. */
|
||||||
|
export function pgArrayLiteral(items: unknown[]): string {
|
||||||
|
return `{${items
|
||||||
|
.map((v) =>
|
||||||
|
v === null || v === undefined
|
||||||
|
? 'NULL'
|
||||||
|
: `"${String(typeof v === 'object' ? JSON.stringify(v) : v).replace(/["\\]/g, '\\$&')}"`
|
||||||
|
)
|
||||||
|
.join(',')}}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Coerced = { ok: true; value: string | null } | { ok: false; error: string };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Turn a value from the editor/CSV into the text that is bound as a parameter and
|
||||||
|
* CAST to the column type by Postgres (which does the final validation). null = NULL.
|
||||||
|
* Empty input means NULL for every non-text column.
|
||||||
|
*/
|
||||||
|
export function coerce(kind: Kind, raw: unknown, label: string): Coerced {
|
||||||
|
if (raw === null || raw === undefined) return { ok: true, value: null };
|
||||||
|
if (kind === 'array') {
|
||||||
|
let items: unknown = raw;
|
||||||
|
if (typeof raw === 'string') {
|
||||||
|
const s = raw.trim();
|
||||||
|
if (!s) return { ok: true, value: null };
|
||||||
|
if (s.startsWith('{')) return { ok: true, value: s }; // already a Postgres literal
|
||||||
|
try {
|
||||||
|
items = JSON.parse(s);
|
||||||
|
} catch {
|
||||||
|
items = s
|
||||||
|
.split(',')
|
||||||
|
.map((x) => x.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!Array.isArray(items)) return { ok: false, error: `${label} must be a list.` };
|
||||||
|
return { ok: true, value: pgArrayLiteral(items) };
|
||||||
|
}
|
||||||
|
if (kind === 'json') {
|
||||||
|
if (typeof raw !== 'string') return { ok: true, value: JSON.stringify(raw) };
|
||||||
|
if (!raw.trim()) return { ok: true, value: null };
|
||||||
|
try {
|
||||||
|
JSON.parse(raw);
|
||||||
|
} catch {
|
||||||
|
return { ok: false, error: `${label} is not valid JSON.` };
|
||||||
|
}
|
||||||
|
return { ok: true, value: raw };
|
||||||
|
}
|
||||||
|
const s = typeof raw === 'string' ? raw : String(raw);
|
||||||
|
if (kind === 'text' || kind === 'other') return { ok: true, value: s };
|
||||||
|
const t = s.trim();
|
||||||
|
if (!t) return { ok: true, value: null };
|
||||||
|
switch (kind) {
|
||||||
|
case 'number':
|
||||||
|
if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(t.replace(/_/g, '')))
|
||||||
|
return { ok: false, error: `${label} must be a number.` };
|
||||||
|
return { ok: true, value: t.replace(/_/g, '') };
|
||||||
|
case 'bool': {
|
||||||
|
const v = t.toLowerCase();
|
||||||
|
if (['true', 't', 'yes', 'y', '1', 'on'].includes(v)) return { ok: true, value: 'true' };
|
||||||
|
if (['false', 'f', 'no', 'n', '0', 'off'].includes(v)) return { ok: true, value: 'false' };
|
||||||
|
return { ok: false, error: `${label} must be true or false.` };
|
||||||
|
}
|
||||||
|
case 'date':
|
||||||
|
if (!/^\d{4}-\d{2}-\d{2}/.test(t))
|
||||||
|
return { ok: false, error: `${label} must be a date (YYYY-MM-DD).` };
|
||||||
|
return { ok: true, value: t };
|
||||||
|
case 'datetime':
|
||||||
|
if (Number.isNaN(Date.parse(t)) && !/^\d{4}-\d{2}-\d{2}/.test(t))
|
||||||
|
return { ok: false, error: `${label} must be a date and time.` };
|
||||||
|
return { ok: true, value: t };
|
||||||
|
case 'uuid':
|
||||||
|
if (!/^[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}$/i.test(t))
|
||||||
|
return { ok: false, error: `${label} must be a UUID.` };
|
||||||
|
return { ok: true, value: t };
|
||||||
|
default:
|
||||||
|
return { ok: true, value: t };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Filter operators accepted by the grid (?f=<column>:<op>:<value>). */
|
||||||
|
export const FILTER_OPS = [
|
||||||
|
'contains',
|
||||||
|
'eq',
|
||||||
|
'ne',
|
||||||
|
'gt',
|
||||||
|
'gte',
|
||||||
|
'lt',
|
||||||
|
'lte',
|
||||||
|
'null',
|
||||||
|
'notnull'
|
||||||
|
] as const;
|
||||||
|
export type FilterOp = (typeof FILTER_OPS)[number];
|
||||||
|
export type Filter = { column: string; op: FilterOp; value: string };
|
||||||
|
|
||||||
|
export function parseFilter(s: string): Filter | null {
|
||||||
|
const i = s.indexOf(':');
|
||||||
|
const j = i < 0 ? -1 : s.indexOf(':', i + 1);
|
||||||
|
if (i < 1) return null;
|
||||||
|
const column = s.slice(0, i);
|
||||||
|
const op = (j < 0 ? s.slice(i + 1) : s.slice(i + 1, j)) as FilterOp;
|
||||||
|
const value = j < 0 ? '' : s.slice(j + 1);
|
||||||
|
if (!plausibleIdent(column) || !FILTER_OPS.includes(op) || value.length > 500) return null;
|
||||||
|
return { column, op, value };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- CSV (RFC 4180) ----
|
||||||
|
|
||||||
|
export function csvCell(v: unknown): string {
|
||||||
|
if (v === null || v === undefined) return '';
|
||||||
|
const s = typeof v === 'object' ? JSON.stringify(v) : String(v);
|
||||||
|
return /[",\r\n]/.test(s) || /^\s|\s$/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function toCsv(header: string[], rows: unknown[][]): string {
|
||||||
|
return [header, ...rows].map((r) => r.map(csvCell).join(',')).join('\r\n') + '\r\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseCsv(text: string): string[][] {
|
||||||
|
const rows: string[][] = [];
|
||||||
|
let row: string[] = [];
|
||||||
|
let cell = '';
|
||||||
|
let quoted = false;
|
||||||
|
let i = text.charCodeAt(0) === 0xfeff ? 1 : 0; // BOM
|
||||||
|
for (; i < text.length; i++) {
|
||||||
|
const ch = text[i];
|
||||||
|
if (quoted) {
|
||||||
|
if (ch === '"') {
|
||||||
|
if (text[i + 1] === '"') {
|
||||||
|
cell += '"';
|
||||||
|
i++;
|
||||||
|
} else quoted = false;
|
||||||
|
} else cell += ch;
|
||||||
|
} else if (ch === '"' && cell === '') quoted = true;
|
||||||
|
else if (ch === ',') {
|
||||||
|
row.push(cell);
|
||||||
|
cell = '';
|
||||||
|
} else if (ch === '\n' || ch === '\r') {
|
||||||
|
if (ch === '\r' && text[i + 1] === '\n') i++;
|
||||||
|
row.push(cell);
|
||||||
|
rows.push(row);
|
||||||
|
row = [];
|
||||||
|
cell = '';
|
||||||
|
} else cell += ch;
|
||||||
|
}
|
||||||
|
if (quoted) throw new Error('CSV has an unterminated quoted field.');
|
||||||
|
if (cell !== '' || row.length) {
|
||||||
|
row.push(cell);
|
||||||
|
rows.push(row);
|
||||||
|
}
|
||||||
|
return rows.filter((r) => !(r.length === 1 && r[0] === ''));
|
||||||
|
}
|
||||||
50
src/lib/server/data-route.ts
Normal file
50
src/lib/server/data-route.ts
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
// Request helpers shared by the /lists/db routes. Every route calls ownerTable() (or
|
||||||
|
// requireOwner) itself, so access never depends on the layout load having run.
|
||||||
|
import { error, json } from '@sveltejs/kit';
|
||||||
|
import { requireOwner } from '#lib/server/owner.js';
|
||||||
|
import { parseFilter, type Filter } from '#lib/server/data-policy.js';
|
||||||
|
import { DataError, getTable, type Actor, type RowQuery } from '#lib/server/data-editor.js';
|
||||||
|
|
||||||
|
type Event = { locals: App.Locals; params: Record<string, string>; request: Request; url: URL };
|
||||||
|
|
||||||
|
export async function ownerTable(event: Event) {
|
||||||
|
const user = requireOwner(event);
|
||||||
|
const table = await getTable(event.params.schema ?? '', event.params.table ?? '');
|
||||||
|
if (!table) error(404, 'Not Found');
|
||||||
|
const actor: Actor = { id: user.id, email: user.email };
|
||||||
|
return { table, actor };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Core (app) tables are read-only until the owner unlocks editing in the UI, which sends
|
||||||
|
* this header with every write; the server refuses core-table writes without it. */
|
||||||
|
export const unlocked = (request: Request) => request.headers.get('x-data-unlock') === '1';
|
||||||
|
|
||||||
|
export function rowQuery(url: URL): RowQuery {
|
||||||
|
const p = url.searchParams;
|
||||||
|
const filters = p
|
||||||
|
.getAll('f')
|
||||||
|
.slice(0, 20)
|
||||||
|
.map(parseFilter)
|
||||||
|
.filter((f): f is Filter => f !== null);
|
||||||
|
const limit = Number(p.get('limit') ?? 50);
|
||||||
|
const page = Number(p.get('page') ?? 1);
|
||||||
|
const safeLimit = [25, 50, 100, 250, 500].includes(limit) ? limit : 50;
|
||||||
|
return {
|
||||||
|
q: (p.get('q') ?? '').slice(0, 200),
|
||||||
|
filters,
|
||||||
|
sort: p.get('sort') || undefined,
|
||||||
|
dir: p.get('dir') === 'desc' ? 'desc' : 'asc',
|
||||||
|
limit: safeLimit,
|
||||||
|
offset: Number.isSafeInteger(page) && page > 1 ? (page - 1) * safeLimit : 0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function failure(e: unknown) {
|
||||||
|
if (e instanceof DataError) return json({ message: e.message }, { status: e.status });
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const noStore = {
|
||||||
|
'cache-control': 'private, no-store',
|
||||||
|
'x-robots-tag': 'noindex, nofollow'
|
||||||
|
};
|
||||||
209
src/lib/server/db/schema/books.ts
Normal file
209
src/lib/server/db/schema/books.ts
Normal file
@@ -0,0 +1,209 @@
|
|||||||
|
import { sql } from 'drizzle-orm';
|
||||||
|
import {
|
||||||
|
bigint,
|
||||||
|
check,
|
||||||
|
index,
|
||||||
|
integer,
|
||||||
|
jsonb,
|
||||||
|
primaryKey,
|
||||||
|
real,
|
||||||
|
smallint,
|
||||||
|
text,
|
||||||
|
uniqueIndex
|
||||||
|
} from 'drizzle-orm/pg-core';
|
||||||
|
import { booksSchema } from './schemas';
|
||||||
|
import { id, timestamps, timestamptz } from './helpers';
|
||||||
|
|
||||||
|
// Book tracker + reader. A work is one book (any edition we care about); a work can
|
||||||
|
// exist without files (placeholder for a book Stan wants but doesn't have yet).
|
||||||
|
// Files live on stan-vm under $BOOKS_ROOT/library (target /srv/books/library); covers under $BOOKS_ROOT/covers.
|
||||||
|
|
||||||
|
export const bookMetadataStatus = booksSchema.enum('metadata_status', [
|
||||||
|
'pending', // not looked up yet
|
||||||
|
'matched', // confident match (ISBN or exact title+author)
|
||||||
|
'low_confidence', // fuzzy match, needs a human look
|
||||||
|
'not_found',
|
||||||
|
'manual' // edited by hand; enrichment never overwrites it
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Reading state per work (on books.progress), shown publicly on /trackers/books.
|
||||||
|
export const bookReadingStatus = booksSchema.enum('reading_status', [
|
||||||
|
'want_to_read',
|
||||||
|
'reading',
|
||||||
|
'finished',
|
||||||
|
'abandoned'
|
||||||
|
]);
|
||||||
|
|
||||||
|
export const bookWorks = booksSchema.table(
|
||||||
|
'works',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
title: text('title').notNull(),
|
||||||
|
subtitle: text('subtitle'),
|
||||||
|
authors: text('authors').array().notNull().default([]),
|
||||||
|
isbn10: text('isbn10'),
|
||||||
|
isbn13: text('isbn13'),
|
||||||
|
publisher: text('publisher'),
|
||||||
|
year: integer('year'),
|
||||||
|
language: text('language'), // ISO 639-1 where known, e.g. 'en', 'nl'
|
||||||
|
pageCount: integer('page_count'),
|
||||||
|
description: text('description'),
|
||||||
|
tags: text('tags').array().notNull().default([]),
|
||||||
|
coverUrl: text('cover_url'), // remote source of the cover
|
||||||
|
coverPath: text('cover_path'), // local file, relative to /srv/books/covers
|
||||||
|
// External ids
|
||||||
|
openlibraryWorkId: text('openlibrary_work_id'), // e.g. OL45883W
|
||||||
|
openlibraryEditionId: text('openlibrary_edition_id'), // e.g. OL7353617M
|
||||||
|
googleBooksId: text('google_books_id'),
|
||||||
|
annasMd5: text('annas_md5').array().notNull().default([]), // Anna's Archive / libgen file md5s
|
||||||
|
sourceRefs: jsonb('source_refs').$type<Record<string, unknown>>().notNull().default({}),
|
||||||
|
metadataStatus: bookMetadataStatus('metadata_status').notNull().default('pending'),
|
||||||
|
metadataSource: text('metadata_source'), // 'openlibrary' | 'googlebooks' | 'libgen' | 'manual'
|
||||||
|
metadataFetchedAt: timestamptz('metadata_fetched_at'),
|
||||||
|
notes: text('notes'),
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
uniqueIndex('works_isbn13_key')
|
||||||
|
.on(t.isbn13)
|
||||||
|
.where(sql`${t.isbn13} is not null`),
|
||||||
|
index('works_title_idx').on(sql`lower(${t.title})`),
|
||||||
|
check('works_year_check', sql`${t.year} is null or ${t.year} between 0 and 2100`)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
export const bookFiles = booksSchema.table(
|
||||||
|
'files',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
workId: integer('work_id')
|
||||||
|
.notNull()
|
||||||
|
.references(() => bookWorks.id, { onDelete: 'cascade' }),
|
||||||
|
path: text('path').notNull(), // relative to /srv/books/library
|
||||||
|
format: text('format').notNull(), // epub | pdf | mobi | azw3 | djvu
|
||||||
|
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
|
||||||
|
sha256: text('sha256').notNull(),
|
||||||
|
md5: text('md5'),
|
||||||
|
source: text('source'), // 'gdrive' | 'textbooks-repo' | 'annas' | 'upload'
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
uniqueIndex('files_path_key').on(t.path),
|
||||||
|
uniqueIndex('files_sha256_key').on(t.sha256),
|
||||||
|
index('files_work_id_idx').on(t.workId)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
export const bookShelves = booksSchema.table(
|
||||||
|
'shelves',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
slug: text('slug').notNull(),
|
||||||
|
name: text('name').notNull(),
|
||||||
|
sortOrder: integer('sort_order').notNull().default(0),
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [uniqueIndex('shelves_slug_key').on(t.slug)]
|
||||||
|
);
|
||||||
|
|
||||||
|
export const bookShelfWorks = booksSchema.table(
|
||||||
|
'shelf_works',
|
||||||
|
{
|
||||||
|
shelfId: integer('shelf_id')
|
||||||
|
.notNull()
|
||||||
|
.references(() => bookShelves.id, { onDelete: 'cascade' }),
|
||||||
|
workId: integer('work_id')
|
||||||
|
.notNull()
|
||||||
|
.references(() => bookWorks.id, { onDelete: 'cascade' }),
|
||||||
|
sortOrder: integer('sort_order').notNull().default(0),
|
||||||
|
addedAt: timestamptz('added_at').notNull().defaultNow()
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
primaryKey({ columns: [t.shelfId, t.workId] }),
|
||||||
|
index('shelf_works_work_id_idx').on(t.workId)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
export const bookProgress = booksSchema.table(
|
||||||
|
'progress',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
workId: integer('work_id')
|
||||||
|
.notNull()
|
||||||
|
.references(() => bookWorks.id, { onDelete: 'cascade' }),
|
||||||
|
fileId: integer('file_id').references(() => bookFiles.id, { onDelete: 'set null' }),
|
||||||
|
locator: text('locator'), // EPUB CFI or 'page:123' for PDFs
|
||||||
|
page: integer('page'),
|
||||||
|
percent: real('percent').notNull().default(0),
|
||||||
|
startedAt: timestamptz('started_at'),
|
||||||
|
finishedAt: timestamptz('finished_at'),
|
||||||
|
status: bookReadingStatus('status').notNull().default('want_to_read'),
|
||||||
|
rating: smallint('rating'), // 1-5 stars
|
||||||
|
lastReadAt: timestamptz('last_read_at'), // last time the reader saved a position
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
uniqueIndex('progress_work_id_key').on(t.workId),
|
||||||
|
check('progress_percent_check', sql`${t.percent} between 0 and 100`),
|
||||||
|
check('progress_rating_check', sql`${t.rating} is null or ${t.rating} between 1 and 5`),
|
||||||
|
index('progress_status_idx').on(t.status)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
export const bookNotes = booksSchema.table(
|
||||||
|
'notes',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
workId: integer('work_id')
|
||||||
|
.notNull()
|
||||||
|
.references(() => bookWorks.id, { onDelete: 'cascade' }),
|
||||||
|
fileId: integer('file_id').references(() => bookFiles.id, { onDelete: 'set null' }),
|
||||||
|
locator: text('locator'),
|
||||||
|
page: integer('page'),
|
||||||
|
highlight: text('highlight'), // the selected text, if any
|
||||||
|
note: text('note'),
|
||||||
|
color: text('color'),
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [index('notes_work_id_idx').on(t.workId)]
|
||||||
|
);
|
||||||
|
|
||||||
|
// One reading session: opened in the reader (source 'reader') or logged by hand ('manual').
|
||||||
|
export const bookSessions = booksSchema.table(
|
||||||
|
'sessions',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
workId: integer('work_id')
|
||||||
|
.notNull()
|
||||||
|
.references(() => bookWorks.id, { onDelete: 'cascade' }),
|
||||||
|
fileId: integer('file_id').references(() => bookFiles.id, { onDelete: 'set null' }),
|
||||||
|
startedAt: timestamptz('started_at').notNull().defaultNow(),
|
||||||
|
endedAt: timestamptz('ended_at'),
|
||||||
|
startPercent: real('start_percent'),
|
||||||
|
endPercent: real('end_percent'),
|
||||||
|
startPage: integer('start_page'),
|
||||||
|
endPage: integer('end_page'),
|
||||||
|
pagesRead: integer('pages_read'), // derived on save: page delta, or percent delta x page_count
|
||||||
|
source: text('source').notNull().default('reader'),
|
||||||
|
note: text('note'),
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
index('sessions_work_id_idx').on(t.workId),
|
||||||
|
index('sessions_started_at_idx').on(t.startedAt),
|
||||||
|
check(
|
||||||
|
'sessions_percent_check',
|
||||||
|
sql`(${t.startPercent} is null or ${t.startPercent} between 0 and 100) and (${t.endPercent} is null or ${t.endPercent} between 0 and 100)`
|
||||||
|
),
|
||||||
|
check('sessions_end_check', sql`${t.endedAt} is null or ${t.endedAt} >= ${t.startedAt}`)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
export type BookWork = typeof bookWorks.$inferSelect;
|
||||||
|
export type NewBookWork = typeof bookWorks.$inferInsert;
|
||||||
|
export type BookFile = typeof bookFiles.$inferSelect;
|
||||||
|
export type BookShelf = typeof bookShelves.$inferSelect;
|
||||||
|
export type BookProgress = typeof bookProgress.$inferSelect;
|
||||||
|
export type BookNote = typeof bookNotes.$inferSelect;
|
||||||
|
export type BookSession = typeof bookSessions.$inferSelect;
|
||||||
|
export type BookReadingStatus = (typeof bookReadingStatus.enumValues)[number];
|
||||||
@@ -13,3 +13,5 @@ export * from './money-tracker';
|
|||||||
export * from './cs-prices';
|
export * from './cs-prices';
|
||||||
export * from './house-tracker';
|
export * from './house-tracker';
|
||||||
export * from './health';
|
export * from './health';
|
||||||
|
export * from './books';
|
||||||
|
export * from './tasks';
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import {
|
import {
|
||||||
|
bigint,
|
||||||
boolean,
|
boolean,
|
||||||
check,
|
check,
|
||||||
index,
|
index,
|
||||||
@@ -88,5 +89,29 @@ export const metaColumns = metaSchema.table(
|
|||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/** Audit trail of every write made through the database browser (/lists/db). */
|
||||||
|
export const metaEditLog = metaSchema.table(
|
||||||
|
'edit_log',
|
||||||
|
{
|
||||||
|
id: bigint('id', { mode: 'number' }).primaryKey().generatedAlwaysAsIdentity(),
|
||||||
|
at: ts('at').notNull().defaultNow(),
|
||||||
|
userId: text('user_id'),
|
||||||
|
userEmail: text('user_email'),
|
||||||
|
schemaName: text('schema_name').notNull(),
|
||||||
|
tableName: text('table_name').notNull(),
|
||||||
|
action: text('action').notNull(),
|
||||||
|
pk: jsonb('pk'),
|
||||||
|
before: jsonb('before'),
|
||||||
|
after: jsonb('after')
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
index('edit_log_table_at_index').on(t.schemaName, t.tableName, t.at),
|
||||||
|
check(
|
||||||
|
'edit_log_action_check',
|
||||||
|
sql`action IN ('insert','update','delete','import','add_column')`
|
||||||
|
)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
export type MetaTable = typeof metaTables.$inferSelect;
|
export type MetaTable = typeof metaTables.$inferSelect;
|
||||||
export type MetaColumn = typeof metaColumns.$inferSelect;
|
export type MetaColumn = typeof metaColumns.$inferSelect;
|
||||||
|
|||||||
@@ -1,7 +1,17 @@
|
|||||||
import { boolean, check, date, index, integer, numeric, text, unique } from 'drizzle-orm/pg-core';
|
import {
|
||||||
|
boolean,
|
||||||
|
check,
|
||||||
|
date,
|
||||||
|
index,
|
||||||
|
integer,
|
||||||
|
jsonb,
|
||||||
|
numeric,
|
||||||
|
text,
|
||||||
|
unique
|
||||||
|
} from 'drizzle-orm/pg-core';
|
||||||
import { sql } from 'drizzle-orm';
|
import { sql } from 'drizzle-orm';
|
||||||
import { moneySchema } from './schemas';
|
import { moneySchema } from './schemas';
|
||||||
import { id, timestamps } from './helpers';
|
import { id, timestamps, timestamptz } from './helpers';
|
||||||
import { user } from './auth';
|
import { user } from './auth';
|
||||||
|
|
||||||
export const moneyAccounts = moneySchema.table(
|
export const moneyAccounts = moneySchema.table(
|
||||||
@@ -14,13 +24,30 @@ export const moneyAccounts = moneySchema.table(
|
|||||||
name: text('name').notNull(),
|
name: text('name').notNull(),
|
||||||
openingBalanceCents: integer('opening_balance_cents').notNull().default(0),
|
openingBalanceCents: integer('opening_balance_cents').notNull().default(0),
|
||||||
kind: text('kind').notNull().default('spendable'),
|
kind: text('kind').notNull().default('spendable'),
|
||||||
|
// False for money tracked but not owned (e.g. a savings account parents controlled):
|
||||||
|
// still listed with its own balance, left out of net worth and the total wealth chart.
|
||||||
|
includeInTotal: boolean('include_in_total').notNull().default(true),
|
||||||
color: text('color'),
|
color: text('color'),
|
||||||
|
// Household scope: Stan's own money vs. the joint account with Juhee.
|
||||||
|
scope: text('scope').notNull().default('personal'),
|
||||||
|
owner: text('owner'),
|
||||||
|
iban: text('iban'),
|
||||||
|
// Bank-side account identifier (Enable Banking account uid) for synced accounts.
|
||||||
|
externalUid: text('external_uid'),
|
||||||
|
provider: text('provider').notNull().default('manual'),
|
||||||
|
// Last balance reported by the bank (synced accounts); the source of truth for the total.
|
||||||
|
bankBalanceCents: integer('bank_balance_cents'),
|
||||||
|
bankBalanceType: text('bank_balance_type'),
|
||||||
|
bankBalanceAt: timestamptz('bank_balance_at'),
|
||||||
...timestamps
|
...timestamps
|
||||||
},
|
},
|
||||||
(table) => [
|
(table) => [
|
||||||
unique().on(table.userId, table.name),
|
unique().on(table.userId, table.name),
|
||||||
|
unique('accounts_external_uid_key').on(table.externalUid),
|
||||||
index().on(table.userId),
|
index().on(table.userId),
|
||||||
check('accounts_kind_check', sql`kind IN ('spendable','investment')`)
|
check('accounts_kind_check', sql`kind IN ('spendable','investment')`),
|
||||||
|
check('accounts_scope_check', sql`scope IN ('personal','joint')`),
|
||||||
|
check('accounts_provider_check', sql`provider IN ('manual','enablebanking')`)
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -37,9 +64,53 @@ export const moneyTransactions = moneySchema.table(
|
|||||||
date: date('date', { mode: 'string' }).notNull(),
|
date: date('date', { mode: 'string' }).notNull(),
|
||||||
description: text('description').notNull(),
|
description: text('description').notNull(),
|
||||||
amountCents: integer('amount_cents').notNull(),
|
amountCents: integer('amount_cents').notNull(),
|
||||||
|
// Bank transaction id (Enable Banking entry_reference / transaction_id, or a
|
||||||
|
// content hash for CSV imports). NULLs are distinct, so manual rows never clash.
|
||||||
|
externalId: text('external_id'),
|
||||||
|
source: text('source').notNull().default('manual'),
|
||||||
|
bookingStatus: text('booking_status'),
|
||||||
|
counterparty: text('counterparty'),
|
||||||
|
category: text('category'),
|
||||||
|
raw: jsonb('raw'),
|
||||||
|
valueDate: date('value_date', { mode: 'string' }),
|
||||||
...timestamps
|
...timestamps
|
||||||
},
|
},
|
||||||
(table) => [index().on(table.userId, table.date, table.id), index().on(table.accountId)]
|
(table) => [
|
||||||
|
index().on(table.userId, table.date, table.id),
|
||||||
|
index().on(table.accountId),
|
||||||
|
unique('transactions_account_external_id_key').on(table.accountId, table.externalId),
|
||||||
|
check('transactions_source_check', sql`source IN ('manual','csv','enablebanking')`)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Open-banking consent sessions (Enable Banking). One row per authorised session;
|
||||||
|
// `accounts` keeps the bank-side account list returned when the session was created.
|
||||||
|
export const moneyBankSessions = moneySchema.table(
|
||||||
|
'bank_sessions',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
userId: text('user_id')
|
||||||
|
.notNull()
|
||||||
|
.references(() => user.id, { onDelete: 'cascade' }),
|
||||||
|
provider: text('provider').notNull().default('enablebanking'),
|
||||||
|
sessionId: text('session_id').notNull(),
|
||||||
|
aspspName: text('aspsp_name').notNull(),
|
||||||
|
aspspCountry: text('aspsp_country').notNull(),
|
||||||
|
psuType: text('psu_type'),
|
||||||
|
status: text('status').notNull().default('authorized'),
|
||||||
|
validUntil: timestamptz('valid_until'),
|
||||||
|
accounts: jsonb('accounts')
|
||||||
|
.notNull()
|
||||||
|
.default(sql`'[]'::jsonb`),
|
||||||
|
lastSyncedAt: timestamptz('last_synced_at'),
|
||||||
|
lastError: text('last_error'),
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(table) => [
|
||||||
|
unique('bank_sessions_session_id_key').on(table.sessionId),
|
||||||
|
index().on(table.userId),
|
||||||
|
check('bank_sessions_status_check', sql`status IN ('authorized','expired','revoked','error')`)
|
||||||
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
export const moneyRecurring = moneySchema.table(
|
export const moneyRecurring = moneySchema.table(
|
||||||
|
|||||||
@@ -13,3 +13,5 @@ export const animaldexSchema = pgSchema('animaldex');
|
|||||||
export const siteSchema = pgSchema('site');
|
export const siteSchema = pgSchema('site');
|
||||||
export const metaSchema = pgSchema('meta');
|
export const metaSchema = pgSchema('meta');
|
||||||
export const healthSchema = pgSchema('health');
|
export const healthSchema = pgSchema('health');
|
||||||
|
export const booksSchema = pgSchema('books');
|
||||||
|
export const tasksSchema = pgSchema('tasks');
|
||||||
|
|||||||
145
src/lib/server/db/schema/tasks.ts
Normal file
145
src/lib/server/db/schema/tasks.ts
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
import { sql } from 'drizzle-orm';
|
||||||
|
import {
|
||||||
|
type AnyPgColumn,
|
||||||
|
boolean,
|
||||||
|
check,
|
||||||
|
date,
|
||||||
|
index,
|
||||||
|
integer,
|
||||||
|
numeric,
|
||||||
|
smallint,
|
||||||
|
text,
|
||||||
|
time,
|
||||||
|
uniqueIndex
|
||||||
|
} from 'drizzle-orm/pg-core';
|
||||||
|
import { tasksSchema } from './schemas';
|
||||||
|
import { id, timestamps, timestamptz } from './helpers';
|
||||||
|
|
||||||
|
// Private task + goal tracker (owner only, see requireOwner); meant to replace TickTick.
|
||||||
|
// No user_id: there is exactly one owner, and the schema is granted to personal_app only.
|
||||||
|
// Statuses are text + CHECK (not enums) so new ones are a one-line migration.
|
||||||
|
// Due dates are Europe/Amsterdam wall-clock (date + optional time), like TickTick's
|
||||||
|
// all-day vs timed tasks; reminders are real moments (timestamptz).
|
||||||
|
|
||||||
|
export const TASK_STATUSES = ['todo', 'in_progress', 'done', 'dropped'] as const;
|
||||||
|
export type TaskStatus = (typeof TASK_STATUSES)[number];
|
||||||
|
export const GOAL_STATUSES = ['active', 'achieved', 'dropped'] as const;
|
||||||
|
export type GoalStatus = (typeof GOAL_STATUSES)[number];
|
||||||
|
|
||||||
|
/** Areas group tasks (TickTick "lists"); `slug` is the stable key for imports. */
|
||||||
|
export const todoAreas = tasksSchema.table(
|
||||||
|
'areas',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
slug: text('slug').notNull(),
|
||||||
|
name: text('name').notNull(),
|
||||||
|
description: text('description'),
|
||||||
|
color: text('color'), // CSS colour, optional
|
||||||
|
sortOrder: integer('sort_order').notNull().default(0),
|
||||||
|
archivedAt: timestamptz('archived_at'),
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
uniqueIndex('areas_slug_key').on(t.slug),
|
||||||
|
check('areas_slug_check', sql`${t.slug} ~ '^[a-z0-9][a-z0-9-]{0,62}$'`),
|
||||||
|
check('areas_name_check', sql`length(btrim(${t.name})) > 0`)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
/** Long-term goals, optionally with a target date and a numeric metric. */
|
||||||
|
export const todoGoals = tasksSchema.table(
|
||||||
|
'goals',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
title: text('title').notNull(),
|
||||||
|
notes: text('notes'),
|
||||||
|
status: text('status').$type<GoalStatus>().notNull().default('active'),
|
||||||
|
areaId: integer('area_id').references(() => todoAreas.id, { onDelete: 'set null' }),
|
||||||
|
targetDate: date('target_date', { mode: 'string' }),
|
||||||
|
metric: text('metric'), // what is measured, e.g. "net income per month"
|
||||||
|
metricUnit: text('metric_unit'), // e.g. "EUR"
|
||||||
|
metricTarget: numeric('metric_target', { precision: 14, scale: 2, mode: 'number' }),
|
||||||
|
metricCurrent: numeric('metric_current', { precision: 14, scale: 2, mode: 'number' }),
|
||||||
|
flagged: boolean('flagged').notNull().default(false),
|
||||||
|
sortOrder: integer('sort_order').notNull().default(0),
|
||||||
|
source: text('source'), // e.g. 'daisy-import', 'manual', 'ticktick'
|
||||||
|
achievedAt: timestamptz('achieved_at'),
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
uniqueIndex('goals_title_key').on(sql`lower(${t.title})`),
|
||||||
|
check('goals_title_check', sql`length(btrim(${t.title})) > 0`),
|
||||||
|
check('goals_status_check', sql`${t.status} IN ('active','achieved','dropped')`)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
export const todoTasks = tasksSchema.table(
|
||||||
|
'tasks',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
title: text('title').notNull(),
|
||||||
|
notes: text('notes'),
|
||||||
|
status: text('status').$type<TaskStatus>().notNull().default('todo'),
|
||||||
|
priority: smallint('priority').notNull().default(0), // 0 none, 1 low, 2 medium, 3 high
|
||||||
|
flagged: boolean('flagged').notNull().default(false), // Stan's "(!)"
|
||||||
|
dueDate: date('due_date', { mode: 'string' }),
|
||||||
|
dueTime: time('due_time'), // optional; Europe/Amsterdam wall clock
|
||||||
|
reminderAt: timestamptz('reminder_at'),
|
||||||
|
recurrence: text('recurrence'), // RFC 5545 RRULE, e.g. FREQ=WEEKLY;BYDAY=MO (not acted on yet)
|
||||||
|
areaId: integer('area_id').references(() => todoAreas.id, { onDelete: 'set null' }),
|
||||||
|
goalId: integer('goal_id').references(() => todoGoals.id, { onDelete: 'set null' }),
|
||||||
|
parentId: integer('parent_id').references((): AnyPgColumn => todoTasks.id, {
|
||||||
|
onDelete: 'cascade'
|
||||||
|
}),
|
||||||
|
tags: text('tags').array().notNull().default([]),
|
||||||
|
sortOrder: integer('sort_order').notNull().default(0),
|
||||||
|
source: text('source'), // e.g. 'daisy-import', 'manual', 'ticktick'
|
||||||
|
externalId: text('external_id'), // id in the source system, for future syncs
|
||||||
|
startAt: timestamptz('start_at'), // planned start (TickTick startDate)
|
||||||
|
timeZone: text('time_zone'), // zone the task was planned in (TickTick); due_time stays Amsterdam
|
||||||
|
startedAt: timestamptz('started_at'),
|
||||||
|
completedAt: timestamptz('completed_at'),
|
||||||
|
...timestamps
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
index('tasks_area_idx').on(t.areaId),
|
||||||
|
index('tasks_goal_idx').on(t.goalId),
|
||||||
|
index('tasks_parent_idx').on(t.parentId),
|
||||||
|
index('tasks_status_due_idx').on(t.status, t.dueDate),
|
||||||
|
index('tasks_tags_idx').using('gin', t.tags),
|
||||||
|
uniqueIndex('tasks_source_external_key')
|
||||||
|
.on(t.source, t.externalId)
|
||||||
|
.where(sql`${t.externalId} IS NOT NULL`),
|
||||||
|
check('tasks_title_check', sql`length(btrim(${t.title})) > 0`),
|
||||||
|
check('tasks_status_check', sql`${t.status} IN ('todo','in_progress','done','dropped')`),
|
||||||
|
check('tasks_priority_check', sql`${t.priority} BETWEEN 0 AND 3`),
|
||||||
|
check('tasks_parent_check', sql`${t.parentId} IS NULL OR ${t.parentId} <> ${t.id}`),
|
||||||
|
check('tasks_time_check', sql`${t.dueTime} IS NULL OR ${t.dueDate} IS NOT NULL`)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
/** Append-only history per task (status changes, notes, progress updates). */
|
||||||
|
export const todoActivity = tasksSchema.table(
|
||||||
|
'activity',
|
||||||
|
{
|
||||||
|
id: id(),
|
||||||
|
taskId: integer('task_id').references(() => todoTasks.id, { onDelete: 'cascade' }),
|
||||||
|
goalId: integer('goal_id').references(() => todoGoals.id, { onDelete: 'cascade' }),
|
||||||
|
kind: text('kind').notNull(), // created | status | edit | note | progress
|
||||||
|
fromStatus: text('from_status'),
|
||||||
|
toStatus: text('to_status'),
|
||||||
|
note: text('note'),
|
||||||
|
value: numeric('value', { precision: 14, scale: 2, mode: 'number' }), // goal metric snapshot
|
||||||
|
createdAt: timestamptz('created_at').notNull().defaultNow()
|
||||||
|
},
|
||||||
|
(t) => [
|
||||||
|
index('activity_task_idx').on(t.taskId, t.createdAt),
|
||||||
|
index('activity_goal_idx').on(t.goalId, t.createdAt),
|
||||||
|
check('activity_target_check', sql`${t.taskId} IS NOT NULL OR ${t.goalId} IS NOT NULL`),
|
||||||
|
check('activity_kind_check', sql`${t.kind} IN ('created','status','edit','note','progress')`)
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
export type TodoArea = typeof todoAreas.$inferSelect;
|
||||||
|
export type TodoGoal = typeof todoGoals.$inferSelect;
|
||||||
|
export type TodoTask = typeof todoTasks.$inferSelect;
|
||||||
266
src/lib/server/enable-banking.ts
Normal file
266
src/lib/server/enable-banking.ts
Normal file
@@ -0,0 +1,266 @@
|
|||||||
|
// Minimal Enable Banking (https://enablebanking.com/docs/api/reference/) client for
|
||||||
|
// account information (AIS): consent (POST /auth + POST /sessions), balances and
|
||||||
|
// transactions. Every request carries a short-lived RS256 JWT signed with the
|
||||||
|
// application's private key; the key never leaves this process.
|
||||||
|
//
|
||||||
|
// Framework-free on purpose: used by the SvelteKit callback route and by the
|
||||||
|
// bun scripts in scripts/ (bank-link.ts, bank-sync.ts), which don't load $app/env.
|
||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { createPrivateKey, randomUUID, type KeyObject } from 'node:crypto';
|
||||||
|
import { SignJWT } from 'jose';
|
||||||
|
|
||||||
|
export const ENABLE_BANKING_API = 'https://api.enablebanking.com';
|
||||||
|
|
||||||
|
export type EnableBankingConfig = {
|
||||||
|
appId: string;
|
||||||
|
/** PEM private key (PKCS#1 or PKCS#8). Prefer keyPath so the key isn't in env. */
|
||||||
|
privateKeyPem?: string;
|
||||||
|
keyPath?: string;
|
||||||
|
baseUrl?: string;
|
||||||
|
fetch?: typeof fetch;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type Amount = { currency: string; amount: string };
|
||||||
|
|
||||||
|
export type BankAccount = {
|
||||||
|
uid?: string;
|
||||||
|
account_id?: { iban?: string; other?: { identification: string; scheme_name: string } };
|
||||||
|
name?: string;
|
||||||
|
details?: string;
|
||||||
|
product?: string;
|
||||||
|
currency?: string;
|
||||||
|
cash_account_type?: string;
|
||||||
|
usage?: string;
|
||||||
|
identification_hash?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type BankTransaction = {
|
||||||
|
entry_reference?: string | null;
|
||||||
|
transaction_id?: string | null;
|
||||||
|
transaction_amount: Amount;
|
||||||
|
credit_debit_indicator: 'CRDT' | 'DBIT';
|
||||||
|
status?: string;
|
||||||
|
booking_date?: string | null;
|
||||||
|
value_date?: string | null;
|
||||||
|
transaction_date?: string | null;
|
||||||
|
creditor?: { name?: string | null } | null;
|
||||||
|
debtor?: { name?: string | null } | null;
|
||||||
|
creditor_account?: { iban?: string | null } | null;
|
||||||
|
debtor_account?: { iban?: string | null } | null;
|
||||||
|
remittance_information?: string[] | null;
|
||||||
|
bank_transaction_code?: { description?: string | null } | null;
|
||||||
|
merchant_category_code?: string | null;
|
||||||
|
note?: string | null;
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type Balance = {
|
||||||
|
name?: string;
|
||||||
|
balance_amount: Amount;
|
||||||
|
balance_type: string;
|
||||||
|
reference_date?: string | null;
|
||||||
|
last_change_date_time?: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type StartAuthResponse = { url: string; authorization_id: string; psu_id_hash?: string };
|
||||||
|
export type SessionResponse = {
|
||||||
|
session_id: string;
|
||||||
|
accounts: BankAccount[];
|
||||||
|
aspsp: { name: string; country: string };
|
||||||
|
psu_type: string;
|
||||||
|
access: { valid_until: string };
|
||||||
|
};
|
||||||
|
|
||||||
|
export class EnableBankingError extends Error {
|
||||||
|
constructor(
|
||||||
|
readonly status: number,
|
||||||
|
readonly code: string | undefined,
|
||||||
|
message: string
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = 'EnableBankingError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reads ENABLEBANKING_APP_ID and ENABLEBANKING_KEY_PATH / ENABLEBANKING_PRIVATE_KEY. */
|
||||||
|
export function configFromEnv(env: Record<string, string | undefined> = process.env) {
|
||||||
|
const appId = env.ENABLEBANKING_APP_ID?.trim();
|
||||||
|
if (!appId) throw new Error('ENABLEBANKING_APP_ID is not set');
|
||||||
|
const keyPath = env.ENABLEBANKING_KEY_PATH?.trim();
|
||||||
|
const privateKeyPem = env.ENABLEBANKING_PRIVATE_KEY?.replace(/\\n/g, '\n');
|
||||||
|
if (!keyPath && !privateKeyPem)
|
||||||
|
throw new Error('Set ENABLEBANKING_KEY_PATH (preferred) or ENABLEBANKING_PRIVATE_KEY');
|
||||||
|
return { appId, keyPath, privateKeyPem } satisfies EnableBankingConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createEnableBankingClient(config: EnableBankingConfig) {
|
||||||
|
const baseUrl = config.baseUrl ?? ENABLE_BANKING_API;
|
||||||
|
const doFetch = config.fetch ?? fetch;
|
||||||
|
let key: KeyObject | undefined;
|
||||||
|
let cached: { token: string; expires: number } | undefined;
|
||||||
|
|
||||||
|
function privateKey() {
|
||||||
|
if (!key) {
|
||||||
|
const pem =
|
||||||
|
config.privateKeyPem ?? (config.keyPath ? readFileSync(config.keyPath, 'utf8') : '');
|
||||||
|
if (!pem) throw new Error('No Enable Banking private key configured');
|
||||||
|
key = createPrivateKey(pem);
|
||||||
|
}
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** RS256 JWT, valid for one hour (the API accepts at most 24 h). */
|
||||||
|
async function token() {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
if (cached && cached.expires - 60 > now) return cached.token;
|
||||||
|
const jwt = await new SignJWT({})
|
||||||
|
.setProtectedHeader({ typ: 'JWT', alg: 'RS256', kid: config.appId })
|
||||||
|
.setIssuer('enablebanking.com')
|
||||||
|
.setAudience('api.enablebanking.com')
|
||||||
|
.setIssuedAt(now)
|
||||||
|
.setExpirationTime(now + 3600)
|
||||||
|
.sign(privateKey());
|
||||||
|
cached = { token: jwt, expires: now + 3600 };
|
||||||
|
return jwt;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function request<T>(method: string, path: string, body?: unknown): Promise<T> {
|
||||||
|
const response = await doFetch(baseUrl + path, {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${await token()}`,
|
||||||
|
Accept: 'application/json',
|
||||||
|
...(body === undefined ? {} : { 'Content-Type': 'application/json' })
|
||||||
|
},
|
||||||
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
|
signal: AbortSignal.timeout(60_000)
|
||||||
|
});
|
||||||
|
const text = await response.text();
|
||||||
|
let data: unknown;
|
||||||
|
try {
|
||||||
|
data = text ? JSON.parse(text) : undefined;
|
||||||
|
} catch {
|
||||||
|
data = text;
|
||||||
|
}
|
||||||
|
if (!response.ok) {
|
||||||
|
const detail = (data ?? {}) as { message?: string; error?: string; detail?: unknown };
|
||||||
|
throw new EnableBankingError(
|
||||||
|
response.status,
|
||||||
|
detail.error,
|
||||||
|
`Enable Banking ${method} ${path.split('?')[0]} failed (${response.status}${detail.error ? ' ' + detail.error : ''}): ${detail.message ?? text.slice(0, 300)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return data as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
token,
|
||||||
|
application: () =>
|
||||||
|
request<{
|
||||||
|
name: string;
|
||||||
|
kid: string;
|
||||||
|
environment: string;
|
||||||
|
active: boolean;
|
||||||
|
redirect_urls: string[];
|
||||||
|
countries: string[];
|
||||||
|
services: string[];
|
||||||
|
}>('GET', '/application'),
|
||||||
|
aspsps: (country: string, psuType = 'personal') =>
|
||||||
|
request<{
|
||||||
|
aspsps: {
|
||||||
|
name: string;
|
||||||
|
country: string;
|
||||||
|
maximum_consent_validity: number;
|
||||||
|
psu_types: string[];
|
||||||
|
bic?: string;
|
||||||
|
beta?: boolean;
|
||||||
|
required_psu_headers?: string[];
|
||||||
|
}[];
|
||||||
|
}>('GET', `/aspsps?${new URLSearchParams({ country, psu_type: psuType, service: 'AIS' })}`),
|
||||||
|
startAuth: (input: {
|
||||||
|
aspsp: { name: string; country: string };
|
||||||
|
redirectUrl: string;
|
||||||
|
validUntil: Date;
|
||||||
|
psuType?: 'personal' | 'business';
|
||||||
|
state?: string;
|
||||||
|
language?: string;
|
||||||
|
}) =>
|
||||||
|
request<StartAuthResponse>('POST', '/auth', {
|
||||||
|
access: { valid_until: input.validUntil.toISOString() },
|
||||||
|
aspsp: input.aspsp,
|
||||||
|
state: input.state ?? randomUUID(),
|
||||||
|
redirect_url: input.redirectUrl,
|
||||||
|
psu_type: input.psuType ?? 'personal',
|
||||||
|
...(input.language ? { language: input.language } : {})
|
||||||
|
}),
|
||||||
|
createSession: (code: string) => request<SessionResponse>('POST', '/sessions', { code }),
|
||||||
|
getSession: (sessionId: string) =>
|
||||||
|
request<{
|
||||||
|
status: string;
|
||||||
|
accounts: string[];
|
||||||
|
access: { valid_until: string };
|
||||||
|
aspsp: { name: string; country: string };
|
||||||
|
}>('GET', `/sessions/${encodeURIComponent(sessionId)}`),
|
||||||
|
deleteSession: (sessionId: string) =>
|
||||||
|
request<{ message: string }>('DELETE', `/sessions/${encodeURIComponent(sessionId)}`),
|
||||||
|
accountDetails: (uid: string) =>
|
||||||
|
request<BankAccount>('GET', `/accounts/${encodeURIComponent(uid)}/details`),
|
||||||
|
balances: (uid: string) =>
|
||||||
|
request<{ balances: Balance[] }>('GET', `/accounts/${encodeURIComponent(uid)}/balances`),
|
||||||
|
/** All transactions from dateFrom (inclusive), following continuation_key pages. */
|
||||||
|
async transactions(
|
||||||
|
uid: string,
|
||||||
|
options: {
|
||||||
|
dateFrom?: string;
|
||||||
|
dateTo?: string;
|
||||||
|
status?: 'BOOK' | 'PDNG';
|
||||||
|
/** 'longest' asks Enable Banking to find the longest period the bank serves. */
|
||||||
|
strategy?: 'default' | 'longest';
|
||||||
|
maxPages?: number;
|
||||||
|
} = {}
|
||||||
|
) {
|
||||||
|
const all: BankTransaction[] = [];
|
||||||
|
let continuationKey: string | undefined;
|
||||||
|
for (let page = 0; page < (options.maxPages ?? 200); page++) {
|
||||||
|
const query = new URLSearchParams();
|
||||||
|
if (options.dateFrom) query.set('date_from', options.dateFrom);
|
||||||
|
if (options.dateTo) query.set('date_to', options.dateTo);
|
||||||
|
if (options.status) query.set('transaction_status', options.status);
|
||||||
|
if (options.strategy) query.set('strategy', options.strategy);
|
||||||
|
if (continuationKey) query.set('continuation_key', continuationKey);
|
||||||
|
const result = await request<{
|
||||||
|
transactions: BankTransaction[];
|
||||||
|
continuation_key?: string | null;
|
||||||
|
}>(
|
||||||
|
'GET',
|
||||||
|
`/accounts/${encodeURIComponent(uid)}/transactions${query.size ? '?' + query : ''}`
|
||||||
|
);
|
||||||
|
all.push(...(result.transactions ?? []));
|
||||||
|
continuationKey = result.continuation_key ?? undefined;
|
||||||
|
if (!continuationKey) break;
|
||||||
|
}
|
||||||
|
return all;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export type EnableBankingClient = ReturnType<typeof createEnableBankingClient>;
|
||||||
|
|
||||||
|
/** Extracts the authorisation code from a pasted redirect URL, query string or bare code. */
|
||||||
|
export function parseAuthCode(input: string): { code?: string; error?: string; state?: string } {
|
||||||
|
const value = input.trim();
|
||||||
|
if (!value) return {};
|
||||||
|
if (/^[\w-]+$/.test(value) && !value.includes('=')) return { code: value };
|
||||||
|
let params: URLSearchParams;
|
||||||
|
try {
|
||||||
|
params = new URL(value).searchParams;
|
||||||
|
} catch {
|
||||||
|
params = new URLSearchParams(value.replace(/^.*\?/, ''));
|
||||||
|
}
|
||||||
|
const error = params.get('error');
|
||||||
|
return {
|
||||||
|
code: params.get('code') ?? undefined,
|
||||||
|
state: params.get('state') ?? undefined,
|
||||||
|
error: error ? `${error}: ${params.get('error_description') ?? ''}`.trim() : undefined
|
||||||
|
};
|
||||||
|
}
|
||||||
121
src/lib/server/energy.ts
Normal file
121
src/lib/server/energy.ts
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
import { db } from '#lib/server/db/index.js';
|
||||||
|
import { sql } from 'drizzle-orm';
|
||||||
|
import type { EnergyRow } from '#lib/energy/data.js';
|
||||||
|
|
||||||
|
// Read-only queries over the ned-collector tables in the `energy` schema. The app role
|
||||||
|
// already has USAGE + SELECT there; nothing on the site writes to these tables.
|
||||||
|
|
||||||
|
type Raw = Record<string, unknown>;
|
||||||
|
const num = (value: unknown) => (value === null || value === undefined ? null : Number(value));
|
||||||
|
const time = (value: unknown) => new Date(value as string | Date).getTime();
|
||||||
|
const rows = (result: unknown) => result as Raw[];
|
||||||
|
|
||||||
|
function energyRows(result: unknown): EnergyRow[] {
|
||||||
|
return rows(result).map((row) => ({
|
||||||
|
t: time(row.t),
|
||||||
|
type: String(row.type),
|
||||||
|
value: Number(row.value),
|
||||||
|
pct: num(row.pct),
|
||||||
|
ef: num(row.ef),
|
||||||
|
...(row.hours === undefined ? {} : { hours: Number(row.hours) }),
|
||||||
|
...(row.expected === undefined ? {} : { expected: Number(row.expected) })
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadEnergy() {
|
||||||
|
const [hourly, daily, forecast, monthly, yearly, gasNl, gasSites, runs, coverage] =
|
||||||
|
await Promise.all([
|
||||||
|
// Last 7 days of hourly measurements, counted back from the newest hour NED published.
|
||||||
|
db.execute(sql`WITH latest AS (
|
||||||
|
SELECT max(valid_from) AS t FROM energy.ned_utilization
|
||||||
|
WHERE granularity = 'Hour' AND classification = 'Current')
|
||||||
|
SELECT valid_from AS t, type, volume_kwh::float8 / 1000 AS value,
|
||||||
|
percentage::float8 AS pct, emission_factor::float8 AS ef
|
||||||
|
FROM energy.ned_utilization, latest
|
||||||
|
WHERE granularity = 'Hour' AND classification = 'Current'
|
||||||
|
AND valid_from > latest.t - interval '7 days'
|
||||||
|
ORDER BY 1, 2`),
|
||||||
|
// Daily totals (Amsterdam days) from hourly data, last 31 days. `expected` is 23 or 25
|
||||||
|
// on DST change days so partial days can be flagged.
|
||||||
|
db.execute(sql`WITH latest AS (
|
||||||
|
SELECT max(valid_from) AS t FROM energy.ned_utilization
|
||||||
|
WHERE granularity = 'Hour' AND classification = 'Current'),
|
||||||
|
days AS (
|
||||||
|
SELECT date_trunc('day', valid_from AT TIME ZONE 'Europe/Amsterdam') AS day, type,
|
||||||
|
sum(volume_kwh) AS kwh, sum(emission_kg) AS kg, count(*)::int AS hours
|
||||||
|
FROM energy.ned_utilization, latest
|
||||||
|
WHERE granularity = 'Hour' AND classification = 'Current'
|
||||||
|
AND valid_from > latest.t - interval '31 days'
|
||||||
|
GROUP BY 1, 2)
|
||||||
|
SELECT day AT TIME ZONE 'Europe/Amsterdam' AS t, type, kwh::float8 / 1000 AS value, hours,
|
||||||
|
(extract(epoch FROM (day + interval '1 day') AT TIME ZONE 'Europe/Amsterdam'
|
||||||
|
- day AT TIME ZONE 'Europe/Amsterdam') / 3600)::int AS expected,
|
||||||
|
(kg / nullif(kwh, 0))::float8 AS ef
|
||||||
|
FROM days ORDER BY 1, 2`),
|
||||||
|
db.execute(sql`WITH latest AS (
|
||||||
|
SELECT max(valid_from) AS t FROM energy.ned_utilization
|
||||||
|
WHERE granularity = 'Hour' AND classification = 'Current')
|
||||||
|
SELECT valid_from AS t, type, volume_kwh::float8 / 1000 AS value
|
||||||
|
FROM energy.ned_utilization, latest
|
||||||
|
WHERE granularity = 'Hour' AND classification = 'Forecast'
|
||||||
|
AND valid_from > latest.t AND valid_from <= latest.t + interval '8 days'
|
||||||
|
AND type IN ('Wind', 'WindOffshoreC', 'Solar', 'ElectricityLoad', 'ElectricityMix')
|
||||||
|
ORDER BY 1, 2`),
|
||||||
|
db.execute(sql`SELECT valid_from AS t, type, volume_kwh::float8 / 1e6 AS value,
|
||||||
|
percentage::float8 AS pct, emission_factor::float8 AS ef
|
||||||
|
FROM energy.ned_utilization
|
||||||
|
WHERE granularity = 'Month' AND classification = 'Current'
|
||||||
|
ORDER BY 1, 2`),
|
||||||
|
db.execute(sql`SELECT valid_from AS t, type, volume_kwh::float8 / 1e6 AS value,
|
||||||
|
emission_factor::float8 AS ef
|
||||||
|
FROM energy.ned_utilization
|
||||||
|
WHERE granularity = 'Year' AND classification = 'Current'
|
||||||
|
ORDER BY 1, 2`),
|
||||||
|
db.execute(sql`SELECT gas_day::text AS day, fill_pct::float8 AS fill, volume_twh::float8 AS twh
|
||||||
|
FROM energy.gas_storage_daily WHERE site = 'NL' ORDER BY gas_day`),
|
||||||
|
db.execute(sql`SELECT site, gas_day::text AS day, fill_pct::float8 AS fill,
|
||||||
|
volume_twh::float8 AS twh
|
||||||
|
FROM energy.gas_storage_latest WHERE site <> 'NL' ORDER BY volume_twh DESC`),
|
||||||
|
db.execute(sql`SELECT
|
||||||
|
(SELECT max(started_at) FROM energy.collector_runs) AS last_run,
|
||||||
|
(SELECT max(finished_at) FROM energy.collector_runs WHERE ok) AS last_ok,
|
||||||
|
(SELECT count(*)::int FROM energy.collector_runs
|
||||||
|
WHERE NOT ok AND started_at > now() - interval '7 days') AS failed_week`),
|
||||||
|
db.execute(sql`SELECT min(valid_from) AS first_hour, max(valid_from) AS last_hour,
|
||||||
|
count(DISTINCT valid_from)::int AS hours
|
||||||
|
FROM energy.ned_utilization WHERE granularity = 'Hour' AND classification = 'Current'`)
|
||||||
|
]);
|
||||||
|
|
||||||
|
const run = rows(runs)[0] ?? {};
|
||||||
|
const cover = rows(coverage)[0] ?? {};
|
||||||
|
return {
|
||||||
|
hourly: energyRows(hourly),
|
||||||
|
daily: energyRows(daily),
|
||||||
|
forecast: energyRows(forecast),
|
||||||
|
monthly: energyRows(monthly),
|
||||||
|
yearly: energyRows(yearly),
|
||||||
|
gas: rows(gasNl).map((row) => ({
|
||||||
|
day: String(row.day),
|
||||||
|
fill: Number(row.fill),
|
||||||
|
twh: Number(row.twh)
|
||||||
|
})),
|
||||||
|
gasSites: rows(gasSites).map((row) => ({
|
||||||
|
site: String(row.site),
|
||||||
|
day: String(row.day),
|
||||||
|
fill: Number(row.fill),
|
||||||
|
twh: Number(row.twh)
|
||||||
|
})),
|
||||||
|
collector: {
|
||||||
|
lastRun: run.last_run ? time(run.last_run) : null,
|
||||||
|
lastOk: run.last_ok ? time(run.last_ok) : null,
|
||||||
|
failedWeek: Number(run.failed_week ?? 0)
|
||||||
|
},
|
||||||
|
coverage: {
|
||||||
|
firstHour: cover.first_hour ? time(cover.first_hour) : null,
|
||||||
|
lastHour: cover.last_hour ? time(cover.last_hour) : null,
|
||||||
|
hours: Number(cover.hours ?? 0)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export type EnergyData = Awaited<ReturnType<typeof loadEnergy>>;
|
||||||
@@ -44,6 +44,7 @@ export type ListSummary = {
|
|||||||
checked: number;
|
checked: number;
|
||||||
};
|
};
|
||||||
export type ListView = Omit<ListSummary, 'total' | 'checked'> & {
|
export type ListView = Omit<ListSummary, 'total' | 'checked'> & {
|
||||||
|
tableName: string;
|
||||||
columns: ListColumn[];
|
columns: ListColumn[];
|
||||||
items: ListItem[];
|
items: ListItem[];
|
||||||
};
|
};
|
||||||
@@ -144,6 +145,7 @@ export async function getList(slug: string): Promise<ListView | null> {
|
|||||||
slug: t.slug,
|
slug: t.slug,
|
||||||
pictureUrl: t.pictureUrl,
|
pictureUrl: t.pictureUrl,
|
||||||
isProgress: t.hasDone,
|
isProgress: t.hasDone,
|
||||||
|
tableName: t.tableName,
|
||||||
columns,
|
columns,
|
||||||
items: [...rows].map((r) => toItem(r, columns))
|
items: [...rows].map((r) => toItem(r, columns))
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,14 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
import { cents, csvDate, importCents, parseCsv, parseTransactionImport } from './money-tracker';
|
import {
|
||||||
|
accountNumberKey,
|
||||||
|
cents,
|
||||||
|
csvDate,
|
||||||
|
dropInternalTransfers,
|
||||||
|
importCents,
|
||||||
|
parseCsv,
|
||||||
|
parseTransactionImport,
|
||||||
|
planImport
|
||||||
|
} from './money-tracker';
|
||||||
|
|
||||||
describe('transaction import parsing', () => {
|
describe('transaction import parsing', () => {
|
||||||
it('handles quoted commas and semicolon-separated European amounts', () => {
|
it('handles quoted commas and semicolon-separated European amounts', () => {
|
||||||
@@ -63,3 +72,197 @@ describe('transaction import parsing', () => {
|
|||||||
).toThrow('Quote values containing commas');
|
).toThrow('Quote values containing commas');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('revolut statements and import dedupe', () => {
|
||||||
|
const statement =
|
||||||
|
'Type,Product,Started Date,Completed Date,Description,Amount,Fee,Currency,State,Balance\n' +
|
||||||
|
'CARD_PAYMENT,Current,2026-10-01 10:00:00,2026-10-02 09:00:00,Albert Heijn,-12.34,0.00,EUR,COMPLETED,100.00\n' +
|
||||||
|
'TOPUP,Current,2026-10-03 10:00:00,,Pending top-up,50.00,0.00,EUR,PENDING,\n' +
|
||||||
|
'TRANSFER,Current,2026-10-04 10:00:00,2026-10-04 10:00:01,To Joint,-100.00,0.50,EUR,COMPLETED,0.00\n' +
|
||||||
|
'CARD_PAYMENT,Current,2026-10-05 10:00:00,2026-10-05 11:00:00,Coffee,-3.00,0.00,USD,COMPLETED,0.00\n';
|
||||||
|
|
||||||
|
it('keeps completed EUR rows, uses the completed date and deducts fees', () => {
|
||||||
|
const rows = parseTransactionImport(statement, 'Revolut');
|
||||||
|
expect(rows.map((row) => [row.date, row.account, row.description, row.amountCents])).toEqual([
|
||||||
|
['2026-10-02', 'Revolut', 'Albert Heijn', -1234],
|
||||||
|
['2026-10-04', 'Revolut', 'To Joint', -10050]
|
||||||
|
]);
|
||||||
|
expect(rows[0].externalId).toMatch(/^csv:[0-9a-f]{32}$/);
|
||||||
|
expect(parseTransactionImport(statement, 'Revolut')[0].externalId).toBe(rows[0].externalId);
|
||||||
|
expect(() => parseTransactionImport(statement)).toThrow('Choose which account');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('gives identical rows in one file distinct, stable ids', () => {
|
||||||
|
const file =
|
||||||
|
'date,account,description,amount\n2026-10-01,Cash,Kiosk,-2\n2026-10-01,Cash,Kiosk,-2\n';
|
||||||
|
const [a, b] = parseTransactionImport(file);
|
||||||
|
expect(a.externalId).not.toBe(b.externalId);
|
||||||
|
expect(parseTransactionImport(file).map((row) => row.externalId)).toEqual([
|
||||||
|
a.externalId,
|
||||||
|
b.externalId
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips rows already imported or already entered by hand (±1 day)', () => {
|
||||||
|
const rows = [
|
||||||
|
{ accountId: 1, date: '2026-10-02', amountCents: -1234, externalId: 'csv:a' },
|
||||||
|
{ accountId: 1, date: '2026-10-02', amountCents: -1234, externalId: 'csv:b' },
|
||||||
|
{ accountId: 1, date: '2026-10-04', amountCents: -500, externalId: 'csv:c' },
|
||||||
|
{ accountId: 2, date: '2026-10-04', amountCents: -500, externalId: 'csv:d' }
|
||||||
|
];
|
||||||
|
const existing = [
|
||||||
|
{ id: 10, accountId: 1, date: '2026-10-01', amountCents: -1234, externalId: null },
|
||||||
|
{ id: 11, accountId: 1, date: '2026-10-04', amountCents: -500, externalId: 'csv:c' }
|
||||||
|
];
|
||||||
|
const { insert, duplicates } = planImport(rows, existing);
|
||||||
|
expect(insert.map((row) => row.externalId)).toEqual(['csv:b', 'csv:d']);
|
||||||
|
expect(duplicates).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('knab exports', () => {
|
||||||
|
const header =
|
||||||
|
'Rekeningnummer;Transactiedatum;Valutacode;CreditDebet;Bedrag;Tegenrekeningnummer;Tegenrekeninghouder;Valutadatum;Betaalwijze;Omschrijving;Type betaling;Machtigingsnummer;Incassant ID;Referentie;Boekdatum;\n';
|
||||||
|
const line = (
|
||||||
|
account: string,
|
||||||
|
date: string,
|
||||||
|
cd: string,
|
||||||
|
amount: string,
|
||||||
|
other: string,
|
||||||
|
name: string,
|
||||||
|
method: string,
|
||||||
|
note: string,
|
||||||
|
ref: string
|
||||||
|
) =>
|
||||||
|
`"${account}";"${date}";"EUR";"${cd}";"${amount}";"${other}";"${name}";"${date}";"${method}";"${note}";"";"";"";"${ref}";"${date}";\n`;
|
||||||
|
const current =
|
||||||
|
header +
|
||||||
|
line(
|
||||||
|
'NL98KNAB0408449136',
|
||||||
|
'01-10-2026',
|
||||||
|
'Afschrijvingen',
|
||||||
|
'1',
|
||||||
|
'80000208',
|
||||||
|
'Knab',
|
||||||
|
'Pakketkosten',
|
||||||
|
'Pakketkosten',
|
||||||
|
'R1'
|
||||||
|
) +
|
||||||
|
line(
|
||||||
|
'NL98KNAB0408449136',
|
||||||
|
'02-10-2026',
|
||||||
|
'Bijschrijvingen',
|
||||||
|
'1250,5',
|
||||||
|
'30581727',
|
||||||
|
'Spaar',
|
||||||
|
'Ontvangen betaling',
|
||||||
|
'',
|
||||||
|
'SHARED'
|
||||||
|
) +
|
||||||
|
line(
|
||||||
|
'NL98KNAB0408449136',
|
||||||
|
'03-10-2026',
|
||||||
|
'Afschrijvingen',
|
||||||
|
'2,35',
|
||||||
|
'NL11INGB0000000001',
|
||||||
|
'NS',
|
||||||
|
'iDEAL',
|
||||||
|
'Treinkaartje',
|
||||||
|
'R3'
|
||||||
|
);
|
||||||
|
const savings =
|
||||||
|
header +
|
||||||
|
line(
|
||||||
|
'30581727',
|
||||||
|
'02-10-2026',
|
||||||
|
'Afschrijvingen',
|
||||||
|
'1250,5',
|
||||||
|
'NL98KNAB0408449136',
|
||||||
|
'S Runge',
|
||||||
|
'Overboeking',
|
||||||
|
'',
|
||||||
|
'SHARED'
|
||||||
|
) +
|
||||||
|
line('30581727', '30-09-2026', 'Bijschrijvingen', '0,42', '', '', 'Rente', 'Rente', 'R5');
|
||||||
|
|
||||||
|
it('parses signs, decimal commas and descriptions', () => {
|
||||||
|
const rows = parseTransactionImport(current, 'Knab');
|
||||||
|
expect(
|
||||||
|
rows.map((row) => [row.date, row.description, row.amountCents, row.counterparty])
|
||||||
|
).toEqual([
|
||||||
|
['2026-10-01', 'Knab: Pakketkosten', -100, 'Knab'],
|
||||||
|
['2026-10-02', 'Spaar', 125050, 'Spaar'],
|
||||||
|
['2026-10-03', 'NS: Treinkaartje', -235, 'NS']
|
||||||
|
]);
|
||||||
|
expect(parseTransactionImport(current)[0].account).toBe('NL98KNAB0408449136');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('scopes ids to the booked account because Referentie repeats across accounts', () => {
|
||||||
|
const a = parseTransactionImport(current, 'Knab').find((row) => row.amountCents === 125050)!;
|
||||||
|
const b = parseTransactionImport(savings, 'Knab').find((row) => row.amountCents === -125050)!;
|
||||||
|
expect(a.externalId).toBe('knab:408449136:SHARED');
|
||||||
|
expect(b.externalId).toBe('knab:30581727:SHARED');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('drops transfers between own accounts but keeps interest', () => {
|
||||||
|
const rows = [
|
||||||
|
...parseTransactionImport(current, 'Knab'),
|
||||||
|
...parseTransactionImport(savings, 'Knab')
|
||||||
|
];
|
||||||
|
const { kept, internal } = dropInternalTransfers(rows, ['NL98KNAB0408449136', '30581727']);
|
||||||
|
expect(internal.map((row) => row.amountCents)).toEqual([125050, -125050]);
|
||||||
|
expect(kept.map((row) => row.amountCents)).toEqual([-100, -235, 42]);
|
||||||
|
expect(accountNumberKey('NL98 KNAB 0408 4491 36')).toBe(accountNumberKey('408449136'));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects malformed amounts', () => {
|
||||||
|
const bad =
|
||||||
|
header +
|
||||||
|
line('NL98KNAB0408449136', '01-10-2026', 'Afschrijvingen', '1.234,00', '', '', '', '', 'X');
|
||||||
|
expect(() => parseTransactionImport(bad, 'Knab')).toThrow('Invalid data in Knab row 1');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('import reconcile', () => {
|
||||||
|
it('attaches hand-entered twins and replaces hand sums of several payments', () => {
|
||||||
|
const rows = [
|
||||||
|
{ accountId: 2, date: '2026-03-01', amountCents: -20000, externalId: 'k:1' },
|
||||||
|
{ accountId: 2, date: '2026-03-01', amountCents: -300, externalId: 'k:2' },
|
||||||
|
{ accountId: 2, date: '2026-03-02', amountCents: -600, externalId: 'k:3' },
|
||||||
|
{ accountId: 2, date: '2026-03-03', amountCents: 5000, externalId: 'k:4' }
|
||||||
|
];
|
||||||
|
const existing = [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
accountId: 2,
|
||||||
|
date: '2026-03-01',
|
||||||
|
amountCents: -20900,
|
||||||
|
externalId: null,
|
||||||
|
description: 'To Joint'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 2,
|
||||||
|
accountId: 2,
|
||||||
|
date: '2026-03-03',
|
||||||
|
amountCents: 5000,
|
||||||
|
externalId: null,
|
||||||
|
description: 'From Revolut'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 3,
|
||||||
|
accountId: 2,
|
||||||
|
date: '2026-03-01',
|
||||||
|
amountCents: -20300,
|
||||||
|
externalId: null,
|
||||||
|
description: 'Start balance'
|
||||||
|
}
|
||||||
|
];
|
||||||
|
const { insert, duplicates, attach, replace } = planImport(rows, existing);
|
||||||
|
expect(duplicates).toBe(1);
|
||||||
|
expect(attach.map((item) => [item.id, item.row.externalId])).toEqual([[2, 'k:4']]);
|
||||||
|
expect(insert.map((row) => row.externalId)).toEqual(['k:1', 'k:2', 'k:3']);
|
||||||
|
expect(
|
||||||
|
replace.map((item) => [item.id, item.parts.map((row) => row.externalId).sort()])
|
||||||
|
).toEqual([[1, ['k:1', 'k:2', 'k:3']]]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
import { fail } from '@sveltejs/kit';
|
import { fail } from '@sveltejs/kit';
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { findSumMatch } from './reconcile';
|
||||||
|
|
||||||
export const utilityKinds = ['electricity', 'gas', 'water'] as const;
|
export const utilityKinds = ['electricity', 'gas', 'water'] as const;
|
||||||
export const cadences = ['weekly', 'monthly', 'yearly'] as const;
|
export const cadences = ['weekly', 'monthly', 'yearly'] as const;
|
||||||
@@ -130,21 +132,131 @@ export type ImportedTransaction = {
|
|||||||
account: string;
|
account: string;
|
||||||
description: string;
|
description: string;
|
||||||
amountCents: number;
|
amountCents: number;
|
||||||
|
/** Stable per-row id ("csv:<hash>", "knab:<account>:<ref>") so re-imports add nothing. */
|
||||||
|
externalId: string;
|
||||||
|
counterparty?: string | null;
|
||||||
|
/** Bank account the row was booked on (bank statements that say so, e.g. Knab). */
|
||||||
|
sourceAccount?: string;
|
||||||
|
/** The other side's account number, used to recognise transfers between own accounts. */
|
||||||
|
counterpartyAccount?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function parseTransactionImport(input: string): ImportedTransaction[] {
|
export type ImportFormat = 'tracker' | 'revolut' | 'knab';
|
||||||
const rows = parseCsv(input);
|
|
||||||
if (rows.length < 2 || rows.length > 20_001)
|
/**
|
||||||
throw new Error('The file must have a header and 1–20,000 rows.');
|
* Canonical account number: Dutch IBANs and the bare numbers Knab uses for its savings
|
||||||
const header = rows.shift()!.map((value) =>
|
* accounts compare equal (NL98KNAB0408449136 ~ 408449136).
|
||||||
|
*/
|
||||||
|
export function accountNumberKey(value: string | null | undefined) {
|
||||||
|
const raw = (value ?? '').replace(/[\s.-]/g, '').toUpperCase();
|
||||||
|
const iban = /^[A-Z]{2}\d{2}[A-Z]{4}(\d{6,})$/.exec(raw);
|
||||||
|
return (iban ? iban[1] : raw).replace(/^0+/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Knab "Exporteer transacties" CSV (semicolon separated, Dutch headers). */
|
||||||
|
export function isKnabHeader(header: string[]) {
|
||||||
|
return ['rekeningnummer', 'transactiedatum', 'creditdebet', 'bedrag', 'referentie'].every((key) =>
|
||||||
|
header.includes(key)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drops transfers between the given own accounts (both legs), keeping everything else. */
|
||||||
|
export function dropInternalTransfers<T extends ImportedTransaction>(
|
||||||
|
rows: T[],
|
||||||
|
own: Iterable<string>
|
||||||
|
) {
|
||||||
|
const keys = new Set([...own].map(accountNumberKey).filter(Boolean));
|
||||||
|
const kept: T[] = [];
|
||||||
|
const internal: T[] = [];
|
||||||
|
for (const row of rows) {
|
||||||
|
const other = accountNumberKey(row.counterpartyAccount);
|
||||||
|
(other && keys.has(other) ? internal : kept).push(row);
|
||||||
|
}
|
||||||
|
return { kept, internal };
|
||||||
|
}
|
||||||
|
|
||||||
|
function headerOf(rows: string[][]) {
|
||||||
|
return rows[0].map((value) =>
|
||||||
value
|
value
|
||||||
.replace(/^\uFEFF/, '')
|
.replace(/^\uFEFF/, '')
|
||||||
.trim()
|
.trim()
|
||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Revolut app statement export: Type, Product, Started Date, Completed Date, Description, Amount, Fee, Currency, State, Balance. */
|
||||||
|
export function isRevolutHeader(header: string[]) {
|
||||||
|
return ['type', 'started date', 'description', 'amount', 'currency', 'state'].every((key) =>
|
||||||
|
header.includes(key)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function importId(parts: (string | number)[], seen: Map<string, number>) {
|
||||||
|
const base = parts.join('\u001f');
|
||||||
|
// Identical rows in one file (two coffees, same day) stay distinct but stable.
|
||||||
|
const occurrence = (seen.get(base) ?? 0) + 1;
|
||||||
|
seen.set(base, occurrence);
|
||||||
|
return (
|
||||||
|
'csv:' + createHash('sha256').update(`${base}\u001f${occurrence}`).digest('hex').slice(0, 32)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses either the tracker's own export (date, account, description, amount) or a Revolut
|
||||||
|
* statement. Revolut files have no account column, so `defaultAccount` names the tracker
|
||||||
|
* account they belong to; only COMPLETED EUR rows are kept and the fee is deducted.
|
||||||
|
*/
|
||||||
|
export function parseTransactionImport(
|
||||||
|
input: string,
|
||||||
|
defaultAccount?: string
|
||||||
|
): ImportedTransaction[] {
|
||||||
|
const rows = parseCsv(input);
|
||||||
|
if (rows.length < 2 || rows.length > 20_001)
|
||||||
|
throw new Error('The file must have a header and 1–20,000 rows.');
|
||||||
|
const header = headerOf(rows);
|
||||||
|
rows.shift();
|
||||||
|
const seen = new Map<string, number>();
|
||||||
|
if (isKnabHeader(header)) return parseKnab(header, rows, defaultAccount, seen);
|
||||||
|
if (isRevolutHeader(header)) {
|
||||||
|
const account = defaultAccount?.trim();
|
||||||
|
if (!account) throw new Error('Choose which account this Revolut statement belongs to.');
|
||||||
|
const col = (key: string) => header.indexOf(key);
|
||||||
|
const parsed: ImportedTransaction[] = [];
|
||||||
|
for (const [index, row] of rows.entries()) {
|
||||||
|
const state = (row[col('state')] ?? '').trim().toUpperCase();
|
||||||
|
if (state && state !== 'COMPLETED') continue;
|
||||||
|
const currency = (row[col('currency')] ?? 'EUR').trim().toUpperCase();
|
||||||
|
if (currency && currency !== 'EUR') continue;
|
||||||
|
const started = (row[col('started date')] ?? '').trim();
|
||||||
|
const completed = col('completed date') >= 0 ? (row[col('completed date')] ?? '').trim() : '';
|
||||||
|
const date = csvDate((completed || started).slice(0, 10));
|
||||||
|
const amount = importCents(row[col('amount')]);
|
||||||
|
const fee = col('fee') >= 0 ? (importCents(row[col('fee')] || '0') ?? undefined) : 0;
|
||||||
|
const description = (row[col('description')] ?? '').trim().slice(0, 500) || 'Revolut';
|
||||||
|
if (!date || amount == null || fee == null)
|
||||||
|
throw new Error(`Invalid data in statement row ${index + 1}. No rows were imported.`);
|
||||||
|
const amountCents = amount - fee;
|
||||||
|
if (!amountCents) continue;
|
||||||
|
parsed.push({
|
||||||
|
date,
|
||||||
|
account,
|
||||||
|
description,
|
||||||
|
amountCents,
|
||||||
|
externalId: importId(
|
||||||
|
['revolut', account.toLowerCase(), started, amount, fee, description],
|
||||||
|
seen
|
||||||
|
)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (!parsed.length) throw new Error('No completed EUR transactions found in this statement.');
|
||||||
|
return parsed.sort((a, b) => a.date.localeCompare(b.date));
|
||||||
|
}
|
||||||
const columns = ['date', 'account', 'description', 'amount'].map((key) => header.indexOf(key));
|
const columns = ['date', 'account', 'description', 'amount'].map((key) => header.indexOf(key));
|
||||||
if (columns.some((index) => index < 0))
|
const accountColumn = columns[1];
|
||||||
throw new Error('Headers must include date, account, description and amount.');
|
if (columns[0] < 0 || columns[2] < 0 || columns[3] < 0 || (accountColumn < 0 && !defaultAccount))
|
||||||
|
throw new Error(
|
||||||
|
'Headers must include date, account, description and amount (or choose an account), or be a Revolut statement.'
|
||||||
|
);
|
||||||
const parsed: ImportedTransaction[] = [];
|
const parsed: ImportedTransaction[] = [];
|
||||||
for (const [index, row] of rows.entries()) {
|
for (const [index, row] of rows.entries()) {
|
||||||
if (row.length !== header.length)
|
if (row.length !== header.length)
|
||||||
@@ -152,7 +264,8 @@ export function parseTransactionImport(input: string): ImportedTransaction[] {
|
|||||||
`Data row ${index + 1} has ${row.length} columns; expected ${header.length}. Quote values containing commas. No rows were imported.`
|
`Data row ${index + 1} has ${row.length} columns; expected ${header.length}. Quote values containing commas. No rows were imported.`
|
||||||
);
|
);
|
||||||
const date = csvDate(row[columns[0]] ?? '');
|
const date = csvDate(row[columns[0]] ?? '');
|
||||||
const account = (row[columns[1]] ?? '').trim();
|
const account =
|
||||||
|
(accountColumn >= 0 ? (row[accountColumn] ?? '') : '').trim() || defaultAccount?.trim() || '';
|
||||||
const description = (row[columns[2]] ?? '').trim();
|
const description = (row[columns[2]] ?? '').trim();
|
||||||
const amountCents = importCents(row[columns[3]]);
|
const amountCents = importCents(row[columns[3]]);
|
||||||
if (
|
if (
|
||||||
@@ -164,7 +277,143 @@ export function parseTransactionImport(input: string): ImportedTransaction[] {
|
|||||||
amountCents == null
|
amountCents == null
|
||||||
)
|
)
|
||||||
throw new Error(`Invalid data in transaction ${index + 1}. No rows were imported.`);
|
throw new Error(`Invalid data in transaction ${index + 1}. No rows were imported.`);
|
||||||
parsed.push({ date, account, description, amountCents });
|
parsed.push({
|
||||||
|
date,
|
||||||
|
account,
|
||||||
|
description,
|
||||||
|
amountCents,
|
||||||
|
externalId: importId([date, account.toLowerCase(), description, amountCents], seen)
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return parsed.sort((a, b) => a.date.localeCompare(b.date));
|
return parsed.sort((a, b) => a.date.localeCompare(b.date));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseKnab(
|
||||||
|
header: string[],
|
||||||
|
rows: string[][],
|
||||||
|
defaultAccount: string | undefined,
|
||||||
|
seen: Map<string, number>
|
||||||
|
) {
|
||||||
|
const col = (key: string) => header.indexOf(key);
|
||||||
|
const cell = (row: string[], key: string) => (col(key) >= 0 ? (row[col(key)] ?? '').trim() : '');
|
||||||
|
const parsed: ImportedTransaction[] = [];
|
||||||
|
for (const [index, row] of rows.entries()) {
|
||||||
|
if (row.every((value) => !value.trim())) continue;
|
||||||
|
const sourceAccount = cell(row, 'rekeningnummer').replace(/\s/g, '').toUpperCase();
|
||||||
|
const currency = cell(row, 'valutacode').toUpperCase();
|
||||||
|
if (currency && currency !== 'EUR') continue;
|
||||||
|
const date = csvDate(cell(row, 'transactiedatum'));
|
||||||
|
// Knab writes plain amounts with a decimal comma and never a thousands separator.
|
||||||
|
const amount = /^\d{1,9}(?:,\d{1,2})?$/.test(cell(row, 'bedrag'))
|
||||||
|
? cents(cell(row, 'bedrag'))
|
||||||
|
: undefined;
|
||||||
|
const direction = cell(row, 'creditdebet').toLowerCase();
|
||||||
|
const debit = direction.startsWith('af') || direction === 'd' || direction === 'debet';
|
||||||
|
const credit = direction.startsWith('bij') || direction === 'c' || direction === 'credit';
|
||||||
|
if (!sourceAccount || !date || amount == null || (!debit && !credit))
|
||||||
|
throw new Error(`Invalid data in Knab row ${index + 1}. No rows were imported.`);
|
||||||
|
if (!amount) continue;
|
||||||
|
const counterparty = cell(row, 'tegenrekeninghouder');
|
||||||
|
const note = cell(row, 'omschrijving');
|
||||||
|
const method = cell(row, 'betaalwijze');
|
||||||
|
const description = (
|
||||||
|
counterparty && note && note.toLowerCase() !== counterparty.toLowerCase()
|
||||||
|
? `${counterparty}: ${note}`
|
||||||
|
: counterparty || note || method || 'Knab'
|
||||||
|
).slice(0, 500);
|
||||||
|
const reference = cell(row, 'referentie');
|
||||||
|
const account = defaultAccount?.trim() || sourceAccount;
|
||||||
|
// Referentie repeats across a customer's accounts (both legs of a transfer), so the
|
||||||
|
// id is scoped to the account the row was booked on.
|
||||||
|
const externalId = reference
|
||||||
|
? `knab:${accountNumberKey(sourceAccount)}:${reference}`
|
||||||
|
: importId(['knab', sourceAccount, date, amount, description], seen);
|
||||||
|
parsed.push({
|
||||||
|
date,
|
||||||
|
account,
|
||||||
|
description,
|
||||||
|
amountCents: debit ? -amount : amount,
|
||||||
|
externalId,
|
||||||
|
counterparty: counterparty || null,
|
||||||
|
sourceAccount,
|
||||||
|
counterpartyAccount: cell(row, 'tegenrekeningnummer') || undefined
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (!parsed.length) throw new Error('No EUR transactions found in this Knab export.');
|
||||||
|
return parsed.sort((a, b) => a.date.localeCompare(b.date));
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ExistingImportRow = {
|
||||||
|
id: number;
|
||||||
|
accountId: number;
|
||||||
|
date: string;
|
||||||
|
amountCents: number;
|
||||||
|
externalId: string | null;
|
||||||
|
/** Hand-entered rows (no external id) can be replaced; opening balances never. */
|
||||||
|
description?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const isOpeningBalanceRow = (description: string | undefined) =>
|
||||||
|
/^(start|opening) balance/i.test((description ?? '').trim());
|
||||||
|
|
||||||
|
function daysApart(a: string, b: string) {
|
||||||
|
return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Splits imported rows into new ones and duplicates: same import id already stored, or an
|
||||||
|
* existing row on the same account with the same amount within ±1 day (each existing row
|
||||||
|
* can only absorb one imported row). Hand-entered matches are returned in `attach` so the
|
||||||
|
* caller can give them the import id. A hand-entered row equal to the sum of 2-3 new rows
|
||||||
|
* within a day ("To Joint 209" = 200 + 3 + 6) is returned in `replace`: the new rows go in
|
||||||
|
* and the hand row should be deleted.
|
||||||
|
*/
|
||||||
|
export function planImport<
|
||||||
|
T extends { accountId: number; date: string; amountCents: number; externalId: string }
|
||||||
|
>(rows: T[], existing: ExistingImportRow[]) {
|
||||||
|
const ids = new Set(
|
||||||
|
existing.flatMap((row) => (row.externalId ? [`${row.accountId}:${row.externalId}`] : []))
|
||||||
|
);
|
||||||
|
const byKey = new Map<string, ExistingImportRow[]>();
|
||||||
|
for (const row of existing) {
|
||||||
|
if (!row.externalId && isOpeningBalanceRow(row.description)) continue;
|
||||||
|
const key = `${row.accountId}:${row.amountCents}`;
|
||||||
|
byKey.set(key, [...(byKey.get(key) ?? []), row]);
|
||||||
|
}
|
||||||
|
const claimed = new Set<number>();
|
||||||
|
const insert: T[] = [];
|
||||||
|
const attach: { id: number; row: T }[] = [];
|
||||||
|
let duplicates = 0;
|
||||||
|
for (const row of rows) {
|
||||||
|
const idKey = `${row.accountId}:${row.externalId}`;
|
||||||
|
if (ids.has(idKey)) {
|
||||||
|
duplicates++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const match = (byKey.get(`${row.accountId}:${row.amountCents}`) ?? [])
|
||||||
|
.filter((candidate) => !claimed.has(candidate.id) && daysApart(candidate.date, row.date) <= 1)
|
||||||
|
.sort((a, b) => daysApart(a.date, row.date) - daysApart(b.date, row.date))[0];
|
||||||
|
if (match) {
|
||||||
|
claimed.add(match.id);
|
||||||
|
duplicates++;
|
||||||
|
if (!match.externalId) attach.push({ id: match.id, row });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
ids.add(idKey);
|
||||||
|
insert.push(row);
|
||||||
|
}
|
||||||
|
const replace: { id: number; parts: T[] }[] = [];
|
||||||
|
const taken = new Set<number>();
|
||||||
|
for (const hand of existing) {
|
||||||
|
if (hand.externalId || claimed.has(hand.id) || isOpeningBalanceRow(hand.description)) continue;
|
||||||
|
const pool = insert.map((row) =>
|
||||||
|
row.accountId === hand.accountId ? row : { ...row, amountCents: 0 }
|
||||||
|
);
|
||||||
|
const parts = findSumMatch(hand, pool, taken);
|
||||||
|
if (!parts) continue;
|
||||||
|
parts.forEach((index) => taken.add(index));
|
||||||
|
claimed.add(hand.id);
|
||||||
|
replace.push({ id: hand.id, parts: parts.map((index) => insert[index]) });
|
||||||
|
}
|
||||||
|
return { insert, duplicates, attach, replace };
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it } from 'vitest';
|
||||||
import { isOwner, isOwnerOnlyPath } from './owner';
|
import { isOwner, isOwnerOnlyPath, isOwnerWritePath } from './owner';
|
||||||
|
|
||||||
const event = (user: Partial<NonNullable<App.Locals['user']>> | null) =>
|
const event = (user: Partial<NonNullable<App.Locals['user']>> | null) =>
|
||||||
({ locals: { user, session: null } }) as unknown as { locals: App.Locals };
|
({ locals: { user, session: null } }) as unknown as { locals: App.Locals };
|
||||||
@@ -16,5 +16,26 @@ describe('owner gate', () => {
|
|||||||
expect(isOwnerOnlyPath('/trackers/Health/import')).toBe(true);
|
expect(isOwnerOnlyPath('/trackers/Health/import')).toBe(true);
|
||||||
expect(isOwnerOnlyPath('/trackers/healthy')).toBe(false);
|
expect(isOwnerOnlyPath('/trackers/healthy')).toBe(false);
|
||||||
expect(isOwnerOnlyPath('/trackers')).toBe(false);
|
expect(isOwnerOnlyPath('/trackers')).toBe(false);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/tasks')).toBe(true);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/Tasks/')).toBe(true);
|
||||||
|
expect(isOwnerWritePath('/trackers/tasks')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('gates the book reader, files and APIs but not the catalogue', () => {
|
||||||
|
expect(isOwnerOnlyPath('/trackers/books/files/12')).toBe(true);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/books/12/read')).toBe(true);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/books/12/progress')).toBe(true);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/books/12/notes/3')).toBe(true);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/books')).toBe(false);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/books/12')).toBe(false);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/books/covers/12')).toBe(false);
|
||||||
|
expect(isOwnerOnlyPath('/trackers/books/12/reader-notes-public')).toBe(false);
|
||||||
|
});
|
||||||
|
it('only lets the owner write in owner areas', () => {
|
||||||
|
expect(isOwnerWritePath('/trackers/books/12')).toBe(true);
|
||||||
|
expect(isOwnerWritePath('/trackers/books')).toBe(true);
|
||||||
|
expect(isOwnerWritePath('/trackers/health/import')).toBe(true);
|
||||||
|
expect(isOwnerWritePath('/trackers/bookshelf')).toBe(false);
|
||||||
|
expect(isOwnerWritePath('/lists/books')).toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -19,9 +19,41 @@ export function requireOwner(event: OwnerEvent) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Paths that only the owner may see (checked in hooks.server.ts before any route runs). */
|
/** Paths that only the owner may see (checked in hooks.server.ts before any route runs). */
|
||||||
export const ownerOnlyPrefixes = ['/trackers/health'] as const;
|
export const ownerOnlyPrefixes = [
|
||||||
|
'/trackers/health',
|
||||||
|
'/trackers/tasks',
|
||||||
|
'/trackers/books/files',
|
||||||
|
'/lists/db'
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Owner-only paths inside otherwise public areas: the book reader and its progress/notes
|
||||||
|
* APIs. /trackers/books and /trackers/books/<id> themselves are public catalogue pages.
|
||||||
|
*/
|
||||||
|
export const ownerOnlyPatterns = [/^\/trackers\/books\/\d+\/(read|progress|notes)(\/|$)/] as const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Areas where only the owner may write: any non-GET request from anyone else is refused
|
||||||
|
* (404) in hooks.server.ts, which also covers form actions on the public book pages.
|
||||||
|
*/
|
||||||
|
export const ownerWritePrefixes = [
|
||||||
|
'/trackers/health',
|
||||||
|
'/trackers/tasks',
|
||||||
|
'/trackers/books'
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
const normalise = (pathname: string) => pathname.toLowerCase().replace(/\/+$/, '');
|
||||||
|
const under = (path: string, prefix: string) => path === prefix || path.startsWith(`${prefix}/`);
|
||||||
|
|
||||||
export function isOwnerOnlyPath(pathname: string): boolean {
|
export function isOwnerOnlyPath(pathname: string): boolean {
|
||||||
const path = pathname.toLowerCase().replace(/\/+$/, '');
|
const path = normalise(pathname);
|
||||||
return ownerOnlyPrefixes.some((prefix) => path === prefix || path.startsWith(`${prefix}/`));
|
return (
|
||||||
|
ownerOnlyPrefixes.some((prefix) => under(path, prefix)) ||
|
||||||
|
ownerOnlyPatterns.some((pattern) => pattern.test(path))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isOwnerWritePath(pathname: string): boolean {
|
||||||
|
const path = normalise(pathname);
|
||||||
|
return ownerWritePrefixes.some((prefix) => under(path, prefix));
|
||||||
}
|
}
|
||||||
|
|||||||
30
src/lib/server/reconcile.test.ts
Normal file
30
src/lib/server/reconcile.test.ts
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { findSumMatch } from './reconcile';
|
||||||
|
|
||||||
|
describe('sum matching', () => {
|
||||||
|
const pool = [
|
||||||
|
{ date: '2026-03-01', amountCents: -20000 },
|
||||||
|
{ date: '2026-03-01', amountCents: -300 },
|
||||||
|
{ date: '2026-03-02', amountCents: -600 },
|
||||||
|
{ date: '2026-03-05', amountCents: -900 },
|
||||||
|
{ date: '2026-03-01', amountCents: 900 }
|
||||||
|
];
|
||||||
|
it('finds the payments a hand entry stands for', () => {
|
||||||
|
expect(
|
||||||
|
findSumMatch({ date: '2026-03-01', amountCents: -20900 }, pool, new Set())?.sort()
|
||||||
|
).toEqual([0, 1, 2]);
|
||||||
|
expect(findSumMatch({ date: '2026-03-01', amountCents: -20300 }, pool, new Set())).toEqual([
|
||||||
|
0, 1
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
it('ignores used rows, other days and opposite signs', () => {
|
||||||
|
expect(
|
||||||
|
findSumMatch({ date: '2026-03-01', amountCents: -20900 }, pool, new Set([2]))
|
||||||
|
).toBeNull();
|
||||||
|
expect(findSumMatch({ date: '2026-03-05', amountCents: -1200 }, pool, new Set())).toBeNull();
|
||||||
|
expect(findSumMatch({ date: '2026-03-01', amountCents: -19100 }, pool, new Set())).toBeNull();
|
||||||
|
});
|
||||||
|
it('needs at least two parts', () => {
|
||||||
|
expect(findSumMatch({ date: '2026-03-01', amountCents: -300 }, pool, new Set())).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
49
src/lib/server/reconcile.ts
Normal file
49
src/lib/server/reconcile.ts
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
// Shared reconcile helpers for bank sync and CSV import: a hand-entered row often stands
|
||||||
|
// for several bank payments (e.g. "To Joint 209" = 200 + 3 + 6 on the same day).
|
||||||
|
|
||||||
|
export type DatedAmount = { date: string; amountCents: number };
|
||||||
|
|
||||||
|
export function daysBetween(a: string, b: string) {
|
||||||
|
return Math.abs(Date.parse(a + 'T00:00:00Z') - Date.parse(b + 'T00:00:00Z')) / 86_400_000;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Finds 2..maxParts unused pool entries, each within ±maxDays of `target.date`, whose
|
||||||
|
* amounts add up exactly to `target.amountCents`. Prefers fewer parts, then the smallest
|
||||||
|
* total date distance. Returns pool indexes, or null. Every part must have the same sign as
|
||||||
|
* the target, which keeps coincidental matches rare.
|
||||||
|
*/
|
||||||
|
export function findSumMatch(
|
||||||
|
target: DatedAmount,
|
||||||
|
pool: DatedAmount[],
|
||||||
|
used: Set<number>,
|
||||||
|
maxParts = 3,
|
||||||
|
maxDays = 1
|
||||||
|
): number[] | null {
|
||||||
|
const near = pool
|
||||||
|
.map((row, index) => ({ row, index, distance: daysBetween(row.date, target.date) }))
|
||||||
|
.filter(
|
||||||
|
({ row, index, distance }) =>
|
||||||
|
!used.has(index) &&
|
||||||
|
Math.sign(row.amountCents) === Math.sign(target.amountCents) &&
|
||||||
|
row.amountCents !== 0 &&
|
||||||
|
distance <= maxDays
|
||||||
|
);
|
||||||
|
// Same-day batches are small; cap the search so a busy day can't explode combinatorially.
|
||||||
|
if (near.length > 40) near.length = 40;
|
||||||
|
let best: { indexes: number[]; distance: number } | null = null;
|
||||||
|
const search = (size: number, start: number, chosen: typeof near, sum: number) => {
|
||||||
|
if (chosen.length === size) {
|
||||||
|
if (sum !== target.amountCents) return;
|
||||||
|
const distance = chosen.reduce((total, item) => total + item.distance, 0);
|
||||||
|
if (!best || distance < best.distance)
|
||||||
|
best = { indexes: chosen.map((item) => item.index), distance };
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (let i = start; i < near.length; i++)
|
||||||
|
search(size, i + 1, [...chosen, near[i]], sum + near[i].row.amountCents);
|
||||||
|
};
|
||||||
|
for (let size = 2; size <= maxParts && !best; size++) search(size, 0, [], 0);
|
||||||
|
const found = best as { indexes: number[] } | null;
|
||||||
|
return found ? found.indexes : null;
|
||||||
|
}
|
||||||
@@ -28,10 +28,13 @@ export async function uniqueSlug(title: string): Promise<string> {
|
|||||||
return slug;
|
return slug;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** /lists/db is the database browser, so no list may use that slug. */
|
||||||
|
export const RESERVED_LIST_SLUGS = new Set(['db']);
|
||||||
|
|
||||||
export async function uniqueListSlug(name: string): Promise<string> {
|
export async function uniqueListSlug(name: string): Promise<string> {
|
||||||
const base = slugify(name, 'list');
|
const base = slugify(name, 'list');
|
||||||
let slug = base;
|
let slug = RESERVED_LIST_SLUGS.has(base) ? `${base}-2` : base;
|
||||||
let suffix = 2;
|
let suffix = RESERVED_LIST_SLUGS.has(base) ? 3 : 2;
|
||||||
while (
|
while (
|
||||||
(await db.select({ id: metaTables.id }).from(metaTables).where(eq(metaTables.slug, slug)))
|
(await db.select({ id: metaTables.id }).from(metaTables).where(eq(metaTables.slug, slug)))
|
||||||
.length > 0
|
.length > 0
|
||||||
|
|||||||
62
src/lib/server/tasks.test.ts
Normal file
62
src/lib/server/tasks.test.ts
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import seed from '../../../data/tasks/daisy-import.json';
|
||||||
|
import { parseQuickTitle, seedFile, statusTimestamps, taskInput } from './tasks';
|
||||||
|
|
||||||
|
describe('task tracker', () => {
|
||||||
|
it('parses the seed file', () => {
|
||||||
|
const parsed = seedFile.parse(seed);
|
||||||
|
expect(parsed.tasks.length).toBeGreaterThan(50);
|
||||||
|
expect(parsed.goals.length).toBeGreaterThan(3);
|
||||||
|
const slugs = new Set(parsed.areas.map((a) => a.slug));
|
||||||
|
for (const t of parsed.tasks) expect(slugs.has(t.area!)).toBe(true);
|
||||||
|
const goals = new Set(parsed.goals.map((g) => g.title.toLowerCase()));
|
||||||
|
for (const t of parsed.tasks) if (t.goal) expect(goals.has(t.goal.toLowerCase())).toBe(true);
|
||||||
|
const keys = parsed.tasks.map((t) => `${t.area}|${t.title.toLowerCase()}`);
|
||||||
|
expect(new Set(keys).size).toBe(keys.length);
|
||||||
|
});
|
||||||
|
it('validates form input', () => {
|
||||||
|
expect(taskInput.safeParse({ title: ' ' }).success).toBe(false);
|
||||||
|
expect(taskInput.safeParse({ title: 'x', dueTime: '10:00' }).success).toBe(false);
|
||||||
|
const ok = taskInput.parse({ title: 'x', due: '2026-10-26', dueTime: '10:00', tags: '#a, b' });
|
||||||
|
expect(ok.tags).toEqual(['a', 'b']);
|
||||||
|
expect(ok.status).toBe('todo');
|
||||||
|
});
|
||||||
|
it('parses quick-add flags', () => {
|
||||||
|
expect(parseQuickTitle('Call vet !')).toEqual({
|
||||||
|
title: 'Call vet',
|
||||||
|
flagged: true,
|
||||||
|
priority: 0
|
||||||
|
});
|
||||||
|
expect(parseQuickTitle('Pay rent !!!')).toEqual({
|
||||||
|
title: 'Pay rent',
|
||||||
|
flagged: true,
|
||||||
|
priority: 3
|
||||||
|
});
|
||||||
|
expect(parseQuickTitle('Sign up (!)')).toEqual({
|
||||||
|
title: 'Sign up',
|
||||||
|
flagged: true,
|
||||||
|
priority: 0
|
||||||
|
});
|
||||||
|
expect(parseQuickTitle('Hello!')).toEqual({ title: 'Hello!', flagged: false, priority: 0 });
|
||||||
|
});
|
||||||
|
it('sets completion timestamps from the status', () => {
|
||||||
|
const now = new Date('2026-10-24T10:00:00Z');
|
||||||
|
expect(statusTimestamps('done', null, now).completedAt).toEqual(now);
|
||||||
|
expect(statusTimestamps('todo', { startedAt: null, completedAt: now }, now).completedAt).toBe(
|
||||||
|
null
|
||||||
|
);
|
||||||
|
expect(statusTimestamps('in_progress', null, now).startedAt).toEqual(now);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('local times', () => {
|
||||||
|
it('reads datetime-local values as Amsterdam time', async () => {
|
||||||
|
const { parseLocalMoment } = await import('./tasks');
|
||||||
|
expect(parseLocalMoment('2026-10-24T11:50')?.toISOString()).toBe('2026-10-24T09:50:00.000Z');
|
||||||
|
expect(parseLocalMoment('2026-12-07T09:00')?.toISOString()).toBe('2026-12-07T08:00:00.000Z');
|
||||||
|
expect(parseLocalMoment('2026-10-24T11:50:00+02:00')?.toISOString()).toBe(
|
||||||
|
'2026-10-24T09:50:00.000Z'
|
||||||
|
);
|
||||||
|
expect(parseLocalMoment('nope')).toBe(null);
|
||||||
|
});
|
||||||
|
});
|
||||||
338
src/lib/server/tasks.ts
Normal file
338
src/lib/server/tasks.ts
Normal file
@@ -0,0 +1,338 @@
|
|||||||
|
// Task + goal tracker: validation and the idempotent seed import. Shared by
|
||||||
|
// /trackers/tasks (owner only) and scripts/tasks-import.ts, so it must not import
|
||||||
|
// anything from SvelteKit.
|
||||||
|
import { and, eq, isNull, sql } from 'drizzle-orm';
|
||||||
|
import type { PgAsyncDatabase } from 'drizzle-orm/pg-core';
|
||||||
|
import * as z from 'zod';
|
||||||
|
import {
|
||||||
|
GOAL_STATUSES,
|
||||||
|
TASK_STATUSES,
|
||||||
|
todoActivity,
|
||||||
|
todoAreas,
|
||||||
|
todoGoals,
|
||||||
|
todoTasks
|
||||||
|
} from './db/schema/tasks';
|
||||||
|
|
||||||
|
export const TIME_ZONE = 'Europe/Amsterdam';
|
||||||
|
|
||||||
|
const blank = (v: unknown) => (typeof v === 'string' && v.trim() === '' ? null : v);
|
||||||
|
const optText = z.preprocess(blank, z.string().trim().max(10_000).nullish());
|
||||||
|
const day = z.preprocess(
|
||||||
|
blank,
|
||||||
|
z
|
||||||
|
.string()
|
||||||
|
.regex(/^\d{4}-\d{2}-\d{2}$/, 'Use YYYY-MM-DD')
|
||||||
|
.refine((s) => !Number.isNaN(Date.parse(`${s}T00:00:00Z`)), 'Invalid date')
|
||||||
|
.nullish()
|
||||||
|
);
|
||||||
|
const clock = z.preprocess(
|
||||||
|
blank,
|
||||||
|
z
|
||||||
|
.string()
|
||||||
|
.regex(/^([01]\d|2[0-3]):[0-5]\d(:[0-5]\d)?$/, 'Use HH:MM')
|
||||||
|
.nullish()
|
||||||
|
);
|
||||||
|
const moment = z.preprocess(
|
||||||
|
blank,
|
||||||
|
z
|
||||||
|
.string()
|
||||||
|
.refine((s) => !Number.isNaN(Date.parse(s)), 'Invalid date/time')
|
||||||
|
.nullish()
|
||||||
|
);
|
||||||
|
const tags = z.preprocess(
|
||||||
|
(v) =>
|
||||||
|
typeof v === 'string'
|
||||||
|
? v
|
||||||
|
.split(/[,\s]+/)
|
||||||
|
.map((s) => s.replace(/^#/, ''))
|
||||||
|
.filter(Boolean)
|
||||||
|
: v,
|
||||||
|
z.array(z.string().trim().toLowerCase().min(1).max(40)).max(30).default([])
|
||||||
|
);
|
||||||
|
const num = z.preprocess(
|
||||||
|
(v) => (blank(v) === null ? null : typeof v === 'string' ? Number(v.replace(',', '.')) : v),
|
||||||
|
z.number().finite().nullish()
|
||||||
|
);
|
||||||
|
|
||||||
|
export const areaSeed = z.object({
|
||||||
|
slug: z.string().regex(/^[a-z0-9][a-z0-9-]{0,62}$/),
|
||||||
|
name: z.string().trim().min(1),
|
||||||
|
description: optText,
|
||||||
|
color: optText,
|
||||||
|
sortOrder: z.number().int().default(0)
|
||||||
|
});
|
||||||
|
|
||||||
|
export const goalInput = z.object({
|
||||||
|
title: z.string().trim().min(1).max(300),
|
||||||
|
notes: optText,
|
||||||
|
status: z.enum(GOAL_STATUSES).default('active'),
|
||||||
|
area: z.preprocess(blank, z.string().nullish()), // area slug
|
||||||
|
targetDate: day,
|
||||||
|
metric: optText,
|
||||||
|
metricUnit: optText,
|
||||||
|
metricTarget: num,
|
||||||
|
metricCurrent: num,
|
||||||
|
flagged: z.coerce.boolean().default(false)
|
||||||
|
});
|
||||||
|
|
||||||
|
export const taskInput = z
|
||||||
|
.object({
|
||||||
|
title: z.string().trim().min(1).max(500),
|
||||||
|
notes: optText,
|
||||||
|
status: z.enum(TASK_STATUSES).default('todo'),
|
||||||
|
priority: z.coerce.number().int().min(0).max(3).default(0),
|
||||||
|
flagged: z.coerce.boolean().default(false),
|
||||||
|
due: day,
|
||||||
|
dueTime: clock,
|
||||||
|
reminderAt: moment,
|
||||||
|
recurrence: optText,
|
||||||
|
area: z.preprocess(blank, z.string().nullish()), // area slug
|
||||||
|
goal: z.preprocess(blank, z.string().nullish()), // goal title
|
||||||
|
parent: z.preprocess(blank, z.string().nullish()), // parent task title (same area)
|
||||||
|
tags
|
||||||
|
})
|
||||||
|
.refine((t) => !t.dueTime || t.due, {
|
||||||
|
message: 'A due time needs a due date',
|
||||||
|
path: ['dueTime']
|
||||||
|
});
|
||||||
|
|
||||||
|
export const seedFile = z.object({
|
||||||
|
source: z.string().min(1).default('daisy-import'),
|
||||||
|
areas: z.array(areaSeed).default([]),
|
||||||
|
goals: z.array(goalInput).default([]),
|
||||||
|
tasks: z.array(taskInput).default([])
|
||||||
|
});
|
||||||
|
export type SeedFile = z.infer<typeof seedFile>;
|
||||||
|
export type TaskInput = z.infer<typeof taskInput>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* "Buy milk !" or "! buy milk" → flagged; "!!!" anywhere → priority 3 (TickTick-style).
|
||||||
|
* Used by quick-add.
|
||||||
|
*/
|
||||||
|
export function parseQuickTitle(raw: string): {
|
||||||
|
title: string;
|
||||||
|
flagged: boolean;
|
||||||
|
priority: number;
|
||||||
|
} {
|
||||||
|
let title = raw.trim();
|
||||||
|
let priority = 0;
|
||||||
|
const bangs = title.match(/(^|\s)(!{1,3})(?=\s|$)/);
|
||||||
|
if (bangs) {
|
||||||
|
priority = bangs[2].length === 1 ? 0 : bangs[2].length;
|
||||||
|
title = title.replace(bangs[0], ' ').replace(/\s+/g, ' ').trim();
|
||||||
|
}
|
||||||
|
const flagged = !!bangs || /\(!\)/.test(title);
|
||||||
|
title = title.replace(/\s*\(!\)\s*/g, ' ').trim();
|
||||||
|
return { title, flagged, priority };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Status side effects: completedAt/startedAt follow the status. */
|
||||||
|
export function statusTimestamps(
|
||||||
|
status: (typeof TASK_STATUSES)[number],
|
||||||
|
prev: { startedAt: Date | null; completedAt: Date | null } | null,
|
||||||
|
now = new Date()
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
startedAt:
|
||||||
|
status === 'in_progress' || status === 'done'
|
||||||
|
? (prev?.startedAt ?? (status === 'in_progress' ? now : null))
|
||||||
|
: (prev?.startedAt ?? null),
|
||||||
|
completedAt: status === 'done' || status === 'dropped' ? (prev?.completedAt ?? now) : null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- any drizzle pg driver
|
||||||
|
type Db = PgAsyncDatabase<any, any>;
|
||||||
|
export type ImportResult = {
|
||||||
|
areas: { inserted: number; existing: number };
|
||||||
|
goals: { inserted: number; existing: number; updated: number };
|
||||||
|
tasks: { inserted: number; existing: number; updated: number };
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Idempotent import. Areas dedupe on slug, goals on lower(title), tasks on area + title
|
||||||
|
* (case-insensitive). Existing rows are left alone unless `update` is set, so re-running
|
||||||
|
* never undoes edits made on the site.
|
||||||
|
*/
|
||||||
|
export async function importSeed(
|
||||||
|
db: Db,
|
||||||
|
seed: SeedFile,
|
||||||
|
{ update = false }: { update?: boolean } = {}
|
||||||
|
): Promise<ImportResult> {
|
||||||
|
const result: ImportResult = {
|
||||||
|
areas: { inserted: 0, existing: 0 },
|
||||||
|
goals: { inserted: 0, existing: 0, updated: 0 },
|
||||||
|
tasks: { inserted: 0, existing: 0, updated: 0 }
|
||||||
|
};
|
||||||
|
await db.transaction(async (tx) => {
|
||||||
|
const areaIds = new Map<string, number>();
|
||||||
|
for (const a of seed.areas) {
|
||||||
|
const [row] = await tx
|
||||||
|
.insert(todoAreas)
|
||||||
|
.values({ ...a, description: a.description ?? null, color: a.color ?? null })
|
||||||
|
.onConflictDoNothing({ target: todoAreas.slug })
|
||||||
|
.returning({ id: todoAreas.id });
|
||||||
|
if (row) result.areas.inserted++;
|
||||||
|
else result.areas.existing++;
|
||||||
|
}
|
||||||
|
for (const a of await tx.select({ id: todoAreas.id, slug: todoAreas.slug }).from(todoAreas))
|
||||||
|
areaIds.set(a.slug, a.id);
|
||||||
|
const areaId = (slug: string | null | undefined) => {
|
||||||
|
if (!slug) return null;
|
||||||
|
const id = areaIds.get(slug);
|
||||||
|
if (id === undefined) throw new Error(`Unknown area "${slug}"`);
|
||||||
|
return id;
|
||||||
|
};
|
||||||
|
|
||||||
|
const goalIds = new Map<string, number>();
|
||||||
|
for (const g of seed.goals) {
|
||||||
|
const values = {
|
||||||
|
title: g.title,
|
||||||
|
notes: g.notes ?? null,
|
||||||
|
status: g.status,
|
||||||
|
areaId: areaId(g.area),
|
||||||
|
targetDate: g.targetDate ?? null,
|
||||||
|
metric: g.metric ?? null,
|
||||||
|
metricUnit: g.metricUnit ?? null,
|
||||||
|
metricTarget: g.metricTarget ?? null,
|
||||||
|
metricCurrent: g.metricCurrent ?? null,
|
||||||
|
flagged: g.flagged,
|
||||||
|
source: seed.source
|
||||||
|
};
|
||||||
|
const [existing] = await tx
|
||||||
|
.select({ id: todoGoals.id })
|
||||||
|
.from(todoGoals)
|
||||||
|
.where(sql`lower(${todoGoals.title}) = lower(${g.title})`);
|
||||||
|
if (existing) {
|
||||||
|
result.goals.existing++;
|
||||||
|
if (update) {
|
||||||
|
await tx.update(todoGoals).set(values).where(eq(todoGoals.id, existing.id));
|
||||||
|
result.goals.updated++;
|
||||||
|
}
|
||||||
|
goalIds.set(g.title.toLowerCase(), existing.id);
|
||||||
|
} else {
|
||||||
|
const [row] = await tx.insert(todoGoals).values(values).returning({ id: todoGoals.id });
|
||||||
|
result.goals.inserted++;
|
||||||
|
goalIds.set(g.title.toLowerCase(), row.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const goalId = async (title: string | null | undefined) => {
|
||||||
|
if (!title) return null;
|
||||||
|
const known = goalIds.get(title.toLowerCase());
|
||||||
|
if (known) return known;
|
||||||
|
const [row] = await tx
|
||||||
|
.select({ id: todoGoals.id })
|
||||||
|
.from(todoGoals)
|
||||||
|
.where(sql`lower(${todoGoals.title}) = lower(${title})`);
|
||||||
|
if (!row) throw new Error(`Unknown goal "${title}"`);
|
||||||
|
return row.id;
|
||||||
|
};
|
||||||
|
|
||||||
|
const findTask = async (area: number | null, title: string) => {
|
||||||
|
const [row] = await tx
|
||||||
|
.select({ id: todoTasks.id })
|
||||||
|
.from(todoTasks)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
area === null ? isNull(todoTasks.areaId) : eq(todoTasks.areaId, area),
|
||||||
|
sql`lower(${todoTasks.title}) = lower(${title})`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.limit(1);
|
||||||
|
return row?.id ?? null;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Parents first, so subtasks can point at them.
|
||||||
|
const ordered = [...seed.tasks].sort((a, b) => Number(!!a.parent) - Number(!!b.parent));
|
||||||
|
for (const [i, t] of ordered.entries()) {
|
||||||
|
const area = areaId(t.area);
|
||||||
|
const parentId = t.parent ? await findTask(area, t.parent) : null;
|
||||||
|
if (t.parent && parentId === null) throw new Error(`Unknown parent task "${t.parent}"`);
|
||||||
|
const values = {
|
||||||
|
title: t.title,
|
||||||
|
notes: t.notes ?? null,
|
||||||
|
status: t.status,
|
||||||
|
priority: t.priority,
|
||||||
|
flagged: t.flagged,
|
||||||
|
dueDate: t.due ?? null,
|
||||||
|
dueTime: t.dueTime ?? null,
|
||||||
|
reminderAt: t.reminderAt ? parseLocalMoment(t.reminderAt) : null,
|
||||||
|
recurrence: t.recurrence ?? null,
|
||||||
|
areaId: area,
|
||||||
|
goalId: await goalId(t.goal),
|
||||||
|
parentId,
|
||||||
|
tags: t.tags,
|
||||||
|
sortOrder: (i + 1) * 10,
|
||||||
|
source: seed.source,
|
||||||
|
...statusTimestamps(t.status, null)
|
||||||
|
};
|
||||||
|
const existing = await findTask(area, t.title);
|
||||||
|
if (existing !== null) {
|
||||||
|
result.tasks.existing++;
|
||||||
|
if (update) {
|
||||||
|
const rest: Partial<typeof values> = { ...values };
|
||||||
|
delete rest.startedAt;
|
||||||
|
delete rest.completedAt;
|
||||||
|
await tx.update(todoTasks).set(rest).where(eq(todoTasks.id, existing));
|
||||||
|
result.tasks.updated++;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const [row] = await tx.insert(todoTasks).values(values).returning({ id: todoTasks.id });
|
||||||
|
await tx.insert(todoActivity).values({
|
||||||
|
taskId: row.id,
|
||||||
|
kind: 'created',
|
||||||
|
toStatus: t.status,
|
||||||
|
note: `Imported (${seed.source})`
|
||||||
|
});
|
||||||
|
result.tasks.inserted++;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function offsetMinutes(at: Date, timeZone: string): number {
|
||||||
|
const parts = Object.fromEntries(
|
||||||
|
new Intl.DateTimeFormat('en-US', {
|
||||||
|
timeZone,
|
||||||
|
hourCycle: 'h23',
|
||||||
|
year: 'numeric',
|
||||||
|
month: '2-digit',
|
||||||
|
day: '2-digit',
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
second: '2-digit'
|
||||||
|
})
|
||||||
|
.formatToParts(at)
|
||||||
|
.map((p) => [p.type, p.value])
|
||||||
|
);
|
||||||
|
const wall = Date.UTC(
|
||||||
|
+parts.year,
|
||||||
|
+parts.month - 1,
|
||||||
|
+parts.day,
|
||||||
|
+parts.hour,
|
||||||
|
+parts.minute,
|
||||||
|
+parts.second
|
||||||
|
);
|
||||||
|
return Math.round((wall - at.getTime()) / 60_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "2026-12-07T09:00" (a datetime-local value, Amsterdam wall clock) → Date. ISO with offset passes through. */
|
||||||
|
export function parseLocalMoment(value: string, timeZone = TIME_ZONE): Date | null {
|
||||||
|
const local = value.match(/^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})(:\d{2})?$/);
|
||||||
|
if (!local) {
|
||||||
|
const d = new Date(value);
|
||||||
|
return Number.isNaN(d.getTime()) ? null : d;
|
||||||
|
}
|
||||||
|
const guess = new Date(`${local[1]}T${local[2]}${local[3] ?? ':00'}Z`);
|
||||||
|
if (Number.isNaN(guess.getTime())) return null;
|
||||||
|
let result = new Date(guess.getTime() - offsetMinutes(guess, timeZone) * 60_000);
|
||||||
|
const corrected = offsetMinutes(result, timeZone);
|
||||||
|
result = new Date(guess.getTime() - corrected * 60_000);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Today's date in Amsterdam, YYYY-MM-DD. */
|
||||||
|
export function todayIn(timeZone = TIME_ZONE, now = new Date()): string {
|
||||||
|
return new Intl.DateTimeFormat('en-CA', { timeZone }).format(now);
|
||||||
|
}
|
||||||
140
src/lib/server/ticktick.test.ts
Normal file
140
src/lib/server/ticktick.test.ts
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import {
|
||||||
|
mapTask,
|
||||||
|
mergeCsv,
|
||||||
|
parseTickTickCsv,
|
||||||
|
moreComplete,
|
||||||
|
normTitle,
|
||||||
|
slugify,
|
||||||
|
ttDate,
|
||||||
|
ttDue,
|
||||||
|
ttPriority,
|
||||||
|
ttRecurrence,
|
||||||
|
ttReminder,
|
||||||
|
ttStatus
|
||||||
|
} from './ticktick';
|
||||||
|
|
||||||
|
describe('ticktick mapping', () => {
|
||||||
|
it('parses TickTick timestamps', () => {
|
||||||
|
expect(ttDate('2024-06-11T17:30:36.000+0000')?.toISOString()).toBe('2024-06-11T17:30:36.000Z');
|
||||||
|
expect(ttDate('')).toBeNull();
|
||||||
|
});
|
||||||
|
it('keeps the calendar date of all-day dues and converts timed dues to Amsterdam', () => {
|
||||||
|
expect(ttDue('2025-03-11T23:00:00.000+0000', true, 'Europe/Amsterdam')).toEqual({
|
||||||
|
dueDate: '2025-03-12',
|
||||||
|
dueTime: null
|
||||||
|
});
|
||||||
|
expect(ttDue('2026-05-04T04:00:00.000+0000', false, 'Europe/Berlin')).toEqual({
|
||||||
|
dueDate: '2026-05-04',
|
||||||
|
dueTime: '06:00'
|
||||||
|
});
|
||||||
|
expect(ttDue(undefined, true, undefined)).toEqual({ dueDate: null, dueTime: null });
|
||||||
|
});
|
||||||
|
it('maps priority, status, recurrence and reminders', () => {
|
||||||
|
expect([5, 3, 1, 0, undefined].map(ttPriority)).toEqual([3, 2, 1, 0, 0]);
|
||||||
|
expect([0, 2, -1].map(ttStatus)).toEqual(['todo', 'done', 'dropped']);
|
||||||
|
expect(ttRecurrence('RRULE:FREQ=WEEKLY;INTERVAL=1')).toBe('FREQ=WEEKLY;INTERVAL=1');
|
||||||
|
const due = new Date('2026-01-01T10:00:00Z');
|
||||||
|
expect(ttReminder(['TRIGGER:-PT60M', 'TRIGGER:PT0S'], due)?.toISOString()).toBe(
|
||||||
|
'2026-01-01T09:00:00.000Z'
|
||||||
|
);
|
||||||
|
expect(ttReminder(['TRIGGER:P0DT9H0M0S'], due)?.toISOString()).toBe('2026-01-01T19:00:00.000Z');
|
||||||
|
});
|
||||||
|
it('prefers the more complete status', () => {
|
||||||
|
expect(moreComplete('done', 'todo')).toBe('done');
|
||||||
|
expect(moreComplete('todo', 'done')).toBe('done');
|
||||||
|
expect(moreComplete('in_progress', 'todo')).toBe('in_progress');
|
||||||
|
});
|
||||||
|
it('turns checklist items into subtasks', () => {
|
||||||
|
const rows = mapTask({
|
||||||
|
id: 't1',
|
||||||
|
projectId: 'inbox1',
|
||||||
|
title: ' Groceries ',
|
||||||
|
kind: 'CHECKLIST',
|
||||||
|
desc: 'weekly',
|
||||||
|
status: 2,
|
||||||
|
priority: 5,
|
||||||
|
createdTime: '2025-01-01T10:00:00.000+0000',
|
||||||
|
modifiedTime: '2025-01-02T10:00:00.000+0000',
|
||||||
|
completedTime: '2025-01-02T09:00:00.000+0000',
|
||||||
|
items: [
|
||||||
|
{ id: 'i1', title: 'milk', status: 1, completedTime: '2025-01-02T08:00:00.000+0000' },
|
||||||
|
{ id: 'i2', title: 'eggs', status: 0 },
|
||||||
|
{ id: 'i3', title: 'bread', status: 1, completedTime: 978307200000 }
|
||||||
|
]
|
||||||
|
} as Parameters<typeof mapTask>[0]);
|
||||||
|
expect(rows).toHaveLength(4);
|
||||||
|
// Placeholder completion (2001) falls back to the task's completion.
|
||||||
|
expect(rows[3].completedAt?.toISOString()).toBe('2025-01-02T09:00:00.000Z');
|
||||||
|
expect(rows[0]).toMatchObject({
|
||||||
|
title: 'Groceries',
|
||||||
|
notes: 'weekly',
|
||||||
|
status: 'done',
|
||||||
|
priority: 3
|
||||||
|
});
|
||||||
|
expect(rows[0].completedAt?.toISOString()).toBe('2025-01-02T09:00:00.000Z');
|
||||||
|
expect(rows[1]).toMatchObject({ externalId: 't1:i1', parentExternalId: 't1', status: 'done' });
|
||||||
|
// Unchecked item of a completed task follows the task.
|
||||||
|
expect(rows[2]).toMatchObject({ externalId: 't1:i2', status: 'done' });
|
||||||
|
expect(rows[2].completedAt?.toISOString()).toBe('2025-01-02T09:00:00.000Z');
|
||||||
|
});
|
||||||
|
it('slugs and normalises titles', () => {
|
||||||
|
expect(slugify('Stuhee')).toBe('stuhee');
|
||||||
|
expect(normTitle('Book haircut!')).toBe('book haircut');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('ticktick backup csv', () => {
|
||||||
|
const csv = [
|
||||||
|
'"Date: 2026-10-24+0000"',
|
||||||
|
'"Status: ',
|
||||||
|
'0 Normal',
|
||||||
|
'2 Completed"',
|
||||||
|
'"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId","projectKind"',
|
||||||
|
'"","Inbox","Known","TEXT","","","N","","","","","0","2","2025-01-01T10:00:00+0000","2025-01-02T10:00:00+0000","0","Europe/Amsterdam","false","false",,,"list","1","","TASK"',
|
||||||
|
'"","Inbox","Old won\'t do","TEXT","a, b","line ""one""\nline two","N","","","","","5","-1","2024-01-01T10:00:00+0000","2024-02-01T10:00:00+0000","0","Europe/Amsterdam","false","false",,,"list","2","1","TASK"',
|
||||||
|
'"Old","Trips","Pack","CHECKLIST","","▫Tea\n▪Jerky\n","Y","","","","","0","0","2023-01-01T10:00:00+0000","","0","Europe/Amsterdam","false","false",,,"list","3","","TASK"',
|
||||||
|
''
|
||||||
|
].join('\n');
|
||||||
|
const api = {
|
||||||
|
projects: [],
|
||||||
|
tasks: [
|
||||||
|
{
|
||||||
|
id: 'api1',
|
||||||
|
projectId: 'inbox9',
|
||||||
|
title: 'Known',
|
||||||
|
status: 2,
|
||||||
|
createdTime: '2025-01-01T10:00:00.000+0000'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
};
|
||||||
|
it('parses the backup, including multi-line quoted fields and checklists', () => {
|
||||||
|
const t = parseTickTickCsv(csv);
|
||||||
|
expect(t).toHaveLength(3);
|
||||||
|
expect(t[1]).toMatchObject({ content: 'line "one"\nline two', tags: ['a', 'b'], status: -1 });
|
||||||
|
expect(t[2].items).toEqual([
|
||||||
|
{ id: 'item1', title: 'Tea', status: 0 },
|
||||||
|
{ id: 'item2', title: 'Jerky', status: 1 }
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
it('skips rows the API already has and keys the rest stably', () => {
|
||||||
|
const a = mergeCsv(api, parseTickTickCsv(csv));
|
||||||
|
const b = mergeCsv(api, parseTickTickCsv(csv));
|
||||||
|
expect(a.report).toMatchObject({
|
||||||
|
csvRows: 3,
|
||||||
|
matchedListTitleCreated: 1,
|
||||||
|
added: 2,
|
||||||
|
newLists: ['Trips']
|
||||||
|
});
|
||||||
|
const added = a.data.tasks.slice(1);
|
||||||
|
expect(added.map((t) => t.id)).toEqual(b.data.tasks.slice(1).map((t) => t.id));
|
||||||
|
expect(added[0].id).toMatch(/^csv:/);
|
||||||
|
expect(added[0]).toMatchObject({ projectId: 'inbox9', parentId: 'api1' });
|
||||||
|
expect(added[1].projectId).toBe('csvlist:Trips');
|
||||||
|
expect(a.data.projects).toContainEqual({
|
||||||
|
id: 'csvlist:Trips',
|
||||||
|
name: 'Old / Trips',
|
||||||
|
closed: true
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
723
src/lib/server/ticktick.ts
Normal file
723
src/lib/server/ticktick.ts
Normal file
@@ -0,0 +1,723 @@
|
|||||||
|
// TickTick → tasks schema sync. Pure mapping + an idempotent sync, shared by
|
||||||
|
// scripts/ticktick-import.ts and its tests; must not import anything from SvelteKit.
|
||||||
|
// Input is the JSON written by scripts/ticktick-export.ts (Open API v1 task objects).
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { and, eq, isNotNull, isNull, sql } from 'drizzle-orm';
|
||||||
|
import type { PgAsyncDatabase } from 'drizzle-orm/pg-core';
|
||||||
|
import { todoActivity, todoAreas, todoTasks, type TaskStatus } from './db/schema/tasks';
|
||||||
|
import { TIME_ZONE } from './tasks';
|
||||||
|
|
||||||
|
export const TICKTICK_SOURCE = 'ticktick';
|
||||||
|
|
||||||
|
export type TtItem = {
|
||||||
|
id: string;
|
||||||
|
title?: string;
|
||||||
|
status?: number;
|
||||||
|
completedTime?: string | number;
|
||||||
|
sortOrder?: number;
|
||||||
|
startDate?: string;
|
||||||
|
isAllDay?: boolean;
|
||||||
|
timeZone?: string;
|
||||||
|
};
|
||||||
|
export type TtTask = {
|
||||||
|
id: string;
|
||||||
|
projectId: string;
|
||||||
|
title?: string;
|
||||||
|
content?: string;
|
||||||
|
desc?: string;
|
||||||
|
status?: number;
|
||||||
|
priority?: number;
|
||||||
|
tags?: string[];
|
||||||
|
dueDate?: string;
|
||||||
|
startDate?: string;
|
||||||
|
isAllDay?: boolean;
|
||||||
|
timeZone?: string;
|
||||||
|
repeatFlag?: string;
|
||||||
|
reminders?: string[];
|
||||||
|
items?: TtItem[];
|
||||||
|
parentId?: string;
|
||||||
|
sortOrder?: number;
|
||||||
|
createdTime?: string;
|
||||||
|
modifiedTime?: string;
|
||||||
|
completedTime?: string;
|
||||||
|
};
|
||||||
|
export type TtProject = { id: string; name: string; closed?: boolean };
|
||||||
|
export type TtExport = { projects: TtProject[]; tasks: TtTask[]; inboxId?: string };
|
||||||
|
/** Hand-picked links: a TickTick task that is clearly the same as an existing task. */
|
||||||
|
export type TtMerge = { ticktickId: string; title: string };
|
||||||
|
|
||||||
|
/** TickTick timestamps look like 2024-06-11T17:30:36.000+0000. */
|
||||||
|
export function ttDate(v: string | number | null | undefined): Date | null {
|
||||||
|
if (v === null || v === undefined || v === '') return null;
|
||||||
|
const d =
|
||||||
|
typeof v === 'number' ? new Date(v) : new Date(v.replace(/([+-]\d{2})(\d{2})$/, '$1:$2'));
|
||||||
|
return Number.isNaN(d.getTime()) ? null : d;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parts = (d: Date, timeZone: string) => {
|
||||||
|
const f = new Intl.DateTimeFormat('en-GB', {
|
||||||
|
timeZone,
|
||||||
|
year: 'numeric',
|
||||||
|
month: '2-digit',
|
||||||
|
day: '2-digit',
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
hourCycle: 'h23'
|
||||||
|
}).formatToParts(d);
|
||||||
|
const g = (t: string) => f.find((p) => p.type === t)?.value ?? '';
|
||||||
|
return { date: `${g('year')}-${g('month')}-${g('day')}`, time: `${g('hour')}:${g('minute')}` };
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* All-day dues are midnight in the task's own zone (keep that calendar date);
|
||||||
|
* timed dues become an Amsterdam wall-clock date + time (due_time is Amsterdam).
|
||||||
|
*/
|
||||||
|
export function ttDue(
|
||||||
|
dueDate: string | undefined,
|
||||||
|
isAllDay: boolean | undefined,
|
||||||
|
timeZone: string | undefined
|
||||||
|
): { dueDate: string | null; dueTime: string | null } {
|
||||||
|
const d = ttDate(dueDate);
|
||||||
|
if (!d) return { dueDate: null, dueTime: null };
|
||||||
|
if (isAllDay) return { dueDate: parts(d, validZone(timeZone)).date, dueTime: null };
|
||||||
|
const p = parts(d, TIME_ZONE);
|
||||||
|
return { dueDate: p.date, dueTime: p.time };
|
||||||
|
}
|
||||||
|
|
||||||
|
function validZone(z: string | undefined) {
|
||||||
|
if (!z) return TIME_ZONE;
|
||||||
|
try {
|
||||||
|
new Intl.DateTimeFormat('en', { timeZone: z });
|
||||||
|
return z;
|
||||||
|
} catch {
|
||||||
|
return TIME_ZONE;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** First reminder: TRIGGER:PT0S / -PT60M / P0DT9H0M0S (relative to the due moment). */
|
||||||
|
export function ttReminder(reminders: string[] | undefined, due: Date | null): Date | null {
|
||||||
|
if (!due || !reminders?.length) return null;
|
||||||
|
let best: Date | null = null;
|
||||||
|
for (const r of reminders) {
|
||||||
|
const m = r.match(
|
||||||
|
/^TRIGGER:(-)?P(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?$/
|
||||||
|
);
|
||||||
|
if (!m) continue;
|
||||||
|
const [, neg, w, d, h, min, s] = m;
|
||||||
|
const ms =
|
||||||
|
((Number(w ?? 0) * 7 + Number(d ?? 0)) * 86_400 +
|
||||||
|
Number(h ?? 0) * 3600 +
|
||||||
|
Number(min ?? 0) * 60 +
|
||||||
|
Number(s ?? 0)) *
|
||||||
|
1000;
|
||||||
|
const at = new Date(due.getTime() + (neg ? -ms : ms));
|
||||||
|
if (!best || at < best) best = at;
|
||||||
|
}
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const ttPriority = (p: number | undefined) => (p === 5 ? 3 : p === 3 ? 2 : p === 1 ? 1 : 0);
|
||||||
|
export const ttStatus = (s: number | undefined): TaskStatus =>
|
||||||
|
s === 2 ? 'done' : s === -1 ? 'dropped' : 'todo';
|
||||||
|
/** TickTick stores "RRULE:FREQ=…"; the tracker keeps the bare RFC 5545 rule. */
|
||||||
|
export const ttRecurrence = (r: string | undefined) => r?.replace(/^RRULE:/, '').trim() || null;
|
||||||
|
|
||||||
|
const RANK: Record<TaskStatus, number> = { todo: 0, in_progress: 1, dropped: 2, done: 3 };
|
||||||
|
/** The more complete of two statuses (done > dropped > in progress > to do). */
|
||||||
|
export const moreComplete = (a: TaskStatus, b: TaskStatus) => (RANK[b] > RANK[a] ? b : a);
|
||||||
|
|
||||||
|
export const slugify = (s: string) =>
|
||||||
|
s
|
||||||
|
.normalize('NFKD')
|
||||||
|
.replace(/[^\p{L}\p{N}]+/gu, '-')
|
||||||
|
.replace(/^-+|-+$/g, '')
|
||||||
|
.toLowerCase() || 'list';
|
||||||
|
export const normTitle = (s: string) =>
|
||||||
|
s
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
export type MappedRow = {
|
||||||
|
externalId: string;
|
||||||
|
parentExternalId: string | null;
|
||||||
|
projectId: string;
|
||||||
|
title: string;
|
||||||
|
notes: string | null;
|
||||||
|
status: TaskStatus;
|
||||||
|
priority: number;
|
||||||
|
tags: string[];
|
||||||
|
dueDate: string | null;
|
||||||
|
dueTime: string | null;
|
||||||
|
startAt: Date | null;
|
||||||
|
timeZone: string | null;
|
||||||
|
reminderAt: Date | null;
|
||||||
|
recurrence: string | null;
|
||||||
|
completedAt: Date | null;
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
ttSortOrder: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** One TickTick task → its row plus one row per checklist item (as subtasks). */
|
||||||
|
export function mapTask(t: TtTask): MappedRow[] {
|
||||||
|
const status = ttStatus(t.status);
|
||||||
|
const created =
|
||||||
|
ttDate(t.createdTime) ?? ttDate(t.modifiedTime) ?? ttDate(t.completedTime) ?? new Date(0);
|
||||||
|
const modified = ttDate(t.modifiedTime) ?? created;
|
||||||
|
const due = ttDue(t.dueDate, t.isAllDay, t.timeZone);
|
||||||
|
const notes = (t.content?.trim() || t.desc?.trim() || '').slice(0, 50_000) || null;
|
||||||
|
const done = status === 'done' || status === 'dropped';
|
||||||
|
const row: MappedRow = {
|
||||||
|
externalId: t.id,
|
||||||
|
parentExternalId: t.parentId || null,
|
||||||
|
projectId: t.projectId,
|
||||||
|
title: t.title?.trim() || '(untitled)',
|
||||||
|
notes,
|
||||||
|
status,
|
||||||
|
priority: ttPriority(t.priority),
|
||||||
|
tags: [...new Set((t.tags ?? []).map((x) => x.toLowerCase()))],
|
||||||
|
...due,
|
||||||
|
startAt: ttDate(t.startDate),
|
||||||
|
timeZone: t.timeZone || null,
|
||||||
|
reminderAt: ttReminder(t.reminders, ttDate(t.dueDate)),
|
||||||
|
recurrence: ttRecurrence(t.repeatFlag),
|
||||||
|
completedAt: done ? (ttDate(t.completedTime) ?? modified) : null,
|
||||||
|
createdAt: created,
|
||||||
|
updatedAt: modified,
|
||||||
|
ttSortOrder: t.sortOrder ?? 0
|
||||||
|
};
|
||||||
|
// Some old checklist items carry a placeholder completedTime (2001-01-01): ignore any
|
||||||
|
// completion before the task existed.
|
||||||
|
const itemDone = (v: string | number | undefined) => {
|
||||||
|
const d = ttDate(v);
|
||||||
|
return d && d.getTime() >= created.getTime() - 86_400_000 ? d : null;
|
||||||
|
};
|
||||||
|
const items = (t.items ?? []).map((i): MappedRow => {
|
||||||
|
// Checklist status: 0 open, 1 (or 2) checked. Unchecked items of a finished task
|
||||||
|
// follow the task (done → done, won't do → dropped) so they don't resurface as open.
|
||||||
|
const checked = (i.status ?? 0) !== 0 || status === 'done';
|
||||||
|
return {
|
||||||
|
externalId: `${t.id}:${i.id}`,
|
||||||
|
parentExternalId: t.id,
|
||||||
|
projectId: t.projectId,
|
||||||
|
title: i.title?.trim() || '(untitled)',
|
||||||
|
notes: null,
|
||||||
|
status: checked ? 'done' : status === 'dropped' ? 'dropped' : 'todo',
|
||||||
|
priority: 0,
|
||||||
|
tags: [],
|
||||||
|
...ttDue(i.startDate, i.isAllDay ?? true, i.timeZone ?? t.timeZone),
|
||||||
|
startAt: null,
|
||||||
|
timeZone: i.timeZone || t.timeZone || null,
|
||||||
|
reminderAt: null,
|
||||||
|
recurrence: null,
|
||||||
|
completedAt: checked
|
||||||
|
? (itemDone(i.completedTime) ?? row.completedAt ?? modified)
|
||||||
|
: status === 'dropped'
|
||||||
|
? row.completedAt
|
||||||
|
: null,
|
||||||
|
createdAt: created,
|
||||||
|
updatedAt: modified,
|
||||||
|
ttSortOrder: i.sortOrder ?? 0
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return [row, ...items];
|
||||||
|
}
|
||||||
|
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
type Db = PgAsyncDatabase<any, any>;
|
||||||
|
|
||||||
|
export type SyncReport = {
|
||||||
|
areas: { inserted: string[]; existing: string[] };
|
||||||
|
tasks: { inserted: number; updated: number; unchanged: number; subtasks: number };
|
||||||
|
merged: { ticktick: string; task: string; status: TaskStatus }[];
|
||||||
|
/** Rows inserted by this run. */
|
||||||
|
newByStatus: Partial<Record<TaskStatus, number>>;
|
||||||
|
newByArea: Record<string, number>;
|
||||||
|
byStatus: Record<TaskStatus, number>;
|
||||||
|
byArea: Record<string, number>;
|
||||||
|
range: { created: [string, string] | null; completed: [string, string] | null };
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upsert a TickTick export (idempotent). Rows are keyed on external_id = TickTick id
|
||||||
|
* (checklist items: "<taskId>:<itemId>"). A TickTick row is only overwritten when
|
||||||
|
* TickTick modified it after our last update, so edits made on the site win until
|
||||||
|
* the task changes in TickTick again. Links to existing (e.g. daisy-import) tasks:
|
||||||
|
* the explicit `merges` list, plus an exact title match when the TickTick task is
|
||||||
|
* still open or was finished after that task was created. A linked task keeps its
|
||||||
|
* own fields; only its status moves to the more complete of the two.
|
||||||
|
*/
|
||||||
|
export async function syncTickTick(
|
||||||
|
db: Db,
|
||||||
|
data: TtExport,
|
||||||
|
opts: { merges?: TtMerge[] } = {}
|
||||||
|
): Promise<SyncReport> {
|
||||||
|
const report: SyncReport = {
|
||||||
|
areas: { inserted: [], existing: [] },
|
||||||
|
tasks: { inserted: 0, updated: 0, unchanged: 0, subtasks: 0 },
|
||||||
|
merged: [],
|
||||||
|
newByStatus: {},
|
||||||
|
newByArea: {},
|
||||||
|
byStatus: { todo: 0, in_progress: 0, done: 0, dropped: 0 },
|
||||||
|
byArea: {},
|
||||||
|
range: { created: null, completed: null }
|
||||||
|
};
|
||||||
|
const rows = data.tasks.flatMap(mapTask);
|
||||||
|
report.tasks.subtasks = rows.filter((r) => r.parentExternalId).length;
|
||||||
|
|
||||||
|
return db.transaction(async (tx) => {
|
||||||
|
// Areas: one per TickTick list. The inbox and any list no longer returned
|
||||||
|
// (closed/deleted) get their own area.
|
||||||
|
const projectName = new Map(data.projects.map((p) => [p.id, p.name]));
|
||||||
|
const areaFor = new Map<string, number>();
|
||||||
|
const [{ maxSort }] = await tx
|
||||||
|
.select({ maxSort: sql<number>`coalesce(max(${todoAreas.sortOrder}), 0)` })
|
||||||
|
.from(todoAreas);
|
||||||
|
let sort = Number(maxSort);
|
||||||
|
for (const pid of new Set(rows.map((r) => r.projectId))) {
|
||||||
|
const isInbox = pid.startsWith('inbox');
|
||||||
|
const raw = isInbox ? 'Inbox' : (projectName.get(pid) ?? `TickTick list ${pid.slice(-6)}`);
|
||||||
|
const name = raw.replace(/^[^\p{L}\p{N}]+/u, '').trim() || raw;
|
||||||
|
const slug = slugify(name);
|
||||||
|
const [found] = await tx.select().from(todoAreas).where(eq(todoAreas.slug, slug));
|
||||||
|
if (found) {
|
||||||
|
areaFor.set(pid, found.id);
|
||||||
|
report.areas.existing.push(slug);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const [ins] = await tx
|
||||||
|
.insert(todoAreas)
|
||||||
|
.values({
|
||||||
|
slug,
|
||||||
|
name,
|
||||||
|
description: isInbox ? 'TickTick inbox' : `TickTick list "${raw}"`,
|
||||||
|
sortOrder: (sort += 10)
|
||||||
|
})
|
||||||
|
.returning({ id: todoAreas.id });
|
||||||
|
areaFor.set(pid, ins.id);
|
||||||
|
report.areas.inserted.push(slug);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Site sort order: rank by TickTick sortOrder within each list (TickTick values overflow int4).
|
||||||
|
const siteSort = new Map<string, number>();
|
||||||
|
const perProject = new Map<string, MappedRow[]>();
|
||||||
|
for (const r of rows) {
|
||||||
|
const list = perProject.get(r.projectId) ?? [];
|
||||||
|
list.push(r);
|
||||||
|
perProject.set(r.projectId, list);
|
||||||
|
}
|
||||||
|
for (const list of perProject.values())
|
||||||
|
list
|
||||||
|
.sort(
|
||||||
|
(a, b) => a.ttSortOrder - b.ttSortOrder || a.createdAt.getTime() - b.createdAt.getTime()
|
||||||
|
)
|
||||||
|
.forEach((r, i) => siteSort.set(r.externalId, (i + 1) * 10));
|
||||||
|
|
||||||
|
const existing = new Map(
|
||||||
|
(
|
||||||
|
await tx
|
||||||
|
.select({
|
||||||
|
id: todoTasks.id,
|
||||||
|
source: todoTasks.source,
|
||||||
|
externalId: todoTasks.externalId,
|
||||||
|
status: todoTasks.status,
|
||||||
|
updatedAt: todoTasks.updatedAt
|
||||||
|
})
|
||||||
|
.from(todoTasks)
|
||||||
|
.where(isNotNull(todoTasks.externalId))
|
||||||
|
).map((r) => [r.externalId!, r])
|
||||||
|
);
|
||||||
|
// Unlinked non-TickTick tasks are merge candidates.
|
||||||
|
const candidates = await tx
|
||||||
|
.select({
|
||||||
|
id: todoTasks.id,
|
||||||
|
title: todoTasks.title,
|
||||||
|
status: todoTasks.status,
|
||||||
|
createdAt: todoTasks.createdAt
|
||||||
|
})
|
||||||
|
.from(todoTasks)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
isNull(todoTasks.externalId),
|
||||||
|
sql`coalesce(${todoTasks.source}, '') <> ${TICKTICK_SOURCE}`
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const byTitle = new Map<string, typeof candidates>();
|
||||||
|
for (const c of candidates) {
|
||||||
|
const k = normTitle(c.title);
|
||||||
|
byTitle.set(k, [...(byTitle.get(k) ?? []), c]);
|
||||||
|
}
|
||||||
|
const explicit = new Map((opts.merges ?? []).map((m) => [m.ticktickId, normTitle(m.title)]));
|
||||||
|
const linked = new Set<number>();
|
||||||
|
|
||||||
|
const idFor = new Map<string, number>();
|
||||||
|
const activity: (typeof todoActivity.$inferInsert)[] = [];
|
||||||
|
|
||||||
|
for (const r of rows) {
|
||||||
|
let prev = existing.get(r.externalId);
|
||||||
|
// A task first imported from the backup CSV (csv:… key) that the API now returns:
|
||||||
|
// move it onto the API id instead of inserting a duplicate.
|
||||||
|
if (!prev && !r.externalId.startsWith('csv:') && !r.parentExternalId) {
|
||||||
|
const alias = existing.get(
|
||||||
|
csvKey(listNameOf(data.projects, r.projectId), r.title, r.createdAt)
|
||||||
|
);
|
||||||
|
if (alias && alias.source === TICKTICK_SOURCE) {
|
||||||
|
await tx
|
||||||
|
.update(todoTasks)
|
||||||
|
.set({ externalId: r.externalId })
|
||||||
|
.where(eq(todoTasks.id, alias.id));
|
||||||
|
existing.delete(alias.externalId!);
|
||||||
|
prev = { ...alias, externalId: r.externalId };
|
||||||
|
existing.set(r.externalId, prev);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const fields = {
|
||||||
|
title: r.title,
|
||||||
|
notes: r.notes,
|
||||||
|
status: r.status,
|
||||||
|
priority: r.priority,
|
||||||
|
tags: r.tags,
|
||||||
|
dueDate: r.dueDate,
|
||||||
|
dueTime: r.dueTime,
|
||||||
|
startAt: r.startAt,
|
||||||
|
timeZone: r.timeZone,
|
||||||
|
reminderAt: r.reminderAt,
|
||||||
|
recurrence: r.recurrence,
|
||||||
|
completedAt: r.completedAt,
|
||||||
|
areaId: areaFor.get(r.projectId) ?? null
|
||||||
|
};
|
||||||
|
if (prev && prev.source === TICKTICK_SOURCE) {
|
||||||
|
idFor.set(r.externalId, prev.id);
|
||||||
|
if (r.updatedAt > prev.updatedAt) {
|
||||||
|
await tx.update(todoTasks).set(fields).where(eq(todoTasks.id, prev.id));
|
||||||
|
if (prev.status !== r.status)
|
||||||
|
activity.push({
|
||||||
|
taskId: prev.id,
|
||||||
|
kind: 'status',
|
||||||
|
fromStatus: prev.status,
|
||||||
|
toStatus: r.status,
|
||||||
|
note: 'TickTick sync',
|
||||||
|
createdAt: r.updatedAt
|
||||||
|
});
|
||||||
|
report.tasks.updated++;
|
||||||
|
} else report.tasks.unchanged++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (prev) {
|
||||||
|
// Linked earlier to a task from another source: only upgrade its status.
|
||||||
|
idFor.set(r.externalId, prev.id);
|
||||||
|
const next = moreComplete(prev.status, r.status);
|
||||||
|
if (next !== prev.status) {
|
||||||
|
await tx
|
||||||
|
.update(todoTasks)
|
||||||
|
.set({ status: next, completedAt: r.completedAt })
|
||||||
|
.where(eq(todoTasks.id, prev.id));
|
||||||
|
report.tasks.updated++;
|
||||||
|
} else report.tasks.unchanged++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!r.parentExternalId) {
|
||||||
|
const want = explicit.get(r.externalId);
|
||||||
|
const pool = (want ? byTitle.get(want) : byTitle.get(normTitle(r.title))) ?? [];
|
||||||
|
const match = pool.filter(
|
||||||
|
(c) =>
|
||||||
|
!linked.has(c.id) &&
|
||||||
|
(want || r.status === 'todo' || (r.completedAt && r.completedAt >= c.createdAt))
|
||||||
|
);
|
||||||
|
if (match.length === 1) {
|
||||||
|
const c = match[0];
|
||||||
|
linked.add(c.id);
|
||||||
|
const next = moreComplete(c.status, r.status);
|
||||||
|
await tx
|
||||||
|
.update(todoTasks)
|
||||||
|
.set({
|
||||||
|
externalId: r.externalId,
|
||||||
|
status: next,
|
||||||
|
...(next !== c.status ? { completedAt: r.completedAt } : {})
|
||||||
|
})
|
||||||
|
.where(eq(todoTasks.id, c.id));
|
||||||
|
activity.push({
|
||||||
|
taskId: c.id,
|
||||||
|
kind: 'note',
|
||||||
|
note: `Linked to TickTick task "${r.title}" (${r.status})`
|
||||||
|
});
|
||||||
|
idFor.set(r.externalId, c.id);
|
||||||
|
report.merged.push({ ticktick: r.title, task: c.title, status: next });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const [ins] = await tx
|
||||||
|
.insert(todoTasks)
|
||||||
|
.values({
|
||||||
|
...fields,
|
||||||
|
sortOrder: siteSort.get(r.externalId) ?? 0,
|
||||||
|
source: TICKTICK_SOURCE,
|
||||||
|
externalId: r.externalId,
|
||||||
|
createdAt: r.createdAt,
|
||||||
|
updatedAt: r.updatedAt
|
||||||
|
})
|
||||||
|
.returning({ id: todoTasks.id });
|
||||||
|
idFor.set(r.externalId, ins.id);
|
||||||
|
activity.push({
|
||||||
|
taskId: ins.id,
|
||||||
|
kind: 'created',
|
||||||
|
toStatus: r.status,
|
||||||
|
note: 'Imported (ticktick)',
|
||||||
|
createdAt: r.createdAt
|
||||||
|
});
|
||||||
|
report.tasks.inserted++;
|
||||||
|
report.newByStatus[r.status] = (report.newByStatus[r.status] ?? 0) + 1;
|
||||||
|
const an = listNameOf(data.projects, r.projectId);
|
||||||
|
report.newByArea[an] = (report.newByArea[an] ?? 0) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parents (TickTick subtasks and checklist items), set after every row exists.
|
||||||
|
for (const r of rows) {
|
||||||
|
if (!r.parentExternalId) continue;
|
||||||
|
const me = idFor.get(r.externalId);
|
||||||
|
const parent = idFor.get(r.parentExternalId);
|
||||||
|
const mine = existing.get(r.externalId);
|
||||||
|
if (!me || !parent || me === parent) continue;
|
||||||
|
if (mine && mine.source !== TICKTICK_SOURCE) continue;
|
||||||
|
await tx
|
||||||
|
.update(todoTasks)
|
||||||
|
.set({ parentId: parent })
|
||||||
|
.where(and(eq(todoTasks.id, me), sql`${todoTasks.parentId} IS DISTINCT FROM ${parent}`));
|
||||||
|
}
|
||||||
|
for (let i = 0; i < activity.length; i += 500)
|
||||||
|
await tx.insert(todoActivity).values(activity.slice(i, i + 500));
|
||||||
|
|
||||||
|
// Report from the database: everything that came from (or is linked to) TickTick.
|
||||||
|
const stats = await tx
|
||||||
|
.select({
|
||||||
|
status: todoTasks.status,
|
||||||
|
area: todoAreas.name,
|
||||||
|
n: sql<number>`count(*)::int`
|
||||||
|
})
|
||||||
|
.from(todoTasks)
|
||||||
|
.leftJoin(todoAreas, eq(todoAreas.id, todoTasks.areaId))
|
||||||
|
.where(isNotNull(todoTasks.externalId))
|
||||||
|
.groupBy(todoTasks.status, todoAreas.name);
|
||||||
|
for (const s of stats) {
|
||||||
|
report.byStatus[s.status] += s.n;
|
||||||
|
const a = s.area ?? 'No area';
|
||||||
|
report.byArea[a] = (report.byArea[a] ?? 0) + s.n;
|
||||||
|
}
|
||||||
|
const iso = (d: Date) => d.toISOString().slice(0, 10);
|
||||||
|
const created = rows
|
||||||
|
.map((r) => r.createdAt)
|
||||||
|
.filter((d) => d.getTime() > 0)
|
||||||
|
.sort((a, b) => +a - +b);
|
||||||
|
const completed = rows
|
||||||
|
.map((r) => r.completedAt)
|
||||||
|
.filter((d): d is Date => !!d)
|
||||||
|
.sort((a, b) => +a - +b);
|
||||||
|
if (created.length) report.range.created = [iso(created[0]), iso(created.at(-1)!)];
|
||||||
|
if (completed.length) report.range.completed = [iso(completed[0]), iso(completed.at(-1)!)];
|
||||||
|
return report;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- TickTick backup CSV (Settings → Backup & Import) --------------------------------
|
||||||
|
// The backup has local row numbers as taskId, so CSV-only tasks get a stable key:
|
||||||
|
// "csv:" + hash(list, title, created second) (+ ":n" for repeats). Rows that also came
|
||||||
|
// through the API are dropped in favour of the API copy (matched on list + title +
|
||||||
|
// created time, falling back to title + created time).
|
||||||
|
|
||||||
|
/** Minimal RFC 4180 parser (quoted fields may contain commas, quotes and newlines). */
|
||||||
|
export function parseCsv(text: string): string[][] {
|
||||||
|
const out: string[][] = [];
|
||||||
|
let row: string[] = [];
|
||||||
|
let field = '';
|
||||||
|
let quoted = false;
|
||||||
|
for (let i = 0; i < text.length; i++) {
|
||||||
|
const c = text[i];
|
||||||
|
if (quoted) {
|
||||||
|
if (c === '"') {
|
||||||
|
if (text[i + 1] === '"') {
|
||||||
|
field += '"';
|
||||||
|
i++;
|
||||||
|
} else quoted = false;
|
||||||
|
} else field += c;
|
||||||
|
} else if (c === '"') quoted = true;
|
||||||
|
else if (c === ',') {
|
||||||
|
row.push(field);
|
||||||
|
field = '';
|
||||||
|
} else if (c === '\n' || c === '\r') {
|
||||||
|
if (c === '\r' && text[i + 1] === '\n') i++;
|
||||||
|
row.push(field);
|
||||||
|
out.push(row);
|
||||||
|
row = [];
|
||||||
|
field = '';
|
||||||
|
} else field += c;
|
||||||
|
}
|
||||||
|
if (field || row.length) {
|
||||||
|
row.push(field);
|
||||||
|
out.push(row);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const csvKey = (list: string, title: string, created: Date | null) =>
|
||||||
|
'csv:' +
|
||||||
|
createHash('sha1')
|
||||||
|
.update(
|
||||||
|
`${list}\u0000${title.trim()}\u0000${created ? Math.floor(created.getTime() / 1000) : ''}`
|
||||||
|
)
|
||||||
|
.digest('hex')
|
||||||
|
.slice(0, 20);
|
||||||
|
|
||||||
|
export type CsvTask = TtTask & {
|
||||||
|
csvId: string;
|
||||||
|
csvParent: string;
|
||||||
|
listName: string;
|
||||||
|
folder: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Backup CSV → API-shaped tasks (ids still unset; see mergeCsv). */
|
||||||
|
export function parseTickTickCsv(text: string): CsvTask[] {
|
||||||
|
const records = parseCsv(text.replace(/^\uFEFF/, ''));
|
||||||
|
const h = records.findIndex((r) => r[0] === 'Folder Name');
|
||||||
|
if (h < 0) throw new Error('Not a TickTick backup: no "Folder Name" header row');
|
||||||
|
const cols = records[h];
|
||||||
|
const tasks: CsvTask[] = [];
|
||||||
|
for (const rec of records.slice(h + 1)) {
|
||||||
|
if (rec.length < cols.length - 1) continue;
|
||||||
|
const r = Object.fromEntries(cols.map((c, i) => [c, rec[i] ?? '']));
|
||||||
|
const isChecklist = r['Kind'] === 'CHECKLIST' || r['Is Check list'] === 'Y';
|
||||||
|
const lines = (r['Content'] ?? '').split('\n');
|
||||||
|
const items: TtItem[] = [];
|
||||||
|
const noteLines: string[] = [];
|
||||||
|
if (isChecklist)
|
||||||
|
for (const l of lines) {
|
||||||
|
const m = l.match(/^\s*([▫▪])\s*(.*)$/u);
|
||||||
|
if (m)
|
||||||
|
items.push({ id: `item${items.length + 1}`, title: m[2], status: m[1] === '▪' ? 1 : 0 });
|
||||||
|
else noteLines.push(l);
|
||||||
|
}
|
||||||
|
const num = (s: string) => (s === '' || Number.isNaN(Number(s)) ? undefined : Number(s));
|
||||||
|
tasks.push({
|
||||||
|
id: '',
|
||||||
|
csvId: r['taskId'],
|
||||||
|
csvParent: r['parentId'],
|
||||||
|
listName: r['List Name'],
|
||||||
|
folder: r['Folder Name'],
|
||||||
|
projectId: '',
|
||||||
|
title: r['Title'],
|
||||||
|
content: isChecklist ? noteLines.join('\n') : r['Content'],
|
||||||
|
status: num(r['Status']),
|
||||||
|
priority: num(r['Priority']),
|
||||||
|
tags: r['Tags']
|
||||||
|
? r['Tags']
|
||||||
|
.split(',')
|
||||||
|
.map((t) => t.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
: [],
|
||||||
|
startDate: r['Start Date'] || undefined,
|
||||||
|
dueDate: r['Due Date'] || undefined,
|
||||||
|
isAllDay: r['Is All Day'] === 'true',
|
||||||
|
timeZone: r['Timezone'] || undefined,
|
||||||
|
repeatFlag: r['Repeat'] || undefined,
|
||||||
|
reminders: r['Reminder']
|
||||||
|
? r['Reminder']
|
||||||
|
.split(',')
|
||||||
|
.map((x) => (x.startsWith('TRIGGER:') ? x : `TRIGGER:${x.trim()}`))
|
||||||
|
: [],
|
||||||
|
items,
|
||||||
|
sortOrder: num(r['Order']),
|
||||||
|
createdTime: r['Created Time'] || undefined,
|
||||||
|
modifiedTime: r['Completed Time'] || r['Created Time'] || undefined,
|
||||||
|
completedTime: r['Completed Time'] || undefined
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return tasks;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const listNameOf = (projects: TtProject[], projectId: string) =>
|
||||||
|
projectId.startsWith('inbox')
|
||||||
|
? 'Inbox'
|
||||||
|
: (projects.find((p) => p.id === projectId)?.name ?? projectId);
|
||||||
|
|
||||||
|
export type CsvMergeReport = {
|
||||||
|
csvRows: number;
|
||||||
|
matchedListTitleCreated: number;
|
||||||
|
matchedTitleCreated: number;
|
||||||
|
added: number;
|
||||||
|
newLists: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Add the CSV-only tasks to an API export (returns a new export + what matched). */
|
||||||
|
export function mergeCsv(
|
||||||
|
data: TtExport,
|
||||||
|
csv: CsvTask[]
|
||||||
|
): { data: TtExport; report: CsvMergeReport } {
|
||||||
|
const sec = (v: string | undefined) => {
|
||||||
|
const d = ttDate(v);
|
||||||
|
return d ? String(Math.floor(d.getTime() / 1000)) : '';
|
||||||
|
};
|
||||||
|
const k1 = new Map<string, string[]>();
|
||||||
|
const k2 = new Map<string, string[]>();
|
||||||
|
for (const t of data.tasks) {
|
||||||
|
const title = (t.title ?? '').trim();
|
||||||
|
const a = `${listNameOf(data.projects, t.projectId)}\u0000${title}\u0000${sec(t.createdTime)}`;
|
||||||
|
const b = `${title}\u0000${sec(t.createdTime)}`;
|
||||||
|
k1.set(a, [...(k1.get(a) ?? []), t.id]);
|
||||||
|
k2.set(b, [...(k2.get(b) ?? []), t.id]);
|
||||||
|
}
|
||||||
|
const used = new Set<string>();
|
||||||
|
const take = (m: Map<string, string[]>, k: string) => {
|
||||||
|
const id = m.get(k)?.find((x) => !used.has(x));
|
||||||
|
if (id) used.add(id);
|
||||||
|
return id;
|
||||||
|
};
|
||||||
|
const report: CsvMergeReport = {
|
||||||
|
csvRows: csv.length,
|
||||||
|
matchedListTitleCreated: 0,
|
||||||
|
matchedTitleCreated: 0,
|
||||||
|
added: 0,
|
||||||
|
newLists: []
|
||||||
|
};
|
||||||
|
const projects = [...data.projects];
|
||||||
|
const inboxId =
|
||||||
|
data.inboxId ?? data.tasks.find((t) => t.projectId.startsWith('inbox'))?.projectId ?? 'inbox';
|
||||||
|
const projectFor = (list: string, folder: string) => {
|
||||||
|
if (list === 'Inbox') return inboxId;
|
||||||
|
const p = projects.find((x) => x.name === list);
|
||||||
|
if (p) return p.id;
|
||||||
|
const id = `csvlist:${list}`;
|
||||||
|
projects.push({ id, name: folder ? `${folder} / ${list}` : list, closed: true });
|
||||||
|
report.newLists.push(list);
|
||||||
|
return id;
|
||||||
|
};
|
||||||
|
const idOf = new Map<string, string>(); // csv taskId → API id or csv key
|
||||||
|
const keyCount = new Map<string, number>();
|
||||||
|
const added: CsvTask[] = [];
|
||||||
|
for (const t of csv) {
|
||||||
|
const title = (t.title ?? '').trim();
|
||||||
|
let id = take(k1, `${t.listName}\u0000${title}\u0000${sec(t.createdTime)}`);
|
||||||
|
if (id) report.matchedListTitleCreated++;
|
||||||
|
else if ((id = take(k2, `${title}\u0000${sec(t.createdTime)}`))) report.matchedTitleCreated++;
|
||||||
|
if (!id) {
|
||||||
|
const base = csvKey(t.listName, title, ttDate(t.createdTime));
|
||||||
|
const n = (keyCount.get(base) ?? 0) + 1;
|
||||||
|
keyCount.set(base, n);
|
||||||
|
id = n === 1 ? base : `${base}:${n}`;
|
||||||
|
added.push(t);
|
||||||
|
}
|
||||||
|
idOf.set(t.csvId, id);
|
||||||
|
t.id = id;
|
||||||
|
}
|
||||||
|
const tasks: TtTask[] = [...data.tasks];
|
||||||
|
for (const t of added) {
|
||||||
|
const { csvId, csvParent, listName, folder, ...task } = t;
|
||||||
|
void csvId;
|
||||||
|
tasks.push({
|
||||||
|
...task,
|
||||||
|
projectId: projectFor(listName, folder),
|
||||||
|
parentId: (csvParent && idOf.get(csvParent)) || undefined
|
||||||
|
});
|
||||||
|
}
|
||||||
|
report.added = added.length;
|
||||||
|
return { data: { ...data, projects, tasks }, report };
|
||||||
|
}
|
||||||
@@ -18,6 +18,7 @@ export function getTitle(path: string, environment: SiteEnvironment = null): str
|
|||||||
'/': 'Home',
|
'/': 'Home',
|
||||||
'/projects': 'Projects',
|
'/projects': 'Projects',
|
||||||
'/trackers': 'Trackers',
|
'/trackers': 'Trackers',
|
||||||
|
'/trackers/books': 'Books',
|
||||||
'/blog': 'Blog',
|
'/blog': 'Blog',
|
||||||
'/lists': 'Lists',
|
'/lists': 'Lists',
|
||||||
'/login': 'Sign in',
|
'/login': 'Sign in',
|
||||||
@@ -30,12 +31,13 @@ export function getTitle(path: string, environment: SiteEnvironment = null): str
|
|||||||
'/projects/animaldex/admin': 'Animaldex admin',
|
'/projects/animaldex/admin': 'Animaldex admin',
|
||||||
'/projects/derby': 'Derby Lens',
|
'/projects/derby': 'Derby Lens',
|
||||||
'/projects/cs-prices': 'CS Prices',
|
'/projects/cs-prices': 'CS Prices',
|
||||||
|
'/projects/energy': 'Energy',
|
||||||
'/projects/game-tracker': 'Game Tracker',
|
'/projects/game-tracker': 'Game Tracker',
|
||||||
'/projects/language-learner': 'Language Learner',
|
'/projects/language-learner': 'Language Learner',
|
||||||
'/projects/money-tracker': 'Money Tracker',
|
|
||||||
'/projects/spotify-beatmap-finder': 'osu! map finder',
|
'/projects/spotify-beatmap-finder': 'osu! map finder',
|
||||||
'/projects/progress': 'Progress',
|
'/projects/progress': 'Progress',
|
||||||
'/projects/world-opinions': 'World Opinions'
|
'/projects/world-opinions': 'World Opinions',
|
||||||
|
'/trackers/money': 'Money'
|
||||||
};
|
};
|
||||||
const cleaned = path.replace(/\/$/, '') || '/';
|
const cleaned = path.replace(/\/$/, '') || '/';
|
||||||
const title =
|
const title =
|
||||||
|
|||||||
@@ -30,6 +30,8 @@
|
|||||||
|
|
||||||
let { data, children } = $props();
|
let { data, children } = $props();
|
||||||
const seo = $derived(getSeo(page.url.pathname, page.data, data.environment));
|
const seo = $derived(getSeo(page.url.pathname, page.data, data.environment));
|
||||||
|
// The book reader is a full-screen view; the site header and footer would sit on top of it.
|
||||||
|
const immersive = $derived(/^\/trackers\/books\/\d+\/read\/?$/.test(page.url.pathname));
|
||||||
let menuOpen = $state(false);
|
let menuOpen = $state(false);
|
||||||
// Which header dropdown is open (desktop): '/projects', '/trackers' or none.
|
// Which header dropdown is open (desktop): '/projects', '/trackers' or none.
|
||||||
let openMenu = $state<string | null>(null);
|
let openMenu = $state<string | null>(null);
|
||||||
@@ -184,6 +186,7 @@
|
|||||||
Fix: lift the header (relative z-50) and isolate <main> so page z-indexes stay inside it.
|
Fix: lift the header (relative z-50) and isolate <main> so page z-indexes stay inside it.
|
||||||
Dialogs/sheets/popovers are portalled to <body> and still render above both.
|
Dialogs/sheets/popovers are portalled to <body> and still render above both.
|
||||||
-->
|
-->
|
||||||
|
{#if !immersive}
|
||||||
<header class="relative z-50 border-b border-border bg-background/95 backdrop-blur">
|
<header class="relative z-50 border-b border-border bg-background/95 backdrop-blur">
|
||||||
<div class="mx-auto flex max-w-6xl items-center gap-3 px-4 py-3 sm:px-6">
|
<div class="mx-auto flex max-w-6xl items-center gap-3 px-4 py-3 sm:px-6">
|
||||||
<a
|
<a
|
||||||
@@ -256,7 +259,10 @@
|
|||||||
>{project.subtitle}</span
|
>{project.subtitle}</span
|
||||||
></span
|
></span
|
||||||
>
|
>
|
||||||
{#if project.external}<ExternalLink size={14} class="ml-auto shrink-0" />{/if}
|
{#if project.external}<ExternalLink
|
||||||
|
size={14}
|
||||||
|
class="ml-auto shrink-0"
|
||||||
|
/>{/if}
|
||||||
</a>
|
</a>
|
||||||
<!-- eslint-enable svelte/no-navigation-without-resolve -->
|
<!-- eslint-enable svelte/no-navigation-without-resolve -->
|
||||||
{/each}
|
{/each}
|
||||||
@@ -337,8 +343,10 @@
|
|||||||
aria-label={`Theme: ${theme}. Switch mode`}>{@render themeIcon()}</Button
|
aria-label={`Theme: ${theme}. Switch mode`}>{@render themeIcon()}</Button
|
||||||
>
|
>
|
||||||
{#if data.user}
|
{#if data.user}
|
||||||
<Button href={resolve('account')} variant="outline" onclick={() => (menuOpen = false)}
|
<Button
|
||||||
>account</Button
|
href={resolve('account')}
|
||||||
|
variant="outline"
|
||||||
|
onclick={() => (menuOpen = false)}>account</Button
|
||||||
>
|
>
|
||||||
<form method="POST" action="/logout">
|
<form method="POST" action="/logout">
|
||||||
<Button type="submit" variant="outline">sign out</Button>
|
<Button type="submit" variant="outline">sign out</Button>
|
||||||
@@ -358,7 +366,9 @@
|
|||||||
</Sheet.Root>
|
</Sheet.Root>
|
||||||
</div>
|
</div>
|
||||||
</header>
|
</header>
|
||||||
|
{/if}
|
||||||
<main id="main" class="isolate min-w-0 flex-1">{@render children()}</main>
|
<main id="main" class="isolate min-w-0 flex-1">{@render children()}</main>
|
||||||
|
{#if !immersive}
|
||||||
<footer
|
<footer
|
||||||
class="mt-16 border-t border-[var(--site-border)]"
|
class="mt-16 border-t border-[var(--site-border)]"
|
||||||
class:tracker-footer={page.route.id === '/projects/game-tracker'}
|
class:tracker-footer={page.route.id === '/projects/game-tracker'}
|
||||||
@@ -393,6 +403,7 @@
|
|||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
</footer>
|
</footer>
|
||||||
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
|
|||||||
@@ -2,10 +2,21 @@ import { requireAdmin } from '#lib/server/authorization.js';
|
|||||||
import { uniqueListSlug } from '#lib/server/slugify.js';
|
import { uniqueListSlug } from '#lib/server/slugify.js';
|
||||||
import { listImageUrl } from '#lib/server/r2.js';
|
import { listImageUrl } from '#lib/server/r2.js';
|
||||||
import { createList, listSummaries } from '#lib/server/lists.js';
|
import { createList, listSummaries } from '#lib/server/lists.js';
|
||||||
|
import { listTables, type TableSummary } from '#lib/server/data-editor.js';
|
||||||
|
import { isOwner } from '#lib/server/owner.js';
|
||||||
import { fail, redirect } from '@sveltejs/kit';
|
import { fail, redirect } from '@sveltejs/kit';
|
||||||
import type { Actions, PageServerLoad } from './$types';
|
import type { Actions, PageServerLoad } from './$types';
|
||||||
|
|
||||||
export const load: PageServerLoad = async () => ({ lists: await listSummaries() });
|
export const load: PageServerLoad = async (event) => {
|
||||||
|
const owner = isOwner(event);
|
||||||
|
// The database browser (every schema/table) is only ever loaded for the owner.
|
||||||
|
const [lists, tables] = await Promise.all([
|
||||||
|
listSummaries(),
|
||||||
|
owner ? listTables() : Promise.resolve([] as TableSummary[])
|
||||||
|
]);
|
||||||
|
if (owner) event.setHeaders({ 'cache-control': 'private, no-store' });
|
||||||
|
return { lists, tables };
|
||||||
|
};
|
||||||
|
|
||||||
export const actions: Actions = {
|
export const actions: Actions = {
|
||||||
newList: async (event) => {
|
newList: async (event) => {
|
||||||
|
|||||||
@@ -14,6 +14,30 @@
|
|||||||
import { Input } from '#lib/components/ui/input/index.js';
|
import { Input } from '#lib/components/ui/input/index.js';
|
||||||
import type { PageProps } from './$types';
|
import type { PageProps } from './$types';
|
||||||
let { data, form }: PageProps = $props();
|
let { data, form }: PageProps = $props();
|
||||||
|
let tableFilter = $state('');
|
||||||
|
const schemas = $derived.by(() => {
|
||||||
|
const q = tableFilter.trim().toLowerCase();
|
||||||
|
// eslint-disable-next-line svelte/prefer-svelte-reactivity -- local, rebuilt on every derive
|
||||||
|
const groups = new Map<string, typeof data.tables>();
|
||||||
|
for (const t of data.tables) {
|
||||||
|
if (q && !`${t.schema}.${t.name} ${t.displayName}`.toLowerCase().includes(q)) continue;
|
||||||
|
groups.set(t.schema, [...(groups.get(t.schema) ?? []), t]);
|
||||||
|
}
|
||||||
|
// lists first, then alphabetical
|
||||||
|
return [...groups].sort(([a], [b]) =>
|
||||||
|
a === 'lists' ? -1 : b === 'lists' ? 1 : a.localeCompare(b)
|
||||||
|
);
|
||||||
|
});
|
||||||
|
const totalRows = $derived(data.tables.reduce((n, t) => n + (t.rows ?? 0), 0));
|
||||||
|
const badge = (access: string, relkind: string) =>
|
||||||
|
access === 'open'
|
||||||
|
? { text: 'editable', cls: 'border-emerald-600/40 text-emerald-500' }
|
||||||
|
: access === 'core'
|
||||||
|
? { text: 'app table', cls: 'border-amber-500/40 text-amber-500' }
|
||||||
|
: {
|
||||||
|
text: relkind === 'v' || relkind === 'm' ? 'view' : 'read-only',
|
||||||
|
cls: 'border-[var(--site-border)] text-[var(--site-muted)]'
|
||||||
|
};
|
||||||
let uploadError = $state('');
|
let uploadError = $state('');
|
||||||
let uploading = $state(false);
|
let uploading = $state(false);
|
||||||
const uploadPicture: SubmitFunction = async ({ formData, cancel }) => {
|
const uploadPicture: SubmitFunction = async ({ formData, cancel }) => {
|
||||||
@@ -137,4 +161,60 @@
|
|||||||
</li>
|
</li>
|
||||||
{/each}
|
{/each}
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
|
{#if data.isOwner && data.tables.length}
|
||||||
|
<section id="database" class="mt-16">
|
||||||
|
<div class="mb-4 flex flex-wrap items-end justify-between gap-4">
|
||||||
|
<div>
|
||||||
|
<h2 class="text-2xl font-semibold tracking-tight">database</h2>
|
||||||
|
<p class="mt-1 text-sm text-[var(--site-muted)]">
|
||||||
|
{data.tables.length} tables in {new Set(data.tables.map((t) => t.schema)).size} schemas, ~{totalRows.toLocaleString(
|
||||||
|
'en-GB'
|
||||||
|
)} rows. Only you can see this. Lists are editable; app tables are read-only until you unlock
|
||||||
|
them; a new list is a new table.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
type="search"
|
||||||
|
bind:value={tableFilter}
|
||||||
|
placeholder="find a table"
|
||||||
|
aria-label="Find a table"
|
||||||
|
class="w-56 rounded border border-[var(--site-border)] bg-[var(--site-bg)] px-3 py-1.5 text-sm"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{#each schemas as [schema, tables] (schema)}
|
||||||
|
<details open class="border-t border-[var(--site-border)] py-3" id="schema-{schema}">
|
||||||
|
<summary class="cursor-pointer text-lg font-medium">
|
||||||
|
{schema}
|
||||||
|
<span class="ml-2 text-sm font-normal text-[var(--site-muted)]"
|
||||||
|
>{tables.length} {tables.length === 1 ? 'table' : 'tables'}</span
|
||||||
|
>
|
||||||
|
</summary>
|
||||||
|
<ul class="mt-2 grid gap-x-6 sm:grid-cols-2 lg:grid-cols-3">
|
||||||
|
{#each tables as t (t.name)}
|
||||||
|
{@const b = badge(t.access, t.relkind)}
|
||||||
|
<li>
|
||||||
|
<a
|
||||||
|
href={resolve('/lists/db/[schema]/[table]', { schema: t.schema, table: t.name })}
|
||||||
|
class="flex items-center gap-2 rounded px-2 py-1.5 hover:bg-[var(--site-hover)]"
|
||||||
|
>
|
||||||
|
<span class="min-w-0 flex-1 truncate" title={t.description ?? t.displayName}
|
||||||
|
>{t.name}</span
|
||||||
|
>
|
||||||
|
<span class={['rounded-full border px-1.5 text-[10px]', b.cls]}>{b.text}</span>
|
||||||
|
<span class="w-16 text-right text-xs text-[var(--site-muted)] tabular-nums">
|
||||||
|
{t.rows === null
|
||||||
|
? '–'
|
||||||
|
: `${t.approximate ? '~' : ''}${t.rows.toLocaleString('en-GB')}`}
|
||||||
|
</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
{/each}
|
||||||
|
</ul>
|
||||||
|
</details>
|
||||||
|
{:else}
|
||||||
|
<p class="text-[var(--site-muted)]">no tables match</p>
|
||||||
|
{/each}
|
||||||
|
</section>
|
||||||
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { ORIGIN } from '$app/env/private';
|
|||||||
import { metaTables } from '#lib/server/db/schema/index.js';
|
import { metaTables } from '#lib/server/db/schema/index.js';
|
||||||
import { requireAdmin } from '#lib/server/authorization.js';
|
import { requireAdmin } from '#lib/server/authorization.js';
|
||||||
import { deleteObject, listImageKey, listImageUrl } from '#lib/server/r2.js';
|
import { deleteObject, listImageKey, listImageUrl } from '#lib/server/r2.js';
|
||||||
import { validSlug } from '#lib/server/slugify.js';
|
import { RESERVED_LIST_SLUGS, validSlug } from '#lib/server/slugify.js';
|
||||||
import {
|
import {
|
||||||
archiveList,
|
archiveList,
|
||||||
deleteItem,
|
deleteItem,
|
||||||
@@ -109,7 +109,7 @@ export const actions: Actions = {
|
|||||||
const picture = form.get('picture');
|
const picture = form.get('picture');
|
||||||
if (!name || name.length > 200)
|
if (!name || name.length > 200)
|
||||||
return fail(400, { message: 'Enter a name of 1–200 characters' });
|
return fail(400, { message: 'Enter a name of 1–200 characters' });
|
||||||
if (!validSlug(slug) || slug.length > 63)
|
if (!validSlug(slug) || slug.length > 63 || RESERVED_LIST_SLUGS.has(slug))
|
||||||
return fail(400, { message: 'Use a lowercase slug with letters, numbers, and hyphens.' });
|
return fail(400, { message: 'Use a lowercase slug with letters, numbers, and hyphens.' });
|
||||||
if (picture instanceof File && picture.size > 0)
|
if (picture instanceof File && picture.size > 0)
|
||||||
return fail(400, { message: 'Enable JavaScript to upload an image.' });
|
return fail(400, { message: 'Enable JavaScript to upload an image.' });
|
||||||
|
|||||||
@@ -99,6 +99,13 @@
|
|||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{#if data.isOwner}<a
|
||||||
|
href={resolve('/lists/db/[schema]/[table]', {
|
||||||
|
schema: 'lists',
|
||||||
|
table: data.list.tableName
|
||||||
|
})}
|
||||||
|
class="text-base underline underline-offset-4">table view</a
|
||||||
|
>{/if}
|
||||||
{#if data.isAdmin}<Button
|
{#if data.isAdmin}<Button
|
||||||
type="button"
|
type="button"
|
||||||
onclick={() => (managing = !managing)}
|
onclick={() => (managing = !managing)}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user