trains: stricter Commons photo matching (category files must name the type; no maps, class interiors, trucks)

- model the existing hardware schema in Drizzle (devices, components, new sensitivities)
- idempotent reconciling migration: new columns, checks, setup_overview view, grants, meta.tables
- seed: main PC labels/hostnames, planned AMD X3D + RTX 5090 build, stan-ovh (OVH RISE-S),
  other devices, sensitivity placeholders (no invented cm/360)
- owner-only add/edit via modals; public view hides notes/order numbers
- trackers dropdown entry + icon, seo/title/sitemap, owner write prefix

Dockerfile

@@ -17,7 +17,11 @@ COPY --from=builder /app/drizzle ./drizzle
 COPY --from=builder /app/data ./data
 COPY --from=builder /app/src/lib/server/db/schema ./src/lib/server/db/schema
 COPY --from=builder /app/src/lib/cs-prices ./src/lib/cs-prices
+# scripts/house-listing.ts (manual listings CLI) imports these pure helpers.
+COPY --from=builder /app/src/lib/house-tracker ./src/lib/house-tracker
 COPY --from=builder /app/src/lib/availability.ts ./src/lib/availability.ts
+# scripts/music-*.ts (Spotify poll, Last.fm and streaming-history imports) import these.
+COPY --from=builder /app/src/lib/music ./src/lib/music
 COPY package.json ./
 EXPOSE 3000
 ENV NODE_ENV=production

README.md

@@ -255,18 +255,30 @@ Files are read from `BOOKS_ROOT` (default `/books` when it exists, otherwise `~/
 - `BOOKS_ROOT/library/<books.files.path>` for book files (written by `scripts/books-ingest.py`)
 - `BOOKS_ROOT/covers/<books.works.cover_path>` for downloaded covers
 
-In production, `scripts/deploy-remote.sh` writes `/opt/personal-website/compose.books.yml`, which
+In production, `scripts/deploy-remote.sh` writes `~deploy/compose.books.yml` (the deploy account may only sudo-tee `/opt/personal-website/.env`), which
 bind-mounts the host's `BOOKS_HOST_DIR` (default `/home/stan/books`) read-only at `/books` and sets
 `BOOKS_ROOT=/books`. Without a `library` directory there the deploy still succeeds, but files can't
 be opened.
 
-## Money tracker
+## Money
 
-`/projects/money-tracker` is restricted to administrator accounts. Its tables
+`/trackers/money` (formerly `/projects/money-tracker`, which now redirects) is restricted to administrator accounts. Its tables
 are created by `npm run db:migrate:money-tracker`; apply that migration to each
 environment before deploying the new route. The deployment preflight checks for
 all nine tracker tables and the projection columns.
 
+`/trackers/money/documents` lists private documents (contracts, pension rules) with a
+written summary, an inline viewer and a download. Nothing about them is in git: the
+files and the summaries live on the host in `MONEY_DOCUMENTS_ROOT` (default
+`/money-documents` when it exists, otherwise `~/money-documents`) as
+`documents.json` (`[{ id, title, category, date, pages, file, summary }]`, summary in
+Markdown) plus `files/<file>`. `scripts/deploy-remote.sh` bind-mounts
+`/home/stan/money-documents` read-only at `/money-documents` when it has a
+`documents.json`. Files are only streamed to admins via
+`/trackers/money/documents/<id>/file` (401 logged out, 403 for non-admins,
+`private, no-store`). To add one, copy the file into `files/` and add a manifest entry;
+no redeploy is needed.
+
 Transactions can be entered manually or imported from a CSV or tab-separated
 export with `date,account,description,amount` headers. Dates accept
 `YYYY-MM-DD`, `DD/MM/YYYY`, or `Fri 1 Aug 2025`; amounts accept signed euros

data/tasks/daisy-import.json

@@ -0,0 +1,643 @@
+{
+	"source": "daisy-import",
+	"areas": [
+		{
+			"slug": "deadlines",
+			"name": "Deadlines and dates",
+			"sortOrder": 10
+		},
+		{
+			"slug": "house",
+			"name": "House and moving",
+			"description": "Jan–Feb 2027 with Juhee",
+			"sortOrder": 20
+		},
+		{
+			"slug": "kibi",
+			"name": "Kibi",
+			"sortOrder": 30
+		},
+		{
+			"slug": "admin",
+			"name": "Phone, subscriptions and admin",
+			"sortOrder": 40
+		},
+		{
+			"slug": "clients",
+			"name": "Clients",
+			"sortOrder": 50
+		},
+		{
+			"slug": "website",
+			"name": "Website (stanrunge.dev)",
+			"sortOrder": 60
+		},
+		{
+			"slug": "setup",
+			"name": "Setup and security",
+			"description": "Perma evolvable setup, before 2027",
+			"sortOrder": 70
+		},
+		{
+			"slug": "home",
+			"name": "Home",
+			"sortOrder": 80
+		},
+		{
+			"slug": "health",
+			"name": "Health and self-care",
+			"sortOrder": 90
+		},
+		{
+			"slug": "reading",
+			"name": "Reading",
+			"sortOrder": 100
+		},
+		{
+			"slug": "planning",
+			"name": "Planning and goals",
+			"sortOrder": 110
+		}
+	],
+	"goals": [
+		{
+			"title": "€10k net/month into own Revolut by 2030",
+			"area": "planning",
+			"targetDate": "2030-01-01",
+			"metric": "Net income per month",
+			"metricUnit": "EUR",
+			"metricTarget": 10000
+		},
+		{
+			"title": "Own a really nice Ford Mustang GT"
+		},
+		{
+			"title": "PC overhaul",
+			"notes": "Top AMD X3D CPU, RTX 5090.",
+			"metric": "Budget",
+			"metricUnit": "EUR",
+			"metricTarget": 10000
+		},
+		{
+			"title": "Organise goals into a plan",
+			"area": "planning",
+			"notes": "Social media, Grok Bot, IT company."
+		},
+		{
+			"title": "Perma evolvable setup foundation ready before 2027",
+			"area": "setup",
+			"targetDate": "2026-12-31"
+		},
+		{
+			"title": "Get in shape long term",
+			"area": "health"
+		}
+	],
+	"tasks": [
+		{
+			"area": "deadlines",
+			"title": "Jan Loman printer visit",
+			"status": "done",
+			"notes": "Settled.",
+			"due": "2026-10-24",
+			"dueTime": "14:00"
+		},
+		{
+			"area": "deadlines",
+			"title": "Call BackHome about Kibi's chip",
+			"status": "todo",
+			"due": "2026-10-26",
+			"dueTime": "10:00",
+			"tags": ["kibi", "call"]
+		},
+		{
+			"area": "deadlines",
+			"title": "Cargill collective health insurance (OHRA, code 114)",
+			"status": "todo",
+			"notes": "Needs current insurer + premium to compare.",
+			"due": "2026-12-31",
+			"tags": ["insurance"]
+		},
+		{
+			"area": "deadlines",
+			"title": "Reply to mom about insurance info",
+			"status": "todo",
+			"notes": "Waiting since Oct 9.",
+			"tags": ["insurance"]
+		},
+		{
+			"area": "deadlines",
+			"title": "Stop zorgtoeslag from Jan 2027",
+			"status": "todo",
+			"due": "2027-01-01",
+			"reminderAt": "2026-12-07T09:00:00+01:00",
+			"tags": ["admin"]
+		},
+		{
+			"area": "deadlines",
+			"title": "Supermarket product list",
+			"status": "todo",
+			"reminderAt": "2026-10-24T11:50:00+02:00"
+		},
+		{
+			"area": "deadlines",
+			"title": "Book haircut",
+			"status": "todo",
+			"notes": "Around Nov 14.",
+			"due": "2026-11-14"
+		},
+		{
+			"area": "house",
+			"title": "House overview",
+			"status": "in_progress",
+			"flagged": true,
+			"notes": "Applied only to Graaf Hendrikstraat 16 so far."
+		},
+		{
+			"area": "house",
+			"title": "React to house options",
+			"status": "todo",
+			"notes": "Raam €1,475; Burg. van Dijkesingel 214-B; Groenhovenweg 161; Spoorstraat €795; Noothoven van Goorstraat 1-206."
+		},
+		{
+			"area": "house",
+			"title": "Tell Daisy budget, move date, must-haves, Funda/Pararius applications",
+			"status": "todo"
+		},
+		{
+			"area": "house",
+			"title": "Mijndak Midden-Holland sign-up",
+			"status": "todo",
+			"flagged": true,
+			"notes": "€23 option."
+		},
+		{
+			"area": "house",
+			"title": "Sign up: socialehuurwoning.nl, Vesteda Goudshof, VBT, Viramah, agency Jesse's mom uses",
+			"status": "todo"
+		},
+		{
+			"area": "house",
+			"title": "House tracker/scraper on the site",
+			"status": "todo",
+			"tags": ["website"]
+		},
+		{
+			"area": "house",
+			"title": "Update address everywhere",
+			"status": "todo",
+			"notes": "Checklist ready."
+		},
+		{
+			"area": "house",
+			"title": "UniFi network for the new place",
+			"status": "todo",
+			"tags": ["network"]
+		},
+		{
+			"area": "kibi",
+			"title": "Food ordered",
+			"status": "done"
+		},
+		{
+			"area": "kibi",
+			"title": "Chip scanner ordered",
+			"status": "done"
+		},
+		{
+			"area": "kibi",
+			"title": "Chip number found",
+			"status": "done"
+		},
+		{
+			"area": "kibi",
+			"title": "Chip owner change at BackHome",
+			"status": "todo",
+			"notes": "Still 'Kwiebus' under E Zwart."
+		},
+		{
+			"area": "kibi",
+			"title": "Call vet about Kibi's patient card",
+			"status": "todo",
+			"tags": ["call"]
+		},
+		{
+			"area": "kibi",
+			"title": "GPS tracker",
+			"status": "todo",
+			"notes": "Dad may have done it already."
+		},
+		{
+			"area": "kibi",
+			"title": "Dash KIBI",
+			"status": "todo",
+			"flagged": true,
+			"notes": "Meaning unclear, ask Stan."
+		},
+		{
+			"area": "admin",
+			"title": "Basic-Fit cancelled",
+			"status": "done"
+		},
+		{
+			"area": "admin",
+			"title": "Ohra inboedelverzekering cancelled",
+			"status": "done"
+		},
+		{
+			"area": "admin",
+			"title": "Zorgtoeslag income updated for 2026",
+			"status": "done"
+		},
+		{
+			"area": "admin",
+			"title": "Move number to Revolut eSIM, then iPad eSIM, cancel Odido",
+			"status": "in_progress"
+		},
+		{
+			"area": "admin",
+			"title": "Move SIM + WhatsApp from iPhone to Pixel 11 Pro XL",
+			"status": "todo"
+		},
+		{
+			"area": "admin",
+			"title": "Check Oasen and Delta Energie fully cancelled",
+			"status": "todo"
+		},
+		{
+			"area": "admin",
+			"title": "Cancel other unused subscriptions",
+			"status": "todo"
+		},
+		{
+			"area": "admin",
+			"title": "Merge banks into Revolut",
+			"status": "todo"
+		},
+		{
+			"area": "clients",
+			"title": "Quote dack €450 for vet clinic site fix",
+			"status": "done",
+			"tags": ["dack"]
+		},
+		{
+			"area": "clients",
+			"title": "Reply to dack",
+			"status": "done",
+			"tags": ["dack"]
+		},
+		{
+			"area": "clients",
+			"title": "Cloudflare DNS/SSL for dack",
+			"status": "in_progress",
+			"tags": ["dack"],
+			"notes": "Grind-week goal."
+		},
+		{
+			"area": "clients",
+			"title": "Send dack's invoice via Moneybird",
+			"status": "todo",
+			"tags": ["dack"]
+		},
+		{
+			"area": "clients",
+			"title": "Cheap shared WordPress hosting for dack's clients on own VMs",
+			"status": "todo",
+			"tags": ["dack"]
+		},
+		{
+			"area": "clients",
+			"title": "Karolina's Wix site, Master Purpose (NL/PL)",
+			"status": "todo",
+			"tags": ["karolina"]
+		},
+		{
+			"area": "clients",
+			"title": "Wix Studio + Velo/CLI so an AI can edit code",
+			"status": "todo",
+			"tags": ["karolina"]
+		},
+		{
+			"area": "website",
+			"title": "DB reorganised into schemas, staging retired",
+			"status": "done"
+		},
+		{
+			"area": "website",
+			"title": "Energy tracker (NED data) live",
+			"status": "done"
+		},
+		{
+			"area": "website",
+			"title": "Health tracker (private) + 15 RIVM vaccinations",
+			"status": "done"
+		},
+		{
+			"area": "website",
+			"title": "Book tracker, reader, 362 books live",
+			"status": "done"
+		},
+		{
+			"area": "website",
+			"title": "Reader zoom, 2-page view, scroll/paging",
+			"status": "in_progress"
+		},
+		{
+			"area": "website",
+			"title": "Money tracker with MoneyMan",
+			"status": "in_progress"
+		},
+		{
+			"area": "website",
+			"title": "NED backfill",
+			"status": "in_progress"
+		},
+		{
+			"area": "website",
+			"title": "Music listening tracker + now-playing",
+			"status": "todo"
+		},
+		{
+			"area": "website",
+			"title": "Task/goal tracker to replace TickTick",
+			"status": "in_progress"
+		},
+		{
+			"area": "website",
+			"title": "Hardware DB (current PC parts first)",
+			"status": "todo"
+		},
+		{
+			"area": "website",
+			"title": "List of every organisation holding my data and what they know",
+			"status": "todo"
+		},
+		{
+			"area": "website",
+			"title": "Move to SvelteKit 3",
+			"status": "todo"
+		},
+		{
+			"area": "website",
+			"title": "Juhee website data",
+			"status": "todo"
+		},
+		{
+			"area": "setup",
+			"title": "AWS cleanup ($0/month)",
+			"status": "done",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Google Cloud cleanup",
+			"status": "done",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "stan-vm hardened (ufw, Tailscale-only SSH)",
+			"status": "done",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Nightly restic backups to B2 + Storage Box",
+			"status": "done",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "WhatsApp bridge on stan-vm",
+			"status": "done",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "gcloud + moneybird-cli on stan-vm",
+			"status": "done",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Domains Squarespace to Cloudflare",
+			"status": "in_progress",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"notes": "Both transfers pending.",
+			"due": "2026-10-29"
+		},
+		{
+			"area": "setup",
+			"title": "Syncthing: Introducer + Auto Accept on Arch and Windows",
+			"status": "in_progress",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Backup follow-up: restic passwords into 1Password",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"due": "2026-10-24",
+			"dueTime": "11:00"
+		},
+		{
+			"area": "setup",
+			"title": "Delete empty B2 bucket",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"due": "2026-10-25"
+		},
+		{
+			"area": "setup",
+			"title": "Rotate weak admin secret + DB password",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"tags": ["security"]
+		},
+		{
+			"area": "setup",
+			"title": "1Password overhaul",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"notes": "Imports, 2FA, passkeys, dedupe, delete unused accounts.",
+			"tags": ["security"]
+		},
+		{
+			"area": "setup",
+			"title": "Moneybird API token, then clean up Moneybird",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Authentik SSO on stan-vm",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "vash-prod memory warnings + finish security audit",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"tags": ["security"]
+		},
+		{
+			"area": "setup",
+			"title": "Remove old grok-bot devices from Tailscale",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Close empty GCP billing account; decide on AWS 'Vash Software'",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"notes": "AWS part optional."
+		},
+		{
+			"area": "setup",
+			"title": "Raise WSL RAM",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"notes": "Sees 8 of 16 GB."
+		},
+		{
+			"area": "setup",
+			"title": "Home server rebuild with ZFS",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Analytics, monitoring, alerts across devices",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Mouse sensitivity cm/360 across devices",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Buy Logitech G840 XL mousepad",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"tags": ["buy"]
+		},
+		{
+			"area": "setup",
+			"title": "Router and networking setup",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"tags": ["network"]
+		},
+		{
+			"area": "setup",
+			"title": "J5 Hexagon research, then move wachtboden",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "Agent-deck + AI/Linux dev setup",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027"
+		},
+		{
+			"area": "setup",
+			"title": "MITM/certificate lab",
+			"status": "todo",
+			"goal": "Perma evolvable setup foundation ready before 2027",
+			"notes": "Idea.",
+			"tags": ["idea"]
+		},
+		{
+			"area": "home",
+			"title": "Unpack boxes; organise batteries and tools; buy thermometer",
+			"status": "todo"
+		},
+		{
+			"area": "home",
+			"title": "Clean desk, laundry, clothes",
+			"status": "todo"
+		},
+		{
+			"area": "home",
+			"title": "Connect vinyl player, printer, monitor, switch, osu! pad, tablet, fridge, microwave",
+			"status": "todo"
+		},
+		{
+			"area": "home",
+			"title": "Hue lights + Home Assistant + Apple Home",
+			"status": "todo"
+		},
+		{
+			"area": "home",
+			"title": "Fan controller",
+			"status": "todo"
+		},
+		{
+			"area": "home",
+			"title": "Opa's stuff (WhatsApp, sensor install)",
+			"status": "todo"
+		},
+		{
+			"area": "health",
+			"title": "Getting in shape with Doc",
+			"status": "in_progress",
+			"notes": "Lesson 1 done; ~2,300–2,400 kcal and 160–190 g protein a day.",
+			"goal": "Get in shape long term"
+		},
+		{
+			"area": "health",
+			"title": "Join the gym my parents use",
+			"status": "todo",
+			"goal": "Get in shape long term"
+		},
+		{
+			"area": "health",
+			"title": "Brush teeth twice a day, shower, hair care",
+			"status": "todo",
+			"tags": ["habit"]
+		},
+		{
+			"area": "health",
+			"title": "Dentist checkup",
+			"status": "todo"
+		},
+		{
+			"area": "health",
+			"title": "Pick hardware for Withings/Garmin syncs",
+			"status": "todo"
+		},
+		{
+			"area": "health",
+			"title": "Curaçao vaccines",
+			"status": "todo",
+			"notes": "Via Doc."
+		},
+		{
+			"area": "reading",
+			"title": "Read 'De meeste mensen deugen' (Bregman)",
+			"status": "todo"
+		},
+		{
+			"area": "reading",
+			"title": "Re-download 'Haves and Have-Nots'",
+			"status": "todo"
+		},
+		{
+			"area": "reading",
+			"title": "Book questions: duplicates, 6 unknown files, Nick's repo 404",
+			"status": "todo"
+		},
+		{
+			"area": "planning",
+			"title": "Schedule sync (paper agenda, TickTick, Daisy)",
+			"status": "todo",
+			"goal": "Organise goals into a plan"
+		}
+	]
+}

data/tasks/ticktick-merges.json

@@ -0,0 +1,7 @@
+[
+	{ "ticktickId": "6ad52191ebcd7400000000ce", "title": "Jan Loman printer visit" },
+	{ "ticktickId": "690d2fd98c824268f297a676", "title": "Router and networking setup" },
+	{ "ticktickId": "6844769df471d103461fa9f1", "title": "Home server rebuild with ZFS" },
+	{ "ticktickId": "6ab6f32759b6090811e91a00", "title": "Dentist checkup" },
+	{ "ticktickId": "68225a8dbfe81104eac49a7a", "title": "Merge banks into Revolut" }
+]

deploy/home-collector/README.md

@@ -0,0 +1,31 @@
+# home-collector
+
+Feeds the private dashboard at `/start` (also the browser new-tab page). The app container has
+no Docker socket and must not get one, so a small stdlib-only Python script runs on the host
+every minute (systemd **user** timer `home-collector.timer`) and upserts two rows into
+`home.snapshots`:
+
+| key    | contents                                                                                                                                                                                                                                                                                                                                                                                             |
+| ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `host` | CPU %, load, uptime, RAM, swap, disk; every Docker container (state, health, restart count, CPU/mem via `sudo docker`); systemd timers (`stan-backup*`, `postgres-backup*`, `disk-guard-*`, `house-scrape`, `b2-cost`, `groceries-import`, `money-sync`, `ned-collector-*`) with last/next run and result; HTTP up/down + latency for each self-hosted service; latest `~/backups-cost/daily/*.json` |
+| `ntfy` | the last 40 messages (72 h) from the ntfy topics in `~/.config/disk-guard.env` (`NTFY_URL`) and `~/.config/house-scrape.env` (`HOUSE_NTFY_URL`), plus `NTFY_EXTRA`. Only a label is stored, never the topic name.                                                                                                                                                                                    |
+
+The page reads both rows in the same parallel query batch as tasks and links, so a slow
+upstream never delays it; a snapshot older than 3 minutes is shown as stale.
+
+## Setup
+
+1. A login role that may only upsert snapshots (created once, cluster superuser):
+   `CREATE ROLE home_collector LOGIN CONNECTION LIMIT 3 PASSWORD '…'; GRANT CONNECT ON DATABASE personal TO home_collector;`
+   The `home` migration grants `USAGE` on the schema and `SELECT, INSERT, UPDATE` on
+   `home.snapshots` when the role exists (re-run those two GRANTs by hand if the role is created later).
+2. `~/.config/home-collector.env` (mode 600):
+   ```
+   HOME_DB_URL=postgresql://home_collector:…@127.0.0.1:5440/personal
+   # optional extra topics, e.g. the backup alerts from /etc/stan-backup:
+   # NTFY_EXTRA=backups=https://ntfy.sh/<topic>
+   ```
+3. `bash deploy/home-collector/install.sh` (copies the script to `~/.local/bin/home-collector`
+   and enables the timer). Re-run after editing the script, e.g. to add a service to `SERVICES`.
+
+Debug: `python3 ~/.local/bin/home-collector --print`, `journalctl --user -u home-collector -n 20`.

deploy/home-collector/home-collector.py

@@ -0,0 +1,343 @@
+#!/usr/bin/env python3
+"""Host-side collector for the /start dashboard on stanrunge.dev.
+
+Runs every minute as a systemd user timer on the server (home-collector.timer) and upserts
+two rows into home.snapshots with the limited `home_collector` role:
+  host  CPU, RAM, swap, disk, load, uptime, Docker containers (state, health, restarts,
+        CPU/mem), systemd timer results (backups, disk-guard, scrapers) and service checks
+  ntfy  the most recent messages from the owner's ntfy topics
+
+The app container has no Docker socket and never gets one; this script uses the host's
+passwordless `sudo docker`. ntfy topic names stay on the host: only a label is stored.
+
+Config: ~/.config/home-collector.env (mode 600)
+  HOME_DB_URL=postgres://home_collector:...@127.0.0.1:5440/<db>
+  NTFY_EXTRA=label=https://ntfy.sh/topic,...   (optional extra topics, e.g. backups)
+Topics are also read from ~/.config/disk-guard.env (NTFY_URL) and
+~/.config/house-scrape.env (HOUSE_NTFY_URL).
+Stdlib only. `--print` prints the snapshot instead of writing it; HOME_DB_URL in the
+environment overrides the env file (e.g. to fill the dev database).
+"""
+import concurrent.futures as cf
+import json
+import os
+import secrets
+import shutil
+import socket
+import ssl
+import subprocess
+import sys
+import time
+import urllib.request
+
+HOME = os.path.expanduser('~')
+CFG = os.path.join(HOME, '.config')
+TS = 'stan-ovh.tail665403.ts.net'
+
+# (name, link shown on the dashboard, url checked from the host or None = link only)
+SERVICES = [
+    ('Website', 'https://stanrunge.dev', 'http://127.0.0.1:3000/login'),
+    ('Gitea', 'https://git.stanrunge.dev', 'http://127.0.0.1:3010/'),
+    ('Immich', 'https://photos.stanrunge.dev', 'http://127.0.0.1:2283/'),
+    ('Umami', 'https://analytics.stanrunge.dev', 'http://127.0.0.1:3110/'),
+    ('pgAdmin', 'https://pgadmin.stanrunge.dev', 'http://127.0.0.1:5050/misc/ping'),
+    ('Vash esports', 'https://esports.vash.software', 'http://127.0.0.1:3100/'),
+    ('Vash staging', 'https://esports-staging.vash.software', 'http://127.0.0.1:3101/'),
+    ('Dozzle logs', 'https://logs.vash.software', 'http://127.0.0.1:3111/'),
+    ('Beeriokart', 'https://beeriokart.stanrunge.dev', 'http://127.0.0.1:3003/'),
+    ('Stuhee', 'https://stuhee.stanrunge.dev', 'http://127.0.0.1:3004/'),
+    ('Dev site', 'https://dev.stanrunge.dev', 'http://127.0.0.1:3002/'),
+    ('Netdata', f'http://{TS}:19999', 'http://127.0.0.1:19999/api/v1/info'),
+    ('Portainer', f'https://{TS}:9443', 'https://127.0.0.1:9443/'),
+    ('ntfy', 'https://ntfy.sh/app', 'https://ntfy.sh/v1/health'),
+]
+
+# (label, unit, scope)
+TIMERS = [
+    ('Backup (restic → B2)', 'stan-backup', 'system'),
+    ('Backup maintenance', 'stan-backup-maintain', 'system'),
+    ('Postgres dump', 'postgres-backup', 'system'),
+    ('Postgres dump prune', 'postgres-backup-prune', 'system'),
+    ('Disk guard check', 'disk-guard-check', 'user'),
+    ('Disk guard cleanup', 'disk-guard-cleanup', 'user'),
+    ('House scrape', 'house-scrape', 'user'),
+    ('B2 cost', 'b2-cost', 'user'),
+    ('Groceries import', 'groceries-import', 'user'),
+    ('Money sync', 'money-sync', 'user'),
+    ('Energy (NED) hourly', 'ned-collector-hourly', 'user'),
+    ('Energy (NED) daily', 'ned-collector-daily', 'user'),
+]
+
+
+def read_env(path):
+    out = {}
+    try:
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith('#') or '=' not in line:
+                    continue
+                k, v = line.split('=', 1)
+                out[k.strip()] = v.strip().strip('"').strip("'")
+    except OSError:
+        pass
+    return out
+
+
+def run(cmd, timeout=15):
+    try:
+        r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+        return r.stdout if r.returncode == 0 else ''
+    except Exception:
+        return ''
+
+
+def cpu_times():
+    with open('/proc/stat') as f:
+        parts = [int(x) for x in f.readline().split()[1:]]
+    idle = parts[3] + (parts[4] if len(parts) > 4 else 0)
+    return idle, sum(parts[:8])
+
+
+def meminfo():
+    m = {}
+    with open('/proc/meminfo') as f:
+        for line in f:
+            k, v = line.split(':', 1)
+            m[k] = int(v.split()[0]) * 1024
+    return m
+
+
+def system_stats():
+    i1, t1 = cpu_times()
+    time.sleep(1)
+    i2, t2 = cpu_times()
+    cpu = 100.0 * (1 - (i2 - i1) / max(1, t2 - t1))
+    m = meminfo()
+    with open('/proc/uptime') as f:
+        uptime = float(f.read().split()[0])
+    load = list(os.getloadavg())
+    disks = []
+    for mount in ('/',):
+        u = shutil.disk_usage(mount)
+        disks.append({'mount': mount, 'total': u.total, 'used': u.used, 'free': u.free,
+                      'pct': round(100 * u.used / u.total, 1)})
+    return {
+        'hostname': socket.gethostname(),
+        'cpus': os.cpu_count(),
+        'cpuPct': round(cpu, 1),
+        'load': [round(x, 2) for x in load],
+        'uptimeS': int(uptime),
+        'mem': {'total': m['MemTotal'], 'available': m['MemAvailable'],
+                'used': m['MemTotal'] - m['MemAvailable']},
+        'swap': {'total': m.get('SwapTotal', 0), 'used': m.get('SwapTotal', 0) - m.get('SwapFree', 0)},
+        'disks': disks,
+    }
+
+
+def containers():
+    ids = run(['sudo', '-n', 'docker', 'ps', '-aq']).split()
+    if not ids:
+        return []
+    try:
+        info = json.loads(run(['sudo', '-n', 'docker', 'inspect', *ids]) or '[]')
+    except ValueError:
+        info = []
+    stats = {}
+    for line in run(['sudo', '-n', 'docker', 'stats', '--no-stream', '--format', '{{json .}}'], 20).splitlines():
+        try:
+            s = json.loads(line)
+            stats[s['Name']] = {'cpu': s.get('CPUPerc'), 'mem': s.get('MemUsage', '').split(' / ')[0]}
+        except (ValueError, KeyError):
+            pass
+    out = []
+    for c in info:
+        st = c.get('State', {})
+        name = c.get('Name', '').lstrip('/')
+        out.append({
+            'name': name,
+            'image': c.get('Config', {}).get('Image', '').split('@')[0][:80],
+            'state': st.get('Status'),
+            'health': (st.get('Health') or {}).get('Status'),
+            'restarts': c.get('RestartCount', 0),
+            'exitCode': st.get('ExitCode'),
+            'startedAt': st.get('StartedAt'),
+            'cpu': stats.get(name, {}).get('cpu'),
+            'mem': stats.get(name, {}).get('mem'),
+        })
+    out.sort(key=lambda c: c['name'])
+    return out
+
+
+def unit_props(unit, scope, props):
+    cmd = ['systemctl'] + (['--user'] if scope == 'user' else []) + [
+        'show', unit, '-p', ','.join(props)]
+    d = {}
+    for line in run(cmd).splitlines():
+        if '=' in line:
+            k, v = line.split('=', 1)
+            d[k] = v
+    return d
+
+
+def list_timers(scope):
+    cmd = ['systemctl'] + (['--user'] if scope == 'user' else []) + ['list-timers', '--all', '-o', 'json']
+    try:
+        return {t['unit']: t for t in json.loads(run(cmd) or '[]')}
+    except (ValueError, KeyError, TypeError):
+        return {}
+
+
+def sec(usec):
+    try:
+        n = int(usec or 0)
+        return n // 1_000_000 if n > 0 else None
+    except (ValueError, TypeError):
+        return None
+
+
+def timers():
+    lists = {'user': list_timers('user'), 'system': list_timers('system')}
+    out = []
+    for label, unit, scope in TIMERS + [('Home collector', 'home-collector', 'user')]:
+        t = lists[scope].get(unit + '.timer')
+        if not t:
+            continue
+        s = unit_props(unit + '.service', scope, ['ActiveState', 'Result', 'ExecMainStatus'])
+        out.append({
+            'label': label, 'unit': unit, 'scope': scope,
+            'last': sec(t.get('last')),
+            'next': sec(t.get('next')),
+            'running': s.get('ActiveState') in ('activating', 'active', 'deactivating'),
+            'result': s.get('Result'),
+            'exitStatus': int(s.get('ExecMainStatus') or 0),
+        })
+    return out
+
+
+_insecure = ssl.create_default_context()
+_insecure.check_hostname = False
+_insecure.verify_mode = ssl.CERT_NONE
+
+
+def check(service):
+    name, link, url = service
+    res = {'name': name, 'url': link, 'ok': None, 'code': None, 'ms': None}
+    if not url:
+        return res
+    start = time.monotonic()
+    try:
+        req = urllib.request.Request(url, headers={'User-Agent': 'home-collector'})
+        with urllib.request.urlopen(req, timeout=5, context=_insecure) as r:
+            r.read(256)
+            res['code'] = r.status
+    except urllib.error.HTTPError as e:
+        res['code'] = e.code
+    except Exception:
+        res['code'] = 0
+    res['ms'] = round((time.monotonic() - start) * 1000)
+    res['ok'] = bool(res['code']) and res['code'] < 500
+    return res
+
+
+def services():
+    with cf.ThreadPoolExecutor(8) as ex:
+        return list(ex.map(check, SERVICES))
+
+
+def b2_cost():
+    d = os.path.join(HOME, 'backups-cost', 'daily')
+    try:
+        files = sorted(f for f in os.listdir(d) if f.endswith('.json'))
+        if files:
+            with open(os.path.join(d, files[-1])) as f:
+                return json.load(f)
+    except (OSError, ValueError):
+        pass
+    return None
+
+
+def ntfy_topics(cfg):
+    topics = []
+    for label, path, key in (('disk-guard', 'disk-guard.env', 'NTFY_URL'),
+                             ('houses', 'house-scrape.env', 'HOUSE_NTFY_URL')):
+        url = read_env(os.path.join(CFG, path)).get(key)
+        if url:
+            topics.append((label, url))
+    for item in (cfg.get('NTFY_EXTRA') or '').split(','):
+        if '=' in item:
+            label, url = item.split('=', 1)
+            topics.append((label.strip(), url.strip()))
+    return topics
+
+
+def ntfy(cfg):
+    msgs, seen, errors = [], set(), []
+    for label, url in ntfy_topics(cfg):
+        if url in seen:
+            continue
+        seen.add(url)
+        try:
+            req = urllib.request.Request(url.rstrip('/') + '/json?poll=1&since=72h',
+                                         headers={'User-Agent': 'home-collector'})
+            with urllib.request.urlopen(req, timeout=8) as r:
+                for line in r.read().decode('utf-8', 'replace').splitlines():
+                    try:
+                        m = json.loads(line)
+                    except ValueError:
+                        continue
+                    if m.get('event') != 'message':
+                        continue
+                    msgs.append({'id': m.get('id'), 'topic': label, 'time': m.get('time'),
+                                 'title': (m.get('title') or '')[:200],
+                                 'message': (m.get('message') or '')[:600],
+                                 'priority': m.get('priority', 3), 'tags': m.get('tags') or [],
+                                 'click': m.get('click')})
+        except Exception:
+            errors.append(label)
+    msgs.sort(key=lambda m: m['time'] or 0, reverse=True)
+    return {'messages': msgs[:40], 'topics': [t for t, _ in ntfy_topics(cfg)], 'errors': errors}
+
+
+def write(db_url, rows):
+    tag = 'j' + secrets.token_hex(8)
+    sql = ['BEGIN;']
+    for key, data in rows.items():
+        payload = json.dumps(data, separators=(',', ':'))
+        sql.append(
+            f"INSERT INTO home.snapshots (key, data, collected_at) VALUES ('{key}', ${tag}${payload}${tag}$::jsonb, now()) "
+            "ON CONFLICT (key) DO UPDATE SET data = excluded.data, collected_at = excluded.collected_at;")
+    sql.append('COMMIT;')
+    r = subprocess.run(['psql', '-X', '-q', '-v', 'ON_ERROR_STOP=1', db_url], input='\n'.join(sql),
+                       capture_output=True, text=True, timeout=20)
+    if r.returncode != 0:
+        # psql errors never contain the password; the URL is not echoed.
+        sys.stderr.write(r.stderr[-500:])
+        sys.exit(1)
+
+
+def main():
+    cfg = read_env(os.path.join(CFG, 'home-collector.env'))
+    with cf.ThreadPoolExecutor(4) as ex:
+        f_sys = ex.submit(system_stats)
+        f_ct = ex.submit(containers)
+        f_svc = ex.submit(services)
+        f_ntfy = ex.submit(ntfy, cfg)
+        host = f_sys.result()
+        host['containers'] = f_ct.result()
+        host['services'] = f_svc.result()
+        host['timers'] = timers()
+        host['b2'] = b2_cost()
+        notes = f_ntfy.result()
+    rows = {'host': host, 'ntfy': notes}
+    if '--print' in sys.argv:
+        print(json.dumps(rows, indent=1))
+        return
+    url = os.environ.get('HOME_DB_URL') or cfg.get('HOME_DB_URL')
+    if not url:
+        sys.exit('HOME_DB_URL missing in ~/.config/home-collector.env')
+    write(url, rows)
+
+
+if __name__ == '__main__':
+    main()

deploy/home-collector/home-collector.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Snapshot server stats, containers, timers and ntfy for stanrunge.dev/start
+Wants=network-online.target
+After=network-online.target
+ConditionPathExists=%h/.config/home-collector.env
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 %h/.local/bin/home-collector
+TimeoutStartSec=50
+Nice=10

deploy/home-collector/home-collector.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run home-collector every minute (stanrunge.dev/start)
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec=60s
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target

deploy/home-collector/install.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Install/refresh the /start collector as a systemd user timer (run as stan on the server).
+# Needs ~/.config/home-collector.env with HOME_DB_URL for the home_collector role (see README).
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+install -D -m 755 "$here/home-collector.py" "$HOME/.local/bin/home-collector"
+install -D -m 644 "$here/home-collector.service" "$HOME/.config/systemd/user/home-collector.service"
+install -D -m 644 "$here/home-collector.timer" "$HOME/.config/systemd/user/home-collector.timer"
+systemctl --user daemon-reload
+systemctl --user enable --now home-collector.timer
+systemctl --user start home-collector.service || true
+systemctl --user --no-pager status home-collector.service | head -5

deploy/music-collector/README.md

@@ -0,0 +1,66 @@
+# music (self-hosted Last.fm)
+
+`/trackers/music` (stats) and the "listening now" card on the homepage read the `music`
+schema in the `personal` database:
+
+| table / view              | contents                                                                                        |
+| ------------------------- | ----------------------------------------------------------------------------------------------- |
+| `artists`, `albums`       | Spotify ids when known (imports are matched by name), album art URLs (640 px + small)           |
+| `tracks`, `track_artists` | `match_key` = normalised `artist\|title` (`src/lib/music/normalize.ts`) for cross-source dedupe |
+| `plays`                   | one row per play: `source` spotify / lastfm / youtube / local, `origin` api / export / import   |
+| `plays_deduped` (view)    | `plays` minus Last.fm scrobbles that Spotify also has (what every stat reads)                   |
+| `now_playing`             | row 1, refreshed every minute                                                                   |
+| `spotify_auth`            | row 1, refresh token + poller state. **Not** granted to `personal_ro`                           |
+| `import_state`            | resumable backfill progress (`lastfm:<user>`)                                                   |
+
+## Spotify (live)
+
+1. In the Spotify developer dashboard (the existing app, `SPOTIFY_CLIENT_ID`) add the redirect
+   URIs `https://stanrunge.dev/api/spotify/callback` and
+   `https://dev.stanrunge.dev/api/spotify/callback`.
+2. Logged in as the owner, open <https://stanrunge.dev/api/spotify/connect> once. Scopes:
+   `user-read-currently-playing user-read-recently-played user-read-playback-state user-top-read user-library-read playlist-read-private`.
+3. That's it: the **app itself** polls every minute (`src/lib/server/music/poller.ts`, started
+   from `hooks.server.ts` `init`) with its own role and Spotify credentials. Status (last poll,
+   last error) is shown to the owner on `/trackers/music`.
+
+### Optional: poll from the host instead
+
+Only if the poller should keep running independent of the app container. Needs its own role:
+
+```sh
+sudo docker exec -i postgres-db-1 psql -U postgres -d personal <<'SQL'
+CREATE ROLE music_collector LOGIN CONNECTION LIMIT 3 PASSWORD '<pick one>';
+GRANT CONNECT ON DATABASE personal TO music_collector;
+GRANT USAGE ON SCHEMA music TO music_collector;
+GRANT SELECT, INSERT, UPDATE ON music.artists, music.albums, music.tracks, music.track_artists,
+  music.plays, music.now_playing, music.import_state TO music_collector;
+GRANT SELECT, UPDATE ON music.spotify_auth TO music_collector;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA music TO music_collector;
+SQL
+```
+
+`~/.config/music-collector.env` (mode 600): `MUSIC_DB_URL=postgresql://music_collector:…@127.0.0.1:5440/personal`,
+`SPOTIFY_CLIENT_ID=…`, `SPOTIFY_CLIENT_SECRET=…`. Then `bash deploy/music-collector/install.sh --enable`
+and set `MUSIC_POLLER=off` in prod.env. (The music migration grants the same when the role
+already exists.) Debug: `journalctl --user -u music-collector -n 20`.
+
+## Backfills
+
+Both run from the deployed image with the app role, so no extra credentials:
+
+```sh
+tag=$(sudo sed -n 's/^IMAGE_TAG=//p' /opt/personal-website/.env)
+# Last.fm (needs an API key from https://www.last.fm/api/account/create); resumable, re-run any time
+sudo docker run --rm --network backend --env-file /etc/personal-website/prod.env \
+  -e LASTFM_API_KEY=<key> git.stanrunge.dev/stan/website:$tag bun scripts/music-import-lastfm.ts --user stanrunge
+# Spotify Extended Streaming History (unzipped export in ~/spotify-export)
+sudo docker run --rm --network backend --env-file /etc/personal-website/prod.env \
+  -v ~/spotify-export:/data:ro git.stanrunge.dev/stan/website:$tag bun scripts/music-import-history.ts /data
+```
+
+Dedupe: within one source a play within 60 s of the same `match_key` is skipped on insert;
+across sources `plays_deduped` hides a Last.fm scrobble when a Spotify play of the same
+`match_key` ended between 3 min before and track length + 3 min after the scrobble time
+(Last.fm records when a track started, Spotify when it ended). Listening time uses
+`ms_played` when known, else the track length.

deploy/music-collector/install.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Install the optional host-side Spotify poller as a systemd user timer (run as stan on the
+# server). Copies the units; enables the timer only with --enable (needs
+# ~/.config/music-collector.env, see README.md). Normally not needed: the app polls itself.
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+install -D -m 644 "$here/music-collector.service" "$HOME/.config/systemd/user/music-collector.service"
+install -D -m 644 "$here/music-collector.timer" "$HOME/.config/systemd/user/music-collector.timer"
+systemctl --user daemon-reload
+if [ "${1:-}" = --enable ]; then
+  test -f "$HOME/.config/music-collector.env" || { echo 'Missing ~/.config/music-collector.env' >&2; exit 1; }
+  systemctl --user enable --now music-collector.timer
+  systemctl --user start music-collector.service || true
+  systemctl --user --no-pager status music-collector.service | head -5
+else
+  echo 'Installed (not enabled). Enable with: bash deploy/music-collector/install.sh --enable'
+fi

deploy/music-collector/music-collector.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Spotify scrobbler poll (now playing + recently played -> music.*) for stanrunge.dev/trackers/music
+Wants=network-online.target
+After=network-online.target
+ConditionPathExists=%h/.config/music-collector.env
+
+[Service]
+Type=oneshot
+# A checkout of stan/website on main. Not the live dev tree.
+WorkingDirectory=%h/dev/website-music
+# MUSIC_DB_URL (music_collector role), SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET; mode 600.
+EnvironmentFile=%h/.config/music-collector.env
+ExecStart=/usr/bin/bun scripts/music-poll.ts
+TimeoutStartSec=50
+Nice=10

deploy/music-collector/music-collector.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Spotify scrobbler poll every minute (optional; the app polls by itself)
+
+[Timer]
+OnBootSec=45s
+OnUnitActiveSec=60s
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target

deploy/systemd/house-scrape.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=House tracker nightly scrape (Kamernet, Vesteda -> house.listings)
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+# A checkout of stan/website on main. Not the live dev tree.
+WorkingDirectory=%h/dev/website-houses
+# DB_URL (house_scraper role: house schema only), optional HOUSE_NTFY_URL; mode 600, never committed.
+EnvironmentFile=%h/.config/house-scrape.env
+ExecStart=/usr/bin/bun scripts/house-scrape.ts
+TimeoutStartSec=900

deploy/systemd/house-scrape.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=House tracker scrape, once a night
+
+[Timer]
+OnCalendar=*-*-* 03:40:00 Europe/Amsterdam
+RandomizedDelaySec=1200
+Persistent=true
+
+[Install]
+WantedBy=timers.target

docs/house-tracker.md

@@ -1,7 +1,54 @@
 # House tracker
 
+## Nightly scrape on stan-vm (2026-10)
+
+`scripts/house-scrape.ts` runs once a night from a systemd user timer
+(`deploy/systemd/house-scrape.{service,timer}`, 03:40 Amsterdam + up to 20 min jitter) and
+writes `house.listings` directly with the limited `house_scraper` role
+(`~/.config/house-scrape.env`, mode 600). One request at a time, 6–10 s apart, honest
+User-Agent, robots.txt-allowed paths only. A 403/429/challenge stops that source for the
+night and is logged as `challenged`; nothing is retried or worked around.
+
+| source                 | how                                                           | status                               |
+| ---------------------- | ------------------------------------------------------------- | ------------------------------------ |
+| Kamernet               | `/huren/huurwoningen-<city>` pages, `__NEXT_DATA__` JSON      | scraped (10 cities around Gouda)     |
+| Vesteda                | `POST /api/units/search/facet` (what their site calls), 20 km | scraped; rented units count as gone  |
+| Pararius               | Cloudflare 403 from the Hetzner IP                            | not scraped here; home worker below  |
+| huurwoningen.nl        | same Cloudflare 403 (Pararius group)                          | skipped                              |
+| Funda                  | bot-check page ("Je bent bijna op de pagina…")                | skipped                              |
+| Huurstunt              | Cloudflare challenge                                          | skipped                              |
+| Mijndak Midden-Holland | OutSystems app behind a login, no public listing feed         | skipped; use their own e-mail alerts |
+
+Same home on two sources: `address_key` (street + number + city, normalised, see
+`src/lib/house-tracker/address.ts`) links the copy to the original via `duplicate_of`; a
+manual entry wins. The page shows one row with "also on …" links. Kamernet never shows house
+numbers, so its listings are never merged. Alerts: new non-copy listings matching an alert
+rule go to `HOUSE_NTFY_URL` (its own topic, max 5 per night plus one summary); the first run
+of a source only seeds.
+
+No caps at scrape time: every listing in the region is stored (Kamernet city pages, Vesteda
+with an open rent range, Pararius without a price band). The €2,000 / 2-bedroom / 50 m²
+limits are only the page's default filters (clear a field to drop it) and the alert rule.
+
+Liveability: every listing gets a Leefbaarometer 3.0 score (BZK, measurement 2024, published
+July 2026; `src/lib/house-tracker/liveability.ts`). The address is geocoded with PDOK
+Locatieserver (exact address, else 6-char postcode, else street centroid → marked approx.),
+then the 100 m grid cell and the buurt containing the point are read from the official WFS
+(`geo.leefbaarometer.nl/lbm3`, layers `clippedgridscore24` / `buurtscore24`). The cell score
+is used when there is one (and the location is not street-level), else the buurt. Stored on
+the row: `lbm_class` (1-9, zeer onvoldoende … uitstekend), `lbm_onv_class` (overlast &
+onveiligheid, 1-9 where 5 = Dutch average), buurt, level, precision, year, and all
+sub-scores in `lbm_detail`. Scored by the nightly scrape (new and changed addresses), the
+CLI and the page's add form; backfill or re-score with `scripts/house-liveability.ts
+[--force]`. For a new measurement, bump `LBM_YEAR` and run it with `--force`.
+
+Manual listings: `scripts/house-listing.ts` (CLI) or the form on the page. Against prod use
+`scripts/house-db.sh <command> …`, which runs the CLI as `house_cli` (SELECT/INSERT/UPDATE/
+DELETE on house.listings, SELECT/INSERT on house.price_history, nothing else) from
+`~/.config/house-cli.env`.
+
 Personal tool: collects Dutch housing listings into Postgres, shows them on
-`/projects/house-tracker` (admin only) and sends an alert when a new listing
+`/trackers/houses` (owner only, 404 for everyone else; the old URL redirects) and sends an alert when a new listing
 matches a rule. Only Pararius for now; other sources plug in the same way.
 
 ## How it fits together
@@ -176,3 +223,29 @@ fixture still covers pagination and price on request. Rental `priceEur` is the
 base/listed rent used by the search filter; a separately shown total can be
 higher (Raam: €1,475 base, €1,750 total). If Pararius changes markup, the worker
 reports `parse-error` in the worker log on the page.
+
+## Adding listings by hand
+
+`scripts/house-listing.ts` writes `house.listings` directly (links from chats,
+screenshots). No restart or deploy is needed for new rows.
+
+```sh
+# DATABASE_URL or DB_URL must be set; locally: bun --env-file=.env run house …
+bun run house add --url https://vbtverhuurmakelaars.nl/woning/gouda-vest-7-d \
+  --city Gouda --price "€ 1.422,- p.m." --area 84 --rooms 3 --bedrooms 2 --energy-label A
+echo '{"url":"…","city":"Gouda","price_eur":1350,"area_m2":81}' | bun run house add --json -
+bun run house add --street "Vest 7-D" --city Gouda --price 1422   # no link (screenshot)
+bun run house update <id|url> --status viewing --notes "za 14:00"  # "" clears a field
+bun run house list [--all] [--city gouda] [--offer buy] [--json]
+bun run house remove <id|url> [--hard]   # soft remove marks it gone; restore undoes it
+```
+
+- `source` is the site (`funda`, `pararius`, `vbt`, `vesteda`, … or `manual`
+  without a link); `source_id` comes from the URL (Funda's number, Pararius'
+  worker key, otherwise the path), or a hash of the address. Adding the same
+  link again updates that row and clears `removed_at`.
+- Fields follow `listingDetailSchema` (camelCase, snake_case or the flag names)
+  and are lenient: "€ 1.422,- p.m." and "84 m²" are parsed. Only given fields are
+  written. `city_slug` is derived from `city` in Pararius' slug style.
+- `--first-seen` sets when it was shared; price changes go to `price_history`.
+- Manual rows use `first_seen_lane = 'manual'` and never trigger alerts.

drizzle.config.ts

@@ -4,7 +4,6 @@ import { defineConfig } from 'drizzle-kit';
 //  - lists:  runtime tables created through meta.create_list() (owned by personal_ddl)
 //  - legacy: pre-reorg tables kept read-only for rollback
 //  - energy: owned by ned-collector
-//  - hardware: not modelled yet
 // Migrations run as the schema owner (`personal`), never as the app role.
 export default defineConfig({
 	dialect: 'postgresql',
@@ -21,7 +20,16 @@ export default defineConfig({
 		'site',
 		'meta',
 		'health',
-		'books'
+		'books',
+		'tasks',
+		'groceries',
+		'home',
+		'cars',
+		'music',
+		'hardware',
+		'planes',
+		'wildlife',
+		'trains'
 	],
 	migrations: { schema: 'drizzle', table: '__drizzle_migrations' },
 	verbose: true,

drizzle/20261024010518_books_reading/migration.sql

@@ -1,43 +0,0 @@
-CREATE TYPE "books"."reading_status" AS ENUM('want_to_read', 'reading', 'finished', 'abandoned');--> statement-breakpoint
-CREATE TABLE "books"."sessions" (
-	"id" serial PRIMARY KEY,
-	"work_id" integer NOT NULL,
-	"file_id" integer,
-	"started_at" timestamp with time zone DEFAULT now() NOT NULL,
-	"ended_at" timestamp with time zone,
-	"start_percent" real,
-	"end_percent" real,
-	"start_page" integer,
-	"end_page" integer,
-	"pages_read" integer,
-	"source" text DEFAULT 'reader' NOT NULL,
-	"note" text,
-	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
-	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
-	CONSTRAINT "sessions_percent_check" CHECK (("start_percent" is null or "start_percent" between 0 and 100) and ("end_percent" is null or "end_percent" between 0 and 100)),
-	CONSTRAINT "sessions_end_check" CHECK ("ended_at" is null or "ended_at" >= "started_at")
-);
---> statement-breakpoint
-ALTER TABLE "books"."progress" ADD COLUMN "status" "books"."reading_status" DEFAULT 'want_to_read'::"books"."reading_status" NOT NULL;--> statement-breakpoint
-ALTER TABLE "books"."progress" ADD COLUMN "rating" smallint;--> statement-breakpoint
-ALTER TABLE "books"."progress" ADD COLUMN "last_read_at" timestamp with time zone;--> statement-breakpoint
-CREATE INDEX "progress_status_idx" ON "books"."progress" ("status");--> statement-breakpoint
-CREATE INDEX "sessions_work_id_idx" ON "books"."sessions" ("work_id");--> statement-breakpoint
-CREATE INDEX "sessions_started_at_idx" ON "books"."sessions" ("started_at");--> statement-breakpoint
-ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
-ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
-ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_rating_check" CHECK ("rating" is null or "rating" between 1 and 5);
---> statement-breakpoint
--- Hand-written: updated_at trigger and explicit grants (default privileges from the books migration cover new tables too).
-CREATE TRIGGER touch_updated_at BEFORE UPDATE ON books.sessions FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at();
---> statement-breakpoint
-DO $$
-BEGIN
-  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
-    GRANT SELECT, INSERT, UPDATE, DELETE ON books.sessions TO personal_app;
-    GRANT USAGE, SELECT ON SEQUENCE books.sessions_id_seq TO personal_app;
-  END IF;
-  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
-    GRANT SELECT ON books.sessions TO personal_ro;
-  END IF;
-END $$;

drizzle/20261024011919_money_bank_balance/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_cents" integer;--> statement-breakpoint
+ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_type" text;--> statement-breakpoint
+ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_at" timestamp with time zone;

drizzle/20261024013208_books/migration.sql

@@ -1,6 +1,7 @@
 CREATE SCHEMA "books";
 --> statement-breakpoint
 CREATE TYPE "books"."metadata_status" AS ENUM('pending', 'matched', 'low_confidence', 'not_found', 'manual');--> statement-breakpoint
+CREATE TYPE "books"."reading_status" AS ENUM('want_to_read', 'reading', 'finished', 'abandoned');--> statement-breakpoint
 CREATE TABLE "books"."files" (
 	"id" serial PRIMARY KEY,
 	"work_id" integer NOT NULL,
@@ -36,9 +37,32 @@ CREATE TABLE "books"."progress" (
 	"percent" real DEFAULT 0 NOT NULL,
 	"started_at" timestamp with time zone,
 	"finished_at" timestamp with time zone,
+	"status" "books"."reading_status" DEFAULT 'want_to_read'::"books"."reading_status" NOT NULL,
+	"rating" smallint,
+	"last_read_at" timestamp with time zone,
 	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
 	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
-	CONSTRAINT "progress_percent_check" CHECK ("percent" between 0 and 100)
+	CONSTRAINT "progress_percent_check" CHECK ("percent" between 0 and 100),
+	CONSTRAINT "progress_rating_check" CHECK ("rating" is null or "rating" between 1 and 5)
+);
+--> statement-breakpoint
+CREATE TABLE "books"."sessions" (
+	"id" serial PRIMARY KEY,
+	"work_id" integer NOT NULL,
+	"file_id" integer,
+	"started_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"ended_at" timestamp with time zone,
+	"start_percent" real,
+	"end_percent" real,
+	"start_page" integer,
+	"end_page" integer,
+	"pages_read" integer,
+	"source" text DEFAULT 'reader' NOT NULL,
+	"note" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "sessions_percent_check" CHECK (("start_percent" is null or "start_percent" between 0 and 100) and ("end_percent" is null or "end_percent" between 0 and 100)),
+	CONSTRAINT "sessions_end_check" CHECK ("ended_at" is null or "ended_at" >= "started_at")
 );
 --> statement-breakpoint
 CREATE TABLE "books"."shelf_works" (
@@ -92,6 +116,9 @@ CREATE UNIQUE INDEX "files_sha256_key" ON "books"."files" ("sha256");--> stateme
 CREATE INDEX "files_work_id_idx" ON "books"."files" ("work_id");--> statement-breakpoint
 CREATE INDEX "notes_work_id_idx" ON "books"."notes" ("work_id");--> statement-breakpoint
 CREATE UNIQUE INDEX "progress_work_id_key" ON "books"."progress" ("work_id");--> statement-breakpoint
+CREATE INDEX "progress_status_idx" ON "books"."progress" ("status");--> statement-breakpoint
+CREATE INDEX "sessions_work_id_idx" ON "books"."sessions" ("work_id");--> statement-breakpoint
+CREATE INDEX "sessions_started_at_idx" ON "books"."sessions" ("started_at");--> statement-breakpoint
 CREATE INDEX "shelf_works_work_id_idx" ON "books"."shelf_works" ("work_id");--> statement-breakpoint
 CREATE UNIQUE INDEX "shelves_slug_key" ON "books"."shelves" ("slug");--> statement-breakpoint
 CREATE UNIQUE INDEX "works_isbn13_key" ON "books"."works" ("isbn13") WHERE "isbn13" is not null;--> statement-breakpoint
@@ -101,6 +128,8 @@ ALTER TABLE "books"."notes" ADD CONSTRAINT "notes_work_id_works_id_fkey" FOREIGN
 ALTER TABLE "books"."notes" ADD CONSTRAINT "notes_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
 ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
 ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
 ALTER TABLE "books"."shelf_works" ADD CONSTRAINT "shelf_works_shelf_id_shelves_id_fkey" FOREIGN KEY ("shelf_id") REFERENCES "books"."shelves"("id") ON DELETE CASCADE;--> statement-breakpoint
 ALTER TABLE "books"."shelf_works" ADD CONSTRAINT "shelf_works_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;
 --> statement-breakpoint
@@ -141,6 +170,25 @@ BEGIN
            ('books','shelves','books-shelves','Book shelves','core',true,false,1000),
            ('books','files','books-files','Book files','core',false,false,1000)
     ON CONFLICT DO NOTHING;
-    PERFORM meta.sync();
+    -- meta.sync() is executable by personal_app/personal_ddl, not by the migration role
+    -- (personal) in prod; deploy-remote.sh runs it as the app role after migrating.
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;
+
+--> statement-breakpoint
+-- Hand-written: updated_at trigger and explicit grants (default privileges from the books migration cover new tables too).
+CREATE TRIGGER touch_updated_at BEFORE UPDATE ON books.sessions FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at();
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON books.sessions TO personal_app;
+    GRANT USAGE, SELECT ON SEQUENCE books.sessions_id_seq TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT SELECT ON books.sessions TO personal_ro;
   END IF;
 END $$;

drizzle/20261024013208_books/snapshot.json

@@ -1,9 +1,9 @@
 {
   "version": "8",
   "dialect": "postgres",
-  "id": "f239c655-63c8-4807-b44f-b7698330d30a",
+  "id": "b9c155ef-74b1-4481-acaa-7649155ee04f",
   "prevIds": [
-    "ffad1d0e-51e0-4302-9ab2-022eba184390"
+    "46aa2cce-9f94-40c6-ae0d-1489f5164c74"
   ],
   "ddl": [
     {
@@ -5265,6 +5265,45 @@
       "schema": "money",
       "table": "accounts"
     },
+    {
+      "type": "integer",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "bank_balance_cents",
+      "entityType": "columns",
+      "schema": "money",
+      "table": "accounts"
+    },
+    {
+      "type": "text",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "bank_balance_type",
+      "entityType": "columns",
+      "schema": "money",
+      "table": "accounts"
+    },
+    {
+      "type": "timestamp with time zone",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "bank_balance_at",
+      "entityType": "columns",
+      "schema": "money",
+      "table": "accounts"
+    },
     {
       "type": "timestamp with time zone",
       "typeSchema": null,

drizzle/20261024021157_money_include_in_total/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "money"."accounts" ADD COLUMN "include_in_total" boolean DEFAULT true NOT NULL;

drizzle/20261024021157_money_include_in_total/snapshot.json

@@ -1,9 +1,9 @@
 {
   "version": "8",
   "dialect": "postgres",
-  "id": "ffad1d0e-51e0-4302-9ab2-022eba184390",
+  "id": "d7b58ea0-d24b-4872-b34b-d03d7e202744",
   "prevIds": [
-    "2aa940a3-e75f-4f6f-b12f-6935c9bd4f9c"
+    "b9c155ef-74b1-4481-acaa-7649155ee04f"
   ],
   "ddl": [
     {
@@ -85,6 +85,17 @@
       "entityType": "enums",
       "schema": "books"
     },
+    {
+      "values": [
+        "want_to_read",
+        "reading",
+        "finished",
+        "abandoned"
+      ],
+      "name": "reading_status",
+      "entityType": "enums",
+      "schema": "books"
+    },
     {
       "isRlsEnabled": false,
       "name": "account",
@@ -493,6 +504,12 @@
       "entityType": "tables",
       "schema": "books"
     },
+    {
+      "isRlsEnabled": false,
+      "name": "sessions",
+      "entityType": "tables",
+      "schema": "books"
+    },
     {
       "isRlsEnabled": false,
       "name": "shelf_works",
@@ -5170,6 +5187,19 @@
       "schema": "money",
       "table": "accounts"
     },
+    {
+      "type": "boolean",
+      "typeSchema": null,
+      "notNull": true,
+      "dimensions": 0,
+      "default": "true",
+      "generated": null,
+      "identity": null,
+      "name": "include_in_total",
+      "entityType": "columns",
+      "schema": "money",
+      "table": "accounts"
+    },
     {
       "type": "text",
       "typeSchema": null,
@@ -5248,6 +5278,45 @@
       "schema": "money",
       "table": "accounts"
     },
+    {
+      "type": "integer",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "bank_balance_cents",
+      "entityType": "columns",
+      "schema": "money",
+      "table": "accounts"
+    },
+    {
+      "type": "text",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "bank_balance_type",
+      "entityType": "columns",
+      "schema": "money",
+      "table": "accounts"
+    },
+    {
+      "type": "timestamp with time zone",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "bank_balance_at",
+      "entityType": "columns",
+      "schema": "money",
+      "table": "accounts"
+    },
     {
       "type": "timestamp with time zone",
       "typeSchema": null,
@@ -8831,6 +8900,45 @@
       "schema": "books",
       "table": "progress"
     },
+    {
+      "type": "reading_status",
+      "typeSchema": "books",
+      "notNull": true,
+      "dimensions": 0,
+      "default": "'want_to_read'",
+      "generated": null,
+      "identity": null,
+      "name": "status",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "progress"
+    },
+    {
+      "type": "smallint",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "rating",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "progress"
+    },
+    {
+      "type": "timestamp with time zone",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "last_read_at",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "progress"
+    },
     {
       "type": "timestamp with time zone",
       "typeSchema": null,
@@ -8857,6 +8965,188 @@
       "schema": "books",
       "table": "progress"
     },
+    {
+      "type": "serial",
+      "typeSchema": null,
+      "notNull": true,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "id",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "integer",
+      "typeSchema": null,
+      "notNull": true,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "work_id",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "integer",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "file_id",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "timestamp with time zone",
+      "typeSchema": null,
+      "notNull": true,
+      "dimensions": 0,
+      "default": "now()",
+      "generated": null,
+      "identity": null,
+      "name": "started_at",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "timestamp with time zone",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "ended_at",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "real",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "start_percent",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "real",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "end_percent",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "integer",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "start_page",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "integer",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "end_page",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "integer",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "pages_read",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "text",
+      "typeSchema": null,
+      "notNull": true,
+      "dimensions": 0,
+      "default": "'reader'",
+      "generated": null,
+      "identity": null,
+      "name": "source",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "text",
+      "typeSchema": null,
+      "notNull": false,
+      "dimensions": 0,
+      "default": null,
+      "generated": null,
+      "identity": null,
+      "name": "note",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "timestamp with time zone",
+      "typeSchema": null,
+      "notNull": true,
+      "dimensions": 0,
+      "default": "now()",
+      "generated": null,
+      "identity": null,
+      "name": "created_at",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "type": "timestamp with time zone",
+      "typeSchema": null,
+      "notNull": true,
+      "dimensions": 0,
+      "default": "now()",
+      "generated": null,
+      "identity": null,
+      "name": "updated_at",
+      "entityType": "columns",
+      "schema": "books",
+      "table": "sessions"
+    },
     {
       "type": "integer",
       "typeSchema": null,
@@ -10502,6 +10792,69 @@
       "schema": "books",
       "table": "progress"
     },
+    {
+      "nameExplicit": true,
+      "columns": [
+        {
+          "value": "status",
+          "isExpression": false,
+          "asc": true,
+          "nullsFirst": false,
+          "opclass": null
+        }
+      ],
+      "isUnique": false,
+      "where": null,
+      "with": "",
+      "method": "btree",
+      "concurrently": false,
+      "name": "progress_status_idx",
+      "entityType": "indexes",
+      "schema": "books",
+      "table": "progress"
+    },
+    {
+      "nameExplicit": true,
+      "columns": [
+        {
+          "value": "work_id",
+          "isExpression": false,
+          "asc": true,
+          "nullsFirst": false,
+          "opclass": null
+        }
+      ],
+      "isUnique": false,
+      "where": null,
+      "with": "",
+      "method": "btree",
+      "concurrently": false,
+      "name": "sessions_work_id_idx",
+      "entityType": "indexes",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "nameExplicit": true,
+      "columns": [
+        {
+          "value": "started_at",
+          "isExpression": false,
+          "asc": true,
+          "nullsFirst": false,
+          "opclass": null
+        }
+      ],
+      "isUnique": false,
+      "where": null,
+      "with": "",
+      "method": "btree",
+      "concurrently": false,
+      "name": "sessions_started_at_idx",
+      "entityType": "indexes",
+      "schema": "books",
+      "table": "sessions"
+    },
     {
       "nameExplicit": true,
       "columns": [
@@ -11742,6 +12095,40 @@
       "schema": "books",
       "table": "progress"
     },
+    {
+      "nameExplicit": false,
+      "columns": [
+        "work_id"
+      ],
+      "schemaTo": "books",
+      "tableTo": "works",
+      "columnsTo": [
+        "id"
+      ],
+      "onUpdate": "NO ACTION",
+      "onDelete": "CASCADE",
+      "name": "sessions_work_id_works_id_fkey",
+      "entityType": "fks",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "nameExplicit": false,
+      "columns": [
+        "file_id"
+      ],
+      "schemaTo": "books",
+      "tableTo": "files",
+      "columnsTo": [
+        "id"
+      ],
+      "onUpdate": "NO ACTION",
+      "onDelete": "SET NULL",
+      "name": "sessions_file_id_files_id_fkey",
+      "entityType": "fks",
+      "schema": "books",
+      "table": "sessions"
+    },
     {
       "nameExplicit": false,
       "columns": [
@@ -12471,6 +12858,16 @@
       "table": "progress",
       "entityType": "pks"
     },
+    {
+      "columns": [
+        "id"
+      ],
+      "nameExplicit": false,
+      "name": "sessions_pkey",
+      "schema": "books",
+      "table": "sessions",
+      "entityType": "pks"
+    },
     {
       "columns": [
         "id"
@@ -13305,6 +13702,27 @@
       "schema": "books",
       "table": "progress"
     },
+    {
+      "value": "\"rating\" is null or \"rating\" between 1 and 5",
+      "name": "progress_rating_check",
+      "entityType": "checks",
+      "schema": "books",
+      "table": "progress"
+    },
+    {
+      "value": "(\"start_percent\" is null or \"start_percent\" between 0 and 100) and (\"end_percent\" is null or \"end_percent\" between 0 and 100)",
+      "name": "sessions_percent_check",
+      "entityType": "checks",
+      "schema": "books",
+      "table": "sessions"
+    },
+    {
+      "value": "\"ended_at\" is null or \"ended_at\" >= \"started_at\"",
+      "name": "sessions_end_check",
+      "entityType": "checks",
+      "schema": "books",
+      "table": "sessions"
+    },
     {
       "value": "\"year\" is null or \"year\" between 0 and 2100",
       "name": "works_year_check",

drizzle/20261024022836_tasks/migration.sql

@@ -0,0 +1,142 @@
+CREATE SCHEMA "tasks";
+--> statement-breakpoint
+CREATE TABLE "tasks"."activity" (
+	"id" serial PRIMARY KEY,
+	"task_id" integer,
+	"goal_id" integer,
+	"kind" text NOT NULL,
+	"from_status" text,
+	"to_status" text,
+	"note" text,
+	"value" numeric(14,2),
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "activity_target_check" CHECK ("task_id" IS NOT NULL OR "goal_id" IS NOT NULL),
+	CONSTRAINT "activity_kind_check" CHECK ("kind" IN ('created','status','edit','note','progress'))
+);
+--> statement-breakpoint
+CREATE TABLE "tasks"."areas" (
+	"id" serial PRIMARY KEY,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"description" text,
+	"color" text,
+	"sort_order" integer DEFAULT 0 NOT NULL,
+	"archived_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "areas_slug_check" CHECK ("slug" ~ '^[a-z0-9][a-z0-9-]{0,62}$'),
+	CONSTRAINT "areas_name_check" CHECK (length(btrim("name")) > 0)
+);
+--> statement-breakpoint
+CREATE TABLE "tasks"."goals" (
+	"id" serial PRIMARY KEY,
+	"title" text NOT NULL,
+	"notes" text,
+	"status" text DEFAULT 'active' NOT NULL,
+	"area_id" integer,
+	"target_date" date,
+	"metric" text,
+	"metric_unit" text,
+	"metric_target" numeric(14,2),
+	"metric_current" numeric(14,2),
+	"flagged" boolean DEFAULT false NOT NULL,
+	"sort_order" integer DEFAULT 0 NOT NULL,
+	"source" text,
+	"achieved_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "goals_title_check" CHECK (length(btrim("title")) > 0),
+	CONSTRAINT "goals_status_check" CHECK ("status" IN ('active','achieved','dropped'))
+);
+--> statement-breakpoint
+CREATE TABLE "tasks"."tasks" (
+	"id" serial PRIMARY KEY,
+	"title" text NOT NULL,
+	"notes" text,
+	"status" text DEFAULT 'todo' NOT NULL,
+	"priority" smallint DEFAULT 0 NOT NULL,
+	"flagged" boolean DEFAULT false NOT NULL,
+	"due_date" date,
+	"due_time" time,
+	"reminder_at" timestamp with time zone,
+	"recurrence" text,
+	"area_id" integer,
+	"goal_id" integer,
+	"parent_id" integer,
+	"tags" text[] DEFAULT '{}'::text[] NOT NULL,
+	"sort_order" integer DEFAULT 0 NOT NULL,
+	"source" text,
+	"external_id" text,
+	"started_at" timestamp with time zone,
+	"completed_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "tasks_title_check" CHECK (length(btrim("title")) > 0),
+	CONSTRAINT "tasks_status_check" CHECK ("status" IN ('todo','in_progress','done','dropped')),
+	CONSTRAINT "tasks_priority_check" CHECK ("priority" BETWEEN 0 AND 3),
+	CONSTRAINT "tasks_parent_check" CHECK ("parent_id" IS NULL OR "parent_id" <> "id"),
+	CONSTRAINT "tasks_time_check" CHECK ("due_time" IS NULL OR "due_date" IS NOT NULL)
+);
+--> statement-breakpoint
+CREATE INDEX "activity_task_idx" ON "tasks"."activity" ("task_id","created_at");--> statement-breakpoint
+CREATE INDEX "activity_goal_idx" ON "tasks"."activity" ("goal_id","created_at");--> statement-breakpoint
+CREATE UNIQUE INDEX "areas_slug_key" ON "tasks"."areas" ("slug");--> statement-breakpoint
+CREATE UNIQUE INDEX "goals_title_key" ON "tasks"."goals" (lower("title"));--> statement-breakpoint
+CREATE INDEX "tasks_area_idx" ON "tasks"."tasks" ("area_id");--> statement-breakpoint
+CREATE INDEX "tasks_goal_idx" ON "tasks"."tasks" ("goal_id");--> statement-breakpoint
+CREATE INDEX "tasks_parent_idx" ON "tasks"."tasks" ("parent_id");--> statement-breakpoint
+CREATE INDEX "tasks_status_due_idx" ON "tasks"."tasks" ("status","due_date");--> statement-breakpoint
+CREATE INDEX "tasks_tags_idx" ON "tasks"."tasks" USING gin ("tags");--> statement-breakpoint
+CREATE UNIQUE INDEX "tasks_source_external_key" ON "tasks"."tasks" ("source","external_id") WHERE "external_id" IS NOT NULL;--> statement-breakpoint
+ALTER TABLE "tasks"."activity" ADD CONSTRAINT "activity_task_id_tasks_id_fkey" FOREIGN KEY ("task_id") REFERENCES "tasks"."tasks"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "tasks"."activity" ADD CONSTRAINT "activity_goal_id_goals_id_fkey" FOREIGN KEY ("goal_id") REFERENCES "tasks"."goals"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "tasks"."goals" ADD CONSTRAINT "goals_area_id_areas_id_fkey" FOREIGN KEY ("area_id") REFERENCES "tasks"."areas"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_area_id_areas_id_fkey" FOREIGN KEY ("area_id") REFERENCES "tasks"."areas"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_goal_id_goals_id_fkey" FOREIGN KEY ("goal_id") REFERENCES "tasks"."goals"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_parent_id_tasks_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "tasks"."tasks"("id") ON DELETE CASCADE;
+--> statement-breakpoint
+-- Hand-written below this line (not generated by drizzle-kit).
+-- updated_at triggers, same as every other core table.
+DO $$
+DECLARE t text;
+BEGIN
+  IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
+    FOREACH t IN ARRAY ARRAY['areas','goals','tasks'] LOOP
+      EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON tasks.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+    END LOOP;
+  END IF;
+END $$;
+--> statement-breakpoint
+-- Privacy: like health, the task tracker is readable/writable by the app role only
+-- (no personal_ro grant). Grants only run on prod, where migrations run as `personal`.
+REVOKE ALL ON SCHEMA tasks FROM PUBLIC;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA tasks TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA tasks TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA tasks TO personal_app;
+    ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA tasks
+      GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA tasks
+      GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+END $$;
+--> statement-breakpoint
+-- Registered in meta.tables (structure only; the lists GUI only shows kind = 'list').
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('tasks','tasks','tasks-tasks','Tasks','core',true,true,1000),
+           ('tasks','goals','tasks-goals','Goals','core',true,false,1000),
+           ('tasks','areas','tasks-areas','Task areas','core',true,false,1000),
+           ('tasks','activity','tasks-activity','Task activity','core',false,false,1000)
+    ON CONFLICT DO NOTHING;
+    -- deploy-remote.sh runs meta.sync() as the app role after migrating.
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;

drizzle/20261024024249_meta_edit_log/migration.sql

@@ -0,0 +1,29 @@
+CREATE TABLE "meta"."edit_log" (
+	"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "meta"."edit_log_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
+	"at" timestamp with time zone DEFAULT now() NOT NULL,
+	"user_id" text,
+	"user_email" text,
+	"schema_name" text NOT NULL,
+	"table_name" text NOT NULL,
+	"action" text NOT NULL,
+	"pk" jsonb,
+	"before" jsonb,
+	"after" jsonb,
+	CONSTRAINT "edit_log_action_check" CHECK (action IN ('insert','update','delete','import','add_column'))
+);
+--> statement-breakpoint
+CREATE INDEX "edit_log_table_at_index" ON "meta"."edit_log" ("schema_name","table_name","at");--> statement-breakpoint
+-- Hand-written below this line (not generated by drizzle-kit).
+-- The app role appends to and reads the log; nobody updates or deletes through the app.
+-- personal_ro gets no access: before/after values can hold private (health) data.
+-- Grants only run on prod (as `personal`); the dev DB is owned by personal_dev.
+DO $$
+BEGIN
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT ON meta.edit_log TO personal_app;
+    GRANT USAGE ON SEQUENCE meta.edit_log_id_seq TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    REVOKE ALL ON meta.edit_log FROM personal_ro;
+  END IF;
+END $$;

drizzle/20261024030132_tasks_ticktick/migration.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "tasks"."tasks" ADD COLUMN "start_at" timestamp with time zone;--> statement-breakpoint
+ALTER TABLE "tasks"."tasks" ADD COLUMN "time_zone" text;

drizzle/20261024132343_groceries/migration.sql

@@ -0,0 +1,112 @@
+CREATE SCHEMA "groceries";
+--> statement-breakpoint
+CREATE TABLE "groceries"."prices" (
+	"id" serial PRIMARY KEY,
+	"product_id" integer NOT NULL,
+	"store_id" integer NOT NULL,
+	"price_cents" integer NOT NULL,
+	"unit_price_cents" integer,
+	"promo" boolean DEFAULT false NOT NULL,
+	"observed_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"source" text DEFAULT 'manual' NOT NULL,
+	"store_product_id" text,
+	"url" text,
+	"notes" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "prices_price_check" CHECK ("price_cents" >= 0),
+	CONSTRAINT "prices_unit_price_check" CHECK ("unit_price_cents" IS NULL OR "unit_price_cents" >= 0)
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."products" (
+	"id" serial PRIMARY KEY,
+	"name" text NOT NULL,
+	"brand" text,
+	"category" text,
+	"size" text,
+	"unit_amount" numeric(10,3),
+	"unit" text,
+	"favorite" boolean DEFAULT false NOT NULL,
+	"notes" text,
+	"archived_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "products_name_check" CHECK (length(btrim("name")) > 0),
+	CONSTRAINT "products_unit_check" CHECK ("unit" IS NULL OR "unit" IN ('piece','kg','l')),
+	CONSTRAINT "products_unit_amount_check" CHECK ("unit_amount" IS NULL OR "unit_amount" > 0)
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."stores" (
+	"id" serial PRIMARY KEY,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"website" text,
+	"color" text,
+	"sort_order" integer DEFAULT 0 NOT NULL,
+	"archived_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "stores_slug_check" CHECK ("slug" ~ '^[a-z0-9][a-z0-9-]{0,62}$'),
+	CONSTRAINT "stores_name_check" CHECK (length(btrim("name")) > 0)
+);
+--> statement-breakpoint
+CREATE INDEX "prices_product_store_idx" ON "groceries"."prices" ("product_id","store_id","observed_at");--> statement-breakpoint
+CREATE INDEX "prices_store_idx" ON "groceries"."prices" ("store_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "products_name_brand_key" ON "groceries"."products" (lower("name"),lower(coalesce("brand", '')));--> statement-breakpoint
+CREATE INDEX "products_category_idx" ON "groceries"."products" ("category");--> statement-breakpoint
+CREATE UNIQUE INDEX "stores_slug_key" ON "groceries"."stores" ("slug");--> statement-breakpoint
+ALTER TABLE "groceries"."prices" ADD CONSTRAINT "prices_product_id_products_id_fkey" FOREIGN KEY ("product_id") REFERENCES "groceries"."products"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "groceries"."prices" ADD CONSTRAINT "prices_store_id_stores_id_fkey" FOREIGN KEY ("store_id") REFERENCES "groceries"."stores"("id") ON DELETE CASCADE;
+--> statement-breakpoint
+-- Hand-written below this line (not generated by drizzle-kit).
+-- updated_at triggers, same as every other core table.
+DO $$
+DECLARE t text;
+BEGIN
+  IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
+    FOREACH t IN ARRAY ARRAY['stores','products','prices'] LOOP
+      EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON groceries.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+    END LOOP;
+  END IF;
+END $$;
+--> statement-breakpoint
+-- Private like tasks/health: app role only, no personal_ro. Grants only run on prod,
+-- where migrations run as `personal`.
+REVOKE ALL ON SCHEMA groceries FROM PUBLIC;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA groceries TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA groceries TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA groceries TO personal_app;
+    ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA groceries
+      GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA groceries
+      GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+END $$;
+--> statement-breakpoint
+INSERT INTO groceries.stores (slug, name, website, color, sort_order) VALUES
+  ('ah', 'Albert Heijn', 'https://www.ah.nl', '#00a0e2', 10),
+  ('jumbo', 'Jumbo', 'https://www.jumbo.com', '#eeb717', 20),
+  ('lidl', 'Lidl', 'https://www.lidl.nl', '#0050aa', 30),
+  ('plus', 'Plus', 'https://www.plus.nl', '#e2001a', 40)
+ON CONFLICT DO NOTHING;
+--> statement-breakpoint
+-- Registered in meta.tables so the /lists database browser shows them (kind 'core').
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('groceries','products','groceries-products','Grocery products','core',true,false,1100),
+           ('groceries','prices','groceries-prices','Grocery prices','core',true,false,1100),
+           ('groceries','stores','groceries-stores','Grocery stores','core',true,false,1100)
+    ON CONFLICT DO NOTHING;
+    -- deploy-remote.sh runs meta.sync() as the app role after migrating.
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;
+

drizzle/20261024134401_groceries_drop_manual/migration.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "groceries"."prices" DROP CONSTRAINT "prices_product_id_products_id_fkey";--> statement-breakpoint
+DROP TABLE "groceries"."prices";--> statement-breakpoint
+DROP TABLE "groceries"."products";--> statement-breakpoint
+-- The hand-entered products/prices tables were never used (0 rows); replaced by store items
+-- and groups in the next migration. Unregister them from the /lists database browser.
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    DELETE FROM meta.tables WHERE schema_name = 'groceries' AND table_name IN ('products','prices');
+  END IF;
+END $$;

drizzle/20261024134411_groceries_compare/migration.sql

@@ -0,0 +1,138 @@
+-- Trigram search for group/item names (trusted extension; the database owner may create it).
+CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
+--> statement-breakpoint
+CREATE TABLE "groceries"."groups" (
+	"id" serial PRIMARY KEY,
+	"key" text,
+	"name" text NOT NULL,
+	"brand" text,
+	"category" text,
+	"unit" text,
+	"family_key" text,
+	"favorite" boolean DEFAULT false NOT NULL,
+	"manual" boolean DEFAULT false NOT NULL,
+	"merged_into_id" integer,
+	"notes" text,
+	"item_count" integer DEFAULT 0 NOT NULL,
+	"store_count" integer DEFAULT 0 NOT NULL,
+	"store_ids" integer[] DEFAULT '{}'::integer[] NOT NULL,
+	"min_unit_price_cents" integer,
+	"min_price_cents" integer,
+	"promo_count" integer DEFAULT 0 NOT NULL,
+	"image_url" text,
+	"search_text" text DEFAULT '' NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "groups_name_check" CHECK (length(btrim("name")) > 0),
+	CONSTRAINT "groups_unit_check" CHECK ("unit" IS NULL OR "unit" IN ('piece','kg','l')),
+	CONSTRAINT "groups_not_self_merged" CHECK ("merged_into_id" IS NULL OR "merged_into_id" <> "id")
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."import_runs" (
+	"id" serial PRIMARY KEY,
+	"store_id" integer NOT NULL,
+	"started_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"finished_at" timestamp with time zone,
+	"status" text DEFAULT 'running' NOT NULL,
+	"requests" integer DEFAULT 0 NOT NULL,
+	"fetched" integer DEFAULT 0 NOT NULL,
+	"inserted" integer DEFAULT 0 NOT NULL,
+	"price_changes" integer DEFAULT 0 NOT NULL,
+	"error" text
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."item_prices" (
+	"id" serial PRIMARY KEY,
+	"item_id" integer NOT NULL,
+	"observed_on" date NOT NULL,
+	"price_cents" integer,
+	"promo_price_cents" integer,
+	"promo_label" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."items" (
+	"id" serial PRIMARY KEY,
+	"store_id" integer NOT NULL,
+	"store_product_id" text NOT NULL,
+	"name" text NOT NULL,
+	"brand" text,
+	"size_text" text,
+	"unit" text,
+	"unit_amount" numeric(12,4),
+	"unit_source" text,
+	"price_cents" integer,
+	"promo_price_cents" integer,
+	"promo_label" text,
+	"unit_price_cents" integer,
+	"promo_unit_price_cents" integer,
+	"category" text,
+	"category_path" text,
+	"gtin" text,
+	"gtin_checked_at" timestamp with time zone,
+	"image_url" text,
+	"url" text,
+	"available" boolean DEFAULT true NOT NULL,
+	"first_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"last_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"group_id" integer,
+	"match_key" text,
+	"match_method" text,
+	"match_confidence" real,
+	"match_locked" boolean DEFAULT false NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "items_unit_check" CHECK ("unit" IS NULL OR "unit" IN ('piece','kg','l')),
+	CONSTRAINT "items_price_check" CHECK ("price_cents" IS NULL OR "price_cents" >= 0)
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "groups_key_key" ON "groceries"."groups" ("key");--> statement-breakpoint
+CREATE INDEX "groups_family_idx" ON "groceries"."groups" ("family_key");--> statement-breakpoint
+CREATE INDEX "groups_category_idx" ON "groceries"."groups" ("category");--> statement-breakpoint
+CREATE INDEX "groups_store_ids_idx" ON "groceries"."groups" USING gin ("store_ids");--> statement-breakpoint
+CREATE INDEX "groups_search_trgm_idx" ON "groceries"."groups" USING gin ("search_text" gin_trgm_ops);--> statement-breakpoint
+CREATE INDEX "groups_list_idx" ON "groceries"."groups" ("store_count","name");--> statement-breakpoint
+CREATE INDEX "import_runs_store_idx" ON "groceries"."import_runs" ("store_id","started_at");--> statement-breakpoint
+CREATE UNIQUE INDEX "item_prices_item_day_key" ON "groceries"."item_prices" ("item_id","observed_on");--> statement-breakpoint
+CREATE UNIQUE INDEX "items_store_product_key" ON "groceries"."items" ("store_id","store_product_id");--> statement-breakpoint
+CREATE INDEX "items_group_idx" ON "groceries"."items" ("group_id");--> statement-breakpoint
+CREATE INDEX "items_gtin_idx" ON "groceries"."items" ("gtin");--> statement-breakpoint
+CREATE INDEX "items_name_trgm_idx" ON "groceries"."items" USING gin ("name" gin_trgm_ops);--> statement-breakpoint
+ALTER TABLE "groceries"."groups" ADD CONSTRAINT "groups_merged_into_id_groups_id_fkey" FOREIGN KEY ("merged_into_id") REFERENCES "groceries"."groups"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "groceries"."import_runs" ADD CONSTRAINT "import_runs_store_id_stores_id_fkey" FOREIGN KEY ("store_id") REFERENCES "groceries"."stores"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "groceries"."item_prices" ADD CONSTRAINT "item_prices_item_id_items_id_fkey" FOREIGN KEY ("item_id") REFERENCES "groceries"."items"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "groceries"."items" ADD CONSTRAINT "items_store_id_stores_id_fkey" FOREIGN KEY ("store_id") REFERENCES "groceries"."stores"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "groceries"."items" ADD CONSTRAINT "items_group_id_groups_id_fkey" FOREIGN KEY ("group_id") REFERENCES "groceries"."groups"("id") ON DELETE SET NULL;--> statement-breakpoint
+DO $$
+DECLARE t text;
+BEGIN
+  IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
+    FOREACH t IN ARRAY ARRAY['groups','items'] LOOP
+      EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON groceries.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+    END LOOP;
+  END IF;
+END $$;
+--> statement-breakpoint
+-- Private: app role only (default privileges from the first groceries migration also apply).
+DO $$
+BEGIN
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA groceries TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA groceries TO personal_app;
+  END IF;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('groceries','groups','groceries-groups','Grocery groups','core',true,false,1100),
+           ('groceries','items','groceries-items','Grocery store items','core',true,false,1100),
+           ('groceries','item_prices','groceries-item-prices','Grocery price history','core',false,false,1100),
+           ('groceries','import_runs','groceries-import-runs','Grocery import runs','core',false,false,1100)
+    ON CONFLICT DO NOTHING;
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;

drizzle/20261024140745_house_scrapers/migration.sql

@@ -0,0 +1,5 @@
+ALTER TABLE "house"."listings" ADD COLUMN "address_key" text;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "duplicate_of" integer;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "total_price_eur" integer;--> statement-breakpoint
+CREATE INDEX "listings_address_idx" ON "house"."listings" ("address_key") WHERE address_key IS NOT NULL;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD CONSTRAINT "listings_duplicate_of_listings_id_fkey" FOREIGN KEY ("duplicate_of") REFERENCES "house"."listings"("id") ON DELETE SET NULL;

drizzle/20261024141609_health_scale_metrics/migration.sql

@@ -0,0 +1,20 @@
+ALTER TABLE "health"."body_measurements" ADD COLUMN "muscle_mass_kg" numeric(5,2);--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "skeletal_muscle_kg" numeric(5,2);--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "bone_mass_kg" numeric(4,2);--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "visceral_fat" numeric(4,1);--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "body_water_pct" numeric(4,1);--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "protein_pct" numeric(4,1);--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "bmr_kcal" integer;--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "bmi" numeric(4,1);--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "measured_at" timestamp with time zone;--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD COLUMN "source" text;--> statement-breakpoint
+CREATE UNIQUE INDEX "body_measurements_date_weight_key" ON "health"."body_measurements" ("date","weight_kg");--> statement-breakpoint
+CREATE UNIQUE INDEX "body_measurements_measured_at_key" ON "health"."body_measurements" ("measured_at");--> statement-breakpoint
+ALTER TABLE "health"."body_measurements" ADD CONSTRAINT "body_measurements_scale_check" CHECK (("muscle_mass_kg" IS NULL OR "muscle_mass_kg" BETWEEN 5 AND 250)
+			AND ("skeletal_muscle_kg" IS NULL OR "skeletal_muscle_kg" BETWEEN 5 AND 200)
+			AND ("bone_mass_kg" IS NULL OR "bone_mass_kg" BETWEEN 0.5 AND 20)
+			AND ("visceral_fat" IS NULL OR "visceral_fat" BETWEEN 1 AND 60)
+			AND ("body_water_pct" IS NULL OR "body_water_pct" BETWEEN 20 AND 85)
+			AND ("protein_pct" IS NULL OR "protein_pct" BETWEEN 5 AND 40)
+			AND ("bmr_kcal" IS NULL OR "bmr_kcal" BETWEEN 500 AND 6000)
+			AND ("bmi" IS NULL OR "bmi" BETWEEN 10 AND 80));

drizzle/20261024150925_home/migration.sql

@@ -0,0 +1,66 @@
+CREATE SCHEMA "home";
+--> statement-breakpoint
+CREATE TABLE "home"."links" (
+	"id" serial PRIMARY KEY,
+	"title" text NOT NULL,
+	"url" text NOT NULL,
+	"section" text DEFAULT 'links' NOT NULL,
+	"sort_order" integer DEFAULT 0 NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "links_title_check" CHECK (length(btrim("title")) between 1 and 80),
+	CONSTRAINT "links_url_check" CHECK ("url" ~ '^https?://' and length("url") <= 2000)
+);
+--> statement-breakpoint
+CREATE TABLE "home"."snapshots" (
+	"key" text PRIMARY KEY,
+	"data" jsonb NOT NULL,
+	"collected_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX "links_sort_idx" ON "home"."links" ("section","sort_order");--> statement-breakpoint
+-- Privacy: like tasks and health, the start page is the app role only (no personal_ro).
+-- Grants only run on prod, where migrations run as `personal`.
+REVOKE ALL ON SCHEMA home FROM PUBLIC;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA home TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA home TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA home TO personal_app;
+    ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA home
+      GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA home
+      GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+  -- The host-side collector (deploy/home-collector) may only upsert snapshots. The role is
+  -- created out of band (it has a password); see deploy/home-collector/README.md.
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'home_collector') THEN
+    GRANT USAGE ON SCHEMA home TO home_collector;
+    GRANT SELECT, INSERT, UPDATE ON home.snapshots TO home_collector;
+  END IF;
+END $$;
+--> statement-breakpoint
+-- Starter links; edited inline on /start.
+INSERT INTO "home"."links" ("title", "url", "section", "sort_order") VALUES
+  ('Gmail', 'https://mail.google.com/mail/u/0/', 'daily', 10),
+  ('Calendar', 'https://calendar.google.com/', 'daily', 20),
+  ('WhatsApp', 'https://web.whatsapp.com/', 'daily', 30),
+  ('Discord', 'https://discord.com/app', 'daily', 40),
+  ('X', 'https://x.com/home', 'daily', 50),
+  ('YouTube', 'https://www.youtube.com/', 'daily', 60),
+  ('Spotify', 'https://open.spotify.com/', 'daily', 70),
+  ('Drive', 'https://drive.google.com/', 'daily', 80),
+  ('Moneybird', 'https://moneybird.com/390525278507501515', 'money', 110),
+  ('Revolut', 'https://app.revolut.com/', 'money', 120),
+  ('Knab', 'https://www.knab.nl/', 'money', 130),
+  ('Tikkie', 'https://www.tikkie.me/', 'money', 140),
+  ('Gitea', 'https://git.stanrunge.dev/', 'dev', 210),
+  ('GitHub', 'https://github.com/', 'dev', 220),
+  ('Cloudflare', 'https://dash.cloudflare.com/', 'dev', 230),
+  ('Tailscale', 'https://login.tailscale.com/admin/machines', 'dev', 240),
+  ('Backblaze B2', 'https://secure.backblaze.com/b2_buckets.htm', 'dev', 250),
+  ('1Password', 'https://my.1password.com/', 'dev', 260),
+  ('osu!', 'https://osu.ppy.sh/', 'fun', 310),
+  ('Twitch', 'https://www.twitch.tv/', 'fun', 320);

drizzle/20261024153902_cars/migration.sql

@@ -0,0 +1,134 @@
+CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
+--> statement-breakpoint
+CREATE SCHEMA "cars";
+--> statement-breakpoint
+CREATE TABLE "cars"."brands" (
+	"id" serial PRIMARY KEY,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"country" text,
+	"model_count" integer DEFAULT 0 NOT NULL,
+	"nl_count" integer DEFAULT 0 NOT NULL,
+	"first_year" smallint,
+	"last_year" smallint,
+	"sources" text[] DEFAULT '{}'::text[] NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "cars"."models" (
+	"id" serial PRIMARY KEY,
+	"dex" integer NOT NULL,
+	"brand_id" integer NOT NULL,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"search" text NOT NULL,
+	"first_year" smallint,
+	"last_year" smallint,
+	"bodies" text[] DEFAULT '{}'::text[] NOT NULL,
+	"size_classes" text[] DEFAULT '{}'::text[] NOT NULL,
+	"fuels" text[] DEFAULT '{}'::text[] NOT NULL,
+	"drives" text[] DEFAULT '{}'::text[] NOT NULL,
+	"transmissions" text[] DEFAULT '{}'::text[] NOT NULL,
+	"engine_l_min" real,
+	"engine_l_max" real,
+	"cylinders_min" smallint,
+	"cylinders_max" smallint,
+	"turbo" boolean,
+	"power_kw_min" real,
+	"power_kw_max" real,
+	"l100_min" real,
+	"l100_max" real,
+	"co2_min" smallint,
+	"co2_max" smallint,
+	"mpg_min" real,
+	"mpg_max" real,
+	"range_km_max" smallint,
+	"seats" smallint,
+	"doors" smallint,
+	"mass_kg" smallint,
+	"nl_count" integer DEFAULT 0 NOT NULL,
+	"nl_rank" integer,
+	"nl_first_year" smallint,
+	"nl_last_year" smallint,
+	"nl_price" integer,
+	"nl_price_latest" integer,
+	"nl_price_latest_year" smallint,
+	"nl_years" jsonb DEFAULT '[]' NOT NULL,
+	"nl_names" jsonb DEFAULT '[]' NOT NULL,
+	"us_years" smallint[] DEFAULT '{}'::smallint[] NOT NULL,
+	"variant_count" integer DEFAULT 0 NOT NULL,
+	"sources" text[] DEFAULT '{}'::text[] NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "cars"."variants" (
+	"id" serial PRIMARY KEY,
+	"model_id" integer NOT NULL,
+	"source" text DEFAULT 'fueleconomy.gov' NOT NULL,
+	"source_id" integer,
+	"year" smallint NOT NULL,
+	"name" text NOT NULL,
+	"engine_l" real,
+	"cylinders" smallint,
+	"transmission" text,
+	"drive" text,
+	"fuel" text,
+	"size_class" text,
+	"mpg_city" real,
+	"mpg_hwy" real,
+	"mpg_comb" real,
+	"co2_gpm" smallint,
+	"range_mi" smallint,
+	"ev_motor" text,
+	"turbo" boolean DEFAULT false NOT NULL,
+	"supercharged" boolean DEFAULT false NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "brands_slug_key" ON "cars"."brands" ("slug");--> statement-breakpoint
+CREATE UNIQUE INDEX "models_dex_key" ON "cars"."models" ("dex");--> statement-breakpoint
+CREATE UNIQUE INDEX "models_brand_slug_key" ON "cars"."models" ("brand_id","slug");--> statement-breakpoint
+CREATE INDEX "models_nl_count_idx" ON "cars"."models" ("nl_count");--> statement-breakpoint
+CREATE INDEX "models_search_trgm_idx" ON "cars"."models" USING gin ("search" public.gin_trgm_ops);--> statement-breakpoint
+CREATE INDEX "variants_model_year_idx" ON "cars"."variants" ("model_id","year");--> statement-breakpoint
+ALTER TABLE "cars"."models" ADD CONSTRAINT "models_brand_id_brands_id_fkey" FOREIGN KEY ("brand_id") REFERENCES "cars"."brands"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "cars"."variants" ADD CONSTRAINT "variants_model_id_models_id_fkey" FOREIGN KEY ("model_id") REFERENCES "cars"."models"("id") ON DELETE CASCADE;
+--> statement-breakpoint
+-- Hand-written: updated_at triggers, grants, GUI registration. Data comes from scripts/cars-import.ts.
+DO $$
+DECLARE t text;
+BEGIN
+  FOREACH t IN ARRAY ARRAY['brands','models'] LOOP
+    EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON cars.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+  END LOOP;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA cars TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA cars TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA cars TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA cars GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA cars GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT USAGE ON SCHEMA cars TO personal_ro;
+    GRANT SELECT ON ALL TABLES IN SCHEMA cars TO personal_ro;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA cars GRANT SELECT ON TABLES TO personal_ro;
+  END IF;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('cars','brands','cars-brands','Car brands','core',false,false,1000),
+           ('cars','models','cars-models','Car models','core',false,false,1000)
+    ON CONFLICT DO NOTHING;
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;

drizzle/20261024154032_house_liveability/migration.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_score" double precision;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_class" integer;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_onv_class" integer;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_level" text;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_precision" text;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_buurt_code" text;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_buurt_name" text;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_year" integer;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_detail" jsonb;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_query" text;--> statement-breakpoint
+ALTER TABLE "house"."listings" ADD COLUMN "lbm_scored_at" timestamp with time zone;

drizzle/20261024155715_music/migration.sql

@@ -0,0 +1,159 @@
+CREATE SCHEMA "music";
+--> statement-breakpoint
+CREATE TABLE "music"."albums" (
+	"id" serial PRIMARY KEY,
+	"spotify_id" text,
+	"name" text NOT NULL,
+	"artist_id" integer,
+	"image_url" text,
+	"image_url_small" text,
+	"release_date" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "music"."artists" (
+	"id" serial PRIMARY KEY,
+	"spotify_id" text,
+	"name" text NOT NULL,
+	"image_url" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "music"."import_state" (
+	"key" text PRIMARY KEY,
+	"data" jsonb NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "music"."now_playing" (
+	"id" smallint PRIMARY KEY DEFAULT 1,
+	"track_id" integer,
+	"is_playing" boolean DEFAULT false NOT NULL,
+	"progress_ms" integer,
+	"device_name" text,
+	"device_type" text,
+	"context_uri" text,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "now_playing_single_row" CHECK ("id" = 1)
+);
+--> statement-breakpoint
+CREATE TABLE "music"."plays" (
+	"id" serial PRIMARY KEY,
+	"track_id" integer NOT NULL,
+	"played_at" timestamp with time zone NOT NULL,
+	"source" text DEFAULT 'spotify' NOT NULL,
+	"origin" text DEFAULT 'api' NOT NULL,
+	"ms_played" integer,
+	"context_type" text,
+	"context_uri" text,
+	"platform" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "plays_source_check" CHECK ("source" in ('spotify', 'lastfm', 'youtube', 'local')),
+	CONSTRAINT "plays_origin_check" CHECK ("origin" in ('api', 'export', 'import', 'manual'))
+);
+--> statement-breakpoint
+CREATE TABLE "music"."spotify_auth" (
+	"id" smallint PRIMARY KEY DEFAULT 1,
+	"spotify_user_id" text NOT NULL,
+	"display_name" text,
+	"refresh_token" text NOT NULL,
+	"access_token" text,
+	"access_expires_at" timestamp with time zone,
+	"scope" text,
+	"recent_cursor" bigint,
+	"needs_reauth" boolean DEFAULT false NOT NULL,
+	"lease_until" timestamp with time zone,
+	"last_poll_at" timestamp with time zone,
+	"last_error" text,
+	"connected_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "spotify_auth_single_row" CHECK ("id" = 1)
+);
+--> statement-breakpoint
+CREATE TABLE "music"."track_artists" (
+	"track_id" integer,
+	"artist_id" integer,
+	"position" smallint DEFAULT 0 NOT NULL,
+	CONSTRAINT "track_artists_pkey" PRIMARY KEY("track_id","artist_id")
+);
+--> statement-breakpoint
+CREATE TABLE "music"."tracks" (
+	"id" serial PRIMARY KEY,
+	"spotify_id" text,
+	"name" text NOT NULL,
+	"album_id" integer,
+	"artist_id" integer,
+	"duration_ms" integer,
+	"explicit" boolean,
+	"match_key" text NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "albums_spotify_id_key" ON "music"."albums" ("spotify_id") WHERE "spotify_id" is not null;--> statement-breakpoint
+CREATE INDEX "albums_name_idx" ON "music"."albums" (lower("name"));--> statement-breakpoint
+CREATE UNIQUE INDEX "artists_spotify_id_key" ON "music"."artists" ("spotify_id") WHERE "spotify_id" is not null;--> statement-breakpoint
+CREATE INDEX "artists_name_idx" ON "music"."artists" (lower("name"));--> statement-breakpoint
+CREATE UNIQUE INDEX "plays_dedupe_key" ON "music"."plays" ("source","track_id","played_at");--> statement-breakpoint
+CREATE INDEX "plays_played_at_idx" ON "music"."plays" ("played_at");--> statement-breakpoint
+CREATE INDEX "plays_source_played_idx" ON "music"."plays" ("source","played_at");--> statement-breakpoint
+CREATE INDEX "plays_track_played_idx" ON "music"."plays" ("track_id","played_at");--> statement-breakpoint
+CREATE INDEX "track_artists_artist_idx" ON "music"."track_artists" ("artist_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "tracks_spotify_id_key" ON "music"."tracks" ("spotify_id") WHERE "spotify_id" is not null;--> statement-breakpoint
+CREATE INDEX "tracks_name_idx" ON "music"."tracks" (lower("name"));--> statement-breakpoint
+CREATE INDEX "tracks_match_key_idx" ON "music"."tracks" ("match_key");--> statement-breakpoint
+ALTER TABLE "music"."albums" ADD CONSTRAINT "albums_artist_id_artists_id_fkey" FOREIGN KEY ("artist_id") REFERENCES "music"."artists"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "music"."now_playing" ADD CONSTRAINT "now_playing_track_id_tracks_id_fkey" FOREIGN KEY ("track_id") REFERENCES "music"."tracks"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "music"."plays" ADD CONSTRAINT "plays_track_id_tracks_id_fkey" FOREIGN KEY ("track_id") REFERENCES "music"."tracks"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "music"."track_artists" ADD CONSTRAINT "track_artists_track_id_tracks_id_fkey" FOREIGN KEY ("track_id") REFERENCES "music"."tracks"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "music"."track_artists" ADD CONSTRAINT "track_artists_artist_id_artists_id_fkey" FOREIGN KEY ("artist_id") REFERENCES "music"."artists"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "music"."tracks" ADD CONSTRAINT "tracks_album_id_albums_id_fkey" FOREIGN KEY ("album_id") REFERENCES "music"."albums"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "music"."tracks" ADD CONSTRAINT "tracks_artist_id_artists_id_fkey" FOREIGN KEY ("artist_id") REFERENCES "music"."artists"("id") ON DELETE SET NULL;--> statement-breakpoint
+-- Hand-written below. Cross-source dedupe: every stat reads plays_deduped, which hides a
+-- Last.fm scrobble when Spotify (live API or the streaming-history export) has the same
+-- normalised artist+title around the same time. Last.fm stores when a track started,
+-- Spotify when it ended, so the Spotify end may fall from 3 min before the scrobble to
+-- track length + 3 min after it. Spotify rows always win; Last.fm fills the gaps.
+CREATE VIEW "music"."plays_deduped" AS
+SELECT p.* FROM "music"."plays" p
+WHERE p.source <> 'lastfm' OR NOT EXISTS (
+	SELECT 1
+	FROM "music"."tracks" lt, "music"."plays" s
+	JOIN "music"."tracks" st ON st.id = s.track_id
+	WHERE lt.id = p.track_id
+	  AND s.source = 'spotify'
+	  AND s.played_at >= p.played_at - interval '3 minutes'
+	  AND s.played_at <= p.played_at + interval '25 minutes'
+	  AND s.played_at <= p.played_at + make_interval(secs => coalesce(st.duration_ms, 600000) / 1000.0) + interval '3 minutes'
+	  AND st.match_key = lt.match_key
+);
+--> statement-breakpoint
+-- Grants only run on prod, where migrations run as `personal`. Listening history is public
+-- (personal_ro may read it); the Spotify refresh token is for the app role only.
+REVOKE ALL ON SCHEMA music FROM PUBLIC;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA music TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA music TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA music TO personal_app;
+    ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA music
+      GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA music
+      GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT USAGE ON SCHEMA music TO personal_ro;
+    GRANT SELECT ON music.artists, music.albums, music.tracks, music.track_artists, music.plays,
+      music.plays_deduped, music.now_playing TO personal_ro;
+  END IF;
+  -- Optional host-side poller (deploy/music-collector). The role is created out of band
+  -- (it has a password); see deploy/music-collector/README.md.
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'music_collector') THEN
+    GRANT USAGE ON SCHEMA music TO music_collector;
+    GRANT SELECT, INSERT, UPDATE ON music.artists, music.albums, music.tracks, music.track_artists,
+      music.plays, music.now_playing, music.import_state TO music_collector;
+    GRANT SELECT, UPDATE ON music.spotify_auth TO music_collector;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA music TO music_collector;
+  END IF;
+END $$;

drizzle/20261024160855_tasks_auto_handle/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "tasks"."tasks" ADD COLUMN "auto_handle" boolean DEFAULT false NOT NULL;

drizzle/20261024160937_osu_match_cache/migration.sql

@@ -0,0 +1,7 @@
+CREATE TABLE "music"."osu_match_cache" (
+	"key" text PRIMARY KEY,
+	"result" jsonb NOT NULL,
+	"searched_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+DO $$ BEGIN IF current_user = 'personal' THEN GRANT SELECT, INSERT, UPDATE, DELETE ON music.osu_match_cache TO personal_app; END IF; END $$;

drizzle/20261024161853_cars_photos/migration.sql

@@ -0,0 +1,50 @@
+CREATE TABLE "cars"."photos" (
+	"id" serial PRIMARY KEY,
+	"model_id" integer NOT NULL,
+	"position" smallint NOT NULL,
+	"kind" text NOT NULL,
+	"file" text NOT NULL,
+	"caption" text,
+	"thumb" text NOT NULL,
+	"width" integer NOT NULL,
+	"height" integer NOT NULL,
+	"author" text,
+	"license" text,
+	"license_url" text
+);
+--> statement-breakpoint
+ALTER TABLE "cars"."brands" ADD COLUMN "wikidata" text;--> statement-breakpoint
+ALTER TABLE "cars"."brands" ADD COLUMN "wikipedia" text;--> statement-breakpoint
+ALTER TABLE "cars"."brands" ADD COLUMN "logo" text;--> statement-breakpoint
+ALTER TABLE "cars"."brands" ADD COLUMN "logo_file" text;--> statement-breakpoint
+ALTER TABLE "cars"."brands" ADD COLUMN "logo_author" text;--> statement-breakpoint
+ALTER TABLE "cars"."brands" ADD COLUMN "logo_license" text;--> statement-breakpoint
+ALTER TABLE "cars"."brands" ADD COLUMN "photo" text;--> statement-breakpoint
+ALTER TABLE "cars"."models" ADD COLUMN "wikidata" text;--> statement-breakpoint
+ALTER TABLE "cars"."models" ADD COLUMN "wikipedia" text;--> statement-breakpoint
+ALTER TABLE "cars"."models" ADD COLUMN "photo" text;--> statement-breakpoint
+ALTER TABLE "cars"."models" ADD COLUMN "photo_count" smallint DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE "cars"."models" ADD COLUMN "photos_checked_at" timestamp with time zone;--> statement-breakpoint
+CREATE UNIQUE INDEX "photos_model_position_key" ON "cars"."photos" ("model_id","position");--> statement-breakpoint
+ALTER TABLE "cars"."photos" ADD CONSTRAINT "photos_model_id_models_id_fkey" FOREIGN KEY ("model_id") REFERENCES "cars"."models"("id") ON DELETE CASCADE;
+--> statement-breakpoint
+-- hand-written: grants + registry for the photos table
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON cars.photos TO personal_app;
+    GRANT USAGE, SELECT ON SEQUENCE cars.photos_id_seq TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT SELECT ON cars.photos TO personal_ro;
+  END IF;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('cars','photos','cars-photos','Car photos','core',false,false,1000)
+    ON CONFLICT DO NOTHING;
+  END IF;
+END $$;

drizzle/20261024172700_music_lease_holder/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "music"."spotify_auth" ADD COLUMN "lease_holder" text;

drizzle/20261024174805_hardware/migration.sql

@@ -0,0 +1,290 @@
+-- Hand-reconciled: the hardware schema already existed (hand-made 2026-10-22 from Stan's list +
+-- order mails, see ~/dev/hardware/schema.sql; prices later moved to cents). This brings an existing
+-- schema to the Drizzle model, and creates it from scratch on an empty database.
+CREATE SCHEMA IF NOT EXISTS "hardware";
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "hardware"."devices" (
+	"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "hardware"."devices_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
+	"name" text NOT NULL UNIQUE,
+	"label" text,
+	"hostnames" text[] DEFAULT '{}'::text[] NOT NULL,
+	"kind" text NOT NULL,
+	"status" text DEFAULT 'active' NOT NULL,
+	"brand" text,
+	"model" text,
+	"specs" jsonb DEFAULT '{}' NOT NULL,
+	"price_cents" integer,
+	"monthly_cents" integer,
+	"budget_cents" integer,
+	"shop" text,
+	"purchase_date" date,
+	"url" text,
+	"image_url" text,
+	"sort_order" integer DEFAULT 100 NOT NULL,
+	"notes" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "devices_kind_check" CHECK (kind in ('desktop', 'laptop', 'server', 'vm', 'phone', 'tablet', 'monitor', 'other')),
+	CONSTRAINT "devices_status_check" CHECK (status in ('active', 'spare', 'planned', 'retired', 'sold')),
+	CONSTRAINT "devices_cents_check" CHECK ((price_cents is null or price_cents >= 0) and (monthly_cents is null or monthly_cents >= 0) and (budget_cents is null or budget_cents >= 0))
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "hardware"."components" (
+	"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "hardware"."components_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
+	"device_id" bigint NOT NULL,
+	"category" text NOT NULL,
+	"brand" text,
+	"model" text NOT NULL,
+	"quantity" integer DEFAULT 1 NOT NULL,
+	"specs" jsonb DEFAULT '{}' NOT NULL,
+	"unit_price_cents" integer,
+	"total_price_cents" integer,
+	"shop" text,
+	"order_number" text,
+	"order_date" date,
+	"warranty_until" date,
+	"invoice_number" text,
+	"status" text DEFAULT 'in_use' NOT NULL,
+	"verified" boolean DEFAULT true NOT NULL,
+	"dpi" integer,
+	"measured_dpi" integer,
+	"url" text,
+	"image_url" text,
+	"notes" text,
+	"source" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "components_dedupe_uq" UNIQUE NULLS NOT DISTINCT("device_id","model","order_number"),
+	CONSTRAINT "components_category_check" CHECK (category in ('case', 'cpu', 'motherboard', 'ram', 'cpu_cooler', 'gpu', 'psu', 'storage', 'fan', 'lighting', 'monitor', 'mouse', 'keyboard', 'headset', 'audio', 'controller', 'accessory', 'other')),
+	CONSTRAINT "components_status_check" CHECK (status in ('in_use', 'planned', 'spare', 'retired', 'sold')),
+	CONSTRAINT "components_quantity_check" CHECK (quantity > 0),
+	CONSTRAINT "components_unit_price_cents_check" CHECK (unit_price_cents >= 0),
+	CONSTRAINT "components_total_price_cents_check" CHECK (total_price_cents >= 0),
+	CONSTRAINT "components_dpi_check" CHECK ((dpi is null or dpi > 0) and (measured_dpi is null or measured_dpi > 0))
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "hardware"."sensitivities" (
+	"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "hardware"."sensitivities_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
+	"game" text NOT NULL UNIQUE,
+	"cm_per_360" numeric(7,2),
+	"yaw" numeric(10,6),
+	"in_game_sens" numeric(10,4),
+	"mouse_id" bigint,
+	"fov" text,
+	"notes" text,
+	"sort_order" integer DEFAULT 100 NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "sensitivities_cm_check" CHECK (cm_per_360 is null or cm_per_360 > 0),
+	CONSTRAINT "sensitivities_yaw_check" CHECK (yaw is null or yaw > 0)
+);
+--> statement-breakpoint
+-- Existing tables: new columns.
+ALTER TABLE "hardware"."devices"
+	ADD COLUMN IF NOT EXISTS "label" text,
+	ADD COLUMN IF NOT EXISTS "hostnames" text[] DEFAULT '{}'::text[] NOT NULL,
+	ADD COLUMN IF NOT EXISTS "status" text DEFAULT 'active' NOT NULL,
+	ADD COLUMN IF NOT EXISTS "brand" text,
+	ADD COLUMN IF NOT EXISTS "model" text,
+	ADD COLUMN IF NOT EXISTS "specs" jsonb DEFAULT '{}' NOT NULL,
+	ADD COLUMN IF NOT EXISTS "price_cents" integer,
+	ADD COLUMN IF NOT EXISTS "monthly_cents" integer,
+	ADD COLUMN IF NOT EXISTS "budget_cents" integer,
+	ADD COLUMN IF NOT EXISTS "shop" text,
+	ADD COLUMN IF NOT EXISTS "purchase_date" date,
+	ADD COLUMN IF NOT EXISTS "url" text,
+	ADD COLUMN IF NOT EXISTS "image_url" text,
+	ADD COLUMN IF NOT EXISTS "sort_order" integer DEFAULT 100 NOT NULL,
+	ADD COLUMN IF NOT EXISTS "updated_at" timestamp with time zone DEFAULT now() NOT NULL;
+--> statement-breakpoint
+ALTER TABLE "hardware"."components"
+	ADD COLUMN IF NOT EXISTS "dpi" integer,
+	ADD COLUMN IF NOT EXISTS "measured_dpi" integer,
+	ADD COLUMN IF NOT EXISTS "url" text,
+	ADD COLUMN IF NOT EXISTS "image_url" text;
+--> statement-breakpoint
+-- Checks: widen kinds/categories/statuses (drop + re-add by name).
+ALTER TABLE "hardware"."devices" DROP CONSTRAINT IF EXISTS "devices_kind_check";
+ALTER TABLE "hardware"."devices" ADD CONSTRAINT "devices_kind_check" CHECK (kind in ('desktop', 'laptop', 'server', 'vm', 'phone', 'tablet', 'monitor', 'other'));
+ALTER TABLE "hardware"."devices" DROP CONSTRAINT IF EXISTS "devices_status_check";
+ALTER TABLE "hardware"."devices" ADD CONSTRAINT "devices_status_check" CHECK (status in ('active', 'spare', 'planned', 'retired', 'sold'));
+ALTER TABLE "hardware"."devices" DROP CONSTRAINT IF EXISTS "devices_cents_check";
+ALTER TABLE "hardware"."devices" ADD CONSTRAINT "devices_cents_check" CHECK ((price_cents is null or price_cents >= 0) and (monthly_cents is null or monthly_cents >= 0) and (budget_cents is null or budget_cents >= 0));
+ALTER TABLE "hardware"."components" DROP CONSTRAINT IF EXISTS "components_category_check";
+ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_category_check" CHECK (category in ('case', 'cpu', 'motherboard', 'ram', 'cpu_cooler', 'gpu', 'psu', 'storage', 'fan', 'lighting', 'monitor', 'mouse', 'keyboard', 'headset', 'audio', 'controller', 'accessory', 'other'));
+ALTER TABLE "hardware"."components" DROP CONSTRAINT IF EXISTS "components_status_check";
+ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_status_check" CHECK (status in ('in_use', 'planned', 'spare', 'retired', 'sold'));
+ALTER TABLE "hardware"."components" DROP CONSTRAINT IF EXISTS "components_dpi_check";
+ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_dpi_check" CHECK ((dpi is null or dpi > 0) and (measured_dpi is null or measured_dpi > 0));
+--> statement-breakpoint
+-- Dedupe key: unique index -> constraint (same name, same NULLS NOT DISTINCT semantics);
+-- device FK -> Drizzle's name.
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'components_dedupe_uq' AND connamespace = 'hardware'::regnamespace) THEN
+    DROP INDEX IF EXISTS "hardware"."components_dedupe_uq";
+    ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_dedupe_uq" UNIQUE NULLS NOT DISTINCT ("device_id","model","order_number");
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'components_device_id_fkey' AND connamespace = 'hardware'::regnamespace) THEN
+    ALTER TABLE "hardware"."components" RENAME CONSTRAINT "components_device_id_fkey" TO "components_device_id_devices_id_fkey";
+  END IF;
+  IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'components_device_id_devices_id_fkey' AND connamespace = 'hardware'::regnamespace) THEN
+    ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_device_id_devices_id_fkey" FOREIGN KEY ("device_id") REFERENCES "hardware"."devices"("id") ON DELETE CASCADE;
+  END IF;
+  IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'sensitivities_mouse_id_components_id_fkey' AND connamespace = 'hardware'::regnamespace) THEN
+    ALTER TABLE "hardware"."sensitivities" ADD CONSTRAINT "sensitivities_mouse_id_components_id_fkey" FOREIGN KEY ("mouse_id") REFERENCES "hardware"."components"("id") ON DELETE SET NULL;
+  END IF;
+END $$;
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "components_device_idx" ON "hardware"."components" ("device_id");
+CREATE INDEX IF NOT EXISTS "components_category_idx" ON "hardware"."components" ("category");
+CREATE INDEX IF NOT EXISTS "components_specs_gin" ON "hardware"."components" USING gin ("specs");
+--> statement-breakpoint
+-- Hand-written: updated_at triggers, overview view, grants, GUI registration, seed.
+DO $$
+DECLARE t text;
+BEGIN
+  FOREACH t IN ARRAY ARRAY['devices','components','sensitivities'] LOOP
+    IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = 'touch_updated_at' AND tgrelid = format('hardware.%I', t)::regclass) THEN
+      EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON hardware.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+    END IF;
+  END LOOP;
+END $$;
+--> statement-breakpoint
+DROP VIEW IF EXISTS "hardware"."setup_overview";
+--> statement-breakpoint
+CREATE VIEW "hardware"."setup_overview" AS
+SELECT d.name AS device,
+       d.label AS device_label,
+       d.kind AS device_kind,
+       d.status AS device_status,
+       c.category,
+       c.brand,
+       c.model,
+       c.quantity,
+       c.total_price_cents,
+       round(c.total_price_cents::numeric / 100.0, 2) AS total_price_eur,
+       c.shop,
+       c.order_date,
+       c.warranty_until,
+       (c.warranty_until >= current_date) AS under_warranty,
+       c.status,
+       c.verified,
+       c.specs,
+       c.dpi,
+       c.measured_dpi,
+       c.notes,
+       c.order_number,
+       c.invoice_number,
+       c.source,
+       c.id AS component_id
+FROM hardware.components c
+JOIN hardware.devices d ON d.id = c.device_id
+ORDER BY d.sort_order, d.name,
+         array_position(ARRAY['case','cpu','motherboard','ram','cpu_cooler','gpu','psu','storage','fan','lighting','monitor','mouse','keyboard','headset','audio','controller','accessory','other'], c.category),
+         c.order_date NULLS LAST, c.model;
+--> statement-breakpoint
+COMMENT ON VIEW "hardware"."setup_overview" IS 'Flat view of every component per device (build order), with a live under_warranty flag. status = ''in_use'' is the current setup, ''planned'' the upgrade plan.';
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA hardware TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA hardware TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA hardware TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA hardware GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA hardware GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT USAGE ON SCHEMA hardware TO personal_ro;
+    GRANT SELECT ON ALL TABLES IN SCHEMA hardware TO personal_ro;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA hardware GRANT SELECT ON TABLES TO personal_ro;
+  END IF;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('hardware','devices','hardware-devices','Hardware devices','core',true,false,1000),
+           ('hardware','components','hardware-components','Hardware components','core',true,false,1000),
+           ('hardware','sensitivities','hardware-sensitivities','Game sensitivities (cm/360)','core',true,false,1000)
+    ON CONFLICT DO NOTHING;
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;
+--> statement-breakpoint
+-- ===== Seed (idempotent). Facts from Stan's own list, order mails and what he told Daisy;
+-- unknowns stay null. No home address in public-facing notes.
+INSERT INTO hardware.devices (name, kind) VALUES ('stan-pc','desktop'), ('old-pc-2021','desktop')
+ON CONFLICT (name) DO NOTHING;
+--> statement-breakpoint
+UPDATE hardware.devices SET
+  label = 'Main PC', hostnames = '{stan-arch,stan-win11}', status = 'active', model = 'Custom build',
+  shop = 'Megekko', sort_order = 10,
+  notes = 'Stan''s main PC: Lian Li O11 Dynamic XL ROG White, i9-12900KF, RX 7900 XT, 10 Lian Li fans around the case + 3 NZXT Aer on the radiator. Dual-boots Arch (stan-arch) and Windows 11 + WSL2 (stan-win11). Desk peripherals and the Odyssey G9 are tracked here. Mostly bought at Megekko 2021-2022.'
+WHERE name = 'stan-pc' AND label IS NULL;
+--> statement-breakpoint
+UPDATE hardware.devices SET
+  label = 'Previous PC (2021)', status = 'retired', model = 'Budget build', shop = 'Megekko',
+  purchase_date = '2021-01-31', sort_order = 90
+WHERE name = 'old-pc-2021' AND label IS NULL;
+--> statement-breakpoint
+INSERT INTO hardware.devices (name, label, hostnames, kind, status, brand, model, specs, price_cents, monthly_cents, budget_cents, shop, purchase_date, url, sort_order, notes) VALUES
+ ('stan-pc-next','Next PC (planned)','{}','desktop','planned',NULL,'High-end AMD + RTX 5090 build','{}',NULL,NULL,1000000,NULL,NULL,NULL,15,
+  'Planned upgrade, budget up to about EUR 10k: top AMD X3D CPU, RTX 5090, fast DDR5, PCIe Gen5 SSD, high-end motherboard and a cooler that fits. Parts not picked yet.'),
+ ('stan-ovh','Server','{}','server','active','OVHcloud','RISE-S',
+  '{"cpu":"AMD Ryzen 7 9700X","ram":"64 GB DDR5 ECC","storage":"2x 512 GB NVMe (software RAID1)","os":"Ubuntu 26.04"}',
+  NULL,6174,NULL,'OVHcloud','2026-10-24','https://www.ovhcloud.com/en/bare-metal/rise/',20,
+  'Dedicated server, prod since 2026-10-24 (replaced stan-vm). EUR 61.74/month excl. VAT. Runs Postgres, Gitea, the website and the bots.'),
+ ('stan-vm','Old VM','{}','vm','retired','Hetzner Cloud',NULL,'{"vcpu":4,"ram":"8 GB"}',NULL,2000,NULL,'Hetzner',NULL,NULL,80,
+  'Old always-on Hetzner VM (about EUR 20/month, bought before the price rises). Replaced by stan-ovh on 2026-10-24; stopped, kept as a fallback until Stan cancels it.'),
+ ('stan-macbookpro','MacBook Pro','{}','laptop','active','Apple','MacBook Pro','{}',NULL,NULL,NULL,NULL,NULL,NULL,30,
+  'Exact model/chip not recorded yet. An M5 Max MacBook Pro is planned.'),
+ ('stan-pixel','Phone','{}','phone','active','Google','Pixel 11 Pro XL','{"storage":"256 GB"}',NULL,NULL,NULL,'Belsimpel','2026-10-18',NULL,40,
+  'Main phone since 2026-10-19, stock Android. About EUR 1,299 SIM-free (exact amount not recorded).'),
+ ('stan-iphone','Old phone','{}','phone','spare','Apple','iPhone 14 Pro Max','{}',NULL,NULL,NULL,NULL,NULL,NULL,45,
+  'Previous phone, kept (weak battery and speed).'),
+ ('stan-ipad','iPad','{}','tablet','active','Apple','iPad Air 13-inch (M2)','{"screen_in":13,"chip":"Apple M2"}',NULL,NULL,NULL,NULL,NULL,NULL,50,
+  'With a keyboard case; used for typing on the go and Termius SSH.'),
+ ('cargill-laptop','Work laptop','{}','laptop','active',NULL,NULL,'{"ram":"16 GB"}',NULL,NULL,NULL,NULL,NULL,NULL,60,
+  'Cargill work laptop (not personal; Windows + WSL2). Model not recorded.')
+ON CONFLICT (name) DO NOTHING;
+--> statement-breakpoint
+-- Peripherals get real categories instead of "accessory".
+UPDATE hardware.components SET category = 'mouse' WHERE category = 'accessory' AND model ILIKE 'MX Master%';
+--> statement-breakpoint
+UPDATE hardware.components SET category = 'keyboard' WHERE category = 'accessory' AND (model ILIKE 'Alloy Origins%' OR model ILIKE 'Pudding Keycaps%' OR model ILIKE 'UwU%');
+--> statement-breakpoint
+UPDATE hardware.components SET category = 'controller' WHERE category = 'accessory' AND brand = 'Thrustmaster';
+--> statement-breakpoint
+UPDATE hardware.components SET category = 'audio' WHERE category = 'accessory' AND model ILIKE '%mic boom arm%';
+--> statement-breakpoint
+UPDATE hardware.components SET notes = replace(notes, ', delivered 2026-03-03 to Houtenstraat 15 Gouda.', ', delivered 2026-03-03.') WHERE notes LIKE '%Houtenstraat%';
+--> statement-breakpoint
+INSERT INTO hardware.components (device_id, category, brand, model, quantity, specs, status, verified, notes, source)
+SELECT d.id, v.category, v.brand, v.model, v.quantity, v.specs::jsonb, v.status, true, v.notes, v.source
+FROM (VALUES
+ ('stan-ovh','cpu','AMD','Ryzen 7 9700X',1,'{"cores":8,"threads":16,"socket":"AM5"}','in_use',NULL,'manual'),
+ ('stan-ovh','ram',NULL,'64 GB DDR5 ECC',1,'{"type":"DDR5 ECC","capacity_gb":64}','in_use',NULL,'manual'),
+ ('stan-ovh','storage',NULL,'512 GB NVMe SSD',2,'{"interface":"NVMe","capacity_gb":512,"raid":"software RAID1"}','in_use','Mirrored (RAID1), about 440 GB usable for data.','manual'),
+ ('stan-pc-next','cpu','AMD','Top Ryzen X3D (model TBD)',1,'{"socket":"AM5"}','planned','Pick the fastest gaming X3D chip available when buying.','plan'),
+ ('stan-pc-next','gpu','NVIDIA','GeForce RTX 5090',1,'{"vram_gb":32}','planned','Board partner TBD.','plan'),
+ ('stan-pc-next','motherboard',NULL,'High-end AM5 motherboard (TBD)',1,'{"socket":"AM5"}','planned',NULL,'plan'),
+ ('stan-pc-next','ram',NULL,'Fast DDR5 kit (TBD)',1,'{"type":"DDR5"}','planned',NULL,'plan'),
+ ('stan-pc-next','storage',NULL,'PCIe 5.0 NVMe SSD (TBD)',1,'{"interface":"PCIe 5.0 x4 NVMe"}','planned',NULL,'plan'),
+ ('stan-pc-next','cpu_cooler',NULL,'Cooler for the new CPU (TBD)',1,'{}','planned','Must fit the new socket and the O11 Dynamic XL (or the new case).','plan')
+) AS v(device, category, brand, model, quantity, specs, status, notes, source)
+JOIN hardware.devices d ON d.name = v.device
+ON CONFLICT (device_id, model, order_number) DO NOTHING;
+--> statement-breakpoint
+-- Games Stan plays; cm/360 left empty for him to fill. CS2's yaw (0.022 deg/count) is the
+-- Source engine constant, so its in-game value is computed from a mouse's measured DPI.
+INSERT INTO hardware.sensitivities (game, yaw, sort_order) VALUES
+ ('Counter-Strike 2', 0.022, 10),
+ ('Call of Duty', NULL, 20),
+ ('osu!', NULL, 30),
+ ('Sea of Thieves', NULL, 40)
+ON CONFLICT (game) DO NOTHING;

drizzle/20261024193304_planes/migration.sql

@@ -0,0 +1,140 @@
+CREATE SCHEMA "planes";
+--> statement-breakpoint
+CREATE TABLE "planes"."makers" (
+	"id" serial PRIMARY KEY,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"full_name" text,
+	"country" text,
+	"wikidata" text,
+	"wikipedia" text,
+	"type_count" integer DEFAULT 0 NOT NULL,
+	"first_year" smallint,
+	"last_year" smallint,
+	"built_total" integer DEFAULT 0 NOT NULL,
+	"sources" text[] DEFAULT '{}'::text[] NOT NULL,
+	"logo" text,
+	"logo_file" text,
+	"logo_author" text,
+	"logo_license" text,
+	"photo" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "planes"."photos" (
+	"id" serial PRIMARY KEY,
+	"type_id" integer NOT NULL,
+	"position" smallint NOT NULL,
+	"kind" text NOT NULL,
+	"file" text NOT NULL,
+	"caption" text,
+	"thumb" text NOT NULL,
+	"width" integer NOT NULL,
+	"height" integer NOT NULL,
+	"author" text,
+	"license" text,
+	"license_url" text
+);
+--> statement-breakpoint
+CREATE TABLE "planes"."types" (
+	"id" serial PRIMARY KEY,
+	"dex" integer NOT NULL,
+	"maker_id" integer NOT NULL,
+	"family_id" integer,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"full_name" text NOT NULL,
+	"search" text NOT NULL,
+	"wikidata" text NOT NULL,
+	"wikipedia" text,
+	"kind" text,
+	"description" text,
+	"summary" text,
+	"role" text,
+	"roles" text[] DEFAULT '{}'::text[] NOT NULL,
+	"role_label" text,
+	"country" text,
+	"first_flight" text,
+	"first_flight_year" smallint,
+	"introduced_year" smallint,
+	"retired_year" smallint,
+	"status" text,
+	"in_service" boolean,
+	"number_built" integer,
+	"makers" text[] DEFAULT '{}'::text[] NOT NULL,
+	"engine_type" text,
+	"engine_count" smallint,
+	"engines" text[] DEFAULT '{}'::text[] NOT NULL,
+	"crew" smallint,
+	"passengers" smallint,
+	"length_m" real,
+	"wingspan_m" real,
+	"height_m" real,
+	"rotor_m" real,
+	"mtow_kg" integer,
+	"max_speed_kmh" smallint,
+	"cruise_kmh" smallint,
+	"range_km" integer,
+	"ceiling_m" integer,
+	"wing" text,
+	"gear" text,
+	"icao" text[] DEFAULT '{}'::text[] NOT NULL,
+	"recognize" text,
+	"sources" text[] DEFAULT '{}'::text[] NOT NULL,
+	"photo" text,
+	"photo_count" smallint DEFAULT 0 NOT NULL,
+	"photos_checked_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "makers_slug_key" ON "planes"."makers" ("slug");--> statement-breakpoint
+CREATE UNIQUE INDEX "photos_type_position_key" ON "planes"."photos" ("type_id","position");--> statement-breakpoint
+CREATE UNIQUE INDEX "types_dex_key" ON "planes"."types" ("dex");--> statement-breakpoint
+CREATE UNIQUE INDEX "types_wikidata_key" ON "planes"."types" ("wikidata");--> statement-breakpoint
+CREATE UNIQUE INDEX "types_maker_slug_key" ON "planes"."types" ("maker_id","slug");--> statement-breakpoint
+CREATE INDEX "types_number_built_idx" ON "planes"."types" ("number_built");--> statement-breakpoint
+CREATE INDEX "types_first_flight_year_idx" ON "planes"."types" ("first_flight_year");--> statement-breakpoint
+CREATE INDEX "types_search_trgm_idx" ON "planes"."types" USING gin ("search" public.gin_trgm_ops);--> statement-breakpoint
+ALTER TABLE "planes"."photos" ADD CONSTRAINT "photos_type_id_types_id_fkey" FOREIGN KEY ("type_id") REFERENCES "planes"."types"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "planes"."types" ADD CONSTRAINT "types_maker_id_makers_id_fkey" FOREIGN KEY ("maker_id") REFERENCES "planes"."makers"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "planes"."types" ADD CONSTRAINT "types_family_id_types_id_fkey" FOREIGN KEY ("family_id") REFERENCES "planes"."types"("id") ON DELETE SET NULL;--> statement-breakpoint
+-- Hand-written: updated_at triggers, grants, GUI registration. Data comes from scripts/planes-import.ts and scripts/planes-photos.ts.
+DO $$
+DECLARE t text;
+BEGIN
+  FOREACH t IN ARRAY ARRAY['makers','types'] LOOP
+    EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON planes.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+  END LOOP;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA planes TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA planes TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA planes TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA planes GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA planes GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT USAGE ON SCHEMA planes TO personal_ro;
+    GRANT SELECT ON ALL TABLES IN SCHEMA planes TO personal_ro;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA planes GRANT SELECT ON TABLES TO personal_ro;
+  END IF;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('planes','makers','planes-makers','Plane makers','core',false,false,1000),
+           ('planes','types','planes-types','Plane types','core',false,false,1000),
+           ('planes','photos','planes-photos','Plane photos','core',false,false,1000)
+    ON CONFLICT DO NOTHING;
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;

drizzle/20261024215408_wildlife/migration.sql

@@ -0,0 +1,122 @@
+CREATE SCHEMA "wildlife";
+--> statement-breakpoint
+CREATE TABLE "wildlife"."groups" (
+	"id" serial PRIMARY KEY,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"name_nl" text NOT NULL,
+	"emoji" text NOT NULL,
+	"sort_order" smallint DEFAULT 0 NOT NULL,
+	"species_count" integer DEFAULT 0 NOT NULL,
+	"nl_count" integer DEFAULT 0 NOT NULL,
+	"photo" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "wildlife"."photos" (
+	"id" serial PRIMARY KEY,
+	"species_id" integer NOT NULL,
+	"position" smallint NOT NULL,
+	"kind" text NOT NULL,
+	"photo_id" integer NOT NULL,
+	"url" text NOT NULL,
+	"width" integer,
+	"height" integer,
+	"author" text,
+	"license" text NOT NULL,
+	"page_url" text,
+	"place" text
+);
+--> statement-breakpoint
+CREATE TABLE "wildlife"."species" (
+	"id" serial PRIMARY KEY,
+	"dex" integer NOT NULL,
+	"group_id" integer NOT NULL,
+	"slug" text NOT NULL,
+	"scientific_name" text NOT NULL,
+	"name_en" text,
+	"name_nl" text,
+	"search" text NOT NULL,
+	"inat_id" integer NOT NULL,
+	"gbif_key" integer,
+	"wikidata" text,
+	"wikipedia_en" text,
+	"wikipedia_nl" text,
+	"class_name" text,
+	"order_name" text,
+	"family_name" text,
+	"iucn" text,
+	"nl_status" text,
+	"nl_establishment" text,
+	"extinct" boolean DEFAULT false NOT NULL,
+	"nl_obs" integer DEFAULT 0 NOT NULL,
+	"europe_obs" integer DEFAULT 0 NOT NULL,
+	"world_obs" integer DEFAULT 0 NOT NULL,
+	"gbif_nl" integer,
+	"scope" text NOT NULL,
+	"summary" text,
+	"summary_nl" text,
+	"facts" text[] DEFAULT '{}'::text[] NOT NULL,
+	"mass_kg" real,
+	"length_m" real,
+	"lifespan_years" real,
+	"named_year" smallint,
+	"named_by" text,
+	"sources" text[] DEFAULT '{}'::text[] NOT NULL,
+	"photo" text,
+	"photo_count" smallint DEFAULT 0 NOT NULL,
+	"photos_checked_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "groups_slug_key" ON "wildlife"."groups" ("slug");--> statement-breakpoint
+CREATE UNIQUE INDEX "photos_species_position_key" ON "wildlife"."photos" ("species_id","position");--> statement-breakpoint
+CREATE UNIQUE INDEX "photos_species_photo_key" ON "wildlife"."photos" ("species_id","photo_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "species_dex_key" ON "wildlife"."species" ("dex");--> statement-breakpoint
+CREATE UNIQUE INDEX "species_inat_key" ON "wildlife"."species" ("inat_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "species_slug_key" ON "wildlife"."species" ("slug");--> statement-breakpoint
+CREATE INDEX "species_group_idx" ON "wildlife"."species" ("group_id");--> statement-breakpoint
+CREATE INDEX "species_nl_obs_idx" ON "wildlife"."species" ("nl_obs");--> statement-breakpoint
+CREATE INDEX "species_search_trgm_idx" ON "wildlife"."species" USING gin ("search" public.gin_trgm_ops);--> statement-breakpoint
+ALTER TABLE "wildlife"."photos" ADD CONSTRAINT "photos_species_id_species_id_fkey" FOREIGN KEY ("species_id") REFERENCES "wildlife"."species"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "wildlife"."species" ADD CONSTRAINT "species_group_id_groups_id_fkey" FOREIGN KEY ("group_id") REFERENCES "wildlife"."groups"("id") ON DELETE CASCADE;--> statement-breakpoint
+-- Hand-written: updated_at triggers, grants, GUI registration. Data comes from scripts/wildlife-import.ts and scripts/wildlife-photos.ts.
+DO $$
+DECLARE t text;
+BEGIN
+  FOREACH t IN ARRAY ARRAY['groups','species'] LOOP
+    EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON wildlife.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+  END LOOP;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA wildlife TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA wildlife TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA wildlife TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA wildlife GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA wildlife GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT USAGE ON SCHEMA wildlife TO personal_ro;
+    GRANT SELECT ON ALL TABLES IN SCHEMA wildlife TO personal_ro;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA wildlife GRANT SELECT ON TABLES TO personal_ro;
+  END IF;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('wildlife','groups','wildlife-groups','Wildlife groups','core',false,false,1000),
+           ('wildlife','species','wildlife-species','Wildlife species','core',false,false,1000),
+           ('wildlife','photos','wildlife-photos','Wildlife photos','core',false,false,1000)
+    ON CONFLICT DO NOTHING;
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;

drizzle/20261024232143_trains/migration.sql

@@ -0,0 +1,167 @@
+CREATE SCHEMA "trains";
+--> statement-breakpoint
+CREATE TABLE "trains"."makers" (
+	"id" serial PRIMARY KEY,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"full_name" text,
+	"country" text,
+	"wikidata" text,
+	"wikipedia" text,
+	"type_count" integer DEFAULT 0 NOT NULL,
+	"first_year" smallint,
+	"last_year" smallint,
+	"built_total" integer DEFAULT 0 NOT NULL,
+	"sources" text[] DEFAULT '{}'::text[] NOT NULL,
+	"logo" text,
+	"logo_file" text,
+	"logo_author" text,
+	"logo_license" text,
+	"photo" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "trains"."operators" (
+	"id" serial PRIMARY KEY,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"full_name" text,
+	"country" text,
+	"wikidata" text,
+	"wikipedia" text,
+	"type_count" integer DEFAULT 0 NOT NULL,
+	"logo" text,
+	"logo_file" text,
+	"logo_author" text,
+	"logo_license" text,
+	"photo" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "trains"."photos" (
+	"id" serial PRIMARY KEY,
+	"type_id" integer NOT NULL,
+	"position" smallint NOT NULL,
+	"kind" text NOT NULL,
+	"file" text NOT NULL,
+	"caption" text,
+	"thumb" text NOT NULL,
+	"width" integer NOT NULL,
+	"height" integer NOT NULL,
+	"author" text,
+	"license" text,
+	"license_url" text
+);
+--> statement-breakpoint
+CREATE TABLE "trains"."type_operators" (
+	"type_id" integer,
+	"operator_id" integer,
+	"position" smallint DEFAULT 0 NOT NULL,
+	CONSTRAINT "type_operators_pkey" PRIMARY KEY("type_id","operator_id")
+);
+--> statement-breakpoint
+CREATE TABLE "trains"."types" (
+	"id" serial PRIMARY KEY,
+	"dex" integer NOT NULL,
+	"maker_id" integer NOT NULL,
+	"family_id" integer,
+	"slug" text NOT NULL,
+	"name" text NOT NULL,
+	"full_name" text NOT NULL,
+	"nl_name" text,
+	"search" text NOT NULL,
+	"wikidata" text NOT NULL,
+	"wikipedia" text,
+	"description" text,
+	"summary" text,
+	"category" text NOT NULL,
+	"category_label" text,
+	"traction" text,
+	"country" text,
+	"countries" text[] DEFAULT '{}'::text[] NOT NULL,
+	"makers" text[] DEFAULT '{}'::text[] NOT NULL,
+	"operators" text[] DEFAULT '{}'::text[] NOT NULL,
+	"first_built" smallint,
+	"last_built" smallint,
+	"introduced_year" smallint,
+	"retired_year" smallint,
+	"status" text,
+	"in_service" boolean,
+	"number_built" integer,
+	"max_speed_kmh" smallint,
+	"power_kw" integer,
+	"gauge" text,
+	"gauge_mm" smallint,
+	"electric_system" text,
+	"wheels" text,
+	"length_m" real,
+	"weight_t" real,
+	"capacity" smallint,
+	"formation" text,
+	"designer" text,
+	"nicknames" text,
+	"sources" text[] DEFAULT '{}'::text[] NOT NULL,
+	"photo" text,
+	"photo_count" smallint DEFAULT 0 NOT NULL,
+	"photos_checked_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "makers_slug_key" ON "trains"."makers" ("slug");--> statement-breakpoint
+CREATE UNIQUE INDEX "operators_slug_key" ON "trains"."operators" ("slug");--> statement-breakpoint
+CREATE UNIQUE INDEX "photos_type_position_key" ON "trains"."photos" ("type_id","position");--> statement-breakpoint
+CREATE INDEX "type_operators_operator_idx" ON "trains"."type_operators" ("operator_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "types_dex_key" ON "trains"."types" ("dex");--> statement-breakpoint
+CREATE UNIQUE INDEX "types_wikidata_key" ON "trains"."types" ("wikidata");--> statement-breakpoint
+CREATE UNIQUE INDEX "types_maker_slug_key" ON "trains"."types" ("maker_id","slug");--> statement-breakpoint
+CREATE INDEX "types_number_built_idx" ON "trains"."types" ("number_built");--> statement-breakpoint
+CREATE INDEX "types_first_built_idx" ON "trains"."types" ("first_built");--> statement-breakpoint
+CREATE INDEX "types_category_idx" ON "trains"."types" ("category");--> statement-breakpoint
+CREATE INDEX "types_search_trgm_idx" ON "trains"."types" USING gin ("search" public.gin_trgm_ops);--> statement-breakpoint
+ALTER TABLE "trains"."photos" ADD CONSTRAINT "photos_type_id_types_id_fkey" FOREIGN KEY ("type_id") REFERENCES "trains"."types"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "trains"."type_operators" ADD CONSTRAINT "type_operators_type_id_types_id_fkey" FOREIGN KEY ("type_id") REFERENCES "trains"."types"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "trains"."type_operators" ADD CONSTRAINT "type_operators_operator_id_operators_id_fkey" FOREIGN KEY ("operator_id") REFERENCES "trains"."operators"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "trains"."types" ADD CONSTRAINT "types_maker_id_makers_id_fkey" FOREIGN KEY ("maker_id") REFERENCES "trains"."makers"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "trains"."types" ADD CONSTRAINT "types_family_id_types_id_fkey" FOREIGN KEY ("family_id") REFERENCES "trains"."types"("id") ON DELETE SET NULL;--> statement-breakpoint
+-- Hand-written: updated_at triggers, grants, GUI registration. Data comes from scripts/trains-import.ts and scripts/trains-photos.ts.
+DO $$
+DECLARE t text;
+BEGIN
+  FOREACH t IN ARRAY ARRAY['makers','operators','types'] LOOP
+    EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON trains.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+  END LOOP;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT USAGE ON SCHEMA trains TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA trains TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA trains TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA trains GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA trains GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT USAGE ON SCHEMA trains TO personal_ro;
+    GRANT SELECT ON ALL TABLES IN SCHEMA trains TO personal_ro;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA trains GRANT SELECT ON TABLES TO personal_ro;
+  END IF;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('trains','makers','trains-makers','Train makers','core',false,false,1000),
+           ('trains','operators','trains-operators','Train operators','core',false,false,1000),
+           ('trains','types','trains-types','Train types','core',false,false,1000),
+           ('trains','photos','trains-photos','Train photos','core',false,false,1000)
+    ON CONFLICT DO NOTHING;
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;

extras/newtab-extension/README.md

@@ -0,0 +1,19 @@
+# New-tab extension
+
+A minimal Chrome / Edge / Brave (Manifest V3) extension that makes every new tab open
+[stanrunge.dev/start](https://stanrunge.dev/start), the private dashboard. No permissions,
+no tracking: the new-tab page just redirects.
+
+## Install (load unpacked)
+
+1. Download and unzip <https://stanrunge.dev/newtab-extension.zip> (or use this folder from the repo).
+2. Open `chrome://extensions` and switch on **Developer mode** (top right).
+3. Click **Load unpacked** and pick the unzipped `newtab-extension` folder.
+4. Open a new tab. Chrome asks once whether to keep the changed new-tab page: choose **Keep it**.
+
+Log in on stanrunge.dev once in that browser; after that the dashboard opens straight away.
+To point it elsewhere, edit the URL in `newtab.html` and `newtab.js`, then press the reload
+icon on the extension card. Firefox: set the homepage/new tab with an add-on such as
+"New Tab Override" instead.
+
+Rebuild the zip after changes: `bash extras/newtab-extension/build.sh`.

extras/newtab-extension/build.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Zip the extension into static/newtab-extension.zip (served at /newtab-extension.zip).
+set -euo pipefail
+cd "$(dirname "$0")/.."
+python3 - <<'PY'
+import os, zipfile
+out = '../static/newtab-extension.zip'
+with zipfile.ZipFile(out, 'w', zipfile.ZIP_DEFLATED) as z:
+    for name in sorted(os.listdir('newtab-extension')):
+        if name != 'build.sh':
+            z.write(os.path.join('newtab-extension', name))
+print('wrote static/newtab-extension.zip')
+PY

extras/newtab-extension/manifest.json

@@ -0,0 +1,8 @@
+{
+	"manifest_version": 3,
+	"name": "Stan's start page",
+	"version": "1.0.0",
+	"description": "Opens stanrunge.dev/start in every new tab.",
+	"chrome_url_overrides": { "newtab": "newtab.html" },
+	"icons": { "128": "icon128.png" }
+}

extras/newtab-extension/newtab.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+	<head>
+		<meta charset="utf-8" />
+		<title>New tab</title>
+		<!-- Redirect before anything paints; newtab.js does the same as a fallback. -->
+		<meta http-equiv="refresh" content="0; url=https://stanrunge.dev/start" />
+		<style>
+			html {
+				background: #000;
+				color-scheme: dark;
+			}
+		</style>
+		<script src="newtab.js"></script>
+	</head>
+	<body></body>
+</html>

extras/newtab-extension/newtab.js

@@ -0,0 +1,3 @@
+// MV3 pages can't run inline scripts, so the redirect lives here. Change the URL to point the
+// new tab somewhere else (then reload the extension on chrome://extensions).
+location.replace('https://stanrunge.dev/start');

package.json

@@ -20,9 +20,24 @@
 		"preview:branch": "bash scripts/preview.sh",
 		"db:studio": "drizzle-kit studio",
 		"db:seed": "bun scripts/seed-catalog.ts",
+		"cars:import": "bun scripts/cars-import.ts",
+		"cars:photos": "bun scripts/cars-photos.ts",
+		"cars:refresh": "scripts/cars-refresh.sh",
+		"planes:import": "bun scripts/planes-import.ts",
+		"planes:photos": "bun scripts/planes-photos.ts",
+		"planes:refresh": "scripts/planes-refresh.sh",
+		"trains:import": "bun scripts/trains-import.ts",
+		"trains:photos": "bun scripts/trains-photos.ts",
+		"trains:refresh": "scripts/trains-refresh.sh",
+		"wildlife:import": "bun scripts/wildlife-import.ts",
+		"wildlife:photos": "bun scripts/wildlife-photos.ts",
+		"wildlife:refresh": "scripts/wildlife-refresh.sh",
 		"db:seed:cod": "bun scripts/seed-cod-catalog.ts",
 		"cs-prices:ingest": "bun scripts/ingest-cs-prices.ts",
 		"house:worker": "bun scripts/house-worker.ts",
+		"house": "bun scripts/house-listing.ts",
+		"house:add": "bun scripts/house-listing.ts add",
+		"house:list": "bun scripts/house-listing.ts list",
 		"auth:role": "bun scripts/set-admin.ts",
 		"catalog:build": "python3 scripts/build-pokemon-catalog.py",
 		"catalog:cod": "python3 scripts/build-cod-catalog.py && prettier --write data/call-of-duty.json",

scripts/bank-link.ts

@@ -1,10 +1,16 @@
-// Link Revolut (personal + joint) to the money tracker through Enable Banking.
+// Link a bank to the money tracker through Enable Banking. Each bank gets its own session;
+// linking one never touches another bank's session.
 //
-//   bun --env-file=$HOME/.config/money-sync.env scripts/bank-link.ts            interactive consent
+//   bun --env-file=$HOME/.config/money-sync.env scripts/bank-link.ts            Revolut (personal + joint)
+//   ... scripts/bank-link.ts --bank abn                                          ABN AMRO -> "ABN Amro"
+//   ... scripts/bank-link.ts --bank knab                                         Knab -> "Knab"
 //   ... scripts/bank-link.ts --check                                             verify key + app id only
 //   ... scripts/bank-link.ts --code '<redirect URL or code>' [--yes]             finish a started consent
 //   ... scripts/bank-link.ts --map <uid|IBAN>=<Tracker account> [--map ...]      override the mapping
-//   ... scripts/bank-link.ts --aspsp Revolut --country NL --user <auth user id>
+//   ... scripts/bank-link.ts --aspsp '<ASPSP name>' --account <Tracker account> --country NL --user <id>
+//
+// Then sync within the hour: the first sync of an account asks for the longest history the
+// bank serves, and most banks only serve ~90 days once consent is older than an hour.
 //
 // Needs ENABLEBANKING_APP_ID, ENABLEBANKING_KEY_PATH, DB_URL (and optionally
 // ENABLEBANKING_REDIRECT_URL). Never prints the private key.
@@ -33,11 +39,37 @@ args.forEach((arg, i) => {
 	overrides[key.replace(/\s/g, '').toUpperCase()] = rest.join('=');
 });
 
+// Enable Banking ASPSP names (GET /aspsps?country=NL) and the tracker account they feed.
+const BANKS: Record<string, { aspsp: string; account: string; hint: string }> = {
+	revolut: {
+		aspsp: 'Revolut',
+		account: 'Revolut',
+		hint: 'Tick BOTH your personal account and the joint account with Juhee.'
+	},
+	abn: {
+		aspsp: 'ABN AMRO',
+		account: 'ABN Amro',
+		hint: 'Log in with the ABN AMRO app or identifier and approve your current account.'
+	},
+	knab: {
+		aspsp: 'Knab',
+		account: 'Knab',
+		hint: 'Approve your Knab current account (savings accounts are not needed).'
+	}
+};
+const bankKey = (value('--bank') ?? 'revolut').toLowerCase().replace(/[\s-]|amro/g, '');
+const preset = value('--aspsp') && !value('--bank') ? undefined : BANKS[bankKey];
+if (!preset && !value('--aspsp'))
+	throw new Error(`Unknown --bank ${value('--bank')}; use ${Object.keys(BANKS).join(', ')}`);
 const client = createEnableBankingClient(configFromEnv());
-const aspsp = { name: value('--aspsp') ?? 'Revolut', country: value('--country') ?? 'NL' };
+const aspsp = {
+	name: value('--aspsp') ?? preset!.aspsp,
+	country: value('--country') ?? 'NL'
+};
+const defaultAccount = value('--account') ?? preset?.account ?? aspsp.name;
 const redirectUrl =
 	process.env.ENABLEBANKING_REDIRECT_URL ??
-	'https://stanrunge.dev/projects/money-tracker/bank-callback';
+	'https://stanrunge.dev/projects/money-tracker/bank-callback'; // registered with Enable Banking; 308s to /trackers/money/bank-callback with the query kept
 
 const app = await client.application();
 console.log(
@@ -47,7 +79,15 @@ const { aspsps } = await client.aspsps(aspsp.country);
 const bank = aspsps.find((a) => a.name.toLowerCase() === aspsp.name.toLowerCase());
 if (!bank) throw new Error(`${aspsp.name} (${aspsp.country}) is not available for this app`);
 console.log(
-	`${bank.name} ${bank.country}: max consent ${Math.floor(bank.maximum_consent_validity / 86400)} days`
+	`${bank.name} ${bank.country}${bank.bic ? ' ' + bank.bic : ''}: max consent ` +
+		`${Math.floor(bank.maximum_consent_validity / 86400)} days${bank.beta ? ' (beta)' : ''}` +
+		(bank.required_psu_headers?.length
+			? `; requires PSU headers ${bank.required_psu_headers.join(', ')} (background syncs ` +
+				'run as "PSU not present", which PSD2 limits to ~4 calls a day per account)'
+			: '')
+);
+console.log(
+	`Linked accounts go to tracker account "${defaultAccount}" unless --map says otherwise.`
 );
 if (flag('--check')) process.exit(0);
 
@@ -63,8 +103,8 @@ if (!pasted) {
 		psuType: 'personal',
 		state: randomUUID()
 	});
-	console.log('\n1. Open this link and approve access in the Revolut app.');
+	console.log(`\n1. Open this link and approve access at ${bank.name}.`);
-	console.log('   Tick BOTH your personal account and the joint account with Juhee:\n');
+	console.log(`   ${preset?.hint ?? 'Approve the account(s) you want to track.'}\n`);
 	console.log('   ' + auth.url + '\n');
 	console.log(`2. You'll land on ${redirectUrl}?code=...`);
 	console.log(
@@ -88,7 +128,10 @@ try {
 	const tracker = await sql<
 		{ id: number; name: string; scope: string; iban: string | null; external_uid: string | null }[]
 	>`select id, name, scope, iban, external_uid from money.accounts where user_id = ${userId} order by id`;
-	const plan = planAccountMapping(session.accounts, tracker, overrides);
+	const plan = planAccountMapping(session.accounts, tracker, overrides, {
+		defaultAccount,
+		joint: bank.name.toLowerCase() === 'revolut'
+	});
 	for (const { bank: account, tracker: target, reason } of plan) {
 		const iban = account.account_id?.iban ?? '(no IBAN)';
 		console.log(
@@ -103,8 +146,14 @@ try {
 		process.exit(1);
 	}
 	await saveSession(sql, userId, session, plan);
+	const linked = plan.filter((p) => p.tracker).map((p) => p.tracker!.name);
+	const only = linked.length === 1 ? ` --account '${linked[0]}'` : '';
+	console.log('Saved. Within the next hour (full history is only served right after consent):');
 	console.log(
-		'Saved. Next: bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile --dry-run'
+		`  bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile --dry-run${only}`
+	);
+	console.log(
+		`  bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile${only}`
 	);
 } finally {
 	rl.close();

scripts/bank-sync.ts

@@ -7,6 +7,8 @@
 //                        run once after linking (with --dry-run first)
 //     --from YYYY-MM-DD  override the start date (default: last synced date - 7 days, or the
 //                        account's first transaction on the first run)
+//     --full             ask for the longest history the bank serves (automatic on an
+//                        account's first sync; only works within an hour of consent at most banks)
 //     --account <name>   only this tracker account
 //
 // Dedupe: unique (account_id, external_id); re-running is always safe.
@@ -31,12 +33,15 @@ try {
 		dryRun,
 		reconcile: args.includes('--reconcile'),
 		from,
+		full: args.includes('--full'),
 		accountName: value('--account')
 	});
 	for (const r of results) {
 		console.log(
 			`${r.account}: fetched ${r.fetched}, booked ${r.booked}, new ${r.inserted}, ` +
-				`matched to existing ${r.matched}, already synced ${r.alreadyKnown}` +
+				`matched to existing ${r.matched}, replaced sum entries ${r.absorbed}, ` +
+				`already synced ${r.alreadyKnown}` +
+				(r.historyFrom ? `, history from ${r.historyFrom}` : '') +
 				(r.balance ? `, bank balance ${r.balance}` : '') +
 				(r.error ? `\n  ERROR ${r.error}` : '')
 		);

scripts/books-ingest.py

@@ -7,7 +7,7 @@ Builds books.works / books.files from:
   * ebook files dropped in $BOOKS_ROOT/inbox (sha256-deduped)   -> works + files
 
 Metadata: embedded (EPUB OPF / PDF info + ISBN scan) -> Open Library -> Google Books
-(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/<work_id>.jpg.
+(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/<sha1(cover_url)[:16]>.jpg (stable across dev/prod).
 Files are moved to $BOOKS_ROOT/library/<Author>/<Title>/.
 
 Usage:
@@ -24,6 +24,9 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from xml.etree import ElementTree as ET
 
+def log(*a):
+    print(*a, flush=True)
+
 UA = "stan-books-ingest/0.1 (personal library catalogue; low volume)"
 LANG = {"eng": "en", "dut": "nl", "nld": "nl", "ger": "de", "deu": "de", "fre": "fr", "fra": "fr", "spa": "es", "ita": "it",
         "por": "pt", "rus": "ru", "jpn": "ja", "kor": "ko", "chi": "zh", "zho": "zh", "lat": "la", "gre": "el", "swe": "sv"}
@@ -541,6 +544,7 @@ def main():
     ap.add_argument("--extra", action="append", default=[], help='extra placeholder "Title|Author|isbn13|origin"')
     ap.add_argument("--reset", action="store_true", help="truncate books.works/files first (scratch DBs only)")
     ap.add_argument("--dry-run", action="store_true", help="no DB writes, no file moves; write plan json")
+    ap.add_argument("--no-move", action="store_true", help="write DB rows (and covers) but leave files in the inbox; for a dev rehearsal before the prod run")
     ap.add_argument("--no-network", action="store_true")
     ap.add_argument("--allow-prod", action="store_true")
     ap.add_argument("--source", default="gdrive")
@@ -562,7 +566,7 @@ def main():
     http = Http(root / ".cache" / "http", delay=0.0 if args.no_network else 1.0)
     en = Enricher(http)
     works: list[Work] = []
-    log = lambda *a: print(*a, flush=True)  # noqa
+
 
     # ---- 1. placeholders: lists.books
     with conn.cursor() as cur:
@@ -889,7 +893,7 @@ def main():
                 if s in shelves:
                     cur.execute("insert into books.shelf_works (shelf_id, work_id) values (%s,%s) on conflict do nothing", (shelves[s], w.db_id))
             if "done" in w.shelves and w.refs.get("done_at"):
-                cur.execute("insert into books.progress (work_id, percent, finished_at) values (%s, 100, %s) on conflict do nothing", (w.db_id, w.refs["done_at"]))
+                cur.execute("insert into books.progress (work_id, percent, finished_at, status) values (%s, 100, %s, 'finished') on conflict do nothing", (w.db_id, w.refs["done_at"]))
 
     # ---- 7. covers
     got = 0
@@ -897,7 +901,7 @@ def main():
         for w in works:
             if not w.cover_url:
                 continue
-            dest = covers / f"{w.db_id}.jpg"
+            dest = covers / (hashlib.sha1(w.cover_url.encode()).hexdigest()[:16] + ".jpg")
             if dest.exists() or http.download(w.cover_url, dest):
                 got += 1
                 with conn.cursor() as cur:
@@ -905,6 +909,9 @@ def main():
         conn.commit()
 
     # ---- 8. move files (inbox -> library); write undo manifest
+    if args.no_move:
+        log("--no-move: DB rows written, files left in inbox")
+        return
     manifest = root / ".cache" / f"moves-{time.strftime('%Y%m%d-%H%M%S')}.tsv"
     dupdir = root / "duplicates"
     qdir = root / "quarantine"

scripts/cars-refresh.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Weekly-ish refresh of the car pokedex (/trackers/cars) on stan-vm.
+#
+#   ssh stan-vm 'cd ~/dev/website-cars && git fetch -q && git checkout -q --detach origin/main \
+#     && bun install --frozen-lockfile >/dev/null && scripts/cars-refresh.sh'
+#
+# 1. bun scripts/cars-import.ts --report-new   rebuilds brands/models/variants from RDW +
+#    fueleconomy.gov and notes which brands/models did not exist before.
+# 2. bun scripts/cars-photos.ts --missing       finds Wikimedia Commons photos for models
+#    that have none yet (new ones included).
+# stdout: one JSON report {generatedAt, totals, newBrands, newModels[] (with photo)}.
+# Progress goes to stderr. DB_URL is read from the running app container and never printed.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [ -z "${DB_URL:-}" ]; then
+	DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' |
+		sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
+	export DB_URL
+fi
+
+report=$(mktemp /tmp/cars-report.XXXXXX.json)
+trap 'rm -f "$report"' EXIT
+
+bun scripts/cars-import.ts --report-new "$@" >"$report"
+bun scripts/cars-photos.ts --missing --enrich "$report" --fetch

scripts/db-meta-sync.ts

@@ -0,0 +1,19 @@
+// Refreshes meta.columns from the live catalog for every table in meta.tables
+// (meta.sync() is SECURITY DEFINER and granted to the app role). Idempotent.
+//   bun scripts/db-meta-sync.ts      uses DB_URL (personal_app)
+import postgres from 'postgres';
+
+const url = process.env.DB_URL;
+if (!url) throw new Error('Set DB_URL (app role)');
+const client = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} });
+try {
+	const [registry] = await client`select to_regprocedure('meta.sync(integer)') is not null as ok`;
+	if (!registry.ok) {
+		console.log('meta.sync() not present; skipping');
+	} else {
+		const [row] = await client`select meta.sync() as added`;
+		console.log(`meta.sync(): ${row.added} column(s) added`);
+	}
+} finally {
+	await client.end();
+}

scripts/deploy-preflight.ts

@@ -129,6 +129,50 @@ try {
 		health.public_usage
 	)
 		throw new Error(`health schema grants are wrong: ${JSON.stringify(health)}`);
+	// Private task tracker: same rules as health (app role read/write, personal_ro none).
+	const [tasks] = await client`
+		select
+			count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
+			count(*) filter (
+				where exists (
+					select 1 from aclexplode(c.relacl) a join pg_roles r on r.oid = a.grantee
+					where r.rolname = 'personal_ro'
+				)
+			) as ro_read,
+			count(*) as tables,
+			has_schema_privilege('public', 'tasks', 'usage') as public_usage
+		from pg_class c join pg_namespace n on n.oid = c.relnamespace
+		where n.nspname = 'tasks' and c.relkind = 'r'
+	`;
+	if (
+		Number(tasks.tables) !== 4 ||
+		Number(tasks.app_rw) !== 4 ||
+		Number(tasks.ro_read) !== 0 ||
+		tasks.public_usage
+	)
+		throw new Error(`tasks schema grants are wrong: ${JSON.stringify(tasks)}`);
+	// Private grocery tracker: same rules (app role read/write, personal_ro none).
+	const [groceries] = await client`
+		select
+			count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
+			count(*) filter (
+				where exists (
+					select 1 from aclexplode(c.relacl) a join pg_roles r on r.oid = a.grantee
+					where r.rolname = 'personal_ro'
+				)
+			) as ro_read,
+			count(*) as tables,
+			has_schema_privilege('public', 'groceries', 'usage') as public_usage
+		from pg_class c join pg_namespace n on n.oid = c.relnamespace
+		where n.nspname = 'groceries' and c.relkind = 'r'
+	`;
+	if (
+		Number(groceries.tables) !== 5 ||
+		Number(groceries.app_rw) !== 5 ||
+		Number(groceries.ro_read) !== 0 ||
+		groceries.public_usage
+	)
+		throw new Error(`groceries schema grants are wrong: ${JSON.stringify(groceries)}`);
 	// Energy tracker reads the ned-collector tables (SELECT only).
 	const [energy] = await client`
 		select count(*) as hours, max(valid_from) as latest from energy.ned_utilization
@@ -138,7 +182,7 @@ try {
 	if (!Number(energy.hours))
 		console.warn('Warning: energy.ned_utilization has no hourly data from the last 7 days');
 	console.log(
-		`health: ${health.tables} tables, app read/write, personal_ro none; energy: ${energy.hours} hourly rows, latest ${energy.latest ? new Date(energy.latest).toISOString() : 'none'}`
+		`health: ${health.tables} tables, tasks: ${tasks.tables} tables, groceries: ${groceries.tables} tables, app read/write, personal_ro none; energy: ${energy.hours} hourly rows, latest ${energy.latest ? new Date(energy.latest).toISOString() : 'none'}`
 	);
 	const [catalog] = await client`
 		select

scripts/deploy-remote.sh

@@ -19,14 +19,31 @@ environment_name=production
 image="git.stanrunge.dev/stan/website:$sha"
 compose="$directory/compose.prod.yml"
 compose_files=(-f "$compose")
+# Images built before the repo rename (stan/personal-website -> stan/website) keep
+# their old local name; retag so rollbacks to those SHAs still work.
+legacy_image="git.stanrunge.dev/stan/personal-website:$sha"
+if ! sudo docker image inspect "$image" >/dev/null 2>&1 &&
+  sudo docker image inspect "$legacy_image" >/dev/null 2>&1; then
+  sudo docker tag "$legacy_image" "$image"
+fi
+if ! sudo docker image inspect "$image" >/dev/null 2>&1; then
+  echo "Image $image is not on this host" >&2; exit 1
+fi
 # Book files (ebooks + covers) live on the host and are bind-mounted read-only at /books;
 # the app streams them only to the owner via /trackers/books/files/<id> (no static path).
 # Layout: $books_host_dir/library/<files.path> and $books_host_dir/covers/<works.cover_path>.
-# compose.prod.yml is host-managed, so the mount goes in a generated override file.
+# compose.prod.yml is host-managed, so the mount goes in a generated override file (~deploy).
 books_host_dir=${BOOKS_HOST_DIR:-/home/stan/books}
 case "$books_host_dir" in /*) ;; *) echo 'BOOKS_HOST_DIR must be absolute' >&2; exit 2 ;; esac
-if [ -d "$books_host_dir/library" ]; then
+# The deploy account can't traverse /home/stan (0750); check from a root container instead.
-  sudo tee "$directory/compose.books.yml" >/dev/null <<YAML
+if sudo docker run --rm --entrypoint sh \
+  --mount "type=bind,source=$books_host_dir,target=/books,readonly" "$image" \
+  -c 'test -d /books/library' 2>/dev/null; then
+  # The deploy account may only sudo-tee $directory/.env, so the override lives in its home;
+  # the compose CLI runs as root and can read it from there.
+  books_override="$HOME/compose.books.yml"
+  rm -f "$books_override"
+  cat >"$books_override" <<YAML
 services:
   app:
     environment:
@@ -37,19 +54,34 @@ services:
         target: /books
         read_only: true
 YAML
-  compose_files+=(-f "$directory/compose.books.yml")
+  compose_files+=(-f "$books_override")
 else
   echo "No $books_host_dir/library on this host; book files will not be readable" >&2
 fi
-# Images built before the repo rename (stan/personal-website -> stan/website) keep
+# Private money documents (contracts, pension rules) + their summaries also live on the host,
-# their old local name; retag so rollbacks to those SHAs still work.
+# outside the repo, bind-mounted read-only at /money-documents. The app streams them only to an
-legacy_image="git.stanrunge.dev/stan/personal-website:$sha"
+# admin via /trackers/money/documents/<id>/file. Layout: documents.json + files/<file>.
-if ! sudo docker image inspect "$image" >/dev/null 2>&1 &&
+money_documents_host_dir=${MONEY_DOCUMENTS_HOST_DIR:-/home/stan/money-documents}
-  sudo docker image inspect "$legacy_image" >/dev/null 2>&1; then
+case "$money_documents_host_dir" in /*) ;; *) echo 'MONEY_DOCUMENTS_HOST_DIR must be absolute' >&2; exit 2 ;; esac
-  sudo docker tag "$legacy_image" "$image"
+if sudo docker run --rm --entrypoint sh \
-fi
+  --mount "type=bind,source=$money_documents_host_dir,target=/money-documents,readonly" "$image" \
-if ! sudo docker image inspect "$image" >/dev/null 2>&1; then
+  -c 'test -f /money-documents/documents.json' 2>/dev/null; then
-  echo "Image $image is not on this host" >&2; exit 1
+  money_documents_override="$HOME/compose.money-documents.yml"
+  rm -f "$money_documents_override"
+  cat >"$money_documents_override" <<YAML
+services:
+  app:
+    environment:
+      MONEY_DOCUMENTS_ROOT: /money-documents
+    volumes:
+      - type: bind
+        source: $money_documents_host_dir
+        target: /money-documents
+        read_only: true
+YAML
+  compose_files+=(-f "$money_documents_override")
+else
+  echo "No $money_documents_host_dir/documents.json on this host; money documents will not be readable" >&2
 fi
 previous_tag=$(sed -n 's/^IMAGE_TAG=//p' "$directory/.env" | head -n 1)
 switched=0
@@ -103,6 +135,12 @@ run_database_script() {
 if [ "$mode" = deploy ]; then
   # Schema changes: Drizzle migrations from ./drizzle, as the schema owner.
   run_database_script scripts/db-migrate.ts
+  # Refresh the generic table GUI's column registry (meta.columns) for tables that migrations
+  # registered in meta.tables. meta.sync() is SECURITY DEFINER and granted to the app role, not
+  # to the schema owner, so run it as DB_URL (personal_app).
+  echo "Running meta.sync()"
+  sudo docker run --rm --network backend --env-file "$environment" "$image" \
+    bun scripts/db-meta-sync.ts
   run_database_script scripts/seed-catalog.ts
   run_database_script scripts/seed-cod-catalog.ts
   run_database_script scripts/seed-pokemon-research.ts

scripts/groceries-import.ts

@@ -0,0 +1,63 @@
+// Nightly supermarket catalogue import (Albert Heijn + Jumbo) and product regrouping.
+//
+//   bun --env-file=$HOME/.config/groceries-import.env scripts/groceries-import.ts [options]
+//     --store ah|jumbo     only this store (repeatable; default: all)
+//     --skip-fetch         no store requests; only regroup what is in the DB
+//     --gtin-limit N       AH product details to fetch for EANs this run (default 1500, 0 = off)
+//     --delay-ms N         minimum delay between requests (default 2000)
+//
+// Politeness: one request at a time, >= 2 s apart, backoff on 429/5xx, identifying User-Agent,
+// and a same-day cache in $GROCERIES_CACHE_DIR (default ~/.cache/groceries, kept 7 days).
+// Plus is not imported: plus.nl blocks this server's IP range (Incapsula), and we do not evade it.
+import postgres from 'postgres';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
+import { enrichAhGtins, importStore, regroup } from '../src/lib/server/groceries/import';
+import { SOURCES } from '../src/lib/server/groceries/sources';
+
+const args = process.argv.slice(2);
+const values = (name: string) =>
+	args.flatMap((a, i) => (a === name && args[i + 1] ? [args[i + 1]] : []));
+const num = (name: string, d: number) => {
+	const v = values(name)[0];
+	const n = v === undefined ? d : Number(v);
+	if (!Number.isFinite(n) || n < 0) throw new Error(`${name} must be a number`);
+	return n;
+};
+if (!process.env.DB_URL) throw new Error('DB_URL is not set');
+const stores = values('--store').length ? values('--store') : Object.keys(SOURCES);
+for (const s of stores) if (!SOURCES[s]) throw new Error(`unknown store ${s}`);
+const delay = num('--delay-ms', 2000);
+const gtinLimit = num('--gtin-limit', 1500);
+const cacheDir = process.env.GROCERIES_CACHE_DIR || join(homedir(), '.cache', 'groceries');
+const log = (m: string) => console.log(`[${new Date().toISOString().slice(11, 19)}Z] ${m}`);
+
+const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
+let failed = false;
+try {
+	pruneCache(cacheDir);
+	if (!args.includes('--skip-fetch')) {
+		for (const slug of stores) {
+			const f = new PoliteFetcher({ store: slug, minDelayMs: delay, cacheDir, log });
+			try {
+				log(`${slug}: importing`);
+				const r = await importStore(sql, SOURCES[slug], f, log);
+				log(
+					`${slug}: ${r.fetched} products (${r.inserted} new, ${r.priceChanges} price changes, ${r.delisted} delisted); ${r.requests} requests, ${r.cacheHits} from cache`
+				);
+				if (slug === 'ah' && gtinLimit > 0) {
+					const g = await enrichAhGtins(sql, f, gtinLimit, log);
+					log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
+				}
+			} catch (e) {
+				failed = true;
+				log(`${slug}: FAILED ${e instanceof Error ? e.message : e}`);
+			}
+		}
+	}
+	await regroup(sql, log);
+} finally {
+	await sql.end();
+}
+if (failed) process.exitCode = 1;

scripts/health-import.ts

@@ -7,8 +7,9 @@
  *   add --dry-run to validate without writing
  *
  * Column names are the snake_case DB columns (see src/lib/health/tables.ts). Rows with an
- * `id` update that row; nutrition_daily and sleep upsert on `date`; everything else is
+ * `id` update that row; nutrition_daily and sleep upsert on `date`; body_measurements dedupe
- * inserted (importing the same file twice duplicates those rows). All-or-nothing: if any
+ * on measured_at or (date, weight_kg) and fill smart-scale columns from the notes, so
+ * re-importing is a no-op; everything else is inserted (importing twice duplicates those). All-or-nothing: if any
  * row is invalid, nothing is written. Uses DB_URL (the app role on prod, personal_dev on dev).
  */
 import { readFile } from 'node:fs/promises';
@@ -66,7 +67,7 @@ const client = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
 try {
 	const counts = await writeRows(drizzle({ client }), clean);
 	for (const [t, c] of Object.entries(counts))
-		console.log(`${t}: ${c.inserted} inserted, ${c.updated} updated`);
+		console.log(`${t}: ${c.inserted} inserted, ${c.updated} updated, ${c.unchanged} unchanged`);
 } finally {
 	await client.end();
 }

scripts/house-db.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Run a house-tracker script against prod with the limited house_cli role
+# (house.listings + house.price_history only), without the app's credentials.
+#   scripts/house-db.sh list --city gouda
+#   scripts/house-db.sh add --street "Raam 12" --city Gouda --price 1475
+# Credentials: ~/.config/house-cli.env (DB_URL=..., mode 600, readable by stan only).
+set -euo pipefail
+cd "$(dirname "$0")/.."
+env_file="${HOUSE_CLI_ENV:-$HOME/.config/house-cli.env}"
+[ -r "$env_file" ] || { echo "missing $env_file" >&2; exit 1; }
+unset DATABASE_URL DB_URL
+exec bun --env-file="$env_file" scripts/house-listing.ts "$@"

scripts/house-listing.ts

@@ -0,0 +1,399 @@
+// Add, update, list and remove house tracker listings by hand (screenshots,
+// links from chats). Writes house.listings directly; no website restart needed.
+//
+//   bun run house add --url <link> --city Gouda --price 1422 --area 84 --bedrooms 2
+//   bun run house add --json listing.json        (or `--json -` for stdin; may be an array)
+//   bun run house update <id|url> --status viewing --notes "Sat 14:00"
+//   bun run house list [--all] [--city gouda] [--offer buy] [--json]
+//   bun run house remove <id|url> [--hard]      (soft remove = marked gone)
+//   bun run house restore <id|url>
+//
+// DATABASE_URL or DB_URL must point at the database, e.g. `bun --env-file=.env run house list`.
+import { readFileSync } from 'node:fs';
+import { parseArgs } from 'node:util';
+import postgres from 'postgres';
+import { lbmClassLabel, scoreListings } from '../src/lib/house-tracker/liveability';
+import {
+	inferOffer,
+	inferPropertyType,
+	keyFromAddress,
+	keyFromUrl,
+	manualListingSchema,
+	normalizeKeys,
+	toColumns,
+	type ListingKey,
+	type ManualListing
+} from '../src/lib/house-tracker/manual';
+import {
+	containsPattern,
+	linkHost,
+	normalizeListingUrl,
+	pickLinkedListing,
+	siteKeyFromUrl,
+	type LinkCandidate
+} from '../src/lib/house-tracker/link-key';
+
+const usage = `usage: bun run house <add|update|list|remove|restore> [options]
+
+add      [--json <file|->] [field flags]   insert, or update when the same link/address exists
+update   <id|url> [field flags]            change only the given fields (use "" to clear one)
+list     [--all] [--city <slug>] [--offer rent|buy] [--status <s>] [--limit n] [--json]
+remove   <id|url> [--hard]                 mark gone (or delete with --hard)
+restore  <id|url>                          undo a soft remove
+
+field flags: --url --offer rent|buy --type apartment|house|room|studio|other --title
+  --street --postal-code --city --neighbourhood --price --area --rooms --bedrooms
+  --bathrooms --year --energy-label --interior --deposit --available-from
+  --contract-type --agent --image-url --description --notes --status
+  interested|contacted|viewing|applied|rejected --lat --lng --offered-at
+  --first-seen (when it was shared) --source --source-id
+other:   --dry-run (print, don't write)`;
+
+const fieldFlags = [
+	'url',
+	'offer',
+	'type',
+	'title',
+	'street',
+	'postal-code',
+	'city',
+	'neighbourhood',
+	'price',
+	'area',
+	'rooms',
+	'bedrooms',
+	'bathrooms',
+	'year',
+	'energy-label',
+	'interior',
+	'deposit',
+	'available-from',
+	'contract-type',
+	'agent',
+	'image-url',
+	'description',
+	'notes',
+	'status',
+	'lat',
+	'lng',
+	'offered-at',
+	'first-seen',
+	'source',
+	'source-id',
+	'external-id'
+] as const;
+
+const { values: flags, positionals } = parseArgs({
+	allowPositionals: true,
+	options: {
+		...Object.fromEntries(fieldFlags.map((name) => [name, { type: 'string' as const }])),
+		json: { type: 'string' },
+		all: { type: 'boolean' },
+		hard: { type: 'boolean' },
+		'dry-run': { type: 'boolean' },
+		limit: { type: 'string' },
+		help: { type: 'boolean', short: 'h' }
+	}
+});
+const [command, target] = positionals;
+const flag = (name: string) => (flags as Record<string, string | boolean | undefined>)[name];
+
+if (!command || flag('help')) {
+	console.log(usage);
+	process.exit(command ? 0 : 1);
+}
+
+function fieldInput(): Record<string, unknown> {
+	const raw: Record<string, unknown> = {};
+	for (const name of fieldFlags) {
+		const value = flag(name);
+		// An empty flag clears the column on update.
+		if (typeof value === 'string') raw[name] = value === '' ? null : value;
+	}
+	return normalizeKeys(raw);
+}
+
+async function jsonInput(): Promise<Record<string, unknown>[]> {
+	const source = flag('json');
+	if (typeof source !== 'string') return [];
+	const text = source === '-' ? readFileSync(0, 'utf8') : readFileSync(source, 'utf8');
+	const parsed = JSON.parse(text) as unknown;
+	return (Array.isArray(parsed) ? parsed : [parsed]) as Record<string, unknown>[];
+}
+
+function parseListing(raw: Record<string, unknown>): ManualListing {
+	const result = manualListingSchema.safeParse(normalizeKeys(raw));
+	if (!result.success) {
+		const issues = result.error.issues.map((i) => `  ${i.path.join('.')}: ${i.message}`);
+		throw new Error(`invalid listing:\n${issues.join('\n')}`);
+	}
+	return result.data;
+}
+
+function keyFor(listing: ManualListing): ListingKey {
+	const key = listing.url ? keyFromUrl(listing.url) : keyFromAddress(listing);
+	return {
+		...key,
+		source: listing.source ?? key.source,
+		sourceId: listing.sourceId ?? key.sourceId
+	};
+}
+
+const dbUrl = process.env.DATABASE_URL ?? process.env.DB_URL;
+if (!dbUrl) throw new Error('Set DATABASE_URL (or DB_URL), e.g. bun --env-file=.env run house …');
+const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
+type Tx = postgres.TransactionSql;
+
+const summary = sql`id, source, source_id, offer, property_type, title, city, city_slug,
+	price_eur, area_m2, rooms, bedrooms, status, removed_at, url`;
+
+async function findId(ref: string | undefined): Promise<number> {
+	if (!ref) throw new Error('give a listing id or url');
+	if (/^\d+$/.test(ref)) return Number(ref);
+	const key = keyFromUrl(ref);
+	const [row] = await sql<{ id: number }[]>`select id from house.listings
+		where (source = ${key.source} and source_id = ${key.sourceId}) or url = ${ref} limit 1`;
+	if (row) return row.id;
+	// Same page under a slightly different link (tracking params, www., trailing slash).
+	const wanted = normalizeListingUrl(ref);
+	const byLink = (await candidatesFor(sql, ref, '')).find(
+		(c) => normalizeListingUrl(c.url) === wanted
+	);
+	if (!byLink) throw new Error(`no listing for ${ref}`);
+	return byLink.id;
+}
+
+/** Rows on the same host as `url`, except those under `source`, for pickLinkedListing. */
+async function candidatesFor(
+	db: postgres.Sql | Tx,
+	url: string | null | undefined,
+	source: string
+): Promise<LinkCandidate[]> {
+	const host = linkHost(url);
+	if (!host) return [];
+	return db<LinkCandidate[]>`select id, source, source_id as "sourceId", url,
+			duplicate_of as "duplicateOf"
+		from house.listings where source <> ${source} and url ilike ${containsPattern(host)}`;
+}
+
+/** Id of the row already keyed (source, source_id), other than `id`. */
+async function keyOwner(tx: Tx, source: string, sourceId: string, id: number) {
+	const [row] = await tx<{ id: number }[]>`select id from house.listings
+		where source = ${source} and source_id = ${sourceId} and id <> ${id}`;
+	return row?.id ?? null;
+}
+
+// Same convention as the worker: a price_history row whenever the price is new or changes.
+async function recordPrice(tx: Tx, id: number, previous: number | null, next: unknown) {
+	if (typeof next !== 'number' || next === previous) return;
+	await tx`insert into house.price_history (listing_id, price_eur) values (${id}, ${next})`;
+}
+
+async function add() {
+	const fromFlags = fieldInput();
+	const inputs = (await jsonInput()).map((item) => ({ ...item, ...fromFlags }));
+	if (!inputs.length) inputs.push(fromFlags);
+
+	for (const raw of inputs) {
+		const listing = parseListing(raw);
+		const key = keyFor(listing);
+		const hint = `${key.url} ${listing.title ?? ''} ${listing.description?.slice(0, 300) ?? ''}`;
+		const given = toColumns(listing);
+		if (listing.url) given.url = key.url;
+		if (!given.city_slug && key.citySlug) given.city_slug = key.citySlug;
+
+		const insert = {
+			source: key.source,
+			source_id: key.sourceId,
+			url: key.url,
+			offer: listing.offer ?? key.offer ?? inferOffer(hint) ?? 'rent',
+			property_type: listing.propertyType ?? key.propertyType ?? inferPropertyType(hint) ?? 'other',
+			first_seen_lane: 'manual',
+			// Nothing for the worker to fetch: we filled in what we know.
+			detail_fetched_at: new Date(),
+			...given
+		};
+
+		if (flag('dry-run')) {
+			console.log(insert);
+			continue;
+		}
+
+		const row = await sql.begin(async (tx) => {
+			let [before] = await tx<{ id: number; price_eur: number | null }[]>`
+				select id, price_eur from house.listings
+				where source = ${key.source} and source_id = ${key.sourceId} for update`;
+			let rekey = false;
+			if (!before && key.source !== 'manual' && listing.url) {
+				// The same link already on a hand-added row: update it and give it the site's key.
+				const linked = pickLinkedListing(
+					await candidatesFor(tx, key.url, key.source),
+					key,
+					key.url
+				);
+				if (linked) {
+					[before] = await tx<{ id: number; price_eur: number | null }[]>`
+						select id, price_eur from house.listings where id = ${linked.id} for update`;
+					rekey = Boolean(before);
+				}
+			}
+			if (before) {
+				const update: Record<string, unknown> = {
+					...given,
+					last_seen_at: new Date(),
+					removed_at: null
+				};
+				if (rekey) Object.assign(update, { source: key.source, source_id: key.sourceId });
+				if (listing.offer) Object.assign(update, { offer: listing.offer });
+				const [updated] = await tx`update house.listings set ${tx(update)}
+					where id = ${before.id} returning ${summary}`;
+				await recordPrice(tx, before.id, before.price_eur, given.price_eur);
+				return { ...updated, action: 'updated' };
+			}
+			const [inserted] = await tx`insert into house.listings ${tx(insert)} returning ${summary}`;
+			await recordPrice(tx, inserted.id, null, given.price_eur);
+			return { ...inserted, action: 'added' };
+		});
+		print([row]);
+		if (row?.id) await liveability(row.id as number);
+	}
+}
+
+/** Leefbaarometer score for a listing that was just added or changed (skips if unchanged). */
+async function liveability(id: number) {
+	try {
+		const r = await scoreListings(sql, { ids: [id] });
+		if (!r.checked) return;
+		const [l] = await sql<
+			{
+				lbm_class: number | null;
+				lbm_onv_class: number | null;
+				lbm_buurt_name: string | null;
+				lbm_precision: string | null;
+			}[]
+		>`select lbm_class, lbm_onv_class, lbm_buurt_name, lbm_precision from house.listings where id = ${id}`;
+		console.log(
+			l?.lbm_class != null
+				? `    leefbaarometer ${l.lbm_class}/9 ${lbmClassLabel(l.lbm_class)} · overlast ${l.lbm_onv_class}/9 · ${l.lbm_buurt_name}${l.lbm_precision === 'address' ? '' : ` (approx., ${l.lbm_precision})`}`
+				: r.failed
+					? '    leefbaarometer: lookup failed, the nightly run retries'
+					: '    leefbaarometer: no location found for this address'
+		);
+	} catch (error) {
+		console.warn(`    leefbaarometer: ${(error as Error).message}`);
+	}
+}
+
+async function update() {
+	const id = await findId(target);
+	const listing = parseListing(fieldInput());
+	const given = toColumns(listing);
+	if (listing.url) given.url = listing.url;
+	if (listing.offer) given.offer = listing.offer;
+	if (listing.propertyType) given.property_type = listing.propertyType;
+	if (listing.source) given.source = listing.source;
+	if (listing.sourceId) given.source_id = listing.sourceId;
+	if (!Object.keys(given).length) throw new Error('nothing to update');
+	const [row] = await sql.begin(async (tx) => {
+		const [before] = await tx<
+			{ price_eur: number | null; source: string; source_id: string; url: string }[]
+		>`select price_eur, source, source_id, url from house.listings where id = ${id} for update`;
+		if (!before) throw new Error(`no listing ${id}`);
+		// A manual row with a Funda/Pararius/Kamernet/Vesteda link gets that site's key, so a
+		// later scrape of the listing updates this row instead of adding a copy.
+		const url = (given.url as string | undefined) ?? before.url;
+		const site = siteKeyFromUrl(url);
+		if (before.source === 'manual' && !given.source && !given.source_id && site) {
+			const owner = await keyOwner(tx, site.source, site.sourceId, id);
+			if (owner == null) Object.assign(given, { source: site.source, source_id: site.sourceId });
+			else
+				console.warn(
+					`kept source=manual: ${site.source}/${site.sourceId} is already #${owner} (merge them by hand)`
+				);
+		} else if (given.source || given.source_id) {
+			const source = (given.source as string | undefined) ?? before.source;
+			const sourceId = (given.source_id as string | undefined) ?? before.source_id;
+			const owner = await keyOwner(tx, source, sourceId, id);
+			if (owner != null) throw new Error(`${source}/${sourceId} is already #${owner}`);
+		}
+		if (flag('dry-run')) {
+			console.log({ id, ...given });
+			return [];
+		}
+		const rows =
+			await tx`update house.listings set ${tx(given)} where id = ${id} returning ${summary}`;
+		await recordPrice(tx, id, before.price_eur, given.price_eur);
+		return rows;
+	});
+	if (row) {
+		print([row]);
+		await liveability(id);
+	}
+}
+
+async function list() {
+	const limit = Number(flag('limit') ?? 100);
+	const city = flag('city');
+	const offer = flag('offer');
+	const status = flag('status');
+	const rows = await sql`select ${summary}, first_seen_at, notes from house.listings
+		where true
+		${flag('all') ? sql`` : sql`and removed_at is null`}
+		${typeof city === 'string' ? sql`and city_slug = ${city}` : sql``}
+		${typeof offer === 'string' ? sql`and offer = ${offer}` : sql``}
+		${typeof status === 'string' ? sql`and status = ${status}` : sql``}
+		order by first_seen_at desc limit ${limit}`;
+	if (flag('json')) console.log(JSON.stringify(rows, null, 2));
+	else print(rows);
+}
+
+async function remove(hard: boolean) {
+	const id = await findId(target);
+	const rows = hard
+		? await sql`delete from house.listings where id = ${id} returning ${summary}`
+		: await sql`update house.listings set removed_at = now() where id = ${id} returning ${summary}`;
+	if (!rows.length) throw new Error(`no listing ${id}`);
+	console.log(hard ? 'deleted:' : 'marked gone:');
+	print(rows);
+}
+
+async function restore() {
+	const id = await findId(target);
+	const rows =
+		await sql`update house.listings set removed_at = null where id = ${id} returning ${summary}`;
+	print(rows);
+}
+
+function print(rows: readonly Record<string, unknown>[]) {
+	if (!rows.length) return console.log('(no listings)');
+	for (const r of rows) {
+		const bits = [
+			r.price_eur != null && `€${r.price_eur}${r.offer === 'rent' ? '/m' : ''}`,
+			r.area_m2 != null && `${r.area_m2} m²`,
+			r.rooms != null && `${r.rooms} rooms`,
+			r.bedrooms != null && `${r.bedrooms} bed`,
+			r.status,
+			r.removed_at && 'GONE'
+		].filter(Boolean);
+		console.log(
+			`${r.action ? `${r.action} ` : ''}#${r.id} [${r.source}] ${r.title ?? '(no title)'}, ${r.city ?? '?'} · ${bits.join(' · ')}\n    ${r.url}`
+		);
+	}
+}
+
+try {
+	if (command === 'add') await add();
+	else if (command === 'update') await update();
+	else if (command === 'list' || command === 'ls') await list();
+	else if (command === 'remove' || command === 'rm') await remove(Boolean(flag('hard')));
+	else if (command === 'restore') await restore();
+	else {
+		console.error(usage);
+		process.exitCode = 1;
+	}
+} catch (error) {
+	console.error(error instanceof Error ? error.message : error);
+	process.exitCode = 1;
+} finally {
+	await sql.end();
+}

scripts/house-liveability.ts

@@ -0,0 +1,25 @@
+// (Re)computes Leefbaarometer scores for house.listings. The nightly scrape, the CLI and the
+// page already score new and changed listings; this is for backfills and a new LBM year.
+//
+//   bun --env-file=$HOME/.config/house-scrape.env scripts/house-liveability.ts [--force] [--id 12]
+import postgres from 'postgres';
+import { scoreListings } from '../src/lib/house-tracker/liveability';
+
+const args = process.argv.slice(2);
+const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
+if (!dbUrl) throw new Error('DB_URL is not set');
+const sql = postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} });
+const ids = args.includes('--id') ? [Number(args[args.indexOf('--id') + 1])] : undefined;
+try {
+	const result = await scoreListings(sql, {
+		ids,
+		force: args.includes('--force'),
+		log: (line) => console.log(line)
+	});
+	console.log(
+		`liveability: ${result.checked} checked, ${result.scored} scored, ${result.notFound} without a location, ${result.failed} failed`
+	);
+	if (result.failed) process.exitCode = 1;
+} finally {
+	await sql.end();
+}

scripts/house-rekey.ts

@@ -0,0 +1,41 @@
+// One-off/backfill: give manual house.listings rows with a Funda, Pararius, Kamernet or
+// Vesteda link that site's (source, source_id), so a later scrape updates them instead
+// of adding copies. Rows whose key another row already has are reported, not changed.
+//
+//   DB_URL=... bun scripts/house-rekey.ts [--dry-run]
+//   scripts/house-db.sh is for house-listing.ts; run this with a role that may update listings.
+import postgres from 'postgres';
+import { siteKeyFromUrl } from '../src/lib/house-tracker/link-key';
+
+const dryRun = process.argv.includes('--dry-run');
+const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
+if (!dbUrl) throw new Error('DB_URL is not set');
+const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
+
+try {
+	await sql.begin(async (tx) => {
+		const rows = await tx<{ id: number; source_id: string; url: string }[]>`
+			SELECT id, source_id, url FROM house.listings WHERE source = 'manual' ORDER BY id FOR UPDATE`;
+		const claimed = new Set<string>();
+		for (const row of rows) {
+			const key = siteKeyFromUrl(row.url);
+			if (!key) continue;
+			const pair = `${key.source}/${key.sourceId}`;
+			const [owner] = await tx<{ id: number }[]>`SELECT id FROM house.listings
+				WHERE source = ${key.source} AND source_id = ${key.sourceId}`;
+			if (owner || claimed.has(pair)) {
+				console.log(
+					`#${row.id} conflict: ${pair} is already #${owner?.id ?? '(another manual row)'}, kept manual`
+				);
+				continue;
+			}
+			claimed.add(pair);
+			console.log(`#${row.id} ${row.source_id} -> ${pair}${dryRun ? ' (dry run)' : ''}`);
+			if (!dryRun)
+				await tx`UPDATE house.listings SET source = ${key.source}, source_id = ${key.sourceId}
+					WHERE id = ${row.id}`;
+		}
+	});
+} finally {
+	await sql.end();
+}

scripts/house-scrape.ts

@@ -0,0 +1,307 @@
+// Nightly house scraper, run on stan-vm by a systemd user timer (deploy/systemd/house-scrape.*).
+// Fetches the sources that allow it, writes house.listings directly and pushes an ntfy alert
+// for new listings that match an alert rule. See docs/house-tracker.md.
+//
+//   bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet]
+//
+// Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed
+// paths only. A 403/429/challenge stops that source for the night; nothing is retried or
+// worked around. Funda, Pararius, huurwoningen.nl and Huurstunt challenge this server's
+// (datacenter) IP, so they are not scraped from here.
+import postgres from 'postgres';
+import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
+import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address';
+import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
+import { scoreListings } from '../src/lib/house-tracker/liveability';
+import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
+import { isChallenge } from '../src/lib/house-tracker/pararius';
+import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
+import {
+	parseVestedaSearch,
+	VESTEDA_SEARCH_URL,
+	vestedaSearchBody
+} from '../src/lib/house-tracker/vesteda';
+
+const args = process.argv.slice(2);
+const dryRun = args.includes('--dry-run');
+const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null;
+
+const USER_AGENT =
+	'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
+const KAMERNET_CITIES = (
+	process.env.HOUSE_KAMERNET_CITIES ??
+	'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,haastrecht,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
+)
+	.split(',')
+	.map((c) => c.trim())
+	.filter(Boolean);
+// Everything in the region is stored (no price/size caps); paging stops early once a page has
+// none of our cities, so this is only a safety limit.
+const KAMERNET_MAX_PAGES = 25;
+// Gouda centre; Vesteda's own search uses km radius around a point.
+const VESTEDA = {
+	lat: 52.0115,
+	lng: 4.7105,
+	radiusKm: Number(process.env.HOUSE_VESTEDA_RADIUS_KM ?? 20),
+	place: 'Gouda'
+};
+const MAX_ALERTS = 5;
+
+const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
+if (!dbUrl && !dryRun) throw new Error('DB_URL is not set');
+const sql = dbUrl ? postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
+
+class Blocked extends Error {
+	constructor(
+		public status: number,
+		public url: string
+	) {
+		super(`blocked (${status}) on ${url}`);
+	}
+}
+
+let lastRequest = 0;
+async function politeFetch(url: string, init: RequestInit = {}): Promise<Response> {
+	const wait = lastRequest + 6000 + Math.random() * 4000 - Date.now();
+	if (wait > 0) await Bun.sleep(wait);
+	lastRequest = Date.now();
+	const response = await fetch(url, {
+		...init,
+		redirect: 'follow',
+		signal: AbortSignal.timeout(30_000),
+		headers: {
+			'user-agent': USER_AGENT,
+			'accept-language': 'nl-NL,nl;q=0.9,en;q=0.5',
+			...(init.headers ?? {})
+		}
+	});
+	return response;
+}
+
+async function logRun(
+	source: string,
+	url: string,
+	status: string,
+	extra: { httpStatus?: number; found?: number; durationMs?: number; message?: string } = {}
+) {
+	console.log(
+		`${source} ${status}${extra.httpStatus ? ` ${extra.httpStatus}` : ''} ${extra.found ?? ''} ${url}${extra.message ? ` (${extra.message})` : ''}`
+	);
+	if (!sql || dryRun) return;
+	await sql`INSERT INTO house.worker_runs (source, lane, url, status, http_status, found, duration_ms, message)
+		VALUES (${source}, 'nightly', ${url}, ${status}, ${extra.httpStatus ?? null}, ${extra.found ?? null},
+			${extra.durationMs ?? null}, ${extra.message?.slice(0, 500) ?? null})`;
+}
+
+type SourceResult = { listings: ScrapedListing[]; completed: boolean };
+
+async function scrapeKamernet(): Promise<SourceResult> {
+	const listings: ScrapedListing[] = [];
+	const seen = new Set<string>();
+	const kamernetAllowed = new Set<string>([...GOUDA_REGION, ...KAMERNET_CITIES]);
+	let completed = true;
+	for (const city of KAMERNET_CITIES) {
+		for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) {
+			const url = kamernetSearchUrl(city, page);
+			const started = Date.now();
+			const response = await politeFetch(url, { headers: { accept: 'text/html' } });
+			const html = await response.text();
+			if (isChallenge(html, response.status)) {
+				await logRun('kamernet', url, 'challenged', { httpStatus: response.status });
+				throw new Blocked(response.status, url);
+			}
+			if (response.status === 404) {
+				// Kamernet has no page for some small places; that is not an error.
+				await logRun('kamernet', url, 'ok', {
+					httpStatus: 404,
+					found: 0,
+					message: 'no such city page'
+				});
+				break;
+			}
+			if (!response.ok) {
+				await logRun('kamernet', url, 'http-error', { httpStatus: response.status });
+				completed = false;
+				break;
+			}
+			try {
+				const result = parseKamernetSearch(html);
+				// Small places get padded with listings from all over the country; keep the
+				// region only, each listing once, and stop paging once a page has none of ours.
+				const local = result.listings.filter((l) => kamernetAllowed.has(citySlug(l.city)));
+				for (const l of local) {
+					if (seen.has(l.sourceId)) continue;
+					seen.add(l.sourceId);
+					listings.push(l);
+				}
+				await logRun('kamernet', url, 'ok', {
+					httpStatus: 200,
+					found: local.length,
+					durationMs: Date.now() - started,
+					message:
+						local.length < result.listings.length
+							? `${result.listings.length - local.length} outside the region ignored`
+							: undefined
+				});
+				const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1;
+				if (page >= pages || local.length === 0) break;
+			} catch (error) {
+				await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) });
+				completed = false;
+				break;
+			}
+		}
+	}
+	return { listings, completed };
+}
+
+async function scrapeVesteda(): Promise<SourceResult> {
+	const started = Date.now();
+	const response = await politeFetch(VESTEDA_SEARCH_URL, {
+		method: 'POST',
+		headers: { 'content-type': 'application/json', accept: 'application/json' },
+		body: JSON.stringify(vestedaSearchBody(VESTEDA))
+	});
+	const text = await response.text();
+	if (isChallenge(text, response.status)) {
+		await logRun('vesteda', VESTEDA_SEARCH_URL, 'challenged', { httpStatus: response.status });
+		throw new Blocked(response.status, VESTEDA_SEARCH_URL);
+	}
+	if (!response.ok) {
+		await logRun('vesteda', VESTEDA_SEARCH_URL, 'http-error', { httpStatus: response.status });
+		return { listings: [], completed: false };
+	}
+	try {
+		const listings = parseVestedaSearch(JSON.parse(text));
+		await logRun('vesteda', VESTEDA_SEARCH_URL, 'ok', {
+			httpStatus: 200,
+			found: listings.length,
+			durationMs: Date.now() - started
+		});
+		return { listings, completed: true };
+	} catch (error) {
+		await logRun('vesteda', VESTEDA_SEARCH_URL, 'parse-error', {
+			httpStatus: 200,
+			message: String(error)
+		});
+		return { listings: [], completed: false };
+	}
+}
+
+async function ntfy(title: string, body: string, click: string) {
+	const target = process.env.HOUSE_NTFY_URL;
+	if (!target) return false;
+	const url = new URL(target);
+	const response = await fetch(url.origin, {
+		method: 'POST',
+		headers: {
+			'content-type': 'application/json',
+			...(process.env.HOUSE_NTFY_TOKEN
+				? { authorization: `Bearer ${process.env.HOUSE_NTFY_TOKEN}` }
+				: {})
+		},
+		body: JSON.stringify({
+			topic: url.pathname.replace(/^\/+|\/+$/g, ''),
+			title,
+			message: body,
+			click,
+			tags: ['house'],
+			priority: 3
+		}),
+		signal: AbortSignal.timeout(10_000)
+	}).catch(() => null);
+	return Boolean(response?.ok);
+}
+
+const sources: Record<string, () => Promise<SourceResult>> = {
+	kamernet: scrapeKamernet,
+	vesteda: scrapeVesteda
+};
+const fresh: { id: number; source: string; listing: ScrapedListing }[] = [];
+let failures = 0;
+try {
+	for (const [name, scrape] of Object.entries(sources)) {
+		if (only && only !== name) continue;
+		const runStartedAt = new Date();
+		let result: SourceResult;
+		try {
+			result = await scrape();
+		} catch (error) {
+			failures++;
+			console.error(
+				`${name}: ${error instanceof Error ? error.message : error}; skipped until tomorrow`
+			);
+			continue;
+		}
+		if (dryRun || !sql) {
+			for (const l of result.listings)
+				console.log(
+					`  ${l.sourceId} €${l.priceEur ?? '?'} ${l.areaM2 ?? '?'}m² ${l.title} (${l.city}) ${l.url}`
+				);
+			continue;
+		}
+		// An empty result from a "complete" run is more likely a broken page than an empty market.
+		const completed = result.completed && result.listings.length > 0;
+		const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed });
+		console.log(
+			`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated${ingest.adopted ? ` (${ingest.adopted} hand-added rows taken over)` : ''}, ${ingest.removed} gone`
+		);
+		if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
+	}
+
+	if (sql && !dryRun) {
+		const { keyed, copies } = await dedupe(sql);
+		console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);
+
+		// Leefbaarometer score for new listings and changed addresses (PDOK + the LBM WFS).
+		try {
+			const lbm = await scoreListings(sql, { limit: 300 });
+			console.log(
+				`liveability: ${lbm.checked} checked, ${lbm.scored} scored, ${lbm.notFound} without a location, ${lbm.failed} failed`
+			);
+		} catch (error) {
+			console.error(`liveability: ${(error as Error).message}`);
+		}
+
+		const rules =
+			(await sql`SELECT id, name, enabled, offer, min_price AS "minPrice", max_price AS "maxPrice",
+			min_area AS "minArea", min_bedrooms AS "minBedrooms", cities, property_types AS "propertyTypes"
+			FROM house.alert_rules WHERE enabled`) as unknown as AlertRule[];
+		const copyIds = new Set(
+			fresh.length
+				? (
+						await sql<
+							{ id: number }[]
+						>`SELECT id FROM house.listings WHERE duplicate_of IS NOT NULL AND id IN ${sql(fresh.map((f) => f.id))}`
+					).map((r) => r.id)
+				: []
+		);
+		const matches = fresh.flatMap((f) => {
+			if (copyIds.has(f.id)) return [];
+			const listing = { ...f.listing, citySlug: citySlug(f.listing.city), source: f.source };
+			const rule = rules.find((r) => matchesRule(listing, r));
+			return rule ? [{ ...f, message: alertMessage(listing, rule) }] : [];
+		});
+		let sent = 0;
+		for (const match of matches.slice(0, MAX_ALERTS)) {
+			if (await ntfy(match.message.title, match.message.body, match.message.url)) {
+				sent++;
+				await sql`UPDATE house.listings SET alerted_at = now() WHERE id = ${match.id}`;
+			}
+		}
+		if (matches.length > MAX_ALERTS)
+			await ntfy(
+				`🏠 ${matches.length - MAX_ALERTS} more new listings`,
+				'See the house tracker.',
+				'https://stanrunge.dev/trackers/houses?status='
+			);
+		console.log(
+			`alerts: ${matches.length} new matches, ${sent} sent${process.env.HOUSE_NTFY_URL ? '' : ' (HOUSE_NTFY_URL not set)'}`
+		);
+	}
+} finally {
+	await sql?.end();
+}
+
+// Non-zero so systemd shows the unit as failed when a source broke or blocked us.
+if (failures > 0) process.exitCode = 1;

scripts/house-worker.ts

@@ -50,11 +50,29 @@ const config = {
 	api: (process.env.HOUSE_API_URL ?? '').replace(/\/$/, ''),
 	token: process.env.HOUSE_WORKER_TOKEN ?? '',
 	fastSearches: list(process.env.HOUSE_FAST_SEARCHES, [
-		'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
+		// No price band: every Gouda rental is stored; the alert rule decides what pings.
+		'https://www.pararius.nl/huurwoningen/gouda'
 	]),
 	slowSearches: list(
 		process.env.HOUSE_SLOW_SEARCHES,
-		['gouda', 'rotterdam', 'den-haag', 'utrecht', 'amsterdam', 'eindhoven'].flatMap((city) => [
+		[
+			'gouda',
+			'waddinxveen',
+			'reeuwijk',
+			'bodegraven',
+			'boskoop',
+			'moordrecht',
+			'nieuwerkerk-aan-den-ijssel',
+			'zevenhuizen',
+			'oudewater',
+			'schoonhoven',
+			'haastrecht',
+			'rotterdam',
+			'den-haag',
+			'utrecht',
+			'amsterdam',
+			'eindhoven'
+		].flatMap((city) => [
 			`https://www.pararius.nl/huurwoningen/${city}`,
 			`https://www.pararius.nl/koopwoningen/${city}`
 		])

scripts/music-db.ts

@@ -0,0 +1,16 @@
+// Shared database connection for the music scripts: MUSIC_DB_URL (e.g. the optional
+// music_collector role) or DB_URL (the app role, as in the image's prod.env).
+import postgres from 'postgres';
+
+export function musicDb() {
+	const url = process.env.MUSIC_DB_URL || process.env.DB_URL;
+	if (!url) throw new Error('Set MUSIC_DB_URL or DB_URL');
+	return postgres(url, { max: 1, prepare: false, connect_timeout: 10, onnotice: () => {} });
+}
+
+export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export function arg(name: string): string | undefined {
+	const i = process.argv.indexOf(`--${name}`);
+	return i >= 0 ? process.argv[i + 1] : undefined;
+}

scripts/music-import-history.ts

@@ -0,0 +1,69 @@
+// Import a Spotify data export into music.plays (source 'spotify', origin 'export').
+//
+//   bun scripts/music-import-history.ts <file-or-dir> [...] [--min-ms 30000] [--dry-run]
+//
+// Accepts the Extended Streaming History (Streaming_History_Audio_*.json, older
+// endsong_*.json) and the short account-data export (StreamingHistory_music_*.json).
+// Podcasts/audiobooks and plays under --min-ms are skipped. Safe to re-run: a play already
+// stored (same artist+title within a minute, also from the live poller) is not added twice.
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { DEFAULT_MIN_MS, parseHistoryEntry, type HistoryPlay } from '../src/lib/music/history';
+import { insertPlay, newCache, upsertTrack } from '../src/lib/music/store.server';
+import { arg, musicDb } from './music-db';
+
+const minMs = Number(arg('min-ms') ?? DEFAULT_MIN_MS);
+const dryRun = process.argv.includes('--dry-run');
+const skip = new Set(['--min-ms', arg('min-ms')]);
+const inputs = process.argv.slice(2).filter((a) => !a.startsWith('--') && !skip.has(a));
+if (!inputs.length) throw new Error('Usage: bun scripts/music-import-history.ts <file-or-dir> ...');
+
+const NAME = /^(Streaming_History_Audio_.*|endsong_?\d*|StreamingHistory(_music)?_?\d*)\.json$/i;
+function files(path: string): string[] {
+	if (statSync(path).isDirectory())
+		return readdirSync(path).flatMap((name) => {
+			const child = join(path, name);
+			return statSync(child).isDirectory() ? files(child) : NAME.test(name) ? [child] : [];
+		});
+	return [path];
+}
+
+const plays: HistoryPlay[] = [];
+let entries = 0;
+for (const file of inputs.flatMap(files)) {
+	const data = JSON.parse(readFileSync(file, 'utf8')) as unknown[];
+	if (!Array.isArray(data)) continue;
+	entries += data.length;
+	for (const entry of data) {
+		const play = parseHistoryEntry(entry, minMs);
+		if (play) plays.push(play);
+	}
+	console.log(`${file}: ${data.length} entries`);
+}
+plays.sort((a, b) => a.playedAt.getTime() - b.playedAt.getTime());
+console.log(`${plays.length} music plays of ${minMs / 1000}s or more out of ${entries} entries.`);
+if (dryRun || !plays.length) process.exit(0);
+
+const sql = musicDb();
+try {
+	const cache = newCache();
+	let stored = 0;
+	for (const [i, play] of plays.entries()) {
+		const trackId = await upsertTrack(sql, play.track, false, cache);
+		if (
+			await insertPlay(sql, {
+				trackId,
+				playedAt: play.playedAt,
+				source: 'spotify',
+				origin: 'export',
+				msPlayed: play.msPlayed,
+				platform: play.platform
+			})
+		)
+			stored++;
+		if ((i + 1) % 5000 === 0) console.log(`  ${i + 1}/${plays.length} (${stored} new)`);
+	}
+	console.log(`Done: ${stored} new plays stored.`);
+} finally {
+	await sql.end();
+}

scripts/music-import-lastfm.ts

@@ -0,0 +1,132 @@
+// Backfill the full Last.fm history of a user into music.plays (source 'lastfm').
+//
+//   LASTFM_API_KEY=... bun scripts/music-import-lastfm.ts [--user stanrunge] [--sleep 300]
+//                                                        [--max-pages N] [--reset]
+//
+// Walks user.getRecentTracks newest → oldest in 200-scrobble pages, always asking for page 1
+// of a shrinking [from, to] window, so new scrobbles never shift the pages under it.
+// Progress is saved after every page in music.import_state ('lastfm:<user>'): stop it any
+// time and re-run to resume; once complete, a re-run only fetches scrobbles newer than the
+// last completed run. Overlap with Spotify plays is hidden by the music.plays_deduped view.
+// Rate limit: one request per --sleep ms (default 300, Last.fm asks for at most 5/s),
+// exponential back-off on 429/5xx and Last.fm errors 8, 11, 16 and 29.
+import { parseRecentTracks } from '../src/lib/music/lastfm-parse';
+import {
+	getImportState,
+	insertPlay,
+	newCache,
+	setImportState,
+	upsertTrack
+} from '../src/lib/music/store.server';
+import { arg, musicDb, sleep } from './music-db';
+
+type State = {
+	completeUntil: number;
+	walk: { from: number; to: number; top: number } | null;
+	imported: number;
+	seen: number;
+};
+
+const apiKey = process.env.LASTFM_API_KEY;
+if (!apiKey) throw new Error('Set LASTFM_API_KEY (https://www.last.fm/api/account/create)');
+const user = arg('user') ?? 'stanrunge';
+const pause = Number(arg('sleep') ?? 300);
+const maxPages = Number(arg('max-pages') ?? Infinity);
+const key = `lastfm:${user.toLowerCase()}`;
+const RETRYABLE = new Set([8, 11, 16, 29]);
+
+async function fetchPage(from: number, to: number) {
+	const url = `https://ws.audioscrobbler.com/2.0/?${new URLSearchParams({
+		method: 'user.getrecenttracks',
+		user,
+		api_key: apiKey!,
+		format: 'json',
+		limit: '200',
+		page: '1',
+		from: String(from),
+		to: String(to)
+	})}`;
+	for (let attempt = 0; ; attempt++) {
+		let retry: boolean;
+		let message: string;
+		try {
+			const response = await fetch(url, {
+				headers: { 'User-Agent': 'stanrunge.dev music importer (stan@stanrunge.dev)' },
+				signal: AbortSignal.timeout(30_000)
+			});
+			const body = (await response.json().catch(() => null)) as Record<string, unknown> | null;
+			if (response.ok && body && !('error' in body)) return body;
+			const code = Number(body?.error);
+			message = `HTTP ${response.status}${body?.message ? `: ${String(body.message)}` : ''}`;
+			retry = response.status === 429 || response.status >= 500 || RETRYABLE.has(code);
+		} catch (e) {
+			message = e instanceof Error ? e.message : String(e);
+			retry = true;
+		}
+		if (!retry || attempt >= 6) throw new Error(`Last.fm request failed: ${message}`);
+		const wait = 2 ** attempt * 1000;
+		console.warn(`  ${message}; retrying in ${wait / 1000}s`);
+		await sleep(wait);
+	}
+}
+
+const sql = musicDb();
+try {
+	let state: State = (process.argv.includes('--reset')
+		? null
+		: await getImportState<State>(sql, key)) ?? {
+		completeUntil: 0,
+		walk: null,
+		imported: 0,
+		seen: 0
+	};
+	if (!state.walk) {
+		const now = Math.floor(Date.now() / 1000);
+		state = { ...state, walk: { from: state.completeUntil + 1, to: now, top: now } };
+	}
+	const cache = newCache();
+	let pages = 0;
+	console.log(
+		`Last.fm ${user}: importing ${new Date(state.walk!.from * 1000).toISOString()} → ${new Date(state.walk!.to * 1000).toISOString()}`
+	);
+	while (state.walk && pages < maxPages) {
+		const walk = state.walk;
+		const page =
+			walk.from > walk.to ? null : parseRecentTracks(await fetchPage(walk.from, walk.to));
+		pages++;
+		if (!page || page.plays.length === 0) {
+			state = { ...state, completeUntil: walk.top, walk: null };
+			await setImportState(sql, key, state);
+			console.log(`Done: ${state.imported} new plays stored (${state.seen} scrobbles seen).`);
+			break;
+		}
+		let stored = 0;
+		for (const play of page.plays) {
+			const trackId = await upsertTrack(sql, play.track, false, cache);
+			if (
+				await insertPlay(sql, {
+					trackId,
+					playedAt: play.playedAt,
+					source: 'lastfm',
+					origin: 'import'
+				})
+			)
+				stored++;
+		}
+		const oldest = Math.min(...page.plays.map((p) => p.uts));
+		state = {
+			...state,
+			imported: state.imported + stored,
+			seen: state.seen + page.plays.length,
+			walk: { ...walk, to: oldest >= walk.to ? walk.to - 1 : oldest }
+		};
+		await setImportState(sql, key, state);
+		console.log(
+			`  page ${pages}: ${page.plays.length} scrobbles, ${stored} new, back to ${new Date(oldest * 1000).toISOString()} (~${page.total} left)`
+		);
+		await sleep(pause);
+	}
+	if (state.walk) console.log(`Stopped after ${pages} pages; re-run to resume.`);
+} finally {
+	await sql.end();
+}

scripts/music-poll.ts

@@ -0,0 +1,18 @@
+// One Spotify poll (now playing + recently played), for the optional host timer in
+// deploy/music-collector. The app normally does this itself every minute (see
+// src/lib/server/music/poller.ts); the lease in music.spotify_auth stops double polling.
+//   MUSIC_DB_URL=... SPOTIFY_CLIENT_ID=... SPOTIFY_CLIENT_SECRET=... bun scripts/music-poll.ts
+import { pollSpotify } from '../src/lib/music/collector.server';
+import { musicDb } from './music-db';
+
+const clientId = process.env.SPOTIFY_CLIENT_ID;
+const clientSecret = process.env.SPOTIFY_CLIENT_SECRET;
+if (!clientId || !clientSecret) throw new Error('Set SPOTIFY_CLIENT_ID and SPOTIFY_CLIENT_SECRET');
+const sql = musicDb();
+try {
+	const result = await pollSpotify(sql, { clientId, clientSecret });
+	console.log(JSON.stringify(result));
+	if (result.status === 'error') process.exitCode = 1;
+} finally {
+	await sql.end();
+}

scripts/planes-photos.ts

@@ -0,0 +1,680 @@
+// Finds freely licensed photos (Wikimedia Commons) for the plane pokedex (/trackers/planes)
+// and logos for its makers.
+//
+//   DB_URL=... bun scripts/planes-photos.ts              every type, replace all photos + logos
+//   DB_URL=... bun scripts/planes-photos.ts --missing    only types without photos / makers without a logo
+//   DB_URL=... bun scripts/planes-photos.ts --logos      maker logos only
+//   DB_URL=... bun scripts/planes-photos.ts --enrich r.json [--fetch]
+//                                    add lead photos to a planes-import --report-new file (stdout);
+//                                    with --fetch, first match photos for types that have none
+//   bun scripts/planes-photos.ts --dry-run --types built.json [--cache dir] [--out x.json]
+//
+// Every type already is a Wikidata item (from planes-import), so there is no fuzzy item
+// matching. Photos, in order (max 10, low-confidence files are skipped, never guessed):
+//  1. The item's own image (P18).
+//  2. Images of its variants (items that are "subclass of" it), captioned with the variant.
+//  3. Images used in its English Wikipedia article and files in its Commons category (P373).
+//     These must name the type in the file name: its designator ("AH-64", "A320", "Boeing 737"
+//     / "B737" for number-only designators) or, for names without one, a distinctive word
+//     ("Spitfire") that no other type in the dex uses. Files naming a longer sibling type
+//     of the same maker that is not a variant ("737 MAX" on a "737 Classic") are rejected,
+//     as are cockpits, cabins, engines, crashes, drawings, maps, logos and small files.
+// Logos: the maker item's logo (P154, current/preferred first), else the infobox logo of its
+// article (Commons files only), else a plain "<Maker> logo" file in "<Maker> logos".
+// Nothing is downloaded: thumbnail URLs (standard widths) plus author and licence are stored.
+import postgres from 'postgres';
+import { readFile, writeFile } from 'node:fs/promises';
+import {
+	chunks,
+	cmp,
+	fileName,
+	log,
+	mw,
+	qid,
+	saveCache,
+	sparql,
+	stripHtml,
+	tokens,
+	useCache,
+	type ImageInfo
+} from './planes-wiki';
+
+const args = process.argv.slice(2);
+const flag = (name: string) => args.includes(name);
+const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
+const dryRun = flag('--dry-run');
+const missingOnly = flag('--missing');
+const logosOnly = flag('--logos');
+const enrichFile = opt('--enrich');
+const typesFile = opt('--types');
+/** Only types whose full name matches this regex (for testing). */
+const only = opt('--only');
+const cacheDir = opt('--cache');
+const outFile = opt('--out');
+const MAX_PHOTOS = 10;
+const LOGO = 250;
+const THUMB = 960;
+
+type Photo = {
+	file: string;
+	kind: 'lead' | 'variant' | 'article' | 'commons' | 'logo';
+	caption: string | null;
+	thumb: string;
+	width: number;
+	height: number;
+	author: string | null;
+	license: string | null;
+	licenseUrl: string | null;
+};
+
+// ---------------------------------------------------------------- commons
+
+const BAD_FILE =
+	/\.(svg|gif|tiff?|pdf|djvu|webm|ogv|stl)$|logo|emblem|badge|insignia|roundel|patch\b|wordmark|map\b|karte|locator|diagram|chart|graph|schema|schematic|drawing|sketch|blueprint|patent|3[- ]?view|three[- ]view|icon|flag|coat of arms|wappen|signature|symbol|cutaway|silhouette|commons-|wiki|stub|poster|advert|brochure|stamp|timeline|cockpit|flight deck|instrument|panel|cabin|interior|seats?\b|galley|lavatory|engine\b|nacelle|propeller\b|landing gear|undercarriage|wheel|tyre|\btire\b|crash|wreck|accident|debris|incident|\bfire\b|burnt|scrap|graveyard|boneyard|\btoys?\b|scale model|model aircraft|lego|diecast|plastic model|kit\b|simulator|mock-?up|ticket|menu|uniform|crew\b|pilot\b|portrait|ceremony|memorial|grave|plaque|statue|stamp|\bvs\b|versus|deliveries|render|cgi\b|\btug\b|tow cart|comparison|3 ?-?seiten|zeichnung|fahrwerk|atterrissage|tail ?gear|tail ?bumper|wn.trze|innenraum|first class|business class|radar\b/i;
+
+const infoCache = new Map<string, ImageInfo | null>();
+async function imageInfo(files: string[], width = THUMB) {
+	const todo = [...new Set(files)].filter((f) => !infoCache.has(`${width}|${f}`));
+	for (const chunk of chunks(todo, 50)) {
+		const res = await mw('commons.wikimedia.org', {
+			action: 'query',
+			prop: 'imageinfo',
+			iiprop: 'url|size|mime|extmetadata',
+			iiurlwidth: String(width),
+			iiextmetadatafilter: `Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree${
+				width === LOGO ? '|Restrictions' : ''
+			}`,
+			titles: chunk.map((f) => `File:${f}`).join('|')
+		});
+		const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
+		for (const p of res.query?.pages ?? []) {
+			const original = (norm.get(p.title) ?? p.title).replace(/^File:/, '');
+			infoCache.set(`${width}|${original}`, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]);
+		}
+		for (const f of chunk)
+			if (!infoCache.has(`${width}|${f}`)) infoCache.set(`${width}|${f}`, null);
+	}
+}
+function toPhoto(
+	file: string,
+	kind: Photo['kind'],
+	caption: string | null,
+	width = THUMB
+): Photo | null {
+	const info = infoCache.get(`${width}|${file}`);
+	if (!info?.thumburl) return null;
+	const meta = info.extmetadata ?? {};
+	if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
+	let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
+	if (!license || /fair use|non-free/i.test(license)) return null;
+	if (kind === 'logo' && /trademark/i.test(meta.Restrictions?.value ?? ''))
+		license += ' · trademark';
+	if (kind !== 'logo') {
+		if (!/^image\/(jpeg|png|webp)$/.test(info.mime)) return null;
+		if (info.width < THUMB || info.height < 400) return null;
+		if (info.width / info.height > 3 || info.height / info.width > 1.6) return null;
+	}
+	const author = meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null;
+	return {
+		file,
+		kind,
+		caption,
+		thumb: info.thumburl
+			.replace(/\?.*$/, '')
+			.replace('//thumb.wikimedia.org/', '//upload.wikimedia.org/'),
+		width: info.width,
+		height: info.height,
+		author,
+		license,
+		licenseUrl: meta.LicenseUrl?.value ?? null
+	};
+}
+
+async function articleImages(titles: string[]) {
+	const out = new Map<string, string[]>();
+	for (const chunk of chunks([...new Set(titles)], 50)) {
+		let cont: Record<string, string> = {};
+		for (let guard = 0; guard < 30; guard++) {
+			const res = await mw('en.wikipedia.org', {
+				action: 'query',
+				prop: 'images',
+				imlimit: 'max',
+				redirects: '1',
+				titles: chunk.join('|'),
+				...cont
+			});
+			const norm = new Map<string, string>();
+			for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])])
+				norm.set(n.to, norm.get(n.from) ?? n.from);
+			for (const p of res.query?.pages ?? []) {
+				const key = norm.get(p.title) ?? p.title;
+				out.set(key, [
+					...(out.get(key) ?? []),
+					...(p.images ?? []).map((im) => im.title.replace(/^File:/, ''))
+				]);
+			}
+			if (!res.continue) break;
+			cont = res.continue;
+		}
+	}
+	return out;
+}
+async function categoryFiles(category: string, limit = 200) {
+	const files: string[] = [];
+	let cont: Record<string, string> = {};
+	for (let guard = 0; guard < 4 && files.length < limit; guard++) {
+		const res = await mw('commons.wikimedia.org', {
+			action: 'query',
+			list: 'categorymembers',
+			cmtitle: `Category:${category}`,
+			cmtype: 'file',
+			cmlimit: '100',
+			...cont
+		});
+		files.push(...(res.query?.categorymembers ?? []).map((m) => m.title.replace(/^File:/, '')));
+		if (!res.continue) break;
+		cont = res.continue;
+	}
+	return files;
+}
+
+// ---------------------------------------------------------------- matching
+
+type Target = {
+	id: number | null;
+	wikidata: string;
+	wikipedia: string | null;
+	name: string;
+	fullName: string;
+	maker: string;
+	makerSlug: string;
+	numberBuilt: number;
+	photos: number;
+};
+
+/** File name in a comparable form: uppercase, "AH-64D" -> "AH64D", "Boeing 737" -> "BOEING737". */
+const norm = (s: string) =>
+	` ${s
+		.replace(/\.[a-z0-9]+$/i, '')
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/[_]+/g, ' ')
+		.replace(/([A-Z])[\s.-]+(?=\d)/g, '$1')
+		.replace(/(\d)[\s-]+(?=\d)/g, '$1 ')
+		.replace(/[^A-Z0-9]+/g, ' ')} `;
+const GENERIC = new Set(
+	'FAMILY SERIES AIRCRAFT AIRPLANE AEROPLANE HELICOPTER GLIDER BOMBER FIGHTER TRAINER TRANSPORT CARGO SUPER MODEL TYPE CLASS CLASSIC NEXT GENERATION VARIANT VARIANTS COMMANDER STANDARD ROYAL AMERICAN BRITISH FLYING SPORT SPORTS SPECIAL MARK LIGHT HEAVY MEDIUM'.split(
+		' '
+	)
+);
+const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+type Rule = {
+	test: (f: string) => boolean;
+	loose: (f: string) => boolean;
+	designators: string[];
+	required: string[];
+};
+/** Name-based tests for one type: designators like AH64 / A320 / 737 (with maker) and rare words. */
+function ruleFor(t: Target, wordUse: Map<string, number>): Rule {
+	const makerWords = tokens(t.maker).filter((w) => w.length >= 3 && !GENERIC.has(w));
+	const prefixes = [...new Set([...makerWords, ...makerWords.map((w) => w[0])])];
+	const T = norm(t.name).trim().split(' ').filter(Boolean);
+	const designators = T.filter((x) => /^[A-Z]{0,5}\d{1,5}[A-Z]{0,4}$/.test(x));
+	// Short extra words that tell a sub-type apart ("737 MAX", "A320 NEO").
+	const required = T.filter(
+		(x) => /^[A-Z]{2,4}$/.test(x) && !GENERIC.has(x) && !prefixes.includes(x)
+	);
+	const words = T.filter((x) => /^[A-Z]{5,}$/.test(x) && !GENERIC.has(x));
+	const tests: RegExp[] = [];
+	for (const d of designators) {
+		if (/^\d+$/.test(d)) {
+			// Number-only designator: needs the maker ("Boeing 737", "B737", "Cessna 172").
+			if (prefixes.length)
+				tests.push(new RegExp(` (?:${prefixes.map(esc).join('|')}) ?${d}(?![0-9])`));
+		} else if (d.length >= 3)
+			tests.push(new RegExp(` (?:${prefixes.map(esc).join('|') || 'X'})?${esc(d)}(?![0-9])`));
+		else if (makerWords.length)
+			tests.push(new RegExp(` (?:${makerWords.map(esc).join('|')}) ${esc(d)}(?![0-9])`));
+	}
+	for (const w of words) {
+		if ((wordUse.get(w) ?? 0) <= 1 && w.length >= 6) tests.push(new RegExp(` ${esc(w)} `));
+		else if (makerWords.length)
+			tests.push(new RegExp(`(?=.* (?:${makerWords.map(esc).join('|')}) ).* ${esc(w)} `));
+	}
+	// Inside the type's own article or category, a telling name word alone is enough ("Spitfire").
+	const looseRe = words.length ? new RegExp(words.map((w) => ` ${esc(w)} `).join('|')) : null;
+	return {
+		designators,
+		required,
+		loose: (f) => !!looseRe && looseRe.test(norm(f)),
+		test: (f) => {
+			const F = norm(f);
+			return (
+				tests.some((re) => re.test(F)) &&
+				(!designators.length || required.every((r) => F.includes(r)))
+			);
+		}
+	};
+}
+
+async function loadWikidata(targets: Target[]) {
+	const ids = targets.map((t) => t.wikidata);
+	const own = new Map<string, { image: string | null; category: string | null }>();
+	const children = new Map<string, { label: string; image: string }[]>();
+	for (const c of chunks(ids, 300)) {
+		for (const r of await sparql(`SELECT ?i ?img ?cat WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} }
+			OPTIONAL { ?i wdt:P18 ?img } OPTIONAL { ?i wdt:P373 ?cat } }`)) {
+			const id = qid(r.i);
+			const cur = own.get(id) ?? { image: null, category: null };
+			cur.image ??= r.img ? fileName(r.img) : null;
+			cur.category ??= r.cat ?? null;
+			own.set(id, cur);
+		}
+		for (const r of await sparql(`SELECT ?i ?c ?l ?img WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} }
+			?c wdt:P279 ?i ; wdt:P18 ?img . ?c rdfs:label ?l FILTER(LANG(?l) = "en") }`)) {
+			const id = qid(r.i);
+			const list = children.get(id) ?? [];
+			if (!list.some((x) => x.label === r.l)) list.push({ label: r.l, image: fileName(r.img) });
+			children.set(id, list);
+		}
+	}
+	await saveCache(true);
+	return { own, children };
+}
+
+async function photosFor(targets: Target[], all: Target[]) {
+	const wd = await loadWikidata(targets);
+	const wordUse = new Map<string, number>();
+	for (const t of all)
+		for (const w of new Set(tokens(t.name))) wordUse.set(w, (wordUse.get(w) ?? 0) + 1);
+	const byMaker = new Map<string, Target[]>();
+	for (const t of all) byMaker.set(t.makerSlug, [...(byMaker.get(t.makerSlug) ?? []), t]);
+	const childLabels = (t: Target) => (wd.children.get(t.wikidata) ?? []).map((c) => cmp(c.label));
+	const articles = await articleImages(
+		targets.map((t) => t.wikipedia).filter((a): a is string => !!a)
+	);
+	const result = new Map<Target, Photo[]>();
+	let done = 0;
+	for (const t of targets) {
+		const rule = ruleFor(t, wordUse);
+		const ownCmp = cmp(t.name);
+		const variants = new Set(childLabels(t));
+		// Longer-named types of the same maker that are not variants of this one.
+		const siblings = (byMaker.get(t.makerSlug) ?? [])
+			.filter(
+				(o) => o !== t && cmp(o.name).length > ownCmp.length && cmp(o.name).startsWith(ownCmp)
+			)
+			.filter((o) => ![...variants].some((v) => v.includes(cmp(o.name))))
+			.map((o) => ruleFor(o, wordUse))
+			// Only siblings with an extra telling word ("MAX") can claim a file from us.
+			.filter(
+				(o) =>
+					o.required.some((r) => !rule.required.includes(r)) ||
+					o.designators.some((d) => !rule.designators.includes(d))
+			);
+		const ok = (f: string) =>
+			!BAD_FILE.test(f) &&
+			!/\.png$/i.test(f) &&
+			(rule.test(f) || rule.loose(f)) &&
+			!siblings.some((s) => s.test(f));
+		const own = wd.own.get(t.wikidata);
+		const picks: { file: string; kind: Photo['kind']; caption: string | null }[] = [];
+		if (own?.image && !/\.(svg|gif)$/i.test(own.image))
+			picks.push({ file: own.image, kind: 'lead', caption: null });
+		for (const c of (wd.children.get(t.wikidata) ?? []).slice(0, 12))
+			if (!BAD_FILE.test(c.image)) picks.push({ file: c.image, kind: 'variant', caption: c.label });
+		for (const f of (t.wikipedia ? (articles.get(t.wikipedia) ?? []) : []).filter(ok))
+			picks.push({ file: f, kind: 'article', caption: null });
+		const unique = () => [...new Map(picks.map((p) => [p.file, p])).values()];
+		await imageInfo(unique().map((p) => p.file));
+		let photos = unique()
+			.map((p) => toPhoto(p.file, p.kind, p.caption))
+			.filter((p): p is Photo => !!p);
+		if (photos.length < MAX_PHOTOS && own?.category) {
+			const files = (await categoryFiles(own.category)).filter(ok);
+			const want = MAX_PHOTOS * 2;
+			const step = Math.max(1, files.length / want);
+			const spread = Array.from(
+				{ length: Math.min(want, files.length) },
+				(_, i) => files[Math.floor(i * step)]
+			);
+			await imageInfo(spread);
+			const seen = new Set(photos.map((p) => p.file));
+			for (const f of spread) {
+				if (seen.has(f)) continue;
+				const p = toPhoto(f, 'commons', null);
+				if (p) photos.push(p);
+				seen.add(f);
+			}
+		}
+		photos = photos.slice(0, MAX_PHOTOS);
+		if (photos.length) result.set(t, photos);
+		if (++done % 200 === 0) {
+			log(`photos: ${done}/${targets.length} types`);
+			await saveCache(true);
+		}
+	}
+	return result;
+}
+
+// ---------------------------------------------------------------- logos
+
+type Maker = {
+	slug: string;
+	name: string;
+	fullName: string | null;
+	wikidata: string | null;
+	hasLogo: boolean;
+};
+const LOGO_WORD = /logo|wordmark|crest|emblem|marque|brand|logotip|insignia/i;
+const FREE_PD = /public domain|^pd\b|cc0/i;
+function logoFile(file: string, keys: string[]): Photo | null {
+	const photo = toPhoto(file, 'logo', null, LOGO);
+	if (!photo) return null;
+	const named = keys.some((k) => cmp(file).includes(k));
+	if (/\.svg$/i.test(file)) return LOGO_WORD.test(file) || named ? photo : null;
+	if (/\.(png|gif)$/i.test(file))
+		return LOGO_WORD.test(file) || (FREE_PD.test(photo.license ?? '') && named) ? photo : null;
+	if (/\.(jpe?g|webp)$/i.test(file))
+		return LOGO_WORD.test(file) && FREE_PD.test(photo.license ?? '') ? photo : null;
+	return null;
+}
+const PLAIN_WORDS = new Set(
+	'LOGO LOGOS LOGOTYPE WORDMARK EMBLEM BRAND MARQUE SYMBOL ICON BADGE CREST NEW OLD CURRENT SIMPLE FLAT COLOR COLOUR COLOURED COLORED BLACK WHITE GREY GRAY RED BLUE SILVER VERTICAL HORIZONTAL STACKED TRANSPARENT AIRCRAFT AVIATION AEROSPACE COMPANY CORPORATION INC SVG PNG'.split(
+		' '
+	)
+);
+function plainLogoName(file: string, names: string[]) {
+	const t = tokens(file.replace(/\.[a-z]+$/i, ''));
+	for (const n of names) {
+		const nt = tokens(n);
+		const at = t.findIndex((_, i) => nt.every((x, j) => t[i + j] === x));
+		if (at < 0) continue;
+		const rest = [...t.slice(0, at), ...t.slice(at + nt.length)];
+		if (
+			rest.some((x) => x.startsWith('LOGO')) &&
+			rest.every((x) => PLAIN_WORDS.has(x) || /^\d+$/.test(x))
+		)
+			return true;
+	}
+	return false;
+}
+async function infoboxLogo(article: string) {
+	const res = await mw('en.wikipedia.org', {
+		action: 'query',
+		prop: 'revisions',
+		rvprop: 'content',
+		rvslots: 'main',
+		rvsection: '0',
+		redirects: '1',
+		titles: article
+	});
+	const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? '';
+	const m = text.match(
+		/\|\s*logo\s*=\s*(?:\[\[)?(?:File:|Image:)?\s*([^|\]\n{}<]+?\.(?:svg|png|gif))/i
+	);
+	return m ? m[1].trim().replace(/_/g, ' ') : null;
+}
+
+async function findLogos(makers: Maker[]) {
+	const ids = makers.map((m) => m.wikidata).filter((x): x is string => !!x);
+	const info = new Map<
+		string,
+		{ article: string | null; category: string | null; logos: string[] }
+	>();
+	for (const c of chunks(ids, 200)) {
+		const rows = await sparql(`
+SELECT ?m (SAMPLE(?a) AS ?article) (SAMPLE(?cat) AS ?category)
+  (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(YEAR(?start)), ""), ";", COALESCE(STR(YEAR(?end)), "")); separator="|") AS ?logos)
+WHERE {
+  VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} }
+  OPTIONAL { ?a schema:about ?m ; schema:isPartOf <https://en.wikipedia.org/> }
+  OPTIONAL { ?m wdt:P373 ?cat }
+  OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank)
+    OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end } }
+} GROUP BY ?m`);
+		for (const r of rows) {
+			const logos = (r.logos ? r.logos.split('|').filter(Boolean) : [])
+				.map((x) => {
+					const [uri, rank, start, end] = x.split(';');
+					return {
+						file: fileName(uri),
+						current: !end,
+						score:
+							(rank.endsWith('PreferredRank') ? 10000 : 0) + (end ? 0 : 5000) + (Number(start) || 0)
+					};
+				})
+				.sort((a, b) => b.score - a.score);
+			const current = logos.filter((x) => x.current);
+			info.set(qid(r.m), {
+				article: r.article
+					? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
+					: null,
+				category: r.category ?? null,
+				logos: [...new Set((current.length ? current : logos).map((x) => x.file))]
+			});
+		}
+	}
+	await imageInfo(
+		[...info.values()].flatMap((i) => i.logos),
+		LOGO
+	);
+	const out = new Map<string, { photo: Photo | null; article: string | null }>();
+	for (const m of makers) {
+		const i = m.wikidata ? info.get(m.wikidata) : undefined;
+		const names = [...new Set([m.name, m.fullName ?? m.name])];
+		const keys = names.map(cmp);
+		let photo: Photo | null = null;
+		for (const f of i?.logos ?? []) if ((photo = logoFile(f, keys))) break;
+		if (!photo && i?.article) {
+			const f = await infoboxLogo(i.article);
+			if (f) {
+				await imageInfo([f], LOGO);
+				const p = logoFile(f, keys);
+				if (p && (LOGO_WORD.test(f) || keys.some((k) => cmp(f).includes(k)))) photo = p;
+			}
+		}
+		if (!photo && i?.category) {
+			const files = (await categoryFiles(`${i.category} logos`, 100))
+				.filter((f) => /\.(svg|png)$/i.test(f) && /logo/i.test(f) && plainLogoName(f, names))
+				.sort(
+					(x, y) =>
+						Math.max(0, ...(y.match(/(?:19|20)\d\d/g) ?? []).map(Number)) -
+							Math.max(0, ...(x.match(/(?:19|20)\d\d/g) ?? []).map(Number)) ||
+						Number(/\.svg$/i.test(y)) - Number(/\.svg$/i.test(x)) ||
+						x.length - y.length
+				)
+				.slice(0, 5);
+			await imageInfo(files, LOGO);
+			for (const f of files) if ((photo = logoFile(f, keys))) break;
+		}
+		out.set(m.slug, { photo, article: i?.article ?? null });
+	}
+	await saveCache(true);
+	log(`logos: ${[...out.values()].filter((x) => x.photo).length}/${makers.length} makers`);
+	return out;
+}
+
+// ---------------------------------------------------------------- main
+
+let sqlClient: ReturnType<typeof postgres> | null = null;
+function db() {
+	if (!process.env.DB_URL) throw new Error('Set DB_URL (or use --dry-run --types <built.json>)');
+	sqlClient ??= postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
+	return sqlClient;
+}
+
+async function loadTargets(): Promise<{ targets: Target[]; makers: Maker[] }> {
+	if (typesFile) {
+		const built = JSON.parse(await readFile(typesFile, 'utf8')) as {
+			slug: string;
+			name: string;
+			fullName: string;
+			wikidata: string;
+			types: {
+				wikidata: string;
+				wikipedia: string;
+				name: string;
+				fullName: string;
+				numberBuilt: number | null;
+			}[];
+		}[];
+		return {
+			targets: built.flatMap((m) =>
+				m.types.map((t) => ({
+					id: null,
+					wikidata: t.wikidata,
+					wikipedia: t.wikipedia,
+					name: t.name,
+					fullName: t.fullName,
+					maker: m.name,
+					makerSlug: m.slug,
+					numberBuilt: t.numberBuilt ?? 0,
+					photos: 0
+				}))
+			),
+			makers: built.map((m) => ({
+				slug: m.slug,
+				name: m.name,
+				fullName: m.fullName,
+				wikidata: m.wikidata,
+				hasLogo: false
+			}))
+		};
+	}
+	const sql = db();
+	const rows = await sql<
+		{
+			id: number;
+			wikidata: string;
+			wikipedia: string | null;
+			name: string;
+			full_name: string;
+			maker: string;
+			maker_slug: string;
+			built: number | null;
+			photos: number;
+		}[]
+	>`select t.id, t.wikidata, t.wikipedia, t.name, t.full_name, m.name as maker, m.slug as maker_slug,
+		t.number_built as built, t.photo_count as photos
+		from planes.types t join planes.makers m on m.id = t.maker_id`;
+	const makers = await sql<
+		{
+			slug: string;
+			name: string;
+			full_name: string | null;
+			wikidata: string | null;
+			logo: string | null;
+		}[]
+	>`
+		select slug, name, full_name, wikidata, logo from planes.makers`;
+	return {
+		targets: rows.map((r) => ({
+			id: r.id,
+			wikidata: r.wikidata,
+			wikipedia: r.wikipedia,
+			name: r.name,
+			fullName: r.full_name,
+			maker: r.maker,
+			makerSlug: r.maker_slug,
+			numberBuilt: r.built ?? 0,
+			photos: r.photos
+		})),
+		makers: makers.map((m) => ({
+			slug: m.slug,
+			name: m.name,
+			fullName: m.full_name,
+			wikidata: m.wikidata,
+			hasLogo: !!m.logo
+		}))
+	};
+}
+
+async function enrich(file: string) {
+	const report = JSON.parse(await readFile(file, 'utf8')) as {
+		newTypes?: { url: string; photo?: string | null }[];
+	};
+	const sql = db();
+	for (const t of report.newTypes ?? []) {
+		const [, maker, type] =
+			new URL(t.url).pathname.match(/\/trackers\/planes\/([^/]+)\/([^/]+)/) ?? [];
+		const [row] = await sql<{ photo: string | null }[]>`
+			select t.photo from planes.types t join planes.makers m on m.id = t.maker_id
+			where m.slug = ${maker} and t.slug = ${type}`;
+		t.photo = row?.photo ?? null;
+	}
+	console.log(JSON.stringify(report, null, 2));
+}
+
+async function main() {
+	await useCache(cacheDir, 'photos.json');
+	if (enrichFile && !flag('--fetch')) return enrich(enrichFile);
+	const loaded = await loadTargets();
+	const targets = loaded.targets
+		.filter((t) => !missingOnly || t.photos === 0)
+		.filter((t) => !only || new RegExp(only, 'i').test(t.fullName))
+		.sort((a, b) => b.numberBuilt - a.numberBuilt);
+	const photos = logosOnly ? new Map<Target, Photo[]>() : await photosFor(targets, loaded.targets);
+	const makers = loaded.makers.filter(
+		(m) => (!missingOnly || !m.hasLogo) && (!only || targets.some((t) => t.makerSlug === m.slug))
+	);
+	const logos = await findLogos(makers);
+	await saveCache(true);
+	const total = [...photos.values()].reduce((a, p) => a + p.length, 0);
+	log(`photos: ${photos.size}/${targets.length} types with photos, ${total} photos`);
+	if (outFile)
+		await writeFile(
+			outFile,
+			JSON.stringify({
+				types: targets.map((t) => ({
+					type: t.fullName,
+					photos: (photos.get(t) ?? []).map((p) => `${p.kind}: ${p.file}`)
+				})),
+				logos: makers.map((m) => ({ maker: m.name, logo: logos.get(m.slug)?.photo?.file ?? null }))
+			})
+		);
+	if (dryRun) return;
+	const sql = db();
+	await sql.begin(async (tx) => {
+		const ids = logosOnly ? [] : targets.map((t) => t.id!).filter(Boolean);
+		await tx`delete from planes.photos where type_id = any(${ids})`;
+		await tx`update planes.types set photo = null, photo_count = 0, photos_checked_at = now() where id = any(${ids})`;
+		for (const t of logosOnly ? [] : targets) {
+			const list = photos.get(t) ?? [];
+			if (!t.id || !list.length) continue;
+			await tx`update planes.types set photo = ${list[0].thumb}, photo_count = ${list.length} where id = ${t.id}`;
+			await tx`insert into planes.photos ${tx(
+				list.map((p, i) => ({
+					type_id: t.id,
+					position: i,
+					kind: p.kind,
+					file: p.file,
+					caption: p.caption,
+					thumb: p.thumb,
+					width: p.width,
+					height: p.height,
+					author: p.author,
+					license: p.license,
+					license_url: p.licenseUrl
+				}))
+			)}`;
+		}
+		for (const m of makers) {
+			const l = logos.get(m.slug);
+			await tx`update planes.makers set wikipedia = ${l?.article ?? null},
+				logo = ${l?.photo?.thumb ?? null}, logo_file = ${l?.photo?.file ?? null},
+				logo_license = ${l?.photo?.license ?? null}, logo_author = ${l?.photo?.author ?? null}
+				where slug = ${m.slug}`;
+		}
+		await tx`update planes.makers k set photo = (
+			select t.photo from planes.types t where t.maker_id = k.id and t.photo is not null
+			order by t.family_id is null desc, t.number_built desc nulls last limit 1)`;
+	});
+	log('written');
+	if (enrichFile) await enrich(enrichFile);
+}
+
+await main();
+await sqlClient?.end();

scripts/planes-refresh.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Refresh of the plane pokedex (/trackers/planes) on stan-ovh.
+#
+#   ssh stan@37.187.157.191 'cd ~/dev/website-planes && git fetch -q && git checkout -q --detach origin/main \
+#     && bun install --frozen-lockfile >/dev/null && scripts/planes-refresh.sh'
+#
+# 1. bun scripts/planes-import.ts --report-new   rebuilds makers/types from Wikidata + English
+#    Wikipedia (infobox + specs) and notes which makers/types did not exist before. Dex numbers
+#    are stable (types are matched by Wikidata id).
+# 2. bun scripts/planes-photos.ts --missing       finds Wikimedia Commons photos for types that
+#    have none yet (new ones included) and maker logos that are missing.
+# stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}.
+# Progress goes to stderr. DB_URL is read from the running app container and never printed.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [ -z "${DB_URL:-}" ]; then
+	DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' |
+		sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
+	export DB_URL
+fi
+
+report=$(mktemp /tmp/planes-report.XXXXXX.json)
+trap 'rm -f "$report"' EXIT
+
+bun scripts/planes-import.ts --report-new "$@" >"$report"
+bun scripts/planes-photos.ts --missing --enrich "$report" --fetch

scripts/planes-wiki.ts

@@ -0,0 +1,184 @@
+// Shared HTTP helpers for the plane pokedex scripts (scripts/planes-import.ts,
+// scripts/planes-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries,
+// maxlag handling and an optional on-disk cache (--cache <dir>).
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export const UA =
+	'StanRungePlaneDex/1.0 (https://stanrunge.dev/trackers/planes; plane pokedex importer) bun';
+export const log = (...a: unknown[]) => console.error(...a);
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+let cacheDir: string | null = null;
+let cache: Record<string, unknown> = {};
+let dirty = 0;
+let cacheName = 'api.json';
+
+/** Use `<dir>/<name>` as a persistent response cache (only for dev runs and refreshes). */
+export async function useCache(dir: string | null, name = 'api.json') {
+	cacheDir = dir;
+	cacheName = name;
+	if (!dir) return;
+	await mkdir(dir, { recursive: true });
+	try {
+		cache = JSON.parse(await readFile(join(dir, name), 'utf8'));
+	} catch {
+		cache = {};
+	}
+}
+export async function saveCache(force = false) {
+	if (!cacheDir || (!force && dirty < 200)) return;
+	dirty = 0;
+	await writeFile(join(cacheDir, cacheName), JSON.stringify(cache));
+}
+
+export async function getJson<T>(url: string, init?: RequestInit, useCached = true): Promise<T> {
+	const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
+	if (useCached && key in cache) return cache[key] as T;
+	for (let attempt = 0; ; attempt++) {
+		let res: Response;
+		try {
+			res = await fetch(url, {
+				...init,
+				headers: {
+					'user-agent': UA,
+					'api-user-agent': UA,
+					accept: 'application/json',
+					...init?.headers
+				},
+				signal: AbortSignal.timeout(120_000)
+			});
+		} catch (e) {
+			if (attempt < 6) {
+				await sleep(2000 * 2 ** attempt);
+				continue;
+			}
+			throw e;
+		}
+		const retryAfter = Number(res.headers.get('retry-after')) || 0;
+		if (res.ok) {
+			const body = (await res.json()) as T & { error?: { code?: string } };
+			if (body?.error?.code === 'maxlag' && attempt < 8) {
+				await sleep(Math.max(retryAfter, 5) * 1000);
+				continue;
+			}
+			if (useCached && cacheDir) {
+				cache[key] = body;
+				dirty++;
+				await saveCache();
+			}
+			return body;
+		}
+		if ((res.status === 429 || res.status >= 500) && attempt < 6) {
+			await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt));
+			continue;
+		}
+		throw new Error(`${res.status} ${url.slice(0, 200)}`);
+	}
+}
+
+let lastCall = 0;
+/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
+export async function mw<T = MwResponse>(
+	host: string,
+	params: Record<string, string>,
+	useCached = true
+): Promise<T> {
+	const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
+	const key = `https://${host}/w/api.php?${body}`;
+	if (!useCached || !(key in cache)) {
+		const wait = lastCall + 150 - Date.now();
+		if (wait > 0) await sleep(wait);
+		lastCall = Date.now();
+	}
+	return getJson<T>(key, undefined, useCached);
+}
+export type MwPage = {
+	pageid?: number;
+	title: string;
+	missing?: boolean;
+	lastrevid?: number;
+	extract?: string;
+	images?: { title: string }[];
+	imageinfo?: ImageInfo[];
+	revisions?: { revid?: number; slots?: { main?: { content?: string } } }[];
+	categorymembers?: { title: string }[];
+};
+export type ImageInfo = {
+	width: number;
+	height: number;
+	mime: string;
+	thumburl?: string;
+	descriptionurl?: string;
+	extmetadata?: Record<string, { value: string }>;
+};
+export type MwResponse = {
+	query?: {
+		pages?: MwPage[];
+		normalized?: { from: string; to: string }[];
+		redirects?: { from: string; to: string }[];
+		categorymembers?: { title: string }[];
+	};
+	continue?: Record<string, string>;
+};
+
+let lastSparql = 0;
+export async function sparql<T = Record<string, string>>(query: string): Promise<T[]> {
+	const wait = lastSparql + 400 - Date.now();
+	if (wait > 0) await sleep(wait);
+	lastSparql = Date.now();
+	const res = await getJson<{ results: { bindings: Record<string, { value: string }>[] } }>(
+		'https://query.wikidata.org/sparql',
+		{
+			method: 'POST',
+			headers: {
+				'content-type': 'application/x-www-form-urlencoded',
+				accept: 'application/sparql-results+json'
+			},
+			body: new URLSearchParams({ query }).toString()
+		}
+	);
+	return res.results.bindings.map(
+		(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
+	);
+}
+
+export const qid = (uri: string) => uri.split('/').at(-1)!;
+export const fileName = (uri: string) =>
+	decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
+/** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */
+export const cmp = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/&/g, ' AND ')
+		.replace(/[^A-Z0-9]/g, '');
+export const tokens = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/&/g, ' ')
+		.split(/[^A-Z0-9]+/)
+		.filter(Boolean);
+export const slugify = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toLowerCase()
+		.replace(/&/g, ' and ')
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-+|-+$/g, '')
+		.slice(0, 80) || 'x';
+export const chunks = <T>(xs: T[], n: number) =>
+	Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n));
+export const stripHtml = (s: string) =>
+	s
+		.replace(/<[^>]+>/g, ' ')
+		.replace(/&amp;/g, '&')
+		.replace(/&quot;/g, '"')
+		.replace(/&#0?39;/g, "'")
+		.replace(/&nbsp;/g, ' ')
+		.replace(/\s+/g, ' ')
+		.trim();

scripts/revolut-import.ts

@@ -0,0 +1,337 @@
+// Import a Revolut statement (all-time CSV, Excel or PDF export) into one tracker account,
+// deduped against what's already there (bank-sync rows, Daisy's earlier imports, hand entries).
+//
+//   bun --env-file=$HOME/.config/money-sync.env scripts/revolut-import.ts \
+//       --file statement.csv --account Joint [options]
+//     (default)             dry run: report new rows, duplicates, conflicts, resulting balances
+//     --product Current     Revolut sub-account to import (Savings/pockets are reported, skipped)
+//     --currency EUR        only this currency (other currency pockets are reported, skipped)
+//     --opening <euros>     PDF only: the statement's opening balance (default 0)
+//     --list                print every new row / conflict / tracker-only row
+//     --insert-conflicts    also insert rows that fall in the already-covered period but match nothing
+//     --insert-conflicts-until YYYY-MM-DD   same, but only conflicts up to that date (e.g. a row
+//                           the old "Start balance" had absorbed on its own date)
+//     --transfer-pattern <regex>   statement rows of type Transfer whose description matches get
+//                           category 'transfer' (e.g. Personal <-> Joint legs)
+//     --tag-matched-transfers      with --transfer-pattern: also set category 'transfer' on the
+//                           existing tracker rows those statement rows matched
+//     --apply               write (needs a money-schema backup in ~/backups from the last 2 hours)
+//     --drop-start-balance  with --apply: delete the account's "Start balance" row(s), only if the
+//                           statement opens at €0 and, after the import, the only difference
+//                           to the synced bank balance is the tracker-only rows (hand entries
+//                           the bank doesn't have). The sync's balance-adjustment row is then
+//                           set to that difference (removed when €0), so the total stays equal
+//                           to the bank balance.
+//
+// Matching: same amount, date within ±2 days (booking or started date), closest first,
+// description overlap as tie-breaker. Inserted rows get source 'csv' and a stable
+// external_id 'revolut-stmt:<hash>', so re-running never duplicates.
+import { createHash } from 'node:crypto';
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { homedir } from 'node:os';
+import postgres from 'postgres';
+import {
+	checkBalances,
+	parseRevolutCsv,
+	parseRevolutPdfLines,
+	planImport,
+	rowsFromTable,
+	selectRows,
+	statementExternalId,
+	type StatementRow,
+	type TrackerRow
+} from '../src/lib/server/revolut-statement';
+import { BALANCE_ADJUSTMENT_ID, isOpeningBalance } from '../src/lib/server/bank-sync';
+
+const args = process.argv.slice(2);
+const value = (name: string) => {
+	const i = args.indexOf(name);
+	return i >= 0 ? args[i + 1] : undefined;
+};
+const flag = (name: string) => args.includes(name);
+const file = value('--file');
+const accountName = value('--account');
+if (!file || !accountName)
+	throw new Error('usage: --file <statement> --account <tracker account name>');
+if (!process.env.DB_URL) throw new Error('DB_URL is not set');
+const apply = flag('--apply');
+const euro = (c: number | null | undefined) => (c == null ? '—' : `€${(c / 100).toFixed(2)}`);
+
+function xlsxTable(path: string): string[][] {
+	const read = (entry: string) => {
+		try {
+			return execFileSync('unzip', ['-p', path, entry], { maxBuffer: 1 << 28 }).toString('utf8');
+		} catch {
+			return '';
+		}
+	};
+	const unxml = (s: string) =>
+		s
+			.replace(/&lt;/g, '<')
+			.replace(/&gt;/g, '>')
+			.replace(/&quot;/g, '"')
+			.replace(/&apos;/g, "'")
+			.replace(/&amp;/g, '&');
+	const shared = [...read('xl/sharedStrings.xml').matchAll(/<si>([\s\S]*?)<\/si>/g)].map((m) =>
+		unxml([...m[1].matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((t) => t[1]).join(''))
+	);
+	const sheet = read('xl/worksheets/sheet1.xml');
+	const colIndex = (ref: string) =>
+		[...ref.replace(/\d+/g, '')].reduce((n, ch) => n * 26 + ch.charCodeAt(0) - 64, 0) - 1;
+	return [...sheet.matchAll(/<row[^>]*>([\s\S]*?)<\/row>/g)].map((row) => {
+		const cells: string[] = [];
+		for (const c of row[1].matchAll(/<c r="([A-Z]+\d+)"([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) {
+			const type = c[2].match(/t="(\w+)"/)?.[1];
+			const v =
+				c[3]?.match(/<v>([\s\S]*?)<\/v>/)?.[1] ?? c[3]?.match(/<t[^>]*>([\s\S]*?)<\/t>/)?.[1] ?? '';
+			cells[colIndex(c[1])] = type === 's' ? (shared[Number(v)] ?? '') : unxml(v);
+		}
+		return Array.from(cells, (x) => x ?? '');
+	});
+}
+
+async function pdfLines(path: string): Promise<string[]> {
+	const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
+	const doc = await pdfjs.getDocument({ data: new Uint8Array(readFileSync(path)) }).promise;
+	const lines: string[] = [];
+	for (let p = 1; p <= doc.numPages; p++) {
+		const content = await (await doc.getPage(p)).getTextContent();
+		const byY = new Map<number, { x: number; s: string }[]>();
+		for (const item of content.items as { str: string; transform: number[] }[]) {
+			const y = Math.round(item.transform[5]);
+			byY.set(y, [...(byY.get(y) ?? []), { x: item.transform[4], s: item.str }]);
+		}
+		for (const y of [...byY.keys()].sort((a, b) => b - a))
+			lines.push(
+				byY
+					.get(y)!
+					.sort((a, b) => a.x - b.x)
+					.map((i) => i.s)
+					.join(' ')
+			);
+	}
+	return lines;
+}
+
+function loadStatement(path: string): StatementRow[] {
+	const lower = path.toLowerCase();
+	if (lower.endsWith('.xlsx')) return rowsFromTable(xlsxTable(path));
+	if (lower.endsWith('.csv') || lower.endsWith('.txt'))
+		return parseRevolutCsv(readFileSync(path, 'utf8'));
+	throw new Error('Use .csv, .xlsx or .pdf');
+}
+
+const all = file.toLowerCase().endsWith('.pdf')
+	? parseRevolutPdfLines(await pdfLines(file), Math.round(Number(value('--opening') ?? 0) * 100))
+	: loadStatement(file);
+const { kept, skipped } = selectRows(
+	all,
+	value('--product') ?? 'Current',
+	value('--currency') ?? 'EUR'
+);
+// Chronological by booking date; file order breaks ties (it follows the running balance).
+const rows = kept.slice().sort((a, b) => a.date.localeCompare(b.date) || a.line - b.line);
+const balances = checkBalances(kept);
+
+const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
+try {
+	const [account] = await sql<
+		{
+			id: number;
+			user_id: string;
+			name: string;
+			opening_balance_cents: number;
+			bank_balance_cents: number | null;
+			bank_balance_at: Date | null;
+		}[]
+	>`select id, user_id, name, opening_balance_cents, bank_balance_cents, bank_balance_at
+		from money.accounts where lower(name) = lower(${accountName})`;
+	if (!account) throw new Error(`No tracker account named ${accountName}`);
+	const existing = await sql<
+		{
+			id: number;
+			date: string;
+			amount_cents: number;
+			description: string;
+			source: string;
+			external_id: string | null;
+			text: string;
+		}[]
+	>`select id, date::text, amount_cents, description, source, external_id,
+			concat_ws(' ', description, counterparty, raw->'creditor'->>'name', raw->'debtor'->>'name',
+				raw->>'remittance_information', raw->'revolut'->>'description') as text
+		from money.transactions where account_id = ${account.id} order by date, id`;
+	const startRows = existing.filter(
+		(t) => isOpeningBalance(t.description) && t.source === 'manual'
+	);
+	const adjustment = existing.find((t) => t.external_id === BALANCE_ADJUSTMENT_ID);
+	const tracker: TrackerRow[] = existing
+		.filter((t) => !startRows.includes(t) && t !== adjustment)
+		.map((t) => ({
+			id: t.id,
+			date: t.date,
+			amountCents: t.amount_cents,
+			description: t.description,
+			text: t.text,
+			source: t.source,
+			externalId: t.external_id
+		}));
+	const plan = planImport(rows, tracker);
+	const conflictsUntil = value('--insert-conflicts-until');
+	const toInsert = flag('--insert-conflicts')
+		? [...plan.fresh, ...plan.conflicts]
+		: conflictsUntil
+			? [...plan.fresh, ...plan.conflicts.filter((r) => r.date <= conflictsUntil)]
+			: plan.fresh;
+	const transferRe = value('--transfer-pattern')
+		? new RegExp(value('--transfer-pattern')!, 'i')
+		: null;
+	const isTransfer = (r: StatementRow) =>
+		!!transferRe && /transfer/i.test(r.type) && transferRe.test(r.description);
+
+	const today = new Date().toISOString().slice(0, 10);
+	const sum = (xs: { amountCents: number; date: string }[]) =>
+		xs.filter((x) => x.date <= today).reduce((s, x) => s + x.amountCents, 0);
+	const trackedNow =
+		account.opening_balance_cents +
+		sum(tracker) +
+		sum(startRows.map((t) => ({ amountCents: t.amount_cents, date: t.date }))) +
+		(adjustment?.amount_cents ?? 0);
+	const afterNoStart = account.opening_balance_cents + sum(tracker) + sum(toInsert);
+	const bank = account.bank_balance_cents;
+	const residual = bank == null ? null : bank - afterNoStart;
+	const trackerOnlyCents = sum(plan.trackerOnly);
+	const dropOk = balances.openingCents === 0 && residual !== null && residual === -trackerOnlyCents;
+	const boundary = plan.coverageFrom
+		? (rows.filter((r) => r.date < plan.coverageFrom! && r.balanceCents != null).at(-1)
+				?.balanceCents ?? 0)
+		: null;
+
+	console.log(
+		`Statement ${file}: ${all.length} rows; importing ${account.name} = ${value('--product') ?? 'Current'} ${value('--currency') ?? 'EUR'}: ${kept.length} completed rows ${rows[0]?.date ?? ''}..${rows.at(-1)?.date ?? ''}`
+	);
+	const skippedBy = new Map<string, { n: number; cents: number }>();
+	for (const r of skipped) {
+		const key = `${r.product}/${r.currency}/${r.state}`;
+		const e = skippedBy.get(key) ?? { n: 0, cents: 0 };
+		skippedBy.set(key, { n: e.n + 1, cents: e.cents + r.amountCents });
+	}
+	for (const [k, v] of skippedBy) console.log(`  skipped ${k}: ${v.n} rows (net ${euro(v.cents)})`);
+	console.log(
+		`  statement opening ${euro(balances.openingCents)}, closing ${euro(balances.closingCents)}, running-balance breaks ${balances.breaks.length}`
+	);
+	console.log(
+		`Tracker ${account.name} (#${account.id}): ${tracker.length} rows from ${plan.coverageFrom ?? '-'}; start balance rows ${startRows.map((t) => `#${t.id} ${t.date} ${euro(t.amount_cents)}`).join(', ') || 'none'}; adjustment ${euro(adjustment?.amount_cents)}`
+	);
+	console.log(
+		`  statement balance just before ${plan.coverageFrom}: ${euro(boundary)} (tracker start + adjustment: ${euro(sum(startRows.map((t) => ({ amountCents: t.amount_cents, date: t.date }))) + (adjustment?.amount_cents ?? 0))})`
+	);
+	console.log(
+		`Plan: new ${plan.fresh.length} (${euro(sum(plan.fresh))}), duplicates ${plan.duplicates.length} (${plan.duplicates.filter((d) => !d.descMatch).length} with differing description), conflicts ${plan.conflicts.length} (${euro(sum(plan.conflicts))}), tracker-only ${plan.trackerOnly.length} (${euro(sum(plan.trackerOnly))})`
+	);
+	console.log(
+		`Balances: bank ${euro(bank)} (synced ${account.bank_balance_at?.toISOString() ?? 'never'}), tracker now ${euro(trackedNow)}, after import without start balance/adjustment ${euro(afterNoStart)}, gap ${euro(residual)}${residual !== null && residual === -trackerOnlyCents ? ' = tracker-only rows, OK to drop the start balance' : ` (tracker-only rows ${euro(trackerOnlyCents)}; start balance must stay)`}${transferRe ? `; transfers matching pattern: ${rows.filter(isTransfer).length}` : ''}`
+	);
+	if (flag('--list') || !apply) {
+		const show = (
+			label: string,
+			xs: {
+				date: string;
+				amountCents: number;
+				description: string;
+				id?: number;
+				source?: string;
+			}[],
+			limit: number
+		) => {
+			if (!xs.length) return;
+			console.log(`  ${label}:`);
+			for (const x of xs.slice(0, flag('--list') ? Infinity : limit))
+				console.log(
+					`    ${x.date} ${euro(x.amountCents).padStart(10)} ${x.description}${x.id ? ` (#${x.id} ${x.source})` : ''}`
+				);
+			if (!flag('--list') && xs.length > limit)
+				console.log(`    … ${xs.length - limit} more (--list)`);
+		};
+		show('conflicts (statement only, in covered period)', plan.conflicts, 30);
+		show('tracker-only rows (covered period, no statement row)', plan.trackerOnly, 30);
+		show(
+			'duplicates with differing description',
+			plan.duplicates
+				.filter((d) => !d.descMatch)
+				.map((d) => ({
+					...d.row,
+					description: `${d.row.description}  ==  ${d.match.description}`
+				})),
+			10
+		);
+		show('new rows', plan.fresh, 5);
+	}
+	if (!apply) {
+		console.log('Dry run: nothing written. Re-run with --apply (after a backup) to import.');
+	} else {
+		const dir = `${homedir()}/backups`;
+		const fresh = readdirSync(dir).some(
+			(f) =>
+				f.startsWith('money-before-') &&
+				f.endsWith('.dump') &&
+				Date.now() - statSync(`${dir}/${f}`).mtimeMs < 2 * 3600_000
+		);
+		if (!fresh)
+			throw new Error(`No money-before-*.dump in ${dir} from the last 2 hours; back up first`);
+		const drop = flag('--drop-start-balance');
+		if (drop && !dropOk)
+			throw new Error(
+				`Refusing --drop-start-balance: statement opening ${euro(balances.openingCents)}, gap ${euro(residual)} vs tracker-only rows ${euro(trackerOnlyCents)}`
+			);
+		const hash = (s: string) => createHash('sha256').update(s).digest('hex');
+		await sql.begin(async (q) => {
+			let inserted = 0;
+			for (const r of toInsert) {
+				const res = await q`
+					insert into money.transactions
+						(user_id, account_id, date, value_date, description, amount_cents, external_id, source, booking_status, category, raw)
+					values (${account.user_id}, ${account.id}, ${r.date}, ${r.startedDate}, ${r.description.slice(0, 500)},
+						${r.amountCents}, ${statementExternalId(r, hash)}, 'csv', 'BOOK', ${isTransfer(r) ? 'transfer' : null},
+						${q.json({ revolut: { type: r.type, product: r.product, started: r.startedDate, completed: r.completedAt, description: r.description, fee_cents: r.feeCents, currency: r.currency, balance_cents: r.balanceCents, file_line: r.line } })})
+					on conflict do nothing`;
+				inserted += res.count;
+			}
+			console.log(`Inserted ${inserted} rows into ${account.name}.`);
+			if (transferRe && flag('--tag-matched-transfers')) {
+				const ids = plan.duplicates.filter((d) => isTransfer(d.row)).map((d) => d.match.id);
+				const tagged = ids.length
+					? await q`update money.transactions set category = 'transfer'
+						where account_id = ${account.id} and id in ${q(ids)} and category is distinct from 'transfer'`
+					: { count: 0 };
+				console.log(`Tagged ${tagged.count} existing transfer rows.`);
+			}
+			if (drop) {
+				for (const t of startRows)
+					await q`delete from money.transactions where id = ${t.id} and account_id = ${account.id}`;
+				const needed = residual!;
+				if (adjustment && needed === 0)
+					await q`delete from money.transactions where id = ${adjustment.id}`;
+				else if (adjustment)
+					await q`update money.transactions set amount_cents = ${needed} where id = ${adjustment.id}`;
+				else if (needed !== 0)
+					await q`insert into money.transactions
+						(user_id, account_id, date, description, amount_cents, external_id, source, booking_status)
+						values (${account.user_id}, ${account.id}, ${plan.coverageFrom ?? rows[0].date}, 'Bank balance adjustment',
+							${needed}, ${BALANCE_ADJUSTMENT_ID}, 'enablebanking', 'ADJ')`;
+				const [check] = await q<{ total: number }[]>`
+					select (a.opening_balance_cents + coalesce(sum(t.amount_cents) filter (where t.date <= ${today}), 0))::int as total
+					from money.accounts a left join money.transactions t on t.account_id = a.id
+					where a.id = ${account.id} group by a.id`;
+				if (bank != null && check.total !== bank)
+					throw new Error(`Total would be ${euro(check.total)}, bank ${euro(bank)}; rolled back`);
+				console.log(
+					`Removed start balance ${startRows.map((t) => '#' + t.id).join(', ') || '-'}; adjustment ${euro(adjustment?.amount_cents)} -> ${euro(needed)}; total ${euro(check.total)} = bank.`
+				);
+			}
+		});
+	}
+} finally {
+	await sql.end();
+}

scripts/tasks-import.ts

@@ -0,0 +1,46 @@
+/**
+ * Import tasks, goals and areas into the private `tasks` schema (idempotent).
+ *
+ *   bun scripts/tasks-import.ts data/tasks/daisy-import.json             # insert what is missing
+ *   bun scripts/tasks-import.ts --dry-run data/tasks/daisy-import.json   # validate only
+ *   bun scripts/tasks-import.ts --update  data/tasks/daisy-import.json   # also overwrite existing rows
+ *
+ * Dedupe: areas on slug, goals on title, tasks on area + title (case-insensitive).
+ * Uses DB_URL (the app role on prod, personal_dev on dev).
+ */
+import { readFile } from 'node:fs/promises';
+import { drizzle } from 'drizzle-orm/postgres-js';
+import postgres from 'postgres';
+import { importSeed, seedFile } from '../src/lib/server/tasks';
+
+const args = process.argv.slice(2);
+const dryRun = args.includes('--dry-run');
+const update = args.includes('--update');
+const files = args.filter((a) => !a.startsWith('--'));
+if (files.length !== 1) {
+	console.error('usage: bun scripts/tasks-import.ts [--dry-run] [--update] <seed.json>');
+	process.exit(2);
+}
+const parsed = seedFile.safeParse(JSON.parse(await readFile(files[0], 'utf8')));
+if (!parsed.success) {
+	for (const issue of parsed.error.issues.slice(0, 50))
+		console.error(`${issue.path.join('.')}: ${issue.message}`);
+	process.exit(1);
+}
+const seed = parsed.data;
+console.log(
+	`Valid: ${seed.areas.length} areas, ${seed.goals.length} goals, ${seed.tasks.length} tasks`
+);
+if (dryRun) process.exit(0);
+if (!process.env.DB_URL) throw new Error('DB_URL is required');
+const client = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
+try {
+	const r = await importSeed(drizzle({ client }), seed, { update });
+	console.log(
+		`areas: ${r.areas.inserted} inserted, ${r.areas.existing} existing; ` +
+			`goals: ${r.goals.inserted} inserted, ${r.goals.existing} existing, ${r.goals.updated} updated; ` +
+			`tasks: ${r.tasks.inserted} inserted, ${r.tasks.existing} existing, ${r.tasks.updated} updated`
+	);
+} finally {
+	await client.end();
+}

scripts/ticktick-export.ts

@@ -0,0 +1,79 @@
+/**
+ * Export every TickTick task reachable through the official CLI (Open API v1):
+ * open, won't-do and completed tasks of the inbox and every list, plus completed
+ * history month by month (windows are split further when they hit the API's
+ * 200-result cap). Needs `ticktick auth login` done once; never reads the token.
+ *
+ *   bun scripts/ticktick-export.ts ~/.cache/ticktick-export/all.json
+ *
+ * Then: bun scripts/ticktick-import.ts ~/.cache/ticktick-export/all.json
+ */
+import { $ } from 'bun';
+
+const out = process.argv[2];
+if (!out) {
+	console.error('usage: bun scripts/ticktick-export.ts <out.json> [--since YYYY]');
+	process.exit(2);
+}
+const sinceArg = process.argv.indexOf('--since');
+const since = sinceArg > 0 ? Number(process.argv[sinceArg + 1]) : 2015;
+const CAP = 200;
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type Json = any;
+async function tt(...args: string[]): Promise<Json> {
+	for (let attempt = 0; ; attempt++) {
+		const r = await $`ticktick ${args} --json`.quiet().nothrow();
+		if (r.exitCode === 0) return JSON.parse(r.stdout.toString() || '[]');
+		if (attempt >= 3)
+			throw new Error(
+				`ticktick ${args[0]} ${args[1]} failed: ${r.stderr.toString().slice(0, 300)}`
+			);
+		await Bun.sleep(2000 * (attempt + 1));
+	}
+}
+const fmt = (d: Date) => d.toISOString().replace(/\.\d{3}Z$/, '+0000');
+const byId = new Map<string, Json>();
+const add = (list: Json[]) => list.filter((t) => !byId.has(t.id) && byId.set(t.id, t)).length;
+
+const projects: Json[] = await tt('project', 'list');
+// The inbox is not a project; its id shows up on tasks (inbox<userId>).
+const probe: Json[] = await tt('task', 'filter', '--status=0,2,-1');
+const inboxId = probe.map((t) => t.projectId).find((p: string) => p?.startsWith('inbox'));
+add(probe);
+const ids = [...(inboxId ? [inboxId] : []), ...projects.map((p) => p.id)];
+let capped = 0;
+for (const status of ['0', '-1', '2'])
+	for (const p of ids) {
+		const r = await tt('task', 'filter', '--projects', p, `--status=${status}`);
+		if (r.length >= CAP) capped++;
+		add(r);
+	}
+for (const p of projects) add((await tt('project', 'data', p.id)).tasks ?? []);
+
+async function completed(start: Date, end: Date): Promise<void> {
+	const r = await tt('task', 'completed', '--start-date', fmt(start), '--end-date', fmt(end));
+	if (r.length >= CAP && end.getTime() - start.getTime() > 3_600_000) {
+		const mid = new Date(Math.floor((start.getTime() + end.getTime()) / 2000) * 1000);
+		await completed(start, mid);
+		await completed(new Date(mid.getTime() + 1000), end);
+		return;
+	}
+	add(r);
+}
+const now = new Date();
+for (let y = since; y <= now.getUTCFullYear(); y++)
+	for (let m = 0; m < 12; m++) {
+		const start = new Date(Date.UTC(y, m, 1));
+		if (start > now) break;
+		await completed(start, new Date(Date.UTC(y, m + 1, 1) - 1000));
+	}
+
+const tasks = [...byId.values()];
+await Bun.write(out, JSON.stringify({ exportedAt: now.toISOString(), inboxId, projects, tasks }));
+const count = (s: number) => tasks.filter((t) => t.status === s).length;
+console.log(
+	`${tasks.length} tasks (${count(0)} open, ${count(2)} completed, ${count(-1)} won't do) from ` +
+		`${projects.length} lists + inbox` +
+		(capped ? `; ${capped} status/list queries hit the ${CAP} cap (older ones may be missing)` : '')
+);

scripts/ticktick-import.ts

@@ -0,0 +1,65 @@
+/**
+ * Sync TickTick into the private `tasks` schema. Idempotent; rerun after a new export.
+ *
+ *   bun scripts/ticktick-import.ts [--dry-run] [--csv backup.csv] ~/.cache/ticktick-export/all.json
+ *
+ * all.json comes from scripts/ticktick-export.ts (Open API). --csv adds the tasks from a
+ * TickTick backup (Settings → Backup & Import) that the API no longer returns (closed lists,
+ * old won't-do tasks); rows the API also has are matched and skipped (see mergeCsv).
+ * Links listed in data/tasks/ticktick-merges.json (plus exact title matches, see
+ * syncTickTick) attach a TickTick task to an existing task instead of duplicating it.
+ * Uses DB_URL (the app role on prod, personal_dev on dev).
+ */
+import { readFile } from 'node:fs/promises';
+import { drizzle } from 'drizzle-orm/postgres-js';
+import postgres from 'postgres';
+import {
+	mergeCsv,
+	parseTickTickCsv,
+	syncTickTick,
+	type TtExport,
+	type TtMerge
+} from '../src/lib/server/ticktick';
+
+const args = process.argv.slice(2);
+const dryRun = args.includes('--dry-run');
+const csvAt = args.indexOf('--csv');
+const csvPath = csvAt >= 0 ? args[csvAt + 1] : undefined;
+const files = args.filter((a, i) => !a.startsWith('--') && (csvAt < 0 || i !== csvAt + 1));
+if (files.length !== 1 || (csvAt >= 0 && !csvPath)) {
+	console.error(
+		'usage: bun scripts/ticktick-import.ts [--dry-run] [--csv backup.csv] <export.json>'
+	);
+	process.exit(2);
+}
+let data = JSON.parse(await readFile(files[0], 'utf8')) as TtExport;
+if (csvPath) {
+	const merged = mergeCsv(data, parseTickTickCsv(await readFile(csvPath, 'utf8')));
+	data = merged.data;
+	console.log(JSON.stringify({ csv: merged.report }));
+}
+const merges = JSON.parse(
+	await readFile(new URL('../data/tasks/ticktick-merges.json', import.meta.url), 'utf8')
+) as TtMerge[];
+if (!process.env.DB_URL) throw new Error('DB_URL is required');
+const client = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
+class DryRun extends Error {}
+try {
+	const db = drizzle({ client });
+	try {
+		const report = await db.transaction(async (tx) => {
+			const r = await syncTickTick(tx, data, { merges });
+			if (dryRun) {
+				console.log(JSON.stringify(r, null, 2));
+				throw new DryRun();
+			}
+			return r;
+		});
+		console.log(JSON.stringify(report, null, 2));
+	} catch (e) {
+		if (!(e instanceof DryRun)) throw e;
+		console.log('(dry run, rolled back)');
+	}
+} finally {
+	await client.end();
+}