Compare commits
85 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 704880dc98 | |||
| 5f940e8196 | |||
| 8c0fc74283 | |||
| ef9524aa0f | |||
| 8515cf0e9c | |||
| 602ea9c8e1 | |||
| 42b6b6568a | |||
| ba44365c71 | |||
| 3f251145be | |||
| 59b35c0886 | |||
| eccae362ec | |||
| 6696bc7162 | |||
| 2259965a28 | |||
| cc6c7c8ede | |||
| 6249e6daeb | |||
| 085a30bec8 | |||
| 8442d59b68 | |||
| fac49b57ba | |||
| 7636f0b2bc | |||
| bf0a1e1d2a | |||
|
|
0612270545 | ||
| 901d12f71d | |||
| 771f64c168 | |||
| 46703f1a6c | |||
| 0aa14cebf3 | |||
| 3903317a59 | |||
| 4738f10ad1 | |||
| bc58e97776 | |||
| b757269151 | |||
| 45c124a365 | |||
| 9e30e4246f | |||
| a04e5faebe | |||
| 113eb54cd9 | |||
| e95b7da1e3 | |||
| cc4b10f239 | |||
|
|
112c31d194 | ||
| 2bd668bf5e | |||
| 61ae72e50b | |||
| c4689adda4 | |||
| 807f8af167 | |||
| 8c9a08afa1 | |||
| b5748e5ae5 | |||
| c827deb395 | |||
| 8a29d5ad59 | |||
| 35f78ec829 | |||
| 8818dde99d | |||
| 07d2c70973 | |||
| b51fef14fd | |||
| ea7d64a556 | |||
| 6e528da403 | |||
| 85da5afdfe | |||
| 427f4ed095 | |||
| 8dde5aa302 | |||
| 5f67ad36e7 | |||
| b77c3f8cf5 | |||
| 1754ea9316 | |||
| 6a127de950 | |||
| 9d08e6af23 | |||
| 941a9f16e1 | |||
| c8205e474e | |||
| 81ee02f11a | |||
| eb80a830f5 | |||
| 988c6c31a9 | |||
| 2af279e8a9 | |||
| faeb76a7e7 | |||
| a022c45499 | |||
| 6f82d1350e | |||
| 7a039b23f4 | |||
| 8445eef3cf | |||
| 1d4cfd58c5 | |||
| 763d7881fb | |||
| 2cada618bf | |||
| 7914b0140b | |||
| 178fa8ddce | |||
| 9c8f20a618 | |||
| 9c46e281c2 | |||
| 8b2dfed239 | |||
| 2da375451a | |||
| e4de2dcec3 | |||
| 00c1e8b1e7 | |||
| 9a1b21ec39 | |||
| b1ce715ec7 | |||
| 064d41595f | |||
| fa2456b99e | |||
| 0cf5d44062 |
@@ -39,3 +39,9 @@ HOUSE_WORKER_TOKEN=
|
||||
HOUSE_DISCORD_WEBHOOK_URL=
|
||||
HOUSE_NTFY_URL=
|
||||
HOUSE_NTFY_TOKEN=
|
||||
|
||||
# Money tracker bank sync (scripts/bank-link.ts, scripts/bank-sync.ts; not read by the web app).
|
||||
# Keep these in ~/.config/money-sync.env (mode 600), never in git.
|
||||
ENABLEBANKING_APP_ID=
|
||||
ENABLEBANKING_KEY_PATH=
|
||||
ENABLEBANKING_REDIRECT_URL=https://stanrunge.dev/projects/money-tracker/bank-callback
|
||||
|
||||
@@ -19,3 +19,9 @@ data/cod-challenges.json
|
||||
data/game-images.json
|
||||
data/cod-achievement-art.json
|
||||
src/lib/house-tracker/fixtures/
|
||||
|
||||
# Vendored third-party code
|
||||
static/vendor/
|
||||
|
||||
# Generated drizzle snapshots
|
||||
drizzle/*/snapshot.json
|
||||
|
||||
@@ -17,7 +17,11 @@ COPY --from=builder /app/drizzle ./drizzle
|
||||
COPY --from=builder /app/data ./data
|
||||
COPY --from=builder /app/src/lib/server/db/schema ./src/lib/server/db/schema
|
||||
COPY --from=builder /app/src/lib/cs-prices ./src/lib/cs-prices
|
||||
# scripts/house-listing.ts (manual listings CLI) imports these pure helpers.
|
||||
COPY --from=builder /app/src/lib/house-tracker ./src/lib/house-tracker
|
||||
COPY --from=builder /app/src/lib/availability.ts ./src/lib/availability.ts
|
||||
# scripts/music-*.ts (Spotify poll, Last.fm and streaming-history imports) import these.
|
||||
COPY --from=builder /app/src/lib/music ./src/lib/music
|
||||
COPY package.json ./
|
||||
EXPOSE 3000
|
||||
ENV NODE_ENV=production
|
||||
|
||||
40
README.md
40
README.md
@@ -236,13 +236,49 @@ local database and migrations, and `db/reorg/README.md` for the one-time move of
|
||||
old flat schema to this layout. Catalog seeds (`npm run db:seed`, `db:seed:cod`, …)
|
||||
upsert reference data and run on every production deploy.
|
||||
|
||||
## Money tracker
|
||||
## Book tracker
|
||||
|
||||
`/projects/money-tracker` is restricted to administrator accounts. Its tables
|
||||
`/trackers/books` is the public catalogue (covers, status, progress, stats). `/trackers/books/<id>`
|
||||
shows a book; the owner also sees files, notes, highlights and reading sessions there and can
|
||||
edit status, rating and progress. `/trackers/books/<id>/read` is the owner-only reader:
|
||||
EPUB, MOBI/AZW3 and FB2 open in [foliate-js](https://github.com/johnfactotum/foliate-js)
|
||||
(vendored in `static/vendor/foliate-js`, no conversion), PDFs in pdf.js. The reader remembers the
|
||||
position per file, autosaves progress and sessions, and stores highlights with notes.
|
||||
|
||||
Files are never public. They are streamed (with Range support) by
|
||||
`/trackers/books/files/<id>` to the owner only; everyone else gets a 404. The reader, progress
|
||||
and notes endpoints are owner-only too (`src/lib/server/owner.ts`, backstopped in
|
||||
`hooks.server.ts`).
|
||||
|
||||
Files are read from `BOOKS_ROOT` (default `/books` when it exists, otherwise `~/books`):
|
||||
|
||||
- `BOOKS_ROOT/library/<books.files.path>` for book files (written by `scripts/books-ingest.py`)
|
||||
- `BOOKS_ROOT/covers/<books.works.cover_path>` for downloaded covers
|
||||
|
||||
In production, `scripts/deploy-remote.sh` writes `~deploy/compose.books.yml` (the deploy account may only sudo-tee `/opt/personal-website/.env`), which
|
||||
bind-mounts the host's `BOOKS_HOST_DIR` (default `/home/stan/books`) read-only at `/books` and sets
|
||||
`BOOKS_ROOT=/books`. Without a `library` directory there the deploy still succeeds, but files can't
|
||||
be opened.
|
||||
|
||||
## Money
|
||||
|
||||
`/trackers/money` (formerly `/projects/money-tracker`, which now redirects) is restricted to administrator accounts. Its tables
|
||||
are created by `npm run db:migrate:money-tracker`; apply that migration to each
|
||||
environment before deploying the new route. The deployment preflight checks for
|
||||
all nine tracker tables and the projection columns.
|
||||
|
||||
`/trackers/money/documents` lists private documents (contracts, pension rules) with a
|
||||
written summary, an inline viewer and a download. Nothing about them is in git: the
|
||||
files and the summaries live on the host in `MONEY_DOCUMENTS_ROOT` (default
|
||||
`/money-documents` when it exists, otherwise `~/money-documents`) as
|
||||
`documents.json` (`[{ id, title, category, date, pages, file, summary }]`, summary in
|
||||
Markdown) plus `files/<file>`. `scripts/deploy-remote.sh` bind-mounts
|
||||
`/home/stan/money-documents` read-only at `/money-documents` when it has a
|
||||
`documents.json`. Files are only streamed to admins via
|
||||
`/trackers/money/documents/<id>/file` (401 logged out, 403 for non-admins,
|
||||
`private, no-store`). To add one, copy the file into `files/` and add a manifest entry;
|
||||
no redeploy is needed.
|
||||
|
||||
Transactions can be entered manually or imported from a CSV or tab-separated
|
||||
export with `date,account,description,amount` headers. Dates accept
|
||||
`YYYY-MM-DD`, `DD/MM/YYYY`, or `Fri 1 Aug 2025`; amounts accept signed euros
|
||||
|
||||
28
bun.lock
28
bun.lock
@@ -14,9 +14,11 @@
|
||||
"drizzle-orm": "^1.0.0-rc.4",
|
||||
"echarts": "^6.1.0",
|
||||
"highlight.js": "^11.12.0",
|
||||
"jose": "^6.2.10",
|
||||
"marked": "^18.0.11",
|
||||
"marked-highlight": "^2.2.4",
|
||||
"mode-watcher": "^1.1.0",
|
||||
"pdfjs-dist": "^6.4.299",
|
||||
"postgres": "^3.4.9",
|
||||
"topojson-client": "^3.1.0",
|
||||
"world-atlas": "^2.0.2",
|
||||
@@ -325,6 +327,30 @@
|
||||
|
||||
"@melt-ui/svelte": ["@melt-ui/svelte@0.76.2", "", { "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", "@internationalized/date": "^3.5.0", "dequal": "^2.0.3", "focus-trap": "^7.5.2", "nanoid": "^5.0.4" }, "peerDependencies": { "svelte": ">=3 <5" } }, "sha512-7SbOa11tXUS95T3fReL+dwDs5FyJtCEqrqG3inRziDws346SYLsxOQ6HmX+4BkIsQh1R8U3XNa+EMmdMt38lMA=="],
|
||||
|
||||
"@napi-rs/canvas": ["@napi-rs/canvas@1.0.13", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.13", "@napi-rs/canvas-darwin-arm64": "1.0.13", "@napi-rs/canvas-darwin-x64": "1.0.13", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.13", "@napi-rs/canvas-linux-arm64-gnu": "1.0.13", "@napi-rs/canvas-linux-arm64-musl": "1.0.13", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.13", "@napi-rs/canvas-linux-x64-gnu": "1.0.13", "@napi-rs/canvas-linux-x64-musl": "1.0.13", "@napi-rs/canvas-win32-arm64-msvc": "1.0.13", "@napi-rs/canvas-win32-x64-msvc": "1.0.13" } }, "sha512-b87ugZpQ5ViwqVBkjervxpwa0GmJPsKHLBNoZN6GpjWeHzTbDUGryriq+2DYW2XbB+xCqSIINQysC0/I87nPbw=="],
|
||||
|
||||
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.13", "", { "os": "android", "cpu": "arm64" }, "sha512-trn7AaDzQ+yeeA0ezhkC7cB1ZDPtp4UYk0EvB5F8qtYL9jEVWtVrU30WOpI52DszoTxaTsCgf5PP4ZVaOlOIHQ=="],
|
||||
|
||||
"@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.13", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6Y2djEetL8copxfNK6hg7oig/xqTwOchv19hBOm+zBNdKc/02Z3kjZN5wrLzv/l5O04/bAQS6I/AlQpwBlXgPg=="],
|
||||
|
||||
"@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.13", "", { "os": "darwin", "cpu": "x64" }, "sha512-OZoPwwu4TMMg8GTCFBrg8UFExwQTU/55i3uTC/izDat6PHE+ro9dimyQoOboFf91+NY7Pnn70ScR8f8I8+eZXw=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.13", "", { "os": "linux", "cpu": "arm" }, "sha512-dZSYaKROuswU2eUWG5aBKndNVe68elo1YX93xtRax7q658tj8rYE+HwduovWJgs6qwv1r1/CLR60nDeBbyjvyg=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.13", "", { "os": "linux", "cpu": "arm64" }, "sha512-W6FzYE8ADHI93In1vi80KBL6btDtAMuj50Ch5ptkWS8xuntyzbR4h5JDUmXoakxZn2yMInksudTEqVSWoFqFGg=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.13", "", { "os": "linux", "cpu": "arm64" }, "sha512-+6fQ+rR49hkAa4voh2stLIwQ+4t+gmoseM8Bc53/DlMXb73IKymDPWK9A/NtWeqTW4ijWppYuyuxS2hPI8tzNA=="],
|
||||
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.13", "", { "os": "linux", "cpu": "none" }, "sha512-17XEMO3t9RW/TtHtSFf2AkaSWuCWdzbPeidz7zrOFawHxoLSJCnpTI1hN5aUvrZNnkNNmt/vZVJCh0ryUaik3A=="],
|
||||
|
||||
"@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.13", "", { "os": "linux", "cpu": "x64" }, "sha512-lc48IwknVHBsNzXn9Xs5OkUsRX38Ml3Z1aOybQ51mMEzZckm8+4A0qROlyYHEoBPzr1kL3IU8ZUAqDb7wtdX8w=="],
|
||||
|
||||
"@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.13", "", { "os": "linux", "cpu": "x64" }, "sha512-bGY1r26S2bs8UNFJcPWcCSNdCFbKep5PATsvtgtC9A5u6SRgr2/jJE/RlhHL5m9ZpixNDhVkLY/5h16Kx5V9YA=="],
|
||||
|
||||
"@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.13", "", { "os": "win32", "cpu": "arm64" }, "sha512-2vRPBZoaIvJrUkDhZTE2Hn0YrnpJRCWVM3EJACoBKjuWI6WgPxGtUoRMsxRviCsyks8bKTSFdyw14CellnG2qA=="],
|
||||
|
||||
"@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.13", "", { "os": "win32", "cpu": "x64" }, "sha512-DUlW3lZPz63kYsf8mxgyyZuevhlH0T3/BGP9l+NazhB0F1+gL6D4hqZdnhZpw24DlWmTpATDcyYSZlJOi+DdfQ=="],
|
||||
|
||||
"@noble/ciphers": ["@noble/ciphers@2.4.0", "", {}, "sha512-AnjFn0Jv92laAkvMrghlFZq4qQCIN/4DxFV/eooqtC2YTjB7kBeLMS2T9KJX4Dn+ZVXLOwK0lSgqDtx9gvxtiw=="],
|
||||
|
||||
"@noble/hashes": ["@noble/hashes@2.4.0", "", {}, "sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA=="],
|
||||
@@ -837,6 +863,8 @@
|
||||
|
||||
"pbf": ["pbf@4.0.2", "", { "dependencies": { "resolve-protobuf-schema": "^2.1.0" }, "bin": "bin/pbf" }, "sha512-J0ajxARhZfpUEebxYs1vhMGMuLSXtBe1e+fFPDrf2uA2hgo+UshKfNUWOz92HJNz6/NFEXseQPddnHkTreWRqg=="],
|
||||
|
||||
"pdfjs-dist": ["pdfjs-dist@6.4.299", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.10" } }, "sha512-AVl138zALtfaAPvADulE0PZThbYzCBS79nL4pOSL/6Sm/4AH5A21BD9VHt97OlCuzJuCpmeZtAtkinisF4Vb1g=="],
|
||||
|
||||
"pg-int8": ["pg-int8@1.0.1", "", {}, "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="],
|
||||
|
||||
"pg-protocol": ["pg-protocol@1.16.0", "", {}, "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg=="],
|
||||
|
||||
643
data/tasks/daisy-import.json
Normal file
643
data/tasks/daisy-import.json
Normal file
@@ -0,0 +1,643 @@
|
||||
{
|
||||
"source": "daisy-import",
|
||||
"areas": [
|
||||
{
|
||||
"slug": "deadlines",
|
||||
"name": "Deadlines and dates",
|
||||
"sortOrder": 10
|
||||
},
|
||||
{
|
||||
"slug": "house",
|
||||
"name": "House and moving",
|
||||
"description": "Jan–Feb 2027 with Juhee",
|
||||
"sortOrder": 20
|
||||
},
|
||||
{
|
||||
"slug": "kibi",
|
||||
"name": "Kibi",
|
||||
"sortOrder": 30
|
||||
},
|
||||
{
|
||||
"slug": "admin",
|
||||
"name": "Phone, subscriptions and admin",
|
||||
"sortOrder": 40
|
||||
},
|
||||
{
|
||||
"slug": "clients",
|
||||
"name": "Clients",
|
||||
"sortOrder": 50
|
||||
},
|
||||
{
|
||||
"slug": "website",
|
||||
"name": "Website (stanrunge.dev)",
|
||||
"sortOrder": 60
|
||||
},
|
||||
{
|
||||
"slug": "setup",
|
||||
"name": "Setup and security",
|
||||
"description": "Perma evolvable setup, before 2027",
|
||||
"sortOrder": 70
|
||||
},
|
||||
{
|
||||
"slug": "home",
|
||||
"name": "Home",
|
||||
"sortOrder": 80
|
||||
},
|
||||
{
|
||||
"slug": "health",
|
||||
"name": "Health and self-care",
|
||||
"sortOrder": 90
|
||||
},
|
||||
{
|
||||
"slug": "reading",
|
||||
"name": "Reading",
|
||||
"sortOrder": 100
|
||||
},
|
||||
{
|
||||
"slug": "planning",
|
||||
"name": "Planning and goals",
|
||||
"sortOrder": 110
|
||||
}
|
||||
],
|
||||
"goals": [
|
||||
{
|
||||
"title": "€10k net/month into own Revolut by 2030",
|
||||
"area": "planning",
|
||||
"targetDate": "2030-01-01",
|
||||
"metric": "Net income per month",
|
||||
"metricUnit": "EUR",
|
||||
"metricTarget": 10000
|
||||
},
|
||||
{
|
||||
"title": "Own a really nice Ford Mustang GT"
|
||||
},
|
||||
{
|
||||
"title": "PC overhaul",
|
||||
"notes": "Top AMD X3D CPU, RTX 5090.",
|
||||
"metric": "Budget",
|
||||
"metricUnit": "EUR",
|
||||
"metricTarget": 10000
|
||||
},
|
||||
{
|
||||
"title": "Organise goals into a plan",
|
||||
"area": "planning",
|
||||
"notes": "Social media, Grok Bot, IT company."
|
||||
},
|
||||
{
|
||||
"title": "Perma evolvable setup foundation ready before 2027",
|
||||
"area": "setup",
|
||||
"targetDate": "2026-12-31"
|
||||
},
|
||||
{
|
||||
"title": "Get in shape long term",
|
||||
"area": "health"
|
||||
}
|
||||
],
|
||||
"tasks": [
|
||||
{
|
||||
"area": "deadlines",
|
||||
"title": "Jan Loman printer visit",
|
||||
"status": "done",
|
||||
"notes": "Settled.",
|
||||
"due": "2026-10-24",
|
||||
"dueTime": "14:00"
|
||||
},
|
||||
{
|
||||
"area": "deadlines",
|
||||
"title": "Call BackHome about Kibi's chip",
|
||||
"status": "todo",
|
||||
"due": "2026-10-26",
|
||||
"dueTime": "10:00",
|
||||
"tags": ["kibi", "call"]
|
||||
},
|
||||
{
|
||||
"area": "deadlines",
|
||||
"title": "Cargill collective health insurance (OHRA, code 114)",
|
||||
"status": "todo",
|
||||
"notes": "Needs current insurer + premium to compare.",
|
||||
"due": "2026-12-31",
|
||||
"tags": ["insurance"]
|
||||
},
|
||||
{
|
||||
"area": "deadlines",
|
||||
"title": "Reply to mom about insurance info",
|
||||
"status": "todo",
|
||||
"notes": "Waiting since Oct 9.",
|
||||
"tags": ["insurance"]
|
||||
},
|
||||
{
|
||||
"area": "deadlines",
|
||||
"title": "Stop zorgtoeslag from Jan 2027",
|
||||
"status": "todo",
|
||||
"due": "2027-01-01",
|
||||
"reminderAt": "2026-12-07T09:00:00+01:00",
|
||||
"tags": ["admin"]
|
||||
},
|
||||
{
|
||||
"area": "deadlines",
|
||||
"title": "Supermarket product list",
|
||||
"status": "todo",
|
||||
"reminderAt": "2026-10-24T11:50:00+02:00"
|
||||
},
|
||||
{
|
||||
"area": "deadlines",
|
||||
"title": "Book haircut",
|
||||
"status": "todo",
|
||||
"notes": "Around Nov 14.",
|
||||
"due": "2026-11-14"
|
||||
},
|
||||
{
|
||||
"area": "house",
|
||||
"title": "House overview",
|
||||
"status": "in_progress",
|
||||
"flagged": true,
|
||||
"notes": "Applied only to Graaf Hendrikstraat 16 so far."
|
||||
},
|
||||
{
|
||||
"area": "house",
|
||||
"title": "React to house options",
|
||||
"status": "todo",
|
||||
"notes": "Raam €1,475; Burg. van Dijkesingel 214-B; Groenhovenweg 161; Spoorstraat €795; Noothoven van Goorstraat 1-206."
|
||||
},
|
||||
{
|
||||
"area": "house",
|
||||
"title": "Tell Daisy budget, move date, must-haves, Funda/Pararius applications",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "house",
|
||||
"title": "Mijndak Midden-Holland sign-up",
|
||||
"status": "todo",
|
||||
"flagged": true,
|
||||
"notes": "€23 option."
|
||||
},
|
||||
{
|
||||
"area": "house",
|
||||
"title": "Sign up: socialehuurwoning.nl, Vesteda Goudshof, VBT, Viramah, agency Jesse's mom uses",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "house",
|
||||
"title": "House tracker/scraper on the site",
|
||||
"status": "todo",
|
||||
"tags": ["website"]
|
||||
},
|
||||
{
|
||||
"area": "house",
|
||||
"title": "Update address everywhere",
|
||||
"status": "todo",
|
||||
"notes": "Checklist ready."
|
||||
},
|
||||
{
|
||||
"area": "house",
|
||||
"title": "UniFi network for the new place",
|
||||
"status": "todo",
|
||||
"tags": ["network"]
|
||||
},
|
||||
{
|
||||
"area": "kibi",
|
||||
"title": "Food ordered",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "kibi",
|
||||
"title": "Chip scanner ordered",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "kibi",
|
||||
"title": "Chip number found",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "kibi",
|
||||
"title": "Chip owner change at BackHome",
|
||||
"status": "todo",
|
||||
"notes": "Still 'Kwiebus' under E Zwart."
|
||||
},
|
||||
{
|
||||
"area": "kibi",
|
||||
"title": "Call vet about Kibi's patient card",
|
||||
"status": "todo",
|
||||
"tags": ["call"]
|
||||
},
|
||||
{
|
||||
"area": "kibi",
|
||||
"title": "GPS tracker",
|
||||
"status": "todo",
|
||||
"notes": "Dad may have done it already."
|
||||
},
|
||||
{
|
||||
"area": "kibi",
|
||||
"title": "Dash KIBI",
|
||||
"status": "todo",
|
||||
"flagged": true,
|
||||
"notes": "Meaning unclear, ask Stan."
|
||||
},
|
||||
{
|
||||
"area": "admin",
|
||||
"title": "Basic-Fit cancelled",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "admin",
|
||||
"title": "Ohra inboedelverzekering cancelled",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "admin",
|
||||
"title": "Zorgtoeslag income updated for 2026",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "admin",
|
||||
"title": "Move number to Revolut eSIM, then iPad eSIM, cancel Odido",
|
||||
"status": "in_progress"
|
||||
},
|
||||
{
|
||||
"area": "admin",
|
||||
"title": "Move SIM + WhatsApp from iPhone to Pixel 11 Pro XL",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "admin",
|
||||
"title": "Check Oasen and Delta Energie fully cancelled",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "admin",
|
||||
"title": "Cancel other unused subscriptions",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "admin",
|
||||
"title": "Merge banks into Revolut",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "clients",
|
||||
"title": "Quote dack €450 for vet clinic site fix",
|
||||
"status": "done",
|
||||
"tags": ["dack"]
|
||||
},
|
||||
{
|
||||
"area": "clients",
|
||||
"title": "Reply to dack",
|
||||
"status": "done",
|
||||
"tags": ["dack"]
|
||||
},
|
||||
{
|
||||
"area": "clients",
|
||||
"title": "Cloudflare DNS/SSL for dack",
|
||||
"status": "in_progress",
|
||||
"tags": ["dack"],
|
||||
"notes": "Grind-week goal."
|
||||
},
|
||||
{
|
||||
"area": "clients",
|
||||
"title": "Send dack's invoice via Moneybird",
|
||||
"status": "todo",
|
||||
"tags": ["dack"]
|
||||
},
|
||||
{
|
||||
"area": "clients",
|
||||
"title": "Cheap shared WordPress hosting for dack's clients on own VMs",
|
||||
"status": "todo",
|
||||
"tags": ["dack"]
|
||||
},
|
||||
{
|
||||
"area": "clients",
|
||||
"title": "Karolina's Wix site, Master Purpose (NL/PL)",
|
||||
"status": "todo",
|
||||
"tags": ["karolina"]
|
||||
},
|
||||
{
|
||||
"area": "clients",
|
||||
"title": "Wix Studio + Velo/CLI so an AI can edit code",
|
||||
"status": "todo",
|
||||
"tags": ["karolina"]
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "DB reorganised into schemas, staging retired",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Energy tracker (NED data) live",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Health tracker (private) + 15 RIVM vaccinations",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Book tracker, reader, 362 books live",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Reader zoom, 2-page view, scroll/paging",
|
||||
"status": "in_progress"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Money tracker with MoneyMan",
|
||||
"status": "in_progress"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "NED backfill",
|
||||
"status": "in_progress"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Music listening tracker + now-playing",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Task/goal tracker to replace TickTick",
|
||||
"status": "in_progress"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Hardware DB (current PC parts first)",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "List of every organisation holding my data and what they know",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Move to SvelteKit 3",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "website",
|
||||
"title": "Juhee website data",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "AWS cleanup ($0/month)",
|
||||
"status": "done",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Google Cloud cleanup",
|
||||
"status": "done",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "stan-vm hardened (ufw, Tailscale-only SSH)",
|
||||
"status": "done",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Nightly restic backups to B2 + Storage Box",
|
||||
"status": "done",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "WhatsApp bridge on stan-vm",
|
||||
"status": "done",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "gcloud + moneybird-cli on stan-vm",
|
||||
"status": "done",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Domains Squarespace to Cloudflare",
|
||||
"status": "in_progress",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"notes": "Both transfers pending.",
|
||||
"due": "2026-10-29"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Syncthing: Introducer + Auto Accept on Arch and Windows",
|
||||
"status": "in_progress",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Backup follow-up: restic passwords into 1Password",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"due": "2026-10-24",
|
||||
"dueTime": "11:00"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Delete empty B2 bucket",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"due": "2026-10-25"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Rotate weak admin secret + DB password",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"tags": ["security"]
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "1Password overhaul",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"notes": "Imports, 2FA, passkeys, dedupe, delete unused accounts.",
|
||||
"tags": ["security"]
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Moneybird API token, then clean up Moneybird",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Authentik SSO on stan-vm",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "vash-prod memory warnings + finish security audit",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"tags": ["security"]
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Remove old grok-bot devices from Tailscale",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Close empty GCP billing account; decide on AWS 'Vash Software'",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"notes": "AWS part optional."
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Raise WSL RAM",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"notes": "Sees 8 of 16 GB."
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Home server rebuild with ZFS",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Analytics, monitoring, alerts across devices",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Mouse sensitivity cm/360 across devices",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Buy Logitech G840 XL mousepad",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"tags": ["buy"]
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Router and networking setup",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"tags": ["network"]
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "J5 Hexagon research, then move wachtboden",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "Agent-deck + AI/Linux dev setup",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027"
|
||||
},
|
||||
{
|
||||
"area": "setup",
|
||||
"title": "MITM/certificate lab",
|
||||
"status": "todo",
|
||||
"goal": "Perma evolvable setup foundation ready before 2027",
|
||||
"notes": "Idea.",
|
||||
"tags": ["idea"]
|
||||
},
|
||||
{
|
||||
"area": "home",
|
||||
"title": "Unpack boxes; organise batteries and tools; buy thermometer",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "home",
|
||||
"title": "Clean desk, laundry, clothes",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "home",
|
||||
"title": "Connect vinyl player, printer, monitor, switch, osu! pad, tablet, fridge, microwave",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "home",
|
||||
"title": "Hue lights + Home Assistant + Apple Home",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "home",
|
||||
"title": "Fan controller",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "home",
|
||||
"title": "Opa's stuff (WhatsApp, sensor install)",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "health",
|
||||
"title": "Getting in shape with Doc",
|
||||
"status": "in_progress",
|
||||
"notes": "Lesson 1 done; ~2,300–2,400 kcal and 160–190 g protein a day.",
|
||||
"goal": "Get in shape long term"
|
||||
},
|
||||
{
|
||||
"area": "health",
|
||||
"title": "Join the gym my parents use",
|
||||
"status": "todo",
|
||||
"goal": "Get in shape long term"
|
||||
},
|
||||
{
|
||||
"area": "health",
|
||||
"title": "Brush teeth twice a day, shower, hair care",
|
||||
"status": "todo",
|
||||
"tags": ["habit"]
|
||||
},
|
||||
{
|
||||
"area": "health",
|
||||
"title": "Dentist checkup",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "health",
|
||||
"title": "Pick hardware for Withings/Garmin syncs",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "health",
|
||||
"title": "Curaçao vaccines",
|
||||
"status": "todo",
|
||||
"notes": "Via Doc."
|
||||
},
|
||||
{
|
||||
"area": "reading",
|
||||
"title": "Read 'De meeste mensen deugen' (Bregman)",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "reading",
|
||||
"title": "Re-download 'Haves and Have-Nots'",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "reading",
|
||||
"title": "Book questions: duplicates, 6 unknown files, Nick's repo 404",
|
||||
"status": "todo"
|
||||
},
|
||||
{
|
||||
"area": "planning",
|
||||
"title": "Schedule sync (paper agenda, TickTick, Daisy)",
|
||||
"status": "todo",
|
||||
"goal": "Organise goals into a plan"
|
||||
}
|
||||
]
|
||||
}
|
||||
7
data/tasks/ticktick-merges.json
Normal file
7
data/tasks/ticktick-merges.json
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{ "ticktickId": "6ad52191ebcd7400000000ce", "title": "Jan Loman printer visit" },
|
||||
{ "ticktickId": "690d2fd98c824268f297a676", "title": "Router and networking setup" },
|
||||
{ "ticktickId": "6844769df471d103461fa9f1", "title": "Home server rebuild with ZFS" },
|
||||
{ "ticktickId": "6ab6f32759b6090811e91a00", "title": "Dentist checkup" },
|
||||
{ "ticktickId": "68225a8dbfe81104eac49a7a", "title": "Merge banks into Revolut" }
|
||||
]
|
||||
31
deploy/home-collector/README.md
Normal file
31
deploy/home-collector/README.md
Normal file
@@ -0,0 +1,31 @@
|
||||
# home-collector
|
||||
|
||||
Feeds the private dashboard at `/start` (also the browser new-tab page). The app container has
|
||||
no Docker socket and must not get one, so a small stdlib-only Python script runs on the host
|
||||
every minute (systemd **user** timer `home-collector.timer`) and upserts two rows into
|
||||
`home.snapshots`:
|
||||
|
||||
| key | contents |
|
||||
| ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `host` | CPU %, load, uptime, RAM, swap, disk; every Docker container (state, health, restart count, CPU/mem via `sudo docker`); systemd timers (`stan-backup*`, `postgres-backup*`, `disk-guard-*`, `house-scrape`, `b2-cost`, `groceries-import`, `money-sync`, `ned-collector-*`) with last/next run and result; HTTP up/down + latency for each self-hosted service; latest `~/backups-cost/daily/*.json` |
|
||||
| `ntfy` | the last 40 messages (72 h) from the ntfy topics in `~/.config/disk-guard.env` (`NTFY_URL`) and `~/.config/house-scrape.env` (`HOUSE_NTFY_URL`), plus `NTFY_EXTRA`. Only a label is stored, never the topic name. |
|
||||
|
||||
The page reads both rows in the same parallel query batch as tasks and links, so a slow
|
||||
upstream never delays it; a snapshot older than 3 minutes is shown as stale.
|
||||
|
||||
## Setup
|
||||
|
||||
1. A login role that may only upsert snapshots (created once, cluster superuser):
|
||||
`CREATE ROLE home_collector LOGIN CONNECTION LIMIT 3 PASSWORD '…'; GRANT CONNECT ON DATABASE personal TO home_collector;`
|
||||
The `home` migration grants `USAGE` on the schema and `SELECT, INSERT, UPDATE` on
|
||||
`home.snapshots` when the role exists (re-run those two GRANTs by hand if the role is created later).
|
||||
2. `~/.config/home-collector.env` (mode 600):
|
||||
```
|
||||
HOME_DB_URL=postgresql://home_collector:…@127.0.0.1:5440/personal
|
||||
# optional extra topics, e.g. the backup alerts from /etc/stan-backup:
|
||||
# NTFY_EXTRA=backups=https://ntfy.sh/<topic>
|
||||
```
|
||||
3. `bash deploy/home-collector/install.sh` (copies the script to `~/.local/bin/home-collector`
|
||||
and enables the timer). Re-run after editing the script, e.g. to add a service to `SERVICES`.
|
||||
|
||||
Debug: `python3 ~/.local/bin/home-collector --print`, `journalctl --user -u home-collector -n 20`.
|
||||
343
deploy/home-collector/home-collector.py
Normal file
343
deploy/home-collector/home-collector.py
Normal file
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Host-side collector for the /start dashboard on stanrunge.dev.
|
||||
|
||||
Runs every minute as a systemd user timer on the server (home-collector.timer) and upserts
|
||||
two rows into home.snapshots with the limited `home_collector` role:
|
||||
host CPU, RAM, swap, disk, load, uptime, Docker containers (state, health, restarts,
|
||||
CPU/mem), systemd timer results (backups, disk-guard, scrapers) and service checks
|
||||
ntfy the most recent messages from the owner's ntfy topics
|
||||
|
||||
The app container has no Docker socket and never gets one; this script uses the host's
|
||||
passwordless `sudo docker`. ntfy topic names stay on the host: only a label is stored.
|
||||
|
||||
Config: ~/.config/home-collector.env (mode 600)
|
||||
HOME_DB_URL=postgres://home_collector:...@127.0.0.1:5440/<db>
|
||||
NTFY_EXTRA=label=https://ntfy.sh/topic,... (optional extra topics, e.g. backups)
|
||||
Topics are also read from ~/.config/disk-guard.env (NTFY_URL) and
|
||||
~/.config/house-scrape.env (HOUSE_NTFY_URL).
|
||||
Stdlib only. `--print` prints the snapshot instead of writing it; HOME_DB_URL in the
|
||||
environment overrides the env file (e.g. to fill the dev database).
|
||||
"""
|
||||
import concurrent.futures as cf
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
HOME = os.path.expanduser('~')
|
||||
CFG = os.path.join(HOME, '.config')
|
||||
TS = 'stan-ovh.tail665403.ts.net'
|
||||
|
||||
# (name, link shown on the dashboard, url checked from the host or None = link only)
|
||||
SERVICES = [
|
||||
('Website', 'https://stanrunge.dev', 'http://127.0.0.1:3000/login'),
|
||||
('Gitea', 'https://git.stanrunge.dev', 'http://127.0.0.1:3010/'),
|
||||
('Immich', 'https://photos.stanrunge.dev', 'http://127.0.0.1:2283/'),
|
||||
('Umami', 'https://analytics.stanrunge.dev', 'http://127.0.0.1:3110/'),
|
||||
('pgAdmin', 'https://pgadmin.stanrunge.dev', 'http://127.0.0.1:5050/misc/ping'),
|
||||
('Vash esports', 'https://esports.vash.software', 'http://127.0.0.1:3100/'),
|
||||
('Vash staging', 'https://esports-staging.vash.software', 'http://127.0.0.1:3101/'),
|
||||
('Dozzle logs', 'https://logs.vash.software', 'http://127.0.0.1:3111/'),
|
||||
('Beeriokart', 'https://beeriokart.stanrunge.dev', 'http://127.0.0.1:3003/'),
|
||||
('Stuhee', 'https://stuhee.stanrunge.dev', 'http://127.0.0.1:3004/'),
|
||||
('Dev site', 'https://dev.stanrunge.dev', 'http://127.0.0.1:3002/'),
|
||||
('Netdata', f'http://{TS}:19999', 'http://127.0.0.1:19999/api/v1/info'),
|
||||
('Portainer', f'https://{TS}:9443', 'https://127.0.0.1:9443/'),
|
||||
('ntfy', 'https://ntfy.sh/app', 'https://ntfy.sh/v1/health'),
|
||||
]
|
||||
|
||||
# (label, unit, scope)
|
||||
TIMERS = [
|
||||
('Backup (restic → B2)', 'stan-backup', 'system'),
|
||||
('Backup maintenance', 'stan-backup-maintain', 'system'),
|
||||
('Postgres dump', 'postgres-backup', 'system'),
|
||||
('Postgres dump prune', 'postgres-backup-prune', 'system'),
|
||||
('Disk guard check', 'disk-guard-check', 'user'),
|
||||
('Disk guard cleanup', 'disk-guard-cleanup', 'user'),
|
||||
('House scrape', 'house-scrape', 'user'),
|
||||
('B2 cost', 'b2-cost', 'user'),
|
||||
('Groceries import', 'groceries-import', 'user'),
|
||||
('Money sync', 'money-sync', 'user'),
|
||||
('Energy (NED) hourly', 'ned-collector-hourly', 'user'),
|
||||
('Energy (NED) daily', 'ned-collector-daily', 'user'),
|
||||
]
|
||||
|
||||
|
||||
def read_env(path):
|
||||
out = {}
|
||||
try:
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
k, v = line.split('=', 1)
|
||||
out[k.strip()] = v.strip().strip('"').strip("'")
|
||||
except OSError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def run(cmd, timeout=15):
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||||
return r.stdout if r.returncode == 0 else ''
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
def cpu_times():
|
||||
with open('/proc/stat') as f:
|
||||
parts = [int(x) for x in f.readline().split()[1:]]
|
||||
idle = parts[3] + (parts[4] if len(parts) > 4 else 0)
|
||||
return idle, sum(parts[:8])
|
||||
|
||||
|
||||
def meminfo():
|
||||
m = {}
|
||||
with open('/proc/meminfo') as f:
|
||||
for line in f:
|
||||
k, v = line.split(':', 1)
|
||||
m[k] = int(v.split()[0]) * 1024
|
||||
return m
|
||||
|
||||
|
||||
def system_stats():
|
||||
i1, t1 = cpu_times()
|
||||
time.sleep(1)
|
||||
i2, t2 = cpu_times()
|
||||
cpu = 100.0 * (1 - (i2 - i1) / max(1, t2 - t1))
|
||||
m = meminfo()
|
||||
with open('/proc/uptime') as f:
|
||||
uptime = float(f.read().split()[0])
|
||||
load = list(os.getloadavg())
|
||||
disks = []
|
||||
for mount in ('/',):
|
||||
u = shutil.disk_usage(mount)
|
||||
disks.append({'mount': mount, 'total': u.total, 'used': u.used, 'free': u.free,
|
||||
'pct': round(100 * u.used / u.total, 1)})
|
||||
return {
|
||||
'hostname': socket.gethostname(),
|
||||
'cpus': os.cpu_count(),
|
||||
'cpuPct': round(cpu, 1),
|
||||
'load': [round(x, 2) for x in load],
|
||||
'uptimeS': int(uptime),
|
||||
'mem': {'total': m['MemTotal'], 'available': m['MemAvailable'],
|
||||
'used': m['MemTotal'] - m['MemAvailable']},
|
||||
'swap': {'total': m.get('SwapTotal', 0), 'used': m.get('SwapTotal', 0) - m.get('SwapFree', 0)},
|
||||
'disks': disks,
|
||||
}
|
||||
|
||||
|
||||
def containers():
|
||||
ids = run(['sudo', '-n', 'docker', 'ps', '-aq']).split()
|
||||
if not ids:
|
||||
return []
|
||||
try:
|
||||
info = json.loads(run(['sudo', '-n', 'docker', 'inspect', *ids]) or '[]')
|
||||
except ValueError:
|
||||
info = []
|
||||
stats = {}
|
||||
for line in run(['sudo', '-n', 'docker', 'stats', '--no-stream', '--format', '{{json .}}'], 20).splitlines():
|
||||
try:
|
||||
s = json.loads(line)
|
||||
stats[s['Name']] = {'cpu': s.get('CPUPerc'), 'mem': s.get('MemUsage', '').split(' / ')[0]}
|
||||
except (ValueError, KeyError):
|
||||
pass
|
||||
out = []
|
||||
for c in info:
|
||||
st = c.get('State', {})
|
||||
name = c.get('Name', '').lstrip('/')
|
||||
out.append({
|
||||
'name': name,
|
||||
'image': c.get('Config', {}).get('Image', '').split('@')[0][:80],
|
||||
'state': st.get('Status'),
|
||||
'health': (st.get('Health') or {}).get('Status'),
|
||||
'restarts': c.get('RestartCount', 0),
|
||||
'exitCode': st.get('ExitCode'),
|
||||
'startedAt': st.get('StartedAt'),
|
||||
'cpu': stats.get(name, {}).get('cpu'),
|
||||
'mem': stats.get(name, {}).get('mem'),
|
||||
})
|
||||
out.sort(key=lambda c: c['name'])
|
||||
return out
|
||||
|
||||
|
||||
def unit_props(unit, scope, props):
|
||||
cmd = ['systemctl'] + (['--user'] if scope == 'user' else []) + [
|
||||
'show', unit, '-p', ','.join(props)]
|
||||
d = {}
|
||||
for line in run(cmd).splitlines():
|
||||
if '=' in line:
|
||||
k, v = line.split('=', 1)
|
||||
d[k] = v
|
||||
return d
|
||||
|
||||
|
||||
def list_timers(scope):
|
||||
cmd = ['systemctl'] + (['--user'] if scope == 'user' else []) + ['list-timers', '--all', '-o', 'json']
|
||||
try:
|
||||
return {t['unit']: t for t in json.loads(run(cmd) or '[]')}
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
def sec(usec):
|
||||
try:
|
||||
n = int(usec or 0)
|
||||
return n // 1_000_000 if n > 0 else None
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def timers():
|
||||
lists = {'user': list_timers('user'), 'system': list_timers('system')}
|
||||
out = []
|
||||
for label, unit, scope in TIMERS + [('Home collector', 'home-collector', 'user')]:
|
||||
t = lists[scope].get(unit + '.timer')
|
||||
if not t:
|
||||
continue
|
||||
s = unit_props(unit + '.service', scope, ['ActiveState', 'Result', 'ExecMainStatus'])
|
||||
out.append({
|
||||
'label': label, 'unit': unit, 'scope': scope,
|
||||
'last': sec(t.get('last')),
|
||||
'next': sec(t.get('next')),
|
||||
'running': s.get('ActiveState') in ('activating', 'active', 'deactivating'),
|
||||
'result': s.get('Result'),
|
||||
'exitStatus': int(s.get('ExecMainStatus') or 0),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
_insecure = ssl.create_default_context()
|
||||
_insecure.check_hostname = False
|
||||
_insecure.verify_mode = ssl.CERT_NONE
|
||||
|
||||
|
||||
def check(service):
|
||||
name, link, url = service
|
||||
res = {'name': name, 'url': link, 'ok': None, 'code': None, 'ms': None}
|
||||
if not url:
|
||||
return res
|
||||
start = time.monotonic()
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={'User-Agent': 'home-collector'})
|
||||
with urllib.request.urlopen(req, timeout=5, context=_insecure) as r:
|
||||
r.read(256)
|
||||
res['code'] = r.status
|
||||
except urllib.error.HTTPError as e:
|
||||
res['code'] = e.code
|
||||
except Exception:
|
||||
res['code'] = 0
|
||||
res['ms'] = round((time.monotonic() - start) * 1000)
|
||||
res['ok'] = bool(res['code']) and res['code'] < 500
|
||||
return res
|
||||
|
||||
|
||||
def services():
|
||||
with cf.ThreadPoolExecutor(8) as ex:
|
||||
return list(ex.map(check, SERVICES))
|
||||
|
||||
|
||||
def b2_cost():
|
||||
d = os.path.join(HOME, 'backups-cost', 'daily')
|
||||
try:
|
||||
files = sorted(f for f in os.listdir(d) if f.endswith('.json'))
|
||||
if files:
|
||||
with open(os.path.join(d, files[-1])) as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def ntfy_topics(cfg):
|
||||
topics = []
|
||||
for label, path, key in (('disk-guard', 'disk-guard.env', 'NTFY_URL'),
|
||||
('houses', 'house-scrape.env', 'HOUSE_NTFY_URL')):
|
||||
url = read_env(os.path.join(CFG, path)).get(key)
|
||||
if url:
|
||||
topics.append((label, url))
|
||||
for item in (cfg.get('NTFY_EXTRA') or '').split(','):
|
||||
if '=' in item:
|
||||
label, url = item.split('=', 1)
|
||||
topics.append((label.strip(), url.strip()))
|
||||
return topics
|
||||
|
||||
|
||||
def ntfy(cfg):
|
||||
msgs, seen, errors = [], set(), []
|
||||
for label, url in ntfy_topics(cfg):
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
try:
|
||||
req = urllib.request.Request(url.rstrip('/') + '/json?poll=1&since=72h',
|
||||
headers={'User-Agent': 'home-collector'})
|
||||
with urllib.request.urlopen(req, timeout=8) as r:
|
||||
for line in r.read().decode('utf-8', 'replace').splitlines():
|
||||
try:
|
||||
m = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if m.get('event') != 'message':
|
||||
continue
|
||||
msgs.append({'id': m.get('id'), 'topic': label, 'time': m.get('time'),
|
||||
'title': (m.get('title') or '')[:200],
|
||||
'message': (m.get('message') or '')[:600],
|
||||
'priority': m.get('priority', 3), 'tags': m.get('tags') or [],
|
||||
'click': m.get('click')})
|
||||
except Exception:
|
||||
errors.append(label)
|
||||
msgs.sort(key=lambda m: m['time'] or 0, reverse=True)
|
||||
return {'messages': msgs[:40], 'topics': [t for t, _ in ntfy_topics(cfg)], 'errors': errors}
|
||||
|
||||
|
||||
def write(db_url, rows):
|
||||
tag = 'j' + secrets.token_hex(8)
|
||||
sql = ['BEGIN;']
|
||||
for key, data in rows.items():
|
||||
payload = json.dumps(data, separators=(',', ':'))
|
||||
sql.append(
|
||||
f"INSERT INTO home.snapshots (key, data, collected_at) VALUES ('{key}', ${tag}${payload}${tag}$::jsonb, now()) "
|
||||
"ON CONFLICT (key) DO UPDATE SET data = excluded.data, collected_at = excluded.collected_at;")
|
||||
sql.append('COMMIT;')
|
||||
r = subprocess.run(['psql', '-X', '-q', '-v', 'ON_ERROR_STOP=1', db_url], input='\n'.join(sql),
|
||||
capture_output=True, text=True, timeout=20)
|
||||
if r.returncode != 0:
|
||||
# psql errors never contain the password; the URL is not echoed.
|
||||
sys.stderr.write(r.stderr[-500:])
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
cfg = read_env(os.path.join(CFG, 'home-collector.env'))
|
||||
with cf.ThreadPoolExecutor(4) as ex:
|
||||
f_sys = ex.submit(system_stats)
|
||||
f_ct = ex.submit(containers)
|
||||
f_svc = ex.submit(services)
|
||||
f_ntfy = ex.submit(ntfy, cfg)
|
||||
host = f_sys.result()
|
||||
host['containers'] = f_ct.result()
|
||||
host['services'] = f_svc.result()
|
||||
host['timers'] = timers()
|
||||
host['b2'] = b2_cost()
|
||||
notes = f_ntfy.result()
|
||||
rows = {'host': host, 'ntfy': notes}
|
||||
if '--print' in sys.argv:
|
||||
print(json.dumps(rows, indent=1))
|
||||
return
|
||||
url = os.environ.get('HOME_DB_URL') or cfg.get('HOME_DB_URL')
|
||||
if not url:
|
||||
sys.exit('HOME_DB_URL missing in ~/.config/home-collector.env')
|
||||
write(url, rows)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
11
deploy/home-collector/home-collector.service
Normal file
11
deploy/home-collector/home-collector.service
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Snapshot server stats, containers, timers and ntfy for stanrunge.dev/start
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
ConditionPathExists=%h/.config/home-collector.env
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 %h/.local/bin/home-collector
|
||||
TimeoutStartSec=50
|
||||
Nice=10
|
||||
10
deploy/home-collector/home-collector.timer
Normal file
10
deploy/home-collector/home-collector.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run home-collector every minute (stanrunge.dev/start)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
12
deploy/home-collector/install.sh
Executable file
12
deploy/home-collector/install.sh
Executable file
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install/refresh the /start collector as a systemd user timer (run as stan on the server).
|
||||
# Needs ~/.config/home-collector.env with HOME_DB_URL for the home_collector role (see README).
|
||||
set -euo pipefail
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
install -D -m 755 "$here/home-collector.py" "$HOME/.local/bin/home-collector"
|
||||
install -D -m 644 "$here/home-collector.service" "$HOME/.config/systemd/user/home-collector.service"
|
||||
install -D -m 644 "$here/home-collector.timer" "$HOME/.config/systemd/user/home-collector.timer"
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now home-collector.timer
|
||||
systemctl --user start home-collector.service || true
|
||||
systemctl --user --no-pager status home-collector.service | head -5
|
||||
66
deploy/music-collector/README.md
Normal file
66
deploy/music-collector/README.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# music (self-hosted Last.fm)
|
||||
|
||||
`/trackers/music` (stats) and the "listening now" card on the homepage read the `music`
|
||||
schema in the `personal` database:
|
||||
|
||||
| table / view | contents |
|
||||
| ------------------------- | ----------------------------------------------------------------------------------------------- |
|
||||
| `artists`, `albums` | Spotify ids when known (imports are matched by name), album art URLs (640 px + small) |
|
||||
| `tracks`, `track_artists` | `match_key` = normalised `artist\|title` (`src/lib/music/normalize.ts`) for cross-source dedupe |
|
||||
| `plays` | one row per play: `source` spotify / lastfm / youtube / local, `origin` api / export / import |
|
||||
| `plays_deduped` (view) | `plays` minus Last.fm scrobbles that Spotify also has (what every stat reads) |
|
||||
| `now_playing` | row 1, refreshed every minute |
|
||||
| `spotify_auth` | row 1, refresh token + poller state. **Not** granted to `personal_ro` |
|
||||
| `import_state` | resumable backfill progress (`lastfm:<user>`) |
|
||||
|
||||
## Spotify (live)
|
||||
|
||||
1. In the Spotify developer dashboard (the existing app, `SPOTIFY_CLIENT_ID`) add the redirect
|
||||
URIs `https://stanrunge.dev/api/spotify/callback` and
|
||||
`https://dev.stanrunge.dev/api/spotify/callback`.
|
||||
2. Logged in as the owner, open <https://stanrunge.dev/api/spotify/connect> once. Scopes:
|
||||
`user-read-currently-playing user-read-recently-played user-read-playback-state user-top-read user-library-read playlist-read-private`.
|
||||
3. That's it: the **app itself** polls every minute (`src/lib/server/music/poller.ts`, started
|
||||
from `hooks.server.ts` `init`) with its own role and Spotify credentials. Status (last poll,
|
||||
last error) is shown to the owner on `/trackers/music`.
|
||||
|
||||
### Optional: poll from the host instead
|
||||
|
||||
Only if the poller should keep running independent of the app container. Needs its own role:
|
||||
|
||||
```sh
|
||||
sudo docker exec -i postgres-db-1 psql -U postgres -d personal <<'SQL'
|
||||
CREATE ROLE music_collector LOGIN CONNECTION LIMIT 3 PASSWORD '<pick one>';
|
||||
GRANT CONNECT ON DATABASE personal TO music_collector;
|
||||
GRANT USAGE ON SCHEMA music TO music_collector;
|
||||
GRANT SELECT, INSERT, UPDATE ON music.artists, music.albums, music.tracks, music.track_artists,
|
||||
music.plays, music.now_playing, music.import_state TO music_collector;
|
||||
GRANT SELECT, UPDATE ON music.spotify_auth TO music_collector;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA music TO music_collector;
|
||||
SQL
|
||||
```
|
||||
|
||||
`~/.config/music-collector.env` (mode 600): `MUSIC_DB_URL=postgresql://music_collector:…@127.0.0.1:5440/personal`,
|
||||
`SPOTIFY_CLIENT_ID=…`, `SPOTIFY_CLIENT_SECRET=…`. Then `bash deploy/music-collector/install.sh --enable`
|
||||
and set `MUSIC_POLLER=off` in prod.env. (The music migration grants the same when the role
|
||||
already exists.) Debug: `journalctl --user -u music-collector -n 20`.
|
||||
|
||||
## Backfills
|
||||
|
||||
Both run from the deployed image with the app role, so no extra credentials:
|
||||
|
||||
```sh
|
||||
tag=$(sudo sed -n 's/^IMAGE_TAG=//p' /opt/personal-website/.env)
|
||||
# Last.fm (needs an API key from https://www.last.fm/api/account/create); resumable, re-run any time
|
||||
sudo docker run --rm --network backend --env-file /etc/personal-website/prod.env \
|
||||
-e LASTFM_API_KEY=<key> git.stanrunge.dev/stan/website:$tag bun scripts/music-import-lastfm.ts --user stanrunge
|
||||
# Spotify Extended Streaming History (unzipped export in ~/spotify-export)
|
||||
sudo docker run --rm --network backend --env-file /etc/personal-website/prod.env \
|
||||
-v ~/spotify-export:/data:ro git.stanrunge.dev/stan/website:$tag bun scripts/music-import-history.ts /data
|
||||
```
|
||||
|
||||
Dedupe: within one source a play within 60 s of the same `match_key` is skipped on insert;
|
||||
across sources `plays_deduped` hides a Last.fm scrobble when a Spotify play of the same
|
||||
`match_key` ended between 3 min before and track length + 3 min after the scrobble time
|
||||
(Last.fm records when a track started, Spotify when it ended). Listening time uses
|
||||
`ms_played` when known, else the track length.
|
||||
17
deploy/music-collector/install.sh
Executable file
17
deploy/music-collector/install.sh
Executable file
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the optional host-side Spotify poller as a systemd user timer (run as stan on the
|
||||
# server). Copies the units; enables the timer only with --enable (needs
|
||||
# ~/.config/music-collector.env, see README.md). Normally not needed: the app polls itself.
|
||||
set -euo pipefail
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
install -D -m 644 "$here/music-collector.service" "$HOME/.config/systemd/user/music-collector.service"
|
||||
install -D -m 644 "$here/music-collector.timer" "$HOME/.config/systemd/user/music-collector.timer"
|
||||
systemctl --user daemon-reload
|
||||
if [ "${1:-}" = --enable ]; then
|
||||
test -f "$HOME/.config/music-collector.env" || { echo 'Missing ~/.config/music-collector.env' >&2; exit 1; }
|
||||
systemctl --user enable --now music-collector.timer
|
||||
systemctl --user start music-collector.service || true
|
||||
systemctl --user --no-pager status music-collector.service | head -5
|
||||
else
|
||||
echo 'Installed (not enabled). Enable with: bash deploy/music-collector/install.sh --enable'
|
||||
fi
|
||||
15
deploy/music-collector/music-collector.service
Normal file
15
deploy/music-collector/music-collector.service
Normal file
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=Spotify scrobbler poll (now playing + recently played -> music.*) for stanrunge.dev/trackers/music
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
ConditionPathExists=%h/.config/music-collector.env
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
# A checkout of stan/website on main. Not the live dev tree.
|
||||
WorkingDirectory=%h/dev/website-music
|
||||
# MUSIC_DB_URL (music_collector role), SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET; mode 600.
|
||||
EnvironmentFile=%h/.config/music-collector.env
|
||||
ExecStart=/usr/bin/bun scripts/music-poll.ts
|
||||
TimeoutStartSec=50
|
||||
Nice=10
|
||||
10
deploy/music-collector/music-collector.timer
Normal file
10
deploy/music-collector/music-collector.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Spotify scrobbler poll every minute (optional; the app polls by itself)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=45s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
13
deploy/systemd/house-scrape.service
Normal file
13
deploy/systemd/house-scrape.service
Normal file
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=House tracker nightly scrape (Kamernet, Vesteda -> house.listings)
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
# A checkout of stan/website on main. Not the live dev tree.
|
||||
WorkingDirectory=%h/dev/website-houses
|
||||
# DB_URL (house_scraper role: house schema only), optional HOUSE_NTFY_URL; mode 600, never committed.
|
||||
EnvironmentFile=%h/.config/house-scrape.env
|
||||
ExecStart=/usr/bin/bun scripts/house-scrape.ts
|
||||
TimeoutStartSec=900
|
||||
10
deploy/systemd/house-scrape.timer
Normal file
10
deploy/systemd/house-scrape.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=House tracker scrape, once a night
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:40:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=1200
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
13
deploy/systemd/money-sync.service
Normal file
13
deploy/systemd/money-sync.service
Normal file
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Money tracker bank sync (Enable Banking -> money.transactions)
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
# A checkout of stan/website (main once merged). Not the live dev tree.
|
||||
WorkingDirectory=%h/dev/website-money
|
||||
# DB_URL (app role), ENABLEBANKING_APP_ID, ENABLEBANKING_KEY_PATH; mode 600, never committed.
|
||||
EnvironmentFile=%h/.config/money-sync.env
|
||||
ExecStart=/usr/bin/bun scripts/bank-sync.ts
|
||||
TimeoutStartSec=600
|
||||
10
deploy/systemd/money-sync.timer
Normal file
10
deploy/systemd/money-sync.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Money tracker bank sync, 4x per day (PSD2 allows ~4 unattended fetches/day)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 06,11,16,21:20:00 Europe/Amsterdam
|
||||
RandomizedDelaySec=300
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -1,7 +1,54 @@
|
||||
# House tracker
|
||||
|
||||
## Nightly scrape on stan-vm (2026-10)
|
||||
|
||||
`scripts/house-scrape.ts` runs once a night from a systemd user timer
|
||||
(`deploy/systemd/house-scrape.{service,timer}`, 03:40 Amsterdam + up to 20 min jitter) and
|
||||
writes `house.listings` directly with the limited `house_scraper` role
|
||||
(`~/.config/house-scrape.env`, mode 600). One request at a time, 6–10 s apart, honest
|
||||
User-Agent, robots.txt-allowed paths only. A 403/429/challenge stops that source for the
|
||||
night and is logged as `challenged`; nothing is retried or worked around.
|
||||
|
||||
| source | how | status |
|
||||
| ---------------------- | ------------------------------------------------------------- | ------------------------------------ |
|
||||
| Kamernet | `/huren/huurwoningen-<city>` pages, `__NEXT_DATA__` JSON | scraped (10 cities around Gouda) |
|
||||
| Vesteda | `POST /api/units/search/facet` (what their site calls), 20 km | scraped; rented units count as gone |
|
||||
| Pararius | Cloudflare 403 from the Hetzner IP | not scraped here; home worker below |
|
||||
| huurwoningen.nl | same Cloudflare 403 (Pararius group) | skipped |
|
||||
| Funda | bot-check page ("Je bent bijna op de pagina…") | skipped |
|
||||
| Huurstunt | Cloudflare challenge | skipped |
|
||||
| Mijndak Midden-Holland | OutSystems app behind a login, no public listing feed | skipped; use their own e-mail alerts |
|
||||
|
||||
Same home on two sources: `address_key` (street + number + city, normalised, see
|
||||
`src/lib/house-tracker/address.ts`) links the copy to the original via `duplicate_of`; a
|
||||
manual entry wins. The page shows one row with "also on …" links. Kamernet never shows house
|
||||
numbers, so its listings are never merged. Alerts: new non-copy listings matching an alert
|
||||
rule go to `HOUSE_NTFY_URL` (its own topic, max 5 per night plus one summary); the first run
|
||||
of a source only seeds.
|
||||
|
||||
No caps at scrape time: every listing in the region is stored (Kamernet city pages, Vesteda
|
||||
with an open rent range, Pararius without a price band). The €2,000 / 2-bedroom / 50 m²
|
||||
limits are only the page's default filters (clear a field to drop it) and the alert rule.
|
||||
|
||||
Liveability: every listing gets a Leefbaarometer 3.0 score (BZK, measurement 2024, published
|
||||
July 2026; `src/lib/house-tracker/liveability.ts`). The address is geocoded with PDOK
|
||||
Locatieserver (exact address, else 6-char postcode, else street centroid → marked approx.),
|
||||
then the 100 m grid cell and the buurt containing the point are read from the official WFS
|
||||
(`geo.leefbaarometer.nl/lbm3`, layers `clippedgridscore24` / `buurtscore24`). The cell score
|
||||
is used when there is one (and the location is not street-level), else the buurt. Stored on
|
||||
the row: `lbm_class` (1-9, zeer onvoldoende … uitstekend), `lbm_onv_class` (overlast &
|
||||
onveiligheid, 1-9 where 5 = Dutch average), buurt, level, precision, year, and all
|
||||
sub-scores in `lbm_detail`. Scored by the nightly scrape (new and changed addresses), the
|
||||
CLI and the page's add form; backfill or re-score with `scripts/house-liveability.ts
|
||||
[--force]`. For a new measurement, bump `LBM_YEAR` and run it with `--force`.
|
||||
|
||||
Manual listings: `scripts/house-listing.ts` (CLI) or the form on the page. Against prod use
|
||||
`scripts/house-db.sh <command> …`, which runs the CLI as `house_cli` (SELECT/INSERT/UPDATE/
|
||||
DELETE on house.listings, SELECT/INSERT on house.price_history, nothing else) from
|
||||
`~/.config/house-cli.env`.
|
||||
|
||||
Personal tool: collects Dutch housing listings into Postgres, shows them on
|
||||
`/projects/house-tracker` (admin only) and sends an alert when a new listing
|
||||
`/trackers/houses` (owner only, 404 for everyone else; the old URL redirects) and sends an alert when a new listing
|
||||
matches a rule. Only Pararius for now; other sources plug in the same way.
|
||||
|
||||
## How it fits together
|
||||
@@ -176,3 +223,29 @@ fixture still covers pagination and price on request. Rental `priceEur` is the
|
||||
base/listed rent used by the search filter; a separately shown total can be
|
||||
higher (Raam: €1,475 base, €1,750 total). If Pararius changes markup, the worker
|
||||
reports `parse-error` in the worker log on the page.
|
||||
|
||||
## Adding listings by hand
|
||||
|
||||
`scripts/house-listing.ts` writes `house.listings` directly (links from chats,
|
||||
screenshots). No restart or deploy is needed for new rows.
|
||||
|
||||
```sh
|
||||
# DATABASE_URL or DB_URL must be set; locally: bun --env-file=.env run house …
|
||||
bun run house add --url https://vbtverhuurmakelaars.nl/woning/gouda-vest-7-d \
|
||||
--city Gouda --price "€ 1.422,- p.m." --area 84 --rooms 3 --bedrooms 2 --energy-label A
|
||||
echo '{"url":"…","city":"Gouda","price_eur":1350,"area_m2":81}' | bun run house add --json -
|
||||
bun run house add --street "Vest 7-D" --city Gouda --price 1422 # no link (screenshot)
|
||||
bun run house update <id|url> --status viewing --notes "za 14:00" # "" clears a field
|
||||
bun run house list [--all] [--city gouda] [--offer buy] [--json]
|
||||
bun run house remove <id|url> [--hard] # soft remove marks it gone; restore undoes it
|
||||
```
|
||||
|
||||
- `source` is the site (`funda`, `pararius`, `vbt`, `vesteda`, … or `manual`
|
||||
without a link); `source_id` comes from the URL (Funda's number, Pararius'
|
||||
worker key, otherwise the path), or a hash of the address. Adding the same
|
||||
link again updates that row and clears `removed_at`.
|
||||
- Fields follow `listingDetailSchema` (camelCase, snake_case or the flag names)
|
||||
and are lenient: "€ 1.422,- p.m." and "84 m²" are parsed. Only given fields are
|
||||
written. `city_slug` is derived from `city` in Pararius' slug style.
|
||||
- `--first-seen` sets when it was shared; price changes go to `price_history`.
|
||||
- Manual rows use `first_seen_lane = 'manual'` and never trigger alerts.
|
||||
|
||||
@@ -4,7 +4,6 @@ import { defineConfig } from 'drizzle-kit';
|
||||
// - lists: runtime tables created through meta.create_list() (owned by personal_ddl)
|
||||
// - legacy: pre-reorg tables kept read-only for rollback
|
||||
// - energy: owned by ned-collector
|
||||
// - hardware: not modelled yet
|
||||
// Migrations run as the schema owner (`personal`), never as the app role.
|
||||
export default defineConfig({
|
||||
dialect: 'postgresql',
|
||||
@@ -20,7 +19,17 @@ export default defineConfig({
|
||||
'animaldex',
|
||||
'site',
|
||||
'meta',
|
||||
'health'
|
||||
'health',
|
||||
'books',
|
||||
'tasks',
|
||||
'groceries',
|
||||
'home',
|
||||
'cars',
|
||||
'music',
|
||||
'hardware',
|
||||
'planes',
|
||||
'wildlife',
|
||||
'trains'
|
||||
],
|
||||
migrations: { schema: 'drizzle', table: '__drizzle_migrations' },
|
||||
verbose: true,
|
||||
|
||||
65
drizzle/20261024003809_money_bank_sync/migration.sql
Normal file
65
drizzle/20261024003809_money_bank_sync/migration.sql
Normal file
@@ -0,0 +1,65 @@
|
||||
CREATE TABLE "money"."bank_sessions" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"user_id" text NOT NULL,
|
||||
"provider" text DEFAULT 'enablebanking' NOT NULL,
|
||||
"session_id" text NOT NULL CONSTRAINT "bank_sessions_session_id_key" UNIQUE,
|
||||
"aspsp_name" text NOT NULL,
|
||||
"aspsp_country" text NOT NULL,
|
||||
"psu_type" text,
|
||||
"status" text DEFAULT 'authorized' NOT NULL,
|
||||
"valid_until" timestamp with time zone,
|
||||
"accounts" jsonb DEFAULT '[]' NOT NULL,
|
||||
"last_synced_at" timestamp with time zone,
|
||||
"last_error" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "bank_sessions_status_check" CHECK (status IN ('authorized','expired','revoked','error'))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "scope" text DEFAULT 'personal' NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "owner" text;--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "iban" text;--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "external_uid" text;--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "provider" text DEFAULT 'manual' NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD COLUMN "external_id" text;--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD COLUMN "source" text DEFAULT 'manual' NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD COLUMN "booking_status" text;--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD COLUMN "counterparty" text;--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD COLUMN "category" text;--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD COLUMN "raw" jsonb;--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD COLUMN "value_date" date;--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD CONSTRAINT "accounts_external_uid_key" UNIQUE("external_uid");--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD CONSTRAINT "transactions_account_external_id_key" UNIQUE("account_id","external_id");--> statement-breakpoint
|
||||
CREATE INDEX "bank_sessions_user_id_index" ON "money"."bank_sessions" ("user_id");--> statement-breakpoint
|
||||
ALTER TABLE "money"."bank_sessions" ADD CONSTRAINT "bank_sessions_user_id_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "auth"."user"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD CONSTRAINT "accounts_scope_check" CHECK (scope IN ('personal','joint'));--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD CONSTRAINT "accounts_provider_check" CHECK (provider IN ('manual','enablebanking'));--> statement-breakpoint
|
||||
ALTER TABLE "money"."transactions" ADD CONSTRAINT "transactions_source_check" CHECK (source IN ('manual','csv','enablebanking'));--> statement-breakpoint
|
||||
-- Hand-written below this line (not generated by drizzle-kit).
|
||||
-- Backfill: the shared account with Juhee is the only joint one; everything else stays personal.
|
||||
UPDATE "money"."accounts" SET "scope" = 'joint', "owner" = 'Stan & Juhee' WHERE lower("name") = 'joint';
|
||||
--> statement-breakpoint
|
||||
UPDATE "money"."accounts" SET "owner" = 'Stan' WHERE "scope" = 'personal' AND "owner" IS NULL;
|
||||
--> statement-breakpoint
|
||||
-- updated_at trigger, same as every other core table (07_triggers_meta_grants.sql).
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
|
||||
CREATE TRIGGER touch_updated_at BEFORE UPDATE ON money.bank_sessions
|
||||
FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at();
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
-- Grants only run on prod (migrations run as the schema owner `personal`); the dev DB
|
||||
-- (owned by personal_dev) carries no prod grants, matching db-refresh-dev.sh. Default
|
||||
-- privileges on `money` already cover new tables; this is explicit for clarity.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON money.bank_sessions TO personal_app;
|
||||
GRANT USAGE, SELECT ON SEQUENCE money.bank_sessions_id_seq TO personal_app;
|
||||
END IF;
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT SELECT ON money.bank_sessions TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
11543
drizzle/20261024003809_money_bank_sync/snapshot.json
Normal file
11543
drizzle/20261024003809_money_bank_sync/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
3
drizzle/20261024011919_money_bank_balance/migration.sql
Normal file
3
drizzle/20261024011919_money_bank_balance/migration.sql
Normal file
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_cents" integer;--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_type" text;--> statement-breakpoint
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "bank_balance_at" timestamp with time zone;
|
||||
12076
drizzle/20261024011919_money_bank_balance/snapshot.json
Normal file
12076
drizzle/20261024011919_money_bank_balance/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
194
drizzle/20261024013208_books/migration.sql
Normal file
194
drizzle/20261024013208_books/migration.sql
Normal file
@@ -0,0 +1,194 @@
|
||||
CREATE SCHEMA "books";
|
||||
--> statement-breakpoint
|
||||
CREATE TYPE "books"."metadata_status" AS ENUM('pending', 'matched', 'low_confidence', 'not_found', 'manual');--> statement-breakpoint
|
||||
CREATE TYPE "books"."reading_status" AS ENUM('want_to_read', 'reading', 'finished', 'abandoned');--> statement-breakpoint
|
||||
CREATE TABLE "books"."files" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"work_id" integer NOT NULL,
|
||||
"path" text NOT NULL,
|
||||
"format" text NOT NULL,
|
||||
"size_bytes" bigint NOT NULL,
|
||||
"sha256" text NOT NULL,
|
||||
"md5" text,
|
||||
"source" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "books"."notes" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"work_id" integer NOT NULL,
|
||||
"file_id" integer,
|
||||
"locator" text,
|
||||
"page" integer,
|
||||
"highlight" text,
|
||||
"note" text,
|
||||
"color" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "books"."progress" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"work_id" integer NOT NULL,
|
||||
"file_id" integer,
|
||||
"locator" text,
|
||||
"page" integer,
|
||||
"percent" real DEFAULT 0 NOT NULL,
|
||||
"started_at" timestamp with time zone,
|
||||
"finished_at" timestamp with time zone,
|
||||
"status" "books"."reading_status" DEFAULT 'want_to_read'::"books"."reading_status" NOT NULL,
|
||||
"rating" smallint,
|
||||
"last_read_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "progress_percent_check" CHECK ("percent" between 0 and 100),
|
||||
CONSTRAINT "progress_rating_check" CHECK ("rating" is null or "rating" between 1 and 5)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "books"."sessions" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"work_id" integer NOT NULL,
|
||||
"file_id" integer,
|
||||
"started_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"ended_at" timestamp with time zone,
|
||||
"start_percent" real,
|
||||
"end_percent" real,
|
||||
"start_page" integer,
|
||||
"end_page" integer,
|
||||
"pages_read" integer,
|
||||
"source" text DEFAULT 'reader' NOT NULL,
|
||||
"note" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "sessions_percent_check" CHECK (("start_percent" is null or "start_percent" between 0 and 100) and ("end_percent" is null or "end_percent" between 0 and 100)),
|
||||
CONSTRAINT "sessions_end_check" CHECK ("ended_at" is null or "ended_at" >= "started_at")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "books"."shelf_works" (
|
||||
"shelf_id" integer,
|
||||
"work_id" integer,
|
||||
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||
"added_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "shelf_works_pkey" PRIMARY KEY("shelf_id","work_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "books"."shelves" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "books"."works" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"title" text NOT NULL,
|
||||
"subtitle" text,
|
||||
"authors" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"isbn10" text,
|
||||
"isbn13" text,
|
||||
"publisher" text,
|
||||
"year" integer,
|
||||
"language" text,
|
||||
"page_count" integer,
|
||||
"description" text,
|
||||
"tags" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"cover_url" text,
|
||||
"cover_path" text,
|
||||
"openlibrary_work_id" text,
|
||||
"openlibrary_edition_id" text,
|
||||
"google_books_id" text,
|
||||
"annas_md5" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"source_refs" jsonb DEFAULT '{}' NOT NULL,
|
||||
"metadata_status" "books"."metadata_status" DEFAULT 'pending'::"books"."metadata_status" NOT NULL,
|
||||
"metadata_source" text,
|
||||
"metadata_fetched_at" timestamp with time zone,
|
||||
"notes" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "works_year_check" CHECK ("year" is null or "year" between 0 and 2100)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "files_path_key" ON "books"."files" ("path");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "files_sha256_key" ON "books"."files" ("sha256");--> statement-breakpoint
|
||||
CREATE INDEX "files_work_id_idx" ON "books"."files" ("work_id");--> statement-breakpoint
|
||||
CREATE INDEX "notes_work_id_idx" ON "books"."notes" ("work_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "progress_work_id_key" ON "books"."progress" ("work_id");--> statement-breakpoint
|
||||
CREATE INDEX "progress_status_idx" ON "books"."progress" ("status");--> statement-breakpoint
|
||||
CREATE INDEX "sessions_work_id_idx" ON "books"."sessions" ("work_id");--> statement-breakpoint
|
||||
CREATE INDEX "sessions_started_at_idx" ON "books"."sessions" ("started_at");--> statement-breakpoint
|
||||
CREATE INDEX "shelf_works_work_id_idx" ON "books"."shelf_works" ("work_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "shelves_slug_key" ON "books"."shelves" ("slug");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "works_isbn13_key" ON "books"."works" ("isbn13") WHERE "isbn13" is not null;--> statement-breakpoint
|
||||
CREATE INDEX "works_title_idx" ON "books"."works" (lower("title"));--> statement-breakpoint
|
||||
ALTER TABLE "books"."files" ADD CONSTRAINT "files_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "books"."notes" ADD CONSTRAINT "notes_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "books"."notes" ADD CONSTRAINT "notes_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "books"."progress" ADD CONSTRAINT "progress_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "books"."sessions" ADD CONSTRAINT "sessions_file_id_files_id_fkey" FOREIGN KEY ("file_id") REFERENCES "books"."files"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "books"."shelf_works" ADD CONSTRAINT "shelf_works_shelf_id_shelves_id_fkey" FOREIGN KEY ("shelf_id") REFERENCES "books"."shelves"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "books"."shelf_works" ADD CONSTRAINT "shelf_works_work_id_works_id_fkey" FOREIGN KEY ("work_id") REFERENCES "books"."works"("id") ON DELETE CASCADE;
|
||||
--> statement-breakpoint
|
||||
-- Hand-written: updated_at triggers, grants, default shelves, GUI registration.
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
FOREACH t IN ARRAY ARRAY['works','files','shelves','progress','notes'] LOOP
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON books.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END LOOP;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA books TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA books TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA books TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA books GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA books GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT USAGE ON SCHEMA books TO personal_ro;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA books TO personal_ro;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA books GRANT SELECT ON TABLES TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
INSERT INTO books.shelves (slug, name, sort_order) VALUES
|
||||
('to-read', 'To read', 10), ('reading', 'Reading', 20), ('done', 'Done', 30), ('reference', 'Reference', 40)
|
||||
ON CONFLICT DO NOTHING;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('books','works','books-works','Books','core',true,false,1000),
|
||||
('books','shelves','books-shelves','Book shelves','core',true,false,1000),
|
||||
('books','files','books-files','Book files','core',false,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
-- meta.sync() is executable by personal_app/personal_ddl, not by the migration role
|
||||
-- (personal) in prod; deploy-remote.sh runs it as the app role after migrating.
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
--> statement-breakpoint
|
||||
-- Hand-written: updated_at trigger and explicit grants (default privileges from the books migration cover new tables too).
|
||||
CREATE TRIGGER touch_updated_at BEFORE UPDATE ON books.sessions FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at();
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON books.sessions TO personal_app;
|
||||
GRANT USAGE, SELECT ON SEQUENCE books.sessions_id_seq TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT SELECT ON books.sessions TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
13722
drizzle/20261024013208_books/snapshot.json
Normal file
13722
drizzle/20261024013208_books/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
ALTER TABLE "money"."accounts" ADD COLUMN "include_in_total" boolean DEFAULT true NOT NULL;
|
||||
13735
drizzle/20261024021157_money_include_in_total/snapshot.json
Normal file
13735
drizzle/20261024021157_money_include_in_total/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
142
drizzle/20261024022836_tasks/migration.sql
Normal file
142
drizzle/20261024022836_tasks/migration.sql
Normal file
@@ -0,0 +1,142 @@
|
||||
CREATE SCHEMA "tasks";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "tasks"."activity" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"task_id" integer,
|
||||
"goal_id" integer,
|
||||
"kind" text NOT NULL,
|
||||
"from_status" text,
|
||||
"to_status" text,
|
||||
"note" text,
|
||||
"value" numeric(14,2),
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "activity_target_check" CHECK ("task_id" IS NOT NULL OR "goal_id" IS NOT NULL),
|
||||
CONSTRAINT "activity_kind_check" CHECK ("kind" IN ('created','status','edit','note','progress'))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "tasks"."areas" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"description" text,
|
||||
"color" text,
|
||||
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||
"archived_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "areas_slug_check" CHECK ("slug" ~ '^[a-z0-9][a-z0-9-]{0,62}$'),
|
||||
CONSTRAINT "areas_name_check" CHECK (length(btrim("name")) > 0)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "tasks"."goals" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"title" text NOT NULL,
|
||||
"notes" text,
|
||||
"status" text DEFAULT 'active' NOT NULL,
|
||||
"area_id" integer,
|
||||
"target_date" date,
|
||||
"metric" text,
|
||||
"metric_unit" text,
|
||||
"metric_target" numeric(14,2),
|
||||
"metric_current" numeric(14,2),
|
||||
"flagged" boolean DEFAULT false NOT NULL,
|
||||
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||
"source" text,
|
||||
"achieved_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "goals_title_check" CHECK (length(btrim("title")) > 0),
|
||||
CONSTRAINT "goals_status_check" CHECK ("status" IN ('active','achieved','dropped'))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "tasks"."tasks" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"title" text NOT NULL,
|
||||
"notes" text,
|
||||
"status" text DEFAULT 'todo' NOT NULL,
|
||||
"priority" smallint DEFAULT 0 NOT NULL,
|
||||
"flagged" boolean DEFAULT false NOT NULL,
|
||||
"due_date" date,
|
||||
"due_time" time,
|
||||
"reminder_at" timestamp with time zone,
|
||||
"recurrence" text,
|
||||
"area_id" integer,
|
||||
"goal_id" integer,
|
||||
"parent_id" integer,
|
||||
"tags" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||
"source" text,
|
||||
"external_id" text,
|
||||
"started_at" timestamp with time zone,
|
||||
"completed_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "tasks_title_check" CHECK (length(btrim("title")) > 0),
|
||||
CONSTRAINT "tasks_status_check" CHECK ("status" IN ('todo','in_progress','done','dropped')),
|
||||
CONSTRAINT "tasks_priority_check" CHECK ("priority" BETWEEN 0 AND 3),
|
||||
CONSTRAINT "tasks_parent_check" CHECK ("parent_id" IS NULL OR "parent_id" <> "id"),
|
||||
CONSTRAINT "tasks_time_check" CHECK ("due_time" IS NULL OR "due_date" IS NOT NULL)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX "activity_task_idx" ON "tasks"."activity" ("task_id","created_at");--> statement-breakpoint
|
||||
CREATE INDEX "activity_goal_idx" ON "tasks"."activity" ("goal_id","created_at");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "areas_slug_key" ON "tasks"."areas" ("slug");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "goals_title_key" ON "tasks"."goals" (lower("title"));--> statement-breakpoint
|
||||
CREATE INDEX "tasks_area_idx" ON "tasks"."tasks" ("area_id");--> statement-breakpoint
|
||||
CREATE INDEX "tasks_goal_idx" ON "tasks"."tasks" ("goal_id");--> statement-breakpoint
|
||||
CREATE INDEX "tasks_parent_idx" ON "tasks"."tasks" ("parent_id");--> statement-breakpoint
|
||||
CREATE INDEX "tasks_status_due_idx" ON "tasks"."tasks" ("status","due_date");--> statement-breakpoint
|
||||
CREATE INDEX "tasks_tags_idx" ON "tasks"."tasks" USING gin ("tags");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "tasks_source_external_key" ON "tasks"."tasks" ("source","external_id") WHERE "external_id" IS NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "tasks"."activity" ADD CONSTRAINT "activity_task_id_tasks_id_fkey" FOREIGN KEY ("task_id") REFERENCES "tasks"."tasks"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "tasks"."activity" ADD CONSTRAINT "activity_goal_id_goals_id_fkey" FOREIGN KEY ("goal_id") REFERENCES "tasks"."goals"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "tasks"."goals" ADD CONSTRAINT "goals_area_id_areas_id_fkey" FOREIGN KEY ("area_id") REFERENCES "tasks"."areas"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_area_id_areas_id_fkey" FOREIGN KEY ("area_id") REFERENCES "tasks"."areas"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_goal_id_goals_id_fkey" FOREIGN KEY ("goal_id") REFERENCES "tasks"."goals"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "tasks"."tasks" ADD CONSTRAINT "tasks_parent_id_tasks_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "tasks"."tasks"("id") ON DELETE CASCADE;
|
||||
--> statement-breakpoint
|
||||
-- Hand-written below this line (not generated by drizzle-kit).
|
||||
-- updated_at triggers, same as every other core table.
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
|
||||
FOREACH t IN ARRAY ARRAY['areas','goals','tasks'] LOOP
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON tasks.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END LOOP;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
-- Privacy: like health, the task tracker is readable/writable by the app role only
|
||||
-- (no personal_ro grant). Grants only run on prod, where migrations run as `personal`.
|
||||
REVOKE ALL ON SCHEMA tasks FROM PUBLIC;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA tasks TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA tasks TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA tasks TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA tasks
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA tasks
|
||||
GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
-- Registered in meta.tables (structure only; the lists GUI only shows kind = 'list').
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('tasks','tasks','tasks-tasks','Tasks','core',true,true,1000),
|
||||
('tasks','goals','tasks-goals','Goals','core',true,false,1000),
|
||||
('tasks','areas','tasks-areas','Task areas','core',true,false,1000),
|
||||
('tasks','activity','tasks-activity','Task activity','core',false,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
-- deploy-remote.sh runs meta.sync() as the app role after migrating.
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
14935
drizzle/20261024022836_tasks/snapshot.json
Normal file
14935
drizzle/20261024022836_tasks/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
29
drizzle/20261024024249_meta_edit_log/migration.sql
Normal file
29
drizzle/20261024024249_meta_edit_log/migration.sql
Normal file
@@ -0,0 +1,29 @@
|
||||
CREATE TABLE "meta"."edit_log" (
|
||||
"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "meta"."edit_log_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
|
||||
"at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"user_id" text,
|
||||
"user_email" text,
|
||||
"schema_name" text NOT NULL,
|
||||
"table_name" text NOT NULL,
|
||||
"action" text NOT NULL,
|
||||
"pk" jsonb,
|
||||
"before" jsonb,
|
||||
"after" jsonb,
|
||||
CONSTRAINT "edit_log_action_check" CHECK (action IN ('insert','update','delete','import','add_column'))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX "edit_log_table_at_index" ON "meta"."edit_log" ("schema_name","table_name","at");--> statement-breakpoint
|
||||
-- Hand-written below this line (not generated by drizzle-kit).
|
||||
-- The app role appends to and reads the log; nobody updates or deletes through the app.
|
||||
-- personal_ro gets no access: before/after values can hold private (health) data.
|
||||
-- Grants only run on prod (as `personal`); the dev DB is owned by personal_dev.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT SELECT, INSERT ON meta.edit_log TO personal_app;
|
||||
GRANT USAGE ON SEQUENCE meta.edit_log_id_seq TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
REVOKE ALL ON meta.edit_log FROM personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
15132
drizzle/20261024024249_meta_edit_log/snapshot.json
Normal file
15132
drizzle/20261024024249_meta_edit_log/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
2
drizzle/20261024030132_tasks_ticktick/migration.sql
Normal file
2
drizzle/20261024030132_tasks_ticktick/migration.sql
Normal file
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE "tasks"."tasks" ADD COLUMN "start_at" timestamp with time zone;--> statement-breakpoint
|
||||
ALTER TABLE "tasks"."tasks" ADD COLUMN "time_zone" text;
|
||||
15158
drizzle/20261024030132_tasks_ticktick/snapshot.json
Normal file
15158
drizzle/20261024030132_tasks_ticktick/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
112
drizzle/20261024132343_groceries/migration.sql
Normal file
112
drizzle/20261024132343_groceries/migration.sql
Normal file
@@ -0,0 +1,112 @@
|
||||
CREATE SCHEMA "groceries";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "groceries"."prices" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"product_id" integer NOT NULL,
|
||||
"store_id" integer NOT NULL,
|
||||
"price_cents" integer NOT NULL,
|
||||
"unit_price_cents" integer,
|
||||
"promo" boolean DEFAULT false NOT NULL,
|
||||
"observed_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"source" text DEFAULT 'manual' NOT NULL,
|
||||
"store_product_id" text,
|
||||
"url" text,
|
||||
"notes" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "prices_price_check" CHECK ("price_cents" >= 0),
|
||||
CONSTRAINT "prices_unit_price_check" CHECK ("unit_price_cents" IS NULL OR "unit_price_cents" >= 0)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "groceries"."products" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"name" text NOT NULL,
|
||||
"brand" text,
|
||||
"category" text,
|
||||
"size" text,
|
||||
"unit_amount" numeric(10,3),
|
||||
"unit" text,
|
||||
"favorite" boolean DEFAULT false NOT NULL,
|
||||
"notes" text,
|
||||
"archived_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "products_name_check" CHECK (length(btrim("name")) > 0),
|
||||
CONSTRAINT "products_unit_check" CHECK ("unit" IS NULL OR "unit" IN ('piece','kg','l')),
|
||||
CONSTRAINT "products_unit_amount_check" CHECK ("unit_amount" IS NULL OR "unit_amount" > 0)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "groceries"."stores" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"website" text,
|
||||
"color" text,
|
||||
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||
"archived_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "stores_slug_check" CHECK ("slug" ~ '^[a-z0-9][a-z0-9-]{0,62}$'),
|
||||
CONSTRAINT "stores_name_check" CHECK (length(btrim("name")) > 0)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX "prices_product_store_idx" ON "groceries"."prices" ("product_id","store_id","observed_at");--> statement-breakpoint
|
||||
CREATE INDEX "prices_store_idx" ON "groceries"."prices" ("store_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "products_name_brand_key" ON "groceries"."products" (lower("name"),lower(coalesce("brand", '')));--> statement-breakpoint
|
||||
CREATE INDEX "products_category_idx" ON "groceries"."products" ("category");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "stores_slug_key" ON "groceries"."stores" ("slug");--> statement-breakpoint
|
||||
ALTER TABLE "groceries"."prices" ADD CONSTRAINT "prices_product_id_products_id_fkey" FOREIGN KEY ("product_id") REFERENCES "groceries"."products"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "groceries"."prices" ADD CONSTRAINT "prices_store_id_stores_id_fkey" FOREIGN KEY ("store_id") REFERENCES "groceries"."stores"("id") ON DELETE CASCADE;
|
||||
--> statement-breakpoint
|
||||
-- Hand-written below this line (not generated by drizzle-kit).
|
||||
-- updated_at triggers, same as every other core table.
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
|
||||
FOREACH t IN ARRAY ARRAY['stores','products','prices'] LOOP
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON groceries.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END LOOP;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
-- Private like tasks/health: app role only, no personal_ro. Grants only run on prod,
|
||||
-- where migrations run as `personal`.
|
||||
REVOKE ALL ON SCHEMA groceries FROM PUBLIC;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA groceries TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA groceries TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA groceries TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA groceries
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA groceries
|
||||
GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
INSERT INTO groceries.stores (slug, name, website, color, sort_order) VALUES
|
||||
('ah', 'Albert Heijn', 'https://www.ah.nl', '#00a0e2', 10),
|
||||
('jumbo', 'Jumbo', 'https://www.jumbo.com', '#eeb717', 20),
|
||||
('lidl', 'Lidl', 'https://www.lidl.nl', '#0050aa', 30),
|
||||
('plus', 'Plus', 'https://www.plus.nl', '#e2001a', 40)
|
||||
ON CONFLICT DO NOTHING;
|
||||
--> statement-breakpoint
|
||||
-- Registered in meta.tables so the /lists database browser shows them (kind 'core').
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('groceries','products','groceries-products','Grocery products','core',true,false,1100),
|
||||
('groceries','prices','groceries-prices','Grocery prices','core',true,false,1100),
|
||||
('groceries','stores','groceries-stores','Grocery stores','core',true,false,1100)
|
||||
ON CONFLICT DO NOTHING;
|
||||
-- deploy-remote.sh runs meta.sync() as the app role after migrating.
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
15861
drizzle/20261024132343_groceries/snapshot.json
Normal file
15861
drizzle/20261024132343_groceries/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
11
drizzle/20261024134401_groceries_drop_manual/migration.sql
Normal file
11
drizzle/20261024134401_groceries_drop_manual/migration.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
ALTER TABLE "groceries"."prices" DROP CONSTRAINT "prices_product_id_products_id_fkey";--> statement-breakpoint
|
||||
DROP TABLE "groceries"."prices";--> statement-breakpoint
|
||||
DROP TABLE "groceries"."products";--> statement-breakpoint
|
||||
-- The hand-entered products/prices tables were never used (0 rows); replaced by store items
|
||||
-- and groups in the next migration. Unregister them from the /lists database browser.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
DELETE FROM meta.tables WHERE schema_name = 'groceries' AND table_name IN ('products','prices');
|
||||
END IF;
|
||||
END $$;
|
||||
15330
drizzle/20261024134401_groceries_drop_manual/snapshot.json
Normal file
15330
drizzle/20261024134401_groceries_drop_manual/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
138
drizzle/20261024134411_groceries_compare/migration.sql
Normal file
138
drizzle/20261024134411_groceries_compare/migration.sql
Normal file
@@ -0,0 +1,138 @@
|
||||
-- Trigram search for group/item names (trusted extension; the database owner may create it).
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "groceries"."groups" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"key" text,
|
||||
"name" text NOT NULL,
|
||||
"brand" text,
|
||||
"category" text,
|
||||
"unit" text,
|
||||
"family_key" text,
|
||||
"favorite" boolean DEFAULT false NOT NULL,
|
||||
"manual" boolean DEFAULT false NOT NULL,
|
||||
"merged_into_id" integer,
|
||||
"notes" text,
|
||||
"item_count" integer DEFAULT 0 NOT NULL,
|
||||
"store_count" integer DEFAULT 0 NOT NULL,
|
||||
"store_ids" integer[] DEFAULT '{}'::integer[] NOT NULL,
|
||||
"min_unit_price_cents" integer,
|
||||
"min_price_cents" integer,
|
||||
"promo_count" integer DEFAULT 0 NOT NULL,
|
||||
"image_url" text,
|
||||
"search_text" text DEFAULT '' NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "groups_name_check" CHECK (length(btrim("name")) > 0),
|
||||
CONSTRAINT "groups_unit_check" CHECK ("unit" IS NULL OR "unit" IN ('piece','kg','l')),
|
||||
CONSTRAINT "groups_not_self_merged" CHECK ("merged_into_id" IS NULL OR "merged_into_id" <> "id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "groceries"."import_runs" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"store_id" integer NOT NULL,
|
||||
"started_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"finished_at" timestamp with time zone,
|
||||
"status" text DEFAULT 'running' NOT NULL,
|
||||
"requests" integer DEFAULT 0 NOT NULL,
|
||||
"fetched" integer DEFAULT 0 NOT NULL,
|
||||
"inserted" integer DEFAULT 0 NOT NULL,
|
||||
"price_changes" integer DEFAULT 0 NOT NULL,
|
||||
"error" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "groceries"."item_prices" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"item_id" integer NOT NULL,
|
||||
"observed_on" date NOT NULL,
|
||||
"price_cents" integer,
|
||||
"promo_price_cents" integer,
|
||||
"promo_label" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "groceries"."items" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"store_id" integer NOT NULL,
|
||||
"store_product_id" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"brand" text,
|
||||
"size_text" text,
|
||||
"unit" text,
|
||||
"unit_amount" numeric(12,4),
|
||||
"unit_source" text,
|
||||
"price_cents" integer,
|
||||
"promo_price_cents" integer,
|
||||
"promo_label" text,
|
||||
"unit_price_cents" integer,
|
||||
"promo_unit_price_cents" integer,
|
||||
"category" text,
|
||||
"category_path" text,
|
||||
"gtin" text,
|
||||
"gtin_checked_at" timestamp with time zone,
|
||||
"image_url" text,
|
||||
"url" text,
|
||||
"available" boolean DEFAULT true NOT NULL,
|
||||
"first_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"last_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"group_id" integer,
|
||||
"match_key" text,
|
||||
"match_method" text,
|
||||
"match_confidence" real,
|
||||
"match_locked" boolean DEFAULT false NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "items_unit_check" CHECK ("unit" IS NULL OR "unit" IN ('piece','kg','l')),
|
||||
CONSTRAINT "items_price_check" CHECK ("price_cents" IS NULL OR "price_cents" >= 0)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "groups_key_key" ON "groceries"."groups" ("key");--> statement-breakpoint
|
||||
CREATE INDEX "groups_family_idx" ON "groceries"."groups" ("family_key");--> statement-breakpoint
|
||||
CREATE INDEX "groups_category_idx" ON "groceries"."groups" ("category");--> statement-breakpoint
|
||||
CREATE INDEX "groups_store_ids_idx" ON "groceries"."groups" USING gin ("store_ids");--> statement-breakpoint
|
||||
CREATE INDEX "groups_search_trgm_idx" ON "groceries"."groups" USING gin ("search_text" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "groups_list_idx" ON "groceries"."groups" ("store_count","name");--> statement-breakpoint
|
||||
CREATE INDEX "import_runs_store_idx" ON "groceries"."import_runs" ("store_id","started_at");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "item_prices_item_day_key" ON "groceries"."item_prices" ("item_id","observed_on");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "items_store_product_key" ON "groceries"."items" ("store_id","store_product_id");--> statement-breakpoint
|
||||
CREATE INDEX "items_group_idx" ON "groceries"."items" ("group_id");--> statement-breakpoint
|
||||
CREATE INDEX "items_gtin_idx" ON "groceries"."items" ("gtin");--> statement-breakpoint
|
||||
CREATE INDEX "items_name_trgm_idx" ON "groceries"."items" USING gin ("name" gin_trgm_ops);--> statement-breakpoint
|
||||
ALTER TABLE "groceries"."groups" ADD CONSTRAINT "groups_merged_into_id_groups_id_fkey" FOREIGN KEY ("merged_into_id") REFERENCES "groceries"."groups"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "groceries"."import_runs" ADD CONSTRAINT "import_runs_store_id_stores_id_fkey" FOREIGN KEY ("store_id") REFERENCES "groceries"."stores"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "groceries"."item_prices" ADD CONSTRAINT "item_prices_item_id_items_id_fkey" FOREIGN KEY ("item_id") REFERENCES "groceries"."items"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "groceries"."items" ADD CONSTRAINT "items_store_id_stores_id_fkey" FOREIGN KEY ("store_id") REFERENCES "groceries"."stores"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "groceries"."items" ADD CONSTRAINT "items_group_id_groups_id_fkey" FOREIGN KEY ("group_id") REFERENCES "groceries"."groups"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
|
||||
FOREACH t IN ARRAY ARRAY['groups','items'] LOOP
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON groceries.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END LOOP;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
-- Private: app role only (default privileges from the first groceries migration also apply).
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA groceries TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA groceries TO personal_app;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('groceries','groups','groceries-groups','Grocery groups','core',true,false,1100),
|
||||
('groceries','items','groceries-items','Grocery store items','core',true,false,1100),
|
||||
('groceries','item_prices','groceries-item-prices','Grocery price history','core',false,false,1100),
|
||||
('groceries','import_runs','groceries-import-runs','Grocery import runs','core',false,false,1100)
|
||||
ON CONFLICT DO NOTHING;
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
16678
drizzle/20261024134411_groceries_compare/snapshot.json
Normal file
16678
drizzle/20261024134411_groceries_compare/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
5
drizzle/20261024140745_house_scrapers/migration.sql
Normal file
5
drizzle/20261024140745_house_scrapers/migration.sql
Normal file
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "address_key" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "duplicate_of" integer;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "total_price_eur" integer;--> statement-breakpoint
|
||||
CREATE INDEX "listings_address_idx" ON "house"."listings" ("address_key") WHERE address_key IS NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD CONSTRAINT "listings_duplicate_of_listings_id_fkey" FOREIGN KEY ("duplicate_of") REFERENCES "house"."listings"("id") ON DELETE SET NULL;
|
||||
16755
drizzle/20261024140745_house_scrapers/snapshot.json
Normal file
16755
drizzle/20261024140745_house_scrapers/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
20
drizzle/20261024141609_health_scale_metrics/migration.sql
Normal file
20
drizzle/20261024141609_health_scale_metrics/migration.sql
Normal file
@@ -0,0 +1,20 @@
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "muscle_mass_kg" numeric(5,2);--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "skeletal_muscle_kg" numeric(5,2);--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "bone_mass_kg" numeric(4,2);--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "visceral_fat" numeric(4,1);--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "body_water_pct" numeric(4,1);--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "protein_pct" numeric(4,1);--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "bmr_kcal" integer;--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "bmi" numeric(4,1);--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "measured_at" timestamp with time zone;--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD COLUMN "source" text;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "body_measurements_date_weight_key" ON "health"."body_measurements" ("date","weight_kg");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "body_measurements_measured_at_key" ON "health"."body_measurements" ("measured_at");--> statement-breakpoint
|
||||
ALTER TABLE "health"."body_measurements" ADD CONSTRAINT "body_measurements_scale_check" CHECK (("muscle_mass_kg" IS NULL OR "muscle_mass_kg" BETWEEN 5 AND 250)
|
||||
AND ("skeletal_muscle_kg" IS NULL OR "skeletal_muscle_kg" BETWEEN 5 AND 200)
|
||||
AND ("bone_mass_kg" IS NULL OR "bone_mass_kg" BETWEEN 0.5 AND 20)
|
||||
AND ("visceral_fat" IS NULL OR "visceral_fat" BETWEEN 1 AND 60)
|
||||
AND ("body_water_pct" IS NULL OR "body_water_pct" BETWEEN 20 AND 85)
|
||||
AND ("protein_pct" IS NULL OR "protein_pct" BETWEEN 5 AND 40)
|
||||
AND ("bmr_kcal" IS NULL OR "bmr_kcal" BETWEEN 500 AND 6000)
|
||||
AND ("bmi" IS NULL OR "bmi" BETWEEN 10 AND 80));
|
||||
16941
drizzle/20261024141609_health_scale_metrics/snapshot.json
Normal file
16941
drizzle/20261024141609_health_scale_metrics/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
66
drizzle/20261024150925_home/migration.sql
Normal file
66
drizzle/20261024150925_home/migration.sql
Normal file
@@ -0,0 +1,66 @@
|
||||
CREATE SCHEMA "home";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "home"."links" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"title" text NOT NULL,
|
||||
"url" text NOT NULL,
|
||||
"section" text DEFAULT 'links' NOT NULL,
|
||||
"sort_order" integer DEFAULT 0 NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "links_title_check" CHECK (length(btrim("title")) between 1 and 80),
|
||||
CONSTRAINT "links_url_check" CHECK ("url" ~ '^https?://' and length("url") <= 2000)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "home"."snapshots" (
|
||||
"key" text PRIMARY KEY,
|
||||
"data" jsonb NOT NULL,
|
||||
"collected_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX "links_sort_idx" ON "home"."links" ("section","sort_order");--> statement-breakpoint
|
||||
-- Privacy: like tasks and health, the start page is the app role only (no personal_ro).
|
||||
-- Grants only run on prod, where migrations run as `personal`.
|
||||
REVOKE ALL ON SCHEMA home FROM PUBLIC;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA home TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA home TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA home TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA home
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA home
|
||||
GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
-- The host-side collector (deploy/home-collector) may only upsert snapshots. The role is
|
||||
-- created out of band (it has a password); see deploy/home-collector/README.md.
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'home_collector') THEN
|
||||
GRANT USAGE ON SCHEMA home TO home_collector;
|
||||
GRANT SELECT, INSERT, UPDATE ON home.snapshots TO home_collector;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
-- Starter links; edited inline on /start.
|
||||
INSERT INTO "home"."links" ("title", "url", "section", "sort_order") VALUES
|
||||
('Gmail', 'https://mail.google.com/mail/u/0/', 'daily', 10),
|
||||
('Calendar', 'https://calendar.google.com/', 'daily', 20),
|
||||
('WhatsApp', 'https://web.whatsapp.com/', 'daily', 30),
|
||||
('Discord', 'https://discord.com/app', 'daily', 40),
|
||||
('X', 'https://x.com/home', 'daily', 50),
|
||||
('YouTube', 'https://www.youtube.com/', 'daily', 60),
|
||||
('Spotify', 'https://open.spotify.com/', 'daily', 70),
|
||||
('Drive', 'https://drive.google.com/', 'daily', 80),
|
||||
('Moneybird', 'https://moneybird.com/390525278507501515', 'money', 110),
|
||||
('Revolut', 'https://app.revolut.com/', 'money', 120),
|
||||
('Knab', 'https://www.knab.nl/', 'money', 130),
|
||||
('Tikkie', 'https://www.tikkie.me/', 'money', 140),
|
||||
('Gitea', 'https://git.stanrunge.dev/', 'dev', 210),
|
||||
('GitHub', 'https://github.com/', 'dev', 220),
|
||||
('Cloudflare', 'https://dash.cloudflare.com/', 'dev', 230),
|
||||
('Tailscale', 'https://login.tailscale.com/admin/machines', 'dev', 240),
|
||||
('Backblaze B2', 'https://secure.backblaze.com/b2_buckets.htm', 'dev', 250),
|
||||
('1Password', 'https://my.1password.com/', 'dev', 260),
|
||||
('osu!', 'https://osu.ppy.sh/', 'fun', 310),
|
||||
('Twitch', 'https://www.twitch.tv/', 'fun', 320);
|
||||
17149
drizzle/20261024150925_home/snapshot.json
Normal file
17149
drizzle/20261024150925_home/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
134
drizzle/20261024153902_cars/migration.sql
Normal file
134
drizzle/20261024153902_cars/migration.sql
Normal file
@@ -0,0 +1,134 @@
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
|
||||
--> statement-breakpoint
|
||||
CREATE SCHEMA "cars";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "cars"."brands" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"country" text,
|
||||
"model_count" integer DEFAULT 0 NOT NULL,
|
||||
"nl_count" integer DEFAULT 0 NOT NULL,
|
||||
"first_year" smallint,
|
||||
"last_year" smallint,
|
||||
"sources" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "cars"."models" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"dex" integer NOT NULL,
|
||||
"brand_id" integer NOT NULL,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"search" text NOT NULL,
|
||||
"first_year" smallint,
|
||||
"last_year" smallint,
|
||||
"bodies" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"size_classes" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"fuels" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"drives" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"transmissions" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"engine_l_min" real,
|
||||
"engine_l_max" real,
|
||||
"cylinders_min" smallint,
|
||||
"cylinders_max" smallint,
|
||||
"turbo" boolean,
|
||||
"power_kw_min" real,
|
||||
"power_kw_max" real,
|
||||
"l100_min" real,
|
||||
"l100_max" real,
|
||||
"co2_min" smallint,
|
||||
"co2_max" smallint,
|
||||
"mpg_min" real,
|
||||
"mpg_max" real,
|
||||
"range_km_max" smallint,
|
||||
"seats" smallint,
|
||||
"doors" smallint,
|
||||
"mass_kg" smallint,
|
||||
"nl_count" integer DEFAULT 0 NOT NULL,
|
||||
"nl_rank" integer,
|
||||
"nl_first_year" smallint,
|
||||
"nl_last_year" smallint,
|
||||
"nl_price" integer,
|
||||
"nl_price_latest" integer,
|
||||
"nl_price_latest_year" smallint,
|
||||
"nl_years" jsonb DEFAULT '[]' NOT NULL,
|
||||
"nl_names" jsonb DEFAULT '[]' NOT NULL,
|
||||
"us_years" smallint[] DEFAULT '{}'::smallint[] NOT NULL,
|
||||
"variant_count" integer DEFAULT 0 NOT NULL,
|
||||
"sources" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "cars"."variants" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"model_id" integer NOT NULL,
|
||||
"source" text DEFAULT 'fueleconomy.gov' NOT NULL,
|
||||
"source_id" integer,
|
||||
"year" smallint NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"engine_l" real,
|
||||
"cylinders" smallint,
|
||||
"transmission" text,
|
||||
"drive" text,
|
||||
"fuel" text,
|
||||
"size_class" text,
|
||||
"mpg_city" real,
|
||||
"mpg_hwy" real,
|
||||
"mpg_comb" real,
|
||||
"co2_gpm" smallint,
|
||||
"range_mi" smallint,
|
||||
"ev_motor" text,
|
||||
"turbo" boolean DEFAULT false NOT NULL,
|
||||
"supercharged" boolean DEFAULT false NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "brands_slug_key" ON "cars"."brands" ("slug");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "models_dex_key" ON "cars"."models" ("dex");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "models_brand_slug_key" ON "cars"."models" ("brand_id","slug");--> statement-breakpoint
|
||||
CREATE INDEX "models_nl_count_idx" ON "cars"."models" ("nl_count");--> statement-breakpoint
|
||||
CREATE INDEX "models_search_trgm_idx" ON "cars"."models" USING gin ("search" public.gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "variants_model_year_idx" ON "cars"."variants" ("model_id","year");--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD CONSTRAINT "models_brand_id_brands_id_fkey" FOREIGN KEY ("brand_id") REFERENCES "cars"."brands"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."variants" ADD CONSTRAINT "variants_model_id_models_id_fkey" FOREIGN KEY ("model_id") REFERENCES "cars"."models"("id") ON DELETE CASCADE;
|
||||
--> statement-breakpoint
|
||||
-- Hand-written: updated_at triggers, grants, GUI registration. Data comes from scripts/cars-import.ts.
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
FOREACH t IN ARRAY ARRAY['brands','models'] LOOP
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON cars.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END LOOP;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA cars TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA cars TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA cars TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA cars GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA cars GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT USAGE ON SCHEMA cars TO personal_ro;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA cars TO personal_ro;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA cars GRANT SELECT ON TABLES TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('cars','brands','cars-brands','Car brands','core',false,false,1000),
|
||||
('cars','models','cars-models','Car models','core',false,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
18350
drizzle/20261024153902_cars/snapshot.json
Normal file
18350
drizzle/20261024153902_cars/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
11
drizzle/20261024154032_house_liveability/migration.sql
Normal file
11
drizzle/20261024154032_house_liveability/migration.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_score" double precision;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_class" integer;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_onv_class" integer;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_level" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_precision" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_buurt_code" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_buurt_name" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_year" integer;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_detail" jsonb;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_query" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_scored_at" timestamp with time zone;
|
||||
18493
drizzle/20261024154032_house_liveability/snapshot.json
Normal file
18493
drizzle/20261024154032_house_liveability/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
159
drizzle/20261024155715_music/migration.sql
Normal file
159
drizzle/20261024155715_music/migration.sql
Normal file
@@ -0,0 +1,159 @@
|
||||
CREATE SCHEMA "music";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "music"."albums" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"spotify_id" text,
|
||||
"name" text NOT NULL,
|
||||
"artist_id" integer,
|
||||
"image_url" text,
|
||||
"image_url_small" text,
|
||||
"release_date" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "music"."artists" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"spotify_id" text,
|
||||
"name" text NOT NULL,
|
||||
"image_url" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "music"."import_state" (
|
||||
"key" text PRIMARY KEY,
|
||||
"data" jsonb NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "music"."now_playing" (
|
||||
"id" smallint PRIMARY KEY DEFAULT 1,
|
||||
"track_id" integer,
|
||||
"is_playing" boolean DEFAULT false NOT NULL,
|
||||
"progress_ms" integer,
|
||||
"device_name" text,
|
||||
"device_type" text,
|
||||
"context_uri" text,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "now_playing_single_row" CHECK ("id" = 1)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "music"."plays" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"track_id" integer NOT NULL,
|
||||
"played_at" timestamp with time zone NOT NULL,
|
||||
"source" text DEFAULT 'spotify' NOT NULL,
|
||||
"origin" text DEFAULT 'api' NOT NULL,
|
||||
"ms_played" integer,
|
||||
"context_type" text,
|
||||
"context_uri" text,
|
||||
"platform" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "plays_source_check" CHECK ("source" in ('spotify', 'lastfm', 'youtube', 'local')),
|
||||
CONSTRAINT "plays_origin_check" CHECK ("origin" in ('api', 'export', 'import', 'manual'))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "music"."spotify_auth" (
|
||||
"id" smallint PRIMARY KEY DEFAULT 1,
|
||||
"spotify_user_id" text NOT NULL,
|
||||
"display_name" text,
|
||||
"refresh_token" text NOT NULL,
|
||||
"access_token" text,
|
||||
"access_expires_at" timestamp with time zone,
|
||||
"scope" text,
|
||||
"recent_cursor" bigint,
|
||||
"needs_reauth" boolean DEFAULT false NOT NULL,
|
||||
"lease_until" timestamp with time zone,
|
||||
"last_poll_at" timestamp with time zone,
|
||||
"last_error" text,
|
||||
"connected_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "spotify_auth_single_row" CHECK ("id" = 1)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "music"."track_artists" (
|
||||
"track_id" integer,
|
||||
"artist_id" integer,
|
||||
"position" smallint DEFAULT 0 NOT NULL,
|
||||
CONSTRAINT "track_artists_pkey" PRIMARY KEY("track_id","artist_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "music"."tracks" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"spotify_id" text,
|
||||
"name" text NOT NULL,
|
||||
"album_id" integer,
|
||||
"artist_id" integer,
|
||||
"duration_ms" integer,
|
||||
"explicit" boolean,
|
||||
"match_key" text NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "albums_spotify_id_key" ON "music"."albums" ("spotify_id") WHERE "spotify_id" is not null;--> statement-breakpoint
|
||||
CREATE INDEX "albums_name_idx" ON "music"."albums" (lower("name"));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "artists_spotify_id_key" ON "music"."artists" ("spotify_id") WHERE "spotify_id" is not null;--> statement-breakpoint
|
||||
CREATE INDEX "artists_name_idx" ON "music"."artists" (lower("name"));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "plays_dedupe_key" ON "music"."plays" ("source","track_id","played_at");--> statement-breakpoint
|
||||
CREATE INDEX "plays_played_at_idx" ON "music"."plays" ("played_at");--> statement-breakpoint
|
||||
CREATE INDEX "plays_source_played_idx" ON "music"."plays" ("source","played_at");--> statement-breakpoint
|
||||
CREATE INDEX "plays_track_played_idx" ON "music"."plays" ("track_id","played_at");--> statement-breakpoint
|
||||
CREATE INDEX "track_artists_artist_idx" ON "music"."track_artists" ("artist_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "tracks_spotify_id_key" ON "music"."tracks" ("spotify_id") WHERE "spotify_id" is not null;--> statement-breakpoint
|
||||
CREATE INDEX "tracks_name_idx" ON "music"."tracks" (lower("name"));--> statement-breakpoint
|
||||
CREATE INDEX "tracks_match_key_idx" ON "music"."tracks" ("match_key");--> statement-breakpoint
|
||||
ALTER TABLE "music"."albums" ADD CONSTRAINT "albums_artist_id_artists_id_fkey" FOREIGN KEY ("artist_id") REFERENCES "music"."artists"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "music"."now_playing" ADD CONSTRAINT "now_playing_track_id_tracks_id_fkey" FOREIGN KEY ("track_id") REFERENCES "music"."tracks"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "music"."plays" ADD CONSTRAINT "plays_track_id_tracks_id_fkey" FOREIGN KEY ("track_id") REFERENCES "music"."tracks"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "music"."track_artists" ADD CONSTRAINT "track_artists_track_id_tracks_id_fkey" FOREIGN KEY ("track_id") REFERENCES "music"."tracks"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "music"."track_artists" ADD CONSTRAINT "track_artists_artist_id_artists_id_fkey" FOREIGN KEY ("artist_id") REFERENCES "music"."artists"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "music"."tracks" ADD CONSTRAINT "tracks_album_id_albums_id_fkey" FOREIGN KEY ("album_id") REFERENCES "music"."albums"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
ALTER TABLE "music"."tracks" ADD CONSTRAINT "tracks_artist_id_artists_id_fkey" FOREIGN KEY ("artist_id") REFERENCES "music"."artists"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
-- Hand-written below. Cross-source dedupe: every stat reads plays_deduped, which hides a
|
||||
-- Last.fm scrobble when Spotify (live API or the streaming-history export) has the same
|
||||
-- normalised artist+title around the same time. Last.fm stores when a track started,
|
||||
-- Spotify when it ended, so the Spotify end may fall from 3 min before the scrobble to
|
||||
-- track length + 3 min after it. Spotify rows always win; Last.fm fills the gaps.
|
||||
CREATE VIEW "music"."plays_deduped" AS
|
||||
SELECT p.* FROM "music"."plays" p
|
||||
WHERE p.source <> 'lastfm' OR NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM "music"."tracks" lt, "music"."plays" s
|
||||
JOIN "music"."tracks" st ON st.id = s.track_id
|
||||
WHERE lt.id = p.track_id
|
||||
AND s.source = 'spotify'
|
||||
AND s.played_at >= p.played_at - interval '3 minutes'
|
||||
AND s.played_at <= p.played_at + interval '25 minutes'
|
||||
AND s.played_at <= p.played_at + make_interval(secs => coalesce(st.duration_ms, 600000) / 1000.0) + interval '3 minutes'
|
||||
AND st.match_key = lt.match_key
|
||||
);
|
||||
--> statement-breakpoint
|
||||
-- Grants only run on prod, where migrations run as `personal`. Listening history is public
|
||||
-- (personal_ro may read it); the Spotify refresh token is for the app role only.
|
||||
REVOKE ALL ON SCHEMA music FROM PUBLIC;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA music TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA music TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA music TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA music
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE personal IN SCHEMA music
|
||||
GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT USAGE ON SCHEMA music TO personal_ro;
|
||||
GRANT SELECT ON music.artists, music.albums, music.tracks, music.track_artists, music.plays,
|
||||
music.plays_deduped, music.now_playing TO personal_ro;
|
||||
END IF;
|
||||
-- Optional host-side poller (deploy/music-collector). The role is created out of band
|
||||
-- (it has a password); see deploy/music-collector/README.md.
|
||||
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'music_collector') THEN
|
||||
GRANT USAGE ON SCHEMA music TO music_collector;
|
||||
GRANT SELECT, INSERT, UPDATE ON music.artists, music.albums, music.tracks, music.track_artists,
|
||||
music.plays, music.now_playing, music.import_state TO music_collector;
|
||||
GRANT SELECT, UPDATE ON music.spotify_auth TO music_collector;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA music TO music_collector;
|
||||
END IF;
|
||||
END $$;
|
||||
19833
drizzle/20261024155715_music/snapshot.json
Normal file
19833
drizzle/20261024155715_music/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
1
drizzle/20261024160855_tasks_auto_handle/migration.sql
Normal file
1
drizzle/20261024160855_tasks_auto_handle/migration.sql
Normal file
@@ -0,0 +1 @@
|
||||
ALTER TABLE "tasks"."tasks" ADD COLUMN "auto_handle" boolean DEFAULT false NOT NULL;
|
||||
19846
drizzle/20261024160855_tasks_auto_handle/snapshot.json
Normal file
19846
drizzle/20261024160855_tasks_auto_handle/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
7
drizzle/20261024160937_osu_match_cache/migration.sql
Normal file
7
drizzle/20261024160937_osu_match_cache/migration.sql
Normal file
@@ -0,0 +1,7 @@
|
||||
CREATE TABLE "music"."osu_match_cache" (
|
||||
"key" text PRIMARY KEY,
|
||||
"result" jsonb NOT NULL,
|
||||
"searched_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
DO $$ BEGIN IF current_user = 'personal' THEN GRANT SELECT, INSERT, UPDATE, DELETE ON music.osu_match_cache TO personal_app; END IF; END $$;
|
||||
19901
drizzle/20261024160937_osu_match_cache/snapshot.json
Normal file
19901
drizzle/20261024160937_osu_match_cache/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
50
drizzle/20261024161853_cars_photos/migration.sql
Normal file
50
drizzle/20261024161853_cars_photos/migration.sql
Normal file
@@ -0,0 +1,50 @@
|
||||
CREATE TABLE "cars"."photos" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"model_id" integer NOT NULL,
|
||||
"position" smallint NOT NULL,
|
||||
"kind" text NOT NULL,
|
||||
"file" text NOT NULL,
|
||||
"caption" text,
|
||||
"thumb" text NOT NULL,
|
||||
"width" integer NOT NULL,
|
||||
"height" integer NOT NULL,
|
||||
"author" text,
|
||||
"license" text,
|
||||
"license_url" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "wikidata" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "wikipedia" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "logo" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "logo_file" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "logo_author" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "logo_license" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "photo" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "wikidata" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "wikipedia" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "photo" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "photo_count" smallint DEFAULT 0 NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "photos_checked_at" timestamp with time zone;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "photos_model_position_key" ON "cars"."photos" ("model_id","position");--> statement-breakpoint
|
||||
ALTER TABLE "cars"."photos" ADD CONSTRAINT "photos_model_id_models_id_fkey" FOREIGN KEY ("model_id") REFERENCES "cars"."models"("id") ON DELETE CASCADE;
|
||||
--> statement-breakpoint
|
||||
-- hand-written: grants + registry for the photos table
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON cars.photos TO personal_app;
|
||||
GRANT USAGE, SELECT ON SEQUENCE cars.photos_id_seq TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT SELECT ON cars.photos TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('cars','photos','cars-photos','Car photos','core',false,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
END IF;
|
||||
END $$;
|
||||
20274
drizzle/20261024161853_cars_photos/snapshot.json
Normal file
20274
drizzle/20261024161853_cars_photos/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
1
drizzle/20261024172700_music_lease_holder/migration.sql
Normal file
1
drizzle/20261024172700_music_lease_holder/migration.sql
Normal file
@@ -0,0 +1 @@
|
||||
ALTER TABLE "music"."spotify_auth" ADD COLUMN "lease_holder" text;
|
||||
20287
drizzle/20261024172700_music_lease_holder/snapshot.json
Normal file
20287
drizzle/20261024172700_music_lease_holder/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
290
drizzle/20261024174805_hardware/migration.sql
Normal file
290
drizzle/20261024174805_hardware/migration.sql
Normal file
@@ -0,0 +1,290 @@
|
||||
-- Hand-reconciled: the hardware schema already existed (hand-made 2026-10-22 from Stan's list +
|
||||
-- order mails, see ~/dev/hardware/schema.sql; prices later moved to cents). This brings an existing
|
||||
-- schema to the Drizzle model, and creates it from scratch on an empty database.
|
||||
CREATE SCHEMA IF NOT EXISTS "hardware";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE IF NOT EXISTS "hardware"."devices" (
|
||||
"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "hardware"."devices_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
|
||||
"name" text NOT NULL UNIQUE,
|
||||
"label" text,
|
||||
"hostnames" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"kind" text NOT NULL,
|
||||
"status" text DEFAULT 'active' NOT NULL,
|
||||
"brand" text,
|
||||
"model" text,
|
||||
"specs" jsonb DEFAULT '{}' NOT NULL,
|
||||
"price_cents" integer,
|
||||
"monthly_cents" integer,
|
||||
"budget_cents" integer,
|
||||
"shop" text,
|
||||
"purchase_date" date,
|
||||
"url" text,
|
||||
"image_url" text,
|
||||
"sort_order" integer DEFAULT 100 NOT NULL,
|
||||
"notes" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "devices_kind_check" CHECK (kind in ('desktop', 'laptop', 'server', 'vm', 'phone', 'tablet', 'monitor', 'other')),
|
||||
CONSTRAINT "devices_status_check" CHECK (status in ('active', 'spare', 'planned', 'retired', 'sold')),
|
||||
CONSTRAINT "devices_cents_check" CHECK ((price_cents is null or price_cents >= 0) and (monthly_cents is null or monthly_cents >= 0) and (budget_cents is null or budget_cents >= 0))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE IF NOT EXISTS "hardware"."components" (
|
||||
"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "hardware"."components_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
|
||||
"device_id" bigint NOT NULL,
|
||||
"category" text NOT NULL,
|
||||
"brand" text,
|
||||
"model" text NOT NULL,
|
||||
"quantity" integer DEFAULT 1 NOT NULL,
|
||||
"specs" jsonb DEFAULT '{}' NOT NULL,
|
||||
"unit_price_cents" integer,
|
||||
"total_price_cents" integer,
|
||||
"shop" text,
|
||||
"order_number" text,
|
||||
"order_date" date,
|
||||
"warranty_until" date,
|
||||
"invoice_number" text,
|
||||
"status" text DEFAULT 'in_use' NOT NULL,
|
||||
"verified" boolean DEFAULT true NOT NULL,
|
||||
"dpi" integer,
|
||||
"measured_dpi" integer,
|
||||
"url" text,
|
||||
"image_url" text,
|
||||
"notes" text,
|
||||
"source" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "components_dedupe_uq" UNIQUE NULLS NOT DISTINCT("device_id","model","order_number"),
|
||||
CONSTRAINT "components_category_check" CHECK (category in ('case', 'cpu', 'motherboard', 'ram', 'cpu_cooler', 'gpu', 'psu', 'storage', 'fan', 'lighting', 'monitor', 'mouse', 'keyboard', 'headset', 'audio', 'controller', 'accessory', 'other')),
|
||||
CONSTRAINT "components_status_check" CHECK (status in ('in_use', 'planned', 'spare', 'retired', 'sold')),
|
||||
CONSTRAINT "components_quantity_check" CHECK (quantity > 0),
|
||||
CONSTRAINT "components_unit_price_cents_check" CHECK (unit_price_cents >= 0),
|
||||
CONSTRAINT "components_total_price_cents_check" CHECK (total_price_cents >= 0),
|
||||
CONSTRAINT "components_dpi_check" CHECK ((dpi is null or dpi > 0) and (measured_dpi is null or measured_dpi > 0))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE IF NOT EXISTS "hardware"."sensitivities" (
|
||||
"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "hardware"."sensitivities_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
|
||||
"game" text NOT NULL UNIQUE,
|
||||
"cm_per_360" numeric(7,2),
|
||||
"yaw" numeric(10,6),
|
||||
"in_game_sens" numeric(10,4),
|
||||
"mouse_id" bigint,
|
||||
"fov" text,
|
||||
"notes" text,
|
||||
"sort_order" integer DEFAULT 100 NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
CONSTRAINT "sensitivities_cm_check" CHECK (cm_per_360 is null or cm_per_360 > 0),
|
||||
CONSTRAINT "sensitivities_yaw_check" CHECK (yaw is null or yaw > 0)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
-- Existing tables: new columns.
|
||||
ALTER TABLE "hardware"."devices"
|
||||
ADD COLUMN IF NOT EXISTS "label" text,
|
||||
ADD COLUMN IF NOT EXISTS "hostnames" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
ADD COLUMN IF NOT EXISTS "status" text DEFAULT 'active' NOT NULL,
|
||||
ADD COLUMN IF NOT EXISTS "brand" text,
|
||||
ADD COLUMN IF NOT EXISTS "model" text,
|
||||
ADD COLUMN IF NOT EXISTS "specs" jsonb DEFAULT '{}' NOT NULL,
|
||||
ADD COLUMN IF NOT EXISTS "price_cents" integer,
|
||||
ADD COLUMN IF NOT EXISTS "monthly_cents" integer,
|
||||
ADD COLUMN IF NOT EXISTS "budget_cents" integer,
|
||||
ADD COLUMN IF NOT EXISTS "shop" text,
|
||||
ADD COLUMN IF NOT EXISTS "purchase_date" date,
|
||||
ADD COLUMN IF NOT EXISTS "url" text,
|
||||
ADD COLUMN IF NOT EXISTS "image_url" text,
|
||||
ADD COLUMN IF NOT EXISTS "sort_order" integer DEFAULT 100 NOT NULL,
|
||||
ADD COLUMN IF NOT EXISTS "updated_at" timestamp with time zone DEFAULT now() NOT NULL;
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "hardware"."components"
|
||||
ADD COLUMN IF NOT EXISTS "dpi" integer,
|
||||
ADD COLUMN IF NOT EXISTS "measured_dpi" integer,
|
||||
ADD COLUMN IF NOT EXISTS "url" text,
|
||||
ADD COLUMN IF NOT EXISTS "image_url" text;
|
||||
--> statement-breakpoint
|
||||
-- Checks: widen kinds/categories/statuses (drop + re-add by name).
|
||||
ALTER TABLE "hardware"."devices" DROP CONSTRAINT IF EXISTS "devices_kind_check";
|
||||
ALTER TABLE "hardware"."devices" ADD CONSTRAINT "devices_kind_check" CHECK (kind in ('desktop', 'laptop', 'server', 'vm', 'phone', 'tablet', 'monitor', 'other'));
|
||||
ALTER TABLE "hardware"."devices" DROP CONSTRAINT IF EXISTS "devices_status_check";
|
||||
ALTER TABLE "hardware"."devices" ADD CONSTRAINT "devices_status_check" CHECK (status in ('active', 'spare', 'planned', 'retired', 'sold'));
|
||||
ALTER TABLE "hardware"."devices" DROP CONSTRAINT IF EXISTS "devices_cents_check";
|
||||
ALTER TABLE "hardware"."devices" ADD CONSTRAINT "devices_cents_check" CHECK ((price_cents is null or price_cents >= 0) and (monthly_cents is null or monthly_cents >= 0) and (budget_cents is null or budget_cents >= 0));
|
||||
ALTER TABLE "hardware"."components" DROP CONSTRAINT IF EXISTS "components_category_check";
|
||||
ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_category_check" CHECK (category in ('case', 'cpu', 'motherboard', 'ram', 'cpu_cooler', 'gpu', 'psu', 'storage', 'fan', 'lighting', 'monitor', 'mouse', 'keyboard', 'headset', 'audio', 'controller', 'accessory', 'other'));
|
||||
ALTER TABLE "hardware"."components" DROP CONSTRAINT IF EXISTS "components_status_check";
|
||||
ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_status_check" CHECK (status in ('in_use', 'planned', 'spare', 'retired', 'sold'));
|
||||
ALTER TABLE "hardware"."components" DROP CONSTRAINT IF EXISTS "components_dpi_check";
|
||||
ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_dpi_check" CHECK ((dpi is null or dpi > 0) and (measured_dpi is null or measured_dpi > 0));
|
||||
--> statement-breakpoint
|
||||
-- Dedupe key: unique index -> constraint (same name, same NULLS NOT DISTINCT semantics);
|
||||
-- device FK -> Drizzle's name.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'components_dedupe_uq' AND connamespace = 'hardware'::regnamespace) THEN
|
||||
DROP INDEX IF EXISTS "hardware"."components_dedupe_uq";
|
||||
ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_dedupe_uq" UNIQUE NULLS NOT DISTINCT ("device_id","model","order_number");
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'components_device_id_fkey' AND connamespace = 'hardware'::regnamespace) THEN
|
||||
ALTER TABLE "hardware"."components" RENAME CONSTRAINT "components_device_id_fkey" TO "components_device_id_devices_id_fkey";
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'components_device_id_devices_id_fkey' AND connamespace = 'hardware'::regnamespace) THEN
|
||||
ALTER TABLE "hardware"."components" ADD CONSTRAINT "components_device_id_devices_id_fkey" FOREIGN KEY ("device_id") REFERENCES "hardware"."devices"("id") ON DELETE CASCADE;
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'sensitivities_mouse_id_components_id_fkey' AND connamespace = 'hardware'::regnamespace) THEN
|
||||
ALTER TABLE "hardware"."sensitivities" ADD CONSTRAINT "sensitivities_mouse_id_components_id_fkey" FOREIGN KEY ("mouse_id") REFERENCES "hardware"."components"("id") ON DELETE SET NULL;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "components_device_idx" ON "hardware"."components" ("device_id");
|
||||
CREATE INDEX IF NOT EXISTS "components_category_idx" ON "hardware"."components" ("category");
|
||||
CREATE INDEX IF NOT EXISTS "components_specs_gin" ON "hardware"."components" USING gin ("specs");
|
||||
--> statement-breakpoint
|
||||
-- Hand-written: updated_at triggers, overview view, grants, GUI registration, seed.
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
FOREACH t IN ARRAY ARRAY['devices','components','sensitivities'] LOOP
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = 'touch_updated_at' AND tgrelid = format('hardware.%I', t)::regclass) THEN
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON hardware.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DROP VIEW IF EXISTS "hardware"."setup_overview";
|
||||
--> statement-breakpoint
|
||||
CREATE VIEW "hardware"."setup_overview" AS
|
||||
SELECT d.name AS device,
|
||||
d.label AS device_label,
|
||||
d.kind AS device_kind,
|
||||
d.status AS device_status,
|
||||
c.category,
|
||||
c.brand,
|
||||
c.model,
|
||||
c.quantity,
|
||||
c.total_price_cents,
|
||||
round(c.total_price_cents::numeric / 100.0, 2) AS total_price_eur,
|
||||
c.shop,
|
||||
c.order_date,
|
||||
c.warranty_until,
|
||||
(c.warranty_until >= current_date) AS under_warranty,
|
||||
c.status,
|
||||
c.verified,
|
||||
c.specs,
|
||||
c.dpi,
|
||||
c.measured_dpi,
|
||||
c.notes,
|
||||
c.order_number,
|
||||
c.invoice_number,
|
||||
c.source,
|
||||
c.id AS component_id
|
||||
FROM hardware.components c
|
||||
JOIN hardware.devices d ON d.id = c.device_id
|
||||
ORDER BY d.sort_order, d.name,
|
||||
array_position(ARRAY['case','cpu','motherboard','ram','cpu_cooler','gpu','psu','storage','fan','lighting','monitor','mouse','keyboard','headset','audio','controller','accessory','other'], c.category),
|
||||
c.order_date NULLS LAST, c.model;
|
||||
--> statement-breakpoint
|
||||
COMMENT ON VIEW "hardware"."setup_overview" IS 'Flat view of every component per device (build order), with a live under_warranty flag. status = ''in_use'' is the current setup, ''planned'' the upgrade plan.';
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA hardware TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA hardware TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA hardware TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA hardware GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA hardware GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT USAGE ON SCHEMA hardware TO personal_ro;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA hardware TO personal_ro;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA hardware GRANT SELECT ON TABLES TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('hardware','devices','hardware-devices','Hardware devices','core',true,false,1000),
|
||||
('hardware','components','hardware-components','Hardware components','core',true,false,1000),
|
||||
('hardware','sensitivities','hardware-sensitivities','Game sensitivities (cm/360)','core',true,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
-- ===== Seed (idempotent). Facts from Stan's own list, order mails and what he told Daisy;
|
||||
-- unknowns stay null. No home address in public-facing notes.
|
||||
INSERT INTO hardware.devices (name, kind) VALUES ('stan-pc','desktop'), ('old-pc-2021','desktop')
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
--> statement-breakpoint
|
||||
UPDATE hardware.devices SET
|
||||
label = 'Main PC', hostnames = '{stan-arch,stan-win11}', status = 'active', model = 'Custom build',
|
||||
shop = 'Megekko', sort_order = 10,
|
||||
notes = 'Stan''s main PC: Lian Li O11 Dynamic XL ROG White, i9-12900KF, RX 7900 XT, 10 Lian Li fans around the case + 3 NZXT Aer on the radiator. Dual-boots Arch (stan-arch) and Windows 11 + WSL2 (stan-win11). Desk peripherals and the Odyssey G9 are tracked here. Mostly bought at Megekko 2021-2022.'
|
||||
WHERE name = 'stan-pc' AND label IS NULL;
|
||||
--> statement-breakpoint
|
||||
UPDATE hardware.devices SET
|
||||
label = 'Previous PC (2021)', status = 'retired', model = 'Budget build', shop = 'Megekko',
|
||||
purchase_date = '2021-01-31', sort_order = 90
|
||||
WHERE name = 'old-pc-2021' AND label IS NULL;
|
||||
--> statement-breakpoint
|
||||
INSERT INTO hardware.devices (name, label, hostnames, kind, status, brand, model, specs, price_cents, monthly_cents, budget_cents, shop, purchase_date, url, sort_order, notes) VALUES
|
||||
('stan-pc-next','Next PC (planned)','{}','desktop','planned',NULL,'High-end AMD + RTX 5090 build','{}',NULL,NULL,1000000,NULL,NULL,NULL,15,
|
||||
'Planned upgrade, budget up to about EUR 10k: top AMD X3D CPU, RTX 5090, fast DDR5, PCIe Gen5 SSD, high-end motherboard and a cooler that fits. Parts not picked yet.'),
|
||||
('stan-ovh','Server','{}','server','active','OVHcloud','RISE-S',
|
||||
'{"cpu":"AMD Ryzen 7 9700X","ram":"64 GB DDR5 ECC","storage":"2x 512 GB NVMe (software RAID1)","os":"Ubuntu 26.04"}',
|
||||
NULL,6174,NULL,'OVHcloud','2026-10-24','https://www.ovhcloud.com/en/bare-metal/rise/',20,
|
||||
'Dedicated server, prod since 2026-10-24 (replaced stan-vm). EUR 61.74/month excl. VAT. Runs Postgres, Gitea, the website and the bots.'),
|
||||
('stan-vm','Old VM','{}','vm','retired','Hetzner Cloud',NULL,'{"vcpu":4,"ram":"8 GB"}',NULL,2000,NULL,'Hetzner',NULL,NULL,80,
|
||||
'Old always-on Hetzner VM (about EUR 20/month, bought before the price rises). Replaced by stan-ovh on 2026-10-24; stopped, kept as a fallback until Stan cancels it.'),
|
||||
('stan-macbookpro','MacBook Pro','{}','laptop','active','Apple','MacBook Pro','{}',NULL,NULL,NULL,NULL,NULL,NULL,30,
|
||||
'Exact model/chip not recorded yet. An M5 Max MacBook Pro is planned.'),
|
||||
('stan-pixel','Phone','{}','phone','active','Google','Pixel 11 Pro XL','{"storage":"256 GB"}',NULL,NULL,NULL,'Belsimpel','2026-10-18',NULL,40,
|
||||
'Main phone since 2026-10-19, stock Android. About EUR 1,299 SIM-free (exact amount not recorded).'),
|
||||
('stan-iphone','Old phone','{}','phone','spare','Apple','iPhone 14 Pro Max','{}',NULL,NULL,NULL,NULL,NULL,NULL,45,
|
||||
'Previous phone, kept (weak battery and speed).'),
|
||||
('stan-ipad','iPad','{}','tablet','active','Apple','iPad Air 13-inch (M2)','{"screen_in":13,"chip":"Apple M2"}',NULL,NULL,NULL,NULL,NULL,NULL,50,
|
||||
'With a keyboard case; used for typing on the go and Termius SSH.'),
|
||||
('cargill-laptop','Work laptop','{}','laptop','active',NULL,NULL,'{"ram":"16 GB"}',NULL,NULL,NULL,NULL,NULL,NULL,60,
|
||||
'Cargill work laptop (not personal; Windows + WSL2). Model not recorded.')
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
--> statement-breakpoint
|
||||
-- Peripherals get real categories instead of "accessory".
|
||||
UPDATE hardware.components SET category = 'mouse' WHERE category = 'accessory' AND model ILIKE 'MX Master%';
|
||||
--> statement-breakpoint
|
||||
UPDATE hardware.components SET category = 'keyboard' WHERE category = 'accessory' AND (model ILIKE 'Alloy Origins%' OR model ILIKE 'Pudding Keycaps%' OR model ILIKE 'UwU%');
|
||||
--> statement-breakpoint
|
||||
UPDATE hardware.components SET category = 'controller' WHERE category = 'accessory' AND brand = 'Thrustmaster';
|
||||
--> statement-breakpoint
|
||||
UPDATE hardware.components SET category = 'audio' WHERE category = 'accessory' AND model ILIKE '%mic boom arm%';
|
||||
--> statement-breakpoint
|
||||
UPDATE hardware.components SET notes = replace(notes, ', delivered 2026-03-03 to Houtenstraat 15 Gouda.', ', delivered 2026-03-03.') WHERE notes LIKE '%Houtenstraat%';
|
||||
--> statement-breakpoint
|
||||
INSERT INTO hardware.components (device_id, category, brand, model, quantity, specs, status, verified, notes, source)
|
||||
SELECT d.id, v.category, v.brand, v.model, v.quantity, v.specs::jsonb, v.status, true, v.notes, v.source
|
||||
FROM (VALUES
|
||||
('stan-ovh','cpu','AMD','Ryzen 7 9700X',1,'{"cores":8,"threads":16,"socket":"AM5"}','in_use',NULL,'manual'),
|
||||
('stan-ovh','ram',NULL,'64 GB DDR5 ECC',1,'{"type":"DDR5 ECC","capacity_gb":64}','in_use',NULL,'manual'),
|
||||
('stan-ovh','storage',NULL,'512 GB NVMe SSD',2,'{"interface":"NVMe","capacity_gb":512,"raid":"software RAID1"}','in_use','Mirrored (RAID1), about 440 GB usable for data.','manual'),
|
||||
('stan-pc-next','cpu','AMD','Top Ryzen X3D (model TBD)',1,'{"socket":"AM5"}','planned','Pick the fastest gaming X3D chip available when buying.','plan'),
|
||||
('stan-pc-next','gpu','NVIDIA','GeForce RTX 5090',1,'{"vram_gb":32}','planned','Board partner TBD.','plan'),
|
||||
('stan-pc-next','motherboard',NULL,'High-end AM5 motherboard (TBD)',1,'{"socket":"AM5"}','planned',NULL,'plan'),
|
||||
('stan-pc-next','ram',NULL,'Fast DDR5 kit (TBD)',1,'{"type":"DDR5"}','planned',NULL,'plan'),
|
||||
('stan-pc-next','storage',NULL,'PCIe 5.0 NVMe SSD (TBD)',1,'{"interface":"PCIe 5.0 x4 NVMe"}','planned',NULL,'plan'),
|
||||
('stan-pc-next','cpu_cooler',NULL,'Cooler for the new CPU (TBD)',1,'{}','planned','Must fit the new socket and the O11 Dynamic XL (or the new case).','plan')
|
||||
) AS v(device, category, brand, model, quantity, specs, status, notes, source)
|
||||
JOIN hardware.devices d ON d.name = v.device
|
||||
ON CONFLICT (device_id, model, order_number) DO NOTHING;
|
||||
--> statement-breakpoint
|
||||
-- Games Stan plays; cm/360 left empty for him to fill. CS2's yaw (0.022 deg/count) is the
|
||||
-- Source engine constant, so its in-game value is computed from a mouse's measured DPI.
|
||||
INSERT INTO hardware.sensitivities (game, yaw, sort_order) VALUES
|
||||
('Counter-Strike 2', 0.022, 10),
|
||||
('Call of Duty', NULL, 20),
|
||||
('osu!', NULL, 30),
|
||||
('Sea of Thieves', NULL, 40)
|
||||
ON CONFLICT (game) DO NOTHING;
|
||||
21290
drizzle/20261024174805_hardware/snapshot.json
Normal file
21290
drizzle/20261024174805_hardware/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
140
drizzle/20261024193304_planes/migration.sql
Normal file
140
drizzle/20261024193304_planes/migration.sql
Normal file
@@ -0,0 +1,140 @@
|
||||
CREATE SCHEMA "planes";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "planes"."makers" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"full_name" text,
|
||||
"country" text,
|
||||
"wikidata" text,
|
||||
"wikipedia" text,
|
||||
"type_count" integer DEFAULT 0 NOT NULL,
|
||||
"first_year" smallint,
|
||||
"last_year" smallint,
|
||||
"built_total" integer DEFAULT 0 NOT NULL,
|
||||
"sources" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"logo" text,
|
||||
"logo_file" text,
|
||||
"logo_author" text,
|
||||
"logo_license" text,
|
||||
"photo" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "planes"."photos" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"type_id" integer NOT NULL,
|
||||
"position" smallint NOT NULL,
|
||||
"kind" text NOT NULL,
|
||||
"file" text NOT NULL,
|
||||
"caption" text,
|
||||
"thumb" text NOT NULL,
|
||||
"width" integer NOT NULL,
|
||||
"height" integer NOT NULL,
|
||||
"author" text,
|
||||
"license" text,
|
||||
"license_url" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "planes"."types" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"dex" integer NOT NULL,
|
||||
"maker_id" integer NOT NULL,
|
||||
"family_id" integer,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"full_name" text NOT NULL,
|
||||
"search" text NOT NULL,
|
||||
"wikidata" text NOT NULL,
|
||||
"wikipedia" text,
|
||||
"kind" text,
|
||||
"description" text,
|
||||
"summary" text,
|
||||
"role" text,
|
||||
"roles" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"role_label" text,
|
||||
"country" text,
|
||||
"first_flight" text,
|
||||
"first_flight_year" smallint,
|
||||
"introduced_year" smallint,
|
||||
"retired_year" smallint,
|
||||
"status" text,
|
||||
"in_service" boolean,
|
||||
"number_built" integer,
|
||||
"makers" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"engine_type" text,
|
||||
"engine_count" smallint,
|
||||
"engines" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"crew" smallint,
|
||||
"passengers" smallint,
|
||||
"length_m" real,
|
||||
"wingspan_m" real,
|
||||
"height_m" real,
|
||||
"rotor_m" real,
|
||||
"mtow_kg" integer,
|
||||
"max_speed_kmh" smallint,
|
||||
"cruise_kmh" smallint,
|
||||
"range_km" integer,
|
||||
"ceiling_m" integer,
|
||||
"wing" text,
|
||||
"gear" text,
|
||||
"icao" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"recognize" text,
|
||||
"sources" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"photo" text,
|
||||
"photo_count" smallint DEFAULT 0 NOT NULL,
|
||||
"photos_checked_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "makers_slug_key" ON "planes"."makers" ("slug");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "photos_type_position_key" ON "planes"."photos" ("type_id","position");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "types_dex_key" ON "planes"."types" ("dex");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "types_wikidata_key" ON "planes"."types" ("wikidata");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "types_maker_slug_key" ON "planes"."types" ("maker_id","slug");--> statement-breakpoint
|
||||
CREATE INDEX "types_number_built_idx" ON "planes"."types" ("number_built");--> statement-breakpoint
|
||||
CREATE INDEX "types_first_flight_year_idx" ON "planes"."types" ("first_flight_year");--> statement-breakpoint
|
||||
CREATE INDEX "types_search_trgm_idx" ON "planes"."types" USING gin ("search" public.gin_trgm_ops);--> statement-breakpoint
|
||||
ALTER TABLE "planes"."photos" ADD CONSTRAINT "photos_type_id_types_id_fkey" FOREIGN KEY ("type_id") REFERENCES "planes"."types"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "planes"."types" ADD CONSTRAINT "types_maker_id_makers_id_fkey" FOREIGN KEY ("maker_id") REFERENCES "planes"."makers"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "planes"."types" ADD CONSTRAINT "types_family_id_types_id_fkey" FOREIGN KEY ("family_id") REFERENCES "planes"."types"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
-- Hand-written: updated_at triggers, grants, GUI registration. Data comes from scripts/planes-import.ts and scripts/planes-photos.ts.
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
FOREACH t IN ARRAY ARRAY['makers','types'] LOOP
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON planes.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END LOOP;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA planes TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA planes TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA planes TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA planes GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA planes GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT USAGE ON SCHEMA planes TO personal_ro;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA planes TO personal_ro;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA planes GRANT SELECT ON TABLES TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('planes','makers','planes-makers','Plane makers','core',false,false,1000),
|
||||
('planes','types','planes-types','Plane types','core',false,false,1000),
|
||||
('planes','photos','planes-photos','Plane photos','core',false,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
22615
drizzle/20261024193304_planes/snapshot.json
Normal file
22615
drizzle/20261024193304_planes/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
122
drizzle/20261024215408_wildlife/migration.sql
Normal file
122
drizzle/20261024215408_wildlife/migration.sql
Normal file
@@ -0,0 +1,122 @@
|
||||
CREATE SCHEMA "wildlife";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "wildlife"."groups" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"name_nl" text NOT NULL,
|
||||
"emoji" text NOT NULL,
|
||||
"sort_order" smallint DEFAULT 0 NOT NULL,
|
||||
"species_count" integer DEFAULT 0 NOT NULL,
|
||||
"nl_count" integer DEFAULT 0 NOT NULL,
|
||||
"photo" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "wildlife"."photos" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"species_id" integer NOT NULL,
|
||||
"position" smallint NOT NULL,
|
||||
"kind" text NOT NULL,
|
||||
"photo_id" integer NOT NULL,
|
||||
"url" text NOT NULL,
|
||||
"width" integer,
|
||||
"height" integer,
|
||||
"author" text,
|
||||
"license" text NOT NULL,
|
||||
"page_url" text,
|
||||
"place" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "wildlife"."species" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"dex" integer NOT NULL,
|
||||
"group_id" integer NOT NULL,
|
||||
"slug" text NOT NULL,
|
||||
"scientific_name" text NOT NULL,
|
||||
"name_en" text,
|
||||
"name_nl" text,
|
||||
"search" text NOT NULL,
|
||||
"inat_id" integer NOT NULL,
|
||||
"gbif_key" integer,
|
||||
"wikidata" text,
|
||||
"wikipedia_en" text,
|
||||
"wikipedia_nl" text,
|
||||
"class_name" text,
|
||||
"order_name" text,
|
||||
"family_name" text,
|
||||
"iucn" text,
|
||||
"nl_status" text,
|
||||
"nl_establishment" text,
|
||||
"extinct" boolean DEFAULT false NOT NULL,
|
||||
"nl_obs" integer DEFAULT 0 NOT NULL,
|
||||
"europe_obs" integer DEFAULT 0 NOT NULL,
|
||||
"world_obs" integer DEFAULT 0 NOT NULL,
|
||||
"gbif_nl" integer,
|
||||
"scope" text NOT NULL,
|
||||
"summary" text,
|
||||
"summary_nl" text,
|
||||
"facts" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"mass_kg" real,
|
||||
"length_m" real,
|
||||
"lifespan_years" real,
|
||||
"named_year" smallint,
|
||||
"named_by" text,
|
||||
"sources" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"photo" text,
|
||||
"photo_count" smallint DEFAULT 0 NOT NULL,
|
||||
"photos_checked_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "groups_slug_key" ON "wildlife"."groups" ("slug");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "photos_species_position_key" ON "wildlife"."photos" ("species_id","position");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "photos_species_photo_key" ON "wildlife"."photos" ("species_id","photo_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "species_dex_key" ON "wildlife"."species" ("dex");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "species_inat_key" ON "wildlife"."species" ("inat_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "species_slug_key" ON "wildlife"."species" ("slug");--> statement-breakpoint
|
||||
CREATE INDEX "species_group_idx" ON "wildlife"."species" ("group_id");--> statement-breakpoint
|
||||
CREATE INDEX "species_nl_obs_idx" ON "wildlife"."species" ("nl_obs");--> statement-breakpoint
|
||||
CREATE INDEX "species_search_trgm_idx" ON "wildlife"."species" USING gin ("search" public.gin_trgm_ops);--> statement-breakpoint
|
||||
ALTER TABLE "wildlife"."photos" ADD CONSTRAINT "photos_species_id_species_id_fkey" FOREIGN KEY ("species_id") REFERENCES "wildlife"."species"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "wildlife"."species" ADD CONSTRAINT "species_group_id_groups_id_fkey" FOREIGN KEY ("group_id") REFERENCES "wildlife"."groups"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
-- Hand-written: updated_at triggers, grants, GUI registration. Data comes from scripts/wildlife-import.ts and scripts/wildlife-photos.ts.
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
FOREACH t IN ARRAY ARRAY['groups','species'] LOOP
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON wildlife.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END LOOP;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA wildlife TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA wildlife TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA wildlife TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA wildlife GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA wildlife GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT USAGE ON SCHEMA wildlife TO personal_ro;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA wildlife TO personal_ro;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA wildlife GRANT SELECT ON TABLES TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('wildlife','groups','wildlife-groups','Wildlife groups','core',false,false,1000),
|
||||
('wildlife','species','wildlife-species','Wildlife species','core',false,false,1000),
|
||||
('wildlife','photos','wildlife-photos','Wildlife photos','core',false,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
23710
drizzle/20261024215408_wildlife/snapshot.json
Normal file
23710
drizzle/20261024215408_wildlife/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
167
drizzle/20261024232143_trains/migration.sql
Normal file
167
drizzle/20261024232143_trains/migration.sql
Normal file
@@ -0,0 +1,167 @@
|
||||
CREATE SCHEMA "trains";
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "trains"."makers" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"full_name" text,
|
||||
"country" text,
|
||||
"wikidata" text,
|
||||
"wikipedia" text,
|
||||
"type_count" integer DEFAULT 0 NOT NULL,
|
||||
"first_year" smallint,
|
||||
"last_year" smallint,
|
||||
"built_total" integer DEFAULT 0 NOT NULL,
|
||||
"sources" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"logo" text,
|
||||
"logo_file" text,
|
||||
"logo_author" text,
|
||||
"logo_license" text,
|
||||
"photo" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "trains"."operators" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"full_name" text,
|
||||
"country" text,
|
||||
"wikidata" text,
|
||||
"wikipedia" text,
|
||||
"type_count" integer DEFAULT 0 NOT NULL,
|
||||
"logo" text,
|
||||
"logo_file" text,
|
||||
"logo_author" text,
|
||||
"logo_license" text,
|
||||
"photo" text,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "trains"."photos" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"type_id" integer NOT NULL,
|
||||
"position" smallint NOT NULL,
|
||||
"kind" text NOT NULL,
|
||||
"file" text NOT NULL,
|
||||
"caption" text,
|
||||
"thumb" text NOT NULL,
|
||||
"width" integer NOT NULL,
|
||||
"height" integer NOT NULL,
|
||||
"author" text,
|
||||
"license" text,
|
||||
"license_url" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "trains"."type_operators" (
|
||||
"type_id" integer,
|
||||
"operator_id" integer,
|
||||
"position" smallint DEFAULT 0 NOT NULL,
|
||||
CONSTRAINT "type_operators_pkey" PRIMARY KEY("type_id","operator_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "trains"."types" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"dex" integer NOT NULL,
|
||||
"maker_id" integer NOT NULL,
|
||||
"family_id" integer,
|
||||
"slug" text NOT NULL,
|
||||
"name" text NOT NULL,
|
||||
"full_name" text NOT NULL,
|
||||
"nl_name" text,
|
||||
"search" text NOT NULL,
|
||||
"wikidata" text NOT NULL,
|
||||
"wikipedia" text,
|
||||
"description" text,
|
||||
"summary" text,
|
||||
"category" text NOT NULL,
|
||||
"category_label" text,
|
||||
"traction" text,
|
||||
"country" text,
|
||||
"countries" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"makers" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"operators" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"first_built" smallint,
|
||||
"last_built" smallint,
|
||||
"introduced_year" smallint,
|
||||
"retired_year" smallint,
|
||||
"status" text,
|
||||
"in_service" boolean,
|
||||
"number_built" integer,
|
||||
"max_speed_kmh" smallint,
|
||||
"power_kw" integer,
|
||||
"gauge" text,
|
||||
"gauge_mm" smallint,
|
||||
"electric_system" text,
|
||||
"wheels" text,
|
||||
"length_m" real,
|
||||
"weight_t" real,
|
||||
"capacity" smallint,
|
||||
"formation" text,
|
||||
"designer" text,
|
||||
"nicknames" text,
|
||||
"sources" text[] DEFAULT '{}'::text[] NOT NULL,
|
||||
"photo" text,
|
||||
"photo_count" smallint DEFAULT 0 NOT NULL,
|
||||
"photos_checked_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "makers_slug_key" ON "trains"."makers" ("slug");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "operators_slug_key" ON "trains"."operators" ("slug");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "photos_type_position_key" ON "trains"."photos" ("type_id","position");--> statement-breakpoint
|
||||
CREATE INDEX "type_operators_operator_idx" ON "trains"."type_operators" ("operator_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "types_dex_key" ON "trains"."types" ("dex");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "types_wikidata_key" ON "trains"."types" ("wikidata");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "types_maker_slug_key" ON "trains"."types" ("maker_id","slug");--> statement-breakpoint
|
||||
CREATE INDEX "types_number_built_idx" ON "trains"."types" ("number_built");--> statement-breakpoint
|
||||
CREATE INDEX "types_first_built_idx" ON "trains"."types" ("first_built");--> statement-breakpoint
|
||||
CREATE INDEX "types_category_idx" ON "trains"."types" ("category");--> statement-breakpoint
|
||||
CREATE INDEX "types_search_trgm_idx" ON "trains"."types" USING gin ("search" public.gin_trgm_ops);--> statement-breakpoint
|
||||
ALTER TABLE "trains"."photos" ADD CONSTRAINT "photos_type_id_types_id_fkey" FOREIGN KEY ("type_id") REFERENCES "trains"."types"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "trains"."type_operators" ADD CONSTRAINT "type_operators_type_id_types_id_fkey" FOREIGN KEY ("type_id") REFERENCES "trains"."types"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "trains"."type_operators" ADD CONSTRAINT "type_operators_operator_id_operators_id_fkey" FOREIGN KEY ("operator_id") REFERENCES "trains"."operators"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "trains"."types" ADD CONSTRAINT "types_maker_id_makers_id_fkey" FOREIGN KEY ("maker_id") REFERENCES "trains"."makers"("id") ON DELETE CASCADE;--> statement-breakpoint
|
||||
ALTER TABLE "trains"."types" ADD CONSTRAINT "types_family_id_types_id_fkey" FOREIGN KEY ("family_id") REFERENCES "trains"."types"("id") ON DELETE SET NULL;--> statement-breakpoint
|
||||
-- Hand-written: updated_at triggers, grants, GUI registration. Data comes from scripts/trains-import.ts and scripts/trains-photos.ts.
|
||||
DO $$
|
||||
DECLARE t text;
|
||||
BEGIN
|
||||
FOREACH t IN ARRAY ARRAY['makers','operators','types'] LOOP
|
||||
EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON trains.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
|
||||
END LOOP;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT USAGE ON SCHEMA trains TO personal_app;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA trains TO personal_app;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA trains TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA trains GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA trains GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT USAGE ON SCHEMA trains TO personal_ro;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA trains TO personal_ro;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA trains GRANT SELECT ON TABLES TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('trains','makers','trains-makers','Train makers','core',false,false,1000),
|
||||
('trains','operators','trains-operators','Train operators','core',false,false,1000),
|
||||
('trains','types','trains-types','Train types','core',false,false,1000),
|
||||
('trains','photos','trains-photos','Train photos','core',false,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
|
||||
PERFORM meta.sync();
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
25347
drizzle/20261024232143_trains/snapshot.json
Normal file
25347
drizzle/20261024232143_trains/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -34,6 +34,6 @@ export default [
|
||||
}
|
||||
},
|
||||
{
|
||||
ignores: ['build/', '.svelte-kit/', 'dist/']
|
||||
ignores: ['build/', '.svelte-kit/', 'dist/', 'static/vendor/']
|
||||
}
|
||||
];
|
||||
|
||||
19
extras/newtab-extension/README.md
Normal file
19
extras/newtab-extension/README.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# New-tab extension
|
||||
|
||||
A minimal Chrome / Edge / Brave (Manifest V3) extension that makes every new tab open
|
||||
[stanrunge.dev/start](https://stanrunge.dev/start), the private dashboard. No permissions,
|
||||
no tracking: the new-tab page just redirects.
|
||||
|
||||
## Install (load unpacked)
|
||||
|
||||
1. Download and unzip <https://stanrunge.dev/newtab-extension.zip> (or use this folder from the repo).
|
||||
2. Open `chrome://extensions` and switch on **Developer mode** (top right).
|
||||
3. Click **Load unpacked** and pick the unzipped `newtab-extension` folder.
|
||||
4. Open a new tab. Chrome asks once whether to keep the changed new-tab page: choose **Keep it**.
|
||||
|
||||
Log in on stanrunge.dev once in that browser; after that the dashboard opens straight away.
|
||||
To point it elsewhere, edit the URL in `newtab.html` and `newtab.js`, then press the reload
|
||||
icon on the extension card. Firefox: set the homepage/new tab with an add-on such as
|
||||
"New Tab Override" instead.
|
||||
|
||||
Rebuild the zip after changes: `bash extras/newtab-extension/build.sh`.
|
||||
13
extras/newtab-extension/build.sh
Executable file
13
extras/newtab-extension/build.sh
Executable file
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# Zip the extension into static/newtab-extension.zip (served at /newtab-extension.zip).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 - <<'PY'
|
||||
import os, zipfile
|
||||
out = '../static/newtab-extension.zip'
|
||||
with zipfile.ZipFile(out, 'w', zipfile.ZIP_DEFLATED) as z:
|
||||
for name in sorted(os.listdir('newtab-extension')):
|
||||
if name != 'build.sh':
|
||||
z.write(os.path.join('newtab-extension', name))
|
||||
print('wrote static/newtab-extension.zip')
|
||||
PY
|
||||
BIN
extras/newtab-extension/icon128.png
Normal file
BIN
extras/newtab-extension/icon128.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 527 B |
8
extras/newtab-extension/manifest.json
Normal file
8
extras/newtab-extension/manifest.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "Stan's start page",
|
||||
"version": "1.0.0",
|
||||
"description": "Opens stanrunge.dev/start in every new tab.",
|
||||
"chrome_url_overrides": { "newtab": "newtab.html" },
|
||||
"icons": { "128": "icon128.png" }
|
||||
}
|
||||
17
extras/newtab-extension/newtab.html
Normal file
17
extras/newtab-extension/newtab.html
Normal file
@@ -0,0 +1,17 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>New tab</title>
|
||||
<!-- Redirect before anything paints; newtab.js does the same as a fallback. -->
|
||||
<meta http-equiv="refresh" content="0; url=https://stanrunge.dev/start" />
|
||||
<style>
|
||||
html {
|
||||
background: #000;
|
||||
color-scheme: dark;
|
||||
}
|
||||
</style>
|
||||
<script src="newtab.js"></script>
|
||||
</head>
|
||||
<body></body>
|
||||
</html>
|
||||
3
extras/newtab-extension/newtab.js
Normal file
3
extras/newtab-extension/newtab.js
Normal file
@@ -0,0 +1,3 @@
|
||||
// MV3 pages can't run inline scripts, so the redirect lives here. Change the URL to point the
|
||||
// new tab somewhere else (then reload the extension on chrome://extensions).
|
||||
location.replace('https://stanrunge.dev/start');
|
||||
17
package.json
17
package.json
@@ -20,9 +20,24 @@
|
||||
"preview:branch": "bash scripts/preview.sh",
|
||||
"db:studio": "drizzle-kit studio",
|
||||
"db:seed": "bun scripts/seed-catalog.ts",
|
||||
"cars:import": "bun scripts/cars-import.ts",
|
||||
"cars:photos": "bun scripts/cars-photos.ts",
|
||||
"cars:refresh": "scripts/cars-refresh.sh",
|
||||
"planes:import": "bun scripts/planes-import.ts",
|
||||
"planes:photos": "bun scripts/planes-photos.ts",
|
||||
"planes:refresh": "scripts/planes-refresh.sh",
|
||||
"trains:import": "bun scripts/trains-import.ts",
|
||||
"trains:photos": "bun scripts/trains-photos.ts",
|
||||
"trains:refresh": "scripts/trains-refresh.sh",
|
||||
"wildlife:import": "bun scripts/wildlife-import.ts",
|
||||
"wildlife:photos": "bun scripts/wildlife-photos.ts",
|
||||
"wildlife:refresh": "scripts/wildlife-refresh.sh",
|
||||
"db:seed:cod": "bun scripts/seed-cod-catalog.ts",
|
||||
"cs-prices:ingest": "bun scripts/ingest-cs-prices.ts",
|
||||
"house:worker": "bun scripts/house-worker.ts",
|
||||
"house": "bun scripts/house-listing.ts",
|
||||
"house:add": "bun scripts/house-listing.ts add",
|
||||
"house:list": "bun scripts/house-listing.ts list",
|
||||
"auth:role": "bun scripts/set-admin.ts",
|
||||
"catalog:build": "python3 scripts/build-pokemon-catalog.py",
|
||||
"catalog:cod": "python3 scripts/build-cod-catalog.py && prettier --write data/call-of-duty.json",
|
||||
@@ -92,9 +107,11 @@
|
||||
"drizzle-orm": "^1.0.0-rc.4",
|
||||
"echarts": "^6.1.0",
|
||||
"highlight.js": "^11.12.0",
|
||||
"jose": "^6.2.10",
|
||||
"marked": "^18.0.11",
|
||||
"marked-highlight": "^2.2.4",
|
||||
"mode-watcher": "^1.1.0",
|
||||
"pdfjs-dist": "^6.4.299",
|
||||
"postgres": "^3.4.9",
|
||||
"topojson-client": "^3.1.0",
|
||||
"world-atlas": "^2.0.2",
|
||||
|
||||
161
scripts/bank-link.ts
Normal file
161
scripts/bank-link.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
// Link a bank to the money tracker through Enable Banking. Each bank gets its own session;
|
||||
// linking one never touches another bank's session.
|
||||
//
|
||||
// bun --env-file=$HOME/.config/money-sync.env scripts/bank-link.ts Revolut (personal + joint)
|
||||
// ... scripts/bank-link.ts --bank abn ABN AMRO -> "ABN Amro"
|
||||
// ... scripts/bank-link.ts --bank knab Knab -> "Knab"
|
||||
// ... scripts/bank-link.ts --check verify key + app id only
|
||||
// ... scripts/bank-link.ts --code '<redirect URL or code>' [--yes] finish a started consent
|
||||
// ... scripts/bank-link.ts --map <uid|IBAN>=<Tracker account> [--map ...] override the mapping
|
||||
// ... scripts/bank-link.ts --aspsp '<ASPSP name>' --account <Tracker account> --country NL --user <id>
|
||||
//
|
||||
// Then sync within the hour: the first sync of an account asks for the longest history the
|
||||
// bank serves, and most banks only serve ~90 days once consent is older than an hour.
|
||||
//
|
||||
// Needs ENABLEBANKING_APP_ID, ENABLEBANKING_KEY_PATH, DB_URL (and optionally
|
||||
// ENABLEBANKING_REDIRECT_URL). Never prints the private key.
|
||||
import { createInterface } from 'node:readline/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import postgres from 'postgres';
|
||||
import {
|
||||
configFromEnv,
|
||||
createEnableBankingClient,
|
||||
parseAuthCode
|
||||
} from '../src/lib/server/enable-banking';
|
||||
import { planAccountMapping, saveSession, trackerUserId } from '../src/lib/server/bank-sync';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const flag = (name: string) => args.includes(name);
|
||||
const value = (name: string) => {
|
||||
const i = args.indexOf(name);
|
||||
return i >= 0 ? args[i + 1] : undefined;
|
||||
};
|
||||
const overrides: Record<string, string> = {};
|
||||
args.forEach((arg, i) => {
|
||||
if (arg !== '--map') return;
|
||||
const [key, ...rest] = (args[i + 1] ?? '').split('=');
|
||||
if (!key) return;
|
||||
overrides[key] = rest.join('=');
|
||||
overrides[key.replace(/\s/g, '').toUpperCase()] = rest.join('=');
|
||||
});
|
||||
|
||||
// Enable Banking ASPSP names (GET /aspsps?country=NL) and the tracker account they feed.
|
||||
const BANKS: Record<string, { aspsp: string; account: string; hint: string }> = {
|
||||
revolut: {
|
||||
aspsp: 'Revolut',
|
||||
account: 'Revolut',
|
||||
hint: 'Tick BOTH your personal account and the joint account with Juhee.'
|
||||
},
|
||||
abn: {
|
||||
aspsp: 'ABN AMRO',
|
||||
account: 'ABN Amro',
|
||||
hint: 'Log in with the ABN AMRO app or identifier and approve your current account.'
|
||||
},
|
||||
knab: {
|
||||
aspsp: 'Knab',
|
||||
account: 'Knab',
|
||||
hint: 'Approve your Knab current account (savings accounts are not needed).'
|
||||
}
|
||||
};
|
||||
const bankKey = (value('--bank') ?? 'revolut').toLowerCase().replace(/[\s-]|amro/g, '');
|
||||
const preset = value('--aspsp') && !value('--bank') ? undefined : BANKS[bankKey];
|
||||
if (!preset && !value('--aspsp'))
|
||||
throw new Error(`Unknown --bank ${value('--bank')}; use ${Object.keys(BANKS).join(', ')}`);
|
||||
const client = createEnableBankingClient(configFromEnv());
|
||||
const aspsp = {
|
||||
name: value('--aspsp') ?? preset!.aspsp,
|
||||
country: value('--country') ?? 'NL'
|
||||
};
|
||||
const defaultAccount = value('--account') ?? preset?.account ?? aspsp.name;
|
||||
const redirectUrl =
|
||||
process.env.ENABLEBANKING_REDIRECT_URL ??
|
||||
'https://stanrunge.dev/projects/money-tracker/bank-callback'; // registered with Enable Banking; 308s to /trackers/money/bank-callback with the query kept
|
||||
|
||||
const app = await client.application();
|
||||
console.log(
|
||||
`Enable Banking app "${app.name}" (${app.environment}, ${app.active ? 'active' : 'INACTIVE'}), services ${app.services.join(',')}`
|
||||
);
|
||||
const { aspsps } = await client.aspsps(aspsp.country);
|
||||
const bank = aspsps.find((a) => a.name.toLowerCase() === aspsp.name.toLowerCase());
|
||||
if (!bank) throw new Error(`${aspsp.name} (${aspsp.country}) is not available for this app`);
|
||||
console.log(
|
||||
`${bank.name} ${bank.country}${bank.bic ? ' ' + bank.bic : ''}: max consent ` +
|
||||
`${Math.floor(bank.maximum_consent_validity / 86400)} days${bank.beta ? ' (beta)' : ''}` +
|
||||
(bank.required_psu_headers?.length
|
||||
? `; requires PSU headers ${bank.required_psu_headers.join(', ')} (background syncs ` +
|
||||
'run as "PSU not present", which PSD2 limits to ~4 calls a day per account)'
|
||||
: '')
|
||||
);
|
||||
console.log(
|
||||
`Linked accounts go to tracker account "${defaultAccount}" unless --map says otherwise.`
|
||||
);
|
||||
if (flag('--check')) process.exit(0);
|
||||
|
||||
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
||||
let pasted = value('--code');
|
||||
if (!pasted) {
|
||||
// Stay one hour under the bank's maximum so the API never rejects valid_until.
|
||||
const validUntil = new Date(Date.now() + (bank.maximum_consent_validity - 3600) * 1000);
|
||||
const auth = await client.startAuth({
|
||||
aspsp: { name: bank.name, country: bank.country },
|
||||
redirectUrl,
|
||||
validUntil,
|
||||
psuType: 'personal',
|
||||
state: randomUUID()
|
||||
});
|
||||
console.log(`\n1. Open this link and approve access at ${bank.name}.`);
|
||||
console.log(` ${preset?.hint ?? 'Approve the account(s) you want to track.'}\n`);
|
||||
console.log(' ' + auth.url + '\n');
|
||||
console.log(`2. You'll land on ${redirectUrl}?code=...`);
|
||||
console.log(
|
||||
' Copy the full address from the browser bar (or just the code) and paste it here.'
|
||||
);
|
||||
console.log(' The code expires quickly, so paste it within a few minutes.\n');
|
||||
pasted = await rl.question('Redirect URL or code: ');
|
||||
}
|
||||
const parsed = parseAuthCode(pasted ?? '');
|
||||
if (parsed.error) throw new Error(`Consent failed: ${parsed.error}`);
|
||||
if (!parsed.code) throw new Error('No authorisation code found in what was pasted');
|
||||
|
||||
const session = await client.createSession(parsed.code);
|
||||
console.log(
|
||||
`\nSession authorised until ${session.access.valid_until}; ${session.accounts.length} account(s):`
|
||||
);
|
||||
|
||||
const sql = postgres(process.env.DB_URL!, { max: 1, onnotice: () => {} });
|
||||
try {
|
||||
const userId = await trackerUserId(sql, value('--user'));
|
||||
const tracker = await sql<
|
||||
{ id: number; name: string; scope: string; iban: string | null; external_uid: string | null }[]
|
||||
>`select id, name, scope, iban, external_uid from money.accounts where user_id = ${userId} order by id`;
|
||||
const plan = planAccountMapping(session.accounts, tracker, overrides, {
|
||||
defaultAccount,
|
||||
joint: bank.name.toLowerCase() === 'revolut'
|
||||
});
|
||||
for (const { bank: account, tracker: target, reason } of plan) {
|
||||
const iban = account.account_id?.iban ?? '(no IBAN)';
|
||||
console.log(
|
||||
` ${account.uid} ${iban} ${account.currency ?? ''} ${account.name ?? ''}${account.details ? ' / ' + account.details : ''}` +
|
||||
`\n -> ${target ? target.name : 'not linked'} (${reason})`
|
||||
);
|
||||
}
|
||||
const answer = flag('--yes') ? 'y' : await rl.question('\nSave this session and mapping? [y/N] ');
|
||||
if (!/^y(es)?$/i.test(answer.trim())) {
|
||||
console.log('Nothing saved. Re-run with --code is not possible (codes are single-use);');
|
||||
console.log('start again and use --map <uid>=<Tracker account> to fix the mapping.');
|
||||
process.exit(1);
|
||||
}
|
||||
await saveSession(sql, userId, session, plan);
|
||||
const linked = plan.filter((p) => p.tracker).map((p) => p.tracker!.name);
|
||||
const only = linked.length === 1 ? ` --account '${linked[0]}'` : '';
|
||||
console.log('Saved. Within the next hour (full history is only served right after consent):');
|
||||
console.log(
|
||||
` bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile --dry-run${only}`
|
||||
);
|
||||
console.log(
|
||||
` bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts --reconcile${only}`
|
||||
);
|
||||
} finally {
|
||||
rl.close();
|
||||
await sql.end();
|
||||
}
|
||||
53
scripts/bank-sync.ts
Normal file
53
scripts/bank-sync.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
// Pull booked bank transactions into money.transactions (Enable Banking).
|
||||
//
|
||||
// bun --env-file=$HOME/.config/money-sync.env scripts/bank-sync.ts [options]
|
||||
// --dry-run fetch and report, write nothing
|
||||
// --reconcile match against ALL hand-entered rows (account + amount + date ±1 day)
|
||||
// and attach the bank id to them instead of inserting duplicates;
|
||||
// run once after linking (with --dry-run first)
|
||||
// --from YYYY-MM-DD override the start date (default: last synced date - 7 days, or the
|
||||
// account's first transaction on the first run)
|
||||
// --full ask for the longest history the bank serves (automatic on an
|
||||
// account's first sync; only works within an hour of consent at most banks)
|
||||
// --account <name> only this tracker account
|
||||
//
|
||||
// Dedupe: unique (account_id, external_id); re-running is always safe.
|
||||
import postgres from 'postgres';
|
||||
import { configFromEnv, createEnableBankingClient } from '../src/lib/server/enable-banking';
|
||||
import { syncBankAccounts } from '../src/lib/server/bank-sync';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const value = (name: string) => {
|
||||
const i = args.indexOf(name);
|
||||
return i >= 0 ? args[i + 1] : undefined;
|
||||
};
|
||||
const from = value('--from');
|
||||
if (from && !/^\d{4}-\d{2}-\d{2}$/.test(from)) throw new Error('--from must be YYYY-MM-DD');
|
||||
if (!process.env.DB_URL) throw new Error('DB_URL is not set');
|
||||
|
||||
const client = createEnableBankingClient(configFromEnv());
|
||||
const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
||||
const dryRun = args.includes('--dry-run');
|
||||
try {
|
||||
const results = await syncBankAccounts(sql, client, {
|
||||
dryRun,
|
||||
reconcile: args.includes('--reconcile'),
|
||||
from,
|
||||
full: args.includes('--full'),
|
||||
accountName: value('--account')
|
||||
});
|
||||
for (const r of results) {
|
||||
console.log(
|
||||
`${r.account}: fetched ${r.fetched}, booked ${r.booked}, new ${r.inserted}, ` +
|
||||
`matched to existing ${r.matched}, replaced sum entries ${r.absorbed}, ` +
|
||||
`already synced ${r.alreadyKnown}` +
|
||||
(r.historyFrom ? `, history from ${r.historyFrom}` : '') +
|
||||
(r.balance ? `, bank balance ${r.balance}` : '') +
|
||||
(r.error ? `\n ERROR ${r.error}` : '')
|
||||
);
|
||||
}
|
||||
if (dryRun) console.log('(dry run: nothing written)');
|
||||
if (results.some((r) => r.error)) process.exitCode = 1;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
952
scripts/books-ingest.py
Normal file
952
scripts/books-ingest.py
Normal file
@@ -0,0 +1,952 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Books ingest + metadata enrichment.
|
||||
|
||||
Builds books.works / books.files from:
|
||||
* lists.books rows (the old GUI list) -> placeholders
|
||||
* Obsidian "Book links.md" md5s resolved via the libgen JSON API -> placeholders
|
||||
* ebook files dropped in $BOOKS_ROOT/inbox (sha256-deduped) -> works + files
|
||||
|
||||
Metadata: embedded (EPUB OPF / PDF info + ISBN scan) -> Open Library -> Google Books
|
||||
(-> arXiv for anything tagged paper). Covers go to $BOOKS_ROOT/covers/<sha1(cover_url)[:16]>.jpg (stable across dev/prod).
|
||||
Files are moved to $BOOKS_ROOT/library/<Author>/<Title>/.
|
||||
|
||||
Usage:
|
||||
DB_URL=postgresql://... BOOKS_ROOT=~/books \
|
||||
python scripts/books-ingest.py --obsidian-json resolved.json [--reset] [--dry-run]
|
||||
|
||||
Network calls are cached in $BOOKS_ROOT/.cache/http so reruns are cheap and polite.
|
||||
Refuses to run against the prod database name ('personal') unless --allow-prod.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse, difflib, hashlib, html, json, os, re, shutil, sys, time, unicodedata, urllib.parse, urllib.request, zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
def log(*a):
|
||||
print(*a, flush=True)
|
||||
|
||||
UA = "stan-books-ingest/0.1 (personal library catalogue; low volume)"
|
||||
LANG = {"eng": "en", "dut": "nl", "nld": "nl", "ger": "de", "deu": "de", "fre": "fr", "fra": "fr", "spa": "es", "ita": "it",
|
||||
"por": "pt", "rus": "ru", "jpn": "ja", "kor": "ko", "chi": "zh", "zho": "zh", "lat": "la", "gre": "el", "swe": "sv"}
|
||||
BOOK_EXT = {".pdf", ".epub", ".mobi", ".azw3", ".azw", ".lit", ".djvu", ".docx", ".fb2", ".cbz"}
|
||||
|
||||
# ---------------------------------------------------------------- helpers
|
||||
|
||||
def strip_accents(s: str) -> str:
|
||||
s = (s or "").replace("'", "").replace("’", "")
|
||||
return "".join(c for c in unicodedata.normalize("NFKD", s) if not unicodedata.combining(c))
|
||||
|
||||
def norm_title(t: str) -> str:
|
||||
t = strip_accents(html.unescape(t or "")).lower().replace("&", " and ")
|
||||
t = re.split(r"\s*[:(\[]\s*|\s+[-–—]\s+|_\s", t, maxsplit=1)[0] # main title only
|
||||
t = re.sub(r"[^a-z0-9]+", " ", t).strip()
|
||||
t = re.sub(r"^(the|a|an|de|het|een)\s+", "", t)
|
||||
t = re.sub(r"\s+(\d+(st|nd|rd|th)?\s+edition|ed)$", "", t)
|
||||
return t
|
||||
|
||||
def full_norm(t: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", strip_accents(t or "").lower()).strip()
|
||||
|
||||
def sim(a: str, b: str) -> float:
|
||||
na, nb = norm_title(a), norm_title(b)
|
||||
a, b = full_norm(a), full_norm(b)
|
||||
if not a or not b:
|
||||
return 0.0
|
||||
if a == b:
|
||||
return 1.0
|
||||
# volume / part / edition markers must agree ("Finance I" vs "Finance II", "Part 1" vs "Part 2")
|
||||
marks = lambda x: {t for t in x.split() if re.fullmatch(r"\d{1,2}|[ivx]{1,4}", t)}
|
||||
if marks(a) != marks(b) and marks(a) and marks(b):
|
||||
return 0.5
|
||||
r = difflib.SequenceMatcher(None, a, b).ratio()
|
||||
if na and na == nb and len(na) >= 15:
|
||||
r = max(r, 0.93)
|
||||
return r
|
||||
|
||||
def surnames(authors: list[str]) -> set[str]:
|
||||
out = set()
|
||||
for a in authors or []:
|
||||
a = strip_accents(a).lower()
|
||||
if "," in a: # "King, Michael"
|
||||
a = a.split(",")[0]
|
||||
parts = re.findall(r"[a-z]{3,}", a)
|
||||
if parts:
|
||||
out.add(parts[-1] if "," not in a else parts[0])
|
||||
out.update(parts)
|
||||
return out
|
||||
|
||||
JUNK_AUTHORS = {"admin", "administrator", "user", "owner", "unknown", "oreily", "oreilly", "o'reilly", "author", "john", "mr2"}
|
||||
|
||||
def clean_authors(names) -> list[str]:
|
||||
out = []
|
||||
for n in names or []:
|
||||
n = re.sub(r"\s+", " ", (n or "").strip(" ,;"))
|
||||
low = n.lower()
|
||||
if not n or low in JUNK_AUTHORS or re.search(r"\d|@|\.(com|org|net)\b", n) or " " not in n and "," not in n and "." not in n:
|
||||
continue
|
||||
if n.isupper():
|
||||
n = n.title()
|
||||
out.append(n)
|
||||
return out
|
||||
|
||||
def authors_compatible(a: list[str], b: list[str]) -> bool:
|
||||
if not a or not b:
|
||||
return True
|
||||
return bool(surnames(a) & surnames(b))
|
||||
|
||||
def isbn_ok(s: str) -> str | None:
|
||||
s = re.sub(r"[^0-9Xx]", "", s or "").upper()
|
||||
if len(s) == 13 and s.isdigit() and s[:3] in ("978", "979"):
|
||||
tot = sum(int(c) * (1 if i % 2 == 0 else 3) for i, c in enumerate(s[:12]))
|
||||
return s if (10 - tot % 10) % 10 == int(s[12]) else None
|
||||
if len(s) == 10 and s[:9].isdigit():
|
||||
tot = sum((10 - i) * int(c) for i, c in enumerate(s[:9]))
|
||||
chk = (11 - tot % 11) % 11
|
||||
return s if (("X" if chk == 10 else str(chk)) == s[9]) else None
|
||||
return None
|
||||
|
||||
def to13(s: str) -> str | None:
|
||||
s = isbn_ok(s)
|
||||
if not s:
|
||||
return None
|
||||
if len(s) == 13:
|
||||
return s
|
||||
core = "978" + s[:9]
|
||||
tot = sum(int(c) * (1 if i % 2 == 0 else 3) for i, c in enumerate(core))
|
||||
return core + str((10 - tot % 10) % 10)
|
||||
|
||||
def to10(s13: str) -> str | None:
|
||||
if not s13 or not s13.startswith("978"):
|
||||
return None
|
||||
core = s13[3:12]
|
||||
tot = sum((10 - i) * int(c) for i, c in enumerate(core))
|
||||
chk = (11 - tot % 11) % 11
|
||||
return core + ("X" if chk == 10 else str(chk))
|
||||
|
||||
ISBN_RE = re.compile(r"ISBN(?:-1[03])?[:\s]*((?:97[89][\s-]?)?(?:\d[\s-]?){9}[\dXx])", re.I)
|
||||
|
||||
def find_isbns(text: str) -> list[str]:
|
||||
out = []
|
||||
for m in ISBN_RE.finditer(text or ""):
|
||||
i = to13(m.group(1))
|
||||
if i and i not in out:
|
||||
out.append(i)
|
||||
return out
|
||||
|
||||
def safe_name(s: str, n: int = 80) -> str:
|
||||
s = strip_accents(s or "").replace("/", "-").replace("\\", "-")
|
||||
s = re.sub(r'[<>:"|?*\x00-\x1f]', "", s)
|
||||
s = re.sub(r"\s+", " ", s).strip(" .")
|
||||
return (s[:n].rstrip(" .") or "Untitled")
|
||||
|
||||
def year_of(v) -> int | None:
|
||||
m = re.search(r"(1[5-9]\d\d|20\d\d)", str(v or ""))
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
# ---------------------------------------------------------------- http with cache
|
||||
|
||||
class Http:
|
||||
def __init__(self, cache: Path, delay: float = 1.0):
|
||||
self.cache, self.delay, self.last = cache, delay, 0.0
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
self.calls = 0
|
||||
self.dead_hosts: set[str] = set()
|
||||
|
||||
def _get(self, url: str, binary=False):
|
||||
host = urllib.parse.urlparse(url).netloc
|
||||
if host in self.dead_hosts: # daily quota hit (Google Books): stop hammering it this run
|
||||
return 429, b""
|
||||
wait = self.delay - (time.time() - self.last)
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
self.last = time.time()
|
||||
self.calls += 1
|
||||
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "*/*"})
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429 and "googleapis" in host:
|
||||
self.dead_hosts.add(host)
|
||||
log(f" {host}: quota exhausted (429), skipping it for the rest of this run")
|
||||
return 429, b""
|
||||
if e.code in (429, 503) and attempt < 2:
|
||||
time.sleep(5 * (attempt + 1))
|
||||
continue
|
||||
return e.code, b""
|
||||
except Exception:
|
||||
if attempt < 2:
|
||||
time.sleep(3)
|
||||
continue
|
||||
return 0, b""
|
||||
return 0, b""
|
||||
|
||||
def json(self, url: str):
|
||||
key = self.cache / (hashlib.sha1(url.encode()).hexdigest() + ".json")
|
||||
if key.exists():
|
||||
return json.loads(key.read_text())
|
||||
status, body = self._get(url)
|
||||
data = None
|
||||
if status == 200:
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except Exception:
|
||||
data = None
|
||||
if status in (200, 404): # don't cache transient errors
|
||||
key.write_text(json.dumps(data))
|
||||
return data
|
||||
|
||||
def text(self, url: str):
|
||||
key = self.cache / (hashlib.sha1(url.encode()).hexdigest() + ".txt")
|
||||
if key.exists():
|
||||
return key.read_text()
|
||||
status, body = self._get(url)
|
||||
if status == 200:
|
||||
key.write_text(body.decode("utf-8", "replace"))
|
||||
return body.decode("utf-8", "replace")
|
||||
return None
|
||||
|
||||
def download(self, url: str, dest: Path) -> bool:
|
||||
status, body = self._get(url, binary=True)
|
||||
if status == 200 and len(body) > 1500: # skip 1x1 placeholders
|
||||
dest.write_bytes(body)
|
||||
return True
|
||||
return False
|
||||
|
||||
# ---------------------------------------------------------------- model
|
||||
|
||||
@dataclass
|
||||
class Work:
|
||||
title: str
|
||||
authors: list[str] = field(default_factory=list)
|
||||
subtitle: str | None = None
|
||||
isbns: list[str] = field(default_factory=list) # isbn13s, best first
|
||||
publisher: str | None = None
|
||||
year: int | None = None
|
||||
language: str | None = None
|
||||
page_count: int | None = None
|
||||
description: str | None = None
|
||||
tags: list[str] = field(default_factory=list)
|
||||
cover_url: str | None = None
|
||||
cover_path: str | None = None
|
||||
ol_work: str | None = None
|
||||
ol_edition: str | None = None
|
||||
gb_id: str | None = None
|
||||
md5s: list[str] = field(default_factory=list)
|
||||
refs: dict = field(default_factory=dict)
|
||||
status: str = "pending"
|
||||
source: str | None = None
|
||||
notes: str | None = None
|
||||
files: list = field(default_factory=list)
|
||||
shelves: set = field(default_factory=set)
|
||||
origins: set = field(default_factory=set)
|
||||
db_id: int | None = None
|
||||
|
||||
def add_ref(self, k, v):
|
||||
self.refs.setdefault(k, [])
|
||||
if v not in self.refs[k]:
|
||||
self.refs[k].append(v)
|
||||
|
||||
@dataclass
|
||||
class FileRec:
|
||||
path: Path
|
||||
rel_inbox: str
|
||||
sha256: str
|
||||
md5: str
|
||||
size: int
|
||||
fmt: str
|
||||
title: str | None = None
|
||||
authors: list[str] = field(default_factory=list)
|
||||
isbns: list[str] = field(default_factory=list)
|
||||
year: int | None = None
|
||||
publisher: str | None = None
|
||||
language: str | None = None
|
||||
pages: int | None = None
|
||||
is_paper: bool = False
|
||||
broken: str | None = None
|
||||
dest_rel: str | None = None
|
||||
|
||||
# ---------------------------------------------------------------- embedded metadata
|
||||
|
||||
def epub_meta(p: Path) -> dict:
|
||||
out: dict = {}
|
||||
try:
|
||||
with zipfile.ZipFile(p) as z:
|
||||
cont = ET.fromstring(z.read("META-INF/container.xml"))
|
||||
opf_path = next(e.get("full-path") for e in cont.iter() if e.tag.endswith("rootfile"))
|
||||
opf = ET.fromstring(z.read(opf_path))
|
||||
md = next((e for e in opf.iter() if e.tag.endswith("metadata")), None)
|
||||
if md is not None:
|
||||
for e in md:
|
||||
tag = e.tag.split("}")[-1]
|
||||
txt = (e.text or "").strip()
|
||||
if not txt:
|
||||
continue
|
||||
if tag == "title" and "title" not in out:
|
||||
out["title"] = txt
|
||||
elif tag == "creator":
|
||||
out.setdefault("authors", []).append(txt)
|
||||
elif tag == "identifier":
|
||||
i = to13(txt.replace("urn:isbn:", ""))
|
||||
if i:
|
||||
out.setdefault("isbns", []).append(i)
|
||||
elif tag == "publisher":
|
||||
out.setdefault("publisher", txt)
|
||||
elif tag == "date":
|
||||
out.setdefault("year", year_of(txt))
|
||||
elif tag == "language":
|
||||
out.setdefault("language", txt[:2].lower())
|
||||
# scan the first few content docs (copyright page) for ISBNs
|
||||
names = [n for n in z.namelist() if n.lower().endswith((".xhtml", ".html", ".htm"))][:12]
|
||||
for n in names:
|
||||
for i in find_isbns(re.sub(r"<[^>]+>", " ", z.read(n).decode("utf-8", "ignore"))):
|
||||
out.setdefault("isbns", [])
|
||||
if i not in out["isbns"]:
|
||||
out["isbns"].append(i)
|
||||
except Exception as e: # noqa
|
||||
out["error"] = f"epub: {e.__class__.__name__}"
|
||||
return out
|
||||
|
||||
def pdf_meta(p: Path) -> dict:
|
||||
out: dict = {}
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
import logging
|
||||
logging.getLogger("pypdf").setLevel(logging.ERROR)
|
||||
r = PdfReader(str(p), strict=False)
|
||||
out["pages"] = len(r.pages)
|
||||
info = r.metadata or {}
|
||||
t = (info.get("/Title") or "").strip() if info else ""
|
||||
a = (info.get("/Author") or "").strip() if info else ""
|
||||
if t:
|
||||
out["title"] = t
|
||||
if a:
|
||||
out["authors"] = clean_authors(re.split(r";|\band\b|&", a))
|
||||
text = ""
|
||||
for i in range(min(len(r.pages), 10)):
|
||||
try:
|
||||
text += (r.pages[i].extract_text() or "") + "\n"
|
||||
except Exception:
|
||||
pass
|
||||
out["isbns"] = find_isbns(text)
|
||||
if re.search(r"arXiv:\d{4}\.\d{4,5}", text):
|
||||
out["arxiv"] = re.search(r"arXiv:(\d{4}\.\d{4,5})", text).group(1)
|
||||
except Exception as e: # noqa
|
||||
out["error"] = f"pdf: {e.__class__.__name__}"
|
||||
return out
|
||||
|
||||
ANNAS_RE = re.compile(r"^(?P<title>.+?) -- (?P<author>.+?) -- (?P<rest>.+?) -- (?P<md5>[0-9a-f]{32}) -- Anna’s Archive$")
|
||||
|
||||
def filename_meta(stem: str) -> dict:
|
||||
out: dict = {}
|
||||
m = ANNAS_RE.match(stem)
|
||||
if m:
|
||||
out["title"] = m["title"].replace("_ ", ": ").replace("_", " ")
|
||||
out["authors"] = [a.strip().replace("_", ".") for a in m["author"].split(",") if a.strip()]
|
||||
out["md5"] = m["md5"]
|
||||
for part in m["rest"].split(" -- "):
|
||||
i = to13(part)
|
||||
if i:
|
||||
out.setdefault("isbns", []).append(i)
|
||||
elif year_of(part) and "year" not in out:
|
||||
out["year"] = year_of(part)
|
||||
return out
|
||||
s = re.sub(r"\s*-\s*libgen\.(li|rs|is)$", "", stem)
|
||||
s = re.sub(r"\s*\((z-library|z-lib\.org|1lib\.\w+)\)", "", s, flags=re.I)
|
||||
s = re.sub(r"\s+(penguin (modern )?classics)$", "", s, flags=re.I)
|
||||
s = re.sub(r"\s+0$", "", s)
|
||||
mm = re.match(r"^([^-]+?) : ([^-]+?) - (.+)$", s) # "A : B - Title" (libgen multi-author)
|
||||
if mm:
|
||||
out["authors"] = [mm[1].strip(), mm[2].strip()]
|
||||
s = mm[3]
|
||||
pm = re.search(r"\s*\((\d{4}),\s*([^)]+)\)\s*$", s)
|
||||
if pm:
|
||||
out["year"], out["publisher"] = int(pm[1]), pm[2].strip()
|
||||
s = s[: pm.start()]
|
||||
# "Author - Title" (only when the left part looks like a person name)
|
||||
mm = re.match(r"^([A-Z][a-z]+(?: [A-Z][a-z]+){1,2}) - (.+)$", s)
|
||||
if mm:
|
||||
out["authors"] = [mm[1]]
|
||||
s = mm[2]
|
||||
mm = re.match(r"^(.+?) by ([A-Z][\w.]+(?: [A-Z][\w.]+){1,2})$", s)
|
||||
if mm:
|
||||
s, out["authors"] = mm[1], [mm[2]]
|
||||
if s.count("-") >= 4 and " " not in s: # The-Lean-Startup_-How-...-Eric-Ries-2011-Crown-Business-030788791X-
|
||||
for tok in s.split("-"):
|
||||
i = to13(tok)
|
||||
if i:
|
||||
out.setdefault("isbns", []).append(i)
|
||||
s = s.split("_")[0].replace("-", " ")
|
||||
s = s.replace("_ ", ": ").replace("_", " ")
|
||||
if s.count(".") >= 3 and s.count(" ") <= 1: # Sipser_Introduction.to.the.Theory.of.Computation.3E
|
||||
s = s.replace(".", " ")
|
||||
s = re.sub(r"\s+", " ", s).strip(" -")
|
||||
out["title"] = s
|
||||
return out
|
||||
|
||||
def good_embedded_title(t: str | None, fname_title: str) -> bool:
|
||||
if not t or len(t) < 4:
|
||||
return False
|
||||
bad = ("untitled", "microsoft word", ".doc", ".pdf", ".indd", "layout 1", "unknown", "title", "cover")
|
||||
tl = t.lower()
|
||||
if any(b in tl for b in bad) or re.fullmatch(r"[\w-]+\.\w+", t) or re.fullmatch(r"[0-9a-f\-_ ]+", tl):
|
||||
return False
|
||||
return sim(t, fname_title) > 0.55
|
||||
|
||||
# ---------------------------------------------------------------- enrichment
|
||||
|
||||
class Enricher:
|
||||
def __init__(self, http: Http):
|
||||
self.h = http
|
||||
|
||||
def ol_isbn(self, isbn: str):
|
||||
ed = self.h.json(f"https://openlibrary.org/isbn/{isbn}.json")
|
||||
if not ed or "title" not in ed:
|
||||
return None
|
||||
rec = {"title": ed.get("title"), "subtitle": ed.get("subtitle"), "publisher": (ed.get("publishers") or [None])[0],
|
||||
"year": year_of(ed.get("publish_date")), "pages": ed.get("number_of_pages"),
|
||||
"ol_edition": ed.get("key", "").split("/")[-1] or None, "isbn": isbn,
|
||||
"cover": f"https://covers.openlibrary.org/b/id/{ed['covers'][0]}-L.jpg" if ed.get("covers") and ed["covers"][0] > 0 else None,
|
||||
"lang": LANG.get(((ed.get("languages") or [{}])[0].get("key") or "").split("/")[-1])}
|
||||
wk = (ed.get("works") or [{}])[0].get("key")
|
||||
if wk:
|
||||
w = self.h.json(f"https://openlibrary.org{wk}.json") or {}
|
||||
rec["ol_work"] = wk.split("/")[-1]
|
||||
d = w.get("description")
|
||||
rec["description"] = d.get("value") if isinstance(d, dict) else d
|
||||
rec["subjects"] = (w.get("subjects") or [])[:8]
|
||||
names = []
|
||||
for a in (w.get("authors") or [])[:4]:
|
||||
k = (a.get("author") or {}).get("key")
|
||||
if k:
|
||||
aj = self.h.json(f"https://openlibrary.org{k}.json") or {}
|
||||
if aj.get("name"):
|
||||
names.append(aj["name"])
|
||||
rec["authors"] = names
|
||||
if not rec["cover"] and w.get("covers") and w["covers"][0] > 0:
|
||||
rec["cover"] = f"https://covers.openlibrary.org/b/id/{w['covers'][0]}-L.jpg"
|
||||
return rec
|
||||
|
||||
def ol_search(self, title: str, authors: list[str]):
|
||||
q = {"title": norm_title(title) if len(norm_title(title)) > 3 else title, "limit": 8,
|
||||
"fields": "key,title,subtitle,author_name,first_publish_year,isbn,cover_i,cover_edition_key,language,number_of_pages_median,publisher,subject"}
|
||||
if authors:
|
||||
q["author"] = re.sub(r"[^\w\s.-]", " ", authors[0])
|
||||
d = self.h.json("https://openlibrary.org/search.json?" + urllib.parse.urlencode(q))
|
||||
docs = (d or {}).get("docs") or []
|
||||
if not docs and authors:
|
||||
q.pop("author")
|
||||
d = self.h.json("https://openlibrary.org/search.json?" + urllib.parse.urlencode(q))
|
||||
docs = (d or {}).get("docs") or []
|
||||
best, bs = None, 0.0
|
||||
for doc in docs:
|
||||
t = doc.get("title", "")
|
||||
s = max(sim(title, t), sim(title, f"{t}: {doc.get('subtitle', '')}"))
|
||||
if norm_title(title) and norm_title(title) == norm_title(t) and (
|
||||
authors and doc.get("author_name") and authors_compatible(authors, doc["author_name"]) or len(norm_title(t)) >= 15):
|
||||
s = max(s, 0.92)
|
||||
if authors and doc.get("author_name"):
|
||||
s += 0.05 if authors_compatible(authors, doc["author_name"]) else -0.25
|
||||
if s > bs:
|
||||
best, bs = doc, s
|
||||
if not best:
|
||||
return None, 0.0
|
||||
isbns = [to13(i) for i in (best.get("isbn") or [])]
|
||||
isbns = [i for i in isbns if i]
|
||||
rec = {"title": best.get("title"), "subtitle": best.get("subtitle"), "authors": best.get("author_name") or [],
|
||||
"year": best.get("first_publish_year"), "ol_work": best.get("key", "").split("/")[-1] or None,
|
||||
"ol_edition": best.get("cover_edition_key"), "pages": best.get("number_of_pages_median"),
|
||||
"publisher": (best.get("publisher") or [None])[0],
|
||||
"cover": f"https://covers.openlibrary.org/b/id/{best['cover_i']}-L.jpg" if best.get("cover_i") else None,
|
||||
"lang": LANG.get((best.get("language") or [""])[0]), "subjects": (best.get("subject") or [])[:8],
|
||||
"isbn": next((i for i in isbns if i.startswith("978")), isbns[0] if isbns else None)}
|
||||
if rec["ol_work"]:
|
||||
w = self.h.json(f"https://openlibrary.org/works/{rec['ol_work']}.json") or {}
|
||||
dsc = w.get("description")
|
||||
rec["description"] = dsc.get("value") if isinstance(dsc, dict) else dsc
|
||||
return rec, min(bs, 1.0)
|
||||
|
||||
def gb(self, title: str | None, authors: list[str], isbn: str | None = None):
|
||||
if isbn:
|
||||
q = f"isbn:{isbn}"
|
||||
else:
|
||||
q = f'intitle:"{norm_title(title) or title}"' + (f" inauthor:{surnames(authors[:1]).pop()}" if authors and surnames(authors[:1]) else "")
|
||||
d = self.h.json("https://www.googleapis.com/books/v1/volumes?" + urllib.parse.urlencode({"q": q, "maxResults": 8, "printType": "books"}))
|
||||
items = (d or {}).get("items") or []
|
||||
best, bs = None, 0.0
|
||||
for it in items:
|
||||
v = it.get("volumeInfo", {})
|
||||
if isbn:
|
||||
s = 1.0
|
||||
else:
|
||||
s = max(sim(title, v.get("title", "")), sim(title, f"{v.get('title', '')}: {v.get('subtitle', '')}"))
|
||||
if norm_title(title) and norm_title(title) == norm_title(v.get("title", "")) and (
|
||||
authors and v.get("authors") and authors_compatible(authors, v["authors"]) or len(norm_title(title)) >= 15):
|
||||
s = max(s, 0.92)
|
||||
if authors and v.get("authors"):
|
||||
s += 0.05 if authors_compatible(authors, v["authors"]) else -0.25
|
||||
if s > bs:
|
||||
best, bs = it, s
|
||||
if isbn:
|
||||
break
|
||||
if not best:
|
||||
return None, 0.0
|
||||
v = best["volumeInfo"]
|
||||
ids = {x.get("type"): x.get("identifier") for x in v.get("industryIdentifiers", [])}
|
||||
img = (v.get("imageLinks") or {})
|
||||
cover = img.get("extraLarge") or img.get("large") or img.get("medium") or img.get("thumbnail")
|
||||
if cover:
|
||||
cover = cover.replace("http://", "https://").replace("&edge=curl", "")
|
||||
return {"title": v.get("title"), "subtitle": v.get("subtitle"), "authors": v.get("authors") or [],
|
||||
"publisher": v.get("publisher"), "year": year_of(v.get("publishedDate")), "pages": v.get("pageCount"),
|
||||
"description": re.sub(r"<[^>]+>", "", v.get("description") or "") or None,
|
||||
"lang": (v.get("language") or "")[:2] or None, "gb_id": best.get("id"), "cover": cover,
|
||||
"isbn": to13(ids.get("ISBN_13") or ids.get("ISBN_10") or ""), "subjects": v.get("categories") or []}, min(bs, 1.0)
|
||||
|
||||
def arxiv(self, title: str, arxiv_id: str | None = None):
|
||||
if arxiv_id:
|
||||
url = f"https://export.arxiv.org/api/query?id_list={arxiv_id}"
|
||||
else:
|
||||
t = re.sub(r"[^\w\s]", " ", title)
|
||||
url = "https://export.arxiv.org/api/query?" + urllib.parse.urlencode({"search_query": f'ti:"{t}"', "max_results": 5})
|
||||
x = self.h.text(url)
|
||||
if not x:
|
||||
return None, 0.0
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
try:
|
||||
root = ET.fromstring(x)
|
||||
except ET.ParseError:
|
||||
return None, 0.0
|
||||
best, bs = None, 0.0
|
||||
for e in root.findall("a:entry", ns):
|
||||
t = re.sub(r"\s+", " ", e.findtext("a:title", "", ns)).strip()
|
||||
if not t or t == "Error":
|
||||
continue
|
||||
s = 1.0 if arxiv_id else sim(title, t)
|
||||
if s > bs:
|
||||
best, bs = (e, t), s
|
||||
if not best:
|
||||
return None, 0.0
|
||||
e, t = best
|
||||
return {"title": t, "authors": [a.findtext("a:name", "", ns) for a in e.findall("a:author", ns)],
|
||||
"year": year_of(e.findtext("a:published", "", ns)),
|
||||
"description": re.sub(r"\s+", " ", e.findtext("a:summary", "", ns)).strip(),
|
||||
"arxiv": e.findtext("a:id", "", ns).split("/abs/")[-1]}, bs
|
||||
|
||||
# ---------------------------------------------------------------- main
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--obsidian-json", help="md5 -> libgen record map from the resolver")
|
||||
ap.add_argument("--extra", action="append", default=[], help='extra placeholder "Title|Author|isbn13|origin"')
|
||||
ap.add_argument("--reset", action="store_true", help="truncate books.works/files first (scratch DBs only)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="no DB writes, no file moves; write plan json")
|
||||
ap.add_argument("--no-move", action="store_true", help="write DB rows (and covers) but leave files in the inbox; for a dev rehearsal before the prod run")
|
||||
ap.add_argument("--no-network", action="store_true")
|
||||
ap.add_argument("--allow-prod", action="store_true")
|
||||
ap.add_argument("--source", default="gdrive")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = Path(os.path.expanduser(os.environ.get("BOOKS_ROOT", "/srv/books")))
|
||||
inbox, library, covers = root / "inbox", root / "library", root / "covers"
|
||||
for d in (inbox, library, covers):
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
db_url = os.environ.get("DB_URL")
|
||||
if not db_url:
|
||||
sys.exit("DB_URL required")
|
||||
dbname = urllib.parse.urlparse(db_url).path.lstrip("/")
|
||||
if dbname == "personal" and not args.allow_prod:
|
||||
sys.exit("refusing to run against prod DB 'personal' (use --allow-prod)")
|
||||
|
||||
import psycopg
|
||||
conn = psycopg.connect(db_url)
|
||||
http = Http(root / ".cache" / "http", delay=0.0 if args.no_network else 1.0)
|
||||
en = Enricher(http)
|
||||
works: list[Work] = []
|
||||
|
||||
|
||||
# ---- 1. placeholders: lists.books
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("select to_regclass('lists.books')")
|
||||
has_lists = cur.fetchone()[0] is not None
|
||||
rows = []
|
||||
if has_lists:
|
||||
cur.execute("select id, name, author, notes, done_at from lists.books order by sort_order nulls last, id")
|
||||
rows = cur.fetchall()
|
||||
for bid, name, author, notes, done_at in rows:
|
||||
w = Work(title=name.strip(), authors=[author.strip()] if author else [], notes=notes or None)
|
||||
w.add_ref("lists_books_id", bid)
|
||||
w.origins.add("lists")
|
||||
w.shelves.add("done" if done_at else "to-read")
|
||||
if done_at:
|
||||
w.refs["done_at"] = done_at.isoformat()
|
||||
works.append(w)
|
||||
n_lists = len(rows)
|
||||
|
||||
def find_work(title, authors, isbns, md5=None):
|
||||
for w in works:
|
||||
if md5 and md5 in w.md5s:
|
||||
return w
|
||||
if isbns and set(isbns) & set(w.isbns) and (not title or not w.title or sim(title, w.title) >= 0.85):
|
||||
return w
|
||||
best, bs = None, 0.0
|
||||
for w in works:
|
||||
s = sim(title, w.title)
|
||||
if s >= 0.9 and authors_compatible(authors, w.authors) and s > bs:
|
||||
best, bs = w, s
|
||||
return best
|
||||
|
||||
def merge(w: Work, title, authors, isbns, **kw):
|
||||
if authors and not w.authors:
|
||||
w.authors = authors
|
||||
for i in isbns or []:
|
||||
if i not in w.isbns:
|
||||
w.isbns.append(i)
|
||||
for k in ("year", "publisher", "language"):
|
||||
if kw.get(k) and not getattr(w, k):
|
||||
setattr(w, k, kw[k])
|
||||
# prefer the longer, properly-cased title when the list name is a lowercase shorthand
|
||||
if title and (w.title.islower() or len(norm_title(w.title)) < len(norm_title(title)) and sim(w.title, title) > 0.9 and w.title == w.title.lower()):
|
||||
w.title = title
|
||||
|
||||
# ---- 2. placeholders: Obsidian md5s
|
||||
obs_total = obs_resolved = obs_overlap = 0
|
||||
if args.obsidian_json:
|
||||
data = json.load(open(args.obsidian_json))
|
||||
for md5, rec in data.items():
|
||||
obs_total += 1
|
||||
eds = (rec or {}).get("editions") or []
|
||||
if not eds:
|
||||
continue
|
||||
obs_resolved += 1
|
||||
ed = next((e for e in eds if e.get("isbns")), eds[0])
|
||||
isbns = []
|
||||
for e in eds:
|
||||
for i in e.get("isbns") or []:
|
||||
i13 = to13(i)
|
||||
if i13 and i13 not in isbns:
|
||||
isbns.append(i13)
|
||||
title = (ed.get("title") or rec.get("locator") or md5).strip()
|
||||
authors = [a.strip() for a in re.split(r";|, (?=[A-Z][a-z]+ [A-Z])", ed.get("author") or "") if a.strip()]
|
||||
lang = LANG.get((ed.get("lang") or [""])[0], None) if isinstance(ed.get("lang"), list) else None
|
||||
w = find_work(title, authors, isbns, md5)
|
||||
if w:
|
||||
if "obsidian" not in w.origins:
|
||||
obs_overlap += 1
|
||||
merge(w, title, authors, isbns, year=year_of(ed.get("year")), publisher=ed.get("publisher") or None, language=lang)
|
||||
else:
|
||||
w = Work(title=title, authors=authors, isbns=isbns, year=year_of(ed.get("year")), publisher=ed.get("publisher") or None, language=lang)
|
||||
works.append(w)
|
||||
w.shelves.add("to-read")
|
||||
w.origins.add("obsidian")
|
||||
if md5 not in w.md5s:
|
||||
w.md5s.append(md5)
|
||||
w.add_ref("obsidian", {"md5": md5, "ext": rec.get("ext"), "filesize": rec.get("filesize")})
|
||||
for ex in args.extra:
|
||||
t, a, i, origin = (ex.split("|") + ["", "", "", ""])[:4]
|
||||
isbns = [to13(i)] if to13(i) else []
|
||||
authors = [a] if a else []
|
||||
w = find_work(t, authors, isbns)
|
||||
if w:
|
||||
merge(w, t, authors, isbns)
|
||||
else:
|
||||
w = Work(title=t, authors=authors, isbns=isbns)
|
||||
works.append(w)
|
||||
w.shelves.add("to-read")
|
||||
w.origins.add("obsidian")
|
||||
w.add_ref("obsidian_link", origin)
|
||||
n_placeholders = len(works)
|
||||
|
||||
# ---- 3. files in inbox
|
||||
files: list[FileRec] = []
|
||||
for p in sorted(inbox.rglob("*")):
|
||||
if not p.is_file() or p.name.startswith("."):
|
||||
continue
|
||||
h256, h5 = hashlib.sha256(), hashlib.md5()
|
||||
with open(p, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h256.update(chunk)
|
||||
h5.update(chunk)
|
||||
rel = str(p.relative_to(inbox))
|
||||
ext = p.suffix.lower()
|
||||
with open(p, "rb") as hf:
|
||||
head = hf.read(8)
|
||||
fmt = ext.lstrip(".") or "bin"
|
||||
if head.startswith(b"%PDF"):
|
||||
fmt = "pdf"
|
||||
elif head.startswith(b"PK") and ext in (".epub", ""):
|
||||
fmt = "epub"
|
||||
fr = FileRec(path=p, rel_inbox=rel, sha256=h256.hexdigest(), md5=h5.hexdigest(), size=p.stat().st_size, fmt=fmt)
|
||||
fr.is_paper = rel.split("/")[0].lower() == "papers"
|
||||
if ext == ".crdownload":
|
||||
fr.broken = "partial Chrome download (.crdownload)"
|
||||
fr.fmt = "pdf" if head.startswith(b"%PDF") else "partial"
|
||||
files.append(fr)
|
||||
by_sha: dict[str, FileRec] = {}
|
||||
dups: list[tuple[FileRec, FileRec]] = []
|
||||
uniq: list[FileRec] = []
|
||||
for fr in files:
|
||||
if fr.sha256 in by_sha:
|
||||
dups.append((fr, by_sha[fr.sha256]))
|
||||
else:
|
||||
by_sha[fr.sha256] = fr
|
||||
uniq.append(fr)
|
||||
log(f"files: {len(files)} total, {len(uniq)} unique, {len(dups)} sha256 duplicates")
|
||||
|
||||
# already-ingested shas (rerun without --reset)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("select sha256 from books.files")
|
||||
known = {r[0] for r in cur.fetchall()} if not args.reset else set()
|
||||
|
||||
matched_existing = 0
|
||||
for fr in uniq:
|
||||
if fr.sha256 in known:
|
||||
continue
|
||||
stem = Path(fr.rel_inbox).name
|
||||
stem = stem[: -len(Path(stem).suffix)] if Path(stem).suffix.lower() in BOOK_EXT | {".crdownload"} else stem
|
||||
fm = filename_meta(stem)
|
||||
em = epub_meta(fr.path) if fr.fmt == "epub" else pdf_meta(fr.path) if fr.fmt == "pdf" and not fr.broken else {}
|
||||
fr.title = fm.get("title") or stem
|
||||
if fr.fmt == "epub" and good_embedded_title(em.get("title"), fr.title):
|
||||
fr.title = em["title"]
|
||||
fr.authors = fm.get("authors") or [a for a in em.get("authors", []) if len(a) > 2 and "unknown" not in a.lower()][:4]
|
||||
fr.isbns = list(dict.fromkeys((fm.get("isbns") or []) + (em.get("isbns") or [])))[:6]
|
||||
fr.year = fm.get("year") or em.get("year")
|
||||
fr.publisher = fm.get("publisher") or em.get("publisher")
|
||||
fr.language = em.get("language")
|
||||
fr.pages = em.get("pages")
|
||||
md5 = fm.get("md5") or fr.md5
|
||||
w = find_work(fr.title, fr.authors, fr.isbns[:2], md5) or (find_work(fr.title, fr.authors, [], fr.md5))
|
||||
if w and w.files == [] and w.origins & {"lists", "obsidian"} and "drive" not in w.origins:
|
||||
matched_existing += 1
|
||||
if w:
|
||||
merge(w, fr.title, fr.authors, fr.isbns[:2], year=fr.year, publisher=fr.publisher, language=fr.language)
|
||||
else:
|
||||
w = Work(title=fr.title, authors=fr.authors, isbns=fr.isbns[:2], year=fr.year, publisher=fr.publisher, language=fr.language, page_count=fr.pages)
|
||||
works.append(w)
|
||||
if fr.is_paper and "paper" not in w.tags:
|
||||
w.tags.append("paper")
|
||||
if fr.md5 not in w.md5s and fm.get("md5"):
|
||||
w.md5s.append(fm["md5"])
|
||||
w.origins.add("drive")
|
||||
w.add_ref("drive", fr.rel_inbox)
|
||||
if em.get("arxiv"):
|
||||
w.refs["arxiv"] = em["arxiv"]
|
||||
if fr.broken:
|
||||
w.notes = ((w.notes + "\n") if w.notes else "") + f"Drive copy is broken: {fr.broken}; needs a fresh copy."
|
||||
w.add_ref("broken_files", fr.rel_inbox)
|
||||
else:
|
||||
w.files.append(fr)
|
||||
for d, orig in dups:
|
||||
for w in works:
|
||||
if orig in w.files:
|
||||
w.add_ref("drive_duplicates", d.rel_inbox)
|
||||
log(f"works: {len(works)} ({n_placeholders} placeholders before files; {matched_existing} placeholders got files)")
|
||||
|
||||
# ---- 4. enrichment
|
||||
stats = {"matched": 0, "low_confidence": 0, "not_found": 0}
|
||||
for n, w in enumerate(works, 1):
|
||||
if args.no_network:
|
||||
break
|
||||
rec, score, src = None, 0.0, None
|
||||
if "paper" in w.tags:
|
||||
rec, score = en.arxiv(w.title, w.refs.get("arxiv"))
|
||||
src = "arxiv"
|
||||
else:
|
||||
for isbn in w.isbns[:3]:
|
||||
rec = en.ol_isbn(isbn)
|
||||
if rec and (sim(w.title, rec["title"]) > 0.6 or sim(w.title, f"{rec['title']}: {rec.get('subtitle') or ''}") > 0.6 or "drive" not in w.origins):
|
||||
score, src = 1.0, "openlibrary"
|
||||
break
|
||||
rec = None
|
||||
if not rec:
|
||||
rec, score = en.ol_search(w.title, w.authors)
|
||||
src = "openlibrary"
|
||||
if score < 0.9:
|
||||
g, gs = en.gb(None, [], w.isbns[0]) if w.isbns else (None, 0.0)
|
||||
if g and sim(w.title, g["title"] or "") < 0.6 and sim(w.title, f"{g['title']}: {g.get('subtitle') or ''}") < 0.6:
|
||||
g, gs = None, 0.0
|
||||
if not g:
|
||||
g, gs = en.gb(w.title, w.authors)
|
||||
if g and gs > score:
|
||||
rec, score, src = g, gs, "googlebooks"
|
||||
if rec and score >= 0.9:
|
||||
w.status = "matched"
|
||||
elif rec and score >= 0.75:
|
||||
w.status = "low_confidence"
|
||||
else:
|
||||
w.status = "not_found"
|
||||
stats[w.status] += 1
|
||||
if rec:
|
||||
w.refs["match"] = {"source": src, "score": round(score, 3), "title": rec.get("title"), "authors": rec.get("authors")}
|
||||
if w.status == "matched":
|
||||
w.source = src
|
||||
if src == "openlibrary" or src == "googlebooks":
|
||||
# keep Stan's/the file's title wording unless it was a lowercase list shorthand or a filename
|
||||
# keep Stan's own list wording; otherwise take the catalogue's properly-cased title
|
||||
if rec.get("title") and ("lists" not in w.origins or w.title.islower()):
|
||||
w.title = rec["title"]
|
||||
w.subtitle = w.subtitle or rec.get("subtitle")
|
||||
elif src == "arxiv":
|
||||
w.title = rec["title"]
|
||||
if rec.get("authors"):
|
||||
w.authors = rec["authors"][:6]
|
||||
w.year = w.year or rec.get("year")
|
||||
if src == "openlibrary" and rec.get("year") and w.year and rec["year"] < w.year:
|
||||
w.year = w.year # edition year stays; first-publish year recorded in refs
|
||||
w.refs["first_publish_year"] = rec["year"]
|
||||
w.publisher = w.publisher or rec.get("publisher")
|
||||
w.page_count = w.page_count or rec.get("pages")
|
||||
w.language = w.language or rec.get("lang")
|
||||
w.description = w.description or rec.get("description")
|
||||
if rec.get("isbn") and rec["isbn"] not in w.isbns:
|
||||
w.isbns.insert(0 if not w.isbns else len(w.isbns), rec["isbn"])
|
||||
w.ol_work = w.ol_work or rec.get("ol_work")
|
||||
w.ol_edition = w.ol_edition or rec.get("ol_edition")
|
||||
w.gb_id = w.gb_id or rec.get("gb_id")
|
||||
w.cover_url = w.cover_url or rec.get("cover")
|
||||
if rec.get("arxiv"):
|
||||
w.refs["arxiv"] = rec["arxiv"]
|
||||
for s in (rec.get("subjects") or [])[:5]:
|
||||
s = s.lower().strip()
|
||||
if 2 < len(s) < 40 and s not in w.tags:
|
||||
w.tags.append(s)
|
||||
if n % 25 == 0:
|
||||
log(f" enriched {n}/{len(works)} ({http.calls} http calls)")
|
||||
if args.no_network:
|
||||
stats = {"skipped": len(works)}
|
||||
log("metadata:", stats)
|
||||
|
||||
# cover fallback for matched works lacking one: Google Books by isbn
|
||||
if not args.no_network:
|
||||
for w in works:
|
||||
if not w.cover_url and w.status == "matched" and "paper" not in w.tags and w.isbns:
|
||||
g, _ = en.gb(None, [], w.isbns[0])
|
||||
if g and g.get("cover"):
|
||||
w.cover_url = g["cover"]
|
||||
w.gb_id = w.gb_id or g.get("gb_id")
|
||||
|
||||
# ---- 5. plan destinations
|
||||
used: set[str] = set()
|
||||
for w in works:
|
||||
author_dir = safe_name(w.authors[0] if w.authors else "Unknown", 60)
|
||||
title_dir = safe_name(w.title, 90)
|
||||
for fr in w.files:
|
||||
base = f"{author_dir}/{title_dir}/{title_dir}"
|
||||
rel = f"{base}.{fr.fmt}"
|
||||
k = 2
|
||||
while rel in used or (library / rel).exists() and not args.reset:
|
||||
rel = f"{base} ({k}).{fr.fmt}"
|
||||
k += 1
|
||||
used.add(rel)
|
||||
fr.dest_rel = rel
|
||||
|
||||
plan = [{"title": w.title, "authors": w.authors, "isbns": w.isbns, "status": w.status, "origins": sorted(w.origins),
|
||||
"files": [{"from": f.rel_inbox, "to": f.dest_rel, "sha256": f.sha256} for f in w.files], "refs": w.refs,
|
||||
"cover": w.cover_url} for w in works]
|
||||
(root / ".cache").mkdir(exist_ok=True)
|
||||
(root / ".cache" / "ingest-plan.json").write_text(json.dumps(plan, indent=1, default=str))
|
||||
if args.dry_run:
|
||||
log("dry run: plan written to", root / ".cache" / "ingest-plan.json")
|
||||
return
|
||||
|
||||
# ---- 6. write DB
|
||||
def isbn13(w):
|
||||
return next((i for i in w.isbns if i.startswith("978") or i.startswith("979")), None)
|
||||
seen13 = set()
|
||||
from datetime import datetime, timezone
|
||||
now = datetime.now(timezone.utc)
|
||||
with conn.transaction():
|
||||
cur = conn.cursor()
|
||||
if args.reset:
|
||||
cur.execute("truncate books.notes, books.progress, books.shelf_works, books.files, books.works restart identity cascade")
|
||||
cur.execute("select slug, id from books.shelves")
|
||||
shelves = dict(cur.fetchall())
|
||||
for w in works:
|
||||
i13 = isbn13(w)
|
||||
if i13 in seen13:
|
||||
w.refs["isbn13_shared_with_other_work"] = i13
|
||||
i13 = None
|
||||
if i13:
|
||||
seen13.add(i13)
|
||||
refs = dict(w.refs)
|
||||
refs["origins"] = sorted(w.origins)
|
||||
if len(w.isbns) > 1:
|
||||
refs["all_isbns"] = w.isbns
|
||||
cur.execute(
|
||||
"""insert into books.works (title, subtitle, authors, isbn10, isbn13, publisher, year, language, page_count,
|
||||
description, tags, cover_url, openlibrary_work_id, openlibrary_edition_id, google_books_id, annas_md5,
|
||||
source_refs, metadata_status, metadata_source, metadata_fetched_at, notes)
|
||||
values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) returning id""",
|
||||
(w.title, w.subtitle, w.authors, to10(i13) if i13 else None, i13, w.publisher,
|
||||
w.year if w.year and 1400 < w.year < 2100 else None, w.language, w.page_count, w.description, w.tags,
|
||||
w.cover_url, w.ol_work, w.ol_edition, w.gb_id, w.md5s, json.dumps(refs, default=str), w.status,
|
||||
w.source, now if w.status != "pending" else None, w.notes))
|
||||
w.db_id = cur.fetchone()[0]
|
||||
for fr in w.files:
|
||||
cur.execute("insert into books.files (work_id, path, format, size_bytes, sha256, md5, source) values (%s,%s,%s,%s,%s,%s,%s)",
|
||||
(w.db_id, fr.dest_rel, fr.fmt, fr.size, fr.sha256, fr.md5, args.source))
|
||||
for s in sorted(w.shelves):
|
||||
if s in shelves:
|
||||
cur.execute("insert into books.shelf_works (shelf_id, work_id) values (%s,%s) on conflict do nothing", (shelves[s], w.db_id))
|
||||
if "done" in w.shelves and w.refs.get("done_at"):
|
||||
cur.execute("insert into books.progress (work_id, percent, finished_at, status) values (%s, 100, %s, 'finished') on conflict do nothing", (w.db_id, w.refs["done_at"]))
|
||||
|
||||
# ---- 7. covers
|
||||
got = 0
|
||||
if not args.no_network:
|
||||
for w in works:
|
||||
if not w.cover_url:
|
||||
continue
|
||||
dest = covers / (hashlib.sha1(w.cover_url.encode()).hexdigest()[:16] + ".jpg")
|
||||
if dest.exists() or http.download(w.cover_url, dest):
|
||||
got += 1
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("update books.works set cover_path=%s where id=%s", (dest.name, w.db_id))
|
||||
conn.commit()
|
||||
|
||||
# ---- 8. move files (inbox -> library); write undo manifest
|
||||
if args.no_move:
|
||||
log("--no-move: DB rows written, files left in inbox")
|
||||
return
|
||||
manifest = root / ".cache" / f"moves-{time.strftime('%Y%m%d-%H%M%S')}.tsv"
|
||||
dupdir = root / "duplicates"
|
||||
qdir = root / "quarantine"
|
||||
with open(manifest, "w") as mf:
|
||||
for w in works:
|
||||
for fr in w.files:
|
||||
dest = library / fr.dest_rel
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(fr.path), dest)
|
||||
mf.write(f"{fr.path}\t{dest}\n")
|
||||
for d, _ in dups:
|
||||
dest = dupdir / d.rel_inbox
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(d.path), dest)
|
||||
mf.write(f"{d.path}\t{dest}\n")
|
||||
for fr in uniq:
|
||||
if fr.broken and fr.path.exists():
|
||||
dest = qdir / fr.rel_inbox
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(fr.path), dest)
|
||||
mf.write(f"{fr.path}\t{dest}\n")
|
||||
for d in sorted(inbox.rglob("*"), reverse=True):
|
||||
if d.is_dir() and not any(d.iterdir()):
|
||||
d.rmdir()
|
||||
|
||||
summary = {
|
||||
"lists_rows": n_lists, "obsidian_md5s": obs_total, "obsidian_resolved": obs_resolved,
|
||||
"obsidian_overlapping_lists": obs_overlap, "placeholders": n_placeholders,
|
||||
"files_total": len(files), "files_unique": len(uniq), "sha256_duplicates": len(dups),
|
||||
"placeholders_with_files": matched_existing, "works_total": len(works),
|
||||
"works_with_files": sum(1 for w in works if w.files), "metadata": stats, "covers": got,
|
||||
"papers": sum(1 for w in works if "paper" in w.tags), "http_calls": http.calls, "manifest": str(manifest),
|
||||
}
|
||||
(root / ".cache" / "ingest-summary.json").write_text(json.dumps(summary, indent=1))
|
||||
print(json.dumps(summary, indent=1))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1177
scripts/cars-import.ts
Normal file
1177
scripts/cars-import.ts
Normal file
File diff suppressed because it is too large
Load Diff
1129
scripts/cars-photos.ts
Normal file
1129
scripts/cars-photos.ts
Normal file
File diff suppressed because it is too large
Load Diff
26
scripts/cars-refresh.sh
Executable file
26
scripts/cars-refresh.sh
Executable file
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# Weekly-ish refresh of the car pokedex (/trackers/cars) on stan-vm.
|
||||
#
|
||||
# ssh stan-vm 'cd ~/dev/website-cars && git fetch -q && git checkout -q --detach origin/main \
|
||||
# && bun install --frozen-lockfile >/dev/null && scripts/cars-refresh.sh'
|
||||
#
|
||||
# 1. bun scripts/cars-import.ts --report-new rebuilds brands/models/variants from RDW +
|
||||
# fueleconomy.gov and notes which brands/models did not exist before.
|
||||
# 2. bun scripts/cars-photos.ts --missing finds Wikimedia Commons photos for models
|
||||
# that have none yet (new ones included).
|
||||
# stdout: one JSON report {generatedAt, totals, newBrands, newModels[] (with photo)}.
|
||||
# Progress goes to stderr. DB_URL is read from the running app container and never printed.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ -z "${DB_URL:-}" ]; then
|
||||
DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' |
|
||||
sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
|
||||
export DB_URL
|
||||
fi
|
||||
|
||||
report=$(mktemp /tmp/cars-report.XXXXXX.json)
|
||||
trap 'rm -f "$report"' EXIT
|
||||
|
||||
bun scripts/cars-import.ts --report-new "$@" >"$report"
|
||||
bun scripts/cars-photos.ts --missing --enrich "$report" --fetch
|
||||
19
scripts/db-meta-sync.ts
Normal file
19
scripts/db-meta-sync.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
// Refreshes meta.columns from the live catalog for every table in meta.tables
|
||||
// (meta.sync() is SECURITY DEFINER and granted to the app role). Idempotent.
|
||||
// bun scripts/db-meta-sync.ts uses DB_URL (personal_app)
|
||||
import postgres from 'postgres';
|
||||
|
||||
const url = process.env.DB_URL;
|
||||
if (!url) throw new Error('Set DB_URL (app role)');
|
||||
const client = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
||||
try {
|
||||
const [registry] = await client`select to_regprocedure('meta.sync(integer)') is not null as ok`;
|
||||
if (!registry.ok) {
|
||||
console.log('meta.sync() not present; skipping');
|
||||
} else {
|
||||
const [row] = await client`select meta.sync() as added`;
|
||||
console.log(`meta.sync(): ${row.added} column(s) added`);
|
||||
}
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
@@ -107,6 +107,83 @@ try {
|
||||
}
|
||||
if (schema.app_has_ddl)
|
||||
throw new Error('DB_URL must use the app role (personal_app), not the schema owner');
|
||||
// Private health schema: the app role reads and writes it, nobody else (not even personal_ro).
|
||||
const [health] = await client`
|
||||
select
|
||||
count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
|
||||
count(*) filter (
|
||||
where exists (
|
||||
select 1 from aclexplode(c.relacl) a join pg_roles r on r.oid = a.grantee
|
||||
where r.rolname = 'personal_ro'
|
||||
)
|
||||
) as ro_read,
|
||||
count(*) as tables,
|
||||
has_schema_privilege('public', 'health', 'usage') as public_usage
|
||||
from pg_class c join pg_namespace n on n.oid = c.relnamespace
|
||||
where n.nspname = 'health' and c.relkind = 'r'
|
||||
`;
|
||||
if (
|
||||
Number(health.tables) !== 7 ||
|
||||
Number(health.app_rw) !== 7 ||
|
||||
Number(health.ro_read) !== 0 ||
|
||||
health.public_usage
|
||||
)
|
||||
throw new Error(`health schema grants are wrong: ${JSON.stringify(health)}`);
|
||||
// Private task tracker: same rules as health (app role read/write, personal_ro none).
|
||||
const [tasks] = await client`
|
||||
select
|
||||
count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
|
||||
count(*) filter (
|
||||
where exists (
|
||||
select 1 from aclexplode(c.relacl) a join pg_roles r on r.oid = a.grantee
|
||||
where r.rolname = 'personal_ro'
|
||||
)
|
||||
) as ro_read,
|
||||
count(*) as tables,
|
||||
has_schema_privilege('public', 'tasks', 'usage') as public_usage
|
||||
from pg_class c join pg_namespace n on n.oid = c.relnamespace
|
||||
where n.nspname = 'tasks' and c.relkind = 'r'
|
||||
`;
|
||||
if (
|
||||
Number(tasks.tables) !== 4 ||
|
||||
Number(tasks.app_rw) !== 4 ||
|
||||
Number(tasks.ro_read) !== 0 ||
|
||||
tasks.public_usage
|
||||
)
|
||||
throw new Error(`tasks schema grants are wrong: ${JSON.stringify(tasks)}`);
|
||||
// Private grocery tracker: same rules (app role read/write, personal_ro none).
|
||||
const [groceries] = await client`
|
||||
select
|
||||
count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
|
||||
count(*) filter (
|
||||
where exists (
|
||||
select 1 from aclexplode(c.relacl) a join pg_roles r on r.oid = a.grantee
|
||||
where r.rolname = 'personal_ro'
|
||||
)
|
||||
) as ro_read,
|
||||
count(*) as tables,
|
||||
has_schema_privilege('public', 'groceries', 'usage') as public_usage
|
||||
from pg_class c join pg_namespace n on n.oid = c.relnamespace
|
||||
where n.nspname = 'groceries' and c.relkind = 'r'
|
||||
`;
|
||||
if (
|
||||
Number(groceries.tables) !== 5 ||
|
||||
Number(groceries.app_rw) !== 5 ||
|
||||
Number(groceries.ro_read) !== 0 ||
|
||||
groceries.public_usage
|
||||
)
|
||||
throw new Error(`groceries schema grants are wrong: ${JSON.stringify(groceries)}`);
|
||||
// Energy tracker reads the ned-collector tables (SELECT only).
|
||||
const [energy] = await client`
|
||||
select count(*) as hours, max(valid_from) as latest from energy.ned_utilization
|
||||
where granularity = 'Hour' and classification = 'Current'
|
||||
and valid_from > now() - interval '7 days'
|
||||
`;
|
||||
if (!Number(energy.hours))
|
||||
console.warn('Warning: energy.ned_utilization has no hourly data from the last 7 days');
|
||||
console.log(
|
||||
`health: ${health.tables} tables, tasks: ${tasks.tables} tables, groceries: ${groceries.tables} tables, app read/write, personal_ro none; energy: ${energy.hours} hourly rows, latest ${energy.latest ? new Date(energy.latest).toISOString() : 'none'}`
|
||||
);
|
||||
const [catalog] = await client`
|
||||
select
|
||||
(select count(*) from ref.pokemon) as pokemon,
|
||||
|
||||
@@ -18,6 +18,7 @@ environment_name=production
|
||||
|
||||
image="git.stanrunge.dev/stan/website:$sha"
|
||||
compose="$directory/compose.prod.yml"
|
||||
compose_files=(-f "$compose")
|
||||
# Images built before the repo rename (stan/personal-website -> stan/website) keep
|
||||
# their old local name; retag so rollbacks to those SHAs still work.
|
||||
legacy_image="git.stanrunge.dev/stan/personal-website:$sha"
|
||||
@@ -28,13 +29,67 @@ fi
|
||||
if ! sudo docker image inspect "$image" >/dev/null 2>&1; then
|
||||
echo "Image $image is not on this host" >&2; exit 1
|
||||
fi
|
||||
# Book files (ebooks + covers) live on the host and are bind-mounted read-only at /books;
|
||||
# the app streams them only to the owner via /trackers/books/files/<id> (no static path).
|
||||
# Layout: $books_host_dir/library/<files.path> and $books_host_dir/covers/<works.cover_path>.
|
||||
# compose.prod.yml is host-managed, so the mount goes in a generated override file (~deploy).
|
||||
books_host_dir=${BOOKS_HOST_DIR:-/home/stan/books}
|
||||
case "$books_host_dir" in /*) ;; *) echo 'BOOKS_HOST_DIR must be absolute' >&2; exit 2 ;; esac
|
||||
# The deploy account can't traverse /home/stan (0750); check from a root container instead.
|
||||
if sudo docker run --rm --entrypoint sh \
|
||||
--mount "type=bind,source=$books_host_dir,target=/books,readonly" "$image" \
|
||||
-c 'test -d /books/library' 2>/dev/null; then
|
||||
# The deploy account may only sudo-tee $directory/.env, so the override lives in its home;
|
||||
# the compose CLI runs as root and can read it from there.
|
||||
books_override="$HOME/compose.books.yml"
|
||||
rm -f "$books_override"
|
||||
cat >"$books_override" <<YAML
|
||||
services:
|
||||
app:
|
||||
environment:
|
||||
BOOKS_ROOT: /books
|
||||
volumes:
|
||||
- type: bind
|
||||
source: $books_host_dir
|
||||
target: /books
|
||||
read_only: true
|
||||
YAML
|
||||
compose_files+=(-f "$books_override")
|
||||
else
|
||||
echo "No $books_host_dir/library on this host; book files will not be readable" >&2
|
||||
fi
|
||||
# Private money documents (contracts, pension rules) + their summaries also live on the host,
|
||||
# outside the repo, bind-mounted read-only at /money-documents. The app streams them only to an
|
||||
# admin via /trackers/money/documents/<id>/file. Layout: documents.json + files/<file>.
|
||||
money_documents_host_dir=${MONEY_DOCUMENTS_HOST_DIR:-/home/stan/money-documents}
|
||||
case "$money_documents_host_dir" in /*) ;; *) echo 'MONEY_DOCUMENTS_HOST_DIR must be absolute' >&2; exit 2 ;; esac
|
||||
if sudo docker run --rm --entrypoint sh \
|
||||
--mount "type=bind,source=$money_documents_host_dir,target=/money-documents,readonly" "$image" \
|
||||
-c 'test -f /money-documents/documents.json' 2>/dev/null; then
|
||||
money_documents_override="$HOME/compose.money-documents.yml"
|
||||
rm -f "$money_documents_override"
|
||||
cat >"$money_documents_override" <<YAML
|
||||
services:
|
||||
app:
|
||||
environment:
|
||||
MONEY_DOCUMENTS_ROOT: /money-documents
|
||||
volumes:
|
||||
- type: bind
|
||||
source: $money_documents_host_dir
|
||||
target: /money-documents
|
||||
read_only: true
|
||||
YAML
|
||||
compose_files+=(-f "$money_documents_override")
|
||||
else
|
||||
echo "No $money_documents_host_dir/documents.json on this host; money documents will not be readable" >&2
|
||||
fi
|
||||
previous_tag=$(sed -n 's/^IMAGE_TAG=//p' "$directory/.env" | head -n 1)
|
||||
switched=0
|
||||
rollback() {
|
||||
if [ "$switched" -eq 1 ] && [ -n "$previous_tag" ]; then
|
||||
echo "Health check failed; restoring prior image tag $previous_tag" >&2
|
||||
printf 'IMAGE_TAG=%s\n' "$previous_tag" | sudo tee "$directory/.env" >/dev/null
|
||||
sudo docker compose --env-file "$directory/.env" -f "$compose" up -d
|
||||
sudo docker compose --env-file "$directory/.env" "${compose_files[@]}" up -d
|
||||
fi
|
||||
}
|
||||
trap rollback ERR
|
||||
@@ -80,6 +135,12 @@ run_database_script() {
|
||||
if [ "$mode" = deploy ]; then
|
||||
# Schema changes: Drizzle migrations from ./drizzle, as the schema owner.
|
||||
run_database_script scripts/db-migrate.ts
|
||||
# Refresh the generic table GUI's column registry (meta.columns) for tables that migrations
|
||||
# registered in meta.tables. meta.sync() is SECURITY DEFINER and granted to the app role, not
|
||||
# to the schema owner, so run it as DB_URL (personal_app).
|
||||
echo "Running meta.sync()"
|
||||
sudo docker run --rm --network backend --env-file "$environment" "$image" \
|
||||
bun scripts/db-meta-sync.ts
|
||||
run_database_script scripts/seed-catalog.ts
|
||||
run_database_script scripts/seed-cod-catalog.ts
|
||||
run_database_script scripts/seed-pokemon-research.ts
|
||||
@@ -92,9 +153,9 @@ sudo docker run --rm --network backend --env-file "$environment" "$image" \
|
||||
|
||||
printf 'IMAGE_TAG=%s\n' "$sha" | sudo tee "$directory/.env" >/dev/null
|
||||
switched=1
|
||||
sudo docker compose --env-file "$directory/.env" -f "$compose" up -d
|
||||
sudo docker compose --env-file "$directory/.env" "${compose_files[@]}" up -d
|
||||
|
||||
container=$(sudo docker compose --env-file "$directory/.env" -f "$compose" ps -q app)
|
||||
container=$(sudo docker compose --env-file "$directory/.env" "${compose_files[@]}" ps -q app)
|
||||
expected_image=$(sudo docker image inspect -f '{{.Id}}' "$image")
|
||||
actual_image=$(sudo docker inspect -f '{{.Image}}' "$container")
|
||||
if [ "$actual_image" != "$expected_image" ]; then
|
||||
|
||||
63
scripts/groceries-import.ts
Normal file
63
scripts/groceries-import.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
// Nightly supermarket catalogue import (Albert Heijn + Jumbo) and product regrouping.
|
||||
//
|
||||
// bun --env-file=$HOME/.config/groceries-import.env scripts/groceries-import.ts [options]
|
||||
// --store ah|jumbo only this store (repeatable; default: all)
|
||||
// --skip-fetch no store requests; only regroup what is in the DB
|
||||
// --gtin-limit N AH product details to fetch for EANs this run (default 1500, 0 = off)
|
||||
// --delay-ms N minimum delay between requests (default 2000)
|
||||
//
|
||||
// Politeness: one request at a time, >= 2 s apart, backoff on 429/5xx, identifying User-Agent,
|
||||
// and a same-day cache in $GROCERIES_CACHE_DIR (default ~/.cache/groceries, kept 7 days).
|
||||
// Plus is not imported: plus.nl blocks this server's IP range (Incapsula), and we do not evade it.
|
||||
import postgres from 'postgres';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
|
||||
import { enrichAhGtins, importStore, regroup } from '../src/lib/server/groceries/import';
|
||||
import { SOURCES } from '../src/lib/server/groceries/sources';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const values = (name: string) =>
|
||||
args.flatMap((a, i) => (a === name && args[i + 1] ? [args[i + 1]] : []));
|
||||
const num = (name: string, d: number) => {
|
||||
const v = values(name)[0];
|
||||
const n = v === undefined ? d : Number(v);
|
||||
if (!Number.isFinite(n) || n < 0) throw new Error(`${name} must be a number`);
|
||||
return n;
|
||||
};
|
||||
if (!process.env.DB_URL) throw new Error('DB_URL is not set');
|
||||
const stores = values('--store').length ? values('--store') : Object.keys(SOURCES);
|
||||
for (const s of stores) if (!SOURCES[s]) throw new Error(`unknown store ${s}`);
|
||||
const delay = num('--delay-ms', 2000);
|
||||
const gtinLimit = num('--gtin-limit', 1500);
|
||||
const cacheDir = process.env.GROCERIES_CACHE_DIR || join(homedir(), '.cache', 'groceries');
|
||||
const log = (m: string) => console.log(`[${new Date().toISOString().slice(11, 19)}Z] ${m}`);
|
||||
|
||||
const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
||||
let failed = false;
|
||||
try {
|
||||
pruneCache(cacheDir);
|
||||
if (!args.includes('--skip-fetch')) {
|
||||
for (const slug of stores) {
|
||||
const f = new PoliteFetcher({ store: slug, minDelayMs: delay, cacheDir, log });
|
||||
try {
|
||||
log(`${slug}: importing`);
|
||||
const r = await importStore(sql, SOURCES[slug], f, log);
|
||||
log(
|
||||
`${slug}: ${r.fetched} products (${r.inserted} new, ${r.priceChanges} price changes, ${r.delisted} delisted); ${r.requests} requests, ${r.cacheHits} from cache`
|
||||
);
|
||||
if (slug === 'ah' && gtinLimit > 0) {
|
||||
const g = await enrichAhGtins(sql, f, gtinLimit, log);
|
||||
log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
|
||||
}
|
||||
} catch (e) {
|
||||
failed = true;
|
||||
log(`${slug}: FAILED ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
await regroup(sql, log);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
if (failed) process.exitCode = 1;
|
||||
@@ -7,8 +7,9 @@
|
||||
* add --dry-run to validate without writing
|
||||
*
|
||||
* Column names are the snake_case DB columns (see src/lib/health/tables.ts). Rows with an
|
||||
* `id` update that row; nutrition_daily and sleep upsert on `date`; everything else is
|
||||
* inserted (importing the same file twice duplicates those rows). All-or-nothing: if any
|
||||
* `id` update that row; nutrition_daily and sleep upsert on `date`; body_measurements dedupe
|
||||
* on measured_at or (date, weight_kg) and fill smart-scale columns from the notes, so
|
||||
* re-importing is a no-op; everything else is inserted (importing twice duplicates those). All-or-nothing: if any
|
||||
* row is invalid, nothing is written. Uses DB_URL (the app role on prod, personal_dev on dev).
|
||||
*/
|
||||
import { readFile } from 'node:fs/promises';
|
||||
@@ -66,7 +67,7 @@ const client = postgres(process.env.DB_URL, { max: 1, onnotice: () => {} });
|
||||
try {
|
||||
const counts = await writeRows(drizzle({ client }), clean);
|
||||
for (const [t, c] of Object.entries(counts))
|
||||
console.log(`${t}: ${c.inserted} inserted, ${c.updated} updated`);
|
||||
console.log(`${t}: ${c.inserted} inserted, ${c.updated} updated, ${c.unchanged} unchanged`);
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
|
||||
12
scripts/house-db.sh
Executable file
12
scripts/house-db.sh
Executable file
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run a house-tracker script against prod with the limited house_cli role
|
||||
# (house.listings + house.price_history only), without the app's credentials.
|
||||
# scripts/house-db.sh list --city gouda
|
||||
# scripts/house-db.sh add --street "Raam 12" --city Gouda --price 1475
|
||||
# Credentials: ~/.config/house-cli.env (DB_URL=..., mode 600, readable by stan only).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
env_file="${HOUSE_CLI_ENV:-$HOME/.config/house-cli.env}"
|
||||
[ -r "$env_file" ] || { echo "missing $env_file" >&2; exit 1; }
|
||||
unset DATABASE_URL DB_URL
|
||||
exec bun --env-file="$env_file" scripts/house-listing.ts "$@"
|
||||
399
scripts/house-listing.ts
Normal file
399
scripts/house-listing.ts
Normal file
@@ -0,0 +1,399 @@
|
||||
// Add, update, list and remove house tracker listings by hand (screenshots,
|
||||
// links from chats). Writes house.listings directly; no website restart needed.
|
||||
//
|
||||
// bun run house add --url <link> --city Gouda --price 1422 --area 84 --bedrooms 2
|
||||
// bun run house add --json listing.json (or `--json -` for stdin; may be an array)
|
||||
// bun run house update <id|url> --status viewing --notes "Sat 14:00"
|
||||
// bun run house list [--all] [--city gouda] [--offer buy] [--json]
|
||||
// bun run house remove <id|url> [--hard] (soft remove = marked gone)
|
||||
// bun run house restore <id|url>
|
||||
//
|
||||
// DATABASE_URL or DB_URL must point at the database, e.g. `bun --env-file=.env run house list`.
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { parseArgs } from 'node:util';
|
||||
import postgres from 'postgres';
|
||||
import { lbmClassLabel, scoreListings } from '../src/lib/house-tracker/liveability';
|
||||
import {
|
||||
inferOffer,
|
||||
inferPropertyType,
|
||||
keyFromAddress,
|
||||
keyFromUrl,
|
||||
manualListingSchema,
|
||||
normalizeKeys,
|
||||
toColumns,
|
||||
type ListingKey,
|
||||
type ManualListing
|
||||
} from '../src/lib/house-tracker/manual';
|
||||
import {
|
||||
containsPattern,
|
||||
linkHost,
|
||||
normalizeListingUrl,
|
||||
pickLinkedListing,
|
||||
siteKeyFromUrl,
|
||||
type LinkCandidate
|
||||
} from '../src/lib/house-tracker/link-key';
|
||||
|
||||
const usage = `usage: bun run house <add|update|list|remove|restore> [options]
|
||||
|
||||
add [--json <file|->] [field flags] insert, or update when the same link/address exists
|
||||
update <id|url> [field flags] change only the given fields (use "" to clear one)
|
||||
list [--all] [--city <slug>] [--offer rent|buy] [--status <s>] [--limit n] [--json]
|
||||
remove <id|url> [--hard] mark gone (or delete with --hard)
|
||||
restore <id|url> undo a soft remove
|
||||
|
||||
field flags: --url --offer rent|buy --type apartment|house|room|studio|other --title
|
||||
--street --postal-code --city --neighbourhood --price --area --rooms --bedrooms
|
||||
--bathrooms --year --energy-label --interior --deposit --available-from
|
||||
--contract-type --agent --image-url --description --notes --status
|
||||
interested|contacted|viewing|applied|rejected --lat --lng --offered-at
|
||||
--first-seen (when it was shared) --source --source-id
|
||||
other: --dry-run (print, don't write)`;
|
||||
|
||||
const fieldFlags = [
|
||||
'url',
|
||||
'offer',
|
||||
'type',
|
||||
'title',
|
||||
'street',
|
||||
'postal-code',
|
||||
'city',
|
||||
'neighbourhood',
|
||||
'price',
|
||||
'area',
|
||||
'rooms',
|
||||
'bedrooms',
|
||||
'bathrooms',
|
||||
'year',
|
||||
'energy-label',
|
||||
'interior',
|
||||
'deposit',
|
||||
'available-from',
|
||||
'contract-type',
|
||||
'agent',
|
||||
'image-url',
|
||||
'description',
|
||||
'notes',
|
||||
'status',
|
||||
'lat',
|
||||
'lng',
|
||||
'offered-at',
|
||||
'first-seen',
|
||||
'source',
|
||||
'source-id',
|
||||
'external-id'
|
||||
] as const;
|
||||
|
||||
const { values: flags, positionals } = parseArgs({
|
||||
allowPositionals: true,
|
||||
options: {
|
||||
...Object.fromEntries(fieldFlags.map((name) => [name, { type: 'string' as const }])),
|
||||
json: { type: 'string' },
|
||||
all: { type: 'boolean' },
|
||||
hard: { type: 'boolean' },
|
||||
'dry-run': { type: 'boolean' },
|
||||
limit: { type: 'string' },
|
||||
help: { type: 'boolean', short: 'h' }
|
||||
}
|
||||
});
|
||||
const [command, target] = positionals;
|
||||
const flag = (name: string) => (flags as Record<string, string | boolean | undefined>)[name];
|
||||
|
||||
if (!command || flag('help')) {
|
||||
console.log(usage);
|
||||
process.exit(command ? 0 : 1);
|
||||
}
|
||||
|
||||
function fieldInput(): Record<string, unknown> {
|
||||
const raw: Record<string, unknown> = {};
|
||||
for (const name of fieldFlags) {
|
||||
const value = flag(name);
|
||||
// An empty flag clears the column on update.
|
||||
if (typeof value === 'string') raw[name] = value === '' ? null : value;
|
||||
}
|
||||
return normalizeKeys(raw);
|
||||
}
|
||||
|
||||
async function jsonInput(): Promise<Record<string, unknown>[]> {
|
||||
const source = flag('json');
|
||||
if (typeof source !== 'string') return [];
|
||||
const text = source === '-' ? readFileSync(0, 'utf8') : readFileSync(source, 'utf8');
|
||||
const parsed = JSON.parse(text) as unknown;
|
||||
return (Array.isArray(parsed) ? parsed : [parsed]) as Record<string, unknown>[];
|
||||
}
|
||||
|
||||
function parseListing(raw: Record<string, unknown>): ManualListing {
|
||||
const result = manualListingSchema.safeParse(normalizeKeys(raw));
|
||||
if (!result.success) {
|
||||
const issues = result.error.issues.map((i) => ` ${i.path.join('.')}: ${i.message}`);
|
||||
throw new Error(`invalid listing:\n${issues.join('\n')}`);
|
||||
}
|
||||
return result.data;
|
||||
}
|
||||
|
||||
function keyFor(listing: ManualListing): ListingKey {
|
||||
const key = listing.url ? keyFromUrl(listing.url) : keyFromAddress(listing);
|
||||
return {
|
||||
...key,
|
||||
source: listing.source ?? key.source,
|
||||
sourceId: listing.sourceId ?? key.sourceId
|
||||
};
|
||||
}
|
||||
|
||||
const dbUrl = process.env.DATABASE_URL ?? process.env.DB_URL;
|
||||
if (!dbUrl) throw new Error('Set DATABASE_URL (or DB_URL), e.g. bun --env-file=.env run house …');
|
||||
const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
|
||||
type Tx = postgres.TransactionSql;
|
||||
|
||||
const summary = sql`id, source, source_id, offer, property_type, title, city, city_slug,
|
||||
price_eur, area_m2, rooms, bedrooms, status, removed_at, url`;
|
||||
|
||||
async function findId(ref: string | undefined): Promise<number> {
|
||||
if (!ref) throw new Error('give a listing id or url');
|
||||
if (/^\d+$/.test(ref)) return Number(ref);
|
||||
const key = keyFromUrl(ref);
|
||||
const [row] = await sql<{ id: number }[]>`select id from house.listings
|
||||
where (source = ${key.source} and source_id = ${key.sourceId}) or url = ${ref} limit 1`;
|
||||
if (row) return row.id;
|
||||
// Same page under a slightly different link (tracking params, www., trailing slash).
|
||||
const wanted = normalizeListingUrl(ref);
|
||||
const byLink = (await candidatesFor(sql, ref, '')).find(
|
||||
(c) => normalizeListingUrl(c.url) === wanted
|
||||
);
|
||||
if (!byLink) throw new Error(`no listing for ${ref}`);
|
||||
return byLink.id;
|
||||
}
|
||||
|
||||
/** Rows on the same host as `url`, except those under `source`, for pickLinkedListing. */
|
||||
async function candidatesFor(
|
||||
db: postgres.Sql | Tx,
|
||||
url: string | null | undefined,
|
||||
source: string
|
||||
): Promise<LinkCandidate[]> {
|
||||
const host = linkHost(url);
|
||||
if (!host) return [];
|
||||
return db<LinkCandidate[]>`select id, source, source_id as "sourceId", url,
|
||||
duplicate_of as "duplicateOf"
|
||||
from house.listings where source <> ${source} and url ilike ${containsPattern(host)}`;
|
||||
}
|
||||
|
||||
/** Id of the row already keyed (source, source_id), other than `id`. */
|
||||
async function keyOwner(tx: Tx, source: string, sourceId: string, id: number) {
|
||||
const [row] = await tx<{ id: number }[]>`select id from house.listings
|
||||
where source = ${source} and source_id = ${sourceId} and id <> ${id}`;
|
||||
return row?.id ?? null;
|
||||
}
|
||||
|
||||
// Same convention as the worker: a price_history row whenever the price is new or changes.
|
||||
async function recordPrice(tx: Tx, id: number, previous: number | null, next: unknown) {
|
||||
if (typeof next !== 'number' || next === previous) return;
|
||||
await tx`insert into house.price_history (listing_id, price_eur) values (${id}, ${next})`;
|
||||
}
|
||||
|
||||
async function add() {
|
||||
const fromFlags = fieldInput();
|
||||
const inputs = (await jsonInput()).map((item) => ({ ...item, ...fromFlags }));
|
||||
if (!inputs.length) inputs.push(fromFlags);
|
||||
|
||||
for (const raw of inputs) {
|
||||
const listing = parseListing(raw);
|
||||
const key = keyFor(listing);
|
||||
const hint = `${key.url} ${listing.title ?? ''} ${listing.description?.slice(0, 300) ?? ''}`;
|
||||
const given = toColumns(listing);
|
||||
if (listing.url) given.url = key.url;
|
||||
if (!given.city_slug && key.citySlug) given.city_slug = key.citySlug;
|
||||
|
||||
const insert = {
|
||||
source: key.source,
|
||||
source_id: key.sourceId,
|
||||
url: key.url,
|
||||
offer: listing.offer ?? key.offer ?? inferOffer(hint) ?? 'rent',
|
||||
property_type: listing.propertyType ?? key.propertyType ?? inferPropertyType(hint) ?? 'other',
|
||||
first_seen_lane: 'manual',
|
||||
// Nothing for the worker to fetch: we filled in what we know.
|
||||
detail_fetched_at: new Date(),
|
||||
...given
|
||||
};
|
||||
|
||||
if (flag('dry-run')) {
|
||||
console.log(insert);
|
||||
continue;
|
||||
}
|
||||
|
||||
const row = await sql.begin(async (tx) => {
|
||||
let [before] = await tx<{ id: number; price_eur: number | null }[]>`
|
||||
select id, price_eur from house.listings
|
||||
where source = ${key.source} and source_id = ${key.sourceId} for update`;
|
||||
let rekey = false;
|
||||
if (!before && key.source !== 'manual' && listing.url) {
|
||||
// The same link already on a hand-added row: update it and give it the site's key.
|
||||
const linked = pickLinkedListing(
|
||||
await candidatesFor(tx, key.url, key.source),
|
||||
key,
|
||||
key.url
|
||||
);
|
||||
if (linked) {
|
||||
[before] = await tx<{ id: number; price_eur: number | null }[]>`
|
||||
select id, price_eur from house.listings where id = ${linked.id} for update`;
|
||||
rekey = Boolean(before);
|
||||
}
|
||||
}
|
||||
if (before) {
|
||||
const update: Record<string, unknown> = {
|
||||
...given,
|
||||
last_seen_at: new Date(),
|
||||
removed_at: null
|
||||
};
|
||||
if (rekey) Object.assign(update, { source: key.source, source_id: key.sourceId });
|
||||
if (listing.offer) Object.assign(update, { offer: listing.offer });
|
||||
const [updated] = await tx`update house.listings set ${tx(update)}
|
||||
where id = ${before.id} returning ${summary}`;
|
||||
await recordPrice(tx, before.id, before.price_eur, given.price_eur);
|
||||
return { ...updated, action: 'updated' };
|
||||
}
|
||||
const [inserted] = await tx`insert into house.listings ${tx(insert)} returning ${summary}`;
|
||||
await recordPrice(tx, inserted.id, null, given.price_eur);
|
||||
return { ...inserted, action: 'added' };
|
||||
});
|
||||
print([row]);
|
||||
if (row?.id) await liveability(row.id as number);
|
||||
}
|
||||
}
|
||||
|
||||
/** Leefbaarometer score for a listing that was just added or changed (skips if unchanged). */
|
||||
async function liveability(id: number) {
|
||||
try {
|
||||
const r = await scoreListings(sql, { ids: [id] });
|
||||
if (!r.checked) return;
|
||||
const [l] = await sql<
|
||||
{
|
||||
lbm_class: number | null;
|
||||
lbm_onv_class: number | null;
|
||||
lbm_buurt_name: string | null;
|
||||
lbm_precision: string | null;
|
||||
}[]
|
||||
>`select lbm_class, lbm_onv_class, lbm_buurt_name, lbm_precision from house.listings where id = ${id}`;
|
||||
console.log(
|
||||
l?.lbm_class != null
|
||||
? ` leefbaarometer ${l.lbm_class}/9 ${lbmClassLabel(l.lbm_class)} · overlast ${l.lbm_onv_class}/9 · ${l.lbm_buurt_name}${l.lbm_precision === 'address' ? '' : ` (approx., ${l.lbm_precision})`}`
|
||||
: r.failed
|
||||
? ' leefbaarometer: lookup failed, the nightly run retries'
|
||||
: ' leefbaarometer: no location found for this address'
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(` leefbaarometer: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function update() {
|
||||
const id = await findId(target);
|
||||
const listing = parseListing(fieldInput());
|
||||
const given = toColumns(listing);
|
||||
if (listing.url) given.url = listing.url;
|
||||
if (listing.offer) given.offer = listing.offer;
|
||||
if (listing.propertyType) given.property_type = listing.propertyType;
|
||||
if (listing.source) given.source = listing.source;
|
||||
if (listing.sourceId) given.source_id = listing.sourceId;
|
||||
if (!Object.keys(given).length) throw new Error('nothing to update');
|
||||
const [row] = await sql.begin(async (tx) => {
|
||||
const [before] = await tx<
|
||||
{ price_eur: number | null; source: string; source_id: string; url: string }[]
|
||||
>`select price_eur, source, source_id, url from house.listings where id = ${id} for update`;
|
||||
if (!before) throw new Error(`no listing ${id}`);
|
||||
// A manual row with a Funda/Pararius/Kamernet/Vesteda link gets that site's key, so a
|
||||
// later scrape of the listing updates this row instead of adding a copy.
|
||||
const url = (given.url as string | undefined) ?? before.url;
|
||||
const site = siteKeyFromUrl(url);
|
||||
if (before.source === 'manual' && !given.source && !given.source_id && site) {
|
||||
const owner = await keyOwner(tx, site.source, site.sourceId, id);
|
||||
if (owner == null) Object.assign(given, { source: site.source, source_id: site.sourceId });
|
||||
else
|
||||
console.warn(
|
||||
`kept source=manual: ${site.source}/${site.sourceId} is already #${owner} (merge them by hand)`
|
||||
);
|
||||
} else if (given.source || given.source_id) {
|
||||
const source = (given.source as string | undefined) ?? before.source;
|
||||
const sourceId = (given.source_id as string | undefined) ?? before.source_id;
|
||||
const owner = await keyOwner(tx, source, sourceId, id);
|
||||
if (owner != null) throw new Error(`${source}/${sourceId} is already #${owner}`);
|
||||
}
|
||||
if (flag('dry-run')) {
|
||||
console.log({ id, ...given });
|
||||
return [];
|
||||
}
|
||||
const rows =
|
||||
await tx`update house.listings set ${tx(given)} where id = ${id} returning ${summary}`;
|
||||
await recordPrice(tx, id, before.price_eur, given.price_eur);
|
||||
return rows;
|
||||
});
|
||||
if (row) {
|
||||
print([row]);
|
||||
await liveability(id);
|
||||
}
|
||||
}
|
||||
|
||||
async function list() {
|
||||
const limit = Number(flag('limit') ?? 100);
|
||||
const city = flag('city');
|
||||
const offer = flag('offer');
|
||||
const status = flag('status');
|
||||
const rows = await sql`select ${summary}, first_seen_at, notes from house.listings
|
||||
where true
|
||||
${flag('all') ? sql`` : sql`and removed_at is null`}
|
||||
${typeof city === 'string' ? sql`and city_slug = ${city}` : sql``}
|
||||
${typeof offer === 'string' ? sql`and offer = ${offer}` : sql``}
|
||||
${typeof status === 'string' ? sql`and status = ${status}` : sql``}
|
||||
order by first_seen_at desc limit ${limit}`;
|
||||
if (flag('json')) console.log(JSON.stringify(rows, null, 2));
|
||||
else print(rows);
|
||||
}
|
||||
|
||||
async function remove(hard: boolean) {
|
||||
const id = await findId(target);
|
||||
const rows = hard
|
||||
? await sql`delete from house.listings where id = ${id} returning ${summary}`
|
||||
: await sql`update house.listings set removed_at = now() where id = ${id} returning ${summary}`;
|
||||
if (!rows.length) throw new Error(`no listing ${id}`);
|
||||
console.log(hard ? 'deleted:' : 'marked gone:');
|
||||
print(rows);
|
||||
}
|
||||
|
||||
async function restore() {
|
||||
const id = await findId(target);
|
||||
const rows =
|
||||
await sql`update house.listings set removed_at = null where id = ${id} returning ${summary}`;
|
||||
print(rows);
|
||||
}
|
||||
|
||||
function print(rows: readonly Record<string, unknown>[]) {
|
||||
if (!rows.length) return console.log('(no listings)');
|
||||
for (const r of rows) {
|
||||
const bits = [
|
||||
r.price_eur != null && `€${r.price_eur}${r.offer === 'rent' ? '/m' : ''}`,
|
||||
r.area_m2 != null && `${r.area_m2} m²`,
|
||||
r.rooms != null && `${r.rooms} rooms`,
|
||||
r.bedrooms != null && `${r.bedrooms} bed`,
|
||||
r.status,
|
||||
r.removed_at && 'GONE'
|
||||
].filter(Boolean);
|
||||
console.log(
|
||||
`${r.action ? `${r.action} ` : ''}#${r.id} [${r.source}] ${r.title ?? '(no title)'}, ${r.city ?? '?'} · ${bits.join(' · ')}\n ${r.url}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (command === 'add') await add();
|
||||
else if (command === 'update') await update();
|
||||
else if (command === 'list' || command === 'ls') await list();
|
||||
else if (command === 'remove' || command === 'rm') await remove(Boolean(flag('hard')));
|
||||
else if (command === 'restore') await restore();
|
||||
else {
|
||||
console.error(usage);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
25
scripts/house-liveability.ts
Normal file
25
scripts/house-liveability.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
// (Re)computes Leefbaarometer scores for house.listings. The nightly scrape, the CLI and the
|
||||
// page already score new and changed listings; this is for backfills and a new LBM year.
|
||||
//
|
||||
// bun --env-file=$HOME/.config/house-scrape.env scripts/house-liveability.ts [--force] [--id 12]
|
||||
import postgres from 'postgres';
|
||||
import { scoreListings } from '../src/lib/house-tracker/liveability';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
|
||||
if (!dbUrl) throw new Error('DB_URL is not set');
|
||||
const sql = postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
||||
const ids = args.includes('--id') ? [Number(args[args.indexOf('--id') + 1])] : undefined;
|
||||
try {
|
||||
const result = await scoreListings(sql, {
|
||||
ids,
|
||||
force: args.includes('--force'),
|
||||
log: (line) => console.log(line)
|
||||
});
|
||||
console.log(
|
||||
`liveability: ${result.checked} checked, ${result.scored} scored, ${result.notFound} without a location, ${result.failed} failed`
|
||||
);
|
||||
if (result.failed) process.exitCode = 1;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
41
scripts/house-rekey.ts
Normal file
41
scripts/house-rekey.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
// One-off/backfill: give manual house.listings rows with a Funda, Pararius, Kamernet or
|
||||
// Vesteda link that site's (source, source_id), so a later scrape updates them instead
|
||||
// of adding copies. Rows whose key another row already has are reported, not changed.
|
||||
//
|
||||
// DB_URL=... bun scripts/house-rekey.ts [--dry-run]
|
||||
// scripts/house-db.sh is for house-listing.ts; run this with a role that may update listings.
|
||||
import postgres from 'postgres';
|
||||
import { siteKeyFromUrl } from '../src/lib/house-tracker/link-key';
|
||||
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
|
||||
if (!dbUrl) throw new Error('DB_URL is not set');
|
||||
const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
|
||||
|
||||
try {
|
||||
await sql.begin(async (tx) => {
|
||||
const rows = await tx<{ id: number; source_id: string; url: string }[]>`
|
||||
SELECT id, source_id, url FROM house.listings WHERE source = 'manual' ORDER BY id FOR UPDATE`;
|
||||
const claimed = new Set<string>();
|
||||
for (const row of rows) {
|
||||
const key = siteKeyFromUrl(row.url);
|
||||
if (!key) continue;
|
||||
const pair = `${key.source}/${key.sourceId}`;
|
||||
const [owner] = await tx<{ id: number }[]>`SELECT id FROM house.listings
|
||||
WHERE source = ${key.source} AND source_id = ${key.sourceId}`;
|
||||
if (owner || claimed.has(pair)) {
|
||||
console.log(
|
||||
`#${row.id} conflict: ${pair} is already #${owner?.id ?? '(another manual row)'}, kept manual`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
claimed.add(pair);
|
||||
console.log(`#${row.id} ${row.source_id} -> ${pair}${dryRun ? ' (dry run)' : ''}`);
|
||||
if (!dryRun)
|
||||
await tx`UPDATE house.listings SET source = ${key.source}, source_id = ${key.sourceId}
|
||||
WHERE id = ${row.id}`;
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
307
scripts/house-scrape.ts
Normal file
307
scripts/house-scrape.ts
Normal file
@@ -0,0 +1,307 @@
|
||||
// Nightly house scraper, run on stan-vm by a systemd user timer (deploy/systemd/house-scrape.*).
|
||||
// Fetches the sources that allow it, writes house.listings directly and pushes an ntfy alert
|
||||
// for new listings that match an alert rule. See docs/house-tracker.md.
|
||||
//
|
||||
// bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet]
|
||||
//
|
||||
// Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed
|
||||
// paths only. A 403/429/challenge stops that source for the night; nothing is retried or
|
||||
// worked around. Funda, Pararius, huurwoningen.nl and Huurstunt challenge this server's
|
||||
// (datacenter) IP, so they are not scraped from here.
|
||||
import postgres from 'postgres';
|
||||
import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
|
||||
import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address';
|
||||
import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
|
||||
import { scoreListings } from '../src/lib/house-tracker/liveability';
|
||||
import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
|
||||
import { isChallenge } from '../src/lib/house-tracker/pararius';
|
||||
import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
|
||||
import {
|
||||
parseVestedaSearch,
|
||||
VESTEDA_SEARCH_URL,
|
||||
vestedaSearchBody
|
||||
} from '../src/lib/house-tracker/vesteda';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null;
|
||||
|
||||
const USER_AGENT =
|
||||
'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
|
||||
const KAMERNET_CITIES = (
|
||||
process.env.HOUSE_KAMERNET_CITIES ??
|
||||
'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,haastrecht,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
|
||||
)
|
||||
.split(',')
|
||||
.map((c) => c.trim())
|
||||
.filter(Boolean);
|
||||
// Everything in the region is stored (no price/size caps); paging stops early once a page has
|
||||
// none of our cities, so this is only a safety limit.
|
||||
const KAMERNET_MAX_PAGES = 25;
|
||||
// Gouda centre; Vesteda's own search uses km radius around a point.
|
||||
const VESTEDA = {
|
||||
lat: 52.0115,
|
||||
lng: 4.7105,
|
||||
radiusKm: Number(process.env.HOUSE_VESTEDA_RADIUS_KM ?? 20),
|
||||
place: 'Gouda'
|
||||
};
|
||||
const MAX_ALERTS = 5;
|
||||
|
||||
const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
|
||||
if (!dbUrl && !dryRun) throw new Error('DB_URL is not set');
|
||||
const sql = dbUrl ? postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
|
||||
|
||||
class Blocked extends Error {
|
||||
constructor(
|
||||
public status: number,
|
||||
public url: string
|
||||
) {
|
||||
super(`blocked (${status}) on ${url}`);
|
||||
}
|
||||
}
|
||||
|
||||
let lastRequest = 0;
|
||||
async function politeFetch(url: string, init: RequestInit = {}): Promise<Response> {
|
||||
const wait = lastRequest + 6000 + Math.random() * 4000 - Date.now();
|
||||
if (wait > 0) await Bun.sleep(wait);
|
||||
lastRequest = Date.now();
|
||||
const response = await fetch(url, {
|
||||
...init,
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
headers: {
|
||||
'user-agent': USER_AGENT,
|
||||
'accept-language': 'nl-NL,nl;q=0.9,en;q=0.5',
|
||||
...(init.headers ?? {})
|
||||
}
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
async function logRun(
|
||||
source: string,
|
||||
url: string,
|
||||
status: string,
|
||||
extra: { httpStatus?: number; found?: number; durationMs?: number; message?: string } = {}
|
||||
) {
|
||||
console.log(
|
||||
`${source} ${status}${extra.httpStatus ? ` ${extra.httpStatus}` : ''} ${extra.found ?? ''} ${url}${extra.message ? ` (${extra.message})` : ''}`
|
||||
);
|
||||
if (!sql || dryRun) return;
|
||||
await sql`INSERT INTO house.worker_runs (source, lane, url, status, http_status, found, duration_ms, message)
|
||||
VALUES (${source}, 'nightly', ${url}, ${status}, ${extra.httpStatus ?? null}, ${extra.found ?? null},
|
||||
${extra.durationMs ?? null}, ${extra.message?.slice(0, 500) ?? null})`;
|
||||
}
|
||||
|
||||
type SourceResult = { listings: ScrapedListing[]; completed: boolean };
|
||||
|
||||
async function scrapeKamernet(): Promise<SourceResult> {
|
||||
const listings: ScrapedListing[] = [];
|
||||
const seen = new Set<string>();
|
||||
const kamernetAllowed = new Set<string>([...GOUDA_REGION, ...KAMERNET_CITIES]);
|
||||
let completed = true;
|
||||
for (const city of KAMERNET_CITIES) {
|
||||
for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) {
|
||||
const url = kamernetSearchUrl(city, page);
|
||||
const started = Date.now();
|
||||
const response = await politeFetch(url, { headers: { accept: 'text/html' } });
|
||||
const html = await response.text();
|
||||
if (isChallenge(html, response.status)) {
|
||||
await logRun('kamernet', url, 'challenged', { httpStatus: response.status });
|
||||
throw new Blocked(response.status, url);
|
||||
}
|
||||
if (response.status === 404) {
|
||||
// Kamernet has no page for some small places; that is not an error.
|
||||
await logRun('kamernet', url, 'ok', {
|
||||
httpStatus: 404,
|
||||
found: 0,
|
||||
message: 'no such city page'
|
||||
});
|
||||
break;
|
||||
}
|
||||
if (!response.ok) {
|
||||
await logRun('kamernet', url, 'http-error', { httpStatus: response.status });
|
||||
completed = false;
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const result = parseKamernetSearch(html);
|
||||
// Small places get padded with listings from all over the country; keep the
|
||||
// region only, each listing once, and stop paging once a page has none of ours.
|
||||
const local = result.listings.filter((l) => kamernetAllowed.has(citySlug(l.city)));
|
||||
for (const l of local) {
|
||||
if (seen.has(l.sourceId)) continue;
|
||||
seen.add(l.sourceId);
|
||||
listings.push(l);
|
||||
}
|
||||
await logRun('kamernet', url, 'ok', {
|
||||
httpStatus: 200,
|
||||
found: local.length,
|
||||
durationMs: Date.now() - started,
|
||||
message:
|
||||
local.length < result.listings.length
|
||||
? `${result.listings.length - local.length} outside the region ignored`
|
||||
: undefined
|
||||
});
|
||||
const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1;
|
||||
if (page >= pages || local.length === 0) break;
|
||||
} catch (error) {
|
||||
await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) });
|
||||
completed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { listings, completed };
|
||||
}
|
||||
|
||||
async function scrapeVesteda(): Promise<SourceResult> {
|
||||
const started = Date.now();
|
||||
const response = await politeFetch(VESTEDA_SEARCH_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', accept: 'application/json' },
|
||||
body: JSON.stringify(vestedaSearchBody(VESTEDA))
|
||||
});
|
||||
const text = await response.text();
|
||||
if (isChallenge(text, response.status)) {
|
||||
await logRun('vesteda', VESTEDA_SEARCH_URL, 'challenged', { httpStatus: response.status });
|
||||
throw new Blocked(response.status, VESTEDA_SEARCH_URL);
|
||||
}
|
||||
if (!response.ok) {
|
||||
await logRun('vesteda', VESTEDA_SEARCH_URL, 'http-error', { httpStatus: response.status });
|
||||
return { listings: [], completed: false };
|
||||
}
|
||||
try {
|
||||
const listings = parseVestedaSearch(JSON.parse(text));
|
||||
await logRun('vesteda', VESTEDA_SEARCH_URL, 'ok', {
|
||||
httpStatus: 200,
|
||||
found: listings.length,
|
||||
durationMs: Date.now() - started
|
||||
});
|
||||
return { listings, completed: true };
|
||||
} catch (error) {
|
||||
await logRun('vesteda', VESTEDA_SEARCH_URL, 'parse-error', {
|
||||
httpStatus: 200,
|
||||
message: String(error)
|
||||
});
|
||||
return { listings: [], completed: false };
|
||||
}
|
||||
}
|
||||
|
||||
async function ntfy(title: string, body: string, click: string) {
|
||||
const target = process.env.HOUSE_NTFY_URL;
|
||||
if (!target) return false;
|
||||
const url = new URL(target);
|
||||
const response = await fetch(url.origin, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(process.env.HOUSE_NTFY_TOKEN
|
||||
? { authorization: `Bearer ${process.env.HOUSE_NTFY_TOKEN}` }
|
||||
: {})
|
||||
},
|
||||
body: JSON.stringify({
|
||||
topic: url.pathname.replace(/^\/+|\/+$/g, ''),
|
||||
title,
|
||||
message: body,
|
||||
click,
|
||||
tags: ['house'],
|
||||
priority: 3
|
||||
}),
|
||||
signal: AbortSignal.timeout(10_000)
|
||||
}).catch(() => null);
|
||||
return Boolean(response?.ok);
|
||||
}
|
||||
|
||||
const sources: Record<string, () => Promise<SourceResult>> = {
|
||||
kamernet: scrapeKamernet,
|
||||
vesteda: scrapeVesteda
|
||||
};
|
||||
const fresh: { id: number; source: string; listing: ScrapedListing }[] = [];
|
||||
let failures = 0;
|
||||
try {
|
||||
for (const [name, scrape] of Object.entries(sources)) {
|
||||
if (only && only !== name) continue;
|
||||
const runStartedAt = new Date();
|
||||
let result: SourceResult;
|
||||
try {
|
||||
result = await scrape();
|
||||
} catch (error) {
|
||||
failures++;
|
||||
console.error(
|
||||
`${name}: ${error instanceof Error ? error.message : error}; skipped until tomorrow`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (dryRun || !sql) {
|
||||
for (const l of result.listings)
|
||||
console.log(
|
||||
` ${l.sourceId} €${l.priceEur ?? '?'} ${l.areaM2 ?? '?'}m² ${l.title} (${l.city}) ${l.url}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// An empty result from a "complete" run is more likely a broken page than an empty market.
|
||||
const completed = result.completed && result.listings.length > 0;
|
||||
const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed });
|
||||
console.log(
|
||||
`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated${ingest.adopted ? ` (${ingest.adopted} hand-added rows taken over)` : ''}, ${ingest.removed} gone`
|
||||
);
|
||||
if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
|
||||
}
|
||||
|
||||
if (sql && !dryRun) {
|
||||
const { keyed, copies } = await dedupe(sql);
|
||||
console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);
|
||||
|
||||
// Leefbaarometer score for new listings and changed addresses (PDOK + the LBM WFS).
|
||||
try {
|
||||
const lbm = await scoreListings(sql, { limit: 300 });
|
||||
console.log(
|
||||
`liveability: ${lbm.checked} checked, ${lbm.scored} scored, ${lbm.notFound} without a location, ${lbm.failed} failed`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`liveability: ${(error as Error).message}`);
|
||||
}
|
||||
|
||||
const rules =
|
||||
(await sql`SELECT id, name, enabled, offer, min_price AS "minPrice", max_price AS "maxPrice",
|
||||
min_area AS "minArea", min_bedrooms AS "minBedrooms", cities, property_types AS "propertyTypes"
|
||||
FROM house.alert_rules WHERE enabled`) as unknown as AlertRule[];
|
||||
const copyIds = new Set(
|
||||
fresh.length
|
||||
? (
|
||||
await sql<
|
||||
{ id: number }[]
|
||||
>`SELECT id FROM house.listings WHERE duplicate_of IS NOT NULL AND id IN ${sql(fresh.map((f) => f.id))}`
|
||||
).map((r) => r.id)
|
||||
: []
|
||||
);
|
||||
const matches = fresh.flatMap((f) => {
|
||||
if (copyIds.has(f.id)) return [];
|
||||
const listing = { ...f.listing, citySlug: citySlug(f.listing.city), source: f.source };
|
||||
const rule = rules.find((r) => matchesRule(listing, r));
|
||||
return rule ? [{ ...f, message: alertMessage(listing, rule) }] : [];
|
||||
});
|
||||
let sent = 0;
|
||||
for (const match of matches.slice(0, MAX_ALERTS)) {
|
||||
if (await ntfy(match.message.title, match.message.body, match.message.url)) {
|
||||
sent++;
|
||||
await sql`UPDATE house.listings SET alerted_at = now() WHERE id = ${match.id}`;
|
||||
}
|
||||
}
|
||||
if (matches.length > MAX_ALERTS)
|
||||
await ntfy(
|
||||
`🏠 ${matches.length - MAX_ALERTS} more new listings`,
|
||||
'See the house tracker.',
|
||||
'https://stanrunge.dev/trackers/houses?status='
|
||||
);
|
||||
console.log(
|
||||
`alerts: ${matches.length} new matches, ${sent} sent${process.env.HOUSE_NTFY_URL ? '' : ' (HOUSE_NTFY_URL not set)'}`
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await sql?.end();
|
||||
}
|
||||
|
||||
// Non-zero so systemd shows the unit as failed when a source broke or blocked us.
|
||||
if (failures > 0) process.exitCode = 1;
|
||||
@@ -50,11 +50,29 @@ const config = {
|
||||
api: (process.env.HOUSE_API_URL ?? '').replace(/\/$/, ''),
|
||||
token: process.env.HOUSE_WORKER_TOKEN ?? '',
|
||||
fastSearches: list(process.env.HOUSE_FAST_SEARCHES, [
|
||||
'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
|
||||
// No price band: every Gouda rental is stored; the alert rule decides what pings.
|
||||
'https://www.pararius.nl/huurwoningen/gouda'
|
||||
]),
|
||||
slowSearches: list(
|
||||
process.env.HOUSE_SLOW_SEARCHES,
|
||||
['gouda', 'rotterdam', 'den-haag', 'utrecht', 'amsterdam', 'eindhoven'].flatMap((city) => [
|
||||
[
|
||||
'gouda',
|
||||
'waddinxveen',
|
||||
'reeuwijk',
|
||||
'bodegraven',
|
||||
'boskoop',
|
||||
'moordrecht',
|
||||
'nieuwerkerk-aan-den-ijssel',
|
||||
'zevenhuizen',
|
||||
'oudewater',
|
||||
'schoonhoven',
|
||||
'haastrecht',
|
||||
'rotterdam',
|
||||
'den-haag',
|
||||
'utrecht',
|
||||
'amsterdam',
|
||||
'eindhoven'
|
||||
].flatMap((city) => [
|
||||
`https://www.pararius.nl/huurwoningen/${city}`,
|
||||
`https://www.pararius.nl/koopwoningen/${city}`
|
||||
])
|
||||
|
||||
16
scripts/music-db.ts
Normal file
16
scripts/music-db.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
// Shared database connection for the music scripts: MUSIC_DB_URL (e.g. the optional
|
||||
// music_collector role) or DB_URL (the app role, as in the image's prod.env).
|
||||
import postgres from 'postgres';
|
||||
|
||||
export function musicDb() {
|
||||
const url = process.env.MUSIC_DB_URL || process.env.DB_URL;
|
||||
if (!url) throw new Error('Set MUSIC_DB_URL or DB_URL');
|
||||
return postgres(url, { max: 1, prepare: false, connect_timeout: 10, onnotice: () => {} });
|
||||
}
|
||||
|
||||
export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export function arg(name: string): string | undefined {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 ? process.argv[i + 1] : undefined;
|
||||
}
|
||||
69
scripts/music-import-history.ts
Normal file
69
scripts/music-import-history.ts
Normal file
@@ -0,0 +1,69 @@
|
||||
// Import a Spotify data export into music.plays (source 'spotify', origin 'export').
|
||||
//
|
||||
// bun scripts/music-import-history.ts <file-or-dir> [...] [--min-ms 30000] [--dry-run]
|
||||
//
|
||||
// Accepts the Extended Streaming History (Streaming_History_Audio_*.json, older
|
||||
// endsong_*.json) and the short account-data export (StreamingHistory_music_*.json).
|
||||
// Podcasts/audiobooks and plays under --min-ms are skipped. Safe to re-run: a play already
|
||||
// stored (same artist+title within a minute, also from the live poller) is not added twice.
|
||||
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { DEFAULT_MIN_MS, parseHistoryEntry, type HistoryPlay } from '../src/lib/music/history';
|
||||
import { insertPlay, newCache, upsertTrack } from '../src/lib/music/store.server';
|
||||
import { arg, musicDb } from './music-db';
|
||||
|
||||
const minMs = Number(arg('min-ms') ?? DEFAULT_MIN_MS);
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
const skip = new Set(['--min-ms', arg('min-ms')]);
|
||||
const inputs = process.argv.slice(2).filter((a) => !a.startsWith('--') && !skip.has(a));
|
||||
if (!inputs.length) throw new Error('Usage: bun scripts/music-import-history.ts <file-or-dir> ...');
|
||||
|
||||
const NAME = /^(Streaming_History_Audio_.*|endsong_?\d*|StreamingHistory(_music)?_?\d*)\.json$/i;
|
||||
function files(path: string): string[] {
|
||||
if (statSync(path).isDirectory())
|
||||
return readdirSync(path).flatMap((name) => {
|
||||
const child = join(path, name);
|
||||
return statSync(child).isDirectory() ? files(child) : NAME.test(name) ? [child] : [];
|
||||
});
|
||||
return [path];
|
||||
}
|
||||
|
||||
const plays: HistoryPlay[] = [];
|
||||
let entries = 0;
|
||||
for (const file of inputs.flatMap(files)) {
|
||||
const data = JSON.parse(readFileSync(file, 'utf8')) as unknown[];
|
||||
if (!Array.isArray(data)) continue;
|
||||
entries += data.length;
|
||||
for (const entry of data) {
|
||||
const play = parseHistoryEntry(entry, minMs);
|
||||
if (play) plays.push(play);
|
||||
}
|
||||
console.log(`${file}: ${data.length} entries`);
|
||||
}
|
||||
plays.sort((a, b) => a.playedAt.getTime() - b.playedAt.getTime());
|
||||
console.log(`${plays.length} music plays of ${minMs / 1000}s or more out of ${entries} entries.`);
|
||||
if (dryRun || !plays.length) process.exit(0);
|
||||
|
||||
const sql = musicDb();
|
||||
try {
|
||||
const cache = newCache();
|
||||
let stored = 0;
|
||||
for (const [i, play] of plays.entries()) {
|
||||
const trackId = await upsertTrack(sql, play.track, false, cache);
|
||||
if (
|
||||
await insertPlay(sql, {
|
||||
trackId,
|
||||
playedAt: play.playedAt,
|
||||
source: 'spotify',
|
||||
origin: 'export',
|
||||
msPlayed: play.msPlayed,
|
||||
platform: play.platform
|
||||
})
|
||||
)
|
||||
stored++;
|
||||
if ((i + 1) % 5000 === 0) console.log(` ${i + 1}/${plays.length} (${stored} new)`);
|
||||
}
|
||||
console.log(`Done: ${stored} new plays stored.`);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
132
scripts/music-import-lastfm.ts
Normal file
132
scripts/music-import-lastfm.ts
Normal file
@@ -0,0 +1,132 @@
|
||||
// Backfill the full Last.fm history of a user into music.plays (source 'lastfm').
|
||||
//
|
||||
// LASTFM_API_KEY=... bun scripts/music-import-lastfm.ts [--user stanrunge] [--sleep 300]
|
||||
// [--max-pages N] [--reset]
|
||||
//
|
||||
// Walks user.getRecentTracks newest → oldest in 200-scrobble pages, always asking for page 1
|
||||
// of a shrinking [from, to] window, so new scrobbles never shift the pages under it.
|
||||
// Progress is saved after every page in music.import_state ('lastfm:<user>'): stop it any
|
||||
// time and re-run to resume; once complete, a re-run only fetches scrobbles newer than the
|
||||
// last completed run. Overlap with Spotify plays is hidden by the music.plays_deduped view.
|
||||
// Rate limit: one request per --sleep ms (default 300, Last.fm asks for at most 5/s),
|
||||
// exponential back-off on 429/5xx and Last.fm errors 8, 11, 16 and 29.
|
||||
import { parseRecentTracks } from '../src/lib/music/lastfm-parse';
|
||||
import {
|
||||
getImportState,
|
||||
insertPlay,
|
||||
newCache,
|
||||
setImportState,
|
||||
upsertTrack
|
||||
} from '../src/lib/music/store.server';
|
||||
import { arg, musicDb, sleep } from './music-db';
|
||||
|
||||
type State = {
|
||||
completeUntil: number;
|
||||
walk: { from: number; to: number; top: number } | null;
|
||||
imported: number;
|
||||
seen: number;
|
||||
};
|
||||
|
||||
const apiKey = process.env.LASTFM_API_KEY;
|
||||
if (!apiKey) throw new Error('Set LASTFM_API_KEY (https://www.last.fm/api/account/create)');
|
||||
const user = arg('user') ?? 'stanrunge';
|
||||
const pause = Number(arg('sleep') ?? 300);
|
||||
const maxPages = Number(arg('max-pages') ?? Infinity);
|
||||
const key = `lastfm:${user.toLowerCase()}`;
|
||||
const RETRYABLE = new Set([8, 11, 16, 29]);
|
||||
|
||||
async function fetchPage(from: number, to: number) {
|
||||
const url = `https://ws.audioscrobbler.com/2.0/?${new URLSearchParams({
|
||||
method: 'user.getrecenttracks',
|
||||
user,
|
||||
api_key: apiKey!,
|
||||
format: 'json',
|
||||
limit: '200',
|
||||
page: '1',
|
||||
from: String(from),
|
||||
to: String(to)
|
||||
})}`;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
let retry: boolean;
|
||||
let message: string;
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: { 'User-Agent': 'stanrunge.dev music importer (stan@stanrunge.dev)' },
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
});
|
||||
const body = (await response.json().catch(() => null)) as Record<string, unknown> | null;
|
||||
if (response.ok && body && !('error' in body)) return body;
|
||||
const code = Number(body?.error);
|
||||
message = `HTTP ${response.status}${body?.message ? `: ${String(body.message)}` : ''}`;
|
||||
retry = response.status === 429 || response.status >= 500 || RETRYABLE.has(code);
|
||||
} catch (e) {
|
||||
message = e instanceof Error ? e.message : String(e);
|
||||
retry = true;
|
||||
}
|
||||
if (!retry || attempt >= 6) throw new Error(`Last.fm request failed: ${message}`);
|
||||
const wait = 2 ** attempt * 1000;
|
||||
console.warn(` ${message}; retrying in ${wait / 1000}s`);
|
||||
await sleep(wait);
|
||||
}
|
||||
}
|
||||
|
||||
const sql = musicDb();
|
||||
try {
|
||||
let state: State = (process.argv.includes('--reset')
|
||||
? null
|
||||
: await getImportState<State>(sql, key)) ?? {
|
||||
completeUntil: 0,
|
||||
walk: null,
|
||||
imported: 0,
|
||||
seen: 0
|
||||
};
|
||||
if (!state.walk) {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
state = { ...state, walk: { from: state.completeUntil + 1, to: now, top: now } };
|
||||
}
|
||||
const cache = newCache();
|
||||
let pages = 0;
|
||||
console.log(
|
||||
`Last.fm ${user}: importing ${new Date(state.walk!.from * 1000).toISOString()} → ${new Date(state.walk!.to * 1000).toISOString()}`
|
||||
);
|
||||
while (state.walk && pages < maxPages) {
|
||||
const walk = state.walk;
|
||||
const page =
|
||||
walk.from > walk.to ? null : parseRecentTracks(await fetchPage(walk.from, walk.to));
|
||||
pages++;
|
||||
if (!page || page.plays.length === 0) {
|
||||
state = { ...state, completeUntil: walk.top, walk: null };
|
||||
await setImportState(sql, key, state);
|
||||
console.log(`Done: ${state.imported} new plays stored (${state.seen} scrobbles seen).`);
|
||||
break;
|
||||
}
|
||||
let stored = 0;
|
||||
for (const play of page.plays) {
|
||||
const trackId = await upsertTrack(sql, play.track, false, cache);
|
||||
if (
|
||||
await insertPlay(sql, {
|
||||
trackId,
|
||||
playedAt: play.playedAt,
|
||||
source: 'lastfm',
|
||||
origin: 'import'
|
||||
})
|
||||
)
|
||||
stored++;
|
||||
}
|
||||
const oldest = Math.min(...page.plays.map((p) => p.uts));
|
||||
state = {
|
||||
...state,
|
||||
imported: state.imported + stored,
|
||||
seen: state.seen + page.plays.length,
|
||||
walk: { ...walk, to: oldest >= walk.to ? walk.to - 1 : oldest }
|
||||
};
|
||||
await setImportState(sql, key, state);
|
||||
console.log(
|
||||
` page ${pages}: ${page.plays.length} scrobbles, ${stored} new, back to ${new Date(oldest * 1000).toISOString()} (~${page.total} left)`
|
||||
);
|
||||
await sleep(pause);
|
||||
}
|
||||
if (state.walk) console.log(`Stopped after ${pages} pages; re-run to resume.`);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
18
scripts/music-poll.ts
Normal file
18
scripts/music-poll.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
// One Spotify poll (now playing + recently played), for the optional host timer in
|
||||
// deploy/music-collector. The app normally does this itself every minute (see
|
||||
// src/lib/server/music/poller.ts); the lease in music.spotify_auth stops double polling.
|
||||
// MUSIC_DB_URL=... SPOTIFY_CLIENT_ID=... SPOTIFY_CLIENT_SECRET=... bun scripts/music-poll.ts
|
||||
import { pollSpotify } from '../src/lib/music/collector.server';
|
||||
import { musicDb } from './music-db';
|
||||
|
||||
const clientId = process.env.SPOTIFY_CLIENT_ID;
|
||||
const clientSecret = process.env.SPOTIFY_CLIENT_SECRET;
|
||||
if (!clientId || !clientSecret) throw new Error('Set SPOTIFY_CLIENT_ID and SPOTIFY_CLIENT_SECRET');
|
||||
const sql = musicDb();
|
||||
try {
|
||||
const result = await pollSpotify(sql, { clientId, clientSecret });
|
||||
console.log(JSON.stringify(result));
|
||||
if (result.status === 'error') process.exitCode = 1;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
1464
scripts/planes-import.ts
Normal file
1464
scripts/planes-import.ts
Normal file
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user