stats: fix role-name quoting in the stats migration grants; add deploy/stats-collector (systemd user timers for CBS, Eurostat, FRED, BIS)

deploy/stats-collector/README.md

@@ -0,0 +1,28 @@
+# stats collector
+
+Host-side collector for the public [/stats](https://stanrunge.dev/stats) section. It pulls open
+statistics (no API keys) into the `stats` schema with `scripts/stats-fetch.ts`:
+
+| source | timer | schedule (Europe/Amsterdam) | what |
+|---|---|---|---|
+| `cbs` | `stats-cbs.timer` | nightly 05:40 | CBS StatLine OData v4: checks every table's `Modified`, re-pulls changed ones, flags discontinued tables (base-year swaps) |
+| `eurostat` | `stats-eurostat.timer` | Mondays 06:20 | Eurostat JSON-stat datasets (energy prices, yields, HICP, government finance…) |
+| `fred` | `stats-fred.timer` | daily 07:20 | FRED graph CSV (Brent, TTF, Fed, Treasuries, US energy prices, indices) |
+| `bis` | `stats-bis.timer` | daily 07:50 | BIS central bank policy rates, monthly + daily |
+
+Each timer starts `stats-collector@<source>.service`, which runs
+`~/.local/bin/stats-collector-run.sh <source>`: it updates a detached checkout of `origin/main`
+in `~/dev/stats-runner`, reads `DB_URL` (app role) from the running app container, and runs the
+fetch. Every job is logged in `stats.fetch_run` (shown as "data freshness" on /stats).
+
+Politeness: one shared User-Agent, a per-host gap between requests (CBS 250 ms ≈ 4 req/s,
+OECD 3.1 s ≈ 19/min, FRED/BIS 1 s), retries with backoff on 429/5xx.
+
+```sh
+bash deploy/stats-collector/install.sh                     # install units + enable timers
+systemctl --user start stats-collector@fred.service        # run one source now
+# long backfills: detached, survive SSH drops; logs in journalctl --user -u stats-backfill-cbs
+systemd-run --user --unit=stats-backfill-cbs ~/.local/bin/stats-collector-run.sh cbs --backfill
+```
+
+Never touches the `energy` schema or ned-collector.

deploy/stats-collector/install.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Install the /stats collector on the host as systemd user units (run as stan on stan-ovh):
+# the runner script, the stats-collector@.service template and one timer per source.
+#   bash deploy/stats-collector/install.sh            install + enable every timer
+#   bash deploy/stats-collector/install.sh --no-enable install only
+set -euo pipefail
+here="$(cd "$(dirname "$0")" && pwd)"
+units="$HOME/.config/systemd/user"
+install -D -m 755 "$here/stats-collector-run.sh" "$HOME/.local/bin/stats-collector-run.sh"
+install -D -m 644 "$here/stats-collector@.service" "$units/stats-collector@.service"
+timers=()
+for timer in "$here"/stats-*.timer; do
+	install -m 644 "$timer" "$units/$(basename "$timer")"
+	timers+=("$(basename "$timer")")
+done
+systemctl --user daemon-reload
+if [ "${1:-}" != --no-enable ]; then
+	systemctl --user enable --now "${timers[@]}"
+fi
+systemctl --user list-timers --no-pager 'stats-*'

deploy/stats-collector/stats-bis.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: BIS policy rates daily
+
+[Timer]
+OnCalendar=*-*-* 07:50:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@bis.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-cbs.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: CBS StatLine nightly (re-pulls tables whose Modified changed)
+
+[Timer]
+OnCalendar=*-*-* 05:40:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@cbs.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-collector-run.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# /stats collector runner (systemd: stats-collector@<source>.service, one timer per source).
+#   stats-collector-run.sh <source> [--backfill|--force|--only=…]
+# Runs scripts/stats-fetch.ts from a detached checkout of origin/main (~/dev/stats-runner), with
+# DB_URL read fresh from the app container each run (never stored on disk). One lock per source,
+# so a slow backfill and its timer never overlap; a shared lock guards the checkout update.
+set -euo pipefail
+source_name=${1:?source required (cbs, eurostat, fred, bis, …)}
+case "$source_name" in *[!a-z0-9-]*) echo "bad source name" >&2; exit 2 ;; esac
+runner="${STATS_RUNNER_DIR:-$HOME/dev/stats-runner}"
+locks="${XDG_RUNTIME_DIR:-/tmp}/stats-collector"
+mkdir -p "$locks"
+exec 9>"$locks/$source_name.lock"
+if ! flock -w "${STATS_LOCK_WAIT:-3600}" 9; then
+	echo "another $source_name run still holds the lock; giving up" >&2
+	exit 75
+fi
+if [ ! -e "$runner/.git" ]; then
+	git -C "$HOME/dev/personal-website" fetch -q origin
+	git -C "$HOME/dev/personal-website" worktree add -q --detach "$runner" origin/main
+fi
+(
+	flock -w 600 8
+	cd "$runner"
+	git fetch -q origin
+	git checkout -q --detach origin/main
+	bun install --frozen-lockfile --silent
+) 8>"$locks/checkout.lock"
+cd "$runner"
+DB_URL=$(sudo -n docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' \
+	| sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
+[ -n "$DB_URL" ] || { echo "no DB_URL from app container" >&2; exit 1; }
+export DB_URL
+exec nice -n 10 bun scripts/stats-fetch.ts "$@"

deploy/stats-collector/stats-collector@.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=stanrunge.dev /stats collector: %i (open data → stats.*)
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=%h/.local/bin/stats-collector-run.sh %i
+TimeoutStartSec=3h
+Nice=10

deploy/stats-collector/stats-eurostat.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: Eurostat weekly
+
+[Timer]
+OnCalendar=Mon *-*-* 06:20:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@eurostat.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-fred.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: FRED graph CSV daily
+
+[Timer]
+OnCalendar=*-*-* 07:20:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@fred.service
+
+[Install]
+WantedBy=timers.target