From c4397e9d594b9544542e5cf3ace7178ae63c65f1 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sun, 25 Oct 2026 20:36:00 +0000 Subject: [PATCH] stats: fix role-name quoting in the stats migration grants; add deploy/stats-collector (systemd user timers for CBS, Eurostat, FRED, BIS) --- deploy/stats-collector/README.md | 28 +++++++++++++++ deploy/stats-collector/install.sh | 20 +++++++++++ deploy/stats-collector/stats-bis.timer | 11 ++++++ deploy/stats-collector/stats-cbs.timer | 11 ++++++ deploy/stats-collector/stats-collector-run.sh | 34 +++++++++++++++++++ .../stats-collector/stats-collector@.service | 10 ++++++ deploy/stats-collector/stats-eurostat.timer | 11 ++++++ deploy/stats-collector/stats-fred.timer | 11 ++++++ .../20261025202715_stats_core/migration.sql | 4 +-- 9 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 deploy/stats-collector/README.md create mode 100644 deploy/stats-collector/install.sh create mode 100644 deploy/stats-collector/stats-bis.timer create mode 100644 deploy/stats-collector/stats-cbs.timer create mode 100644 deploy/stats-collector/stats-collector-run.sh create mode 100644 deploy/stats-collector/stats-collector@.service create mode 100644 deploy/stats-collector/stats-eurostat.timer create mode 100644 deploy/stats-collector/stats-fred.timer diff --git a/deploy/stats-collector/README.md b/deploy/stats-collector/README.md new file mode 100644 index 0000000..88baad3 --- /dev/null +++ b/deploy/stats-collector/README.md @@ -0,0 +1,28 @@ +# stats collector + +Host-side collector for the public [/stats](https://stanrunge.dev/stats) section. It pulls open +statistics (no API keys) into the `stats` schema with `scripts/stats-fetch.ts`: + +| source | timer | schedule (Europe/Amsterdam) | what | +|---|---|---|---| +| `cbs` | `stats-cbs.timer` | nightly 05:40 | CBS StatLine OData v4: checks every table's `Modified`, re-pulls changed ones, flags discontinued tables (base-year swaps) | +| `eurostat` | `stats-eurostat.timer` | Mondays 06:20 | Eurostat JSON-stat datasets (energy prices, yields, HICP, government finance…) | +| `fred` | `stats-fred.timer` | daily 07:20 | FRED graph CSV (Brent, TTF, Fed, Treasuries, US energy prices, indices) | +| `bis` | `stats-bis.timer` | daily 07:50 | BIS central bank policy rates, monthly + daily | + +Each timer starts `stats-collector@.service`, which runs +`~/.local/bin/stats-collector-run.sh `: it updates a detached checkout of `origin/main` +in `~/dev/stats-runner`, reads `DB_URL` (app role) from the running app container, and runs the +fetch. Every job is logged in `stats.fetch_run` (shown as "data freshness" on /stats). + +Politeness: one shared User-Agent, a per-host gap between requests (CBS 250 ms ≈ 4 req/s, +OECD 3.1 s ≈ 19/min, FRED/BIS 1 s), retries with backoff on 429/5xx. + +```sh +bash deploy/stats-collector/install.sh # install units + enable timers +systemctl --user start stats-collector@fred.service # run one source now +# long backfills: detached, survive SSH drops; logs in journalctl --user -u stats-backfill-cbs +systemd-run --user --unit=stats-backfill-cbs ~/.local/bin/stats-collector-run.sh cbs --backfill +``` + +Never touches the `energy` schema or ned-collector. diff --git a/deploy/stats-collector/install.sh b/deploy/stats-collector/install.sh new file mode 100644 index 0000000..6e17a86 --- /dev/null +++ b/deploy/stats-collector/install.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Install the /stats collector on the host as systemd user units (run as stan on stan-ovh): +# the runner script, the stats-collector@.service template and one timer per source. +# bash deploy/stats-collector/install.sh install + enable every timer +# bash deploy/stats-collector/install.sh --no-enable install only +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +units="$HOME/.config/systemd/user" +install -D -m 755 "$here/stats-collector-run.sh" "$HOME/.local/bin/stats-collector-run.sh" +install -D -m 644 "$here/stats-collector@.service" "$units/stats-collector@.service" +timers=() +for timer in "$here"/stats-*.timer; do + install -m 644 "$timer" "$units/$(basename "$timer")" + timers+=("$(basename "$timer")") +done +systemctl --user daemon-reload +if [ "${1:-}" != --no-enable ]; then + systemctl --user enable --now "${timers[@]}" +fi +systemctl --user list-timers --no-pager 'stats-*' diff --git a/deploy/stats-collector/stats-bis.timer b/deploy/stats-collector/stats-bis.timer new file mode 100644 index 0000000..f4e823d --- /dev/null +++ b/deploy/stats-collector/stats-bis.timer @@ -0,0 +1,11 @@ +[Unit] +Description=/stats collector: BIS policy rates daily + +[Timer] +OnCalendar=*-*-* 07:50:00 Europe/Amsterdam +RandomizedDelaySec=10min +Persistent=true +Unit=stats-collector@bis.service + +[Install] +WantedBy=timers.target diff --git a/deploy/stats-collector/stats-cbs.timer b/deploy/stats-collector/stats-cbs.timer new file mode 100644 index 0000000..0b9bd5e --- /dev/null +++ b/deploy/stats-collector/stats-cbs.timer @@ -0,0 +1,11 @@ +[Unit] +Description=/stats collector: CBS StatLine nightly (re-pulls tables whose Modified changed) + +[Timer] +OnCalendar=*-*-* 05:40:00 Europe/Amsterdam +RandomizedDelaySec=10min +Persistent=true +Unit=stats-collector@cbs.service + +[Install] +WantedBy=timers.target diff --git a/deploy/stats-collector/stats-collector-run.sh b/deploy/stats-collector/stats-collector-run.sh new file mode 100644 index 0000000..79e8b79 --- /dev/null +++ b/deploy/stats-collector/stats-collector-run.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# /stats collector runner (systemd: stats-collector@.service, one timer per source). +# stats-collector-run.sh [--backfill|--force|--only=…] +# Runs scripts/stats-fetch.ts from a detached checkout of origin/main (~/dev/stats-runner), with +# DB_URL read fresh from the app container each run (never stored on disk). One lock per source, +# so a slow backfill and its timer never overlap; a shared lock guards the checkout update. +set -euo pipefail +source_name=${1:?source required (cbs, eurostat, fred, bis, …)} +case "$source_name" in *[!a-z0-9-]*) echo "bad source name" >&2; exit 2 ;; esac +runner="${STATS_RUNNER_DIR:-$HOME/dev/stats-runner}" +locks="${XDG_RUNTIME_DIR:-/tmp}/stats-collector" +mkdir -p "$locks" +exec 9>"$locks/$source_name.lock" +if ! flock -w "${STATS_LOCK_WAIT:-3600}" 9; then + echo "another $source_name run still holds the lock; giving up" >&2 + exit 75 +fi +if [ ! -e "$runner/.git" ]; then + git -C "$HOME/dev/personal-website" fetch -q origin + git -C "$HOME/dev/personal-website" worktree add -q --detach "$runner" origin/main +fi +( + flock -w 600 8 + cd "$runner" + git fetch -q origin + git checkout -q --detach origin/main + bun install --frozen-lockfile --silent +) 8>"$locks/checkout.lock" +cd "$runner" +DB_URL=$(sudo -n docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' \ + | sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/') +[ -n "$DB_URL" ] || { echo "no DB_URL from app container" >&2; exit 1; } +export DB_URL +exec nice -n 10 bun scripts/stats-fetch.ts "$@" diff --git a/deploy/stats-collector/stats-collector@.service b/deploy/stats-collector/stats-collector@.service new file mode 100644 index 0000000..c7e3b8d --- /dev/null +++ b/deploy/stats-collector/stats-collector@.service @@ -0,0 +1,10 @@ +[Unit] +Description=stanrunge.dev /stats collector: %i (open data → stats.*) +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +ExecStart=%h/.local/bin/stats-collector-run.sh %i +TimeoutStartSec=3h +Nice=10 diff --git a/deploy/stats-collector/stats-eurostat.timer b/deploy/stats-collector/stats-eurostat.timer new file mode 100644 index 0000000..5822f8c --- /dev/null +++ b/deploy/stats-collector/stats-eurostat.timer @@ -0,0 +1,11 @@ +[Unit] +Description=/stats collector: Eurostat weekly + +[Timer] +OnCalendar=Mon *-*-* 06:20:00 Europe/Amsterdam +RandomizedDelaySec=10min +Persistent=true +Unit=stats-collector@eurostat.service + +[Install] +WantedBy=timers.target diff --git a/deploy/stats-collector/stats-fred.timer b/deploy/stats-collector/stats-fred.timer new file mode 100644 index 0000000..b411a69 --- /dev/null +++ b/deploy/stats-collector/stats-fred.timer @@ -0,0 +1,11 @@ +[Unit] +Description=/stats collector: FRED graph CSV daily + +[Timer] +OnCalendar=*-*-* 07:20:00 Europe/Amsterdam +RandomizedDelaySec=10min +Persistent=true +Unit=stats-collector@fred.service + +[Install] +WantedBy=timers.target diff --git a/drizzle/20261025202715_stats_core/migration.sql b/drizzle/20261025202715_stats_core/migration.sql index c568f6d..86087d0 100644 --- a/drizzle/20261025202715_stats_core/migration.sql +++ b/drizzle/20261025202715_stats_core/migration.sql @@ -130,14 +130,14 @@ ALTER TABLE "stats"."series" ADD CONSTRAINT "series_region_code_region_region_co -- personal_ro may read. Not registered in meta.tables (no generic table GUI for it). DO $$ BEGIN - IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = personal_app) THEN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN GRANT USAGE ON SCHEMA stats TO personal_app; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA stats TO personal_app; GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA stats TO personal_app; ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app; ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT USAGE, SELECT ON SEQUENCES TO personal_app; END IF; - IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = personal_ro) THEN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN GRANT USAGE ON SCHEMA stats TO personal_ro; GRANT SELECT ON ALL TABLES IN SCHEMA stats TO personal_ro; ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT SELECT ON TABLES TO personal_ro;