stats: new public stats schema (source/series/obs/region/fetch_run + raw CBS, Eurostat, FRED, BIS tables) and the stats collector (scripts/stats-fetch.ts); preflight checks the stats tables

drizzle.config.ts

@@ -30,7 +30,8 @@ export default defineConfig({
 		'planes',
 		'wildlife',
 		'trains',
-		'dataholders'
+		'dataholders',
+		'stats'
 	],
 	migrations: { schema: 'drizzle', table: '__drizzle_migrations' },
 	verbose: true,

drizzle/20261025202715_stats_core/migration.sql

@@ -0,0 +1,145 @@
+CREATE SCHEMA "stats";
+--> statement-breakpoint
+CREATE TABLE "stats"."bis_policy_rate" (
+	"ref_area" text,
+	"freq" char(1),
+	"period" date,
+	"rate" double precision,
+	CONSTRAINT "bis_policy_rate_pkey" PRIMARY KEY("ref_area","freq","period")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."cbs_code" (
+	"table_id" text,
+	"dimension" text,
+	"code" text,
+	"title" text NOT NULL,
+	"unit" text,
+	"decimals" integer,
+	"group_id" text,
+	"sort_index" integer,
+	CONSTRAINT "cbs_code_pkey" PRIMARY KEY("table_id","dimension","code")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."cbs_obs" (
+	"table_id" text,
+	"obs_id" bigint,
+	"measure" text NOT NULL,
+	"dims" jsonb NOT NULL,
+	"period" text NOT NULL,
+	"value" double precision,
+	"value_attribute" text,
+	CONSTRAINT "cbs_obs_pkey" PRIMARY KEY("table_id","obs_id")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."cbs_table" (
+	"table_id" text PRIMARY KEY,
+	"title" text NOT NULL,
+	"modified" timestamp with time zone,
+	"status" text,
+	"filter" text,
+	"obs_count" integer,
+	"fetched_at" timestamp with time zone
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."eurostat_dataset" (
+	"code" text PRIMARY KEY,
+	"label" text NOT NULL,
+	"updated" timestamp with time zone,
+	"query" text NOT NULL,
+	"dims" jsonb DEFAULT '{}' NOT NULL,
+	"obs_count" integer,
+	"fetched_at" timestamp with time zone
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."eurostat_obs" (
+	"dataset" text,
+	"key" text,
+	"geo" text NOT NULL,
+	"period" text,
+	"value" double precision,
+	"status" text,
+	CONSTRAINT "eurostat_obs_pkey" PRIMARY KEY("dataset","key","period")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."fetch_run" (
+	"run_id" integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "stats"."fetch_run_run_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1),
+	"source_id" text NOT NULL,
+	"job" text DEFAULT 'refresh' NOT NULL,
+	"started_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"finished_at" timestamp with time zone,
+	"rows" integer,
+	"ok" boolean,
+	"error" text
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."fred_obs" (
+	"series_id" text,
+	"period" date,
+	"value" double precision,
+	CONSTRAINT "fred_obs_pkey" PRIMARY KEY("series_id","period")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."obs" (
+	"series_id" text,
+	"period" date,
+	"value" double precision,
+	"status" text,
+	"fetched_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "obs_pkey" PRIMARY KEY("series_id","period")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."region" (
+	"region_code" text PRIMARY KEY,
+	"name" text NOT NULL,
+	"kind" text NOT NULL,
+	"parent" text
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."series" (
+	"series_id" text PRIMARY KEY,
+	"source_id" text NOT NULL,
+	"source_code" text NOT NULL,
+	"title" text NOT NULL,
+	"unit" text NOT NULL,
+	"freq" char(1) NOT NULL,
+	"region_code" text,
+	"theme" text NOT NULL,
+	"meta" jsonb DEFAULT '{}' NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."source" (
+	"source_id" text PRIMARY KEY,
+	"name" text NOT NULL,
+	"base_url" text NOT NULL,
+	"licence" text NOT NULL,
+	"key_required" boolean DEFAULT false NOT NULL,
+	"attribution" text NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX "cbs_obs_table_measure_idx" ON "stats"."cbs_obs" ("table_id","measure");--> statement-breakpoint
+CREATE INDEX "eurostat_obs_dataset_geo_idx" ON "stats"."eurostat_obs" ("dataset","geo");--> statement-breakpoint
+CREATE INDEX "fetch_run_source_idx" ON "stats"."fetch_run" ("source_id","started_at");--> statement-breakpoint
+CREATE INDEX "series_theme_idx" ON "stats"."series" ("theme");--> statement-breakpoint
+CREATE INDEX "series_source_idx" ON "stats"."series" ("source_id");--> statement-breakpoint
+ALTER TABLE "stats"."fetch_run" ADD CONSTRAINT "fetch_run_source_id_source_source_id_fkey" FOREIGN KEY ("source_id") REFERENCES "stats"."source"("source_id");--> statement-breakpoint
+ALTER TABLE "stats"."obs" ADD CONSTRAINT "obs_series_id_series_series_id_fkey" FOREIGN KEY ("series_id") REFERENCES "stats"."series"("series_id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "stats"."series" ADD CONSTRAINT "series_source_id_source_source_id_fkey" FOREIGN KEY ("source_id") REFERENCES "stats"."source"("source_id");--> statement-breakpoint
+ALTER TABLE "stats"."series" ADD CONSTRAINT "series_region_code_region_region_code_fkey" FOREIGN KEY ("region_code") REFERENCES "stats"."region"("region_code");--> statement-breakpoint
+-- Hand-written below this line (not generated by drizzle-kit).
+-- Public open data: the app role (also used by the host-side stats collector) reads and writes,
+-- personal_ro may read. Not registered in meta.tables (no generic table GUI for it).
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = personal_app) THEN
+    GRANT USAGE ON SCHEMA stats TO personal_app;
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA stats TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA stats TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO personal_app;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT USAGE, SELECT ON SEQUENCES TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = personal_ro) THEN
+    GRANT USAGE ON SCHEMA stats TO personal_ro;
+    GRANT SELECT ON ALL TABLES IN SCHEMA stats TO personal_ro;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA stats GRANT SELECT ON TABLES TO personal_ro;
+  END IF;
+END $$;

scripts/deploy-preflight.ts

@@ -173,6 +173,18 @@ try {
 		groceries.public_usage
 	)
 		throw new Error(`groceries schema grants are wrong: ${JSON.stringify(groceries)}`);
+	// Public /stats section: open data, so the app role (and the host-side stats collector, which
+	// uses it) reads and writes, and personal_ro may read. Bump the count with every new stats table.
+	const STATS_TABLES = 12;
+	const [stats] = await client`
+		select
+			count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
+			count(*) as tables
+		from pg_class c join pg_namespace n on n.oid = c.relnamespace
+		where n.nspname = 'stats' and c.relkind = 'r'
+	`;
+	if (Number(stats.tables) !== STATS_TABLES || Number(stats.app_rw) !== STATS_TABLES)
+		throw new Error(`stats schema tables/grants are wrong: ${JSON.stringify(stats)}`);
 	// Energy tracker reads the ned-collector tables (SELECT only).
 	const [energy] = await client`
 		select count(*) as hours, max(valid_from) as latest from energy.ned_utilization
@@ -182,7 +194,7 @@ try {
 	if (!Number(energy.hours))
 		console.warn('Warning: energy.ned_utilization has no hourly data from the last 7 days');
 	console.log(
-		`health: ${health.tables} tables, tasks: ${tasks.tables} tables, groceries: ${groceries.tables} tables, app read/write, personal_ro none; energy: ${energy.hours} hourly rows, latest ${energy.latest ? new Date(energy.latest).toISOString() : 'none'}`
+		`health: ${health.tables} tables, tasks: ${tasks.tables} tables, groceries: ${groceries.tables} tables, stats: ${stats.tables} tables, app read/write, personal_ro none; energy: ${energy.hours} hourly rows, latest ${energy.latest ? new Date(energy.latest).toISOString() : 'none'}`
 	);
 	const [catalog] = await client`
 		select

scripts/stats-fetch.ts

@@ -0,0 +1,44 @@
+// /stats collector. Pulls open statistics into the `stats` schema; every job is logged in
+// stats.fetch_run. Run on the host by systemd user timers (deploy/stats-collector), with DB_URL
+// read from the app container (app role personal_app).
+//
+//   bun scripts/stats-fetch.ts cbs [--force] [--only=86141NED,85592NED]
+//   bun scripts/stats-fetch.ts eurostat [--only=nrg_pc_204]
+//   bun scripts/stats-fetch.ts fred [--only=DCOILBRENTEU]
+//   bun scripts/stats-fetch.ts bis
+//   bun scripts/stats-fetch.ts normalise-cbs     rebuild CBS chart series from the raw rows
+import { ensureMeta, sql } from './stats/common';
+import { cbs, normaliseCbs } from './stats/cbs';
+import { eurostat } from './stats/eurostat';
+import { fred } from './stats/fred';
+import { bis } from './stats/bis';
+
+const [source, ...args] = process.argv.slice(2);
+const flag = (name: string) => args.includes(`--${name}`);
+const value = (name: string) => args.find((a) => a.startsWith(`--${name}=`))?.split('=')[1];
+const job = flag('backfill') ? 'backfill' : value('only') ? `only:${value('only')}` : 'refresh';
+
+try {
+	await ensureMeta();
+	switch (source) {
+		case 'cbs':
+			await cbs({ force: flag('force') || flag('backfill'), only: value('only'), job });
+			break;
+		case 'normalise-cbs':
+			console.log(`${await normaliseCbs()} CBS observations normalised`);
+			break;
+		case 'eurostat':
+			await eurostat({ only: value('only'), job });
+			break;
+		case 'fred':
+			await fred({ only: value('only'), job });
+			break;
+		case 'bis':
+			await bis({ job });
+			break;
+		default:
+			throw new Error(`Unknown source "${source}" (cbs, eurostat, fred, bis, normalise-cbs)`);
+	}
+} finally {
+	await sql.end({ timeout: 5 });
+}

scripts/stats/bis.ts

@@ -0,0 +1,78 @@
+// A5 BIS central bank policy rates (WS_CBPOL), monthly and daily, in one call each.
+import { getText, log, run, sql, writeSeries, type ObsRow, type SeriesRow } from './common';
+
+const AREAS: Record<string, { name: string; region: string | null }> = {
+	US: { name: 'Federal Reserve (US)', region: 'USA' },
+	XM: { name: 'ECB (euro area)', region: 'EA' },
+	KR: { name: 'Bank of Korea', region: 'KOR' },
+	CN: { name: "People's Bank of China", region: 'CHN' },
+	GB: { name: 'Bank of England', region: 'GBR' },
+	JP: { name: 'Bank of Japan', region: 'JPN' }
+};
+
+function parseCsv(text: string): Record<string, string>[] {
+	const lines = text.trim().split(/\r?\n/);
+	const split = (line: string) => {
+		const out: string[] = [];
+		let cur = '';
+		let q = false;
+		for (const ch of line) {
+			if (ch === '"') q = !q;
+			else if (ch === ',' && !q) {
+				out.push(cur);
+				cur = '';
+			} else cur += ch;
+		}
+		out.push(cur);
+		return out;
+	};
+	const head = split(lines[0]);
+	return lines.slice(1).map((l) => Object.fromEntries(split(l).map((v, i) => [head[i], v])));
+}
+
+export async function bis(opts: { job: string }) {
+	await run('bis', opts.job, async () => {
+		let total = 0;
+		for (const freq of ['M', 'D'] as const) {
+			const csv = await getText(
+				`https://stats.bis.org/api/v2/data/dataflow/BIS/WS_CBPOL/1.0/${freq}.${Object.keys(AREAS).join('+')}?format=csv`
+			);
+			const rows = parseCsv(csv)
+				.filter((r) => r.TIME_PERIOD && r.REF_AREA)
+				.map((r) => ({
+					ref_area: r.REF_AREA,
+					freq,
+					period: freq === 'M' ? `${r.TIME_PERIOD.slice(0, 7)}-01` : r.TIME_PERIOD.slice(0, 10),
+					rate: r.OBS_VALUE === '' || r.OBS_VALUE === 'NaN' ? null : Number(r.OBS_VALUE)
+				}));
+			await sql.begin(async (tx) => {
+				await tx`delete from stats.bis_policy_rate where freq = ${freq}`;
+				for (let i = 0; i < rows.length; i += 5000)
+					await tx`insert into stats.bis_policy_rate ${tx(rows.slice(i, i + 5000))} on conflict do nothing`;
+			});
+			const series: SeriesRow[] = Object.entries(AREAS).map(([code, a]) => ({
+				series_id: `bis:${freq}:${code}`,
+				source_id: 'bis',
+				source_code: `WS_CBPOL/${freq}.${code}`,
+				title: `${a.name} policy rate`,
+				unit: '%',
+				freq,
+				region_code: a.region,
+				theme: 'economy',
+				meta: {}
+			}));
+			const obs: ObsRow[] = rows
+				.filter((r) => r.rate != null)
+				.map((r) => ({
+					series_id: `bis:${freq}:${r.ref_area}`,
+					period: r.period,
+					value: r.rate,
+					status: null
+				}));
+			await writeSeries(series, obs);
+			log(`bis ${freq}: ${rows.length} rows`);
+			total += rows.length;
+		}
+		return total;
+	});
+}

scripts/stats/cbs-series.ts

@@ -0,0 +1,26 @@
+// Chart series normalised from the raw CBS rows (stats.cbs_obs → stats.series/obs).
+// `dims` must match the observation's dimension codes exactly (see stats.cbs_code for titles).
+import { GOUDA_RING } from './common';
+
+export type CbsSeriesDef = {
+	id: string;
+	table: string;
+	measure: string;
+	dims?: Record<string, string>;
+	freq: 'M' | 'Q' | 'S' | 'A';
+	title: string;
+	unit: string;
+	theme: string;
+	region?: string;
+	/** Expand into one series per region (id suffixed with :<code>). */
+	regionDim?: string;
+	regions?: { code: string; region: string }[];
+	scale?: number;
+};
+
+export const RING = [
+	{ code: 'NL01', region: 'NLD' },
+	...GOUDA_RING.map((code) => ({ code, region: code }))
+];
+
+export const CBS_SERIES: CbsSeriesDef[] = [];

scripts/stats/cbs.ts

@@ -0,0 +1,240 @@
+// A1 CBS StatLine via the OData v4 API (long format: one row per measure × dimensions × period).
+// Raw rows go to stats.cbs_obs (+ code lists in stats.cbs_code); the chart series in
+// CBS_SERIES are then normalised into stats.series / stats.obs.
+// Nightly: re-pull a table only when its `Modified` changed (or with --force).
+import {
+	GOUDA_RING,
+	getJson,
+	log,
+	parsePeriod,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+import { CBS_SERIES } from './cbs-series';
+
+const BASE = 'https://datasets.cbs.nl/odata/v1/CBS';
+const NL = ['NL01', 'NL00'];
+const regionIn = (dim: string, extra: string[] = []) =>
+	'(' + [...NL, ...GOUDA_RING, 'CR28', ...extra].map((c) => `${dim} eq '${c}'`).join(' or ') + ')';
+const kwbFilter =
+	`(startswith(WijkenEnBuurten,'WK0513') or startswith(WijkenEnBuurten,'BU0513') or ` +
+	[...NL, ...GOUDA_RING].map((c) => `WijkenEnBuurten eq '${c}'`).join(' or ') +
+	')';
+
+export type CbsTable = { id: string; theme: string; filter?: string; note?: string };
+
+/** Tables pulled in build step 1. Big tables are filtered to NL + Gouda + its ring. */
+export const CBS_TABLES: CbsTable[] = [
+	// energy
+	{ id: '85592NED', theme: 'energy' },
+	{ id: '84672NED', theme: 'energy', note: 'discontinued 2018–2023 tariffs (history)' },
+	{ id: '81309NED', theme: 'energy', note: 'discontinued 2007–2022 prices (history)' },
+	{ id: '83140NED', theme: 'energy' },
+	{ id: '82610NED', theme: 'energy' },
+	{ id: '85523NED', theme: 'energy' },
+	{ id: '85005NED', theme: 'energy' },
+	{ id: '85237NED', theme: 'energy' },
+	// economy
+	{ id: '86141NED', theme: 'economy' },
+	{ id: '83136NED', theme: 'economy' },
+	{ id: '86144NED', theme: 'economy', filter: "Bestedingscategorieen eq 'T001112'" },
+	{ id: '85865NED', theme: 'economy' },
+	{ id: '85773NED', theme: 'economy' },
+	// budget
+	{ id: '84120NED', theme: 'budget' },
+	{ id: '84117NED', theme: 'budget' },
+	{ id: '85590NED', theme: 'budget' },
+	{ id: '80509ned', theme: 'budget' },
+	{ id: '83642NED', theme: 'budget' },
+	{ id: '83641NED', theme: 'budget' },
+	// gouda
+	{ id: '37230ned', theme: 'gouda', filter: regionIn('RegioS') },
+	{ id: '37259ned', theme: 'gouda', filter: regionIn('RegioS') },
+	{ id: '70072ned', theme: 'gouda', filter: regionIn('RegioS') },
+	{ id: '83625NED', theme: 'gouda' },
+	{ id: '85819NED', theme: 'gouda' },
+	{ id: '86079NED', theme: 'gouda', filter: regionIn('RegioS') },
+	{ id: '83162NED', theme: 'gouda' },
+	{ id: '70675ned', theme: 'gouda' },
+	// Kerncijfers wijken en buurten, one table per year (Gouda wijken/buurten + ring + NL)
+	{ id: '86358NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2026' },
+	{ id: '86165NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2025' },
+	{ id: '85984NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2024' },
+	{ id: '85618NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2023' },
+	{ id: '85318NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2022' },
+	{ id: '85039NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2021' },
+	{ id: '84799NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2020' },
+	{ id: '84583NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2019' }
+];
+
+type Props = { Title: string; Modified: string; Status: string; ObservationCount?: number };
+type Page<T> = { value: T[]; '@odata.nextLink'?: string };
+
+async function all<T>(url: string): Promise<T[]> {
+	const out: T[] = [];
+	let next: string | undefined = url;
+	while (next) {
+		const page: Page<T> = await getJson<Page<T>>(next);
+		out.push(...page.value);
+		next = page['@odata.nextLink'];
+	}
+	return out;
+}
+
+async function pullTable(t: CbsTable, force: boolean, note: (w: string) => void): Promise<number> {
+	const props = await getJson<Props>(`${BASE}/${t.id}/Properties`);
+	const [known] = await sql<{ modified: Date | null; filter: string | null }[]>`
+		select modified, filter from stats.cbs_table where table_id = ${t.id}`;
+	if (props.Status && props.Status !== 'Regulier' && !t.note?.includes('discontinued'))
+		note(
+			`${t.id} status is ${props.Status}: check the CBS catalog for its successor (base-year swap)`
+		);
+	if (
+		!force &&
+		known?.modified &&
+		new Date(props.Modified).getTime() === known.modified.getTime() &&
+		(known.filter ?? null) === (t.filter ?? null)
+	) {
+		log(`cbs ${t.id}: unchanged since ${props.Modified}`);
+		return 0;
+	}
+	log(`cbs ${t.id}: pulling "${props.Title.trim()}"${t.filter ? ' (filtered)' : ''}`);
+	const dims = await getJson<Page<{ Identifier: string; Kind: string; CodesUrl: string }>>(
+		`${BASE}/${t.id}/Dimensions`
+	);
+	const codes: Record<string, unknown>[] = [];
+	const measures = await all<{
+		Identifier: string;
+		Title: string;
+		Unit?: string;
+		Decimals?: number;
+		MeasureGroupId?: string;
+		Index?: number;
+	}>(`${BASE}/${t.id}/MeasureCodes`);
+	for (const m of measures)
+		codes.push({
+			table_id: t.id,
+			dimension: 'Measure',
+			code: m.Identifier.trim(),
+			title: (m.Title ?? '').trim() || m.Identifier,
+			unit: m.Unit?.trim() || null,
+			decimals: m.Decimals ?? null,
+			group_id: m.MeasureGroupId ?? null,
+			sort_index: m.Index ?? null
+		});
+	const dimIds: string[] = [];
+	for (const d of dims.value) {
+		dimIds.push(d.Identifier);
+		const list = await all<{
+			Identifier: string;
+			Title: string;
+			DimensionGroupId?: string;
+			Index?: number;
+		}>(d.CodesUrl);
+		for (const c of list)
+			codes.push({
+				table_id: t.id,
+				dimension: d.Identifier,
+				code: c.Identifier.trim(),
+				title: (c.Title ?? '').trim() || c.Identifier,
+				unit: null,
+				decimals: null,
+				group_id: c.DimensionGroupId ?? null,
+				sort_index: c.Index ?? null
+			});
+	}
+	const timeDim = dims.value.find((d) => d.Kind === 'TimeDimension')?.Identifier;
+	const url =
+		`${BASE}/${t.id}/Observations` + (t.filter ? `?$filter=${encodeURIComponent(t.filter)}` : '');
+	const raw = await all<Record<string, unknown>>(url);
+	const rows = raw.map((o) => {
+		const d: Record<string, string> = {};
+		for (const id of dimIds) if (id !== timeDim && o[id] != null) d[id] = String(o[id]).trim();
+		return {
+			table_id: t.id,
+			obs_id: Number(o.Id),
+			measure: String(o.Measure).trim(),
+			dims: sql.json(d),
+			period: timeDim ? String(o[timeDim]).trim() : '',
+			value: typeof o.Value === 'number' ? o.Value : null,
+			value_attribute:
+				o.ValueAttribute && o.ValueAttribute !== 'None' ? String(o.ValueAttribute) : null
+		};
+	});
+	await sql.begin(async (tx) => {
+		await tx`delete from stats.cbs_obs where table_id = ${t.id}`;
+		await tx`delete from stats.cbs_code where table_id = ${t.id}`;
+		for (let i = 0; i < codes.length; i += 5000)
+			await tx`insert into stats.cbs_code ${tx(codes.slice(i, i + 5000))} on conflict do nothing`;
+		for (let i = 0; i < rows.length; i += 5000)
+			await tx`insert into stats.cbs_obs ${tx(rows.slice(i, i + 5000))}`;
+		await tx`insert into stats.cbs_table (table_id, title, modified, status, filter, obs_count, fetched_at)
+			values (${t.id}, ${props.Title.trim()}, ${props.Modified}, ${props.Status ?? null}, ${t.filter ?? null}, ${rows.length}, now())
+			on conflict (table_id) do update set title = excluded.title, modified = excluded.modified,
+			status = excluded.status, filter = excluded.filter, obs_count = excluded.obs_count, fetched_at = now()`;
+	});
+	log(`cbs ${t.id}: ${rows.length} observations, ${codes.length} codes`);
+	return rows.length;
+}
+
+/** Normalise CBS_SERIES from the raw rows into stats.series / stats.obs. */
+export async function normaliseCbs(): Promise<number> {
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	for (const def of CBS_SERIES) {
+		const regions = def.regions ?? [null];
+		for (const region of regions) {
+			const dims = { ...(def.dims ?? {}) };
+			if (region && def.regionDim) dims[def.regionDim] = region.code;
+			const rows = await sql<
+				{ period: string; value: number | null; value_attribute: string | null }[]
+			>`
+				select period, value, value_attribute from stats.cbs_obs
+				where table_id = ${def.table} and measure = ${def.measure} and dims @> ${sql.json(dims)}`;
+			const id = region ? `${def.id}:${region.code}` : def.id;
+			let n = 0;
+			const seen = new Set<string>();
+			for (const r of rows) {
+				const p = parsePeriod(r.period);
+				if (!p || p.freq !== def.freq || seen.has(p.date)) continue;
+				seen.add(p.date);
+				const value = r.value == null ? null : r.value * (def.scale ?? 1);
+				obs.push({ series_id: id, period: p.date, value, status: r.value_attribute });
+				n++;
+			}
+			if (!n) continue;
+			series.push({
+				series_id: id,
+				source_id: 'cbs',
+				source_code: `${def.table}/${def.measure}/${JSON.stringify(dims)}`,
+				title: def.title,
+				unit: def.unit,
+				freq: def.freq,
+				region_code: region?.region ?? def.region ?? 'NLD',
+				theme: def.theme,
+				meta: { table: def.table, measure: def.measure, dims }
+			});
+		}
+	}
+	return writeSeries(series, obs);
+}
+
+export async function cbs(opts: { force: boolean; only?: string; job: string }) {
+	let total = 0;
+	await run('cbs', opts.job, async (note) => {
+		for (const t of CBS_TABLES) {
+			if (opts.only && !opts.only.split(',').includes(t.id)) continue;
+			try {
+				total += await pullTable(t, opts.force, note);
+			} catch (error) {
+				note(`${t.id}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		const normalised = await normaliseCbs();
+		log(`cbs: ${normalised} normalised observations`);
+		return total;
+	});
+}

scripts/stats/common.ts

@@ -0,0 +1,237 @@
+// Shared plumbing for the /stats collector (scripts/stats-fetch.ts): DB client, polite HTTP,
+// fetch_run logging, and the normalised layer (stats.source / region / series / obs).
+// Runs on the host (systemd user timers, see deploy/stats-collector) with DB_URL = app role.
+import postgres from 'postgres';
+
+if (!process.env.DB_URL) throw new Error('DB_URL is missing');
+export const sql = postgres(process.env.DB_URL, {
+	max: 2,
+	connect_timeout: 15,
+	onnotice: () => {},
+	prepare: false
+});
+
+export const USER_AGENT =
+	'stanrunge.dev-stats/1.0 (+https://stanrunge.dev/stats; open-data collector; stan@stanrunge.dev)';
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/** Minimum gap between requests per host (CBS ≤5 req/s → 250 ms; OECD ≤20/min → 3.1 s). */
+const hostGapMs: Record<string, number> = {
+	'datasets.cbs.nl': 250,
+	'opendata.cbs.nl': 250,
+	'sdmx.oecd.org': 3100,
+	'fred.stlouisfed.org': 1000,
+	'ec.europa.eu': 500,
+	'stats.bis.org': 1000
+};
+const lastHit = new Map<string, number>();
+
+export async function http(url: string, init: RequestInit = {}, tries = 4): Promise<Response> {
+	const host = new URL(url).host;
+	for (let attempt = 1; ; attempt++) {
+		const gap = hostGapMs[host] ?? 300;
+		const wait = (lastHit.get(host) ?? 0) + gap - Date.now();
+		if (wait > 0) await sleep(wait);
+		lastHit.set(host, Date.now());
+		let response: Response | undefined;
+		let failure: unknown;
+		try {
+			response = await fetch(url, {
+				...init,
+				headers: { 'user-agent': USER_AGENT, accept: '*/*', ...(init.headers ?? {}) },
+				signal: AbortSignal.timeout(180_000)
+			});
+			if (response.ok) return response;
+			if (response.status !== 429 && response.status < 500) {
+				const body = (await response.text()).slice(0, 300);
+				throw new Error(`HTTP ${response.status} for ${url}: ${body}`);
+			}
+			failure = new Error(`HTTP ${response.status} for ${url}`);
+		} catch (error) {
+			if (error instanceof Error && error.message.startsWith('HTTP 4')) throw error;
+			failure = error;
+		}
+		if (attempt >= tries) throw failure;
+		const retryAfter = Number(response?.headers.get('retry-after'));
+		await sleep(
+			Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : 2000 * 2 ** attempt
+		);
+	}
+}
+
+export const getJson = async <T = unknown>(url: string) => (await (await http(url)).json()) as T;
+export const getText = async (url: string) => (await http(url)).text();
+
+export function log(...parts: unknown[]) {
+	console.log(new Date().toISOString(), ...parts);
+}
+
+/** Wraps one collector job in a stats.fetch_run row (rows, ok, error). */
+export async function run(
+	sourceId: string,
+	job: string,
+	fn: (note: (warning: string) => void) => Promise<number>
+): Promise<number> {
+	const [{ run_id }] = await sql`
+		insert into stats.fetch_run (source_id, job) values (${sourceId}, ${job}) returning run_id`;
+	const warnings: string[] = [];
+	try {
+		const rows = await fn((w) => warnings.push(w));
+		await sql`update stats.fetch_run set finished_at = now(), rows = ${rows}, ok = true,
+			error = ${warnings.length ? warnings.join('; ').slice(0, 4000) : null} where run_id = ${run_id}`;
+		log(
+			`${sourceId}/${job}: ok, ${rows} rows${warnings.length ? `, ${warnings.length} warnings` : ''}`
+		);
+		return rows;
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		await sql`update stats.fetch_run set finished_at = now(), ok = false,
+			error = ${[message, ...warnings].join('; ').slice(0, 4000)} where run_id = ${run_id}`;
+		log(`${sourceId}/${job}: FAILED ${message}`);
+		throw error;
+	}
+}
+
+// ── Sources and regions ──────────────────────────────────────────────────────────────────
+
+export const SOURCES = [
+	{
+		source_id: 'cbs',
+		name: 'CBS StatLine',
+		base_url: 'https://datasets.cbs.nl/odata/v1/CBS',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: CBS StatLine (Statistics Netherlands), CC BY 4.0'
+	},
+	{
+		source_id: 'eurostat',
+		name: 'Eurostat',
+		base_url: 'https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: Eurostat, CC BY 4.0'
+	},
+	{
+		source_id: 'fred',
+		name: 'FRED (Federal Reserve Bank of St. Louis)',
+		base_url: 'https://fred.stlouisfed.org/graph/fredgraph.csv',
+		licence: 'Per series (EIA/BLS/Fed public domain, OECD CC BY 4.0)',
+		key_required: false,
+		attribution: 'Source: FRED, Federal Reserve Bank of St. Louis (original series sources apply)'
+	},
+	{
+		source_id: 'bis',
+		name: 'BIS central bank policy rates',
+		base_url: 'https://stats.bis.org/api/v2/data/dataflow/BIS/WS_CBPOL/1.0',
+		licence: 'Free reuse with attribution',
+		key_required: false,
+		attribution: 'Source: BIS, central bank policy rates (WS_CBPOL)'
+	}
+] as const;
+
+export const REGIONS: { region_code: string; name: string; kind: string; parent: string | null }[] =
+	[
+		{ region_code: 'WLD', name: 'World', kind: 'aggregate', parent: null },
+		{ region_code: 'EU27', name: 'European Union (27)', kind: 'aggregate', parent: 'WLD' },
+		{ region_code: 'EA', name: 'Euro area', kind: 'aggregate', parent: 'EU27' },
+		{ region_code: 'NLD', name: 'Netherlands', kind: 'country', parent: 'EU27' },
+		{ region_code: 'USA', name: 'United States', kind: 'country', parent: 'WLD' },
+		{ region_code: 'KOR', name: 'South Korea', kind: 'country', parent: 'WLD' },
+		{ region_code: 'CHN', name: 'China', kind: 'country', parent: 'WLD' },
+		{ region_code: 'CUW', name: 'Curaçao', kind: 'country', parent: 'WLD' },
+		{ region_code: 'DEU', name: 'Germany', kind: 'country', parent: 'EU27' },
+		{ region_code: 'FRA', name: 'France', kind: 'country', parent: 'EU27' },
+		{ region_code: 'BEL', name: 'Belgium', kind: 'country', parent: 'EU27' },
+		{ region_code: 'ITA', name: 'Italy', kind: 'country', parent: 'EU27' },
+		{ region_code: 'ESP', name: 'Spain', kind: 'country', parent: 'EU27' },
+		{ region_code: 'POL', name: 'Poland', kind: 'country', parent: 'EU27' },
+		{ region_code: 'SWE', name: 'Sweden', kind: 'country', parent: 'EU27' },
+		{ region_code: 'DNK', name: 'Denmark', kind: 'country', parent: 'EU27' },
+		{ region_code: 'AUT', name: 'Austria', kind: 'country', parent: 'EU27' },
+		{ region_code: 'GBR', name: 'United Kingdom', kind: 'country', parent: 'WLD' },
+		{ region_code: 'JPN', name: 'Japan', kind: 'country', parent: 'WLD' },
+		{ region_code: 'IND', name: 'India', kind: 'country', parent: 'WLD' },
+		{ region_code: 'CR28', name: 'Oost-Zuid-Holland (COROP)', kind: 'corop', parent: 'NLD' },
+		{ region_code: 'GM0513', name: 'Gouda', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM0627', name: 'Waddinxveen', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM1901', name: 'Bodegraven-Reeuwijk', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM1892', name: 'Zuidplas', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM1931', name: 'Krimpenerwaard', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM0484', name: 'Alphen aan den Rijn', kind: 'municipality', parent: 'NLD' },
+		{ region_code: 'GM0632', name: 'Woerden', kind: 'municipality', parent: 'NLD' }
+	];
+
+/** Gouda and its house-hunting ring (CBS gemeente codes). */
+export const GOUDA_RING = ['GM0513', 'GM0627', 'GM1901', 'GM1892', 'GM1931', 'GM0484', 'GM0632'];
+
+export async function ensureMeta() {
+	await sql`insert into stats.source ${sql(SOURCES as unknown as Record<string, unknown>[])}
+		on conflict (source_id) do update set name = excluded.name, base_url = excluded.base_url,
+		licence = excluded.licence, key_required = excluded.key_required, attribution = excluded.attribution`;
+	await sql`insert into stats.region ${sql(REGIONS)}
+		on conflict (region_code) do update set name = excluded.name, kind = excluded.kind, parent = excluded.parent`;
+}
+
+export type SeriesRow = {
+	series_id: string;
+	source_id: string;
+	source_code: string;
+	title: string;
+	unit: string;
+	freq: string;
+	region_code: string | null;
+	theme: string;
+	meta: Record<string, unknown>;
+};
+export type ObsRow = {
+	series_id: string;
+	period: string;
+	value: number | null;
+	status: string | null;
+};
+
+/** Replaces the normalised series + observations for the given series ids. */
+export async function writeSeries(series: SeriesRow[], obs: ObsRow[]): Promise<number> {
+	if (!series.length) return 0;
+	const known = new Set(REGIONS.map((r) => r.region_code));
+	for (const s of series) if (s.region_code && !known.has(s.region_code)) s.region_code = null;
+	await sql.begin(async (tx) => {
+		for (let i = 0; i < series.length; i += 2000) {
+			const chunk = series
+				.slice(i, i + 2000)
+				.map((s) => ({ ...s, meta: tx.json(s.meta as never) }));
+			await tx`insert into stats.series ${tx(chunk)} on conflict (series_id) do update set
+				source_id = excluded.source_id, source_code = excluded.source_code, title = excluded.title,
+				unit = excluded.unit, freq = excluded.freq, region_code = excluded.region_code,
+				theme = excluded.theme, meta = excluded.meta`;
+		}
+		const ids = series.map((s) => s.series_id);
+		await tx`delete from stats.obs where series_id = any(${ids})`;
+		for (let i = 0; i < obs.length; i += 5000) {
+			await tx`insert into stats.obs ${tx(obs.slice(i, i + 5000), 'series_id', 'period', 'value', 'status')}
+				on conflict (series_id, period) do update set value = excluded.value, status = excluded.status, fetched_at = now()`;
+		}
+	});
+	return obs.length;
+}
+
+/** Period code → period start date (YYYY-MM-DD) and frequency. Handles CBS, Eurostat, ISO forms. */
+export function parsePeriod(code: string): { date: string; freq: string } | null {
+	const c = code.trim();
+	let m: RegExpMatchArray | null;
+	if ((m = c.match(/^(\d{4})JJ00$/)) || (m = c.match(/^(\d{4})$/)))
+		return { date: `${m[1]}-01-01`, freq: 'A' };
+	if (
+		(m = c.match(/^(\d{4})MM(\d{2})$/)) ||
+		(m = c.match(/^(\d{4})-(\d{2})$/)) ||
+		(m = c.match(/^(\d{4})M(\d{2})$/))
+	)
+		return { date: `${m[1]}-${m[2]}-01`, freq: 'M' };
+	if ((m = c.match(/^(\d{4})KW0?(\d)$/)) || (m = c.match(/^(\d{4})-?Q(\d)$/)))
+		return { date: `${m[1]}-${String((Number(m[2]) - 1) * 3 + 1).padStart(2, '0')}-01`, freq: 'Q' };
+	if ((m = c.match(/^(\d{4})-?S(\d)$/)) || (m = c.match(/^(\d{4})HJ0?(\d)$/)))
+		return { date: `${m[1]}-${m[2] === '1' ? '01' : '07'}-01`, freq: 'S' };
+	if (/^\d{4}-\d{2}-\d{2}$/.test(c)) return { date: c, freq: 'D' };
+	return null;
+}

scripts/stats/eurostat.ts

@@ -0,0 +1,249 @@
+// A2 Eurostat dissemination API (JSON-stat 2.0). Each dataset is pulled with a fixed filter
+// (12 countries + EU/EA aggregates, one consumption band / unit) into stats.eurostat_obs, then
+// every (dataset, key) becomes a normalised series `eurostat:<dataset>:<key>`.
+import {
+	getJson,
+	log,
+	parsePeriod,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const BASE = 'https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data';
+export const EU_GEOS = [
+	'NL',
+	'DE',
+	'FR',
+	'BE',
+	'IT',
+	'ES',
+	'PL',
+	'SE',
+	'DK',
+	'AT',
+	'EU27_2020',
+	'EA20'
+];
+const GEO_REGION: Record<string, string> = {
+	NL: 'NLD',
+	DE: 'DEU',
+	FR: 'FRA',
+	BE: 'BEL',
+	IT: 'ITA',
+	ES: 'ESP',
+	PL: 'POL',
+	SE: 'SWE',
+	DK: 'DNK',
+	AT: 'AUT',
+	EU27_2020: 'EU27',
+	EA20: 'EA',
+	EA: 'EA',
+	UK: 'GBR',
+	JP: 'JPN',
+	US: 'USA'
+};
+
+type Dataset = {
+	code: string;
+	theme: string;
+	params: Record<string, string | string[]>;
+	geos?: string[];
+};
+
+export const EUROSTAT_DATASETS: Dataset[] = [
+	{
+		code: 'nrg_pc_204',
+		theme: 'energy',
+		params: { nrg_cons: 'KWH2500-4999', currency: 'EUR', unit: 'KWH' }
+	},
+	{
+		code: 'nrg_pc_202',
+		theme: 'energy',
+		params: { nrg_cons: 'GJ20-199', currency: 'EUR', unit: 'KWH' }
+	},
+	{ code: 'nrg_pc_204_h', theme: 'energy', params: { consom: '4161902', currency: 'EUR' } },
+	{ code: 'nrg_pc_202_h', theme: 'energy', params: { consom: '4141902', currency: 'EUR' } },
+	{ code: 'nrg_ind_id', theme: 'energy', params: { siec: 'TOTAL' } },
+	{
+		code: 'irt_lt_mcby_m',
+		theme: 'economy',
+		params: {},
+		geos: ['NL', 'DE', 'FR', 'IT', 'BE', 'ES', 'AT', 'EA']
+	},
+	{ code: 'prc_hicp_minr', theme: 'economy', params: { coicop18: 'TOTAL', unit: 'RCH_A' } },
+	{ code: 'prc_hicp_manr', theme: 'economy', params: { coicop: 'CP00', unit: 'RCH_A' } },
+	{ code: 'prc_hpi_q', theme: 'economy', params: { purchase: 'TOTAL', unit: ['I15_Q', 'RCH_A'] } },
+	{ code: 'road_eqr_carpda', theme: 'energy', params: { unit: 'NR' } },
+	{
+		code: 'migr_imm1ctz',
+		theme: 'countries',
+		params: { citizen: 'TOTAL', age: 'TOTAL', sex: 'T', agedef: 'COMPLET', unit: 'NR' }
+	},
+	{
+		code: 'gov_10a_main',
+		theme: 'budget',
+		params: { unit: 'PC_GDP', sector: 'S13', na_item: ['TR', 'TE', 'B9'] }
+	},
+	{
+		code: 'gov_10a_exp',
+		theme: 'budget',
+		params: { unit: 'PC_GDP', sector: 'S13', na_item: 'TE' }
+	},
+	{
+		code: 'gov_10a_taxag',
+		theme: 'budget',
+		params: { unit: 'PC_GDP', sector: 'S13_S212', na_item: ['D2_REC', 'D5_REC', 'D61_REC', 'D995'] }
+	},
+	{
+		code: 'gov_10dd_edpt1',
+		theme: 'budget',
+		params: { unit: 'PC_GDP', sector: 'S13', na_item: ['GD', 'B9'] }
+	},
+	{
+		code: 'nama_10_gdp',
+		theme: 'economy',
+		params: { unit: ['CP_MEUR', 'CLV_PCH_PRE'], na_item: 'B1GQ' }
+	},
+	{ code: 'demo_pjan', theme: 'countries', params: { sex: 'T', age: 'TOTAL', unit: 'NR' } }
+];
+
+type JsonStat = {
+	label: string;
+	updated?: string;
+	id: string[];
+	size: number[];
+	dimension: Record<
+		string,
+		{ label: string; category: { index: Record<string, number>; label: Record<string, string> } }
+	>;
+	value: Record<string, number> | number[];
+	status?: Record<string, string> | string[];
+};
+
+function query(d: Dataset) {
+	const q = new URLSearchParams({ format: 'JSON', lang: 'EN' });
+	for (const g of d.geos ?? EU_GEOS) q.append('geo', g);
+	for (const [k, v] of Object.entries(d.params)) for (const x of [v].flat()) q.append(k, x);
+	return q.toString();
+}
+
+async function pull(d: Dataset): Promise<number> {
+	const qs = query(d);
+	const js = await getJson<JsonStat>(`${BASE}/${d.code}?${qs}`);
+	const order = js.id;
+	const codesByDim = order.map((id) => {
+		const idx = js.dimension[id].category.index;
+		const arr: string[] = [];
+		for (const [code, i] of Object.entries(idx)) arr[i] = code;
+		return arr;
+	});
+	const timePos = order.indexOf('time');
+	const geoPos = order.indexOf('geo');
+	const rows: {
+		dataset: string;
+		key: string;
+		geo: string;
+		period: string;
+		value: number | null;
+		status: string | null;
+	}[] = [];
+	const values = js.value;
+	const status = js.status ?? {};
+	const entries: [number, number][] = Array.isArray(values)
+		? values.map((v, i) => [i, v] as [number, number]).filter(([, v]) => v != null)
+		: Object.entries(values).map(([k, v]) => [Number(k), v]);
+	for (const [flat, value] of entries) {
+		let rest = flat;
+		const coord: number[] = new Array(order.length);
+		for (let p = order.length - 1; p >= 0; p--) {
+			coord[p] = rest % js.size[p];
+			rest = Math.floor(rest / js.size[p]);
+		}
+		const keyParts: string[] = [];
+		for (let p = 0; p < order.length; p++)
+			if (p !== timePos && order[p] !== 'freq') keyParts.push(codesByDim[p][coord[p]]);
+		rows.push({
+			dataset: d.code,
+			key: keyParts.join('.'),
+			geo: geoPos >= 0 ? codesByDim[geoPos][coord[geoPos]] : '',
+			period: codesByDim[timePos][coord[timePos]],
+			value: value ?? null,
+			status: (Array.isArray(status) ? status[flat] : status[String(flat)]) ?? null
+		});
+	}
+	const dims: Record<string, unknown> = {};
+	for (const id of order)
+		dims[id] = { label: js.dimension[id].label, labels: js.dimension[id].category.label };
+	await sql.begin(async (tx) => {
+		await tx`delete from stats.eurostat_obs where dataset = ${d.code}`;
+		for (let i = 0; i < rows.length; i += 5000)
+			await tx`insert into stats.eurostat_obs ${tx(rows.slice(i, i + 5000))} on conflict do nothing`;
+		await tx`insert into stats.eurostat_dataset (code, label, updated, query, dims, obs_count, fetched_at)
+			values (${d.code}, ${js.label}, ${js.updated ?? null}, ${qs}, ${tx.json(dims as never)}, ${rows.length}, now())
+			on conflict (code) do update set label = excluded.label, updated = excluded.updated, query = excluded.query,
+			dims = excluded.dims, obs_count = excluded.obs_count, fetched_at = now()`;
+	});
+	// Normalise: one series per key.
+	const order2 = order.filter((id) => id !== 'time' && id !== 'freq');
+	const byKey = new Map<string, typeof rows>();
+	for (const r of rows) byKey.set(r.key, [...(byKey.get(r.key) ?? []), r]);
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	for (const [key, list] of byKey) {
+		const parts = key.split('.');
+		const labels = order2.map((id, i) => ({
+			id,
+			code: parts[i],
+			label: js.dimension[id].category.label[parts[i]] ?? parts[i]
+		}));
+		const unit =
+			labels.find((l) => l.id === 'unit')?.label ??
+			labels.find((l) => l.id === 'currency')?.label ??
+			'';
+		const geo = labels.find((l) => l.id === 'geo');
+		const id = `eurostat:${d.code}:${key}`;
+		let freq = 'A';
+		for (const r of list) {
+			const p = parsePeriod(r.period);
+			if (!p) continue;
+			freq = p.freq;
+			obs.push({ series_id: id, period: p.date, value: r.value, status: r.status });
+		}
+		series.push({
+			series_id: id,
+			source_id: 'eurostat',
+			source_code: `${d.code}/${key}`,
+			title: `${js.label} · ${labels
+				.filter((l) => l.id !== 'geo' && l.id !== 'unit')
+				.map((l) => l.label)
+				.join(', ')}`.replace(/ · $/, ''),
+			unit,
+			freq,
+			region_code: geo ? (GEO_REGION[geo.code] ?? null) : null,
+			theme: d.theme,
+			meta: { dataset: d.code, dims: Object.fromEntries(labels.map((l) => [l.id, l.code])) }
+		});
+	}
+	await sql`delete from stats.series where source_id = 'eurostat' and meta->>'dataset' = ${d.code}`;
+	await writeSeries(series, obs);
+	log(`eurostat ${d.code}: ${rows.length} observations, ${series.length} series`);
+	return rows.length;
+}
+
+export async function eurostat(opts: { only?: string; job: string }) {
+	await run('eurostat', opts.job, async (note) => {
+		let total = 0;
+		for (const d of EUROSTAT_DATASETS) {
+			if (opts.only && !opts.only.split(',').includes(d.code)) continue;
+			try {
+				total += await pull(d);
+			} catch (error) {
+				note(`${d.code}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		return total;
+	});
+}

scripts/stats/fred.ts

@@ -0,0 +1,230 @@
+// A7 FRED graph CSV (no key). One call per series → stats.fred_obs and series `fred:<id>`.
+import { getText, log, run, sql, writeSeries, type ObsRow } from './common';
+
+type Fred = {
+	id: string;
+	title: string;
+	unit: string;
+	freq: 'D' | 'M' | 'Q' | 'A';
+	region: string;
+	theme: string;
+};
+export const FRED_SERIES: Fred[] = [
+	{
+		id: 'DCOILBRENTEU',
+		title: 'Brent crude oil spot price',
+		unit: 'USD per barrel',
+		freq: 'D',
+		region: 'WLD',
+		theme: 'energy'
+	},
+	{
+		id: 'PNGASEUUSDM',
+		title: 'EU natural gas price (TTF, IMF)',
+		unit: 'USD per MMBtu',
+		freq: 'M',
+		region: 'EU27',
+		theme: 'energy'
+	},
+	{
+		id: 'DFEDTARU',
+		title: 'Fed funds target range, upper limit',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'DFEDTARL',
+		title: 'Fed funds target range, lower limit',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'DFF',
+		title: 'Effective federal funds rate',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'DGS10',
+		title: 'US Treasury 10-year yield',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'DGS2',
+		title: 'US Treasury 2-year yield',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'IRLTLT01KRM156N',
+		title: 'South Korea 10-year government bond yield',
+		unit: '%',
+		freq: 'M',
+		region: 'KOR',
+		theme: 'economy'
+	},
+	{
+		id: 'IRLTLT01FRM156N',
+		title: 'France 10-year government bond yield',
+		unit: '%',
+		freq: 'M',
+		region: 'FRA',
+		theme: 'economy'
+	},
+	{
+		id: 'IRLTLT01DEM156N',
+		title: 'Germany 10-year government bond yield',
+		unit: '%',
+		freq: 'M',
+		region: 'DEU',
+		theme: 'economy'
+	},
+	{
+		id: 'APU000072610',
+		title: 'US average residential electricity price',
+		unit: 'USD per kWh',
+		freq: 'M',
+		region: 'USA',
+		theme: 'energy'
+	},
+	{
+		id: 'APU000072620',
+		title: 'US average utility (piped) gas price',
+		unit: 'USD per therm',
+		freq: 'M',
+		region: 'USA',
+		theme: 'energy'
+	},
+	{
+		id: 'SPASTT01NLM661N',
+		title: 'Netherlands share prices (OECD index)',
+		unit: 'Index 2015=100',
+		freq: 'M',
+		region: 'NLD',
+		theme: 'economy'
+	},
+	{
+		id: 'SPASTT01KRM661N',
+		title: 'South Korea share prices (OECD index)',
+		unit: 'Index 2015=100',
+		freq: 'M',
+		region: 'KOR',
+		theme: 'economy'
+	},
+	{
+		id: 'SPASTT01USM661N',
+		title: 'United States share prices (OECD index)',
+		unit: 'Index 2015=100',
+		freq: 'M',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'SPASTT01CNM661N',
+		title: 'China share prices (OECD index)',
+		unit: 'Index 2015=100',
+		freq: 'M',
+		region: 'CHN',
+		theme: 'economy'
+	},
+	{ id: 'SP500', title: 'S&P 500', unit: 'Index', freq: 'D', region: 'USA', theme: 'economy' },
+	{
+		id: 'NASDAQCOM',
+		title: 'NASDAQ Composite',
+		unit: 'Index',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'CPIAUCSL',
+		title: 'US consumer price index (CPI-U)',
+		unit: 'Index 1982-84=100',
+		freq: 'M',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'UNRATE',
+		title: 'US unemployment rate',
+		unit: '%',
+		freq: 'M',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'GDPC1',
+		title: 'US real GDP',
+		unit: 'Billions of chained 2017 USD (SAAR)',
+		freq: 'Q',
+		region: 'USA',
+		theme: 'economy'
+	}
+];
+
+export async function fred(opts: { only?: string; job: string }) {
+	await run('fred', opts.job, async (note) => {
+		let total = 0;
+		for (const s of FRED_SERIES) {
+			if (opts.only && !opts.only.split(',').includes(s.id)) continue;
+			try {
+				const csv = await getText(`https://fred.stlouisfed.org/graph/fredgraph.csv?id=${s.id}`);
+				const lines = csv.trim().split(/\r?\n/).slice(1);
+				const rows = lines
+					.map((l) => l.split(','))
+					.filter(([d]) => /^\d{4}-\d{2}-\d{2}$/.test(d))
+					.map(([d, v]) => ({
+						series_id: s.id,
+						period: d,
+						value: v === '.' || v === '' ? null : Number(v)
+					}));
+				if (!rows.length) throw new Error('empty CSV');
+				await sql.begin(async (tx) => {
+					await tx`delete from stats.fred_obs where series_id = ${s.id}`;
+					for (let i = 0; i < rows.length; i += 5000)
+						await tx`insert into stats.fred_obs ${tx(rows.slice(i, i + 5000))}`;
+				});
+				const obs: ObsRow[] = rows
+					.filter((r) => r.value != null)
+					.map((r) => ({
+						series_id: `fred:${s.id}`,
+						period: r.period,
+						value: r.value,
+						status: null
+					}));
+				await writeSeries(
+					[
+						{
+							series_id: `fred:${s.id}`,
+							source_id: 'fred',
+							source_code: s.id,
+							title: s.title,
+							unit: s.unit,
+							freq: s.freq,
+							region_code: s.region,
+							theme: s.theme,
+							meta: { url: `https://fred.stlouisfed.org/series/${s.id}` }
+						}
+					],
+					obs
+				);
+				log(`fred ${s.id}: ${rows.length} rows`);
+				total += rows.length;
+			} catch (error) {
+				note(`${s.id}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		return total;
+	});
+}

src/lib/server/db/schema/index.ts

@@ -26,3 +26,4 @@ export * from './trains';
 export * from './music';
 export * from './hardware';
 export * from './dataholders';
+export * from './stats';

src/lib/server/db/schema/schemas.ts

@@ -24,3 +24,4 @@ export const planesSchema = pgSchema('planes');
 export const wildlifeSchema = pgSchema('wildlife');
 export const trainsSchema = pgSchema('trains');
 export const dataholdersSchema = pgSchema('dataholders');
+export const statsSchema = pgSchema('stats');

src/lib/server/db/schema/stats.ts

@@ -0,0 +1,204 @@
+import {
+	bigint,
+	boolean,
+	char,
+	date,
+	doublePrecision,
+	index,
+	integer,
+	jsonb,
+	primaryKey,
+	text
+} from 'drizzle-orm/pg-core';
+import { statsSchema } from './schemas';
+import { timestamptz } from './helpers';
+
+// Public /stats section: open statistics from official sources (CBS, Eurostat, FRED, BIS, …).
+// Two layers (see docs/stats.md):
+//  - raw tables per source (`cbs_*`, `eurostat_*`, `fred_*`, `bis_*`), keeping the source's own codes;
+//  - a normalised layer (`source`, `series`, `obs`, `region`) that every chart reads.
+// Written only by the stats collector (scripts/stats-fetch.ts, systemd user timers on the host);
+// every run is logged in `fetch_run`. Everything here is open data, so personal_ro may read it.
+
+export const statsSource = statsSchema.table('source', {
+	sourceId: text('source_id').primaryKey(),
+	name: text('name').notNull(),
+	baseUrl: text('base_url').notNull(),
+	licence: text('licence').notNull(),
+	keyRequired: boolean('key_required').notNull().default(false),
+	/** Footer line under every chart that uses this source. */
+	attribution: text('attribution').notNull()
+});
+
+export const statsRegion = statsSchema.table('region', {
+	/** ISO3 for countries, 'EU27' / 'EA' / 'WLD' aggregates, CBS codes (GM0513, CR28) below that. */
+	regionCode: text('region_code').primaryKey(),
+	name: text('name').notNull(),
+	/** country, municipality, corop, aggregate */
+	kind: text('kind').notNull(),
+	parent: text('parent')
+});
+
+export const statsSeries = statsSchema.table(
+	'series',
+	{
+		/** Stable id, e.g. `cbs:86141NED:cpi-yoy`, `fred:DCOILBRENTEU`, `bis:XM`. */
+		seriesId: text('series_id').primaryKey(),
+		sourceId: text('source_id')
+			.notNull()
+			.references(() => statsSource.sourceId),
+		/** The source's own code (table + measure + dimension keys, FRED id, …). */
+		sourceCode: text('source_code').notNull(),
+		title: text('title').notNull(),
+		unit: text('unit').notNull(),
+		/** D, M, Q, S (semester) or A */
+		freq: char('freq', { length: 1 }).notNull(),
+		regionCode: text('region_code').references(() => statsRegion.regionCode),
+		/** economy, energy, gouda, budget, countries, … */
+		theme: text('theme').notNull(),
+		meta: jsonb('meta').$type<Record<string, unknown>>().notNull().default({})
+	},
+	(t) => [index('series_theme_idx').on(t.theme), index('series_source_idx').on(t.sourceId)]
+);
+
+export const statsObs = statsSchema.table(
+	'obs',
+	{
+		seriesId: text('series_id')
+			.notNull()
+			.references(() => statsSeries.seriesId, { onDelete: 'cascade' }),
+		/** Period start date: 2026-S1 → 2026-01-01, 2026Q3 → 2026-07-01, 2026 → 2026-01-01. */
+		period: date('period', { mode: 'string' }).notNull(),
+		value: doublePrecision('value'),
+		/** Source status flag (provisional, estimated, …) where given. */
+		status: text('status'),
+		fetchedAt: timestamptz('fetched_at').notNull().defaultNow()
+	},
+	(t) => [primaryKey({ columns: [t.seriesId, t.period] })]
+);
+
+export const statsFetchRun = statsSchema.table(
+	'fetch_run',
+	{
+		runId: integer('run_id').primaryKey().generatedAlwaysAsIdentity(),
+		sourceId: text('source_id')
+			.notNull()
+			.references(() => statsSource.sourceId),
+		/** What the run did: 'refresh', 'backfill', or a single table/dataset code. */
+		job: text('job').notNull().default('refresh'),
+		startedAt: timestamptz('started_at').notNull().defaultNow(),
+		finishedAt: timestamptz('finished_at'),
+		rows: integer('rows'),
+		ok: boolean('ok'),
+		error: text('error')
+	},
+	(t) => [index('fetch_run_source_idx').on(t.sourceId, t.startedAt)]
+);
+
+// ── A1 CBS StatLine (OData v4, long format) ───────────────────────────────────────────────
+
+export const statsCbsTable = statsSchema.table('cbs_table', {
+	tableId: text('table_id').primaryKey(),
+	title: text('title').notNull(),
+	/** `Modified` from the table's Properties; the nightly run re-pulls only when it changes. */
+	modified: timestamptz('modified'),
+	status: text('status'),
+	/** OData $filter used for big tables (regions of interest only); null = full table. */
+	filter: text('filter'),
+	obsCount: integer('obs_count'),
+	fetchedAt: timestamptz('fetched_at')
+});
+
+export const statsCbsCode = statsSchema.table(
+	'cbs_code',
+	{
+		tableId: text('table_id').notNull(),
+		/** 'Measure' for MeasureCodes, otherwise the dimension identifier (RegioS, Perioden, …). */
+		dimension: text('dimension').notNull(),
+		code: text('code').notNull(),
+		title: text('title').notNull(),
+		unit: text('unit'),
+		decimals: integer('decimals'),
+		groupId: text('group_id'),
+		sortIndex: integer('sort_index')
+	},
+	(t) => [primaryKey({ columns: [t.tableId, t.dimension, t.code] })]
+);
+
+export const statsCbsObs = statsSchema.table(
+	'cbs_obs',
+	{
+		tableId: text('table_id').notNull(),
+		/** CBS observation Id (unique per table). */
+		obsId: bigint('obs_id', { mode: 'number' }).notNull(),
+		measure: text('measure').notNull(),
+		/** Every non-time dimension, trimmed: {"RegioS":"GM0513","Geslacht":"T001038"}. */
+		dims: jsonb('dims').$type<Record<string, string>>().notNull(),
+		/** CBS period code as-is: 2026MM09, 2026KW03, 2026JJ00. */
+		period: text('period').notNull(),
+		value: doublePrecision('value'),
+		valueAttribute: text('value_attribute')
+	},
+	(t) => [
+		primaryKey({ columns: [t.tableId, t.obsId] }),
+		index('cbs_obs_table_measure_idx').on(t.tableId, t.measure)
+	]
+);
+
+// ── A2 Eurostat dissemination API (JSON-stat) ─────────────────────────────────────────────
+
+export const statsEurostatDataset = statsSchema.table('eurostat_dataset', {
+	code: text('code').primaryKey(),
+	label: text('label').notNull(),
+	/** `updated` from the JSON-stat payload. */
+	updated: timestamptz('updated'),
+	/** Query used (filters), so a refresh repeats it exactly. */
+	query: text('query').notNull(),
+	/** Dimension ids in order plus their category labels. */
+	dims: jsonb('dims').$type<Record<string, unknown>>().notNull().default({}),
+	obsCount: integer('obs_count'),
+	fetchedAt: timestamptz('fetched_at')
+});
+
+export const statsEurostatObs = statsSchema.table(
+	'eurostat_obs',
+	{
+		dataset: text('dataset').notNull(),
+		/** Non-time dimension codes joined with '.', in the dataset's dimension order. */
+		key: text('key').notNull(),
+		geo: text('geo').notNull(),
+		/** Eurostat time code as-is: 2026-S1, 2026-09, 2026-Q2, 2025. */
+		period: text('period').notNull(),
+		value: doublePrecision('value'),
+		status: text('status')
+	},
+	(t) => [
+		primaryKey({ columns: [t.dataset, t.key, t.period] }),
+		index('eurostat_obs_dataset_geo_idx').on(t.dataset, t.geo)
+	]
+);
+
+// ── A7 FRED graph CSV, A5 BIS policy rates ────────────────────────────────────────────────
+
+export const statsFredObs = statsSchema.table(
+	'fred_obs',
+	{
+		seriesId: text('series_id').notNull(),
+		period: date('period', { mode: 'string' }).notNull(),
+		value: doublePrecision('value')
+	},
+	(t) => [primaryKey({ columns: [t.seriesId, t.period] })]
+);
+
+export const statsBisPolicyRate = statsSchema.table(
+	'bis_policy_rate',
+	{
+		/** BIS area code: US, XM (euro area), KR, CN, GB, JP. */
+		refArea: text('ref_area').notNull(),
+		/** D or M */
+		freq: char('freq', { length: 1 }).notNull(),
+		period: date('period', { mode: 'string' }).notNull(),
+		rate: doublePrecision('rate')
+	},
+	(t) => [primaryKey({ columns: [t.refArea, t.freq, t.period] })]
+);