stats: new public stats schema (source/series/obs/region/fetch_run + raw CBS, Eurostat, FRED, BIS tables) and the stats collector (scripts/stats-fetch.ts); preflight checks the stats tables

2026-10-25 20:27:04 +00:00
parent 902b1cadd5
commit 8ef291f586
14 changed files with 33128 additions and 2 deletions
--- a/scripts/stats/bis.ts
+++ b/scripts/stats/bis.ts
@@ -0,0 +1,78 @@
+// A5 BIS central bank policy rates (WS_CBPOL), monthly and daily, in one call each.
+import { getText, log, run, sql, writeSeries, type ObsRow, type SeriesRow } from './common';
+
+const AREAS: Record<string, { name: string; region: string | null }> = {
+	US: { name: 'Federal Reserve (US)', region: 'USA' },
+	XM: { name: 'ECB (euro area)', region: 'EA' },
+	KR: { name: 'Bank of Korea', region: 'KOR' },
+	CN: { name: "People's Bank of China", region: 'CHN' },
+	GB: { name: 'Bank of England', region: 'GBR' },
+	JP: { name: 'Bank of Japan', region: 'JPN' }
+};
+
+function parseCsv(text: string): Record<string, string>[] {
+	const lines = text.trim().split(/\r?\n/);
+	const split = (line: string) => {
+		const out: string[] = [];
+		let cur = '';
+		let q = false;
+		for (const ch of line) {
+			if (ch === '"') q = !q;
+			else if (ch === ',' && !q) {
+				out.push(cur);
+				cur = '';
+			} else cur += ch;
+		}
+		out.push(cur);
+		return out;
+	};
+	const head = split(lines[0]);
+	return lines.slice(1).map((l) => Object.fromEntries(split(l).map((v, i) => [head[i], v])));
+}
+
+export async function bis(opts: { job: string }) {
+	await run('bis', opts.job, async () => {
+		let total = 0;
+		for (const freq of ['M', 'D'] as const) {
+			const csv = await getText(
+				`https://stats.bis.org/api/v2/data/dataflow/BIS/WS_CBPOL/1.0/${freq}.${Object.keys(AREAS).join('+')}?format=csv`
+			);
+			const rows = parseCsv(csv)
+				.filter((r) => r.TIME_PERIOD && r.REF_AREA)
+				.map((r) => ({
+					ref_area: r.REF_AREA,
+					freq,
+					period: freq === 'M' ? `${r.TIME_PERIOD.slice(0, 7)}-01` : r.TIME_PERIOD.slice(0, 10),
+					rate: r.OBS_VALUE === '' || r.OBS_VALUE === 'NaN' ? null : Number(r.OBS_VALUE)
+				}));
+			await sql.begin(async (tx) => {
+				await tx`delete from stats.bis_policy_rate where freq = ${freq}`;
+				for (let i = 0; i < rows.length; i += 5000)
+					await tx`insert into stats.bis_policy_rate ${tx(rows.slice(i, i + 5000))} on conflict do nothing`;
+			});
+			const series: SeriesRow[] = Object.entries(AREAS).map(([code, a]) => ({
+				series_id: `bis:${freq}:${code}`,
+				source_id: 'bis',
+				source_code: `WS_CBPOL/${freq}.${code}`,
+				title: `${a.name} policy rate`,
+				unit: '%',
+				freq,
+				region_code: a.region,
+				theme: 'economy',
+				meta: {}
+			}));
+			const obs: ObsRow[] = rows
+				.filter((r) => r.rate != null)
+				.map((r) => ({
+					series_id: `bis:${freq}:${r.ref_area}`,
+					period: r.period,
+					value: r.rate,
+					status: null
+				}));
+			await writeSeries(series, obs);
+			log(`bis ${freq}: ${rows.length} rows`);
+			total += rows.length;
+		}
+		return total;
+	});
+}
--- a/scripts/stats/cbs-series.ts
+++ b/scripts/stats/cbs-series.ts
@@ -0,0 +1,26 @@
+// Chart series normalised from the raw CBS rows (stats.cbs_obs → stats.series/obs).
+// `dims` must match the observation's dimension codes exactly (see stats.cbs_code for titles).
+import { GOUDA_RING } from './common';
+
+export type CbsSeriesDef = {
+	id: string;
+	table: string;
+	measure: string;
+	dims?: Record<string, string>;
+	freq: 'M' | 'Q' | 'S' | 'A';
+	title: string;
+	unit: string;
+	theme: string;
+	region?: string;
+	/** Expand into one series per region (id suffixed with :<code>). */
+	regionDim?: string;
+	regions?: { code: string; region: string }[];
+	scale?: number;
+};
+
+export const RING = [
+	{ code: 'NL01', region: 'NLD' },
+	...GOUDA_RING.map((code) => ({ code, region: code }))
+];
+
+export const CBS_SERIES: CbsSeriesDef[] = [];
--- a/scripts/stats/cbs.ts
+++ b/scripts/stats/cbs.ts
@@ -0,0 +1,240 @@
+// A1 CBS StatLine via the OData v4 API (long format: one row per measure × dimensions × period).
+// Raw rows go to stats.cbs_obs (+ code lists in stats.cbs_code); the chart series in
+// CBS_SERIES are then normalised into stats.series / stats.obs.
+// Nightly: re-pull a table only when its `Modified` changed (or with --force).
+import {
+	GOUDA_RING,
+	getJson,
+	log,
+	parsePeriod,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+import { CBS_SERIES } from './cbs-series';
+
+const BASE = 'https://datasets.cbs.nl/odata/v1/CBS';
+const NL = ['NL01', 'NL00'];
+const regionIn = (dim: string, extra: string[] = []) =>
+	'(' + [...NL, ...GOUDA_RING, 'CR28', ...extra].map((c) => `${dim} eq '${c}'`).join(' or ') + ')';
+const kwbFilter =
+	`(startswith(WijkenEnBuurten,'WK0513') or startswith(WijkenEnBuurten,'BU0513') or ` +
+	[...NL, ...GOUDA_RING].map((c) => `WijkenEnBuurten eq '${c}'`).join(' or ') +
+	')';
+
+export type CbsTable = { id: string; theme: string; filter?: string; note?: string };
+
+/** Tables pulled in build step 1. Big tables are filtered to NL + Gouda + its ring. */
+export const CBS_TABLES: CbsTable[] = [
+	// energy
+	{ id: '85592NED', theme: 'energy' },
+	{ id: '84672NED', theme: 'energy', note: 'discontinued 2018–2023 tariffs (history)' },
+	{ id: '81309NED', theme: 'energy', note: 'discontinued 2007–2022 prices (history)' },
+	{ id: '83140NED', theme: 'energy' },
+	{ id: '82610NED', theme: 'energy' },
+	{ id: '85523NED', theme: 'energy' },
+	{ id: '85005NED', theme: 'energy' },
+	{ id: '85237NED', theme: 'energy' },
+	// economy
+	{ id: '86141NED', theme: 'economy' },
+	{ id: '83136NED', theme: 'economy' },
+	{ id: '86144NED', theme: 'economy', filter: "Bestedingscategorieen eq 'T001112'" },
+	{ id: '85865NED', theme: 'economy' },
+	{ id: '85773NED', theme: 'economy' },
+	// budget
+	{ id: '84120NED', theme: 'budget' },
+	{ id: '84117NED', theme: 'budget' },
+	{ id: '85590NED', theme: 'budget' },
+	{ id: '80509ned', theme: 'budget' },
+	{ id: '83642NED', theme: 'budget' },
+	{ id: '83641NED', theme: 'budget' },
+	// gouda
+	{ id: '37230ned', theme: 'gouda', filter: regionIn('RegioS') },
+	{ id: '37259ned', theme: 'gouda', filter: regionIn('RegioS') },
+	{ id: '70072ned', theme: 'gouda', filter: regionIn('RegioS') },
+	{ id: '83625NED', theme: 'gouda' },
+	{ id: '85819NED', theme: 'gouda' },
+	{ id: '86079NED', theme: 'gouda', filter: regionIn('RegioS') },
+	{ id: '83162NED', theme: 'gouda' },
+	{ id: '70675ned', theme: 'gouda' },
+	// Kerncijfers wijken en buurten, one table per year (Gouda wijken/buurten + ring + NL)
+	{ id: '86358NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2026' },
+	{ id: '86165NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2025' },
+	{ id: '85984NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2024' },
+	{ id: '85618NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2023' },
+	{ id: '85318NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2022' },
+	{ id: '85039NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2021' },
+	{ id: '84799NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2020' },
+	{ id: '84583NED', theme: 'gouda', filter: kwbFilter, note: 'kwb 2019' }
+];
+
+type Props = { Title: string; Modified: string; Status: string; ObservationCount?: number };
+type Page<T> = { value: T[]; '@odata.nextLink'?: string };
+
+async function all<T>(url: string): Promise<T[]> {
+	const out: T[] = [];
+	let next: string | undefined = url;
+	while (next) {
+		const page: Page<T> = await getJson<Page<T>>(next);
+		out.push(...page.value);
+		next = page['@odata.nextLink'];
+	}
+	return out;
+}
+
+async function pullTable(t: CbsTable, force: boolean, note: (w: string) => void): Promise<number> {
+	const props = await getJson<Props>(`${BASE}/${t.id}/Properties`);
+	const [known] = await sql<{ modified: Date | null; filter: string | null }[]>`
+		select modified, filter from stats.cbs_table where table_id = ${t.id}`;
+	if (props.Status && props.Status !== 'Regulier' && !t.note?.includes('discontinued'))
+		note(
+			`${t.id} status is ${props.Status}: check the CBS catalog for its successor (base-year swap)`
+		);
+	if (
+		!force &&
+		known?.modified &&
+		new Date(props.Modified).getTime() === known.modified.getTime() &&
+		(known.filter ?? null) === (t.filter ?? null)
+	) {
+		log(`cbs ${t.id}: unchanged since ${props.Modified}`);
+		return 0;
+	}
+	log(`cbs ${t.id}: pulling "${props.Title.trim()}"${t.filter ? ' (filtered)' : ''}`);
+	const dims = await getJson<Page<{ Identifier: string; Kind: string; CodesUrl: string }>>(
+		`${BASE}/${t.id}/Dimensions`
+	);
+	const codes: Record<string, unknown>[] = [];
+	const measures = await all<{
+		Identifier: string;
+		Title: string;
+		Unit?: string;
+		Decimals?: number;
+		MeasureGroupId?: string;
+		Index?: number;
+	}>(`${BASE}/${t.id}/MeasureCodes`);
+	for (const m of measures)
+		codes.push({
+			table_id: t.id,
+			dimension: 'Measure',
+			code: m.Identifier.trim(),
+			title: (m.Title ?? '').trim() || m.Identifier,
+			unit: m.Unit?.trim() || null,
+			decimals: m.Decimals ?? null,
+			group_id: m.MeasureGroupId ?? null,
+			sort_index: m.Index ?? null
+		});
+	const dimIds: string[] = [];
+	for (const d of dims.value) {
+		dimIds.push(d.Identifier);
+		const list = await all<{
+			Identifier: string;
+			Title: string;
+			DimensionGroupId?: string;
+			Index?: number;
+		}>(d.CodesUrl);
+		for (const c of list)
+			codes.push({
+				table_id: t.id,
+				dimension: d.Identifier,
+				code: c.Identifier.trim(),
+				title: (c.Title ?? '').trim() || c.Identifier,
+				unit: null,
+				decimals: null,
+				group_id: c.DimensionGroupId ?? null,
+				sort_index: c.Index ?? null
+			});
+	}
+	const timeDim = dims.value.find((d) => d.Kind === 'TimeDimension')?.Identifier;
+	const url =
+		`${BASE}/${t.id}/Observations` + (t.filter ? `?$filter=${encodeURIComponent(t.filter)}` : '');
+	const raw = await all<Record<string, unknown>>(url);
+	const rows = raw.map((o) => {
+		const d: Record<string, string> = {};
+		for (const id of dimIds) if (id !== timeDim && o[id] != null) d[id] = String(o[id]).trim();
+		return {
+			table_id: t.id,
+			obs_id: Number(o.Id),
+			measure: String(o.Measure).trim(),
+			dims: sql.json(d),
+			period: timeDim ? String(o[timeDim]).trim() : '',
+			value: typeof o.Value === 'number' ? o.Value : null,
+			value_attribute:
+				o.ValueAttribute && o.ValueAttribute !== 'None' ? String(o.ValueAttribute) : null
+		};
+	});
+	await sql.begin(async (tx) => {
+		await tx`delete from stats.cbs_obs where table_id = ${t.id}`;
+		await tx`delete from stats.cbs_code where table_id = ${t.id}`;
+		for (let i = 0; i < codes.length; i += 5000)
+			await tx`insert into stats.cbs_code ${tx(codes.slice(i, i + 5000))} on conflict do nothing`;
+		for (let i = 0; i < rows.length; i += 5000)
+			await tx`insert into stats.cbs_obs ${tx(rows.slice(i, i + 5000))}`;
+		await tx`insert into stats.cbs_table (table_id, title, modified, status, filter, obs_count, fetched_at)
+			values (${t.id}, ${props.Title.trim()}, ${props.Modified}, ${props.Status ?? null}, ${t.filter ?? null}, ${rows.length}, now())
+			on conflict (table_id) do update set title = excluded.title, modified = excluded.modified,
+			status = excluded.status, filter = excluded.filter, obs_count = excluded.obs_count, fetched_at = now()`;
+	});
+	log(`cbs ${t.id}: ${rows.length} observations, ${codes.length} codes`);
+	return rows.length;
+}
+
+/** Normalise CBS_SERIES from the raw rows into stats.series / stats.obs. */
+export async function normaliseCbs(): Promise<number> {
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	for (const def of CBS_SERIES) {
+		const regions = def.regions ?? [null];
+		for (const region of regions) {
+			const dims = { ...(def.dims ?? {}) };
+			if (region && def.regionDim) dims[def.regionDim] = region.code;
+			const rows = await sql<
+				{ period: string; value: number | null; value_attribute: string | null }[]
+			>`
+				select period, value, value_attribute from stats.cbs_obs
+				where table_id = ${def.table} and measure = ${def.measure} and dims @> ${sql.json(dims)}`;
+			const id = region ? `${def.id}:${region.code}` : def.id;
+			let n = 0;
+			const seen = new Set<string>();
+			for (const r of rows) {
+				const p = parsePeriod(r.period);
+				if (!p || p.freq !== def.freq || seen.has(p.date)) continue;
+				seen.add(p.date);
+				const value = r.value == null ? null : r.value * (def.scale ?? 1);
+				obs.push({ series_id: id, period: p.date, value, status: r.value_attribute });
+				n++;
+			}
+			if (!n) continue;
+			series.push({
+				series_id: id,
+				source_id: 'cbs',
+				source_code: `${def.table}/${def.measure}/${JSON.stringify(dims)}`,
+				title: def.title,
+				unit: def.unit,
+				freq: def.freq,
+				region_code: region?.region ?? def.region ?? 'NLD',
+				theme: def.theme,
+				meta: { table: def.table, measure: def.measure, dims }
+			});
+		}
+	}
+	return writeSeries(series, obs);
+}
+
+export async function cbs(opts: { force: boolean; only?: string; job: string }) {
+	let total = 0;
+	await run('cbs', opts.job, async (note) => {
+		for (const t of CBS_TABLES) {
+			if (opts.only && !opts.only.split(',').includes(t.id)) continue;
+			try {
+				total += await pullTable(t, opts.force, note);
+			} catch (error) {
+				note(`${t.id}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		const normalised = await normaliseCbs();
+		log(`cbs: ${normalised} normalised observations`);
+		return total;
+	});
+}
--- a/scripts/stats/common.ts
+++ b/scripts/stats/common.ts
@@ -0,0 +1,237 @@
+// Shared plumbing for the /stats collector (scripts/stats-fetch.ts): DB client, polite HTTP,
+// fetch_run logging, and the normalised layer (stats.source / region / series / obs).
+// Runs on the host (systemd user timers, see deploy/stats-collector) with DB_URL = app role.
+import postgres from 'postgres';
+
+if (!process.env.DB_URL) throw new Error('DB_URL is missing');
+export const sql = postgres(process.env.DB_URL, {
+	max: 2,
+	connect_timeout: 15,
+	onnotice: () => {},
+	prepare: false
+});
+
+export const USER_AGENT =
+	'stanrunge.dev-stats/1.0 (+https://stanrunge.dev/stats; open-data collector; stan@stanrunge.dev)';
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/** Minimum gap between requests per host (CBS ≤5 req/s → 250 ms; OECD ≤20/min → 3.1 s). */
+const hostGapMs: Record<string, number> = {
+	'datasets.cbs.nl': 250,
+	'opendata.cbs.nl': 250,
+	'sdmx.oecd.org': 3100,
+	'fred.stlouisfed.org': 1000,
+	'ec.europa.eu': 500,
+	'stats.bis.org': 1000
+};
+const lastHit = new Map<string, number>();
+
+export async function http(url: string, init: RequestInit = {}, tries = 4): Promise<Response> {
+	const host = new URL(url).host;
+	for (let attempt = 1; ; attempt++) {
+		const gap = hostGapMs[host] ?? 300;
+		const wait = (lastHit.get(host) ?? 0) + gap - Date.now();
+		if (wait > 0) await sleep(wait);
+		lastHit.set(host, Date.now());
+		let response: Response | undefined;
+		let failure: unknown;
+		try {
+			response = await fetch(url, {
+				...init,
+				headers: { 'user-agent': USER_AGENT, accept: '*/*', ...(init.headers ?? {}) },
+				signal: AbortSignal.timeout(180_000)
+			});
+			if (response.ok) return response;
+			if (response.status !== 429 && response.status < 500) {
+				const body = (await response.text()).slice(0, 300);
+				throw new Error(`HTTP ${response.status} for ${url}: ${body}`);
+			}
+			failure = new Error(`HTTP ${response.status} for ${url}`);
+		} catch (error) {
+			if (error instanceof Error && error.message.startsWith('HTTP 4')) throw error;
+			failure = error;
+		}
+		if (attempt >= tries) throw failure;
+		const retryAfter = Number(response?.headers.get('retry-after'));
+		await sleep(
+			Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : 2000 * 2 ** attempt
+		);
+	}
+}
+
+export const getJson = async <T = unknown>(url: string) => (await (await http(url)).json()) as T;
+export const getText = async (url: string) => (await http(url)).text();
+
+export function log(...parts: unknown[]) {
+	console.log(new Date().toISOString(), ...parts);
+}
+
+/** Wraps one collector job in a stats.fetch_run row (rows, ok, error). */
+export async function run(
+	sourceId: string,
+	job: string,
+	fn: (note: (warning: string) => void) => Promise<number>
+): Promise<number> {
+	const [{ run_id }] = await sql`
+		insert into stats.fetch_run (source_id, job) values (${sourceId}, ${job}) returning run_id`;
+	const warnings: string[] = [];
+	try {
+		const rows = await fn((w) => warnings.push(w));
+		await sql`update stats.fetch_run set finished_at = now(), rows = ${rows}, ok = true,
+			error = ${warnings.length ? warnings.join('; ').slice(0, 4000) : null} where run_id = ${run_id}`;
+		log(
+			`${sourceId}/${job}: ok, ${rows} rows${warnings.length ? `, ${warnings.length} warnings` : ''}`
+		);
+		return rows;
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		await sql`update stats.fetch_run set finished_at = now(), ok = false,
+			error = ${[message, ...warnings].join('; ').slice(0, 4000)} where run_id = ${run_id}`;
+		log(`${sourceId}/${job}: FAILED ${message}`);
+		throw error;
+	}
+}
+
+// ── Sources and regions ──────────────────────────────────────────────────────────────────
+
+export const SOURCES = [
+	{
+		source_id: 'cbs',
+		name: 'CBS StatLine',
+		base_url: 'https://datasets.cbs.nl/odata/v1/CBS',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: CBS StatLine (Statistics Netherlands), CC BY 4.0'
+	},
+	{
+		source_id: 'eurostat',
+		name: 'Eurostat',
+		base_url: 'https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: Eurostat, CC BY 4.0'
+	},
+	{
+		source_id: 'fred',
+		name: 'FRED (Federal Reserve Bank of St. Louis)',
+		base_url: 'https://fred.stlouisfed.org/graph/fredgraph.csv',
+		licence: 'Per series (EIA/BLS/Fed public domain, OECD CC BY 4.0)',
+		key_required: false,
+		attribution: 'Source: FRED, Federal Reserve Bank of St. Louis (original series sources apply)'
+	},
+	{
+		source_id: 'bis',
+		name: 'BIS central bank policy rates',
+		base_url: 'https://stats.bis.org/api/v2/data/dataflow/BIS/WS_CBPOL/1.0',
+		licence: 'Free reuse with attribution',
+		key_required: false,
+		attribution: 'Source: BIS, central bank policy rates (WS_CBPOL)'
+	}
+] as const;
+
+export const REGIONS: { region_code: string; name: string; kind: string; parent: string | null }[] =
+	[
+		{ region_code: 'WLD', name: 'World', kind: 'aggregate', parent: null },
+		{ region_code: 'EU27', name: 'European Union (27)', kind: 'aggregate', parent: 'WLD' },
+		{ region_code: 'EA', name: 'Euro area', kind: 'aggregate', parent: 'EU27' },
+		{ region_code: 'NLD', name: 'Netherlands', kind: 'country', parent: 'EU27' },
+		{ region_code: 'USA', name: 'United States', kind: 'country', parent: 'WLD' },
+		{ region_code: 'KOR', name: 'South Korea', kind: 'country', parent: 'WLD' },
+		{ region_code: 'CHN', name: 'China', kind: 'country', parent: 'WLD' },
+		{ region_code: 'CUW', name: 'Curaçao', kind: 'country', parent: 'WLD' },
+		{ region_code: 'DEU', name: 'Germany', kind: 'country', parent: 'EU27' },
+		{ region_code: 'FRA', name: 'France', kind: 'country', parent: 'EU27' },
+		{ region_code: 'BEL', name: 'Belgium', kind: 'country', parent: 'EU27' },
+		{ region_code: 'ITA', name: 'Italy', kind: 'country', parent: 'EU27' },
+		{ region_code: 'ESP', name: 'Spain', kind: 'country', parent: 'EU27' },
+		{ region_code: 'POL', name: 'Poland', kind: 'country', parent: 'EU27' },
+		{ region_code: 'SWE', name: 'Sweden', kind: 'country', parent: 'EU27' },
+		{ region_code: 'DNK', name: 'Denmark', kind: 'country', parent: 'EU27' },
+		{ region_code: 'AUT', name: 'Austria', kind: 'country', parent: 'EU27' },
+		{ region_code: 'GBR', name: 'United Kingdom', kind: 'country', parent: 'WLD' },
+		{ region_code: 'JPN', name: 'Japan', kind: 'country', parent: 'WLD' },
+		{ region_code: 'IND', name: 'India', kind: 'country', parent: 'WLD' },
+		{ region_code: 'CR28', name: 'Oost-Zuid-Holland (COROP)', kind: 'corop', parent: 'NLD' },
+		{ region_code: 'GM0513', name: 'Gouda', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM0627', name: 'Waddinxveen', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM1901', name: 'Bodegraven-Reeuwijk', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM1892', name: 'Zuidplas', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM1931', name: 'Krimpenerwaard', kind: 'municipality', parent: 'CR28' },
+		{ region_code: 'GM0484', name: 'Alphen aan den Rijn', kind: 'municipality', parent: 'NLD' },
+		{ region_code: 'GM0632', name: 'Woerden', kind: 'municipality', parent: 'NLD' }
+	];
+
+/** Gouda and its house-hunting ring (CBS gemeente codes). */
+export const GOUDA_RING = ['GM0513', 'GM0627', 'GM1901', 'GM1892', 'GM1931', 'GM0484', 'GM0632'];
+
+export async function ensureMeta() {
+	await sql`insert into stats.source ${sql(SOURCES as unknown as Record<string, unknown>[])}
+		on conflict (source_id) do update set name = excluded.name, base_url = excluded.base_url,
+		licence = excluded.licence, key_required = excluded.key_required, attribution = excluded.attribution`;
+	await sql`insert into stats.region ${sql(REGIONS)}
+		on conflict (region_code) do update set name = excluded.name, kind = excluded.kind, parent = excluded.parent`;
+}
+
+export type SeriesRow = {
+	series_id: string;
+	source_id: string;
+	source_code: string;
+	title: string;
+	unit: string;
+	freq: string;
+	region_code: string | null;
+	theme: string;
+	meta: Record<string, unknown>;
+};
+export type ObsRow = {
+	series_id: string;
+	period: string;
+	value: number | null;
+	status: string | null;
+};
+
+/** Replaces the normalised series + observations for the given series ids. */
+export async function writeSeries(series: SeriesRow[], obs: ObsRow[]): Promise<number> {
+	if (!series.length) return 0;
+	const known = new Set(REGIONS.map((r) => r.region_code));
+	for (const s of series) if (s.region_code && !known.has(s.region_code)) s.region_code = null;
+	await sql.begin(async (tx) => {
+		for (let i = 0; i < series.length; i += 2000) {
+			const chunk = series
+				.slice(i, i + 2000)
+				.map((s) => ({ ...s, meta: tx.json(s.meta as never) }));
+			await tx`insert into stats.series ${tx(chunk)} on conflict (series_id) do update set
+				source_id = excluded.source_id, source_code = excluded.source_code, title = excluded.title,
+				unit = excluded.unit, freq = excluded.freq, region_code = excluded.region_code,
+				theme = excluded.theme, meta = excluded.meta`;
+		}
+		const ids = series.map((s) => s.series_id);
+		await tx`delete from stats.obs where series_id = any(${ids})`;
+		for (let i = 0; i < obs.length; i += 5000) {
+			await tx`insert into stats.obs ${tx(obs.slice(i, i + 5000), 'series_id', 'period', 'value', 'status')}
+				on conflict (series_id, period) do update set value = excluded.value, status = excluded.status, fetched_at = now()`;
+		}
+	});
+	return obs.length;
+}
+
+/** Period code → period start date (YYYY-MM-DD) and frequency. Handles CBS, Eurostat, ISO forms. */
+export function parsePeriod(code: string): { date: string; freq: string } | null {
+	const c = code.trim();
+	let m: RegExpMatchArray | null;
+	if ((m = c.match(/^(\d{4})JJ00$/)) || (m = c.match(/^(\d{4})$/)))
+		return { date: `${m[1]}-01-01`, freq: 'A' };
+	if (
+		(m = c.match(/^(\d{4})MM(\d{2})$/)) ||
+		(m = c.match(/^(\d{4})-(\d{2})$/)) ||
+		(m = c.match(/^(\d{4})M(\d{2})$/))
+	)
+		return { date: `${m[1]}-${m[2]}-01`, freq: 'M' };
+	if ((m = c.match(/^(\d{4})KW0?(\d)$/)) || (m = c.match(/^(\d{4})-?Q(\d)$/)))
+		return { date: `${m[1]}-${String((Number(m[2]) - 1) * 3 + 1).padStart(2, '0')}-01`, freq: 'Q' };
+	if ((m = c.match(/^(\d{4})-?S(\d)$/)) || (m = c.match(/^(\d{4})HJ0?(\d)$/)))
+		return { date: `${m[1]}-${m[2] === '1' ? '01' : '07'}-01`, freq: 'S' };
+	if (/^\d{4}-\d{2}-\d{2}$/.test(c)) return { date: c, freq: 'D' };
+	return null;
+}
--- a/scripts/stats/eurostat.ts
+++ b/scripts/stats/eurostat.ts
@@ -0,0 +1,249 @@
+// A2 Eurostat dissemination API (JSON-stat 2.0). Each dataset is pulled with a fixed filter
+// (12 countries + EU/EA aggregates, one consumption band / unit) into stats.eurostat_obs, then
+// every (dataset, key) becomes a normalised series `eurostat:<dataset>:<key>`.
+import {
+	getJson,
+	log,
+	parsePeriod,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const BASE = 'https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data';
+export const EU_GEOS = [
+	'NL',
+	'DE',
+	'FR',
+	'BE',
+	'IT',
+	'ES',
+	'PL',
+	'SE',
+	'DK',
+	'AT',
+	'EU27_2020',
+	'EA20'
+];
+const GEO_REGION: Record<string, string> = {
+	NL: 'NLD',
+	DE: 'DEU',
+	FR: 'FRA',
+	BE: 'BEL',
+	IT: 'ITA',
+	ES: 'ESP',
+	PL: 'POL',
+	SE: 'SWE',
+	DK: 'DNK',
+	AT: 'AUT',
+	EU27_2020: 'EU27',
+	EA20: 'EA',
+	EA: 'EA',
+	UK: 'GBR',
+	JP: 'JPN',
+	US: 'USA'
+};
+
+type Dataset = {
+	code: string;
+	theme: string;
+	params: Record<string, string | string[]>;
+	geos?: string[];
+};
+
+export const EUROSTAT_DATASETS: Dataset[] = [
+	{
+		code: 'nrg_pc_204',
+		theme: 'energy',
+		params: { nrg_cons: 'KWH2500-4999', currency: 'EUR', unit: 'KWH' }
+	},
+	{
+		code: 'nrg_pc_202',
+		theme: 'energy',
+		params: { nrg_cons: 'GJ20-199', currency: 'EUR', unit: 'KWH' }
+	},
+	{ code: 'nrg_pc_204_h', theme: 'energy', params: { consom: '4161902', currency: 'EUR' } },
+	{ code: 'nrg_pc_202_h', theme: 'energy', params: { consom: '4141902', currency: 'EUR' } },
+	{ code: 'nrg_ind_id', theme: 'energy', params: { siec: 'TOTAL' } },
+	{
+		code: 'irt_lt_mcby_m',
+		theme: 'economy',
+		params: {},
+		geos: ['NL', 'DE', 'FR', 'IT', 'BE', 'ES', 'AT', 'EA']
+	},
+	{ code: 'prc_hicp_minr', theme: 'economy', params: { coicop18: 'TOTAL', unit: 'RCH_A' } },
+	{ code: 'prc_hicp_manr', theme: 'economy', params: { coicop: 'CP00', unit: 'RCH_A' } },
+	{ code: 'prc_hpi_q', theme: 'economy', params: { purchase: 'TOTAL', unit: ['I15_Q', 'RCH_A'] } },
+	{ code: 'road_eqr_carpda', theme: 'energy', params: { unit: 'NR' } },
+	{
+		code: 'migr_imm1ctz',
+		theme: 'countries',
+		params: { citizen: 'TOTAL', age: 'TOTAL', sex: 'T', agedef: 'COMPLET', unit: 'NR' }
+	},
+	{
+		code: 'gov_10a_main',
+		theme: 'budget',
+		params: { unit: 'PC_GDP', sector: 'S13', na_item: ['TR', 'TE', 'B9'] }
+	},
+	{
+		code: 'gov_10a_exp',
+		theme: 'budget',
+		params: { unit: 'PC_GDP', sector: 'S13', na_item: 'TE' }
+	},
+	{
+		code: 'gov_10a_taxag',
+		theme: 'budget',
+		params: { unit: 'PC_GDP', sector: 'S13_S212', na_item: ['D2_REC', 'D5_REC', 'D61_REC', 'D995'] }
+	},
+	{
+		code: 'gov_10dd_edpt1',
+		theme: 'budget',
+		params: { unit: 'PC_GDP', sector: 'S13', na_item: ['GD', 'B9'] }
+	},
+	{
+		code: 'nama_10_gdp',
+		theme: 'economy',
+		params: { unit: ['CP_MEUR', 'CLV_PCH_PRE'], na_item: 'B1GQ' }
+	},
+	{ code: 'demo_pjan', theme: 'countries', params: { sex: 'T', age: 'TOTAL', unit: 'NR' } }
+];
+
+type JsonStat = {
+	label: string;
+	updated?: string;
+	id: string[];
+	size: number[];
+	dimension: Record<
+		string,
+		{ label: string; category: { index: Record<string, number>; label: Record<string, string> } }
+	>;
+	value: Record<string, number> | number[];
+	status?: Record<string, string> | string[];
+};
+
+function query(d: Dataset) {
+	const q = new URLSearchParams({ format: 'JSON', lang: 'EN' });
+	for (const g of d.geos ?? EU_GEOS) q.append('geo', g);
+	for (const [k, v] of Object.entries(d.params)) for (const x of [v].flat()) q.append(k, x);
+	return q.toString();
+}
+
+async function pull(d: Dataset): Promise<number> {
+	const qs = query(d);
+	const js = await getJson<JsonStat>(`${BASE}/${d.code}?${qs}`);
+	const order = js.id;
+	const codesByDim = order.map((id) => {
+		const idx = js.dimension[id].category.index;
+		const arr: string[] = [];
+		for (const [code, i] of Object.entries(idx)) arr[i] = code;
+		return arr;
+	});
+	const timePos = order.indexOf('time');
+	const geoPos = order.indexOf('geo');
+	const rows: {
+		dataset: string;
+		key: string;
+		geo: string;
+		period: string;
+		value: number | null;
+		status: string | null;
+	}[] = [];
+	const values = js.value;
+	const status = js.status ?? {};
+	const entries: [number, number][] = Array.isArray(values)
+		? values.map((v, i) => [i, v] as [number, number]).filter(([, v]) => v != null)
+		: Object.entries(values).map(([k, v]) => [Number(k), v]);
+	for (const [flat, value] of entries) {
+		let rest = flat;
+		const coord: number[] = new Array(order.length);
+		for (let p = order.length - 1; p >= 0; p--) {
+			coord[p] = rest % js.size[p];
+			rest = Math.floor(rest / js.size[p]);
+		}
+		const keyParts: string[] = [];
+		for (let p = 0; p < order.length; p++)
+			if (p !== timePos && order[p] !== 'freq') keyParts.push(codesByDim[p][coord[p]]);
+		rows.push({
+			dataset: d.code,
+			key: keyParts.join('.'),
+			geo: geoPos >= 0 ? codesByDim[geoPos][coord[geoPos]] : '',
+			period: codesByDim[timePos][coord[timePos]],
+			value: value ?? null,
+			status: (Array.isArray(status) ? status[flat] : status[String(flat)]) ?? null
+		});
+	}
+	const dims: Record<string, unknown> = {};
+	for (const id of order)
+		dims[id] = { label: js.dimension[id].label, labels: js.dimension[id].category.label };
+	await sql.begin(async (tx) => {
+		await tx`delete from stats.eurostat_obs where dataset = ${d.code}`;
+		for (let i = 0; i < rows.length; i += 5000)
+			await tx`insert into stats.eurostat_obs ${tx(rows.slice(i, i + 5000))} on conflict do nothing`;
+		await tx`insert into stats.eurostat_dataset (code, label, updated, query, dims, obs_count, fetched_at)
+			values (${d.code}, ${js.label}, ${js.updated ?? null}, ${qs}, ${tx.json(dims as never)}, ${rows.length}, now())
+			on conflict (code) do update set label = excluded.label, updated = excluded.updated, query = excluded.query,
+			dims = excluded.dims, obs_count = excluded.obs_count, fetched_at = now()`;
+	});
+	// Normalise: one series per key.
+	const order2 = order.filter((id) => id !== 'time' && id !== 'freq');
+	const byKey = new Map<string, typeof rows>();
+	for (const r of rows) byKey.set(r.key, [...(byKey.get(r.key) ?? []), r]);
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	for (const [key, list] of byKey) {
+		const parts = key.split('.');
+		const labels = order2.map((id, i) => ({
+			id,
+			code: parts[i],
+			label: js.dimension[id].category.label[parts[i]] ?? parts[i]
+		}));
+		const unit =
+			labels.find((l) => l.id === 'unit')?.label ??
+			labels.find((l) => l.id === 'currency')?.label ??
+			'';
+		const geo = labels.find((l) => l.id === 'geo');
+		const id = `eurostat:${d.code}:${key}`;
+		let freq = 'A';
+		for (const r of list) {
+			const p = parsePeriod(r.period);
+			if (!p) continue;
+			freq = p.freq;
+			obs.push({ series_id: id, period: p.date, value: r.value, status: r.status });
+		}
+		series.push({
+			series_id: id,
+			source_id: 'eurostat',
+			source_code: `${d.code}/${key}`,
+			title: `${js.label} · ${labels
+				.filter((l) => l.id !== 'geo' && l.id !== 'unit')
+				.map((l) => l.label)
+				.join(', ')}`.replace(/ · $/, ''),
+			unit,
+			freq,
+			region_code: geo ? (GEO_REGION[geo.code] ?? null) : null,
+			theme: d.theme,
+			meta: { dataset: d.code, dims: Object.fromEntries(labels.map((l) => [l.id, l.code])) }
+		});
+	}
+	await sql`delete from stats.series where source_id = 'eurostat' and meta->>'dataset' = ${d.code}`;
+	await writeSeries(series, obs);
+	log(`eurostat ${d.code}: ${rows.length} observations, ${series.length} series`);
+	return rows.length;
+}
+
+export async function eurostat(opts: { only?: string; job: string }) {
+	await run('eurostat', opts.job, async (note) => {
+		let total = 0;
+		for (const d of EUROSTAT_DATASETS) {
+			if (opts.only && !opts.only.split(',').includes(d.code)) continue;
+			try {
+				total += await pull(d);
+			} catch (error) {
+				note(`${d.code}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		return total;
+	});
+}
--- a/scripts/stats/fred.ts
+++ b/scripts/stats/fred.ts
@@ -0,0 +1,230 @@
+// A7 FRED graph CSV (no key). One call per series → stats.fred_obs and series `fred:<id>`.
+import { getText, log, run, sql, writeSeries, type ObsRow } from './common';
+
+type Fred = {
+	id: string;
+	title: string;
+	unit: string;
+	freq: 'D' | 'M' | 'Q' | 'A';
+	region: string;
+	theme: string;
+};
+export const FRED_SERIES: Fred[] = [
+	{
+		id: 'DCOILBRENTEU',
+		title: 'Brent crude oil spot price',
+		unit: 'USD per barrel',
+		freq: 'D',
+		region: 'WLD',
+		theme: 'energy'
+	},
+	{
+		id: 'PNGASEUUSDM',
+		title: 'EU natural gas price (TTF, IMF)',
+		unit: 'USD per MMBtu',
+		freq: 'M',
+		region: 'EU27',
+		theme: 'energy'
+	},
+	{
+		id: 'DFEDTARU',
+		title: 'Fed funds target range, upper limit',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'DFEDTARL',
+		title: 'Fed funds target range, lower limit',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'DFF',
+		title: 'Effective federal funds rate',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'DGS10',
+		title: 'US Treasury 10-year yield',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'DGS2',
+		title: 'US Treasury 2-year yield',
+		unit: '%',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'IRLTLT01KRM156N',
+		title: 'South Korea 10-year government bond yield',
+		unit: '%',
+		freq: 'M',
+		region: 'KOR',
+		theme: 'economy'
+	},
+	{
+		id: 'IRLTLT01FRM156N',
+		title: 'France 10-year government bond yield',
+		unit: '%',
+		freq: 'M',
+		region: 'FRA',
+		theme: 'economy'
+	},
+	{
+		id: 'IRLTLT01DEM156N',
+		title: 'Germany 10-year government bond yield',
+		unit: '%',
+		freq: 'M',
+		region: 'DEU',
+		theme: 'economy'
+	},
+	{
+		id: 'APU000072610',
+		title: 'US average residential electricity price',
+		unit: 'USD per kWh',
+		freq: 'M',
+		region: 'USA',
+		theme: 'energy'
+	},
+	{
+		id: 'APU000072620',
+		title: 'US average utility (piped) gas price',
+		unit: 'USD per therm',
+		freq: 'M',
+		region: 'USA',
+		theme: 'energy'
+	},
+	{
+		id: 'SPASTT01NLM661N',
+		title: 'Netherlands share prices (OECD index)',
+		unit: 'Index 2015=100',
+		freq: 'M',
+		region: 'NLD',
+		theme: 'economy'
+	},
+	{
+		id: 'SPASTT01KRM661N',
+		title: 'South Korea share prices (OECD index)',
+		unit: 'Index 2015=100',
+		freq: 'M',
+		region: 'KOR',
+		theme: 'economy'
+	},
+	{
+		id: 'SPASTT01USM661N',
+		title: 'United States share prices (OECD index)',
+		unit: 'Index 2015=100',
+		freq: 'M',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'SPASTT01CNM661N',
+		title: 'China share prices (OECD index)',
+		unit: 'Index 2015=100',
+		freq: 'M',
+		region: 'CHN',
+		theme: 'economy'
+	},
+	{ id: 'SP500', title: 'S&P 500', unit: 'Index', freq: 'D', region: 'USA', theme: 'economy' },
+	{
+		id: 'NASDAQCOM',
+		title: 'NASDAQ Composite',
+		unit: 'Index',
+		freq: 'D',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'CPIAUCSL',
+		title: 'US consumer price index (CPI-U)',
+		unit: 'Index 1982-84=100',
+		freq: 'M',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'UNRATE',
+		title: 'US unemployment rate',
+		unit: '%',
+		freq: 'M',
+		region: 'USA',
+		theme: 'economy'
+	},
+	{
+		id: 'GDPC1',
+		title: 'US real GDP',
+		unit: 'Billions of chained 2017 USD (SAAR)',
+		freq: 'Q',
+		region: 'USA',
+		theme: 'economy'
+	}
+];
+
+export async function fred(opts: { only?: string; job: string }) {
+	await run('fred', opts.job, async (note) => {
+		let total = 0;
+		for (const s of FRED_SERIES) {
+			if (opts.only && !opts.only.split(',').includes(s.id)) continue;
+			try {
+				const csv = await getText(`https://fred.stlouisfed.org/graph/fredgraph.csv?id=${s.id}`);
+				const lines = csv.trim().split(/\r?\n/).slice(1);
+				const rows = lines
+					.map((l) => l.split(','))
+					.filter(([d]) => /^\d{4}-\d{2}-\d{2}$/.test(d))
+					.map(([d, v]) => ({
+						series_id: s.id,
+						period: d,
+						value: v === '.' || v === '' ? null : Number(v)
+					}));
+				if (!rows.length) throw new Error('empty CSV');
+				await sql.begin(async (tx) => {
+					await tx`delete from stats.fred_obs where series_id = ${s.id}`;
+					for (let i = 0; i < rows.length; i += 5000)
+						await tx`insert into stats.fred_obs ${tx(rows.slice(i, i + 5000))}`;
+				});
+				const obs: ObsRow[] = rows
+					.filter((r) => r.value != null)
+					.map((r) => ({
+						series_id: `fred:${s.id}`,
+						period: r.period,
+						value: r.value,
+						status: null
+					}));
+				await writeSeries(
+					[
+						{
+							series_id: `fred:${s.id}`,
+							source_id: 'fred',
+							source_code: s.id,
+							title: s.title,
+							unit: s.unit,
+							freq: s.freq,
+							region_code: s.region,
+							theme: s.theme,
+							meta: { url: `https://fred.stlouisfed.org/series/${s.id}` }
+						}
+					],
+					obs
+				);
+				log(`fred ${s.id}: ${rows.length} rows`);
+				total += rows.length;
+			} catch (error) {
+				note(`${s.id}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		return total;
+	});
+}