stats step 2: World Bank, IMF WEO, OWID, Energy-Charts, Rijksfinanciën, US Treasury, USAspending collectors + tables, timers

deploy/stats-collector/README.md

@@ -9,6 +9,13 @@ statistics (no API keys) into the `stats` schema with `scripts/stats-fetch.ts`:
 | `eurostat` | `stats-eurostat.timer` | Mondays 06:20 | Eurostat JSON-stat datasets (energy prices, yields, HICP, government finance…) |
 | `fred` | `stats-fred.timer` | daily 07:20 | FRED graph CSV (Brent, TTF, Fed, Treasuries, US energy prices, indices) |
 | `bis` | `stats-bis.timer` | daily 07:50 | BIS central bank policy rates, monthly + daily |
+| `worldbank` | `stats-worldbank.timer` | Mondays 04:15 | World Bank WDI indicators for the compared countries |
+| `imf` | `stats-imf.timer` | 1st of the month 04:05 | IMF WEO (DataMapper), stored per WEO vintage with forecast flags |
+| `owid` | `stats-owid.timer` | 2nd of the month 04:35 | Our World in Data: Maddison GDP per capita, energy dataset |
+| `energycharts` | `stats-energycharts.timer` | hourly at :19 | Energy-Charts NL day-ahead prices + cross-border flows (last 3 days; `--backfill` from 2015) |
+| `rijksfinancien` | `stats-rijksfinancien.timer` | 3rd of the month 05:05 | Rijksfinanciën budget tables, last year to next year (`--backfill` from 2015) |
+| `ustreasury` | `stats-ustreasury.timer` | daily 23:25 | US Treasury MTS receipts/outlays + Debt to the Penny (last 400 days; `--backfill` all) |
+| `usaspending` | `stats-usaspending.timer` | Sundays 06:10 | USAspending budgetary resources per agency and fiscal year |
 
 Each timer starts `stats-collector@<source>.service`, which runs
 `~/.local/bin/stats-collector-run.sh <source>`: it updates a detached checkout of `origin/main`

deploy/stats-collector/stats-energycharts.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: Energy-Charts hourly at :19 (off ned-collector's :07/:37)
+
+[Timer]
+OnCalendar=*-*-* *:19:00 Europe/Amsterdam
+RandomizedDelaySec=2min
+Persistent=true
+Unit=stats-collector@energycharts.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-imf.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: IMF WEO monthly (new rounds in April and October)
+
+[Timer]
+OnCalendar=*-*-01 04:05:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@imf.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-owid.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: Our World in Data monthly (Maddison, energy dataset)
+
+[Timer]
+OnCalendar=*-*-02 04:35:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@owid.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-rijksfinancien.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: Rijksfinanciën monthly (budget tables)
+
+[Timer]
+OnCalendar=*-*-03 05:05:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@rijksfinancien.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-usaspending.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: USAspending weekly (Sunday)
+
+[Timer]
+OnCalendar=Sun *-*-* 06:10:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@usaspending.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-ustreasury.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: US Treasury Fiscal Data daily (after the US afternoon release)
+
+[Timer]
+OnCalendar=*-*-* 23:25:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@ustreasury.service
+
+[Install]
+WantedBy=timers.target

deploy/stats-collector/stats-worldbank.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=/stats collector: World Bank WDI weekly (Monday)
+
+[Timer]
+OnCalendar=Mon *-*-* 04:15:00 Europe/Amsterdam
+RandomizedDelaySec=10min
+Persistent=true
+Unit=stats-collector@worldbank.service
+
+[Install]
+WantedBy=timers.target

drizzle/20261025210436_stats_step2/migration.sql

@@ -0,0 +1,119 @@
+CREATE TABLE "stats"."ec_crossborder" (
+	"country" text,
+	"neighbour" text,
+	"ts" timestamp with time zone,
+	"gw" double precision,
+	CONSTRAINT "ec_crossborder_pkey" PRIMARY KEY("country","neighbour","ts")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."ec_dayahead_price" (
+	"bzn" text,
+	"ts" timestamp with time zone,
+	"eur_mwh" double precision,
+	CONSTRAINT "ec_dayahead_price_pkey" PRIMARY KEY("bzn","ts")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."imf_weo" (
+	"vintage" text,
+	"indicator" text,
+	"country_iso3" text,
+	"year" integer,
+	"value" double precision,
+	"is_forecast" boolean NOT NULL,
+	CONSTRAINT "imf_weo_pkey" PRIMARY KEY("vintage","indicator","country_iso3","year")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."owid_energy" (
+	"country" text,
+	"iso3" text,
+	"year" integer,
+	"values" jsonb NOT NULL,
+	CONSTRAINT "owid_energy_pkey" PRIMARY KEY("country","year")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."owid_maddison" (
+	"entity" text,
+	"iso3" text,
+	"year" integer,
+	"gdp_pc" double precision,
+	CONSTRAINT "owid_maddison_pkey" PRIMARY KEY("entity","year")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."rf_budget_table" (
+	"year" integer,
+	"phase" text,
+	"row_key" text,
+	"vuo" char(1) NOT NULL,
+	"chapter" text NOT NULL,
+	"chapter_name" text NOT NULL,
+	"article" text,
+	"article_name" text,
+	"detail" jsonb DEFAULT '{}' NOT NULL,
+	"amount_eur_k" double precision,
+	CONSTRAINT "rf_budget_table_pkey" PRIMARY KEY("year","phase","row_key")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."usas_agency_fy" (
+	"toptier_code" text,
+	"fiscal_year" integer,
+	"agency_name" text NOT NULL,
+	"abbreviation" text,
+	"budgetary_resources" double precision,
+	"obligated" double precision,
+	"outlayed" double precision,
+	CONSTRAINT "usas_agency_fy_pkey" PRIMARY KEY("toptier_code","fiscal_year")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."ust_debt_daily" (
+	"record_date" date PRIMARY KEY,
+	"debt_held_public" double precision,
+	"intragov_holdings" double precision,
+	"total_debt" double precision
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."ust_mts_outlays" (
+	"record_date" date,
+	"classification_id" text,
+	"parent_id" text,
+	"description" text NOT NULL,
+	"level" integer,
+	"print_order" integer,
+	"net_month" double precision,
+	"net_fytd" double precision,
+	"gross_month" double precision,
+	CONSTRAINT "ust_mts_outlays_pkey" PRIMARY KEY("record_date","classification_id")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."ust_mts_receipts" (
+	"record_date" date,
+	"classification_id" text,
+	"parent_id" text,
+	"description" text NOT NULL,
+	"level" integer,
+	"print_order" integer,
+	"net_month" double precision,
+	"net_fytd" double precision,
+	"gross_month" double precision,
+	CONSTRAINT "ust_mts_receipts_pkey" PRIMARY KEY("record_date","classification_id")
+);
+--> statement-breakpoint
+CREATE TABLE "stats"."wb_indicator" (
+	"indicator_id" text,
+	"country_iso3" text,
+	"year" integer,
+	"value" double precision,
+	CONSTRAINT "wb_indicator_pkey" PRIMARY KEY("indicator_id","country_iso3","year")
+);
+--> statement-breakpoint
+CREATE INDEX "rf_budget_table_chapter_idx" ON "stats"."rf_budget_table" ("year","vuo","chapter");--> statement-breakpoint
+-- Hand-written below this line (not generated by drizzle-kit).
+-- Same grants as stats_core (default privileges already cover new tables; kept idempotent).
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA stats TO personal_app;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
+    GRANT SELECT ON ALL TABLES IN SCHEMA stats TO personal_ro;
+  END IF;
+END $$;

scripts/deploy-preflight.ts

@@ -177,7 +177,7 @@ try {
 		throw new Error(`groceries schema grants are wrong: ${JSON.stringify(groceries)}`);
 	// Public /stats section: open data, so the app role (and the host-side stats collector, which
 	// uses it) reads and writes, and personal_ro may read. Bump the count with every new stats table.
-	const STATS_TABLES = 12;
+	const STATS_TABLES = 23;
 	const [stats] = await client`
 		select
 			count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,

scripts/stats-fetch.ts

@@ -6,12 +6,22 @@
 //   bun scripts/stats-fetch.ts eurostat [--only=nrg_pc_204]
 //   bun scripts/stats-fetch.ts fred [--only=DCOILBRENTEU]
 //   bun scripts/stats-fetch.ts bis
+//   bun scripts/stats-fetch.ts worldbank|imf|owid [--only=…]
+//   bun scripts/stats-fetch.ts energycharts|rijksfinancien|ustreasury [--backfill]
+//   bun scripts/stats-fetch.ts usaspending
 //   bun scripts/stats-fetch.ts normalise-cbs     rebuild CBS chart series from the raw rows
 import { ensureMeta, sql } from './stats/common';
 import { cbs, normaliseCbs } from './stats/cbs';
 import { eurostat } from './stats/eurostat';
 import { fred } from './stats/fred';
 import { bis } from './stats/bis';
+import { worldbank } from './stats/worldbank';
+import { imf } from './stats/imf';
+import { owid } from './stats/owid';
+import { energycharts } from './stats/energycharts';
+import { rijksfinancien } from './stats/rijksfinancien';
+import { ustreasury } from './stats/ustreasury';
+import { usaspending } from './stats/usaspending';
 
 const [source, ...args] = process.argv.slice(2);
 const flag = (name: string) => args.includes(`--${name}`);
@@ -36,8 +46,31 @@ try {
 		case 'bis':
 			await bis({ job });
 			break;
+		case 'worldbank':
+			await worldbank({ only: value('only'), job });
+			break;
+		case 'imf':
+			await imf({ only: value('only'), job });
+			break;
+		case 'owid':
+			await owid({ only: value('only'), job });
+			break;
+		case 'energycharts':
+			await energycharts({ backfill: flag('backfill'), job });
+			break;
+		case 'rijksfinancien':
+			await rijksfinancien({ backfill: flag('backfill'), job });
+			break;
+		case 'ustreasury':
+			await ustreasury({ backfill: flag('backfill'), job });
+			break;
+		case 'usaspending':
+			await usaspending({ job });
+			break;
 		default:
-			throw new Error(`Unknown source "${source}" (cbs, eurostat, fred, bis, normalise-cbs)`);
+			throw new Error(
+				`Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, normalise-cbs)`
+			);
 	}
 } finally {
 	await sql.end({ timeout: 5 });

scripts/stats/common.ts

@@ -23,7 +23,15 @@ const hostGapMs: Record<string, number> = {
 	'sdmx.oecd.org': 3100,
 	'fred.stlouisfed.org': 1000,
 	'ec.europa.eu': 500,
-	'stats.bis.org': 1000
+	'stats.bis.org': 1000,
+	'api.worldbank.org': 1000,
+	'www.imf.org': 1500,
+	'ourworldindata.org': 2000,
+	'owid-public.owid.io': 2000,
+	'api.energy-charts.info': 2000,
+	'www.rijksfinancien.nl': 1000,
+	'api.fiscaldata.treasury.gov': 1000,
+	'api.usaspending.gov': 500
 };
 const lastHit = new Map<string, number>();
 
@@ -127,6 +135,64 @@ export const SOURCES = [
 		licence: 'Free reuse with attribution',
 		key_required: false,
 		attribution: 'Source: BIS, central bank policy rates (WS_CBPOL)'
+	},
+	{
+		source_id: 'worldbank',
+		name: 'World Bank World Development Indicators',
+		base_url: 'https://api.worldbank.org/v2',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: World Bank, World Development Indicators, CC BY 4.0'
+	},
+	{
+		source_id: 'imf',
+		name: 'IMF World Economic Outlook (DataMapper)',
+		base_url: 'https://www.imf.org/external/datamapper/api/v1',
+		licence: 'IMF copyright, free reuse with attribution',
+		key_required: false,
+		attribution: 'Source: IMF, World Economic Outlook (DataMapper)'
+	},
+	{
+		source_id: 'owid',
+		name: 'Our World in Data',
+		base_url: 'https://ourworldindata.org',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution:
+			'Source: Our World in Data (Maddison Project Database 2023; Energy Institute, Ember), CC BY 4.0'
+	},
+	{
+		source_id: 'energycharts',
+		name: 'Energy-Charts (Fraunhofer ISE)',
+		base_url: 'https://api.energy-charts.info',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: Energy-Charts.info (Fraunhofer ISE; ENTSO-E, SMARD), CC BY 4.0'
+	},
+	{
+		source_id: 'rijksfinancien',
+		name: 'Rijksfinanciën (Ministerie van Financiën)',
+		base_url: 'https://www.rijksfinancien.nl/open-data/api',
+		licence: 'CC0',
+		key_required: false,
+		attribution: 'Source: Rijksfinanciën open data, Ministerie van Financiën (CC0)'
+	},
+	{
+		source_id: 'ustreasury',
+		name: 'US Treasury Fiscal Data',
+		base_url: 'https://api.fiscaldata.treasury.gov/services/api/fiscal_service',
+		licence: 'Public domain (US government work)',
+		key_required: false,
+		attribution:
+			'Source: U.S. Treasury, Fiscal Data (Monthly Treasury Statement, Debt to the Penny)'
+	},
+	{
+		source_id: 'usaspending',
+		name: 'USAspending.gov',
+		base_url: 'https://api.usaspending.gov/api/v2',
+		licence: 'Public domain (US government work)',
+		key_required: false,
+		attribution: 'Source: USAspending.gov (U.S. Department of the Treasury)'
 	}
 ] as const;
 
@@ -152,6 +218,8 @@ export const REGIONS: { region_code: string; name: string; kind: string; parent:
 		{ region_code: 'GBR', name: 'United Kingdom', kind: 'country', parent: 'WLD' },
 		{ region_code: 'JPN', name: 'Japan', kind: 'country', parent: 'WLD' },
 		{ region_code: 'IND', name: 'India', kind: 'country', parent: 'WLD' },
+		{ region_code: 'BRA', name: 'Brazil', kind: 'country', parent: 'WLD' },
+		{ region_code: 'RUS', name: 'Russia', kind: 'country', parent: 'WLD' },
 		{ region_code: 'CR28', name: 'Oost-Zuid-Holland (COROP)', kind: 'corop', parent: 'NLD' },
 		{ region_code: 'GM0513', name: 'Gouda', kind: 'municipality', parent: 'CR28' },
 		{ region_code: 'GM0627', name: 'Waddinxveen', kind: 'municipality', parent: 'CR28' },
@@ -162,6 +230,46 @@ export const REGIONS: { region_code: string; name: string; kind: string; parent:
 		{ region_code: 'GM0632', name: 'Woerden', kind: 'municipality', parent: 'NLD' }
 	];
 
+/** Countries compared on /stats/countries (ISO3), in display order. */
+export const COMPARE_ISO3 = [
+	'NLD',
+	'USA',
+	'KOR',
+	'CHN',
+	'CUW',
+	'DEU',
+	'FRA',
+	'BEL',
+	'GBR',
+	'JPN',
+	'IND',
+	'BRA',
+	'RUS'
+];
+
+/**
+ * Batch insert inside an open transaction: one jsonb parameter per chunk, so big tables load fast
+ * without hitting the bind-parameter limit. Row keys are column names (snake_case).
+ */
+export async function insertChunks(
+	tx: postgres.TransactionSql,
+	table: string,
+	rows: Record<string, unknown>[],
+	conflict = '',
+	size = 20000
+) {
+	if (!rows.length) return;
+	const cols = Object.keys(rows[0])
+		.map((k) => `"${k}"`)
+		.join(', ');
+	for (let i = 0; i < rows.length; i += size) {
+		await tx.unsafe(
+			`insert into stats.${table} (${cols}) select ${cols} from jsonb_populate_recordset(null::stats.${table}, $1::jsonb) ${conflict}`,
+			[JSON.stringify(rows.slice(i, i + size))] as never
+		);
+	}
+}
+
 /** Gouda and its house-hunting ring (CBS gemeente codes). */
 export const GOUDA_RING = ['GM0513', 'GM0627', 'GM1901', 'GM1892', 'GM1931', 'GM0484', 'GM0632'];
 

scripts/stats/energycharts.ts

@@ -0,0 +1,120 @@
+// A10 Energy-Charts (Fraunhofer ISE): NL day-ahead prices and NL cross-border physical flows.
+// Raw rows keep the API resolution (hourly until Oct 2025, 15-minute after); the normalised
+// layer holds daily / monthly averages (prices) and monthly net TWh per neighbour (flows).
+import { getJson, insertChunks, log, run, sql } from './common';
+
+const BASE = 'https://api.energy-charts.info';
+const BACKFILL_FROM = '2015-01-01';
+
+const ymd = (d: Date) => d.toISOString().slice(0, 10);
+function monthChunks(from: string, to: Date) {
+	const out: [string, string][] = [];
+	const d = new Date(`${from}T00:00:00Z`);
+	while (d < to) {
+		const start = ymd(d);
+		d.setUTCMonth(d.getUTCMonth() + 1);
+		const end = new Date(d.getTime() - 86_400_000);
+		out.push([start, ymd(end < to ? end : to)]);
+	}
+	return out;
+}
+
+type PriceBody = { unix_seconds?: number[]; price?: (number | null)[] };
+type CbetBody = {
+	unix_seconds?: number[];
+	countries?: { name: string; data: (number | null)[] }[];
+};
+
+async function pullRange(start: string, end: string) {
+	let rows = 0;
+	const price = await getJson<PriceBody>(`${BASE}/price?bzn=NL&start=${start}&end=${end}`);
+	const p = (price.unix_seconds ?? []).map((t, i) => ({
+		bzn: 'NL',
+		ts: new Date(t * 1000).toISOString(),
+		eur_mwh: price.price?.[i] ?? null
+	}));
+	const cbet = await getJson<CbetBody>(`${BASE}/cbet?country=nl&start=${start}&end=${end}`);
+	const c = (cbet.countries ?? []).flatMap((n) =>
+		(cbet.unix_seconds ?? []).map((t, i) => ({
+			country: 'nl',
+			neighbour: n.name,
+			ts: new Date(t * 1000).toISOString(),
+			gw: n.data[i] ?? null
+		}))
+	);
+	await sql.begin(async (tx) => {
+		await insertChunks(
+			tx,
+			'ec_dayahead_price',
+			p,
+			'on conflict (bzn, ts) do update set eur_mwh = excluded.eur_mwh'
+		);
+		await insertChunks(
+			tx,
+			'ec_crossborder',
+			c,
+			'on conflict (country, neighbour, ts) do update set gw = excluded.gw'
+		);
+	});
+	rows += p.length + c.length;
+	return rows;
+}
+
+/** Rebuild the chart series from the raw tables (cheap: a few hundred thousand rows). */
+export async function normaliseEnergyCharts() {
+	const source = { url: 'https://www.energy-charts.info/charts/price_spot_market/chart.htm?c=NL' };
+	await sql.begin(async (tx) => {
+		const series = [
+			['ec:price:NL:D', 'Day-ahead electricity price, daily average', 'EUR/MWh', 'D'],
+			['ec:price:NL:M', 'Day-ahead electricity price, monthly average', 'EUR/MWh', 'M']
+		];
+		for (const [id, title, unit, freq] of series)
+			await tx`insert into stats.series (series_id, source_id, source_code, title, unit, freq, region_code, theme, meta)
+				values (${id}, 'energycharts', 'price/NL', ${title}, ${unit}, ${freq}, 'NLD', 'energy', ${tx.json(source)})
+				on conflict (series_id) do update set title = excluded.title, unit = excluded.unit, meta = excluded.meta`;
+		await tx`insert into stats.series (series_id, source_id, source_code, title, unit, freq, region_code, theme, meta)
+			select 'ec:cbet:nl:' || lower(replace(neighbour, ' ', '-')) || ':M', 'energycharts', 'cbet/nl/' || neighbour,
+				'Net physical electricity import from ' || neighbour, 'TWh', 'M', 'NLD', 'energy',
+				${tx.json({ url: 'https://www.energy-charts.info/charts/import_export/chart.htm?c=NL', sign: 'positive = import into NL' })}
+			from (select distinct neighbour from stats.ec_crossborder where country = 'nl') n
+			on conflict (series_id) do update set title = excluded.title, meta = excluded.meta`;
+		await tx`delete from stats.obs where series_id like 'ec:%'`;
+		await tx`insert into stats.obs (series_id, period, value)
+			select 'ec:price:NL:D', (ts at time zone 'Europe/Amsterdam')::date, avg(eur_mwh)
+			from stats.ec_dayahead_price where bzn = 'NL' and eur_mwh is not null group by 2`;
+		await tx`insert into stats.obs (series_id, period, value)
+			select 'ec:price:NL:M', date_trunc('month', ts at time zone 'Europe/Amsterdam')::date, avg(eur_mwh)
+			from stats.ec_dayahead_price where bzn = 'NL' and eur_mwh is not null group by 2`;
+		// Average GW over the month × hours in the month / 1000 = TWh (robust to the resolution change).
+		await tx`insert into stats.obs (series_id, period, value)
+			select 'ec:cbet:nl:' || lower(replace(neighbour, ' ', '-')) || ':M', m,
+				avg(gw) * extract(epoch from (m + interval '1 month') - m) / 3600 / 1000
+			from (select neighbour, gw, date_trunc('month', ts at time zone 'Europe/Amsterdam')::date as m
+				from stats.ec_crossborder where country = 'nl' and gw is not null) x
+			group by neighbour, m`;
+	});
+	const [{ n }] = await sql`select count(*)::int as n from stats.obs where series_id like 'ec:%'`;
+	return n as number;
+}
+
+export async function energycharts(opts: { backfill?: boolean; job: string }) {
+	await run('energycharts', opts.job, async (note) => {
+		const now = new Date();
+		const tomorrow = new Date(now.getTime() + 86_400_000);
+		const chunks = opts.backfill
+			? monthChunks(BACKFILL_FROM, tomorrow)
+			: [[ymd(new Date(now.getTime() - 3 * 86_400_000)), ymd(tomorrow)] as [string, string]];
+		let rows = 0;
+		for (const [start, end] of chunks) {
+			try {
+				rows += await pullRange(start, end);
+				if (opts.backfill) log(`energycharts ${start}..${end}: ${rows} rows so far`);
+			} catch (error) {
+				note(`${start}..${end}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		const obs = await normaliseEnergyCharts();
+		log(`energycharts: ${rows} raw rows, ${obs} normalised obs`);
+		return rows;
+	});
+}

scripts/stats/imf.ts

@@ -0,0 +1,128 @@
+// A4 IMF World Economic Outlook via DataMapper. The country filter in the path is ignored, so
+// each call returns every country; keep the compared ones. Each WEO round is kept as a vintage
+// (April / October) so forecasts can later be compared with outcomes.
+import {
+	COMPARE_ISO3,
+	getJson,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+export const IMF_INDICATORS: { id: string; title: string; unit: string; theme: string }[] = [
+	{ id: 'NGDP_RPCH', title: 'Real GDP growth', unit: '%', theme: 'countries' },
+	{ id: 'NGDPDPC', title: 'GDP per capita', unit: 'current USD', theme: 'countries' },
+	{
+		id: 'PPPPC',
+		title: 'GDP per capita, PPP',
+		unit: 'current international $',
+		theme: 'countries'
+	},
+	{ id: 'PCPIPCH', title: 'Inflation (average consumer prices)', unit: '%', theme: 'countries' },
+	{ id: 'LUR', title: 'Unemployment rate', unit: '%', theme: 'countries' },
+	{ id: 'GGXWDG_NGDP', title: 'General government gross debt', unit: '% of GDP', theme: 'budget' },
+	{
+		id: 'GGXCNL_NGDP',
+		title: 'General government net lending/borrowing',
+		unit: '% of GDP',
+		theme: 'budget'
+	},
+	{ id: 'GGR_NGDP', title: 'General government revenue', unit: '% of GDP', theme: 'budget' },
+	{
+		id: 'GGX_NGDP',
+		title: 'General government total expenditure',
+		unit: '% of GDP',
+		theme: 'budget'
+	}
+];
+// IMF aggregate codes kept besides the countries.
+const AGGREGATES: Record<string, string> = { EU: 'EU27', WEOWORLD: 'WLD' };
+
+/** WEO round in force on a date: October and April releases. */
+export function currentVintage(now = new Date()) {
+	const y = now.getUTCFullYear();
+	const m = now.getUTCMonth() + 1;
+	return m >= 10 ? `${y}-10` : m >= 4 ? `${y}-04` : `${y - 1}-10`;
+}
+
+export async function imf(opts: { only?: string; job: string }) {
+	const vintage = currentVintage();
+	const vintageYear = Number(vintage.slice(0, 4));
+	await run('imf', opts.job, async (note) => {
+		let total = 0;
+		for (const ind of IMF_INDICATORS) {
+			if (opts.only && !opts.only.split(',').includes(ind.id)) continue;
+			try {
+				const body = await getJson<{
+					values?: Record<string, Record<string, Record<string, number>>>;
+				}>(`https://www.imf.org/external/datamapper/api/v1/${ind.id}`);
+				const byCountry = body.values?.[ind.id] ?? {};
+				const rows: {
+					vintage: string;
+					indicator: string;
+					country_iso3: string;
+					year: number;
+					value: number;
+					is_forecast: boolean;
+				}[] = [];
+				for (const [code, years] of Object.entries(byCountry)) {
+					const region = COMPARE_ISO3.includes(code) ? code : AGGREGATES[code];
+					if (!region) continue;
+					for (const [y, v] of Object.entries(years))
+						if (v != null)
+							rows.push({
+								vintage,
+								indicator: ind.id,
+								country_iso3: region,
+								year: Number(y),
+								value: v,
+								is_forecast: Number(y) >= vintageYear
+							});
+				}
+				if (!rows.length) throw new Error('no rows');
+				await sql.begin(async (tx) => {
+					await tx`delete from stats.imf_weo where vintage = ${vintage} and indicator = ${ind.id}`;
+					await insertChunks(tx, 'imf_weo', rows);
+				});
+				const series: SeriesRow[] = [];
+				const obs: ObsRow[] = [];
+				for (const region of new Set(rows.map((r) => r.country_iso3))) {
+					const id = `imf:${ind.id}:${region}`;
+					series.push({
+						series_id: id,
+						source_id: 'imf',
+						source_code: `${ind.id}/${region}`,
+						title: ind.title,
+						unit: ind.unit,
+						freq: 'A',
+						region_code: region,
+						theme: ind.theme,
+						meta: {
+							vintage,
+							forecastFrom: vintageYear,
+							url: `https://www.imf.org/external/datamapper/${ind.id}`
+						}
+					});
+					for (const r of rows)
+						if (r.country_iso3 === region)
+							obs.push({
+								series_id: id,
+								period: `${r.year}-01-01`,
+								value: r.value,
+								status: r.is_forecast ? 'forecast' : null
+							});
+				}
+				await writeSeries(series, obs);
+				log(`imf ${ind.id}: ${rows.length} rows (vintage ${vintage})`);
+				total += rows.length;
+			} catch (error) {
+				note(`${ind.id}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		return total;
+	});
+}

scripts/stats/owid.ts

@@ -0,0 +1,218 @@
+// A9 Our World in Data: Maddison GDP per capita (NL back to 1348) and the OWID energy dataset
+// (energy mix per country). Both are full CSV downloads, replaced on every run.
+import {
+	COMPARE_ISO3,
+	getText,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const MADDISON_URL =
+	'https://ourworldindata.org/grapher/gdp-per-capita-maddison-project-database.csv?csvType=full&useColumnShortNames=true';
+const ENERGY_URL = 'https://owid-public.owid.io/data/energy/owid-energy-data.csv';
+
+/** OWID energy columns kept (raw) and the ones normalised into series for compared countries. */
+export const ENERGY_COLUMNS: Record<string, { title: string; unit: string; series?: boolean }> = {
+	population: { title: 'Population', unit: 'people' },
+	energy_per_capita: { title: 'Primary energy per person', unit: 'kWh', series: true },
+	primary_energy_consumption: { title: 'Primary energy consumption', unit: 'TWh', series: true },
+	per_capita_electricity: { title: 'Electricity generation per person', unit: 'kWh', series: true },
+	electricity_generation: { title: 'Electricity generation', unit: 'TWh' },
+	carbon_intensity_elec: {
+		title: 'Carbon intensity of electricity',
+		unit: 'gCO₂/kWh',
+		series: true
+	},
+	fossil_share_energy: { title: 'Fossil fuels, share of primary energy', unit: '%', series: true },
+	renewables_share_energy: {
+		title: 'Renewables, share of primary energy',
+		unit: '%',
+		series: true
+	},
+	low_carbon_share_elec: { title: 'Low-carbon share of electricity', unit: '%', series: true },
+	coal_share_elec: { title: 'Coal, share of electricity', unit: '%', series: true },
+	gas_share_elec: { title: 'Gas, share of electricity', unit: '%', series: true },
+	oil_share_elec: { title: 'Oil, share of electricity', unit: '%', series: true },
+	nuclear_share_elec: { title: 'Nuclear, share of electricity', unit: '%', series: true },
+	hydro_share_elec: { title: 'Hydro, share of electricity', unit: '%', series: true },
+	wind_share_elec: { title: 'Wind, share of electricity', unit: '%', series: true },
+	solar_share_elec: { title: 'Solar, share of electricity', unit: '%', series: true },
+	biofuel_share_elec: { title: 'Bioenergy, share of electricity', unit: '%', series: true },
+	other_renewables_share_elec_exc_biofuel: {
+		title: 'Other renewables, share of electricity',
+		unit: '%'
+	},
+	coal_consumption: { title: 'Coal consumption', unit: 'TWh' },
+	oil_consumption: { title: 'Oil consumption', unit: 'TWh' },
+	gas_consumption: { title: 'Gas consumption', unit: 'TWh' },
+	nuclear_consumption: { title: 'Nuclear consumption', unit: 'TWh' },
+	renewables_consumption: { title: 'Renewables consumption', unit: 'TWh' },
+	net_elec_imports_share_demand: { title: 'Net electricity imports, share of demand', unit: '%' }
+};
+
+/** Minimal CSV parser (quoted fields, commas inside quotes). */
+export function parseCsv(text: string): string[][] {
+	const out: string[][] = [];
+	let row: string[] = [];
+	let field = '';
+	let quoted = false;
+	for (let i = 0; i < text.length; i++) {
+		const ch = text[i];
+		if (quoted) {
+			if (ch === '"' && text[i + 1] === '"') {
+				field += '"';
+				i++;
+			} else if (ch === '"') quoted = false;
+			else field += ch;
+		} else if (ch === '"') quoted = true;
+		else if (ch === ',') {
+			row.push(field);
+			field = '';
+		} else if (ch === '\n' || ch === '\r') {
+			if (ch === '\r' && text[i + 1] === '\n') i++;
+			row.push(field);
+			out.push(row);
+			row = [];
+			field = '';
+		} else field += ch;
+	}
+	if (field || row.length) {
+		row.push(field);
+		out.push(row);
+	}
+	return out;
+}
+
+const regionOf = (iso3: string, entity: string) =>
+	COMPARE_ISO3.includes(iso3)
+		? iso3
+		: entity === 'World'
+			? 'WLD'
+			: entity === 'European Union (27)'
+				? 'EU27'
+				: null;
+
+export async function owid(opts: { only?: string; job: string }) {
+	await run('owid', opts.job, async (note) => {
+		let total = 0;
+		if (!opts.only || opts.only.includes('maddison')) {
+			try {
+				const [header, ...lines] = parseCsv(await getText(MADDISON_URL));
+				const col = (n: string) => header.indexOf(n);
+				const [e, c, y, g] = [col('entity'), col('code'), col('year'), col('gdp_per_capita')];
+				const rows = lines
+					.filter((l) => l.length > g && l[g] !== '')
+					.map((l) => ({
+						entity: l[e],
+						iso3: l[c] || null,
+						year: Number(l[y]),
+						gdp_pc: Number(l[g])
+					}));
+				if (rows.length < 1000) throw new Error(`only ${rows.length} rows`);
+				await sql.begin(async (tx) => {
+					await tx`delete from stats.owid_maddison`;
+					await insertChunks(tx, 'owid_maddison', rows);
+				});
+				const series: SeriesRow[] = [];
+				const obs: ObsRow[] = [];
+				const seen = new Set<string>();
+				for (const r of rows) {
+					const region = regionOf(r.iso3 ?? '', r.entity);
+					if (!region) continue;
+					const id = `owid:maddison:${region}`;
+					if (!seen.has(id)) {
+						seen.add(id);
+						series.push({
+							series_id: id,
+							source_id: 'owid',
+							source_code: `maddison/${r.entity}`,
+							title: 'GDP per capita (Maddison)',
+							unit: 'international-$ at 2011 prices',
+							freq: 'A',
+							region_code: region,
+							theme: 'countries',
+							meta: {
+								url: 'https://ourworldindata.org/grapher/gdp-per-capita-maddison-project-database'
+							}
+						});
+					}
+					obs.push({
+						series_id: id,
+						period: `${String(r.year).padStart(4, '0')}-01-01`,
+						value: r.gdp_pc,
+						status: null
+					});
+				}
+				await writeSeries(series, obs);
+				log(`owid maddison: ${rows.length} rows`);
+				total += rows.length;
+			} catch (error) {
+				note(`maddison: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		if (!opts.only || opts.only.includes('energy')) {
+			try {
+				const [header, ...lines] = parseCsv(await getText(ENERGY_URL));
+				const idx = Object.fromEntries(header.map((h, i) => [h, i]));
+				const keep = Object.keys(ENERGY_COLUMNS).filter((k) => k in idx);
+				const rows = lines
+					.filter((l) => l.length >= header.length - 1 && l[idx.country])
+					.map((l) => {
+						const values: Record<string, number> = {};
+						for (const k of keep)
+							if (l[idx[k]] !== '' && l[idx[k]] != null) values[k] = Number(l[idx[k]]);
+						return {
+							country: l[idx.country],
+							iso3: l[idx.iso_code] || null,
+							year: Number(l[idx.year]),
+							values
+						};
+					})
+					.filter((r) => Object.keys(r.values).length > 0);
+				if (rows.length < 1000) throw new Error(`only ${rows.length} rows`);
+				await sql.begin(async (tx) => {
+					await tx`delete from stats.owid_energy`;
+					await insertChunks(tx, 'owid_energy', rows);
+				});
+				const series: SeriesRow[] = [];
+				const obs: ObsRow[] = [];
+				const seen = new Set<string>();
+				for (const r of rows) {
+					const region = regionOf(r.iso3 ?? '', r.country);
+					if (!region) continue;
+					for (const [k, v] of Object.entries(r.values)) {
+						const meta = ENERGY_COLUMNS[k];
+						if (!meta.series) continue;
+						const id = `owid:energy:${k}:${region}`;
+						if (!seen.has(id)) {
+							seen.add(id);
+							series.push({
+								series_id: id,
+								source_id: 'owid',
+								source_code: `energy/${k}/${r.country}`,
+								title: meta.title,
+								unit: meta.unit,
+								freq: 'A',
+								region_code: region,
+								theme: 'energy',
+								meta: { url: 'https://github.com/owid/energy-data' }
+							});
+						}
+						obs.push({ series_id: id, period: `${r.year}-01-01`, value: v, status: null });
+					}
+				}
+				await writeSeries(series, obs);
+				log(`owid energy: ${rows.length} rows`);
+				total += rows.length;
+			} catch (error) {
+				note(`energy: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		return total;
+	});
+}

scripts/stats/rijksfinancien.ts

@@ -0,0 +1,163 @@
+// A13 Rijksfinanciën: the central-government budget tables (begroting OWB, suppletoire
+// begrotingen O1/O2, jaarverslag JV) per year, down to regeling level. Open data starts in 2015.
+// Normalised: annual totals per chapter and overall, from the most final phase available.
+import {
+	getJson,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const FIRST_YEAR = 2015;
+const PHASES = ['OWB', 'O1', 'O2', 'JV'] as const;
+const PHASE_RANK: Record<string, number> = { OWB: 1, O1: 2, O2: 3, JV: 4 };
+
+type RfRow = {
+	index: string;
+	jaar: number;
+	fase: string;
+	hoofdstuk_naam: string;
+	hoofdstuk_nummer: string;
+	artikel_naam: string | null;
+	artikel_nummer: string | null;
+	vuo: string;
+	bedrag: number | null;
+	[key: string]: unknown;
+};
+
+export async function rijksfinancien(opts: { backfill?: boolean; job: string }) {
+	await run('rijksfinancien', opts.job, async (note) => {
+		const now = new Date().getUTCFullYear();
+		// Refresh: last year (JV lands in May) up to next year (OWB on Prinsjesdag).
+		const years = [];
+		for (let y = opts.backfill ? FIRST_YEAR : now - 1; y <= now + 1; y++) years.push(y);
+		let total = 0;
+		for (const year of years) {
+			for (const phase of PHASES) {
+				try {
+					const data = await getJson<RfRow[] | Record<string, unknown>>(
+						`https://www.rijksfinancien.nl/open-data/api/json/budgettaire_tabellen?year=${year}&phase=${phase}`
+					);
+					const list = Array.isArray(data) ? data : [];
+					if (!list.length) continue;
+					const rows = list.map((r, i) => {
+						const {
+							index,
+							jaar,
+							fase,
+							hoofdstuk_naam,
+							hoofdstuk_nummer,
+							artikel_naam,
+							artikel_nummer,
+							vuo,
+							bedrag,
+							...detail
+						} = r;
+						void jaar;
+						void fase;
+						return {
+							year,
+							phase,
+							row_key: index ?? `row-${i}`,
+							vuo: (vuo || '?').slice(0, 1),
+							chapter: hoofdstuk_nummer ?? '?',
+							chapter_name: hoofdstuk_naam ?? '',
+							article: artikel_nummer,
+							article_name: artikel_naam,
+							detail,
+							amount_eur_k: bedrag
+						};
+					});
+					await sql.begin(async (tx) => {
+						await tx`delete from stats.rf_budget_table where year = ${year} and phase = ${phase}`;
+						await insertChunks(tx, 'rf_budget_table', rows, 'on conflict do nothing');
+					});
+					log(`rijksfinancien ${year} ${phase}: ${rows.length} rows`);
+					total += rows.length;
+				} catch (error) {
+					const msg = error instanceof Error ? error.message : String(error);
+					if (!/HTTP 404/.test(msg)) note(`${year} ${phase}: ${msg}`);
+				}
+			}
+		}
+		await normaliseRf();
+		return total;
+	});
+}
+
+/** Annual U (expenditure) and O (revenue) per chapter + totals, from the most final phase. */
+export async function normaliseRf() {
+	const agg = await sql<
+		{
+			year: number;
+			phase: string;
+			vuo: string;
+			chapter: string;
+			chapter_name: string;
+			eur_bn: number;
+		}[]
+	>`
+		select year, phase, vuo, chapter, max(chapter_name) as chapter_name, sum(amount_eur_k) / 1e6 as eur_bn
+		from stats.rf_budget_table where vuo in ('U', 'O') group by 1, 2, 3, 4`;
+	const best = new Map<string, (typeof agg)[number]>();
+	for (const r of agg) {
+		const key = `${r.vuo}|${r.chapter}|${r.year}`;
+		const prev = best.get(key);
+		if (!prev || PHASE_RANK[r.phase] > PHASE_RANK[prev.phase]) best.set(key, r);
+	}
+	const series = new Map<string, SeriesRow>();
+	const obs: ObsRow[] = [];
+	const totals = new Map<string, { value: number; phase: string }>();
+	const label = { U: 'expenditure', O: 'revenue' } as Record<string, string>;
+	for (const r of best.values()) {
+		const id = `rf:${r.vuo}:${r.chapter}`;
+		series.set(id, {
+			series_id: id,
+			source_id: 'rijksfinancien',
+			source_code: `${r.vuo}/${r.chapter}`,
+			title: `${r.chapter_name}: ${label[r.vuo]}`,
+			unit: 'EUR bn',
+			freq: 'A',
+			region_code: 'NLD',
+			theme: 'budget',
+			meta: {
+				chapter: r.chapter,
+				chapterName: r.chapter_name,
+				vuo: r.vuo,
+				url: 'https://www.rijksfinancien.nl/open-data'
+			}
+		});
+		obs.push({
+			series_id: id,
+			period: `${r.year}-01-01`,
+			value: Number(r.eur_bn),
+			status: r.phase
+		});
+		const tk = `${r.vuo}|${r.year}`;
+		const t = totals.get(tk) ?? { value: 0, phase: r.phase };
+		t.value += Number(r.eur_bn);
+		if (PHASE_RANK[r.phase] < PHASE_RANK[t.phase]) t.phase = r.phase;
+		totals.set(tk, t);
+	}
+	for (const [tk, t] of totals) {
+		const [vuo, year] = tk.split('|');
+		const id = `rf:${vuo}:total`;
+		series.set(id, {
+			series_id: id,
+			source_id: 'rijksfinancien',
+			source_code: `${vuo}/total`,
+			title: `Rijksbegroting ${label[vuo]}, all chapters`,
+			unit: 'EUR bn',
+			freq: 'A',
+			region_code: 'NLD',
+			theme: 'budget',
+			meta: { vuo, url: 'https://www.rijksfinancien.nl/open-data' }
+		});
+		obs.push({ series_id: id, period: `${year}-01-01`, value: t.value, status: t.phase });
+	}
+	return writeSeries([...series.values()], obs);
+}

scripts/stats/usaspending.ts

@@ -0,0 +1,86 @@
+// A12 USAspending: budgetary resources, obligations and outlays per top-tier agency and fiscal
+// year (one call for the agency list, one per agency for its history). Small: ~110 agencies.
+import {
+	getJson,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const BASE = 'https://api.usaspending.gov/api/v2';
+type Agency = { toptier_code: string; agency_name: string; abbreviation: string | null };
+type Resources = {
+	agency_data_by_year: {
+		fiscal_year: number;
+		agency_budgetary_resources: number | null;
+		agency_total_obligated: number | null;
+		agency_total_outlayed: number | null;
+	}[];
+};
+
+export async function usaspending(opts: { job: string }) {
+	await run('usaspending', opts.job, async (note) => {
+		const { results } = await getJson<{ results: Agency[] }>(
+			`${BASE}/references/toptier_agencies/`
+		);
+		const rows: Record<string, unknown>[] = [];
+		for (const a of results) {
+			try {
+				const r = await getJson<Resources>(`${BASE}/agency/${a.toptier_code}/budgetary_resources/`);
+				for (const y of r.agency_data_by_year ?? [])
+					rows.push({
+						toptier_code: a.toptier_code,
+						fiscal_year: y.fiscal_year,
+						agency_name: a.agency_name,
+						abbreviation: a.abbreviation,
+						budgetary_resources: y.agency_budgetary_resources,
+						obligated: y.agency_total_obligated,
+						outlayed: y.agency_total_outlayed
+					});
+			} catch (error) {
+				note(`${a.toptier_code}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		if (!rows.length) throw new Error('no rows');
+		await sql.begin(async (tx) => {
+			await tx`delete from stats.usas_agency_fy`;
+			await insertChunks(tx, 'usas_agency_fy', rows);
+		});
+		const series: SeriesRow[] = [];
+		const obs: ObsRow[] = [];
+		for (const a of results) {
+			const id = `usas:outlays:${a.toptier_code}`;
+			const mine = rows.filter((r) => r.toptier_code === a.toptier_code);
+			if (!mine.length) continue;
+			series.push({
+				series_id: id,
+				source_id: 'usaspending',
+				source_code: a.toptier_code,
+				title: `Outlays: ${a.agency_name}`,
+				unit: 'USD bn',
+				freq: 'A',
+				region_code: 'USA',
+				theme: 'budget',
+				meta: {
+					abbreviation: a.abbreviation,
+					url: `https://www.usaspending.gov/agency/${a.toptier_code}`
+				}
+			});
+			for (const r of mine)
+				if (r.outlayed != null)
+					obs.push({
+						series_id: id,
+						period: `${r.fiscal_year}-01-01`,
+						value: Number(r.outlayed) / 1e9,
+						status: null
+					});
+		}
+		await writeSeries(series, obs);
+		log(`usaspending: ${rows.length} agency-years`);
+		return rows.length;
+	});
+}

scripts/stats/ustreasury.ts

@@ -0,0 +1,307 @@
+// A11 US Treasury Fiscal Data: Monthly Treasury Statement receipts (table 4) and outlays
+// (table 5), and the daily Debt to the Penny. Normalised: monthly and fiscal-year receipts by
+// type (customs duties = tariffs), outlays (total and per agency), deficit, and debt.
+import {
+	getJson,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const BASE = 'https://api.fiscaldata.treasury.gov/services/api/fiscal_service';
+type Page = { data: Record<string, string>[]; meta: { 'total-pages': number } };
+
+async function pullAll(endpoint: string, filter: string | null) {
+	const out: Record<string, string>[] = [];
+	for (let page = 1; ; page++) {
+		const q = new URLSearchParams({
+			'page[size]': '10000',
+			'page[number]': String(page),
+			sort: 'record_date'
+		});
+		if (filter) q.set('filter', filter);
+		const body = await getJson<Page>(`${BASE}/${endpoint}?${q}`);
+		out.push(...body.data);
+		if (page >= (body.meta['total-pages'] ?? 1)) break;
+	}
+	return out;
+}
+const num = (v: string | undefined) => (v == null || v === 'null' || v === '' ? null : Number(v));
+const int = (v: string | undefined) => (num(v) == null ? null : Math.trunc(Number(v)));
+
+/** Receipt lines kept as series (MTS table 4 descriptions, normalised to lower case). */
+const RECEIPTS: [string, RegExp, string][] = [
+	['individual', /^total -- individual income taxes$/, 'Individual income taxes'],
+	['corporate', /^(total -- )?corporation income taxes$/, 'Corporation income taxes'],
+	[
+		'social',
+		/^total -- social insurance and retirement receipts$/,
+		'Social insurance and retirement receipts'
+	],
+	['excise', /^total -- excise taxes$/, 'Excise taxes'],
+	['estate', /^(total -- )?estate and gift taxes$/, 'Estate and gift taxes'],
+	['customs', /^(total -- )?customs duties$/, 'Customs duties (tariffs)'],
+	['misc', /^total -- miscellaneous receipts$/, 'Miscellaneous receipts'],
+	['total', /^total -- receipts$/, 'Total receipts']
+];
+const clean = (s: string) =>
+	s
+		.toLowerCase()
+		.replace(/\s+/g, ' ')
+		.replace(/\s*--\s*/g, ' -- ')
+		.trim();
+const slug = (s: string) =>
+	s
+		.toLowerCase()
+		.replace(/&/g, 'and')
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-|-$/g, '');
+/** US fiscal year of a record date (FY2026 = Oct 2025 – Sep 2026). */
+const fiscalYear = (d: string) => Number(d.slice(0, 4)) + (Number(d.slice(5, 7)) >= 10 ? 1 : 0);
+
+export async function ustreasury(opts: { backfill?: boolean; job: string }) {
+	await run('ustreasury', opts.job, async (note) => {
+		const since = new Date(Date.now() - 400 * 86_400_000).toISOString().slice(0, 10);
+		const filter = opts.backfill ? null : `record_date:gte:${since}`;
+		let total = 0;
+		const mts = async (
+			table: 'mts_table_4' | 'mts_table_5',
+			target: string,
+			kind: 'rcpt' | 'outly'
+		) => {
+			const rows = (await pullAll(`v1/accounting/mts/${table}`, filter)).map((r) => ({
+				record_date: r.record_date,
+				classification_id: r.classification_id,
+				parent_id: r.parent_id === 'null' ? null : r.parent_id,
+				description: r.classification_desc,
+				level: int(r.sequence_level_nbr),
+				print_order: int(r.print_order_nbr),
+				net_month: num(r[`current_month_net_${kind}_amt`]),
+				net_fytd: num(r[`current_fytd_net_${kind}_amt`]),
+				gross_month: num(r[`current_month_gross_${kind}_amt`])
+			}));
+			await sql.begin(async (tx) => {
+				await insertChunks(
+					tx,
+					target,
+					rows,
+					`on conflict (record_date, classification_id) do update set
+					parent_id = excluded.parent_id, description = excluded.description, level = excluded.level,
+					print_order = excluded.print_order, net_month = excluded.net_month, net_fytd = excluded.net_fytd,
+					gross_month = excluded.gross_month`
+				);
+			});
+			log(`ustreasury ${table}: ${rows.length} rows`);
+			return rows.length;
+		};
+		for (const [table, target, kind] of [
+			['mts_table_4', 'ust_mts_receipts', 'rcpt'],
+			['mts_table_5', 'ust_mts_outlays', 'outly']
+		] as const) {
+			try {
+				total += await mts(table, target, kind);
+			} catch (error) {
+				note(`${table}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		try {
+			const rows = (await pullAll('v2/accounting/od/debt_to_penny', filter)).map((r) => ({
+				record_date: r.record_date,
+				debt_held_public: num(r.debt_held_public_amt),
+				intragov_holdings: num(r.intragov_hold_amt),
+				total_debt: num(r.tot_pub_debt_out_amt)
+			}));
+			await sql.begin(async (tx) => {
+				await insertChunks(
+					tx,
+					'ust_debt_daily',
+					rows,
+					`on conflict (record_date) do update set
+					debt_held_public = excluded.debt_held_public, intragov_holdings = excluded.intragov_holdings,
+					total_debt = excluded.total_debt`
+				);
+			});
+			log(`ustreasury debt_to_penny: ${rows.length} rows`);
+			total += rows.length;
+		} catch (error) {
+			note(`debt_to_penny: ${error instanceof Error ? error.message : String(error)}`);
+		}
+		await normaliseUst();
+		return total;
+	});
+}
+
+export async function normaliseUst() {
+	const meta = { url: 'https://fiscaldata.treasury.gov/datasets/monthly-treasury-statement/' };
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	const add = (
+		id: string,
+		title: string,
+		unit: string,
+		freq: string,
+		rows: [string, number | null][],
+		m = meta
+	) => {
+		series.push({
+			series_id: id,
+			source_id: 'ustreasury',
+			source_code: id.slice(4),
+			title,
+			unit,
+			freq,
+			region_code: 'USA',
+			theme: 'budget',
+			meta: m
+		});
+		for (const [period, value] of rows)
+			if (value != null) obs.push({ series_id: id, period, value, status: null });
+	};
+	const receipts = await sql<
+		{
+			record_date: string;
+			description: string;
+			net_month: number | null;
+			net_fytd: number | null;
+		}[]
+	>`
+		select record_date::text, description, net_month, net_fytd from stats.ust_mts_receipts order by record_date`;
+	for (const [key, re, title] of RECEIPTS) {
+		const rows = receipts.filter((r) => re.test(clean(r.description)));
+		// Some months repeat a line (sub-totals); keep the first per date.
+		const month = new Map<string, (typeof rows)[number]>();
+		for (const r of rows) if (!month.has(r.record_date)) month.set(r.record_date, r);
+		const list = [...month.values()];
+		add(
+			`ust:receipts:${key}:M`,
+			title,
+			'USD bn',
+			'M',
+			list.map((r) => [
+				`${r.record_date.slice(0, 7)}-01`,
+				r.net_month == null ? null : r.net_month / 1e9
+			])
+		);
+		add(
+			`ust:receipts:${key}:FY`,
+			`${title}, fiscal year`,
+			'USD bn',
+			'A',
+			list
+				.filter((r) => r.record_date.slice(5, 7) === '09')
+				.map((r) => [
+					`${fiscalYear(r.record_date)}-01-01`,
+					r.net_fytd == null ? null : r.net_fytd / 1e9
+				])
+		);
+	}
+	const outlays = await sql<
+		{
+			record_date: string;
+			description: string;
+			level: number | null;
+			net_month: number | null;
+			net_fytd: number | null;
+		}[]
+	>`
+		select record_date::text, description, level, net_month, net_fytd from stats.ust_mts_outlays order by record_date, print_order`;
+	const pick = (re: RegExp) => {
+		const m = new Map<string, (typeof outlays)[number]>();
+		for (const r of outlays)
+			if (re.test(clean(r.description)) && !m.has(r.record_date)) m.set(r.record_date, r);
+		return [...m.values()];
+	};
+	const totalOut = pick(/^total outlays$/);
+	add(
+		'ust:outlays:total:M',
+		'Total outlays',
+		'USD bn',
+		'M',
+		totalOut.map((r) => [
+			`${r.record_date.slice(0, 7)}-01`,
+			r.net_month == null ? null : r.net_month / 1e9
+		])
+	);
+	add(
+		'ust:outlays:total:FY',
+		'Total outlays, fiscal year',
+		'USD bn',
+		'A',
+		totalOut
+			.filter((r) => r.record_date.slice(5, 7) === '09')
+			.map((r) => [
+				`${fiscalYear(r.record_date)}-01-01`,
+				r.net_fytd == null ? null : r.net_fytd / 1e9
+			])
+	);
+	const deficit = pick(/^total surplus \(\+\) or deficit \(-\)$/);
+	add(
+		'ust:deficit:M',
+		'Surplus (+) or deficit (−)',
+		'USD bn',
+		'M',
+		deficit.map((r) => [
+			`${r.record_date.slice(0, 7)}-01`,
+			r.net_month == null ? null : r.net_month / 1e9
+		])
+	);
+	add(
+		'ust:deficit:FY',
+		'Surplus (+) or deficit (−), fiscal year',
+		'USD bn',
+		'A',
+		deficit
+			.filter((r) => r.record_date.slice(5, 7) === '09')
+			.map((r) => [
+				`${fiscalYear(r.record_date)}-01-01`,
+				r.net_fytd == null ? null : r.net_fytd / 1e9
+			])
+	);
+	// Agency totals: "Total--Department of Defense--Military Programs" etc.
+	const agencies = new Map<string, Map<string, number | null>>();
+	for (const r of outlays) {
+		const d = r.description.replace(/\s+/g, ' ').trim();
+		const m = /^Total\s*--\s*(.+)$/.exec(d);
+		if (!m || r.level !== 2 || r.record_date.slice(5, 7) !== '09') continue;
+		const name = m[1].trim();
+		if (!agencies.has(name)) agencies.set(name, new Map());
+		const fy = `${fiscalYear(r.record_date)}-01-01`;
+		if (!agencies.get(name)!.has(fy))
+			agencies.get(name)!.set(fy, r.net_fytd == null ? null : r.net_fytd / 1e9);
+	}
+	for (const [name, years] of agencies)
+		add(
+			`ust:outlays:${slug(name)}:FY`,
+			`Outlays: ${name}, fiscal year`,
+			'USD bn',
+			'A',
+			[...years.entries()],
+			{ ...meta, agency: name }
+		);
+	const debt = await sql<
+		{ d: string; total_debt: number | null; debt_held_public: number | null }[]
+	>`
+		select record_date::text as d, total_debt, debt_held_public from stats.ust_debt_daily order by 1`;
+	const dm = { url: 'https://fiscaldata.treasury.gov/datasets/debt-to-the-penny/' };
+	add(
+		'ust:debt:D',
+		'Total public debt outstanding',
+		'USD tn',
+		'D',
+		debt.map((r) => [r.d, r.total_debt == null ? null : r.total_debt / 1e12]),
+		dm
+	);
+	add(
+		'ust:debt:public:D',
+		'Debt held by the public',
+		'USD tn',
+		'D',
+		debt.map((r) => [r.d, r.debt_held_public == null ? null : r.debt_held_public / 1e12]),
+		dm
+	);
+	return writeSeries(series, obs);
+}

scripts/stats/worldbank.ts

@@ -0,0 +1,111 @@
+// A3 World Bank World Development Indicators: one call per indicator for every compared country
+// (plus EU and World aggregates) → stats.wb_indicator, normalised as `wb:<indicator>:<ISO3>`.
+import {
+	COMPARE_ISO3,
+	getJson,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const COUNTRIES = [...COMPARE_ISO3, 'EUU', 'WLD'];
+// EUU (World Bank code for the EU) is stored as EU27 in the normalised layer.
+const regionOf = (iso3: string) => (iso3 === 'EUU' ? 'EU27' : iso3);
+
+export const WB_INDICATORS: { id: string; title: string; unit: string; theme: string }[] = [
+	{ id: 'NY.GDP.PCAP.KD', title: 'GDP per capita', unit: 'constant 2015 USD', theme: 'countries' },
+	{
+		id: 'NY.GDP.PCAP.PP.KD',
+		title: 'GDP per capita, PPP',
+		unit: 'constant 2021 international $',
+		theme: 'countries'
+	},
+	{ id: 'NY.GDP.MKTP.CD', title: 'GDP', unit: 'current USD', theme: 'countries' },
+	{ id: 'NY.GDP.MKTP.KD.ZG', title: 'GDP growth', unit: '%', theme: 'countries' },
+	{ id: 'FP.CPI.TOTL.ZG', title: 'Inflation, consumer prices', unit: '%', theme: 'countries' },
+	{ id: 'SP.POP.TOTL', title: 'Population', unit: 'people', theme: 'countries' },
+	{ id: 'SM.POP.NETM', title: 'Net migration', unit: 'people', theme: 'countries' },
+	{
+		id: 'NE.TRD.GNFS.ZS',
+		title: 'Trade (exports + imports)',
+		unit: '% of GDP',
+		theme: 'countries'
+	},
+	{
+		id: 'TM.TAX.MRCH.WM.AR.ZS',
+		title: 'Applied tariff rate, weighted mean, all products',
+		unit: '%',
+		theme: 'economy'
+	},
+	{ id: 'EG.IMP.CONS.ZS', title: 'Energy imports, net', unit: '% of energy use', theme: 'energy' },
+	{
+		id: 'GC.REV.XGRT.GD.ZS',
+		title: 'Government revenue, excluding grants',
+		unit: '% of GDP',
+		theme: 'budget'
+	},
+	{ id: 'GC.XPN.TOTL.GD.ZS', title: 'Government expense', unit: '% of GDP', theme: 'budget' },
+	{ id: 'GC.DOD.TOTL.GD.ZS', title: 'Central government debt', unit: '% of GDP', theme: 'budget' }
+];
+
+type WbRow = {
+	countryiso3code: string;
+	date: string;
+	value: number | null;
+	indicator: { id: string };
+};
+
+export async function worldbank(opts: { only?: string; job: string }) {
+	await run('worldbank', opts.job, async (note) => {
+		let total = 0;
+		for (const ind of WB_INDICATORS) {
+			if (opts.only && !opts.only.split(',').includes(ind.id)) continue;
+			try {
+				const url = `https://api.worldbank.org/v2/country/${COUNTRIES.join(';')}/indicator/${ind.id}?format=json&per_page=20000`;
+				const body = await getJson<[{ pages: number }, WbRow[] | null]>(url);
+				const rows = (body[1] ?? [])
+					.filter((r) => r.countryiso3code && /^\d{4}$/.test(r.date))
+					.map((r) => ({
+						indicator_id: ind.id,
+						country_iso3: r.countryiso3code,
+						year: Number(r.date),
+						value: r.value
+					}));
+				if (!rows.length) throw new Error('no rows');
+				await sql.begin(async (tx) => {
+					await tx`delete from stats.wb_indicator where indicator_id = ${ind.id}`;
+					await insertChunks(tx, 'wb_indicator', rows);
+				});
+				const series: SeriesRow[] = [];
+				const obs: ObsRow[] = [];
+				for (const iso3 of new Set(rows.map((r) => r.country_iso3))) {
+					const id = `wb:${ind.id}:${regionOf(iso3)}`;
+					series.push({
+						series_id: id,
+						source_id: 'worldbank',
+						source_code: `${ind.id}/${iso3}`,
+						title: ind.title,
+						unit: ind.unit,
+						freq: 'A',
+						region_code: regionOf(iso3),
+						theme: ind.theme,
+						meta: { url: `https://data.worldbank.org/indicator/${ind.id}` }
+					});
+					for (const r of rows)
+						if (r.country_iso3 === iso3 && r.value != null)
+							obs.push({ series_id: id, period: `${r.year}-01-01`, value: r.value, status: null });
+				}
+				await writeSeries(series, obs);
+				log(`worldbank ${ind.id}: ${rows.length} rows`);
+				total += rows.length;
+			} catch (error) {
+				note(`${ind.id}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		return total;
+	});
+}

src/lib/server/db/schema/stats.ts

@@ -202,3 +202,155 @@ export const statsBisPolicyRate = statsSchema.table(
 	},
 	(t) => [primaryKey({ columns: [t.refArea, t.freq, t.period] })]
 );
+
+// ── A3 World Bank indicators, A4 IMF WEO (DataMapper) ─────────────────────────────────────
+
+export const statsWbIndicator = statsSchema.table(
+	'wb_indicator',
+	{
+		indicatorId: text('indicator_id').notNull(),
+		countryIso3: text('country_iso3').notNull(),
+		year: integer('year').notNull(),
+		value: doublePrecision('value')
+	},
+	(t) => [primaryKey({ columns: [t.indicatorId, t.countryIso3, t.year] })]
+);
+
+export const statsImfWeo = statsSchema.table(
+	'imf_weo',
+	{
+		/** WEO round the values come from, e.g. '2026-10' (October 2026 WEO). */
+		vintage: text('vintage').notNull(),
+		indicator: text('indicator').notNull(),
+		countryIso3: text('country_iso3').notNull(),
+		year: integer('year').notNull(),
+		value: doublePrecision('value'),
+		isForecast: boolean('is_forecast').notNull()
+	},
+	(t) => [primaryKey({ columns: [t.vintage, t.indicator, t.countryIso3, t.year] })]
+);
+
+// ── A9 Our World in Data (Maddison GDP per capita, energy dataset) ───────────────────────
+
+export const statsOwidMaddison = statsSchema.table(
+	'owid_maddison',
+	{
+		entity: text('entity').notNull(),
+		/** null for OWID regions (Western Europe, World, …). */
+		iso3: text('iso3'),
+		year: integer('year').notNull(),
+		/** International-$ at 2011 prices. */
+		gdpPc: doublePrecision('gdp_pc')
+	},
+	(t) => [primaryKey({ columns: [t.entity, t.year] })]
+);
+
+export const statsOwidEnergy = statsSchema.table(
+	'owid_energy',
+	{
+		country: text('country').notNull(),
+		iso3: text('iso3'),
+		year: integer('year').notNull(),
+		/** Selected OWID columns as-is: {"solar_share_elec": 21.3, "energy_per_capita": 40000, …}. */
+		values: jsonb('values').$type<Record<string, number>>().notNull()
+	},
+	(t) => [primaryKey({ columns: [t.country, t.year] })]
+);
+
+// ── A10 Energy-Charts (Fraunhofer ISE): day-ahead prices, cross-border physical flows ─────
+
+export const statsEcDayaheadPrice = statsSchema.table(
+	'ec_dayahead_price',
+	{
+		bzn: text('bzn').notNull(),
+		ts: timestamptz('ts').notNull(),
+		eurMwh: doublePrecision('eur_mwh')
+	},
+	(t) => [primaryKey({ columns: [t.bzn, t.ts] })]
+);
+
+export const statsEcCrossborder = statsSchema.table(
+	'ec_crossborder',
+	{
+		/** Lower-case Energy-Charts country code, e.g. 'nl'. */
+		country: text('country').notNull(),
+		/** Neighbour name as Energy-Charts reports it ('Belgium', …, 'sum'). */
+		neighbour: text('neighbour').notNull(),
+		ts: timestamptz('ts').notNull(),
+		/** GW; positive = import into `country`, negative = export. */
+		gw: doublePrecision('gw')
+	},
+	(t) => [primaryKey({ columns: [t.country, t.neighbour, t.ts] })]
+);
+
+// ── A13 Rijksfinanciën (NL central-government budget tables) ──────────────────────────────
+
+export const statsRfBudgetTable = statsSchema.table(
+	'rf_budget_table',
+	{
+		year: integer('year').notNull(),
+		/** OWB (budget), O1/O2 (supplementary budgets), JV (annual report). */
+		phase: text('phase').notNull(),
+		/** Rijksfinanciën row index, unique within year + phase (includes chapter/article/vuo). */
+		rowKey: text('row_key').notNull(),
+		/** U = expenditure, O = revenue, V = commitments. */
+		vuo: char('vuo', { length: 1 }).notNull(),
+		chapter: text('chapter').notNull(),
+		chapterName: text('chapter_name').notNull(),
+		article: text('article'),
+		articleName: text('article_name'),
+		detail: jsonb('detail').$type<Record<string, unknown>>().notNull().default({}),
+		/** Thousand euro. */
+		amountEurK: doublePrecision('amount_eur_k')
+	},
+	(t) => [
+		primaryKey({ columns: [t.year, t.phase, t.rowKey] }),
+		index('rf_budget_table_chapter_idx').on(t.year, t.vuo, t.chapter)
+	]
+);
+
+// ── A11 US Treasury Fiscal Data (Monthly Treasury Statement, Debt to the Penny) ───────────
+
+const mtsColumns = {
+	recordDate: date('record_date', { mode: 'string' }).notNull(),
+	classificationId: text('classification_id').notNull(),
+	parentId: text('parent_id'),
+	description: text('description').notNull(),
+	level: integer('level'),
+	printOrder: integer('print_order'),
+	/** Net amount for the month and fiscal year to date, USD. */
+	netMonth: doublePrecision('net_month'),
+	netFytd: doublePrecision('net_fytd'),
+	grossMonth: doublePrecision('gross_month')
+};
+
+export const statsUstMtsReceipts = statsSchema.table('ust_mts_receipts', mtsColumns, (t) => [
+	primaryKey({ columns: [t.recordDate, t.classificationId] })
+]);
+
+export const statsUstMtsOutlays = statsSchema.table('ust_mts_outlays', { ...mtsColumns }, (t) => [
+	primaryKey({ columns: [t.recordDate, t.classificationId] })
+]);
+
+export const statsUstDebtDaily = statsSchema.table('ust_debt_daily', {
+	recordDate: date('record_date', { mode: 'string' }).primaryKey(),
+	debtHeldPublic: doublePrecision('debt_held_public'),
+	intragovHoldings: doublePrecision('intragov_holdings'),
+	totalDebt: doublePrecision('total_debt')
+});
+
+// ── A12 USAspending (budgetary resources per top-tier agency and fiscal year) ─────────────
+
+export const statsUsasAgencyFy = statsSchema.table(
+	'usas_agency_fy',
+	{
+		toptierCode: text('toptier_code').notNull(),
+		fiscalYear: integer('fiscal_year').notNull(),
+		agencyName: text('agency_name').notNull(),
+		abbreviation: text('abbreviation'),
+		budgetaryResources: doublePrecision('budgetary_resources'),
+		obligated: doublePrecision('obligated'),
+		outlayed: doublePrecision('outlayed')
+	},
+	(t) => [primaryKey({ columns: [t.toptierCode, t.fiscalYear] })]
+);