stats step 2: World Bank, IMF WEO, OWID, Energy-Charts, Rijksfinanciën, US Treasury, USAspending collectors + tables, timers

2026-10-25 21:05:51 +00:00
parent 7e905b1651
commit 8958d3ff8f
21 changed files with 34655 additions and 3 deletions
--- a/scripts/stats/worldbank.ts
+++ b/scripts/stats/worldbank.ts
@@ -0,0 +1,111 @@
+// A3 World Bank World Development Indicators: one call per indicator for every compared country
+// (plus EU and World aggregates) → stats.wb_indicator, normalised as `wb:<indicator>:<ISO3>`.
+import {
+	COMPARE_ISO3,
+	getJson,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const COUNTRIES = [...COMPARE_ISO3, 'EUU', 'WLD'];
+// EUU (World Bank code for the EU) is stored as EU27 in the normalised layer.
+const regionOf = (iso3: string) => (iso3 === 'EUU' ? 'EU27' : iso3);
+
+export const WB_INDICATORS: { id: string; title: string; unit: string; theme: string }[] = [
+	{ id: 'NY.GDP.PCAP.KD', title: 'GDP per capita', unit: 'constant 2015 USD', theme: 'countries' },
+	{
+		id: 'NY.GDP.PCAP.PP.KD',
+		title: 'GDP per capita, PPP',
+		unit: 'constant 2021 international $',
+		theme: 'countries'
+	},
+	{ id: 'NY.GDP.MKTP.CD', title: 'GDP', unit: 'current USD', theme: 'countries' },
+	{ id: 'NY.GDP.MKTP.KD.ZG', title: 'GDP growth', unit: '%', theme: 'countries' },
+	{ id: 'FP.CPI.TOTL.ZG', title: 'Inflation, consumer prices', unit: '%', theme: 'countries' },
+	{ id: 'SP.POP.TOTL', title: 'Population', unit: 'people', theme: 'countries' },
+	{ id: 'SM.POP.NETM', title: 'Net migration', unit: 'people', theme: 'countries' },
+	{
+		id: 'NE.TRD.GNFS.ZS',
+		title: 'Trade (exports + imports)',
+		unit: '% of GDP',
+		theme: 'countries'
+	},
+	{
+		id: 'TM.TAX.MRCH.WM.AR.ZS',
+		title: 'Applied tariff rate, weighted mean, all products',
+		unit: '%',
+		theme: 'economy'
+	},
+	{ id: 'EG.IMP.CONS.ZS', title: 'Energy imports, net', unit: '% of energy use', theme: 'energy' },
+	{
+		id: 'GC.REV.XGRT.GD.ZS',
+		title: 'Government revenue, excluding grants',
+		unit: '% of GDP',
+		theme: 'budget'
+	},
+	{ id: 'GC.XPN.TOTL.GD.ZS', title: 'Government expense', unit: '% of GDP', theme: 'budget' },
+	{ id: 'GC.DOD.TOTL.GD.ZS', title: 'Central government debt', unit: '% of GDP', theme: 'budget' }
+];
+
+type WbRow = {
+	countryiso3code: string;
+	date: string;
+	value: number | null;
+	indicator: { id: string };
+};
+
+export async function worldbank(opts: { only?: string; job: string }) {
+	await run('worldbank', opts.job, async (note) => {
+		let total = 0;
+		for (const ind of WB_INDICATORS) {
+			if (opts.only && !opts.only.split(',').includes(ind.id)) continue;
+			try {
+				const url = `https://api.worldbank.org/v2/country/${COUNTRIES.join(';')}/indicator/${ind.id}?format=json&per_page=20000`;
+				const body = await getJson<[{ pages: number }, WbRow[] | null]>(url);
+				const rows = (body[1] ?? [])
+					.filter((r) => r.countryiso3code && /^\d{4}$/.test(r.date))
+					.map((r) => ({
+						indicator_id: ind.id,
+						country_iso3: r.countryiso3code,
+						year: Number(r.date),
+						value: r.value
+					}));
+				if (!rows.length) throw new Error('no rows');
+				await sql.begin(async (tx) => {
+					await tx`delete from stats.wb_indicator where indicator_id = ${ind.id}`;
+					await insertChunks(tx, 'wb_indicator', rows);
+				});
+				const series: SeriesRow[] = [];
+				const obs: ObsRow[] = [];
+				for (const iso3 of new Set(rows.map((r) => r.country_iso3))) {
+					const id = `wb:${ind.id}:${regionOf(iso3)}`;
+					series.push({
+						series_id: id,
+						source_id: 'worldbank',
+						source_code: `${ind.id}/${iso3}`,
+						title: ind.title,
+						unit: ind.unit,
+						freq: 'A',
+						region_code: regionOf(iso3),
+						theme: ind.theme,
+						meta: { url: `https://data.worldbank.org/indicator/${ind.id}` }
+					});
+					for (const r of rows)
+						if (r.country_iso3 === iso3 && r.value != null)
+							obs.push({ series_id: id, period: `${r.year}-01-01`, value: r.value, status: null });
+				}
+				await writeSeries(series, obs);
+				log(`worldbank ${ind.id}: ${rows.length} rows`);
+				total += rows.length;
+			} catch (error) {
+				note(`${ind.id}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		return total;
+	});
+}