website/scripts/stats/worldbank.ts

// A3 World Bank World Development Indicators: one call per indicator for every compared country
// (plus EU and World aggregates) → stats.wb_indicator, normalised as `wb:<indicator>:<ISO3>`.
import {
	COMPARE_ISO3,
	getJson,
	insertChunks,
	log,
	run,
	sql,
	writeSeries,
	type ObsRow,
	type SeriesRow
} from './common';

const COUNTRIES = [...COMPARE_ISO3, 'EUU', 'WLD'];
// EUU (World Bank code for the EU) is stored as EU27 in the normalised layer.
const regionOf = (iso3: string) => (iso3 === 'EUU' ? 'EU27' : iso3);

export const WB_INDICATORS: { id: string; title: string; unit: string; theme: string }[] = [
	{ id: 'NY.GDP.PCAP.KD', title: 'GDP per capita', unit: 'constant 2015 USD', theme: 'countries' },
	{
		id: 'NY.GDP.PCAP.PP.KD',
		title: 'GDP per capita, PPP',
		unit: 'constant 2021 international $',
		theme: 'countries'
	},
	{ id: 'NY.GDP.MKTP.CD', title: 'GDP', unit: 'current USD', theme: 'countries' },
	{ id: 'NY.GDP.MKTP.KD.ZG', title: 'GDP growth', unit: '%', theme: 'countries' },
	{ id: 'FP.CPI.TOTL.ZG', title: 'Inflation, consumer prices', unit: '%', theme: 'countries' },
	{ id: 'FP.CPI.TOTL', title: 'Consumer price index', unit: 'Index 2010=100', theme: 'economy' },
	{ id: 'SP.POP.TOTL', title: 'Population', unit: 'people', theme: 'countries' },
	{ id: 'SM.POP.NETM', title: 'Net migration', unit: 'people', theme: 'countries' },
	{
		id: 'NE.TRD.GNFS.ZS',
		title: 'Trade (exports + imports)',
		unit: '% of GDP',
		theme: 'countries'
	},
	{
		id: 'TM.TAX.MRCH.WM.AR.ZS',
		title: 'Applied tariff rate, weighted mean, all products',
		unit: '%',
		theme: 'economy'
	},
	{ id: 'EG.IMP.CONS.ZS', title: 'Energy imports, net', unit: '% of energy use', theme: 'energy' },
	{
		id: 'GC.REV.XGRT.GD.ZS',
		title: 'Government revenue, excluding grants',
		unit: '% of GDP',
		theme: 'budget'
	},
	{ id: 'GC.XPN.TOTL.GD.ZS', title: 'Government expense', unit: '% of GDP', theme: 'budget' },
	{ id: 'GC.DOD.TOTL.GD.ZS', title: 'Central government debt', unit: '% of GDP', theme: 'budget' },
	// Country profiles (/stats/countries/[iso3]); most of these also cover Curaçao.
	{
		id: 'NY.GNP.PCAP.CD',
		title: 'GNI per capita (Atlas)',
		unit: 'current USD',
		theme: 'countries'
	},
	{
		id: 'SL.UEM.TOTL.ZS',
		title: 'Unemployment (ILO modelled)',
		unit: '% of labour force',
		theme: 'countries'
	},
	{ id: 'SP.DYN.LE00.IN', title: 'Life expectancy at birth', unit: 'years', theme: 'countries' },
	{ id: 'SP.DYN.TFRT.IN', title: 'Fertility rate', unit: 'births per woman', theme: 'countries' },
	{
		id: 'SP.POP.65UP.TO.ZS',
		title: 'Population aged 65+',
		unit: '% of population',
		theme: 'countries'
	},
	{
		id: 'NE.EXP.GNFS.ZS',
		title: 'Exports of goods and services',
		unit: '% of GDP',
		theme: 'countries'
	},
	{
		id: 'NE.IMP.GNFS.ZS',
		title: 'Imports of goods and services',
		unit: '% of GDP',
		theme: 'countries'
	},
	{ id: 'NV.SRV.TOTL.ZS', title: 'Services, value added', unit: '% of GDP', theme: 'countries' },
	{
		id: 'ST.INT.ARVL',
		title: 'International tourism, arrivals',
		unit: 'arrivals',
		theme: 'countries'
	},
	{
		id: 'ST.INT.RCPT.CD',
		title: 'International tourism, receipts',
		unit: 'current USD',
		theme: 'countries'
	},
	{
		id: 'EG.ELC.RNEW.ZS',
		title: 'Renewable electricity output',
		unit: '% of electricity',
		theme: 'energy'
	}
];

type WbRow = {
	countryiso3code: string;
	date: string;
	value: number | null;
	indicator: { id: string };
};

export async function worldbank(opts: { only?: string; job: string }) {
	await run('worldbank', opts.job, async (note) => {
		let total = 0;
		for (const ind of WB_INDICATORS) {
			if (opts.only && !opts.only.split(',').includes(ind.id)) continue;
			try {
				const url = `https://api.worldbank.org/v2/country/${COUNTRIES.join(';')}/indicator/${ind.id}?format=json&per_page=20000`;
				const body = await getJson<[{ pages: number }, WbRow[] | null]>(url);
				const rows = (body[1] ?? [])
					.filter((r) => r.countryiso3code && /^\d{4}$/.test(r.date))
					.map((r) => ({
						indicator_id: ind.id,
						country_iso3: r.countryiso3code,
						year: Number(r.date),
						value: r.value
					}));
				if (!rows.length) throw new Error('no rows');
				await sql.begin(async (tx) => {
					await tx`delete from stats.wb_indicator where indicator_id = ${ind.id}`;
					await insertChunks(tx, 'wb_indicator', rows);
				});
				const series: SeriesRow[] = [];
				const obs: ObsRow[] = [];
				for (const iso3 of new Set(rows.map((r) => r.country_iso3))) {
					const id = `wb:${ind.id}:${regionOf(iso3)}`;
					series.push({
						series_id: id,
						source_id: 'worldbank',
						source_code: `${ind.id}/${iso3}`,
						title: ind.title,
						unit: ind.unit,
						freq: 'A',
						region_code: regionOf(iso3),
						theme: ind.theme,
						meta: { url: `https://data.worldbank.org/indicator/${ind.id}` }
					});
					for (const r of rows)
						if (r.country_iso3 === iso3 && r.value != null)
							obs.push({ series_id: id, period: `${r.year}-01-01`, value: r.value, status: null });
				}
				await writeSeries(series, obs);
				log(`worldbank ${ind.id}: ${rows.length} rows`);
				total += rows.length;
			} catch (error) {
				note(`${ind.id}: ${error instanceof Error ? error.message : String(error)}`);
			}
		}
		return total;
	});
}