website/scripts/stats/eurostat.ts

// A2 Eurostat dissemination API (JSON-stat 2.0). Each dataset is pulled with a fixed filter
// (12 countries + EU/EA aggregates, one consumption band / unit) into stats.eurostat_obs, then
// every (dataset, key) becomes a normalised series `eurostat:<dataset>:<key>`.
import {
	getJson,
	log,
	parsePeriod,
	run,
	sql,
	writeSeries,
	type ObsRow,
	type SeriesRow
} from './common';

const BASE = 'https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data';
export const EU_GEOS = [
	'NL',
	'DE',
	'FR',
	'BE',
	'IT',
	'ES',
	'PL',
	'SE',
	'DK',
	'AT',
	'EU27_2020',
	'EA20'
];
const GEO_REGION: Record<string, string> = {
	NL: 'NLD',
	DE: 'DEU',
	FR: 'FRA',
	BE: 'BEL',
	IT: 'ITA',
	ES: 'ESP',
	PL: 'POL',
	SE: 'SWE',
	DK: 'DNK',
	AT: 'AUT',
	EU27_2020: 'EU27',
	EA20: 'EA',
	EA: 'EA',
	UK: 'GBR',
	JP: 'JPN',
	US: 'USA'
};

type Dataset = {
	code: string;
	theme: string;
	params: Record<string, string | string[]>;
	geos?: string[];
	/** Region for every series of the dataset (partner datasets: the partner, not the reporter). */
	region?: string;
};

export const EUROSTAT_DATASETS: Dataset[] = [
	{
		code: 'nrg_pc_204',
		theme: 'energy',
		// KWH3500 (band Dc) carries 1991–2007; the old nrg_pc_204_h table was folded into nrg_pc_204.
		params: { nrg_cons: ['KWH2500-4999', 'KWH3500'], currency: 'EUR', unit: 'KWH' }
	},
	{
		code: 'nrg_pc_202',
		theme: 'energy',
		// GJ83P70 (band D3, 83.7 GJ) carries the pre-2007 history (was nrg_pc_202_h); it is only published per GJ (gross calorific value).
		params: { nrg_cons: ['GJ20-199', 'GJ83P70'], currency: 'EUR', unit: ['KWH', 'GJ_GCV'] }
	},
	{
		// §8 CW12: EU27 trade in goods with Curaçao (million EUR).
		code: 'ext_lt_maineu',
		theme: 'curacao',
		params: {
			partner: 'CW',
			sitc06: 'TOTAL',
			indic_et: ['MIO_EXP_VAL', 'MIO_IMP_VAL', 'MIO_BAL_VAL', 'PC_EXP_PART', 'PC_IMP_PART']
		},
		geos: ['EU27_2020'],
		region: 'CUW'
	},
	{ code: 'nrg_ind_id', theme: 'energy', params: { siec: 'TOTAL' } },
	{
		code: 'irt_lt_mcby_m',
		theme: 'economy',
		params: {},
		geos: ['NL', 'DE', 'FR', 'IT', 'BE', 'ES', 'AT', 'EA']
	},
	{
		// Rates for every geo; index levels (2015 = 100) and the main categories feed the CPI charts.
		code: 'prc_hicp_minr',
		theme: 'economy',
		params: {
			coicop18: ['TOTAL', 'CP01', 'CP04', 'NRG', 'TOT_X_NRG_FOOD'],
			unit: ['RCH_A', 'I15']
		}
	},
	{ code: 'prc_hicp_manr', theme: 'economy', params: { coicop: 'CP00', unit: 'RCH_A' } },
	{ code: 'prc_hpi_q', theme: 'economy', params: { purchase: 'TOTAL', unit: ['I15_Q', 'RCH_A'] } },
	{ code: 'road_eqr_carpda', theme: 'energy', params: { unit: 'NR' } },
	{
		code: 'migr_imm1ctz',
		theme: 'countries',
		params: { citizen: 'TOTAL', age: 'TOTAL', sex: 'T', agedef: 'COMPLET', unit: 'NR' }
	},
	{
		code: 'gov_10a_main',
		theme: 'budget',
		params: { unit: 'PC_GDP', sector: 'S13', na_item: ['TR', 'TE', 'B9'] }
	},
	{
		code: 'gov_10a_exp',
		theme: 'budget',
		params: { unit: 'PC_GDP', sector: 'S13', na_item: 'TE' }
	},
	{
		code: 'gov_10a_taxag',
		theme: 'budget',
		params: { unit: 'PC_GDP', sector: 'S13_S212', na_item: ['D2_REC', 'D5_REC', 'D61_REC', 'D995'] }
	},
	{
		code: 'gov_10dd_edpt1',
		theme: 'budget',
		params: { unit: 'PC_GDP', sector: 'S13', na_item: ['GD', 'B9'] }
	},
	{
		code: 'nama_10_gdp',
		theme: 'economy',
		params: { unit: ['CP_MEUR', 'CLV_PCH_PRE'], na_item: 'B1GQ' }
	},
	{ code: 'demo_pjan', theme: 'countries', params: { sex: 'T', age: 'TOTAL', unit: 'NR' } },
	// ── Jobs & cost of living (/stats/<place>/jobs) ──
	{
		code: 'une_rt_m',
		theme: 'jobs',
		params: { s_adj: 'SA', age: ['TOTAL', 'Y_LT25', 'Y25-74'], unit: 'PC_ACT', sex: 'T' },
		geos: [...EU_GEOS, 'US', 'JP']
	},
	{
		code: 'lfsa_urgan',
		theme: 'jobs',
		params: { unit: 'PC', sex: 'T', age: ['Y15-24', 'Y25-29', 'Y15-74'], citizen: 'TOTAL' }
	},
	{
		code: 'jvs_q_nace2',
		theme: 'jobs',
		params: { s_adj: 'NSA', nace_r2: 'B-S', sizeclas: 'TOTAL', indic_em: 'JVR' }
	},
	{
		// Employment rate of recent graduates (left education 1–3 years ago), 20–34.
		code: 'edat_lfse_24',
		theme: 'jobs',
		params: { duration: 'Y1-3', isced11: ['ED3-8', 'ED5-8'], age: 'Y20-34', sex: 'T', unit: 'PC' }
	},
	{
		// NEET rate: not employed, not in education or training.
		code: 'edat_lfse_20',
		theme: 'jobs',
		params: {
			sex: 'T',
			age: ['Y15-29', 'Y15-24'],
			training: 'NO_FE_NO_NFE',
			wstatus: 'NEMP',
			unit: 'PC'
		}
	},
	{
		code: 'lfsa_etpgan',
		theme: 'jobs',
		params: { unit: 'PC', sex: 'T', age: ['Y15-64', 'Y15-24', 'Y25-34'], citizen: 'TOTAL' }
	},
	{
		code: 'lfsa_eppga',
		theme: 'jobs',
		params: { unit: 'PC', sex: 'T', age: ['Y15-64', 'Y15-24'] }
	},
	{
		code: 'lfsa_ewhan2',
		theme: 'jobs',
		params: {
			nace_r2: 'TOTAL',
			wstatus: 'EMP',
			worktime: ['TOTAL', 'FT'],
			age: 'Y15-64',
			sex: 'T',
			unit: 'HR'
		}
	},
	{
		code: 'earn_mw_cur',
		theme: 'jobs',
		params: { currency: ['EUR', 'PPS'] },
		geos: [...EU_GEOS, 'US']
	},
	{
		// Price level indices (EU27 = 100): everything, food, alcohol & tobacco, housing, transport, restaurants.
		code: 'prc_ppp_ind',
		theme: 'jobs',
		params: {
			na_item: 'PLI_EU27_2020',
			ppp_cat: ['GDP', 'A01', 'A0101', 'A0104', 'A0107', 'A0111']
		},
		geos: [...EU_GEOS, 'US', 'JP']
	},
	{
		// Housing cost overburden: households spending > 40 % of disposable income on housing.
		code: 'ilc_lvho07a',
		theme: 'jobs',
		params: { unit: 'PC', rskpovth: 'TOTAL', age: ['TOTAL', 'Y18-24', 'Y25-29'], sex: 'T' }
	},
	{
		code: 'une_ltu_a',
		theme: 'jobs',
		params: { indic_em: 'LTU', age: 'Y15-74', sex: 'T', unit: 'PC_UNE' }
	},
	{
		// Share of young adults still living with their parents.
		code: 'ilc_lvps08',
		theme: 'jobs',
		params: { age: ['Y25-29', 'Y25-34'], sex: 'T', unit: 'PC' }
	},
	{
		// Structure of earnings survey (every 4 years): mean and median earnings, firms with 10+ staff.
		code: 'earn_ses_pub2s',
		theme: 'jobs',
		params: { sizeclas: 'GE10', sex: 'T', unit: ['EUR', 'PPS'] }
	}
];

type JsonStat = {
	label: string;
	updated?: string;
	id: string[];
	size: number[];
	dimension: Record<
		string,
		{ label: string; category: { index: Record<string, number>; label: Record<string, string> } }
	>;
	value: Record<string, number> | number[];
	status?: Record<string, string> | string[];
};

function query(d: Dataset) {
	const q = new URLSearchParams({ format: 'JSON', lang: 'EN' });
	for (const g of d.geos ?? EU_GEOS) q.append('geo', g);
	for (const [k, v] of Object.entries(d.params)) for (const x of [v].flat()) q.append(k, x);
	return q.toString();
}

async function pull(d: Dataset): Promise<number> {
	const qs = query(d);
	const js = await getJson<JsonStat>(`${BASE}/${d.code}?${qs}`);
	const order = js.id;
	const codesByDim = order.map((id) => {
		const idx = js.dimension[id].category.index;
		const arr: string[] = [];
		for (const [code, i] of Object.entries(idx)) arr[i] = code;
		return arr;
	});
	const timePos = order.indexOf('time');
	const geoPos = order.indexOf('geo');
	const rows: {
		dataset: string;
		key: string;
		geo: string;
		period: string;
		value: number | null;
		status: string | null;
	}[] = [];
	const values = js.value;
	const status = js.status ?? {};
	const entries: [number, number][] = Array.isArray(values)
		? values.map((v, i) => [i, v] as [number, number]).filter(([, v]) => v != null)
		: Object.entries(values).map(([k, v]) => [Number(k), v]);
	for (const [flat, value] of entries) {
		let rest = flat;
		const coord: number[] = new Array(order.length);
		for (let p = order.length - 1; p >= 0; p--) {
			coord[p] = rest % js.size[p];
			rest = Math.floor(rest / js.size[p]);
		}
		const keyParts: string[] = [];
		for (let p = 0; p < order.length; p++)
			if (p !== timePos && order[p] !== 'freq') keyParts.push(codesByDim[p][coord[p]]);
		rows.push({
			dataset: d.code,
			key: keyParts.join('.'),
			geo: geoPos >= 0 ? codesByDim[geoPos][coord[geoPos]] : '',
			period: codesByDim[timePos][coord[timePos]],
			value: value ?? null,
			status: (Array.isArray(status) ? status[flat] : status[String(flat)]) ?? null
		});
	}
	const dims: Record<string, unknown> = {};
	for (const id of order)
		dims[id] = { label: js.dimension[id].label, labels: js.dimension[id].category.label };
	await sql.begin(async (tx) => {
		await tx`delete from stats.eurostat_obs where dataset = ${d.code}`;
		for (let i = 0; i < rows.length; i += 5000)
			await tx`insert into stats.eurostat_obs ${tx(rows.slice(i, i + 5000))} on conflict do nothing`;
		await tx`insert into stats.eurostat_dataset (code, label, updated, query, dims, obs_count, fetched_at)
			values (${d.code}, ${js.label}, ${js.updated ?? null}, ${qs}, ${tx.json(dims as never)}, ${rows.length}, now())
			on conflict (code) do update set label = excluded.label, updated = excluded.updated, query = excluded.query,
			dims = excluded.dims, obs_count = excluded.obs_count, fetched_at = now()`;
	});
	// Normalise: one series per key.
	const order2 = order.filter((id) => id !== 'time' && id !== 'freq');
	const byKey = new Map<string, typeof rows>();
	for (const r of rows) byKey.set(r.key, [...(byKey.get(r.key) ?? []), r]);
	const series: SeriesRow[] = [];
	const obs: ObsRow[] = [];
	for (const [key, list] of byKey) {
		const parts = key.split('.');
		const labels = order2.map((id, i) => ({
			id,
			code: parts[i],
			label: js.dimension[id].category.label[parts[i]] ?? parts[i]
		}));
		const unit =
			labels.find((l) => l.id === 'unit')?.label ??
			labels.find((l) => l.id === 'currency')?.label ??
			'';
		const geo = labels.find((l) => l.id === 'geo');
		const id = `eurostat:${d.code}:${key}`;
		let freq = 'A';
		for (const r of list) {
			const p = parsePeriod(r.period);
			if (!p) continue;
			freq = p.freq;
			obs.push({ series_id: id, period: p.date, value: r.value, status: r.status });
		}
		series.push({
			series_id: id,
			source_id: 'eurostat',
			source_code: `${d.code}/${key}`,
			title: `${js.label} · ${labels
				.filter((l) => l.id !== 'geo' && l.id !== 'unit')
				.map((l) => l.label)
				.join(', ')}`.replace(/ · $/, ''),
			unit,
			freq,
			region_code: d.region ?? (geo ? (GEO_REGION[geo.code] ?? null) : null),
			theme: d.theme,
			meta: { dataset: d.code, dims: Object.fromEntries(labels.map((l) => [l.id, l.code])) }
		});
	}
	await sql`delete from stats.series where source_id = 'eurostat' and meta->>'dataset' = ${d.code}`;
	await writeSeries(series, obs);
	log(`eurostat ${d.code}: ${rows.length} observations, ${series.length} series`);
	return rows.length;
}

export async function eurostat(opts: { only?: string; job: string }) {
	await run('eurostat', opts.job, async (note) => {
		let total = 0;
		for (const d of EUROSTAT_DATASETS) {
			if (opts.only && !opts.only.split(',').includes(d.code)) continue;
			try {
				total += await pull(d);
			} catch (error) {
				note(`${d.code}: ${error instanceof Error ? error.message : String(error)}`);
			}
		}
		return total;
	});
}