stats §8: Curaçao from ILOSTAT (SDMX), OWID, UN SDG 531, UN Comtrade 531 (cwintl + weekly timer), EU trade with CW (Eurostat ext_lt_maineu); CBS Curaçao LFS file pick fix; Curaçao detail panels on /stats/countries/CUW

2026-10-25 21:47:05 +00:00
parent a6b4f9931d
commit 031ee4baf2
10 changed files with 795 additions and 52 deletions
--- a/scripts/stats/common.ts
+++ b/scripts/stats/common.ts
@@ -34,7 +34,10 @@ const hostGapMs: Record<string, number> = {
 	'api.usaspending.gov': 500,
 	'public.ep-online.nl': 1000,
 	'www.cbs.cw': 2000,
-	'cuatro.sim-cdn.nl': 1000
+	'cuatro.sim-cdn.nl': 1000,
+	'sdmx.ilo.org': 3000,
+	'unstats.un.org': 1500,
+	'comtradeapi.un.org': 2500
 };
 const lastHit = new Map<string, number>();

@@ -221,6 +224,30 @@ export const SOURCES = [
 		licence: 'No open licence published; all rights reserved, shown with attribution',
 		key_required: false,
 		attribution: 'Source: Centrale Bank van Curaçao en Sint Maarten (CBCS), all rights reserved'
+	},
+	{
+		source_id: 'ilostat',
+		name: 'ILOSTAT (International Labour Organization)',
+		base_url: 'https://sdmx.ilo.org/rest',
+		licence: 'CC BY 4.0',
+		key_required: false,
+		attribution: 'Source: ILOSTAT, International Labour Organization, CC BY 4.0'
+	},
+	{
+		source_id: 'unsdg',
+		name: 'UN SDG Global Database',
+		base_url: 'https://unstats.un.org/sdgs/UNSDGAPIV5',
+		licence: 'Free use with attribution (UN terms of use)',
+		key_required: false,
+		attribution: 'Source: United Nations Global SDG Database (UN Statistics Division)'
+	},
+	{
+		source_id: 'comtrade',
+		name: 'UN Comtrade',
+		base_url: 'https://comtradeapi.un.org/public/v1/preview',
+		licence: 'Free use with attribution (UN Comtrade public preview)',
+		key_required: false,
+		attribution: 'Source: UN Comtrade Database (United Nations Statistics Division)'
 	}
 ] as const;

--- a/scripts/stats/cwcbs.ts
+++ b/scripts/stats/cwcbs.ts
@@ -359,7 +359,9 @@ export const TABLES: Table[] = [
 		match: /labour force survey.*age.groups.*\.xlsx$/i,
 		table: 'cw_cbs_lfs',
 		owns: `age_group <> '15+'`,
-		parse: parseLfs
+		// The 15+ totals come from the by-sex file; older age-group uploads repeat them per sheet.
+		parse: async (path: string) =>
+			(await parseLfs(path)).filter((r) => (r as { age_group: string }).age_group !== '15+')
 	},
 	{
 		key: 'tourism_cruise',
@@ -421,7 +423,12 @@ async function links(page: string): Promise<string[]> {
 	const html = await (await http(`${SITE}/${page}`)).text();
 	const urls = new Set<string>();
 	for (const m of html.replace(/&amp;/g, '&').matchAll(CDN)) urls.add(m[0]);
-	return [...urls].sort((a, b) => decodeURIComponent(b).localeCompare(decodeURIComponent(a)));
+	// Plain code-point order (not localeCompare, which ignores punctuation): the CMS re-upload
+	// suffix `_0` then sorts ahead of the original, so the newest copy of a file wins.
+	return [...urls]
+		.map((u) => [u, decodeURIComponent(u)] as const)
+		.sort((a, b) => (a[1] < b[1] ? 1 : a[1] > b[1] ? -1 : 0))
+		.map(([u]) => u);
 }
 const fileName = (url: string) => decodeURIComponent(url.split('/uploads/')[1] ?? url);

--- a/scripts/stats/cwintl.ts
+++ b/scripts/stats/cwintl.ts
@@ -0,0 +1,306 @@
+// §8 CW13–CW16: international open APIs that cover Curaçao (M49 531, ISO3 CUW). All small,
+// keyless and replaced on every run; each source is its own fetch_run so one flaky API
+// (ILOSTAT, Comtrade preview) does not hide the others.
+import { parseCsv } from './owid';
+import { getJson, getText, log, run, writeSeries, type ObsRow, type SeriesRow } from './common';
+
+const REGION = 'CUW';
+const THEME = 'curacao';
+const year = (y: string | number) => `${String(y).slice(0, 4)}-01-01`;
+const toNum = (v: unknown) => {
+	const n = typeof v === 'number' ? v : Number(String(v ?? '').trim());
+	return String(v ?? '').trim() !== '' && Number.isFinite(n) ? n : null;
+};
+const slug = (s: string) =>
+	s
+		.toLowerCase()
+		.replace(/\+/g, 'plus')
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-|-$/g, '');
+
+/** Sequential retries on top of http()'s own: ILOSTAT answers 200 with an empty body or 403 at times. */
+async function retry<T>(what: string, fn: () => Promise<T>, ok: (t: T) => boolean, tries = 4) {
+	let last: unknown;
+	for (let i = 1; i <= tries; i++) {
+		try {
+			const t = await fn();
+			if (ok(t)) return t;
+			last = new Error(`${what}: empty response`);
+		} catch (error) {
+			last = error;
+		}
+		await new Promise((r) => setTimeout(r, 5000 * i));
+	}
+	throw last;
+}
+
+// ── ILOSTAT (SDMX; the rplumber CSV endpoint in the spec returns empty bodies) ─────────────
+const ILO_BASE = 'https://sdmx.ilo.org/rest/data/ILO';
+export const ILO_FLOWS: Record<string, { title: string; unit: string }> = {
+	DF_UNE_DEAP_SEX_AGE_RT: { title: 'Unemployment rate', unit: '%' },
+	DF_EAP_DWAP_SEX_AGE_RT: { title: 'Labour force participation rate', unit: '%' },
+	DF_EMP_DWAP_SEX_AGE_RT: { title: 'Employment-to-population ratio', unit: '%' }
+};
+const ILO_AGES: Record<string, string> = {
+	AGE_YTHADULT_YGE15: '15plus',
+	'AGE_YTHADULT_Y15-24': '15-24',
+	AGE_YTHADULT_YGE25: '25plus'
+};
+const ILO_SEX: Record<string, string> = { SEX_T: 'total', SEX_M: 'm', SEX_F: 'f' };
+
+async function ilostat(note: (w: string) => void) {
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	for (const [flow, meta] of Object.entries(ILO_FLOWS)) {
+		const url = `${ILO_BASE},${flow},1.0/${REGION}.A....?format=csv`;
+		let text: string;
+		try {
+			text = await retry(
+				flow,
+				() => getText(url),
+				(t) => t.startsWith('DATAFLOW')
+			);
+		} catch (error) {
+			note(`${flow}: ${error instanceof Error ? error.message : String(error)}`);
+			continue;
+		}
+		const [head, ...rows] = parseCsv(text);
+		const col = (n: string) => head.indexOf(n);
+		const [cSex, cAge, cTime, cVal, cSrc] = [
+			'SEX',
+			'AGE',
+			'TIME_PERIOD',
+			'OBS_VALUE',
+			'SOURCE'
+		].map(col);
+		const byId = new Map<string, ObsRow[]>();
+		for (const r of rows) {
+			const age = ILO_AGES[r[cAge]];
+			const sex = ILO_SEX[r[cSex]];
+			if (!age || !sex || !r[cTime]) continue;
+			const id = `ilostat:${flow.replace(/^DF_/, '').toLowerCase()}:${sex}:${age}`;
+			if (!byId.has(id)) {
+				byId.set(id, []);
+				series.push({
+					series_id: id,
+					source_id: 'ilostat',
+					source_code: `${flow}/${r[cSex]}.${r[cAge]}`,
+					title: `Curaçao · ${meta.title} (${sex === 'total' ? 'total' : sex === 'm' ? 'men' : 'women'}, ${age.replace('plus', '+')})`,
+					unit: meta.unit,
+					freq: 'A',
+					region_code: REGION,
+					theme: THEME,
+					meta: { flow, sex: r[cSex], age: r[cAge], survey: r[cSrc] }
+				});
+			}
+			byId
+				.get(id)!
+				.push({ series_id: id, period: year(r[cTime]), value: toNum(r[cVal]), status: null });
+		}
+		for (const list of byId.values()) obs.push(...list);
+	}
+	const n = await writeSeries(series, obs);
+	log(`ilostat CUW: ${series.length} series, ${n} obs`);
+	return n;
+}
+
+// ── OWID grapher CSVs (UN WPP / GCB via OWID, CC BY 4.0) ──────────────────────────────────
+export const OWID_CW: Record<string, { col: string; title: string; unit: string }> = {
+	population: { col: 'population_historical', title: 'Population (UN WPP)', unit: 'people' },
+	'life-expectancy': { col: 'life_expectancy_0', title: 'Life expectancy at birth', unit: 'years' },
+	'median-age': {
+		col: 'median_age__sex_all__age_all__variant_estimates',
+		title: 'Median age',
+		unit: 'years'
+	},
+	'co-emissions-per-capita': {
+		col: 'emissions_total_per_capita',
+		title: 'CO₂ emissions per person',
+		unit: 't'
+	}
+};
+
+async function owidCw(note: (w: string) => void) {
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	for (const [slugName, meta] of Object.entries(OWID_CW)) {
+		try {
+			const text = await getText(
+				`https://ourworldindata.org/grapher/${slugName}.csv?country=${REGION}&csvType=filtered&useColumnShortNames=true`
+			);
+			const [head, ...rows] = parseCsv(text);
+			const cCode = head.indexOf('code');
+			const cYear = head.indexOf('year');
+			const cVal = head.indexOf(meta.col);
+			if (cVal < 0) throw new Error(`column ${meta.col} missing`);
+			const id = `owid:cw:${slugName}`;
+			const list = rows
+				.filter((r) => r[cCode] === REGION && toNum(r[cVal]) !== null)
+				.map((r) => ({
+					series_id: id,
+					period: year(r[cYear]),
+					value: toNum(r[cVal]),
+					status: null
+				}));
+			if (!list.length) throw new Error('no CUW rows');
+			obs.push(...list);
+			series.push({
+				series_id: id,
+				source_id: 'owid',
+				source_code: `grapher/${slugName}`,
+				title: `Curaçao · ${meta.title}`,
+				unit: meta.unit,
+				freq: 'A',
+				region_code: REGION,
+				theme: THEME,
+				meta: { url: `https://ourworldindata.org/grapher/${slugName}` }
+			});
+		} catch (error) {
+			note(`owid ${slugName}: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+	const n = await writeSeries(series, obs);
+	log(`owid CUW: ${series.length} series, ${n} obs`);
+	return n;
+}
+
+// ── UN SDG API (area 531) ─────────────────────────────────────────────────────────────────
+export const SDG_SERIES: Record<string, { title: string; unit: string }> = {
+	EN_ATM_CO2: { title: 'CO₂ emissions from fuel combustion', unit: 'Mt' },
+	EG_FEC_RNEW: { title: 'Renewable share of final energy consumption', unit: '%' },
+	SP_DYN_ADKL: { title: 'Adolescent birth rate (per 1,000 women 15–19)', unit: 'per 1,000' },
+	ER_PTD_TERR: { title: 'Terrestrial key biodiversity areas protected', unit: '%' }
+};
+type SdgRow = {
+	timePeriodStart: number;
+	value: string;
+	dimensions: Record<string, string>;
+	seriesDescription: string;
+};
+
+async function unsdg(note: (w: string) => void) {
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	for (const [code, meta] of Object.entries(SDG_SERIES)) {
+		try {
+			const d = await getJson<{ data: SdgRow[] }>(
+				`https://unstats.un.org/sdgs/UNSDGAPIV5/v1/sdg/Series/Data?seriesCode=${code}&areaCode=531&pageSize=2000`
+			);
+			const byId = new Map<string, Map<string, number | null>>();
+			for (const r of d.data) {
+				const dims = Object.entries(r.dimensions)
+					.filter(([k]) => k !== 'Reporting Type')
+					.sort(([a], [b]) => a.localeCompare(b));
+				// SP_DYN_ADKL also carries the 10–14 group; keep the headline 15–19.
+				if (code === 'SP_DYN_ADKL' && r.dimensions.Age !== '15-19') continue;
+				const key = dims.map(([, v]) => slug(v)).join(':');
+				const id = `unsdg:${code.toLowerCase()}${key ? `:${key}` : ''}`;
+				if (!byId.has(id)) {
+					byId.set(id, new Map());
+					series.push({
+						series_id: id,
+						source_id: 'unsdg',
+						source_code: `${code}/${dims.map(([k, v]) => `${k}=${v}`).join(',')}`,
+						title: `Curaçao · ${meta.title}${dims.length && code !== 'SP_DYN_ADKL' ? ` (${dims.map(([, v]) => v).join(', ')})` : ''}`,
+						unit: meta.unit,
+						freq: 'A',
+						region_code: REGION,
+						theme: THEME,
+						meta: { code, description: r.seriesDescription, dims: Object.fromEntries(dims) }
+					});
+				}
+				// Global (G) and national (N) reporting can both appear; the first wins.
+				const p = year(r.timePeriodStart);
+				if (!byId.get(id)!.has(p)) byId.get(id)!.set(p, toNum(r.value));
+			}
+			for (const [id, m] of byId)
+				for (const [period, value] of m) obs.push({ series_id: id, period, value, status: null });
+		} catch (error) {
+			note(`${code}: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+	const n = await writeSeries(series, obs);
+	log(`unsdg 531: ${series.length} series, ${n} obs`);
+	return n;
+}
+
+// ── UN Comtrade public preview (one period per call, ≤500 rows) ───────────────────────────
+const COMTRADE_PARTNERS: Record<number, string> = {
+	0: 'world',
+	842: 'usa',
+	528: 'nld',
+	76: 'bra',
+	862: 'ven',
+	156: 'chn'
+};
+type ComtradeRow = { period: string; flowCode: string; partnerCode: number; primaryValue: number };
+
+async function comtrade(note: (w: string) => void) {
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	const seen = new Set<string>();
+	const last = new Date().getFullYear() - 1;
+	for (let y = 2010; y <= last; y++) {
+		let d: { data?: ComtradeRow[]; error?: string };
+		try {
+			d = await getJson(
+				`https://comtradeapi.un.org/public/v1/preview/C/A/HS?reporterCode=531&period=${y}&flowCode=M,X&partnerCode=${Object.keys(COMTRADE_PARTNERS).join(',')}&cmdCode=TOTAL`
+			);
+		} catch (error) {
+			note(`${y}: ${error instanceof Error ? error.message : String(error)}`);
+			continue;
+		}
+		if (d.error) note(`${y}: ${d.error}`);
+		for (const r of d.data ?? []) {
+			const partner = COMTRADE_PARTNERS[r.partnerCode];
+			if (!partner) continue;
+			const flow = r.flowCode === 'M' ? 'imports' : r.flowCode === 'X' ? 'exports' : null;
+			if (!flow) continue;
+			const id = `comtrade:cw:${flow}:${partner}`;
+			if (!seen.has(id)) {
+				seen.add(id);
+				series.push({
+					series_id: id,
+					source_id: 'comtrade',
+					source_code: `531/${r.flowCode}/${r.partnerCode}/TOTAL`,
+					title: `Curaçao · Goods ${flow}${partner === 'world' ? '' : ` (partner ${partner.toUpperCase()})`}`,
+					unit: 'US$',
+					freq: 'A',
+					region_code: REGION,
+					theme: THEME,
+					meta: { reporter: 531, flow: r.flowCode, partner: r.partnerCode }
+				});
+			}
+			obs.push({ series_id: id, period: year(r.period), value: r.primaryValue, status: null });
+		}
+	}
+	const n = await writeSeries(series, obs);
+	log(`comtrade 531: ${series.length} series, ${n} obs`);
+	return n;
+}
+
+export const CW_INTL_JOBS = { ilostat, owid: owidCw, unsdg, comtrade } as const;
+const SOURCE_OF: Record<keyof typeof CW_INTL_JOBS, string> = {
+	ilostat: 'ilostat',
+	owid: 'owid',
+	unsdg: 'unsdg',
+	comtrade: 'comtrade'
+};
+
+export async function cwintl(opts: { only?: string; job: string }) {
+	let total = 0;
+	let failed = 0;
+	for (const [name, fn] of Object.entries(CW_INTL_JOBS) as [
+		keyof typeof CW_INTL_JOBS,
+		typeof ilostat
+	][]) {
+		if (opts.only && !opts.only.split(',').includes(name)) continue;
+		try {
+			total += await run(SOURCE_OF[name], `cw-${opts.job}`, fn);
+		} catch {
+			failed++;
+		}
+	}
+	if (failed) throw new Error(`cwintl: ${failed} job(s) failed`);
+	return total;
+}
--- a/scripts/stats/eurostat.ts
+++ b/scripts/stats/eurostat.ts
@@ -51,6 +51,8 @@ type Dataset = {
 	theme: string;
 	params: Record<string, string | string[]>;
 	geos?: string[];
+	/** Region for every series of the dataset (partner datasets: the partner, not the reporter). */
+	region?: string;
 };

 export const EUROSTAT_DATASETS: Dataset[] = [
@@ -66,6 +68,18 @@ export const EUROSTAT_DATASETS: Dataset[] = [
 		// GJ83P70 (band D3, 83.7 GJ) carries the pre-2007 history (was nrg_pc_202_h); it is only published per GJ (gross calorific value).
 		params: { nrg_cons: ['GJ20-199', 'GJ83P70'], currency: 'EUR', unit: ['KWH', 'GJ_GCV'] }
 	},
+	{
+		// §8 CW12: EU27 trade in goods with Curaçao (million EUR).
+		code: 'ext_lt_maineu',
+		theme: 'curacao',
+		params: {
+			partner: 'CW',
+			sitc06: 'TOTAL',
+			indic_et: ['MIO_EXP_VAL', 'MIO_IMP_VAL', 'MIO_BAL_VAL', 'PC_EXP_PART', 'PC_IMP_PART']
+		},
+		geos: ['EU27_2020'],
+		region: 'CUW'
+	},
 	{ code: 'nrg_ind_id', theme: 'energy', params: { siec: 'TOTAL' } },
 	{
 		code: 'irt_lt_mcby_m',
@@ -222,7 +236,7 @@ async function pull(d: Dataset): Promise<number> {
 				.join(', ')}`.replace(/ · $/, ''),
 			unit,
 			freq,
-			region_code: geo ? (GEO_REGION[geo.code] ?? null) : null,
+			region_code: d.region ?? (geo ? (GEO_REGION[geo.code] ?? null) : null),
 			theme: d.theme,
 			meta: { dataset: d.code, dims: Object.fromEntries(labels.map((l) => [l.id, l.code])) }
 		});