stats §8: CBS Curaçao collector (topic-page scrape, sha256 manifest, CPI since 1970 from PDF, GDP by industry, government finance, population since 1900, LFS, tourism) into cw_cbs_* + stats.obs region CUW; cw_cbcs_bop for manual uploads; weekly timer

scripts/stats/common.ts

@@ -32,7 +32,9 @@ const hostGapMs: Record<string, number> = {
 	'www.rijksfinancien.nl': 1000,
 	'api.fiscaldata.treasury.gov': 1000,
 	'api.usaspending.gov': 500,
-	'public.ep-online.nl': 1000
+	'public.ep-online.nl': 1000,
+	'www.cbs.cw': 2000,
+	'cuatro.sim-cdn.nl': 1000
 };
 const lastHit = new Map<string, number>();
 
@@ -203,6 +205,22 @@ export const SOURCES = [
 		key_required: true,
 		attribution:
 			'Source: EP-Online, Netherlands Enterprise Agency (RVO), aggregated by stanrunge.dev'
+	},
+	{
+		source_id: 'cbscw',
+		name: 'CBS Curaçao (Central Bureau of Statistics Curaçao)',
+		base_url: 'https://www.cbs.cw',
+		licence: 'Free reuse on condition that the source is mentioned (cbs.cw/disclaimer)',
+		key_required: false,
+		attribution: 'Source: Central Bureau of Statistics Curaçao (CBS Curaçao)'
+	},
+	{
+		source_id: 'cbcs',
+		name: 'Centrale Bank van Curaçao en Sint Maarten (CBCS)',
+		base_url: 'https://www.centralbank.cw',
+		licence: 'No open licence published; all rights reserved, shown with attribution',
+		key_required: false,
+		attribution: 'Source: Centrale Bank van Curaçao en Sint Maarten (CBCS), all rights reserved'
 	}
 ] as const;
 

scripts/stats/cwcbs.ts

@@ -0,0 +1,699 @@
+// §8 CW1–CW8: CBS Curaçao (Central Bureau of Statistics Curaçao). The statistics are xlsx/PDF
+// files on a CDN whose names change with every update, so each run scrapes the topic pages,
+// matches each table by a filename regex, and re-imports a table only when the file's sha256
+// changed (manifest: stats.cw_cbs_file). Each table has its own parser (hand-formatted sheets);
+// a table that fails to parse keeps its last good rows and is reported as a warning.
+// Licence: free to use on condition that the source (Central Bureau of Statistics Curaçao or
+// CBS Curaçao) is mentioned (cbs.cw/disclaimer).
+import { createHash } from 'node:crypto';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+	http,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+import { readXlsx, type Cell } from './xlsx';
+
+const SITE = 'https://www.cbs.cw';
+const CDN =
+	/https:\/\/cuatro\.sim-cdn\.nl\/cbscuracao\/uploads\/[^"'<>\\]+?\.(?:xlsx?|pdf)(?=[?"'<>\\])/gi;
+
+type Row = Record<string, unknown>;
+type Table = {
+	key: string;
+	page: string;
+	match: RegExp;
+	/** Raw table the parser fills, and the slice of it this file owns (for replace-on-import). */
+	table: string;
+	owns: string;
+	parse: (path: string) => Promise<Row[]>;
+};
+
+// ── cell helpers ─────────────────────────────────────────────────────────────────────────
+
+/** Number from a cell: handles "*2019", "1.096,470" (thousand separators), "." for n/a. */
+function num(v: Cell): number | null {
+	if (typeof v === 'number') return Number.isFinite(v) ? v : null;
+	if (v == null) return null;
+	let s = v.replace(/[*\s]/g, '');
+	if (!s || s === '.' || s === '-' || s === '–') return null;
+	if (/^-?\d{1,3}([.,]\d{3})+$/.test(s)) s = s.replace(/[.,]/g, '');
+	else if (/^-?\d{1,3}(\.\d{3})+,\d+$/.test(s)) s = s.replace(/\./g, '').replace(',', '.');
+	else s = s.replace(',', '.');
+	const n = Number(s);
+	return Number.isFinite(n) ? n : null;
+}
+/** Year header cell: 2019, "*2019", "2021*" → { year, provisional }. */
+function yearCell(v: Cell): { year: number; provisional: boolean } | null {
+	const s = String(v ?? '').trim();
+	const m = s.match(/^\*?\s*((?:19|20)\d\d)\s*\*?$/);
+	return m ? { year: Number(m[1]), provisional: s.includes('*') } : null;
+}
+/** Columns of a header row that hold years. */
+function yearColumns(row: Cell[]) {
+	const cols: { col: number; year: number; provisional: boolean }[] = [];
+	row.forEach((v, col) => {
+		const y = yearCell(v);
+		if (y) cols.push({ col, ...y });
+	});
+	return cols;
+}
+const text = (v: Cell) => (v == null ? '' : String(v).replace(/\s+/g, ' ').trim());
+/** Counts typed as 834.922 for 834,922 (Excel turned a thousands dot into a decimal). */
+const count = (v: number | null) =>
+	v != null &&
+	!Number.isInteger(v) &&
+	v < 10_000 &&
+	Number.isInteger(Math.round(v * 1000 * 1e6) / 1e6)
+		? Math.round(v * 1000)
+		: v;
+
+// ── parsers ──────────────────────────────────────────────────────────────────────────────
+
+/** CW1: monthly CPI since 1970 (PDF): index, 12-month average, inflation of the 12-month average. */
+export async function parseCpi(path: string): Promise<Row[]> {
+	const { getDocument } = await import('pdfjs-dist/legacy/build/pdf.mjs');
+	const data = new Uint8Array(await Bun.file(path).arrayBuffer());
+	const task = getDocument({ data, isEvalSupported: false, useSystemFonts: false });
+	const doc = await task.promise;
+	const lines: string[] = [];
+	let base = '';
+	for (let p = 1; p <= doc.numPages; p++) {
+		const tc = await (await doc.getPage(p)).getTextContent();
+		// Cluster text items into lines (superscript footnote marks sit a little off the baseline).
+		const items = (tc.items as { str?: string; transform: number[] }[])
+			.filter((it) => it.str?.trim())
+			.map((it) => ({ x: it.transform[4], y: it.transform[5], s: it.str!.trim() }))
+			.sort((a, b) => b.y - a.y);
+		const clusters: { y: number; items: typeof items }[] = [];
+		for (const it of items) {
+			const c = clusters.at(-1);
+			if (c && c.y - it.y <= 3) c.items.push(it);
+			else clusters.push({ y: it.y, items: [it] });
+		}
+		for (const c of clusters)
+			lines.push(
+				c.items
+					.sort((a, b) => a.x - b.x)
+					.map((i) => i.s)
+					.join(' ')
+			);
+	}
+	await task.destroy();
+	const MONTHS = [
+		'Jan',
+		'Feb',
+		'Mar',
+		'Apr',
+		'May',
+		'Jun',
+		'Jul',
+		'Aug',
+		'Sep',
+		'Oct',
+		'Nov',
+		'Dec'
+	];
+	const byPeriod = new Map<string, { index: number; avg: number | null; infl: number | null }>();
+	for (const line of lines) {
+		const b = line.match(/average (\d{4}) = 100/);
+		if (b) base = `${b[1]}=100`;
+		const m = line.match(
+			/^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s*(?:\d\))?\s+(\d{4})\s+(.*)$/
+		);
+		if (!m) continue;
+		const vals = m[3].split(/\s+/).filter((t) => /^(-?\d+(\.\d+)?|\.)$/.test(t));
+		const index = num(vals[0] ?? null);
+		if (index == null) continue;
+		const month = String(MONTHS.indexOf(m[1]) + 1).padStart(2, '0');
+		byPeriod.set(`${m[2]}-${month}-01`, {
+			index,
+			avg: num(vals[1] ?? null),
+			infl: num(vals[2] ?? null)
+		});
+	}
+	const periods = [...byPeriod.keys()].sort();
+	if (periods.length < 600) throw new Error(`CPI PDF: only ${periods.length} months parsed`);
+	const shift = (period: string, months: number) => {
+		const d = new Date(`${period}T00:00:00Z`);
+		d.setUTCMonth(d.getUTCMonth() - months);
+		return d.toISOString().slice(0, 10);
+	};
+	return periods.map((period) => {
+		const r = byPeriod.get(period)!;
+		const prev = byPeriod.get(shift(period, 1))?.index;
+		const lastYear = byPeriod.get(shift(period, 12))?.index;
+		return {
+			period,
+			cpi_index: r.index,
+			infl_mom: prev ? (100 * (r.index - prev)) / prev : null,
+			infl_yoy: lastYear ? (100 * (r.index - lastYear)) / lastYear : null,
+			infl_12m_avg: r.infl,
+			base: base || null
+		};
+	});
+}
+
+/** CW3: GDP by industry (ISIC 4), mln ANG, plus totals and real growth (stored as %). */
+async function parseGdp(path: string): Promise<Row[]> {
+	const [sheet] = await readXlsx(path);
+	const h = sheet.rows.findIndex((r) => /isic/i.test(text(r[0])));
+	if (h < 0) throw new Error('GDP: no ISIC header');
+	const years = yearColumns(sheet.rows[h]);
+	const out: Row[] = [];
+	const totals: [RegExp, string][] = [
+		[/^gross value added/i, 'GVA'],
+		[/taxes less subsidies/i, 'TLS'],
+		[/domestic product gross|gross domestic product/i, 'GDP'],
+		[/^real gdp growth/i, 'REAL_GROWTH_PCT'],
+		[/^nominal gdp growth/i, 'NOMINAL_GROWTH_PCT']
+	];
+	for (const row of sheet.rows.slice(h + 1)) {
+		const label = text(row[1]);
+		let isic = text(row[0]).replace(/\s+/g, '');
+		if (!isic) isic = totals.find(([re]) => re.test(label))?.[1] ?? '';
+		if (!isic || !label) continue;
+		for (const y of years) {
+			const v = num(row[y.col]);
+			if (v != null)
+				out.push({ year: y.year, isic, label, mln_ang: v, provisional: y.provisional });
+		}
+	}
+	if (!out.some((r) => r.isic === 'GDP')) throw new Error('GDP: no GDP total row');
+	return out;
+}
+
+/** CW4: government revenue and expenditure (ESA codes), mln ANG, all provisional. */
+async function parseGovFinance(path: string): Promise<Row[]> {
+	const [sheet] = await readXlsx(path);
+	const h = sheet.rows.findIndex((r) => yearColumns(r).length >= 5);
+	if (h < 0) throw new Error('government finance: no year header');
+	const years = yearColumns(sheet.rows[h]);
+	const out: Row[] = [];
+	let section = '';
+	for (const row of sheet.rows.slice(h + 1)) {
+		const label = text(row[0]);
+		if (/^revenues?$/i.test(label)) section = 'R';
+		else if (/^expenditures?$/i.test(label)) section = 'E';
+		if (!section || !label || /saldo|balance/i.test(label)) continue;
+		const code = /^total/i.test(label) ? 'TOTAL' : text(row[1]);
+		if (!code) continue;
+		for (const y of years) {
+			const v = num(row[y.col]);
+			if (v != null)
+				out.push({
+					year: y.year,
+					item_code: `${section}.${code}`,
+					label,
+					mln_ang: v,
+					provisional: true
+				});
+		}
+	}
+	if (!out.some((r) => r.item_code === 'R.TOTAL') || !out.some((r) => r.item_code === 'E.TOTAL'))
+		throw new Error('government finance: totals missing');
+	return out;
+}
+
+/** CW5: population 1900→ (thousands before 1951; 31 Dec until 1991, 1 Jan from 1992; censuses). */
+async function parsePopulationHistory(path: string): Promise<Row[]> {
+	const [sheet] = await readXlsx(path);
+	const out: Row[] = [];
+	for (const row of sheet.rows) {
+		const y = yearCell(row[1]);
+		let v = num(row[2]);
+		if (!y || v == null) continue;
+		if (v < 1000) v = Math.round(v * 1000);
+		const tag = text(row[0]).toLowerCase();
+		const kind = tag.startsWith('census') ? 'census' : y.year <= 1991 ? 'dec31' : 'jan1';
+		out.push({ year: y.year, sex: 'T', age_group: 'total', value: v, kind });
+	}
+	if (out.length < 100) throw new Error(`population history: only ${out.length} rows`);
+	// Two rows for one year and kind (1921): keep the last.
+	return [...new Map(out.map((r) => [`${r.year}|${r.kind}`, r])).values()];
+}
+
+const LFS_MEASURES: [RegExp, string][] = [
+	[/^population/i, 'population'],
+	[/^employed/i, 'employed'],
+	[/^unemployed/i, 'unemployed'],
+	[/not active/i, 'inactive'],
+	[/^labour force$/i, 'labour_force'],
+	[/^gross participation/i, 'gross_participation_pct'],
+	[/^net participation/i, 'net_participation_pct'],
+	[/^unemployment rate/i, 'unemployment_rate_pct']
+];
+/** CW7: LFS blocks ("Labour Force of Curacao 2014 – 2025 (Men)" / "(15-24 years)"). */
+async function parseLfs(path: string): Promise<Row[]> {
+	const out: Row[] = [];
+	for (const sheet of await readXlsx(path)) {
+		let years: ReturnType<typeof yearColumns> = [];
+		let sex = 'T';
+		let age = '15+';
+		sheet.rows.forEach((row, i) => {
+			const ys = yearColumns(row);
+			if (ys.length >= 3 && !text(row[0])) {
+				years = ys;
+				const title = text(sheet.rows[i - 1]?.[0]);
+				const tag = title.match(/\(([^)]+)\)\s*$/)?.[1] ?? '';
+				sex = /^(men|male)/i.test(tag) ? 'M' : /^(women|female)/i.test(tag) ? 'F' : 'T';
+				age =
+					tag
+						.match(/(\d+\s*[-–]\s*\d+|\d+\+)/)?.[1]
+						.replace(/\s/g, '')
+						.replace('–', '-') ?? '15+';
+				return;
+			}
+			const measure = LFS_MEASURES.find(([re]) => re.test(text(row[0])))?.[1];
+			if (!measure || !years.length) return;
+			for (const y of years) {
+				const v = num(row[y.col]);
+				if (v != null) out.push({ year: y.year, sex, age_group: age, measure, value: v });
+			}
+		});
+	}
+	if (!out.length) throw new Error('LFS: no rows');
+	return out;
+}
+
+/** CW8 tourism sheets: a year header, then one row per measure (or source market). */
+function tourismParser(map: (label: string) => { measure: string; market: string } | null) {
+	return async (path: string): Promise<Row[]> => {
+		const out: Row[] = [];
+		const seen = new Set<string>();
+		for (const sheet of await readXlsx(path)) {
+			let years: ReturnType<typeof yearColumns> = [];
+			for (const row of sheet.rows) {
+				const ys = yearColumns(row);
+				if (ys.length >= 3) {
+					years = ys;
+					continue;
+				}
+				const m = map(text(row[0]));
+				if (!m || !years.length) continue;
+				for (const y of years) {
+					const key = `${y.year}|${m.measure}|${m.market}`;
+					const v = count(num(row[y.col]));
+					if (v == null || seen.has(key)) continue;
+					seen.add(key);
+					out.push({ period: `${y.year}-01-01`, measure: m.measure, market: m.market, value: v });
+				}
+			}
+		}
+		if (!out.length) throw new Error('tourism: no rows');
+		return out;
+	};
+}
+
+export const TABLES: Table[] = [
+	{
+		key: 'cpi',
+		page: 'inflationprices',
+		match: /consumer price index and inflation rates.*since 1970.*\.pdf$/i,
+		table: 'cw_cbs_cpi',
+		owns: 'true',
+		parse: parseCpi
+	},
+	{
+		key: 'gdp',
+		page: 'economy-0',
+		match: /gdp.*\.xlsx$/i,
+		table: 'cw_cbs_gdp_industry',
+		owns: 'true',
+		parse: parseGdp
+	},
+	{
+		key: 'gov_finance',
+		page: 'economy-0',
+		match: /revenues.*expenditures.*government.*\.xlsx$/i,
+		table: 'cw_cbs_gov_finance',
+		owns: 'true',
+		parse: parseGovFinance
+	},
+	{
+		key: 'population_history',
+		page: 'population',
+		match: /historical.*population.*\.xlsx$/i,
+		table: 'cw_cbs_population',
+		owns: `age_group = 'total' and sex = 'T'`,
+		parse: parsePopulationHistory
+	},
+	{
+		key: 'lfs_sex',
+		page: 'labour',
+		match: /labour force of cura.*by sex.*\.xlsx$/i,
+		table: 'cw_cbs_lfs',
+		owns: `age_group = '15+'`,
+		parse: parseLfs
+	},
+	{
+		key: 'lfs_age',
+		page: 'labour',
+		match: /labour force survey.*age.groups.*\.xlsx$/i,
+		table: 'cw_cbs_lfs',
+		owns: `age_group <> '15+'`,
+		parse: parseLfs
+	},
+	{
+		key: 'tourism_cruise',
+		page: 'tourism',
+		match: /cruise arrivals.*\.xlsx$/i,
+		table: 'cw_cbs_tourism',
+		owns: `measure in ('cruise_calls', 'cruise_pax')`,
+		parse: tourismParser((l) =>
+			/^cruise calls/i.test(l)
+				? { measure: 'cruise_calls', market: 'total' }
+				: /^cruise passengers/i.test(l)
+					? { measure: 'cruise_pax', market: 'total' }
+					: null
+		)
+	},
+	{
+		key: 'tourism_daytrippers',
+		page: 'tourism',
+		match: /daytrippers.*\.xlsx$/i,
+		table: 'cw_cbs_tourism',
+		owns: `measure = 'daytrippers'`,
+		parse: tourismParser((l) =>
+			/^daytrippers/i.test(l) ? { measure: 'daytrippers', market: 'total' } : null
+		)
+	},
+	{
+		key: 'tourism_nights',
+		page: 'tourism',
+		match: /nights.*region.*\.xlsx$/i,
+		table: 'cw_cbs_tourism',
+		owns: `measure = 'nights'`,
+		parse: tourismParser((l) => {
+			if (/^total/i.test(l)) return { measure: 'nights', market: 'total' };
+			const market = l.match(/^(north america|south america|caribbean|europe|others?)$/i)?.[1];
+			return market
+				? { measure: 'nights', market: market.toLowerCase().replace(/\s+/g, '_') }
+				: null;
+		})
+	},
+	{
+		key: 'tourism_air',
+		page: 'tourism',
+		match: /air.traffic.movements.*\.xlsx$/i,
+		table: 'cw_cbs_tourism',
+		owns: `measure in ('air_landings_scheduled', 'air_landings_civil')`,
+		// The landings block comes first; the take-off block repeats the labels (skipped as seen).
+		parse: tourismParser((l) =>
+			/^scheduled pax flights/i.test(l)
+				? { measure: 'air_landings_scheduled', market: 'total' }
+				: /^total civil landings/i.test(l)
+					? { measure: 'air_landings_civil', market: 'total' }
+					: null
+		)
+	}
+];
+
+/** File links on a topic page, decoded, newest-looking first (…_0.xlsx before ….xlsx). */
+async function links(page: string): Promise<string[]> {
+	const html = await (await http(`${SITE}/${page}`)).text();
+	const urls = new Set<string>();
+	for (const m of html.replace(/&amp;/g, '&').matchAll(CDN)) urls.add(m[0]);
+	return [...urls].sort((a, b) => decodeURIComponent(b).localeCompare(decodeURIComponent(a)));
+}
+const fileName = (url: string) => decodeURIComponent(url.split('/uploads/')[1] ?? url);
+
+export async function cwcbs(opts: { force: boolean; only?: string; job: string }) {
+	await run('cbscw', opts.job, async (note) => {
+		const only = opts.only?.split(',');
+		const dir = await mkdtemp(join(tmpdir(), 'cwcbs-'));
+		let rows = 0;
+		try {
+			const pages = new Map<string, string[]>();
+			for (const t of TABLES) {
+				if (only && !only.includes(t.key)) continue;
+				try {
+					if (!pages.has(t.page)) pages.set(t.page, await links(t.page));
+					const url = pages.get(t.page)!.find((u) => t.match.test(fileName(u)));
+					if (!url) throw new Error(`no file matching ${t.match} on /${t.page}`);
+					const bytes = new Uint8Array(await (await http(url)).arrayBuffer());
+					const sha256 = createHash('sha256').update(bytes).digest('hex');
+					const [prev] = await sql<{ sha256: string }[]>`
+						select sha256 from stats.cw_cbs_file where topic = ${t.key}`;
+					if (prev?.sha256 === sha256 && !opts.force) {
+						log(`cbscw ${t.key}: unchanged (${fileName(url)})`);
+						continue;
+					}
+					const path = join(dir, `${t.key}${url.match(/\.\w+$/)?.[0] ?? ''}`);
+					await Bun.write(path, bytes);
+					const parsed = await t.parse(path);
+					await sql.begin(async (tx) => {
+						await tx.unsafe(`delete from stats.${t.table} where ${t.owns}`);
+						await insertChunks(tx, t.table, parsed);
+						await tx`delete from stats.cw_cbs_file where topic = ${t.key}`;
+						await tx`insert into stats.cw_cbs_file ${tx({ url, topic: t.key, filename: fileName(url), sha256 })}
+							on conflict (url) do update set topic = excluded.topic, filename = excluded.filename,
+							sha256 = excluded.sha256, fetched_at = now()`;
+					});
+					rows += parsed.length;
+					log(`cbscw ${t.key}: ${parsed.length} rows from ${fileName(url)}`);
+				} catch (error) {
+					note(`${t.key}: ${error instanceof Error ? error.message : String(error)}`);
+				}
+			}
+		} finally {
+			await rm(dir, { recursive: true, force: true });
+		}
+		const obs = await rebuildSeries();
+		log(`cbscw: ${obs} observations in stats.obs`);
+		return rows;
+	});
+}
+
+// ── normalised series (region CUW) ───────────────────────────────────────────────────────
+
+const LFS_TITLE: Record<string, [string, string]> = {
+	population: ['Population 15+', 'persons'],
+	employed: ['Employed', 'persons'],
+	unemployed: ['Unemployed', 'persons'],
+	inactive: ['Not in the labour force', 'persons'],
+	labour_force: ['Labour force', 'persons'],
+	gross_participation_pct: ['Gross participation rate', '%'],
+	net_participation_pct: ['Net participation rate (employment rate)', '%'],
+	unemployment_rate_pct: ['Unemployment rate', '%']
+};
+const TOURISM_TITLE: Record<string, [string, string]> = {
+	cruise_calls: ['Cruise ship calls', 'calls'],
+	cruise_pax: ['Cruise passengers', 'passengers'],
+	daytrippers: ['Daytrippers', 'visitors'],
+	nights: ['Stay-over tourist nights', 'nights'],
+	air_landings_scheduled: ['Scheduled passenger flight landings', 'landings'],
+	air_landings_civil: ['Civil aircraft landings', 'landings']
+};
+
+export async function rebuildSeries(): Promise<number> {
+	const series: SeriesRow[] = [];
+	const obs: ObsRow[] = [];
+	const add = (
+		id: string,
+		title: string,
+		unit: string,
+		freq: string,
+		code: string,
+		points: { period: string; value: number | null; status?: string | null }[],
+		meta: Record<string, unknown> = {}
+	) => {
+		if (!points.length) return;
+		series.push({
+			series_id: id,
+			source_id: 'cbscw',
+			source_code: code,
+			title,
+			unit,
+			freq,
+			region_code: 'CUW',
+			theme: 'curacao',
+			meta: { url: SITE, ...meta }
+		});
+		for (const p of points)
+			obs.push({ series_id: id, period: p.period, value: p.value, status: p.status ?? null });
+	};
+
+	const cpi = await sql<
+		{
+			period: string;
+			cpi_index: number;
+			infl_yoy: number | null;
+			infl_12m_avg: number | null;
+			base: string | null;
+		}[]
+	>`
+		select period::text, cpi_index, infl_yoy, infl_12m_avg, base from stats.cw_cbs_cpi order by period`;
+	const base = cpi.at(-1)?.base ?? '2023=100';
+	add(
+		'cw:cpi:index',
+		`Consumer price index (${base})`,
+		'index',
+		'M',
+		'CPI',
+		cpi.map((r) => ({ period: r.period, value: r.cpi_index })),
+		{ base }
+	);
+	add(
+		'cw:cpi:yoy',
+		'Inflation, CPI year on year',
+		'%',
+		'M',
+		'CPI_YOY',
+		cpi.filter((r) => r.infl_yoy != null).map((r) => ({ period: r.period, value: r.infl_yoy }))
+	);
+	add(
+		'cw:cpi:avg12',
+		'Inflation, 12-month average CPI',
+		'%',
+		'M',
+		'CPI_AVG12',
+		cpi
+			.filter((r) => r.infl_12m_avg != null)
+			.map((r) => ({ period: r.period, value: r.infl_12m_avg }))
+	);
+
+	const gdp = await sql<
+		{ year: number; isic: string; label: string; mln_ang: number; provisional: boolean }[]
+	>`
+		select year, isic, label, mln_ang, provisional from stats.cw_cbs_gdp_industry order by year`;
+	for (const isic of [...new Set(gdp.map((r) => r.isic))]) {
+		const mine = gdp.filter((r) => r.isic === isic);
+		const pct = isic.endsWith('_PCT');
+		add(
+			`cw:gdp:${isic.toLowerCase()}`,
+			pct ? mine[0].label : `GDP by industry: ${mine[0].label}`,
+			pct ? '%' : 'mln ANG',
+			'A',
+			isic,
+			mine.map((r) => ({
+				period: `${r.year}-01-01`,
+				value: r.mln_ang,
+				status: r.provisional ? 'provisional' : null
+			})),
+			{ isic }
+		);
+	}
+
+	const gov = await sql<{ year: number; item_code: string; label: string; mln_ang: number }[]>`
+		select year, item_code, label, mln_ang from stats.cw_cbs_gov_finance order by year`;
+	for (const code of [...new Set(gov.map((r) => r.item_code))]) {
+		const mine = gov.filter((r) => r.item_code === code);
+		const side = code.startsWith('R.') ? 'revenue' : 'expenditure';
+		add(
+			`cw:gov:${code.toLowerCase()}`,
+			code.endsWith('TOTAL') ? `Government ${side}, total` : `Government ${side}: ${mine[0].label}`,
+			'mln ANG',
+			'A',
+			code,
+			mine.map((r) => ({ period: `${r.year}-01-01`, value: r.mln_ang, status: 'provisional' }))
+		);
+	}
+	const rev = new Map(gov.filter((r) => r.item_code === 'R.TOTAL').map((r) => [r.year, r.mln_ang]));
+	add(
+		'cw:gov:balance',
+		'Government balance (revenue minus expenditure)',
+		'mln ANG',
+		'A',
+		'BALANCE',
+		gov
+			.filter((r) => r.item_code === 'E.TOTAL' && rev.has(r.year))
+			.map((r) => ({
+				period: `${r.year}-01-01`,
+				value: rev.get(r.year)! - r.mln_ang,
+				status: 'provisional'
+			}))
+	);
+
+	const pop = await sql<{ year: number; value: number; kind: string }[]>`
+		select year, value, kind from stats.cw_cbs_population where sex = 'T' and age_group = 'total' order by year`;
+	// 31 December of year y is 1 January of y + 1; censuses are a separate series.
+	const jan1 = new Map<number, number>();
+	for (const r of pop) if (r.kind === 'dec31') jan1.set(r.year + 1, r.value);
+	for (const r of pop) if (r.kind === 'jan1') jan1.set(r.year, r.value);
+	add(
+		'cw:pop:jan1',
+		'Population on 1 January (31 December before 1992)',
+		'persons',
+		'A',
+		'POP',
+		[...jan1].sort((a, b) => a[0] - b[0]).map(([y, v]) => ({ period: `${y}-01-01`, value: v })),
+		{ note: 'Until 1955 from "Olie als water"; CBS estimates from 1956' }
+	);
+	add(
+		'cw:pop:census',
+		'Population at census',
+		'persons',
+		'A',
+		'CENSUS',
+		pop
+			.filter((r) => r.kind === 'census')
+			.map((r) => ({ period: `${r.year}-01-01`, value: r.value }))
+	);
+
+	const lfs = await sql<
+		{ year: number; sex: string; age_group: string; measure: string; value: number }[]
+	>`
+		select year, sex, age_group, measure, value from stats.cw_cbs_lfs order by year`;
+	// Both sexes together: add up the counts, recompute the rates.
+	const both = new Map<string, Record<string, number>>();
+	for (const r of lfs.filter((x) => x.age_group === '15+' && x.sex !== 'T')) {
+		const m = both.get(String(r.year)) ?? {};
+		if (!r.measure.endsWith('_pct')) m[r.measure] = (m[r.measure] ?? 0) + r.value;
+		both.set(String(r.year), m);
+	}
+	for (const [year, m] of both) {
+		const push = (measure: string, value: number) =>
+			lfs.push({ year: Number(year), sex: 'T', age_group: '15+', measure, value });
+		for (const k of Object.keys(m)) push(k, m[k]);
+		if (m.labour_force) push('unemployment_rate_pct', (100 * m.unemployed) / m.labour_force);
+		if (m.population) {
+			push('gross_participation_pct', (100 * m.labour_force) / m.population);
+			push('net_participation_pct', (100 * m.employed) / m.population);
+		}
+	}
+	const lfsKeys = [...new Set(lfs.map((r) => `${r.measure}|${r.sex}|${r.age_group}`))];
+	for (const key of lfsKeys) {
+		const [measure, sex, age] = key.split('|');
+		const [title, unit] = LFS_TITLE[measure] ?? [measure, ''];
+		const who = sex === 'M' ? 'men' : sex === 'F' ? 'women' : age === '15+' ? '' : `aged ${age}`;
+		const slug =
+			sex !== 'T' ? sex.toLowerCase() : age === '15+' ? 'total' : age.replace('+', 'plus');
+		add(
+			`cw:lfs:${measure}:${slug}`,
+			`${title}${who ? `, ${who}` : ''} (Labour Force Survey)`,
+			unit,
+			'A',
+			`LFS.${measure}.${sex}.${age}`,
+			lfs
+				.filter((r) => `${r.measure}|${r.sex}|${r.age_group}` === key)
+				.map((r) => ({ period: `${r.year}-01-01`, value: r.value }))
+		);
+	}
+
+	const tour = await sql<{ period: string; measure: string; market: string; value: number }[]>`
+		select period::text, measure, market, value from stats.cw_cbs_tourism order by period`;
+	for (const key of [...new Set(tour.map((r) => `${r.measure}|${r.market}`))]) {
+		const [measure, market] = key.split('|');
+		const [title, unit] = TOURISM_TITLE[measure] ?? [measure, ''];
+		add(
+			`cw:tour:${measure}${market === 'total' ? '' : `:${market}`}`,
+			`${title}${market === 'total' ? '' : `: ${market.replace(/_/g, ' ')}`}`,
+			unit,
+			'A',
+			`${measure}.${market}`,
+			tour
+				.filter((r) => r.measure === measure && r.market === market)
+				.map((r) => ({ period: r.period, value: r.value })),
+			{ origin: measure.startsWith('air') ? 'Curaçao Airport Partners' : 'Curaçao Tourist Board' }
+		);
+	}
+	return writeSeries(series, obs);
+}

scripts/stats/xlsx.ts

@@ -0,0 +1,78 @@
+// Minimal xlsx reader for the stats collector (no dependency): unzips the workbook with the
+// host's `unzip` and reads cell values from the sheet XML. Enough for hand-formatted statistical
+// tables: returns each sheet as a grid of strings/numbers (merged cells are not expanded).
+export type Cell = string | number | null;
+export type Sheet = { name: string; rows: Cell[][] };
+
+async function unzipEntry(path: string, entry: string): Promise<string | null> {
+	const proc = Bun.spawn(['unzip', '-p', path, entry], { stdout: 'pipe', stderr: 'ignore' });
+	const text = await new Response(proc.stdout).text();
+	return (await proc.exited) === 0 ? text : null;
+}
+
+const decode = (s: string) =>
+	s
+		.replace(/&lt;/g, '<')
+		.replace(/&gt;/g, '>')
+		.replace(/&quot;/g, '"')
+		.replace(/&apos;/g, "'")
+		.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n)))
+		.replace(/&#x([0-9a-f]+);/gi, (_, n) => String.fromCodePoint(parseInt(n, 16)))
+		.replace(/&amp;/g, '&');
+
+/** Text of all <t> runs inside a fragment (shared strings and inline strings). */
+const textOf = (xml: string) =>
+	decode([...xml.matchAll(/<t(?:\s[^>]*)?>([\s\S]*?)<\/t>/g)].map((m) => m[1]).join(''));
+
+function colIndex(ref: string) {
+	let n = 0;
+	for (const ch of ref.replace(/\d+$/, '')) n = n * 26 + (ch.charCodeAt(0) - 64);
+	return n - 1;
+}
+
+export async function readXlsx(path: string): Promise<Sheet[]> {
+	const workbook = await unzipEntry(path, 'xl/workbook.xml');
+	const rels = await unzipEntry(path, 'xl/_rels/workbook.xml.rels');
+	if (!workbook || !rels) throw new Error('not an xlsx workbook');
+	const shared = await unzipEntry(path, 'xl/sharedStrings.xml');
+	const strings = shared
+		? [...shared.matchAll(/<si>([\s\S]*?)<\/si>/g)].map((m) => textOf(m[1]))
+		: [];
+	const target = new Map(
+		[...rels.matchAll(/<Relationship\b[^>]*>/g)].map((m) => [
+			m[0].match(/Id="([^"]+)"/)?.[1],
+			m[0].match(/Target="([^"]+)"/)?.[1]
+		])
+	);
+	const sheets: Sheet[] = [];
+	for (const m of workbook.matchAll(/<sheet\b[^>]*>/g)) {
+		const name = decode(m[0].match(/name="([^"]*)"/)?.[1] ?? '');
+		const rid = m[0].match(/r:id="([^"]+)"/)?.[1];
+		let file = target.get(rid) ?? '';
+		file = file.startsWith('/') ? file.slice(1) : `xl/${file}`;
+		const xml = await unzipEntry(path, file);
+		if (!xml) continue;
+		const rows: Cell[][] = [];
+		for (const r of xml.matchAll(/<row\b[^>]*?r="(\d+)"[^>]*?(?:\/>|>([\s\S]*?)<\/row>)/g)) {
+			const row: Cell[] = [];
+			for (const c of (r[2] ?? '').matchAll(/<c\b([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) {
+				const ref = c[1].match(/r="([A-Z]+\d+)"/)?.[1];
+				if (!ref) continue;
+				const type = c[1].match(/t="(\w+)"/)?.[1];
+				const body = c[2] ?? '';
+				const v = body.match(/<v>([\s\S]*?)<\/v>/)?.[1];
+				let value: Cell = null;
+				if (type === 's' && v != null) value = strings[Number(v)] ?? null;
+				else if (type === 'inlineStr') value = textOf(body);
+				else if (type === 'str' || type === 'e') value = v != null ? decode(v) : null;
+				else if (type === 'b') value = v === '1' ? 'TRUE' : 'FALSE';
+				else if (v != null && v !== '') value = Number(v);
+				if (typeof value === 'string') value = value.trim() || null;
+				row[colIndex(ref)] = value;
+			}
+			rows[Number(r[1]) - 1] = Array.from(row, (x) => x ?? null);
+		}
+		sheets.push({ name, rows: Array.from(rows, (x) => x ?? []) });
+	}
+	return sheets;
+}