From 031ee4baf2bc20ba72f955a5b76924ad75e4b1cf Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sun, 25 Oct 2026 21:47:05 +0000 Subject: [PATCH] =?UTF-8?q?stats=20=C2=A78:=20Cura=C3=A7ao=20from=20ILOSTA?= =?UTF-8?q?T=20(SDMX),=20OWID,=20UN=20SDG=20531,=20UN=20Comtrade=20531=20(?= =?UTF-8?q?cwintl=20+=20weekly=20timer),=20EU=20trade=20with=20CW=20(Euros?= =?UTF-8?q?tat=20ext=5Flt=5Fmaineu);=20CBS=20Cura=C3=A7ao=20LFS=20file=20p?= =?UTF-8?q?ick=20fix;=20Cura=C3=A7ao=20detail=20panels=20on=20/stats/count?= =?UTF-8?q?ries/CUW?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/stats-collector/README.md | 32 +- deploy/stats-collector/stats-cwintl.timer | 11 + scripts/stats-fetch.ts | 7 +- scripts/stats/common.ts | 29 +- scripts/stats/cwcbs.ts | 11 +- scripts/stats/cwintl.ts | 306 +++++++++++++++ scripts/stats/eurostat.ts | 16 +- src/lib/stats/curacao.ts | 64 +++ .../stats/countries/[iso3]/+page.server.ts | 5 +- .../stats/countries/[iso3]/+page.svelte | 366 ++++++++++++++++-- 10 files changed, 795 insertions(+), 52 deletions(-) create mode 100644 deploy/stats-collector/stats-cwintl.timer create mode 100644 scripts/stats/cwintl.ts create mode 100644 src/lib/stats/curacao.ts diff --git a/deploy/stats-collector/README.md b/deploy/stats-collector/README.md index fd6d799..2917b64 100644 --- a/deploy/stats-collector/README.md +++ b/deploy/stats-collector/README.md @@ -3,21 +3,23 @@ Host-side collector for the public [/stats](https://stanrunge.dev/stats) section. It pulls open statistics into the `stats` schema with `scripts/stats-fetch.ts`: -| source | timer | schedule (Europe/Amsterdam) | what | -|---|---|---|---| -| `cbs` | `stats-cbs.timer` | nightly 05:40 | CBS StatLine OData v4: checks every table's `Modified`, re-pulls changed ones, flags discontinued tables (base-year swaps) | -| `eurostat` | `stats-eurostat.timer` | Mondays 06:20 | Eurostat JSON-stat datasets (energy prices, yields, HICP, government finance…) | -| `fred` | `stats-fred.timer` | daily 07:20 | FRED graph CSV (Brent, TTF, Fed, Treasuries, US energy prices, indices) | -| `bis` | `stats-bis.timer` | daily 07:50 | BIS central bank policy rates, monthly + daily | -| `worldbank` | `stats-worldbank.timer` | Mondays 04:15 | World Bank WDI indicators for the compared countries | -| `imf` | `stats-imf.timer` | 1st of the month 04:05 | IMF WEO (DataMapper), stored per WEO vintage with forecast flags | -| `owid` | `stats-owid.timer` | 2nd of the month 04:35 | Our World in Data: Maddison GDP per capita, energy dataset | -| `energycharts` | `stats-energycharts.timer` | hourly at :19 | Energy-Charts NL day-ahead prices + cross-border flows (last 3 days; `--backfill` from 2015) | -| `rijksfinancien` | `stats-rijksfinancien.timer` | 3rd of the month 05:05 | Rijksfinanciën budget tables, last year to next year (`--backfill` from 2015) | -| `ustreasury` | `stats-ustreasury.timer` | daily 23:25 | US Treasury MTS receipts/outlays + Debt to the Penny (last 400 days; `--backfill` all) | -| `usaspending` | `stats-usaspending.timer` | Sundays 06:10 | USAspending budgetary resources per agency and fiscal year | -| `eponline` | `stats-eponline.timer` | 3rd of the month 05:20 | EP-Online monthly total file (RVO energy labels): homes per label for NL, Gouda and Gouda PC4s; aggregates only. Needs `EP_ONLINE_API_KEY` in `~/.config/stats/ep-online.env` (mode 600), which the runner exports for this source only | -| `cbscw` | `stats-cbscw.timer` | Tuesdays 05:50 | CBS Curaçao: scrapes the topic pages (cbs.cw), matches each table's file by regex, re-imports only when its sha256 changes (`stats.cw_cbs_file`); CPI from PDF (pdfjs), GDP, government finance, population since 1900, LFS, tourism from xlsx | +| source | timer | schedule (Europe/Amsterdam) | what | +| ---------------- | ---------------------------- | --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `cbs` | `stats-cbs.timer` | nightly 05:40 | CBS StatLine OData v4: checks every table's `Modified`, re-pulls changed ones, flags discontinued tables (base-year swaps) | +| `eurostat` | `stats-eurostat.timer` | Mondays 06:20 | Eurostat JSON-stat datasets (energy prices, yields, HICP, government finance…) | +| `fred` | `stats-fred.timer` | daily 07:20 | FRED graph CSV (Brent, TTF, Fed, Treasuries, US energy prices, indices) | +| `bis` | `stats-bis.timer` | daily 07:50 | BIS central bank policy rates, monthly + daily | +| `worldbank` | `stats-worldbank.timer` | Mondays 04:15 | World Bank WDI indicators for the compared countries | +| `imf` | `stats-imf.timer` | 1st of the month 04:05 | IMF WEO (DataMapper), stored per WEO vintage with forecast flags | +| `owid` | `stats-owid.timer` | 2nd of the month 04:35 | Our World in Data: Maddison GDP per capita, energy dataset | +| `energycharts` | `stats-energycharts.timer` | hourly at :19 | Energy-Charts NL day-ahead prices + cross-border flows (last 3 days; `--backfill` from 2015) | +| `rijksfinancien` | `stats-rijksfinancien.timer` | 3rd of the month 05:05 | Rijksfinanciën budget tables, last year to next year (`--backfill` from 2015) | +| `ustreasury` | `stats-ustreasury.timer` | daily 23:25 | US Treasury MTS receipts/outlays + Debt to the Penny (last 400 days; `--backfill` all) | +| `usaspending` | `stats-usaspending.timer` | Sundays 06:10 | USAspending budgetary resources per agency and fiscal year | +| `eponline` | `stats-eponline.timer` | 3rd of the month 05:20 | EP-Online monthly total file (RVO energy labels): homes per label for NL, Gouda and Gouda PC4s; aggregates only. Needs `EP_ONLINE_API_KEY` in `~/.config/stats/ep-online.env` (mode 600), which the runner exports for this source only | +| `cbscw` | `stats-cbscw.timer` | Tuesdays 05:50 | CBS Curaçao: scrapes the topic pages (cbs.cw), matches each table's file by regex, re-imports only when its sha256 changes (`stats.cw_cbs_file`); CPI from PDF (pdfjs), GDP, government finance, population since 1900, LFS, tourism from xlsx | +| `cwintl` | `stats-cwintl.timer` | Wednesdays 05:50 | Curaçao from international APIs: ILOSTAT SDMX (labour, with retries), OWID grapher CSVs (population, life expectancy, median age, CO₂), UN SDG area 531, UN Comtrade public preview reporter 531. EU trade with Curaçao (Eurostat `ext_lt_maineu`) runs with the `eurostat` timer | +| – | – | manual | CBCS (centralbank.cw) blocks scripts with a Cloudflare challenge: download the BoP xlsx by hand and load it into `stats.cw_cbcs_bop` (period, item, mln_ang) | Each timer starts `stats-collector@.service`, which runs `~/.local/bin/stats-collector-run.sh `: it updates a detached checkout of `origin/main` diff --git a/deploy/stats-collector/stats-cwintl.timer b/deploy/stats-collector/stats-cwintl.timer new file mode 100644 index 0000000..cd8c38c --- /dev/null +++ b/deploy/stats-collector/stats-cwintl.timer @@ -0,0 +1,11 @@ +[Unit] +Description=/stats collector: Curaçao from ILOSTAT, OWID, UN SDG and UN Comtrade + +[Timer] +OnCalendar=Wed *-*-* 05:50:00 Europe/Amsterdam +RandomizedDelaySec=20min +Persistent=true +Unit=stats-collector@cwintl.service + +[Install] +WantedBy=timers.target diff --git a/scripts/stats-fetch.ts b/scripts/stats-fetch.ts index 75d0607..662d243 100644 --- a/scripts/stats-fetch.ts +++ b/scripts/stats-fetch.ts @@ -11,6 +11,7 @@ // bun scripts/stats-fetch.ts usaspending // bun scripts/stats-fetch.ts eponline [--force] (needs EP_ONLINE_API_KEY) // bun scripts/stats-fetch.ts cbscw [--force] [--only=cpi,gdp,…] CBS Curaçao files +// bun scripts/stats-fetch.ts cwintl [--only=ilostat,owid,unsdg,comtrade] Curaçao via international APIs // bun scripts/stats-fetch.ts normalise-cbs rebuild CBS chart series from the raw rows import { ensureMeta, sql } from './stats/common'; import { cbs, normaliseCbs } from './stats/cbs'; @@ -25,6 +26,7 @@ import { rijksfinancien } from './stats/rijksfinancien'; import { ustreasury } from './stats/ustreasury'; import { usaspending } from './stats/usaspending'; import { eponline } from './stats/eponline'; +import { cwintl } from './stats/cwintl'; import { cwcbs } from './stats/cwcbs'; const [source, ...args] = process.argv.slice(2); @@ -74,12 +76,15 @@ try { case 'cbscw': await cwcbs({ force: flag('force') || flag('backfill'), only: value('only'), job }); break; + case 'cwintl': + await cwintl({ only: value('only'), job }); + break; case 'eponline': await eponline({ force: flag('force') || flag('backfill'), job }); break; default: throw new Error( - `Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, eponline, cbscw, normalise-cbs)` + `Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, eponline, cbscw, cwintl, normalise-cbs)` ); } } finally { diff --git a/scripts/stats/common.ts b/scripts/stats/common.ts index 8c7e30c..c505a3c 100644 --- a/scripts/stats/common.ts +++ b/scripts/stats/common.ts @@ -34,7 +34,10 @@ const hostGapMs: Record = { 'api.usaspending.gov': 500, 'public.ep-online.nl': 1000, 'www.cbs.cw': 2000, - 'cuatro.sim-cdn.nl': 1000 + 'cuatro.sim-cdn.nl': 1000, + 'sdmx.ilo.org': 3000, + 'unstats.un.org': 1500, + 'comtradeapi.un.org': 2500 }; const lastHit = new Map(); @@ -221,6 +224,30 @@ export const SOURCES = [ licence: 'No open licence published; all rights reserved, shown with attribution', key_required: false, attribution: 'Source: Centrale Bank van Curaçao en Sint Maarten (CBCS), all rights reserved' + }, + { + source_id: 'ilostat', + name: 'ILOSTAT (International Labour Organization)', + base_url: 'https://sdmx.ilo.org/rest', + licence: 'CC BY 4.0', + key_required: false, + attribution: 'Source: ILOSTAT, International Labour Organization, CC BY 4.0' + }, + { + source_id: 'unsdg', + name: 'UN SDG Global Database', + base_url: 'https://unstats.un.org/sdgs/UNSDGAPIV5', + licence: 'Free use with attribution (UN terms of use)', + key_required: false, + attribution: 'Source: United Nations Global SDG Database (UN Statistics Division)' + }, + { + source_id: 'comtrade', + name: 'UN Comtrade', + base_url: 'https://comtradeapi.un.org/public/v1/preview', + licence: 'Free use with attribution (UN Comtrade public preview)', + key_required: false, + attribution: 'Source: UN Comtrade Database (United Nations Statistics Division)' } ] as const; diff --git a/scripts/stats/cwcbs.ts b/scripts/stats/cwcbs.ts index ba5a831..75c454f 100644 --- a/scripts/stats/cwcbs.ts +++ b/scripts/stats/cwcbs.ts @@ -359,7 +359,9 @@ export const TABLES: Table[] = [ match: /labour force survey.*age.groups.*\.xlsx$/i, table: 'cw_cbs_lfs', owns: `age_group <> '15+'`, - parse: parseLfs + // The 15+ totals come from the by-sex file; older age-group uploads repeat them per sheet. + parse: async (path: string) => + (await parseLfs(path)).filter((r) => (r as { age_group: string }).age_group !== '15+') }, { key: 'tourism_cruise', @@ -421,7 +423,12 @@ async function links(page: string): Promise { const html = await (await http(`${SITE}/${page}`)).text(); const urls = new Set(); for (const m of html.replace(/&/g, '&').matchAll(CDN)) urls.add(m[0]); - return [...urls].sort((a, b) => decodeURIComponent(b).localeCompare(decodeURIComponent(a))); + // Plain code-point order (not localeCompare, which ignores punctuation): the CMS re-upload + // suffix `_0` then sorts ahead of the original, so the newest copy of a file wins. + return [...urls] + .map((u) => [u, decodeURIComponent(u)] as const) + .sort((a, b) => (a[1] < b[1] ? 1 : a[1] > b[1] ? -1 : 0)) + .map(([u]) => u); } const fileName = (url: string) => decodeURIComponent(url.split('/uploads/')[1] ?? url); diff --git a/scripts/stats/cwintl.ts b/scripts/stats/cwintl.ts new file mode 100644 index 0000000..4c85e64 --- /dev/null +++ b/scripts/stats/cwintl.ts @@ -0,0 +1,306 @@ +// §8 CW13–CW16: international open APIs that cover Curaçao (M49 531, ISO3 CUW). All small, +// keyless and replaced on every run; each source is its own fetch_run so one flaky API +// (ILOSTAT, Comtrade preview) does not hide the others. +import { parseCsv } from './owid'; +import { getJson, getText, log, run, writeSeries, type ObsRow, type SeriesRow } from './common'; + +const REGION = 'CUW'; +const THEME = 'curacao'; +const year = (y: string | number) => `${String(y).slice(0, 4)}-01-01`; +const toNum = (v: unknown) => { + const n = typeof v === 'number' ? v : Number(String(v ?? '').trim()); + return String(v ?? '').trim() !== '' && Number.isFinite(n) ? n : null; +}; +const slug = (s: string) => + s + .toLowerCase() + .replace(/\+/g, 'plus') + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + +/** Sequential retries on top of http()'s own: ILOSTAT answers 200 with an empty body or 403 at times. */ +async function retry(what: string, fn: () => Promise, ok: (t: T) => boolean, tries = 4) { + let last: unknown; + for (let i = 1; i <= tries; i++) { + try { + const t = await fn(); + if (ok(t)) return t; + last = new Error(`${what}: empty response`); + } catch (error) { + last = error; + } + await new Promise((r) => setTimeout(r, 5000 * i)); + } + throw last; +} + +// ── ILOSTAT (SDMX; the rplumber CSV endpoint in the spec returns empty bodies) ───────────── +const ILO_BASE = 'https://sdmx.ilo.org/rest/data/ILO'; +export const ILO_FLOWS: Record = { + DF_UNE_DEAP_SEX_AGE_RT: { title: 'Unemployment rate', unit: '%' }, + DF_EAP_DWAP_SEX_AGE_RT: { title: 'Labour force participation rate', unit: '%' }, + DF_EMP_DWAP_SEX_AGE_RT: { title: 'Employment-to-population ratio', unit: '%' } +}; +const ILO_AGES: Record = { + AGE_YTHADULT_YGE15: '15plus', + 'AGE_YTHADULT_Y15-24': '15-24', + AGE_YTHADULT_YGE25: '25plus' +}; +const ILO_SEX: Record = { SEX_T: 'total', SEX_M: 'm', SEX_F: 'f' }; + +async function ilostat(note: (w: string) => void) { + const series: SeriesRow[] = []; + const obs: ObsRow[] = []; + for (const [flow, meta] of Object.entries(ILO_FLOWS)) { + const url = `${ILO_BASE},${flow},1.0/${REGION}.A....?format=csv`; + let text: string; + try { + text = await retry( + flow, + () => getText(url), + (t) => t.startsWith('DATAFLOW') + ); + } catch (error) { + note(`${flow}: ${error instanceof Error ? error.message : String(error)}`); + continue; + } + const [head, ...rows] = parseCsv(text); + const col = (n: string) => head.indexOf(n); + const [cSex, cAge, cTime, cVal, cSrc] = [ + 'SEX', + 'AGE', + 'TIME_PERIOD', + 'OBS_VALUE', + 'SOURCE' + ].map(col); + const byId = new Map(); + for (const r of rows) { + const age = ILO_AGES[r[cAge]]; + const sex = ILO_SEX[r[cSex]]; + if (!age || !sex || !r[cTime]) continue; + const id = `ilostat:${flow.replace(/^DF_/, '').toLowerCase()}:${sex}:${age}`; + if (!byId.has(id)) { + byId.set(id, []); + series.push({ + series_id: id, + source_id: 'ilostat', + source_code: `${flow}/${r[cSex]}.${r[cAge]}`, + title: `Curaçao · ${meta.title} (${sex === 'total' ? 'total' : sex === 'm' ? 'men' : 'women'}, ${age.replace('plus', '+')})`, + unit: meta.unit, + freq: 'A', + region_code: REGION, + theme: THEME, + meta: { flow, sex: r[cSex], age: r[cAge], survey: r[cSrc] } + }); + } + byId + .get(id)! + .push({ series_id: id, period: year(r[cTime]), value: toNum(r[cVal]), status: null }); + } + for (const list of byId.values()) obs.push(...list); + } + const n = await writeSeries(series, obs); + log(`ilostat CUW: ${series.length} series, ${n} obs`); + return n; +} + +// ── OWID grapher CSVs (UN WPP / GCB via OWID, CC BY 4.0) ────────────────────────────────── +export const OWID_CW: Record = { + population: { col: 'population_historical', title: 'Population (UN WPP)', unit: 'people' }, + 'life-expectancy': { col: 'life_expectancy_0', title: 'Life expectancy at birth', unit: 'years' }, + 'median-age': { + col: 'median_age__sex_all__age_all__variant_estimates', + title: 'Median age', + unit: 'years' + }, + 'co-emissions-per-capita': { + col: 'emissions_total_per_capita', + title: 'CO₂ emissions per person', + unit: 't' + } +}; + +async function owidCw(note: (w: string) => void) { + const series: SeriesRow[] = []; + const obs: ObsRow[] = []; + for (const [slugName, meta] of Object.entries(OWID_CW)) { + try { + const text = await getText( + `https://ourworldindata.org/grapher/${slugName}.csv?country=${REGION}&csvType=filtered&useColumnShortNames=true` + ); + const [head, ...rows] = parseCsv(text); + const cCode = head.indexOf('code'); + const cYear = head.indexOf('year'); + const cVal = head.indexOf(meta.col); + if (cVal < 0) throw new Error(`column ${meta.col} missing`); + const id = `owid:cw:${slugName}`; + const list = rows + .filter((r) => r[cCode] === REGION && toNum(r[cVal]) !== null) + .map((r) => ({ + series_id: id, + period: year(r[cYear]), + value: toNum(r[cVal]), + status: null + })); + if (!list.length) throw new Error('no CUW rows'); + obs.push(...list); + series.push({ + series_id: id, + source_id: 'owid', + source_code: `grapher/${slugName}`, + title: `Curaçao · ${meta.title}`, + unit: meta.unit, + freq: 'A', + region_code: REGION, + theme: THEME, + meta: { url: `https://ourworldindata.org/grapher/${slugName}` } + }); + } catch (error) { + note(`owid ${slugName}: ${error instanceof Error ? error.message : String(error)}`); + } + } + const n = await writeSeries(series, obs); + log(`owid CUW: ${series.length} series, ${n} obs`); + return n; +} + +// ── UN SDG API (area 531) ───────────────────────────────────────────────────────────────── +export const SDG_SERIES: Record = { + EN_ATM_CO2: { title: 'CO₂ emissions from fuel combustion', unit: 'Mt' }, + EG_FEC_RNEW: { title: 'Renewable share of final energy consumption', unit: '%' }, + SP_DYN_ADKL: { title: 'Adolescent birth rate (per 1,000 women 15–19)', unit: 'per 1,000' }, + ER_PTD_TERR: { title: 'Terrestrial key biodiversity areas protected', unit: '%' } +}; +type SdgRow = { + timePeriodStart: number; + value: string; + dimensions: Record; + seriesDescription: string; +}; + +async function unsdg(note: (w: string) => void) { + const series: SeriesRow[] = []; + const obs: ObsRow[] = []; + for (const [code, meta] of Object.entries(SDG_SERIES)) { + try { + const d = await getJson<{ data: SdgRow[] }>( + `https://unstats.un.org/sdgs/UNSDGAPIV5/v1/sdg/Series/Data?seriesCode=${code}&areaCode=531&pageSize=2000` + ); + const byId = new Map>(); + for (const r of d.data) { + const dims = Object.entries(r.dimensions) + .filter(([k]) => k !== 'Reporting Type') + .sort(([a], [b]) => a.localeCompare(b)); + // SP_DYN_ADKL also carries the 10–14 group; keep the headline 15–19. + if (code === 'SP_DYN_ADKL' && r.dimensions.Age !== '15-19') continue; + const key = dims.map(([, v]) => slug(v)).join(':'); + const id = `unsdg:${code.toLowerCase()}${key ? `:${key}` : ''}`; + if (!byId.has(id)) { + byId.set(id, new Map()); + series.push({ + series_id: id, + source_id: 'unsdg', + source_code: `${code}/${dims.map(([k, v]) => `${k}=${v}`).join(',')}`, + title: `Curaçao · ${meta.title}${dims.length && code !== 'SP_DYN_ADKL' ? ` (${dims.map(([, v]) => v).join(', ')})` : ''}`, + unit: meta.unit, + freq: 'A', + region_code: REGION, + theme: THEME, + meta: { code, description: r.seriesDescription, dims: Object.fromEntries(dims) } + }); + } + // Global (G) and national (N) reporting can both appear; the first wins. + const p = year(r.timePeriodStart); + if (!byId.get(id)!.has(p)) byId.get(id)!.set(p, toNum(r.value)); + } + for (const [id, m] of byId) + for (const [period, value] of m) obs.push({ series_id: id, period, value, status: null }); + } catch (error) { + note(`${code}: ${error instanceof Error ? error.message : String(error)}`); + } + } + const n = await writeSeries(series, obs); + log(`unsdg 531: ${series.length} series, ${n} obs`); + return n; +} + +// ── UN Comtrade public preview (one period per call, ≤500 rows) ─────────────────────────── +const COMTRADE_PARTNERS: Record = { + 0: 'world', + 842: 'usa', + 528: 'nld', + 76: 'bra', + 862: 'ven', + 156: 'chn' +}; +type ComtradeRow = { period: string; flowCode: string; partnerCode: number; primaryValue: number }; + +async function comtrade(note: (w: string) => void) { + const series: SeriesRow[] = []; + const obs: ObsRow[] = []; + const seen = new Set(); + const last = new Date().getFullYear() - 1; + for (let y = 2010; y <= last; y++) { + let d: { data?: ComtradeRow[]; error?: string }; + try { + d = await getJson( + `https://comtradeapi.un.org/public/v1/preview/C/A/HS?reporterCode=531&period=${y}&flowCode=M,X&partnerCode=${Object.keys(COMTRADE_PARTNERS).join(',')}&cmdCode=TOTAL` + ); + } catch (error) { + note(`${y}: ${error instanceof Error ? error.message : String(error)}`); + continue; + } + if (d.error) note(`${y}: ${d.error}`); + for (const r of d.data ?? []) { + const partner = COMTRADE_PARTNERS[r.partnerCode]; + if (!partner) continue; + const flow = r.flowCode === 'M' ? 'imports' : r.flowCode === 'X' ? 'exports' : null; + if (!flow) continue; + const id = `comtrade:cw:${flow}:${partner}`; + if (!seen.has(id)) { + seen.add(id); + series.push({ + series_id: id, + source_id: 'comtrade', + source_code: `531/${r.flowCode}/${r.partnerCode}/TOTAL`, + title: `Curaçao · Goods ${flow}${partner === 'world' ? '' : ` (partner ${partner.toUpperCase()})`}`, + unit: 'US$', + freq: 'A', + region_code: REGION, + theme: THEME, + meta: { reporter: 531, flow: r.flowCode, partner: r.partnerCode } + }); + } + obs.push({ series_id: id, period: year(r.period), value: r.primaryValue, status: null }); + } + } + const n = await writeSeries(series, obs); + log(`comtrade 531: ${series.length} series, ${n} obs`); + return n; +} + +export const CW_INTL_JOBS = { ilostat, owid: owidCw, unsdg, comtrade } as const; +const SOURCE_OF: Record = { + ilostat: 'ilostat', + owid: 'owid', + unsdg: 'unsdg', + comtrade: 'comtrade' +}; + +export async function cwintl(opts: { only?: string; job: string }) { + let total = 0; + let failed = 0; + for (const [name, fn] of Object.entries(CW_INTL_JOBS) as [ + keyof typeof CW_INTL_JOBS, + typeof ilostat + ][]) { + if (opts.only && !opts.only.split(',').includes(name)) continue; + try { + total += await run(SOURCE_OF[name], `cw-${opts.job}`, fn); + } catch { + failed++; + } + } + if (failed) throw new Error(`cwintl: ${failed} job(s) failed`); + return total; +} diff --git a/scripts/stats/eurostat.ts b/scripts/stats/eurostat.ts index ba02ee5..a8761e6 100644 --- a/scripts/stats/eurostat.ts +++ b/scripts/stats/eurostat.ts @@ -51,6 +51,8 @@ type Dataset = { theme: string; params: Record; geos?: string[]; + /** Region for every series of the dataset (partner datasets: the partner, not the reporter). */ + region?: string; }; export const EUROSTAT_DATASETS: Dataset[] = [ @@ -66,6 +68,18 @@ export const EUROSTAT_DATASETS: Dataset[] = [ // GJ83P70 (band D3, 83.7 GJ) carries the pre-2007 history (was nrg_pc_202_h); it is only published per GJ (gross calorific value). params: { nrg_cons: ['GJ20-199', 'GJ83P70'], currency: 'EUR', unit: ['KWH', 'GJ_GCV'] } }, + { + // §8 CW12: EU27 trade in goods with Curaçao (million EUR). + code: 'ext_lt_maineu', + theme: 'curacao', + params: { + partner: 'CW', + sitc06: 'TOTAL', + indic_et: ['MIO_EXP_VAL', 'MIO_IMP_VAL', 'MIO_BAL_VAL', 'PC_EXP_PART', 'PC_IMP_PART'] + }, + geos: ['EU27_2020'], + region: 'CUW' + }, { code: 'nrg_ind_id', theme: 'energy', params: { siec: 'TOTAL' } }, { code: 'irt_lt_mcby_m', @@ -222,7 +236,7 @@ async function pull(d: Dataset): Promise { .join(', ')}`.replace(/ · $/, ''), unit, freq, - region_code: geo ? (GEO_REGION[geo.code] ?? null) : null, + region_code: d.region ?? (geo ? (GEO_REGION[geo.code] ?? null) : null), theme: d.theme, meta: { dataset: d.code, dims: Object.fromEntries(labels.map((l) => [l.id, l.code])) } }); diff --git a/src/lib/stats/curacao.ts b/src/lib/stats/curacao.ts new file mode 100644 index 0000000..8f32cd7 --- /dev/null +++ b/src/lib/stats/curacao.ts @@ -0,0 +1,64 @@ +// §8 Curaçao detail series (CBS Curaçao, Eurostat, ILOSTAT, OWID, UN SDG, UN Comtrade), all with +// region CUW in stats.series. Loaded by /stats/countries/CUW next to the World Bank profile. + +/** CBS Curaçao GDP by industry (ISIC 4 sections as published, mln ANG) with short labels. */ +export const CW_INDUSTRIES: Record = { + 'a+b': 'Agriculture & mining', + c: 'Manufacturing', + 'd+e': 'Utilities', + f: 'Construction', + g: 'Trade', + h: 'Transport', + i: 'Hotels & restaurants', + j: 'ICT', + k: 'Finance', + l: 'Real estate', + m: 'Professional services', + n: 'Admin services', + o: 'Public administration', + p: 'Education', + q: 'Health', + r: 'Arts & recreation', + 's+t': 'Other services' +}; + +export const CW_NIGHT_MARKETS: Record = { + europe: 'Europe', + north_america: 'North America', + south_america: 'South America', + caribbean: 'Caribbean', + others: 'Other' +}; + +export const EU_TRADE = (indic: string) => `eurostat:ext_lt_maineu:${indic}.TOTAL.CW.EU27_2020`; + +export const CW_SERIES = [ + 'cw:cpi:index', + 'cw:cpi:yoy', + 'cw:gdp:gdp', + 'cw:gdp:real_growth_pct', + ...Object.keys(CW_INDUSTRIES).map((k) => `cw:gdp:${k}`), + 'cw:gov:r.total', + 'cw:gov:e.total', + 'cw:gov:balance', + 'cw:pop:jan1', + 'cw:pop:census', + ...['total', 'm', 'f', '15-24'].map((g) => `cw:lfs:unemployment_rate_pct:${g}`), + 'cw:lfs:net_participation_pct:total', + 'cw:tour:cruise_pax', + 'cw:tour:daytrippers', + 'cw:tour:nights', + ...Object.keys(CW_NIGHT_MARKETS).map((m) => `cw:tour:nights:${m}`), + EU_TRADE('MIO_EXP_VAL'), + EU_TRADE('MIO_IMP_VAL'), + EU_TRADE('MIO_BAL_VAL'), + 'ilostat:une_deap_sex_age_rt:total:15plus', + 'owid:cw:population', + 'owid:cw:co-emissions-per-capita', + 'owid:cw:median-age', + 'unsdg:eg_fec_rnew', + 'comtrade:cw:imports:world', + 'comtrade:cw:exports:world', + 'comtrade:cw:imports:usa', + 'comtrade:cw:imports:nld' +]; diff --git a/src/routes/stats/countries/[iso3]/+page.server.ts b/src/routes/stats/countries/[iso3]/+page.server.ts index d5ea01c..8a19e29 100644 --- a/src/routes/stats/countries/[iso3]/+page.server.ts +++ b/src/routes/stats/countries/[iso3]/+page.server.ts @@ -1,6 +1,7 @@ import { error, redirect } from '@sveltejs/kit'; import { cached, loadSeries } from '#lib/server/stats.js'; import { isCountryPage, PROFILE_SERIES } from '#lib/stats/countries.js'; +import { CW_SERIES } from '#lib/stats/curacao.js'; import type { PageServerLoad } from './$types'; export const load: PageServerLoad = async ({ params, url }) => { @@ -16,7 +17,9 @@ export const load: PageServerLoad = async ({ params, url }) => { `wb:NY.GDP.PCAP.PP.KD:${c}`, `wb:SP.DYN.LE00.IN:${c}`, `wb:FP.CPI.TOTL.ZG:${c}` - ]) + ]), + // §8: Curaçao detail from CBS Curaçao and the international APIs (ids carry no code suffix). + ...(code === 'CUW' ? CW_SERIES : []) ]) ); return { code, series }; diff --git a/src/routes/stats/countries/[iso3]/+page.svelte b/src/routes/stats/countries/[iso3]/+page.svelte index 14ce7c0..d265520 100644 --- a/src/routes/stats/countries/[iso3]/+page.svelte +++ b/src/routes/stats/countries/[iso3]/+page.svelte @@ -2,7 +2,15 @@ import { resolve } from '$app/paths'; import StatChart from '#lib/stats/StatChart.svelte'; import Kpi from '#lib/stats/Kpi.svelte'; - import { change, latest, lineOption, type Line, type StatPoint } from '#lib/stats/chart.js'; + import { + barRankOption, + change, + latest, + lineOption, + type Line, + type StatPoint + } from '#lib/stats/chart.js'; + import { CW_INDUSTRIES, CW_NIGHT_MARKETS, EU_TRADE } from '#lib/stats/curacao.js'; import { compact, num, period } from '#lib/stats/format.js'; import { COUNTRY } from '#lib/stats/regions.js'; import { COUNTRY_COLOR, FLAG } from '#lib/stats/countries.js'; @@ -54,6 +62,23 @@ const kDebt = $derived(lastOf(pick('imf:GGXWDG_NGDP', 'wb:GC.DOD.TOTL.GD.ZS'))); const popDelta = $derived(change(pts('wb:SP.POP.TOTL'), 10)?.pct ?? null); const imfCovered = $derived(has('imf:NGDP_RPCH')); + + // §8 Curaçao detail series: stored without the `:CUW` suffix. + const cw = (id: string): StatPoint[] => s[id]?.points ?? []; + const isCw = $derived(c === 'CUW' && cw('cw:cpi:index').length > 0); + const cwGdpYear = $derived(latest(cw('cw:gdp:gdp'))?.[0]); + const cwIndustries = $derived( + Object.entries(CW_INDUSTRIES) + .map(([k, label]) => ({ + label, + value: cw(`cw:gdp:${k}`).find(([d]) => d === cwGdpYear)?.[1] ?? NaN + })) + .filter((r) => Number.isFinite(r.value)) + ); + const cwCpi = $derived(latest(cw('cw:cpi:yoy'))); + const cwPop = $derived(latest(cw('cw:pop:jan1'))); + const cwUnemp = $derived(latest(cw('cw:lfs:unemployment_rate_pct:total'))); + const cwGrowth = $derived(latest(cw('cw:gdp:real_growth_pct'))); @@ -73,9 +98,11 @@

{#if c === 'CUW'} Curaçao has been a country within the Kingdom of the Netherlands since 2010, with about - 150,000 inhabitants. The IMF's World Economic Outlook doesn't cover it, so the headline - numbers here come from the World Bank; detailed CBS Curaçao and central bank series are on - their way. + 158,000 inhabitants. The IMF's World Economic Outlook doesn't cover it, so the headline + numbers come from the World Bank; the detail further down comes from CBS Curaçao (the island's + statistics office), Eurostat, ILOSTAT, Our World in Data and the UN. Amounts in guilders (ANG, + pegged at 1.79 to the US dollar). The central bank's balance-of-payments tables can't be + fetched automatically, so they aren't here yet. {:else} {name} at a glance, from the World Bank's development indicators and the IMF World Economic Outlook{imfCovered ? ' (dashed lines are IMF forecasts)' @@ -365,33 +392,35 @@

Government and energy

- - lineOption( - t, - has('imf:GGXWDG_NGDP') - ? [ - ...imfLines('imf:GGXWDG_NGDP', 'Debt', '#e11d48'), - ...imfLines('imf:GGR_G01_GDP_PT', 'Revenue', '#10b981'), - ...imfLines('imf:G_X_G01_GDP_PT', 'Spending', '#3b82f6') - ] - : [ - { name: 'Debt', points: pts('wb:GC.DOD.TOTL.GD.ZS'), color: '#e11d48' }, - { name: 'Revenue', points: pts('wb:GC.REV.XGRT.GD.ZS'), color: '#10b981' }, - { name: 'Expense', points: pts('wb:GC.XPN.TOTL.GD.ZS'), color: '#3b82f6' } - ], - { unit: '% GDP', decimals: 1 } - )} - note={!has('imf:GGXWDG_NGDP') && !has('wb:GC.DOD.TOTL.GD.ZS') - ? 'No government finance data for this country yet.' - : ''} - /> + {#if !isCw || has('imf:GGXWDG_NGDP') || has('wb:GC.DOD.TOTL.GD.ZS')} + + lineOption( + t, + has('imf:GGXWDG_NGDP') + ? [ + ...imfLines('imf:GGXWDG_NGDP', 'Debt', '#e11d48'), + ...imfLines('imf:GGR_G01_GDP_PT', 'Revenue', '#10b981'), + ...imfLines('imf:G_X_G01_GDP_PT', 'Spending', '#3b82f6') + ] + : [ + { name: 'Debt', points: pts('wb:GC.DOD.TOTL.GD.ZS'), color: '#e11d48' }, + { name: 'Revenue', points: pts('wb:GC.REV.XGRT.GD.ZS'), color: '#10b981' }, + { name: 'Expense', points: pts('wb:GC.XPN.TOTL.GD.ZS'), color: '#3b82f6' } + ], + { unit: '% GDP', decimals: 1 } + )} + note={!has('imf:GGXWDG_NGDP') && !has('wb:GC.DOD.TOTL.GD.ZS') + ? 'No government finance data for this country yet.' + : ''} + /> + {/if}
+{#if isCw} +

Curaçao in detail

+
+ + + + +
+
+ + lineOption( + t, + [ + { name: 'CPI', points: cw('cw:cpi:index'), color: '#64748b', kind: 'area' }, + { name: 'Inflation', points: cw('cw:cpi:yoy'), color: '#e11d48', right: true } + ], + { unit: 'index', rightUnit: '%', decimals: 1, zoom: true } + )} + /> + + lineOption( + t, + [ + { name: 'GDP', points: cw('cw:gdp:gdp'), color, kind: 'bar' }, + { + name: 'Real growth', + points: cw('cw:gdp:real_growth_pct'), + color: '#3b82f6', + right: true + } + ], + { unit: 'mln ANG', rightUnit: '%', decimals: 1 } + )} + /> + barRankOption(t, cwIndustries, { decimals: 0, unit: 'mln ANG', color })} + /> + + lineOption( + t, + [ + { name: 'Population', points: cw('cw:pop:jan1'), color, kind: 'area' }, + { name: 'Census', points: cw('cw:pop:census'), color: '#0f172a', kind: 'bar' }, + { + name: 'UN WPP (via OWID)', + points: cw('owid:cw:population'), + color: '#94a3b8', + dashed: true + } + ], + { decimals: 0, unit: 'people' } + )} + /> + + lineOption( + t, + [ + { name: 'Total', points: cw('cw:lfs:unemployment_rate_pct:total'), color: '#a855f7' }, + { name: 'Men', points: cw('cw:lfs:unemployment_rate_pct:m'), color: '#3b82f6' }, + { name: 'Women', points: cw('cw:lfs:unemployment_rate_pct:f'), color: '#ec4899' }, + { + name: 'Aged 15–24', + points: cw('cw:lfs:unemployment_rate_pct:15-24'), + color: '#f97316' + }, + { + name: 'ILO estimate (15+)', + points: cw('ilostat:une_deap_sex_age_rt:total:15plus'), + color: '#94a3b8', + dashed: true + } + ], + { unit: '%', decimals: 1 } + )} + /> + + lineOption( + t, + [ + { + name: 'Cruise passengers', + points: cw('cw:tour:cruise_pax'), + color: '#06b6d4', + kind: 'bar' + }, + { + name: 'Daytrippers', + points: cw('cw:tour:daytrippers'), + color: '#eab308', + kind: 'bar' + }, + { + name: 'Stay-over nights', + points: cw('cw:tour:nights'), + color: '#e11d48', + right: true + } + ], + { decimals: 0, rightUnit: 'nights' } + )} + /> + + lineOption( + t, + Object.entries(CW_NIGHT_MARKETS).map(([k, label], i) => ({ + name: label, + points: cw(`cw:tour:nights:${k}`), + color: ['#3b82f6', '#e11d48', '#10b981', '#eab308', '#94a3b8'][i], + kind: 'bar' as const, + stack: 'nights' + })), + { decimals: 0, unit: 'nights' } + )} + /> + + lineOption( + t, + [ + { name: 'Revenue', points: cw('cw:gov:r.total'), color: '#10b981' }, + { name: 'Expenditure', points: cw('cw:gov:e.total'), color: '#3b82f6' }, + { name: 'Balance', points: cw('cw:gov:balance'), color: '#e11d48', kind: 'bar' } + ], + { unit: 'mln ANG', decimals: 0 } + )} + /> + + lineOption( + t, + [ + { + name: 'EU exports to Curaçao', + points: cw(EU_TRADE('MIO_EXP_VAL')), + color: '#3b82f6' + }, + { + name: 'EU imports from Curaçao', + points: cw(EU_TRADE('MIO_IMP_VAL')), + color: '#f97316' + }, + { + name: 'EU balance', + points: cw(EU_TRADE('MIO_BAL_VAL')), + color: '#94a3b8', + kind: 'bar' + } + ], + { unit: 'mln €', decimals: 0 } + )} + /> + + lineOption( + t, + [ + { + name: 'Imports', + points: cw('comtrade:cw:imports:world'), + color: '#e11d48', + kind: 'bar' + }, + { + name: 'Exports', + points: cw('comtrade:cw:exports:world'), + color: '#10b981', + kind: 'bar' + }, + { + name: 'Imports from the US', + points: cw('comtrade:cw:imports:usa'), + color: '#3b82f6' + }, + { name: 'Imports from NL', points: cw('comtrade:cw:imports:nld'), color: '#f97316' } + ], + { decimals: 0, unit: 'US$' } + )} + /> + + lineOption( + t, + [ + { + name: 'CO₂ per person', + points: cw('owid:cw:co-emissions-per-capita'), + color: '#64748b' + }, + { + name: 'Renewable share', + points: cw('unsdg:eg_fec_rnew'), + color: '#10b981', + right: true + } + ], + { unit: 't', rightUnit: '%', decimals: 1 } + )} + /> +
+{/if} +