stats §8: Curaçao from ILOSTAT (SDMX), OWID, UN SDG 531, UN Comtrade 531 (cwintl + weekly timer), EU trade with CW (Eurostat ext_lt_maineu); CBS Curaçao LFS file pick fix; Curaçao detail panels on /stats/countries/CUW
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
This commit is contained in:
306
scripts/stats/cwintl.ts
Normal file
306
scripts/stats/cwintl.ts
Normal file
@@ -0,0 +1,306 @@
|
||||
// §8 CW13–CW16: international open APIs that cover Curaçao (M49 531, ISO3 CUW). All small,
|
||||
// keyless and replaced on every run; each source is its own fetch_run so one flaky API
|
||||
// (ILOSTAT, Comtrade preview) does not hide the others.
|
||||
import { parseCsv } from './owid';
|
||||
import { getJson, getText, log, run, writeSeries, type ObsRow, type SeriesRow } from './common';
|
||||
|
||||
const REGION = 'CUW';
|
||||
const THEME = 'curacao';
|
||||
const year = (y: string | number) => `${String(y).slice(0, 4)}-01-01`;
|
||||
const toNum = (v: unknown) => {
|
||||
const n = typeof v === 'number' ? v : Number(String(v ?? '').trim());
|
||||
return String(v ?? '').trim() !== '' && Number.isFinite(n) ? n : null;
|
||||
};
|
||||
const slug = (s: string) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/\+/g, 'plus')
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
|
||||
/** Sequential retries on top of http()'s own: ILOSTAT answers 200 with an empty body or 403 at times. */
|
||||
async function retry<T>(what: string, fn: () => Promise<T>, ok: (t: T) => boolean, tries = 4) {
|
||||
let last: unknown;
|
||||
for (let i = 1; i <= tries; i++) {
|
||||
try {
|
||||
const t = await fn();
|
||||
if (ok(t)) return t;
|
||||
last = new Error(`${what}: empty response`);
|
||||
} catch (error) {
|
||||
last = error;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 5000 * i));
|
||||
}
|
||||
throw last;
|
||||
}
|
||||
|
||||
// ── ILOSTAT (SDMX; the rplumber CSV endpoint in the spec returns empty bodies) ─────────────
|
||||
const ILO_BASE = 'https://sdmx.ilo.org/rest/data/ILO';
|
||||
export const ILO_FLOWS: Record<string, { title: string; unit: string }> = {
|
||||
DF_UNE_DEAP_SEX_AGE_RT: { title: 'Unemployment rate', unit: '%' },
|
||||
DF_EAP_DWAP_SEX_AGE_RT: { title: 'Labour force participation rate', unit: '%' },
|
||||
DF_EMP_DWAP_SEX_AGE_RT: { title: 'Employment-to-population ratio', unit: '%' }
|
||||
};
|
||||
const ILO_AGES: Record<string, string> = {
|
||||
AGE_YTHADULT_YGE15: '15plus',
|
||||
'AGE_YTHADULT_Y15-24': '15-24',
|
||||
AGE_YTHADULT_YGE25: '25plus'
|
||||
};
|
||||
const ILO_SEX: Record<string, string> = { SEX_T: 'total', SEX_M: 'm', SEX_F: 'f' };
|
||||
|
||||
async function ilostat(note: (w: string) => void) {
|
||||
const series: SeriesRow[] = [];
|
||||
const obs: ObsRow[] = [];
|
||||
for (const [flow, meta] of Object.entries(ILO_FLOWS)) {
|
||||
const url = `${ILO_BASE},${flow},1.0/${REGION}.A....?format=csv`;
|
||||
let text: string;
|
||||
try {
|
||||
text = await retry(
|
||||
flow,
|
||||
() => getText(url),
|
||||
(t) => t.startsWith('DATAFLOW')
|
||||
);
|
||||
} catch (error) {
|
||||
note(`${flow}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
continue;
|
||||
}
|
||||
const [head, ...rows] = parseCsv(text);
|
||||
const col = (n: string) => head.indexOf(n);
|
||||
const [cSex, cAge, cTime, cVal, cSrc] = [
|
||||
'SEX',
|
||||
'AGE',
|
||||
'TIME_PERIOD',
|
||||
'OBS_VALUE',
|
||||
'SOURCE'
|
||||
].map(col);
|
||||
const byId = new Map<string, ObsRow[]>();
|
||||
for (const r of rows) {
|
||||
const age = ILO_AGES[r[cAge]];
|
||||
const sex = ILO_SEX[r[cSex]];
|
||||
if (!age || !sex || !r[cTime]) continue;
|
||||
const id = `ilostat:${flow.replace(/^DF_/, '').toLowerCase()}:${sex}:${age}`;
|
||||
if (!byId.has(id)) {
|
||||
byId.set(id, []);
|
||||
series.push({
|
||||
series_id: id,
|
||||
source_id: 'ilostat',
|
||||
source_code: `${flow}/${r[cSex]}.${r[cAge]}`,
|
||||
title: `Curaçao · ${meta.title} (${sex === 'total' ? 'total' : sex === 'm' ? 'men' : 'women'}, ${age.replace('plus', '+')})`,
|
||||
unit: meta.unit,
|
||||
freq: 'A',
|
||||
region_code: REGION,
|
||||
theme: THEME,
|
||||
meta: { flow, sex: r[cSex], age: r[cAge], survey: r[cSrc] }
|
||||
});
|
||||
}
|
||||
byId
|
||||
.get(id)!
|
||||
.push({ series_id: id, period: year(r[cTime]), value: toNum(r[cVal]), status: null });
|
||||
}
|
||||
for (const list of byId.values()) obs.push(...list);
|
||||
}
|
||||
const n = await writeSeries(series, obs);
|
||||
log(`ilostat CUW: ${series.length} series, ${n} obs`);
|
||||
return n;
|
||||
}
|
||||
|
||||
// ── OWID grapher CSVs (UN WPP / GCB via OWID, CC BY 4.0) ──────────────────────────────────
|
||||
export const OWID_CW: Record<string, { col: string; title: string; unit: string }> = {
|
||||
population: { col: 'population_historical', title: 'Population (UN WPP)', unit: 'people' },
|
||||
'life-expectancy': { col: 'life_expectancy_0', title: 'Life expectancy at birth', unit: 'years' },
|
||||
'median-age': {
|
||||
col: 'median_age__sex_all__age_all__variant_estimates',
|
||||
title: 'Median age',
|
||||
unit: 'years'
|
||||
},
|
||||
'co-emissions-per-capita': {
|
||||
col: 'emissions_total_per_capita',
|
||||
title: 'CO₂ emissions per person',
|
||||
unit: 't'
|
||||
}
|
||||
};
|
||||
|
||||
async function owidCw(note: (w: string) => void) {
|
||||
const series: SeriesRow[] = [];
|
||||
const obs: ObsRow[] = [];
|
||||
for (const [slugName, meta] of Object.entries(OWID_CW)) {
|
||||
try {
|
||||
const text = await getText(
|
||||
`https://ourworldindata.org/grapher/${slugName}.csv?country=${REGION}&csvType=filtered&useColumnShortNames=true`
|
||||
);
|
||||
const [head, ...rows] = parseCsv(text);
|
||||
const cCode = head.indexOf('code');
|
||||
const cYear = head.indexOf('year');
|
||||
const cVal = head.indexOf(meta.col);
|
||||
if (cVal < 0) throw new Error(`column ${meta.col} missing`);
|
||||
const id = `owid:cw:${slugName}`;
|
||||
const list = rows
|
||||
.filter((r) => r[cCode] === REGION && toNum(r[cVal]) !== null)
|
||||
.map((r) => ({
|
||||
series_id: id,
|
||||
period: year(r[cYear]),
|
||||
value: toNum(r[cVal]),
|
||||
status: null
|
||||
}));
|
||||
if (!list.length) throw new Error('no CUW rows');
|
||||
obs.push(...list);
|
||||
series.push({
|
||||
series_id: id,
|
||||
source_id: 'owid',
|
||||
source_code: `grapher/${slugName}`,
|
||||
title: `Curaçao · ${meta.title}`,
|
||||
unit: meta.unit,
|
||||
freq: 'A',
|
||||
region_code: REGION,
|
||||
theme: THEME,
|
||||
meta: { url: `https://ourworldindata.org/grapher/${slugName}` }
|
||||
});
|
||||
} catch (error) {
|
||||
note(`owid ${slugName}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
const n = await writeSeries(series, obs);
|
||||
log(`owid CUW: ${series.length} series, ${n} obs`);
|
||||
return n;
|
||||
}
|
||||
|
||||
// ── UN SDG API (area 531) ─────────────────────────────────────────────────────────────────
|
||||
export const SDG_SERIES: Record<string, { title: string; unit: string }> = {
|
||||
EN_ATM_CO2: { title: 'CO₂ emissions from fuel combustion', unit: 'Mt' },
|
||||
EG_FEC_RNEW: { title: 'Renewable share of final energy consumption', unit: '%' },
|
||||
SP_DYN_ADKL: { title: 'Adolescent birth rate (per 1,000 women 15–19)', unit: 'per 1,000' },
|
||||
ER_PTD_TERR: { title: 'Terrestrial key biodiversity areas protected', unit: '%' }
|
||||
};
|
||||
type SdgRow = {
|
||||
timePeriodStart: number;
|
||||
value: string;
|
||||
dimensions: Record<string, string>;
|
||||
seriesDescription: string;
|
||||
};
|
||||
|
||||
async function unsdg(note: (w: string) => void) {
|
||||
const series: SeriesRow[] = [];
|
||||
const obs: ObsRow[] = [];
|
||||
for (const [code, meta] of Object.entries(SDG_SERIES)) {
|
||||
try {
|
||||
const d = await getJson<{ data: SdgRow[] }>(
|
||||
`https://unstats.un.org/sdgs/UNSDGAPIV5/v1/sdg/Series/Data?seriesCode=${code}&areaCode=531&pageSize=2000`
|
||||
);
|
||||
const byId = new Map<string, Map<string, number | null>>();
|
||||
for (const r of d.data) {
|
||||
const dims = Object.entries(r.dimensions)
|
||||
.filter(([k]) => k !== 'Reporting Type')
|
||||
.sort(([a], [b]) => a.localeCompare(b));
|
||||
// SP_DYN_ADKL also carries the 10–14 group; keep the headline 15–19.
|
||||
if (code === 'SP_DYN_ADKL' && r.dimensions.Age !== '15-19') continue;
|
||||
const key = dims.map(([, v]) => slug(v)).join(':');
|
||||
const id = `unsdg:${code.toLowerCase()}${key ? `:${key}` : ''}`;
|
||||
if (!byId.has(id)) {
|
||||
byId.set(id, new Map());
|
||||
series.push({
|
||||
series_id: id,
|
||||
source_id: 'unsdg',
|
||||
source_code: `${code}/${dims.map(([k, v]) => `${k}=${v}`).join(',')}`,
|
||||
title: `Curaçao · ${meta.title}${dims.length && code !== 'SP_DYN_ADKL' ? ` (${dims.map(([, v]) => v).join(', ')})` : ''}`,
|
||||
unit: meta.unit,
|
||||
freq: 'A',
|
||||
region_code: REGION,
|
||||
theme: THEME,
|
||||
meta: { code, description: r.seriesDescription, dims: Object.fromEntries(dims) }
|
||||
});
|
||||
}
|
||||
// Global (G) and national (N) reporting can both appear; the first wins.
|
||||
const p = year(r.timePeriodStart);
|
||||
if (!byId.get(id)!.has(p)) byId.get(id)!.set(p, toNum(r.value));
|
||||
}
|
||||
for (const [id, m] of byId)
|
||||
for (const [period, value] of m) obs.push({ series_id: id, period, value, status: null });
|
||||
} catch (error) {
|
||||
note(`${code}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
const n = await writeSeries(series, obs);
|
||||
log(`unsdg 531: ${series.length} series, ${n} obs`);
|
||||
return n;
|
||||
}
|
||||
|
||||
// ── UN Comtrade public preview (one period per call, ≤500 rows) ───────────────────────────
|
||||
const COMTRADE_PARTNERS: Record<number, string> = {
|
||||
0: 'world',
|
||||
842: 'usa',
|
||||
528: 'nld',
|
||||
76: 'bra',
|
||||
862: 'ven',
|
||||
156: 'chn'
|
||||
};
|
||||
type ComtradeRow = { period: string; flowCode: string; partnerCode: number; primaryValue: number };
|
||||
|
||||
async function comtrade(note: (w: string) => void) {
|
||||
const series: SeriesRow[] = [];
|
||||
const obs: ObsRow[] = [];
|
||||
const seen = new Set<string>();
|
||||
const last = new Date().getFullYear() - 1;
|
||||
for (let y = 2010; y <= last; y++) {
|
||||
let d: { data?: ComtradeRow[]; error?: string };
|
||||
try {
|
||||
d = await getJson(
|
||||
`https://comtradeapi.un.org/public/v1/preview/C/A/HS?reporterCode=531&period=${y}&flowCode=M,X&partnerCode=${Object.keys(COMTRADE_PARTNERS).join(',')}&cmdCode=TOTAL`
|
||||
);
|
||||
} catch (error) {
|
||||
note(`${y}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
continue;
|
||||
}
|
||||
if (d.error) note(`${y}: ${d.error}`);
|
||||
for (const r of d.data ?? []) {
|
||||
const partner = COMTRADE_PARTNERS[r.partnerCode];
|
||||
if (!partner) continue;
|
||||
const flow = r.flowCode === 'M' ? 'imports' : r.flowCode === 'X' ? 'exports' : null;
|
||||
if (!flow) continue;
|
||||
const id = `comtrade:cw:${flow}:${partner}`;
|
||||
if (!seen.has(id)) {
|
||||
seen.add(id);
|
||||
series.push({
|
||||
series_id: id,
|
||||
source_id: 'comtrade',
|
||||
source_code: `531/${r.flowCode}/${r.partnerCode}/TOTAL`,
|
||||
title: `Curaçao · Goods ${flow}${partner === 'world' ? '' : ` (partner ${partner.toUpperCase()})`}`,
|
||||
unit: 'US$',
|
||||
freq: 'A',
|
||||
region_code: REGION,
|
||||
theme: THEME,
|
||||
meta: { reporter: 531, flow: r.flowCode, partner: r.partnerCode }
|
||||
});
|
||||
}
|
||||
obs.push({ series_id: id, period: year(r.period), value: r.primaryValue, status: null });
|
||||
}
|
||||
}
|
||||
const n = await writeSeries(series, obs);
|
||||
log(`comtrade 531: ${series.length} series, ${n} obs`);
|
||||
return n;
|
||||
}
|
||||
|
||||
export const CW_INTL_JOBS = { ilostat, owid: owidCw, unsdg, comtrade } as const;
|
||||
const SOURCE_OF: Record<keyof typeof CW_INTL_JOBS, string> = {
|
||||
ilostat: 'ilostat',
|
||||
owid: 'owid',
|
||||
unsdg: 'unsdg',
|
||||
comtrade: 'comtrade'
|
||||
};
|
||||
|
||||
export async function cwintl(opts: { only?: string; job: string }) {
|
||||
let total = 0;
|
||||
let failed = 0;
|
||||
for (const [name, fn] of Object.entries(CW_INTL_JOBS) as [
|
||||
keyof typeof CW_INTL_JOBS,
|
||||
typeof ilostat
|
||||
][]) {
|
||||
if (opts.only && !opts.only.split(',').includes(name)) continue;
|
||||
try {
|
||||
total += await run(SOURCE_OF[name], `cw-${opts.job}`, fn);
|
||||
} catch {
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
if (failed) throw new Error(`cwintl: ${failed} job(s) failed`);
|
||||
return total;
|
||||
}
|
||||
Reference in New Issue
Block a user