Stan Runge
031ee4baf2
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
307 lines
11 KiB
TypeScript
307 lines
11 KiB
TypeScript
// §8 CW13–CW16: international open APIs that cover Curaçao (M49 531, ISO3 CUW). All small,
|
||
// keyless and replaced on every run; each source is its own fetch_run so one flaky API
|
||
// (ILOSTAT, Comtrade preview) does not hide the others.
|
||
import { parseCsv } from './owid';
|
||
import { getJson, getText, log, run, writeSeries, type ObsRow, type SeriesRow } from './common';
|
||
|
||
const REGION = 'CUW';
|
||
const THEME = 'curacao';
|
||
const year = (y: string | number) => `${String(y).slice(0, 4)}-01-01`;
|
||
const toNum = (v: unknown) => {
|
||
const n = typeof v === 'number' ? v : Number(String(v ?? '').trim());
|
||
return String(v ?? '').trim() !== '' && Number.isFinite(n) ? n : null;
|
||
};
|
||
const slug = (s: string) =>
|
||
s
|
||
.toLowerCase()
|
||
.replace(/\+/g, 'plus')
|
||
.replace(/[^a-z0-9]+/g, '-')
|
||
.replace(/^-|-$/g, '');
|
||
|
||
/** Sequential retries on top of http()'s own: ILOSTAT answers 200 with an empty body or 403 at times. */
|
||
async function retry<T>(what: string, fn: () => Promise<T>, ok: (t: T) => boolean, tries = 4) {
|
||
let last: unknown;
|
||
for (let i = 1; i <= tries; i++) {
|
||
try {
|
||
const t = await fn();
|
||
if (ok(t)) return t;
|
||
last = new Error(`${what}: empty response`);
|
||
} catch (error) {
|
||
last = error;
|
||
}
|
||
await new Promise((r) => setTimeout(r, 5000 * i));
|
||
}
|
||
throw last;
|
||
}
|
||
|
||
// ── ILOSTAT (SDMX; the rplumber CSV endpoint in the spec returns empty bodies) ─────────────
|
||
const ILO_BASE = 'https://sdmx.ilo.org/rest/data/ILO';
|
||
export const ILO_FLOWS: Record<string, { title: string; unit: string }> = {
|
||
DF_UNE_DEAP_SEX_AGE_RT: { title: 'Unemployment rate', unit: '%' },
|
||
DF_EAP_DWAP_SEX_AGE_RT: { title: 'Labour force participation rate', unit: '%' },
|
||
DF_EMP_DWAP_SEX_AGE_RT: { title: 'Employment-to-population ratio', unit: '%' }
|
||
};
|
||
const ILO_AGES: Record<string, string> = {
|
||
AGE_YTHADULT_YGE15: '15plus',
|
||
'AGE_YTHADULT_Y15-24': '15-24',
|
||
AGE_YTHADULT_YGE25: '25plus'
|
||
};
|
||
const ILO_SEX: Record<string, string> = { SEX_T: 'total', SEX_M: 'm', SEX_F: 'f' };
|
||
|
||
async function ilostat(note: (w: string) => void) {
|
||
const series: SeriesRow[] = [];
|
||
const obs: ObsRow[] = [];
|
||
for (const [flow, meta] of Object.entries(ILO_FLOWS)) {
|
||
const url = `${ILO_BASE},${flow},1.0/${REGION}.A....?format=csv`;
|
||
let text: string;
|
||
try {
|
||
text = await retry(
|
||
flow,
|
||
() => getText(url),
|
||
(t) => t.startsWith('DATAFLOW')
|
||
);
|
||
} catch (error) {
|
||
note(`${flow}: ${error instanceof Error ? error.message : String(error)}`);
|
||
continue;
|
||
}
|
||
const [head, ...rows] = parseCsv(text);
|
||
const col = (n: string) => head.indexOf(n);
|
||
const [cSex, cAge, cTime, cVal, cSrc] = [
|
||
'SEX',
|
||
'AGE',
|
||
'TIME_PERIOD',
|
||
'OBS_VALUE',
|
||
'SOURCE'
|
||
].map(col);
|
||
const byId = new Map<string, ObsRow[]>();
|
||
for (const r of rows) {
|
||
const age = ILO_AGES[r[cAge]];
|
||
const sex = ILO_SEX[r[cSex]];
|
||
if (!age || !sex || !r[cTime]) continue;
|
||
const id = `ilostat:${flow.replace(/^DF_/, '').toLowerCase()}:${sex}:${age}`;
|
||
if (!byId.has(id)) {
|
||
byId.set(id, []);
|
||
series.push({
|
||
series_id: id,
|
||
source_id: 'ilostat',
|
||
source_code: `${flow}/${r[cSex]}.${r[cAge]}`,
|
||
title: `Curaçao · ${meta.title} (${sex === 'total' ? 'total' : sex === 'm' ? 'men' : 'women'}, ${age.replace('plus', '+')})`,
|
||
unit: meta.unit,
|
||
freq: 'A',
|
||
region_code: REGION,
|
||
theme: THEME,
|
||
meta: { flow, sex: r[cSex], age: r[cAge], survey: r[cSrc] }
|
||
});
|
||
}
|
||
byId
|
||
.get(id)!
|
||
.push({ series_id: id, period: year(r[cTime]), value: toNum(r[cVal]), status: null });
|
||
}
|
||
for (const list of byId.values()) obs.push(...list);
|
||
}
|
||
const n = await writeSeries(series, obs);
|
||
log(`ilostat CUW: ${series.length} series, ${n} obs`);
|
||
return n;
|
||
}
|
||
|
||
// ── OWID grapher CSVs (UN WPP / GCB via OWID, CC BY 4.0) ──────────────────────────────────
|
||
export const OWID_CW: Record<string, { col: string; title: string; unit: string }> = {
|
||
population: { col: 'population_historical', title: 'Population (UN WPP)', unit: 'people' },
|
||
'life-expectancy': { col: 'life_expectancy_0', title: 'Life expectancy at birth', unit: 'years' },
|
||
'median-age': {
|
||
col: 'median_age__sex_all__age_all__variant_estimates',
|
||
title: 'Median age',
|
||
unit: 'years'
|
||
},
|
||
'co-emissions-per-capita': {
|
||
col: 'emissions_total_per_capita',
|
||
title: 'CO₂ emissions per person',
|
||
unit: 't'
|
||
}
|
||
};
|
||
|
||
async function owidCw(note: (w: string) => void) {
|
||
const series: SeriesRow[] = [];
|
||
const obs: ObsRow[] = [];
|
||
for (const [slugName, meta] of Object.entries(OWID_CW)) {
|
||
try {
|
||
const text = await getText(
|
||
`https://ourworldindata.org/grapher/${slugName}.csv?country=${REGION}&csvType=filtered&useColumnShortNames=true`
|
||
);
|
||
const [head, ...rows] = parseCsv(text);
|
||
const cCode = head.indexOf('code');
|
||
const cYear = head.indexOf('year');
|
||
const cVal = head.indexOf(meta.col);
|
||
if (cVal < 0) throw new Error(`column ${meta.col} missing`);
|
||
const id = `owid:cw:${slugName}`;
|
||
const list = rows
|
||
.filter((r) => r[cCode] === REGION && toNum(r[cVal]) !== null)
|
||
.map((r) => ({
|
||
series_id: id,
|
||
period: year(r[cYear]),
|
||
value: toNum(r[cVal]),
|
||
status: null
|
||
}));
|
||
if (!list.length) throw new Error('no CUW rows');
|
||
obs.push(...list);
|
||
series.push({
|
||
series_id: id,
|
||
source_id: 'owid',
|
||
source_code: `grapher/${slugName}`,
|
||
title: `Curaçao · ${meta.title}`,
|
||
unit: meta.unit,
|
||
freq: 'A',
|
||
region_code: REGION,
|
||
theme: THEME,
|
||
meta: { url: `https://ourworldindata.org/grapher/${slugName}` }
|
||
});
|
||
} catch (error) {
|
||
note(`owid ${slugName}: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
}
|
||
const n = await writeSeries(series, obs);
|
||
log(`owid CUW: ${series.length} series, ${n} obs`);
|
||
return n;
|
||
}
|
||
|
||
// ── UN SDG API (area 531) ─────────────────────────────────────────────────────────────────
|
||
export const SDG_SERIES: Record<string, { title: string; unit: string }> = {
|
||
EN_ATM_CO2: { title: 'CO₂ emissions from fuel combustion', unit: 'Mt' },
|
||
EG_FEC_RNEW: { title: 'Renewable share of final energy consumption', unit: '%' },
|
||
SP_DYN_ADKL: { title: 'Adolescent birth rate (per 1,000 women 15–19)', unit: 'per 1,000' },
|
||
ER_PTD_TERR: { title: 'Terrestrial key biodiversity areas protected', unit: '%' }
|
||
};
|
||
type SdgRow = {
|
||
timePeriodStart: number;
|
||
value: string;
|
||
dimensions: Record<string, string>;
|
||
seriesDescription: string;
|
||
};
|
||
|
||
async function unsdg(note: (w: string) => void) {
|
||
const series: SeriesRow[] = [];
|
||
const obs: ObsRow[] = [];
|
||
for (const [code, meta] of Object.entries(SDG_SERIES)) {
|
||
try {
|
||
const d = await getJson<{ data: SdgRow[] }>(
|
||
`https://unstats.un.org/sdgs/UNSDGAPIV5/v1/sdg/Series/Data?seriesCode=${code}&areaCode=531&pageSize=2000`
|
||
);
|
||
const byId = new Map<string, Map<string, number | null>>();
|
||
for (const r of d.data) {
|
||
const dims = Object.entries(r.dimensions)
|
||
.filter(([k]) => k !== 'Reporting Type')
|
||
.sort(([a], [b]) => a.localeCompare(b));
|
||
// SP_DYN_ADKL also carries the 10–14 group; keep the headline 15–19.
|
||
if (code === 'SP_DYN_ADKL' && r.dimensions.Age !== '15-19') continue;
|
||
const key = dims.map(([, v]) => slug(v)).join(':');
|
||
const id = `unsdg:${code.toLowerCase()}${key ? `:${key}` : ''}`;
|
||
if (!byId.has(id)) {
|
||
byId.set(id, new Map());
|
||
series.push({
|
||
series_id: id,
|
||
source_id: 'unsdg',
|
||
source_code: `${code}/${dims.map(([k, v]) => `${k}=${v}`).join(',')}`,
|
||
title: `Curaçao · ${meta.title}${dims.length && code !== 'SP_DYN_ADKL' ? ` (${dims.map(([, v]) => v).join(', ')})` : ''}`,
|
||
unit: meta.unit,
|
||
freq: 'A',
|
||
region_code: REGION,
|
||
theme: THEME,
|
||
meta: { code, description: r.seriesDescription, dims: Object.fromEntries(dims) }
|
||
});
|
||
}
|
||
// Global (G) and national (N) reporting can both appear; the first wins.
|
||
const p = year(r.timePeriodStart);
|
||
if (!byId.get(id)!.has(p)) byId.get(id)!.set(p, toNum(r.value));
|
||
}
|
||
for (const [id, m] of byId)
|
||
for (const [period, value] of m) obs.push({ series_id: id, period, value, status: null });
|
||
} catch (error) {
|
||
note(`${code}: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
}
|
||
const n = await writeSeries(series, obs);
|
||
log(`unsdg 531: ${series.length} series, ${n} obs`);
|
||
return n;
|
||
}
|
||
|
||
// ── UN Comtrade public preview (one period per call, ≤500 rows) ───────────────────────────
|
||
const COMTRADE_PARTNERS: Record<number, string> = {
|
||
0: 'world',
|
||
842: 'usa',
|
||
528: 'nld',
|
||
76: 'bra',
|
||
862: 'ven',
|
||
156: 'chn'
|
||
};
|
||
type ComtradeRow = { period: string; flowCode: string; partnerCode: number; primaryValue: number };
|
||
|
||
async function comtrade(note: (w: string) => void) {
|
||
const series: SeriesRow[] = [];
|
||
const obs: ObsRow[] = [];
|
||
const seen = new Set<string>();
|
||
const last = new Date().getFullYear() - 1;
|
||
for (let y = 2010; y <= last; y++) {
|
||
let d: { data?: ComtradeRow[]; error?: string };
|
||
try {
|
||
d = await getJson(
|
||
`https://comtradeapi.un.org/public/v1/preview/C/A/HS?reporterCode=531&period=${y}&flowCode=M,X&partnerCode=${Object.keys(COMTRADE_PARTNERS).join(',')}&cmdCode=TOTAL`
|
||
);
|
||
} catch (error) {
|
||
note(`${y}: ${error instanceof Error ? error.message : String(error)}`);
|
||
continue;
|
||
}
|
||
if (d.error) note(`${y}: ${d.error}`);
|
||
for (const r of d.data ?? []) {
|
||
const partner = COMTRADE_PARTNERS[r.partnerCode];
|
||
if (!partner) continue;
|
||
const flow = r.flowCode === 'M' ? 'imports' : r.flowCode === 'X' ? 'exports' : null;
|
||
if (!flow) continue;
|
||
const id = `comtrade:cw:${flow}:${partner}`;
|
||
if (!seen.has(id)) {
|
||
seen.add(id);
|
||
series.push({
|
||
series_id: id,
|
||
source_id: 'comtrade',
|
||
source_code: `531/${r.flowCode}/${r.partnerCode}/TOTAL`,
|
||
title: `Curaçao · Goods ${flow}${partner === 'world' ? '' : ` (partner ${partner.toUpperCase()})`}`,
|
||
unit: 'US$',
|
||
freq: 'A',
|
||
region_code: REGION,
|
||
theme: THEME,
|
||
meta: { reporter: 531, flow: r.flowCode, partner: r.partnerCode }
|
||
});
|
||
}
|
||
obs.push({ series_id: id, period: year(r.period), value: r.primaryValue, status: null });
|
||
}
|
||
}
|
||
const n = await writeSeries(series, obs);
|
||
log(`comtrade 531: ${series.length} series, ${n} obs`);
|
||
return n;
|
||
}
|
||
|
||
export const CW_INTL_JOBS = { ilostat, owid: owidCw, unsdg, comtrade } as const;
|
||
const SOURCE_OF: Record<keyof typeof CW_INTL_JOBS, string> = {
|
||
ilostat: 'ilostat',
|
||
owid: 'owid',
|
||
unsdg: 'unsdg',
|
||
comtrade: 'comtrade'
|
||
};
|
||
|
||
export async function cwintl(opts: { only?: string; job: string }) {
|
||
let total = 0;
|
||
let failed = 0;
|
||
for (const [name, fn] of Object.entries(CW_INTL_JOBS) as [
|
||
keyof typeof CW_INTL_JOBS,
|
||
typeof ilostat
|
||
][]) {
|
||
if (opts.only && !opts.only.split(',').includes(name)) continue;
|
||
try {
|
||
total += await run(SOURCE_OF[name], `cw-${opts.job}`, fn);
|
||
} catch {
|
||
failed++;
|
||
}
|
||
}
|
||
if (failed) throw new Error(`cwintl: ${failed} job(s) failed`);
|
||
return total;
|
||
}
|