Stan Runge
031ee4baf2
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
707 lines
25 KiB
TypeScript
707 lines
25 KiB
TypeScript
// §8 CW1–CW8: CBS Curaçao (Central Bureau of Statistics Curaçao). The statistics are xlsx/PDF
|
||
// files on a CDN whose names change with every update, so each run scrapes the topic pages,
|
||
// matches each table by a filename regex, and re-imports a table only when the file's sha256
|
||
// changed (manifest: stats.cw_cbs_file). Each table has its own parser (hand-formatted sheets);
|
||
// a table that fails to parse keeps its last good rows and is reported as a warning.
|
||
// Licence: free to use on condition that the source (Central Bureau of Statistics Curaçao or
|
||
// CBS Curaçao) is mentioned (cbs.cw/disclaimer).
|
||
import { createHash } from 'node:crypto';
|
||
import { mkdtemp, rm } from 'node:fs/promises';
|
||
import { tmpdir } from 'node:os';
|
||
import { join } from 'node:path';
|
||
import {
|
||
http,
|
||
insertChunks,
|
||
log,
|
||
run,
|
||
sql,
|
||
writeSeries,
|
||
type ObsRow,
|
||
type SeriesRow
|
||
} from './common';
|
||
import { readXlsx, type Cell } from './xlsx';
|
||
|
||
const SITE = 'https://www.cbs.cw';
|
||
const CDN =
|
||
/https:\/\/cuatro\.sim-cdn\.nl\/cbscuracao\/uploads\/[^"'<>\\]+?\.(?:xlsx?|pdf)(?=[?"'<>\\])/gi;
|
||
|
||
type Row = Record<string, unknown>;
|
||
type Table = {
|
||
key: string;
|
||
page: string;
|
||
match: RegExp;
|
||
/** Raw table the parser fills, and the slice of it this file owns (for replace-on-import). */
|
||
table: string;
|
||
owns: string;
|
||
parse: (path: string) => Promise<Row[]>;
|
||
};
|
||
|
||
// ── cell helpers ─────────────────────────────────────────────────────────────────────────
|
||
|
||
/** Number from a cell: handles "*2019", "1.096,470" (thousand separators), "." for n/a. */
|
||
function num(v: Cell): number | null {
|
||
if (typeof v === 'number') return Number.isFinite(v) ? v : null;
|
||
if (v == null) return null;
|
||
let s = v.replace(/[*\s]/g, '');
|
||
if (!s || s === '.' || s === '-' || s === '–') return null;
|
||
if (/^-?\d{1,3}([.,]\d{3})+$/.test(s)) s = s.replace(/[.,]/g, '');
|
||
else if (/^-?\d{1,3}(\.\d{3})+,\d+$/.test(s)) s = s.replace(/\./g, '').replace(',', '.');
|
||
else s = s.replace(',', '.');
|
||
const n = Number(s);
|
||
return Number.isFinite(n) ? n : null;
|
||
}
|
||
/** Year header cell: 2019, "*2019", "2021*" → { year, provisional }. */
|
||
function yearCell(v: Cell): { year: number; provisional: boolean } | null {
|
||
const s = String(v ?? '').trim();
|
||
const m = s.match(/^\*?\s*((?:19|20)\d\d)\s*\*?$/);
|
||
return m ? { year: Number(m[1]), provisional: s.includes('*') } : null;
|
||
}
|
||
/** Columns of a header row that hold years. */
|
||
function yearColumns(row: Cell[]) {
|
||
const cols: { col: number; year: number; provisional: boolean }[] = [];
|
||
row.forEach((v, col) => {
|
||
const y = yearCell(v);
|
||
if (y) cols.push({ col, ...y });
|
||
});
|
||
return cols;
|
||
}
|
||
const text = (v: Cell) => (v == null ? '' : String(v).replace(/\s+/g, ' ').trim());
|
||
/** Counts typed as 834.922 for 834,922 (Excel turned a thousands dot into a decimal). */
|
||
const count = (v: number | null) =>
|
||
v != null &&
|
||
!Number.isInteger(v) &&
|
||
v < 10_000 &&
|
||
Number.isInteger(Math.round(v * 1000 * 1e6) / 1e6)
|
||
? Math.round(v * 1000)
|
||
: v;
|
||
|
||
// ── parsers ──────────────────────────────────────────────────────────────────────────────
|
||
|
||
/** CW1: monthly CPI since 1970 (PDF): index, 12-month average, inflation of the 12-month average. */
|
||
export async function parseCpi(path: string): Promise<Row[]> {
|
||
const { getDocument } = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
||
const data = new Uint8Array(await Bun.file(path).arrayBuffer());
|
||
const task = getDocument({ data, isEvalSupported: false, useSystemFonts: false });
|
||
const doc = await task.promise;
|
||
const lines: string[] = [];
|
||
let base = '';
|
||
for (let p = 1; p <= doc.numPages; p++) {
|
||
const tc = await (await doc.getPage(p)).getTextContent();
|
||
// Cluster text items into lines (superscript footnote marks sit a little off the baseline).
|
||
const items = (tc.items as { str?: string; transform: number[] }[])
|
||
.filter((it) => it.str?.trim())
|
||
.map((it) => ({ x: it.transform[4], y: it.transform[5], s: it.str!.trim() }))
|
||
.sort((a, b) => b.y - a.y);
|
||
const clusters: { y: number; items: typeof items }[] = [];
|
||
for (const it of items) {
|
||
const c = clusters.at(-1);
|
||
if (c && c.y - it.y <= 3) c.items.push(it);
|
||
else clusters.push({ y: it.y, items: [it] });
|
||
}
|
||
for (const c of clusters)
|
||
lines.push(
|
||
c.items
|
||
.sort((a, b) => a.x - b.x)
|
||
.map((i) => i.s)
|
||
.join(' ')
|
||
);
|
||
}
|
||
await task.destroy();
|
||
const MONTHS = [
|
||
'Jan',
|
||
'Feb',
|
||
'Mar',
|
||
'Apr',
|
||
'May',
|
||
'Jun',
|
||
'Jul',
|
||
'Aug',
|
||
'Sep',
|
||
'Oct',
|
||
'Nov',
|
||
'Dec'
|
||
];
|
||
const byPeriod = new Map<string, { index: number; avg: number | null; infl: number | null }>();
|
||
for (const line of lines) {
|
||
const b = line.match(/average (\d{4}) = 100/);
|
||
if (b) base = `${b[1]}=100`;
|
||
const m = line.match(
|
||
/^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s*(?:\d\))?\s+(\d{4})\s+(.*)$/
|
||
);
|
||
if (!m) continue;
|
||
const vals = m[3].split(/\s+/).filter((t) => /^(-?\d+(\.\d+)?|\.)$/.test(t));
|
||
const index = num(vals[0] ?? null);
|
||
if (index == null) continue;
|
||
const month = String(MONTHS.indexOf(m[1]) + 1).padStart(2, '0');
|
||
byPeriod.set(`${m[2]}-${month}-01`, {
|
||
index,
|
||
avg: num(vals[1] ?? null),
|
||
infl: num(vals[2] ?? null)
|
||
});
|
||
}
|
||
const periods = [...byPeriod.keys()].sort();
|
||
if (periods.length < 600) throw new Error(`CPI PDF: only ${periods.length} months parsed`);
|
||
const shift = (period: string, months: number) => {
|
||
const d = new Date(`${period}T00:00:00Z`);
|
||
d.setUTCMonth(d.getUTCMonth() - months);
|
||
return d.toISOString().slice(0, 10);
|
||
};
|
||
return periods.map((period) => {
|
||
const r = byPeriod.get(period)!;
|
||
const prev = byPeriod.get(shift(period, 1))?.index;
|
||
const lastYear = byPeriod.get(shift(period, 12))?.index;
|
||
return {
|
||
period,
|
||
cpi_index: r.index,
|
||
infl_mom: prev ? (100 * (r.index - prev)) / prev : null,
|
||
infl_yoy: lastYear ? (100 * (r.index - lastYear)) / lastYear : null,
|
||
infl_12m_avg: r.infl,
|
||
base: base || null
|
||
};
|
||
});
|
||
}
|
||
|
||
/** CW3: GDP by industry (ISIC 4), mln ANG, plus totals and real growth (stored as %). */
|
||
async function parseGdp(path: string): Promise<Row[]> {
|
||
const [sheet] = await readXlsx(path);
|
||
const h = sheet.rows.findIndex((r) => /isic/i.test(text(r[0])));
|
||
if (h < 0) throw new Error('GDP: no ISIC header');
|
||
const years = yearColumns(sheet.rows[h]);
|
||
const out: Row[] = [];
|
||
const totals: [RegExp, string][] = [
|
||
[/^gross value added/i, 'GVA'],
|
||
[/taxes less subsidies/i, 'TLS'],
|
||
[/domestic product gross|gross domestic product/i, 'GDP'],
|
||
[/^real gdp growth/i, 'REAL_GROWTH_PCT'],
|
||
[/^nominal gdp growth/i, 'NOMINAL_GROWTH_PCT']
|
||
];
|
||
for (const row of sheet.rows.slice(h + 1)) {
|
||
const label = text(row[1]);
|
||
let isic = text(row[0]).replace(/\s+/g, '');
|
||
if (!isic) isic = totals.find(([re]) => re.test(label))?.[1] ?? '';
|
||
if (!isic || !label) continue;
|
||
for (const y of years) {
|
||
const v = num(row[y.col]);
|
||
if (v != null)
|
||
out.push({ year: y.year, isic, label, mln_ang: v, provisional: y.provisional });
|
||
}
|
||
}
|
||
if (!out.some((r) => r.isic === 'GDP')) throw new Error('GDP: no GDP total row');
|
||
return out;
|
||
}
|
||
|
||
/** CW4: government revenue and expenditure (ESA codes), mln ANG, all provisional. */
|
||
async function parseGovFinance(path: string): Promise<Row[]> {
|
||
const [sheet] = await readXlsx(path);
|
||
const h = sheet.rows.findIndex((r) => yearColumns(r).length >= 5);
|
||
if (h < 0) throw new Error('government finance: no year header');
|
||
const years = yearColumns(sheet.rows[h]);
|
||
const out: Row[] = [];
|
||
let section = '';
|
||
for (const row of sheet.rows.slice(h + 1)) {
|
||
const label = text(row[0]);
|
||
if (/^revenues?$/i.test(label)) section = 'R';
|
||
else if (/^expenditures?$/i.test(label)) section = 'E';
|
||
if (!section || !label || /saldo|balance/i.test(label)) continue;
|
||
const code = /^total/i.test(label) ? 'TOTAL' : text(row[1]);
|
||
if (!code) continue;
|
||
for (const y of years) {
|
||
const v = num(row[y.col]);
|
||
if (v != null)
|
||
out.push({
|
||
year: y.year,
|
||
item_code: `${section}.${code}`,
|
||
label,
|
||
mln_ang: v,
|
||
provisional: true
|
||
});
|
||
}
|
||
}
|
||
if (!out.some((r) => r.item_code === 'R.TOTAL') || !out.some((r) => r.item_code === 'E.TOTAL'))
|
||
throw new Error('government finance: totals missing');
|
||
return out;
|
||
}
|
||
|
||
/** CW5: population 1900→ (thousands before 1951; 31 Dec until 1991, 1 Jan from 1992; censuses). */
|
||
async function parsePopulationHistory(path: string): Promise<Row[]> {
|
||
const [sheet] = await readXlsx(path);
|
||
const out: Row[] = [];
|
||
for (const row of sheet.rows) {
|
||
const y = yearCell(row[1]);
|
||
let v = num(row[2]);
|
||
if (!y || v == null) continue;
|
||
if (v < 1000) v = Math.round(v * 1000);
|
||
const tag = text(row[0]).toLowerCase();
|
||
const kind = tag.startsWith('census') ? 'census' : y.year <= 1991 ? 'dec31' : 'jan1';
|
||
out.push({ year: y.year, sex: 'T', age_group: 'total', value: v, kind });
|
||
}
|
||
if (out.length < 100) throw new Error(`population history: only ${out.length} rows`);
|
||
// Two rows for one year and kind (1921): keep the last.
|
||
return [...new Map(out.map((r) => [`${r.year}|${r.kind}`, r])).values()];
|
||
}
|
||
|
||
const LFS_MEASURES: [RegExp, string][] = [
|
||
[/^population/i, 'population'],
|
||
[/^employed/i, 'employed'],
|
||
[/^unemployed/i, 'unemployed'],
|
||
[/not active/i, 'inactive'],
|
||
[/^labour force$/i, 'labour_force'],
|
||
[/^gross participation/i, 'gross_participation_pct'],
|
||
[/^net participation/i, 'net_participation_pct'],
|
||
[/^unemployment rate/i, 'unemployment_rate_pct']
|
||
];
|
||
/** CW7: LFS blocks ("Labour Force of Curacao 2014 – 2025 (Men)" / "(15-24 years)"). */
|
||
async function parseLfs(path: string): Promise<Row[]> {
|
||
const out: Row[] = [];
|
||
for (const sheet of await readXlsx(path)) {
|
||
let years: ReturnType<typeof yearColumns> = [];
|
||
let sex = 'T';
|
||
let age = '15+';
|
||
sheet.rows.forEach((row, i) => {
|
||
const ys = yearColumns(row);
|
||
if (ys.length >= 3 && !text(row[0])) {
|
||
years = ys;
|
||
const title = text(sheet.rows[i - 1]?.[0]);
|
||
const tag = title.match(/\(([^)]+)\)\s*$/)?.[1] ?? '';
|
||
sex = /^(men|male)/i.test(tag) ? 'M' : /^(women|female)/i.test(tag) ? 'F' : 'T';
|
||
age =
|
||
tag
|
||
.match(/(\d+\s*[-–]\s*\d+|\d+\+)/)?.[1]
|
||
.replace(/\s/g, '')
|
||
.replace('–', '-') ?? '15+';
|
||
return;
|
||
}
|
||
const measure = LFS_MEASURES.find(([re]) => re.test(text(row[0])))?.[1];
|
||
if (!measure || !years.length) return;
|
||
for (const y of years) {
|
||
const v = num(row[y.col]);
|
||
if (v != null) out.push({ year: y.year, sex, age_group: age, measure, value: v });
|
||
}
|
||
});
|
||
}
|
||
if (!out.length) throw new Error('LFS: no rows');
|
||
return out;
|
||
}
|
||
|
||
/** CW8 tourism sheets: a year header, then one row per measure (or source market). */
|
||
function tourismParser(map: (label: string) => { measure: string; market: string } | null) {
|
||
return async (path: string): Promise<Row[]> => {
|
||
const out: Row[] = [];
|
||
const seen = new Set<string>();
|
||
for (const sheet of await readXlsx(path)) {
|
||
let years: ReturnType<typeof yearColumns> = [];
|
||
for (const row of sheet.rows) {
|
||
const ys = yearColumns(row);
|
||
if (ys.length >= 3) {
|
||
years = ys;
|
||
continue;
|
||
}
|
||
const m = map(text(row[0]));
|
||
if (!m || !years.length) continue;
|
||
for (const y of years) {
|
||
const key = `${y.year}|${m.measure}|${m.market}`;
|
||
const v = count(num(row[y.col]));
|
||
if (v == null || seen.has(key)) continue;
|
||
seen.add(key);
|
||
out.push({ period: `${y.year}-01-01`, measure: m.measure, market: m.market, value: v });
|
||
}
|
||
}
|
||
}
|
||
if (!out.length) throw new Error('tourism: no rows');
|
||
return out;
|
||
};
|
||
}
|
||
|
||
export const TABLES: Table[] = [
|
||
{
|
||
key: 'cpi',
|
||
page: 'inflationprices',
|
||
match: /consumer price index and inflation rates.*since 1970.*\.pdf$/i,
|
||
table: 'cw_cbs_cpi',
|
||
owns: 'true',
|
||
parse: parseCpi
|
||
},
|
||
{
|
||
key: 'gdp',
|
||
page: 'economy-0',
|
||
match: /gdp.*\.xlsx$/i,
|
||
table: 'cw_cbs_gdp_industry',
|
||
owns: 'true',
|
||
parse: parseGdp
|
||
},
|
||
{
|
||
key: 'gov_finance',
|
||
page: 'economy-0',
|
||
match: /revenues.*expenditures.*government.*\.xlsx$/i,
|
||
table: 'cw_cbs_gov_finance',
|
||
owns: 'true',
|
||
parse: parseGovFinance
|
||
},
|
||
{
|
||
key: 'population_history',
|
||
page: 'population',
|
||
match: /historical.*population.*\.xlsx$/i,
|
||
table: 'cw_cbs_population',
|
||
owns: `age_group = 'total' and sex = 'T'`,
|
||
parse: parsePopulationHistory
|
||
},
|
||
{
|
||
key: 'lfs_sex',
|
||
page: 'labour',
|
||
match: /labour force of cura.*by sex.*\.xlsx$/i,
|
||
table: 'cw_cbs_lfs',
|
||
owns: `age_group = '15+'`,
|
||
parse: parseLfs
|
||
},
|
||
{
|
||
key: 'lfs_age',
|
||
page: 'labour',
|
||
match: /labour force survey.*age.groups.*\.xlsx$/i,
|
||
table: 'cw_cbs_lfs',
|
||
owns: `age_group <> '15+'`,
|
||
// The 15+ totals come from the by-sex file; older age-group uploads repeat them per sheet.
|
||
parse: async (path: string) =>
|
||
(await parseLfs(path)).filter((r) => (r as { age_group: string }).age_group !== '15+')
|
||
},
|
||
{
|
||
key: 'tourism_cruise',
|
||
page: 'tourism',
|
||
match: /cruise arrivals.*\.xlsx$/i,
|
||
table: 'cw_cbs_tourism',
|
||
owns: `measure in ('cruise_calls', 'cruise_pax')`,
|
||
parse: tourismParser((l) =>
|
||
/^cruise calls/i.test(l)
|
||
? { measure: 'cruise_calls', market: 'total' }
|
||
: /^cruise passengers/i.test(l)
|
||
? { measure: 'cruise_pax', market: 'total' }
|
||
: null
|
||
)
|
||
},
|
||
{
|
||
key: 'tourism_daytrippers',
|
||
page: 'tourism',
|
||
match: /daytrippers.*\.xlsx$/i,
|
||
table: 'cw_cbs_tourism',
|
||
owns: `measure = 'daytrippers'`,
|
||
parse: tourismParser((l) =>
|
||
/^daytrippers/i.test(l) ? { measure: 'daytrippers', market: 'total' } : null
|
||
)
|
||
},
|
||
{
|
||
key: 'tourism_nights',
|
||
page: 'tourism',
|
||
match: /nights.*region.*\.xlsx$/i,
|
||
table: 'cw_cbs_tourism',
|
||
owns: `measure = 'nights'`,
|
||
parse: tourismParser((l) => {
|
||
if (/^total/i.test(l)) return { measure: 'nights', market: 'total' };
|
||
const market = l.match(/^(north america|south america|caribbean|europe|others?)$/i)?.[1];
|
||
return market
|
||
? { measure: 'nights', market: market.toLowerCase().replace(/\s+/g, '_') }
|
||
: null;
|
||
})
|
||
},
|
||
{
|
||
key: 'tourism_air',
|
||
page: 'tourism',
|
||
match: /air.traffic.movements.*\.xlsx$/i,
|
||
table: 'cw_cbs_tourism',
|
||
owns: `measure in ('air_landings_scheduled', 'air_landings_civil')`,
|
||
// The landings block comes first; the take-off block repeats the labels (skipped as seen).
|
||
parse: tourismParser((l) =>
|
||
/^scheduled pax flights/i.test(l)
|
||
? { measure: 'air_landings_scheduled', market: 'total' }
|
||
: /^total civil landings/i.test(l)
|
||
? { measure: 'air_landings_civil', market: 'total' }
|
||
: null
|
||
)
|
||
}
|
||
];
|
||
|
||
/** File links on a topic page, decoded, newest-looking first (…_0.xlsx before ….xlsx). */
|
||
async function links(page: string): Promise<string[]> {
|
||
const html = await (await http(`${SITE}/${page}`)).text();
|
||
const urls = new Set<string>();
|
||
for (const m of html.replace(/&/g, '&').matchAll(CDN)) urls.add(m[0]);
|
||
// Plain code-point order (not localeCompare, which ignores punctuation): the CMS re-upload
|
||
// suffix `_0` then sorts ahead of the original, so the newest copy of a file wins.
|
||
return [...urls]
|
||
.map((u) => [u, decodeURIComponent(u)] as const)
|
||
.sort((a, b) => (a[1] < b[1] ? 1 : a[1] > b[1] ? -1 : 0))
|
||
.map(([u]) => u);
|
||
}
|
||
const fileName = (url: string) => decodeURIComponent(url.split('/uploads/')[1] ?? url);
|
||
|
||
export async function cwcbs(opts: { force: boolean; only?: string; job: string }) {
|
||
await run('cbscw', opts.job, async (note) => {
|
||
const only = opts.only?.split(',');
|
||
const dir = await mkdtemp(join(tmpdir(), 'cwcbs-'));
|
||
let rows = 0;
|
||
try {
|
||
const pages = new Map<string, string[]>();
|
||
for (const t of TABLES) {
|
||
if (only && !only.includes(t.key)) continue;
|
||
try {
|
||
if (!pages.has(t.page)) pages.set(t.page, await links(t.page));
|
||
const url = pages.get(t.page)!.find((u) => t.match.test(fileName(u)));
|
||
if (!url) throw new Error(`no file matching ${t.match} on /${t.page}`);
|
||
const bytes = new Uint8Array(await (await http(url)).arrayBuffer());
|
||
const sha256 = createHash('sha256').update(bytes).digest('hex');
|
||
const [prev] = await sql<{ sha256: string }[]>`
|
||
select sha256 from stats.cw_cbs_file where topic = ${t.key}`;
|
||
if (prev?.sha256 === sha256 && !opts.force) {
|
||
log(`cbscw ${t.key}: unchanged (${fileName(url)})`);
|
||
continue;
|
||
}
|
||
const path = join(dir, `${t.key}${url.match(/\.\w+$/)?.[0] ?? ''}`);
|
||
await Bun.write(path, bytes);
|
||
const parsed = await t.parse(path);
|
||
await sql.begin(async (tx) => {
|
||
await tx.unsafe(`delete from stats.${t.table} where ${t.owns}`);
|
||
await insertChunks(tx, t.table, parsed);
|
||
await tx`delete from stats.cw_cbs_file where topic = ${t.key}`;
|
||
await tx`insert into stats.cw_cbs_file ${tx({ url, topic: t.key, filename: fileName(url), sha256 })}
|
||
on conflict (url) do update set topic = excluded.topic, filename = excluded.filename,
|
||
sha256 = excluded.sha256, fetched_at = now()`;
|
||
});
|
||
rows += parsed.length;
|
||
log(`cbscw ${t.key}: ${parsed.length} rows from ${fileName(url)}`);
|
||
} catch (error) {
|
||
note(`${t.key}: ${error instanceof Error ? error.message : String(error)}`);
|
||
}
|
||
}
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true });
|
||
}
|
||
const obs = await rebuildSeries();
|
||
log(`cbscw: ${obs} observations in stats.obs`);
|
||
return rows;
|
||
});
|
||
}
|
||
|
||
// ── normalised series (region CUW) ───────────────────────────────────────────────────────
|
||
|
||
const LFS_TITLE: Record<string, [string, string]> = {
|
||
population: ['Population 15+', 'persons'],
|
||
employed: ['Employed', 'persons'],
|
||
unemployed: ['Unemployed', 'persons'],
|
||
inactive: ['Not in the labour force', 'persons'],
|
||
labour_force: ['Labour force', 'persons'],
|
||
gross_participation_pct: ['Gross participation rate', '%'],
|
||
net_participation_pct: ['Net participation rate (employment rate)', '%'],
|
||
unemployment_rate_pct: ['Unemployment rate', '%']
|
||
};
|
||
const TOURISM_TITLE: Record<string, [string, string]> = {
|
||
cruise_calls: ['Cruise ship calls', 'calls'],
|
||
cruise_pax: ['Cruise passengers', 'passengers'],
|
||
daytrippers: ['Daytrippers', 'visitors'],
|
||
nights: ['Stay-over tourist nights', 'nights'],
|
||
air_landings_scheduled: ['Scheduled passenger flight landings', 'landings'],
|
||
air_landings_civil: ['Civil aircraft landings', 'landings']
|
||
};
|
||
|
||
export async function rebuildSeries(): Promise<number> {
|
||
const series: SeriesRow[] = [];
|
||
const obs: ObsRow[] = [];
|
||
const add = (
|
||
id: string,
|
||
title: string,
|
||
unit: string,
|
||
freq: string,
|
||
code: string,
|
||
points: { period: string; value: number | null; status?: string | null }[],
|
||
meta: Record<string, unknown> = {}
|
||
) => {
|
||
if (!points.length) return;
|
||
series.push({
|
||
series_id: id,
|
||
source_id: 'cbscw',
|
||
source_code: code,
|
||
title,
|
||
unit,
|
||
freq,
|
||
region_code: 'CUW',
|
||
theme: 'curacao',
|
||
meta: { url: SITE, ...meta }
|
||
});
|
||
for (const p of points)
|
||
obs.push({ series_id: id, period: p.period, value: p.value, status: p.status ?? null });
|
||
};
|
||
|
||
const cpi = await sql<
|
||
{
|
||
period: string;
|
||
cpi_index: number;
|
||
infl_yoy: number | null;
|
||
infl_12m_avg: number | null;
|
||
base: string | null;
|
||
}[]
|
||
>`
|
||
select period::text, cpi_index, infl_yoy, infl_12m_avg, base from stats.cw_cbs_cpi order by period`;
|
||
const base = cpi.at(-1)?.base ?? '2023=100';
|
||
add(
|
||
'cw:cpi:index',
|
||
`Consumer price index (${base})`,
|
||
'index',
|
||
'M',
|
||
'CPI',
|
||
cpi.map((r) => ({ period: r.period, value: r.cpi_index })),
|
||
{ base }
|
||
);
|
||
add(
|
||
'cw:cpi:yoy',
|
||
'Inflation, CPI year on year',
|
||
'%',
|
||
'M',
|
||
'CPI_YOY',
|
||
cpi.filter((r) => r.infl_yoy != null).map((r) => ({ period: r.period, value: r.infl_yoy }))
|
||
);
|
||
add(
|
||
'cw:cpi:avg12',
|
||
'Inflation, 12-month average CPI',
|
||
'%',
|
||
'M',
|
||
'CPI_AVG12',
|
||
cpi
|
||
.filter((r) => r.infl_12m_avg != null)
|
||
.map((r) => ({ period: r.period, value: r.infl_12m_avg }))
|
||
);
|
||
|
||
const gdp = await sql<
|
||
{ year: number; isic: string; label: string; mln_ang: number; provisional: boolean }[]
|
||
>`
|
||
select year, isic, label, mln_ang, provisional from stats.cw_cbs_gdp_industry order by year`;
|
||
for (const isic of [...new Set(gdp.map((r) => r.isic))]) {
|
||
const mine = gdp.filter((r) => r.isic === isic);
|
||
const pct = isic.endsWith('_PCT');
|
||
add(
|
||
`cw:gdp:${isic.toLowerCase()}`,
|
||
pct ? mine[0].label : `GDP by industry: ${mine[0].label}`,
|
||
pct ? '%' : 'mln ANG',
|
||
'A',
|
||
isic,
|
||
mine.map((r) => ({
|
||
period: `${r.year}-01-01`,
|
||
value: r.mln_ang,
|
||
status: r.provisional ? 'provisional' : null
|
||
})),
|
||
{ isic }
|
||
);
|
||
}
|
||
|
||
const gov = await sql<{ year: number; item_code: string; label: string; mln_ang: number }[]>`
|
||
select year, item_code, label, mln_ang from stats.cw_cbs_gov_finance order by year`;
|
||
for (const code of [...new Set(gov.map((r) => r.item_code))]) {
|
||
const mine = gov.filter((r) => r.item_code === code);
|
||
const side = code.startsWith('R.') ? 'revenue' : 'expenditure';
|
||
add(
|
||
`cw:gov:${code.toLowerCase()}`,
|
||
code.endsWith('TOTAL') ? `Government ${side}, total` : `Government ${side}: ${mine[0].label}`,
|
||
'mln ANG',
|
||
'A',
|
||
code,
|
||
mine.map((r) => ({ period: `${r.year}-01-01`, value: r.mln_ang, status: 'provisional' }))
|
||
);
|
||
}
|
||
const rev = new Map(gov.filter((r) => r.item_code === 'R.TOTAL').map((r) => [r.year, r.mln_ang]));
|
||
add(
|
||
'cw:gov:balance',
|
||
'Government balance (revenue minus expenditure)',
|
||
'mln ANG',
|
||
'A',
|
||
'BALANCE',
|
||
gov
|
||
.filter((r) => r.item_code === 'E.TOTAL' && rev.has(r.year))
|
||
.map((r) => ({
|
||
period: `${r.year}-01-01`,
|
||
value: rev.get(r.year)! - r.mln_ang,
|
||
status: 'provisional'
|
||
}))
|
||
);
|
||
|
||
const pop = await sql<{ year: number; value: number; kind: string }[]>`
|
||
select year, value, kind from stats.cw_cbs_population where sex = 'T' and age_group = 'total' order by year`;
|
||
// 31 December of year y is 1 January of y + 1; censuses are a separate series.
|
||
const jan1 = new Map<number, number>();
|
||
for (const r of pop) if (r.kind === 'dec31') jan1.set(r.year + 1, r.value);
|
||
for (const r of pop) if (r.kind === 'jan1') jan1.set(r.year, r.value);
|
||
add(
|
||
'cw:pop:jan1',
|
||
'Population on 1 January (31 December before 1992)',
|
||
'persons',
|
||
'A',
|
||
'POP',
|
||
[...jan1].sort((a, b) => a[0] - b[0]).map(([y, v]) => ({ period: `${y}-01-01`, value: v })),
|
||
{ note: 'Until 1955 from "Olie als water"; CBS estimates from 1956' }
|
||
);
|
||
add(
|
||
'cw:pop:census',
|
||
'Population at census',
|
||
'persons',
|
||
'A',
|
||
'CENSUS',
|
||
pop
|
||
.filter((r) => r.kind === 'census')
|
||
.map((r) => ({ period: `${r.year}-01-01`, value: r.value }))
|
||
);
|
||
|
||
const lfs = await sql<
|
||
{ year: number; sex: string; age_group: string; measure: string; value: number }[]
|
||
>`
|
||
select year, sex, age_group, measure, value from stats.cw_cbs_lfs order by year`;
|
||
// Both sexes together: add up the counts, recompute the rates.
|
||
const both = new Map<string, Record<string, number>>();
|
||
for (const r of lfs.filter((x) => x.age_group === '15+' && x.sex !== 'T')) {
|
||
const m = both.get(String(r.year)) ?? {};
|
||
if (!r.measure.endsWith('_pct')) m[r.measure] = (m[r.measure] ?? 0) + r.value;
|
||
both.set(String(r.year), m);
|
||
}
|
||
for (const [year, m] of both) {
|
||
const push = (measure: string, value: number) =>
|
||
lfs.push({ year: Number(year), sex: 'T', age_group: '15+', measure, value });
|
||
for (const k of Object.keys(m)) push(k, m[k]);
|
||
if (m.labour_force) push('unemployment_rate_pct', (100 * m.unemployed) / m.labour_force);
|
||
if (m.population) {
|
||
push('gross_participation_pct', (100 * m.labour_force) / m.population);
|
||
push('net_participation_pct', (100 * m.employed) / m.population);
|
||
}
|
||
}
|
||
const lfsKeys = [...new Set(lfs.map((r) => `${r.measure}|${r.sex}|${r.age_group}`))];
|
||
for (const key of lfsKeys) {
|
||
const [measure, sex, age] = key.split('|');
|
||
const [title, unit] = LFS_TITLE[measure] ?? [measure, ''];
|
||
const who = sex === 'M' ? 'men' : sex === 'F' ? 'women' : age === '15+' ? '' : `aged ${age}`;
|
||
const slug =
|
||
sex !== 'T' ? sex.toLowerCase() : age === '15+' ? 'total' : age.replace('+', 'plus');
|
||
add(
|
||
`cw:lfs:${measure}:${slug}`,
|
||
`${title}${who ? `, ${who}` : ''} (Labour Force Survey)`,
|
||
unit,
|
||
'A',
|
||
`LFS.${measure}.${sex}.${age}`,
|
||
lfs
|
||
.filter((r) => `${r.measure}|${r.sex}|${r.age_group}` === key)
|
||
.map((r) => ({ period: `${r.year}-01-01`, value: r.value }))
|
||
);
|
||
}
|
||
|
||
const tour = await sql<{ period: string; measure: string; market: string; value: number }[]>`
|
||
select period::text, measure, market, value from stats.cw_cbs_tourism order by period`;
|
||
for (const key of [...new Set(tour.map((r) => `${r.measure}|${r.market}`))]) {
|
||
const [measure, market] = key.split('|');
|
||
const [title, unit] = TOURISM_TITLE[measure] ?? [measure, ''];
|
||
add(
|
||
`cw:tour:${measure}${market === 'total' ? '' : `:${market}`}`,
|
||
`${title}${market === 'total' ? '' : `: ${market.replace(/_/g, ' ')}`}`,
|
||
unit,
|
||
'A',
|
||
`${measure}.${market}`,
|
||
tour
|
||
.filter((r) => r.measure === measure && r.market === market)
|
||
.map((r) => ({ period: r.period, value: r.value })),
|
||
{ origin: measure.startsWith('air') ? 'Curaçao Airport Partners' : 'Curaçao Tourist Board' }
|
||
);
|
||
}
|
||
return writeSeries(series, obs);
|
||
}
|