website/scripts/stats-fetch.ts

// /stats collector. Pulls open statistics into the `stats` schema; every job is logged in
// stats.fetch_run. Run on the host by systemd user timers (deploy/stats-collector), with DB_URL
// read from the app container (app role personal_app).
//
//   bun scripts/stats-fetch.ts cbs [--force] [--only=86141NED,85592NED]
//   bun scripts/stats-fetch.ts eurostat [--only=nrg_pc_204]
//   bun scripts/stats-fetch.ts fred [--only=DCOILBRENTEU]
//   bun scripts/stats-fetch.ts bis
//   bun scripts/stats-fetch.ts oecd [--only=cpi,irlt]   OECD SDMX (CPI categories, 10y yields)
//   bun scripts/stats-fetch.ts statcan                 Statistics Canada CPI (WDS)
//   bun scripts/stats-fetch.ts worldbank|imf|owid [--only=…]
//   bun scripts/stats-fetch.ts energycharts|rijksfinancien|ustreasury [--backfill]
//   bun scripts/stats-fetch.ts usaspending
//   bun scripts/stats-fetch.ts eponline [--force]   (needs EP_ONLINE_API_KEY)
//   bun scripts/stats-fetch.ts cbscw [--force] [--only=cpi,gdp,…]   CBS Curaçao files
//   bun scripts/stats-fetch.ts cwintl [--only=ilostat,owid,unsdg,comtrade]   Curaçao via international APIs
//   bun scripts/stats-fetch.ts elections [--force] [--only=kiesraad,cbs,mit]   A14
//   bun scripts/stats-fetch.ts parties            CHES + V-Party + Party Facts (A15)
//   bun scripts/stats-fetch.ts wikipolls|eurobarometer [--force]|whr|gtrends|gdelt   A16
//   bun scripts/stats-fetch.ts normalise-cbs     rebuild CBS chart series from the raw rows
import { ensureMeta, sql } from './stats/common';
import { cbs, normaliseCbs } from './stats/cbs';
import { eurostat } from './stats/eurostat';
import { fred } from './stats/fred';
import { bis } from './stats/bis';
import { oecd } from './stats/oecd';
import { statcan } from './stats/statcan';
import { worldbank } from './stats/worldbank';
import { imf } from './stats/imf';
import { owid } from './stats/owid';
import { energycharts } from './stats/energycharts';
import { rijksfinancien } from './stats/rijksfinancien';
import { ustreasury } from './stats/ustreasury';
import { usaspending } from './stats/usaspending';
import { eponline } from './stats/eponline';
import { cwintl } from './stats/cwintl';
import { cwcbs } from './stats/cwcbs';
import { elections } from './stats/elections';
import { parties } from './stats/parties';
import { wikipolls } from './stats/wikipolls';
import { eurobarometer } from './stats/eurobarometer';
import { whr } from './stats/whr';
import { gtrends } from './stats/gtrends';
import { gdelt } from './stats/gdelt';
import { history } from './stats/history';

const [source, ...args] = process.argv.slice(2);
const flag = (name: string) => args.includes(`--${name}`);
const value = (name: string) => args.find((a) => a.startsWith(`--${name}=`))?.split('=')[1];
const job = flag('backfill') ? 'backfill' : value('only') ? `only:${value('only')}` : 'refresh';

try {
	await ensureMeta();
	switch (source) {
		case 'cbs':
			await cbs({ force: flag('force') || flag('backfill'), only: value('only'), job });
			break;
		case 'normalise-cbs':
			console.log(`${await normaliseCbs()} CBS observations normalised`);
			break;
		case 'eurostat':
			await eurostat({ only: value('only'), job });
			break;
		case 'fred':
			await fred({ only: value('only'), job });
			break;
		case 'bis':
			await bis({ job });
			break;
		case 'oecd':
			await oecd({ only: value('only'), job });
			break;
		case 'statcan':
			await statcan({ job });
			break;
		case 'worldbank':
			await worldbank({ only: value('only'), job });
			break;
		case 'imf':
			await imf({ only: value('only'), job });
			break;
		case 'owid':
			await owid({ only: value('only'), job });
			break;
		case 'energycharts':
			await energycharts({ backfill: flag('backfill'), job });
			break;
		case 'rijksfinancien':
			await rijksfinancien({ backfill: flag('backfill'), job });
			break;
		case 'ustreasury':
			await ustreasury({ backfill: flag('backfill'), job });
			break;
		case 'usaspending':
			await usaspending({ job });
			break;
		case 'cbscw':
			await cwcbs({ force: flag('force') || flag('backfill'), only: value('only'), job });
			break;
		case 'cwintl':
			await cwintl({ only: value('only'), job });
			break;
		case 'eponline':
			await eponline({ force: flag('force') || flag('backfill'), job });
			break;
		case 'elections':
			await elections({ force: flag('force') || flag('backfill'), only: value('only'), job });
			break;
		case 'parties':
			await parties({ job });
			break;
		case 'wikipolls':
			await wikipolls({ job });
			break;
		case 'eurobarometer':
			await eurobarometer({ force: flag('force') || flag('backfill'), job });
			break;
		case 'whr':
			await whr({ job });
			break;
		case 'gtrends':
			await gtrends({ job });
			break;
		case 'history':
			await history({ only: value('only'), job });
			break;
		case 'gdelt':
			await gdelt({ job, timespan: value('timespan') });
			break;
		default:
			throw new Error(
				`Unknown source "${source}" (cbs, eurostat, fred, bis, oecd, statcan, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, eponline, cbscw, cwintl, elections, parties, wikipolls, eurobarometer, whr, gtrends, gdelt, history, normalise-cbs)`
			);
	}
} finally {
	await sql.end({ timeout: 5 });
}