stats step 2: World Bank, IMF WEO, OWID, Energy-Charts, Rijksfinanciën, US Treasury, USAspending collectors + tables, timers

scripts/stats/rijksfinancien.ts

@@ -0,0 +1,163 @@
+// A13 Rijksfinanciën: the central-government budget tables (begroting OWB, suppletoire
+// begrotingen O1/O2, jaarverslag JV) per year, down to regeling level. Open data starts in 2015.
+// Normalised: annual totals per chapter and overall, from the most final phase available.
+import {
+	getJson,
+	insertChunks,
+	log,
+	run,
+	sql,
+	writeSeries,
+	type ObsRow,
+	type SeriesRow
+} from './common';
+
+const FIRST_YEAR = 2015;
+const PHASES = ['OWB', 'O1', 'O2', 'JV'] as const;
+const PHASE_RANK: Record<string, number> = { OWB: 1, O1: 2, O2: 3, JV: 4 };
+
+type RfRow = {
+	index: string;
+	jaar: number;
+	fase: string;
+	hoofdstuk_naam: string;
+	hoofdstuk_nummer: string;
+	artikel_naam: string | null;
+	artikel_nummer: string | null;
+	vuo: string;
+	bedrag: number | null;
+	[key: string]: unknown;
+};
+
+export async function rijksfinancien(opts: { backfill?: boolean; job: string }) {
+	await run('rijksfinancien', opts.job, async (note) => {
+		const now = new Date().getUTCFullYear();
+		// Refresh: last year (JV lands in May) up to next year (OWB on Prinsjesdag).
+		const years = [];
+		for (let y = opts.backfill ? FIRST_YEAR : now - 1; y <= now + 1; y++) years.push(y);
+		let total = 0;
+		for (const year of years) {
+			for (const phase of PHASES) {
+				try {
+					const data = await getJson<RfRow[] | Record<string, unknown>>(
+						`https://www.rijksfinancien.nl/open-data/api/json/budgettaire_tabellen?year=${year}&phase=${phase}`
+					);
+					const list = Array.isArray(data) ? data : [];
+					if (!list.length) continue;
+					const rows = list.map((r, i) => {
+						const {
+							index,
+							jaar,
+							fase,
+							hoofdstuk_naam,
+							hoofdstuk_nummer,
+							artikel_naam,
+							artikel_nummer,
+							vuo,
+							bedrag,
+							...detail
+						} = r;
+						void jaar;
+						void fase;
+						return {
+							year,
+							phase,
+							row_key: index ?? `row-${i}`,
+							vuo: (vuo || '?').slice(0, 1),
+							chapter: hoofdstuk_nummer ?? '?',
+							chapter_name: hoofdstuk_naam ?? '',
+							article: artikel_nummer,
+							article_name: artikel_naam,
+							detail,
+							amount_eur_k: bedrag
+						};
+					});
+					await sql.begin(async (tx) => {
+						await tx`delete from stats.rf_budget_table where year = ${year} and phase = ${phase}`;
+						await insertChunks(tx, 'rf_budget_table', rows, 'on conflict do nothing');
+					});
+					log(`rijksfinancien ${year} ${phase}: ${rows.length} rows`);
+					total += rows.length;
+				} catch (error) {
+					const msg = error instanceof Error ? error.message : String(error);
+					if (!/HTTP 404/.test(msg)) note(`${year} ${phase}: ${msg}`);
+				}
+			}
+		}
+		await normaliseRf();
+		return total;
+	});
+}
+
+/** Annual U (expenditure) and O (revenue) per chapter + totals, from the most final phase. */
+export async function normaliseRf() {
+	const agg = await sql<
+		{
+			year: number;
+			phase: string;
+			vuo: string;
+			chapter: string;
+			chapter_name: string;
+			eur_bn: number;
+		}[]
+	>`
+		select year, phase, vuo, chapter, max(chapter_name) as chapter_name, sum(amount_eur_k) / 1e6 as eur_bn
+		from stats.rf_budget_table where vuo in ('U', 'O') group by 1, 2, 3, 4`;
+	const best = new Map<string, (typeof agg)[number]>();
+	for (const r of agg) {
+		const key = `${r.vuo}|${r.chapter}|${r.year}`;
+		const prev = best.get(key);
+		if (!prev || PHASE_RANK[r.phase] > PHASE_RANK[prev.phase]) best.set(key, r);
+	}
+	const series = new Map<string, SeriesRow>();
+	const obs: ObsRow[] = [];
+	const totals = new Map<string, { value: number; phase: string }>();
+	const label = { U: 'expenditure', O: 'revenue' } as Record<string, string>;
+	for (const r of best.values()) {
+		const id = `rf:${r.vuo}:${r.chapter}`;
+		series.set(id, {
+			series_id: id,
+			source_id: 'rijksfinancien',
+			source_code: `${r.vuo}/${r.chapter}`,
+			title: `${r.chapter_name}: ${label[r.vuo]}`,
+			unit: 'EUR bn',
+			freq: 'A',
+			region_code: 'NLD',
+			theme: 'budget',
+			meta: {
+				chapter: r.chapter,
+				chapterName: r.chapter_name,
+				vuo: r.vuo,
+				url: 'https://www.rijksfinancien.nl/open-data'
+			}
+		});
+		obs.push({
+			series_id: id,
+			period: `${r.year}-01-01`,
+			value: Number(r.eur_bn),
+			status: r.phase
+		});
+		const tk = `${r.vuo}|${r.year}`;
+		const t = totals.get(tk) ?? { value: 0, phase: r.phase };
+		t.value += Number(r.eur_bn);
+		if (PHASE_RANK[r.phase] < PHASE_RANK[t.phase]) t.phase = r.phase;
+		totals.set(tk, t);
+	}
+	for (const [tk, t] of totals) {
+		const [vuo, year] = tk.split('|');
+		const id = `rf:${vuo}:total`;
+		series.set(id, {
+			series_id: id,
+			source_id: 'rijksfinancien',
+			source_code: `${vuo}/total`,
+			title: `Rijksbegroting ${label[vuo]}, all chapters`,
+			unit: 'EUR bn',
+			freq: 'A',
+			region_code: 'NLD',
+			theme: 'budget',
+			meta: { vuo, url: 'https://www.rijksfinancien.nl/open-data' }
+		});
+		obs.push({ series_id: id, period: `${year}-01-01`, value: t.value, status: t.phase });
+	}
+	return writeSeries([...series.values()], obs);
+}