website/scripts/stats/rijksfinancien.ts

// A13 Rijksfinanciën: the central-government budget tables (begroting OWB, suppletoire
// begrotingen O1/O2, jaarverslag JV) per year, down to regeling level. Open data starts in 2015.
// Normalised: annual totals per chapter and overall, from the most final phase available.
import {
	getJson,
	insertChunks,
	log,
	run,
	sql,
	writeSeries,
	type ObsRow,
	type SeriesRow
} from './common';

const FIRST_YEAR = 2015;
const PHASES = ['OWB', 'O1', 'O2', 'JV'] as const;
const PHASE_RANK: Record<string, number> = { OWB: 1, O1: 2, O2: 3, JV: 4 };

type RfRow = {
	index: string;
	jaar: number;
	fase: string;
	hoofdstuk_naam: string;
	hoofdstuk_nummer: string;
	artikel_naam: string | null;
	artikel_nummer: string | null;
	vuo: string;
	bedrag: number | null;
	[key: string]: unknown;
};

export async function rijksfinancien(opts: { backfill?: boolean; job: string }) {
	await run('rijksfinancien', opts.job, async (note) => {
		const now = new Date().getUTCFullYear();
		// Refresh: last year (JV lands in May) up to next year (OWB on Prinsjesdag).
		const years = [];
		for (let y = opts.backfill ? FIRST_YEAR : now - 1; y <= now + 1; y++) years.push(y);
		let total = 0;
		for (const year of years) {
			for (const phase of PHASES) {
				try {
					const data = await getJson<RfRow[] | Record<string, unknown>>(
						`https://www.rijksfinancien.nl/open-data/api/json/budgettaire_tabellen?year=${year}&phase=${phase}`
					);
					const list = Array.isArray(data) ? data : [];
					if (!list.length) continue;
					const rows = list.map((r, i) => {
						const {
							index,
							jaar,
							fase,
							hoofdstuk_naam,
							hoofdstuk_nummer,
							artikel_naam,
							artikel_nummer,
							vuo,
							bedrag,
							...detail
						} = r;
						void jaar;
						void fase;
						return {
							year,
							phase,
							row_key: index ?? `row-${i}`,
							vuo: (vuo || '?').slice(0, 1),
							chapter: hoofdstuk_nummer ?? '?',
							chapter_name: hoofdstuk_naam ?? '',
							article: artikel_nummer,
							article_name: artikel_naam,
							detail,
							amount_eur_k: bedrag
						};
					});
					await sql.begin(async (tx) => {
						await tx`delete from stats.rf_budget_table where year = ${year} and phase = ${phase}`;
						await insertChunks(tx, 'rf_budget_table', rows, 'on conflict do nothing');
					});
					log(`rijksfinancien ${year} ${phase}: ${rows.length} rows`);
					total += rows.length;
				} catch (error) {
					const msg = error instanceof Error ? error.message : String(error);
					if (!/HTTP 404/.test(msg)) note(`${year} ${phase}: ${msg}`);
				}
			}
		}
		await normaliseRf();
		return total;
	});
}

/** Annual U (expenditure) and O (revenue) per chapter + totals, from the most final phase. */
export async function normaliseRf() {
	const agg = await sql<
		{
			year: number;
			phase: string;
			vuo: string;
			chapter: string;
			chapter_name: string;
			eur_bn: number;
		}[]
	>`
		select year, phase, vuo, chapter, max(chapter_name) as chapter_name, sum(amount_eur_k) / 1e6 as eur_bn
		from stats.rf_budget_table where vuo in ('U', 'O') group by 1, 2, 3, 4`;
	const best = new Map<string, (typeof agg)[number]>();
	for (const r of agg) {
		const key = `${r.vuo}|${r.chapter}|${r.year}`;
		const prev = best.get(key);
		if (!prev || PHASE_RANK[r.phase] > PHASE_RANK[prev.phase]) best.set(key, r);
	}
	const series = new Map<string, SeriesRow>();
	const obs: ObsRow[] = [];
	const totals = new Map<string, { value: number; phase: string }>();
	const label = { U: 'expenditure', O: 'revenue' } as Record<string, string>;
	for (const r of best.values()) {
		const id = `rf:${r.vuo}:${r.chapter}`;
		series.set(id, {
			series_id: id,
			source_id: 'rijksfinancien',
			source_code: `${r.vuo}/${r.chapter}`,
			title: `${r.chapter_name}: ${label[r.vuo]}`,
			unit: 'EUR bn',
			freq: 'A',
			region_code: 'NLD',
			theme: 'budget',
			meta: {
				chapter: r.chapter,
				chapterName: r.chapter_name,
				vuo: r.vuo,
				url: 'https://www.rijksfinancien.nl/open-data'
			}
		});
		obs.push({
			series_id: id,
			period: `${r.year}-01-01`,
			value: Number(r.eur_bn),
			status: r.phase
		});
		const tk = `${r.vuo}|${r.year}`;
		const t = totals.get(tk) ?? { value: 0, phase: r.phase };
		t.value += Number(r.eur_bn);
		if (PHASE_RANK[r.phase] < PHASE_RANK[t.phase]) t.phase = r.phase;
		totals.set(tk, t);
	}
	for (const [tk, t] of totals) {
		const [vuo, year] = tk.split('|');
		const id = `rf:${vuo}:total`;
		series.set(id, {
			series_id: id,
			source_id: 'rijksfinancien',
			source_code: `${vuo}/total`,
			title: `Rijksbegroting ${label[vuo]}, all chapters`,
			unit: 'EUR bn',
			freq: 'A',
			region_code: 'NLD',
			theme: 'budget',
			meta: { vuo, url: 'https://www.rijksfinancien.nl/open-data' }
		});
		obs.push({ series_id: id, period: `${year}-01-01`, value: t.value, status: t.phase });
	}
	return writeSeries([...series.values()], obs);
}