stats step 2: World Bank, IMF WEO, OWID, Energy-Charts, Rijksfinanciën, US Treasury, USAspending collectors + tables, timers

2026-10-25 21:05:51 +00:00
parent 7e905b1651
commit 8958d3ff8f
21 changed files with 34655 additions and 3 deletions
--- a/scripts/stats/energycharts.ts
+++ b/scripts/stats/energycharts.ts
@@ -0,0 +1,120 @@
+// A10 Energy-Charts (Fraunhofer ISE): NL day-ahead prices and NL cross-border physical flows.
+// Raw rows keep the API resolution (hourly until Oct 2025, 15-minute after); the normalised
+// layer holds daily / monthly averages (prices) and monthly net TWh per neighbour (flows).
+import { getJson, insertChunks, log, run, sql } from './common';
+
+const BASE = 'https://api.energy-charts.info';
+const BACKFILL_FROM = '2015-01-01';
+
+const ymd = (d: Date) => d.toISOString().slice(0, 10);
+function monthChunks(from: string, to: Date) {
+	const out: [string, string][] = [];
+	const d = new Date(`${from}T00:00:00Z`);
+	while (d < to) {
+		const start = ymd(d);
+		d.setUTCMonth(d.getUTCMonth() + 1);
+		const end = new Date(d.getTime() - 86_400_000);
+		out.push([start, ymd(end < to ? end : to)]);
+	}
+	return out;
+}
+
+type PriceBody = { unix_seconds?: number[]; price?: (number | null)[] };
+type CbetBody = {
+	unix_seconds?: number[];
+	countries?: { name: string; data: (number | null)[] }[];
+};
+
+async function pullRange(start: string, end: string) {
+	let rows = 0;
+	const price = await getJson<PriceBody>(`${BASE}/price?bzn=NL&start=${start}&end=${end}`);
+	const p = (price.unix_seconds ?? []).map((t, i) => ({
+		bzn: 'NL',
+		ts: new Date(t * 1000).toISOString(),
+		eur_mwh: price.price?.[i] ?? null
+	}));
+	const cbet = await getJson<CbetBody>(`${BASE}/cbet?country=nl&start=${start}&end=${end}`);
+	const c = (cbet.countries ?? []).flatMap((n) =>
+		(cbet.unix_seconds ?? []).map((t, i) => ({
+			country: 'nl',
+			neighbour: n.name,
+			ts: new Date(t * 1000).toISOString(),
+			gw: n.data[i] ?? null
+		}))
+	);
+	await sql.begin(async (tx) => {
+		await insertChunks(
+			tx,
+			'ec_dayahead_price',
+			p,
+			'on conflict (bzn, ts) do update set eur_mwh = excluded.eur_mwh'
+		);
+		await insertChunks(
+			tx,
+			'ec_crossborder',
+			c,
+			'on conflict (country, neighbour, ts) do update set gw = excluded.gw'
+		);
+	});
+	rows += p.length + c.length;
+	return rows;
+}
+
+/** Rebuild the chart series from the raw tables (cheap: a few hundred thousand rows). */
+export async function normaliseEnergyCharts() {
+	const source = { url: 'https://www.energy-charts.info/charts/price_spot_market/chart.htm?c=NL' };
+	await sql.begin(async (tx) => {
+		const series = [
+			['ec:price:NL:D', 'Day-ahead electricity price, daily average', 'EUR/MWh', 'D'],
+			['ec:price:NL:M', 'Day-ahead electricity price, monthly average', 'EUR/MWh', 'M']
+		];
+		for (const [id, title, unit, freq] of series)
+			await tx`insert into stats.series (series_id, source_id, source_code, title, unit, freq, region_code, theme, meta)
+				values (${id}, 'energycharts', 'price/NL', ${title}, ${unit}, ${freq}, 'NLD', 'energy', ${tx.json(source)})
+				on conflict (series_id) do update set title = excluded.title, unit = excluded.unit, meta = excluded.meta`;
+		await tx`insert into stats.series (series_id, source_id, source_code, title, unit, freq, region_code, theme, meta)
+			select 'ec:cbet:nl:' || lower(replace(neighbour, ' ', '-')) || ':M', 'energycharts', 'cbet/nl/' || neighbour,
+				'Net physical electricity import from ' || neighbour, 'TWh', 'M', 'NLD', 'energy',
+				${tx.json({ url: 'https://www.energy-charts.info/charts/import_export/chart.htm?c=NL', sign: 'positive = import into NL' })}
+			from (select distinct neighbour from stats.ec_crossborder where country = 'nl') n
+			on conflict (series_id) do update set title = excluded.title, meta = excluded.meta`;
+		await tx`delete from stats.obs where series_id like 'ec:%'`;
+		await tx`insert into stats.obs (series_id, period, value)
+			select 'ec:price:NL:D', (ts at time zone 'Europe/Amsterdam')::date, avg(eur_mwh)
+			from stats.ec_dayahead_price where bzn = 'NL' and eur_mwh is not null group by 2`;
+		await tx`insert into stats.obs (series_id, period, value)
+			select 'ec:price:NL:M', date_trunc('month', ts at time zone 'Europe/Amsterdam')::date, avg(eur_mwh)
+			from stats.ec_dayahead_price where bzn = 'NL' and eur_mwh is not null group by 2`;
+		// Average GW over the month × hours in the month / 1000 = TWh (robust to the resolution change).
+		await tx`insert into stats.obs (series_id, period, value)
+			select 'ec:cbet:nl:' || lower(replace(neighbour, ' ', '-')) || ':M', m,
+				avg(gw) * extract(epoch from (m + interval '1 month') - m) / 3600 / 1000
+			from (select neighbour, gw, date_trunc('month', ts at time zone 'Europe/Amsterdam')::date as m
+				from stats.ec_crossborder where country = 'nl' and gw is not null) x
+			group by neighbour, m`;
+	});
+	const [{ n }] = await sql`select count(*)::int as n from stats.obs where series_id like 'ec:%'`;
+	return n as number;
+}
+
+export async function energycharts(opts: { backfill?: boolean; job: string }) {
+	await run('energycharts', opts.job, async (note) => {
+		const now = new Date();
+		const tomorrow = new Date(now.getTime() + 86_400_000);
+		const chunks = opts.backfill
+			? monthChunks(BACKFILL_FROM, tomorrow)
+			: [[ymd(new Date(now.getTime() - 3 * 86_400_000)), ymd(tomorrow)] as [string, string]];
+		let rows = 0;
+		for (const [start, end] of chunks) {
+			try {
+				rows += await pullRange(start, end);
+				if (opts.backfill) log(`energycharts ${start}..${end}: ${rows} rows so far`);
+			} catch (error) {
+				note(`${start}..${end}: ${error instanceof Error ? error.message : String(error)}`);
+			}
+		}
+		const obs = await normaliseEnergyCharts();
+		log(`energycharts: ${rows} raw rows, ${obs} normalised obs`);
+		return rows;
+	});
+}