website/scripts/stats/energycharts.ts

// A10 Energy-Charts (Fraunhofer ISE): NL day-ahead prices and NL cross-border physical flows.
// Raw rows keep the API resolution (hourly until Oct 2025, 15-minute after); the normalised
// layer holds daily / monthly averages (prices) and monthly net TWh per neighbour (flows).
import { getJson, insertChunks, log, run, sql } from './common';

const BASE = 'https://api.energy-charts.info';
const BACKFILL_FROM = '2015-01-01';

const ymd = (d: Date) => d.toISOString().slice(0, 10);
function monthChunks(from: string, to: Date) {
	const out: [string, string][] = [];
	const d = new Date(`${from}T00:00:00Z`);
	while (d < to) {
		const start = ymd(d);
		d.setUTCMonth(d.getUTCMonth() + 1);
		const end = new Date(d.getTime() - 86_400_000);
		out.push([start, ymd(end < to ? end : to)]);
	}
	return out;
}

type PriceBody = { unix_seconds?: number[]; price?: (number | null)[] };
type CbetBody = {
	unix_seconds?: number[];
	countries?: { name: string; data: (number | null)[] }[];
};

async function pullRange(start: string, end: string) {
	let rows = 0;
	const price = await getJson<PriceBody>(`${BASE}/price?bzn=NL&start=${start}&end=${end}`);
	const p = (price.unix_seconds ?? []).map((t, i) => ({
		bzn: 'NL',
		ts: new Date(t * 1000).toISOString(),
		eur_mwh: price.price?.[i] ?? null
	}));
	const cbet = await getJson<CbetBody>(`${BASE}/cbet?country=nl&start=${start}&end=${end}`);
	const c = (cbet.countries ?? []).flatMap((n) =>
		(cbet.unix_seconds ?? []).map((t, i) => ({
			country: 'nl',
			neighbour: n.name,
			ts: new Date(t * 1000).toISOString(),
			gw: n.data[i] ?? null
		}))
	);
	await sql.begin(async (tx) => {
		await insertChunks(
			tx,
			'ec_dayahead_price',
			p,
			'on conflict (bzn, ts) do update set eur_mwh = excluded.eur_mwh'
		);
		await insertChunks(
			tx,
			'ec_crossborder',
			c,
			'on conflict (country, neighbour, ts) do update set gw = excluded.gw'
		);
	});
	rows += p.length + c.length;
	return rows;
}

/** Rebuild the chart series from the raw tables (cheap: a few hundred thousand rows). */
export async function normaliseEnergyCharts() {
	const source = { url: 'https://www.energy-charts.info/charts/price_spot_market/chart.htm?c=NL' };
	await sql.begin(async (tx) => {
		const series = [
			['ec:price:NL:D', 'Day-ahead electricity price, daily average', 'EUR/MWh', 'D'],
			['ec:price:NL:M', 'Day-ahead electricity price, monthly average', 'EUR/MWh', 'M']
		];
		for (const [id, title, unit, freq] of series)
			await tx`insert into stats.series (series_id, source_id, source_code, title, unit, freq, region_code, theme, meta)
				values (${id}, 'energycharts', 'price/NL', ${title}, ${unit}, ${freq}, 'NLD', 'energy', ${tx.json(source)})
				on conflict (series_id) do update set title = excluded.title, unit = excluded.unit, meta = excluded.meta`;
		await tx`insert into stats.series (series_id, source_id, source_code, title, unit, freq, region_code, theme, meta)
			select 'ec:cbet:nl:' || lower(replace(neighbour, ' ', '-')) || ':M', 'energycharts', 'cbet/nl/' || neighbour,
				'Net physical electricity import from ' || neighbour, 'TWh', 'M', 'NLD', 'energy',
				${tx.json({ url: 'https://www.energy-charts.info/charts/import_export/chart.htm?c=NL', sign: 'positive = import into NL' })}
			from (select distinct neighbour from stats.ec_crossborder where country = 'nl') n
			on conflict (series_id) do update set title = excluded.title, meta = excluded.meta`;
		await tx`delete from stats.obs where series_id like 'ec:%'`;
		await tx`insert into stats.obs (series_id, period, value)
			select 'ec:price:NL:D', (ts at time zone 'Europe/Amsterdam')::date, avg(eur_mwh)
			from stats.ec_dayahead_price where bzn = 'NL' and eur_mwh is not null group by 2`;
		await tx`insert into stats.obs (series_id, period, value)
			select 'ec:price:NL:M', date_trunc('month', ts at time zone 'Europe/Amsterdam')::date, avg(eur_mwh)
			from stats.ec_dayahead_price where bzn = 'NL' and eur_mwh is not null group by 2`;
		// Average GW over the month × hours in the month / 1000 = TWh (robust to the resolution change).
		await tx`insert into stats.obs (series_id, period, value)
			select 'ec:cbet:nl:' || lower(replace(neighbour, ' ', '-')) || ':M', m,
				avg(gw) * extract(epoch from (m + interval '1 month') - m) / 3600 / 1000
			from (select neighbour, gw, date_trunc('month', ts at time zone 'Europe/Amsterdam')::date as m
				from stats.ec_crossborder where country = 'nl' and gw is not null) x
			group by neighbour, m`;
	});
	const [{ n }] = await sql`select count(*)::int as n from stats.obs where series_id like 'ec:%'`;
	return n as number;
}

export async function energycharts(opts: { backfill?: boolean; job: string }) {
	await run('energycharts', opts.job, async (note) => {
		const now = new Date();
		const tomorrow = new Date(now.getTime() + 86_400_000);
		const chunks = opts.backfill
			? monthChunks(BACKFILL_FROM, tomorrow)
			: [[ymd(new Date(now.getTime() - 3 * 86_400_000)), ymd(tomorrow)] as [string, string]];
		let rows = 0;
		for (const [start, end] of chunks) {
			try {
				rows += await pullRange(start, end);
				if (opts.backfill) log(`energycharts ${start}..${end}: ${rows} rows so far`);
			} catch (error) {
				note(`${start}..${end}: ${error instanceof Error ? error.message : String(error)}`);
			}
		}
		const obs = await normaliseEnergyCharts();
		log(`energycharts: ${rows} raw rows, ${obs} normalised obs`);
		return rows;
	});
}