Add energy tracker (/projects/energy)

Public, read-only page over the ned-collector energy schema: latest-hour
mix and per-source utilization, generation vs consumption with CO2
intensity (day/week/month/12 months/all years), NED forecast, gas storage
by year and per site, and gas-fired power share vs storage fill.
Incomplete hours (sources not summing to the NED total) are left out and
partial days/months/years are marked.

src/lib/energy/EnergyChart.svelte

@@ -0,0 +1,104 @@
+<script lang="ts" module>
+	export type ChartTheme = {
+		text: string;
+		muted: string;
+		border: string;
+		surface: string;
+		dark: boolean;
+	};
+</script>
+
+<script lang="ts">
+	import { onMount } from 'svelte';
+	import type { ECharts, EChartsCoreOption } from 'echarts/core';
+
+	// Generic ECharts wrapper for the energy tracker: same lazy-loading and theme handling as
+	// the CS price chart, but the caller builds the option from the current theme colours.
+	let {
+		option,
+		label,
+		height = 'min(52vh, 440px)'
+	}: {
+		option: (theme: ChartTheme) => EChartsCoreOption;
+		label: string;
+		height?: string;
+	} = $props();
+	let element: HTMLDivElement;
+	let chart: ECharts | undefined;
+	let ready = $state(false);
+
+	function update() {
+		if (!chart) return;
+		const style = getComputedStyle(document.documentElement);
+		const read = (name: string) => style.getPropertyValue(name).trim();
+		chart.setOption(
+			option({
+				text: read('--site-text'),
+				muted: read('--site-muted'),
+				border: read('--site-border'),
+				surface: read('--site-surface'),
+				dark: read('--site-bg') === '#000'
+			}),
+			true
+		);
+	}
+
+	onMount(() => {
+		let disposed = false;
+		let resize: ResizeObserver | undefined;
+		let theme: MutationObserver | undefined;
+		import('echarts/core').then(async (echarts) => {
+			const [
+				{ LineChart, BarChart, PieChart },
+				{ GridComponent, TooltipComponent, LegendComponent, DataZoomComponent, MarkAreaComponent },
+				{ CanvasRenderer }
+			] = await Promise.all([
+				import('echarts/charts'),
+				import('echarts/components'),
+				import('echarts/renderers')
+			]);
+			if (disposed) return;
+			echarts.use([
+				LineChart,
+				BarChart,
+				PieChart,
+				GridComponent,
+				TooltipComponent,
+				LegendComponent,
+				DataZoomComponent,
+				MarkAreaComponent,
+				CanvasRenderer
+			]);
+			chart = echarts.init(element, undefined, { renderer: 'canvas' });
+			ready = true;
+			update();
+			resize = new ResizeObserver(() => chart?.resize());
+			resize.observe(element);
+			theme = new MutationObserver(update);
+			theme.observe(document.documentElement, {
+				attributes: true,
+				attributeFilter: ['data-theme']
+			});
+		});
+		return () => {
+			disposed = true;
+			resize?.disconnect();
+			theme?.disconnect();
+			chart?.dispose();
+		};
+	});
+	$effect(() => {
+		// Re-render when the option builder (and therefore its inputs) changes.
+		void option;
+		if (ready) update();
+	});
+</script>
+
+<div class="chart" style:height bind:this={element} role="img" aria-label={label}></div>
+
+<style>
+	.chart {
+		width: 100%;
+		min-height: 260px;
+	}
+</style>

src/lib/energy/data.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from 'vitest';
+import { fillGaps, gaps, latestSnapshot, seasonalTime, toBuckets, type EnergyRow } from './data';
+
+const hour = 3_600_000;
+const rows: EnergyRow[] = [
+	{ t: 0, type: 'ElectricityMix', value: 100, ef: 0.2 },
+	{ t: 0, type: 'ElectricityLoad', value: 90 },
+	{ t: 0, type: 'Wind', value: 60, pct: 0.5 },
+	{ t: 0, type: 'FossilGasPower', value: 30, pct: 0.1 },
+	{ t: 0, type: 'Nuclear', value: 10, pct: 1 },
+	{ t: hour, type: 'ElectricityMix', value: 100 },
+	{ t: hour, type: 'Wind', value: 20 }
+];
+
+describe('energy buckets', () => {
+	it('flags buckets whose sources do not add up to the total', () => {
+		const buckets = toBuckets(rows);
+		expect(buckets.map((bucket) => bucket.incomplete)).toEqual([false, true]);
+		expect(buckets[0].sum).toBe(100);
+		expect(buckets[0].load).toBe(90);
+	});
+
+	it('summarises the latest complete bucket', () => {
+		const snapshot = latestSnapshot(toBuckets(rows))!;
+		expect(snapshot.t).toBe(0);
+		expect(snapshot.renewableShare).toBeCloseTo(0.6);
+		expect(snapshot.lowCarbonShare).toBeCloseTo(0.7);
+		expect(snapshot.fossilShare).toBeCloseTo(0.3);
+		expect(snapshot.sources[0]).toMatchObject({ type: 'Wind', share: 0.6, utilization: 0.5 });
+	});
+
+	it('finds and fills missing hours', () => {
+		expect(gaps([0, hour, 4 * hour])).toEqual([{ from: 2 * hour, to: 4 * hour, hours: 2 }]);
+		const filled = fillGaps(toBuckets([...rows, { t: 3 * hour, type: 'Wind', value: 1 }]), hour);
+		expect(filled.map((bucket) => 'missing' in bucket)).toEqual([false, false, true, false]);
+	});
+
+	it('maps dates onto one seasonal year', () => {
+		expect(seasonalTime('2023-02-28')).toBe(Date.UTC(2000, 1, 28));
+		expect(seasonalTime('2024-12-31')).toBe(Date.UTC(2000, 11, 31));
+	});
+});

src/lib/energy/data.ts

@@ -0,0 +1,195 @@
+// Pure helpers for the energy tracker. Inputs come from the ned-collector tables in the
+// `energy` schema (NED.nl electricity data and AGSI/GIE gas storage).
+
+export type SourceType =
+	| 'Nuclear'
+	| 'WindOffshoreC'
+	| 'Wind'
+	| 'Solar'
+	| 'BiomassPower'
+	| 'WastePower'
+	| 'WKK Total'
+	| 'FossilGasPower'
+	| 'FossilHardCoal'
+	| 'OtherPower';
+
+export type Source = {
+	type: SourceType;
+	label: string;
+	color: string;
+	renewable: boolean;
+	fossil: boolean;
+};
+
+// Stacking order, bottom to top: steady baseload first, weather-driven next, fossil on top.
+export const SOURCES: readonly Source[] = [
+	{ type: 'Nuclear', label: 'Nuclear', color: '#a78bfa', renewable: false, fossil: false },
+	{
+		type: 'WindOffshoreC',
+		label: 'Wind offshore',
+		color: '#0284c7',
+		renewable: true,
+		fossil: false
+	},
+	{ type: 'Wind', label: 'Wind onshore', color: '#38bdf8', renewable: true, fossil: false },
+	{ type: 'Solar', label: 'Solar', color: '#facc15', renewable: true, fossil: false },
+	{ type: 'BiomassPower', label: 'Biomass', color: '#65a30d', renewable: true, fossil: false },
+	{ type: 'WastePower', label: 'Waste', color: '#a3a3a3', renewable: false, fossil: false },
+	{ type: 'WKK Total', label: 'CHP (WKK)', color: '#fdba74', renewable: false, fossil: true },
+	{ type: 'FossilGasPower', label: 'Fossil gas', color: '#f97316', renewable: false, fossil: true },
+	{ type: 'FossilHardCoal', label: 'Hard coal', color: '#57534e', renewable: false, fossil: true },
+	{ type: 'OtherPower', label: 'Other', color: '#94a3b8', renewable: false, fossil: false }
+];
+
+export const MIX = 'ElectricityMix';
+export const LOAD = 'ElectricityLoad';
+const sourceTypes = new Set<string>(SOURCES.map((source) => source.type));
+
+/** One row per (time bucket, type). `value` is MWh for hours/days and GWh for months/years. */
+export type EnergyRow = {
+	t: number;
+	type: string;
+	value: number;
+	pct?: number | null;
+	ef?: number | null;
+	hours?: number;
+	expected?: number;
+};
+
+export type Bucket = {
+	t: number;
+	mix: number | null;
+	load: number | null;
+	ef: number | null;
+	sources: Partial<Record<SourceType, { value: number; pct: number | null }>>;
+	sum: number;
+	hours?: number;
+	/** Hours in the period (23 or 25 on DST change days). */
+	expected?: number;
+	/** The source rows don't add up to the NED total, so the bucket is still being filled in. */
+	incomplete: boolean;
+};
+
+/** Group rows into buckets ordered by time. A bucket is incomplete when its sources sum to
+ * less than 98% (or more than 102%) of the published total. */
+export function toBuckets(rows: readonly EnergyRow[]): Bucket[] {
+	const buckets = new Map<number, Bucket>();
+	for (const row of rows) {
+		let bucket = buckets.get(row.t);
+		if (!bucket) {
+			bucket = {
+				t: row.t,
+				mix: null,
+				load: null,
+				ef: null,
+				sources: {},
+				sum: 0,
+				incomplete: false
+			};
+			buckets.set(row.t, bucket);
+		}
+		if (row.hours !== undefined) bucket.hours = Math.max(bucket.hours ?? 0, row.hours);
+		if (row.expected !== undefined) bucket.expected = row.expected;
+		if (row.type === MIX) {
+			bucket.mix = row.value;
+			bucket.ef = row.ef ?? null;
+		} else if (row.type === LOAD) bucket.load = row.value;
+		else if (sourceTypes.has(row.type)) {
+			bucket.sources[row.type as SourceType] = { value: row.value, pct: row.pct ?? null };
+			bucket.sum += row.value;
+		}
+	}
+	const list = [...buckets.values()].sort((a, b) => a.t - b.t);
+	for (const bucket of list) {
+		bucket.incomplete =
+			bucket.mix === null ? bucket.sum === 0 : Math.abs(bucket.sum / bucket.mix - 1) > 0.02;
+	}
+	return list;
+}
+
+export type Snapshot = {
+	t: number;
+	total: number;
+	load: number | null;
+	ef: number | null;
+	renewableShare: number;
+	lowCarbonShare: number;
+	fossilShare: number;
+	sources: {
+		type: SourceType;
+		label: string;
+		color: string;
+		value: number;
+		share: number;
+		utilization: number | null;
+	}[];
+};
+
+/** The latest complete bucket, broken down by source. */
+export function latestSnapshot(buckets: readonly Bucket[]): Snapshot | null {
+	const bucket = [...buckets]
+		.reverse()
+		.find((candidate) => !candidate.incomplete && candidate.sum > 0);
+	if (!bucket) return null;
+	const total = bucket.sum;
+	const share = (filter: (source: Source) => boolean) =>
+		SOURCES.filter(filter).reduce(
+			(sum, source) => sum + (bucket.sources[source.type]?.value ?? 0),
+			0
+		) / total;
+	return {
+		t: bucket.t,
+		total,
+		load: bucket.load,
+		ef: bucket.ef,
+		renewableShare: share((source) => source.renewable),
+		lowCarbonShare: share((source) => source.renewable || source.type === 'Nuclear'),
+		fossilShare: share((source) => source.fossil),
+		sources: SOURCES.map((source) => {
+			const entry = bucket.sources[source.type];
+			return {
+				type: source.type,
+				label: source.label,
+				color: source.color,
+				value: entry?.value ?? 0,
+				share: (entry?.value ?? 0) / total,
+				utilization: entry?.pct ?? null
+			};
+		}).sort((a, b) => b.value - a.value)
+	};
+}
+
+/** Insert empty buckets so charts show gaps where the collector missed data. */
+export function fillGaps(
+	buckets: readonly Bucket[],
+	step: number
+): (Bucket | { t: number; missing: true })[] {
+	const out: (Bucket | { t: number; missing: true })[] = [];
+	for (const bucket of buckets) {
+		const previous = out.at(-1);
+		if (previous)
+			for (let t = previous.t + step; t < bucket.t; t += step) out.push({ t, missing: true });
+		out.push(bucket);
+	}
+	return out;
+}
+
+/** Hours missing between consecutive buckets, as [from, to) ranges in epoch ms. */
+export function gaps(
+	times: readonly number[],
+	step = 3_600_000
+): { from: number; to: number; hours: number }[] {
+	const result = [];
+	for (let index = 1; index < times.length; index++) {
+		const missing = Math.round((times[index] - times[index - 1]) / step) - 1;
+		if (missing > 0)
+			result.push({ from: times[index - 1] + step, to: times[index], hours: missing });
+	}
+	return result;
+}
+
+/** Map a date onto leap year 2000 so different years can share one seasonal x axis. */
+export function seasonalTime(day: string) {
+	const [, month, date] = day.split('-').map(Number);
+	return Date.UTC(2000, month - 1, date);
+}

src/lib/projects.ts

@@ -58,6 +58,12 @@ export const trackers = [
 		href: '/projects/cs-prices',
 		external: false
 	},
+	{
+		name: 'energy',
+		subtitle: 'Dutch electricity mix, utilization and gas storage',
+		href: '/projects/energy',
+		external: false
+	},
 	{
 		name: 'animaldex',
 		subtitle: 'add neighborhood pets to the world map',

src/lib/seo.ts

@@ -38,7 +38,7 @@ const publicPages: Record<string, { title: string; description: string }> = {
 	'/trackers': {
 		title: 'Trackers',
 		description:
-			'Stan Runge’s trackers: games, money, houses, Counter-Strike prices, Animaldex and lists.'
+			'Stan Runge’s trackers: games, money, houses, Counter-Strike prices, Dutch energy, Animaldex and lists.'
 	},
 	'/lists': { title: 'Lists', description: 'Lists collected and maintained by Stan Runge.' },
 	'/projects/game-tracker': {
@@ -69,6 +69,11 @@ const publicPages: Record<string, { title: string; description: string }> = {
 		description:
 			'Compare historical Counter-Strike market prices by listing, weapon, skin, wear, and other item dimensions.'
 	},
+	'/projects/energy': {
+		title: 'Energy',
+		description:
+			'Dutch electricity generation mix, source utilization, consumption trends, CO₂ intensity and gas storage, from NED.nl data.'
+	},
 	'/projects/derby': {
 		title: 'Derby Lens',
 		description: 'Estimate horse racing odds and compare them with market prices.'

src/lib/server/energy.ts

@@ -0,0 +1,121 @@
+import { db } from '#lib/server/db/index.js';
+import { sql } from 'drizzle-orm';
+import type { EnergyRow } from '#lib/energy/data.js';
+
+// Read-only queries over the ned-collector tables in the `energy` schema. The app role
+// already has USAGE + SELECT there; nothing on the site writes to these tables.
+
+type Raw = Record<string, unknown>;
+const num = (value: unknown) => (value === null || value === undefined ? null : Number(value));
+const time = (value: unknown) => new Date(value as string | Date).getTime();
+const rows = (result: unknown) => result as Raw[];
+
+function energyRows(result: unknown): EnergyRow[] {
+	return rows(result).map((row) => ({
+		t: time(row.t),
+		type: String(row.type),
+		value: Number(row.value),
+		pct: num(row.pct),
+		ef: num(row.ef),
+		...(row.hours === undefined ? {} : { hours: Number(row.hours) }),
+		...(row.expected === undefined ? {} : { expected: Number(row.expected) })
+	}));
+}
+
+export async function loadEnergy() {
+	const [hourly, daily, forecast, monthly, yearly, gasNl, gasSites, runs, coverage] =
+		await Promise.all([
+			// Last 7 days of hourly measurements, counted back from the newest hour NED published.
+			db.execute(sql`WITH latest AS (
+					SELECT max(valid_from) AS t FROM energy.ned_utilization
+					WHERE granularity = 'Hour' AND classification = 'Current')
+				SELECT valid_from AS t, type, volume_kwh::float8 / 1000 AS value,
+					percentage::float8 AS pct, emission_factor::float8 AS ef
+				FROM energy.ned_utilization, latest
+				WHERE granularity = 'Hour' AND classification = 'Current'
+					AND valid_from > latest.t - interval '7 days'
+				ORDER BY 1, 2`),
+			// Daily totals (Amsterdam days) from hourly data, last 31 days. `expected` is 23 or 25
+			// on DST change days so partial days can be flagged.
+			db.execute(sql`WITH latest AS (
+					SELECT max(valid_from) AS t FROM energy.ned_utilization
+					WHERE granularity = 'Hour' AND classification = 'Current'),
+				days AS (
+					SELECT date_trunc('day', valid_from AT TIME ZONE 'Europe/Amsterdam') AS day, type,
+						sum(volume_kwh) AS kwh, sum(emission_kg) AS kg, count(*)::int AS hours
+					FROM energy.ned_utilization, latest
+					WHERE granularity = 'Hour' AND classification = 'Current'
+						AND valid_from > latest.t - interval '31 days'
+					GROUP BY 1, 2)
+				SELECT day AT TIME ZONE 'Europe/Amsterdam' AS t, type, kwh::float8 / 1000 AS value, hours,
+					(extract(epoch FROM (day + interval '1 day') AT TIME ZONE 'Europe/Amsterdam'
+						- day AT TIME ZONE 'Europe/Amsterdam') / 3600)::int AS expected,
+					(kg / nullif(kwh, 0))::float8 AS ef
+				FROM days ORDER BY 1, 2`),
+			db.execute(sql`WITH latest AS (
+					SELECT max(valid_from) AS t FROM energy.ned_utilization
+					WHERE granularity = 'Hour' AND classification = 'Current')
+				SELECT valid_from AS t, type, volume_kwh::float8 / 1000 AS value
+				FROM energy.ned_utilization, latest
+				WHERE granularity = 'Hour' AND classification = 'Forecast'
+					AND valid_from > latest.t AND valid_from <= latest.t + interval '8 days'
+					AND type IN ('Wind', 'WindOffshoreC', 'Solar', 'ElectricityLoad', 'ElectricityMix')
+				ORDER BY 1, 2`),
+			db.execute(sql`SELECT valid_from AS t, type, volume_kwh::float8 / 1e6 AS value,
+					percentage::float8 AS pct, emission_factor::float8 AS ef
+				FROM energy.ned_utilization
+				WHERE granularity = 'Month' AND classification = 'Current'
+				ORDER BY 1, 2`),
+			db.execute(sql`SELECT valid_from AS t, type, volume_kwh::float8 / 1e6 AS value,
+					emission_factor::float8 AS ef
+				FROM energy.ned_utilization
+				WHERE granularity = 'Year' AND classification = 'Current'
+				ORDER BY 1, 2`),
+			db.execute(sql`SELECT gas_day::text AS day, fill_pct::float8 AS fill, volume_twh::float8 AS twh
+				FROM energy.gas_storage_daily WHERE site = 'NL' ORDER BY gas_day`),
+			db.execute(sql`SELECT site, gas_day::text AS day, fill_pct::float8 AS fill,
+					volume_twh::float8 AS twh
+				FROM energy.gas_storage_latest WHERE site <> 'NL' ORDER BY volume_twh DESC`),
+			db.execute(sql`SELECT
+					(SELECT max(started_at) FROM energy.collector_runs) AS last_run,
+					(SELECT max(finished_at) FROM energy.collector_runs WHERE ok) AS last_ok,
+					(SELECT count(*)::int FROM energy.collector_runs
+						WHERE NOT ok AND started_at > now() - interval '7 days') AS failed_week`),
+			db.execute(sql`SELECT min(valid_from) AS first_hour, max(valid_from) AS last_hour,
+					count(DISTINCT valid_from)::int AS hours
+				FROM energy.ned_utilization WHERE granularity = 'Hour' AND classification = 'Current'`)
+		]);
+
+	const run = rows(runs)[0] ?? {};
+	const cover = rows(coverage)[0] ?? {};
+	return {
+		hourly: energyRows(hourly),
+		daily: energyRows(daily),
+		forecast: energyRows(forecast),
+		monthly: energyRows(monthly),
+		yearly: energyRows(yearly),
+		gas: rows(gasNl).map((row) => ({
+			day: String(row.day),
+			fill: Number(row.fill),
+			twh: Number(row.twh)
+		})),
+		gasSites: rows(gasSites).map((row) => ({
+			site: String(row.site),
+			day: String(row.day),
+			fill: Number(row.fill),
+			twh: Number(row.twh)
+		})),
+		collector: {
+			lastRun: run.last_run ? time(run.last_run) : null,
+			lastOk: run.last_ok ? time(run.last_ok) : null,
+			failedWeek: Number(run.failed_week ?? 0)
+		},
+		coverage: {
+			firstHour: cover.first_hour ? time(cover.first_hour) : null,
+			lastHour: cover.last_hour ? time(cover.last_hour) : null,
+			hours: Number(cover.hours ?? 0)
+		}
+	};
+}
+
+export type EnergyData = Awaited<ReturnType<typeof loadEnergy>>;

src/lib/title.ts

@@ -30,6 +30,7 @@ export function getTitle(path: string, environment: SiteEnvironment = null): str
 		'/projects/animaldex/admin': 'Animaldex admin',
 		'/projects/derby': 'Derby Lens',
 		'/projects/cs-prices': 'CS Prices',
+		'/projects/energy': 'Energy',
 		'/projects/game-tracker': 'Game Tracker',
 		'/projects/language-learner': 'Language Learner',
 		'/projects/money-tracker': 'Money Tracker',