From 0cf5d440623624e78ec5d0362f8f1368d93f99d6 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Fri, 23 Oct 2026 23:11:27 +0000 Subject: [PATCH] Add energy tracker (/projects/energy) Public, read-only page over the ned-collector energy schema: latest-hour mix and per-source utilization, generation vs consumption with CO2 intensity (day/week/month/12 months/all years), NED forecast, gas storage by year and per site, and gas-fired power share vs storage fill. Incomplete hours (sources not summing to the NED total) are left out and partial days/months/years are marked. --- src/lib/energy/EnergyChart.svelte | 104 +++ src/lib/energy/data.test.ts | 42 + src/lib/energy/data.ts | 195 +++++ src/lib/projects.ts | 6 + src/lib/seo.ts | 7 +- src/lib/server/energy.ts | 121 +++ src/lib/title.ts | 1 + src/routes/projects/energy/+page.server.ts | 4 + src/routes/projects/energy/+page.svelte | 896 +++++++++++++++++++++ src/routes/sitemap.xml/+server.ts | 1 + tests/energy.test.ts | 17 + 11 files changed, 1393 insertions(+), 1 deletion(-) create mode 100644 src/lib/energy/EnergyChart.svelte create mode 100644 src/lib/energy/data.test.ts create mode 100644 src/lib/energy/data.ts create mode 100644 src/lib/server/energy.ts create mode 100644 src/routes/projects/energy/+page.server.ts create mode 100644 src/routes/projects/energy/+page.svelte create mode 100644 tests/energy.test.ts diff --git a/src/lib/energy/EnergyChart.svelte b/src/lib/energy/EnergyChart.svelte new file mode 100644 index 0000000..35b4639 --- /dev/null +++ b/src/lib/energy/EnergyChart.svelte @@ -0,0 +1,104 @@ + + + + + + + diff --git a/src/lib/energy/data.test.ts b/src/lib/energy/data.test.ts new file mode 100644 index 0000000..f2086d2 --- /dev/null +++ b/src/lib/energy/data.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { fillGaps, gaps, latestSnapshot, seasonalTime, toBuckets, type EnergyRow } from './data'; + +const hour = 3_600_000; +const rows: EnergyRow[] = [ + { t: 0, type: 'ElectricityMix', value: 100, ef: 0.2 }, + { t: 0, type: 'ElectricityLoad', value: 90 }, + { t: 0, type: 'Wind', value: 60, pct: 0.5 }, + { t: 0, type: 'FossilGasPower', value: 30, pct: 0.1 }, + { t: 0, type: 'Nuclear', value: 10, pct: 1 }, + { t: hour, type: 'ElectricityMix', value: 100 }, + { t: hour, type: 'Wind', value: 20 } +]; + +describe('energy buckets', () => { + it('flags buckets whose sources do not add up to the total', () => { + const buckets = toBuckets(rows); + expect(buckets.map((bucket) => bucket.incomplete)).toEqual([false, true]); + expect(buckets[0].sum).toBe(100); + expect(buckets[0].load).toBe(90); + }); + + it('summarises the latest complete bucket', () => { + const snapshot = latestSnapshot(toBuckets(rows))!; + expect(snapshot.t).toBe(0); + expect(snapshot.renewableShare).toBeCloseTo(0.6); + expect(snapshot.lowCarbonShare).toBeCloseTo(0.7); + expect(snapshot.fossilShare).toBeCloseTo(0.3); + expect(snapshot.sources[0]).toMatchObject({ type: 'Wind', share: 0.6, utilization: 0.5 }); + }); + + it('finds and fills missing hours', () => { + expect(gaps([0, hour, 4 * hour])).toEqual([{ from: 2 * hour, to: 4 * hour, hours: 2 }]); + const filled = fillGaps(toBuckets([...rows, { t: 3 * hour, type: 'Wind', value: 1 }]), hour); + expect(filled.map((bucket) => 'missing' in bucket)).toEqual([false, false, true, false]); + }); + + it('maps dates onto one seasonal year', () => { + expect(seasonalTime('2023-02-28')).toBe(Date.UTC(2000, 1, 28)); + expect(seasonalTime('2024-12-31')).toBe(Date.UTC(2000, 11, 31)); + }); +}); diff --git a/src/lib/energy/data.ts b/src/lib/energy/data.ts new file mode 100644 index 0000000..ca94dd1 --- /dev/null +++ b/src/lib/energy/data.ts @@ -0,0 +1,195 @@ +// Pure helpers for the energy tracker. Inputs come from the ned-collector tables in the +// `energy` schema (NED.nl electricity data and AGSI/GIE gas storage). + +export type SourceType = + | 'Nuclear' + | 'WindOffshoreC' + | 'Wind' + | 'Solar' + | 'BiomassPower' + | 'WastePower' + | 'WKK Total' + | 'FossilGasPower' + | 'FossilHardCoal' + | 'OtherPower'; + +export type Source = { + type: SourceType; + label: string; + color: string; + renewable: boolean; + fossil: boolean; +}; + +// Stacking order, bottom to top: steady baseload first, weather-driven next, fossil on top. +export const SOURCES: readonly Source[] = [ + { type: 'Nuclear', label: 'Nuclear', color: '#a78bfa', renewable: false, fossil: false }, + { + type: 'WindOffshoreC', + label: 'Wind offshore', + color: '#0284c7', + renewable: true, + fossil: false + }, + { type: 'Wind', label: 'Wind onshore', color: '#38bdf8', renewable: true, fossil: false }, + { type: 'Solar', label: 'Solar', color: '#facc15', renewable: true, fossil: false }, + { type: 'BiomassPower', label: 'Biomass', color: '#65a30d', renewable: true, fossil: false }, + { type: 'WastePower', label: 'Waste', color: '#a3a3a3', renewable: false, fossil: false }, + { type: 'WKK Total', label: 'CHP (WKK)', color: '#fdba74', renewable: false, fossil: true }, + { type: 'FossilGasPower', label: 'Fossil gas', color: '#f97316', renewable: false, fossil: true }, + { type: 'FossilHardCoal', label: 'Hard coal', color: '#57534e', renewable: false, fossil: true }, + { type: 'OtherPower', label: 'Other', color: '#94a3b8', renewable: false, fossil: false } +]; + +export const MIX = 'ElectricityMix'; +export const LOAD = 'ElectricityLoad'; +const sourceTypes = new Set(SOURCES.map((source) => source.type)); + +/** One row per (time bucket, type). `value` is MWh for hours/days and GWh for months/years. */ +export type EnergyRow = { + t: number; + type: string; + value: number; + pct?: number | null; + ef?: number | null; + hours?: number; + expected?: number; +}; + +export type Bucket = { + t: number; + mix: number | null; + load: number | null; + ef: number | null; + sources: Partial>; + sum: number; + hours?: number; + /** Hours in the period (23 or 25 on DST change days). */ + expected?: number; + /** The source rows don't add up to the NED total, so the bucket is still being filled in. */ + incomplete: boolean; +}; + +/** Group rows into buckets ordered by time. A bucket is incomplete when its sources sum to + * less than 98% (or more than 102%) of the published total. */ +export function toBuckets(rows: readonly EnergyRow[]): Bucket[] { + const buckets = new Map(); + for (const row of rows) { + let bucket = buckets.get(row.t); + if (!bucket) { + bucket = { + t: row.t, + mix: null, + load: null, + ef: null, + sources: {}, + sum: 0, + incomplete: false + }; + buckets.set(row.t, bucket); + } + if (row.hours !== undefined) bucket.hours = Math.max(bucket.hours ?? 0, row.hours); + if (row.expected !== undefined) bucket.expected = row.expected; + if (row.type === MIX) { + bucket.mix = row.value; + bucket.ef = row.ef ?? null; + } else if (row.type === LOAD) bucket.load = row.value; + else if (sourceTypes.has(row.type)) { + bucket.sources[row.type as SourceType] = { value: row.value, pct: row.pct ?? null }; + bucket.sum += row.value; + } + } + const list = [...buckets.values()].sort((a, b) => a.t - b.t); + for (const bucket of list) { + bucket.incomplete = + bucket.mix === null ? bucket.sum === 0 : Math.abs(bucket.sum / bucket.mix - 1) > 0.02; + } + return list; +} + +export type Snapshot = { + t: number; + total: number; + load: number | null; + ef: number | null; + renewableShare: number; + lowCarbonShare: number; + fossilShare: number; + sources: { + type: SourceType; + label: string; + color: string; + value: number; + share: number; + utilization: number | null; + }[]; +}; + +/** The latest complete bucket, broken down by source. */ +export function latestSnapshot(buckets: readonly Bucket[]): Snapshot | null { + const bucket = [...buckets] + .reverse() + .find((candidate) => !candidate.incomplete && candidate.sum > 0); + if (!bucket) return null; + const total = bucket.sum; + const share = (filter: (source: Source) => boolean) => + SOURCES.filter(filter).reduce( + (sum, source) => sum + (bucket.sources[source.type]?.value ?? 0), + 0 + ) / total; + return { + t: bucket.t, + total, + load: bucket.load, + ef: bucket.ef, + renewableShare: share((source) => source.renewable), + lowCarbonShare: share((source) => source.renewable || source.type === 'Nuclear'), + fossilShare: share((source) => source.fossil), + sources: SOURCES.map((source) => { + const entry = bucket.sources[source.type]; + return { + type: source.type, + label: source.label, + color: source.color, + value: entry?.value ?? 0, + share: (entry?.value ?? 0) / total, + utilization: entry?.pct ?? null + }; + }).sort((a, b) => b.value - a.value) + }; +} + +/** Insert empty buckets so charts show gaps where the collector missed data. */ +export function fillGaps( + buckets: readonly Bucket[], + step: number +): (Bucket | { t: number; missing: true })[] { + const out: (Bucket | { t: number; missing: true })[] = []; + for (const bucket of buckets) { + const previous = out.at(-1); + if (previous) + for (let t = previous.t + step; t < bucket.t; t += step) out.push({ t, missing: true }); + out.push(bucket); + } + return out; +} + +/** Hours missing between consecutive buckets, as [from, to) ranges in epoch ms. */ +export function gaps( + times: readonly number[], + step = 3_600_000 +): { from: number; to: number; hours: number }[] { + const result = []; + for (let index = 1; index < times.length; index++) { + const missing = Math.round((times[index] - times[index - 1]) / step) - 1; + if (missing > 0) + result.push({ from: times[index - 1] + step, to: times[index], hours: missing }); + } + return result; +} + +/** Map a date onto leap year 2000 so different years can share one seasonal x axis. */ +export function seasonalTime(day: string) { + const [, month, date] = day.split('-').map(Number); + return Date.UTC(2000, month - 1, date); +} diff --git a/src/lib/projects.ts b/src/lib/projects.ts index 7a189d4..f6e5986 100644 --- a/src/lib/projects.ts +++ b/src/lib/projects.ts @@ -58,6 +58,12 @@ export const trackers = [ href: '/projects/cs-prices', external: false }, + { + name: 'energy', + subtitle: 'Dutch electricity mix, utilization and gas storage', + href: '/projects/energy', + external: false + }, { name: 'animaldex', subtitle: 'add neighborhood pets to the world map', diff --git a/src/lib/seo.ts b/src/lib/seo.ts index c936178..bf6f21e 100644 --- a/src/lib/seo.ts +++ b/src/lib/seo.ts @@ -38,7 +38,7 @@ const publicPages: Record = { '/trackers': { title: 'Trackers', description: - 'Stan Runge’s trackers: games, money, houses, Counter-Strike prices, Animaldex and lists.' + 'Stan Runge’s trackers: games, money, houses, Counter-Strike prices, Dutch energy, Animaldex and lists.' }, '/lists': { title: 'Lists', description: 'Lists collected and maintained by Stan Runge.' }, '/projects/game-tracker': { @@ -69,6 +69,11 @@ const publicPages: Record = { description: 'Compare historical Counter-Strike market prices by listing, weapon, skin, wear, and other item dimensions.' }, + '/projects/energy': { + title: 'Energy', + description: + 'Dutch electricity generation mix, source utilization, consumption trends, CO₂ intensity and gas storage, from NED.nl data.' + }, '/projects/derby': { title: 'Derby Lens', description: 'Estimate horse racing odds and compare them with market prices.' diff --git a/src/lib/server/energy.ts b/src/lib/server/energy.ts new file mode 100644 index 0000000..7f2d6fd --- /dev/null +++ b/src/lib/server/energy.ts @@ -0,0 +1,121 @@ +import { db } from '#lib/server/db/index.js'; +import { sql } from 'drizzle-orm'; +import type { EnergyRow } from '#lib/energy/data.js'; + +// Read-only queries over the ned-collector tables in the `energy` schema. The app role +// already has USAGE + SELECT there; nothing on the site writes to these tables. + +type Raw = Record; +const num = (value: unknown) => (value === null || value === undefined ? null : Number(value)); +const time = (value: unknown) => new Date(value as string | Date).getTime(); +const rows = (result: unknown) => result as Raw[]; + +function energyRows(result: unknown): EnergyRow[] { + return rows(result).map((row) => ({ + t: time(row.t), + type: String(row.type), + value: Number(row.value), + pct: num(row.pct), + ef: num(row.ef), + ...(row.hours === undefined ? {} : { hours: Number(row.hours) }), + ...(row.expected === undefined ? {} : { expected: Number(row.expected) }) + })); +} + +export async function loadEnergy() { + const [hourly, daily, forecast, monthly, yearly, gasNl, gasSites, runs, coverage] = + await Promise.all([ + // Last 7 days of hourly measurements, counted back from the newest hour NED published. + db.execute(sql`WITH latest AS ( + SELECT max(valid_from) AS t FROM energy.ned_utilization + WHERE granularity = 'Hour' AND classification = 'Current') + SELECT valid_from AS t, type, volume_kwh::float8 / 1000 AS value, + percentage::float8 AS pct, emission_factor::float8 AS ef + FROM energy.ned_utilization, latest + WHERE granularity = 'Hour' AND classification = 'Current' + AND valid_from > latest.t - interval '7 days' + ORDER BY 1, 2`), + // Daily totals (Amsterdam days) from hourly data, last 31 days. `expected` is 23 or 25 + // on DST change days so partial days can be flagged. + db.execute(sql`WITH latest AS ( + SELECT max(valid_from) AS t FROM energy.ned_utilization + WHERE granularity = 'Hour' AND classification = 'Current'), + days AS ( + SELECT date_trunc('day', valid_from AT TIME ZONE 'Europe/Amsterdam') AS day, type, + sum(volume_kwh) AS kwh, sum(emission_kg) AS kg, count(*)::int AS hours + FROM energy.ned_utilization, latest + WHERE granularity = 'Hour' AND classification = 'Current' + AND valid_from > latest.t - interval '31 days' + GROUP BY 1, 2) + SELECT day AT TIME ZONE 'Europe/Amsterdam' AS t, type, kwh::float8 / 1000 AS value, hours, + (extract(epoch FROM (day + interval '1 day') AT TIME ZONE 'Europe/Amsterdam' + - day AT TIME ZONE 'Europe/Amsterdam') / 3600)::int AS expected, + (kg / nullif(kwh, 0))::float8 AS ef + FROM days ORDER BY 1, 2`), + db.execute(sql`WITH latest AS ( + SELECT max(valid_from) AS t FROM energy.ned_utilization + WHERE granularity = 'Hour' AND classification = 'Current') + SELECT valid_from AS t, type, volume_kwh::float8 / 1000 AS value + FROM energy.ned_utilization, latest + WHERE granularity = 'Hour' AND classification = 'Forecast' + AND valid_from > latest.t AND valid_from <= latest.t + interval '8 days' + AND type IN ('Wind', 'WindOffshoreC', 'Solar', 'ElectricityLoad', 'ElectricityMix') + ORDER BY 1, 2`), + db.execute(sql`SELECT valid_from AS t, type, volume_kwh::float8 / 1e6 AS value, + percentage::float8 AS pct, emission_factor::float8 AS ef + FROM energy.ned_utilization + WHERE granularity = 'Month' AND classification = 'Current' + ORDER BY 1, 2`), + db.execute(sql`SELECT valid_from AS t, type, volume_kwh::float8 / 1e6 AS value, + emission_factor::float8 AS ef + FROM energy.ned_utilization + WHERE granularity = 'Year' AND classification = 'Current' + ORDER BY 1, 2`), + db.execute(sql`SELECT gas_day::text AS day, fill_pct::float8 AS fill, volume_twh::float8 AS twh + FROM energy.gas_storage_daily WHERE site = 'NL' ORDER BY gas_day`), + db.execute(sql`SELECT site, gas_day::text AS day, fill_pct::float8 AS fill, + volume_twh::float8 AS twh + FROM energy.gas_storage_latest WHERE site <> 'NL' ORDER BY volume_twh DESC`), + db.execute(sql`SELECT + (SELECT max(started_at) FROM energy.collector_runs) AS last_run, + (SELECT max(finished_at) FROM energy.collector_runs WHERE ok) AS last_ok, + (SELECT count(*)::int FROM energy.collector_runs + WHERE NOT ok AND started_at > now() - interval '7 days') AS failed_week`), + db.execute(sql`SELECT min(valid_from) AS first_hour, max(valid_from) AS last_hour, + count(DISTINCT valid_from)::int AS hours + FROM energy.ned_utilization WHERE granularity = 'Hour' AND classification = 'Current'`) + ]); + + const run = rows(runs)[0] ?? {}; + const cover = rows(coverage)[0] ?? {}; + return { + hourly: energyRows(hourly), + daily: energyRows(daily), + forecast: energyRows(forecast), + monthly: energyRows(monthly), + yearly: energyRows(yearly), + gas: rows(gasNl).map((row) => ({ + day: String(row.day), + fill: Number(row.fill), + twh: Number(row.twh) + })), + gasSites: rows(gasSites).map((row) => ({ + site: String(row.site), + day: String(row.day), + fill: Number(row.fill), + twh: Number(row.twh) + })), + collector: { + lastRun: run.last_run ? time(run.last_run) : null, + lastOk: run.last_ok ? time(run.last_ok) : null, + failedWeek: Number(run.failed_week ?? 0) + }, + coverage: { + firstHour: cover.first_hour ? time(cover.first_hour) : null, + lastHour: cover.last_hour ? time(cover.last_hour) : null, + hours: Number(cover.hours ?? 0) + } + }; +} + +export type EnergyData = Awaited>; diff --git a/src/lib/title.ts b/src/lib/title.ts index fdd0da8..1b79011 100644 --- a/src/lib/title.ts +++ b/src/lib/title.ts @@ -30,6 +30,7 @@ export function getTitle(path: string, environment: SiteEnvironment = null): str '/projects/animaldex/admin': 'Animaldex admin', '/projects/derby': 'Derby Lens', '/projects/cs-prices': 'CS Prices', + '/projects/energy': 'Energy', '/projects/game-tracker': 'Game Tracker', '/projects/language-learner': 'Language Learner', '/projects/money-tracker': 'Money Tracker', diff --git a/src/routes/projects/energy/+page.server.ts b/src/routes/projects/energy/+page.server.ts new file mode 100644 index 0000000..4b01d10 --- /dev/null +++ b/src/routes/projects/energy/+page.server.ts @@ -0,0 +1,4 @@ +import { loadEnergy } from '#lib/server/energy.js'; +import type { PageServerLoad } from './$types'; + +export const load: PageServerLoad = async () => ({ energy: await loadEnergy() }); diff --git a/src/routes/projects/energy/+page.svelte b/src/routes/projects/energy/+page.svelte new file mode 100644 index 0000000..d56c830 --- /dev/null +++ b/src/routes/projects/energy/+page.svelte @@ -0,0 +1,896 @@ + + + + Energy · Stan Runge + + + +
+
+
+

grid archive / netherlands

+

Energy

+

+ Where Dutch electricity comes from hour by hour, how hard each source is running, and how + full the gas storages are. Data from NED.nl, collected hourly. +

+
+
+ {#if now} +
{percent.format(now.renewableShare)}renewable now
+ {#if now.ef !== null} +
{whole.format(now.ef * 1000)}g CO₂ / kWh
+ {/if} + {/if} + {#if latestGas} +
{one.format(latestGas.fill)}%gas storage full
+ {/if} +
+
+ + {#if !now} +
No electricity data has been collected yet.
+ {:else} +
+
+
+

Current mix

+ {formatHour.format(now.t)} · {whole.format(now.total)} MWh +
+ +
+
+
Low-carbon
+
{percent.format(now.lowCarbonShare)}
+
+
+
Fossil
+
{percent.format(now.fossilShare)}
+
+ {#if now.load !== null} +
+
{now.load > now.total ? 'Net import' : 'Net export'}
+
{whole.format(Math.abs(now.total - now.load))} MWh
+
+ {/if} +
+
+
+
+

Utilization

+ share of installed capacity in use +
+ +

+ NED's load factor per source. Nuclear and "other" are reported near 100% almost + constantly, so treat those as estimates. +

+
+
+ +
+
+

Generation and consumption

+ + + {#each Object.entries(rangeLabels) as [value, label] (value)} + {label} + {/each} + + +
+ +

+ Stacked: generation by source. Dashed: consumption (gap above it is export). Red: grid CO₂ + intensity. * = incomplete (still running or hours missing). + {#if range === 'month' && daily.length < 28} + Hourly collection started {formatDate.format(energy.coverage.firstHour ?? now.t)}, so this + fills up over the coming weeks. + {/if} +

+
+ + {#if forecast.length} +
+
+

Next days

+ NED forecast for wind, solar, generation and consumption +
+ +
+ {/if} + {/if} + +
+
+
+

Gas storage by year

+ {#if latestGas} + {one.format(latestGas.twh)} TWh on {formatDate.format(new Date(latestGas.day))} + {/if} +
+ +
+
+
+

Gas vs electricity

+ monthly +
+ +

+ NED has no gas consumption series here, so this compares how much electricity is made from + gas with how full storage is. +

+
+
+ + {#if energy.gasSites.length} +
+
+

Storage sites

+ latest day per site +
+ +
+ {/if} + +
+

+ {#if energy.coverage.firstHour && energy.coverage.lastHour} + Hourly data {formatHour.format(energy.coverage.firstHour)} – {formatHour.format( + energy.coverage.lastHour + )} ({whole.format(energy.coverage.hours)} hours{missingHours + ? `, ${missingHours} missing in the last week` + : ''}{incompleteHours.length + ? `, ${incompleteHours.length} still incomplete and left out` + : ''}). + {/if} + {#if monthly.length} + Monthly from {formatMonth.format(monthly[0].t)}, yearly from {formatYear.format( + yearly[0]?.t ?? monthly[0].t + )}. + {/if} + {#if energy.gas.length} + Gas storage from {formatDate.format(new Date(energy.gas[0].day))}. + {/if} +

+

+ Source: NED.nl. + {#if energy.collector.lastOk} + Last successful collection {relativeTime(energy.collector.lastOk, clock)}{energy.collector + .failedWeek + ? ` · ${energy.collector.failedWeek} failed runs this week` + : ''}. + {/if} +

+
+
+ + diff --git a/src/routes/sitemap.xml/+server.ts b/src/routes/sitemap.xml/+server.ts index 91ebd60..518f776 100644 --- a/src/routes/sitemap.xml/+server.ts +++ b/src/routes/sitemap.xml/+server.ts @@ -18,6 +18,7 @@ const staticPaths = [ '/projects/game-tracker/osu', '/projects/game-tracker/pokemon/locations', '/projects/animaldex', + '/projects/energy', '/projects/derby', '/projects/spotify-beatmap-finder', '/projects/world-opinions', diff --git a/tests/energy.test.ts b/tests/energy.test.ts new file mode 100644 index 0000000..2c1e3d7 --- /dev/null +++ b/tests/energy.test.ts @@ -0,0 +1,17 @@ +import { expect, test } from '@playwright/test'; + +test('energy tracker renders its charts and is listed under trackers', async ({ page }) => { + await page.goto('/projects/energy'); + await expect(page.getByRole('heading', { level: 1, name: 'Energy' })).toBeVisible(); + await expect(page.getByRole('heading', { name: 'Generation and consumption' })).toBeVisible(); + await expect(page.locator('canvas').first()).toBeVisible(); + for (const range of ['day', 'month', '12 months', 'all years']) { + await page.getByRole('tab', { name: range, exact: true }).click(); + await expect(page.getByRole('tab', { name: range, exact: true })).toHaveAttribute( + 'data-state', + 'active' + ); + } + await page.goto('/trackers'); + await expect(page.locator('a[href="/projects/energy"]').first()).toBeVisible(); +});