stats B11: EP-Online energy labels (monthly total file → homes per label for NL, Gouda, Gouda PC4s; aggregates only), stats.ep_label_agg, monthly timer with key from ~/.config/stats/ep-online.env, charts on /stats/energy and /stats/gouda
Some checks failed
Deploy production / deploy (push) Has been cancelled
Some checks failed
Deploy production / deploy (push) Has been cancelled
This commit is contained in:
@@ -1,7 +1,7 @@
|
|||||||
# stats collector
|
# stats collector
|
||||||
|
|
||||||
Host-side collector for the public [/stats](https://stanrunge.dev/stats) section. It pulls open
|
Host-side collector for the public [/stats](https://stanrunge.dev/stats) section. It pulls open
|
||||||
statistics (no API keys) into the `stats` schema with `scripts/stats-fetch.ts`:
|
statistics into the `stats` schema with `scripts/stats-fetch.ts`:
|
||||||
|
|
||||||
| source | timer | schedule (Europe/Amsterdam) | what |
|
| source | timer | schedule (Europe/Amsterdam) | what |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
@@ -16,6 +16,7 @@ statistics (no API keys) into the `stats` schema with `scripts/stats-fetch.ts`:
|
|||||||
| `rijksfinancien` | `stats-rijksfinancien.timer` | 3rd of the month 05:05 | Rijksfinanciën budget tables, last year to next year (`--backfill` from 2015) |
|
| `rijksfinancien` | `stats-rijksfinancien.timer` | 3rd of the month 05:05 | Rijksfinanciën budget tables, last year to next year (`--backfill` from 2015) |
|
||||||
| `ustreasury` | `stats-ustreasury.timer` | daily 23:25 | US Treasury MTS receipts/outlays + Debt to the Penny (last 400 days; `--backfill` all) |
|
| `ustreasury` | `stats-ustreasury.timer` | daily 23:25 | US Treasury MTS receipts/outlays + Debt to the Penny (last 400 days; `--backfill` all) |
|
||||||
| `usaspending` | `stats-usaspending.timer` | Sundays 06:10 | USAspending budgetary resources per agency and fiscal year |
|
| `usaspending` | `stats-usaspending.timer` | Sundays 06:10 | USAspending budgetary resources per agency and fiscal year |
|
||||||
|
| `eponline` | `stats-eponline.timer` | 3rd of the month 05:20 | EP-Online monthly total file (RVO energy labels): homes per label for NL, Gouda and Gouda PC4s; aggregates only. Needs `EP_ONLINE_API_KEY` in `~/.config/stats/ep-online.env` (mode 600), which the runner exports for this source only |
|
||||||
|
|
||||||
Each timer starts `stats-collector@<source>.service`, which runs
|
Each timer starts `stats-collector@<source>.service`, which runs
|
||||||
`~/.local/bin/stats-collector-run.sh <source>`: it updates a detached checkout of `origin/main`
|
`~/.local/bin/stats-collector-run.sh <source>`: it updates a detached checkout of `origin/main`
|
||||||
|
|||||||
@@ -31,4 +31,16 @@ DB_URL=$(sudo -n docker inspect personal-website-app-1 --format '{{range .Config
|
|||||||
| sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
|
| sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
|
||||||
[ -n "$DB_URL" ] || { echo "no DB_URL from app container" >&2; exit 1; }
|
[ -n "$DB_URL" ] || { echo "no DB_URL from app container" >&2; exit 1; }
|
||||||
export DB_URL
|
export DB_URL
|
||||||
|
# Per-source API keys live in ~/.config/stats/<name>.env (mode 600); exported, never echoed.
|
||||||
|
case "$source_name" in
|
||||||
|
eponline) env_file="$HOME/.config/stats/ep-online.env" ;;
|
||||||
|
*) env_file="" ;;
|
||||||
|
esac
|
||||||
|
if [ -n "$env_file" ]; then
|
||||||
|
[ -r "$env_file" ] || { echo "missing $env_file" >&2; exit 1; }
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
. "$env_file"
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
exec nice -n 10 bun scripts/stats-fetch.ts "$@"
|
exec nice -n 10 bun scripts/stats-fetch.ts "$@"
|
||||||
|
|||||||
11
deploy/stats-collector/stats-eponline.timer
Normal file
11
deploy/stats-collector/stats-eponline.timer
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=/stats collector: EP-Online energy labels, monthly total file (published on the 1st)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-03 05:20:00 Europe/Amsterdam
|
||||||
|
RandomizedDelaySec=20min
|
||||||
|
Persistent=true
|
||||||
|
Unit=stats-collector@eponline.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
17
drizzle/20261025212601_stats_eponline/migration.sql
Normal file
17
drizzle/20261025212601_stats_eponline/migration.sql
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
CREATE TABLE "stats"."ep_label_agg" (
|
||||||
|
"snapshot" date,
|
||||||
|
"area" text,
|
||||||
|
"label" text,
|
||||||
|
"homes" integer NOT NULL,
|
||||||
|
CONSTRAINT "ep_label_agg_pkey" PRIMARY KEY("snapshot","area","label")
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA stats TO personal_app;
|
||||||
|
END IF;
|
||||||
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||||
|
GRANT SELECT ON ALL TABLES IN SCHEMA stats TO personal_ro;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
33093
drizzle/20261025212601_stats_eponline/snapshot.json
Normal file
33093
drizzle/20261025212601_stats_eponline/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -177,7 +177,7 @@ try {
|
|||||||
throw new Error(`groceries schema grants are wrong: ${JSON.stringify(groceries)}`);
|
throw new Error(`groceries schema grants are wrong: ${JSON.stringify(groceries)}`);
|
||||||
// Public /stats section: open data, so the app role (and the host-side stats collector, which
|
// Public /stats section: open data, so the app role (and the host-side stats collector, which
|
||||||
// uses it) reads and writes, and personal_ro may read. Bump the count with every new stats table.
|
// uses it) reads and writes, and personal_ro may read. Bump the count with every new stats table.
|
||||||
const STATS_TABLES = 23;
|
const STATS_TABLES = 24;
|
||||||
const [stats] = await client`
|
const [stats] = await client`
|
||||||
select
|
select
|
||||||
count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
|
count(*) filter (where has_table_privilege(c.oid, 'select, insert, update, delete')) as app_rw,
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
// bun scripts/stats-fetch.ts worldbank|imf|owid [--only=…]
|
// bun scripts/stats-fetch.ts worldbank|imf|owid [--only=…]
|
||||||
// bun scripts/stats-fetch.ts energycharts|rijksfinancien|ustreasury [--backfill]
|
// bun scripts/stats-fetch.ts energycharts|rijksfinancien|ustreasury [--backfill]
|
||||||
// bun scripts/stats-fetch.ts usaspending
|
// bun scripts/stats-fetch.ts usaspending
|
||||||
|
// bun scripts/stats-fetch.ts eponline [--force] (needs EP_ONLINE_API_KEY)
|
||||||
// bun scripts/stats-fetch.ts normalise-cbs rebuild CBS chart series from the raw rows
|
// bun scripts/stats-fetch.ts normalise-cbs rebuild CBS chart series from the raw rows
|
||||||
import { ensureMeta, sql } from './stats/common';
|
import { ensureMeta, sql } from './stats/common';
|
||||||
import { cbs, normaliseCbs } from './stats/cbs';
|
import { cbs, normaliseCbs } from './stats/cbs';
|
||||||
@@ -22,6 +23,7 @@ import { energycharts } from './stats/energycharts';
|
|||||||
import { rijksfinancien } from './stats/rijksfinancien';
|
import { rijksfinancien } from './stats/rijksfinancien';
|
||||||
import { ustreasury } from './stats/ustreasury';
|
import { ustreasury } from './stats/ustreasury';
|
||||||
import { usaspending } from './stats/usaspending';
|
import { usaspending } from './stats/usaspending';
|
||||||
|
import { eponline } from './stats/eponline';
|
||||||
|
|
||||||
const [source, ...args] = process.argv.slice(2);
|
const [source, ...args] = process.argv.slice(2);
|
||||||
const flag = (name: string) => args.includes(`--${name}`);
|
const flag = (name: string) => args.includes(`--${name}`);
|
||||||
@@ -67,9 +69,12 @@ try {
|
|||||||
case 'usaspending':
|
case 'usaspending':
|
||||||
await usaspending({ job });
|
await usaspending({ job });
|
||||||
break;
|
break;
|
||||||
|
case 'eponline':
|
||||||
|
await eponline({ force: flag('force') || flag('backfill'), job });
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, normalise-cbs)`
|
`Unknown source "${source}" (cbs, eurostat, fred, bis, worldbank, imf, owid, energycharts, rijksfinancien, ustreasury, usaspending, eponline, normalise-cbs)`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -31,7 +31,8 @@ const hostGapMs: Record<string, number> = {
|
|||||||
'api.energy-charts.info': 2000,
|
'api.energy-charts.info': 2000,
|
||||||
'www.rijksfinancien.nl': 1000,
|
'www.rijksfinancien.nl': 1000,
|
||||||
'api.fiscaldata.treasury.gov': 1000,
|
'api.fiscaldata.treasury.gov': 1000,
|
||||||
'api.usaspending.gov': 500
|
'api.usaspending.gov': 500,
|
||||||
|
'public.ep-online.nl': 1000
|
||||||
};
|
};
|
||||||
const lastHit = new Map<string, number>();
|
const lastHit = new Map<string, number>();
|
||||||
|
|
||||||
@@ -193,6 +194,15 @@ export const SOURCES = [
|
|||||||
licence: 'Public domain (US government work)',
|
licence: 'Public domain (US government work)',
|
||||||
key_required: false,
|
key_required: false,
|
||||||
attribution: 'Source: USAspending.gov (U.S. Department of the Treasury)'
|
attribution: 'Source: USAspending.gov (U.S. Department of the Treasury)'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
source_id: 'eponline',
|
||||||
|
name: 'EP-Online (RVO energy labels)',
|
||||||
|
base_url: 'https://public.ep-online.nl/api/v5',
|
||||||
|
licence: 'Open data (RVO), aggregated here; API key required',
|
||||||
|
key_required: true,
|
||||||
|
attribution:
|
||||||
|
'Source: EP-Online, Netherlands Enterprise Agency (RVO), aggregated by stanrunge.dev'
|
||||||
}
|
}
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
|
|||||||
238
scripts/stats/eponline.ts
Normal file
238
scripts/stats/eponline.ts
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
// B11 EP-Online (RVO energy labels): the monthly total file ("mutatiebestand", CSV in a zip,
|
||||||
|
// ~1.6 GB unpacked, every currently registered label) aggregated to the number of homes per
|
||||||
|
// energy label for the Netherlands, Gouda (postcodes 2800–2809) and each Gouda PC4. Only these
|
||||||
|
// aggregates are stored; no address-level data leaves the run. Needs EP_ONLINE_API_KEY (loaded
|
||||||
|
// by the runner from ~/.config/stats/ep-online.env; never logged). One snapshot per publication
|
||||||
|
// date, kept over time, so the shares become a monthly series.
|
||||||
|
import { mkdtemp, rm } from 'node:fs/promises';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import {
|
||||||
|
http,
|
||||||
|
insertChunks,
|
||||||
|
log,
|
||||||
|
run,
|
||||||
|
sql,
|
||||||
|
writeSeries,
|
||||||
|
type ObsRow,
|
||||||
|
type SeriesRow
|
||||||
|
} from './common';
|
||||||
|
|
||||||
|
const API = 'https://public.ep-online.nl/api/v5';
|
||||||
|
/** Best to worst (NTA 8800 scale since 2021). */
|
||||||
|
export const LABELS = ['A++++', 'A+++', 'A++', 'A+', 'A', 'B', 'C', 'D', 'E', 'F', 'G'] as const;
|
||||||
|
const GOUDA_PC4 = Array.from({ length: 10 }, (_, i) => String(2800 + i));
|
||||||
|
const AREA_NAME: Record<string, string> = {
|
||||||
|
NL: 'Netherlands',
|
||||||
|
GOUDA: 'Gouda (postcodes 2800–2809)',
|
||||||
|
...Object.fromEntries(GOUDA_PC4.map((p) => [p, `Gouda ${p}`]))
|
||||||
|
};
|
||||||
|
const AREA_REGION: Record<string, string> = { NL: 'NLD', GOUDA: 'GM0513' };
|
||||||
|
|
||||||
|
/** CSV split on ';' with optional "…" quoting ("" escapes). */
|
||||||
|
function splitLine(line: string): string[] {
|
||||||
|
if (!line.includes('"')) return line.split(';');
|
||||||
|
const out: string[] = [];
|
||||||
|
let cur = '';
|
||||||
|
let q = false;
|
||||||
|
for (let i = 0; i < line.length; i++) {
|
||||||
|
const c = line[i];
|
||||||
|
if (q && c === '"' && line[i + 1] === '"') {
|
||||||
|
cur += '"';
|
||||||
|
i++;
|
||||||
|
} else if (c === '"') {
|
||||||
|
q = !q;
|
||||||
|
} else if (!q && c === ';') {
|
||||||
|
out.push(cur);
|
||||||
|
cur = '';
|
||||||
|
} else {
|
||||||
|
cur += c;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.push(cur);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function* lines(stream: ReadableStream<Uint8Array>) {
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
let buf = '';
|
||||||
|
for await (const chunk of stream) {
|
||||||
|
buf += decoder.decode(chunk, { stream: true });
|
||||||
|
let nl: number;
|
||||||
|
let start = 0;
|
||||||
|
while ((nl = buf.indexOf('\n', start)) >= 0) {
|
||||||
|
yield buf.slice(start, nl).replace(/\r$/, '');
|
||||||
|
start = nl + 1;
|
||||||
|
}
|
||||||
|
buf = buf.slice(start);
|
||||||
|
}
|
||||||
|
if (buf) yield buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Streams the CSV out of the zip and counts homes per label and area (one label per home). */
|
||||||
|
export async function aggregate(zipPath: string) {
|
||||||
|
const proc = Bun.spawn(['unzip', '-p', zipPath], { stdout: 'pipe', stderr: 'ignore' });
|
||||||
|
const it = lines(proc.stdout as ReadableStream<Uint8Array>);
|
||||||
|
let published: string | null = null;
|
||||||
|
let ix: Record<string, number> | null = null;
|
||||||
|
// Newest registration per home: BAG verblijfsobject id, or the address when there is none.
|
||||||
|
// Value packs label index (4 bits) + Gouda PC4 slot (4 bits, 15 = elsewhere) + yyyymmdd.
|
||||||
|
const byVbo = new Map<number, number>();
|
||||||
|
const byAddr = new Map<string, number>();
|
||||||
|
let rows = 0;
|
||||||
|
let skipped = 0;
|
||||||
|
for await (const line of it) {
|
||||||
|
if (!ix) {
|
||||||
|
const m = line.match(/^PublicatieDatum;(\d\d)-(\d\d)-(\d{4})/);
|
||||||
|
if (m) published = `${m[3]}-${m[2]}-${m[1]}`;
|
||||||
|
if (line.startsWith('Registratiedatum;'))
|
||||||
|
ix = Object.fromEntries(line.split(';').map((h, i) => [h, i]));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!line) continue;
|
||||||
|
rows++;
|
||||||
|
const f = splitLine(line);
|
||||||
|
if (f[ix.Gebouwklasse] !== 'W') continue;
|
||||||
|
// Labels issued on a building-permit design are not a home's label yet.
|
||||||
|
if (f[ix.Status]?.startsWith('Vergunningsaanvraag')) {
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const label = LABELS.indexOf(f[ix.Energieklasse] as (typeof LABELS)[number]);
|
||||||
|
if (label < 0) {
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const pc = f[ix.Postcode] ?? '';
|
||||||
|
const slot = pc >= '2800' && pc < '2810' ? Number(pc[3]) : 15;
|
||||||
|
const date = Number(f[ix.Registratiedatum]) || 0;
|
||||||
|
const packed = date * 256 + slot * 16 + label;
|
||||||
|
const vbo = f[ix.BAGVerblijfsobjectID];
|
||||||
|
if (vbo && /^\d{16}$/.test(vbo)) {
|
||||||
|
const key = Number(vbo);
|
||||||
|
const prev = byVbo.get(key);
|
||||||
|
if (prev === undefined || prev < packed) byVbo.set(key, packed);
|
||||||
|
} else {
|
||||||
|
const key = [pc, f[ix.Huisnummer], f[ix.Huisletter], f[ix.Huisnummertoevoeging]].join('|');
|
||||||
|
const prev = byAddr.get(key);
|
||||||
|
if (prev === undefined || prev < packed) byAddr.set(key, packed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const code = await proc.exited;
|
||||||
|
if (code !== 0) throw new Error(`unzip exited ${code}`);
|
||||||
|
if (!ix || !published) throw new Error('unexpected EP-Online file header');
|
||||||
|
const counts = new Map<string, number[]>();
|
||||||
|
const bump = (area: string, label: number) => {
|
||||||
|
let c = counts.get(area);
|
||||||
|
if (!c) counts.set(area, (c = LABELS.map(() => 0)));
|
||||||
|
c[label]++;
|
||||||
|
};
|
||||||
|
for (const v of [...byVbo.values(), ...byAddr.values()]) {
|
||||||
|
const label = v % 16;
|
||||||
|
const slot = Math.floor(v / 16) % 16;
|
||||||
|
bump('NL', label);
|
||||||
|
if (slot !== 15) {
|
||||||
|
bump('GOUDA', label);
|
||||||
|
bump(String(2800 + slot), label);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { published, counts, rows, skipped, homes: byVbo.size + byAddr.size };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function eponline(opts: { force: boolean; job: string }) {
|
||||||
|
await run('eponline', opts.job, async (note) => {
|
||||||
|
const key = process.env.EP_ONLINE_API_KEY;
|
||||||
|
if (!key) throw new Error('EP_ONLINE_API_KEY is not set (see ~/.config/stats/ep-online.env)');
|
||||||
|
const info = (await (
|
||||||
|
await http(`${API}/Mutatiebestand/DownloadInfo?fileType=csv`, {
|
||||||
|
headers: { authorization: key, accept: 'application/json' }
|
||||||
|
})
|
||||||
|
).json()) as { bestandsnaam: string; downloadUrl: string };
|
||||||
|
const m = info.bestandsnaam?.match(/v(\d{4})(\d\d)(\d\d)/);
|
||||||
|
if (!m || !info.downloadUrl) throw new Error('DownloadInfo returned no file');
|
||||||
|
const snapshot = `${m[1]}-${m[2]}-${m[3]}`;
|
||||||
|
const [{ have }] = await sql`
|
||||||
|
select count(*)::int as have from stats.ep_label_agg where snapshot = ${snapshot}`;
|
||||||
|
if (have && !opts.force) {
|
||||||
|
log(`eponline: ${info.bestandsnaam} already loaded`);
|
||||||
|
await rebuildSeries();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), 'eponline-'));
|
||||||
|
try {
|
||||||
|
const zip = join(dir, 'labels.zip');
|
||||||
|
// The download URL carries a token: never put it in logs or fetch_run errors.
|
||||||
|
const res = await fetch(info.downloadUrl, {
|
||||||
|
headers: { 'user-agent': 'stanrunge.dev-stats/1.0' },
|
||||||
|
signal: AbortSignal.timeout(30 * 60_000)
|
||||||
|
}).catch((e) => {
|
||||||
|
throw new Error(`download failed (${e instanceof Error ? e.name : 'error'})`);
|
||||||
|
});
|
||||||
|
if (!res.ok || !res.body) throw new Error(`download failed: HTTP ${res.status}`);
|
||||||
|
await Bun.write(zip, res);
|
||||||
|
log(
|
||||||
|
`eponline: downloaded ${info.bestandsnaam} (${(Bun.file(zip).size / 1e6).toFixed(0)} MB)`
|
||||||
|
);
|
||||||
|
const agg = await aggregate(zip);
|
||||||
|
if (agg.published !== snapshot)
|
||||||
|
note(`file says published ${agg.published}, name says ${snapshot}`);
|
||||||
|
const nl = agg.counts.get('NL');
|
||||||
|
if (!nl || nl.reduce((a, b) => a + b, 0) < 1_000_000)
|
||||||
|
throw new Error(`implausible total: ${agg.homes} homes`);
|
||||||
|
const out: Record<string, unknown>[] = [];
|
||||||
|
for (const [area, c] of agg.counts)
|
||||||
|
c.forEach((homes, i) => out.push({ snapshot, area, label: LABELS[i], homes }));
|
||||||
|
await sql.begin(async (tx) => {
|
||||||
|
await tx`delete from stats.ep_label_agg where snapshot = ${snapshot}`;
|
||||||
|
await insertChunks(tx, 'ep_label_agg', out);
|
||||||
|
});
|
||||||
|
log(
|
||||||
|
`eponline: ${agg.rows} rows, ${agg.homes} homes with a label (${agg.skipped} permit/blank skipped)`
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
return rebuildSeries();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** stats.obs: share of homes per label (%), per area, one point per monthly snapshot. */
|
||||||
|
async function rebuildSeries(): Promise<number> {
|
||||||
|
const rows = await sql<{ snapshot: string; area: string; label: string; homes: number }[]>`
|
||||||
|
select snapshot::text, area, label, homes from stats.ep_label_agg`;
|
||||||
|
const totals = new Map<string, number>();
|
||||||
|
for (const r of rows)
|
||||||
|
totals.set(`${r.snapshot}|${r.area}`, (totals.get(`${r.snapshot}|${r.area}`) ?? 0) + r.homes);
|
||||||
|
const series: SeriesRow[] = [];
|
||||||
|
const obs: ObsRow[] = [];
|
||||||
|
const areas = [...new Set(rows.map((r) => r.area))];
|
||||||
|
for (const area of areas)
|
||||||
|
for (const label of [...LABELS, 'TOTAL']) {
|
||||||
|
const id = `eponline:${label === 'TOTAL' ? 'homes' : `share:${label}`}:${area}`;
|
||||||
|
series.push({
|
||||||
|
series_id: id,
|
||||||
|
source_id: 'eponline',
|
||||||
|
source_code: `${area}:${label}`,
|
||||||
|
title:
|
||||||
|
label === 'TOTAL'
|
||||||
|
? `Homes with a registered energy label: ${AREA_NAME[area] ?? area}`
|
||||||
|
: `Share of labelled homes with label ${label}: ${AREA_NAME[area] ?? area}`,
|
||||||
|
unit: label === 'TOTAL' ? 'homes' : '%',
|
||||||
|
freq: 'M',
|
||||||
|
region_code: AREA_REGION[area] ?? null,
|
||||||
|
theme: 'energy',
|
||||||
|
meta: { area, label }
|
||||||
|
});
|
||||||
|
for (const r of rows)
|
||||||
|
if (r.area === area && (label === 'TOTAL' ? r.label === LABELS[0] : r.label === label)) {
|
||||||
|
const total = totals.get(`${r.snapshot}|${area}`) ?? 0;
|
||||||
|
obs.push({
|
||||||
|
series_id: id,
|
||||||
|
period: `${r.snapshot.slice(0, 7)}-01`,
|
||||||
|
value: label === 'TOTAL' ? total : total ? (100 * r.homes) / total : null,
|
||||||
|
status: null
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return writeSeries(series, obs);
|
||||||
|
}
|
||||||
@@ -354,3 +354,18 @@ export const statsUsasAgencyFy = statsSchema.table(
|
|||||||
},
|
},
|
||||||
(t) => [primaryKey({ columns: [t.toptierCode, t.fiscalYear] })]
|
(t) => [primaryKey({ columns: [t.toptierCode, t.fiscalYear] })]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// ── B11 EP-Online (registered energy labels, homes; aggregates only, no addresses) ─────────
|
||||||
|
|
||||||
|
export const statsEpLabelAgg = statsSchema.table(
|
||||||
|
'ep_label_agg',
|
||||||
|
{
|
||||||
|
/** Publication date of the monthly total file. */
|
||||||
|
snapshot: date('snapshot', { mode: 'string' }).notNull(),
|
||||||
|
/** NL, GOUDA (postcodes 2800–2809) or a Gouda PC4. */
|
||||||
|
area: text('area').notNull(),
|
||||||
|
label: text('label').notNull(),
|
||||||
|
homes: integer('homes').notNull()
|
||||||
|
},
|
||||||
|
(t) => [primaryKey({ columns: [t.snapshot, t.area, t.label] })]
|
||||||
|
);
|
||||||
|
|||||||
88
src/lib/stats/energylabels.ts
Normal file
88
src/lib/stats/energylabels.ts
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
// EP-Online energy labels on /stats/energy and /stats/gouda (collector: scripts/stats/eponline.ts).
|
||||||
|
import type { ChartTheme, StatPoint } from './chart';
|
||||||
|
import type { EChartsCoreOption } from 'echarts/core';
|
||||||
|
|
||||||
|
/** Best to worst (NTA 8800 scale). */
|
||||||
|
export const EP_LABELS = ['A++++', 'A+++', 'A++', 'A+', 'A', 'B', 'C', 'D', 'E', 'F', 'G'];
|
||||||
|
/** Dark green → red, like the label sticker. */
|
||||||
|
export const EP_COLORS = [
|
||||||
|
'#00632f',
|
||||||
|
'#00733a',
|
||||||
|
'#008a44',
|
||||||
|
'#00a24f',
|
||||||
|
'#22b14c',
|
||||||
|
'#8cc63e',
|
||||||
|
'#fff200',
|
||||||
|
'#fdb913',
|
||||||
|
'#f37021',
|
||||||
|
'#ed1c24',
|
||||||
|
'#b5121b'
|
||||||
|
];
|
||||||
|
export const GOUDA_PC4 = Array.from({ length: 10 }, (_, i) => String(2800 + i));
|
||||||
|
|
||||||
|
export const epShareId = (label: string, area: string) => `eponline:share:${label}:${area}`;
|
||||||
|
export const epHomesId = (area: string) => `eponline:homes:${area}`;
|
||||||
|
export const epSeriesIds = (areas: string[]) =>
|
||||||
|
areas.flatMap((a) => [epHomesId(a), ...EP_LABELS.map((l) => epShareId(l, a))]);
|
||||||
|
|
||||||
|
/** Share of homes with label A or better per snapshot. */
|
||||||
|
export function aOrBetter(get: (id: string) => StatPoint[], area: string): StatPoint[] {
|
||||||
|
const sum = new Map<string, number>();
|
||||||
|
for (const l of EP_LABELS.slice(0, 5))
|
||||||
|
for (const [p, v] of get(epShareId(l, area))) sum.set(p, (sum.get(p) ?? 0) + v);
|
||||||
|
return [...sum].sort((a, b) => a[0].localeCompare(b[0]));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Horizontal 100% stacked bars: one row per area, one segment per label (latest snapshot). */
|
||||||
|
export function labelStackOption(
|
||||||
|
theme: ChartTheme,
|
||||||
|
rows: { name: string; shares: number[] }[]
|
||||||
|
): EChartsCoreOption {
|
||||||
|
const pct = new Intl.NumberFormat('en-GB', { maximumFractionDigits: 1 });
|
||||||
|
return {
|
||||||
|
animation: false,
|
||||||
|
grid: { left: 8, right: 16, top: 36, bottom: 8, containLabel: true },
|
||||||
|
legend: {
|
||||||
|
top: 0,
|
||||||
|
itemWidth: 12,
|
||||||
|
itemHeight: 10,
|
||||||
|
textStyle: { color: theme.muted, fontSize: 11 }
|
||||||
|
},
|
||||||
|
tooltip: {
|
||||||
|
trigger: 'axis',
|
||||||
|
axisPointer: { type: 'shadow' },
|
||||||
|
backgroundColor: theme.surface,
|
||||||
|
borderColor: theme.border,
|
||||||
|
textStyle: { color: theme.text, fontSize: 12 },
|
||||||
|
valueFormatter: (v: number) => `${pct.format(v)}%`
|
||||||
|
},
|
||||||
|
xAxis: {
|
||||||
|
type: 'value',
|
||||||
|
max: 100,
|
||||||
|
axisLabel: { color: theme.muted, fontSize: 11, formatter: '{value}%' },
|
||||||
|
splitLine: { lineStyle: { color: theme.border, opacity: 0.6 } }
|
||||||
|
},
|
||||||
|
yAxis: {
|
||||||
|
type: 'category',
|
||||||
|
inverse: true,
|
||||||
|
data: rows.map((r) => r.name),
|
||||||
|
axisLabel: { color: theme.text, fontSize: 11 },
|
||||||
|
axisLine: { show: false },
|
||||||
|
axisTick: { show: false }
|
||||||
|
},
|
||||||
|
series: EP_LABELS.map((label, i) => ({
|
||||||
|
name: label,
|
||||||
|
type: 'bar',
|
||||||
|
stack: 'labels',
|
||||||
|
barMaxWidth: 22,
|
||||||
|
itemStyle: { color: EP_COLORS[i] },
|
||||||
|
data: rows.map((r) => r.shares[i] ?? 0),
|
||||||
|
label: {
|
||||||
|
show: true,
|
||||||
|
color: i >= 6 && i <= 8 ? '#111' : '#fff',
|
||||||
|
fontSize: 10,
|
||||||
|
formatter: (p: { value: number }) => (p.value >= 6 ? `${pct.format(p.value)}` : '')
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { cached, loadSeries } from '#lib/server/stats.js';
|
import { cached, loadSeries } from '#lib/server/stats.js';
|
||||||
|
import { epSeriesIds } from '#lib/stats/energylabels.js';
|
||||||
import type { PageServerLoad } from './$types';
|
import type { PageServerLoad } from './$types';
|
||||||
|
|
||||||
const EU = ['NL', 'DE', 'FR', 'BE', 'IT', 'ES', 'PL', 'SE', 'DK', 'AT', 'EU27_2020'];
|
const EU = ['NL', 'DE', 'FR', 'BE', 'IT', 'ES', 'PL', 'SE', 'DK', 'AT', 'EU27_2020'];
|
||||||
@@ -54,7 +55,8 @@ export const load: PageServerLoad = async () => ({
|
|||||||
'cbs:cars:petrol',
|
'cbs:cars:petrol',
|
||||||
'cbs:cars:diesel',
|
'cbs:cars:diesel',
|
||||||
'eurostat:road_eqr_carpda:NR.ELC.NL',
|
'eurostat:road_eqr_carpda:NR.ELC.NL',
|
||||||
'eurostat:road_eqr_carpda:NR.TOTAL.NL'
|
'eurostat:road_eqr_carpda:NR.TOTAL.NL',
|
||||||
|
...epSeriesIds(['NL', 'GOUDA'])
|
||||||
])
|
])
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -5,6 +5,13 @@
|
|||||||
import { barRankOption, latest, lineOption, type StatPoint } from '#lib/stats/chart.js';
|
import { barRankOption, latest, lineOption, type StatPoint } from '#lib/stats/chart.js';
|
||||||
import { compact, num, period } from '#lib/stats/format.js';
|
import { compact, num, period } from '#lib/stats/format.js';
|
||||||
import { COUNTRY } from '#lib/stats/regions.js';
|
import { COUNTRY } from '#lib/stats/regions.js';
|
||||||
|
import {
|
||||||
|
aOrBetter,
|
||||||
|
epHomesId,
|
||||||
|
epShareId,
|
||||||
|
EP_LABELS,
|
||||||
|
labelStackOption
|
||||||
|
} from '#lib/stats/energylabels.js';
|
||||||
import Zap from 'lucide-svelte/icons/zap';
|
import Zap from 'lucide-svelte/icons/zap';
|
||||||
import ArrowRight from 'lucide-svelte/icons/arrow-right';
|
import ArrowRight from 'lucide-svelte/icons/arrow-right';
|
||||||
|
|
||||||
@@ -13,6 +20,11 @@
|
|||||||
const a = $derived(data.attributions);
|
const a = $derived(data.attributions);
|
||||||
const pts = (id: string): StatPoint[] => s[id]?.points ?? [];
|
const pts = (id: string): StatPoint[] => s[id]?.points ?? [];
|
||||||
const last = (id: string) => latest(s[id]?.points);
|
const last = (id: string) => latest(s[id]?.points);
|
||||||
|
// EP-Online: share of homes per energy label, latest monthly snapshot.
|
||||||
|
const epShares = (area: string) => EP_LABELS.map((l) => last(epShareId(l, area))?.[1] ?? 0);
|
||||||
|
const epHomes = $derived(last(epHomesId('NL')));
|
||||||
|
const epAorBetter = $derived(aOrBetter(pts, 'NL'));
|
||||||
|
const epGoudaAorBetter = $derived(aOrBetter(pts, 'GOUDA'));
|
||||||
const EU = ['NL', 'DE', 'FR', 'BE', 'IT', 'ES', 'PL', 'SE', 'DK', 'AT', 'EU27_2020'];
|
const EU = ['NL', 'DE', 'FR', 'BE', 'IT', 'ES', 'PL', 'SE', 'DK', 'AT', 'EU27_2020'];
|
||||||
|
|
||||||
/** a ÷ b × k on shared dates. */
|
/** a ÷ b × k on shared dates. */
|
||||||
@@ -592,6 +604,43 @@
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{#if epHomes}
|
||||||
|
<h2 class="stats-section-title">Energy labels of homes</h2>
|
||||||
|
<div class="stats-grid">
|
||||||
|
<StatChart
|
||||||
|
title="Homes by energy label"
|
||||||
|
subtitle={`% of homes with a registered label · ${num(epHomes[1], 0)} homes in NL · ${period(epHomes[0], 'M')}`}
|
||||||
|
sources={['eponline']}
|
||||||
|
attributions={a}
|
||||||
|
wide
|
||||||
|
height="220px"
|
||||||
|
note="Latest registered label per home (BAG dwelling) in EP-Online's monthly total file; labels on building-permit designs are left out. Gouda = postcodes 2800–2809."
|
||||||
|
option={(t) =>
|
||||||
|
labelStackOption(t, [
|
||||||
|
{ name: 'Netherlands', shares: epShares('NL') },
|
||||||
|
{ name: 'Gouda', shares: epShares('GOUDA') }
|
||||||
|
])}
|
||||||
|
/>
|
||||||
|
{#if epAorBetter.length > 1}
|
||||||
|
<StatChart
|
||||||
|
title="Homes labelled A or better"
|
||||||
|
subtitle="% of homes with a registered label, monthly"
|
||||||
|
sources={['eponline']}
|
||||||
|
attributions={a}
|
||||||
|
option={(t) =>
|
||||||
|
lineOption(
|
||||||
|
t,
|
||||||
|
[
|
||||||
|
{ name: 'Netherlands', points: epAorBetter, color: '#0f172a', dashed: true },
|
||||||
|
{ name: 'Gouda', points: epGoudaAorBetter, color: '#f97316' }
|
||||||
|
],
|
||||||
|
{ unit: '%', decimals: 1 }
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
.live-link {
|
.live-link {
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { cached, loadCbsCross, loadSeries } from '#lib/server/stats.js';
|
import { cached, loadCbsCross, loadSeries } from '#lib/server/stats.js';
|
||||||
|
import { epSeriesIds, GOUDA_PC4 } from '#lib/stats/energylabels.js';
|
||||||
import type { PageServerLoad } from './$types';
|
import type { PageServerLoad } from './$types';
|
||||||
|
|
||||||
// Gouda (GM0513) and its neighbours; NL01 is the national total.
|
// Gouda (GM0513) and its neighbours; NL01 is the national total.
|
||||||
@@ -33,7 +34,8 @@ export const load: PageServerLoad = async () => {
|
|||||||
(k) => `cbs:pop:${k}:GM0513`
|
(k) => `cbs:pop:${k}:GM0513`
|
||||||
),
|
),
|
||||||
'cbs:house:hpi:cr28',
|
'cbs:house:hpi:cr28',
|
||||||
'cbs:house:hpi:nl'
|
'cbs:house:hpi:nl',
|
||||||
|
...epSeriesIds(['NL', 'GOUDA', ...GOUDA_PC4])
|
||||||
])
|
])
|
||||||
),
|
),
|
||||||
cached('stats:gouda:kwb', () =>
|
cached('stats:gouda:kwb', () =>
|
||||||
|
|||||||
@@ -3,12 +3,24 @@
|
|||||||
import Kpi from '#lib/stats/Kpi.svelte';
|
import Kpi from '#lib/stats/Kpi.svelte';
|
||||||
import { barRankOption, change, latest, lineOption, type StatPoint } from '#lib/stats/chart.js';
|
import { barRankOption, change, latest, lineOption, type StatPoint } from '#lib/stats/chart.js';
|
||||||
import { compact, num, period } from '#lib/stats/format.js';
|
import { compact, num, period } from '#lib/stats/format.js';
|
||||||
|
import { epShareId, EP_LABELS, GOUDA_PC4, labelStackOption } from '#lib/stats/energylabels.js';
|
||||||
|
|
||||||
let { data } = $props();
|
let { data } = $props();
|
||||||
const s = $derived(data.series);
|
const s = $derived(data.series);
|
||||||
const a = $derived(data.attributions);
|
const a = $derived(data.attributions);
|
||||||
const pts = (id: string): StatPoint[] => s[id]?.points ?? [];
|
const pts = (id: string): StatPoint[] => s[id]?.points ?? [];
|
||||||
const last = (id: string) => latest(s[id]?.points);
|
const last = (id: string) => latest(s[id]?.points);
|
||||||
|
// EP-Online energy labels per Gouda postcode (PC4), latest monthly snapshot.
|
||||||
|
const epShares = (area: string) => EP_LABELS.map((l) => last(epShareId(l, area))?.[1] ?? 0);
|
||||||
|
const epPeriod = $derived(last(epShareId('A', 'GOUDA'))?.[0]);
|
||||||
|
const epRows = $derived([
|
||||||
|
{ name: 'Netherlands', shares: epShares('NL') },
|
||||||
|
{ name: 'Gouda (2800–2809)', shares: epShares('GOUDA') },
|
||||||
|
...GOUDA_PC4.filter((p) => s[epShareId('A', p)]?.points.length).map((p) => ({
|
||||||
|
name: p,
|
||||||
|
shares: epShares(p)
|
||||||
|
}))
|
||||||
|
]);
|
||||||
|
|
||||||
const NAMES: Record<string, string> = {
|
const NAMES: Record<string, string> = {
|
||||||
NL01: 'Netherlands',
|
NL01: 'Netherlands',
|
||||||
@@ -336,6 +348,18 @@
|
|||||||
{ unit: '€', decimals: 0 }
|
{ unit: '€', decimals: 0 }
|
||||||
)}
|
)}
|
||||||
/>
|
/>
|
||||||
|
{#if epPeriod}
|
||||||
|
<StatChart
|
||||||
|
title="Energy labels by postcode"
|
||||||
|
subtitle={`% of homes with a registered label · ${period(epPeriod, 'M')}`}
|
||||||
|
sources={['eponline']}
|
||||||
|
attributions={a}
|
||||||
|
wide
|
||||||
|
height="440px"
|
||||||
|
note="Latest registered label per home in EP-Online's monthly total file (permit-stage labels left out). 2809 is mostly new-build (Westergouwe)."
|
||||||
|
option={(t) => labelStackOption(t, epRows)}
|
||||||
|
/>
|
||||||
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<h2 class="stats-section-title">Neighbourhoods ({data.kwbYear})</h2>
|
<h2 class="stats-section-title">Neighbourhoods ({data.kwbYear})</h2>
|
||||||
|
|||||||
Reference in New Issue
Block a user