houses: Leefbaarometer liveability per listing (PDOK geocode + LBM 2024 grid/buurt), min-score filter and sort; store every regional listing (no price caps at scrape time)

src/lib/house-tracker/liveability.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from 'vitest';
+import { gridCellId, lbmClassLabel, lbmDimensionLabel, lbmQuery } from './liveability';
+
+describe('leefbaarometer helpers', () => {
+	it('names the 100 m cell by its lower-left corner in hectometres', () => {
+		expect(gridCellId([106050.4, 446899.9])).toBe('E1060N4468');
+		expect(gridCellId([105866.161, 446838.902])).toBe('E1058N4468');
+	});
+	it('labels classes 1-9', () => {
+		expect(lbmClassLabel(1)).toBe('zeer onvoldoende');
+		expect(lbmClassLabel(9)).toBe('uitstekend');
+		expect(lbmClassLabel(0)).toBeNull();
+		expect(lbmDimensionLabel(5)).toBe('geen afwijking');
+	});
+	it('builds the geocode query from street, postcode and city', () => {
+		expect(lbmQuery({ street: 'Raam 12', postalCode: '2801 vk', city: 'Gouda' })).toBe(
+			'Raam 12 | 2801VK | Gouda'
+		);
+		expect(lbmQuery({ street: null, postalCode: null, city: 'Gouda' })).toBeNull();
+		expect(
+			lbmQuery({
+				street: null,
+				postalCode: null,
+				city: 'Gouda',
+				title: 'Turfmarkt 5',
+				source: 'manual'
+			})
+		).toBe('Turfmarkt 5 | Gouda');
+	});
+});

src/lib/house-tracker/liveability.ts

@@ -0,0 +1,374 @@
+// Leefbaarometer 3.0 (Ministerie van BZK) liveability scores for listings.
+// Address -> RD point via PDOK Locatieserver, then the 100 m grid cell and the buurt that
+// contain it from the official Leefbaarometer WFS (geo.leefbaarometer.nl/lbm3). No API keys,
+// no static data in the repo: each listing needs two or three tiny requests, once, and the
+// result is stored on the row. Plain fetch + postgres.js, so the nightly scraper, the CLI
+// and the page can all use it. See docs/house-tracker.md.
+import type postgres from 'postgres';
+
+/** Latest Leefbaarometer measurement (published 1 July 2026). Bump when 2026 comes out. */
+export const LBM_YEAR = 2024;
+const LAYER_SUFFIX = String(LBM_YEAR).slice(2);
+const WFS = 'https://geo.leefbaarometer.nl/lbm3/ows';
+const PDOK = 'https://api.pdok.nl/bzk/locatieserver/search/v3_1/free';
+const USER_AGENT = 'stanrunge.dev-house-tracker/1.0 (personal use; stan@stanrunge.dev)';
+
+/** kscore 1..9 (total score class). */
+export const LBM_CLASS_LABELS = [
+	'zeer onvoldoende',
+	'ruim onvoldoende',
+	'onvoldoende',
+	'zwak',
+	'voldoende',
+	'ruim voldoende',
+	'goed',
+	'zeer goed',
+	'uitstekend'
+] as const;
+/** Sub-dimension classes 1..9: contribution relative to the Dutch average. */
+export const LBM_DIMENSION_LABELS = [
+	'zeer grote negatieve bijdrage',
+	'grote negatieve bijdrage',
+	'negatieve bijdrage',
+	'kleine negatieve bijdrage',
+	'geen afwijking',
+	'kleine positieve bijdrage',
+	'positieve bijdrage',
+	'grote positieve bijdrage',
+	'zeer grote positieve bijdrage'
+] as const;
+export const LBM_DIMENSIONS = {
+	onv: 'overlast & onveiligheid',
+	fys: 'fysieke omgeving',
+	soc: 'sociale samenhang',
+	vrz: 'voorzieningen',
+	won: 'woningvoorraad'
+} as const;
+export type LbmDimension = keyof typeof LBM_DIMENSIONS;
+
+export const lbmClassLabel = (k: number | null | undefined) =>
+	k != null && k >= 1 && k <= 9 ? LBM_CLASS_LABELS[k - 1] : null;
+export const lbmDimensionLabel = (k: number | null | undefined) =>
+	k != null && k >= 1 && k <= 9 ? LBM_DIMENSION_LABELS[k - 1] : null;
+
+/** address: geocoded to the house; postcode: PC6 centroid; street: street centroid. */
+export type LbmPrecision = 'address' | 'postcode' | 'street';
+
+export type LbmArea = {
+	id: string;
+	name: string | null;
+	gemeente: string | null;
+	lbm: number;
+	kscore: number;
+	classes: Record<LbmDimension, number | null>;
+	deviation: Record<LbmDimension | 'afw', number | null>;
+};
+
+export type Liveability = {
+	year: number;
+	/** Which area the main score comes from. */
+	level: 'grid' | 'buurt';
+	precision: LbmPrecision;
+	rd: [number, number];
+	lat: number | null;
+	lng: number | null;
+	matched: string;
+	buurtCode: string | null;
+	buurtName: string | null;
+	grid: LbmArea | null;
+	buurt: LbmArea | null;
+};
+
+type Doc = Record<string, unknown>;
+
+async function getJson(url: string, timeoutMs = 15_000): Promise<Doc> {
+	const response = await fetch(url, {
+		headers: { 'user-agent': USER_AGENT, accept: 'application/json' },
+		signal: AbortSignal.timeout(timeoutMs)
+	});
+	if (!response.ok) throw new Error(`${new URL(url).host}: HTTP ${response.status}`);
+	return (await response.json()) as Doc;
+}
+
+const norm = (s: unknown) =>
+	String(s ?? '')
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toLowerCase()
+		.replace(/[^a-z0-9]+/g, ' ')
+		.trim();
+
+const point = (wkt: unknown): [number, number] | null => {
+	const m = /POINT\(([-\d.]+) ([-\d.]+)\)/.exec(String(wkt ?? ''));
+	return m ? [Number(m[1]), Number(m[2])] : null;
+};
+
+const POSTCODE = /^\s*(\d{4})\s?([a-z]{2})\s*$/i;
+const HAS_NUMBER = /\s\d+\s*[a-z]?(?:[\s-]*\d*[a-z]*)?\s*$/i;
+
+export type AddressParts = {
+	street: string | null;
+	postalCode: string | null;
+	city: string | null;
+	title?: string | null;
+	source?: string | null;
+};
+
+/** The text we geocode; stored so a changed address gets re-scored. */
+export function lbmQuery(parts: AddressParts): string | null {
+	const street =
+		(parts.street ?? '').trim() || (parts.source === 'manual' ? (parts.title ?? '').trim() : '');
+	const pc = POSTCODE.exec(parts.postalCode ?? '');
+	const city = (parts.city ?? '').trim();
+	if (!street && !pc) return null;
+	return [street, pc ? `${pc[1]}${pc[2].toUpperCase()}` : '', city].filter(Boolean).join(' | ');
+}
+
+async function pdok(q: string, type: string): Promise<Doc[]> {
+	const url = new URL(PDOK);
+	url.searchParams.set('q', q);
+	url.searchParams.set('fq', `type:${type}`);
+	url.searchParams.set('rows', '5');
+	url.searchParams.set(
+		'fl',
+		'type,weergavenaam,woonplaatsnaam,gemeentenaam,straatnaam,postcode,huisnummer,huisletter,centroide_rd,centroide_ll,buurtcode,buurtnaam,score'
+	);
+	const data = await getJson(url.toString());
+	return ((data.response as Doc)?.docs as Doc[]) ?? [];
+}
+
+const placeMatches = (doc: Doc, city: string) =>
+	!city ||
+	norm(doc.woonplaatsnaam) === norm(city) ||
+	norm(doc.gemeentenaam) === norm(city) ||
+	norm(doc.weergavenaam).includes(norm(city));
+
+/** Geocode with PDOK: exact address, else 6-char postcode, else street centroid. */
+export async function geocode(parts: AddressParts): Promise<{
+	rd: [number, number];
+	ll: [number, number] | null;
+	precision: LbmPrecision;
+	matched: string;
+	buurtCode: string | null;
+	buurtName: string | null;
+} | null> {
+	const street =
+		(parts.street ?? '').trim() || (parts.source === 'manual' ? (parts.title ?? '').trim() : '');
+	const city = (parts.city ?? '').trim();
+	const pc = POSTCODE.exec(parts.postalCode ?? '');
+	const postcode = pc ? `${pc[1]}${pc[2].toUpperCase()}` : null;
+	const pick = (doc: Doc, precision: LbmPrecision) => {
+		const rd = point(doc.centroide_rd);
+		if (!rd) return null;
+		return {
+			rd,
+			ll: point(doc.centroide_ll),
+			precision,
+			matched: String(doc.weergavenaam ?? ''),
+			buurtCode: (doc.buurtcode as string) ?? null,
+			buurtName: (doc.buurtnaam as string) ?? null
+		};
+	};
+
+	if (street && HAS_NUMBER.test(street)) {
+		const docs = await pdok([street, postcode, city].filter(Boolean).join(' '), 'adres');
+		const streetName = norm(street.replace(HAS_NUMBER, ''));
+		const number = /(\d+)\s*[a-z]?[\s-]*\d*[a-z]*\s*$/i.exec(street)?.[1];
+		const doc = docs.find(
+			(d) =>
+				placeMatches(d, city) &&
+				String(d.huisnummer ?? '') === number &&
+				(norm(d.straatnaam).includes(streetName) || streetName.includes(norm(d.straatnaam)))
+		);
+		if (doc) return pick(doc, 'address');
+	}
+	if (postcode) {
+		const [doc] = await pdok(postcode, 'postcode');
+		if (doc && String(doc.postcode ?? '').replace(/\s/g, '') === postcode) {
+			const hit = pick(doc, 'postcode');
+			if (hit) return hit;
+		}
+	}
+	const streetName = street.replace(HAS_NUMBER, '').trim();
+	if (streetName && city) {
+		const docs = await pdok(`${streetName} ${city}`, 'weg');
+		const doc = docs.find(
+			(d) =>
+				placeMatches(d, city) && norm(d.straatnaam ?? d.weergavenaam).includes(norm(streetName))
+		);
+		if (doc) return pick(doc, 'street');
+	}
+	return null;
+}
+
+const FIELDS = 'id,name,gemeente,lbm,afw,fys,onv,soc,vrz,won,kscore,kfys,konv,ksoc,kvrz,kwon';
+
+async function wfsArea(layer: string, cql: string): Promise<LbmArea | null> {
+	const url = new URL(WFS);
+	url.search = new URLSearchParams({
+		service: 'WFS',
+		version: '2.0.0',
+		request: 'GetFeature',
+		typeNames: `lbm3:${layer}`,
+		outputFormat: 'application/json',
+		propertyName: FIELDS,
+		count: '1',
+		CQL_FILTER: cql
+	}).toString();
+	const data = await getJson(url.toString());
+	const p = ((data.features as Doc[])?.[0]?.properties as Doc) ?? null;
+	if (!p || p.kscore == null || p.lbm == null) return null;
+	const num = (v: unknown) => (v == null || v === '' ? null : Number(v));
+	return {
+		id: String(p.id),
+		name: (p.name as string) ?? null,
+		gemeente: (p.gemeente as string) ?? null,
+		lbm: Number(p.lbm),
+		kscore: Number(p.kscore),
+		classes: {
+			onv: num(p.konv),
+			fys: num(p.kfys),
+			soc: num(p.ksoc),
+			vrz: num(p.kvrz),
+			won: num(p.kwon)
+		},
+		deviation: {
+			afw: num(p.afw),
+			onv: num(p.onv),
+			fys: num(p.fys),
+			soc: num(p.soc),
+			vrz: num(p.vrz),
+			won: num(p.won)
+		}
+	};
+}
+
+/** 100 m cell id as used by the Leefbaarometer grid: lower-left corner in hectometres. */
+export const gridCellId = ([x, y]: [number, number]) =>
+	`E${Math.floor(x / 100)}N${Math.floor(y / 100)}`;
+
+export async function liveabilityAt(parts: AddressParts): Promise<Liveability | null> {
+	const geo = await geocode(parts);
+	if (!geo) return null;
+	const [x, y] = geo.rd;
+	const [buurt, grid] = await Promise.all([
+		wfsArea(`buurtscore${LAYER_SUFFIX}`, `INTERSECTS(geom,POINT(${x} ${y}))`),
+		// A street centroid says little about one 100 m cell; use the buurt for those.
+		geo.precision === 'street'
+			? Promise.resolve(null)
+			: wfsArea(`clippedgridscore${LAYER_SUFFIX}`, `id='${gridCellId(geo.rd)}'`)
+	]);
+	if (!buurt && !grid) return null;
+	return {
+		year: LBM_YEAR,
+		level: grid ? 'grid' : 'buurt',
+		precision: geo.precision,
+		rd: geo.rd,
+		lng: geo.ll?.[0] ?? null,
+		lat: geo.ll?.[1] ?? null,
+		matched: geo.matched,
+		buurtCode: buurt?.id ?? geo.buurtCode,
+		buurtName: buurt?.name ?? geo.buurtName,
+		grid,
+		buurt
+	};
+}
+
+/** Columns written to house.listings for one result (null result = nothing found). */
+export function lbmColumns(query: string | null, l: Liveability | null) {
+	const main = l ? (l.grid ?? l.buurt)! : null;
+	return {
+		lbm_query: query,
+		lbm_scored_at: new Date(),
+		lbm_year: l ? l.year : null,
+		lbm_score: main ? main.lbm : null,
+		lbm_class: main ? main.kscore : null,
+		lbm_onv_class: main ? main.classes.onv : null,
+		lbm_level: l ? l.level : null,
+		lbm_precision: l ? l.precision : null,
+		lbm_buurt_code: l ? l.buurtCode : null,
+		lbm_buurt_name: l ? l.buurtName : null,
+		lbm_detail: l
+	};
+}
+
+export type ScoreResult = { checked: number; scored: number; notFound: number; failed: number };
+
+/**
+ * Scores listings whose address has not been scored yet (or changed since). `ids` limits it
+ * to those rows; `force` re-scores even when the address is unchanged. One listing at a time,
+ * a short pause between them; a network error leaves the row for the next run.
+ */
+export async function scoreListings(
+	sql: postgres.Sql,
+	opts: { ids?: number[]; force?: boolean; limit?: number; log?: (line: string) => void } = {}
+): Promise<ScoreResult> {
+	const rows = await sql<
+		{
+			id: number;
+			source: string;
+			street: string | null;
+			postal_code: string | null;
+			city: string | null;
+			title: string | null;
+			lat: number | null;
+			lbm_query: string | null;
+			lbm_scored_at: Date | null;
+		}[]
+	>`SELECT id, source, street, postal_code, city, title, lat, lbm_query, lbm_scored_at
+		FROM house.listings
+		WHERE ${opts.ids ? sql`id IN ${sql(opts.ids.length ? opts.ids : [-1])}` : sql`duplicate_of IS NULL`}
+		ORDER BY removed_at IS NULL DESC, first_seen_at DESC`;
+	const result: ScoreResult = { checked: 0, scored: 0, notFound: 0, failed: 0 };
+	for (const row of rows) {
+		const parts = {
+			street: row.street,
+			postalCode: row.postal_code,
+			city: row.city,
+			title: row.title,
+			source: row.source
+		};
+		const query = lbmQuery(parts);
+		if (!opts.force && row.lbm_scored_at && row.lbm_query === query) continue;
+		if (opts.limit != null && result.checked >= opts.limit) break;
+		result.checked++;
+		try {
+			const l = query ? await liveabilityAt(parts) : null;
+			await sql`UPDATE house.listings SET ${sql({
+				...lbmColumns(query, l),
+				lbm_detail: l ? sql.json(l as unknown as postgres.JSONValue) : null
+			})} WHERE id = ${row.id}`;
+			// Fill missing coordinates so the map can show the listing too.
+			if (l?.lat != null && row.lat == null)
+				await sql`UPDATE house.listings SET lat = ${l.lat}, lng = ${l.lng} WHERE id = ${row.id} AND lat IS NULL`;
+			if (l) result.scored++;
+			else result.notFound++;
+			opts.log?.(
+				`  #${row.id} ${query ?? '(no address)'} -> ${
+					l
+						? `${(l.grid ?? l.buurt)!.kscore} ${lbmClassLabel((l.grid ?? l.buurt)!.kscore)} (${l.level}, ${l.precision}, ${l.buurtName})`
+						: 'not found'
+				}`
+			);
+		} catch (error) {
+			result.failed++;
+			opts.log?.(`  #${row.id} ${query}: ${(error as Error).message}`);
+		}
+		await new Promise((resolve) => setTimeout(resolve, 300));
+	}
+	return result;
+}
+
+/** Link to the official map, zoomed in on the point. */
+export function lbmMapUrl(lat: number, lng: number) {
+	const u = new URL('https://www.leefbaarometer.nl/kaart/');
+	u.search = new URLSearchParams({
+		indicatorN: 'Leefbaarheidssituatie',
+		periodeN: String(LBM_YEAR),
+		schaalniveauN: 'Grid',
+		latitude: lat.toFixed(6),
+		longitude: lng.toFixed(6),
+		zoom: '16'
+	}).toString();
+	return u.toString();
+}

src/lib/house-tracker/vesteda.ts

@@ -6,7 +6,7 @@ import type { PropertyType } from './types';
 export const VESTEDA_ORIGIN = 'https://www.vesteda.com';
 export const VESTEDA_SEARCH_URL = `${VESTEDA_ORIGIN}/api/units/search/facet`;
 
-/** Same body the site sends: a point + radius (km), rent range. */
+/** Same body the site sends: a point + radius (km), rent range (wide open: we store all). */
 export function vestedaSearchBody(opts: {
 	lat: number;
 	lng: number;
@@ -21,8 +21,8 @@ export function vestedaSearchBody(opts: {
 		placeType: 1,
 		radius: opts.radiusKm,
 		sorting: 0,
-		priceFrom: 500,
-		priceTo: 9999,
+		priceFrom: 0,
+		priceTo: 99999,
 		language: 'nl'
 	};
 }

src/lib/server/db/schema/house-tracker.ts

@@ -14,6 +14,7 @@ import type { AnyPgColumn } from 'drizzle-orm/pg-core';
 import { houseSchema } from './schemas';
 import { sql } from 'drizzle-orm';
 import { id, timestamptz } from './helpers';
+import type { Liveability } from '../../../house-tracker/liveability';
 
 export const houseListings = houseSchema.table(
 	'listings',
@@ -66,7 +67,23 @@ export const houseListings = houseSchema.table(
 			onDelete: 'set null'
 		}),
 		// Monthly rent including service costs, when a source shows it separately.
-		totalPriceEur: integer('total_price_eur')
+		totalPriceEur: integer('total_price_eur'),
+		// Leefbaarometer liveability (src/lib/house-tracker/liveability.ts). lbm_class is the
+		// 1-9 class (zeer onvoldoende .. uitstekend) of the 100 m grid cell, or of the buurt when
+		// there is no cell or only a street-level location; lbm_onv_class the 1-9 class of the
+		// "overlast & onveiligheid" dimension. lbm_query is the address text that was geocoded,
+		// so a changed address is re-scored.
+		lbmScore: doublePrecision('lbm_score'),
+		lbmClass: integer('lbm_class'),
+		lbmOnvClass: integer('lbm_onv_class'),
+		lbmLevel: text('lbm_level'),
+		lbmPrecision: text('lbm_precision'),
+		lbmBuurtCode: text('lbm_buurt_code'),
+		lbmBuurtName: text('lbm_buurt_name'),
+		lbmYear: integer('lbm_year'),
+		lbmDetail: jsonb('lbm_detail').$type<Liveability>(),
+		lbmQuery: text('lbm_query'),
+		lbmScoredAt: timestamptz('lbm_scored_at')
 	},
 	(table) => [
 		check('listings_offer_check', sql`offer IN ('rent', 'buy')`),