houses: Leefbaarometer liveability per listing (PDOK geocode + LBM 2024 grid/buurt), min-score filter and sort; store every regional listing (no price caps at scrape time)

scripts/house-listing.ts

@@ -12,6 +12,7 @@
 import { readFileSync } from 'node:fs';
 import { parseArgs } from 'node:util';
 import postgres from 'postgres';
+import { lbmClassLabel, scoreListings } from '../src/lib/house-tracker/liveability';
 import {
 	inferOffer,
 	inferPropertyType,
@@ -254,6 +255,32 @@ async function add() {
 			return { ...inserted, action: 'added' };
 		});
 		print([row]);
+		if (row?.id) await liveability(row.id as number);
+	}
+}
+
+/** Leefbaarometer score for a listing that was just added or changed (skips if unchanged). */
+async function liveability(id: number) {
+	try {
+		const r = await scoreListings(sql, { ids: [id] });
+		if (!r.checked) return;
+		const [l] = await sql<
+			{
+				lbm_class: number | null;
+				lbm_onv_class: number | null;
+				lbm_buurt_name: string | null;
+				lbm_precision: string | null;
+			}[]
+		>`select lbm_class, lbm_onv_class, lbm_buurt_name, lbm_precision from house.listings where id = ${id}`;
+		console.log(
+			l?.lbm_class != null
+				? `    leefbaarometer ${l.lbm_class}/9 ${lbmClassLabel(l.lbm_class)} · overlast ${l.lbm_onv_class}/9 · ${l.lbm_buurt_name}${l.lbm_precision === 'address' ? '' : ` (approx., ${l.lbm_precision})`}`
+				: r.failed
+					? '    leefbaarometer: lookup failed, the nightly run retries'
+					: '    leefbaarometer: no location found for this address'
+		);
+	} catch (error) {
+		console.warn(`    leefbaarometer: ${(error as Error).message}`);
 	}
 }
 
@@ -298,7 +325,10 @@ async function update() {
 		await recordPrice(tx, id, before.price_eur, given.price_eur);
 		return rows;
 	});
-	if (row) print([row]);
+	if (row) {
+		print([row]);
+		await liveability(id);
+	}
 }
 
 async function list() {

scripts/house-liveability.ts

@@ -0,0 +1,25 @@
+// (Re)computes Leefbaarometer scores for house.listings. The nightly scrape, the CLI and the
+// page already score new and changed listings; this is for backfills and a new LBM year.
+//
+//   bun --env-file=$HOME/.config/house-scrape.env scripts/house-liveability.ts [--force] [--id 12]
+import postgres from 'postgres';
+import { scoreListings } from '../src/lib/house-tracker/liveability';
+
+const args = process.argv.slice(2);
+const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
+if (!dbUrl) throw new Error('DB_URL is not set');
+const sql = postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} });
+const ids = args.includes('--id') ? [Number(args[args.indexOf('--id') + 1])] : undefined;
+try {
+	const result = await scoreListings(sql, {
+		ids,
+		force: args.includes('--force'),
+		log: (line) => console.log(line)
+	});
+	console.log(
+		`liveability: ${result.checked} checked, ${result.scored} scored, ${result.notFound} without a location, ${result.failed} failed`
+	);
+	if (result.failed) process.exitCode = 1;
+} finally {
+	await sql.end();
+}

scripts/house-scrape.ts

@@ -12,6 +12,7 @@ import postgres from 'postgres';
 import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
 import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address';
 import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
+import { scoreListings } from '../src/lib/house-tracker/liveability';
 import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
 import { isChallenge } from '../src/lib/house-tracker/pararius';
 import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
@@ -29,12 +30,14 @@ const USER_AGENT =
 	'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
 const KAMERNET_CITIES = (
 	process.env.HOUSE_KAMERNET_CITIES ??
-	'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
+	'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,haastrecht,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
 )
 	.split(',')
 	.map((c) => c.trim())
 	.filter(Boolean);
-const KAMERNET_MAX_PAGES = 5;
+// Everything in the region is stored (no price/size caps); paging stops early once a page has
+// none of our cities, so this is only a safety limit.
+const KAMERNET_MAX_PAGES = 25;
 // Gouda centre; Vesteda's own search uses km radius around a point.
 const VESTEDA = {
 	lat: 52.0115,
@@ -250,6 +253,16 @@ try {
 		const { keyed, copies } = await dedupe(sql);
 		console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);
 
+		// Leefbaarometer score for new listings and changed addresses (PDOK + the LBM WFS).
+		try {
+			const lbm = await scoreListings(sql, { limit: 300 });
+			console.log(
+				`liveability: ${lbm.checked} checked, ${lbm.scored} scored, ${lbm.notFound} without a location, ${lbm.failed} failed`
+			);
+		} catch (error) {
+			console.error(`liveability: ${(error as Error).message}`);
+		}
+
 		const rules =
 			(await sql`SELECT id, name, enabled, offer, min_price AS "minPrice", max_price AS "maxPrice",
 			min_area AS "minArea", min_bedrooms AS "minBedrooms", cities, property_types AS "propertyTypes"

scripts/house-worker.ts

@@ -50,11 +50,29 @@ const config = {
 	api: (process.env.HOUSE_API_URL ?? '').replace(/\/$/, ''),
 	token: process.env.HOUSE_WORKER_TOKEN ?? '',
 	fastSearches: list(process.env.HOUSE_FAST_SEARCHES, [
-		'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
+		// No price band: every Gouda rental is stored; the alert rule decides what pings.
+		'https://www.pararius.nl/huurwoningen/gouda'
 	]),
 	slowSearches: list(
 		process.env.HOUSE_SLOW_SEARCHES,
-		['gouda', 'rotterdam', 'den-haag', 'utrecht', 'amsterdam', 'eindhoven'].flatMap((city) => [
+		[
+			'gouda',
+			'waddinxveen',
+			'reeuwijk',
+			'bodegraven',
+			'boskoop',
+			'moordrecht',
+			'nieuwerkerk-aan-den-ijssel',
+			'zevenhuizen',
+			'oudewater',
+			'schoonhoven',
+			'haastrecht',
+			'rotterdam',
+			'den-haag',
+			'utrecht',
+			'amsterdam',
+			'eindhoven'
+		].flatMap((city) => [
 			`https://www.pararius.nl/huurwoningen/${city}`,
 			`https://www.pararius.nl/koopwoningen/${city}`
 		])