houses: Leefbaarometer liveability per listing (PDOK geocode + LBM 2024 grid/buurt), min-score filter and sort; store every regional listing (no price caps at scrape time)
All checks were successful
Deploy production / deploy (push) Successful in 2m33s
All checks were successful
Deploy production / deploy (push) Successful in 2m33s
This commit is contained in:
@@ -26,6 +26,22 @@ numbers, so its listings are never merged. Alerts: new non-copy listings matchin
|
||||
rule go to `HOUSE_NTFY_URL` (its own topic, max 5 per night plus one summary); the first run
|
||||
of a source only seeds.
|
||||
|
||||
No caps at scrape time: every listing in the region is stored (Kamernet city pages, Vesteda
|
||||
with an open rent range, Pararius without a price band). The €2,000 / 2-bedroom / 50 m²
|
||||
limits are only the page's default filters (clear a field to drop it) and the alert rule.
|
||||
|
||||
Liveability: every listing gets a Leefbaarometer 3.0 score (BZK, measurement 2024, published
|
||||
July 2026; `src/lib/house-tracker/liveability.ts`). The address is geocoded with PDOK
|
||||
Locatieserver (exact address, else 6-char postcode, else street centroid → marked approx.),
|
||||
then the 100 m grid cell and the buurt containing the point are read from the official WFS
|
||||
(`geo.leefbaarometer.nl/lbm3`, layers `clippedgridscore24` / `buurtscore24`). The cell score
|
||||
is used when there is one (and the location is not street-level), else the buurt. Stored on
|
||||
the row: `lbm_class` (1-9, zeer onvoldoende … uitstekend), `lbm_onv_class` (overlast &
|
||||
onveiligheid, 1-9 where 5 = Dutch average), buurt, level, precision, year, and all
|
||||
sub-scores in `lbm_detail`. Scored by the nightly scrape (new and changed addresses), the
|
||||
CLI and the page's add form; backfill or re-score with `scripts/house-liveability.ts
|
||||
[--force]`. For a new measurement, bump `LBM_YEAR` and run it with `--force`.
|
||||
|
||||
Manual listings: `scripts/house-listing.ts` (CLI) or the form on the page. Against prod use
|
||||
`scripts/house-db.sh <command> …`, which runs the CLI as `house_cli` (SELECT/INSERT/UPDATE/
|
||||
DELETE on house.listings, SELECT/INSERT on house.price_history, nothing else) from
|
||||
|
||||
11
drizzle/20261024154032_house_liveability/migration.sql
Normal file
11
drizzle/20261024154032_house_liveability/migration.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_score" double precision;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_class" integer;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_onv_class" integer;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_level" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_precision" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_buurt_code" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_buurt_name" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_year" integer;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_detail" jsonb;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_query" text;--> statement-breakpoint
|
||||
ALTER TABLE "house"."listings" ADD COLUMN "lbm_scored_at" timestamp with time zone;
|
||||
18493
drizzle/20261024154032_house_liveability/snapshot.json
Normal file
18493
drizzle/20261024154032_house_liveability/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,7 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { parseArgs } from 'node:util';
|
||||
import postgres from 'postgres';
|
||||
import { lbmClassLabel, scoreListings } from '../src/lib/house-tracker/liveability';
|
||||
import {
|
||||
inferOffer,
|
||||
inferPropertyType,
|
||||
@@ -254,6 +255,32 @@ async function add() {
|
||||
return { ...inserted, action: 'added' };
|
||||
});
|
||||
print([row]);
|
||||
if (row?.id) await liveability(row.id as number);
|
||||
}
|
||||
}
|
||||
|
||||
/** Leefbaarometer score for a listing that was just added or changed (skips if unchanged). */
|
||||
async function liveability(id: number) {
|
||||
try {
|
||||
const r = await scoreListings(sql, { ids: [id] });
|
||||
if (!r.checked) return;
|
||||
const [l] = await sql<
|
||||
{
|
||||
lbm_class: number | null;
|
||||
lbm_onv_class: number | null;
|
||||
lbm_buurt_name: string | null;
|
||||
lbm_precision: string | null;
|
||||
}[]
|
||||
>`select lbm_class, lbm_onv_class, lbm_buurt_name, lbm_precision from house.listings where id = ${id}`;
|
||||
console.log(
|
||||
l?.lbm_class != null
|
||||
? ` leefbaarometer ${l.lbm_class}/9 ${lbmClassLabel(l.lbm_class)} · overlast ${l.lbm_onv_class}/9 · ${l.lbm_buurt_name}${l.lbm_precision === 'address' ? '' : ` (approx., ${l.lbm_precision})`}`
|
||||
: r.failed
|
||||
? ' leefbaarometer: lookup failed, the nightly run retries'
|
||||
: ' leefbaarometer: no location found for this address'
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(` leefbaarometer: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -298,7 +325,10 @@ async function update() {
|
||||
await recordPrice(tx, id, before.price_eur, given.price_eur);
|
||||
return rows;
|
||||
});
|
||||
if (row) print([row]);
|
||||
if (row) {
|
||||
print([row]);
|
||||
await liveability(id);
|
||||
}
|
||||
}
|
||||
|
||||
async function list() {
|
||||
|
||||
25
scripts/house-liveability.ts
Normal file
25
scripts/house-liveability.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
// (Re)computes Leefbaarometer scores for house.listings. The nightly scrape, the CLI and the
|
||||
// page already score new and changed listings; this is for backfills and a new LBM year.
|
||||
//
|
||||
// bun --env-file=$HOME/.config/house-scrape.env scripts/house-liveability.ts [--force] [--id 12]
|
||||
import postgres from 'postgres';
|
||||
import { scoreListings } from '../src/lib/house-tracker/liveability';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
|
||||
if (!dbUrl) throw new Error('DB_URL is not set');
|
||||
const sql = postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
||||
const ids = args.includes('--id') ? [Number(args[args.indexOf('--id') + 1])] : undefined;
|
||||
try {
|
||||
const result = await scoreListings(sql, {
|
||||
ids,
|
||||
force: args.includes('--force'),
|
||||
log: (line) => console.log(line)
|
||||
});
|
||||
console.log(
|
||||
`liveability: ${result.checked} checked, ${result.scored} scored, ${result.notFound} without a location, ${result.failed} failed`
|
||||
);
|
||||
if (result.failed) process.exitCode = 1;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import postgres from 'postgres';
|
||||
import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
|
||||
import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address';
|
||||
import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
|
||||
import { scoreListings } from '../src/lib/house-tracker/liveability';
|
||||
import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
|
||||
import { isChallenge } from '../src/lib/house-tracker/pararius';
|
||||
import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
|
||||
@@ -29,12 +30,14 @@ const USER_AGENT =
|
||||
'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
|
||||
const KAMERNET_CITIES = (
|
||||
process.env.HOUSE_KAMERNET_CITIES ??
|
||||
'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
|
||||
'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,haastrecht,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
|
||||
)
|
||||
.split(',')
|
||||
.map((c) => c.trim())
|
||||
.filter(Boolean);
|
||||
const KAMERNET_MAX_PAGES = 5;
|
||||
// Everything in the region is stored (no price/size caps); paging stops early once a page has
|
||||
// none of our cities, so this is only a safety limit.
|
||||
const KAMERNET_MAX_PAGES = 25;
|
||||
// Gouda centre; Vesteda's own search uses km radius around a point.
|
||||
const VESTEDA = {
|
||||
lat: 52.0115,
|
||||
@@ -250,6 +253,16 @@ try {
|
||||
const { keyed, copies } = await dedupe(sql);
|
||||
console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);
|
||||
|
||||
// Leefbaarometer score for new listings and changed addresses (PDOK + the LBM WFS).
|
||||
try {
|
||||
const lbm = await scoreListings(sql, { limit: 300 });
|
||||
console.log(
|
||||
`liveability: ${lbm.checked} checked, ${lbm.scored} scored, ${lbm.notFound} without a location, ${lbm.failed} failed`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`liveability: ${(error as Error).message}`);
|
||||
}
|
||||
|
||||
const rules =
|
||||
(await sql`SELECT id, name, enabled, offer, min_price AS "minPrice", max_price AS "maxPrice",
|
||||
min_area AS "minArea", min_bedrooms AS "minBedrooms", cities, property_types AS "propertyTypes"
|
||||
|
||||
@@ -50,11 +50,29 @@ const config = {
|
||||
api: (process.env.HOUSE_API_URL ?? '').replace(/\/$/, ''),
|
||||
token: process.env.HOUSE_WORKER_TOKEN ?? '',
|
||||
fastSearches: list(process.env.HOUSE_FAST_SEARCHES, [
|
||||
'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
|
||||
// No price band: every Gouda rental is stored; the alert rule decides what pings.
|
||||
'https://www.pararius.nl/huurwoningen/gouda'
|
||||
]),
|
||||
slowSearches: list(
|
||||
process.env.HOUSE_SLOW_SEARCHES,
|
||||
['gouda', 'rotterdam', 'den-haag', 'utrecht', 'amsterdam', 'eindhoven'].flatMap((city) => [
|
||||
[
|
||||
'gouda',
|
||||
'waddinxveen',
|
||||
'reeuwijk',
|
||||
'bodegraven',
|
||||
'boskoop',
|
||||
'moordrecht',
|
||||
'nieuwerkerk-aan-den-ijssel',
|
||||
'zevenhuizen',
|
||||
'oudewater',
|
||||
'schoonhoven',
|
||||
'haastrecht',
|
||||
'rotterdam',
|
||||
'den-haag',
|
||||
'utrecht',
|
||||
'amsterdam',
|
||||
'eindhoven'
|
||||
].flatMap((city) => [
|
||||
`https://www.pararius.nl/huurwoningen/${city}`,
|
||||
`https://www.pararius.nl/koopwoningen/${city}`
|
||||
])
|
||||
|
||||
30
src/lib/house-tracker/liveability.test.ts
Normal file
30
src/lib/house-tracker/liveability.test.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { gridCellId, lbmClassLabel, lbmDimensionLabel, lbmQuery } from './liveability';
|
||||
|
||||
describe('leefbaarometer helpers', () => {
|
||||
it('names the 100 m cell by its lower-left corner in hectometres', () => {
|
||||
expect(gridCellId([106050.4, 446899.9])).toBe('E1060N4468');
|
||||
expect(gridCellId([105866.161, 446838.902])).toBe('E1058N4468');
|
||||
});
|
||||
it('labels classes 1-9', () => {
|
||||
expect(lbmClassLabel(1)).toBe('zeer onvoldoende');
|
||||
expect(lbmClassLabel(9)).toBe('uitstekend');
|
||||
expect(lbmClassLabel(0)).toBeNull();
|
||||
expect(lbmDimensionLabel(5)).toBe('geen afwijking');
|
||||
});
|
||||
it('builds the geocode query from street, postcode and city', () => {
|
||||
expect(lbmQuery({ street: 'Raam 12', postalCode: '2801 vk', city: 'Gouda' })).toBe(
|
||||
'Raam 12 | 2801VK | Gouda'
|
||||
);
|
||||
expect(lbmQuery({ street: null, postalCode: null, city: 'Gouda' })).toBeNull();
|
||||
expect(
|
||||
lbmQuery({
|
||||
street: null,
|
||||
postalCode: null,
|
||||
city: 'Gouda',
|
||||
title: 'Turfmarkt 5',
|
||||
source: 'manual'
|
||||
})
|
||||
).toBe('Turfmarkt 5 | Gouda');
|
||||
});
|
||||
});
|
||||
374
src/lib/house-tracker/liveability.ts
Normal file
374
src/lib/house-tracker/liveability.ts
Normal file
@@ -0,0 +1,374 @@
|
||||
// Leefbaarometer 3.0 (Ministerie van BZK) liveability scores for listings.
|
||||
// Address -> RD point via PDOK Locatieserver, then the 100 m grid cell and the buurt that
|
||||
// contain it from the official Leefbaarometer WFS (geo.leefbaarometer.nl/lbm3). No API keys,
|
||||
// no static data in the repo: each listing needs two or three tiny requests, once, and the
|
||||
// result is stored on the row. Plain fetch + postgres.js, so the nightly scraper, the CLI
|
||||
// and the page can all use it. See docs/house-tracker.md.
|
||||
import type postgres from 'postgres';
|
||||
|
||||
/** Latest Leefbaarometer measurement (published 1 July 2026). Bump when 2026 comes out. */
|
||||
export const LBM_YEAR = 2024;
|
||||
const LAYER_SUFFIX = String(LBM_YEAR).slice(2);
|
||||
const WFS = 'https://geo.leefbaarometer.nl/lbm3/ows';
|
||||
const PDOK = 'https://api.pdok.nl/bzk/locatieserver/search/v3_1/free';
|
||||
const USER_AGENT = 'stanrunge.dev-house-tracker/1.0 (personal use; stan@stanrunge.dev)';
|
||||
|
||||
/** kscore 1..9 (total score class). */
|
||||
export const LBM_CLASS_LABELS = [
|
||||
'zeer onvoldoende',
|
||||
'ruim onvoldoende',
|
||||
'onvoldoende',
|
||||
'zwak',
|
||||
'voldoende',
|
||||
'ruim voldoende',
|
||||
'goed',
|
||||
'zeer goed',
|
||||
'uitstekend'
|
||||
] as const;
|
||||
/** Sub-dimension classes 1..9: contribution relative to the Dutch average. */
|
||||
export const LBM_DIMENSION_LABELS = [
|
||||
'zeer grote negatieve bijdrage',
|
||||
'grote negatieve bijdrage',
|
||||
'negatieve bijdrage',
|
||||
'kleine negatieve bijdrage',
|
||||
'geen afwijking',
|
||||
'kleine positieve bijdrage',
|
||||
'positieve bijdrage',
|
||||
'grote positieve bijdrage',
|
||||
'zeer grote positieve bijdrage'
|
||||
] as const;
|
||||
export const LBM_DIMENSIONS = {
|
||||
onv: 'overlast & onveiligheid',
|
||||
fys: 'fysieke omgeving',
|
||||
soc: 'sociale samenhang',
|
||||
vrz: 'voorzieningen',
|
||||
won: 'woningvoorraad'
|
||||
} as const;
|
||||
export type LbmDimension = keyof typeof LBM_DIMENSIONS;
|
||||
|
||||
export const lbmClassLabel = (k: number | null | undefined) =>
|
||||
k != null && k >= 1 && k <= 9 ? LBM_CLASS_LABELS[k - 1] : null;
|
||||
export const lbmDimensionLabel = (k: number | null | undefined) =>
|
||||
k != null && k >= 1 && k <= 9 ? LBM_DIMENSION_LABELS[k - 1] : null;
|
||||
|
||||
/** address: geocoded to the house; postcode: PC6 centroid; street: street centroid. */
|
||||
export type LbmPrecision = 'address' | 'postcode' | 'street';
|
||||
|
||||
export type LbmArea = {
|
||||
id: string;
|
||||
name: string | null;
|
||||
gemeente: string | null;
|
||||
lbm: number;
|
||||
kscore: number;
|
||||
classes: Record<LbmDimension, number | null>;
|
||||
deviation: Record<LbmDimension | 'afw', number | null>;
|
||||
};
|
||||
|
||||
export type Liveability = {
|
||||
year: number;
|
||||
/** Which area the main score comes from. */
|
||||
level: 'grid' | 'buurt';
|
||||
precision: LbmPrecision;
|
||||
rd: [number, number];
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
matched: string;
|
||||
buurtCode: string | null;
|
||||
buurtName: string | null;
|
||||
grid: LbmArea | null;
|
||||
buurt: LbmArea | null;
|
||||
};
|
||||
|
||||
type Doc = Record<string, unknown>;
|
||||
|
||||
async function getJson(url: string, timeoutMs = 15_000): Promise<Doc> {
|
||||
const response = await fetch(url, {
|
||||
headers: { 'user-agent': USER_AGENT, accept: 'application/json' },
|
||||
signal: AbortSignal.timeout(timeoutMs)
|
||||
});
|
||||
if (!response.ok) throw new Error(`${new URL(url).host}: HTTP ${response.status}`);
|
||||
return (await response.json()) as Doc;
|
||||
}
|
||||
|
||||
const norm = (s: unknown) =>
|
||||
String(s ?? '')
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.trim();
|
||||
|
||||
const point = (wkt: unknown): [number, number] | null => {
|
||||
const m = /POINT\(([-\d.]+) ([-\d.]+)\)/.exec(String(wkt ?? ''));
|
||||
return m ? [Number(m[1]), Number(m[2])] : null;
|
||||
};
|
||||
|
||||
const POSTCODE = /^\s*(\d{4})\s?([a-z]{2})\s*$/i;
|
||||
const HAS_NUMBER = /\s\d+\s*[a-z]?(?:[\s-]*\d*[a-z]*)?\s*$/i;
|
||||
|
||||
export type AddressParts = {
|
||||
street: string | null;
|
||||
postalCode: string | null;
|
||||
city: string | null;
|
||||
title?: string | null;
|
||||
source?: string | null;
|
||||
};
|
||||
|
||||
/** The text we geocode; stored so a changed address gets re-scored. */
|
||||
export function lbmQuery(parts: AddressParts): string | null {
|
||||
const street =
|
||||
(parts.street ?? '').trim() || (parts.source === 'manual' ? (parts.title ?? '').trim() : '');
|
||||
const pc = POSTCODE.exec(parts.postalCode ?? '');
|
||||
const city = (parts.city ?? '').trim();
|
||||
if (!street && !pc) return null;
|
||||
return [street, pc ? `${pc[1]}${pc[2].toUpperCase()}` : '', city].filter(Boolean).join(' | ');
|
||||
}
|
||||
|
||||
async function pdok(q: string, type: string): Promise<Doc[]> {
|
||||
const url = new URL(PDOK);
|
||||
url.searchParams.set('q', q);
|
||||
url.searchParams.set('fq', `type:${type}`);
|
||||
url.searchParams.set('rows', '5');
|
||||
url.searchParams.set(
|
||||
'fl',
|
||||
'type,weergavenaam,woonplaatsnaam,gemeentenaam,straatnaam,postcode,huisnummer,huisletter,centroide_rd,centroide_ll,buurtcode,buurtnaam,score'
|
||||
);
|
||||
const data = await getJson(url.toString());
|
||||
return ((data.response as Doc)?.docs as Doc[]) ?? [];
|
||||
}
|
||||
|
||||
const placeMatches = (doc: Doc, city: string) =>
|
||||
!city ||
|
||||
norm(doc.woonplaatsnaam) === norm(city) ||
|
||||
norm(doc.gemeentenaam) === norm(city) ||
|
||||
norm(doc.weergavenaam).includes(norm(city));
|
||||
|
||||
/** Geocode with PDOK: exact address, else 6-char postcode, else street centroid. */
|
||||
export async function geocode(parts: AddressParts): Promise<{
|
||||
rd: [number, number];
|
||||
ll: [number, number] | null;
|
||||
precision: LbmPrecision;
|
||||
matched: string;
|
||||
buurtCode: string | null;
|
||||
buurtName: string | null;
|
||||
} | null> {
|
||||
const street =
|
||||
(parts.street ?? '').trim() || (parts.source === 'manual' ? (parts.title ?? '').trim() : '');
|
||||
const city = (parts.city ?? '').trim();
|
||||
const pc = POSTCODE.exec(parts.postalCode ?? '');
|
||||
const postcode = pc ? `${pc[1]}${pc[2].toUpperCase()}` : null;
|
||||
const pick = (doc: Doc, precision: LbmPrecision) => {
|
||||
const rd = point(doc.centroide_rd);
|
||||
if (!rd) return null;
|
||||
return {
|
||||
rd,
|
||||
ll: point(doc.centroide_ll),
|
||||
precision,
|
||||
matched: String(doc.weergavenaam ?? ''),
|
||||
buurtCode: (doc.buurtcode as string) ?? null,
|
||||
buurtName: (doc.buurtnaam as string) ?? null
|
||||
};
|
||||
};
|
||||
|
||||
if (street && HAS_NUMBER.test(street)) {
|
||||
const docs = await pdok([street, postcode, city].filter(Boolean).join(' '), 'adres');
|
||||
const streetName = norm(street.replace(HAS_NUMBER, ''));
|
||||
const number = /(\d+)\s*[a-z]?[\s-]*\d*[a-z]*\s*$/i.exec(street)?.[1];
|
||||
const doc = docs.find(
|
||||
(d) =>
|
||||
placeMatches(d, city) &&
|
||||
String(d.huisnummer ?? '') === number &&
|
||||
(norm(d.straatnaam).includes(streetName) || streetName.includes(norm(d.straatnaam)))
|
||||
);
|
||||
if (doc) return pick(doc, 'address');
|
||||
}
|
||||
if (postcode) {
|
||||
const [doc] = await pdok(postcode, 'postcode');
|
||||
if (doc && String(doc.postcode ?? '').replace(/\s/g, '') === postcode) {
|
||||
const hit = pick(doc, 'postcode');
|
||||
if (hit) return hit;
|
||||
}
|
||||
}
|
||||
const streetName = street.replace(HAS_NUMBER, '').trim();
|
||||
if (streetName && city) {
|
||||
const docs = await pdok(`${streetName} ${city}`, 'weg');
|
||||
const doc = docs.find(
|
||||
(d) =>
|
||||
placeMatches(d, city) && norm(d.straatnaam ?? d.weergavenaam).includes(norm(streetName))
|
||||
);
|
||||
if (doc) return pick(doc, 'street');
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const FIELDS = 'id,name,gemeente,lbm,afw,fys,onv,soc,vrz,won,kscore,kfys,konv,ksoc,kvrz,kwon';
|
||||
|
||||
async function wfsArea(layer: string, cql: string): Promise<LbmArea | null> {
|
||||
const url = new URL(WFS);
|
||||
url.search = new URLSearchParams({
|
||||
service: 'WFS',
|
||||
version: '2.0.0',
|
||||
request: 'GetFeature',
|
||||
typeNames: `lbm3:${layer}`,
|
||||
outputFormat: 'application/json',
|
||||
propertyName: FIELDS,
|
||||
count: '1',
|
||||
CQL_FILTER: cql
|
||||
}).toString();
|
||||
const data = await getJson(url.toString());
|
||||
const p = ((data.features as Doc[])?.[0]?.properties as Doc) ?? null;
|
||||
if (!p || p.kscore == null || p.lbm == null) return null;
|
||||
const num = (v: unknown) => (v == null || v === '' ? null : Number(v));
|
||||
return {
|
||||
id: String(p.id),
|
||||
name: (p.name as string) ?? null,
|
||||
gemeente: (p.gemeente as string) ?? null,
|
||||
lbm: Number(p.lbm),
|
||||
kscore: Number(p.kscore),
|
||||
classes: {
|
||||
onv: num(p.konv),
|
||||
fys: num(p.kfys),
|
||||
soc: num(p.ksoc),
|
||||
vrz: num(p.kvrz),
|
||||
won: num(p.kwon)
|
||||
},
|
||||
deviation: {
|
||||
afw: num(p.afw),
|
||||
onv: num(p.onv),
|
||||
fys: num(p.fys),
|
||||
soc: num(p.soc),
|
||||
vrz: num(p.vrz),
|
||||
won: num(p.won)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** 100 m cell id as used by the Leefbaarometer grid: lower-left corner in hectometres. */
|
||||
export const gridCellId = ([x, y]: [number, number]) =>
|
||||
`E${Math.floor(x / 100)}N${Math.floor(y / 100)}`;
|
||||
|
||||
export async function liveabilityAt(parts: AddressParts): Promise<Liveability | null> {
|
||||
const geo = await geocode(parts);
|
||||
if (!geo) return null;
|
||||
const [x, y] = geo.rd;
|
||||
const [buurt, grid] = await Promise.all([
|
||||
wfsArea(`buurtscore${LAYER_SUFFIX}`, `INTERSECTS(geom,POINT(${x} ${y}))`),
|
||||
// A street centroid says little about one 100 m cell; use the buurt for those.
|
||||
geo.precision === 'street'
|
||||
? Promise.resolve(null)
|
||||
: wfsArea(`clippedgridscore${LAYER_SUFFIX}`, `id='${gridCellId(geo.rd)}'`)
|
||||
]);
|
||||
if (!buurt && !grid) return null;
|
||||
return {
|
||||
year: LBM_YEAR,
|
||||
level: grid ? 'grid' : 'buurt',
|
||||
precision: geo.precision,
|
||||
rd: geo.rd,
|
||||
lng: geo.ll?.[0] ?? null,
|
||||
lat: geo.ll?.[1] ?? null,
|
||||
matched: geo.matched,
|
||||
buurtCode: buurt?.id ?? geo.buurtCode,
|
||||
buurtName: buurt?.name ?? geo.buurtName,
|
||||
grid,
|
||||
buurt
|
||||
};
|
||||
}
|
||||
|
||||
/** Columns written to house.listings for one result (null result = nothing found). */
|
||||
export function lbmColumns(query: string | null, l: Liveability | null) {
|
||||
const main = l ? (l.grid ?? l.buurt)! : null;
|
||||
return {
|
||||
lbm_query: query,
|
||||
lbm_scored_at: new Date(),
|
||||
lbm_year: l ? l.year : null,
|
||||
lbm_score: main ? main.lbm : null,
|
||||
lbm_class: main ? main.kscore : null,
|
||||
lbm_onv_class: main ? main.classes.onv : null,
|
||||
lbm_level: l ? l.level : null,
|
||||
lbm_precision: l ? l.precision : null,
|
||||
lbm_buurt_code: l ? l.buurtCode : null,
|
||||
lbm_buurt_name: l ? l.buurtName : null,
|
||||
lbm_detail: l
|
||||
};
|
||||
}
|
||||
|
||||
export type ScoreResult = { checked: number; scored: number; notFound: number; failed: number };
|
||||
|
||||
/**
|
||||
* Scores listings whose address has not been scored yet (or changed since). `ids` limits it
|
||||
* to those rows; `force` re-scores even when the address is unchanged. One listing at a time,
|
||||
* a short pause between them; a network error leaves the row for the next run.
|
||||
*/
|
||||
export async function scoreListings(
|
||||
sql: postgres.Sql,
|
||||
opts: { ids?: number[]; force?: boolean; limit?: number; log?: (line: string) => void } = {}
|
||||
): Promise<ScoreResult> {
|
||||
const rows = await sql<
|
||||
{
|
||||
id: number;
|
||||
source: string;
|
||||
street: string | null;
|
||||
postal_code: string | null;
|
||||
city: string | null;
|
||||
title: string | null;
|
||||
lat: number | null;
|
||||
lbm_query: string | null;
|
||||
lbm_scored_at: Date | null;
|
||||
}[]
|
||||
>`SELECT id, source, street, postal_code, city, title, lat, lbm_query, lbm_scored_at
|
||||
FROM house.listings
|
||||
WHERE ${opts.ids ? sql`id IN ${sql(opts.ids.length ? opts.ids : [-1])}` : sql`duplicate_of IS NULL`}
|
||||
ORDER BY removed_at IS NULL DESC, first_seen_at DESC`;
|
||||
const result: ScoreResult = { checked: 0, scored: 0, notFound: 0, failed: 0 };
|
||||
for (const row of rows) {
|
||||
const parts = {
|
||||
street: row.street,
|
||||
postalCode: row.postal_code,
|
||||
city: row.city,
|
||||
title: row.title,
|
||||
source: row.source
|
||||
};
|
||||
const query = lbmQuery(parts);
|
||||
if (!opts.force && row.lbm_scored_at && row.lbm_query === query) continue;
|
||||
if (opts.limit != null && result.checked >= opts.limit) break;
|
||||
result.checked++;
|
||||
try {
|
||||
const l = query ? await liveabilityAt(parts) : null;
|
||||
await sql`UPDATE house.listings SET ${sql({
|
||||
...lbmColumns(query, l),
|
||||
lbm_detail: l ? sql.json(l as unknown as postgres.JSONValue) : null
|
||||
})} WHERE id = ${row.id}`;
|
||||
// Fill missing coordinates so the map can show the listing too.
|
||||
if (l?.lat != null && row.lat == null)
|
||||
await sql`UPDATE house.listings SET lat = ${l.lat}, lng = ${l.lng} WHERE id = ${row.id} AND lat IS NULL`;
|
||||
if (l) result.scored++;
|
||||
else result.notFound++;
|
||||
opts.log?.(
|
||||
` #${row.id} ${query ?? '(no address)'} -> ${
|
||||
l
|
||||
? `${(l.grid ?? l.buurt)!.kscore} ${lbmClassLabel((l.grid ?? l.buurt)!.kscore)} (${l.level}, ${l.precision}, ${l.buurtName})`
|
||||
: 'not found'
|
||||
}`
|
||||
);
|
||||
} catch (error) {
|
||||
result.failed++;
|
||||
opts.log?.(` #${row.id} ${query}: ${(error as Error).message}`);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 300));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Link to the official map, zoomed in on the point. */
|
||||
export function lbmMapUrl(lat: number, lng: number) {
|
||||
const u = new URL('https://www.leefbaarometer.nl/kaart/');
|
||||
u.search = new URLSearchParams({
|
||||
indicatorN: 'Leefbaarheidssituatie',
|
||||
periodeN: String(LBM_YEAR),
|
||||
schaalniveauN: 'Grid',
|
||||
latitude: lat.toFixed(6),
|
||||
longitude: lng.toFixed(6),
|
||||
zoom: '16'
|
||||
}).toString();
|
||||
return u.toString();
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import type { PropertyType } from './types';
|
||||
export const VESTEDA_ORIGIN = 'https://www.vesteda.com';
|
||||
export const VESTEDA_SEARCH_URL = `${VESTEDA_ORIGIN}/api/units/search/facet`;
|
||||
|
||||
/** Same body the site sends: a point + radius (km), rent range. */
|
||||
/** Same body the site sends: a point + radius (km), rent range (wide open: we store all). */
|
||||
export function vestedaSearchBody(opts: {
|
||||
lat: number;
|
||||
lng: number;
|
||||
@@ -21,8 +21,8 @@ export function vestedaSearchBody(opts: {
|
||||
placeType: 1,
|
||||
radius: opts.radiusKm,
|
||||
sorting: 0,
|
||||
priceFrom: 500,
|
||||
priceTo: 9999,
|
||||
priceFrom: 0,
|
||||
priceTo: 99999,
|
||||
language: 'nl'
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import type { AnyPgColumn } from 'drizzle-orm/pg-core';
|
||||
import { houseSchema } from './schemas';
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { id, timestamptz } from './helpers';
|
||||
import type { Liveability } from '../../../house-tracker/liveability';
|
||||
|
||||
export const houseListings = houseSchema.table(
|
||||
'listings',
|
||||
@@ -66,7 +67,23 @@ export const houseListings = houseSchema.table(
|
||||
onDelete: 'set null'
|
||||
}),
|
||||
// Monthly rent including service costs, when a source shows it separately.
|
||||
totalPriceEur: integer('total_price_eur')
|
||||
totalPriceEur: integer('total_price_eur'),
|
||||
// Leefbaarometer liveability (src/lib/house-tracker/liveability.ts). lbm_class is the
|
||||
// 1-9 class (zeer onvoldoende .. uitstekend) of the 100 m grid cell, or of the buurt when
|
||||
// there is no cell or only a street-level location; lbm_onv_class the 1-9 class of the
|
||||
// "overlast & onveiligheid" dimension. lbm_query is the address text that was geocoded,
|
||||
// so a changed address is re-scored.
|
||||
lbmScore: doublePrecision('lbm_score'),
|
||||
lbmClass: integer('lbm_class'),
|
||||
lbmOnvClass: integer('lbm_onv_class'),
|
||||
lbmLevel: text('lbm_level'),
|
||||
lbmPrecision: text('lbm_precision'),
|
||||
lbmBuurtCode: text('lbm_buurt_code'),
|
||||
lbmBuurtName: text('lbm_buurt_name'),
|
||||
lbmYear: integer('lbm_year'),
|
||||
lbmDetail: jsonb('lbm_detail').$type<Liveability>(),
|
||||
lbmQuery: text('lbm_query'),
|
||||
lbmScoredAt: timestamptz('lbm_scored_at')
|
||||
},
|
||||
(table) => [
|
||||
check('listings_offer_check', sql`offer IN ('rent', 'buy')`),
|
||||
|
||||
@@ -20,6 +20,7 @@ import { sendAlert } from '#lib/server/house-tracker.js';
|
||||
import { normalizeCity } from '#lib/house-tracker/alerts.js';
|
||||
import { listingStatuses, propertyTypes } from '#lib/house-tracker/types.js';
|
||||
import { dedupe } from '#lib/house-tracker/ingest.js';
|
||||
import { scoreListings } from '#lib/house-tracker/liveability.js';
|
||||
import {
|
||||
inferOffer,
|
||||
inferPropertyType,
|
||||
@@ -59,7 +60,9 @@ export const load: PageServerLoad = async (event) => {
|
||||
const showRemoved = params.get('removed') === '1';
|
||||
const status = params.get('status') ?? '';
|
||||
const q = (params.get('q') ?? '').trim().slice(0, 80);
|
||||
const sort = ['new', 'price', 'ppm2', 'area'].includes(params.get('sort') ?? '')
|
||||
// Leefbaarometer class 1-9; listings without a score are hidden while this is set.
|
||||
const minLbm = int(params.get('lbm'));
|
||||
const sort = ['new', 'price', 'ppm2', 'area', 'lbm'].includes(params.get('sort') ?? '')
|
||||
? (params.get('sort') as string)
|
||||
: 'new';
|
||||
|
||||
@@ -77,6 +80,7 @@ export const load: PageServerLoad = async (event) => {
|
||||
if (source) where.push(eq(houseListings.source, source));
|
||||
if (min != null) where.push(gte(houseListings.priceEur, min));
|
||||
if (max != null) where.push(lte(houseListings.priceEur, max));
|
||||
if (minLbm != null && minLbm > 0) where.push(gte(houseListings.lbmClass, minLbm));
|
||||
if (!showRemoved) where.push(isNull(houseListings.removedAt));
|
||||
if (status === 'any') where.push(sql`${houseListings.status} IS NOT NULL`);
|
||||
else if ((listingStatuses as readonly string[]).includes(status))
|
||||
@@ -97,7 +101,12 @@ export const load: PageServerLoad = async (event) => {
|
||||
new: [desc(houseListings.firstSeenAt)],
|
||||
price: [asc(houseListings.priceEur), desc(houseListings.firstSeenAt)],
|
||||
ppm2: [sql`${ppm2} ASC NULLS LAST`, desc(houseListings.firstSeenAt)],
|
||||
area: [sql`${houseListings.areaM2} DESC NULLS LAST`, desc(houseListings.firstSeenAt)]
|
||||
area: [sql`${houseListings.areaM2} DESC NULLS LAST`, desc(houseListings.firstSeenAt)],
|
||||
lbm: [
|
||||
sql`${houseListings.lbmClass} DESC NULLS LAST`,
|
||||
sql`${houseListings.lbmScore} DESC NULLS LAST`,
|
||||
desc(houseListings.firstSeenAt)
|
||||
]
|
||||
}[sort]!;
|
||||
|
||||
const [listings, cities, stats, rules, runs, health] = await Promise.all([
|
||||
@@ -131,7 +140,14 @@ export const load: PageServerLoad = async (event) => {
|
||||
removedAt: houseListings.removedAt,
|
||||
alertedAt: houseListings.alertedAt,
|
||||
status: houseListings.status,
|
||||
notes: houseListings.notes
|
||||
notes: houseListings.notes,
|
||||
lbmClass: houseListings.lbmClass,
|
||||
lbmOnvClass: houseListings.lbmOnvClass,
|
||||
lbmLevel: houseListings.lbmLevel,
|
||||
lbmPrecision: houseListings.lbmPrecision,
|
||||
lbmBuurtName: houseListings.lbmBuurtName,
|
||||
lbmYear: houseListings.lbmYear,
|
||||
lbmDetail: houseListings.lbmDetail
|
||||
})
|
||||
.from(houseListings)
|
||||
.where(and(...where))
|
||||
@@ -194,7 +210,20 @@ export const load: PageServerLoad = async (event) => {
|
||||
}
|
||||
|
||||
return {
|
||||
filters: { offer, city, min, max, minArea, minRooms, source, showRemoved, status, q, sort },
|
||||
filters: {
|
||||
offer,
|
||||
city,
|
||||
min,
|
||||
max,
|
||||
minArea,
|
||||
minRooms,
|
||||
minLbm,
|
||||
source,
|
||||
showRemoved,
|
||||
status,
|
||||
q,
|
||||
sort
|
||||
},
|
||||
defaults: DEFAULTS,
|
||||
listings: listings.map((row) => ({ ...row, copies: copiesOf.get(row.id) ?? [] })),
|
||||
sources: sources as unknown as { source: string; active: number; last_seen: string | Date }[],
|
||||
@@ -283,6 +312,7 @@ export const actions: Actions = {
|
||||
const client = db.$client;
|
||||
const [existing] = await client`SELECT id FROM house.listings
|
||||
WHERE source = ${key.source} AND source_id = ${key.sourceId}`;
|
||||
let listingId: number = existing?.id;
|
||||
if (existing) {
|
||||
await client`UPDATE house.listings SET ${client(columns)}, removed_at = NULL WHERE id = ${existing.id}`;
|
||||
} else {
|
||||
@@ -297,10 +327,13 @@ export const actions: Actions = {
|
||||
detail_fetched_at: new Date(),
|
||||
...columns
|
||||
})} RETURNING id`;
|
||||
listingId = row.id;
|
||||
if (listing.priceEur != null)
|
||||
await client`INSERT INTO house.price_history (listing_id, price_eur) VALUES (${row.id}, ${listing.priceEur})`;
|
||||
}
|
||||
await dedupe(client);
|
||||
// Leefbaarometer score right away; on a network hiccup the nightly run fills it in.
|
||||
await scoreListings(client, { ids: [listingId] }).catch(() => undefined);
|
||||
return { message: existing ? 'Updated the existing listing' : 'Added' };
|
||||
},
|
||||
saveRule: async (event) => {
|
||||
|
||||
@@ -10,6 +10,14 @@
|
||||
import * as Card from '#lib/components/ui/card/index.js';
|
||||
import ListingMap from '#lib/house-tracker/ListingMap.svelte';
|
||||
import { relativeTime } from '#lib/relative-time.js';
|
||||
import {
|
||||
LBM_DIMENSIONS,
|
||||
lbmClassLabel,
|
||||
lbmDimensionLabel,
|
||||
lbmMapUrl,
|
||||
type LbmDimension,
|
||||
type Liveability
|
||||
} from '#lib/house-tracker/liveability.js';
|
||||
import type { PageProps } from './$types';
|
||||
|
||||
let { data, form }: PageProps = $props();
|
||||
@@ -60,6 +68,42 @@
|
||||
? 'healthy'
|
||||
: 'quiet'
|
||||
);
|
||||
// Leefbaarometer class 1-9 -> badge colour (red .. green, like their map).
|
||||
function lbmColour(k: number) {
|
||||
if (k <= 3) return 'bg-red-600 text-white';
|
||||
if (k === 4) return 'bg-orange-500 text-white';
|
||||
if (k === 5) return 'bg-yellow-400 text-black';
|
||||
if (k === 6) return 'bg-lime-400 text-black';
|
||||
if (k === 7) return 'bg-green-500 text-white';
|
||||
return 'bg-green-700 text-white';
|
||||
}
|
||||
function lbmTooltip(l: {
|
||||
lbmClass: number | null;
|
||||
lbmLevel: string | null;
|
||||
lbmPrecision: string | null;
|
||||
lbmBuurtName: string | null;
|
||||
lbmYear: number | null;
|
||||
lbmDetail: Liveability | null;
|
||||
}) {
|
||||
const main = l.lbmDetail ? (l.lbmDetail.grid ?? l.lbmDetail.buurt) : null;
|
||||
const lines = [
|
||||
`Leefbaarometer ${l.lbmYear}: ${l.lbmClass}/9 ${lbmClassLabel(l.lbmClass)}`,
|
||||
l.lbmLevel === 'grid'
|
||||
? `100 m grid cell (buurt ${l.lbmBuurtName}${l.lbmDetail?.buurt ? `: ${l.lbmDetail.buurt.kscore}/9 ${lbmClassLabel(l.lbmDetail.buurt.kscore)}` : ''})`
|
||||
: `buurt average: ${l.lbmBuurtName}`,
|
||||
l.lbmPrecision === 'address'
|
||||
? `location: exact address`
|
||||
: `location: approximate (${l.lbmPrecision} level)`,
|
||||
'',
|
||||
...(main
|
||||
? (Object.keys(LBM_DIMENSIONS) as LbmDimension[]).map(
|
||||
(d) =>
|
||||
`${LBM_DIMENSIONS[d]}: ${main.classes[d] ?? '?'}/9 ${lbmDimensionLabel(main.classes[d]) ?? ''}`
|
||||
)
|
||||
: [])
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
const mappable = $derived(data.listings.filter((row) => row.lat != null).length);
|
||||
</script>
|
||||
|
||||
@@ -129,6 +173,12 @@
|
||||
placeholder="min rooms"
|
||||
value={data.filters.minRooms ?? ''}
|
||||
/>
|
||||
<NativeSelect name="lbm" value={String(data.filters.minLbm ?? '')} aria-label="Min liveability">
|
||||
<option value="">any liveability</option>
|
||||
{#each [5, 6, 7, 8] as k (k)}
|
||||
<option value={String(k)}>≥ {k} {lbmClassLabel(k)}</option>
|
||||
{/each}
|
||||
</NativeSelect>
|
||||
<NativeSelect name="source" value={data.filters.source} aria-label="Source">
|
||||
<option value="">all sources</option>
|
||||
{#each data.sources as row (row.source)}
|
||||
@@ -141,6 +191,7 @@
|
||||
<option value="price">cheapest</option>
|
||||
<option value="ppm2">cheapest per m²</option>
|
||||
<option value="area">biggest</option>
|
||||
<option value="lbm">best neighbourhood</option>
|
||||
</NativeSelect>
|
||||
<NativeSelect name="status" value={data.filters.status} aria-label="My status">
|
||||
<option value="">any status</option>
|
||||
@@ -224,6 +275,32 @@
|
||||
.filter(Boolean)
|
||||
.join(' · ')}
|
||||
</div>
|
||||
{#if listing.lbmClass != null}
|
||||
{@const detail = listing.lbmDetail}
|
||||
<div class="mt-1 flex flex-wrap items-center gap-2 text-xs">
|
||||
<span
|
||||
class="inline-flex cursor-help items-center gap-1 rounded px-1.5 py-0.5 font-medium {lbmColour(
|
||||
listing.lbmClass
|
||||
)}"
|
||||
title={lbmTooltip(listing)}
|
||||
>
|
||||
{listing.lbmClass}/9 {lbmClassLabel(listing.lbmClass)}
|
||||
</span>
|
||||
<span class="text-muted-foreground" title={lbmTooltip(listing)}>
|
||||
{listing.lbmBuurtName ?? ''}{listing.lbmOnvClass != null
|
||||
? ` · overlast ${listing.lbmOnvClass}/9`
|
||||
: ''}{listing.lbmPrecision !== 'address' ? ' · approx.' : ''}
|
||||
</span>
|
||||
{#if detail?.lat != null && detail?.lng != null}
|
||||
<a
|
||||
href={lbmMapUrl(detail.lat, detail.lng)}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
class="text-muted-foreground underline-offset-2 hover:underline">map</a
|
||||
>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
<div
|
||||
class="mt-1 flex flex-wrap items-center gap-x-3 gap-y-1 text-xs text-muted-foreground"
|
||||
>
|
||||
|
||||
Reference in New Issue
Block a user