houses: nightly Kamernet + Vesteda scraper, dedupe by address, owner-only /trackers/houses

- scripts/house-scrape.ts: polite nightly scrape (honest UA, 6-10 s apart, stops a source on 403/challenge), ingest + price history + removals, ntfy alerts on a separate topic (max 5 + summary, first run seeds) - address_key + duplicate_of: copies on another source link to one listing, manual wins - /projects/house-tracker -> /trackers/houses (308), owner only (404 otherwise); filters for min m2, rooms, source, Gouda region default; notes, add-by-hand form, total rent, also-on links - scripts/house-db.sh: run the house-listing CLI as the limited house_cli role - deploy/systemd/house-scrape.{service,timer}
2026-10-24 14:07:33 +00:00
parent ea7d64a556
commit b51fef14fd
24 changed files with 18142 additions and 35 deletions
--- a/scripts/house-scrape.ts
+++ b/scripts/house-scrape.ts
@@ -0,0 +1,282 @@
+// Nightly house scraper, run on stan-vm by a systemd user timer (deploy/systemd/house-scrape.*).
+// Fetches the sources that allow it, writes house.listings directly and pushes an ntfy alert
+// for new listings that match an alert rule. See docs/house-tracker.md.
+//
+//   bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet]
+//
+// Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed
+// paths only. A 403/429/challenge stops that source for the night; nothing is retried or
+// worked around. Funda, Pararius, huurwoningen.nl and Huurstunt challenge this server's
+// (datacenter) IP, so they are not scraped from here.
+import postgres from 'postgres';
+import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
+import { citySlug } from '../src/lib/house-tracker/address';
+import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
+import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
+import { isChallenge } from '../src/lib/house-tracker/pararius';
+import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
+import {
+	parseVestedaSearch,
+	VESTEDA_SEARCH_URL,
+	vestedaSearchBody
+} from '../src/lib/house-tracker/vesteda';
+
+const args = process.argv.slice(2);
+const dryRun = args.includes('--dry-run');
+const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null;
+
+const USER_AGENT =
+	'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
+const KAMERNET_CITIES = (
+	process.env.HOUSE_KAMERNET_CITIES ??
+	'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
+)
+	.split(',')
+	.map((c) => c.trim())
+	.filter(Boolean);
+const KAMERNET_MAX_PAGES = 5;
+// Gouda centre; Vesteda's own search uses km radius around a point.
+const VESTEDA = {
+	lat: 52.0115,
+	lng: 4.7105,
+	radiusKm: Number(process.env.HOUSE_VESTEDA_RADIUS_KM ?? 20),
+	place: 'Gouda'
+};
+const MAX_ALERTS = 5;
+
+const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
+if (!dbUrl && !dryRun) throw new Error('DB_URL is not set');
+const sql = dbUrl ? postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
+
+class Blocked extends Error {
+	constructor(
+		public status: number,
+		public url: string
+	) {
+		super(`blocked (${status}) on ${url}`);
+	}
+}
+
+let lastRequest = 0;
+async function politeFetch(url: string, init: RequestInit = {}): Promise<Response> {
+	const wait = lastRequest + 6000 + Math.random() * 4000 - Date.now();
+	if (wait > 0) await Bun.sleep(wait);
+	lastRequest = Date.now();
+	const response = await fetch(url, {
+		...init,
+		redirect: 'follow',
+		signal: AbortSignal.timeout(30_000),
+		headers: {
+			'user-agent': USER_AGENT,
+			'accept-language': 'nl-NL,nl;q=0.9,en;q=0.5',
+			...(init.headers ?? {})
+		}
+	});
+	return response;
+}
+
+async function logRun(
+	source: string,
+	url: string,
+	status: string,
+	extra: { httpStatus?: number; found?: number; durationMs?: number; message?: string } = {}
+) {
+	console.log(
+		`${source} ${status}${extra.httpStatus ? ` ${extra.httpStatus}` : ''} ${extra.found ?? ''} ${url}${extra.message ? ` (${extra.message})` : ''}`
+	);
+	if (!sql || dryRun) return;
+	await sql`INSERT INTO house.worker_runs (source, lane, url, status, http_status, found, duration_ms, message)
+		VALUES (${source}, 'nightly', ${url}, ${status}, ${extra.httpStatus ?? null}, ${extra.found ?? null},
+			${extra.durationMs ?? null}, ${extra.message?.slice(0, 500) ?? null})`;
+}
+
+type SourceResult = { listings: ScrapedListing[]; completed: boolean };
+
+async function scrapeKamernet(): Promise<SourceResult> {
+	const listings: ScrapedListing[] = [];
+	let completed = true;
+	for (const city of KAMERNET_CITIES) {
+		for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) {
+			const url = kamernetSearchUrl(city, page);
+			const started = Date.now();
+			const response = await politeFetch(url, { headers: { accept: 'text/html' } });
+			const html = await response.text();
+			if (isChallenge(html, response.status)) {
+				await logRun('kamernet', url, 'challenged', { httpStatus: response.status });
+				throw new Blocked(response.status, url);
+			}
+			if (response.status === 404) {
+				// Kamernet has no page for some small places; that is not an error.
+				await logRun('kamernet', url, 'ok', {
+					httpStatus: 404,
+					found: 0,
+					message: 'no such city page'
+				});
+				break;
+			}
+			if (!response.ok) {
+				await logRun('kamernet', url, 'http-error', { httpStatus: response.status });
+				completed = false;
+				break;
+			}
+			try {
+				const result = parseKamernetSearch(html);
+				// Kamernet widens the search to nearby places; keep each city's own listings once.
+				listings.push(...result.listings);
+				await logRun('kamernet', url, 'ok', {
+					httpStatus: 200,
+					found: result.listings.length,
+					durationMs: Date.now() - started
+				});
+				const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1;
+				if (page >= pages) break;
+			} catch (error) {
+				await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) });
+				completed = false;
+				break;
+			}
+		}
+	}
+	return { listings, completed };
+}
+
+async function scrapeVesteda(): Promise<SourceResult> {
+	const started = Date.now();
+	const response = await politeFetch(VESTEDA_SEARCH_URL, {
+		method: 'POST',
+		headers: { 'content-type': 'application/json', accept: 'application/json' },
+		body: JSON.stringify(vestedaSearchBody(VESTEDA))
+	});
+	const text = await response.text();
+	if (isChallenge(text, response.status)) {
+		await logRun('vesteda', VESTEDA_SEARCH_URL, 'challenged', { httpStatus: response.status });
+		throw new Blocked(response.status, VESTEDA_SEARCH_URL);
+	}
+	if (!response.ok) {
+		await logRun('vesteda', VESTEDA_SEARCH_URL, 'http-error', { httpStatus: response.status });
+		return { listings: [], completed: false };
+	}
+	try {
+		const listings = parseVestedaSearch(JSON.parse(text));
+		await logRun('vesteda', VESTEDA_SEARCH_URL, 'ok', {
+			httpStatus: 200,
+			found: listings.length,
+			durationMs: Date.now() - started
+		});
+		return { listings, completed: true };
+	} catch (error) {
+		await logRun('vesteda', VESTEDA_SEARCH_URL, 'parse-error', {
+			httpStatus: 200,
+			message: String(error)
+		});
+		return { listings: [], completed: false };
+	}
+}
+
+async function ntfy(title: string, body: string, click: string) {
+	const target = process.env.HOUSE_NTFY_URL;
+	if (!target) return false;
+	const url = new URL(target);
+	const response = await fetch(url.origin, {
+		method: 'POST',
+		headers: {
+			'content-type': 'application/json',
+			...(process.env.HOUSE_NTFY_TOKEN
+				? { authorization: `Bearer ${process.env.HOUSE_NTFY_TOKEN}` }
+				: {})
+		},
+		body: JSON.stringify({
+			topic: url.pathname.replace(/^\/+|\/+$/g, ''),
+			title,
+			message: body,
+			click,
+			tags: ['house'],
+			priority: 3
+		}),
+		signal: AbortSignal.timeout(10_000)
+	}).catch(() => null);
+	return Boolean(response?.ok);
+}
+
+const sources: Record<string, () => Promise<SourceResult>> = {
+	kamernet: scrapeKamernet,
+	vesteda: scrapeVesteda
+};
+const fresh: { id: number; source: string; listing: ScrapedListing }[] = [];
+let failures = 0;
+try {
+	for (const [name, scrape] of Object.entries(sources)) {
+		if (only && only !== name) continue;
+		const runStartedAt = new Date();
+		let result: SourceResult;
+		try {
+			result = await scrape();
+		} catch (error) {
+			failures++;
+			console.error(
+				`${name}: ${error instanceof Error ? error.message : error}; skipped until tomorrow`
+			);
+			continue;
+		}
+		if (dryRun || !sql) {
+			for (const l of result.listings)
+				console.log(
+					`  ${l.sourceId} €${l.priceEur ?? '?'} ${l.areaM2 ?? '?'}m² ${l.title} (${l.city}) ${l.url}`
+				);
+			continue;
+		}
+		// An empty result from a "complete" run is more likely a broken page than an empty market.
+		const completed = result.completed && result.listings.length > 0;
+		const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed });
+		console.log(
+			`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated, ${ingest.removed} gone`
+		);
+		if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
+	}
+
+	if (sql && !dryRun) {
+		const { keyed, copies } = await dedupe(sql);
+		console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);
+
+		const rules =
+			(await sql`SELECT id, name, enabled, offer, min_price AS "minPrice", max_price AS "maxPrice",
+			min_area AS "minArea", min_bedrooms AS "minBedrooms", cities, property_types AS "propertyTypes"
+			FROM house.alert_rules WHERE enabled`) as unknown as AlertRule[];
+		const copyIds = new Set(
+			fresh.length
+				? (
+						await sql<
+							{ id: number }[]
+						>`SELECT id FROM house.listings WHERE duplicate_of IS NOT NULL AND id IN ${sql(fresh.map((f) => f.id))}`
+					).map((r) => r.id)
+				: []
+		);
+		const matches = fresh.flatMap((f) => {
+			if (copyIds.has(f.id)) return [];
+			const listing = { ...f.listing, citySlug: citySlug(f.listing.city), source: f.source };
+			const rule = rules.find((r) => matchesRule(listing, r));
+			return rule ? [{ ...f, message: alertMessage(listing, rule) }] : [];
+		});
+		let sent = 0;
+		for (const match of matches.slice(0, MAX_ALERTS)) {
+			if (await ntfy(match.message.title, match.message.body, match.message.url)) {
+				sent++;
+				await sql`UPDATE house.listings SET alerted_at = now() WHERE id = ${match.id}`;
+			}
+		}
+		if (matches.length > MAX_ALERTS)
+			await ntfy(
+				`🏠 ${matches.length - MAX_ALERTS} more new listings`,
+				'See the house tracker.',
+				'https://stanrunge.dev/trackers/houses?status='
+			);
+		console.log(
+			`alerts: ${matches.length} new matches, ${sent} sent${process.env.HOUSE_NTFY_URL ? '' : ' (HOUSE_NTFY_URL not set)'}`
+		);
+	}
+} finally {
+	await sql?.end();
+}
+
+// Non-zero so systemd shows the unit as failed when a source broke or blocked us.
+if (failures > 0) process.exitCode = 1;