feat: house tracker with home worker, alerts and listing map

Collects Pararius rent/buy listings into Postgres via a worker that runs on a residential connection, sends Discord/ntfy alerts for new listings matching rules, and shows an admin-only map, market stats and worker log. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013HfF5tnChi8tkCx9svJP6D
2026-10-11 22:04:15 +02:00
parent 0ee9ed0abb
commit 0846cf7d77
30 changed files with 4269 additions and 1 deletions
--- a/scripts/deploy-remote.sh
+++ b/scripts/deploy-remote.sh
@@ -68,6 +68,7 @@ run_database_script scripts/migrate-foundation.ts
 run_database_script scripts/migrate-money-tracker.ts
 run_database_script scripts/migrate-game-tracker.ts
 run_database_script scripts/migrate-tracker-journal.ts
+run_database_script scripts/migrate-house-tracker.ts
 run_database_script scripts/seed-catalog.ts
 run_database_script scripts/seed-cod-catalog.ts
 run_database_script scripts/seed-pokemon-research.ts
--- a/scripts/house-tracker-migration.sql
+++ b/scripts/house-tracker-migration.sql
@@ -0,0 +1,111 @@
+-- House tracker: listings from Dutch housing sites, collected by a home worker.
+-- Idempotent; safe to run on every deploy.
+
+CREATE TABLE IF NOT EXISTS house_listings (
+  id serial PRIMARY KEY,
+  source text NOT NULL,
+  source_id text NOT NULL,
+  external_id text,
+  url text NOT NULL,
+  offer text NOT NULL CHECK (offer IN ('rent', 'buy')),
+  property_type text NOT NULL,
+  title text,
+  street text,
+  postal_code text,
+  city text,
+  city_slug text,
+  neighbourhood text,
+  price_eur integer CHECK (price_eur IS NULL OR price_eur >= 0),
+  area_m2 integer,
+  rooms integer,
+  bedrooms integer,
+  bathrooms integer,
+  construction_year integer,
+  energy_label text,
+  interior text,
+  deposit_eur integer,
+  available_from text,
+  contract_type text,
+  agent text,
+  image_url text,
+  description text,
+  features jsonb NOT NULL DEFAULT '{}'::jsonb,
+  lat double precision,
+  lng double precision,
+  -- When the site says it was published. first_seen_at is when we noticed.
+  offered_at timestamptz,
+  first_seen_at timestamptz NOT NULL DEFAULT now(),
+  -- fast, slow, or seed (already listed when its search was first polled).
+  first_seen_lane text NOT NULL DEFAULT 'slow',
+  last_seen_at timestamptz NOT NULL DEFAULT now(),
+  detail_fetched_at timestamptz,
+  removed_at timestamptz,
+  alerted_at timestamptz,
+  -- Personal tracking.
+  status text,
+  notes text
+);
+CREATE UNIQUE INDEX IF NOT EXISTS house_listings_source_idx ON house_listings (source, source_id);
+CREATE INDEX IF NOT EXISTS house_listings_active_idx
+  ON house_listings (offer, first_seen_at DESC) WHERE removed_at IS NULL;
+CREATE INDEX IF NOT EXISTS house_listings_city_idx ON house_listings (source, offer, city_slug);
+CREATE INDEX IF NOT EXISTS house_listings_details_idx
+  ON house_listings (first_seen_at DESC) WHERE detail_fetched_at IS NULL;
+
+-- Every price we have seen, so price drops and market trends survive edits.
+CREATE TABLE IF NOT EXISTS house_price_history (
+  listing_id integer NOT NULL REFERENCES house_listings(id) ON DELETE CASCADE,
+  observed_at timestamptz NOT NULL DEFAULT now(),
+  price_eur integer NOT NULL
+);
+CREATE INDEX IF NOT EXISTS house_price_history_listing_idx
+  ON house_price_history (listing_id, observed_at);
+
+-- One row per polled search URL. The first sync of a search only seeds the
+-- database: alerting on everything already listed would be noise.
+CREATE TABLE IF NOT EXISTS house_searches (
+  search_url text PRIMARY KEY,
+  source text NOT NULL,
+  lane text NOT NULL,
+  first_synced_at timestamptz NOT NULL DEFAULT now(),
+  last_synced_at timestamptz NOT NULL DEFAULT now(),
+  last_found integer NOT NULL DEFAULT 0
+);
+
+-- Worker health: challenges, HTTP errors, parse failures.
+CREATE TABLE IF NOT EXISTS house_worker_runs (
+  id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+  source text NOT NULL,
+  lane text NOT NULL,
+  url text NOT NULL,
+  status text NOT NULL,
+  http_status integer,
+  found integer,
+  duration_ms integer,
+  message text,
+  created_at timestamptz NOT NULL DEFAULT now()
+);
+CREATE INDEX IF NOT EXISTS house_worker_runs_created_idx ON house_worker_runs (created_at DESC);
+
+CREATE TABLE IF NOT EXISTS house_alert_rules (
+  id serial PRIMARY KEY,
+  name text NOT NULL,
+  enabled boolean NOT NULL DEFAULT true,
+  offer text NOT NULL DEFAULT 'rent' CHECK (offer IN ('rent', 'buy')),
+  min_price integer,
+  max_price integer,
+  min_area integer,
+  min_bedrooms integer,
+  -- Lowercase city names; empty means anywhere.
+  cities text[] NOT NULL DEFAULT '{}',
+  -- Empty means every property type.
+  property_types text[] NOT NULL DEFAULT '{}',
+  created_at timestamptz NOT NULL DEFAULT now()
+);
+
+INSERT INTO house_alert_rules (name, offer, min_price, max_price, cities)
+SELECT 'Gouda area rentals', 'rent', 1300, 1700, ARRAY[
+  'gouda', 'waddinxveen', 'reeuwijk', 'moordrecht', 'nieuwerkerk aan den ijssel',
+  'zevenhuizen', 'bodegraven', 'boskoop', 'oudewater', 'schoonhoven', 'haastrecht'
+]
+WHERE NOT EXISTS (SELECT 1 FROM house_alert_rules);
--- a/scripts/house-worker.ts
+++ b/scripts/house-worker.ts
@@ -0,0 +1,383 @@
+/**
+ * House tracker home worker. Runs on a machine at home (residential IP), polls
+ * Pararius politely and POSTs what it finds to the website, which stores it and
+ * sends the alerts. See docs/house-tracker.md.
+ *
+ *   HOUSE_API_URL=https://stanrunge.dev HOUSE_WORKER_TOKEN=... bun scripts/house-worker.ts
+ *   bun scripts/house-worker.ts --once     # one fast-lane poll, then exit
+ *   bun scripts/house-worker.ts --dry-run  # fetch + parse, print, send nothing
+ *
+ * Politeness rules, in order of importance:
+ * - one request at a time, at least HOUSE_MIN_GAP_SECONDS apart (with jitter)
+ * - a daily request budget
+ * - any challenge / 403 / 429 stops everything and backs off exponentially
+ * - the fast lane only polls often on weekdays 07:00–18:00 Amsterdam time
+ */
+import {
+	isChallenge,
+	parseDetailPage,
+	parseSearchPage,
+	searchPageUrl
+} from '../src/lib/house-tracker/pararius';
+import { backoffMs, nextDelayMs } from '../src/lib/house-tracker/schedule';
+import type {
+	Lane,
+	ListingDetail,
+	ReportRequest,
+	RunStatus,
+	SyncRequest,
+	SyncResponse
+} from '../src/lib/house-tracker/types';
+
+const args = new Set(process.argv.slice(2));
+const once = args.has('--once');
+const dryRun = args.has('--dry-run') || process.env.HOUSE_DRY_RUN === '1';
+
+const list = (value: string | undefined, fallback: string[]) =>
+	value
+		? value
+				.split(/[\s,]+/)
+				.map((item) => item.trim())
+				.filter(Boolean)
+		: fallback;
+const number = (value: string | undefined, fallback: number) =>
+	value && Number.isFinite(Number(value)) ? Number(value) : fallback;
+
+const config = {
+	api: (process.env.HOUSE_API_URL ?? '').replace(/\/$/, ''),
+	token: process.env.HOUSE_WORKER_TOKEN ?? '',
+	fastSearches: list(process.env.HOUSE_FAST_SEARCHES, [
+		'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
+	]),
+	slowSearches: list(
+		process.env.HOUSE_SLOW_SEARCHES,
+		['gouda', 'rotterdam', 'den-haag', 'utrecht', 'amsterdam', 'eindhoven'].flatMap((city) => [
+			`https://www.pararius.nl/huurwoningen/${city}`,
+			`https://www.pararius.nl/koopwoningen/${city}`
+		])
+	),
+	slowEveryHours: number(process.env.HOUSE_SLOW_EVERY_HOURS, 12),
+	slowMaxPages: number(process.env.HOUSE_SLOW_MAX_PAGES, 80),
+	minGapMs: number(process.env.HOUSE_MIN_GAP_SECONDS, 8) * 1000,
+	dailyBudget: number(process.env.HOUSE_DAILY_BUDGET, 1500),
+	fetchMode: (process.env.HOUSE_FETCH_MODE ?? 'fetch') as 'fetch' | 'browser',
+	userAgent:
+		process.env.HOUSE_USER_AGENT ??
+		'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
+};
+
+if (!dryRun && (!config.api || config.token.length < 32))
+	throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required');
+
+const SOURCE = 'pararius';
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+const log = (...parts: unknown[]) => console.log(new Date().toISOString(), ...parts);
+
+// ---------------------------------------------------------------------------
+// Polite fetching
+
+class Challenged extends Error {
+	constructor(
+		readonly url: string,
+		readonly status: number
+	) {
+		super(`Challenged (${status}) on ${url}`);
+	}
+}
+
+const cookies = new Map<string, string>();
+let lastRequestAt = 0;
+let budgetDay = '';
+let budgetUsed = 0;
+
+function remember(response: Response) {
+	for (const header of response.headers.getSetCookie?.() ?? []) {
+		const [pair] = header.split(';');
+		const index = pair.indexOf('=');
+		if (index > 0) cookies.set(pair.slice(0, index).trim(), pair.slice(index + 1).trim());
+	}
+}
+
+function resetBudget() {
+	const day = new Date().toISOString().slice(0, 10);
+	if (day !== budgetDay) {
+		budgetDay = day;
+		budgetUsed = 0;
+	}
+}
+
+/** The slow lane may only use part of the budget so the fast lane never runs dry. */
+function slowBudgetLeft() {
+	resetBudget();
+	return budgetUsed < config.dailyBudget * 0.6;
+}
+
+async function waitForTurn() {
+	resetBudget();
+	if (budgetUsed >= config.dailyBudget) throw new Error('Daily request budget used up');
+	const gap = config.minGapMs * (0.8 + Math.random() * 0.7);
+	const wait = lastRequestAt + gap - Date.now();
+	if (wait > 0) await sleep(wait);
+	lastRequestAt = Date.now();
+	budgetUsed++;
+}
+
+type BrowserPage = {
+	goto(url: string, options: object): Promise<{ status(): number } | null>;
+	content(): Promise<string>;
+};
+let browserPage: Promise<BrowserPage> | null = null;
+
+async function openBrowser(): Promise<BrowserPage> {
+	// Real Chromium for when plain requests get challenged. Needs
+	// `npx playwright install chromium` on the worker machine.
+	const { chromium } = await import('@playwright/test');
+	const context = await chromium.launchPersistentContext(
+		process.env.HOUSE_BROWSER_PROFILE ?? '.house-worker-profile',
+		{ headless: true, locale: 'nl-NL', timezoneId: 'Europe/Amsterdam' }
+	);
+	return context.pages()[0] ?? (await context.newPage());
+}
+
+async function getHtml(url: string, referer?: string): Promise<{ html: string; status: number }> {
+	await waitForTurn();
+	if (config.fetchMode === 'browser') {
+		browserPage ??= openBrowser();
+		const page = await browserPage;
+		const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45_000 });
+		const html = await page.content();
+		const status = response?.status() ?? 0;
+		if (isChallenge(html, status)) throw new Challenged(url, status);
+		return { html, status };
+	}
+	const response = await fetch(url, {
+		redirect: 'follow',
+		signal: AbortSignal.timeout(30_000),
+		headers: {
+			'user-agent': config.userAgent,
+			accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+			'accept-language': 'nl-NL,nl;q=0.9,en-US;q=0.8,en;q=0.7',
+			'cache-control': 'no-cache',
+			...(referer ? { referer } : {}),
+			...(cookies.size
+				? { cookie: [...cookies].map(([key, value]) => `${key}=${value}`).join('; ') }
+				: {})
+		}
+	});
+	remember(response);
+	const html = await response.text();
+	if (isChallenge(html, response.status)) throw new Challenged(url, response.status);
+	return { html, status: response.status };
+}
+
+// ---------------------------------------------------------------------------
+// Website API
+
+async function api<T>(path: string, body: unknown): Promise<T | null> {
+	if (dryRun) return null;
+	for (let attempt = 0; attempt < 4; attempt++) {
+		try {
+			const response = await fetch(`${config.api}/projects/house-tracker/api/worker/${path}`, {
+				method: 'POST',
+				headers: { 'content-type': 'application/json', authorization: `Bearer ${config.token}` },
+				body: JSON.stringify(body),
+				signal: AbortSignal.timeout(30_000)
+			});
+			if (response.ok) return (await response.json()) as T;
+			const text = await response.text();
+			if (response.status < 500)
+				throw new Error(`API ${path} ${response.status}: ${text.slice(0, 300)}`);
+			log(`API ${path} ${response.status}, retrying`);
+		} catch (error) {
+			if (attempt === 3) throw error;
+			log(`API ${path} failed: ${(error as Error).message}, retrying`);
+		}
+		await sleep(5000 * 2 ** attempt);
+	}
+	return null;
+}
+
+async function report(
+	lane: Lane,
+	url: string,
+	status: RunStatus,
+	extra: Partial<ReportRequest> = {}
+) {
+	try {
+		await api('report', { source: SOURCE, lane, url, status, ...extra });
+	} catch (error) {
+		log('Could not report:', (error as Error).message);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Lanes
+
+async function fetchDetails(
+	lane: Lane,
+	wanted: { sourceId: string; url: string }[],
+	referer: string
+) {
+	const details: ListingDetail[] = [];
+	for (const item of wanted) {
+		const { html } = await getHtml(item.url, referer);
+		const detail = parseDetailPage(html, item.url);
+		if (detail) details.push(detail);
+		else await report(lane, item.url, 'parse-error', { message: 'listing page did not parse' });
+	}
+	if (dryRun) {
+		for (const detail of details) {
+			log('detail', { ...detail, description: undefined, features: undefined });
+		}
+	}
+	if (details.length) await api('details', { source: SOURCE, details });
+}
+
+async function pollSearch(
+	lane: Lane,
+	url: string,
+	completedSweep?: SyncRequest['completedSweep']
+): Promise<{ nextPageUrl: string | null; found: number }> {
+	const startedAt = new Date().toISOString();
+	const { html } = await getHtml(url);
+	const page = parseSearchPage(html, url);
+	if (!page.cards.length && page.nextPageUrl)
+		await report(lane, url, 'parse-error', {
+			message: 'no listing cards on a page with a next page'
+		});
+	if (dryRun) {
+		log(`${lane} ${url}: ${page.cards.length} cards, next=${page.nextPageUrl}`);
+		for (const card of page.cards.slice(0, 5)) log('card', card);
+	}
+	const sync = await api<SyncResponse>('sync', {
+		source: SOURCE,
+		lane,
+		searchUrl: url,
+		startedAt,
+		cards: page.cards,
+		completedSweep: page.nextPageUrl ? null : (completedSweep ?? null)
+	} satisfies SyncRequest);
+	if (sync) {
+		if (sync.newIds.length || sync.alerted)
+			log(`${lane} ${url}: ${sync.newIds.length} new, ${sync.alerted} alerted`);
+		if (sync.needDetails.length) await fetchDetails(lane, sync.needDetails, url);
+	} else if (dryRun && page.cards[0]) {
+		await fetchDetails(lane, [page.cards[0]], url);
+	}
+	return { nextPageUrl: page.nextPageUrl, found: page.cards.length };
+}
+
+async function fastTick() {
+	for (const url of config.fastSearches) await pollSearch('fast', url);
+}
+
+/** Only an unfiltered `/huurwoningen/<city>` sweep can tell us what disappeared. */
+function sweepScope(url: string) {
+	const [kind, city, ...rest] = new URL(url).pathname.split('/').filter(Boolean);
+	if (rest.length || !city) return null;
+	if (kind === 'huurwoningen') return { offer: 'rent' as const, city };
+	if (kind === 'koopwoningen') return { offer: 'buy' as const, city };
+	return null;
+}
+
+type Sweep = { index: number; page: number; startedAt: string };
+let sweep: Sweep | null = null;
+
+/** One page of the slow lane, so fast polls can interleave. Returns false when idle. */
+async function slowStep(): Promise<boolean> {
+	if (!sweep || !slowBudgetLeft()) return false;
+	const base = config.slowSearches[sweep.index];
+	if (!base) {
+		log('Slow sweep finished');
+		sweep = null;
+		return false;
+	}
+	const url = searchPageUrl(base, sweep.page);
+	const scope = sweepScope(base);
+	const result = await pollSearch(
+		'slow',
+		url,
+		scope ? { ...scope, sweepStartedAt: sweep.startedAt } : null
+	);
+	if (result.nextPageUrl && sweep.page < config.slowMaxPages) {
+		sweep.page++;
+	} else {
+		sweep = { index: sweep.index + 1, page: 1, startedAt: new Date().toISOString() };
+	}
+	return true;
+}
+
+// ---------------------------------------------------------------------------
+// Main loop
+
+let stopping = false;
+process.on('SIGTERM', () => (stopping = true));
+process.on('SIGINT', () => (stopping = true));
+
+async function guarded(
+	lane: Lane,
+	step: () => Promise<boolean | void>
+): Promise<'ok' | 'idle' | 'failed'> {
+	const started = Date.now();
+	try {
+		return (await step()) === false ? 'idle' : 'ok';
+	} catch (error) {
+		if (error instanceof Challenged) {
+			log(`CHALLENGED on ${error.url} (${error.status})`);
+			await report(lane, error.url, 'challenged', {
+				httpStatus: error.status,
+				durationMs: Date.now() - started
+			});
+		} else {
+			log(`${lane} failed:`, (error as Error).message);
+			await report(lane, config.fastSearches[0], 'network-error', {
+				message: (error as Error).message.slice(0, 500),
+				durationMs: Date.now() - started
+			});
+		}
+		return 'failed';
+	}
+}
+
+async function main() {
+	log(
+		`house worker: ${config.fastSearches.length} fast, ${config.slowSearches.length} slow searches, mode=${config.fetchMode}${dryRun ? ', dry run' : ''}`
+	);
+	if (once) {
+		const result = await guarded('fast', fastTick);
+		process.exit(result === 'ok' ? 0 : 1);
+	}
+
+	let failures = 0;
+	let nextFast = Date.now();
+	let nextSlow = Date.now() + 5 * 60_000; // let the fast lane go first
+	while (!stopping) {
+		const now = Date.now();
+		let result: 'ok' | 'idle' | 'failed' = 'idle';
+		if (now >= nextFast) {
+			result = await guarded('fast', fastTick);
+			nextFast = Date.now() + nextDelayMs(new Date());
+		} else if (sweep || now >= nextSlow) {
+			if (!sweep) {
+				log('Slow sweep starting');
+				sweep = { index: 0, page: 1, startedAt: new Date().toISOString() };
+				nextSlow = now + config.slowEveryHours * 60 * 60_000;
+			}
+			result = await guarded('slow', slowStep);
+		}
+		if (result === 'failed') {
+			failures++;
+			const pause = backoffMs(failures);
+			log(`Backing off ${Math.round(pause / 60_000)} min (failure ${failures})`);
+			nextFast = Math.max(nextFast, Date.now() + pause);
+			await sleep(pause);
+			continue;
+		}
+		if (result === 'ok') failures = 0;
+		else await sleep(Math.min(30_000, Math.max(5_000, nextFast - Date.now())));
+	}
+	log('Stopped');
+	process.exit(0);
+}
+
+await main();
--- a/scripts/migrate-house-tracker.ts
+++ b/scripts/migrate-house-tracker.ts
@@ -0,0 +1,15 @@
+import { readFile } from 'node:fs/promises';
+import postgres from 'postgres';
+
+if (!process.env.DB_URL) throw new Error('DB_URL is required');
+const client = postgres(process.env.DB_URL, { max: 1 });
+try {
+	await client.begin(async (tx) => {
+		await tx.unsafe(
+			await readFile(new URL('./house-tracker-migration.sql', import.meta.url), 'utf8')
+		);
+	});
+	console.log('House tracker migration applied');
+} finally {
+	await client.end();
+}