website/scripts/house-scrape.ts

// Nightly house scraper, run on stan-vm by a systemd user timer (deploy/systemd/house-scrape.*).
// Fetches the sources that allow it, writes house.listings directly and pushes an ntfy alert
// for new listings that match an alert rule. See docs/house-tracker.md.
//
//   bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet]
//
// Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed
// paths only. A 403/429/challenge stops that source for the night; nothing is retried or
// worked around. Funda, Pararius, huurwoningen.nl and Huurstunt challenge this server's
// (datacenter) IP, so they are not scraped from here.
import postgres from 'postgres';
import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
import { citySlug } from '../src/lib/house-tracker/address';
import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
import { isChallenge } from '../src/lib/house-tracker/pararius';
import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
import {
	parseVestedaSearch,
	VESTEDA_SEARCH_URL,
	vestedaSearchBody
} from '../src/lib/house-tracker/vesteda';

const args = process.argv.slice(2);
const dryRun = args.includes('--dry-run');
const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null;

const USER_AGENT =
	'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
const KAMERNET_CITIES = (
	process.env.HOUSE_KAMERNET_CITIES ??
	'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
)
	.split(',')
	.map((c) => c.trim())
	.filter(Boolean);
const KAMERNET_MAX_PAGES = 5;
// Gouda centre; Vesteda's own search uses km radius around a point.
const VESTEDA = {
	lat: 52.0115,
	lng: 4.7105,
	radiusKm: Number(process.env.HOUSE_VESTEDA_RADIUS_KM ?? 20),
	place: 'Gouda'
};
const MAX_ALERTS = 5;

const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
if (!dbUrl && !dryRun) throw new Error('DB_URL is not set');
const sql = dbUrl ? postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;

class Blocked extends Error {
	constructor(
		public status: number,
		public url: string
	) {
		super(`blocked (${status}) on ${url}`);
	}
}

let lastRequest = 0;
async function politeFetch(url: string, init: RequestInit = {}): Promise<Response> {
	const wait = lastRequest + 6000 + Math.random() * 4000 - Date.now();
	if (wait > 0) await Bun.sleep(wait);
	lastRequest = Date.now();
	const response = await fetch(url, {
		...init,
		redirect: 'follow',
		signal: AbortSignal.timeout(30_000),
		headers: {
			'user-agent': USER_AGENT,
			'accept-language': 'nl-NL,nl;q=0.9,en;q=0.5',
			...(init.headers ?? {})
		}
	});
	return response;
}

async function logRun(
	source: string,
	url: string,
	status: string,
	extra: { httpStatus?: number; found?: number; durationMs?: number; message?: string } = {}
) {
	console.log(
		`${source} ${status}${extra.httpStatus ? ` ${extra.httpStatus}` : ''} ${extra.found ?? ''} ${url}${extra.message ? ` (${extra.message})` : ''}`
	);
	if (!sql || dryRun) return;
	await sql`INSERT INTO house.worker_runs (source, lane, url, status, http_status, found, duration_ms, message)
		VALUES (${source}, 'nightly', ${url}, ${status}, ${extra.httpStatus ?? null}, ${extra.found ?? null},
			${extra.durationMs ?? null}, ${extra.message?.slice(0, 500) ?? null})`;
}

type SourceResult = { listings: ScrapedListing[]; completed: boolean };

async function scrapeKamernet(): Promise<SourceResult> {
	const listings: ScrapedListing[] = [];
	let completed = true;
	for (const city of KAMERNET_CITIES) {
		for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) {
			const url = kamernetSearchUrl(city, page);
			const started = Date.now();
			const response = await politeFetch(url, { headers: { accept: 'text/html' } });
			const html = await response.text();
			if (isChallenge(html, response.status)) {
				await logRun('kamernet', url, 'challenged', { httpStatus: response.status });
				throw new Blocked(response.status, url);
			}
			if (response.status === 404) {
				// Kamernet has no page for some small places; that is not an error.
				await logRun('kamernet', url, 'ok', {
					httpStatus: 404,
					found: 0,
					message: 'no such city page'
				});
				break;
			}
			if (!response.ok) {
				await logRun('kamernet', url, 'http-error', { httpStatus: response.status });
				completed = false;
				break;
			}
			try {
				const result = parseKamernetSearch(html);
				// Kamernet widens the search to nearby places; keep each city's own listings once.
				listings.push(...result.listings);
				await logRun('kamernet', url, 'ok', {
					httpStatus: 200,
					found: result.listings.length,
					durationMs: Date.now() - started
				});
				const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1;
				if (page >= pages) break;
			} catch (error) {
				await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) });
				completed = false;
				break;
			}
		}
	}
	return { listings, completed };
}

async function scrapeVesteda(): Promise<SourceResult> {
	const started = Date.now();
	const response = await politeFetch(VESTEDA_SEARCH_URL, {
		method: 'POST',
		headers: { 'content-type': 'application/json', accept: 'application/json' },
		body: JSON.stringify(vestedaSearchBody(VESTEDA))
	});
	const text = await response.text();
	if (isChallenge(text, response.status)) {
		await logRun('vesteda', VESTEDA_SEARCH_URL, 'challenged', { httpStatus: response.status });
		throw new Blocked(response.status, VESTEDA_SEARCH_URL);
	}
	if (!response.ok) {
		await logRun('vesteda', VESTEDA_SEARCH_URL, 'http-error', { httpStatus: response.status });
		return { listings: [], completed: false };
	}
	try {
		const listings = parseVestedaSearch(JSON.parse(text));
		await logRun('vesteda', VESTEDA_SEARCH_URL, 'ok', {
			httpStatus: 200,
			found: listings.length,
			durationMs: Date.now() - started
		});
		return { listings, completed: true };
	} catch (error) {
		await logRun('vesteda', VESTEDA_SEARCH_URL, 'parse-error', {
			httpStatus: 200,
			message: String(error)
		});
		return { listings: [], completed: false };
	}
}

async function ntfy(title: string, body: string, click: string) {
	const target = process.env.HOUSE_NTFY_URL;
	if (!target) return false;
	const url = new URL(target);
	const response = await fetch(url.origin, {
		method: 'POST',
		headers: {
			'content-type': 'application/json',
			...(process.env.HOUSE_NTFY_TOKEN
				? { authorization: `Bearer ${process.env.HOUSE_NTFY_TOKEN}` }
				: {})
		},
		body: JSON.stringify({
			topic: url.pathname.replace(/^\/+|\/+$/g, ''),
			title,
			message: body,
			click,
			tags: ['house'],
			priority: 3
		}),
		signal: AbortSignal.timeout(10_000)
	}).catch(() => null);
	return Boolean(response?.ok);
}

const sources: Record<string, () => Promise<SourceResult>> = {
	kamernet: scrapeKamernet,
	vesteda: scrapeVesteda
};
const fresh: { id: number; source: string; listing: ScrapedListing }[] = [];
let failures = 0;
try {
	for (const [name, scrape] of Object.entries(sources)) {
		if (only && only !== name) continue;
		const runStartedAt = new Date();
		let result: SourceResult;
		try {
			result = await scrape();
		} catch (error) {
			failures++;
			console.error(
				`${name}: ${error instanceof Error ? error.message : error}; skipped until tomorrow`
			);
			continue;
		}
		if (dryRun || !sql) {
			for (const l of result.listings)
				console.log(
					`  ${l.sourceId} €${l.priceEur ?? '?'} ${l.areaM2 ?? '?'}m² ${l.title} (${l.city}) ${l.url}`
				);
			continue;
		}
		// An empty result from a "complete" run is more likely a broken page than an empty market.
		const completed = result.completed && result.listings.length > 0;
		const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed });
		console.log(
			`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated, ${ingest.removed} gone`
		);
		if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
	}

	if (sql && !dryRun) {
		const { keyed, copies } = await dedupe(sql);
		console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);

		const rules =
			(await sql`SELECT id, name, enabled, offer, min_price AS "minPrice", max_price AS "maxPrice",
			min_area AS "minArea", min_bedrooms AS "minBedrooms", cities, property_types AS "propertyTypes"
			FROM house.alert_rules WHERE enabled`) as unknown as AlertRule[];
		const copyIds = new Set(
			fresh.length
				? (
						await sql<
							{ id: number }[]
						>`SELECT id FROM house.listings WHERE duplicate_of IS NOT NULL AND id IN ${sql(fresh.map((f) => f.id))}`
					).map((r) => r.id)
				: []
		);
		const matches = fresh.flatMap((f) => {
			if (copyIds.has(f.id)) return [];
			const listing = { ...f.listing, citySlug: citySlug(f.listing.city), source: f.source };
			const rule = rules.find((r) => matchesRule(listing, r));
			return rule ? [{ ...f, message: alertMessage(listing, rule) }] : [];
		});
		let sent = 0;
		for (const match of matches.slice(0, MAX_ALERTS)) {
			if (await ntfy(match.message.title, match.message.body, match.message.url)) {
				sent++;
				await sql`UPDATE house.listings SET alerted_at = now() WHERE id = ${match.id}`;
			}
		}
		if (matches.length > MAX_ALERTS)
			await ntfy(
				`🏠 ${matches.length - MAX_ALERTS} more new listings`,
				'See the house tracker.',
				'https://stanrunge.dev/trackers/houses?status='
			);
		console.log(
			`alerts: ${matches.length} new matches, ${sent} sent${process.env.HOUSE_NTFY_URL ? '' : ' (HOUSE_NTFY_URL not set)'}`
		);
	}
} finally {
	await sql?.end();
}

// Non-zero so systemd shows the unit as failed when a source broke or blocked us.
if (failures > 0) process.exitCode = 1;