house-scrape: re-check huispedia, huislijn, huurportaal and rentola rows and mark clearly gone ones removed

deploy/systemd/house-scrape.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=House tracker nightly scrape (Kamernet, Vesteda -> house.listings)
+Description=House tracker nightly scrape (Kamernet, Vesteda, re-check of imported rows -> house.listings)
 Wants=network-online.target
 After=network-online.target
 

docs/house-tracker.md

@@ -19,6 +19,24 @@ night and is logged as `challenged`; nothing is retried or worked around.
 | Huurstunt              | Cloudflare challenge                                          | skipped                              |
 | Mijndak Midden-Holland | OutSystems app behind a login, no public listing feed         | skipped; use their own e-mail alerts |
 
+**Re-check of imported rows** (`src/lib/house-tracker/recheck.ts`). Rows added by the CLI or
+one-off backfills from huispedia, huislijn, huurportaal and rentola are not in any search
+feed, so after the sources the scrape fetches each row's own page (all four allow detail
+pages in robots.txt), up to 400 per site, oldest `last_seen_at` first, within
+`HOUSE_RECHECK_MINUTES` (default 8). Each site has its own one-at-a-time 6–10 s queue.
+Alive sets `last_seen_at`; gone sets `removed_at` and needs a clear signal:
+
+| site                 | gone when                                                                       |
+| -------------------- | ------------------------------------------------------------------------------- |
+| huurportaal, rentola | redirect to a search page with `?inactive_property_id=`, or 404/410             |
+| huislijn             | redirect to the province overview (id gone from the URL), 404/410, woningruil   |
+| huispedia            | 404/410 ("niet meer beschikbaar"), or Status verhuurd / verkocht / niet te koop |
+
+Errors, timeouts, off-site redirects and unfamiliar pages are `unknown` and change nothing; a
+403/429/503 or challenge stops that site for the night. If more than half of a site's checks
+in one night (min. 5) say gone, none are applied and the run is logged as `suspicious`.
+Try it with `--dry-run --source recheck --recheck-limit 5`.
+
 Same home on two sources: `address_key` (street + number + city, normalised, see
 `src/lib/house-tracker/address.ts`) links the copy to the original via `duplicate_of`; a
 manual entry wins. The page shows one row with "also on …" links. Kamernet never shows house

scripts/house-scrape.ts

@@ -3,6 +3,12 @@
 // for new listings that match an alert rule. See docs/house-tracker.md.
 //
 //   bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet]
+//       [--source recheck] [--recheck-limit 20]
+//
+// After the search sources it re-checks rows that came from one-off imports or the CLI
+// (huispedia, huislijn, huurportaal, rentola) by fetching each listing's own page, oldest
+// check first, within a time budget, and marks a row gone only on a clear signal (see
+// src/lib/house-tracker/recheck.ts). Each site gets its own one-at-a-time 6-10 s queue.
 //
 // Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed
 // paths only. A 403/429/challenge stops that source for the night; nothing is retried or
@@ -15,6 +21,14 @@ import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
 import { scoreListings } from '../src/lib/house-tracker/liveability';
 import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
 import { isChallenge } from '../src/lib/house-tracker/pararius';
+import {
+	classifyListingPage,
+	RECHECK_SITES,
+	RECHECK_URL_PATTERNS,
+	recheckSite,
+	removalsLookSane,
+	type RecheckSite
+} from '../src/lib/house-tracker/recheck';
 import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
 import {
 	parseVestedaSearch,
@@ -25,6 +39,13 @@ import {
 const args = process.argv.slice(2);
 const dryRun = args.includes('--dry-run');
 const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null;
+const recheckLimit = args.includes('--recheck-limit')
+	? Number(args[args.indexOf('--recheck-limit') + 1])
+	: 400;
+const scriptStarted = Date.now();
+// The unit has TimeoutStartSec=900; leave room for the search sources and the steps after.
+const RECHECK_MINUTES = Number(process.env.HOUSE_RECHECK_MINUTES ?? 8);
+const RECHECK_HARD_STOP_MS = 12 * 60_000;
 
 const USER_AGENT =
 	'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
@@ -60,11 +81,16 @@ class Blocked extends Error {
 	}
 }
 
-let lastRequest = 0;
+// One queue per site: requests to the same site stay one at a time, 6-10 s apart.
-async function politeFetch(url: string, init: RequestInit = {}): Promise<Response> {
+const lastRequest = new Map<string, number>();
-	const wait = lastRequest + 6000 + Math.random() * 4000 - Date.now();
+async function politeFetch(
+	url: string,
+	init: RequestInit = {},
+	queue = 'search'
+): Promise<Response> {
+	const wait = (lastRequest.get(queue) ?? 0) + 6000 + Math.random() * 4000 - Date.now();
 	if (wait > 0) await Bun.sleep(wait);
-	lastRequest = Date.now();
+	lastRequest.set(queue, Date.now());
 	const response = await fetch(url, {
 		...init,
 		redirect: 'follow',
@@ -188,6 +214,106 @@ async function scrapeVesteda(): Promise<SourceResult> {
 	}
 }
 
+type RecheckRow = { id: number; url: string; title: string };
+type RecheckSummary = { checked: number; alive: number; gone: number; unknown: number };
+
+/** Fetch each row's own page for one site until the deadline; a block stops the site. */
+async function recheckSiteRows(site: RecheckSite, rows: RecheckRow[], deadline: number) {
+	const summary: RecheckSummary = { checked: 0, alive: 0, gone: 0, unknown: 0 };
+	const alive: number[] = [];
+	const gone: { row: RecheckRow; reason: string }[] = [];
+	let blocked = false;
+	for (const row of rows) {
+		if (Date.now() + 12_000 > deadline) break;
+		let page: { status: number; finalUrl: string; html: string };
+		try {
+			const response = await politeFetch(row.url, { headers: { accept: 'text/html' } }, site);
+			page = {
+				status: response.status,
+				finalUrl: response.url || row.url,
+				html: await response.text()
+			};
+		} catch (error) {
+			summary.checked++;
+			summary.unknown++;
+			console.log(`  ${site} #${row.id} unknown (${(error as Error).message}) ${row.url}`);
+			continue;
+		}
+		const { status, html } = page;
+		if (isChallenge(html, status)) {
+			await logRun(site, row.url, 'challenged', { httpStatus: status, message: 'recheck' });
+			blocked = true;
+			break;
+		}
+		summary.checked++;
+		const verdict = classifyListingPage(site, row.url, page);
+		summary[verdict.verdict]++;
+		if (verdict.verdict === 'alive') alive.push(row.id);
+		if (verdict.verdict === 'gone') gone.push({ row, reason: verdict.reason });
+		if (verdict.verdict !== 'alive' || dryRun)
+			console.log(
+				`  ${site} #${row.id} ${verdict.verdict} (${verdict.reason}) ${row.title} ${row.url}`
+			);
+	}
+	return { summary, alive, gone, blocked };
+}
+
+async function recheckListings(): Promise<boolean> {
+	if (!sql) return true;
+	const started = Date.now();
+	const deadline = Math.min(
+		started + RECHECK_MINUTES * 60_000,
+		scriptStarted + RECHECK_HARD_STOP_MS
+	);
+	// Per site, oldest-checked first, so every row gets its turn over a few nights.
+	const bySite = new Map<RecheckSite, RecheckRow[]>();
+	for (const site of RECHECK_SITES) {
+		const rows = await sql<RecheckRow[]>`SELECT id, url, title FROM house.listings
+			WHERE removed_at IS NULL AND url ILIKE ${RECHECK_URL_PATTERNS[site]}
+			ORDER BY last_seen_at ASC NULLS FIRST, id
+			LIMIT ${recheckLimit}`;
+		const own = rows.filter((r) => recheckSite(r.url) === site);
+		if (own.length) bySite.set(site, own);
+	}
+	let ok = true;
+	const results = await Promise.all(
+		[...bySite].map(async ([site, list]) => ({
+			site,
+			...(await recheckSiteRows(site, list, deadline))
+		}))
+	);
+	for (const { site, summary, alive, gone, blocked } of results) {
+		const sane = removalsLookSane(summary.checked, gone.length);
+		let applied = 0;
+		if (!dryRun) {
+			if (alive.length)
+				await sql`UPDATE house.listings SET last_seen_at = now() WHERE id IN ${sql(alive)}`;
+			if (gone.length && sane) {
+				const done = await sql`UPDATE house.listings SET removed_at = now()
+					WHERE removed_at IS NULL AND id IN ${sql(gone.map((g) => g.row.id))} RETURNING id`;
+				applied = done.length;
+			}
+		}
+		if (blocked) ok = false;
+		const message = `${summary.alive} alive, ${summary.gone} gone${
+			!sane
+				? ' (NOT applied: too many for one night, check the site)'
+				: dryRun
+					? ' (dry run)'
+					: `, ${applied} marked removed`
+		}, ${summary.unknown} unknown${blocked ? '; blocked, stopped for tonight' : ''}`;
+		await logRun(site, 'recheck', !sane ? 'suspicious' : blocked ? 'challenged' : 'ok', {
+			found: summary.checked,
+			durationMs: Date.now() - started,
+			message
+		});
+		console.log(
+			`recheck ${site}: ${summary.checked} of ${bySite.get(site)?.length ?? 0} checked, ${message}`
+		);
+	}
+	return ok;
+}
+
 async function ntfy(title: string, body: string, click: string) {
 	const target = process.env.HOUSE_NTFY_URL;
 	if (!target) return false;
@@ -249,6 +375,15 @@ try {
 		if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
 	}
 
+	if (!only || only === 'recheck') {
+		try {
+			if (!(await recheckListings())) failures++;
+		} catch (error) {
+			failures++;
+			console.error(`recheck: ${(error as Error).message}`);
+		}
+	}
+
 	if (sql && !dryRun) {
 		const { keyed, copies } = await dedupe(sql);
 		console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);

src/lib/house-tracker/recheck.test.ts

@@ -0,0 +1,112 @@
+import { describe, expect, it } from 'vitest';
+import { classifyListingPage, huispediaStatus, recheckSite, removalsLookSane } from './recheck';
+
+const page = (status: number, finalUrl: string, html = '') => ({ status, finalUrl, html });
+const hpStatus = (s: string) =>
+	`<li><span class="name">Status</span>\n <span class="value">\n   ${s}\n  </span></li>`;
+
+describe('recheckSite', () => {
+	it('knows the four backfill sites, with or without www', () => {
+		expect(recheckSite('https://www.huislijn.nl/huurwoning/x/1')).toBe('huislijn');
+		expect(recheckSite('https://huispedia.nl/gouda/2801gk/kazernestraat/1-a')).toBe('huispedia');
+		expect(recheckSite('https://rentola.nl/listings/a-p1')).toBe('rentola');
+		expect(recheckSite('https://huurportaal.nl/listings/a-p1')).toBe('huurportaal');
+		expect(recheckSite('https://www.pararius.nl/x')).toBeNull();
+		expect(recheckSite(null)).toBeNull();
+	});
+});
+
+describe('classifyListingPage', () => {
+	const hp = 'https://huurportaal.nl/listings/mvgm-te-huur-voorzorg-85-gouda-p7d7387';
+	it('huurportaal/rentola: inactive redirect is gone, own page is alive', () => {
+		expect(
+			classifyListingPage(
+				'huurportaal',
+				hp,
+				page(
+					200,
+					'https://huurportaal.nl/huurwoningen/appartement/gouda?inactive_property_id=40072247'
+				)
+			).verdict
+		).toBe('gone');
+		expect(classifyListingPage('huurportaal', hp, page(200, hp)).verdict).toBe('alive');
+		expect(
+			classifyListingPage(
+				'rentola',
+				'https://rentola.nl/listings/x-p1',
+				page(200, 'https://rentola.nl/')
+			).verdict
+		).toBe('unknown');
+	});
+
+	const hl = 'https://www.huislijn.nl/huurwoning/nederland/zuid-holland/4439788/voorzorg-gouda';
+	it('huislijn: redirect to the overview is gone; woningruil is gone; detail is alive', () => {
+		expect(
+			classifyListingPage(
+				'huislijn',
+				hl,
+				page(200, 'https://www.huislijn.nl/huurwoning/nederland/zuid-holland')
+			)
+		).toMatchObject({ verdict: 'gone' });
+		expect(classifyListingPage('huislijn', hl, page(200, hl, '<p>Mooi huis</p>')).verdict).toBe(
+			'alive'
+		);
+		expect(
+			classifyListingPage(
+				'huislijn',
+				hl,
+				page(200, hl, '<p>Belangrijk! Dit geldt uitsluitend voor een woningruil; je moet</p>')
+			)
+		).toMatchObject({ verdict: 'gone', reason: expect.stringContaining('woningruil') });
+		expect(
+			classifyListingPage('huislijn', hl, page(200, 'https://www.huislijn.nl/inloggen')).verdict
+		).toBe('unknown');
+	});
+
+	const hpd = 'https://huispedia.nl/gouda/2801gk/kazernestraat/1-a';
+	it('huispedia: reads the status field', () => {
+		expect(huispediaStatus(hpStatus('Te huur'))).toBe('Te huur');
+		expect(classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Te huur'))).verdict).toBe(
+			'alive'
+		);
+		for (const s of [
+			'Verkocht',
+			'Verhuurd',
+			'Verhuurd onder voorbehoud',
+			'Niet te koop, laatst verkocht in 2019'
+		])
+			expect(classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus(s))).verdict).toBe(
+				'gone'
+			);
+		expect(
+			classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Verkocht onder voorbehoud')))
+				.verdict
+		).toBe('alive');
+		expect(classifyListingPage('huispedia', hpd, page(200, hpd, '<p>nothing</p>')).verdict).toBe(
+			'unknown'
+		);
+		expect(
+			classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Iets nieuws'))).verdict
+		).toBe('unknown');
+	});
+
+	it('only a 404/410 on the site itself is gone; errors are unknown', () => {
+		expect(classifyListingPage('huispedia', hpd, page(404, hpd)).verdict).toBe('gone');
+		expect(classifyListingPage('huislijn', hl, page(410, hl)).verdict).toBe('gone');
+		expect(classifyListingPage('huispedia', hpd, page(500, hpd)).verdict).toBe('unknown');
+		expect(classifyListingPage('huispedia', hpd, page(404, 'https://example.com/x')).verdict).toBe(
+			'unknown'
+		);
+	});
+});
+
+describe('removalsLookSane', () => {
+	it('refuses a night where most rows look gone', () => {
+		expect(removalsLookSane(60, 6)).toBe(true);
+		expect(removalsLookSane(60, 30)).toBe(true);
+		expect(removalsLookSane(60, 31)).toBe(false);
+		expect(removalsLookSane(5, 3)).toBe(true);
+		expect(removalsLookSane(5, 4)).toBe(false);
+		expect(removalsLookSane(0, 0)).toBe(true);
+	});
+});

src/lib/house-tracker/recheck.ts

@@ -0,0 +1,128 @@
+// Re-checks listings that were added by hand or by a one-off backfill (huispedia, huislijn,
+// huurportaal, rentola) by fetching their own page, so rows whose ad is gone get marked
+// removed. Pure helpers, no #lib / $app imports; the fetching lives in scripts/house-scrape.ts.
+//
+// Only a clear signal counts as gone: a 404/410 on the listing's own page, the site's own
+// "no longer available" redirect, or the page's own status saying rented/sold. Anything else
+// (errors, timeouts, an unfamiliar page) is "unknown" and leaves the row alone.
+
+export const RECHECK_SITES = ['huispedia', 'huislijn', 'huurportaal', 'rentola'] as const;
+export type RecheckSite = (typeof RECHECK_SITES)[number];
+
+export type RecheckVerdict = {
+	verdict: 'alive' | 'gone' | 'unknown';
+	reason: string;
+};
+
+const HOSTS: Record<string, RecheckSite> = {
+	'huispedia.nl': 'huispedia',
+	'huislijn.nl': 'huislijn',
+	'huurportaal.nl': 'huurportaal',
+	'rentola.nl': 'rentola'
+};
+
+function parse(url: string): URL | null {
+	try {
+		return new URL(url);
+	} catch {
+		return null;
+	}
+}
+
+const hostOf = (u: URL) => u.hostname.toLowerCase().replace(/^www\./, '');
+
+/** Which site a listing link belongs to, or null when it is not one we re-check. */
+export function recheckSite(url: string | null | undefined): RecheckSite | null {
+	const u = url ? parse(url) : null;
+	return u ? (HOSTS[hostOf(u)] ?? null) : null;
+}
+
+/** SQL ILIKE pattern per site for rows whose link is on it (narrowed by recheckSite after). */
+export const RECHECK_URL_PATTERNS = Object.fromEntries(
+	Object.entries(HOSTS).map(([host, site]) => [site, `%://%${host}/%`])
+) as Record<RecheckSite, string>;
+
+const text = (html: string) =>
+	html
+		.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, ' ')
+		.replace(/<[^>]+>/g, ' ')
+		.replace(/&nbsp;/g, ' ')
+		.replace(/\s+/g, ' ');
+
+/** Huispedia's "Status" value from the features list ("Te huur", "Verkocht", ...). */
+export function huispediaStatus(html: string): string | null {
+	const m = html.match(
+		/<span class="name">\s*Status\s*<\/span>\s*<span class="value">\s*([^<]{1,120}?)\s*<\/span>/i
+	);
+	return m ? m[1].replace(/\s+/g, ' ').trim() : null;
+}
+
+const HUISPEDIA_ALIVE = /^(te huur|te koop|onder bod|onder optie|verkocht onder voorbehoud)\b/i;
+const HUISPEDIA_GONE =
+	/^(verhuurd|verkocht\b(?! onder voorbehoud)|niet te koop|niet te huur|ingetrokken)/i;
+
+/**
+ * Decide from one fetch of `requestedUrl` whether the ad is still up. `finalUrl` is where
+ * redirects ended. Call only for pages that were not a challenge/block (the caller stops a
+ * site for the night on 403/429/503 or a challenge page).
+ */
+export function classifyListingPage(
+	site: RecheckSite,
+	requestedUrl: string,
+	res: { status: number; finalUrl: string; html: string }
+): RecheckVerdict {
+	const requested = parse(requestedUrl);
+	const final = parse(res.finalUrl) ?? requested;
+	if (!requested || !final) return { verdict: 'unknown', reason: 'bad url' };
+	const sameSite = recheckSite(final.href) === site;
+
+	if ((res.status === 404 || res.status === 410) && sameSite)
+		return { verdict: 'gone', reason: `http ${res.status}` };
+	if (res.status !== 200) return { verdict: 'unknown', reason: `http ${res.status}` };
+	if (!sameSite) return { verdict: 'unknown', reason: `redirected off-site to ${final.host}` };
+
+	switch (site) {
+		case 'huurportaal':
+		case 'rentola': {
+			// Their own "this listing is no longer available" redirect to a search page.
+			if (final.searchParams.has('inactive_property_id'))
+				return { verdict: 'gone', reason: 'redirected as inactive' };
+			if (final.pathname === requested.pathname && final.pathname.startsWith('/listings/'))
+				return { verdict: 'alive', reason: 'listing page' };
+			return { verdict: 'unknown', reason: `redirected to ${final.pathname}` };
+		}
+		case 'huislijn': {
+			const id = requested.pathname.match(/\/(\d{5,})\//)?.[1];
+			if (!id) return { verdict: 'unknown', reason: 'no listing id in link' };
+			// A removed ad redirects to the province/search overview, without the id.
+			if (!final.pathname.includes(`/${id}/`)) {
+				return /\/(huurwoning|koopwoning)\/nederland(\/[a-z-]+)?\/?$/.test(final.pathname)
+					? { verdict: 'gone', reason: 'redirected to the overview' }
+					: { verdict: 'unknown', reason: `redirected to ${final.pathname}` };
+			}
+			if (/geldt uitsluitend voor een woningruil/i.test(text(res.html)))
+				return { verdict: 'gone', reason: 'woningruil only (swap, not a rental)' };
+			return { verdict: 'alive', reason: 'listing page' };
+		}
+		case 'huispedia': {
+			// Huispedia keeps a page for every address; the ad's status is what changes.
+			if (final.pathname.replace(/\/$/, '') !== requested.pathname.replace(/\/$/, ''))
+				return { verdict: 'unknown', reason: `redirected to ${final.pathname}` };
+			const status = huispediaStatus(res.html);
+			if (!status) return { verdict: 'unknown', reason: 'no status on page' };
+			if (HUISPEDIA_ALIVE.test(status)) return { verdict: 'alive', reason: status };
+			if (HUISPEDIA_GONE.test(status)) return { verdict: 'gone', reason: status };
+			return { verdict: 'unknown', reason: `status "${status}"` };
+		}
+	}
+}
+
+/**
+ * Safety valve: a night where most checked rows of a site look gone is more likely a site
+ * change than a real exodus, so none of that night's removals for the site are applied.
+ */
+export function removalsLookSane(checked: number, gone: number): boolean {
+	if (gone === 0) return true;
+	if (checked < 10) return gone <= 3;
+	return gone <= Math.max(5, Math.floor(checked * 0.5));
+}