house-scrape: re-check huispedia, huislijn, huurportaal and rentola rows and mark clearly gone ones removed

scripts/house-scrape.ts

@@ -3,6 +3,12 @@
 // for new listings that match an alert rule. See docs/house-tracker.md.
 //
 //   bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet]
+//       [--source recheck] [--recheck-limit 20]
+//
+// After the search sources it re-checks rows that came from one-off imports or the CLI
+// (huispedia, huislijn, huurportaal, rentola) by fetching each listing's own page, oldest
+// check first, within a time budget, and marks a row gone only on a clear signal (see
+// src/lib/house-tracker/recheck.ts). Each site gets its own one-at-a-time 6-10 s queue.
 //
 // Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed
 // paths only. A 403/429/challenge stops that source for the night; nothing is retried or
@@ -15,6 +21,14 @@ import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
 import { scoreListings } from '../src/lib/house-tracker/liveability';
 import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
 import { isChallenge } from '../src/lib/house-tracker/pararius';
+import {
+	classifyListingPage,
+	RECHECK_SITES,
+	RECHECK_URL_PATTERNS,
+	recheckSite,
+	removalsLookSane,
+	type RecheckSite
+} from '../src/lib/house-tracker/recheck';
 import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
 import {
 	parseVestedaSearch,
@@ -25,6 +39,13 @@ import {
 const args = process.argv.slice(2);
 const dryRun = args.includes('--dry-run');
 const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null;
+const recheckLimit = args.includes('--recheck-limit')
+	? Number(args[args.indexOf('--recheck-limit') + 1])
+	: 400;
+const scriptStarted = Date.now();
+// The unit has TimeoutStartSec=900; leave room for the search sources and the steps after.
+const RECHECK_MINUTES = Number(process.env.HOUSE_RECHECK_MINUTES ?? 8);
+const RECHECK_HARD_STOP_MS = 12 * 60_000;
 
 const USER_AGENT =
 	'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
@@ -60,11 +81,16 @@ class Blocked extends Error {
 	}
 }
 
-let lastRequest = 0;
-async function politeFetch(url: string, init: RequestInit = {}): Promise<Response> {
-	const wait = lastRequest + 6000 + Math.random() * 4000 - Date.now();
+// One queue per site: requests to the same site stay one at a time, 6-10 s apart.
+const lastRequest = new Map<string, number>();
+async function politeFetch(
+	url: string,
+	init: RequestInit = {},
+	queue = 'search'
+): Promise<Response> {
+	const wait = (lastRequest.get(queue) ?? 0) + 6000 + Math.random() * 4000 - Date.now();
 	if (wait > 0) await Bun.sleep(wait);
-	lastRequest = Date.now();
+	lastRequest.set(queue, Date.now());
 	const response = await fetch(url, {
 		...init,
 		redirect: 'follow',
@@ -188,6 +214,106 @@ async function scrapeVesteda(): Promise<SourceResult> {
 	}
 }
 
+type RecheckRow = { id: number; url: string; title: string };
+type RecheckSummary = { checked: number; alive: number; gone: number; unknown: number };
+
+/** Fetch each row's own page for one site until the deadline; a block stops the site. */
+async function recheckSiteRows(site: RecheckSite, rows: RecheckRow[], deadline: number) {
+	const summary: RecheckSummary = { checked: 0, alive: 0, gone: 0, unknown: 0 };
+	const alive: number[] = [];
+	const gone: { row: RecheckRow; reason: string }[] = [];
+	let blocked = false;
+	for (const row of rows) {
+		if (Date.now() + 12_000 > deadline) break;
+		let page: { status: number; finalUrl: string; html: string };
+		try {
+			const response = await politeFetch(row.url, { headers: { accept: 'text/html' } }, site);
+			page = {
+				status: response.status,
+				finalUrl: response.url || row.url,
+				html: await response.text()
+			};
+		} catch (error) {
+			summary.checked++;
+			summary.unknown++;
+			console.log(`  ${site} #${row.id} unknown (${(error as Error).message}) ${row.url}`);
+			continue;
+		}
+		const { status, html } = page;
+		if (isChallenge(html, status)) {
+			await logRun(site, row.url, 'challenged', { httpStatus: status, message: 'recheck' });
+			blocked = true;
+			break;
+		}
+		summary.checked++;
+		const verdict = classifyListingPage(site, row.url, page);
+		summary[verdict.verdict]++;
+		if (verdict.verdict === 'alive') alive.push(row.id);
+		if (verdict.verdict === 'gone') gone.push({ row, reason: verdict.reason });
+		if (verdict.verdict !== 'alive' || dryRun)
+			console.log(
+				`  ${site} #${row.id} ${verdict.verdict} (${verdict.reason}) ${row.title} ${row.url}`
+			);
+	}
+	return { summary, alive, gone, blocked };
+}
+
+async function recheckListings(): Promise<boolean> {
+	if (!sql) return true;
+	const started = Date.now();
+	const deadline = Math.min(
+		started + RECHECK_MINUTES * 60_000,
+		scriptStarted + RECHECK_HARD_STOP_MS
+	);
+	// Per site, oldest-checked first, so every row gets its turn over a few nights.
+	const bySite = new Map<RecheckSite, RecheckRow[]>();
+	for (const site of RECHECK_SITES) {
+		const rows = await sql<RecheckRow[]>`SELECT id, url, title FROM house.listings
+			WHERE removed_at IS NULL AND url ILIKE ${RECHECK_URL_PATTERNS[site]}
+			ORDER BY last_seen_at ASC NULLS FIRST, id
+			LIMIT ${recheckLimit}`;
+		const own = rows.filter((r) => recheckSite(r.url) === site);
+		if (own.length) bySite.set(site, own);
+	}
+	let ok = true;
+	const results = await Promise.all(
+		[...bySite].map(async ([site, list]) => ({
+			site,
+			...(await recheckSiteRows(site, list, deadline))
+		}))
+	);
+	for (const { site, summary, alive, gone, blocked } of results) {
+		const sane = removalsLookSane(summary.checked, gone.length);
+		let applied = 0;
+		if (!dryRun) {
+			if (alive.length)
+				await sql`UPDATE house.listings SET last_seen_at = now() WHERE id IN ${sql(alive)}`;
+			if (gone.length && sane) {
+				const done = await sql`UPDATE house.listings SET removed_at = now()
+					WHERE removed_at IS NULL AND id IN ${sql(gone.map((g) => g.row.id))} RETURNING id`;
+				applied = done.length;
+			}
+		}
+		if (blocked) ok = false;
+		const message = `${summary.alive} alive, ${summary.gone} gone${
+			!sane
+				? ' (NOT applied: too many for one night, check the site)'
+				: dryRun
+					? ' (dry run)'
+					: `, ${applied} marked removed`
+		}, ${summary.unknown} unknown${blocked ? '; blocked, stopped for tonight' : ''}`;
+		await logRun(site, 'recheck', !sane ? 'suspicious' : blocked ? 'challenged' : 'ok', {
+			found: summary.checked,
+			durationMs: Date.now() - started,
+			message
+		});
+		console.log(
+			`recheck ${site}: ${summary.checked} of ${bySite.get(site)?.length ?? 0} checked, ${message}`
+		);
+	}
+	return ok;
+}
+
 async function ntfy(title: string, body: string, click: string) {
 	const target = process.env.HOUSE_NTFY_URL;
 	if (!target) return false;
@@ -249,6 +375,15 @@ try {
 		if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
 	}
 
+	if (!only || only === 'recheck') {
+		try {
+			if (!(await recheckListings())) failures++;
+		} catch (error) {
+			failures++;
+			console.error(`recheck: ${(error as Error).message}`);
+		}
+	}
+
 	if (sql && !dryRun) {
 		const { keyed, copies } = await dedupe(sql);
 		console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);