From fa3aa25f0b3f6f84b6c5f4185dcef672255710fc Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sun, 25 Oct 2026 23:14:42 +0000 Subject: [PATCH] house-scrape: re-check huispedia, huislijn, huurportaal and rentola rows and mark clearly gone ones removed --- deploy/systemd/house-scrape.service | 2 +- docs/house-tracker.md | 18 ++++ scripts/house-scrape.ts | 143 +++++++++++++++++++++++++- src/lib/house-tracker/recheck.test.ts | 112 ++++++++++++++++++++ src/lib/house-tracker/recheck.ts | 128 +++++++++++++++++++++++ 5 files changed, 398 insertions(+), 5 deletions(-) create mode 100644 src/lib/house-tracker/recheck.test.ts create mode 100644 src/lib/house-tracker/recheck.ts diff --git a/deploy/systemd/house-scrape.service b/deploy/systemd/house-scrape.service index 64a6b7d..9aaa3c2 100644 --- a/deploy/systemd/house-scrape.service +++ b/deploy/systemd/house-scrape.service @@ -1,5 +1,5 @@ [Unit] -Description=House tracker nightly scrape (Kamernet, Vesteda -> house.listings) +Description=House tracker nightly scrape (Kamernet, Vesteda, re-check of imported rows -> house.listings) Wants=network-online.target After=network-online.target diff --git a/docs/house-tracker.md b/docs/house-tracker.md index 9020dbd..cea0704 100644 --- a/docs/house-tracker.md +++ b/docs/house-tracker.md @@ -19,6 +19,24 @@ night and is logged as `challenged`; nothing is retried or worked around. | Huurstunt | Cloudflare challenge | skipped | | Mijndak Midden-Holland | OutSystems app behind a login, no public listing feed | skipped; use their own e-mail alerts | +**Re-check of imported rows** (`src/lib/house-tracker/recheck.ts`). Rows added by the CLI or +one-off backfills from huispedia, huislijn, huurportaal and rentola are not in any search +feed, so after the sources the scrape fetches each row's own page (all four allow detail +pages in robots.txt), up to 400 per site, oldest `last_seen_at` first, within +`HOUSE_RECHECK_MINUTES` (default 8). Each site has its own one-at-a-time 6–10 s queue. +Alive sets `last_seen_at`; gone sets `removed_at` and needs a clear signal: + +| site | gone when | +| -------------------- | ------------------------------------------------------------------------------- | +| huurportaal, rentola | redirect to a search page with `?inactive_property_id=`, or 404/410 | +| huislijn | redirect to the province overview (id gone from the URL), 404/410, woningruil | +| huispedia | 404/410 ("niet meer beschikbaar"), or Status verhuurd / verkocht / niet te koop | + +Errors, timeouts, off-site redirects and unfamiliar pages are `unknown` and change nothing; a +403/429/503 or challenge stops that site for the night. If more than half of a site's checks +in one night (min. 5) say gone, none are applied and the run is logged as `suspicious`. +Try it with `--dry-run --source recheck --recheck-limit 5`. + Same home on two sources: `address_key` (street + number + city, normalised, see `src/lib/house-tracker/address.ts`) links the copy to the original via `duplicate_of`; a manual entry wins. The page shows one row with "also on …" links. Kamernet never shows house diff --git a/scripts/house-scrape.ts b/scripts/house-scrape.ts index 828c652..9561702 100644 --- a/scripts/house-scrape.ts +++ b/scripts/house-scrape.ts @@ -3,6 +3,12 @@ // for new listings that match an alert rule. See docs/house-tracker.md. // // bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet] +// [--source recheck] [--recheck-limit 20] +// +// After the search sources it re-checks rows that came from one-off imports or the CLI +// (huispedia, huislijn, huurportaal, rentola) by fetching each listing's own page, oldest +// check first, within a time budget, and marks a row gone only on a clear signal (see +// src/lib/house-tracker/recheck.ts). Each site gets its own one-at-a-time 6-10 s queue. // // Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed // paths only. A 403/429/challenge stops that source for the night; nothing is retried or @@ -15,6 +21,14 @@ import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest'; import { scoreListings } from '../src/lib/house-tracker/liveability'; import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet'; import { isChallenge } from '../src/lib/house-tracker/pararius'; +import { + classifyListingPage, + RECHECK_SITES, + RECHECK_URL_PATTERNS, + recheckSite, + removalsLookSane, + type RecheckSite +} from '../src/lib/house-tracker/recheck'; import type { ScrapedListing } from '../src/lib/house-tracker/scraped'; import { parseVestedaSearch, @@ -25,6 +39,13 @@ import { const args = process.argv.slice(2); const dryRun = args.includes('--dry-run'); const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null; +const recheckLimit = args.includes('--recheck-limit') + ? Number(args[args.indexOf('--recheck-limit') + 1]) + : 400; +const scriptStarted = Date.now(); +// The unit has TimeoutStartSec=900; leave room for the search sources and the steps after. +const RECHECK_MINUTES = Number(process.env.HOUSE_RECHECK_MINUTES ?? 8); +const RECHECK_HARD_STOP_MS = 12 * 60_000; const USER_AGENT = 'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)'; @@ -60,11 +81,16 @@ class Blocked extends Error { } } -let lastRequest = 0; -async function politeFetch(url: string, init: RequestInit = {}): Promise { - const wait = lastRequest + 6000 + Math.random() * 4000 - Date.now(); +// One queue per site: requests to the same site stay one at a time, 6-10 s apart. +const lastRequest = new Map(); +async function politeFetch( + url: string, + init: RequestInit = {}, + queue = 'search' +): Promise { + const wait = (lastRequest.get(queue) ?? 0) + 6000 + Math.random() * 4000 - Date.now(); if (wait > 0) await Bun.sleep(wait); - lastRequest = Date.now(); + lastRequest.set(queue, Date.now()); const response = await fetch(url, { ...init, redirect: 'follow', @@ -188,6 +214,106 @@ async function scrapeVesteda(): Promise { } } +type RecheckRow = { id: number; url: string; title: string }; +type RecheckSummary = { checked: number; alive: number; gone: number; unknown: number }; + +/** Fetch each row's own page for one site until the deadline; a block stops the site. */ +async function recheckSiteRows(site: RecheckSite, rows: RecheckRow[], deadline: number) { + const summary: RecheckSummary = { checked: 0, alive: 0, gone: 0, unknown: 0 }; + const alive: number[] = []; + const gone: { row: RecheckRow; reason: string }[] = []; + let blocked = false; + for (const row of rows) { + if (Date.now() + 12_000 > deadline) break; + let page: { status: number; finalUrl: string; html: string }; + try { + const response = await politeFetch(row.url, { headers: { accept: 'text/html' } }, site); + page = { + status: response.status, + finalUrl: response.url || row.url, + html: await response.text() + }; + } catch (error) { + summary.checked++; + summary.unknown++; + console.log(` ${site} #${row.id} unknown (${(error as Error).message}) ${row.url}`); + continue; + } + const { status, html } = page; + if (isChallenge(html, status)) { + await logRun(site, row.url, 'challenged', { httpStatus: status, message: 'recheck' }); + blocked = true; + break; + } + summary.checked++; + const verdict = classifyListingPage(site, row.url, page); + summary[verdict.verdict]++; + if (verdict.verdict === 'alive') alive.push(row.id); + if (verdict.verdict === 'gone') gone.push({ row, reason: verdict.reason }); + if (verdict.verdict !== 'alive' || dryRun) + console.log( + ` ${site} #${row.id} ${verdict.verdict} (${verdict.reason}) ${row.title} ${row.url}` + ); + } + return { summary, alive, gone, blocked }; +} + +async function recheckListings(): Promise { + if (!sql) return true; + const started = Date.now(); + const deadline = Math.min( + started + RECHECK_MINUTES * 60_000, + scriptStarted + RECHECK_HARD_STOP_MS + ); + // Per site, oldest-checked first, so every row gets its turn over a few nights. + const bySite = new Map(); + for (const site of RECHECK_SITES) { + const rows = await sql`SELECT id, url, title FROM house.listings + WHERE removed_at IS NULL AND url ILIKE ${RECHECK_URL_PATTERNS[site]} + ORDER BY last_seen_at ASC NULLS FIRST, id + LIMIT ${recheckLimit}`; + const own = rows.filter((r) => recheckSite(r.url) === site); + if (own.length) bySite.set(site, own); + } + let ok = true; + const results = await Promise.all( + [...bySite].map(async ([site, list]) => ({ + site, + ...(await recheckSiteRows(site, list, deadline)) + })) + ); + for (const { site, summary, alive, gone, blocked } of results) { + const sane = removalsLookSane(summary.checked, gone.length); + let applied = 0; + if (!dryRun) { + if (alive.length) + await sql`UPDATE house.listings SET last_seen_at = now() WHERE id IN ${sql(alive)}`; + if (gone.length && sane) { + const done = await sql`UPDATE house.listings SET removed_at = now() + WHERE removed_at IS NULL AND id IN ${sql(gone.map((g) => g.row.id))} RETURNING id`; + applied = done.length; + } + } + if (blocked) ok = false; + const message = `${summary.alive} alive, ${summary.gone} gone${ + !sane + ? ' (NOT applied: too many for one night, check the site)' + : dryRun + ? ' (dry run)' + : `, ${applied} marked removed` + }, ${summary.unknown} unknown${blocked ? '; blocked, stopped for tonight' : ''}`; + await logRun(site, 'recheck', !sane ? 'suspicious' : blocked ? 'challenged' : 'ok', { + found: summary.checked, + durationMs: Date.now() - started, + message + }); + console.log( + `recheck ${site}: ${summary.checked} of ${bySite.get(site)?.length ?? 0} checked, ${message}` + ); + } + return ok; +} + async function ntfy(title: string, body: string, click: string) { const target = process.env.HOUSE_NTFY_URL; if (!target) return false; @@ -249,6 +375,15 @@ try { if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name }); } + if (!only || only === 'recheck') { + try { + if (!(await recheckListings())) failures++; + } catch (error) { + failures++; + console.error(`recheck: ${(error as Error).message}`); + } + } + if (sql && !dryRun) { const { keyed, copies } = await dedupe(sql); console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`); diff --git a/src/lib/house-tracker/recheck.test.ts b/src/lib/house-tracker/recheck.test.ts new file mode 100644 index 0000000..852bac3 --- /dev/null +++ b/src/lib/house-tracker/recheck.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from 'vitest'; +import { classifyListingPage, huispediaStatus, recheckSite, removalsLookSane } from './recheck'; + +const page = (status: number, finalUrl: string, html = '') => ({ status, finalUrl, html }); +const hpStatus = (s: string) => + `
  • Status\n \n ${s}\n
    • `; + +describe('recheckSite', () => { + it('knows the four backfill sites, with or without www', () => { + expect(recheckSite('https://www.huislijn.nl/huurwoning/x/1')).toBe('huislijn'); + expect(recheckSite('https://huispedia.nl/gouda/2801gk/kazernestraat/1-a')).toBe('huispedia'); + expect(recheckSite('https://rentola.nl/listings/a-p1')).toBe('rentola'); + expect(recheckSite('https://huurportaal.nl/listings/a-p1')).toBe('huurportaal'); + expect(recheckSite('https://www.pararius.nl/x')).toBeNull(); + expect(recheckSite(null)).toBeNull(); + }); +}); + +describe('classifyListingPage', () => { + const hp = 'https://huurportaal.nl/listings/mvgm-te-huur-voorzorg-85-gouda-p7d7387'; + it('huurportaal/rentola: inactive redirect is gone, own page is alive', () => { + expect( + classifyListingPage( + 'huurportaal', + hp, + page( + 200, + 'https://huurportaal.nl/huurwoningen/appartement/gouda?inactive_property_id=40072247' + ) + ).verdict + ).toBe('gone'); + expect(classifyListingPage('huurportaal', hp, page(200, hp)).verdict).toBe('alive'); + expect( + classifyListingPage( + 'rentola', + 'https://rentola.nl/listings/x-p1', + page(200, 'https://rentola.nl/') + ).verdict + ).toBe('unknown'); + }); + + const hl = 'https://www.huislijn.nl/huurwoning/nederland/zuid-holland/4439788/voorzorg-gouda'; + it('huislijn: redirect to the overview is gone; woningruil is gone; detail is alive', () => { + expect( + classifyListingPage( + 'huislijn', + hl, + page(200, 'https://www.huislijn.nl/huurwoning/nederland/zuid-holland') + ) + ).toMatchObject({ verdict: 'gone' }); + expect(classifyListingPage('huislijn', hl, page(200, hl, '

    Mooi huis

    ')).verdict).toBe( + 'alive' + ); + expect( + classifyListingPage( + 'huislijn', + hl, + page(200, hl, '

    Belangrijk! Dit geldt uitsluitend voor een woningruil; je moet

    ') + ) + ).toMatchObject({ verdict: 'gone', reason: expect.stringContaining('woningruil') }); + expect( + classifyListingPage('huislijn', hl, page(200, 'https://www.huislijn.nl/inloggen')).verdict + ).toBe('unknown'); + }); + + const hpd = 'https://huispedia.nl/gouda/2801gk/kazernestraat/1-a'; + it('huispedia: reads the status field', () => { + expect(huispediaStatus(hpStatus('Te huur'))).toBe('Te huur'); + expect(classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Te huur'))).verdict).toBe( + 'alive' + ); + for (const s of [ + 'Verkocht', + 'Verhuurd', + 'Verhuurd onder voorbehoud', + 'Niet te koop, laatst verkocht in 2019' + ]) + expect(classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus(s))).verdict).toBe( + 'gone' + ); + expect( + classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Verkocht onder voorbehoud'))) + .verdict + ).toBe('alive'); + expect(classifyListingPage('huispedia', hpd, page(200, hpd, '

    nothing

    ')).verdict).toBe( + 'unknown' + ); + expect( + classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Iets nieuws'))).verdict + ).toBe('unknown'); + }); + + it('only a 404/410 on the site itself is gone; errors are unknown', () => { + expect(classifyListingPage('huispedia', hpd, page(404, hpd)).verdict).toBe('gone'); + expect(classifyListingPage('huislijn', hl, page(410, hl)).verdict).toBe('gone'); + expect(classifyListingPage('huispedia', hpd, page(500, hpd)).verdict).toBe('unknown'); + expect(classifyListingPage('huispedia', hpd, page(404, 'https://example.com/x')).verdict).toBe( + 'unknown' + ); + }); +}); + +describe('removalsLookSane', () => { + it('refuses a night where most rows look gone', () => { + expect(removalsLookSane(60, 6)).toBe(true); + expect(removalsLookSane(60, 30)).toBe(true); + expect(removalsLookSane(60, 31)).toBe(false); + expect(removalsLookSane(5, 3)).toBe(true); + expect(removalsLookSane(5, 4)).toBe(false); + expect(removalsLookSane(0, 0)).toBe(true); + }); +}); diff --git a/src/lib/house-tracker/recheck.ts b/src/lib/house-tracker/recheck.ts new file mode 100644 index 0000000..294cc8a --- /dev/null +++ b/src/lib/house-tracker/recheck.ts @@ -0,0 +1,128 @@ +// Re-checks listings that were added by hand or by a one-off backfill (huispedia, huislijn, +// huurportaal, rentola) by fetching their own page, so rows whose ad is gone get marked +// removed. Pure helpers, no #lib / $app imports; the fetching lives in scripts/house-scrape.ts. +// +// Only a clear signal counts as gone: a 404/410 on the listing's own page, the site's own +// "no longer available" redirect, or the page's own status saying rented/sold. Anything else +// (errors, timeouts, an unfamiliar page) is "unknown" and leaves the row alone. + +export const RECHECK_SITES = ['huispedia', 'huislijn', 'huurportaal', 'rentola'] as const; +export type RecheckSite = (typeof RECHECK_SITES)[number]; + +export type RecheckVerdict = { + verdict: 'alive' | 'gone' | 'unknown'; + reason: string; +}; + +const HOSTS: Record = { + 'huispedia.nl': 'huispedia', + 'huislijn.nl': 'huislijn', + 'huurportaal.nl': 'huurportaal', + 'rentola.nl': 'rentola' +}; + +function parse(url: string): URL | null { + try { + return new URL(url); + } catch { + return null; + } +} + +const hostOf = (u: URL) => u.hostname.toLowerCase().replace(/^www\./, ''); + +/** Which site a listing link belongs to, or null when it is not one we re-check. */ +export function recheckSite(url: string | null | undefined): RecheckSite | null { + const u = url ? parse(url) : null; + return u ? (HOSTS[hostOf(u)] ?? null) : null; +} + +/** SQL ILIKE pattern per site for rows whose link is on it (narrowed by recheckSite after). */ +export const RECHECK_URL_PATTERNS = Object.fromEntries( + Object.entries(HOSTS).map(([host, site]) => [site, `%://%${host}/%`]) +) as Record; + +const text = (html: string) => + html + .replace(/|/gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ /g, ' ') + .replace(/\s+/g, ' '); + +/** Huispedia's "Status" value from the features list ("Te huur", "Verkocht", ...). */ +export function huispediaStatus(html: string): string | null { + const m = html.match( + /\s*Status\s*<\/span>\s*\s*([^<]{1,120}?)\s*<\/span>/i + ); + return m ? m[1].replace(/\s+/g, ' ').trim() : null; +} + +const HUISPEDIA_ALIVE = /^(te huur|te koop|onder bod|onder optie|verkocht onder voorbehoud)\b/i; +const HUISPEDIA_GONE = + /^(verhuurd|verkocht\b(?! onder voorbehoud)|niet te koop|niet te huur|ingetrokken)/i; + +/** + * Decide from one fetch of `requestedUrl` whether the ad is still up. `finalUrl` is where + * redirects ended. Call only for pages that were not a challenge/block (the caller stops a + * site for the night on 403/429/503 or a challenge page). + */ +export function classifyListingPage( + site: RecheckSite, + requestedUrl: string, + res: { status: number; finalUrl: string; html: string } +): RecheckVerdict { + const requested = parse(requestedUrl); + const final = parse(res.finalUrl) ?? requested; + if (!requested || !final) return { verdict: 'unknown', reason: 'bad url' }; + const sameSite = recheckSite(final.href) === site; + + if ((res.status === 404 || res.status === 410) && sameSite) + return { verdict: 'gone', reason: `http ${res.status}` }; + if (res.status !== 200) return { verdict: 'unknown', reason: `http ${res.status}` }; + if (!sameSite) return { verdict: 'unknown', reason: `redirected off-site to ${final.host}` }; + + switch (site) { + case 'huurportaal': + case 'rentola': { + // Their own "this listing is no longer available" redirect to a search page. + if (final.searchParams.has('inactive_property_id')) + return { verdict: 'gone', reason: 'redirected as inactive' }; + if (final.pathname === requested.pathname && final.pathname.startsWith('/listings/')) + return { verdict: 'alive', reason: 'listing page' }; + return { verdict: 'unknown', reason: `redirected to ${final.pathname}` }; + } + case 'huislijn': { + const id = requested.pathname.match(/\/(\d{5,})\//)?.[1]; + if (!id) return { verdict: 'unknown', reason: 'no listing id in link' }; + // A removed ad redirects to the province/search overview, without the id. + if (!final.pathname.includes(`/${id}/`)) { + return /\/(huurwoning|koopwoning)\/nederland(\/[a-z-]+)?\/?$/.test(final.pathname) + ? { verdict: 'gone', reason: 'redirected to the overview' } + : { verdict: 'unknown', reason: `redirected to ${final.pathname}` }; + } + if (/geldt uitsluitend voor een woningruil/i.test(text(res.html))) + return { verdict: 'gone', reason: 'woningruil only (swap, not a rental)' }; + return { verdict: 'alive', reason: 'listing page' }; + } + case 'huispedia': { + // Huispedia keeps a page for every address; the ad's status is what changes. + if (final.pathname.replace(/\/$/, '') !== requested.pathname.replace(/\/$/, '')) + return { verdict: 'unknown', reason: `redirected to ${final.pathname}` }; + const status = huispediaStatus(res.html); + if (!status) return { verdict: 'unknown', reason: 'no status on page' }; + if (HUISPEDIA_ALIVE.test(status)) return { verdict: 'alive', reason: status }; + if (HUISPEDIA_GONE.test(status)) return { verdict: 'gone', reason: status }; + return { verdict: 'unknown', reason: `status "${status}"` }; + } + } +} + +/** + * Safety valve: a night where most checked rows of a site look gone is more likely a site + * change than a real exodus, so none of that night's removals for the site are applied. + */ +export function removalsLookSane(checked: number, gone: number): boolean { + if (gone === 0) return true; + if (checked < 10) return gone <= 3; + return gone <= Math.max(5, Math.floor(checked * 0.5)); +}