Stan Runge
fa3aa25f0b
All checks were successful
Deploy production / deploy (push) Successful in 1m15s
443 lines
16 KiB
TypeScript
443 lines
16 KiB
TypeScript
// Nightly house scraper, run on stan-ovh (prod) by a systemd user timer (deploy/systemd/house-scrape.*).
|
|
// Fetches the sources that allow it, writes house.listings directly and pushes an ntfy alert
|
|
// for new listings that match an alert rule. See docs/house-tracker.md.
|
|
//
|
|
// bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet]
|
|
// [--source recheck] [--recheck-limit 20]
|
|
//
|
|
// After the search sources it re-checks rows that came from one-off imports or the CLI
|
|
// (huispedia, huislijn, huurportaal, rentola) by fetching each listing's own page, oldest
|
|
// check first, within a time budget, and marks a row gone only on a clear signal (see
|
|
// src/lib/house-tracker/recheck.ts). Each site gets its own one-at-a-time 6-10 s queue.
|
|
//
|
|
// Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed
|
|
// paths only. A 403/429/challenge stops that source for the night; nothing is retried or
|
|
// worked around. Funda, Pararius, huurwoningen.nl and Huurstunt challenge this server's
|
|
// (datacenter) IP, so they are not scraped from here.
|
|
import postgres from 'postgres';
|
|
import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
|
|
import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address';
|
|
import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
|
|
import { scoreListings } from '../src/lib/house-tracker/liveability';
|
|
import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
|
|
import { isChallenge } from '../src/lib/house-tracker/pararius';
|
|
import {
|
|
classifyListingPage,
|
|
RECHECK_SITES,
|
|
RECHECK_URL_PATTERNS,
|
|
recheckSite,
|
|
removalsLookSane,
|
|
type RecheckSite
|
|
} from '../src/lib/house-tracker/recheck';
|
|
import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
|
|
import {
|
|
parseVestedaSearch,
|
|
VESTEDA_SEARCH_URL,
|
|
vestedaSearchBody
|
|
} from '../src/lib/house-tracker/vesteda';
|
|
|
|
const args = process.argv.slice(2);
|
|
const dryRun = args.includes('--dry-run');
|
|
const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null;
|
|
const recheckLimit = args.includes('--recheck-limit')
|
|
? Number(args[args.indexOf('--recheck-limit') + 1])
|
|
: 400;
|
|
const scriptStarted = Date.now();
|
|
// The unit has TimeoutStartSec=900; leave room for the search sources and the steps after.
|
|
const RECHECK_MINUTES = Number(process.env.HOUSE_RECHECK_MINUTES ?? 8);
|
|
const RECHECK_HARD_STOP_MS = 12 * 60_000;
|
|
|
|
const USER_AGENT =
|
|
'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
|
|
const KAMERNET_CITIES = (
|
|
process.env.HOUSE_KAMERNET_CITIES ??
|
|
'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,haastrecht,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
|
|
)
|
|
.split(',')
|
|
.map((c) => c.trim())
|
|
.filter(Boolean);
|
|
// Everything in the region is stored (no price/size caps); paging stops early once a page has
|
|
// none of our cities, so this is only a safety limit.
|
|
const KAMERNET_MAX_PAGES = 25;
|
|
// Gouda centre; Vesteda's own search uses km radius around a point.
|
|
const VESTEDA = {
|
|
lat: 52.0115,
|
|
lng: 4.7105,
|
|
radiusKm: Number(process.env.HOUSE_VESTEDA_RADIUS_KM ?? 20),
|
|
place: 'Gouda'
|
|
};
|
|
const MAX_ALERTS = 5;
|
|
|
|
const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
|
|
if (!dbUrl && !dryRun) throw new Error('DB_URL is not set');
|
|
const sql = dbUrl ? postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
|
|
|
|
class Blocked extends Error {
|
|
constructor(
|
|
public status: number,
|
|
public url: string
|
|
) {
|
|
super(`blocked (${status}) on ${url}`);
|
|
}
|
|
}
|
|
|
|
// One queue per site: requests to the same site stay one at a time, 6-10 s apart.
|
|
const lastRequest = new Map<string, number>();
|
|
async function politeFetch(
|
|
url: string,
|
|
init: RequestInit = {},
|
|
queue = 'search'
|
|
): Promise<Response> {
|
|
const wait = (lastRequest.get(queue) ?? 0) + 6000 + Math.random() * 4000 - Date.now();
|
|
if (wait > 0) await Bun.sleep(wait);
|
|
lastRequest.set(queue, Date.now());
|
|
const response = await fetch(url, {
|
|
...init,
|
|
redirect: 'follow',
|
|
signal: AbortSignal.timeout(30_000),
|
|
headers: {
|
|
'user-agent': USER_AGENT,
|
|
'accept-language': 'nl-NL,nl;q=0.9,en;q=0.5',
|
|
...(init.headers ?? {})
|
|
}
|
|
});
|
|
return response;
|
|
}
|
|
|
|
async function logRun(
|
|
source: string,
|
|
url: string,
|
|
status: string,
|
|
extra: { httpStatus?: number; found?: number; durationMs?: number; message?: string } = {}
|
|
) {
|
|
console.log(
|
|
`${source} ${status}${extra.httpStatus ? ` ${extra.httpStatus}` : ''} ${extra.found ?? ''} ${url}${extra.message ? ` (${extra.message})` : ''}`
|
|
);
|
|
if (!sql || dryRun) return;
|
|
await sql`INSERT INTO house.worker_runs (source, lane, url, status, http_status, found, duration_ms, message)
|
|
VALUES (${source}, 'nightly', ${url}, ${status}, ${extra.httpStatus ?? null}, ${extra.found ?? null},
|
|
${extra.durationMs ?? null}, ${extra.message?.slice(0, 500) ?? null})`;
|
|
}
|
|
|
|
type SourceResult = { listings: ScrapedListing[]; completed: boolean };
|
|
|
|
async function scrapeKamernet(): Promise<SourceResult> {
|
|
const listings: ScrapedListing[] = [];
|
|
const seen = new Set<string>();
|
|
const kamernetAllowed = new Set<string>([...GOUDA_REGION, ...KAMERNET_CITIES]);
|
|
let completed = true;
|
|
for (const city of KAMERNET_CITIES) {
|
|
for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) {
|
|
const url = kamernetSearchUrl(city, page);
|
|
const started = Date.now();
|
|
const response = await politeFetch(url, { headers: { accept: 'text/html' } });
|
|
const html = await response.text();
|
|
if (isChallenge(html, response.status)) {
|
|
await logRun('kamernet', url, 'challenged', { httpStatus: response.status });
|
|
throw new Blocked(response.status, url);
|
|
}
|
|
if (response.status === 404) {
|
|
// Kamernet has no page for some small places; that is not an error.
|
|
await logRun('kamernet', url, 'ok', {
|
|
httpStatus: 404,
|
|
found: 0,
|
|
message: 'no such city page'
|
|
});
|
|
break;
|
|
}
|
|
if (!response.ok) {
|
|
await logRun('kamernet', url, 'http-error', { httpStatus: response.status });
|
|
completed = false;
|
|
break;
|
|
}
|
|
try {
|
|
const result = parseKamernetSearch(html);
|
|
// Small places get padded with listings from all over the country; keep the
|
|
// region only, each listing once, and stop paging once a page has none of ours.
|
|
const local = result.listings.filter((l) => kamernetAllowed.has(citySlug(l.city)));
|
|
for (const l of local) {
|
|
if (seen.has(l.sourceId)) continue;
|
|
seen.add(l.sourceId);
|
|
listings.push(l);
|
|
}
|
|
await logRun('kamernet', url, 'ok', {
|
|
httpStatus: 200,
|
|
found: local.length,
|
|
durationMs: Date.now() - started,
|
|
message:
|
|
local.length < result.listings.length
|
|
? `${result.listings.length - local.length} outside the region ignored`
|
|
: undefined
|
|
});
|
|
const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1;
|
|
if (page >= pages || local.length === 0) break;
|
|
} catch (error) {
|
|
await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) });
|
|
completed = false;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return { listings, completed };
|
|
}
|
|
|
|
async function scrapeVesteda(): Promise<SourceResult> {
|
|
const started = Date.now();
|
|
const response = await politeFetch(VESTEDA_SEARCH_URL, {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json', accept: 'application/json' },
|
|
body: JSON.stringify(vestedaSearchBody(VESTEDA))
|
|
});
|
|
const text = await response.text();
|
|
if (isChallenge(text, response.status)) {
|
|
await logRun('vesteda', VESTEDA_SEARCH_URL, 'challenged', { httpStatus: response.status });
|
|
throw new Blocked(response.status, VESTEDA_SEARCH_URL);
|
|
}
|
|
if (!response.ok) {
|
|
await logRun('vesteda', VESTEDA_SEARCH_URL, 'http-error', { httpStatus: response.status });
|
|
return { listings: [], completed: false };
|
|
}
|
|
try {
|
|
const listings = parseVestedaSearch(JSON.parse(text));
|
|
await logRun('vesteda', VESTEDA_SEARCH_URL, 'ok', {
|
|
httpStatus: 200,
|
|
found: listings.length,
|
|
durationMs: Date.now() - started
|
|
});
|
|
return { listings, completed: true };
|
|
} catch (error) {
|
|
await logRun('vesteda', VESTEDA_SEARCH_URL, 'parse-error', {
|
|
httpStatus: 200,
|
|
message: String(error)
|
|
});
|
|
return { listings: [], completed: false };
|
|
}
|
|
}
|
|
|
|
type RecheckRow = { id: number; url: string; title: string };
|
|
type RecheckSummary = { checked: number; alive: number; gone: number; unknown: number };
|
|
|
|
/** Fetch each row's own page for one site until the deadline; a block stops the site. */
|
|
async function recheckSiteRows(site: RecheckSite, rows: RecheckRow[], deadline: number) {
|
|
const summary: RecheckSummary = { checked: 0, alive: 0, gone: 0, unknown: 0 };
|
|
const alive: number[] = [];
|
|
const gone: { row: RecheckRow; reason: string }[] = [];
|
|
let blocked = false;
|
|
for (const row of rows) {
|
|
if (Date.now() + 12_000 > deadline) break;
|
|
let page: { status: number; finalUrl: string; html: string };
|
|
try {
|
|
const response = await politeFetch(row.url, { headers: { accept: 'text/html' } }, site);
|
|
page = {
|
|
status: response.status,
|
|
finalUrl: response.url || row.url,
|
|
html: await response.text()
|
|
};
|
|
} catch (error) {
|
|
summary.checked++;
|
|
summary.unknown++;
|
|
console.log(` ${site} #${row.id} unknown (${(error as Error).message}) ${row.url}`);
|
|
continue;
|
|
}
|
|
const { status, html } = page;
|
|
if (isChallenge(html, status)) {
|
|
await logRun(site, row.url, 'challenged', { httpStatus: status, message: 'recheck' });
|
|
blocked = true;
|
|
break;
|
|
}
|
|
summary.checked++;
|
|
const verdict = classifyListingPage(site, row.url, page);
|
|
summary[verdict.verdict]++;
|
|
if (verdict.verdict === 'alive') alive.push(row.id);
|
|
if (verdict.verdict === 'gone') gone.push({ row, reason: verdict.reason });
|
|
if (verdict.verdict !== 'alive' || dryRun)
|
|
console.log(
|
|
` ${site} #${row.id} ${verdict.verdict} (${verdict.reason}) ${row.title} ${row.url}`
|
|
);
|
|
}
|
|
return { summary, alive, gone, blocked };
|
|
}
|
|
|
|
async function recheckListings(): Promise<boolean> {
|
|
if (!sql) return true;
|
|
const started = Date.now();
|
|
const deadline = Math.min(
|
|
started + RECHECK_MINUTES * 60_000,
|
|
scriptStarted + RECHECK_HARD_STOP_MS
|
|
);
|
|
// Per site, oldest-checked first, so every row gets its turn over a few nights.
|
|
const bySite = new Map<RecheckSite, RecheckRow[]>();
|
|
for (const site of RECHECK_SITES) {
|
|
const rows = await sql<RecheckRow[]>`SELECT id, url, title FROM house.listings
|
|
WHERE removed_at IS NULL AND url ILIKE ${RECHECK_URL_PATTERNS[site]}
|
|
ORDER BY last_seen_at ASC NULLS FIRST, id
|
|
LIMIT ${recheckLimit}`;
|
|
const own = rows.filter((r) => recheckSite(r.url) === site);
|
|
if (own.length) bySite.set(site, own);
|
|
}
|
|
let ok = true;
|
|
const results = await Promise.all(
|
|
[...bySite].map(async ([site, list]) => ({
|
|
site,
|
|
...(await recheckSiteRows(site, list, deadline))
|
|
}))
|
|
);
|
|
for (const { site, summary, alive, gone, blocked } of results) {
|
|
const sane = removalsLookSane(summary.checked, gone.length);
|
|
let applied = 0;
|
|
if (!dryRun) {
|
|
if (alive.length)
|
|
await sql`UPDATE house.listings SET last_seen_at = now() WHERE id IN ${sql(alive)}`;
|
|
if (gone.length && sane) {
|
|
const done = await sql`UPDATE house.listings SET removed_at = now()
|
|
WHERE removed_at IS NULL AND id IN ${sql(gone.map((g) => g.row.id))} RETURNING id`;
|
|
applied = done.length;
|
|
}
|
|
}
|
|
if (blocked) ok = false;
|
|
const message = `${summary.alive} alive, ${summary.gone} gone${
|
|
!sane
|
|
? ' (NOT applied: too many for one night, check the site)'
|
|
: dryRun
|
|
? ' (dry run)'
|
|
: `, ${applied} marked removed`
|
|
}, ${summary.unknown} unknown${blocked ? '; blocked, stopped for tonight' : ''}`;
|
|
await logRun(site, 'recheck', !sane ? 'suspicious' : blocked ? 'challenged' : 'ok', {
|
|
found: summary.checked,
|
|
durationMs: Date.now() - started,
|
|
message
|
|
});
|
|
console.log(
|
|
`recheck ${site}: ${summary.checked} of ${bySite.get(site)?.length ?? 0} checked, ${message}`
|
|
);
|
|
}
|
|
return ok;
|
|
}
|
|
|
|
async function ntfy(title: string, body: string, click: string) {
|
|
const target = process.env.HOUSE_NTFY_URL;
|
|
if (!target) return false;
|
|
const url = new URL(target);
|
|
const response = await fetch(url.origin, {
|
|
method: 'POST',
|
|
headers: {
|
|
'content-type': 'application/json',
|
|
...(process.env.HOUSE_NTFY_TOKEN
|
|
? { authorization: `Bearer ${process.env.HOUSE_NTFY_TOKEN}` }
|
|
: {})
|
|
},
|
|
body: JSON.stringify({
|
|
topic: url.pathname.replace(/^\/+|\/+$/g, ''),
|
|
title,
|
|
message: body,
|
|
click,
|
|
tags: ['house'],
|
|
priority: 3
|
|
}),
|
|
signal: AbortSignal.timeout(10_000)
|
|
}).catch(() => null);
|
|
return Boolean(response?.ok);
|
|
}
|
|
|
|
const sources: Record<string, () => Promise<SourceResult>> = {
|
|
kamernet: scrapeKamernet,
|
|
vesteda: scrapeVesteda
|
|
};
|
|
const fresh: { id: number; source: string; listing: ScrapedListing }[] = [];
|
|
let failures = 0;
|
|
try {
|
|
for (const [name, scrape] of Object.entries(sources)) {
|
|
if (only && only !== name) continue;
|
|
const runStartedAt = new Date();
|
|
let result: SourceResult;
|
|
try {
|
|
result = await scrape();
|
|
} catch (error) {
|
|
failures++;
|
|
console.error(
|
|
`${name}: ${error instanceof Error ? error.message : error}; skipped until tomorrow`
|
|
);
|
|
continue;
|
|
}
|
|
if (dryRun || !sql) {
|
|
for (const l of result.listings)
|
|
console.log(
|
|
` ${l.sourceId} €${l.priceEur ?? '?'} ${l.areaM2 ?? '?'}m² ${l.title} (${l.city}) ${l.url}`
|
|
);
|
|
continue;
|
|
}
|
|
// An empty result from a "complete" run is more likely a broken page than an empty market.
|
|
const completed = result.completed && result.listings.length > 0;
|
|
const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed });
|
|
console.log(
|
|
`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated${ingest.adopted ? ` (${ingest.adopted} hand-added rows taken over)` : ''}, ${ingest.removed} gone`
|
|
);
|
|
if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
|
|
}
|
|
|
|
if (!only || only === 'recheck') {
|
|
try {
|
|
if (!(await recheckListings())) failures++;
|
|
} catch (error) {
|
|
failures++;
|
|
console.error(`recheck: ${(error as Error).message}`);
|
|
}
|
|
}
|
|
|
|
if (sql && !dryRun) {
|
|
const { keyed, copies } = await dedupe(sql);
|
|
console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);
|
|
|
|
// Leefbaarometer score for new listings and changed addresses (PDOK + the LBM WFS).
|
|
try {
|
|
const lbm = await scoreListings(sql, { limit: 300 });
|
|
console.log(
|
|
`liveability: ${lbm.checked} checked, ${lbm.scored} scored, ${lbm.notFound} without a location, ${lbm.failed} failed`
|
|
);
|
|
} catch (error) {
|
|
console.error(`liveability: ${(error as Error).message}`);
|
|
}
|
|
|
|
const rules =
|
|
(await sql`SELECT id, name, enabled, offer, min_price AS "minPrice", max_price AS "maxPrice",
|
|
min_area AS "minArea", min_bedrooms AS "minBedrooms", cities, property_types AS "propertyTypes"
|
|
FROM house.alert_rules WHERE enabled`) as unknown as AlertRule[];
|
|
const copyIds = new Set(
|
|
fresh.length
|
|
? (
|
|
await sql<
|
|
{ id: number }[]
|
|
>`SELECT id FROM house.listings WHERE duplicate_of IS NOT NULL AND id IN ${sql(fresh.map((f) => f.id))}`
|
|
).map((r) => r.id)
|
|
: []
|
|
);
|
|
const matches = fresh.flatMap((f) => {
|
|
if (copyIds.has(f.id)) return [];
|
|
const listing = { ...f.listing, citySlug: citySlug(f.listing.city), source: f.source };
|
|
const rule = rules.find((r) => matchesRule(listing, r));
|
|
return rule ? [{ ...f, message: alertMessage(listing, rule) }] : [];
|
|
});
|
|
let sent = 0;
|
|
for (const match of matches.slice(0, MAX_ALERTS)) {
|
|
if (await ntfy(match.message.title, match.message.body, match.message.url)) {
|
|
sent++;
|
|
await sql`UPDATE house.listings SET alerted_at = now() WHERE id = ${match.id}`;
|
|
}
|
|
}
|
|
if (matches.length > MAX_ALERTS)
|
|
await ntfy(
|
|
`🏠 ${matches.length - MAX_ALERTS} more new listings`,
|
|
'See the house tracker.',
|
|
'https://stanrunge.dev/trackers/houses?status='
|
|
);
|
|
console.log(
|
|
`alerts: ${matches.length} new matches, ${sent} sent${process.env.HOUSE_NTFY_URL ? '' : ' (HOUSE_NTFY_URL not set)'}`
|
|
);
|
|
}
|
|
} finally {
|
|
await sql?.end();
|
|
}
|
|
|
|
// Non-zero so systemd shows the unit as failed when a source broke or blocked us.
|
|
if (failures > 0) process.exitCode = 1;
|