Stan Runge
113eb54cd9
All checks were successful
Deploy production / deploy (push) Successful in 2m33s
308 lines
11 KiB
TypeScript
308 lines
11 KiB
TypeScript
// Nightly house scraper, run on stan-vm by a systemd user timer (deploy/systemd/house-scrape.*).
|
|
// Fetches the sources that allow it, writes house.listings directly and pushes an ntfy alert
|
|
// for new listings that match an alert rule. See docs/house-tracker.md.
|
|
//
|
|
// bun --env-file=$HOME/.config/house-scrape.env scripts/house-scrape.ts [--dry-run] [--source kamernet]
|
|
//
|
|
// Politeness: one request at a time, 6-10 s apart, an honest User-Agent, robots.txt-allowed
|
|
// paths only. A 403/429/challenge stops that source for the night; nothing is retried or
|
|
// worked around. Funda, Pararius, huurwoningen.nl and Huurstunt challenge this server's
|
|
// (datacenter) IP, so they are not scraped from here.
|
|
import postgres from 'postgres';
|
|
import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
|
|
import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address';
|
|
import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
|
|
import { scoreListings } from '../src/lib/house-tracker/liveability';
|
|
import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
|
|
import { isChallenge } from '../src/lib/house-tracker/pararius';
|
|
import type { ScrapedListing } from '../src/lib/house-tracker/scraped';
|
|
import {
|
|
parseVestedaSearch,
|
|
VESTEDA_SEARCH_URL,
|
|
vestedaSearchBody
|
|
} from '../src/lib/house-tracker/vesteda';
|
|
|
|
const args = process.argv.slice(2);
|
|
const dryRun = args.includes('--dry-run');
|
|
const only = args.includes('--source') ? args[args.indexOf('--source') + 1] : null;
|
|
|
|
const USER_AGENT =
|
|
'stanrunge.dev-house-tracker/1.0 (personal use, one run per night; stan@stanrunge.dev)';
|
|
const KAMERNET_CITIES = (
|
|
process.env.HOUSE_KAMERNET_CITIES ??
|
|
'gouda,waddinxveen,reeuwijk,bodegraven,boskoop,schoonhoven,oudewater,haastrecht,moordrecht,zevenhuizen,nieuwerkerk-aan-den-ijssel'
|
|
)
|
|
.split(',')
|
|
.map((c) => c.trim())
|
|
.filter(Boolean);
|
|
// Everything in the region is stored (no price/size caps); paging stops early once a page has
|
|
// none of our cities, so this is only a safety limit.
|
|
const KAMERNET_MAX_PAGES = 25;
|
|
// Gouda centre; Vesteda's own search uses km radius around a point.
|
|
const VESTEDA = {
|
|
lat: 52.0115,
|
|
lng: 4.7105,
|
|
radiusKm: Number(process.env.HOUSE_VESTEDA_RADIUS_KM ?? 20),
|
|
place: 'Gouda'
|
|
};
|
|
const MAX_ALERTS = 5;
|
|
|
|
const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
|
|
if (!dbUrl && !dryRun) throw new Error('DB_URL is not set');
|
|
const sql = dbUrl ? postgres(dbUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
|
|
|
|
class Blocked extends Error {
|
|
constructor(
|
|
public status: number,
|
|
public url: string
|
|
) {
|
|
super(`blocked (${status}) on ${url}`);
|
|
}
|
|
}
|
|
|
|
let lastRequest = 0;
|
|
async function politeFetch(url: string, init: RequestInit = {}): Promise<Response> {
|
|
const wait = lastRequest + 6000 + Math.random() * 4000 - Date.now();
|
|
if (wait > 0) await Bun.sleep(wait);
|
|
lastRequest = Date.now();
|
|
const response = await fetch(url, {
|
|
...init,
|
|
redirect: 'follow',
|
|
signal: AbortSignal.timeout(30_000),
|
|
headers: {
|
|
'user-agent': USER_AGENT,
|
|
'accept-language': 'nl-NL,nl;q=0.9,en;q=0.5',
|
|
...(init.headers ?? {})
|
|
}
|
|
});
|
|
return response;
|
|
}
|
|
|
|
async function logRun(
|
|
source: string,
|
|
url: string,
|
|
status: string,
|
|
extra: { httpStatus?: number; found?: number; durationMs?: number; message?: string } = {}
|
|
) {
|
|
console.log(
|
|
`${source} ${status}${extra.httpStatus ? ` ${extra.httpStatus}` : ''} ${extra.found ?? ''} ${url}${extra.message ? ` (${extra.message})` : ''}`
|
|
);
|
|
if (!sql || dryRun) return;
|
|
await sql`INSERT INTO house.worker_runs (source, lane, url, status, http_status, found, duration_ms, message)
|
|
VALUES (${source}, 'nightly', ${url}, ${status}, ${extra.httpStatus ?? null}, ${extra.found ?? null},
|
|
${extra.durationMs ?? null}, ${extra.message?.slice(0, 500) ?? null})`;
|
|
}
|
|
|
|
type SourceResult = { listings: ScrapedListing[]; completed: boolean };
|
|
|
|
async function scrapeKamernet(): Promise<SourceResult> {
|
|
const listings: ScrapedListing[] = [];
|
|
const seen = new Set<string>();
|
|
const kamernetAllowed = new Set<string>([...GOUDA_REGION, ...KAMERNET_CITIES]);
|
|
let completed = true;
|
|
for (const city of KAMERNET_CITIES) {
|
|
for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) {
|
|
const url = kamernetSearchUrl(city, page);
|
|
const started = Date.now();
|
|
const response = await politeFetch(url, { headers: { accept: 'text/html' } });
|
|
const html = await response.text();
|
|
if (isChallenge(html, response.status)) {
|
|
await logRun('kamernet', url, 'challenged', { httpStatus: response.status });
|
|
throw new Blocked(response.status, url);
|
|
}
|
|
if (response.status === 404) {
|
|
// Kamernet has no page for some small places; that is not an error.
|
|
await logRun('kamernet', url, 'ok', {
|
|
httpStatus: 404,
|
|
found: 0,
|
|
message: 'no such city page'
|
|
});
|
|
break;
|
|
}
|
|
if (!response.ok) {
|
|
await logRun('kamernet', url, 'http-error', { httpStatus: response.status });
|
|
completed = false;
|
|
break;
|
|
}
|
|
try {
|
|
const result = parseKamernetSearch(html);
|
|
// Small places get padded with listings from all over the country; keep the
|
|
// region only, each listing once, and stop paging once a page has none of ours.
|
|
const local = result.listings.filter((l) => kamernetAllowed.has(citySlug(l.city)));
|
|
for (const l of local) {
|
|
if (seen.has(l.sourceId)) continue;
|
|
seen.add(l.sourceId);
|
|
listings.push(l);
|
|
}
|
|
await logRun('kamernet', url, 'ok', {
|
|
httpStatus: 200,
|
|
found: local.length,
|
|
durationMs: Date.now() - started,
|
|
message:
|
|
local.length < result.listings.length
|
|
? `${result.listings.length - local.length} outside the region ignored`
|
|
: undefined
|
|
});
|
|
const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1;
|
|
if (page >= pages || local.length === 0) break;
|
|
} catch (error) {
|
|
await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) });
|
|
completed = false;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return { listings, completed };
|
|
}
|
|
|
|
async function scrapeVesteda(): Promise<SourceResult> {
|
|
const started = Date.now();
|
|
const response = await politeFetch(VESTEDA_SEARCH_URL, {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json', accept: 'application/json' },
|
|
body: JSON.stringify(vestedaSearchBody(VESTEDA))
|
|
});
|
|
const text = await response.text();
|
|
if (isChallenge(text, response.status)) {
|
|
await logRun('vesteda', VESTEDA_SEARCH_URL, 'challenged', { httpStatus: response.status });
|
|
throw new Blocked(response.status, VESTEDA_SEARCH_URL);
|
|
}
|
|
if (!response.ok) {
|
|
await logRun('vesteda', VESTEDA_SEARCH_URL, 'http-error', { httpStatus: response.status });
|
|
return { listings: [], completed: false };
|
|
}
|
|
try {
|
|
const listings = parseVestedaSearch(JSON.parse(text));
|
|
await logRun('vesteda', VESTEDA_SEARCH_URL, 'ok', {
|
|
httpStatus: 200,
|
|
found: listings.length,
|
|
durationMs: Date.now() - started
|
|
});
|
|
return { listings, completed: true };
|
|
} catch (error) {
|
|
await logRun('vesteda', VESTEDA_SEARCH_URL, 'parse-error', {
|
|
httpStatus: 200,
|
|
message: String(error)
|
|
});
|
|
return { listings: [], completed: false };
|
|
}
|
|
}
|
|
|
|
async function ntfy(title: string, body: string, click: string) {
|
|
const target = process.env.HOUSE_NTFY_URL;
|
|
if (!target) return false;
|
|
const url = new URL(target);
|
|
const response = await fetch(url.origin, {
|
|
method: 'POST',
|
|
headers: {
|
|
'content-type': 'application/json',
|
|
...(process.env.HOUSE_NTFY_TOKEN
|
|
? { authorization: `Bearer ${process.env.HOUSE_NTFY_TOKEN}` }
|
|
: {})
|
|
},
|
|
body: JSON.stringify({
|
|
topic: url.pathname.replace(/^\/+|\/+$/g, ''),
|
|
title,
|
|
message: body,
|
|
click,
|
|
tags: ['house'],
|
|
priority: 3
|
|
}),
|
|
signal: AbortSignal.timeout(10_000)
|
|
}).catch(() => null);
|
|
return Boolean(response?.ok);
|
|
}
|
|
|
|
const sources: Record<string, () => Promise<SourceResult>> = {
|
|
kamernet: scrapeKamernet,
|
|
vesteda: scrapeVesteda
|
|
};
|
|
const fresh: { id: number; source: string; listing: ScrapedListing }[] = [];
|
|
let failures = 0;
|
|
try {
|
|
for (const [name, scrape] of Object.entries(sources)) {
|
|
if (only && only !== name) continue;
|
|
const runStartedAt = new Date();
|
|
let result: SourceResult;
|
|
try {
|
|
result = await scrape();
|
|
} catch (error) {
|
|
failures++;
|
|
console.error(
|
|
`${name}: ${error instanceof Error ? error.message : error}; skipped until tomorrow`
|
|
);
|
|
continue;
|
|
}
|
|
if (dryRun || !sql) {
|
|
for (const l of result.listings)
|
|
console.log(
|
|
` ${l.sourceId} €${l.priceEur ?? '?'} ${l.areaM2 ?? '?'}m² ${l.title} (${l.city}) ${l.url}`
|
|
);
|
|
continue;
|
|
}
|
|
// An empty result from a "complete" run is more likely a broken page than an empty market.
|
|
const completed = result.completed && result.listings.length > 0;
|
|
const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed });
|
|
console.log(
|
|
`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated${ingest.adopted ? ` (${ingest.adopted} hand-added rows taken over)` : ''}, ${ingest.removed} gone`
|
|
);
|
|
if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
|
|
}
|
|
|
|
if (sql && !dryRun) {
|
|
const { keyed, copies } = await dedupe(sql);
|
|
console.log(`dedupe: ${keyed} new address keys, ${copies} listings are copies of another`);
|
|
|
|
// Leefbaarometer score for new listings and changed addresses (PDOK + the LBM WFS).
|
|
try {
|
|
const lbm = await scoreListings(sql, { limit: 300 });
|
|
console.log(
|
|
`liveability: ${lbm.checked} checked, ${lbm.scored} scored, ${lbm.notFound} without a location, ${lbm.failed} failed`
|
|
);
|
|
} catch (error) {
|
|
console.error(`liveability: ${(error as Error).message}`);
|
|
}
|
|
|
|
const rules =
|
|
(await sql`SELECT id, name, enabled, offer, min_price AS "minPrice", max_price AS "maxPrice",
|
|
min_area AS "minArea", min_bedrooms AS "minBedrooms", cities, property_types AS "propertyTypes"
|
|
FROM house.alert_rules WHERE enabled`) as unknown as AlertRule[];
|
|
const copyIds = new Set(
|
|
fresh.length
|
|
? (
|
|
await sql<
|
|
{ id: number }[]
|
|
>`SELECT id FROM house.listings WHERE duplicate_of IS NOT NULL AND id IN ${sql(fresh.map((f) => f.id))}`
|
|
).map((r) => r.id)
|
|
: []
|
|
);
|
|
const matches = fresh.flatMap((f) => {
|
|
if (copyIds.has(f.id)) return [];
|
|
const listing = { ...f.listing, citySlug: citySlug(f.listing.city), source: f.source };
|
|
const rule = rules.find((r) => matchesRule(listing, r));
|
|
return rule ? [{ ...f, message: alertMessage(listing, rule) }] : [];
|
|
});
|
|
let sent = 0;
|
|
for (const match of matches.slice(0, MAX_ALERTS)) {
|
|
if (await ntfy(match.message.title, match.message.body, match.message.url)) {
|
|
sent++;
|
|
await sql`UPDATE house.listings SET alerted_at = now() WHERE id = ${match.id}`;
|
|
}
|
|
}
|
|
if (matches.length > MAX_ALERTS)
|
|
await ntfy(
|
|
`🏠 ${matches.length - MAX_ALERTS} more new listings`,
|
|
'See the house tracker.',
|
|
'https://stanrunge.dev/trackers/houses?status='
|
|
);
|
|
console.log(
|
|
`alerts: ${matches.length} new matches, ${sent} sent${process.env.HOUSE_NTFY_URL ? '' : ' (HOUSE_NTFY_URL not set)'}`
|
|
);
|
|
}
|
|
} finally {
|
|
await sql?.end();
|
|
}
|
|
|
|
// Non-zero so systemd shows the unit as failed when a source broke or blocked us.
|
|
if (failures > 0) process.exitCode = 1;
|