feat: house tracker with home worker, alerts and listing map
Collects Pararius rent/buy listings into Postgres via a worker that runs on a residential connection, sends Discord/ntfy alerts for new listings matching rules, and shows an admin-only map, market stats and worker log. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013HfF5tnChi8tkCx9svJP6D
This commit is contained in:
383
scripts/house-worker.ts
Normal file
383
scripts/house-worker.ts
Normal file
@@ -0,0 +1,383 @@
|
||||
/**
|
||||
* House tracker home worker. Runs on a machine at home (residential IP), polls
|
||||
* Pararius politely and POSTs what it finds to the website, which stores it and
|
||||
* sends the alerts. See docs/house-tracker.md.
|
||||
*
|
||||
* HOUSE_API_URL=https://stanrunge.dev HOUSE_WORKER_TOKEN=... bun scripts/house-worker.ts
|
||||
* bun scripts/house-worker.ts --once # one fast-lane poll, then exit
|
||||
* bun scripts/house-worker.ts --dry-run # fetch + parse, print, send nothing
|
||||
*
|
||||
* Politeness rules, in order of importance:
|
||||
* - one request at a time, at least HOUSE_MIN_GAP_SECONDS apart (with jitter)
|
||||
* - a daily request budget
|
||||
* - any challenge / 403 / 429 stops everything and backs off exponentially
|
||||
* - the fast lane only polls often on weekdays 07:00–18:00 Amsterdam time
|
||||
*/
|
||||
import {
|
||||
isChallenge,
|
||||
parseDetailPage,
|
||||
parseSearchPage,
|
||||
searchPageUrl
|
||||
} from '../src/lib/house-tracker/pararius';
|
||||
import { backoffMs, nextDelayMs } from '../src/lib/house-tracker/schedule';
|
||||
import type {
|
||||
Lane,
|
||||
ListingDetail,
|
||||
ReportRequest,
|
||||
RunStatus,
|
||||
SyncRequest,
|
||||
SyncResponse
|
||||
} from '../src/lib/house-tracker/types';
|
||||
|
||||
const args = new Set(process.argv.slice(2));
|
||||
const once = args.has('--once');
|
||||
const dryRun = args.has('--dry-run') || process.env.HOUSE_DRY_RUN === '1';
|
||||
|
||||
const list = (value: string | undefined, fallback: string[]) =>
|
||||
value
|
||||
? value
|
||||
.split(/[\s,]+/)
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean)
|
||||
: fallback;
|
||||
const number = (value: string | undefined, fallback: number) =>
|
||||
value && Number.isFinite(Number(value)) ? Number(value) : fallback;
|
||||
|
||||
const config = {
|
||||
api: (process.env.HOUSE_API_URL ?? '').replace(/\/$/, ''),
|
||||
token: process.env.HOUSE_WORKER_TOKEN ?? '',
|
||||
fastSearches: list(process.env.HOUSE_FAST_SEARCHES, [
|
||||
'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
|
||||
]),
|
||||
slowSearches: list(
|
||||
process.env.HOUSE_SLOW_SEARCHES,
|
||||
['gouda', 'rotterdam', 'den-haag', 'utrecht', 'amsterdam', 'eindhoven'].flatMap((city) => [
|
||||
`https://www.pararius.nl/huurwoningen/${city}`,
|
||||
`https://www.pararius.nl/koopwoningen/${city}`
|
||||
])
|
||||
),
|
||||
slowEveryHours: number(process.env.HOUSE_SLOW_EVERY_HOURS, 12),
|
||||
slowMaxPages: number(process.env.HOUSE_SLOW_MAX_PAGES, 80),
|
||||
minGapMs: number(process.env.HOUSE_MIN_GAP_SECONDS, 8) * 1000,
|
||||
dailyBudget: number(process.env.HOUSE_DAILY_BUDGET, 1500),
|
||||
fetchMode: (process.env.HOUSE_FETCH_MODE ?? 'fetch') as 'fetch' | 'browser',
|
||||
userAgent:
|
||||
process.env.HOUSE_USER_AGENT ??
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
|
||||
};
|
||||
|
||||
if (!dryRun && (!config.api || config.token.length < 32))
|
||||
throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required');
|
||||
|
||||
const SOURCE = 'pararius';
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const log = (...parts: unknown[]) => console.log(new Date().toISOString(), ...parts);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Polite fetching
|
||||
|
||||
class Challenged extends Error {
|
||||
constructor(
|
||||
readonly url: string,
|
||||
readonly status: number
|
||||
) {
|
||||
super(`Challenged (${status}) on ${url}`);
|
||||
}
|
||||
}
|
||||
|
||||
const cookies = new Map<string, string>();
|
||||
let lastRequestAt = 0;
|
||||
let budgetDay = '';
|
||||
let budgetUsed = 0;
|
||||
|
||||
function remember(response: Response) {
|
||||
for (const header of response.headers.getSetCookie?.() ?? []) {
|
||||
const [pair] = header.split(';');
|
||||
const index = pair.indexOf('=');
|
||||
if (index > 0) cookies.set(pair.slice(0, index).trim(), pair.slice(index + 1).trim());
|
||||
}
|
||||
}
|
||||
|
||||
function resetBudget() {
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
if (day !== budgetDay) {
|
||||
budgetDay = day;
|
||||
budgetUsed = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** The slow lane may only use part of the budget so the fast lane never runs dry. */
|
||||
function slowBudgetLeft() {
|
||||
resetBudget();
|
||||
return budgetUsed < config.dailyBudget * 0.6;
|
||||
}
|
||||
|
||||
async function waitForTurn() {
|
||||
resetBudget();
|
||||
if (budgetUsed >= config.dailyBudget) throw new Error('Daily request budget used up');
|
||||
const gap = config.minGapMs * (0.8 + Math.random() * 0.7);
|
||||
const wait = lastRequestAt + gap - Date.now();
|
||||
if (wait > 0) await sleep(wait);
|
||||
lastRequestAt = Date.now();
|
||||
budgetUsed++;
|
||||
}
|
||||
|
||||
type BrowserPage = {
|
||||
goto(url: string, options: object): Promise<{ status(): number } | null>;
|
||||
content(): Promise<string>;
|
||||
};
|
||||
let browserPage: Promise<BrowserPage> | null = null;
|
||||
|
||||
async function openBrowser(): Promise<BrowserPage> {
|
||||
// Real Chromium for when plain requests get challenged. Needs
|
||||
// `npx playwright install chromium` on the worker machine.
|
||||
const { chromium } = await import('@playwright/test');
|
||||
const context = await chromium.launchPersistentContext(
|
||||
process.env.HOUSE_BROWSER_PROFILE ?? '.house-worker-profile',
|
||||
{ headless: true, locale: 'nl-NL', timezoneId: 'Europe/Amsterdam' }
|
||||
);
|
||||
return context.pages()[0] ?? (await context.newPage());
|
||||
}
|
||||
|
||||
async function getHtml(url: string, referer?: string): Promise<{ html: string; status: number }> {
|
||||
await waitForTurn();
|
||||
if (config.fetchMode === 'browser') {
|
||||
browserPage ??= openBrowser();
|
||||
const page = await browserPage;
|
||||
const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45_000 });
|
||||
const html = await page.content();
|
||||
const status = response?.status() ?? 0;
|
||||
if (isChallenge(html, status)) throw new Challenged(url, status);
|
||||
return { html, status };
|
||||
}
|
||||
const response = await fetch(url, {
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
headers: {
|
||||
'user-agent': config.userAgent,
|
||||
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'accept-language': 'nl-NL,nl;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
'cache-control': 'no-cache',
|
||||
...(referer ? { referer } : {}),
|
||||
...(cookies.size
|
||||
? { cookie: [...cookies].map(([key, value]) => `${key}=${value}`).join('; ') }
|
||||
: {})
|
||||
}
|
||||
});
|
||||
remember(response);
|
||||
const html = await response.text();
|
||||
if (isChallenge(html, response.status)) throw new Challenged(url, response.status);
|
||||
return { html, status: response.status };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Website API
|
||||
|
||||
async function api<T>(path: string, body: unknown): Promise<T | null> {
|
||||
if (dryRun) return null;
|
||||
for (let attempt = 0; attempt < 4; attempt++) {
|
||||
try {
|
||||
const response = await fetch(`${config.api}/projects/house-tracker/api/worker/${path}`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', authorization: `Bearer ${config.token}` },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
});
|
||||
if (response.ok) return (await response.json()) as T;
|
||||
const text = await response.text();
|
||||
if (response.status < 500)
|
||||
throw new Error(`API ${path} ${response.status}: ${text.slice(0, 300)}`);
|
||||
log(`API ${path} ${response.status}, retrying`);
|
||||
} catch (error) {
|
||||
if (attempt === 3) throw error;
|
||||
log(`API ${path} failed: ${(error as Error).message}, retrying`);
|
||||
}
|
||||
await sleep(5000 * 2 ** attempt);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function report(
|
||||
lane: Lane,
|
||||
url: string,
|
||||
status: RunStatus,
|
||||
extra: Partial<ReportRequest> = {}
|
||||
) {
|
||||
try {
|
||||
await api('report', { source: SOURCE, lane, url, status, ...extra });
|
||||
} catch (error) {
|
||||
log('Could not report:', (error as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Lanes
|
||||
|
||||
async function fetchDetails(
|
||||
lane: Lane,
|
||||
wanted: { sourceId: string; url: string }[],
|
||||
referer: string
|
||||
) {
|
||||
const details: ListingDetail[] = [];
|
||||
for (const item of wanted) {
|
||||
const { html } = await getHtml(item.url, referer);
|
||||
const detail = parseDetailPage(html, item.url);
|
||||
if (detail) details.push(detail);
|
||||
else await report(lane, item.url, 'parse-error', { message: 'listing page did not parse' });
|
||||
}
|
||||
if (dryRun) {
|
||||
for (const detail of details) {
|
||||
log('detail', { ...detail, description: undefined, features: undefined });
|
||||
}
|
||||
}
|
||||
if (details.length) await api('details', { source: SOURCE, details });
|
||||
}
|
||||
|
||||
async function pollSearch(
|
||||
lane: Lane,
|
||||
url: string,
|
||||
completedSweep?: SyncRequest['completedSweep']
|
||||
): Promise<{ nextPageUrl: string | null; found: number }> {
|
||||
const startedAt = new Date().toISOString();
|
||||
const { html } = await getHtml(url);
|
||||
const page = parseSearchPage(html, url);
|
||||
if (!page.cards.length && page.nextPageUrl)
|
||||
await report(lane, url, 'parse-error', {
|
||||
message: 'no listing cards on a page with a next page'
|
||||
});
|
||||
if (dryRun) {
|
||||
log(`${lane} ${url}: ${page.cards.length} cards, next=${page.nextPageUrl}`);
|
||||
for (const card of page.cards.slice(0, 5)) log('card', card);
|
||||
}
|
||||
const sync = await api<SyncResponse>('sync', {
|
||||
source: SOURCE,
|
||||
lane,
|
||||
searchUrl: url,
|
||||
startedAt,
|
||||
cards: page.cards,
|
||||
completedSweep: page.nextPageUrl ? null : (completedSweep ?? null)
|
||||
} satisfies SyncRequest);
|
||||
if (sync) {
|
||||
if (sync.newIds.length || sync.alerted)
|
||||
log(`${lane} ${url}: ${sync.newIds.length} new, ${sync.alerted} alerted`);
|
||||
if (sync.needDetails.length) await fetchDetails(lane, sync.needDetails, url);
|
||||
} else if (dryRun && page.cards[0]) {
|
||||
await fetchDetails(lane, [page.cards[0]], url);
|
||||
}
|
||||
return { nextPageUrl: page.nextPageUrl, found: page.cards.length };
|
||||
}
|
||||
|
||||
async function fastTick() {
|
||||
for (const url of config.fastSearches) await pollSearch('fast', url);
|
||||
}
|
||||
|
||||
/** Only an unfiltered `/huurwoningen/<city>` sweep can tell us what disappeared. */
|
||||
function sweepScope(url: string) {
|
||||
const [kind, city, ...rest] = new URL(url).pathname.split('/').filter(Boolean);
|
||||
if (rest.length || !city) return null;
|
||||
if (kind === 'huurwoningen') return { offer: 'rent' as const, city };
|
||||
if (kind === 'koopwoningen') return { offer: 'buy' as const, city };
|
||||
return null;
|
||||
}
|
||||
|
||||
type Sweep = { index: number; page: number; startedAt: string };
|
||||
let sweep: Sweep | null = null;
|
||||
|
||||
/** One page of the slow lane, so fast polls can interleave. Returns false when idle. */
|
||||
async function slowStep(): Promise<boolean> {
|
||||
if (!sweep || !slowBudgetLeft()) return false;
|
||||
const base = config.slowSearches[sweep.index];
|
||||
if (!base) {
|
||||
log('Slow sweep finished');
|
||||
sweep = null;
|
||||
return false;
|
||||
}
|
||||
const url = searchPageUrl(base, sweep.page);
|
||||
const scope = sweepScope(base);
|
||||
const result = await pollSearch(
|
||||
'slow',
|
||||
url,
|
||||
scope ? { ...scope, sweepStartedAt: sweep.startedAt } : null
|
||||
);
|
||||
if (result.nextPageUrl && sweep.page < config.slowMaxPages) {
|
||||
sweep.page++;
|
||||
} else {
|
||||
sweep = { index: sweep.index + 1, page: 1, startedAt: new Date().toISOString() };
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main loop
|
||||
|
||||
let stopping = false;
|
||||
process.on('SIGTERM', () => (stopping = true));
|
||||
process.on('SIGINT', () => (stopping = true));
|
||||
|
||||
async function guarded(
|
||||
lane: Lane,
|
||||
step: () => Promise<boolean | void>
|
||||
): Promise<'ok' | 'idle' | 'failed'> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
return (await step()) === false ? 'idle' : 'ok';
|
||||
} catch (error) {
|
||||
if (error instanceof Challenged) {
|
||||
log(`CHALLENGED on ${error.url} (${error.status})`);
|
||||
await report(lane, error.url, 'challenged', {
|
||||
httpStatus: error.status,
|
||||
durationMs: Date.now() - started
|
||||
});
|
||||
} else {
|
||||
log(`${lane} failed:`, (error as Error).message);
|
||||
await report(lane, config.fastSearches[0], 'network-error', {
|
||||
message: (error as Error).message.slice(0, 500),
|
||||
durationMs: Date.now() - started
|
||||
});
|
||||
}
|
||||
return 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
log(
|
||||
`house worker: ${config.fastSearches.length} fast, ${config.slowSearches.length} slow searches, mode=${config.fetchMode}${dryRun ? ', dry run' : ''}`
|
||||
);
|
||||
if (once) {
|
||||
const result = await guarded('fast', fastTick);
|
||||
process.exit(result === 'ok' ? 0 : 1);
|
||||
}
|
||||
|
||||
let failures = 0;
|
||||
let nextFast = Date.now();
|
||||
let nextSlow = Date.now() + 5 * 60_000; // let the fast lane go first
|
||||
while (!stopping) {
|
||||
const now = Date.now();
|
||||
let result: 'ok' | 'idle' | 'failed' = 'idle';
|
||||
if (now >= nextFast) {
|
||||
result = await guarded('fast', fastTick);
|
||||
nextFast = Date.now() + nextDelayMs(new Date());
|
||||
} else if (sweep || now >= nextSlow) {
|
||||
if (!sweep) {
|
||||
log('Slow sweep starting');
|
||||
sweep = { index: 0, page: 1, startedAt: new Date().toISOString() };
|
||||
nextSlow = now + config.slowEveryHours * 60 * 60_000;
|
||||
}
|
||||
result = await guarded('slow', slowStep);
|
||||
}
|
||||
if (result === 'failed') {
|
||||
failures++;
|
||||
const pause = backoffMs(failures);
|
||||
log(`Backing off ${Math.round(pause / 60_000)} min (failure ${failures})`);
|
||||
nextFast = Math.max(nextFast, Date.now() + pause);
|
||||
await sleep(pause);
|
||||
continue;
|
||||
}
|
||||
if (result === 'ok') failures = 0;
|
||||
else await sleep(Math.min(30_000, Math.max(5_000, nextFast - Date.now())));
|
||||
}
|
||||
log('Stopped');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
await main();
|
||||
Reference in New Issue
Block a user