383 lines
12 KiB
TypeScript
383 lines
12 KiB
TypeScript
/**
|
||
* House tracker home worker. Runs on a machine at home (residential IP), polls
|
||
* Pararius politely and POSTs what it finds to the website, which stores it and
|
||
* sends the alerts. See docs/house-tracker.md.
|
||
*
|
||
* HOUSE_API_URL=https://stanrunge.dev HOUSE_WORKER_TOKEN=... bun scripts/house-worker.ts
|
||
* bun scripts/house-worker.ts --once # one fast-lane poll, then exit
|
||
* bun scripts/house-worker.ts --dry-run # fetch + parse, print, send nothing
|
||
* bun scripts/house-worker.ts --check-api # check website auth only, no Pararius requests
|
||
*
|
||
* Politeness rules, in order of importance:
|
||
* - one request at a time, at least HOUSE_MIN_GAP_SECONDS apart (with jitter)
|
||
* - a daily request budget
|
||
* - any challenge / 403 / 429 stops everything and backs off exponentially
|
||
* - the fast lane only polls often on weekdays 07:00–18:00 Amsterdam time
|
||
*/
|
||
import {
|
||
isChallenge,
|
||
parseDetailPage,
|
||
parseSearchPage,
|
||
searchPageUrl
|
||
} from '../src/lib/house-tracker/pararius';
|
||
import { backoffMs, nextDelayMs } from '../src/lib/house-tracker/schedule';
|
||
import { createWorkerApi } from '../src/lib/house-tracker/worker-api';
|
||
import type {
|
||
Lane,
|
||
ListingDetail,
|
||
ReportRequest,
|
||
RunStatus,
|
||
SyncRequest,
|
||
SyncResponse
|
||
} from '../src/lib/house-tracker/types';
|
||
|
||
const args = new Set(process.argv.slice(2));
|
||
const once = args.has('--once');
|
||
const checkApi = args.has('--check-api');
|
||
const dryRun = args.has('--dry-run') || process.env.HOUSE_DRY_RUN === '1';
|
||
|
||
const list = (value: string | undefined, fallback: string[]) =>
|
||
value
|
||
? value
|
||
.split(/[\s,]+/)
|
||
.map((item) => item.trim())
|
||
.filter(Boolean)
|
||
: fallback;
|
||
const number = (value: string | undefined, fallback: number) =>
|
||
value && Number.isFinite(Number(value)) ? Number(value) : fallback;
|
||
|
||
const config = {
|
||
api: (process.env.HOUSE_API_URL ?? '').replace(/\/$/, ''),
|
||
token: process.env.HOUSE_WORKER_TOKEN ?? '',
|
||
fastSearches: list(process.env.HOUSE_FAST_SEARCHES, [
|
||
'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
|
||
]),
|
||
slowSearches: list(
|
||
process.env.HOUSE_SLOW_SEARCHES,
|
||
['gouda', 'rotterdam', 'den-haag', 'utrecht', 'amsterdam', 'eindhoven'].flatMap((city) => [
|
||
`https://www.pararius.nl/huurwoningen/${city}`,
|
||
`https://www.pararius.nl/koopwoningen/${city}`
|
||
])
|
||
),
|
||
slowEveryHours: number(process.env.HOUSE_SLOW_EVERY_HOURS, 12),
|
||
slowMaxPages: number(process.env.HOUSE_SLOW_MAX_PAGES, 80),
|
||
minGapMs: Math.max(8, number(process.env.HOUSE_MIN_GAP_SECONDS, 8)) * 1000,
|
||
dailyBudget: number(process.env.HOUSE_DAILY_BUDGET, 1500),
|
||
fetchMode: (process.env.HOUSE_FETCH_MODE ?? 'fetch') as 'fetch' | 'browser',
|
||
userAgent:
|
||
process.env.HOUSE_USER_AGENT ??
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
|
||
};
|
||
|
||
if (checkApi && dryRun) throw new Error('--check-api cannot be combined with dry-run mode');
|
||
const requestApi = dryRun
|
||
? null
|
||
: createWorkerApi(
|
||
{
|
||
url: config.api,
|
||
token: config.token,
|
||
accessClientId: process.env.HOUSE_CF_ACCESS_CLIENT_ID,
|
||
accessClientSecret: process.env.HOUSE_CF_ACCESS_CLIENT_SECRET
|
||
},
|
||
checkApi ? 1 : 4
|
||
);
|
||
|
||
const SOURCE = 'pararius';
|
||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||
const log = (...parts: unknown[]) => console.log(new Date().toISOString(), ...parts);
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Polite fetching
|
||
|
||
class Challenged extends Error {
|
||
constructor(
|
||
readonly url: string,
|
||
readonly status: number
|
||
) {
|
||
super(`Challenged (${status}) on ${url}`);
|
||
}
|
||
}
|
||
|
||
const cookies = new Map<string, string>();
|
||
let lastRequestAt = 0;
|
||
let budgetDay = '';
|
||
let budgetUsed = 0;
|
||
|
||
function remember(response: Response) {
|
||
for (const header of response.headers.getSetCookie?.() ?? []) {
|
||
const [pair] = header.split(';');
|
||
const index = pair.indexOf('=');
|
||
if (index > 0) cookies.set(pair.slice(0, index).trim(), pair.slice(index + 1).trim());
|
||
}
|
||
}
|
||
|
||
function resetBudget() {
|
||
const day = new Date().toISOString().slice(0, 10);
|
||
if (day !== budgetDay) {
|
||
budgetDay = day;
|
||
budgetUsed = 0;
|
||
}
|
||
}
|
||
|
||
/** The slow lane may only use part of the budget so the fast lane never runs dry. */
|
||
function slowBudgetLeft() {
|
||
resetBudget();
|
||
return budgetUsed < config.dailyBudget * 0.6;
|
||
}
|
||
|
||
async function waitForTurn() {
|
||
resetBudget();
|
||
if (budgetUsed >= config.dailyBudget) throw new Error('Daily request budget used up');
|
||
const gap = config.minGapMs * (1 + Math.random() * 0.5);
|
||
const wait = lastRequestAt + gap - Date.now();
|
||
if (wait > 0) await sleep(wait);
|
||
lastRequestAt = Date.now();
|
||
budgetUsed++;
|
||
}
|
||
|
||
type BrowserPage = {
|
||
goto(url: string, options: object): Promise<{ status(): number } | null>;
|
||
content(): Promise<string>;
|
||
};
|
||
let browserPage: Promise<BrowserPage> | null = null;
|
||
|
||
async function openBrowser(): Promise<BrowserPage> {
|
||
// Real Chromium for when plain requests get challenged. Needs
|
||
// `npx playwright install chromium` on the worker machine.
|
||
const { chromium } = await import('@playwright/test');
|
||
const context = await chromium.launchPersistentContext(
|
||
process.env.HOUSE_BROWSER_PROFILE ?? '.house-worker-profile',
|
||
{ headless: true, locale: 'nl-NL', timezoneId: 'Europe/Amsterdam' }
|
||
);
|
||
return context.pages()[0] ?? (await context.newPage());
|
||
}
|
||
|
||
async function getHtml(url: string, referer?: string): Promise<{ html: string; status: number }> {
|
||
await waitForTurn();
|
||
if (config.fetchMode === 'browser') {
|
||
browserPage ??= openBrowser();
|
||
const page = await browserPage;
|
||
const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45_000 });
|
||
const html = await page.content();
|
||
const status = response?.status() ?? 0;
|
||
if (isChallenge(html, status)) throw new Challenged(url, status);
|
||
return { html, status };
|
||
}
|
||
const response = await fetch(url, {
|
||
redirect: 'follow',
|
||
signal: AbortSignal.timeout(30_000),
|
||
headers: {
|
||
'user-agent': config.userAgent,
|
||
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||
'accept-language': 'nl-NL,nl;q=0.9,en-US;q=0.8,en;q=0.7',
|
||
'cache-control': 'no-cache',
|
||
...(referer ? { referer } : {}),
|
||
...(cookies.size
|
||
? { cookie: [...cookies].map(([key, value]) => `${key}=${value}`).join('; ') }
|
||
: {})
|
||
}
|
||
});
|
||
remember(response);
|
||
const html = await response.text();
|
||
if (isChallenge(html, response.status)) throw new Challenged(url, response.status);
|
||
return { html, status: response.status };
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Website API
|
||
|
||
async function api<T>(path: 'sync' | 'details' | 'report', body: unknown): Promise<T | null> {
|
||
return requestApi ? requestApi<T>(path, body) : null;
|
||
}
|
||
|
||
async function report(
|
||
lane: Lane,
|
||
url: string,
|
||
status: RunStatus,
|
||
extra: Partial<ReportRequest> = {}
|
||
) {
|
||
try {
|
||
await api('report', { source: SOURCE, lane, url, status, ...extra });
|
||
} catch (error) {
|
||
log('Could not report:', (error as Error).message);
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Lanes
|
||
|
||
async function fetchDetails(
|
||
lane: Lane,
|
||
wanted: { sourceId: string; url: string }[],
|
||
referer: string
|
||
) {
|
||
const details: ListingDetail[] = [];
|
||
for (const item of wanted) {
|
||
const { html } = await getHtml(item.url, referer);
|
||
const detail = parseDetailPage(html, item.url);
|
||
if (detail) details.push(detail);
|
||
else await report(lane, item.url, 'parse-error', { message: 'listing page did not parse' });
|
||
}
|
||
if (dryRun) {
|
||
for (const detail of details) {
|
||
log('detail', { ...detail, description: undefined, features: undefined });
|
||
}
|
||
}
|
||
if (details.length) await api('details', { source: SOURCE, details });
|
||
}
|
||
|
||
async function pollSearch(
|
||
lane: Lane,
|
||
url: string,
|
||
completedSweep?: SyncRequest['completedSweep']
|
||
): Promise<{ nextPageUrl: string | null; found: number }> {
|
||
const startedAt = new Date().toISOString();
|
||
const { html } = await getHtml(url);
|
||
const page = parseSearchPage(html, url);
|
||
if (!page.cards.length && page.nextPageUrl)
|
||
await report(lane, url, 'parse-error', {
|
||
message: 'no listing cards on a page with a next page'
|
||
});
|
||
if (dryRun) {
|
||
log(`${lane} ${url}: ${page.cards.length} cards, next=${page.nextPageUrl}`);
|
||
for (const card of page.cards.slice(0, 5)) log('card', card);
|
||
}
|
||
const sync = await api<SyncResponse>('sync', {
|
||
source: SOURCE,
|
||
lane,
|
||
searchUrl: url,
|
||
startedAt,
|
||
cards: page.cards,
|
||
completedSweep: page.nextPageUrl ? null : (completedSweep ?? null)
|
||
} satisfies SyncRequest);
|
||
if (sync) {
|
||
if (sync.newIds.length || sync.alerted)
|
||
log(`${lane} ${url}: ${sync.newIds.length} new, ${sync.alerted} alerted`);
|
||
if (sync.needDetails.length) await fetchDetails(lane, sync.needDetails, url);
|
||
} else if (dryRun && page.cards[0]) {
|
||
await fetchDetails(lane, [page.cards[0]], url);
|
||
}
|
||
return { nextPageUrl: page.nextPageUrl, found: page.cards.length };
|
||
}
|
||
|
||
async function fastTick() {
|
||
for (const url of config.fastSearches) await pollSearch('fast', url);
|
||
}
|
||
|
||
/** Only an unfiltered `/huurwoningen/<city>` sweep can tell us what disappeared. */
|
||
function sweepScope(url: string) {
|
||
const [kind, city, ...rest] = new URL(url).pathname.split('/').filter(Boolean);
|
||
if (rest.length || !city) return null;
|
||
if (kind === 'huurwoningen') return { offer: 'rent' as const, city };
|
||
if (kind === 'koopwoningen') return { offer: 'buy' as const, city };
|
||
return null;
|
||
}
|
||
|
||
type Sweep = { index: number; page: number; startedAt: string };
|
||
let sweep: Sweep | null = null;
|
||
|
||
/** One page of the slow lane, so fast polls can interleave. Returns false when idle. */
|
||
async function slowStep(): Promise<boolean> {
|
||
if (!sweep || !slowBudgetLeft()) return false;
|
||
const base = config.slowSearches[sweep.index];
|
||
if (!base) {
|
||
log('Slow sweep finished');
|
||
sweep = null;
|
||
return false;
|
||
}
|
||
const url = searchPageUrl(base, sweep.page);
|
||
const scope = sweepScope(base);
|
||
const result = await pollSearch(
|
||
'slow',
|
||
url,
|
||
scope ? { ...scope, sweepStartedAt: sweep.startedAt } : null
|
||
);
|
||
if (result.nextPageUrl && sweep.page < config.slowMaxPages) {
|
||
sweep.page++;
|
||
} else {
|
||
sweep = { index: sweep.index + 1, page: 1, startedAt: new Date().toISOString() };
|
||
}
|
||
return true;
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main loop
|
||
|
||
let stopping = false;
|
||
process.on('SIGTERM', () => (stopping = true));
|
||
process.on('SIGINT', () => (stopping = true));
|
||
|
||
async function guarded(
|
||
lane: Lane,
|
||
step: () => Promise<boolean | void>
|
||
): Promise<'ok' | 'idle' | 'failed'> {
|
||
const started = Date.now();
|
||
try {
|
||
return (await step()) === false ? 'idle' : 'ok';
|
||
} catch (error) {
|
||
if (error instanceof Challenged) {
|
||
log(`CHALLENGED on ${error.url} (${error.status})`);
|
||
await report(lane, error.url, 'challenged', {
|
||
httpStatus: error.status,
|
||
durationMs: Date.now() - started
|
||
});
|
||
} else {
|
||
log(`${lane} failed:`, (error as Error).message);
|
||
await report(lane, config.fastSearches[0], 'network-error', {
|
||
message: (error as Error).message.slice(0, 500),
|
||
durationMs: Date.now() - started
|
||
});
|
||
}
|
||
return 'failed';
|
||
}
|
||
}
|
||
|
||
async function main() {
|
||
if (checkApi) {
|
||
const result = await requestApi!<{ ok: boolean }>('health');
|
||
if (result?.ok !== true) throw new Error('API health: unexpected response');
|
||
log('API connection OK. Worker authentication passed; no Pararius requests were made.');
|
||
return;
|
||
}
|
||
log(
|
||
`house worker: ${config.fastSearches.length} fast, ${config.slowSearches.length} slow searches, mode=${config.fetchMode}${dryRun ? ', dry run' : ''}`
|
||
);
|
||
if (once) {
|
||
const result = await guarded('fast', fastTick);
|
||
process.exit(result === 'ok' ? 0 : 1);
|
||
}
|
||
|
||
let failures = 0;
|
||
let nextFast = Date.now();
|
||
let nextSlow = Date.now() + 5 * 60_000; // let the fast lane go first
|
||
while (!stopping) {
|
||
const now = Date.now();
|
||
let result: 'ok' | 'idle' | 'failed' = 'idle';
|
||
if (now >= nextFast) {
|
||
result = await guarded('fast', fastTick);
|
||
nextFast = Date.now() + nextDelayMs(new Date());
|
||
} else if (sweep || now >= nextSlow) {
|
||
if (!sweep) {
|
||
log('Slow sweep starting');
|
||
sweep = { index: 0, page: 1, startedAt: new Date().toISOString() };
|
||
nextSlow = now + config.slowEveryHours * 60 * 60_000;
|
||
}
|
||
result = await guarded('slow', slowStep);
|
||
}
|
||
if (result === 'failed') {
|
||
failures++;
|
||
const pause = backoffMs(failures);
|
||
log(`Backing off ${Math.round(pause / 60_000)} min (failure ${failures})`);
|
||
nextFast = Math.max(nextFast, Date.now() + pause);
|
||
await sleep(pause);
|
||
continue;
|
||
}
|
||
if (result === 'ok') failures = 0;
|
||
else await sleep(Math.min(30_000, Math.max(5_000, nextFast - Date.now())));
|
||
}
|
||
log('Stopped');
|
||
process.exit(0);
|
||
}
|
||
|
||
await main();
|