diff --git a/.env.example b/.env.example index 38a5984..8dda5f5 100755 --- a/.env.example +++ b/.env.example @@ -28,3 +28,10 @@ R2_ACCESS_KEY_ID= R2_SECRET_ACCESS_KEY= R2_BUCKET= R2_PUBLIC_URL= + +# House tracker: shared secret for the home worker (openssl rand -hex 32), and +# where alerts go (Discord webhook and/or ntfy topic URL). See docs/house-tracker.md. +HOUSE_WORKER_TOKEN= +HOUSE_DISCORD_WEBHOOK_URL= +HOUSE_NTFY_URL= +HOUSE_NTFY_TOKEN= diff --git a/.prettierignore b/.prettierignore index e4858e7..77c5704 100755 --- a/.prettierignore +++ b/.prettierignore @@ -18,3 +18,4 @@ data/cod-challenges.json # Generated local image download manifest data/game-images.json data/cod-achievement-art.json +src/lib/house-tracker/fixtures/ diff --git a/README.md b/README.md index 60fc03e..521d8d3 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,12 @@ This is my personal website, including: - Blog posts + comments - Projects like Animaldex, CatchEmAll +## House tracker + +Available at `/projects/house-tracker` (admin only). Listings come from a worker +that runs at home; setup, settings and the ban-avoidance rules are in +[docs/house-tracker.md](docs/house-tracker.md). + ## osu! Map Finder Available at `/projects/spotify-beatmap-finder`. Built with the existing Svelte 5, diff --git a/docs/house-tracker.md b/docs/house-tracker.md new file mode 100644 index 0000000..fe694c4 --- /dev/null +++ b/docs/house-tracker.md @@ -0,0 +1,146 @@ +# House tracker + +Personal tool: collects Dutch housing listings into Postgres, shows them on +`/projects/house-tracker` (admin only) and sends an alert when a new listing +matches a rule. Only Pararius for now; other sources plug in the same way. + +## How it fits together + +``` +home box (residential IP) stanrunge.dev (Hetzner) +┌──────────────────────────┐ POST sync ┌──────────────────────────────┐ +│ scripts/house-worker.ts │ ──────────────▶ │ /projects/house-tracker/api/ │ +│ polls pararius.nl │ POST details │ worker/{sync,details,report}│ +│ parses HTML │ ◀────────────── │ stores, dedups, alerts │ +└──────────────────────────┘ needDetails └──────────────┬───────────────┘ + ▼ + Discord webhook / ntfy push +``` + +The server never talks to the listing sites. Pararius sits behind Cloudflare, +which challenges datacenter IPs and plain HTTP clients, so the worker runs at +home where traffic looks like a normal visitor. + +Two lanes: + +- **fast**: one filtered search (default `huurwoningen/gouda/1300-1700`), page 1 + only. Every 3–5 min on weekdays 07:00–18:00 Amsterdam time, every 30–60 min + otherwise. Listings near Gouda were all published in office hours on weekdays. + Alerts are sent from the search card alone, before the detail page is fetched. +- **slow**: full, paginated sweeps of rent and buy listings in Gouda, Rotterdam, + Den Haag, Utrecht, Amsterdam and Eindhoven every 12 h. Unfiltered city sweeps + also mark listings that disappeared, which gives "days listed" in the stats. + +Safety rules in the worker: one request at a time, at least 8 s apart with +jitter, a daily budget (1500 by default, the slow lane may only use 60%), and +any challenge/403/429 stops everything and backs off 15 min → 30 → 60 … 6 h. +The first poll of any search only seeds the database, so you don't get 40 +alerts for listings that were already online. + +## Server setup + +Add to the production env file (`/etc/personal-website/prod.env`): + +```text +# openssl rand -hex 32 +HOUSE_WORKER_TOKEN= +# one or both +HOUSE_DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... +HOUSE_NTFY_URL=https://ntfy.sh/some-long-random-topic +HOUSE_NTFY_TOKEN= # only for a protected ntfy topic +PUBLIC_MAPTILER_API_KEY= # already set for Animaldex +``` + +`scripts/migrate-house-tracker.ts` runs on every deploy (idempotent). Use +"send test alert" on the page to check the alert channel. + +If Cloudflare Bot Fight Mode / WAF challenges are on for stanrunge.dev, add a +WAF skip rule for `URI Path starts with /projects/house-tracker/api/worker/`, +otherwise the worker's POSTs get challenged too. The endpoints are protected by +the bearer token. + +## Home worker setup + +Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with bun. + +```sh +git clone https://git.stanrunge.dev/stan/personal-website ~/house-worker +cd ~/house-worker && bun install +cat > house.env <<'EOF' +HOUSE_API_URL=https://stanrunge.dev +HOUSE_WORKER_TOKEN=same-value-as-the-server +EOF +chmod 600 house.env + +# 1. check parsing without sending anything +bun --env-file=house.env scripts/house-worker.ts --dry-run --once +# 2. one real poll +bun --env-file=house.env scripts/house-worker.ts --once +``` + +Then run it as a service, `/etc/systemd/system/house-worker.service`: + +```ini +[Unit] +Description=House tracker worker +After=network-online.target +Wants=network-online.target + +[Service] +User=stan +WorkingDirectory=/home/stan/house-worker +ExecStart=/home/stan/.bun/bin/bun --env-file=house.env scripts/house-worker.ts +Restart=always +RestartSec=60 + +[Install] +WantedBy=multi-user.target +``` + +```sh +sudo systemctl enable --now house-worker +journalctl -u house-worker -f +``` + +Or with Docker: `docker run -d --restart unless-stopped --name house-worker +--env-file house.env -v "$PWD":/app -w /app oven/bun:1.4.0 bun +scripts/house-worker.ts`. + +## Worker settings + +| variable | default | +| ------------------------ | ------------------------------------------------------------------------ | +| `HOUSE_FAST_SEARCHES` | `https://www.pararius.nl/huurwoningen/gouda/1300-1700` (comma separated) | +| `HOUSE_SLOW_SEARCHES` | rent + buy for the six cities above | +| `HOUSE_SLOW_EVERY_HOURS` | `12` | +| `HOUSE_SLOW_MAX_PAGES` | `80` per search | +| `HOUSE_MIN_GAP_SECONDS` | `8` between any two requests | +| `HOUSE_DAILY_BUDGET` | `1500` requests | +| `HOUSE_FETCH_MODE` | `fetch`, or `browser` (real Chromium) | + +Any Pararius search URL works as a fast search: set the filters on the site +(price, radius, bedrooms) and paste the URL. Every extra fast search is one more +request per poll. + +## When it gets challenged + +You get one "worker was challenged" alert per 6 hours and the page header turns +red. In order: + +1. Do nothing for a day. Backoff handles short blocks. +2. Slow down: raise `HOUSE_MIN_GAP_SECONDS`, lower `HOUSE_DAILY_BUDGET`, or drop + slow searches. +3. `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent profile + (`npx playwright install chromium` once on the worker machine). Heavier, but + passes checks that plain HTTP clients don't. + +Commercial VPN exits are usually datacenter IPs and get challenged more, not less. + +## Parsers + +`src/lib/house-tracker/pararius.ts`, tested against trimmed real pages in +`src/lib/house-tracker/fixtures/`. Listing pages expose an exact publish +timestamp (`odp.offered_since`), coordinates, deposit, energy label and agent. +The search-page fixture is hand-written; replace it with a real saved page when +the markup is confirmed. If Pararius changes markup, the worker reports +`parse-error` in the worker log on the page. diff --git a/package.json b/package.json index 9730005..ec26fb9 100755 --- a/package.json +++ b/package.json @@ -25,6 +25,8 @@ "db:migrate:game-tracker": "bun scripts/migrate-game-tracker.ts", "db:migrate:cs-prices": "bun scripts/migrate-cs-prices.ts", "cs-prices:ingest": "bun scripts/ingest-cs-prices.ts", + "db:migrate:house-tracker": "bun scripts/migrate-house-tracker.ts", + "house:worker": "bun scripts/house-worker.ts", "auth:role": "bun scripts/set-admin.ts", "catalog:build": "python3 scripts/build-pokemon-catalog.py", "catalog:cod": "python3 scripts/build-cod-catalog.py && prettier --write data/call-of-duty.json", diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index e21a27b..ee852a1 100644 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -68,6 +68,7 @@ run_database_script scripts/migrate-foundation.ts run_database_script scripts/migrate-money-tracker.ts run_database_script scripts/migrate-game-tracker.ts run_database_script scripts/migrate-tracker-journal.ts +run_database_script scripts/migrate-house-tracker.ts run_database_script scripts/seed-catalog.ts run_database_script scripts/seed-cod-catalog.ts run_database_script scripts/seed-pokemon-research.ts diff --git a/scripts/house-tracker-migration.sql b/scripts/house-tracker-migration.sql new file mode 100644 index 0000000..6bfa87e --- /dev/null +++ b/scripts/house-tracker-migration.sql @@ -0,0 +1,111 @@ +-- House tracker: listings from Dutch housing sites, collected by a home worker. +-- Idempotent; safe to run on every deploy. + +CREATE TABLE IF NOT EXISTS house_listings ( + id serial PRIMARY KEY, + source text NOT NULL, + source_id text NOT NULL, + external_id text, + url text NOT NULL, + offer text NOT NULL CHECK (offer IN ('rent', 'buy')), + property_type text NOT NULL, + title text, + street text, + postal_code text, + city text, + city_slug text, + neighbourhood text, + price_eur integer CHECK (price_eur IS NULL OR price_eur >= 0), + area_m2 integer, + rooms integer, + bedrooms integer, + bathrooms integer, + construction_year integer, + energy_label text, + interior text, + deposit_eur integer, + available_from text, + contract_type text, + agent text, + image_url text, + description text, + features jsonb NOT NULL DEFAULT '{}'::jsonb, + lat double precision, + lng double precision, + -- When the site says it was published. first_seen_at is when we noticed. + offered_at timestamptz, + first_seen_at timestamptz NOT NULL DEFAULT now(), + -- fast, slow, or seed (already listed when its search was first polled). + first_seen_lane text NOT NULL DEFAULT 'slow', + last_seen_at timestamptz NOT NULL DEFAULT now(), + detail_fetched_at timestamptz, + removed_at timestamptz, + alerted_at timestamptz, + -- Personal tracking. + status text, + notes text +); +CREATE UNIQUE INDEX IF NOT EXISTS house_listings_source_idx ON house_listings (source, source_id); +CREATE INDEX IF NOT EXISTS house_listings_active_idx + ON house_listings (offer, first_seen_at DESC) WHERE removed_at IS NULL; +CREATE INDEX IF NOT EXISTS house_listings_city_idx ON house_listings (source, offer, city_slug); +CREATE INDEX IF NOT EXISTS house_listings_details_idx + ON house_listings (first_seen_at DESC) WHERE detail_fetched_at IS NULL; + +-- Every price we have seen, so price drops and market trends survive edits. +CREATE TABLE IF NOT EXISTS house_price_history ( + listing_id integer NOT NULL REFERENCES house_listings(id) ON DELETE CASCADE, + observed_at timestamptz NOT NULL DEFAULT now(), + price_eur integer NOT NULL +); +CREATE INDEX IF NOT EXISTS house_price_history_listing_idx + ON house_price_history (listing_id, observed_at); + +-- One row per polled search URL. The first sync of a search only seeds the +-- database: alerting on everything already listed would be noise. +CREATE TABLE IF NOT EXISTS house_searches ( + search_url text PRIMARY KEY, + source text NOT NULL, + lane text NOT NULL, + first_synced_at timestamptz NOT NULL DEFAULT now(), + last_synced_at timestamptz NOT NULL DEFAULT now(), + last_found integer NOT NULL DEFAULT 0 +); + +-- Worker health: challenges, HTTP errors, parse failures. +CREATE TABLE IF NOT EXISTS house_worker_runs ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source text NOT NULL, + lane text NOT NULL, + url text NOT NULL, + status text NOT NULL, + http_status integer, + found integer, + duration_ms integer, + message text, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS house_worker_runs_created_idx ON house_worker_runs (created_at DESC); + +CREATE TABLE IF NOT EXISTS house_alert_rules ( + id serial PRIMARY KEY, + name text NOT NULL, + enabled boolean NOT NULL DEFAULT true, + offer text NOT NULL DEFAULT 'rent' CHECK (offer IN ('rent', 'buy')), + min_price integer, + max_price integer, + min_area integer, + min_bedrooms integer, + -- Lowercase city names; empty means anywhere. + cities text[] NOT NULL DEFAULT '{}', + -- Empty means every property type. + property_types text[] NOT NULL DEFAULT '{}', + created_at timestamptz NOT NULL DEFAULT now() +); + +INSERT INTO house_alert_rules (name, offer, min_price, max_price, cities) +SELECT 'Gouda area rentals', 'rent', 1300, 1700, ARRAY[ + 'gouda', 'waddinxveen', 'reeuwijk', 'moordrecht', 'nieuwerkerk aan den ijssel', + 'zevenhuizen', 'bodegraven', 'boskoop', 'oudewater', 'schoonhoven', 'haastrecht' +] +WHERE NOT EXISTS (SELECT 1 FROM house_alert_rules); diff --git a/scripts/house-worker.ts b/scripts/house-worker.ts new file mode 100644 index 0000000..d83ca85 --- /dev/null +++ b/scripts/house-worker.ts @@ -0,0 +1,383 @@ +/** + * House tracker home worker. Runs on a machine at home (residential IP), polls + * Pararius politely and POSTs what it finds to the website, which stores it and + * sends the alerts. See docs/house-tracker.md. + * + * HOUSE_API_URL=https://stanrunge.dev HOUSE_WORKER_TOKEN=... bun scripts/house-worker.ts + * bun scripts/house-worker.ts --once # one fast-lane poll, then exit + * bun scripts/house-worker.ts --dry-run # fetch + parse, print, send nothing + * + * Politeness rules, in order of importance: + * - one request at a time, at least HOUSE_MIN_GAP_SECONDS apart (with jitter) + * - a daily request budget + * - any challenge / 403 / 429 stops everything and backs off exponentially + * - the fast lane only polls often on weekdays 07:00–18:00 Amsterdam time + */ +import { + isChallenge, + parseDetailPage, + parseSearchPage, + searchPageUrl +} from '../src/lib/house-tracker/pararius'; +import { backoffMs, nextDelayMs } from '../src/lib/house-tracker/schedule'; +import type { + Lane, + ListingDetail, + ReportRequest, + RunStatus, + SyncRequest, + SyncResponse +} from '../src/lib/house-tracker/types'; + +const args = new Set(process.argv.slice(2)); +const once = args.has('--once'); +const dryRun = args.has('--dry-run') || process.env.HOUSE_DRY_RUN === '1'; + +const list = (value: string | undefined, fallback: string[]) => + value + ? value + .split(/[\s,]+/) + .map((item) => item.trim()) + .filter(Boolean) + : fallback; +const number = (value: string | undefined, fallback: number) => + value && Number.isFinite(Number(value)) ? Number(value) : fallback; + +const config = { + api: (process.env.HOUSE_API_URL ?? '').replace(/\/$/, ''), + token: process.env.HOUSE_WORKER_TOKEN ?? '', + fastSearches: list(process.env.HOUSE_FAST_SEARCHES, [ + 'https://www.pararius.nl/huurwoningen/gouda/1300-1700' + ]), + slowSearches: list( + process.env.HOUSE_SLOW_SEARCHES, + ['gouda', 'rotterdam', 'den-haag', 'utrecht', 'amsterdam', 'eindhoven'].flatMap((city) => [ + `https://www.pararius.nl/huurwoningen/${city}`, + `https://www.pararius.nl/koopwoningen/${city}` + ]) + ), + slowEveryHours: number(process.env.HOUSE_SLOW_EVERY_HOURS, 12), + slowMaxPages: number(process.env.HOUSE_SLOW_MAX_PAGES, 80), + minGapMs: number(process.env.HOUSE_MIN_GAP_SECONDS, 8) * 1000, + dailyBudget: number(process.env.HOUSE_DAILY_BUDGET, 1500), + fetchMode: (process.env.HOUSE_FETCH_MODE ?? 'fetch') as 'fetch' | 'browser', + userAgent: + process.env.HOUSE_USER_AGENT ?? + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36' +}; + +if (!dryRun && (!config.api || config.token.length < 32)) + throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required'); + +const SOURCE = 'pararius'; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const log = (...parts: unknown[]) => console.log(new Date().toISOString(), ...parts); + +// --------------------------------------------------------------------------- +// Polite fetching + +class Challenged extends Error { + constructor( + readonly url: string, + readonly status: number + ) { + super(`Challenged (${status}) on ${url}`); + } +} + +const cookies = new Map(); +let lastRequestAt = 0; +let budgetDay = ''; +let budgetUsed = 0; + +function remember(response: Response) { + for (const header of response.headers.getSetCookie?.() ?? []) { + const [pair] = header.split(';'); + const index = pair.indexOf('='); + if (index > 0) cookies.set(pair.slice(0, index).trim(), pair.slice(index + 1).trim()); + } +} + +function resetBudget() { + const day = new Date().toISOString().slice(0, 10); + if (day !== budgetDay) { + budgetDay = day; + budgetUsed = 0; + } +} + +/** The slow lane may only use part of the budget so the fast lane never runs dry. */ +function slowBudgetLeft() { + resetBudget(); + return budgetUsed < config.dailyBudget * 0.6; +} + +async function waitForTurn() { + resetBudget(); + if (budgetUsed >= config.dailyBudget) throw new Error('Daily request budget used up'); + const gap = config.minGapMs * (0.8 + Math.random() * 0.7); + const wait = lastRequestAt + gap - Date.now(); + if (wait > 0) await sleep(wait); + lastRequestAt = Date.now(); + budgetUsed++; +} + +type BrowserPage = { + goto(url: string, options: object): Promise<{ status(): number } | null>; + content(): Promise; +}; +let browserPage: Promise | null = null; + +async function openBrowser(): Promise { + // Real Chromium for when plain requests get challenged. Needs + // `npx playwright install chromium` on the worker machine. + const { chromium } = await import('@playwright/test'); + const context = await chromium.launchPersistentContext( + process.env.HOUSE_BROWSER_PROFILE ?? '.house-worker-profile', + { headless: true, locale: 'nl-NL', timezoneId: 'Europe/Amsterdam' } + ); + return context.pages()[0] ?? (await context.newPage()); +} + +async function getHtml(url: string, referer?: string): Promise<{ html: string; status: number }> { + await waitForTurn(); + if (config.fetchMode === 'browser') { + browserPage ??= openBrowser(); + const page = await browserPage; + const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45_000 }); + const html = await page.content(); + const status = response?.status() ?? 0; + if (isChallenge(html, status)) throw new Challenged(url, status); + return { html, status }; + } + const response = await fetch(url, { + redirect: 'follow', + signal: AbortSignal.timeout(30_000), + headers: { + 'user-agent': config.userAgent, + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'accept-language': 'nl-NL,nl;q=0.9,en-US;q=0.8,en;q=0.7', + 'cache-control': 'no-cache', + ...(referer ? { referer } : {}), + ...(cookies.size + ? { cookie: [...cookies].map(([key, value]) => `${key}=${value}`).join('; ') } + : {}) + } + }); + remember(response); + const html = await response.text(); + if (isChallenge(html, response.status)) throw new Challenged(url, response.status); + return { html, status: response.status }; +} + +// --------------------------------------------------------------------------- +// Website API + +async function api(path: string, body: unknown): Promise { + if (dryRun) return null; + for (let attempt = 0; attempt < 4; attempt++) { + try { + const response = await fetch(`${config.api}/projects/house-tracker/api/worker/${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${config.token}` }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(30_000) + }); + if (response.ok) return (await response.json()) as T; + const text = await response.text(); + if (response.status < 500) + throw new Error(`API ${path} ${response.status}: ${text.slice(0, 300)}`); + log(`API ${path} ${response.status}, retrying`); + } catch (error) { + if (attempt === 3) throw error; + log(`API ${path} failed: ${(error as Error).message}, retrying`); + } + await sleep(5000 * 2 ** attempt); + } + return null; +} + +async function report( + lane: Lane, + url: string, + status: RunStatus, + extra: Partial = {} +) { + try { + await api('report', { source: SOURCE, lane, url, status, ...extra }); + } catch (error) { + log('Could not report:', (error as Error).message); + } +} + +// --------------------------------------------------------------------------- +// Lanes + +async function fetchDetails( + lane: Lane, + wanted: { sourceId: string; url: string }[], + referer: string +) { + const details: ListingDetail[] = []; + for (const item of wanted) { + const { html } = await getHtml(item.url, referer); + const detail = parseDetailPage(html, item.url); + if (detail) details.push(detail); + else await report(lane, item.url, 'parse-error', { message: 'listing page did not parse' }); + } + if (dryRun) { + for (const detail of details) { + log('detail', { ...detail, description: undefined, features: undefined }); + } + } + if (details.length) await api('details', { source: SOURCE, details }); +} + +async function pollSearch( + lane: Lane, + url: string, + completedSweep?: SyncRequest['completedSweep'] +): Promise<{ nextPageUrl: string | null; found: number }> { + const startedAt = new Date().toISOString(); + const { html } = await getHtml(url); + const page = parseSearchPage(html, url); + if (!page.cards.length && page.nextPageUrl) + await report(lane, url, 'parse-error', { + message: 'no listing cards on a page with a next page' + }); + if (dryRun) { + log(`${lane} ${url}: ${page.cards.length} cards, next=${page.nextPageUrl}`); + for (const card of page.cards.slice(0, 5)) log('card', card); + } + const sync = await api('sync', { + source: SOURCE, + lane, + searchUrl: url, + startedAt, + cards: page.cards, + completedSweep: page.nextPageUrl ? null : (completedSweep ?? null) + } satisfies SyncRequest); + if (sync) { + if (sync.newIds.length || sync.alerted) + log(`${lane} ${url}: ${sync.newIds.length} new, ${sync.alerted} alerted`); + if (sync.needDetails.length) await fetchDetails(lane, sync.needDetails, url); + } else if (dryRun && page.cards[0]) { + await fetchDetails(lane, [page.cards[0]], url); + } + return { nextPageUrl: page.nextPageUrl, found: page.cards.length }; +} + +async function fastTick() { + for (const url of config.fastSearches) await pollSearch('fast', url); +} + +/** Only an unfiltered `/huurwoningen/` sweep can tell us what disappeared. */ +function sweepScope(url: string) { + const [kind, city, ...rest] = new URL(url).pathname.split('/').filter(Boolean); + if (rest.length || !city) return null; + if (kind === 'huurwoningen') return { offer: 'rent' as const, city }; + if (kind === 'koopwoningen') return { offer: 'buy' as const, city }; + return null; +} + +type Sweep = { index: number; page: number; startedAt: string }; +let sweep: Sweep | null = null; + +/** One page of the slow lane, so fast polls can interleave. Returns false when idle. */ +async function slowStep(): Promise { + if (!sweep || !slowBudgetLeft()) return false; + const base = config.slowSearches[sweep.index]; + if (!base) { + log('Slow sweep finished'); + sweep = null; + return false; + } + const url = searchPageUrl(base, sweep.page); + const scope = sweepScope(base); + const result = await pollSearch( + 'slow', + url, + scope ? { ...scope, sweepStartedAt: sweep.startedAt } : null + ); + if (result.nextPageUrl && sweep.page < config.slowMaxPages) { + sweep.page++; + } else { + sweep = { index: sweep.index + 1, page: 1, startedAt: new Date().toISOString() }; + } + return true; +} + +// --------------------------------------------------------------------------- +// Main loop + +let stopping = false; +process.on('SIGTERM', () => (stopping = true)); +process.on('SIGINT', () => (stopping = true)); + +async function guarded( + lane: Lane, + step: () => Promise +): Promise<'ok' | 'idle' | 'failed'> { + const started = Date.now(); + try { + return (await step()) === false ? 'idle' : 'ok'; + } catch (error) { + if (error instanceof Challenged) { + log(`CHALLENGED on ${error.url} (${error.status})`); + await report(lane, error.url, 'challenged', { + httpStatus: error.status, + durationMs: Date.now() - started + }); + } else { + log(`${lane} failed:`, (error as Error).message); + await report(lane, config.fastSearches[0], 'network-error', { + message: (error as Error).message.slice(0, 500), + durationMs: Date.now() - started + }); + } + return 'failed'; + } +} + +async function main() { + log( + `house worker: ${config.fastSearches.length} fast, ${config.slowSearches.length} slow searches, mode=${config.fetchMode}${dryRun ? ', dry run' : ''}` + ); + if (once) { + const result = await guarded('fast', fastTick); + process.exit(result === 'ok' ? 0 : 1); + } + + let failures = 0; + let nextFast = Date.now(); + let nextSlow = Date.now() + 5 * 60_000; // let the fast lane go first + while (!stopping) { + const now = Date.now(); + let result: 'ok' | 'idle' | 'failed' = 'idle'; + if (now >= nextFast) { + result = await guarded('fast', fastTick); + nextFast = Date.now() + nextDelayMs(new Date()); + } else if (sweep || now >= nextSlow) { + if (!sweep) { + log('Slow sweep starting'); + sweep = { index: 0, page: 1, startedAt: new Date().toISOString() }; + nextSlow = now + config.slowEveryHours * 60 * 60_000; + } + result = await guarded('slow', slowStep); + } + if (result === 'failed') { + failures++; + const pause = backoffMs(failures); + log(`Backing off ${Math.round(pause / 60_000)} min (failure ${failures})`); + nextFast = Math.max(nextFast, Date.now() + pause); + await sleep(pause); + continue; + } + if (result === 'ok') failures = 0; + else await sleep(Math.min(30_000, Math.max(5_000, nextFast - Date.now()))); + } + log('Stopped'); + process.exit(0); +} + +await main(); diff --git a/scripts/migrate-house-tracker.ts b/scripts/migrate-house-tracker.ts new file mode 100644 index 0000000..0858ebf --- /dev/null +++ b/scripts/migrate-house-tracker.ts @@ -0,0 +1,15 @@ +import { readFile } from 'node:fs/promises'; +import postgres from 'postgres'; + +if (!process.env.DB_URL) throw new Error('DB_URL is required'); +const client = postgres(process.env.DB_URL, { max: 1 }); +try { + await client.begin(async (tx) => { + await tx.unsafe( + await readFile(new URL('./house-tracker-migration.sql', import.meta.url), 'utf8') + ); + }); + console.log('House tracker migration applied'); +} finally { + await client.end(); +} diff --git a/src/lib/components/ProjectIcon.svelte b/src/lib/components/ProjectIcon.svelte index 129b7c3..316f79f 100644 --- a/src/lib/components/ProjectIcon.svelte +++ b/src/lib/components/ProjectIcon.svelte @@ -12,6 +12,7 @@ import ChartNoAxesCombined from 'lucide-svelte/icons/chart-no-axes-combined'; import FolderKanban from 'lucide-svelte/icons/folder-kanban'; import Gamepad2 from 'lucide-svelte/icons/gamepad-2'; + import House from 'lucide-svelte/icons/house'; import Globe2 from 'lucide-svelte/icons/globe-2'; import Languages from 'lucide-svelte/icons/languages'; import Music2 from 'lucide-svelte/icons/music-2'; @@ -30,7 +31,8 @@ 'world opinions': Globe2, 'language learner': Languages, 'money tracker': Wallet, - 'cs prices': CandlestickChart + 'cs prices': CandlestickChart, + 'house tracker': House }; const Icon = $derived(icons[name as keyof typeof icons] ?? FolderKanban); diff --git a/src/lib/house-tracker/ListingMap.svelte b/src/lib/house-tracker/ListingMap.svelte new file mode 100644 index 0000000..8695c93 --- /dev/null +++ b/src/lib/house-tracker/ListingMap.svelte @@ -0,0 +1,185 @@ + + + + + + +
+
+ {#if mapError} +
+ {mapError} +
+ {/if} +
+ new (48h) + active + tracked + gone +
+
diff --git a/src/lib/house-tracker/alerts.test.ts b/src/lib/house-tracker/alerts.test.ts new file mode 100644 index 0000000..00528b3 --- /dev/null +++ b/src/lib/house-tracker/alerts.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import { alertMessage, matchesRule, type AlertRule } from './alerts'; +import { backoffMs, isBusy, nextDelayMs } from './schedule'; + +const rule: AlertRule = { + id: 1, + name: 'Gouda area rentals', + enabled: true, + offer: 'rent', + minPrice: 1300, + maxPrice: 1700, + minArea: null, + minBedrooms: null, + cities: ['gouda', 'nieuwerkerk aan den ijssel'], + propertyTypes: [] +}; + +const listing = { + offer: 'rent', + propertyType: 'apartment', + city: 'Gouda', + priceEur: 1558, + areaM2: 65, + rooms: 2 +}; + +describe('matchesRule', () => { + it('matches inside the price band and city list', () => { + expect(matchesRule(listing, rule)).toBe(true); + expect(matchesRule({ ...listing, city: 'Nieuwerkerk aan den IJssel' }, rule)).toBe(true); + expect( + matchesRule({ ...listing, city: null, citySlug: 'nieuwerkerk-aan-den-ijssel' }, rule) + ).toBe(true); + }); + + it('rejects outside price, city, offer or when disabled', () => { + expect(matchesRule({ ...listing, priceEur: 1800 }, rule)).toBe(false); + expect(matchesRule({ ...listing, priceEur: null }, rule)).toBe(false); + expect(matchesRule({ ...listing, city: 'Rotterdam' }, rule)).toBe(false); + expect(matchesRule({ ...listing, offer: 'buy' }, rule)).toBe(false); + expect(matchesRule(listing, { ...rule, enabled: false })).toBe(false); + }); + + it('lets unknown size through but rejects known small ones', () => { + const roomy = { ...rule, minArea: 70, minBedrooms: 2 }; + expect(matchesRule({ ...listing, areaM2: null, rooms: null }, roomy)).toBe(true); + expect(matchesRule(listing, roomy)).toBe(false); + }); + + it('formats a message', () => { + const message = alertMessage( + { ...listing, title: 'Appartement Raam', url: 'https://example.com', source: 'pararius' }, + rule + ); + expect(message.title).toBe('🏠 Appartement Raam'); + expect(message.body).toContain('65 m²'); + expect(message.body).toContain('/mo'); + }); +}); + +describe('schedule', () => { + // 2026-10-07 is a Wednesday; Amsterdam is UTC+2 in October. + const wednesdayNoon = new Date('2026-10-07T10:00:00Z'); + const wednesdayNight = new Date('2026-10-07T21:00:00Z'); + const saturday = new Date('2026-10-10T10:00:00Z'); + + it('knows office hours in Amsterdam', () => { + expect(isBusy(wednesdayNoon)).toBe(true); + expect(isBusy(wednesdayNight)).toBe(false); + expect(isBusy(saturday)).toBe(false); + expect(isBusy(new Date('2026-10-07T04:59:00Z'))).toBe(false); + expect(isBusy(new Date('2026-10-07T05:00:00Z'))).toBe(true); + }); + + it('polls every few minutes when busy and slowly otherwise', () => { + expect(nextDelayMs(wednesdayNoon, undefined, () => 0)).toBe(3 * 60_000); + expect(nextDelayMs(wednesdayNoon, undefined, () => 1)).toBe(5 * 60_000); + expect(nextDelayMs(saturday, undefined, () => 0)).toBe(30 * 60_000); + }); + + it('wakes up for the start of office hours', () => { + const justBefore = new Date('2026-10-07T04:50:00Z'); // 06:50 local + expect(nextDelayMs(justBefore, undefined, () => 1)).toBeLessThanOrEqual(11 * 60_000); + }); + + it('backs off exponentially with a cap', () => { + expect(backoffMs(1, () => 0.5)).toBe(15 * 60_000); + expect(backoffMs(3, () => 0.5)).toBe(60 * 60_000); + expect(backoffMs(20, () => 0.5)).toBe(6 * 60 * 60_000); + }); +}); diff --git a/src/lib/house-tracker/alerts.ts b/src/lib/house-tracker/alerts.ts new file mode 100644 index 0000000..0d4b669 --- /dev/null +++ b/src/lib/house-tracker/alerts.ts @@ -0,0 +1,92 @@ +// Pure alert matching and message formatting, shared by server and tests. + +export type AlertRule = { + id: number; + name: string; + enabled: boolean; + offer: string; + minPrice: number | null; + maxPrice: number | null; + minArea: number | null; + minBedrooms: number | null; + cities: string[]; + propertyTypes: string[]; +}; + +export type AlertListing = { + offer: string; + propertyType: string; + city?: string | null; + citySlug?: string | null; + priceEur?: number | null; + areaM2?: number | null; + bedrooms?: number | null; + rooms?: number | null; +}; + +export function normalizeCity(value: string | null | undefined): string { + return (value ?? '') + .normalize('NFD') + .replace(/[̀-ͯ]/g, '') + .toLowerCase() + .replace(/-/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +export function matchesRule(listing: AlertListing, rule: AlertRule): boolean { + if (!rule.enabled || listing.offer !== rule.offer) return false; + if (rule.minPrice != null || rule.maxPrice != null) { + // "Price on request" can't be checked, so it never matches a price rule. + if (listing.priceEur == null) return false; + if (rule.minPrice != null && listing.priceEur < rule.minPrice) return false; + if (rule.maxPrice != null && listing.priceEur > rule.maxPrice) return false; + } + // Unknown area or bedrooms pass: search cards often lack them, and missing an + // alert is worse than one extra notification. + if (rule.minArea != null && listing.areaM2 != null && listing.areaM2 < rule.minArea) return false; + if (rule.minBedrooms != null) { + const bedrooms = listing.bedrooms ?? (listing.rooms != null ? listing.rooms - 1 : null); + if (bedrooms != null && bedrooms < rule.minBedrooms) return false; + } + if (rule.propertyTypes.length && !rule.propertyTypes.includes(listing.propertyType)) return false; + if (rule.cities.length) { + const wanted = new Set(rule.cities.map(normalizeCity)); + const city = normalizeCity(listing.city); + const slug = normalizeCity(listing.citySlug); + if (!wanted.has(city) && !wanted.has(slug)) return false; + } + return true; +} + +const euros = new Intl.NumberFormat('nl-NL', { + style: 'currency', + currency: 'EUR', + maximumFractionDigits: 0 +}); + +export type AlertMessage = { title: string; body: string; url: string }; + +export function alertMessage( + listing: AlertListing & { + title?: string | null; + url: string; + neighbourhood?: string | null; + source: string; + }, + rule: Pick +): AlertMessage { + const price = listing.priceEur != null ? euros.format(listing.priceEur) : 'price on request'; + const suffix = listing.offer === 'rent' && listing.priceEur != null ? '/mo' : ''; + const facts = [ + `${price}${suffix}`, + listing.areaM2 != null ? `${listing.areaM2} m²` : null, + listing.rooms != null ? `${listing.rooms} rooms` : null, + [listing.city, listing.neighbourhood].filter(Boolean).join(', ') || null + ].filter(Boolean); + return { + title: `🏠 ${listing.title ?? 'New listing'}`, + body: `${facts.join(' · ')}\n${rule.name} · ${listing.source}`, + url: listing.url + }; +} diff --git a/src/lib/house-tracker/fixtures/pararius-detail-apartment.html b/src/lib/house-tracker/fixtures/pararius-detail-apartment.html new file mode 100644 index 0000000..b2fb92b --- /dev/null +++ b/src/lib/house-tracker/fixtures/pararius-detail-apartment.html @@ -0,0 +1,467 @@ + +trimmed fixture + + + + +

Te huur: Appartement Graaf Hendrikstraat 16 in Gouda

+
2805 CK (Ouwe Gouwe)
+ +
+
Huurprijs
+
+ € 1.558 per maand + +
    +
+
+
Prijs per m²
+
+ € 24 +
+
Aangeboden sinds
+
+ 02-10-2026 +
+
Status
+
+ Te huur +
+
Beschikbaar
+
+ Per 15-10-2026 +
+
Contractvorm
+
+ Onbepaalde tijd +
+ +
Borg
+
+ + € 3.316 + + +
+ + + + + + + + + + + + + + +
+
+ +
Interieur
+
+ + Gestoffeerd + + +
+ + + + + + + + + + + + + + +
+
+
Onderhoudsstaat
+
+ Zeer goed +
+
+
+
Woonoppervlakte
+
+ + 65 m² + + + +
+
Inhoud
+
+ + 197 m³ + + + +
+
+
+
Type woning
+
+ + Appartement + + + +
+
Soort woning
+
+ + Portiekwoning, Appartement + + + +
+
Soort bouw
+
+ + Bestaande bouw + + + +
+
Bouwjaar
+
+ + 1954 + + + +
+
Ligging
+
+ + +
    +
  • + + + + + + + + Aan een rustige weg +
    • +
  • + + + + + + + + Aan het water +
    • +
  • + + + + + + + + In een woonwijk +
    • +
  • + + + + + + + + Nabij het openbaar vervoer +
    • +
  • + + + + + + + + Vrij uitzicht +
    • +
+ + + +
+
Portiek
+
Type
+
+ + Gesloten portiek + + + +
+
Deurbelpaneel
+
+ + Niet aanwezig + + + +
+
+
+
Aantal kamers
+
+ + 2 + + + +
+
Aantal slaapkamers
+
+ + 1 + + + +
+
Aantal badkamers
+
+ + 1 + + + +
+
Aantal woonlagen
+
+ + 1 + + + +
+
Verdieping
+
+ + 3 + + + +
+
Voorzieningen
+
+ + +
    +
  • + + + + + + + + Toilet +
    • +
  • + + + + + + + + Douche +
    • +
+ + + +
+
Keuken
+
Apparatuur
+
+ + Inbouwapparatuur + + + +
+
+
+
Balkon
+
Ligging
+
+ + Op het zuiden + + + +
+
+
+
Isolatie
+
+ + Dubbele beglazing + + + +
+
Energielabel
+
+ + D + + + +
+
+
+
Schuur/Berging
+
+ + Aanwezig + + + +
+
Oppervlakte berging
+
+ + 6 m² + + + +
+
+
+
Aanwezig
+
+ + Nee + + + +
+
+
+
Aanwezig
+
+ + Nee + + + +
+
+

ViaDaan

+ \ No newline at end of file diff --git a/src/lib/house-tracker/fixtures/pararius-detail-house.html b/src/lib/house-tracker/fixtures/pararius-detail-house.html new file mode 100644 index 0000000..fc5a016 --- /dev/null +++ b/src/lib/house-tracker/fixtures/pararius-detail-house.html @@ -0,0 +1,304 @@ + +trimmed fixture + + + + +

Te huur: Huis Kongsbergstraat 45 in Gouda

+
2804 XV (De Mammoet)
+ +
+
Huurprijs
+
+ € 1.600 per maand + +
    +
+
+
Prijs per m²
+
+ € 14 +
+
Aangeboden sinds
+
+ 30-09-2026 +
+
Status
+
+ Te huur +
+
Beschikbaar
+
+ In overleg +
+
Huurperiode
+
+ Minimaal 12 maanden +
+ +
Borg
+
+ + € 1.600 + + +
+ + + + + + + + + + + + + + +
+
+
+
+
Woonoppervlakte
+
+ + 113 m² + + + +
+
Inhoud
+
+ + 341 m³ + + + +
+
+
+
Type woning
+
+ + Huis + + + +
+
Soort woning
+
+ + Tussenwoning (rijtjeshuis), Eengezinswoning, Kwadrantwoning + + + +
+
Soort bouw
+
+ + Bestaande bouw + + + +
+
Bouwjaar
+
+ + 1981 + + + +
+
+
+
Aantal kamers
+
+ + 5 + + + +
+
Aantal slaapkamers
+
+ + 4 + + + +
+
Aantal badkamers
+
+ + 1 + + + +
+
Aantal woonlagen
+
+ + 1 + + + +
+
Voorzieningen
+
+ + +
    +
  • + + + + + + + + Airconditioning +
    • +
  • + + + + + + + + Alarm +
    • +
  • + + + + + + + + Glasvezelaansluiting +
    • +
  • + + + + + + + + Internetaansluiting +
    • +
+ + + +
+
+
+
Isolatie
+
+ + Ecoconstructie, Vloerisolatie, Dakisolatie, Muurisolatie + + + +
+
Verwarming
+
+ + Blokverwarming, Stadsverwarming, Electrische verwarming, Volledige vloerverwarming, Gedeeltelijke vloerverwarming, Gaskachel, Warmte terugwininstallatie, Hete lucht verwarming + + + +
+
Warm water
+
+ + Centrale verwarming, CV-ketel, Geiser + + + +
+
Energielabel
+
+ + B + + + +
+
+
+
Aanwezig
+
+ + Ja + + + +
+
+

LIV Residential

+ \ No newline at end of file diff --git a/src/lib/house-tracker/fixtures/pararius-detail-on-request.html b/src/lib/house-tracker/fixtures/pararius-detail-on-request.html new file mode 100644 index 0000000..6234e40 --- /dev/null +++ b/src/lib/house-tracker/fixtures/pararius-detail-on-request.html @@ -0,0 +1,391 @@ + +trimmed fixture + + + + +

Te huur: Huis Nansenstraat in Gouda

+
2806 HK (Vreewijk)
+ +
+
Huurprijs
+
+ Prijs op aanvraag + +
    +
+
+
Aangeboden sinds
+
+ 18-11-2025 +
+
Status
+
+ Te huur +
+
Beschikbaar
+
+ Per direct +
+
Contractvorm
+
+ Tijdelijke verhuur +
+
Huurperiode
+
+ Minimaal 6, maximaal 12 maanden +
+ +
Borg
+
+ + € 2.800 + + +
+ + + + + + + + + + + + + + +
+
+ +
Interieur
+
+ + Gemeubileerd + + +
+ + + + + + + + + + + + + + +
+
+
+
+
Woonoppervlakte
+
+ + 107 m² + + + +
+
+
+
Type woning
+
+ + Huis + + + +
+
Soort woning
+
+ + Tussenwoning (rijtjeshuis), Eengezinswoning + + + +
+
Soort bouw
+
+ + Bestaande bouw + + + +
+
Bouwjaar
+
+ + 2000 + + + +
+
+
+
Aantal kamers
+
+ + 4 + + + +
+
Aantal slaapkamers
+
+ + 3 + + + +
+
Aantal badkamers
+
+ + 2 + + + +
+
Aantal woonlagen
+
+ + 3 + + + +
+
Voorzieningen
+
+ + +
    +
  • + + + + + + + + Douche +
    • +
  • + + + + + + + + Bergruimte +
    • +
  • + + + + + + + + Toilet +
    • +
+ + + +
+
+
+
Tuin
+
Grootte
+
+ + 70 m² + + + +
+
Ligging
+
+ + Op het oosten + + + +
+
+
+
Verwarming
+
+ + CV-ketel + + + +
+
Energielabel
+
+ + D + + + +
+
+
+
Schuur/Berging
+
+ + Aanwezig + + + +
+
+
+
Aanwezig
+
+ + Ja + + + +
+
Soort parkeergelegenheid
+
+ + Openbaar + + + +
+
+
+
Roken toegestaan
+
+ + Nee + + + +
+
Huisdieren toegestaan
+
+ + Niet toegestaan + + + +
+
Doelgroep
+
+ + Werkende + + + +
+
Maximaal aantal bewoners
+
+ + 6 + + + +
+
+

Staffhousing

+ \ No newline at end of file diff --git a/src/lib/house-tracker/fixtures/pararius-search-synthetic.html b/src/lib/house-tracker/fixtures/pararius-search-synthetic.html new file mode 100644 index 0000000..ddfca97 --- /dev/null +++ b/src/lib/house-tracker/fixtures/pararius-search-synthetic.html @@ -0,0 +1,107 @@ + + + + + Huurwoningen Gouda + + +
12
+ + + + diff --git a/src/lib/house-tracker/pararius.test.ts b/src/lib/house-tracker/pararius.test.ts new file mode 100644 index 0000000..2b14868 --- /dev/null +++ b/src/lib/house-tracker/pararius.test.ts @@ -0,0 +1,158 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { + cityFromSlug, + isChallenge, + listingRef, + parseDetailPage, + parseEuros, + parseLocation, + parseSearchPage, + searchPageUrl +} from './pararius'; +import { listingDetailSchema, listingCardSchema } from './types'; + +const fixture = (name: string) => + readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8'); + +describe('pararius helpers', () => { + it('reads listing urls', () => { + expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({ + sourceId: '541493b8', + url: 'https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat', + offer: 'rent', + propertyType: 'apartment', + citySlug: 'gouda' + }); + expect(listingRef('https://www.pararius.nl/huis-te-koop/den-haag/0a1b2c3d/laan')?.offer).toBe( + 'buy' + ); + expect(listingRef('/huurwoningen/gouda')).toBeNull(); + }); + + it('parses Dutch and English prices', () => { + expect(parseEuros('€ 1.558 per maand')).toBe(1558); + expect(parseEuros('€1,600 per month')).toBe(1600); + expect(parseEuros('€ 425.000 k.k.')).toBe(425000); + expect(parseEuros('€ 1.250,50')).toBe(1250); + expect(parseEuros('Prijs op aanvraag')).toBeNull(); + }); + + it('splits a location line', () => { + expect(parseLocation('2801 BC Gouda (Centrum)')).toEqual({ + postalCode: '2801 BC', + city: 'Gouda', + neighbourhood: 'Centrum' + }); + expect(parseLocation('Alphen aan den Rijn')).toEqual({ + postalCode: null, + city: 'Alphen aan den Rijn', + neighbourhood: null + }); + }); + + it('names cities from slugs', () => { + expect(cityFromSlug('gouda')).toBe('Gouda'); + expect(cityFromSlug('den-haag')).toBe('Den Haag'); + expect(cityFromSlug('alphen-aan-den-rijn')).toBe('Alphen aan den Rijn'); + }); + + it('builds page urls', () => { + const base = 'https://www.pararius.nl/huurwoningen/gouda/1300-1700'; + expect(searchPageUrl(base, 1)).toBe(base); + expect(searchPageUrl(base, 3)).toBe(`${base}/page-3`); + expect(searchPageUrl(`${base}/page-3`, 2)).toBe(`${base}/page-2`); + }); + + it('detects Cloudflare challenges', () => { + expect(isChallenge('Just a moment...')).toBe(true); + expect(isChallenge('ok', 403)).toBe(true); + expect(isChallenge(fixture('pararius-detail-house.html'))).toBe(false); + }); +}); + +describe('parseDetailPage', () => { + it('reads a house listing', () => { + const detail = parseDetailPage(fixture('pararius-detail-house.html'))!; + expect(listingDetailSchema.parse(detail)).toBeTruthy(); + expect(detail).toMatchObject({ + sourceId: '8f976237', + offer: 'rent', + propertyType: 'house', + externalId: '8f976237-2ce0-5613-adbe-883495268361', + street: 'Kongsbergstraat 45', + postalCode: '2804 XV', + city: 'Gouda', + neighbourhood: 'De Mammoet', + priceEur: 1600, + areaM2: 113, + rooms: 5, + bedrooms: 4, + constructionYear: 1981, + energyLabel: 'B', + depositEur: 1600, + agent: 'LIV Residential', + offeredAt: '2026-09-30T09:47:00.000Z' + }); + expect(detail.lat).toBeCloseTo(52.0284, 3); + expect(detail.lng).toBeCloseTo(4.7174, 3); + }); + + it('reads interior and availability', () => { + const detail = parseDetailPage(fixture('pararius-detail-apartment.html'))!; + expect(detail).toMatchObject({ + propertyType: 'apartment', + priceEur: 1558, + interior: 'gestoffeerd', + availableFrom: 'Per 15-10-2026', + contractType: 'Onbepaalde tijd', + depositEur: 3316, + offeredAt: '2026-10-02T05:35:37.000Z' + }); + }); + + it('keeps price-on-request listings', () => { + const detail = parseDetailPage(fixture('pararius-detail-on-request.html'))!; + expect(detail.priceEur).toBeNull(); + expect(detail.interior).toBe('gemeubileerd'); + expect(listingDetailSchema.parse(detail)).toBeTruthy(); + }); + + it('rejects pages that are not listings', () => { + expect(parseDetailPage('nothing')).toBeNull(); + }); +}); + +describe('parseSearchPage', () => { + it('reads listing cards and pagination', () => { + const page = parseSearchPage( + fixture('pararius-search-synthetic.html'), + 'https://www.pararius.nl/huurwoningen/gouda' + ); + expect(page.cards).toHaveLength(3); + for (const card of page.cards) expect(listingCardSchema.parse(card)).toBeTruthy(); + expect(page.cards[0]).toMatchObject({ + sourceId: 'd5d8a72c', + title: 'Appartement Raam', + postalCode: '2801 VN', + city: 'Gouda', + neighbourhood: 'Binnenstad', + priceEur: 1475, + areaM2: 58, + rooms: 3, + interior: 'Gestoffeerd', + agent: 'Voorbeeld Makelaardij' + }); + expect(page.cards[1].priceEur).toBe(1600); + expect(page.cards[2].priceEur).toBeNull(); + expect(page.nextPageUrl).toBe('https://www.pararius.nl/huurwoningen/gouda/page-2'); + expect(page.totalResults).toBe(12); + }); + + it('falls back to bare links when card markup changes', () => { + const html = `
Huis Straat 1 € 1.400 per maand 80 m²
`; + const page = parseSearchPage(html, 'https://www.pararius.nl/huurwoningen/gouda'); + expect(page.cards).toHaveLength(1); + expect(page.cards[0]).toMatchObject({ sourceId: '0a1b2c3d', priceEur: 1400, areaM2: 80 }); + }); +}); diff --git a/src/lib/house-tracker/pararius.ts b/src/lib/house-tracker/pararius.ts new file mode 100644 index 0000000..1417d11 --- /dev/null +++ b/src/lib/house-tracker/pararius.ts @@ -0,0 +1,435 @@ +// Pararius HTML parsing. Pure functions, no network: the worker fetches, these parse. +// +// Pararius has no public API, so everything here reads markup that can change +// without notice. Each parser prefers structured data (ld+json, the analytics +// blob) and falls back to looser text matching, so a redesign degrades fields +// instead of dropping listings. +import type { ListingCard, ListingDetail, Offer, PropertyType } from './types'; + +export const PARARIUS_ORIGIN = 'https://www.pararius.nl'; + +const listingPath = + /\/(appartement|huis|kamer|studio|woning|[a-z]+)-te-(huur|koop)\/([a-z0-9-]+)\/([0-9a-f]{8})\/([a-z0-9-]+)/; + +const typeWords: Record = { + appartement: 'apartment', + apartment: 'apartment', + huis: 'house', + house: 'house', + kamer: 'room', + room: 'room', + studio: 'studio' +}; + +const entities: Record = { + amp: '&', + lt: '<', + gt: '>', + quot: '"', + apos: "'", + nbsp: ' ', + euro: '€', + sup2: '²', + sup3: '³' +}; + +export function decodeEntities(value: string): string { + return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => { + if (code[0] === '#') { + const n = + code[1] === 'x' || code[1] === 'X' ? parseInt(code.slice(2), 16) : Number(code.slice(1)); + return Number.isFinite(n) ? String.fromCodePoint(n) : match; + } + return entities[code.toLowerCase()] ?? match; + }); +} + +export function textOf(html: string): string { + return decodeEntities(html.replace(/<[^>]*>/g, ' ')) + .replace(/\s+/g, ' ') + .trim(); +} + +/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */ +export function isChallenge(html: string, status = 200): boolean { + if (status === 403 || status === 429 || status === 503) return true; + const head = html.slice(0, 20_000); + return ( + /\s*Just a moment/i.test(head) || + head.includes('challenges.cloudflare.com/turnstile') || + head.includes('cf-chl-') || + /<title>\s*Attention Required/i.test(head) + ); +} + +export type ListingRef = { + sourceId: string; + url: string; + offer: Offer; + propertyType: PropertyType; + citySlug: string; +}; + +export function listingRef(href: string): ListingRef | null { + const match = listingPath.exec(href); + if (!match) return null; + const [, type, offer, citySlug, shortId, slug] = match; + return { + sourceId: shortId, + url: `${PARARIUS_ORIGIN}/${type}-te-${offer}/${citySlug}/${shortId}/${slug}`, + offer: offer === 'koop' ? 'buy' : 'rent', + propertyType: typeWords[type] ?? 'other', + citySlug + }; +} + +/** Whole euros from "€ 1.558 per maand", "€1,558 per month", "€ 425.000 k.k." */ +export function parseEuros(value: string | null | undefined): number | null { + if (!value) return null; + const match = /€\s*([\d.,]+)/.exec(value) ?? /([\d][\d.,]*)/.exec(value); + if (!match) return null; + const digits = match[1].replace(/[.,]\d{2}$/, '').replace(/[.,]/g, ''); + const n = Number(digits); + return digits && Number.isSafeInteger(n) ? n : null; +} + +export function parseInteger(value: string | null | undefined): number | null { + const match = value ? /(\d[\d.]*)/.exec(value) : null; + if (!match) return null; + const n = Number(match[1].replace(/\./g, '')); + return Number.isSafeInteger(n) ? n : null; +} + +/** "2801 BC Gouda (Centrum)" → postal code, city and neighbourhood. */ +export function parseLocation(value: string | null | undefined) { + const out = { + postalCode: null as string | null, + city: null as string | null, + neighbourhood: null as string | null + }; + if (!value) return out; + let rest = value.trim(); + const hood = /\(([^)]+)\)\s*$/.exec(rest); + if (hood) { + out.neighbourhood = hood[1].trim(); + rest = rest.slice(0, hood.index).trim(); + } + const postal = /^(\d{4})\s?([A-Z]{2})\b/.exec(rest); + if (postal) { + out.postalCode = `${postal[1]} ${postal[2]}`; + rest = rest.slice(postal[0].length).trim(); + } + if (rest) out.city = rest; + return out; +} + +function classText(html: string, className: string): string | null { + const re = new RegExp( + `<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)</\\1>`, + 'i' + ); + const match = re.exec(html); + if (!match) return null; + const value = textOf(match[2]); + return value || null; +} + +function firstImage(html: string): string | null { + const tag = /<img\b[^>]*>/i.exec(html)?.[0]; + if (!tag) return null; + const src = + /\bsrc="(https:[^"]+)"/i.exec(tag)?.[1] ?? + /\bsrcset="(https:[^"\s,]+)/i.exec(tag)?.[1] ?? + /\bdata-src="(https:[^"]+)"/i.exec(tag)?.[1]; + return src ? decodeEntities(src) : null; +} + +export type SearchPage = { + cards: ListingCard[]; + nextPageUrl: string | null; + totalResults: number | null; +}; + +/** + * Parse a search results page (`/huurwoningen/gouda/1300-1700`). + * Splits on the listing card wrappers when present; otherwise falls back to + * one chunk per unique listing link so a markup change still yields ids. + */ +export function parseSearchPage(html: string, pageUrl: string): SearchPage { + const chunks: string[] = []; + const marker = + /<(section|li|article|div)\b[^>]*class="[^"]*\blisting-search-item\b(?!__)[^"]*"/gi; + const starts = [...html.matchAll(marker)].map((m) => m.index!); + if (starts.length) { + for (let i = 0; i < starts.length; i++) { + chunks.push(html.slice(starts[i], starts[i + 1] ?? starts[i] + 20_000)); + } + } else { + const links = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/gi; + const seen = new Set<string>(); + for (const m of html.matchAll(links)) { + const ref = listingRef(m[1]); + if (!ref || seen.has(ref.sourceId)) continue; + seen.add(ref.sourceId); + chunks.push(html.slice(Math.max(0, m.index! - 3000), m.index! + 6000)); + } + } + + const cards: ListingCard[] = []; + const seen = new Set<string>(); + for (const chunk of chunks) { + const href = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/i.exec(chunk)?.[1]; + const ref = href ? listingRef(decodeEntities(href)) : null; + if (!ref || seen.has(ref.sourceId)) continue; + seen.add(ref.sourceId); + + const titleLink = /<a\b[^>]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk); + const title = + (titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title'); + const location = parseLocation( + classText(chunk, 'listing-search-item__sub-title') ?? + classText(chunk, 'listing-search-item__location') + ); + const priceText = + classText(chunk, 'listing-search-item__price') ?? /€\s*[\d.,]+[^<]{0,30}/.exec(chunk)?.[0]; + const area = + classText(chunk, 'illustrated-features__item--surface-area') ?? + /(\d+)\s*m(?:²|²|2)/i.exec(chunk)?.[0]; + const rooms = classText(chunk, 'illustrated-features__item--number-of-rooms'); + const agentBlock = classText(chunk, 'listing-search-item__info'); + + cards.push({ + sourceId: ref.sourceId, + url: ref.url, + offer: ref.offer, + propertyType: ref.propertyType, + title: title?.slice(0, 200) ?? null, + postalCode: location.postalCode, + city: location.city ?? cityFromSlug(ref.citySlug), + neighbourhood: location.neighbourhood, + priceEur: parseEuros(priceText), + areaM2: parseInteger(area), + rooms: parseInteger(rooms), + interior: classText(chunk, 'illustrated-features__item--interior'), + agent: agentBlock?.slice(0, 160) ?? null, + imageUrl: firstImage(chunk) + }); + } + + const nextHref = + /<a\b[^>]*class="[^"]*pagination__link--next[^"]*"[^>]*href="([^"]+)"/i.exec(html)?.[1] ?? + /<a\b[^>]*href="([^"]+)"[^>]*class="[^"]*pagination__link--next[^"]*"/i.exec(html)?.[1] ?? + /<link\b[^>]*rel="next"[^>]*href="([^"]+)"/i.exec(html)?.[1]; + const total = + classText(html, 'search-list-header__count') ?? + /(\d[\d.]*)\s+(?:huurwoningen|koopwoningen|woningen|results|resultaten)/i.exec( + textOf(html.slice(0, 200_000)) + )?.[1]; + + return { + cards, + nextPageUrl: nextHref ? new URL(decodeEntities(nextHref), pageUrl).toString() : null, + totalResults: parseInteger(total) + }; +} + +/** `/huurwoningen/gouda/1300-1700` → page N url (`.../page-2`). */ +export function searchPageUrl(base: string, page: number): string { + const url = new URL(base); + url.pathname = url.pathname.replace(/\/page-\d+\/?$/, '').replace(/\/$/, ''); + if (page > 1) url.pathname += `/page-${page}`; + return url.toString(); +} + +export function cityFromSlug(slug: string): string { + return slug + .split('-') + .map((part) => + part === 'aan' || part === 'den' || part === 'de' + ? part + : part[0]?.toUpperCase() + part.slice(1) + ) + .join(' ') + .replace(/^\w/, (c) => c.toUpperCase()); +} + +function jsonBlocks(html: string): unknown[] { + const out: unknown[] = []; + for (const m of html.matchAll(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) { + try { + out.push(JSON.parse(m[1])); + } catch { + // Ignore malformed blocks; other sources may still have the data. + } + } + return out; +} + +type LdListing = { + '@type'?: string | string[]; + name?: string; + description?: string; + image?: string | string[]; + address?: { + streetAddress?: string; + addressLocality?: string; + addressRegion?: string; + postalCode?: string; + }; + numberOfRooms?: { value?: number }[] | { value?: number } | number; + floorSize?: { value?: number }; + yearBuilt?: number; + numberOfBedrooms?: number; + numberOfBathroomsTotal?: number; + datePosted?: string; + offers?: { price?: number; url?: string }; +}; + +function ldListing(html: string): LdListing | null { + for (const block of jsonBlocks(html)) { + const items = Array.isArray(block) ? block : [block]; + for (const item of items) { + if (!item || typeof item !== 'object') continue; + const type = (item as LdListing)['@type']; + const types = Array.isArray(type) ? type : [type]; + if (types.includes('RealEstateListing')) return item as LdListing; + } + } + return null; +} + +/** The flat `odp.*` analytics object Pararius embeds on listing pages. */ +export function analyticsBlob(html: string): Record<string, string> { + const candidates = [html, decodeEntities(html)]; + for (const source of candidates) { + const match = /\{[^{}]*"odp\.listing_id"[^{}]*\}/.exec(source); + if (!match) continue; + try { + const parsed = JSON.parse(match[0]) as Record<string, unknown>; + const out: Record<string, string> = {}; + for (const [key, value] of Object.entries(parsed)) { + if (key.startsWith('odp.') && (typeof value === 'string' || typeof value === 'number')) + out[key.slice(4)] = String(value); + } + return out; + } catch { + // Fall through to the next candidate. + } + } + return {}; +} + +/** The "Kenmerken" definition list: `Borg → € 1.600`, `Interieur → Gestoffeerd`. */ +export function featureList(html: string): Record<string, string> { + const out: Record<string, string> = {}; + const re = + /<dt\b[^>]*class="[^"]*listing-features__term[^"]*"[^>]*>([\s\S]*?)<\/dt>\s*<dd\b[^>]*>([\s\S]*?)<\/dd>/gi; + for (const m of html.matchAll(re)) { + const key = textOf(m[1]).slice(0, 80); + const value = textOf(m[2].replace(/<[^>]*class="[^"]*tooltip[^"]*"[\s\S]*?<\/[a-z]+>/gi, '')) + .replace(/\s*Meer info$/i, '') + .slice(0, 500); + if (key && value && !(key in out)) out[key] = value; + } + return out; +} + +const interiorWords: Record<string, string> = { + shell: 'kaal', + upholstered: 'gestoffeerd', + furnished: 'gemeubileerd' +}; + +const firstNumber = (value: LdListing['numberOfRooms']): number | null => { + if (typeof value === 'number') return value; + const item = Array.isArray(value) ? value[0] : value; + return typeof item?.value === 'number' ? item.value : null; +}; + +/** Parse a listing page. Returns null when the page is not a listing. */ +export function parseDetailPage(html: string, fallbackUrl?: string): ListingDetail | null { + const canonical = + /<link\b[^>]*rel="canonical"[^>]*href="([^"]+)"/i.exec(html)?.[1] ?? + /<link\b[^>]*href="([^"]+)"[^>]*rel="canonical"/i.exec(html)?.[1]; + const ref = listingRef(canonical ?? '') ?? listingRef(fallbackUrl ?? ''); + if (!ref) return null; + + const ld = ldListing(html); + const odp = analyticsBlob(html); + const features = featureList(html); + const summaryTitle = classText(html, 'listing-detail-summary__title'); + const location = parseLocation(classText(html, 'listing-detail-summary__location')); + + const lat = Number( + /data-latitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"latitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1] + ); + const lng = Number( + /data-longitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"longitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1] + ); + + const offeredAt = + odp.offered_since && !Number.isNaN(Date.parse(odp.offered_since)) + ? new Date(odp.offered_since).toISOString() + : ld?.datePosted + ? new Date(`${ld.datePosted}T00:00:00Z`).toISOString() + : null; + + const price = + (odp.price_pcm ? Number(odp.price_pcm) : null) ?? + (typeof ld?.offers?.price === 'number' ? ld.offers.price : null) ?? + parseEuros(features['Huurprijs'] ?? features['Vraagprijs'] ?? features['Koopprijs']); + + const image = Array.isArray(ld?.image) ? ld.image[0] : ld?.image; + const typeFromOdp = odp.property ? typeWords[odp.property] : undefined; + const interior = + (odp.interior ? (interiorWords[odp.interior] ?? odp.interior) : null) ?? + features['Interieur']?.toLowerCase() ?? + null; + + const street = ld?.address?.streetAddress ?? null; + const title = + summaryTitle?.replace(/^Te (huur|koop):\s*/i, '') ?? (ld?.name ? `${ld.name}` : null); + + return { + sourceId: ref.sourceId, + url: ref.url, + offer: odp.offer === 'buy' || ref.offer === 'buy' ? 'buy' : 'rent', + propertyType: typeFromOdp ?? ref.propertyType, + externalId: odp.listing_id ?? null, + title: title?.slice(0, 200) ?? null, + street, + postalCode: ld?.address?.postalCode + ? ld.address.postalCode.replace(/^(\d{4})\s?([A-Z]{2})$/, '$1 $2') + : location.postalCode, + city: ld?.address?.addressLocality ?? location.city ?? cityFromSlug(ref.citySlug), + neighbourhood: ld?.address?.addressRegion ?? location.neighbourhood, + priceEur: Number.isFinite(price) ? price : null, + areaM2: + parseInteger(odp.floor_space) ?? + (typeof ld?.floorSize?.value === 'number' ? ld.floorSize.value : null) ?? + parseInteger(features['Woonoppervlakte']), + rooms: + parseInteger(odp.rooms) ?? + firstNumber(ld?.numberOfRooms) ?? + parseInteger(features['Aantal kamers']), + bedrooms: + parseInteger(odp.bedrooms) ?? + ld?.numberOfBedrooms ?? + parseInteger(features['Aantal slaapkamers']), + bathrooms: ld?.numberOfBathroomsTotal ?? parseInteger(features['Aantal badkamers']), + constructionYear: + parseInteger(odp.construction_year) ?? ld?.yearBuilt ?? parseInteger(features['Bouwjaar']), + energyLabel: odp.energy_rating ?? features['Energielabel'] ?? null, + interior, + depositEur: parseEuros(features['Borg']), + availableFrom: features['Beschikbaar'] ?? null, + contractType: features['Contractvorm'] ?? features['Huurperiode'] ?? null, + agent: classText(html, 'agent-card__title'), + imageUrl: image ?? null, + lat: Number.isFinite(lat) && lat !== 0 ? lat : null, + lng: Number.isFinite(lng) && lng !== 0 ? lng : null, + offeredAt, + description: ld?.description?.slice(0, 6000) ?? null, + features + }; +} diff --git a/src/lib/house-tracker/schedule.ts b/src/lib/house-tracker/schedule.ts new file mode 100644 index 0000000..f085ccf --- /dev/null +++ b/src/lib/house-tracker/schedule.ts @@ -0,0 +1,70 @@ +// When the home worker should poll next. Pure so it can be tested. +// +// Listings near Gouda were all published on weekdays between 07:30 and 17:30, +// so the fast lane polls often during office hours and backs off otherwise. + +export type ScheduleOptions = { + /** Weekday office-hours interval, in minutes. */ + busyMinutes: [number, number]; + /** Evenings, nights and weekends. */ + quietMinutes: [number, number]; + busyStartHour: number; + busyEndHour: number; + timeZone: string; +}; + +export const fastLaneSchedule: ScheduleOptions = { + busyMinutes: [3, 5], + quietMinutes: [30, 60], + busyStartHour: 7, + busyEndHour: 18, + timeZone: 'Europe/Amsterdam' +}; + +export function localClock(date: Date, timeZone: string) { + const parts = new Intl.DateTimeFormat('en-GB', { + timeZone, + weekday: 'short', + hour: 'numeric', + minute: 'numeric', + hourCycle: 'h23' + }).formatToParts(date); + const get = (type: string) => parts.find((part) => part.type === type)?.value ?? ''; + return { + weekday: get('weekday'), + hour: Number(get('hour')), + minute: Number(get('minute')) + }; +} + +export function isBusy(date: Date, options: ScheduleOptions = fastLaneSchedule): boolean { + const { weekday, hour } = localClock(date, options.timeZone); + if (weekday === 'Sat' || weekday === 'Sun') return false; + return hour >= options.busyStartHour && hour < options.busyEndHour; +} + +/** Milliseconds until the next poll, with jitter so requests never look clockwork. */ +export function nextDelayMs( + now: Date, + options: ScheduleOptions = fastLaneSchedule, + random: () => number = Math.random +): number { + const [min, max] = isBusy(now, options) ? options.busyMinutes : options.quietMinutes; + let delay = (min + (max - min) * random()) * 60_000; + // Don't sleep through the start of office hours. + if (!isBusy(now, options)) { + for (let step = 60_000; step < delay; step += 60_000) { + if (isBusy(new Date(now.getTime() + step), options)) { + delay = step + random() * 60_000; + break; + } + } + } + return Math.round(delay); +} + +/** Exponential backoff after challenges or errors: 15 min, 30, 60 ... capped at 6 h. */ +export function backoffMs(failures: number, random: () => number = Math.random): number { + const base = 15 * 60_000 * 2 ** Math.max(0, failures - 1); + return Math.round(Math.min(base, 6 * 60 * 60_000) * (0.85 + 0.3 * random())); +} diff --git a/src/lib/house-tracker/types.ts b/src/lib/house-tracker/types.ts new file mode 100644 index 0000000..17bbdcb --- /dev/null +++ b/src/lib/house-tracker/types.ts @@ -0,0 +1,118 @@ +// Shared between the home worker (scripts/house-worker.ts), the ingest API and +// the page. Keep this file free of `$lib` / `$env` imports: the worker runs it +// directly with bun, outside SvelteKit. +import { z } from 'zod'; + +export const offers = ['rent', 'buy'] as const; +export type Offer = (typeof offers)[number]; + +export const propertyTypes = ['apartment', 'house', 'room', 'studio', 'other'] as const; +export type PropertyType = (typeof propertyTypes)[number]; + +const text = (max: number) => z.string().trim().max(max); +const optionalText = (max: number) => text(max).nullish(); +const count = z.number().int().min(0).max(10_000).nullish(); + +// What a search results page tells us about a listing. This alone is enough to +// send an alert, so the fast lane never waits for the detail page. +export const listingCardSchema = z.object({ + sourceId: text(80).min(1), + url: z.url().max(500), + offer: z.enum(offers), + propertyType: z.enum(propertyTypes), + title: optionalText(200), + postalCode: optionalText(10), + city: optionalText(80), + neighbourhood: optionalText(120), + priceEur: z.number().int().min(0).max(100_000_000).nullish(), + areaM2: count, + rooms: count, + interior: optionalText(40), + agent: optionalText(160), + imageUrl: z.url().max(1000).nullish() +}); +export type ListingCard = z.infer<typeof listingCardSchema>; + +// Everything else we learn from the listing page itself. +export const listingDetailSchema = listingCardSchema.extend({ + externalId: optionalText(80), + street: optionalText(200), + bedrooms: count, + bathrooms: count, + constructionYear: z.number().int().min(1000).max(2100).nullish(), + energyLabel: optionalText(10), + depositEur: z.number().int().min(0).max(10_000_000).nullish(), + availableFrom: optionalText(80), + contractType: optionalText(80), + lat: z.number().min(-90).max(90).nullish(), + lng: z.number().min(-180).max(180).nullish(), + offeredAt: z.iso.datetime({ offset: true }).nullish(), + description: optionalText(6000), + features: z.record(z.string().max(80), z.string().max(500)).optional() +}); +export type ListingDetail = z.infer<typeof listingDetailSchema>; + +export const listingStatuses = [ + 'interested', + 'contacted', + 'viewing', + 'applied', + 'rejected' +] as const; +export type ListingStatus = (typeof listingStatuses)[number]; + +export const lanes = ['fast', 'slow'] as const; +export type Lane = (typeof lanes)[number]; + +// One poll of one search page. +export const syncRequestSchema = z.object({ + source: z.string().regex(/^[a-z0-9-]{1,40}$/), + lane: z.enum(lanes), + searchUrl: z.url().max(500), + startedAt: z.iso.datetime({ offset: true }), + cards: z.array(listingCardSchema).max(200), + // Set on the last page of a full, unfiltered sweep of one city. Listings in + // that scope that were not seen during the sweep are then marked removed. + completedSweep: z + .object({ + offer: z.enum(offers), + city: text(80).min(1), + sweepStartedAt: z.iso.datetime({ offset: true }) + }) + .nullish() +}); +export type SyncRequest = z.infer<typeof syncRequestSchema>; + +export const detailsRequestSchema = z.object({ + source: z.string().regex(/^[a-z0-9-]{1,40}$/), + details: z.array(listingDetailSchema).max(50) +}); +export type DetailsRequest = z.infer<typeof detailsRequestSchema>; + +export const runStatuses = [ + 'ok', + 'challenged', + 'http-error', + 'parse-error', + 'network-error' +] as const; +export type RunStatus = (typeof runStatuses)[number]; + +export const reportRequestSchema = z.object({ + source: z.string().regex(/^[a-z0-9-]{1,40}$/), + lane: z.enum(lanes), + url: z.url().max(500), + status: z.enum(runStatuses), + httpStatus: z.number().int().min(0).max(999).nullish(), + found: z.number().int().min(0).max(10_000).nullish(), + durationMs: z.number().int().min(0).max(600_000).nullish(), + message: optionalText(500) +}); +export type ReportRequest = z.infer<typeof reportRequestSchema>; + +export type SyncResponse = { + newIds: string[]; + // Listings the server has no detail page for yet, newest first. + needDetails: { sourceId: string; url: string }[]; + alerted: number; +}; diff --git a/src/lib/projects.ts b/src/lib/projects.ts index 54e0b37..01fdbf8 100644 --- a/src/lib/projects.ts +++ b/src/lib/projects.ts @@ -47,6 +47,12 @@ export const projects = [ href: '/projects/money-tracker', external: false }, + { + name: 'house tracker', + subtitle: 'Dutch rental and sale listings, with alerts', + href: '/projects/house-tracker', + external: false + }, { name: 'cs prices', subtitle: 'Counter-Strike market prices across time', diff --git a/src/lib/server/db/schema/house-tracker.ts b/src/lib/server/db/schema/house-tracker.ts new file mode 100644 index 0000000..2fefd8b --- /dev/null +++ b/src/lib/server/db/schema/house-tracker.ts @@ -0,0 +1,128 @@ +import { + bigint, + boolean, + check, + doublePrecision, + index, + integer, + jsonb, + pgTable, + text, + uniqueIndex +} from 'drizzle-orm/pg-core'; +import { sql } from 'drizzle-orm'; +import { id, timestamptz } from './helpers'; + +// Created by scripts/house-tracker-migration.sql; keep both in sync. +export const houseListings = pgTable( + 'house_listings', + { + id: id(), + source: text('source').notNull(), + sourceId: text('source_id').notNull(), + externalId: text('external_id'), + url: text('url').notNull(), + offer: text('offer').notNull(), + propertyType: text('property_type').notNull(), + title: text('title'), + street: text('street'), + postalCode: text('postal_code'), + city: text('city'), + citySlug: text('city_slug'), + neighbourhood: text('neighbourhood'), + priceEur: integer('price_eur'), + areaM2: integer('area_m2'), + rooms: integer('rooms'), + bedrooms: integer('bedrooms'), + bathrooms: integer('bathrooms'), + constructionYear: integer('construction_year'), + energyLabel: text('energy_label'), + interior: text('interior'), + depositEur: integer('deposit_eur'), + availableFrom: text('available_from'), + contractType: text('contract_type'), + agent: text('agent'), + imageUrl: text('image_url'), + description: text('description'), + features: jsonb('features').$type<Record<string, string>>().notNull().default({}), + lat: doublePrecision('lat'), + lng: doublePrecision('lng'), + offeredAt: timestamptz('offered_at'), + firstSeenAt: timestamptz('first_seen_at').notNull().defaultNow(), + firstSeenLane: text('first_seen_lane').notNull().default('slow'), + lastSeenAt: timestamptz('last_seen_at').notNull().defaultNow(), + detailFetchedAt: timestamptz('detail_fetched_at'), + removedAt: timestamptz('removed_at'), + alertedAt: timestamptz('alerted_at'), + status: text('status'), + notes: text('notes') + }, + (table) => [ + check('house_listings_offer_check', sql`offer IN ('rent', 'buy')`), + check('house_listings_price_eur_check', sql`price_eur IS NULL OR price_eur >= 0`), + uniqueIndex('house_listings_source_idx').on(table.source, table.sourceId), + index('house_listings_active_idx') + .on(table.offer, table.firstSeenAt.desc()) + .where(sql`removed_at IS NULL`), + index('house_listings_city_idx').on(table.source, table.offer, table.citySlug), + index('house_listings_details_idx') + .on(table.firstSeenAt.desc()) + .where(sql`detail_fetched_at IS NULL`) + ] +); + +export const housePriceHistory = pgTable( + 'house_price_history', + { + listingId: integer('listing_id') + .notNull() + .references(() => houseListings.id, { onDelete: 'cascade' }), + observedAt: timestamptz('observed_at').notNull().defaultNow(), + priceEur: integer('price_eur').notNull() + }, + (table) => [index('house_price_history_listing_idx').on(table.listingId, table.observedAt)] +); + +export const houseSearches = pgTable('house_searches', { + searchUrl: text('search_url').primaryKey(), + source: text('source').notNull(), + lane: text('lane').notNull(), + firstSyncedAt: timestamptz('first_synced_at').notNull().defaultNow(), + lastSyncedAt: timestamptz('last_synced_at').notNull().defaultNow(), + lastFound: integer('last_found').notNull().default(0) +}); + +export const houseWorkerRuns = pgTable( + 'house_worker_runs', + { + id: bigint('id', { mode: 'number' }).primaryKey().generatedAlwaysAsIdentity(), + source: text('source').notNull(), + lane: text('lane').notNull(), + url: text('url').notNull(), + status: text('status').notNull(), + httpStatus: integer('http_status'), + found: integer('found'), + durationMs: integer('duration_ms'), + message: text('message'), + createdAt: timestamptz('created_at').notNull().defaultNow() + }, + (table) => [index('house_worker_runs_created_idx').on(table.createdAt.desc())] +); + +export const houseAlertRules = pgTable( + 'house_alert_rules', + { + id: id(), + name: text('name').notNull(), + enabled: boolean('enabled').notNull().default(true), + offer: text('offer').notNull().default('rent'), + minPrice: integer('min_price'), + maxPrice: integer('max_price'), + minArea: integer('min_area'), + minBedrooms: integer('min_bedrooms'), + cities: text('cities').array().notNull().default([]), + propertyTypes: text('property_types').array().notNull().default([]), + createdAt: timestamptz('created_at').notNull().defaultNow() + }, + () => [check('house_alert_rules_offer_check', sql`offer IN ('rent', 'buy')`)] +); diff --git a/src/lib/server/db/schema/index.ts b/src/lib/server/db/schema/index.ts index a8f65d9..4265355 100644 --- a/src/lib/server/db/schema/index.ts +++ b/src/lib/server/db/schema/index.ts @@ -10,3 +10,4 @@ export * from './animaldex'; export * from './lists'; export * from './money-tracker'; export * from './cs-prices'; +export * from './house-tracker'; diff --git a/src/lib/server/house-tracker.ts b/src/lib/server/house-tracker.ts new file mode 100644 index 0000000..9cf2260 --- /dev/null +++ b/src/lib/server/house-tracker.ts @@ -0,0 +1,360 @@ +// House tracker ingest: the home worker POSTs what it scraped, this stores it +// and sends alerts. Nothing here talks to the listing sites. +import { timingSafeEqual } from 'node:crypto'; +import { env } from '$env/dynamic/private'; +import { error } from '@sveltejs/kit'; +import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm'; +import { db } from '$lib/server/db'; +import { + houseAlertRules, + houseListings, + housePriceHistory, + houseSearches, + houseWorkerRuns +} from '$lib/server/db/schema'; +import { + alertMessage, + matchesRule, + type AlertMessage, + type AlertRule +} from '$lib/house-tracker/alerts'; +import { listingRef } from '$lib/house-tracker/pararius'; +import type { + DetailsRequest, + ListingCard, + ReportRequest, + SyncRequest, + SyncResponse +} from '$lib/house-tracker/types'; + +export function requireWorker(request: Request) { + const expected = env.HOUSE_WORKER_TOKEN ?? ''; + if (expected.length < 32) error(503, 'HOUSE_WORKER_TOKEN is not configured'); + const given = (request.headers.get('authorization') ?? '').replace(/^Bearer\s+/i, ''); + const a = Buffer.from(given); + const b = Buffer.from(expected); + if (a.length !== b.length || !timingSafeEqual(a, b)) error(401, 'Invalid worker token'); +} + +function citySlug(card: Pick<ListingCard, 'url'>): string | null { + return listingRef(card.url)?.citySlug ?? null; +} + +type Tx = Parameters<Parameters<typeof db.transaction>[0]>[0]; + +async function recordPrice( + tx: Tx, + listingId: number, + previous: number | null, + next: number | null +) { + if (next == null || next === previous) return; + await tx.insert(housePriceHistory).values({ listingId, priceEur: next }); +} + +export async function syncCards(input: SyncRequest): Promise<SyncResponse> { + const sourceIds = input.cards.map((card) => card.sourceId); + const toAlert: { id: number; message: AlertMessage }[] = []; + const newIds: string[] = []; + + const rules = (await db + .select() + .from(houseAlertRules) + .where(eq(houseAlertRules.enabled, true))) as AlertRule[]; + + await db.transaction(async (tx) => { + const [search] = await tx + .select({ firstSyncedAt: houseSearches.firstSyncedAt }) + .from(houseSearches) + .where(eq(houseSearches.searchUrl, input.searchUrl)); + const seeding = !search; + await tx + .insert(houseSearches) + .values({ + searchUrl: input.searchUrl, + source: input.source, + lane: input.lane, + lastFound: input.cards.length + }) + .onConflictDoUpdate({ + target: houseSearches.searchUrl, + set: { lastSyncedAt: sql`now()`, lastFound: input.cards.length, lane: input.lane } + }); + + const existing = sourceIds.length + ? await tx + .select({ + id: houseListings.id, + sourceId: houseListings.sourceId, + priceEur: houseListings.priceEur + }) + .from(houseListings) + .where( + and(eq(houseListings.source, input.source), inArray(houseListings.sourceId, sourceIds)) + ) + : []; + const known = new Map(existing.map((row) => [row.sourceId, row])); + + for (const card of input.cards) { + const previous = known.get(card.sourceId); + if (previous) { + await tx + .update(houseListings) + .set({ + lastSeenAt: sql`now()`, + removedAt: null, + url: card.url, + priceEur: card.priceEur ?? previous.priceEur, + title: sql`coalesce(${houseListings.title}, ${card.title ?? null})`, + postalCode: sql`coalesce(${houseListings.postalCode}, ${card.postalCode ?? null})`, + neighbourhood: sql`coalesce(${houseListings.neighbourhood}, ${card.neighbourhood ?? null})`, + areaM2: sql`coalesce(${houseListings.areaM2}, ${card.areaM2 ?? null}::int)`, + rooms: sql`coalesce(${houseListings.rooms}, ${card.rooms ?? null}::int)`, + agent: sql`coalesce(${houseListings.agent}, ${card.agent ?? null})`, + imageUrl: sql`coalesce(${houseListings.imageUrl}, ${card.imageUrl ?? null})` + }) + .where(eq(houseListings.id, previous.id)); + await recordPrice(tx, previous.id, previous.priceEur, card.priceEur ?? null); + continue; + } + + const [row] = await tx + .insert(houseListings) + .values({ + source: input.source, + sourceId: card.sourceId, + url: card.url, + offer: card.offer, + propertyType: card.propertyType, + title: card.title, + postalCode: card.postalCode, + city: card.city, + citySlug: citySlug(card), + neighbourhood: card.neighbourhood, + priceEur: card.priceEur, + areaM2: card.areaM2, + rooms: card.rooms, + interior: card.interior, + agent: card.agent, + imageUrl: card.imageUrl, + // Seeded listings were already online; their detection delay means nothing. + firstSeenLane: seeding ? 'seed' : input.lane + }) + .onConflictDoNothing() + .returning({ id: houseListings.id }); + if (!row) continue; + newIds.push(card.sourceId); + await recordPrice(tx, row.id, null, card.priceEur ?? null); + if (seeding) continue; + const listing = { ...card, citySlug: citySlug(card), source: input.source }; + const rule = rules.find((candidate) => matchesRule(listing, candidate)); + if (rule) toAlert.push({ id: row.id, message: alertMessage(listing, rule) }); + } + + if (input.completedSweep) { + await tx + .update(houseListings) + .set({ removedAt: sql`now()` }) + .where( + and( + eq(houseListings.source, input.source), + eq(houseListings.offer, input.completedSweep.offer), + eq(houseListings.citySlug, input.completedSweep.city), + isNull(houseListings.removedAt), + lt(houseListings.lastSeenAt, new Date(input.completedSweep.sweepStartedAt)) + ) + ); + } + + await tx.insert(houseWorkerRuns).values({ + source: input.source, + lane: input.lane, + url: input.searchUrl, + status: 'ok', + found: input.cards.length, + durationMs: Math.max(0, Date.now() - Date.parse(input.startedAt)), + message: seeding ? 'first sync: seeded without alerts' : null + }); + }); + + let alerted = 0; + for (const { id, message } of toAlert) { + if (await sendAlert(message)) { + alerted++; + await db + .update(houseListings) + .set({ alertedAt: sql`now()` }) + .where(eq(houseListings.id, id)); + } + } + + // Fast lane: detail pages for this search first. Otherwise slowly backfill. + const pending = await db + .select({ sourceId: houseListings.sourceId, url: houseListings.url }) + .from(houseListings) + .where( + and( + eq(houseListings.source, input.source), + isNull(houseListings.detailFetchedAt), + isNull(houseListings.removedAt), + sourceIds.length && input.lane === 'fast' + ? inArray(houseListings.sourceId, sourceIds) + : undefined + ) + ) + .orderBy(desc(houseListings.firstSeenAt)) + .limit(input.lane === 'fast' ? 10 : 5); + + return { newIds, needDetails: pending, alerted }; +} + +export async function saveDetails(input: DetailsRequest) { + let saved = 0; + await db.transaction(async (tx) => { + for (const detail of input.details) { + const [previous] = await tx + .select({ id: houseListings.id, priceEur: houseListings.priceEur }) + .from(houseListings) + .where( + and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId)) + ); + const values = { + externalId: detail.externalId, + url: detail.url, + offer: detail.offer, + propertyType: detail.propertyType, + title: detail.title, + street: detail.street, + postalCode: detail.postalCode, + city: detail.city, + citySlug: citySlug(detail), + neighbourhood: detail.neighbourhood, + priceEur: detail.priceEur, + areaM2: detail.areaM2, + rooms: detail.rooms, + bedrooms: detail.bedrooms, + bathrooms: detail.bathrooms, + constructionYear: detail.constructionYear, + energyLabel: detail.energyLabel, + interior: detail.interior, + depositEur: detail.depositEur, + availableFrom: detail.availableFrom, + contractType: detail.contractType, + agent: detail.agent, + imageUrl: detail.imageUrl, + description: detail.description, + features: detail.features ?? {}, + lat: detail.lat, + lng: detail.lng, + offeredAt: detail.offeredAt ? new Date(detail.offeredAt) : null, + detailFetchedAt: sql`now()`, + lastSeenAt: sql`now()` + }; + if (previous) { + await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id)); + await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null); + } else { + const [row] = await tx + .insert(houseListings) + .values({ ...values, source: input.source, sourceId: detail.sourceId }) + .returning({ id: houseListings.id }); + await recordPrice(tx, row.id, null, detail.priceEur ?? null); + } + saved++; + } + }); + return { saved }; +} + +export async function recordReport(input: ReportRequest) { + // One "challenged" notification per 6 hours is plenty. + const [recent] = + input.status === 'challenged' + ? await db + .select({ id: houseWorkerRuns.id }) + .from(houseWorkerRuns) + .where( + and( + eq(houseWorkerRuns.status, 'challenged'), + sql`${houseWorkerRuns.createdAt} > now() - interval '6 hours'` + ) + ) + .limit(1) + : []; + await db.insert(houseWorkerRuns).values({ + source: input.source, + lane: input.lane, + url: input.url, + status: input.status, + httpStatus: input.httpStatus, + found: input.found, + durationMs: input.durationMs, + message: input.message + }); + // Keep a month of history; runs are only for spotting trouble. + await db + .delete(houseWorkerRuns) + .where(lt(houseWorkerRuns.createdAt, sql`now() - interval '30 days'`)); + if (input.status === 'challenged' && !recent) { + await sendAlert({ + title: '⚠️ House worker was challenged', + body: `${input.source} answered ${input.httpStatus ?? '?'} on ${input.url}. The worker is backing off.`, + url: input.url + }); + } +} + +/** Discord webhook and/or ntfy. Returns true when at least one channel accepted it. */ +export async function sendAlert(message: AlertMessage): Promise<boolean> { + const jobs: Promise<boolean>[] = []; + if (env.HOUSE_DISCORD_WEBHOOK_URL) { + jobs.push( + fetch(env.HOUSE_DISCORD_WEBHOOK_URL, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + username: 'house tracker', + embeds: [ + { + title: message.title.slice(0, 256), + url: message.url, + description: message.body.slice(0, 2000), + color: 0x2f9e44 + } + ] + }), + signal: AbortSignal.timeout(8000) + }).then( + (response) => response.ok, + () => false + ) + ); + } + if (env.HOUSE_NTFY_URL) { + const url = new URL(env.HOUSE_NTFY_URL); + const topic = url.pathname.replace(/^\/+|\/+$/g, ''); + jobs.push( + fetch(url.origin, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(env.HOUSE_NTFY_TOKEN ? { authorization: `Bearer ${env.HOUSE_NTFY_TOKEN}` } : {}) + }, + body: JSON.stringify({ + topic, + title: message.title, + message: message.body, + click: message.url, + priority: 4, + tags: ['house'] + }), + signal: AbortSignal.timeout(8000) + }).then( + (response) => response.ok, + () => false + ) + ); + } + if (!jobs.length) return false; + return (await Promise.all(jobs)).some(Boolean); +} diff --git a/src/routes/projects/house-tracker/+page.server.ts b/src/routes/projects/house-tracker/+page.server.ts new file mode 100644 index 0000000..004b5d0 --- /dev/null +++ b/src/routes/projects/house-tracker/+page.server.ts @@ -0,0 +1,235 @@ +import { fail, redirect } from '@sveltejs/kit'; +import { and, asc, desc, eq, gte, ilike, isNull, lte, or, sql, type SQL } from 'drizzle-orm'; +import { db } from '$lib/server/db'; +import { houseAlertRules, houseListings, houseWorkerRuns } from '$lib/server/db/schema'; +import { requireAdmin } from '$lib/server/authorization'; +import { sendAlert } from '$lib/server/house-tracker'; +import { normalizeCity } from '$lib/house-tracker/alerts'; +import { listingStatuses, propertyTypes } from '$lib/house-tracker/types'; +import type { Actions, PageServerLoad } from './$types'; + +const int = (value: string | null) => { + const n = Number(value); + return value && Number.isInteger(n) && n >= 0 && n < 100_000_000 ? n : null; +}; + +export const load: PageServerLoad = async (event) => { + if (!event.locals.user) redirect(303, '/login?returnTo=/projects/house-tracker'); + requireAdmin(event); + const params = event.url.searchParams; + const offer = params.get('offer') === 'buy' ? 'buy' : 'rent'; + const city = (params.get('city') ?? '').toLowerCase().replace(/[^a-z0-9-]/g, ''); + const min = int(params.get('min')); + const max = int(params.get('max')); + const showRemoved = params.get('removed') === '1'; + const status = params.get('status') ?? ''; + const q = (params.get('q') ?? '').trim().slice(0, 80); + const sort = ['new', 'price', 'ppm2', 'area'].includes(params.get('sort') ?? '') + ? (params.get('sort') as string) + : 'new'; + + const where: (SQL | undefined)[] = [eq(houseListings.offer, offer)]; + if (city) where.push(eq(houseListings.citySlug, city)); + if (min != null) where.push(gte(houseListings.priceEur, min)); + if (max != null) where.push(lte(houseListings.priceEur, max)); + if (!showRemoved) where.push(isNull(houseListings.removedAt)); + if (status === 'any') where.push(sql`${houseListings.status} IS NOT NULL`); + else if ((listingStatuses as readonly string[]).includes(status)) + where.push(eq(houseListings.status, status)); + if (q) { + const pattern = `%${q.replaceAll('\\', '\\\\').replaceAll('%', '\\%').replaceAll('_', '\\_')}%`; + where.push( + or( + ilike(houseListings.title, pattern), + ilike(houseListings.neighbourhood, pattern), + ilike(houseListings.postalCode, pattern), + ilike(houseListings.agent, pattern) + ) + ); + } + const ppm2 = sql<number>`${houseListings.priceEur}::float / nullif(${houseListings.areaM2}, 0)`; + const order = { + new: [desc(houseListings.firstSeenAt)], + price: [asc(houseListings.priceEur), desc(houseListings.firstSeenAt)], + ppm2: [sql`${ppm2} ASC NULLS LAST`, desc(houseListings.firstSeenAt)], + area: [sql`${houseListings.areaM2} DESC NULLS LAST`, desc(houseListings.firstSeenAt)] + }[sort]!; + + const [listings, cities, stats, rules, runs, health] = await Promise.all([ + db + .select({ + id: houseListings.id, + source: houseListings.source, + url: houseListings.url, + propertyType: houseListings.propertyType, + title: houseListings.title, + city: houseListings.city, + neighbourhood: houseListings.neighbourhood, + postalCode: houseListings.postalCode, + priceEur: houseListings.priceEur, + areaM2: houseListings.areaM2, + rooms: houseListings.rooms, + bedrooms: houseListings.bedrooms, + energyLabel: houseListings.energyLabel, + interior: houseListings.interior, + availableFrom: houseListings.availableFrom, + agent: houseListings.agent, + imageUrl: houseListings.imageUrl, + lat: houseListings.lat, + lng: houseListings.lng, + offeredAt: houseListings.offeredAt, + firstSeenAt: houseListings.firstSeenAt, + firstSeenLane: houseListings.firstSeenLane, + removedAt: houseListings.removedAt, + alertedAt: houseListings.alertedAt, + status: houseListings.status, + notes: houseListings.notes + }) + .from(houseListings) + .where(and(...where)) + .orderBy(...order) + .limit(400), + db.execute(sql`SELECT city_slug AS slug, min(city) AS name, + count(*) FILTER (WHERE removed_at IS NULL)::int AS active + FROM house_listings WHERE offer = ${offer} AND city_slug IS NOT NULL + GROUP BY city_slug ORDER BY active DESC, slug LIMIT 200`), + db.execute(sql`SELECT coalesce(min(city), city_slug) AS city, + count(*) FILTER (WHERE removed_at IS NULL)::int AS active, + count(*) FILTER (WHERE first_seen_at > now() - interval '7 days')::int AS new_week, + percentile_cont(0.5) WITHIN GROUP (ORDER BY price_eur) + FILTER (WHERE removed_at IS NULL AND price_eur > 0) AS median_price, + percentile_cont(0.5) WITHIN GROUP (ORDER BY price_eur::float / area_m2) + FILTER (WHERE removed_at IS NULL AND price_eur > 0 AND area_m2 > 0) AS median_ppm2, + percentile_cont(0.5) WITHIN GROUP ( + ORDER BY extract(epoch FROM removed_at - coalesce(offered_at, first_seen_at)) / 86400) + FILTER (WHERE removed_at > now() - interval '90 days') AS median_days_listed + FROM house_listings WHERE offer = ${offer} AND city_slug IS NOT NULL + GROUP BY city_slug ORDER BY active DESC LIMIT 30`), + db.select().from(houseAlertRules).orderBy(asc(houseAlertRules.id)), + db.select().from(houseWorkerRuns).orderBy(desc(houseWorkerRuns.createdAt)).limit(25), + db.execute(sql`SELECT + (SELECT max(created_at) FROM house_worker_runs WHERE status = 'ok') AS last_ok, + (SELECT max(created_at) FROM house_worker_runs WHERE status = 'challenged') AS last_challenge, + (SELECT count(*)::int FROM house_listings) AS total, + (SELECT count(*)::int FROM house_listings WHERE alerted_at > now() - interval '7 days') AS alerts_week, + (SELECT percentile_cont(0.5) WITHIN GROUP ( + ORDER BY extract(epoch FROM first_seen_at - offered_at) / 60) + FROM house_listings WHERE first_seen_lane = 'fast' AND offered_at IS NOT NULL + AND first_seen_at >= offered_at AND first_seen_at - offered_at < interval '3 days' + AND first_seen_at > now() - interval '30 days') + AS median_detect_minutes`) + ]); + + return { + filters: { offer, city, min, max, showRemoved, status, q, sort }, + listings, + cities: cities as unknown as { slug: string; name: string | null; active: number }[], + stats: stats as unknown as { + city: string; + active: number; + new_week: number; + median_price: number | null; + median_ppm2: number | null; + median_days_listed: number | null; + }[], + rules, + runs, + health: health[0] as { + last_ok: Date | string | null; + last_challenge: Date | string | null; + total: number; + alerts_week: number; + median_detect_minutes: number | null; + }, + statuses: listingStatuses, + propertyTypes + }; +}; + +const nullableInt = (value: FormDataEntryValue | null) => { + const text = String(value ?? '').trim(); + if (!text) return null; + const n = Number(text); + return Number.isInteger(n) && n >= 0 && n < 100_000_000 ? n : undefined; +}; + +export const actions: Actions = { + status: async (event) => { + requireAdmin(event); + const form = await event.request.formData(); + const id = Number(form.get('id')); + const status = String(form.get('status') ?? ''); + const notes = String(form.get('notes') ?? '') + .trim() + .slice(0, 2000); + if (!Number.isInteger(id) || id <= 0) return fail(400, { message: 'Invalid listing' }); + if (status && !(listingStatuses as readonly string[]).includes(status)) + return fail(400, { message: 'Invalid status' }); + await db + .update(houseListings) + .set({ status: status || null, ...(form.has('notes') ? { notes: notes || null } : {}) }) + .where(eq(houseListings.id, id)); + return { ok: true }; + }, + saveRule: async (event) => { + requireAdmin(event); + const form = await event.request.formData(); + const id = Number(form.get('id') || 0); + const name = String(form.get('name') ?? '') + .trim() + .slice(0, 80); + const minPrice = nullableInt(form.get('minPrice')); + const maxPrice = nullableInt(form.get('maxPrice')); + const minArea = nullableInt(form.get('minArea')); + const minBedrooms = nullableInt(form.get('minBedrooms')); + if (!name) return fail(400, { message: 'Give the rule a name' }); + if ( + minPrice === undefined || + maxPrice === undefined || + minArea === undefined || + minBedrooms === undefined + ) + return fail(400, { message: 'Numbers must be whole and positive' }); + const values = { + name, + enabled: form.get('enabled') === 'on', + offer: form.get('offer') === 'buy' ? 'buy' : 'rent', + minPrice, + maxPrice, + minArea, + minBedrooms, + cities: String(form.get('cities') ?? '') + .split(',') + .map(normalizeCity) + .filter(Boolean) + .slice(0, 60), + propertyTypes: form + .getAll('propertyTypes') + .map(String) + .filter((type) => (propertyTypes as readonly string[]).includes(type)) + }; + if (id > 0) await db.update(houseAlertRules).set(values).where(eq(houseAlertRules.id, id)); + else await db.insert(houseAlertRules).values(values); + return { ok: true }; + }, + deleteRule: async (event) => { + requireAdmin(event); + const id = Number((await event.request.formData()).get('id')); + if (!Number.isInteger(id) || id <= 0) return fail(400, { message: 'Invalid rule' }); + await db.delete(houseAlertRules).where(eq(houseAlertRules.id, id)); + return { ok: true }; + }, + testAlert: async (event) => { + requireAdmin(event); + const sent = await sendAlert({ + title: '🏠 House tracker test', + body: 'If you can read this, alerts work.', + url: `${event.url.origin}/projects/house-tracker` + }); + return sent + ? { message: 'Test alert sent' } + : fail(400, { + message: 'No alert channel accepted it. Set HOUSE_DISCORD_WEBHOOK_URL or HOUSE_NTFY_URL.' + }); + } +}; diff --git a/src/routes/projects/house-tracker/+page.svelte b/src/routes/projects/house-tracker/+page.svelte new file mode 100644 index 0000000..5cc7990 --- /dev/null +++ b/src/routes/projects/house-tracker/+page.svelte @@ -0,0 +1,412 @@ +<script lang="ts"> + import { enhance } from '$app/forms'; + import { onMount } from 'svelte'; + import ExternalLink from 'lucide-svelte/icons/external-link'; + import BellRing from 'lucide-svelte/icons/bell-ring'; + import { Button } from '$lib/components/ui/button/index.js'; + import { Input } from '$lib/components/ui/input/index.js'; + import { Badge } from '$lib/components/ui/badge/index.js'; + import { NativeSelect } from '$lib/components/ui/native-select/index.js'; + import * as Card from '$lib/components/ui/card/index.js'; + import ListingMap from '$lib/house-tracker/ListingMap.svelte'; + import { relativeTime } from '$lib/relative-time'; + import type { PageProps } from './$types'; + + let { data, form }: PageProps = $props(); + + let now = $state(Date.now()); + onMount(() => { + const timer = setInterval(() => (now = Date.now()), 30_000); + return () => clearInterval(timer); + }); + + let highlighted = $state<number | null>(null); + const euro = new Intl.NumberFormat('nl-NL', { + style: 'currency', + currency: 'EUR', + maximumFractionDigits: 0 + }); + const clock = new Intl.DateTimeFormat('nl-NL', { + timeZone: 'Europe/Amsterdam', + weekday: 'short', + day: 'numeric', + month: 'short', + hour: '2-digit', + minute: '2-digit' + }); + const suffix = $derived(data.filters.offer === 'rent' ? '/mo' : ''); + + function price(value: number | null) { + return value == null ? 'on request' : `${euro.format(value)}${suffix}`; + } + function minutes(from: Date | string | null, to: Date | string | null) { + if (!from || !to) return null; + return Math.round((new Date(to).getTime() - new Date(from).getTime()) / 60_000); + } + function delayLabel(value: number | null) { + if (value == null || value < 0) return null; + if (value < 60) return `+${value} min`; + if (value < 48 * 60) return `+${Math.round(value / 60)} h`; + return `+${Math.round(value / 1440)} d`; + } + const workerAge = $derived( + data.health.last_ok ? now - new Date(data.health.last_ok).getTime() : Infinity + ); + const workerState = $derived( + data.health.last_challenge && + (!data.health.last_ok || new Date(data.health.last_challenge) > new Date(data.health.last_ok)) + ? 'challenged' + : workerAge < 75 * 60_000 + ? 'healthy' + : 'quiet' + ); + const mappable = $derived(data.listings.filter((row) => row.lat != null).length); +</script> + +<svelte:head> + <title>house tracker + + + +
+
+
+

house tracker

+

+ {data.health.total} listings collected · {data.health.alerts_week} + {data.health.alerts_week === 1 ? 'alert' : 'alerts'} this week + {#if data.health.median_detect_minutes != null} + · the fast lane spots new listings {delayLabel( + Math.round(data.health.median_detect_minutes) + )?.slice(1)} after they go up (median) + {/if} +

+
+
+ + {#if workerState === 'challenged'} + worker was challenged {relativeTime(data.health.last_challenge!, now)}, backing off + {:else if data.health.last_ok} + worker last synced {relativeTime(data.health.last_ok, now)} + {:else} + worker has not synced yet + {/if} +
+
+ +
+ + + + + + + {#each data.cities as city (city.slug)} + + {/each} + + + + + + + + + + + + + + {#each data.statuses as status (status)}{/each} + + + +
+ + (highlighted = id)} + /> +

+ {mappable} of {data.listings.length} shown listings have coordinates (they arrive with the detail + page fetch). +

+ +
    + {#each data.listings as listing (listing.id)} + {@const delay = minutes(listing.offeredAt, listing.firstSeenAt)} +
  • + {#if listing.imageUrl} + + {/if} +
    +
    + + + + {listing.title ?? 'Listing'} + + + {price(listing.priceEur)} +
    +
    + {[ + listing.postalCode, + [listing.city, listing.neighbourhood].filter(Boolean).join(', '), + listing.areaM2 != null && `${listing.areaM2} m²`, + listing.priceEur && + listing.areaM2 && + `${euro.format(listing.priceEur / listing.areaM2)}/m²`, + listing.rooms != null && `${listing.rooms} rooms`, + listing.energyLabel && `label ${listing.energyLabel}`, + listing.interior, + listing.availableFrom && `available ${listing.availableFrom.toLowerCase()}` + ] + .filter(Boolean) + .join(' · ')} +
    +
    + {#if listing.offeredAt} + posted {clock.format(new Date(listing.offeredAt))} + {/if} + + seen {relativeTime(listing.firstSeenAt, now)} + {#if listing.firstSeenLane !== 'seed' && delayLabel(delay)}({delayLabel(delay)}, {listing.firstSeenLane} + lane){/if} + + {#if listing.alertedAt}alerted{/if} + {#if listing.removedAt}gone {relativeTime(listing.removedAt, now)}{/if} + {#if listing.agent}{listing.agent}{/if} + {listing.source} +
    +
    +
    + async ({ update }) => + update({ reset: false, invalidateAll: false })} + class="shrink-0" + > + + (event.currentTarget as HTMLSelectElement).form?.requestSubmit()} + > + + {#each data.statuses as status (status)}{/each} + + +
    +
    • + {:else} +
  • + No listings yet. Start the home worker (see docs/house-tracker.md). +
    • + {/each} +
+ {#if data.listings.length === 400} +

+ Showing the first 400. Narrow the filters to see more. +

+ {/if} + +

market ({data.filters.offer})

+
+ + + + + + + + + + + + + {#each data.stats as row (row.city)} + + + + + + + + + {:else} + + {/each} + +
cityactivenew (7d)median pricemedian €/m²median days listed
{row.city}{row.active}{row.new_week}{row.median_price != null ? price(Math.round(row.median_price)) : '—'}{row.median_ppm2 != null ? euro.format(row.median_ppm2) : '—'}{row.median_days_listed != null ? Math.round(row.median_days_listed) : '—'}
no data yet
+
+

+ "Days listed" needs a few full sweeps before listings can be seen disappearing. +

+ +
+

alerts

+
+ +
+
+ {#if form?.message}

{form.message}

{/if} +
+ {#each [...data.rules, null] as rule (rule?.id ?? 'new')} + + +
+ +
+ + +
+
+ + + + + + + + +
+ +
+ {#each data.propertyTypes as type (type)} + + {/each} +
+
+ + {#if rule} + + {/if} +
+
+ + + {/each} +
+

+ Unchecked property types means all types. Listings without a size pass size limits, so you never + miss one because a search card left it out. +

+ +

worker log

+
+ + + {#each data.runs as run (run.id)} + + + + + + + + {:else} + + {/each} + +
{relativeTime(run.createdAt, now)}{run.lane} + {run.status}{run.httpStatus ? ` ${run.httpStatus}` : ''} + {run.found ?? ''} + {run.message ?? run.url.replace(/^https:\/\/www\./, '')} +
nothing yet
+
+
diff --git a/src/routes/projects/house-tracker/api/worker/details/+server.ts b/src/routes/projects/house-tracker/api/worker/details/+server.ts new file mode 100644 index 0000000..0938fc7 --- /dev/null +++ b/src/routes/projects/house-tracker/api/worker/details/+server.ts @@ -0,0 +1,11 @@ +import { json } from '@sveltejs/kit'; +import { detailsRequestSchema } from '$lib/house-tracker/types'; +import { requireWorker, saveDetails } from '$lib/server/house-tracker'; +import type { RequestHandler } from './$types'; + +export const POST: RequestHandler = async ({ request }) => { + requireWorker(request); + const parsed = detailsRequestSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return json({ error: parsed.error.issues.slice(0, 5) }, { status: 400 }); + return json(await saveDetails(parsed.data)); +}; diff --git a/src/routes/projects/house-tracker/api/worker/report/+server.ts b/src/routes/projects/house-tracker/api/worker/report/+server.ts new file mode 100644 index 0000000..5512675 --- /dev/null +++ b/src/routes/projects/house-tracker/api/worker/report/+server.ts @@ -0,0 +1,12 @@ +import { json } from '@sveltejs/kit'; +import { reportRequestSchema } from '$lib/house-tracker/types'; +import { recordReport, requireWorker } from '$lib/server/house-tracker'; +import type { RequestHandler } from './$types'; + +export const POST: RequestHandler = async ({ request }) => { + requireWorker(request); + const parsed = reportRequestSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return json({ error: parsed.error.issues.slice(0, 5) }, { status: 400 }); + await recordReport(parsed.data); + return json({ ok: true }); +}; diff --git a/src/routes/projects/house-tracker/api/worker/sync/+server.ts b/src/routes/projects/house-tracker/api/worker/sync/+server.ts new file mode 100644 index 0000000..55e7264 --- /dev/null +++ b/src/routes/projects/house-tracker/api/worker/sync/+server.ts @@ -0,0 +1,11 @@ +import { json } from '@sveltejs/kit'; +import { syncRequestSchema } from '$lib/house-tracker/types'; +import { requireWorker, syncCards } from '$lib/server/house-tracker'; +import type { RequestHandler } from './$types'; + +export const POST: RequestHandler = async ({ request }) => { + requireWorker(request); + const parsed = syncRequestSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) return json({ error: parsed.error.issues.slice(0, 5) }, { status: 400 }); + return json(await syncCards(parsed.data)); +};