feat: house tracker with home worker, alerts and listing map

Collects Pararius rent/buy listings into Postgres via a worker that runs on a
residential connection, sends Discord/ntfy alerts for new listings matching
rules, and shows an admin-only map, market stats and worker log.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013HfF5tnChi8tkCx9svJP6D
This commit is contained in:
2026-10-11 22:04:15 +02:00
parent 0ee9ed0abb
commit 0846cf7d77
30 changed files with 4269 additions and 1 deletions

View File

@@ -12,6 +12,7 @@
import ChartNoAxesCombined from 'lucide-svelte/icons/chart-no-axes-combined';
import FolderKanban from 'lucide-svelte/icons/folder-kanban';
import Gamepad2 from 'lucide-svelte/icons/gamepad-2';
import House from 'lucide-svelte/icons/house';
import Globe2 from 'lucide-svelte/icons/globe-2';
import Languages from 'lucide-svelte/icons/languages';
import Music2 from 'lucide-svelte/icons/music-2';
@@ -30,7 +31,8 @@
'world opinions': Globe2,
'language learner': Languages,
'money tracker': Wallet,
'cs prices': CandlestickChart
'cs prices': CandlestickChart,
'house tracker': House
};
const Icon = $derived(icons[name as keyof typeof icons] ?? FolderKanban);

View File

@@ -0,0 +1,185 @@
<script lang="ts">
import { onMount } from 'svelte';
import type { Map as MaptilerMap, GeoJSONSource } from '@maptiler/sdk';
export type MapListing = {
id: number;
title: string | null;
url: string;
priceEur: number | null;
areaM2: number | null;
lat: number | null;
lng: number | null;
firstSeenAt: Date | string;
status: string | null;
removedAt: Date | string | null;
};
let {
listings,
offer,
onselect
}: { listings: MapListing[]; offer: string; onselect?: (id: number) => void } = $props();
let container: HTMLDivElement;
let map: MaptilerMap | null = null;
let ready = $state(false);
let mapError = $state('');
const euro = new Intl.NumberFormat('nl-NL', {
style: 'currency',
currency: 'EUR',
maximumFractionDigits: 0
});
function collection(rows: MapListing[]) {
const now = Date.now();
return {
type: 'FeatureCollection' as const,
features: rows
.filter((row) => row.lat != null && row.lng != null)
.map((row) => ({
type: 'Feature' as const,
geometry: { type: 'Point' as const, coordinates: [row.lng!, row.lat!] },
properties: {
id: row.id,
title: row.title ?? 'Listing',
url: row.url,
label:
row.priceEur != null
? `${euro.format(row.priceEur)}${offer === 'rent' ? '/mo' : ''}`
: 'price on request',
area: row.areaM2 ?? 0,
kind: row.removedAt
? 'removed'
: row.status
? 'tracked'
: now - new Date(row.firstSeenAt).getTime() < 48 * 3_600_000
? 'fresh'
: 'active'
}
}))
};
}
$effect(() => {
const data = collection(listings);
if (!ready || !map) return;
(map.getSource('listings') as GeoJSONSource | undefined)?.setData(data);
});
onMount(() => {
let destroyed = false;
(async () => {
try {
const { Map, Popup, config, LngLatBounds } = await import('@maptiler/sdk');
const { env } = await import('$env/dynamic/public');
if (destroyed) return;
if (!env.PUBLIC_MAPTILER_API_KEY) {
mapError = 'Map needs PUBLIC_MAPTILER_API_KEY.';
return;
}
config.apiKey = env.PUBLIC_MAPTILER_API_KEY;
map = new Map({
container,
style:
'https://api.maptiler.com/maps/streets-v2/style.json?key=' +
env.PUBLIC_MAPTILER_API_KEY,
center: [4.7104, 52.0115],
zoom: 11,
cooperativeGestures: true
});
map.on('load', () => {
if (!map) return;
const data = collection(listings);
map.addSource('listings', { type: 'geojson', data });
map.addLayer({
id: 'listings',
type: 'circle',
source: 'listings',
paint: {
'circle-radius': ['interpolate', ['linear'], ['zoom'], 8, 3, 14, 8],
'circle-color': [
'match',
['get', 'kind'],
'fresh',
'#2f9e44',
'tracked',
'#1c7ed6',
'removed',
'#adb5bd',
'#f08c00'
],
'circle-stroke-color': '#ffffff',
'circle-stroke-width': 1.5
}
});
map.on('click', 'listings', (event) => {
const feature = event.features?.[0];
if (!feature || !map) return;
const props = feature.properties as {
id: number;
title: string;
url: string;
label: string;
area: number;
};
const body = document.createElement('div');
body.className = 'text-sm text-neutral-900';
const title = document.createElement('a');
title.href = props.url;
title.target = '_blank';
title.rel = 'noreferrer';
title.className = 'font-medium underline';
title.textContent = props.title;
const facts = document.createElement('div');
facts.textContent = `${props.label}${props.area ? ` · ${props.area} m²` : ''}`;
body.append(title, facts);
new Popup({ offset: 8 })
.setLngLat(
(feature.geometry as unknown as { coordinates: [number, number] }).coordinates
)
.setDOMContent(body)
.addTo(map);
onselect?.(props.id);
});
map.on('mouseenter', 'listings', () => map && (map.getCanvas().style.cursor = 'pointer'));
map.on('mouseleave', 'listings', () => map && (map.getCanvas().style.cursor = ''));
if (data.features.length > 1) {
const bounds = new LngLatBounds();
for (const feature of data.features) bounds.extend(feature.geometry.coordinates);
map.fitBounds(bounds, { padding: 40, maxZoom: 14, duration: 0 });
}
ready = true;
});
} catch (error) {
mapError = `Map failed to load: ${(error as Error).message}`;
}
})();
return () => {
destroyed = true;
map?.remove();
map = null;
};
});
</script>
<svelte:head>
<link rel="stylesheet" href="https://cdn.maptiler.com/maptiler-sdk-js/latest/maptiler-sdk.css" />
</svelte:head>
<div class="relative h-[420px] w-full overflow-hidden rounded-lg border border-border">
<div bind:this={container} class="h-full w-full"></div>
{#if mapError}
<div
class="absolute inset-0 flex items-center justify-center bg-muted p-4 text-sm text-muted-foreground"
>
{mapError}
</div>
{/if}
<div class="absolute bottom-2 left-2 flex gap-3 rounded-md bg-background/90 px-2 py-1 text-xs">
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#2f9e44]"></span>new (48h)</span>
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#f08c00]"></span>active</span>
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#1c7ed6]"></span>tracked</span>
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#adb5bd]"></span>gone</span>
</div>
</div>

View File

@@ -0,0 +1,91 @@
import { describe, expect, it } from 'vitest';
import { alertMessage, matchesRule, type AlertRule } from './alerts';
import { backoffMs, isBusy, nextDelayMs } from './schedule';
const rule: AlertRule = {
id: 1,
name: 'Gouda area rentals',
enabled: true,
offer: 'rent',
minPrice: 1300,
maxPrice: 1700,
minArea: null,
minBedrooms: null,
cities: ['gouda', 'nieuwerkerk aan den ijssel'],
propertyTypes: []
};
const listing = {
offer: 'rent',
propertyType: 'apartment',
city: 'Gouda',
priceEur: 1558,
areaM2: 65,
rooms: 2
};
describe('matchesRule', () => {
it('matches inside the price band and city list', () => {
expect(matchesRule(listing, rule)).toBe(true);
expect(matchesRule({ ...listing, city: 'Nieuwerkerk aan den IJssel' }, rule)).toBe(true);
expect(
matchesRule({ ...listing, city: null, citySlug: 'nieuwerkerk-aan-den-ijssel' }, rule)
).toBe(true);
});
it('rejects outside price, city, offer or when disabled', () => {
expect(matchesRule({ ...listing, priceEur: 1800 }, rule)).toBe(false);
expect(matchesRule({ ...listing, priceEur: null }, rule)).toBe(false);
expect(matchesRule({ ...listing, city: 'Rotterdam' }, rule)).toBe(false);
expect(matchesRule({ ...listing, offer: 'buy' }, rule)).toBe(false);
expect(matchesRule(listing, { ...rule, enabled: false })).toBe(false);
});
it('lets unknown size through but rejects known small ones', () => {
const roomy = { ...rule, minArea: 70, minBedrooms: 2 };
expect(matchesRule({ ...listing, areaM2: null, rooms: null }, roomy)).toBe(true);
expect(matchesRule(listing, roomy)).toBe(false);
});
it('formats a message', () => {
const message = alertMessage(
{ ...listing, title: 'Appartement Raam', url: 'https://example.com', source: 'pararius' },
rule
);
expect(message.title).toBe('🏠 Appartement Raam');
expect(message.body).toContain('65 m²');
expect(message.body).toContain('/mo');
});
});
describe('schedule', () => {
// 2026-10-07 is a Wednesday; Amsterdam is UTC+2 in October.
const wednesdayNoon = new Date('2026-10-07T10:00:00Z');
const wednesdayNight = new Date('2026-10-07T21:00:00Z');
const saturday = new Date('2026-10-10T10:00:00Z');
it('knows office hours in Amsterdam', () => {
expect(isBusy(wednesdayNoon)).toBe(true);
expect(isBusy(wednesdayNight)).toBe(false);
expect(isBusy(saturday)).toBe(false);
expect(isBusy(new Date('2026-10-07T04:59:00Z'))).toBe(false);
expect(isBusy(new Date('2026-10-07T05:00:00Z'))).toBe(true);
});
it('polls every few minutes when busy and slowly otherwise', () => {
expect(nextDelayMs(wednesdayNoon, undefined, () => 0)).toBe(3 * 60_000);
expect(nextDelayMs(wednesdayNoon, undefined, () => 1)).toBe(5 * 60_000);
expect(nextDelayMs(saturday, undefined, () => 0)).toBe(30 * 60_000);
});
it('wakes up for the start of office hours', () => {
const justBefore = new Date('2026-10-07T04:50:00Z'); // 06:50 local
expect(nextDelayMs(justBefore, undefined, () => 1)).toBeLessThanOrEqual(11 * 60_000);
});
it('backs off exponentially with a cap', () => {
expect(backoffMs(1, () => 0.5)).toBe(15 * 60_000);
expect(backoffMs(3, () => 0.5)).toBe(60 * 60_000);
expect(backoffMs(20, () => 0.5)).toBe(6 * 60 * 60_000);
});
});

View File

@@ -0,0 +1,92 @@
// Pure alert matching and message formatting, shared by server and tests.
export type AlertRule = {
id: number;
name: string;
enabled: boolean;
offer: string;
minPrice: number | null;
maxPrice: number | null;
minArea: number | null;
minBedrooms: number | null;
cities: string[];
propertyTypes: string[];
};
export type AlertListing = {
offer: string;
propertyType: string;
city?: string | null;
citySlug?: string | null;
priceEur?: number | null;
areaM2?: number | null;
bedrooms?: number | null;
rooms?: number | null;
};
export function normalizeCity(value: string | null | undefined): string {
return (value ?? '')
.normalize('NFD')
.replace(/[̀-ͯ]/g, '')
.toLowerCase()
.replace(/-/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
export function matchesRule(listing: AlertListing, rule: AlertRule): boolean {
if (!rule.enabled || listing.offer !== rule.offer) return false;
if (rule.minPrice != null || rule.maxPrice != null) {
// "Price on request" can't be checked, so it never matches a price rule.
if (listing.priceEur == null) return false;
if (rule.minPrice != null && listing.priceEur < rule.minPrice) return false;
if (rule.maxPrice != null && listing.priceEur > rule.maxPrice) return false;
}
// Unknown area or bedrooms pass: search cards often lack them, and missing an
// alert is worse than one extra notification.
if (rule.minArea != null && listing.areaM2 != null && listing.areaM2 < rule.minArea) return false;
if (rule.minBedrooms != null) {
const bedrooms = listing.bedrooms ?? (listing.rooms != null ? listing.rooms - 1 : null);
if (bedrooms != null && bedrooms < rule.minBedrooms) return false;
}
if (rule.propertyTypes.length && !rule.propertyTypes.includes(listing.propertyType)) return false;
if (rule.cities.length) {
const wanted = new Set(rule.cities.map(normalizeCity));
const city = normalizeCity(listing.city);
const slug = normalizeCity(listing.citySlug);
if (!wanted.has(city) && !wanted.has(slug)) return false;
}
return true;
}
const euros = new Intl.NumberFormat('nl-NL', {
style: 'currency',
currency: 'EUR',
maximumFractionDigits: 0
});
export type AlertMessage = { title: string; body: string; url: string };
export function alertMessage(
listing: AlertListing & {
title?: string | null;
url: string;
neighbourhood?: string | null;
source: string;
},
rule: Pick<AlertRule, 'name'>
): AlertMessage {
const price = listing.priceEur != null ? euros.format(listing.priceEur) : 'price on request';
const suffix = listing.offer === 'rent' && listing.priceEur != null ? '/mo' : '';
const facts = [
`${price}${suffix}`,
listing.areaM2 != null ? `${listing.areaM2} m²` : null,
listing.rooms != null ? `${listing.rooms} rooms` : null,
[listing.city, listing.neighbourhood].filter(Boolean).join(', ') || null
].filter(Boolean);
return {
title: `🏠 ${listing.title ?? 'New listing'}`,
body: `${facts.join(' · ')}\n${rule.name} · ${listing.source}`,
url: listing.url
};
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,107 @@
<!doctype html>
<!-- Hand-written in the shape of Pararius search results. Replace with a real saved page. -->
<html>
<head>
<title>Huurwoningen Gouda</title>
</head>
<body>
<div class="search-list-header__count">12</div>
<ul class="search-list">
<li class="search-list__item search-list__item--listing">
<section
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
>
<div class="listing-search-item__depiction">
<a
class="listing-search-item__link listing-search-item__link--depiction"
href="/appartement-te-huur/gouda/d5d8a72c/raam"
>
<img
src="https://casco-media-prod.global.ssl.fastly.net/listings/example-raam.jpg?width=300"
alt=""
/>
</a>
</div>
<div class="listing-search-item__label">
<span class="listing-label listing-label--new">Nieuw</span>
</div>
<h2 class="listing-search-item__title">
<a
class="listing-search-item__link listing-search-item__link--title"
href="/appartement-te-huur/gouda/d5d8a72c/raam"
>
Appartement Raam
</a>
</h2>
<div class="listing-search-item__sub-title'">2801 VN Gouda (Binnenstad)</div>
<div class="listing-search-item__price">€&nbsp;1.475 per maand</div>
<div class="listing-search-item__features">
<ul class="illustrated-features illustrated-features--compact">
<li class="illustrated-features__item illustrated-features__item--surface-area">
58 m²
</li>
<li class="illustrated-features__item illustrated-features__item--number-of-rooms">
3 kamers
</li>
<li class="illustrated-features__item illustrated-features__item--interior">
Gestoffeerd
</li>
</ul>
</div>
<div class="listing-search-item__info">
<a class="listing-search-item__link" href="/makelaars/gouda/example"
>Voorbeeld Makelaardij</a
>
</div>
</section>
</li>
<li class="search-list__item search-list__item--listing">
<section
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
>
<h2 class="listing-search-item__title">
<a
class="listing-search-item__link listing-search-item__link--title"
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
>
Huis Kongsbergstraat 45
</a>
</h2>
<div class="listing-search-item__sub-title'">2804 XV Gouda (De Mammoet)</div>
<div class="listing-search-item__price">€1,600 per month</div>
<ul class="illustrated-features">
<li class="illustrated-features__item illustrated-features__item--surface-area">
113 m²
</li>
<li class="illustrated-features__item illustrated-features__item--number-of-rooms">
5 rooms
</li>
</ul>
</section>
</li>
<li class="search-list__item search-list__item--listing">
<section
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
>
<h2 class="listing-search-item__title">
<a
class="listing-search-item__link listing-search-item__link--title"
href="/huis-te-huur/gouda/8fc6ae25/nansenstraat"
>
Huis Nansenstraat
</a>
</h2>
<div class="listing-search-item__sub-title'">2806 HK Gouda (Vreewijk)</div>
<div class="listing-search-item__price">Prijs op aanvraag</div>
</section>
</li>
</ul>
<ul class="pagination__list">
<li class="pagination__item pagination__item--next">
<a class="pagination__link pagination__link--next" href="/huurwoningen/gouda/page-2"
>Volgende</a
>
</li>
</ul>
</body>
</html>

View File

@@ -0,0 +1,158 @@
import { readFileSync } from 'node:fs';
import { describe, expect, it } from 'vitest';
import {
cityFromSlug,
isChallenge,
listingRef,
parseDetailPage,
parseEuros,
parseLocation,
parseSearchPage,
searchPageUrl
} from './pararius';
import { listingDetailSchema, listingCardSchema } from './types';
const fixture = (name: string) =>
readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8');
describe('pararius helpers', () => {
it('reads listing urls', () => {
expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({
sourceId: '541493b8',
url: 'https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat',
offer: 'rent',
propertyType: 'apartment',
citySlug: 'gouda'
});
expect(listingRef('https://www.pararius.nl/huis-te-koop/den-haag/0a1b2c3d/laan')?.offer).toBe(
'buy'
);
expect(listingRef('/huurwoningen/gouda')).toBeNull();
});
it('parses Dutch and English prices', () => {
expect(parseEuros('€ 1.558 per maand')).toBe(1558);
expect(parseEuros('€1,600 per month')).toBe(1600);
expect(parseEuros('€ 425.000 k.k.')).toBe(425000);
expect(parseEuros('€ 1.250,50')).toBe(1250);
expect(parseEuros('Prijs op aanvraag')).toBeNull();
});
it('splits a location line', () => {
expect(parseLocation('2801 BC Gouda (Centrum)')).toEqual({
postalCode: '2801 BC',
city: 'Gouda',
neighbourhood: 'Centrum'
});
expect(parseLocation('Alphen aan den Rijn')).toEqual({
postalCode: null,
city: 'Alphen aan den Rijn',
neighbourhood: null
});
});
it('names cities from slugs', () => {
expect(cityFromSlug('gouda')).toBe('Gouda');
expect(cityFromSlug('den-haag')).toBe('Den Haag');
expect(cityFromSlug('alphen-aan-den-rijn')).toBe('Alphen aan den Rijn');
});
it('builds page urls', () => {
const base = 'https://www.pararius.nl/huurwoningen/gouda/1300-1700';
expect(searchPageUrl(base, 1)).toBe(base);
expect(searchPageUrl(base, 3)).toBe(`${base}/page-3`);
expect(searchPageUrl(`${base}/page-3`, 2)).toBe(`${base}/page-2`);
});
it('detects Cloudflare challenges', () => {
expect(isChallenge('<html><head><title>Just a moment...</title>')).toBe(true);
expect(isChallenge('<html>ok</html>', 403)).toBe(true);
expect(isChallenge(fixture('pararius-detail-house.html'))).toBe(false);
});
});
describe('parseDetailPage', () => {
it('reads a house listing', () => {
const detail = parseDetailPage(fixture('pararius-detail-house.html'))!;
expect(listingDetailSchema.parse(detail)).toBeTruthy();
expect(detail).toMatchObject({
sourceId: '8f976237',
offer: 'rent',
propertyType: 'house',
externalId: '8f976237-2ce0-5613-adbe-883495268361',
street: 'Kongsbergstraat 45',
postalCode: '2804 XV',
city: 'Gouda',
neighbourhood: 'De Mammoet',
priceEur: 1600,
areaM2: 113,
rooms: 5,
bedrooms: 4,
constructionYear: 1981,
energyLabel: 'B',
depositEur: 1600,
agent: 'LIV Residential',
offeredAt: '2026-09-30T09:47:00.000Z'
});
expect(detail.lat).toBeCloseTo(52.0284, 3);
expect(detail.lng).toBeCloseTo(4.7174, 3);
});
it('reads interior and availability', () => {
const detail = parseDetailPage(fixture('pararius-detail-apartment.html'))!;
expect(detail).toMatchObject({
propertyType: 'apartment',
priceEur: 1558,
interior: 'gestoffeerd',
availableFrom: 'Per 15-10-2026',
contractType: 'Onbepaalde tijd',
depositEur: 3316,
offeredAt: '2026-10-02T05:35:37.000Z'
});
});
it('keeps price-on-request listings', () => {
const detail = parseDetailPage(fixture('pararius-detail-on-request.html'))!;
expect(detail.priceEur).toBeNull();
expect(detail.interior).toBe('gemeubileerd');
expect(listingDetailSchema.parse(detail)).toBeTruthy();
});
it('rejects pages that are not listings', () => {
expect(parseDetailPage('<html><body>nothing</body></html>')).toBeNull();
});
});
describe('parseSearchPage', () => {
it('reads listing cards and pagination', () => {
const page = parseSearchPage(
fixture('pararius-search-synthetic.html'),
'https://www.pararius.nl/huurwoningen/gouda'
);
expect(page.cards).toHaveLength(3);
for (const card of page.cards) expect(listingCardSchema.parse(card)).toBeTruthy();
expect(page.cards[0]).toMatchObject({
sourceId: 'd5d8a72c',
title: 'Appartement Raam',
postalCode: '2801 VN',
city: 'Gouda',
neighbourhood: 'Binnenstad',
priceEur: 1475,
areaM2: 58,
rooms: 3,
interior: 'Gestoffeerd',
agent: 'Voorbeeld Makelaardij'
});
expect(page.cards[1].priceEur).toBe(1600);
expect(page.cards[2].priceEur).toBeNull();
expect(page.nextPageUrl).toBe('https://www.pararius.nl/huurwoningen/gouda/page-2');
expect(page.totalResults).toBe(12);
});
it('falls back to bare links when card markup changes', () => {
const html = `<div><a href="/huis-te-huur/gouda/0a1b2c3d/straat">Huis Straat 1</a> € 1.400 per maand 80 m²</div>`;
const page = parseSearchPage(html, 'https://www.pararius.nl/huurwoningen/gouda');
expect(page.cards).toHaveLength(1);
expect(page.cards[0]).toMatchObject({ sourceId: '0a1b2c3d', priceEur: 1400, areaM2: 80 });
});
});

View File

@@ -0,0 +1,435 @@
// Pararius HTML parsing. Pure functions, no network: the worker fetches, these parse.
//
// Pararius has no public API, so everything here reads markup that can change
// without notice. Each parser prefers structured data (ld+json, the analytics
// blob) and falls back to looser text matching, so a redesign degrades fields
// instead of dropping listings.
import type { ListingCard, ListingDetail, Offer, PropertyType } from './types';
export const PARARIUS_ORIGIN = 'https://www.pararius.nl';
const listingPath =
/\/(appartement|huis|kamer|studio|woning|[a-z]+)-te-(huur|koop)\/([a-z0-9-]+)\/([0-9a-f]{8})\/([a-z0-9-]+)/;
const typeWords: Record<string, PropertyType> = {
appartement: 'apartment',
apartment: 'apartment',
huis: 'house',
house: 'house',
kamer: 'room',
room: 'room',
studio: 'studio'
};
const entities: Record<string, string> = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
euro: '€',
sup2: '²',
sup3: '³'
};
export function decodeEntities(value: string): string {
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => {
if (code[0] === '#') {
const n =
code[1] === 'x' || code[1] === 'X' ? parseInt(code.slice(2), 16) : Number(code.slice(1));
return Number.isFinite(n) ? String.fromCodePoint(n) : match;
}
return entities[code.toLowerCase()] ?? match;
});
}
export function textOf(html: string): string {
return decodeEntities(html.replace(/<[^>]*>/g, ' '))
.replace(/\s+/g, ' ')
.trim();
}
/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */
export function isChallenge(html: string, status = 200): boolean {
if (status === 403 || status === 429 || status === 503) return true;
const head = html.slice(0, 20_000);
return (
/<title>\s*Just a moment/i.test(head) ||
head.includes('challenges.cloudflare.com/turnstile') ||
head.includes('cf-chl-') ||
/<title>\s*Attention Required/i.test(head)
);
}
export type ListingRef = {
sourceId: string;
url: string;
offer: Offer;
propertyType: PropertyType;
citySlug: string;
};
export function listingRef(href: string): ListingRef | null {
const match = listingPath.exec(href);
if (!match) return null;
const [, type, offer, citySlug, shortId, slug] = match;
return {
sourceId: shortId,
url: `${PARARIUS_ORIGIN}/${type}-te-${offer}/${citySlug}/${shortId}/${slug}`,
offer: offer === 'koop' ? 'buy' : 'rent',
propertyType: typeWords[type] ?? 'other',
citySlug
};
}
/** Whole euros from "€ 1.558 per maand", "€1,558 per month", "€ 425.000 k.k." */
export function parseEuros(value: string | null | undefined): number | null {
if (!value) return null;
const match = /€\s*([\d.,]+)/.exec(value) ?? /([\d][\d.,]*)/.exec(value);
if (!match) return null;
const digits = match[1].replace(/[.,]\d{2}$/, '').replace(/[.,]/g, '');
const n = Number(digits);
return digits && Number.isSafeInteger(n) ? n : null;
}
export function parseInteger(value: string | null | undefined): number | null {
const match = value ? /(\d[\d.]*)/.exec(value) : null;
if (!match) return null;
const n = Number(match[1].replace(/\./g, ''));
return Number.isSafeInteger(n) ? n : null;
}
/** "2801 BC Gouda (Centrum)" → postal code, city and neighbourhood. */
export function parseLocation(value: string | null | undefined) {
const out = {
postalCode: null as string | null,
city: null as string | null,
neighbourhood: null as string | null
};
if (!value) return out;
let rest = value.trim();
const hood = /\(([^)]+)\)\s*$/.exec(rest);
if (hood) {
out.neighbourhood = hood[1].trim();
rest = rest.slice(0, hood.index).trim();
}
const postal = /^(\d{4})\s?([A-Z]{2})\b/.exec(rest);
if (postal) {
out.postalCode = `${postal[1]} ${postal[2]}`;
rest = rest.slice(postal[0].length).trim();
}
if (rest) out.city = rest;
return out;
}
function classText(html: string, className: string): string | null {
const re = new RegExp(
`<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)</\\1>`,
'i'
);
const match = re.exec(html);
if (!match) return null;
const value = textOf(match[2]);
return value || null;
}
function firstImage(html: string): string | null {
const tag = /<img\b[^>]*>/i.exec(html)?.[0];
if (!tag) return null;
const src =
/\bsrc="(https:[^"]+)"/i.exec(tag)?.[1] ??
/\bsrcset="(https:[^"\s,]+)/i.exec(tag)?.[1] ??
/\bdata-src="(https:[^"]+)"/i.exec(tag)?.[1];
return src ? decodeEntities(src) : null;
}
export type SearchPage = {
cards: ListingCard[];
nextPageUrl: string | null;
totalResults: number | null;
};
/**
* Parse a search results page (`/huurwoningen/gouda/1300-1700`).
* Splits on the listing card wrappers when present; otherwise falls back to
* one chunk per unique listing link so a markup change still yields ids.
*/
export function parseSearchPage(html: string, pageUrl: string): SearchPage {
const chunks: string[] = [];
const marker =
/<(section|li|article|div)\b[^>]*class="[^"]*\blisting-search-item\b(?!__)[^"]*"/gi;
const starts = [...html.matchAll(marker)].map((m) => m.index!);
if (starts.length) {
for (let i = 0; i < starts.length; i++) {
chunks.push(html.slice(starts[i], starts[i + 1] ?? starts[i] + 20_000));
}
} else {
const links = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/gi;
const seen = new Set<string>();
for (const m of html.matchAll(links)) {
const ref = listingRef(m[1]);
if (!ref || seen.has(ref.sourceId)) continue;
seen.add(ref.sourceId);
chunks.push(html.slice(Math.max(0, m.index! - 3000), m.index! + 6000));
}
}
const cards: ListingCard[] = [];
const seen = new Set<string>();
for (const chunk of chunks) {
const href = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/i.exec(chunk)?.[1];
const ref = href ? listingRef(decodeEntities(href)) : null;
if (!ref || seen.has(ref.sourceId)) continue;
seen.add(ref.sourceId);
const titleLink = /<a\b[^>]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk);
const title =
(titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title');
const location = parseLocation(
classText(chunk, 'listing-search-item__sub-title') ??
classText(chunk, 'listing-search-item__location')
);
const priceText =
classText(chunk, 'listing-search-item__price') ?? /€\s*[\d.,]+[^<]{0,30}/.exec(chunk)?.[0];
const area =
classText(chunk, 'illustrated-features__item--surface-area') ??
/(\d+)\s*m(?:²|&sup2;|2)/i.exec(chunk)?.[0];
const rooms = classText(chunk, 'illustrated-features__item--number-of-rooms');
const agentBlock = classText(chunk, 'listing-search-item__info');
cards.push({
sourceId: ref.sourceId,
url: ref.url,
offer: ref.offer,
propertyType: ref.propertyType,
title: title?.slice(0, 200) ?? null,
postalCode: location.postalCode,
city: location.city ?? cityFromSlug(ref.citySlug),
neighbourhood: location.neighbourhood,
priceEur: parseEuros(priceText),
areaM2: parseInteger(area),
rooms: parseInteger(rooms),
interior: classText(chunk, 'illustrated-features__item--interior'),
agent: agentBlock?.slice(0, 160) ?? null,
imageUrl: firstImage(chunk)
});
}
const nextHref =
/<a\b[^>]*class="[^"]*pagination__link--next[^"]*"[^>]*href="([^"]+)"/i.exec(html)?.[1] ??
/<a\b[^>]*href="([^"]+)"[^>]*class="[^"]*pagination__link--next[^"]*"/i.exec(html)?.[1] ??
/<link\b[^>]*rel="next"[^>]*href="([^"]+)"/i.exec(html)?.[1];
const total =
classText(html, 'search-list-header__count') ??
/(\d[\d.]*)\s+(?:huurwoningen|koopwoningen|woningen|results|resultaten)/i.exec(
textOf(html.slice(0, 200_000))
)?.[1];
return {
cards,
nextPageUrl: nextHref ? new URL(decodeEntities(nextHref), pageUrl).toString() : null,
totalResults: parseInteger(total)
};
}
/** `/huurwoningen/gouda/1300-1700` → page N url (`.../page-2`). */
export function searchPageUrl(base: string, page: number): string {
const url = new URL(base);
url.pathname = url.pathname.replace(/\/page-\d+\/?$/, '').replace(/\/$/, '');
if (page > 1) url.pathname += `/page-${page}`;
return url.toString();
}
export function cityFromSlug(slug: string): string {
return slug
.split('-')
.map((part) =>
part === 'aan' || part === 'den' || part === 'de'
? part
: part[0]?.toUpperCase() + part.slice(1)
)
.join(' ')
.replace(/^\w/, (c) => c.toUpperCase());
}
function jsonBlocks(html: string): unknown[] {
const out: unknown[] = [];
for (const m of html.matchAll(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) {
try {
out.push(JSON.parse(m[1]));
} catch {
// Ignore malformed blocks; other sources may still have the data.
}
}
return out;
}
type LdListing = {
'@type'?: string | string[];
name?: string;
description?: string;
image?: string | string[];
address?: {
streetAddress?: string;
addressLocality?: string;
addressRegion?: string;
postalCode?: string;
};
numberOfRooms?: { value?: number }[] | { value?: number } | number;
floorSize?: { value?: number };
yearBuilt?: number;
numberOfBedrooms?: number;
numberOfBathroomsTotal?: number;
datePosted?: string;
offers?: { price?: number; url?: string };
};
function ldListing(html: string): LdListing | null {
for (const block of jsonBlocks(html)) {
const items = Array.isArray(block) ? block : [block];
for (const item of items) {
if (!item || typeof item !== 'object') continue;
const type = (item as LdListing)['@type'];
const types = Array.isArray(type) ? type : [type];
if (types.includes('RealEstateListing')) return item as LdListing;
}
}
return null;
}
/** The flat `odp.*` analytics object Pararius embeds on listing pages. */
export function analyticsBlob(html: string): Record<string, string> {
const candidates = [html, decodeEntities(html)];
for (const source of candidates) {
const match = /\{[^{}]*"odp\.listing_id"[^{}]*\}/.exec(source);
if (!match) continue;
try {
const parsed = JSON.parse(match[0]) as Record<string, unknown>;
const out: Record<string, string> = {};
for (const [key, value] of Object.entries(parsed)) {
if (key.startsWith('odp.') && (typeof value === 'string' || typeof value === 'number'))
out[key.slice(4)] = String(value);
}
return out;
} catch {
// Fall through to the next candidate.
}
}
return {};
}
/** The "Kenmerken" definition list: `Borg → € 1.600`, `Interieur → Gestoffeerd`. */
export function featureList(html: string): Record<string, string> {
const out: Record<string, string> = {};
const re =
/<dt\b[^>]*class="[^"]*listing-features__term[^"]*"[^>]*>([\s\S]*?)<\/dt>\s*<dd\b[^>]*>([\s\S]*?)<\/dd>/gi;
for (const m of html.matchAll(re)) {
const key = textOf(m[1]).slice(0, 80);
const value = textOf(m[2].replace(/<[^>]*class="[^"]*tooltip[^"]*"[\s\S]*?<\/[a-z]+>/gi, ''))
.replace(/\s*Meer info$/i, '')
.slice(0, 500);
if (key && value && !(key in out)) out[key] = value;
}
return out;
}
const interiorWords: Record<string, string> = {
shell: 'kaal',
upholstered: 'gestoffeerd',
furnished: 'gemeubileerd'
};
const firstNumber = (value: LdListing['numberOfRooms']): number | null => {
if (typeof value === 'number') return value;
const item = Array.isArray(value) ? value[0] : value;
return typeof item?.value === 'number' ? item.value : null;
};
/** Parse a listing page. Returns null when the page is not a listing. */
export function parseDetailPage(html: string, fallbackUrl?: string): ListingDetail | null {
const canonical =
/<link\b[^>]*rel="canonical"[^>]*href="([^"]+)"/i.exec(html)?.[1] ??
/<link\b[^>]*href="([^"]+)"[^>]*rel="canonical"/i.exec(html)?.[1];
const ref = listingRef(canonical ?? '') ?? listingRef(fallbackUrl ?? '');
if (!ref) return null;
const ld = ldListing(html);
const odp = analyticsBlob(html);
const features = featureList(html);
const summaryTitle = classText(html, 'listing-detail-summary__title');
const location = parseLocation(classText(html, 'listing-detail-summary__location'));
const lat = Number(
/data-latitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"latitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1]
);
const lng = Number(
/data-longitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"longitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1]
);
const offeredAt =
odp.offered_since && !Number.isNaN(Date.parse(odp.offered_since))
? new Date(odp.offered_since).toISOString()
: ld?.datePosted
? new Date(`${ld.datePosted}T00:00:00Z`).toISOString()
: null;
const price =
(odp.price_pcm ? Number(odp.price_pcm) : null) ??
(typeof ld?.offers?.price === 'number' ? ld.offers.price : null) ??
parseEuros(features['Huurprijs'] ?? features['Vraagprijs'] ?? features['Koopprijs']);
const image = Array.isArray(ld?.image) ? ld.image[0] : ld?.image;
const typeFromOdp = odp.property ? typeWords[odp.property] : undefined;
const interior =
(odp.interior ? (interiorWords[odp.interior] ?? odp.interior) : null) ??
features['Interieur']?.toLowerCase() ??
null;
const street = ld?.address?.streetAddress ?? null;
const title =
summaryTitle?.replace(/^Te (huur|koop):\s*/i, '') ?? (ld?.name ? `${ld.name}` : null);
return {
sourceId: ref.sourceId,
url: ref.url,
offer: odp.offer === 'buy' || ref.offer === 'buy' ? 'buy' : 'rent',
propertyType: typeFromOdp ?? ref.propertyType,
externalId: odp.listing_id ?? null,
title: title?.slice(0, 200) ?? null,
street,
postalCode: ld?.address?.postalCode
? ld.address.postalCode.replace(/^(\d{4})\s?([A-Z]{2})$/, '$1 $2')
: location.postalCode,
city: ld?.address?.addressLocality ?? location.city ?? cityFromSlug(ref.citySlug),
neighbourhood: ld?.address?.addressRegion ?? location.neighbourhood,
priceEur: Number.isFinite(price) ? price : null,
areaM2:
parseInteger(odp.floor_space) ??
(typeof ld?.floorSize?.value === 'number' ? ld.floorSize.value : null) ??
parseInteger(features['Woonoppervlakte']),
rooms:
parseInteger(odp.rooms) ??
firstNumber(ld?.numberOfRooms) ??
parseInteger(features['Aantal kamers']),
bedrooms:
parseInteger(odp.bedrooms) ??
ld?.numberOfBedrooms ??
parseInteger(features['Aantal slaapkamers']),
bathrooms: ld?.numberOfBathroomsTotal ?? parseInteger(features['Aantal badkamers']),
constructionYear:
parseInteger(odp.construction_year) ?? ld?.yearBuilt ?? parseInteger(features['Bouwjaar']),
energyLabel: odp.energy_rating ?? features['Energielabel'] ?? null,
interior,
depositEur: parseEuros(features['Borg']),
availableFrom: features['Beschikbaar'] ?? null,
contractType: features['Contractvorm'] ?? features['Huurperiode'] ?? null,
agent: classText(html, 'agent-card__title'),
imageUrl: image ?? null,
lat: Number.isFinite(lat) && lat !== 0 ? lat : null,
lng: Number.isFinite(lng) && lng !== 0 ? lng : null,
offeredAt,
description: ld?.description?.slice(0, 6000) ?? null,
features
};
}

View File

@@ -0,0 +1,70 @@
// When the home worker should poll next. Pure so it can be tested.
//
// Listings near Gouda were all published on weekdays between 07:30 and 17:30,
// so the fast lane polls often during office hours and backs off otherwise.
export type ScheduleOptions = {
/** Weekday office-hours interval, in minutes. */
busyMinutes: [number, number];
/** Evenings, nights and weekends. */
quietMinutes: [number, number];
busyStartHour: number;
busyEndHour: number;
timeZone: string;
};
export const fastLaneSchedule: ScheduleOptions = {
busyMinutes: [3, 5],
quietMinutes: [30, 60],
busyStartHour: 7,
busyEndHour: 18,
timeZone: 'Europe/Amsterdam'
};
export function localClock(date: Date, timeZone: string) {
const parts = new Intl.DateTimeFormat('en-GB', {
timeZone,
weekday: 'short',
hour: 'numeric',
minute: 'numeric',
hourCycle: 'h23'
}).formatToParts(date);
const get = (type: string) => parts.find((part) => part.type === type)?.value ?? '';
return {
weekday: get('weekday'),
hour: Number(get('hour')),
minute: Number(get('minute'))
};
}
export function isBusy(date: Date, options: ScheduleOptions = fastLaneSchedule): boolean {
const { weekday, hour } = localClock(date, options.timeZone);
if (weekday === 'Sat' || weekday === 'Sun') return false;
return hour >= options.busyStartHour && hour < options.busyEndHour;
}
/** Milliseconds until the next poll, with jitter so requests never look clockwork. */
export function nextDelayMs(
now: Date,
options: ScheduleOptions = fastLaneSchedule,
random: () => number = Math.random
): number {
const [min, max] = isBusy(now, options) ? options.busyMinutes : options.quietMinutes;
let delay = (min + (max - min) * random()) * 60_000;
// Don't sleep through the start of office hours.
if (!isBusy(now, options)) {
for (let step = 60_000; step < delay; step += 60_000) {
if (isBusy(new Date(now.getTime() + step), options)) {
delay = step + random() * 60_000;
break;
}
}
}
return Math.round(delay);
}
/** Exponential backoff after challenges or errors: 15 min, 30, 60 ... capped at 6 h. */
export function backoffMs(failures: number, random: () => number = Math.random): number {
const base = 15 * 60_000 * 2 ** Math.max(0, failures - 1);
return Math.round(Math.min(base, 6 * 60 * 60_000) * (0.85 + 0.3 * random()));
}

View File

@@ -0,0 +1,118 @@
// Shared between the home worker (scripts/house-worker.ts), the ingest API and
// the page. Keep this file free of `$lib` / `$env` imports: the worker runs it
// directly with bun, outside SvelteKit.
import { z } from 'zod';
export const offers = ['rent', 'buy'] as const;
export type Offer = (typeof offers)[number];
export const propertyTypes = ['apartment', 'house', 'room', 'studio', 'other'] as const;
export type PropertyType = (typeof propertyTypes)[number];
const text = (max: number) => z.string().trim().max(max);
const optionalText = (max: number) => text(max).nullish();
const count = z.number().int().min(0).max(10_000).nullish();
// What a search results page tells us about a listing. This alone is enough to
// send an alert, so the fast lane never waits for the detail page.
export const listingCardSchema = z.object({
sourceId: text(80).min(1),
url: z.url().max(500),
offer: z.enum(offers),
propertyType: z.enum(propertyTypes),
title: optionalText(200),
postalCode: optionalText(10),
city: optionalText(80),
neighbourhood: optionalText(120),
priceEur: z.number().int().min(0).max(100_000_000).nullish(),
areaM2: count,
rooms: count,
interior: optionalText(40),
agent: optionalText(160),
imageUrl: z.url().max(1000).nullish()
});
export type ListingCard = z.infer<typeof listingCardSchema>;
// Everything else we learn from the listing page itself.
export const listingDetailSchema = listingCardSchema.extend({
externalId: optionalText(80),
street: optionalText(200),
bedrooms: count,
bathrooms: count,
constructionYear: z.number().int().min(1000).max(2100).nullish(),
energyLabel: optionalText(10),
depositEur: z.number().int().min(0).max(10_000_000).nullish(),
availableFrom: optionalText(80),
contractType: optionalText(80),
lat: z.number().min(-90).max(90).nullish(),
lng: z.number().min(-180).max(180).nullish(),
offeredAt: z.iso.datetime({ offset: true }).nullish(),
description: optionalText(6000),
features: z.record(z.string().max(80), z.string().max(500)).optional()
});
export type ListingDetail = z.infer<typeof listingDetailSchema>;
export const listingStatuses = [
'interested',
'contacted',
'viewing',
'applied',
'rejected'
] as const;
export type ListingStatus = (typeof listingStatuses)[number];
export const lanes = ['fast', 'slow'] as const;
export type Lane = (typeof lanes)[number];
// One poll of one search page.
export const syncRequestSchema = z.object({
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
lane: z.enum(lanes),
searchUrl: z.url().max(500),
startedAt: z.iso.datetime({ offset: true }),
cards: z.array(listingCardSchema).max(200),
// Set on the last page of a full, unfiltered sweep of one city. Listings in
// that scope that were not seen during the sweep are then marked removed.
completedSweep: z
.object({
offer: z.enum(offers),
city: text(80).min(1),
sweepStartedAt: z.iso.datetime({ offset: true })
})
.nullish()
});
export type SyncRequest = z.infer<typeof syncRequestSchema>;
export const detailsRequestSchema = z.object({
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
details: z.array(listingDetailSchema).max(50)
});
export type DetailsRequest = z.infer<typeof detailsRequestSchema>;
export const runStatuses = [
'ok',
'challenged',
'http-error',
'parse-error',
'network-error'
] as const;
export type RunStatus = (typeof runStatuses)[number];
export const reportRequestSchema = z.object({
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
lane: z.enum(lanes),
url: z.url().max(500),
status: z.enum(runStatuses),
httpStatus: z.number().int().min(0).max(999).nullish(),
found: z.number().int().min(0).max(10_000).nullish(),
durationMs: z.number().int().min(0).max(600_000).nullish(),
message: optionalText(500)
});
export type ReportRequest = z.infer<typeof reportRequestSchema>;
export type SyncResponse = {
newIds: string[];
// Listings the server has no detail page for yet, newest first.
needDetails: { sourceId: string; url: string }[];
alerted: number;
};

View File

@@ -47,6 +47,12 @@ export const projects = [
href: '/projects/money-tracker',
external: false
},
{
name: 'house tracker',
subtitle: 'Dutch rental and sale listings, with alerts',
href: '/projects/house-tracker',
external: false
},
{
name: 'cs prices',
subtitle: 'Counter-Strike market prices across time',

View File

@@ -0,0 +1,128 @@
import {
bigint,
boolean,
check,
doublePrecision,
index,
integer,
jsonb,
pgTable,
text,
uniqueIndex
} from 'drizzle-orm/pg-core';
import { sql } from 'drizzle-orm';
import { id, timestamptz } from './helpers';
// Created by scripts/house-tracker-migration.sql; keep both in sync.
export const houseListings = pgTable(
'house_listings',
{
id: id(),
source: text('source').notNull(),
sourceId: text('source_id').notNull(),
externalId: text('external_id'),
url: text('url').notNull(),
offer: text('offer').notNull(),
propertyType: text('property_type').notNull(),
title: text('title'),
street: text('street'),
postalCode: text('postal_code'),
city: text('city'),
citySlug: text('city_slug'),
neighbourhood: text('neighbourhood'),
priceEur: integer('price_eur'),
areaM2: integer('area_m2'),
rooms: integer('rooms'),
bedrooms: integer('bedrooms'),
bathrooms: integer('bathrooms'),
constructionYear: integer('construction_year'),
energyLabel: text('energy_label'),
interior: text('interior'),
depositEur: integer('deposit_eur'),
availableFrom: text('available_from'),
contractType: text('contract_type'),
agent: text('agent'),
imageUrl: text('image_url'),
description: text('description'),
features: jsonb('features').$type<Record<string, string>>().notNull().default({}),
lat: doublePrecision('lat'),
lng: doublePrecision('lng'),
offeredAt: timestamptz('offered_at'),
firstSeenAt: timestamptz('first_seen_at').notNull().defaultNow(),
firstSeenLane: text('first_seen_lane').notNull().default('slow'),
lastSeenAt: timestamptz('last_seen_at').notNull().defaultNow(),
detailFetchedAt: timestamptz('detail_fetched_at'),
removedAt: timestamptz('removed_at'),
alertedAt: timestamptz('alerted_at'),
status: text('status'),
notes: text('notes')
},
(table) => [
check('house_listings_offer_check', sql`offer IN ('rent', 'buy')`),
check('house_listings_price_eur_check', sql`price_eur IS NULL OR price_eur >= 0`),
uniqueIndex('house_listings_source_idx').on(table.source, table.sourceId),
index('house_listings_active_idx')
.on(table.offer, table.firstSeenAt.desc())
.where(sql`removed_at IS NULL`),
index('house_listings_city_idx').on(table.source, table.offer, table.citySlug),
index('house_listings_details_idx')
.on(table.firstSeenAt.desc())
.where(sql`detail_fetched_at IS NULL`)
]
);
export const housePriceHistory = pgTable(
'house_price_history',
{
listingId: integer('listing_id')
.notNull()
.references(() => houseListings.id, { onDelete: 'cascade' }),
observedAt: timestamptz('observed_at').notNull().defaultNow(),
priceEur: integer('price_eur').notNull()
},
(table) => [index('house_price_history_listing_idx').on(table.listingId, table.observedAt)]
);
export const houseSearches = pgTable('house_searches', {
searchUrl: text('search_url').primaryKey(),
source: text('source').notNull(),
lane: text('lane').notNull(),
firstSyncedAt: timestamptz('first_synced_at').notNull().defaultNow(),
lastSyncedAt: timestamptz('last_synced_at').notNull().defaultNow(),
lastFound: integer('last_found').notNull().default(0)
});
export const houseWorkerRuns = pgTable(
'house_worker_runs',
{
id: bigint('id', { mode: 'number' }).primaryKey().generatedAlwaysAsIdentity(),
source: text('source').notNull(),
lane: text('lane').notNull(),
url: text('url').notNull(),
status: text('status').notNull(),
httpStatus: integer('http_status'),
found: integer('found'),
durationMs: integer('duration_ms'),
message: text('message'),
createdAt: timestamptz('created_at').notNull().defaultNow()
},
(table) => [index('house_worker_runs_created_idx').on(table.createdAt.desc())]
);
export const houseAlertRules = pgTable(
'house_alert_rules',
{
id: id(),
name: text('name').notNull(),
enabled: boolean('enabled').notNull().default(true),
offer: text('offer').notNull().default('rent'),
minPrice: integer('min_price'),
maxPrice: integer('max_price'),
minArea: integer('min_area'),
minBedrooms: integer('min_bedrooms'),
cities: text('cities').array().notNull().default([]),
propertyTypes: text('property_types').array().notNull().default([]),
createdAt: timestamptz('created_at').notNull().defaultNow()
},
() => [check('house_alert_rules_offer_check', sql`offer IN ('rent', 'buy')`)]
);

View File

@@ -10,3 +10,4 @@ export * from './animaldex';
export * from './lists';
export * from './money-tracker';
export * from './cs-prices';
export * from './house-tracker';

View File

@@ -0,0 +1,360 @@
// House tracker ingest: the home worker POSTs what it scraped, this stores it
// and sends alerts. Nothing here talks to the listing sites.
import { timingSafeEqual } from 'node:crypto';
import { env } from '$env/dynamic/private';
import { error } from '@sveltejs/kit';
import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm';
import { db } from '$lib/server/db';
import {
houseAlertRules,
houseListings,
housePriceHistory,
houseSearches,
houseWorkerRuns
} from '$lib/server/db/schema';
import {
alertMessage,
matchesRule,
type AlertMessage,
type AlertRule
} from '$lib/house-tracker/alerts';
import { listingRef } from '$lib/house-tracker/pararius';
import type {
DetailsRequest,
ListingCard,
ReportRequest,
SyncRequest,
SyncResponse
} from '$lib/house-tracker/types';
export function requireWorker(request: Request) {
const expected = env.HOUSE_WORKER_TOKEN ?? '';
if (expected.length < 32) error(503, 'HOUSE_WORKER_TOKEN is not configured');
const given = (request.headers.get('authorization') ?? '').replace(/^Bearer\s+/i, '');
const a = Buffer.from(given);
const b = Buffer.from(expected);
if (a.length !== b.length || !timingSafeEqual(a, b)) error(401, 'Invalid worker token');
}
function citySlug(card: Pick<ListingCard, 'url'>): string | null {
return listingRef(card.url)?.citySlug ?? null;
}
type Tx = Parameters<Parameters<typeof db.transaction>[0]>[0];
async function recordPrice(
tx: Tx,
listingId: number,
previous: number | null,
next: number | null
) {
if (next == null || next === previous) return;
await tx.insert(housePriceHistory).values({ listingId, priceEur: next });
}
export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
const sourceIds = input.cards.map((card) => card.sourceId);
const toAlert: { id: number; message: AlertMessage }[] = [];
const newIds: string[] = [];
const rules = (await db
.select()
.from(houseAlertRules)
.where(eq(houseAlertRules.enabled, true))) as AlertRule[];
await db.transaction(async (tx) => {
const [search] = await tx
.select({ firstSyncedAt: houseSearches.firstSyncedAt })
.from(houseSearches)
.where(eq(houseSearches.searchUrl, input.searchUrl));
const seeding = !search;
await tx
.insert(houseSearches)
.values({
searchUrl: input.searchUrl,
source: input.source,
lane: input.lane,
lastFound: input.cards.length
})
.onConflictDoUpdate({
target: houseSearches.searchUrl,
set: { lastSyncedAt: sql`now()`, lastFound: input.cards.length, lane: input.lane }
});
const existing = sourceIds.length
? await tx
.select({
id: houseListings.id,
sourceId: houseListings.sourceId,
priceEur: houseListings.priceEur
})
.from(houseListings)
.where(
and(eq(houseListings.source, input.source), inArray(houseListings.sourceId, sourceIds))
)
: [];
const known = new Map(existing.map((row) => [row.sourceId, row]));
for (const card of input.cards) {
const previous = known.get(card.sourceId);
if (previous) {
await tx
.update(houseListings)
.set({
lastSeenAt: sql`now()`,
removedAt: null,
url: card.url,
priceEur: card.priceEur ?? previous.priceEur,
title: sql`coalesce(${houseListings.title}, ${card.title ?? null})`,
postalCode: sql`coalesce(${houseListings.postalCode}, ${card.postalCode ?? null})`,
neighbourhood: sql`coalesce(${houseListings.neighbourhood}, ${card.neighbourhood ?? null})`,
areaM2: sql`coalesce(${houseListings.areaM2}, ${card.areaM2 ?? null}::int)`,
rooms: sql`coalesce(${houseListings.rooms}, ${card.rooms ?? null}::int)`,
agent: sql`coalesce(${houseListings.agent}, ${card.agent ?? null})`,
imageUrl: sql`coalesce(${houseListings.imageUrl}, ${card.imageUrl ?? null})`
})
.where(eq(houseListings.id, previous.id));
await recordPrice(tx, previous.id, previous.priceEur, card.priceEur ?? null);
continue;
}
const [row] = await tx
.insert(houseListings)
.values({
source: input.source,
sourceId: card.sourceId,
url: card.url,
offer: card.offer,
propertyType: card.propertyType,
title: card.title,
postalCode: card.postalCode,
city: card.city,
citySlug: citySlug(card),
neighbourhood: card.neighbourhood,
priceEur: card.priceEur,
areaM2: card.areaM2,
rooms: card.rooms,
interior: card.interior,
agent: card.agent,
imageUrl: card.imageUrl,
// Seeded listings were already online; their detection delay means nothing.
firstSeenLane: seeding ? 'seed' : input.lane
})
.onConflictDoNothing()
.returning({ id: houseListings.id });
if (!row) continue;
newIds.push(card.sourceId);
await recordPrice(tx, row.id, null, card.priceEur ?? null);
if (seeding) continue;
const listing = { ...card, citySlug: citySlug(card), source: input.source };
const rule = rules.find((candidate) => matchesRule(listing, candidate));
if (rule) toAlert.push({ id: row.id, message: alertMessage(listing, rule) });
}
if (input.completedSweep) {
await tx
.update(houseListings)
.set({ removedAt: sql`now()` })
.where(
and(
eq(houseListings.source, input.source),
eq(houseListings.offer, input.completedSweep.offer),
eq(houseListings.citySlug, input.completedSweep.city),
isNull(houseListings.removedAt),
lt(houseListings.lastSeenAt, new Date(input.completedSweep.sweepStartedAt))
)
);
}
await tx.insert(houseWorkerRuns).values({
source: input.source,
lane: input.lane,
url: input.searchUrl,
status: 'ok',
found: input.cards.length,
durationMs: Math.max(0, Date.now() - Date.parse(input.startedAt)),
message: seeding ? 'first sync: seeded without alerts' : null
});
});
let alerted = 0;
for (const { id, message } of toAlert) {
if (await sendAlert(message)) {
alerted++;
await db
.update(houseListings)
.set({ alertedAt: sql`now()` })
.where(eq(houseListings.id, id));
}
}
// Fast lane: detail pages for this search first. Otherwise slowly backfill.
const pending = await db
.select({ sourceId: houseListings.sourceId, url: houseListings.url })
.from(houseListings)
.where(
and(
eq(houseListings.source, input.source),
isNull(houseListings.detailFetchedAt),
isNull(houseListings.removedAt),
sourceIds.length && input.lane === 'fast'
? inArray(houseListings.sourceId, sourceIds)
: undefined
)
)
.orderBy(desc(houseListings.firstSeenAt))
.limit(input.lane === 'fast' ? 10 : 5);
return { newIds, needDetails: pending, alerted };
}
export async function saveDetails(input: DetailsRequest) {
let saved = 0;
await db.transaction(async (tx) => {
for (const detail of input.details) {
const [previous] = await tx
.select({ id: houseListings.id, priceEur: houseListings.priceEur })
.from(houseListings)
.where(
and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId))
);
const values = {
externalId: detail.externalId,
url: detail.url,
offer: detail.offer,
propertyType: detail.propertyType,
title: detail.title,
street: detail.street,
postalCode: detail.postalCode,
city: detail.city,
citySlug: citySlug(detail),
neighbourhood: detail.neighbourhood,
priceEur: detail.priceEur,
areaM2: detail.areaM2,
rooms: detail.rooms,
bedrooms: detail.bedrooms,
bathrooms: detail.bathrooms,
constructionYear: detail.constructionYear,
energyLabel: detail.energyLabel,
interior: detail.interior,
depositEur: detail.depositEur,
availableFrom: detail.availableFrom,
contractType: detail.contractType,
agent: detail.agent,
imageUrl: detail.imageUrl,
description: detail.description,
features: detail.features ?? {},
lat: detail.lat,
lng: detail.lng,
offeredAt: detail.offeredAt ? new Date(detail.offeredAt) : null,
detailFetchedAt: sql`now()`,
lastSeenAt: sql`now()`
};
if (previous) {
await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id));
await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null);
} else {
const [row] = await tx
.insert(houseListings)
.values({ ...values, source: input.source, sourceId: detail.sourceId })
.returning({ id: houseListings.id });
await recordPrice(tx, row.id, null, detail.priceEur ?? null);
}
saved++;
}
});
return { saved };
}
export async function recordReport(input: ReportRequest) {
// One "challenged" notification per 6 hours is plenty.
const [recent] =
input.status === 'challenged'
? await db
.select({ id: houseWorkerRuns.id })
.from(houseWorkerRuns)
.where(
and(
eq(houseWorkerRuns.status, 'challenged'),
sql`${houseWorkerRuns.createdAt} > now() - interval '6 hours'`
)
)
.limit(1)
: [];
await db.insert(houseWorkerRuns).values({
source: input.source,
lane: input.lane,
url: input.url,
status: input.status,
httpStatus: input.httpStatus,
found: input.found,
durationMs: input.durationMs,
message: input.message
});
// Keep a month of history; runs are only for spotting trouble.
await db
.delete(houseWorkerRuns)
.where(lt(houseWorkerRuns.createdAt, sql`now() - interval '30 days'`));
if (input.status === 'challenged' && !recent) {
await sendAlert({
title: '⚠️ House worker was challenged',
body: `${input.source} answered ${input.httpStatus ?? '?'} on ${input.url}. The worker is backing off.`,
url: input.url
});
}
}
/** Discord webhook and/or ntfy. Returns true when at least one channel accepted it. */
export async function sendAlert(message: AlertMessage): Promise<boolean> {
const jobs: Promise<boolean>[] = [];
if (env.HOUSE_DISCORD_WEBHOOK_URL) {
jobs.push(
fetch(env.HOUSE_DISCORD_WEBHOOK_URL, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
username: 'house tracker',
embeds: [
{
title: message.title.slice(0, 256),
url: message.url,
description: message.body.slice(0, 2000),
color: 0x2f9e44
}
]
}),
signal: AbortSignal.timeout(8000)
}).then(
(response) => response.ok,
() => false
)
);
}
if (env.HOUSE_NTFY_URL) {
const url = new URL(env.HOUSE_NTFY_URL);
const topic = url.pathname.replace(/^\/+|\/+$/g, '');
jobs.push(
fetch(url.origin, {
method: 'POST',
headers: {
'content-type': 'application/json',
...(env.HOUSE_NTFY_TOKEN ? { authorization: `Bearer ${env.HOUSE_NTFY_TOKEN}` } : {})
},
body: JSON.stringify({
topic,
title: message.title,
message: message.body,
click: message.url,
priority: 4,
tags: ['house']
}),
signal: AbortSignal.timeout(8000)
}).then(
(response) => response.ok,
() => false
)
);
}
if (!jobs.length) return false;
return (await Promise.all(jobs)).some(Boolean);
}