feat: house tracker with home worker, alerts and listing map
Collects Pararius rent/buy listings into Postgres via a worker that runs on a residential connection, sends Discord/ntfy alerts for new listings matching rules, and shows an admin-only map, market stats and worker log. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013HfF5tnChi8tkCx9svJP6D
This commit is contained in:
@@ -12,6 +12,7 @@
|
||||
import ChartNoAxesCombined from 'lucide-svelte/icons/chart-no-axes-combined';
|
||||
import FolderKanban from 'lucide-svelte/icons/folder-kanban';
|
||||
import Gamepad2 from 'lucide-svelte/icons/gamepad-2';
|
||||
import House from 'lucide-svelte/icons/house';
|
||||
import Globe2 from 'lucide-svelte/icons/globe-2';
|
||||
import Languages from 'lucide-svelte/icons/languages';
|
||||
import Music2 from 'lucide-svelte/icons/music-2';
|
||||
@@ -30,7 +31,8 @@
|
||||
'world opinions': Globe2,
|
||||
'language learner': Languages,
|
||||
'money tracker': Wallet,
|
||||
'cs prices': CandlestickChart
|
||||
'cs prices': CandlestickChart,
|
||||
'house tracker': House
|
||||
};
|
||||
|
||||
const Icon = $derived(icons[name as keyof typeof icons] ?? FolderKanban);
|
||||
|
||||
185
src/lib/house-tracker/ListingMap.svelte
Normal file
185
src/lib/house-tracker/ListingMap.svelte
Normal file
@@ -0,0 +1,185 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import type { Map as MaptilerMap, GeoJSONSource } from '@maptiler/sdk';
|
||||
|
||||
export type MapListing = {
|
||||
id: number;
|
||||
title: string | null;
|
||||
url: string;
|
||||
priceEur: number | null;
|
||||
areaM2: number | null;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
firstSeenAt: Date | string;
|
||||
status: string | null;
|
||||
removedAt: Date | string | null;
|
||||
};
|
||||
|
||||
let {
|
||||
listings,
|
||||
offer,
|
||||
onselect
|
||||
}: { listings: MapListing[]; offer: string; onselect?: (id: number) => void } = $props();
|
||||
|
||||
let container: HTMLDivElement;
|
||||
let map: MaptilerMap | null = null;
|
||||
let ready = $state(false);
|
||||
let mapError = $state('');
|
||||
const euro = new Intl.NumberFormat('nl-NL', {
|
||||
style: 'currency',
|
||||
currency: 'EUR',
|
||||
maximumFractionDigits: 0
|
||||
});
|
||||
|
||||
function collection(rows: MapListing[]) {
|
||||
const now = Date.now();
|
||||
return {
|
||||
type: 'FeatureCollection' as const,
|
||||
features: rows
|
||||
.filter((row) => row.lat != null && row.lng != null)
|
||||
.map((row) => ({
|
||||
type: 'Feature' as const,
|
||||
geometry: { type: 'Point' as const, coordinates: [row.lng!, row.lat!] },
|
||||
properties: {
|
||||
id: row.id,
|
||||
title: row.title ?? 'Listing',
|
||||
url: row.url,
|
||||
label:
|
||||
row.priceEur != null
|
||||
? `${euro.format(row.priceEur)}${offer === 'rent' ? '/mo' : ''}`
|
||||
: 'price on request',
|
||||
area: row.areaM2 ?? 0,
|
||||
kind: row.removedAt
|
||||
? 'removed'
|
||||
: row.status
|
||||
? 'tracked'
|
||||
: now - new Date(row.firstSeenAt).getTime() < 48 * 3_600_000
|
||||
? 'fresh'
|
||||
: 'active'
|
||||
}
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
const data = collection(listings);
|
||||
if (!ready || !map) return;
|
||||
(map.getSource('listings') as GeoJSONSource | undefined)?.setData(data);
|
||||
});
|
||||
|
||||
onMount(() => {
|
||||
let destroyed = false;
|
||||
(async () => {
|
||||
try {
|
||||
const { Map, Popup, config, LngLatBounds } = await import('@maptiler/sdk');
|
||||
const { env } = await import('$env/dynamic/public');
|
||||
if (destroyed) return;
|
||||
if (!env.PUBLIC_MAPTILER_API_KEY) {
|
||||
mapError = 'Map needs PUBLIC_MAPTILER_API_KEY.';
|
||||
return;
|
||||
}
|
||||
config.apiKey = env.PUBLIC_MAPTILER_API_KEY;
|
||||
map = new Map({
|
||||
container,
|
||||
style:
|
||||
'https://api.maptiler.com/maps/streets-v2/style.json?key=' +
|
||||
env.PUBLIC_MAPTILER_API_KEY,
|
||||
center: [4.7104, 52.0115],
|
||||
zoom: 11,
|
||||
cooperativeGestures: true
|
||||
});
|
||||
map.on('load', () => {
|
||||
if (!map) return;
|
||||
const data = collection(listings);
|
||||
map.addSource('listings', { type: 'geojson', data });
|
||||
map.addLayer({
|
||||
id: 'listings',
|
||||
type: 'circle',
|
||||
source: 'listings',
|
||||
paint: {
|
||||
'circle-radius': ['interpolate', ['linear'], ['zoom'], 8, 3, 14, 8],
|
||||
'circle-color': [
|
||||
'match',
|
||||
['get', 'kind'],
|
||||
'fresh',
|
||||
'#2f9e44',
|
||||
'tracked',
|
||||
'#1c7ed6',
|
||||
'removed',
|
||||
'#adb5bd',
|
||||
'#f08c00'
|
||||
],
|
||||
'circle-stroke-color': '#ffffff',
|
||||
'circle-stroke-width': 1.5
|
||||
}
|
||||
});
|
||||
map.on('click', 'listings', (event) => {
|
||||
const feature = event.features?.[0];
|
||||
if (!feature || !map) return;
|
||||
const props = feature.properties as {
|
||||
id: number;
|
||||
title: string;
|
||||
url: string;
|
||||
label: string;
|
||||
area: number;
|
||||
};
|
||||
const body = document.createElement('div');
|
||||
body.className = 'text-sm text-neutral-900';
|
||||
const title = document.createElement('a');
|
||||
title.href = props.url;
|
||||
title.target = '_blank';
|
||||
title.rel = 'noreferrer';
|
||||
title.className = 'font-medium underline';
|
||||
title.textContent = props.title;
|
||||
const facts = document.createElement('div');
|
||||
facts.textContent = `${props.label}${props.area ? ` · ${props.area} m²` : ''}`;
|
||||
body.append(title, facts);
|
||||
new Popup({ offset: 8 })
|
||||
.setLngLat(
|
||||
(feature.geometry as unknown as { coordinates: [number, number] }).coordinates
|
||||
)
|
||||
.setDOMContent(body)
|
||||
.addTo(map);
|
||||
onselect?.(props.id);
|
||||
});
|
||||
map.on('mouseenter', 'listings', () => map && (map.getCanvas().style.cursor = 'pointer'));
|
||||
map.on('mouseleave', 'listings', () => map && (map.getCanvas().style.cursor = ''));
|
||||
if (data.features.length > 1) {
|
||||
const bounds = new LngLatBounds();
|
||||
for (const feature of data.features) bounds.extend(feature.geometry.coordinates);
|
||||
map.fitBounds(bounds, { padding: 40, maxZoom: 14, duration: 0 });
|
||||
}
|
||||
ready = true;
|
||||
});
|
||||
} catch (error) {
|
||||
mapError = `Map failed to load: ${(error as Error).message}`;
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
destroyed = true;
|
||||
map?.remove();
|
||||
map = null;
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<link rel="stylesheet" href="https://cdn.maptiler.com/maptiler-sdk-js/latest/maptiler-sdk.css" />
|
||||
</svelte:head>
|
||||
|
||||
<div class="relative h-[420px] w-full overflow-hidden rounded-lg border border-border">
|
||||
<div bind:this={container} class="h-full w-full"></div>
|
||||
{#if mapError}
|
||||
<div
|
||||
class="absolute inset-0 flex items-center justify-center bg-muted p-4 text-sm text-muted-foreground"
|
||||
>
|
||||
{mapError}
|
||||
</div>
|
||||
{/if}
|
||||
<div class="absolute bottom-2 left-2 flex gap-3 rounded-md bg-background/90 px-2 py-1 text-xs">
|
||||
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#2f9e44]"></span>new (48h)</span>
|
||||
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#f08c00]"></span>active</span>
|
||||
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#1c7ed6]"></span>tracked</span>
|
||||
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#adb5bd]"></span>gone</span>
|
||||
</div>
|
||||
</div>
|
||||
91
src/lib/house-tracker/alerts.test.ts
Normal file
91
src/lib/house-tracker/alerts.test.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { alertMessage, matchesRule, type AlertRule } from './alerts';
|
||||
import { backoffMs, isBusy, nextDelayMs } from './schedule';
|
||||
|
||||
const rule: AlertRule = {
|
||||
id: 1,
|
||||
name: 'Gouda area rentals',
|
||||
enabled: true,
|
||||
offer: 'rent',
|
||||
minPrice: 1300,
|
||||
maxPrice: 1700,
|
||||
minArea: null,
|
||||
minBedrooms: null,
|
||||
cities: ['gouda', 'nieuwerkerk aan den ijssel'],
|
||||
propertyTypes: []
|
||||
};
|
||||
|
||||
const listing = {
|
||||
offer: 'rent',
|
||||
propertyType: 'apartment',
|
||||
city: 'Gouda',
|
||||
priceEur: 1558,
|
||||
areaM2: 65,
|
||||
rooms: 2
|
||||
};
|
||||
|
||||
describe('matchesRule', () => {
|
||||
it('matches inside the price band and city list', () => {
|
||||
expect(matchesRule(listing, rule)).toBe(true);
|
||||
expect(matchesRule({ ...listing, city: 'Nieuwerkerk aan den IJssel' }, rule)).toBe(true);
|
||||
expect(
|
||||
matchesRule({ ...listing, city: null, citySlug: 'nieuwerkerk-aan-den-ijssel' }, rule)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects outside price, city, offer or when disabled', () => {
|
||||
expect(matchesRule({ ...listing, priceEur: 1800 }, rule)).toBe(false);
|
||||
expect(matchesRule({ ...listing, priceEur: null }, rule)).toBe(false);
|
||||
expect(matchesRule({ ...listing, city: 'Rotterdam' }, rule)).toBe(false);
|
||||
expect(matchesRule({ ...listing, offer: 'buy' }, rule)).toBe(false);
|
||||
expect(matchesRule(listing, { ...rule, enabled: false })).toBe(false);
|
||||
});
|
||||
|
||||
it('lets unknown size through but rejects known small ones', () => {
|
||||
const roomy = { ...rule, minArea: 70, minBedrooms: 2 };
|
||||
expect(matchesRule({ ...listing, areaM2: null, rooms: null }, roomy)).toBe(true);
|
||||
expect(matchesRule(listing, roomy)).toBe(false);
|
||||
});
|
||||
|
||||
it('formats a message', () => {
|
||||
const message = alertMessage(
|
||||
{ ...listing, title: 'Appartement Raam', url: 'https://example.com', source: 'pararius' },
|
||||
rule
|
||||
);
|
||||
expect(message.title).toBe('🏠 Appartement Raam');
|
||||
expect(message.body).toContain('65 m²');
|
||||
expect(message.body).toContain('/mo');
|
||||
});
|
||||
});
|
||||
|
||||
describe('schedule', () => {
|
||||
// 2026-10-07 is a Wednesday; Amsterdam is UTC+2 in October.
|
||||
const wednesdayNoon = new Date('2026-10-07T10:00:00Z');
|
||||
const wednesdayNight = new Date('2026-10-07T21:00:00Z');
|
||||
const saturday = new Date('2026-10-10T10:00:00Z');
|
||||
|
||||
it('knows office hours in Amsterdam', () => {
|
||||
expect(isBusy(wednesdayNoon)).toBe(true);
|
||||
expect(isBusy(wednesdayNight)).toBe(false);
|
||||
expect(isBusy(saturday)).toBe(false);
|
||||
expect(isBusy(new Date('2026-10-07T04:59:00Z'))).toBe(false);
|
||||
expect(isBusy(new Date('2026-10-07T05:00:00Z'))).toBe(true);
|
||||
});
|
||||
|
||||
it('polls every few minutes when busy and slowly otherwise', () => {
|
||||
expect(nextDelayMs(wednesdayNoon, undefined, () => 0)).toBe(3 * 60_000);
|
||||
expect(nextDelayMs(wednesdayNoon, undefined, () => 1)).toBe(5 * 60_000);
|
||||
expect(nextDelayMs(saturday, undefined, () => 0)).toBe(30 * 60_000);
|
||||
});
|
||||
|
||||
it('wakes up for the start of office hours', () => {
|
||||
const justBefore = new Date('2026-10-07T04:50:00Z'); // 06:50 local
|
||||
expect(nextDelayMs(justBefore, undefined, () => 1)).toBeLessThanOrEqual(11 * 60_000);
|
||||
});
|
||||
|
||||
it('backs off exponentially with a cap', () => {
|
||||
expect(backoffMs(1, () => 0.5)).toBe(15 * 60_000);
|
||||
expect(backoffMs(3, () => 0.5)).toBe(60 * 60_000);
|
||||
expect(backoffMs(20, () => 0.5)).toBe(6 * 60 * 60_000);
|
||||
});
|
||||
});
|
||||
92
src/lib/house-tracker/alerts.ts
Normal file
92
src/lib/house-tracker/alerts.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
// Pure alert matching and message formatting, shared by server and tests.
|
||||
|
||||
export type AlertRule = {
|
||||
id: number;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
offer: string;
|
||||
minPrice: number | null;
|
||||
maxPrice: number | null;
|
||||
minArea: number | null;
|
||||
minBedrooms: number | null;
|
||||
cities: string[];
|
||||
propertyTypes: string[];
|
||||
};
|
||||
|
||||
export type AlertListing = {
|
||||
offer: string;
|
||||
propertyType: string;
|
||||
city?: string | null;
|
||||
citySlug?: string | null;
|
||||
priceEur?: number | null;
|
||||
areaM2?: number | null;
|
||||
bedrooms?: number | null;
|
||||
rooms?: number | null;
|
||||
};
|
||||
|
||||
export function normalizeCity(value: string | null | undefined): string {
|
||||
return (value ?? '')
|
||||
.normalize('NFD')
|
||||
.replace(/[̀-ͯ]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/-/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function matchesRule(listing: AlertListing, rule: AlertRule): boolean {
|
||||
if (!rule.enabled || listing.offer !== rule.offer) return false;
|
||||
if (rule.minPrice != null || rule.maxPrice != null) {
|
||||
// "Price on request" can't be checked, so it never matches a price rule.
|
||||
if (listing.priceEur == null) return false;
|
||||
if (rule.minPrice != null && listing.priceEur < rule.minPrice) return false;
|
||||
if (rule.maxPrice != null && listing.priceEur > rule.maxPrice) return false;
|
||||
}
|
||||
// Unknown area or bedrooms pass: search cards often lack them, and missing an
|
||||
// alert is worse than one extra notification.
|
||||
if (rule.minArea != null && listing.areaM2 != null && listing.areaM2 < rule.minArea) return false;
|
||||
if (rule.minBedrooms != null) {
|
||||
const bedrooms = listing.bedrooms ?? (listing.rooms != null ? listing.rooms - 1 : null);
|
||||
if (bedrooms != null && bedrooms < rule.minBedrooms) return false;
|
||||
}
|
||||
if (rule.propertyTypes.length && !rule.propertyTypes.includes(listing.propertyType)) return false;
|
||||
if (rule.cities.length) {
|
||||
const wanted = new Set(rule.cities.map(normalizeCity));
|
||||
const city = normalizeCity(listing.city);
|
||||
const slug = normalizeCity(listing.citySlug);
|
||||
if (!wanted.has(city) && !wanted.has(slug)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const euros = new Intl.NumberFormat('nl-NL', {
|
||||
style: 'currency',
|
||||
currency: 'EUR',
|
||||
maximumFractionDigits: 0
|
||||
});
|
||||
|
||||
export type AlertMessage = { title: string; body: string; url: string };
|
||||
|
||||
export function alertMessage(
|
||||
listing: AlertListing & {
|
||||
title?: string | null;
|
||||
url: string;
|
||||
neighbourhood?: string | null;
|
||||
source: string;
|
||||
},
|
||||
rule: Pick<AlertRule, 'name'>
|
||||
): AlertMessage {
|
||||
const price = listing.priceEur != null ? euros.format(listing.priceEur) : 'price on request';
|
||||
const suffix = listing.offer === 'rent' && listing.priceEur != null ? '/mo' : '';
|
||||
const facts = [
|
||||
`${price}${suffix}`,
|
||||
listing.areaM2 != null ? `${listing.areaM2} m²` : null,
|
||||
listing.rooms != null ? `${listing.rooms} rooms` : null,
|
||||
[listing.city, listing.neighbourhood].filter(Boolean).join(', ') || null
|
||||
].filter(Boolean);
|
||||
return {
|
||||
title: `🏠 ${listing.title ?? 'New listing'}`,
|
||||
body: `${facts.join(' · ')}\n${rule.name} · ${listing.source}`,
|
||||
url: listing.url
|
||||
};
|
||||
}
|
||||
467
src/lib/house-tracker/fixtures/pararius-detail-apartment.html
Normal file
467
src/lib/house-tracker/fixtures/pararius-detail-apartment.html
Normal file
File diff suppressed because one or more lines are too long
304
src/lib/house-tracker/fixtures/pararius-detail-house.html
Normal file
304
src/lib/house-tracker/fixtures/pararius-detail-house.html
Normal file
File diff suppressed because one or more lines are too long
391
src/lib/house-tracker/fixtures/pararius-detail-on-request.html
Normal file
391
src/lib/house-tracker/fixtures/pararius-detail-on-request.html
Normal file
File diff suppressed because one or more lines are too long
107
src/lib/house-tracker/fixtures/pararius-search-synthetic.html
Normal file
107
src/lib/house-tracker/fixtures/pararius-search-synthetic.html
Normal file
@@ -0,0 +1,107 @@
|
||||
<!doctype html>
|
||||
<!-- Hand-written in the shape of Pararius search results. Replace with a real saved page. -->
|
||||
<html>
|
||||
<head>
|
||||
<title>Huurwoningen Gouda</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="search-list-header__count">12</div>
|
||||
<ul class="search-list">
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<div class="listing-search-item__depiction">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--depiction"
|
||||
href="/appartement-te-huur/gouda/d5d8a72c/raam"
|
||||
>
|
||||
<img
|
||||
src="https://casco-media-prod.global.ssl.fastly.net/listings/example-raam.jpg?width=300"
|
||||
alt=""
|
||||
/>
|
||||
</a>
|
||||
</div>
|
||||
<div class="listing-search-item__label">
|
||||
<span class="listing-label listing-label--new">Nieuw</span>
|
||||
</div>
|
||||
<h2 class="listing-search-item__title">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/appartement-te-huur/gouda/d5d8a72c/raam"
|
||||
>
|
||||
Appartement Raam
|
||||
</a>
|
||||
</h2>
|
||||
<div class="listing-search-item__sub-title'">2801 VN Gouda (Binnenstad)</div>
|
||||
<div class="listing-search-item__price">€ 1.475 per maand</div>
|
||||
<div class="listing-search-item__features">
|
||||
<ul class="illustrated-features illustrated-features--compact">
|
||||
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||
58 m²
|
||||
</li>
|
||||
<li class="illustrated-features__item illustrated-features__item--number-of-rooms">
|
||||
3 kamers
|
||||
</li>
|
||||
<li class="illustrated-features__item illustrated-features__item--interior">
|
||||
Gestoffeerd
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="listing-search-item__info">
|
||||
<a class="listing-search-item__link" href="/makelaars/gouda/example"
|
||||
>Voorbeeld Makelaardij</a
|
||||
>
|
||||
</div>
|
||||
</section>
|
||||
</li>
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<h2 class="listing-search-item__title">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
|
||||
>
|
||||
Huis Kongsbergstraat 45
|
||||
</a>
|
||||
</h2>
|
||||
<div class="listing-search-item__sub-title'">2804 XV Gouda (De Mammoet)</div>
|
||||
<div class="listing-search-item__price">€1,600 per month</div>
|
||||
<ul class="illustrated-features">
|
||||
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||
113 m²
|
||||
</li>
|
||||
<li class="illustrated-features__item illustrated-features__item--number-of-rooms">
|
||||
5 rooms
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
</li>
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<h2 class="listing-search-item__title">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/huis-te-huur/gouda/8fc6ae25/nansenstraat"
|
||||
>
|
||||
Huis Nansenstraat
|
||||
</a>
|
||||
</h2>
|
||||
<div class="listing-search-item__sub-title'">2806 HK Gouda (Vreewijk)</div>
|
||||
<div class="listing-search-item__price">Prijs op aanvraag</div>
|
||||
</section>
|
||||
</li>
|
||||
</ul>
|
||||
<ul class="pagination__list">
|
||||
<li class="pagination__item pagination__item--next">
|
||||
<a class="pagination__link pagination__link--next" href="/huurwoningen/gouda/page-2"
|
||||
>Volgende</a
|
||||
>
|
||||
</li>
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
158
src/lib/house-tracker/pararius.test.ts
Normal file
158
src/lib/house-tracker/pararius.test.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
cityFromSlug,
|
||||
isChallenge,
|
||||
listingRef,
|
||||
parseDetailPage,
|
||||
parseEuros,
|
||||
parseLocation,
|
||||
parseSearchPage,
|
||||
searchPageUrl
|
||||
} from './pararius';
|
||||
import { listingDetailSchema, listingCardSchema } from './types';
|
||||
|
||||
const fixture = (name: string) =>
|
||||
readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8');
|
||||
|
||||
describe('pararius helpers', () => {
|
||||
it('reads listing urls', () => {
|
||||
expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({
|
||||
sourceId: '541493b8',
|
||||
url: 'https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat',
|
||||
offer: 'rent',
|
||||
propertyType: 'apartment',
|
||||
citySlug: 'gouda'
|
||||
});
|
||||
expect(listingRef('https://www.pararius.nl/huis-te-koop/den-haag/0a1b2c3d/laan')?.offer).toBe(
|
||||
'buy'
|
||||
);
|
||||
expect(listingRef('/huurwoningen/gouda')).toBeNull();
|
||||
});
|
||||
|
||||
it('parses Dutch and English prices', () => {
|
||||
expect(parseEuros('€ 1.558 per maand')).toBe(1558);
|
||||
expect(parseEuros('€1,600 per month')).toBe(1600);
|
||||
expect(parseEuros('€ 425.000 k.k.')).toBe(425000);
|
||||
expect(parseEuros('€ 1.250,50')).toBe(1250);
|
||||
expect(parseEuros('Prijs op aanvraag')).toBeNull();
|
||||
});
|
||||
|
||||
it('splits a location line', () => {
|
||||
expect(parseLocation('2801 BC Gouda (Centrum)')).toEqual({
|
||||
postalCode: '2801 BC',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'Centrum'
|
||||
});
|
||||
expect(parseLocation('Alphen aan den Rijn')).toEqual({
|
||||
postalCode: null,
|
||||
city: 'Alphen aan den Rijn',
|
||||
neighbourhood: null
|
||||
});
|
||||
});
|
||||
|
||||
it('names cities from slugs', () => {
|
||||
expect(cityFromSlug('gouda')).toBe('Gouda');
|
||||
expect(cityFromSlug('den-haag')).toBe('Den Haag');
|
||||
expect(cityFromSlug('alphen-aan-den-rijn')).toBe('Alphen aan den Rijn');
|
||||
});
|
||||
|
||||
it('builds page urls', () => {
|
||||
const base = 'https://www.pararius.nl/huurwoningen/gouda/1300-1700';
|
||||
expect(searchPageUrl(base, 1)).toBe(base);
|
||||
expect(searchPageUrl(base, 3)).toBe(`${base}/page-3`);
|
||||
expect(searchPageUrl(`${base}/page-3`, 2)).toBe(`${base}/page-2`);
|
||||
});
|
||||
|
||||
it('detects Cloudflare challenges', () => {
|
||||
expect(isChallenge('<html><head><title>Just a moment...</title>')).toBe(true);
|
||||
expect(isChallenge('<html>ok</html>', 403)).toBe(true);
|
||||
expect(isChallenge(fixture('pararius-detail-house.html'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseDetailPage', () => {
|
||||
it('reads a house listing', () => {
|
||||
const detail = parseDetailPage(fixture('pararius-detail-house.html'))!;
|
||||
expect(listingDetailSchema.parse(detail)).toBeTruthy();
|
||||
expect(detail).toMatchObject({
|
||||
sourceId: '8f976237',
|
||||
offer: 'rent',
|
||||
propertyType: 'house',
|
||||
externalId: '8f976237-2ce0-5613-adbe-883495268361',
|
||||
street: 'Kongsbergstraat 45',
|
||||
postalCode: '2804 XV',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'De Mammoet',
|
||||
priceEur: 1600,
|
||||
areaM2: 113,
|
||||
rooms: 5,
|
||||
bedrooms: 4,
|
||||
constructionYear: 1981,
|
||||
energyLabel: 'B',
|
||||
depositEur: 1600,
|
||||
agent: 'LIV Residential',
|
||||
offeredAt: '2026-09-30T09:47:00.000Z'
|
||||
});
|
||||
expect(detail.lat).toBeCloseTo(52.0284, 3);
|
||||
expect(detail.lng).toBeCloseTo(4.7174, 3);
|
||||
});
|
||||
|
||||
it('reads interior and availability', () => {
|
||||
const detail = parseDetailPage(fixture('pararius-detail-apartment.html'))!;
|
||||
expect(detail).toMatchObject({
|
||||
propertyType: 'apartment',
|
||||
priceEur: 1558,
|
||||
interior: 'gestoffeerd',
|
||||
availableFrom: 'Per 15-10-2026',
|
||||
contractType: 'Onbepaalde tijd',
|
||||
depositEur: 3316,
|
||||
offeredAt: '2026-10-02T05:35:37.000Z'
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps price-on-request listings', () => {
|
||||
const detail = parseDetailPage(fixture('pararius-detail-on-request.html'))!;
|
||||
expect(detail.priceEur).toBeNull();
|
||||
expect(detail.interior).toBe('gemeubileerd');
|
||||
expect(listingDetailSchema.parse(detail)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('rejects pages that are not listings', () => {
|
||||
expect(parseDetailPage('<html><body>nothing</body></html>')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSearchPage', () => {
|
||||
it('reads listing cards and pagination', () => {
|
||||
const page = parseSearchPage(
|
||||
fixture('pararius-search-synthetic.html'),
|
||||
'https://www.pararius.nl/huurwoningen/gouda'
|
||||
);
|
||||
expect(page.cards).toHaveLength(3);
|
||||
for (const card of page.cards) expect(listingCardSchema.parse(card)).toBeTruthy();
|
||||
expect(page.cards[0]).toMatchObject({
|
||||
sourceId: 'd5d8a72c',
|
||||
title: 'Appartement Raam',
|
||||
postalCode: '2801 VN',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'Binnenstad',
|
||||
priceEur: 1475,
|
||||
areaM2: 58,
|
||||
rooms: 3,
|
||||
interior: 'Gestoffeerd',
|
||||
agent: 'Voorbeeld Makelaardij'
|
||||
});
|
||||
expect(page.cards[1].priceEur).toBe(1600);
|
||||
expect(page.cards[2].priceEur).toBeNull();
|
||||
expect(page.nextPageUrl).toBe('https://www.pararius.nl/huurwoningen/gouda/page-2');
|
||||
expect(page.totalResults).toBe(12);
|
||||
});
|
||||
|
||||
it('falls back to bare links when card markup changes', () => {
|
||||
const html = `<div><a href="/huis-te-huur/gouda/0a1b2c3d/straat">Huis Straat 1</a> € 1.400 per maand 80 m²</div>`;
|
||||
const page = parseSearchPage(html, 'https://www.pararius.nl/huurwoningen/gouda');
|
||||
expect(page.cards).toHaveLength(1);
|
||||
expect(page.cards[0]).toMatchObject({ sourceId: '0a1b2c3d', priceEur: 1400, areaM2: 80 });
|
||||
});
|
||||
});
|
||||
435
src/lib/house-tracker/pararius.ts
Normal file
435
src/lib/house-tracker/pararius.ts
Normal file
@@ -0,0 +1,435 @@
|
||||
// Pararius HTML parsing. Pure functions, no network: the worker fetches, these parse.
|
||||
//
|
||||
// Pararius has no public API, so everything here reads markup that can change
|
||||
// without notice. Each parser prefers structured data (ld+json, the analytics
|
||||
// blob) and falls back to looser text matching, so a redesign degrades fields
|
||||
// instead of dropping listings.
|
||||
import type { ListingCard, ListingDetail, Offer, PropertyType } from './types';
|
||||
|
||||
export const PARARIUS_ORIGIN = 'https://www.pararius.nl';
|
||||
|
||||
const listingPath =
|
||||
/\/(appartement|huis|kamer|studio|woning|[a-z]+)-te-(huur|koop)\/([a-z0-9-]+)\/([0-9a-f]{8})\/([a-z0-9-]+)/;
|
||||
|
||||
const typeWords: Record<string, PropertyType> = {
|
||||
appartement: 'apartment',
|
||||
apartment: 'apartment',
|
||||
huis: 'house',
|
||||
house: 'house',
|
||||
kamer: 'room',
|
||||
room: 'room',
|
||||
studio: 'studio'
|
||||
};
|
||||
|
||||
const entities: Record<string, string> = {
|
||||
amp: '&',
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
nbsp: ' ',
|
||||
euro: '€',
|
||||
sup2: '²',
|
||||
sup3: '³'
|
||||
};
|
||||
|
||||
export function decodeEntities(value: string): string {
|
||||
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => {
|
||||
if (code[0] === '#') {
|
||||
const n =
|
||||
code[1] === 'x' || code[1] === 'X' ? parseInt(code.slice(2), 16) : Number(code.slice(1));
|
||||
return Number.isFinite(n) ? String.fromCodePoint(n) : match;
|
||||
}
|
||||
return entities[code.toLowerCase()] ?? match;
|
||||
});
|
||||
}
|
||||
|
||||
export function textOf(html: string): string {
|
||||
return decodeEntities(html.replace(/<[^>]*>/g, ' '))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */
|
||||
export function isChallenge(html: string, status = 200): boolean {
|
||||
if (status === 403 || status === 429 || status === 503) return true;
|
||||
const head = html.slice(0, 20_000);
|
||||
return (
|
||||
/<title>\s*Just a moment/i.test(head) ||
|
||||
head.includes('challenges.cloudflare.com/turnstile') ||
|
||||
head.includes('cf-chl-') ||
|
||||
/<title>\s*Attention Required/i.test(head)
|
||||
);
|
||||
}
|
||||
|
||||
export type ListingRef = {
|
||||
sourceId: string;
|
||||
url: string;
|
||||
offer: Offer;
|
||||
propertyType: PropertyType;
|
||||
citySlug: string;
|
||||
};
|
||||
|
||||
export function listingRef(href: string): ListingRef | null {
|
||||
const match = listingPath.exec(href);
|
||||
if (!match) return null;
|
||||
const [, type, offer, citySlug, shortId, slug] = match;
|
||||
return {
|
||||
sourceId: shortId,
|
||||
url: `${PARARIUS_ORIGIN}/${type}-te-${offer}/${citySlug}/${shortId}/${slug}`,
|
||||
offer: offer === 'koop' ? 'buy' : 'rent',
|
||||
propertyType: typeWords[type] ?? 'other',
|
||||
citySlug
|
||||
};
|
||||
}
|
||||
|
||||
/** Whole euros from "€ 1.558 per maand", "€1,558 per month", "€ 425.000 k.k." */
|
||||
export function parseEuros(value: string | null | undefined): number | null {
|
||||
if (!value) return null;
|
||||
const match = /€\s*([\d.,]+)/.exec(value) ?? /([\d][\d.,]*)/.exec(value);
|
||||
if (!match) return null;
|
||||
const digits = match[1].replace(/[.,]\d{2}$/, '').replace(/[.,]/g, '');
|
||||
const n = Number(digits);
|
||||
return digits && Number.isSafeInteger(n) ? n : null;
|
||||
}
|
||||
|
||||
export function parseInteger(value: string | null | undefined): number | null {
|
||||
const match = value ? /(\d[\d.]*)/.exec(value) : null;
|
||||
if (!match) return null;
|
||||
const n = Number(match[1].replace(/\./g, ''));
|
||||
return Number.isSafeInteger(n) ? n : null;
|
||||
}
|
||||
|
||||
/** "2801 BC Gouda (Centrum)" → postal code, city and neighbourhood. */
|
||||
export function parseLocation(value: string | null | undefined) {
|
||||
const out = {
|
||||
postalCode: null as string | null,
|
||||
city: null as string | null,
|
||||
neighbourhood: null as string | null
|
||||
};
|
||||
if (!value) return out;
|
||||
let rest = value.trim();
|
||||
const hood = /\(([^)]+)\)\s*$/.exec(rest);
|
||||
if (hood) {
|
||||
out.neighbourhood = hood[1].trim();
|
||||
rest = rest.slice(0, hood.index).trim();
|
||||
}
|
||||
const postal = /^(\d{4})\s?([A-Z]{2})\b/.exec(rest);
|
||||
if (postal) {
|
||||
out.postalCode = `${postal[1]} ${postal[2]}`;
|
||||
rest = rest.slice(postal[0].length).trim();
|
||||
}
|
||||
if (rest) out.city = rest;
|
||||
return out;
|
||||
}
|
||||
|
||||
function classText(html: string, className: string): string | null {
|
||||
const re = new RegExp(
|
||||
`<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)</\\1>`,
|
||||
'i'
|
||||
);
|
||||
const match = re.exec(html);
|
||||
if (!match) return null;
|
||||
const value = textOf(match[2]);
|
||||
return value || null;
|
||||
}
|
||||
|
||||
function firstImage(html: string): string | null {
|
||||
const tag = /<img\b[^>]*>/i.exec(html)?.[0];
|
||||
if (!tag) return null;
|
||||
const src =
|
||||
/\bsrc="(https:[^"]+)"/i.exec(tag)?.[1] ??
|
||||
/\bsrcset="(https:[^"\s,]+)/i.exec(tag)?.[1] ??
|
||||
/\bdata-src="(https:[^"]+)"/i.exec(tag)?.[1];
|
||||
return src ? decodeEntities(src) : null;
|
||||
}
|
||||
|
||||
export type SearchPage = {
|
||||
cards: ListingCard[];
|
||||
nextPageUrl: string | null;
|
||||
totalResults: number | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a search results page (`/huurwoningen/gouda/1300-1700`).
|
||||
* Splits on the listing card wrappers when present; otherwise falls back to
|
||||
* one chunk per unique listing link so a markup change still yields ids.
|
||||
*/
|
||||
export function parseSearchPage(html: string, pageUrl: string): SearchPage {
|
||||
const chunks: string[] = [];
|
||||
const marker =
|
||||
/<(section|li|article|div)\b[^>]*class="[^"]*\blisting-search-item\b(?!__)[^"]*"/gi;
|
||||
const starts = [...html.matchAll(marker)].map((m) => m.index!);
|
||||
if (starts.length) {
|
||||
for (let i = 0; i < starts.length; i++) {
|
||||
chunks.push(html.slice(starts[i], starts[i + 1] ?? starts[i] + 20_000));
|
||||
}
|
||||
} else {
|
||||
const links = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/gi;
|
||||
const seen = new Set<string>();
|
||||
for (const m of html.matchAll(links)) {
|
||||
const ref = listingRef(m[1]);
|
||||
if (!ref || seen.has(ref.sourceId)) continue;
|
||||
seen.add(ref.sourceId);
|
||||
chunks.push(html.slice(Math.max(0, m.index! - 3000), m.index! + 6000));
|
||||
}
|
||||
}
|
||||
|
||||
const cards: ListingCard[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const chunk of chunks) {
|
||||
const href = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/i.exec(chunk)?.[1];
|
||||
const ref = href ? listingRef(decodeEntities(href)) : null;
|
||||
if (!ref || seen.has(ref.sourceId)) continue;
|
||||
seen.add(ref.sourceId);
|
||||
|
||||
const titleLink = /<a\b[^>]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk);
|
||||
const title =
|
||||
(titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title');
|
||||
const location = parseLocation(
|
||||
classText(chunk, 'listing-search-item__sub-title') ??
|
||||
classText(chunk, 'listing-search-item__location')
|
||||
);
|
||||
const priceText =
|
||||
classText(chunk, 'listing-search-item__price') ?? /€\s*[\d.,]+[^<]{0,30}/.exec(chunk)?.[0];
|
||||
const area =
|
||||
classText(chunk, 'illustrated-features__item--surface-area') ??
|
||||
/(\d+)\s*m(?:²|²|2)/i.exec(chunk)?.[0];
|
||||
const rooms = classText(chunk, 'illustrated-features__item--number-of-rooms');
|
||||
const agentBlock = classText(chunk, 'listing-search-item__info');
|
||||
|
||||
cards.push({
|
||||
sourceId: ref.sourceId,
|
||||
url: ref.url,
|
||||
offer: ref.offer,
|
||||
propertyType: ref.propertyType,
|
||||
title: title?.slice(0, 200) ?? null,
|
||||
postalCode: location.postalCode,
|
||||
city: location.city ?? cityFromSlug(ref.citySlug),
|
||||
neighbourhood: location.neighbourhood,
|
||||
priceEur: parseEuros(priceText),
|
||||
areaM2: parseInteger(area),
|
||||
rooms: parseInteger(rooms),
|
||||
interior: classText(chunk, 'illustrated-features__item--interior'),
|
||||
agent: agentBlock?.slice(0, 160) ?? null,
|
||||
imageUrl: firstImage(chunk)
|
||||
});
|
||||
}
|
||||
|
||||
const nextHref =
|
||||
/<a\b[^>]*class="[^"]*pagination__link--next[^"]*"[^>]*href="([^"]+)"/i.exec(html)?.[1] ??
|
||||
/<a\b[^>]*href="([^"]+)"[^>]*class="[^"]*pagination__link--next[^"]*"/i.exec(html)?.[1] ??
|
||||
/<link\b[^>]*rel="next"[^>]*href="([^"]+)"/i.exec(html)?.[1];
|
||||
const total =
|
||||
classText(html, 'search-list-header__count') ??
|
||||
/(\d[\d.]*)\s+(?:huurwoningen|koopwoningen|woningen|results|resultaten)/i.exec(
|
||||
textOf(html.slice(0, 200_000))
|
||||
)?.[1];
|
||||
|
||||
return {
|
||||
cards,
|
||||
nextPageUrl: nextHref ? new URL(decodeEntities(nextHref), pageUrl).toString() : null,
|
||||
totalResults: parseInteger(total)
|
||||
};
|
||||
}
|
||||
|
||||
/** `/huurwoningen/gouda/1300-1700` → page N url (`.../page-2`). */
|
||||
export function searchPageUrl(base: string, page: number): string {
|
||||
const url = new URL(base);
|
||||
url.pathname = url.pathname.replace(/\/page-\d+\/?$/, '').replace(/\/$/, '');
|
||||
if (page > 1) url.pathname += `/page-${page}`;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export function cityFromSlug(slug: string): string {
|
||||
return slug
|
||||
.split('-')
|
||||
.map((part) =>
|
||||
part === 'aan' || part === 'den' || part === 'de'
|
||||
? part
|
||||
: part[0]?.toUpperCase() + part.slice(1)
|
||||
)
|
||||
.join(' ')
|
||||
.replace(/^\w/, (c) => c.toUpperCase());
|
||||
}
|
||||
|
||||
function jsonBlocks(html: string): unknown[] {
|
||||
const out: unknown[] = [];
|
||||
for (const m of html.matchAll(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) {
|
||||
try {
|
||||
out.push(JSON.parse(m[1]));
|
||||
} catch {
|
||||
// Ignore malformed blocks; other sources may still have the data.
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
type LdListing = {
|
||||
'@type'?: string | string[];
|
||||
name?: string;
|
||||
description?: string;
|
||||
image?: string | string[];
|
||||
address?: {
|
||||
streetAddress?: string;
|
||||
addressLocality?: string;
|
||||
addressRegion?: string;
|
||||
postalCode?: string;
|
||||
};
|
||||
numberOfRooms?: { value?: number }[] | { value?: number } | number;
|
||||
floorSize?: { value?: number };
|
||||
yearBuilt?: number;
|
||||
numberOfBedrooms?: number;
|
||||
numberOfBathroomsTotal?: number;
|
||||
datePosted?: string;
|
||||
offers?: { price?: number; url?: string };
|
||||
};
|
||||
|
||||
function ldListing(html: string): LdListing | null {
|
||||
for (const block of jsonBlocks(html)) {
|
||||
const items = Array.isArray(block) ? block : [block];
|
||||
for (const item of items) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const type = (item as LdListing)['@type'];
|
||||
const types = Array.isArray(type) ? type : [type];
|
||||
if (types.includes('RealEstateListing')) return item as LdListing;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The flat `odp.*` analytics object Pararius embeds on listing pages. */
|
||||
export function analyticsBlob(html: string): Record<string, string> {
|
||||
const candidates = [html, decodeEntities(html)];
|
||||
for (const source of candidates) {
|
||||
const match = /\{[^{}]*"odp\.listing_id"[^{}]*\}/.exec(source);
|
||||
if (!match) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(match[0]) as Record<string, unknown>;
|
||||
const out: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(parsed)) {
|
||||
if (key.startsWith('odp.') && (typeof value === 'string' || typeof value === 'number'))
|
||||
out[key.slice(4)] = String(value);
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
// Fall through to the next candidate.
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/** The "Kenmerken" definition list: `Borg → € 1.600`, `Interieur → Gestoffeerd`. */
|
||||
export function featureList(html: string): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
const re =
|
||||
/<dt\b[^>]*class="[^"]*listing-features__term[^"]*"[^>]*>([\s\S]*?)<\/dt>\s*<dd\b[^>]*>([\s\S]*?)<\/dd>/gi;
|
||||
for (const m of html.matchAll(re)) {
|
||||
const key = textOf(m[1]).slice(0, 80);
|
||||
const value = textOf(m[2].replace(/<[^>]*class="[^"]*tooltip[^"]*"[\s\S]*?<\/[a-z]+>/gi, ''))
|
||||
.replace(/\s*Meer info$/i, '')
|
||||
.slice(0, 500);
|
||||
if (key && value && !(key in out)) out[key] = value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const interiorWords: Record<string, string> = {
|
||||
shell: 'kaal',
|
||||
upholstered: 'gestoffeerd',
|
||||
furnished: 'gemeubileerd'
|
||||
};
|
||||
|
||||
const firstNumber = (value: LdListing['numberOfRooms']): number | null => {
|
||||
if (typeof value === 'number') return value;
|
||||
const item = Array.isArray(value) ? value[0] : value;
|
||||
return typeof item?.value === 'number' ? item.value : null;
|
||||
};
|
||||
|
||||
/** Parse a listing page. Returns null when the page is not a listing. */
|
||||
export function parseDetailPage(html: string, fallbackUrl?: string): ListingDetail | null {
|
||||
const canonical =
|
||||
/<link\b[^>]*rel="canonical"[^>]*href="([^"]+)"/i.exec(html)?.[1] ??
|
||||
/<link\b[^>]*href="([^"]+)"[^>]*rel="canonical"/i.exec(html)?.[1];
|
||||
const ref = listingRef(canonical ?? '') ?? listingRef(fallbackUrl ?? '');
|
||||
if (!ref) return null;
|
||||
|
||||
const ld = ldListing(html);
|
||||
const odp = analyticsBlob(html);
|
||||
const features = featureList(html);
|
||||
const summaryTitle = classText(html, 'listing-detail-summary__title');
|
||||
const location = parseLocation(classText(html, 'listing-detail-summary__location'));
|
||||
|
||||
const lat = Number(
|
||||
/data-latitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"latitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1]
|
||||
);
|
||||
const lng = Number(
|
||||
/data-longitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"longitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1]
|
||||
);
|
||||
|
||||
const offeredAt =
|
||||
odp.offered_since && !Number.isNaN(Date.parse(odp.offered_since))
|
||||
? new Date(odp.offered_since).toISOString()
|
||||
: ld?.datePosted
|
||||
? new Date(`${ld.datePosted}T00:00:00Z`).toISOString()
|
||||
: null;
|
||||
|
||||
const price =
|
||||
(odp.price_pcm ? Number(odp.price_pcm) : null) ??
|
||||
(typeof ld?.offers?.price === 'number' ? ld.offers.price : null) ??
|
||||
parseEuros(features['Huurprijs'] ?? features['Vraagprijs'] ?? features['Koopprijs']);
|
||||
|
||||
const image = Array.isArray(ld?.image) ? ld.image[0] : ld?.image;
|
||||
const typeFromOdp = odp.property ? typeWords[odp.property] : undefined;
|
||||
const interior =
|
||||
(odp.interior ? (interiorWords[odp.interior] ?? odp.interior) : null) ??
|
||||
features['Interieur']?.toLowerCase() ??
|
||||
null;
|
||||
|
||||
const street = ld?.address?.streetAddress ?? null;
|
||||
const title =
|
||||
summaryTitle?.replace(/^Te (huur|koop):\s*/i, '') ?? (ld?.name ? `${ld.name}` : null);
|
||||
|
||||
return {
|
||||
sourceId: ref.sourceId,
|
||||
url: ref.url,
|
||||
offer: odp.offer === 'buy' || ref.offer === 'buy' ? 'buy' : 'rent',
|
||||
propertyType: typeFromOdp ?? ref.propertyType,
|
||||
externalId: odp.listing_id ?? null,
|
||||
title: title?.slice(0, 200) ?? null,
|
||||
street,
|
||||
postalCode: ld?.address?.postalCode
|
||||
? ld.address.postalCode.replace(/^(\d{4})\s?([A-Z]{2})$/, '$1 $2')
|
||||
: location.postalCode,
|
||||
city: ld?.address?.addressLocality ?? location.city ?? cityFromSlug(ref.citySlug),
|
||||
neighbourhood: ld?.address?.addressRegion ?? location.neighbourhood,
|
||||
priceEur: Number.isFinite(price) ? price : null,
|
||||
areaM2:
|
||||
parseInteger(odp.floor_space) ??
|
||||
(typeof ld?.floorSize?.value === 'number' ? ld.floorSize.value : null) ??
|
||||
parseInteger(features['Woonoppervlakte']),
|
||||
rooms:
|
||||
parseInteger(odp.rooms) ??
|
||||
firstNumber(ld?.numberOfRooms) ??
|
||||
parseInteger(features['Aantal kamers']),
|
||||
bedrooms:
|
||||
parseInteger(odp.bedrooms) ??
|
||||
ld?.numberOfBedrooms ??
|
||||
parseInteger(features['Aantal slaapkamers']),
|
||||
bathrooms: ld?.numberOfBathroomsTotal ?? parseInteger(features['Aantal badkamers']),
|
||||
constructionYear:
|
||||
parseInteger(odp.construction_year) ?? ld?.yearBuilt ?? parseInteger(features['Bouwjaar']),
|
||||
energyLabel: odp.energy_rating ?? features['Energielabel'] ?? null,
|
||||
interior,
|
||||
depositEur: parseEuros(features['Borg']),
|
||||
availableFrom: features['Beschikbaar'] ?? null,
|
||||
contractType: features['Contractvorm'] ?? features['Huurperiode'] ?? null,
|
||||
agent: classText(html, 'agent-card__title'),
|
||||
imageUrl: image ?? null,
|
||||
lat: Number.isFinite(lat) && lat !== 0 ? lat : null,
|
||||
lng: Number.isFinite(lng) && lng !== 0 ? lng : null,
|
||||
offeredAt,
|
||||
description: ld?.description?.slice(0, 6000) ?? null,
|
||||
features
|
||||
};
|
||||
}
|
||||
70
src/lib/house-tracker/schedule.ts
Normal file
70
src/lib/house-tracker/schedule.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
// When the home worker should poll next. Pure so it can be tested.
|
||||
//
|
||||
// Listings near Gouda were all published on weekdays between 07:30 and 17:30,
|
||||
// so the fast lane polls often during office hours and backs off otherwise.
|
||||
|
||||
export type ScheduleOptions = {
|
||||
/** Weekday office-hours interval, in minutes. */
|
||||
busyMinutes: [number, number];
|
||||
/** Evenings, nights and weekends. */
|
||||
quietMinutes: [number, number];
|
||||
busyStartHour: number;
|
||||
busyEndHour: number;
|
||||
timeZone: string;
|
||||
};
|
||||
|
||||
export const fastLaneSchedule: ScheduleOptions = {
|
||||
busyMinutes: [3, 5],
|
||||
quietMinutes: [30, 60],
|
||||
busyStartHour: 7,
|
||||
busyEndHour: 18,
|
||||
timeZone: 'Europe/Amsterdam'
|
||||
};
|
||||
|
||||
export function localClock(date: Date, timeZone: string) {
|
||||
const parts = new Intl.DateTimeFormat('en-GB', {
|
||||
timeZone,
|
||||
weekday: 'short',
|
||||
hour: 'numeric',
|
||||
minute: 'numeric',
|
||||
hourCycle: 'h23'
|
||||
}).formatToParts(date);
|
||||
const get = (type: string) => parts.find((part) => part.type === type)?.value ?? '';
|
||||
return {
|
||||
weekday: get('weekday'),
|
||||
hour: Number(get('hour')),
|
||||
minute: Number(get('minute'))
|
||||
};
|
||||
}
|
||||
|
||||
export function isBusy(date: Date, options: ScheduleOptions = fastLaneSchedule): boolean {
|
||||
const { weekday, hour } = localClock(date, options.timeZone);
|
||||
if (weekday === 'Sat' || weekday === 'Sun') return false;
|
||||
return hour >= options.busyStartHour && hour < options.busyEndHour;
|
||||
}
|
||||
|
||||
/** Milliseconds until the next poll, with jitter so requests never look clockwork. */
|
||||
export function nextDelayMs(
|
||||
now: Date,
|
||||
options: ScheduleOptions = fastLaneSchedule,
|
||||
random: () => number = Math.random
|
||||
): number {
|
||||
const [min, max] = isBusy(now, options) ? options.busyMinutes : options.quietMinutes;
|
||||
let delay = (min + (max - min) * random()) * 60_000;
|
||||
// Don't sleep through the start of office hours.
|
||||
if (!isBusy(now, options)) {
|
||||
for (let step = 60_000; step < delay; step += 60_000) {
|
||||
if (isBusy(new Date(now.getTime() + step), options)) {
|
||||
delay = step + random() * 60_000;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Math.round(delay);
|
||||
}
|
||||
|
||||
/** Exponential backoff after challenges or errors: 15 min, 30, 60 ... capped at 6 h. */
|
||||
export function backoffMs(failures: number, random: () => number = Math.random): number {
|
||||
const base = 15 * 60_000 * 2 ** Math.max(0, failures - 1);
|
||||
return Math.round(Math.min(base, 6 * 60 * 60_000) * (0.85 + 0.3 * random()));
|
||||
}
|
||||
118
src/lib/house-tracker/types.ts
Normal file
118
src/lib/house-tracker/types.ts
Normal file
@@ -0,0 +1,118 @@
|
||||
// Shared between the home worker (scripts/house-worker.ts), the ingest API and
|
||||
// the page. Keep this file free of `$lib` / `$env` imports: the worker runs it
|
||||
// directly with bun, outside SvelteKit.
|
||||
import { z } from 'zod';
|
||||
|
||||
export const offers = ['rent', 'buy'] as const;
|
||||
export type Offer = (typeof offers)[number];
|
||||
|
||||
export const propertyTypes = ['apartment', 'house', 'room', 'studio', 'other'] as const;
|
||||
export type PropertyType = (typeof propertyTypes)[number];
|
||||
|
||||
const text = (max: number) => z.string().trim().max(max);
|
||||
const optionalText = (max: number) => text(max).nullish();
|
||||
const count = z.number().int().min(0).max(10_000).nullish();
|
||||
|
||||
// What a search results page tells us about a listing. This alone is enough to
|
||||
// send an alert, so the fast lane never waits for the detail page.
|
||||
export const listingCardSchema = z.object({
|
||||
sourceId: text(80).min(1),
|
||||
url: z.url().max(500),
|
||||
offer: z.enum(offers),
|
||||
propertyType: z.enum(propertyTypes),
|
||||
title: optionalText(200),
|
||||
postalCode: optionalText(10),
|
||||
city: optionalText(80),
|
||||
neighbourhood: optionalText(120),
|
||||
priceEur: z.number().int().min(0).max(100_000_000).nullish(),
|
||||
areaM2: count,
|
||||
rooms: count,
|
||||
interior: optionalText(40),
|
||||
agent: optionalText(160),
|
||||
imageUrl: z.url().max(1000).nullish()
|
||||
});
|
||||
export type ListingCard = z.infer<typeof listingCardSchema>;
|
||||
|
||||
// Everything else we learn from the listing page itself.
|
||||
export const listingDetailSchema = listingCardSchema.extend({
|
||||
externalId: optionalText(80),
|
||||
street: optionalText(200),
|
||||
bedrooms: count,
|
||||
bathrooms: count,
|
||||
constructionYear: z.number().int().min(1000).max(2100).nullish(),
|
||||
energyLabel: optionalText(10),
|
||||
depositEur: z.number().int().min(0).max(10_000_000).nullish(),
|
||||
availableFrom: optionalText(80),
|
||||
contractType: optionalText(80),
|
||||
lat: z.number().min(-90).max(90).nullish(),
|
||||
lng: z.number().min(-180).max(180).nullish(),
|
||||
offeredAt: z.iso.datetime({ offset: true }).nullish(),
|
||||
description: optionalText(6000),
|
||||
features: z.record(z.string().max(80), z.string().max(500)).optional()
|
||||
});
|
||||
export type ListingDetail = z.infer<typeof listingDetailSchema>;
|
||||
|
||||
export const listingStatuses = [
|
||||
'interested',
|
||||
'contacted',
|
||||
'viewing',
|
||||
'applied',
|
||||
'rejected'
|
||||
] as const;
|
||||
export type ListingStatus = (typeof listingStatuses)[number];
|
||||
|
||||
export const lanes = ['fast', 'slow'] as const;
|
||||
export type Lane = (typeof lanes)[number];
|
||||
|
||||
// One poll of one search page.
|
||||
export const syncRequestSchema = z.object({
|
||||
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
|
||||
lane: z.enum(lanes),
|
||||
searchUrl: z.url().max(500),
|
||||
startedAt: z.iso.datetime({ offset: true }),
|
||||
cards: z.array(listingCardSchema).max(200),
|
||||
// Set on the last page of a full, unfiltered sweep of one city. Listings in
|
||||
// that scope that were not seen during the sweep are then marked removed.
|
||||
completedSweep: z
|
||||
.object({
|
||||
offer: z.enum(offers),
|
||||
city: text(80).min(1),
|
||||
sweepStartedAt: z.iso.datetime({ offset: true })
|
||||
})
|
||||
.nullish()
|
||||
});
|
||||
export type SyncRequest = z.infer<typeof syncRequestSchema>;
|
||||
|
||||
export const detailsRequestSchema = z.object({
|
||||
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
|
||||
details: z.array(listingDetailSchema).max(50)
|
||||
});
|
||||
export type DetailsRequest = z.infer<typeof detailsRequestSchema>;
|
||||
|
||||
export const runStatuses = [
|
||||
'ok',
|
||||
'challenged',
|
||||
'http-error',
|
||||
'parse-error',
|
||||
'network-error'
|
||||
] as const;
|
||||
export type RunStatus = (typeof runStatuses)[number];
|
||||
|
||||
export const reportRequestSchema = z.object({
|
||||
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
|
||||
lane: z.enum(lanes),
|
||||
url: z.url().max(500),
|
||||
status: z.enum(runStatuses),
|
||||
httpStatus: z.number().int().min(0).max(999).nullish(),
|
||||
found: z.number().int().min(0).max(10_000).nullish(),
|
||||
durationMs: z.number().int().min(0).max(600_000).nullish(),
|
||||
message: optionalText(500)
|
||||
});
|
||||
export type ReportRequest = z.infer<typeof reportRequestSchema>;
|
||||
|
||||
export type SyncResponse = {
|
||||
newIds: string[];
|
||||
// Listings the server has no detail page for yet, newest first.
|
||||
needDetails: { sourceId: string; url: string }[];
|
||||
alerted: number;
|
||||
};
|
||||
@@ -47,6 +47,12 @@ export const projects = [
|
||||
href: '/projects/money-tracker',
|
||||
external: false
|
||||
},
|
||||
{
|
||||
name: 'house tracker',
|
||||
subtitle: 'Dutch rental and sale listings, with alerts',
|
||||
href: '/projects/house-tracker',
|
||||
external: false
|
||||
},
|
||||
{
|
||||
name: 'cs prices',
|
||||
subtitle: 'Counter-Strike market prices across time',
|
||||
|
||||
128
src/lib/server/db/schema/house-tracker.ts
Normal file
128
src/lib/server/db/schema/house-tracker.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
import {
|
||||
bigint,
|
||||
boolean,
|
||||
check,
|
||||
doublePrecision,
|
||||
index,
|
||||
integer,
|
||||
jsonb,
|
||||
pgTable,
|
||||
text,
|
||||
uniqueIndex
|
||||
} from 'drizzle-orm/pg-core';
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { id, timestamptz } from './helpers';
|
||||
|
||||
// Created by scripts/house-tracker-migration.sql; keep both in sync.
|
||||
export const houseListings = pgTable(
|
||||
'house_listings',
|
||||
{
|
||||
id: id(),
|
||||
source: text('source').notNull(),
|
||||
sourceId: text('source_id').notNull(),
|
||||
externalId: text('external_id'),
|
||||
url: text('url').notNull(),
|
||||
offer: text('offer').notNull(),
|
||||
propertyType: text('property_type').notNull(),
|
||||
title: text('title'),
|
||||
street: text('street'),
|
||||
postalCode: text('postal_code'),
|
||||
city: text('city'),
|
||||
citySlug: text('city_slug'),
|
||||
neighbourhood: text('neighbourhood'),
|
||||
priceEur: integer('price_eur'),
|
||||
areaM2: integer('area_m2'),
|
||||
rooms: integer('rooms'),
|
||||
bedrooms: integer('bedrooms'),
|
||||
bathrooms: integer('bathrooms'),
|
||||
constructionYear: integer('construction_year'),
|
||||
energyLabel: text('energy_label'),
|
||||
interior: text('interior'),
|
||||
depositEur: integer('deposit_eur'),
|
||||
availableFrom: text('available_from'),
|
||||
contractType: text('contract_type'),
|
||||
agent: text('agent'),
|
||||
imageUrl: text('image_url'),
|
||||
description: text('description'),
|
||||
features: jsonb('features').$type<Record<string, string>>().notNull().default({}),
|
||||
lat: doublePrecision('lat'),
|
||||
lng: doublePrecision('lng'),
|
||||
offeredAt: timestamptz('offered_at'),
|
||||
firstSeenAt: timestamptz('first_seen_at').notNull().defaultNow(),
|
||||
firstSeenLane: text('first_seen_lane').notNull().default('slow'),
|
||||
lastSeenAt: timestamptz('last_seen_at').notNull().defaultNow(),
|
||||
detailFetchedAt: timestamptz('detail_fetched_at'),
|
||||
removedAt: timestamptz('removed_at'),
|
||||
alertedAt: timestamptz('alerted_at'),
|
||||
status: text('status'),
|
||||
notes: text('notes')
|
||||
},
|
||||
(table) => [
|
||||
check('house_listings_offer_check', sql`offer IN ('rent', 'buy')`),
|
||||
check('house_listings_price_eur_check', sql`price_eur IS NULL OR price_eur >= 0`),
|
||||
uniqueIndex('house_listings_source_idx').on(table.source, table.sourceId),
|
||||
index('house_listings_active_idx')
|
||||
.on(table.offer, table.firstSeenAt.desc())
|
||||
.where(sql`removed_at IS NULL`),
|
||||
index('house_listings_city_idx').on(table.source, table.offer, table.citySlug),
|
||||
index('house_listings_details_idx')
|
||||
.on(table.firstSeenAt.desc())
|
||||
.where(sql`detail_fetched_at IS NULL`)
|
||||
]
|
||||
);
|
||||
|
||||
export const housePriceHistory = pgTable(
|
||||
'house_price_history',
|
||||
{
|
||||
listingId: integer('listing_id')
|
||||
.notNull()
|
||||
.references(() => houseListings.id, { onDelete: 'cascade' }),
|
||||
observedAt: timestamptz('observed_at').notNull().defaultNow(),
|
||||
priceEur: integer('price_eur').notNull()
|
||||
},
|
||||
(table) => [index('house_price_history_listing_idx').on(table.listingId, table.observedAt)]
|
||||
);
|
||||
|
||||
export const houseSearches = pgTable('house_searches', {
|
||||
searchUrl: text('search_url').primaryKey(),
|
||||
source: text('source').notNull(),
|
||||
lane: text('lane').notNull(),
|
||||
firstSyncedAt: timestamptz('first_synced_at').notNull().defaultNow(),
|
||||
lastSyncedAt: timestamptz('last_synced_at').notNull().defaultNow(),
|
||||
lastFound: integer('last_found').notNull().default(0)
|
||||
});
|
||||
|
||||
export const houseWorkerRuns = pgTable(
|
||||
'house_worker_runs',
|
||||
{
|
||||
id: bigint('id', { mode: 'number' }).primaryKey().generatedAlwaysAsIdentity(),
|
||||
source: text('source').notNull(),
|
||||
lane: text('lane').notNull(),
|
||||
url: text('url').notNull(),
|
||||
status: text('status').notNull(),
|
||||
httpStatus: integer('http_status'),
|
||||
found: integer('found'),
|
||||
durationMs: integer('duration_ms'),
|
||||
message: text('message'),
|
||||
createdAt: timestamptz('created_at').notNull().defaultNow()
|
||||
},
|
||||
(table) => [index('house_worker_runs_created_idx').on(table.createdAt.desc())]
|
||||
);
|
||||
|
||||
export const houseAlertRules = pgTable(
|
||||
'house_alert_rules',
|
||||
{
|
||||
id: id(),
|
||||
name: text('name').notNull(),
|
||||
enabled: boolean('enabled').notNull().default(true),
|
||||
offer: text('offer').notNull().default('rent'),
|
||||
minPrice: integer('min_price'),
|
||||
maxPrice: integer('max_price'),
|
||||
minArea: integer('min_area'),
|
||||
minBedrooms: integer('min_bedrooms'),
|
||||
cities: text('cities').array().notNull().default([]),
|
||||
propertyTypes: text('property_types').array().notNull().default([]),
|
||||
createdAt: timestamptz('created_at').notNull().defaultNow()
|
||||
},
|
||||
() => [check('house_alert_rules_offer_check', sql`offer IN ('rent', 'buy')`)]
|
||||
);
|
||||
@@ -10,3 +10,4 @@ export * from './animaldex';
|
||||
export * from './lists';
|
||||
export * from './money-tracker';
|
||||
export * from './cs-prices';
|
||||
export * from './house-tracker';
|
||||
|
||||
360
src/lib/server/house-tracker.ts
Normal file
360
src/lib/server/house-tracker.ts
Normal file
@@ -0,0 +1,360 @@
|
||||
// House tracker ingest: the home worker POSTs what it scraped, this stores it
|
||||
// and sends alerts. Nothing here talks to the listing sites.
|
||||
import { timingSafeEqual } from 'node:crypto';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { error } from '@sveltejs/kit';
|
||||
import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm';
|
||||
import { db } from '$lib/server/db';
|
||||
import {
|
||||
houseAlertRules,
|
||||
houseListings,
|
||||
housePriceHistory,
|
||||
houseSearches,
|
||||
houseWorkerRuns
|
||||
} from '$lib/server/db/schema';
|
||||
import {
|
||||
alertMessage,
|
||||
matchesRule,
|
||||
type AlertMessage,
|
||||
type AlertRule
|
||||
} from '$lib/house-tracker/alerts';
|
||||
import { listingRef } from '$lib/house-tracker/pararius';
|
||||
import type {
|
||||
DetailsRequest,
|
||||
ListingCard,
|
||||
ReportRequest,
|
||||
SyncRequest,
|
||||
SyncResponse
|
||||
} from '$lib/house-tracker/types';
|
||||
|
||||
export function requireWorker(request: Request) {
|
||||
const expected = env.HOUSE_WORKER_TOKEN ?? '';
|
||||
if (expected.length < 32) error(503, 'HOUSE_WORKER_TOKEN is not configured');
|
||||
const given = (request.headers.get('authorization') ?? '').replace(/^Bearer\s+/i, '');
|
||||
const a = Buffer.from(given);
|
||||
const b = Buffer.from(expected);
|
||||
if (a.length !== b.length || !timingSafeEqual(a, b)) error(401, 'Invalid worker token');
|
||||
}
|
||||
|
||||
function citySlug(card: Pick<ListingCard, 'url'>): string | null {
|
||||
return listingRef(card.url)?.citySlug ?? null;
|
||||
}
|
||||
|
||||
type Tx = Parameters<Parameters<typeof db.transaction>[0]>[0];
|
||||
|
||||
async function recordPrice(
|
||||
tx: Tx,
|
||||
listingId: number,
|
||||
previous: number | null,
|
||||
next: number | null
|
||||
) {
|
||||
if (next == null || next === previous) return;
|
||||
await tx.insert(housePriceHistory).values({ listingId, priceEur: next });
|
||||
}
|
||||
|
||||
export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
|
||||
const sourceIds = input.cards.map((card) => card.sourceId);
|
||||
const toAlert: { id: number; message: AlertMessage }[] = [];
|
||||
const newIds: string[] = [];
|
||||
|
||||
const rules = (await db
|
||||
.select()
|
||||
.from(houseAlertRules)
|
||||
.where(eq(houseAlertRules.enabled, true))) as AlertRule[];
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
const [search] = await tx
|
||||
.select({ firstSyncedAt: houseSearches.firstSyncedAt })
|
||||
.from(houseSearches)
|
||||
.where(eq(houseSearches.searchUrl, input.searchUrl));
|
||||
const seeding = !search;
|
||||
await tx
|
||||
.insert(houseSearches)
|
||||
.values({
|
||||
searchUrl: input.searchUrl,
|
||||
source: input.source,
|
||||
lane: input.lane,
|
||||
lastFound: input.cards.length
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: houseSearches.searchUrl,
|
||||
set: { lastSyncedAt: sql`now()`, lastFound: input.cards.length, lane: input.lane }
|
||||
});
|
||||
|
||||
const existing = sourceIds.length
|
||||
? await tx
|
||||
.select({
|
||||
id: houseListings.id,
|
||||
sourceId: houseListings.sourceId,
|
||||
priceEur: houseListings.priceEur
|
||||
})
|
||||
.from(houseListings)
|
||||
.where(
|
||||
and(eq(houseListings.source, input.source), inArray(houseListings.sourceId, sourceIds))
|
||||
)
|
||||
: [];
|
||||
const known = new Map(existing.map((row) => [row.sourceId, row]));
|
||||
|
||||
for (const card of input.cards) {
|
||||
const previous = known.get(card.sourceId);
|
||||
if (previous) {
|
||||
await tx
|
||||
.update(houseListings)
|
||||
.set({
|
||||
lastSeenAt: sql`now()`,
|
||||
removedAt: null,
|
||||
url: card.url,
|
||||
priceEur: card.priceEur ?? previous.priceEur,
|
||||
title: sql`coalesce(${houseListings.title}, ${card.title ?? null})`,
|
||||
postalCode: sql`coalesce(${houseListings.postalCode}, ${card.postalCode ?? null})`,
|
||||
neighbourhood: sql`coalesce(${houseListings.neighbourhood}, ${card.neighbourhood ?? null})`,
|
||||
areaM2: sql`coalesce(${houseListings.areaM2}, ${card.areaM2 ?? null}::int)`,
|
||||
rooms: sql`coalesce(${houseListings.rooms}, ${card.rooms ?? null}::int)`,
|
||||
agent: sql`coalesce(${houseListings.agent}, ${card.agent ?? null})`,
|
||||
imageUrl: sql`coalesce(${houseListings.imageUrl}, ${card.imageUrl ?? null})`
|
||||
})
|
||||
.where(eq(houseListings.id, previous.id));
|
||||
await recordPrice(tx, previous.id, previous.priceEur, card.priceEur ?? null);
|
||||
continue;
|
||||
}
|
||||
|
||||
const [row] = await tx
|
||||
.insert(houseListings)
|
||||
.values({
|
||||
source: input.source,
|
||||
sourceId: card.sourceId,
|
||||
url: card.url,
|
||||
offer: card.offer,
|
||||
propertyType: card.propertyType,
|
||||
title: card.title,
|
||||
postalCode: card.postalCode,
|
||||
city: card.city,
|
||||
citySlug: citySlug(card),
|
||||
neighbourhood: card.neighbourhood,
|
||||
priceEur: card.priceEur,
|
||||
areaM2: card.areaM2,
|
||||
rooms: card.rooms,
|
||||
interior: card.interior,
|
||||
agent: card.agent,
|
||||
imageUrl: card.imageUrl,
|
||||
// Seeded listings were already online; their detection delay means nothing.
|
||||
firstSeenLane: seeding ? 'seed' : input.lane
|
||||
})
|
||||
.onConflictDoNothing()
|
||||
.returning({ id: houseListings.id });
|
||||
if (!row) continue;
|
||||
newIds.push(card.sourceId);
|
||||
await recordPrice(tx, row.id, null, card.priceEur ?? null);
|
||||
if (seeding) continue;
|
||||
const listing = { ...card, citySlug: citySlug(card), source: input.source };
|
||||
const rule = rules.find((candidate) => matchesRule(listing, candidate));
|
||||
if (rule) toAlert.push({ id: row.id, message: alertMessage(listing, rule) });
|
||||
}
|
||||
|
||||
if (input.completedSweep) {
|
||||
await tx
|
||||
.update(houseListings)
|
||||
.set({ removedAt: sql`now()` })
|
||||
.where(
|
||||
and(
|
||||
eq(houseListings.source, input.source),
|
||||
eq(houseListings.offer, input.completedSweep.offer),
|
||||
eq(houseListings.citySlug, input.completedSweep.city),
|
||||
isNull(houseListings.removedAt),
|
||||
lt(houseListings.lastSeenAt, new Date(input.completedSweep.sweepStartedAt))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
await tx.insert(houseWorkerRuns).values({
|
||||
source: input.source,
|
||||
lane: input.lane,
|
||||
url: input.searchUrl,
|
||||
status: 'ok',
|
||||
found: input.cards.length,
|
||||
durationMs: Math.max(0, Date.now() - Date.parse(input.startedAt)),
|
||||
message: seeding ? 'first sync: seeded without alerts' : null
|
||||
});
|
||||
});
|
||||
|
||||
let alerted = 0;
|
||||
for (const { id, message } of toAlert) {
|
||||
if (await sendAlert(message)) {
|
||||
alerted++;
|
||||
await db
|
||||
.update(houseListings)
|
||||
.set({ alertedAt: sql`now()` })
|
||||
.where(eq(houseListings.id, id));
|
||||
}
|
||||
}
|
||||
|
||||
// Fast lane: detail pages for this search first. Otherwise slowly backfill.
|
||||
const pending = await db
|
||||
.select({ sourceId: houseListings.sourceId, url: houseListings.url })
|
||||
.from(houseListings)
|
||||
.where(
|
||||
and(
|
||||
eq(houseListings.source, input.source),
|
||||
isNull(houseListings.detailFetchedAt),
|
||||
isNull(houseListings.removedAt),
|
||||
sourceIds.length && input.lane === 'fast'
|
||||
? inArray(houseListings.sourceId, sourceIds)
|
||||
: undefined
|
||||
)
|
||||
)
|
||||
.orderBy(desc(houseListings.firstSeenAt))
|
||||
.limit(input.lane === 'fast' ? 10 : 5);
|
||||
|
||||
return { newIds, needDetails: pending, alerted };
|
||||
}
|
||||
|
||||
export async function saveDetails(input: DetailsRequest) {
|
||||
let saved = 0;
|
||||
await db.transaction(async (tx) => {
|
||||
for (const detail of input.details) {
|
||||
const [previous] = await tx
|
||||
.select({ id: houseListings.id, priceEur: houseListings.priceEur })
|
||||
.from(houseListings)
|
||||
.where(
|
||||
and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId))
|
||||
);
|
||||
const values = {
|
||||
externalId: detail.externalId,
|
||||
url: detail.url,
|
||||
offer: detail.offer,
|
||||
propertyType: detail.propertyType,
|
||||
title: detail.title,
|
||||
street: detail.street,
|
||||
postalCode: detail.postalCode,
|
||||
city: detail.city,
|
||||
citySlug: citySlug(detail),
|
||||
neighbourhood: detail.neighbourhood,
|
||||
priceEur: detail.priceEur,
|
||||
areaM2: detail.areaM2,
|
||||
rooms: detail.rooms,
|
||||
bedrooms: detail.bedrooms,
|
||||
bathrooms: detail.bathrooms,
|
||||
constructionYear: detail.constructionYear,
|
||||
energyLabel: detail.energyLabel,
|
||||
interior: detail.interior,
|
||||
depositEur: detail.depositEur,
|
||||
availableFrom: detail.availableFrom,
|
||||
contractType: detail.contractType,
|
||||
agent: detail.agent,
|
||||
imageUrl: detail.imageUrl,
|
||||
description: detail.description,
|
||||
features: detail.features ?? {},
|
||||
lat: detail.lat,
|
||||
lng: detail.lng,
|
||||
offeredAt: detail.offeredAt ? new Date(detail.offeredAt) : null,
|
||||
detailFetchedAt: sql`now()`,
|
||||
lastSeenAt: sql`now()`
|
||||
};
|
||||
if (previous) {
|
||||
await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id));
|
||||
await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null);
|
||||
} else {
|
||||
const [row] = await tx
|
||||
.insert(houseListings)
|
||||
.values({ ...values, source: input.source, sourceId: detail.sourceId })
|
||||
.returning({ id: houseListings.id });
|
||||
await recordPrice(tx, row.id, null, detail.priceEur ?? null);
|
||||
}
|
||||
saved++;
|
||||
}
|
||||
});
|
||||
return { saved };
|
||||
}
|
||||
|
||||
export async function recordReport(input: ReportRequest) {
|
||||
// One "challenged" notification per 6 hours is plenty.
|
||||
const [recent] =
|
||||
input.status === 'challenged'
|
||||
? await db
|
||||
.select({ id: houseWorkerRuns.id })
|
||||
.from(houseWorkerRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(houseWorkerRuns.status, 'challenged'),
|
||||
sql`${houseWorkerRuns.createdAt} > now() - interval '6 hours'`
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
: [];
|
||||
await db.insert(houseWorkerRuns).values({
|
||||
source: input.source,
|
||||
lane: input.lane,
|
||||
url: input.url,
|
||||
status: input.status,
|
||||
httpStatus: input.httpStatus,
|
||||
found: input.found,
|
||||
durationMs: input.durationMs,
|
||||
message: input.message
|
||||
});
|
||||
// Keep a month of history; runs are only for spotting trouble.
|
||||
await db
|
||||
.delete(houseWorkerRuns)
|
||||
.where(lt(houseWorkerRuns.createdAt, sql`now() - interval '30 days'`));
|
||||
if (input.status === 'challenged' && !recent) {
|
||||
await sendAlert({
|
||||
title: '⚠️ House worker was challenged',
|
||||
body: `${input.source} answered ${input.httpStatus ?? '?'} on ${input.url}. The worker is backing off.`,
|
||||
url: input.url
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Discord webhook and/or ntfy. Returns true when at least one channel accepted it. */
|
||||
export async function sendAlert(message: AlertMessage): Promise<boolean> {
|
||||
const jobs: Promise<boolean>[] = [];
|
||||
if (env.HOUSE_DISCORD_WEBHOOK_URL) {
|
||||
jobs.push(
|
||||
fetch(env.HOUSE_DISCORD_WEBHOOK_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
username: 'house tracker',
|
||||
embeds: [
|
||||
{
|
||||
title: message.title.slice(0, 256),
|
||||
url: message.url,
|
||||
description: message.body.slice(0, 2000),
|
||||
color: 0x2f9e44
|
||||
}
|
||||
]
|
||||
}),
|
||||
signal: AbortSignal.timeout(8000)
|
||||
}).then(
|
||||
(response) => response.ok,
|
||||
() => false
|
||||
)
|
||||
);
|
||||
}
|
||||
if (env.HOUSE_NTFY_URL) {
|
||||
const url = new URL(env.HOUSE_NTFY_URL);
|
||||
const topic = url.pathname.replace(/^\/+|\/+$/g, '');
|
||||
jobs.push(
|
||||
fetch(url.origin, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(env.HOUSE_NTFY_TOKEN ? { authorization: `Bearer ${env.HOUSE_NTFY_TOKEN}` } : {})
|
||||
},
|
||||
body: JSON.stringify({
|
||||
topic,
|
||||
title: message.title,
|
||||
message: message.body,
|
||||
click: message.url,
|
||||
priority: 4,
|
||||
tags: ['house']
|
||||
}),
|
||||
signal: AbortSignal.timeout(8000)
|
||||
}).then(
|
||||
(response) => response.ok,
|
||||
() => false
|
||||
)
|
||||
);
|
||||
}
|
||||
if (!jobs.length) return false;
|
||||
return (await Promise.all(jobs)).some(Boolean);
|
||||
}
|
||||
Reference in New Issue
Block a user