feat: house tracker with home worker, alerts and listing map
Collects Pararius rent/buy listings into Postgres via a worker that runs on a residential connection, sends Discord/ntfy alerts for new listings matching rules, and shows an admin-only map, market stats and worker log. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013HfF5tnChi8tkCx9svJP6D
This commit is contained in:
@@ -12,6 +12,7 @@
|
||||
import ChartNoAxesCombined from 'lucide-svelte/icons/chart-no-axes-combined';
|
||||
import FolderKanban from 'lucide-svelte/icons/folder-kanban';
|
||||
import Gamepad2 from 'lucide-svelte/icons/gamepad-2';
|
||||
import House from 'lucide-svelte/icons/house';
|
||||
import Globe2 from 'lucide-svelte/icons/globe-2';
|
||||
import Languages from 'lucide-svelte/icons/languages';
|
||||
import Music2 from 'lucide-svelte/icons/music-2';
|
||||
@@ -30,7 +31,8 @@
|
||||
'world opinions': Globe2,
|
||||
'language learner': Languages,
|
||||
'money tracker': Wallet,
|
||||
'cs prices': CandlestickChart
|
||||
'cs prices': CandlestickChart,
|
||||
'house tracker': House
|
||||
};
|
||||
|
||||
const Icon = $derived(icons[name as keyof typeof icons] ?? FolderKanban);
|
||||
|
||||
185
src/lib/house-tracker/ListingMap.svelte
Normal file
185
src/lib/house-tracker/ListingMap.svelte
Normal file
@@ -0,0 +1,185 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import type { Map as MaptilerMap, GeoJSONSource } from '@maptiler/sdk';
|
||||
|
||||
export type MapListing = {
|
||||
id: number;
|
||||
title: string | null;
|
||||
url: string;
|
||||
priceEur: number | null;
|
||||
areaM2: number | null;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
firstSeenAt: Date | string;
|
||||
status: string | null;
|
||||
removedAt: Date | string | null;
|
||||
};
|
||||
|
||||
let {
|
||||
listings,
|
||||
offer,
|
||||
onselect
|
||||
}: { listings: MapListing[]; offer: string; onselect?: (id: number) => void } = $props();
|
||||
|
||||
let container: HTMLDivElement;
|
||||
let map: MaptilerMap | null = null;
|
||||
let ready = $state(false);
|
||||
let mapError = $state('');
|
||||
const euro = new Intl.NumberFormat('nl-NL', {
|
||||
style: 'currency',
|
||||
currency: 'EUR',
|
||||
maximumFractionDigits: 0
|
||||
});
|
||||
|
||||
function collection(rows: MapListing[]) {
|
||||
const now = Date.now();
|
||||
return {
|
||||
type: 'FeatureCollection' as const,
|
||||
features: rows
|
||||
.filter((row) => row.lat != null && row.lng != null)
|
||||
.map((row) => ({
|
||||
type: 'Feature' as const,
|
||||
geometry: { type: 'Point' as const, coordinates: [row.lng!, row.lat!] },
|
||||
properties: {
|
||||
id: row.id,
|
||||
title: row.title ?? 'Listing',
|
||||
url: row.url,
|
||||
label:
|
||||
row.priceEur != null
|
||||
? `${euro.format(row.priceEur)}${offer === 'rent' ? '/mo' : ''}`
|
||||
: 'price on request',
|
||||
area: row.areaM2 ?? 0,
|
||||
kind: row.removedAt
|
||||
? 'removed'
|
||||
: row.status
|
||||
? 'tracked'
|
||||
: now - new Date(row.firstSeenAt).getTime() < 48 * 3_600_000
|
||||
? 'fresh'
|
||||
: 'active'
|
||||
}
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
const data = collection(listings);
|
||||
if (!ready || !map) return;
|
||||
(map.getSource('listings') as GeoJSONSource | undefined)?.setData(data);
|
||||
});
|
||||
|
||||
onMount(() => {
|
||||
let destroyed = false;
|
||||
(async () => {
|
||||
try {
|
||||
const { Map, Popup, config, LngLatBounds } = await import('@maptiler/sdk');
|
||||
const { env } = await import('$env/dynamic/public');
|
||||
if (destroyed) return;
|
||||
if (!env.PUBLIC_MAPTILER_API_KEY) {
|
||||
mapError = 'Map needs PUBLIC_MAPTILER_API_KEY.';
|
||||
return;
|
||||
}
|
||||
config.apiKey = env.PUBLIC_MAPTILER_API_KEY;
|
||||
map = new Map({
|
||||
container,
|
||||
style:
|
||||
'https://api.maptiler.com/maps/streets-v2/style.json?key=' +
|
||||
env.PUBLIC_MAPTILER_API_KEY,
|
||||
center: [4.7104, 52.0115],
|
||||
zoom: 11,
|
||||
cooperativeGestures: true
|
||||
});
|
||||
map.on('load', () => {
|
||||
if (!map) return;
|
||||
const data = collection(listings);
|
||||
map.addSource('listings', { type: 'geojson', data });
|
||||
map.addLayer({
|
||||
id: 'listings',
|
||||
type: 'circle',
|
||||
source: 'listings',
|
||||
paint: {
|
||||
'circle-radius': ['interpolate', ['linear'], ['zoom'], 8, 3, 14, 8],
|
||||
'circle-color': [
|
||||
'match',
|
||||
['get', 'kind'],
|
||||
'fresh',
|
||||
'#2f9e44',
|
||||
'tracked',
|
||||
'#1c7ed6',
|
||||
'removed',
|
||||
'#adb5bd',
|
||||
'#f08c00'
|
||||
],
|
||||
'circle-stroke-color': '#ffffff',
|
||||
'circle-stroke-width': 1.5
|
||||
}
|
||||
});
|
||||
map.on('click', 'listings', (event) => {
|
||||
const feature = event.features?.[0];
|
||||
if (!feature || !map) return;
|
||||
const props = feature.properties as {
|
||||
id: number;
|
||||
title: string;
|
||||
url: string;
|
||||
label: string;
|
||||
area: number;
|
||||
};
|
||||
const body = document.createElement('div');
|
||||
body.className = 'text-sm text-neutral-900';
|
||||
const title = document.createElement('a');
|
||||
title.href = props.url;
|
||||
title.target = '_blank';
|
||||
title.rel = 'noreferrer';
|
||||
title.className = 'font-medium underline';
|
||||
title.textContent = props.title;
|
||||
const facts = document.createElement('div');
|
||||
facts.textContent = `${props.label}${props.area ? ` · ${props.area} m²` : ''}`;
|
||||
body.append(title, facts);
|
||||
new Popup({ offset: 8 })
|
||||
.setLngLat(
|
||||
(feature.geometry as unknown as { coordinates: [number, number] }).coordinates
|
||||
)
|
||||
.setDOMContent(body)
|
||||
.addTo(map);
|
||||
onselect?.(props.id);
|
||||
});
|
||||
map.on('mouseenter', 'listings', () => map && (map.getCanvas().style.cursor = 'pointer'));
|
||||
map.on('mouseleave', 'listings', () => map && (map.getCanvas().style.cursor = ''));
|
||||
if (data.features.length > 1) {
|
||||
const bounds = new LngLatBounds();
|
||||
for (const feature of data.features) bounds.extend(feature.geometry.coordinates);
|
||||
map.fitBounds(bounds, { padding: 40, maxZoom: 14, duration: 0 });
|
||||
}
|
||||
ready = true;
|
||||
});
|
||||
} catch (error) {
|
||||
mapError = `Map failed to load: ${(error as Error).message}`;
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
destroyed = true;
|
||||
map?.remove();
|
||||
map = null;
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<link rel="stylesheet" href="https://cdn.maptiler.com/maptiler-sdk-js/latest/maptiler-sdk.css" />
|
||||
</svelte:head>
|
||||
|
||||
<div class="relative h-[420px] w-full overflow-hidden rounded-lg border border-border">
|
||||
<div bind:this={container} class="h-full w-full"></div>
|
||||
{#if mapError}
|
||||
<div
|
||||
class="absolute inset-0 flex items-center justify-center bg-muted p-4 text-sm text-muted-foreground"
|
||||
>
|
||||
{mapError}
|
||||
</div>
|
||||
{/if}
|
||||
<div class="absolute bottom-2 left-2 flex gap-3 rounded-md bg-background/90 px-2 py-1 text-xs">
|
||||
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#2f9e44]"></span>new (48h)</span>
|
||||
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#f08c00]"></span>active</span>
|
||||
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#1c7ed6]"></span>tracked</span>
|
||||
<span><span class="mr-1 inline-block size-2 rounded-full bg-[#adb5bd]"></span>gone</span>
|
||||
</div>
|
||||
</div>
|
||||
91
src/lib/house-tracker/alerts.test.ts
Normal file
91
src/lib/house-tracker/alerts.test.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { alertMessage, matchesRule, type AlertRule } from './alerts';
|
||||
import { backoffMs, isBusy, nextDelayMs } from './schedule';
|
||||
|
||||
const rule: AlertRule = {
|
||||
id: 1,
|
||||
name: 'Gouda area rentals',
|
||||
enabled: true,
|
||||
offer: 'rent',
|
||||
minPrice: 1300,
|
||||
maxPrice: 1700,
|
||||
minArea: null,
|
||||
minBedrooms: null,
|
||||
cities: ['gouda', 'nieuwerkerk aan den ijssel'],
|
||||
propertyTypes: []
|
||||
};
|
||||
|
||||
const listing = {
|
||||
offer: 'rent',
|
||||
propertyType: 'apartment',
|
||||
city: 'Gouda',
|
||||
priceEur: 1558,
|
||||
areaM2: 65,
|
||||
rooms: 2
|
||||
};
|
||||
|
||||
describe('matchesRule', () => {
|
||||
it('matches inside the price band and city list', () => {
|
||||
expect(matchesRule(listing, rule)).toBe(true);
|
||||
expect(matchesRule({ ...listing, city: 'Nieuwerkerk aan den IJssel' }, rule)).toBe(true);
|
||||
expect(
|
||||
matchesRule({ ...listing, city: null, citySlug: 'nieuwerkerk-aan-den-ijssel' }, rule)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects outside price, city, offer or when disabled', () => {
|
||||
expect(matchesRule({ ...listing, priceEur: 1800 }, rule)).toBe(false);
|
||||
expect(matchesRule({ ...listing, priceEur: null }, rule)).toBe(false);
|
||||
expect(matchesRule({ ...listing, city: 'Rotterdam' }, rule)).toBe(false);
|
||||
expect(matchesRule({ ...listing, offer: 'buy' }, rule)).toBe(false);
|
||||
expect(matchesRule(listing, { ...rule, enabled: false })).toBe(false);
|
||||
});
|
||||
|
||||
it('lets unknown size through but rejects known small ones', () => {
|
||||
const roomy = { ...rule, minArea: 70, minBedrooms: 2 };
|
||||
expect(matchesRule({ ...listing, areaM2: null, rooms: null }, roomy)).toBe(true);
|
||||
expect(matchesRule(listing, roomy)).toBe(false);
|
||||
});
|
||||
|
||||
it('formats a message', () => {
|
||||
const message = alertMessage(
|
||||
{ ...listing, title: 'Appartement Raam', url: 'https://example.com', source: 'pararius' },
|
||||
rule
|
||||
);
|
||||
expect(message.title).toBe('🏠 Appartement Raam');
|
||||
expect(message.body).toContain('65 m²');
|
||||
expect(message.body).toContain('/mo');
|
||||
});
|
||||
});
|
||||
|
||||
describe('schedule', () => {
|
||||
// 2026-10-07 is a Wednesday; Amsterdam is UTC+2 in October.
|
||||
const wednesdayNoon = new Date('2026-10-07T10:00:00Z');
|
||||
const wednesdayNight = new Date('2026-10-07T21:00:00Z');
|
||||
const saturday = new Date('2026-10-10T10:00:00Z');
|
||||
|
||||
it('knows office hours in Amsterdam', () => {
|
||||
expect(isBusy(wednesdayNoon)).toBe(true);
|
||||
expect(isBusy(wednesdayNight)).toBe(false);
|
||||
expect(isBusy(saturday)).toBe(false);
|
||||
expect(isBusy(new Date('2026-10-07T04:59:00Z'))).toBe(false);
|
||||
expect(isBusy(new Date('2026-10-07T05:00:00Z'))).toBe(true);
|
||||
});
|
||||
|
||||
it('polls every few minutes when busy and slowly otherwise', () => {
|
||||
expect(nextDelayMs(wednesdayNoon, undefined, () => 0)).toBe(3 * 60_000);
|
||||
expect(nextDelayMs(wednesdayNoon, undefined, () => 1)).toBe(5 * 60_000);
|
||||
expect(nextDelayMs(saturday, undefined, () => 0)).toBe(30 * 60_000);
|
||||
});
|
||||
|
||||
it('wakes up for the start of office hours', () => {
|
||||
const justBefore = new Date('2026-10-07T04:50:00Z'); // 06:50 local
|
||||
expect(nextDelayMs(justBefore, undefined, () => 1)).toBeLessThanOrEqual(11 * 60_000);
|
||||
});
|
||||
|
||||
it('backs off exponentially with a cap', () => {
|
||||
expect(backoffMs(1, () => 0.5)).toBe(15 * 60_000);
|
||||
expect(backoffMs(3, () => 0.5)).toBe(60 * 60_000);
|
||||
expect(backoffMs(20, () => 0.5)).toBe(6 * 60 * 60_000);
|
||||
});
|
||||
});
|
||||
92
src/lib/house-tracker/alerts.ts
Normal file
92
src/lib/house-tracker/alerts.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
// Pure alert matching and message formatting, shared by server and tests.
|
||||
|
||||
export type AlertRule = {
|
||||
id: number;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
offer: string;
|
||||
minPrice: number | null;
|
||||
maxPrice: number | null;
|
||||
minArea: number | null;
|
||||
minBedrooms: number | null;
|
||||
cities: string[];
|
||||
propertyTypes: string[];
|
||||
};
|
||||
|
||||
export type AlertListing = {
|
||||
offer: string;
|
||||
propertyType: string;
|
||||
city?: string | null;
|
||||
citySlug?: string | null;
|
||||
priceEur?: number | null;
|
||||
areaM2?: number | null;
|
||||
bedrooms?: number | null;
|
||||
rooms?: number | null;
|
||||
};
|
||||
|
||||
export function normalizeCity(value: string | null | undefined): string {
|
||||
return (value ?? '')
|
||||
.normalize('NFD')
|
||||
.replace(/[̀-ͯ]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/-/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function matchesRule(listing: AlertListing, rule: AlertRule): boolean {
|
||||
if (!rule.enabled || listing.offer !== rule.offer) return false;
|
||||
if (rule.minPrice != null || rule.maxPrice != null) {
|
||||
// "Price on request" can't be checked, so it never matches a price rule.
|
||||
if (listing.priceEur == null) return false;
|
||||
if (rule.minPrice != null && listing.priceEur < rule.minPrice) return false;
|
||||
if (rule.maxPrice != null && listing.priceEur > rule.maxPrice) return false;
|
||||
}
|
||||
// Unknown area or bedrooms pass: search cards often lack them, and missing an
|
||||
// alert is worse than one extra notification.
|
||||
if (rule.minArea != null && listing.areaM2 != null && listing.areaM2 < rule.minArea) return false;
|
||||
if (rule.minBedrooms != null) {
|
||||
const bedrooms = listing.bedrooms ?? (listing.rooms != null ? listing.rooms - 1 : null);
|
||||
if (bedrooms != null && bedrooms < rule.minBedrooms) return false;
|
||||
}
|
||||
if (rule.propertyTypes.length && !rule.propertyTypes.includes(listing.propertyType)) return false;
|
||||
if (rule.cities.length) {
|
||||
const wanted = new Set(rule.cities.map(normalizeCity));
|
||||
const city = normalizeCity(listing.city);
|
||||
const slug = normalizeCity(listing.citySlug);
|
||||
if (!wanted.has(city) && !wanted.has(slug)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const euros = new Intl.NumberFormat('nl-NL', {
|
||||
style: 'currency',
|
||||
currency: 'EUR',
|
||||
maximumFractionDigits: 0
|
||||
});
|
||||
|
||||
export type AlertMessage = { title: string; body: string; url: string };
|
||||
|
||||
export function alertMessage(
|
||||
listing: AlertListing & {
|
||||
title?: string | null;
|
||||
url: string;
|
||||
neighbourhood?: string | null;
|
||||
source: string;
|
||||
},
|
||||
rule: Pick<AlertRule, 'name'>
|
||||
): AlertMessage {
|
||||
const price = listing.priceEur != null ? euros.format(listing.priceEur) : 'price on request';
|
||||
const suffix = listing.offer === 'rent' && listing.priceEur != null ? '/mo' : '';
|
||||
const facts = [
|
||||
`${price}${suffix}`,
|
||||
listing.areaM2 != null ? `${listing.areaM2} m²` : null,
|
||||
listing.rooms != null ? `${listing.rooms} rooms` : null,
|
||||
[listing.city, listing.neighbourhood].filter(Boolean).join(', ') || null
|
||||
].filter(Boolean);
|
||||
return {
|
||||
title: `🏠 ${listing.title ?? 'New listing'}`,
|
||||
body: `${facts.join(' · ')}\n${rule.name} · ${listing.source}`,
|
||||
url: listing.url
|
||||
};
|
||||
}
|
||||
467
src/lib/house-tracker/fixtures/pararius-detail-apartment.html
Normal file
467
src/lib/house-tracker/fixtures/pararius-detail-apartment.html
Normal file
File diff suppressed because one or more lines are too long
304
src/lib/house-tracker/fixtures/pararius-detail-house.html
Normal file
304
src/lib/house-tracker/fixtures/pararius-detail-house.html
Normal file
File diff suppressed because one or more lines are too long
391
src/lib/house-tracker/fixtures/pararius-detail-on-request.html
Normal file
391
src/lib/house-tracker/fixtures/pararius-detail-on-request.html
Normal file
File diff suppressed because one or more lines are too long
107
src/lib/house-tracker/fixtures/pararius-search-synthetic.html
Normal file
107
src/lib/house-tracker/fixtures/pararius-search-synthetic.html
Normal file
@@ -0,0 +1,107 @@
|
||||
<!doctype html>
|
||||
<!-- Hand-written in the shape of Pararius search results. Replace with a real saved page. -->
|
||||
<html>
|
||||
<head>
|
||||
<title>Huurwoningen Gouda</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="search-list-header__count">12</div>
|
||||
<ul class="search-list">
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<div class="listing-search-item__depiction">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--depiction"
|
||||
href="/appartement-te-huur/gouda/d5d8a72c/raam"
|
||||
>
|
||||
<img
|
||||
src="https://casco-media-prod.global.ssl.fastly.net/listings/example-raam.jpg?width=300"
|
||||
alt=""
|
||||
/>
|
||||
</a>
|
||||
</div>
|
||||
<div class="listing-search-item__label">
|
||||
<span class="listing-label listing-label--new">Nieuw</span>
|
||||
</div>
|
||||
<h2 class="listing-search-item__title">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/appartement-te-huur/gouda/d5d8a72c/raam"
|
||||
>
|
||||
Appartement Raam
|
||||
</a>
|
||||
</h2>
|
||||
<div class="listing-search-item__sub-title'">2801 VN Gouda (Binnenstad)</div>
|
||||
<div class="listing-search-item__price">€ 1.475 per maand</div>
|
||||
<div class="listing-search-item__features">
|
||||
<ul class="illustrated-features illustrated-features--compact">
|
||||
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||
58 m²
|
||||
</li>
|
||||
<li class="illustrated-features__item illustrated-features__item--number-of-rooms">
|
||||
3 kamers
|
||||
</li>
|
||||
<li class="illustrated-features__item illustrated-features__item--interior">
|
||||
Gestoffeerd
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="listing-search-item__info">
|
||||
<a class="listing-search-item__link" href="/makelaars/gouda/example"
|
||||
>Voorbeeld Makelaardij</a
|
||||
>
|
||||
</div>
|
||||
</section>
|
||||
</li>
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<h2 class="listing-search-item__title">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
|
||||
>
|
||||
Huis Kongsbergstraat 45
|
||||
</a>
|
||||
</h2>
|
||||
<div class="listing-search-item__sub-title'">2804 XV Gouda (De Mammoet)</div>
|
||||
<div class="listing-search-item__price">€1,600 per month</div>
|
||||
<ul class="illustrated-features">
|
||||
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||
113 m²
|
||||
</li>
|
||||
<li class="illustrated-features__item illustrated-features__item--number-of-rooms">
|
||||
5 rooms
|
||||
</li>
|
||||
</ul>
|
||||
</section>
|
||||
</li>
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<h2 class="listing-search-item__title">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/huis-te-huur/gouda/8fc6ae25/nansenstraat"
|
||||
>
|
||||
Huis Nansenstraat
|
||||
</a>
|
||||
</h2>
|
||||
<div class="listing-search-item__sub-title'">2806 HK Gouda (Vreewijk)</div>
|
||||
<div class="listing-search-item__price">Prijs op aanvraag</div>
|
||||
</section>
|
||||
</li>
|
||||
</ul>
|
||||
<ul class="pagination__list">
|
||||
<li class="pagination__item pagination__item--next">
|
||||
<a class="pagination__link pagination__link--next" href="/huurwoningen/gouda/page-2"
|
||||
>Volgende</a
|
||||
>
|
||||
</li>
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
158
src/lib/house-tracker/pararius.test.ts
Normal file
158
src/lib/house-tracker/pararius.test.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
cityFromSlug,
|
||||
isChallenge,
|
||||
listingRef,
|
||||
parseDetailPage,
|
||||
parseEuros,
|
||||
parseLocation,
|
||||
parseSearchPage,
|
||||
searchPageUrl
|
||||
} from './pararius';
|
||||
import { listingDetailSchema, listingCardSchema } from './types';
|
||||
|
||||
const fixture = (name: string) =>
|
||||
readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8');
|
||||
|
||||
describe('pararius helpers', () => {
|
||||
it('reads listing urls', () => {
|
||||
expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({
|
||||
sourceId: '541493b8',
|
||||
url: 'https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat',
|
||||
offer: 'rent',
|
||||
propertyType: 'apartment',
|
||||
citySlug: 'gouda'
|
||||
});
|
||||
expect(listingRef('https://www.pararius.nl/huis-te-koop/den-haag/0a1b2c3d/laan')?.offer).toBe(
|
||||
'buy'
|
||||
);
|
||||
expect(listingRef('/huurwoningen/gouda')).toBeNull();
|
||||
});
|
||||
|
||||
it('parses Dutch and English prices', () => {
|
||||
expect(parseEuros('€ 1.558 per maand')).toBe(1558);
|
||||
expect(parseEuros('€1,600 per month')).toBe(1600);
|
||||
expect(parseEuros('€ 425.000 k.k.')).toBe(425000);
|
||||
expect(parseEuros('€ 1.250,50')).toBe(1250);
|
||||
expect(parseEuros('Prijs op aanvraag')).toBeNull();
|
||||
});
|
||||
|
||||
it('splits a location line', () => {
|
||||
expect(parseLocation('2801 BC Gouda (Centrum)')).toEqual({
|
||||
postalCode: '2801 BC',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'Centrum'
|
||||
});
|
||||
expect(parseLocation('Alphen aan den Rijn')).toEqual({
|
||||
postalCode: null,
|
||||
city: 'Alphen aan den Rijn',
|
||||
neighbourhood: null
|
||||
});
|
||||
});
|
||||
|
||||
it('names cities from slugs', () => {
|
||||
expect(cityFromSlug('gouda')).toBe('Gouda');
|
||||
expect(cityFromSlug('den-haag')).toBe('Den Haag');
|
||||
expect(cityFromSlug('alphen-aan-den-rijn')).toBe('Alphen aan den Rijn');
|
||||
});
|
||||
|
||||
it('builds page urls', () => {
|
||||
const base = 'https://www.pararius.nl/huurwoningen/gouda/1300-1700';
|
||||
expect(searchPageUrl(base, 1)).toBe(base);
|
||||
expect(searchPageUrl(base, 3)).toBe(`${base}/page-3`);
|
||||
expect(searchPageUrl(`${base}/page-3`, 2)).toBe(`${base}/page-2`);
|
||||
});
|
||||
|
||||
it('detects Cloudflare challenges', () => {
|
||||
expect(isChallenge('<html><head><title>Just a moment...</title>')).toBe(true);
|
||||
expect(isChallenge('<html>ok</html>', 403)).toBe(true);
|
||||
expect(isChallenge(fixture('pararius-detail-house.html'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseDetailPage', () => {
|
||||
it('reads a house listing', () => {
|
||||
const detail = parseDetailPage(fixture('pararius-detail-house.html'))!;
|
||||
expect(listingDetailSchema.parse(detail)).toBeTruthy();
|
||||
expect(detail).toMatchObject({
|
||||
sourceId: '8f976237',
|
||||
offer: 'rent',
|
||||
propertyType: 'house',
|
||||
externalId: '8f976237-2ce0-5613-adbe-883495268361',
|
||||
street: 'Kongsbergstraat 45',
|
||||
postalCode: '2804 XV',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'De Mammoet',
|
||||
priceEur: 1600,
|
||||
areaM2: 113,
|
||||
rooms: 5,
|
||||
bedrooms: 4,
|
||||
constructionYear: 1981,
|
||||
energyLabel: 'B',
|
||||
depositEur: 1600,
|
||||
agent: 'LIV Residential',
|
||||
offeredAt: '2026-09-30T09:47:00.000Z'
|
||||
});
|
||||
expect(detail.lat).toBeCloseTo(52.0284, 3);
|
||||
expect(detail.lng).toBeCloseTo(4.7174, 3);
|
||||
});
|
||||
|
||||
it('reads interior and availability', () => {
|
||||
const detail = parseDetailPage(fixture('pararius-detail-apartment.html'))!;
|
||||
expect(detail).toMatchObject({
|
||||
propertyType: 'apartment',
|
||||
priceEur: 1558,
|
||||
interior: 'gestoffeerd',
|
||||
availableFrom: 'Per 15-10-2026',
|
||||
contractType: 'Onbepaalde tijd',
|
||||
depositEur: 3316,
|
||||
offeredAt: '2026-10-02T05:35:37.000Z'
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps price-on-request listings', () => {
|
||||
const detail = parseDetailPage(fixture('pararius-detail-on-request.html'))!;
|
||||
expect(detail.priceEur).toBeNull();
|
||||
expect(detail.interior).toBe('gemeubileerd');
|
||||
expect(listingDetailSchema.parse(detail)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('rejects pages that are not listings', () => {
|
||||
expect(parseDetailPage('<html><body>nothing</body></html>')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSearchPage', () => {
|
||||
it('reads listing cards and pagination', () => {
|
||||
const page = parseSearchPage(
|
||||
fixture('pararius-search-synthetic.html'),
|
||||
'https://www.pararius.nl/huurwoningen/gouda'
|
||||
);
|
||||
expect(page.cards).toHaveLength(3);
|
||||
for (const card of page.cards) expect(listingCardSchema.parse(card)).toBeTruthy();
|
||||
expect(page.cards[0]).toMatchObject({
|
||||
sourceId: 'd5d8a72c',
|
||||
title: 'Appartement Raam',
|
||||
postalCode: '2801 VN',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'Binnenstad',
|
||||
priceEur: 1475,
|
||||
areaM2: 58,
|
||||
rooms: 3,
|
||||
interior: 'Gestoffeerd',
|
||||
agent: 'Voorbeeld Makelaardij'
|
||||
});
|
||||
expect(page.cards[1].priceEur).toBe(1600);
|
||||
expect(page.cards[2].priceEur).toBeNull();
|
||||
expect(page.nextPageUrl).toBe('https://www.pararius.nl/huurwoningen/gouda/page-2');
|
||||
expect(page.totalResults).toBe(12);
|
||||
});
|
||||
|
||||
it('falls back to bare links when card markup changes', () => {
|
||||
const html = `<div><a href="/huis-te-huur/gouda/0a1b2c3d/straat">Huis Straat 1</a> € 1.400 per maand 80 m²</div>`;
|
||||
const page = parseSearchPage(html, 'https://www.pararius.nl/huurwoningen/gouda');
|
||||
expect(page.cards).toHaveLength(1);
|
||||
expect(page.cards[0]).toMatchObject({ sourceId: '0a1b2c3d', priceEur: 1400, areaM2: 80 });
|
||||
});
|
||||
});
|
||||
435
src/lib/house-tracker/pararius.ts
Normal file
435
src/lib/house-tracker/pararius.ts
Normal file
@@ -0,0 +1,435 @@
|
||||
// Pararius HTML parsing. Pure functions, no network: the worker fetches, these parse.
|
||||
//
|
||||
// Pararius has no public API, so everything here reads markup that can change
|
||||
// without notice. Each parser prefers structured data (ld+json, the analytics
|
||||
// blob) and falls back to looser text matching, so a redesign degrades fields
|
||||
// instead of dropping listings.
|
||||
import type { ListingCard, ListingDetail, Offer, PropertyType } from './types';
|
||||
|
||||
export const PARARIUS_ORIGIN = 'https://www.pararius.nl';
|
||||
|
||||
const listingPath =
|
||||
/\/(appartement|huis|kamer|studio|woning|[a-z]+)-te-(huur|koop)\/([a-z0-9-]+)\/([0-9a-f]{8})\/([a-z0-9-]+)/;
|
||||
|
||||
const typeWords: Record<string, PropertyType> = {
|
||||
appartement: 'apartment',
|
||||
apartment: 'apartment',
|
||||
huis: 'house',
|
||||
house: 'house',
|
||||
kamer: 'room',
|
||||
room: 'room',
|
||||
studio: 'studio'
|
||||
};
|
||||
|
||||
const entities: Record<string, string> = {
|
||||
amp: '&',
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
nbsp: ' ',
|
||||
euro: '€',
|
||||
sup2: '²',
|
||||
sup3: '³'
|
||||
};
|
||||
|
||||
export function decodeEntities(value: string): string {
|
||||
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => {
|
||||
if (code[0] === '#') {
|
||||
const n =
|
||||
code[1] === 'x' || code[1] === 'X' ? parseInt(code.slice(2), 16) : Number(code.slice(1));
|
||||
return Number.isFinite(n) ? String.fromCodePoint(n) : match;
|
||||
}
|
||||
return entities[code.toLowerCase()] ?? match;
|
||||
});
|
||||
}
|
||||
|
||||
export function textOf(html: string): string {
|
||||
return decodeEntities(html.replace(/<[^>]*>/g, ' '))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */
|
||||
export function isChallenge(html: string, status = 200): boolean {
|
||||
if (status === 403 || status === 429 || status === 503) return true;
|
||||
const head = html.slice(0, 20_000);
|
||||
return (
|
||||
/<title>\s*Just a moment/i.test(head) ||
|
||||
head.includes('challenges.cloudflare.com/turnstile') ||
|
||||
head.includes('cf-chl-') ||
|
||||
/<title>\s*Attention Required/i.test(head)
|
||||
);
|
||||
}
|
||||
|
||||
export type ListingRef = {
|
||||
sourceId: string;
|
||||
url: string;
|
||||
offer: Offer;
|
||||
propertyType: PropertyType;
|
||||
citySlug: string;
|
||||
};
|
||||
|
||||
export function listingRef(href: string): ListingRef | null {
|
||||
const match = listingPath.exec(href);
|
||||
if (!match) return null;
|
||||
const [, type, offer, citySlug, shortId, slug] = match;
|
||||
return {
|
||||
sourceId: shortId,
|
||||
url: `${PARARIUS_ORIGIN}/${type}-te-${offer}/${citySlug}/${shortId}/${slug}`,
|
||||
offer: offer === 'koop' ? 'buy' : 'rent',
|
||||
propertyType: typeWords[type] ?? 'other',
|
||||
citySlug
|
||||
};
|
||||
}
|
||||
|
||||
/** Whole euros from "€ 1.558 per maand", "€1,558 per month", "€ 425.000 k.k." */
|
||||
export function parseEuros(value: string | null | undefined): number | null {
|
||||
if (!value) return null;
|
||||
const match = /€\s*([\d.,]+)/.exec(value) ?? /([\d][\d.,]*)/.exec(value);
|
||||
if (!match) return null;
|
||||
const digits = match[1].replace(/[.,]\d{2}$/, '').replace(/[.,]/g, '');
|
||||
const n = Number(digits);
|
||||
return digits && Number.isSafeInteger(n) ? n : null;
|
||||
}
|
||||
|
||||
export function parseInteger(value: string | null | undefined): number | null {
|
||||
const match = value ? /(\d[\d.]*)/.exec(value) : null;
|
||||
if (!match) return null;
|
||||
const n = Number(match[1].replace(/\./g, ''));
|
||||
return Number.isSafeInteger(n) ? n : null;
|
||||
}
|
||||
|
||||
/** "2801 BC Gouda (Centrum)" → postal code, city and neighbourhood. */
|
||||
export function parseLocation(value: string | null | undefined) {
|
||||
const out = {
|
||||
postalCode: null as string | null,
|
||||
city: null as string | null,
|
||||
neighbourhood: null as string | null
|
||||
};
|
||||
if (!value) return out;
|
||||
let rest = value.trim();
|
||||
const hood = /\(([^)]+)\)\s*$/.exec(rest);
|
||||
if (hood) {
|
||||
out.neighbourhood = hood[1].trim();
|
||||
rest = rest.slice(0, hood.index).trim();
|
||||
}
|
||||
const postal = /^(\d{4})\s?([A-Z]{2})\b/.exec(rest);
|
||||
if (postal) {
|
||||
out.postalCode = `${postal[1]} ${postal[2]}`;
|
||||
rest = rest.slice(postal[0].length).trim();
|
||||
}
|
||||
if (rest) out.city = rest;
|
||||
return out;
|
||||
}
|
||||
|
||||
function classText(html: string, className: string): string | null {
|
||||
const re = new RegExp(
|
||||
`<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)</\\1>`,
|
||||
'i'
|
||||
);
|
||||
const match = re.exec(html);
|
||||
if (!match) return null;
|
||||
const value = textOf(match[2]);
|
||||
return value || null;
|
||||
}
|
||||
|
||||
function firstImage(html: string): string | null {
|
||||
const tag = /<img\b[^>]*>/i.exec(html)?.[0];
|
||||
if (!tag) return null;
|
||||
const src =
|
||||
/\bsrc="(https:[^"]+)"/i.exec(tag)?.[1] ??
|
||||
/\bsrcset="(https:[^"\s,]+)/i.exec(tag)?.[1] ??
|
||||
/\bdata-src="(https:[^"]+)"/i.exec(tag)?.[1];
|
||||
return src ? decodeEntities(src) : null;
|
||||
}
|
||||
|
||||
export type SearchPage = {
|
||||
cards: ListingCard[];
|
||||
nextPageUrl: string | null;
|
||||
totalResults: number | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a search results page (`/huurwoningen/gouda/1300-1700`).
|
||||
* Splits on the listing card wrappers when present; otherwise falls back to
|
||||
* one chunk per unique listing link so a markup change still yields ids.
|
||||
*/
|
||||
export function parseSearchPage(html: string, pageUrl: string): SearchPage {
|
||||
const chunks: string[] = [];
|
||||
const marker =
|
||||
/<(section|li|article|div)\b[^>]*class="[^"]*\blisting-search-item\b(?!__)[^"]*"/gi;
|
||||
const starts = [...html.matchAll(marker)].map((m) => m.index!);
|
||||
if (starts.length) {
|
||||
for (let i = 0; i < starts.length; i++) {
|
||||
chunks.push(html.slice(starts[i], starts[i + 1] ?? starts[i] + 20_000));
|
||||
}
|
||||
} else {
|
||||
const links = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/gi;
|
||||
const seen = new Set<string>();
|
||||
for (const m of html.matchAll(links)) {
|
||||
const ref = listingRef(m[1]);
|
||||
if (!ref || seen.has(ref.sourceId)) continue;
|
||||
seen.add(ref.sourceId);
|
||||
chunks.push(html.slice(Math.max(0, m.index! - 3000), m.index! + 6000));
|
||||
}
|
||||
}
|
||||
|
||||
const cards: ListingCard[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const chunk of chunks) {
|
||||
const href = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/i.exec(chunk)?.[1];
|
||||
const ref = href ? listingRef(decodeEntities(href)) : null;
|
||||
if (!ref || seen.has(ref.sourceId)) continue;
|
||||
seen.add(ref.sourceId);
|
||||
|
||||
const titleLink = /<a\b[^>]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk);
|
||||
const title =
|
||||
(titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title');
|
||||
const location = parseLocation(
|
||||
classText(chunk, 'listing-search-item__sub-title') ??
|
||||
classText(chunk, 'listing-search-item__location')
|
||||
);
|
||||
const priceText =
|
||||
classText(chunk, 'listing-search-item__price') ?? /€\s*[\d.,]+[^<]{0,30}/.exec(chunk)?.[0];
|
||||
const area =
|
||||
classText(chunk, 'illustrated-features__item--surface-area') ??
|
||||
/(\d+)\s*m(?:²|²|2)/i.exec(chunk)?.[0];
|
||||
const rooms = classText(chunk, 'illustrated-features__item--number-of-rooms');
|
||||
const agentBlock = classText(chunk, 'listing-search-item__info');
|
||||
|
||||
cards.push({
|
||||
sourceId: ref.sourceId,
|
||||
url: ref.url,
|
||||
offer: ref.offer,
|
||||
propertyType: ref.propertyType,
|
||||
title: title?.slice(0, 200) ?? null,
|
||||
postalCode: location.postalCode,
|
||||
city: location.city ?? cityFromSlug(ref.citySlug),
|
||||
neighbourhood: location.neighbourhood,
|
||||
priceEur: parseEuros(priceText),
|
||||
areaM2: parseInteger(area),
|
||||
rooms: parseInteger(rooms),
|
||||
interior: classText(chunk, 'illustrated-features__item--interior'),
|
||||
agent: agentBlock?.slice(0, 160) ?? null,
|
||||
imageUrl: firstImage(chunk)
|
||||
});
|
||||
}
|
||||
|
||||
const nextHref =
|
||||
/<a\b[^>]*class="[^"]*pagination__link--next[^"]*"[^>]*href="([^"]+)"/i.exec(html)?.[1] ??
|
||||
/<a\b[^>]*href="([^"]+)"[^>]*class="[^"]*pagination__link--next[^"]*"/i.exec(html)?.[1] ??
|
||||
/<link\b[^>]*rel="next"[^>]*href="([^"]+)"/i.exec(html)?.[1];
|
||||
const total =
|
||||
classText(html, 'search-list-header__count') ??
|
||||
/(\d[\d.]*)\s+(?:huurwoningen|koopwoningen|woningen|results|resultaten)/i.exec(
|
||||
textOf(html.slice(0, 200_000))
|
||||
)?.[1];
|
||||
|
||||
return {
|
||||
cards,
|
||||
nextPageUrl: nextHref ? new URL(decodeEntities(nextHref), pageUrl).toString() : null,
|
||||
totalResults: parseInteger(total)
|
||||
};
|
||||
}
|
||||
|
||||
/** `/huurwoningen/gouda/1300-1700` → page N url (`.../page-2`). */
|
||||
export function searchPageUrl(base: string, page: number): string {
|
||||
const url = new URL(base);
|
||||
url.pathname = url.pathname.replace(/\/page-\d+\/?$/, '').replace(/\/$/, '');
|
||||
if (page > 1) url.pathname += `/page-${page}`;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export function cityFromSlug(slug: string): string {
|
||||
return slug
|
||||
.split('-')
|
||||
.map((part) =>
|
||||
part === 'aan' || part === 'den' || part === 'de'
|
||||
? part
|
||||
: part[0]?.toUpperCase() + part.slice(1)
|
||||
)
|
||||
.join(' ')
|
||||
.replace(/^\w/, (c) => c.toUpperCase());
|
||||
}
|
||||
|
||||
function jsonBlocks(html: string): unknown[] {
|
||||
const out: unknown[] = [];
|
||||
for (const m of html.matchAll(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) {
|
||||
try {
|
||||
out.push(JSON.parse(m[1]));
|
||||
} catch {
|
||||
// Ignore malformed blocks; other sources may still have the data.
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
type LdListing = {
|
||||
'@type'?: string | string[];
|
||||
name?: string;
|
||||
description?: string;
|
||||
image?: string | string[];
|
||||
address?: {
|
||||
streetAddress?: string;
|
||||
addressLocality?: string;
|
||||
addressRegion?: string;
|
||||
postalCode?: string;
|
||||
};
|
||||
numberOfRooms?: { value?: number }[] | { value?: number } | number;
|
||||
floorSize?: { value?: number };
|
||||
yearBuilt?: number;
|
||||
numberOfBedrooms?: number;
|
||||
numberOfBathroomsTotal?: number;
|
||||
datePosted?: string;
|
||||
offers?: { price?: number; url?: string };
|
||||
};
|
||||
|
||||
function ldListing(html: string): LdListing | null {
|
||||
for (const block of jsonBlocks(html)) {
|
||||
const items = Array.isArray(block) ? block : [block];
|
||||
for (const item of items) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const type = (item as LdListing)['@type'];
|
||||
const types = Array.isArray(type) ? type : [type];
|
||||
if (types.includes('RealEstateListing')) return item as LdListing;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The flat `odp.*` analytics object Pararius embeds on listing pages. */
|
||||
export function analyticsBlob(html: string): Record<string, string> {
|
||||
const candidates = [html, decodeEntities(html)];
|
||||
for (const source of candidates) {
|
||||
const match = /\{[^{}]*"odp\.listing_id"[^{}]*\}/.exec(source);
|
||||
if (!match) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(match[0]) as Record<string, unknown>;
|
||||
const out: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(parsed)) {
|
||||
if (key.startsWith('odp.') && (typeof value === 'string' || typeof value === 'number'))
|
||||
out[key.slice(4)] = String(value);
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
// Fall through to the next candidate.
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/** The "Kenmerken" definition list: `Borg → € 1.600`, `Interieur → Gestoffeerd`. */
|
||||
export function featureList(html: string): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
const re =
|
||||
/<dt\b[^>]*class="[^"]*listing-features__term[^"]*"[^>]*>([\s\S]*?)<\/dt>\s*<dd\b[^>]*>([\s\S]*?)<\/dd>/gi;
|
||||
for (const m of html.matchAll(re)) {
|
||||
const key = textOf(m[1]).slice(0, 80);
|
||||
const value = textOf(m[2].replace(/<[^>]*class="[^"]*tooltip[^"]*"[\s\S]*?<\/[a-z]+>/gi, ''))
|
||||
.replace(/\s*Meer info$/i, '')
|
||||
.slice(0, 500);
|
||||
if (key && value && !(key in out)) out[key] = value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const interiorWords: Record<string, string> = {
|
||||
shell: 'kaal',
|
||||
upholstered: 'gestoffeerd',
|
||||
furnished: 'gemeubileerd'
|
||||
};
|
||||
|
||||
const firstNumber = (value: LdListing['numberOfRooms']): number | null => {
|
||||
if (typeof value === 'number') return value;
|
||||
const item = Array.isArray(value) ? value[0] : value;
|
||||
return typeof item?.value === 'number' ? item.value : null;
|
||||
};
|
||||
|
||||
/** Parse a listing page. Returns null when the page is not a listing. */
|
||||
export function parseDetailPage(html: string, fallbackUrl?: string): ListingDetail | null {
|
||||
const canonical =
|
||||
/<link\b[^>]*rel="canonical"[^>]*href="([^"]+)"/i.exec(html)?.[1] ??
|
||||
/<link\b[^>]*href="([^"]+)"[^>]*rel="canonical"/i.exec(html)?.[1];
|
||||
const ref = listingRef(canonical ?? '') ?? listingRef(fallbackUrl ?? '');
|
||||
if (!ref) return null;
|
||||
|
||||
const ld = ldListing(html);
|
||||
const odp = analyticsBlob(html);
|
||||
const features = featureList(html);
|
||||
const summaryTitle = classText(html, 'listing-detail-summary__title');
|
||||
const location = parseLocation(classText(html, 'listing-detail-summary__location'));
|
||||
|
||||
const lat = Number(
|
||||
/data-latitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"latitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1]
|
||||
);
|
||||
const lng = Number(
|
||||
/data-longitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"longitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1]
|
||||
);
|
||||
|
||||
const offeredAt =
|
||||
odp.offered_since && !Number.isNaN(Date.parse(odp.offered_since))
|
||||
? new Date(odp.offered_since).toISOString()
|
||||
: ld?.datePosted
|
||||
? new Date(`${ld.datePosted}T00:00:00Z`).toISOString()
|
||||
: null;
|
||||
|
||||
const price =
|
||||
(odp.price_pcm ? Number(odp.price_pcm) : null) ??
|
||||
(typeof ld?.offers?.price === 'number' ? ld.offers.price : null) ??
|
||||
parseEuros(features['Huurprijs'] ?? features['Vraagprijs'] ?? features['Koopprijs']);
|
||||
|
||||
const image = Array.isArray(ld?.image) ? ld.image[0] : ld?.image;
|
||||
const typeFromOdp = odp.property ? typeWords[odp.property] : undefined;
|
||||
const interior =
|
||||
(odp.interior ? (interiorWords[odp.interior] ?? odp.interior) : null) ??
|
||||
features['Interieur']?.toLowerCase() ??
|
||||
null;
|
||||
|
||||
const street = ld?.address?.streetAddress ?? null;
|
||||
const title =
|
||||
summaryTitle?.replace(/^Te (huur|koop):\s*/i, '') ?? (ld?.name ? `${ld.name}` : null);
|
||||
|
||||
return {
|
||||
sourceId: ref.sourceId,
|
||||
url: ref.url,
|
||||
offer: odp.offer === 'buy' || ref.offer === 'buy' ? 'buy' : 'rent',
|
||||
propertyType: typeFromOdp ?? ref.propertyType,
|
||||
externalId: odp.listing_id ?? null,
|
||||
title: title?.slice(0, 200) ?? null,
|
||||
street,
|
||||
postalCode: ld?.address?.postalCode
|
||||
? ld.address.postalCode.replace(/^(\d{4})\s?([A-Z]{2})$/, '$1 $2')
|
||||
: location.postalCode,
|
||||
city: ld?.address?.addressLocality ?? location.city ?? cityFromSlug(ref.citySlug),
|
||||
neighbourhood: ld?.address?.addressRegion ?? location.neighbourhood,
|
||||
priceEur: Number.isFinite(price) ? price : null,
|
||||
areaM2:
|
||||
parseInteger(odp.floor_space) ??
|
||||
(typeof ld?.floorSize?.value === 'number' ? ld.floorSize.value : null) ??
|
||||
parseInteger(features['Woonoppervlakte']),
|
||||
rooms:
|
||||
parseInteger(odp.rooms) ??
|
||||
firstNumber(ld?.numberOfRooms) ??
|
||||
parseInteger(features['Aantal kamers']),
|
||||
bedrooms:
|
||||
parseInteger(odp.bedrooms) ??
|
||||
ld?.numberOfBedrooms ??
|
||||
parseInteger(features['Aantal slaapkamers']),
|
||||
bathrooms: ld?.numberOfBathroomsTotal ?? parseInteger(features['Aantal badkamers']),
|
||||
constructionYear:
|
||||
parseInteger(odp.construction_year) ?? ld?.yearBuilt ?? parseInteger(features['Bouwjaar']),
|
||||
energyLabel: odp.energy_rating ?? features['Energielabel'] ?? null,
|
||||
interior,
|
||||
depositEur: parseEuros(features['Borg']),
|
||||
availableFrom: features['Beschikbaar'] ?? null,
|
||||
contractType: features['Contractvorm'] ?? features['Huurperiode'] ?? null,
|
||||
agent: classText(html, 'agent-card__title'),
|
||||
imageUrl: image ?? null,
|
||||
lat: Number.isFinite(lat) && lat !== 0 ? lat : null,
|
||||
lng: Number.isFinite(lng) && lng !== 0 ? lng : null,
|
||||
offeredAt,
|
||||
description: ld?.description?.slice(0, 6000) ?? null,
|
||||
features
|
||||
};
|
||||
}
|
||||
70
src/lib/house-tracker/schedule.ts
Normal file
70
src/lib/house-tracker/schedule.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
// When the home worker should poll next. Pure so it can be tested.
|
||||
//
|
||||
// Listings near Gouda were all published on weekdays between 07:30 and 17:30,
|
||||
// so the fast lane polls often during office hours and backs off otherwise.
|
||||
|
||||
export type ScheduleOptions = {
|
||||
/** Weekday office-hours interval, in minutes. */
|
||||
busyMinutes: [number, number];
|
||||
/** Evenings, nights and weekends. */
|
||||
quietMinutes: [number, number];
|
||||
busyStartHour: number;
|
||||
busyEndHour: number;
|
||||
timeZone: string;
|
||||
};
|
||||
|
||||
export const fastLaneSchedule: ScheduleOptions = {
|
||||
busyMinutes: [3, 5],
|
||||
quietMinutes: [30, 60],
|
||||
busyStartHour: 7,
|
||||
busyEndHour: 18,
|
||||
timeZone: 'Europe/Amsterdam'
|
||||
};
|
||||
|
||||
export function localClock(date: Date, timeZone: string) {
|
||||
const parts = new Intl.DateTimeFormat('en-GB', {
|
||||
timeZone,
|
||||
weekday: 'short',
|
||||
hour: 'numeric',
|
||||
minute: 'numeric',
|
||||
hourCycle: 'h23'
|
||||
}).formatToParts(date);
|
||||
const get = (type: string) => parts.find((part) => part.type === type)?.value ?? '';
|
||||
return {
|
||||
weekday: get('weekday'),
|
||||
hour: Number(get('hour')),
|
||||
minute: Number(get('minute'))
|
||||
};
|
||||
}
|
||||
|
||||
export function isBusy(date: Date, options: ScheduleOptions = fastLaneSchedule): boolean {
|
||||
const { weekday, hour } = localClock(date, options.timeZone);
|
||||
if (weekday === 'Sat' || weekday === 'Sun') return false;
|
||||
return hour >= options.busyStartHour && hour < options.busyEndHour;
|
||||
}
|
||||
|
||||
/** Milliseconds until the next poll, with jitter so requests never look clockwork. */
|
||||
export function nextDelayMs(
|
||||
now: Date,
|
||||
options: ScheduleOptions = fastLaneSchedule,
|
||||
random: () => number = Math.random
|
||||
): number {
|
||||
const [min, max] = isBusy(now, options) ? options.busyMinutes : options.quietMinutes;
|
||||
let delay = (min + (max - min) * random()) * 60_000;
|
||||
// Don't sleep through the start of office hours.
|
||||
if (!isBusy(now, options)) {
|
||||
for (let step = 60_000; step < delay; step += 60_000) {
|
||||
if (isBusy(new Date(now.getTime() + step), options)) {
|
||||
delay = step + random() * 60_000;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Math.round(delay);
|
||||
}
|
||||
|
||||
/** Exponential backoff after challenges or errors: 15 min, 30, 60 ... capped at 6 h. */
|
||||
export function backoffMs(failures: number, random: () => number = Math.random): number {
|
||||
const base = 15 * 60_000 * 2 ** Math.max(0, failures - 1);
|
||||
return Math.round(Math.min(base, 6 * 60 * 60_000) * (0.85 + 0.3 * random()));
|
||||
}
|
||||
118
src/lib/house-tracker/types.ts
Normal file
118
src/lib/house-tracker/types.ts
Normal file
@@ -0,0 +1,118 @@
|
||||
// Shared between the home worker (scripts/house-worker.ts), the ingest API and
|
||||
// the page. Keep this file free of `$lib` / `$env` imports: the worker runs it
|
||||
// directly with bun, outside SvelteKit.
|
||||
import { z } from 'zod';
|
||||
|
||||
export const offers = ['rent', 'buy'] as const;
|
||||
export type Offer = (typeof offers)[number];
|
||||
|
||||
export const propertyTypes = ['apartment', 'house', 'room', 'studio', 'other'] as const;
|
||||
export type PropertyType = (typeof propertyTypes)[number];
|
||||
|
||||
const text = (max: number) => z.string().trim().max(max);
|
||||
const optionalText = (max: number) => text(max).nullish();
|
||||
const count = z.number().int().min(0).max(10_000).nullish();
|
||||
|
||||
// What a search results page tells us about a listing. This alone is enough to
|
||||
// send an alert, so the fast lane never waits for the detail page.
|
||||
export const listingCardSchema = z.object({
|
||||
sourceId: text(80).min(1),
|
||||
url: z.url().max(500),
|
||||
offer: z.enum(offers),
|
||||
propertyType: z.enum(propertyTypes),
|
||||
title: optionalText(200),
|
||||
postalCode: optionalText(10),
|
||||
city: optionalText(80),
|
||||
neighbourhood: optionalText(120),
|
||||
priceEur: z.number().int().min(0).max(100_000_000).nullish(),
|
||||
areaM2: count,
|
||||
rooms: count,
|
||||
interior: optionalText(40),
|
||||
agent: optionalText(160),
|
||||
imageUrl: z.url().max(1000).nullish()
|
||||
});
|
||||
export type ListingCard = z.infer<typeof listingCardSchema>;
|
||||
|
||||
// Everything else we learn from the listing page itself.
|
||||
export const listingDetailSchema = listingCardSchema.extend({
|
||||
externalId: optionalText(80),
|
||||
street: optionalText(200),
|
||||
bedrooms: count,
|
||||
bathrooms: count,
|
||||
constructionYear: z.number().int().min(1000).max(2100).nullish(),
|
||||
energyLabel: optionalText(10),
|
||||
depositEur: z.number().int().min(0).max(10_000_000).nullish(),
|
||||
availableFrom: optionalText(80),
|
||||
contractType: optionalText(80),
|
||||
lat: z.number().min(-90).max(90).nullish(),
|
||||
lng: z.number().min(-180).max(180).nullish(),
|
||||
offeredAt: z.iso.datetime({ offset: true }).nullish(),
|
||||
description: optionalText(6000),
|
||||
features: z.record(z.string().max(80), z.string().max(500)).optional()
|
||||
});
|
||||
export type ListingDetail = z.infer<typeof listingDetailSchema>;
|
||||
|
||||
export const listingStatuses = [
|
||||
'interested',
|
||||
'contacted',
|
||||
'viewing',
|
||||
'applied',
|
||||
'rejected'
|
||||
] as const;
|
||||
export type ListingStatus = (typeof listingStatuses)[number];
|
||||
|
||||
export const lanes = ['fast', 'slow'] as const;
|
||||
export type Lane = (typeof lanes)[number];
|
||||
|
||||
// One poll of one search page.
|
||||
export const syncRequestSchema = z.object({
|
||||
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
|
||||
lane: z.enum(lanes),
|
||||
searchUrl: z.url().max(500),
|
||||
startedAt: z.iso.datetime({ offset: true }),
|
||||
cards: z.array(listingCardSchema).max(200),
|
||||
// Set on the last page of a full, unfiltered sweep of one city. Listings in
|
||||
// that scope that were not seen during the sweep are then marked removed.
|
||||
completedSweep: z
|
||||
.object({
|
||||
offer: z.enum(offers),
|
||||
city: text(80).min(1),
|
||||
sweepStartedAt: z.iso.datetime({ offset: true })
|
||||
})
|
||||
.nullish()
|
||||
});
|
||||
export type SyncRequest = z.infer<typeof syncRequestSchema>;
|
||||
|
||||
export const detailsRequestSchema = z.object({
|
||||
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
|
||||
details: z.array(listingDetailSchema).max(50)
|
||||
});
|
||||
export type DetailsRequest = z.infer<typeof detailsRequestSchema>;
|
||||
|
||||
export const runStatuses = [
|
||||
'ok',
|
||||
'challenged',
|
||||
'http-error',
|
||||
'parse-error',
|
||||
'network-error'
|
||||
] as const;
|
||||
export type RunStatus = (typeof runStatuses)[number];
|
||||
|
||||
export const reportRequestSchema = z.object({
|
||||
source: z.string().regex(/^[a-z0-9-]{1,40}$/),
|
||||
lane: z.enum(lanes),
|
||||
url: z.url().max(500),
|
||||
status: z.enum(runStatuses),
|
||||
httpStatus: z.number().int().min(0).max(999).nullish(),
|
||||
found: z.number().int().min(0).max(10_000).nullish(),
|
||||
durationMs: z.number().int().min(0).max(600_000).nullish(),
|
||||
message: optionalText(500)
|
||||
});
|
||||
export type ReportRequest = z.infer<typeof reportRequestSchema>;
|
||||
|
||||
export type SyncResponse = {
|
||||
newIds: string[];
|
||||
// Listings the server has no detail page for yet, newest first.
|
||||
needDetails: { sourceId: string; url: string }[];
|
||||
alerted: number;
|
||||
};
|
||||
@@ -47,6 +47,12 @@ export const projects = [
|
||||
href: '/projects/money-tracker',
|
||||
external: false
|
||||
},
|
||||
{
|
||||
name: 'house tracker',
|
||||
subtitle: 'Dutch rental and sale listings, with alerts',
|
||||
href: '/projects/house-tracker',
|
||||
external: false
|
||||
},
|
||||
{
|
||||
name: 'cs prices',
|
||||
subtitle: 'Counter-Strike market prices across time',
|
||||
|
||||
128
src/lib/server/db/schema/house-tracker.ts
Normal file
128
src/lib/server/db/schema/house-tracker.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
import {
|
||||
bigint,
|
||||
boolean,
|
||||
check,
|
||||
doublePrecision,
|
||||
index,
|
||||
integer,
|
||||
jsonb,
|
||||
pgTable,
|
||||
text,
|
||||
uniqueIndex
|
||||
} from 'drizzle-orm/pg-core';
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { id, timestamptz } from './helpers';
|
||||
|
||||
// Created by scripts/house-tracker-migration.sql; keep both in sync.
|
||||
export const houseListings = pgTable(
|
||||
'house_listings',
|
||||
{
|
||||
id: id(),
|
||||
source: text('source').notNull(),
|
||||
sourceId: text('source_id').notNull(),
|
||||
externalId: text('external_id'),
|
||||
url: text('url').notNull(),
|
||||
offer: text('offer').notNull(),
|
||||
propertyType: text('property_type').notNull(),
|
||||
title: text('title'),
|
||||
street: text('street'),
|
||||
postalCode: text('postal_code'),
|
||||
city: text('city'),
|
||||
citySlug: text('city_slug'),
|
||||
neighbourhood: text('neighbourhood'),
|
||||
priceEur: integer('price_eur'),
|
||||
areaM2: integer('area_m2'),
|
||||
rooms: integer('rooms'),
|
||||
bedrooms: integer('bedrooms'),
|
||||
bathrooms: integer('bathrooms'),
|
||||
constructionYear: integer('construction_year'),
|
||||
energyLabel: text('energy_label'),
|
||||
interior: text('interior'),
|
||||
depositEur: integer('deposit_eur'),
|
||||
availableFrom: text('available_from'),
|
||||
contractType: text('contract_type'),
|
||||
agent: text('agent'),
|
||||
imageUrl: text('image_url'),
|
||||
description: text('description'),
|
||||
features: jsonb('features').$type<Record<string, string>>().notNull().default({}),
|
||||
lat: doublePrecision('lat'),
|
||||
lng: doublePrecision('lng'),
|
||||
offeredAt: timestamptz('offered_at'),
|
||||
firstSeenAt: timestamptz('first_seen_at').notNull().defaultNow(),
|
||||
firstSeenLane: text('first_seen_lane').notNull().default('slow'),
|
||||
lastSeenAt: timestamptz('last_seen_at').notNull().defaultNow(),
|
||||
detailFetchedAt: timestamptz('detail_fetched_at'),
|
||||
removedAt: timestamptz('removed_at'),
|
||||
alertedAt: timestamptz('alerted_at'),
|
||||
status: text('status'),
|
||||
notes: text('notes')
|
||||
},
|
||||
(table) => [
|
||||
check('house_listings_offer_check', sql`offer IN ('rent', 'buy')`),
|
||||
check('house_listings_price_eur_check', sql`price_eur IS NULL OR price_eur >= 0`),
|
||||
uniqueIndex('house_listings_source_idx').on(table.source, table.sourceId),
|
||||
index('house_listings_active_idx')
|
||||
.on(table.offer, table.firstSeenAt.desc())
|
||||
.where(sql`removed_at IS NULL`),
|
||||
index('house_listings_city_idx').on(table.source, table.offer, table.citySlug),
|
||||
index('house_listings_details_idx')
|
||||
.on(table.firstSeenAt.desc())
|
||||
.where(sql`detail_fetched_at IS NULL`)
|
||||
]
|
||||
);
|
||||
|
||||
export const housePriceHistory = pgTable(
|
||||
'house_price_history',
|
||||
{
|
||||
listingId: integer('listing_id')
|
||||
.notNull()
|
||||
.references(() => houseListings.id, { onDelete: 'cascade' }),
|
||||
observedAt: timestamptz('observed_at').notNull().defaultNow(),
|
||||
priceEur: integer('price_eur').notNull()
|
||||
},
|
||||
(table) => [index('house_price_history_listing_idx').on(table.listingId, table.observedAt)]
|
||||
);
|
||||
|
||||
export const houseSearches = pgTable('house_searches', {
|
||||
searchUrl: text('search_url').primaryKey(),
|
||||
source: text('source').notNull(),
|
||||
lane: text('lane').notNull(),
|
||||
firstSyncedAt: timestamptz('first_synced_at').notNull().defaultNow(),
|
||||
lastSyncedAt: timestamptz('last_synced_at').notNull().defaultNow(),
|
||||
lastFound: integer('last_found').notNull().default(0)
|
||||
});
|
||||
|
||||
export const houseWorkerRuns = pgTable(
|
||||
'house_worker_runs',
|
||||
{
|
||||
id: bigint('id', { mode: 'number' }).primaryKey().generatedAlwaysAsIdentity(),
|
||||
source: text('source').notNull(),
|
||||
lane: text('lane').notNull(),
|
||||
url: text('url').notNull(),
|
||||
status: text('status').notNull(),
|
||||
httpStatus: integer('http_status'),
|
||||
found: integer('found'),
|
||||
durationMs: integer('duration_ms'),
|
||||
message: text('message'),
|
||||
createdAt: timestamptz('created_at').notNull().defaultNow()
|
||||
},
|
||||
(table) => [index('house_worker_runs_created_idx').on(table.createdAt.desc())]
|
||||
);
|
||||
|
||||
export const houseAlertRules = pgTable(
|
||||
'house_alert_rules',
|
||||
{
|
||||
id: id(),
|
||||
name: text('name').notNull(),
|
||||
enabled: boolean('enabled').notNull().default(true),
|
||||
offer: text('offer').notNull().default('rent'),
|
||||
minPrice: integer('min_price'),
|
||||
maxPrice: integer('max_price'),
|
||||
minArea: integer('min_area'),
|
||||
minBedrooms: integer('min_bedrooms'),
|
||||
cities: text('cities').array().notNull().default([]),
|
||||
propertyTypes: text('property_types').array().notNull().default([]),
|
||||
createdAt: timestamptz('created_at').notNull().defaultNow()
|
||||
},
|
||||
() => [check('house_alert_rules_offer_check', sql`offer IN ('rent', 'buy')`)]
|
||||
);
|
||||
@@ -10,3 +10,4 @@ export * from './animaldex';
|
||||
export * from './lists';
|
||||
export * from './money-tracker';
|
||||
export * from './cs-prices';
|
||||
export * from './house-tracker';
|
||||
|
||||
360
src/lib/server/house-tracker.ts
Normal file
360
src/lib/server/house-tracker.ts
Normal file
@@ -0,0 +1,360 @@
|
||||
// House tracker ingest: the home worker POSTs what it scraped, this stores it
|
||||
// and sends alerts. Nothing here talks to the listing sites.
|
||||
import { timingSafeEqual } from 'node:crypto';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { error } from '@sveltejs/kit';
|
||||
import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm';
|
||||
import { db } from '$lib/server/db';
|
||||
import {
|
||||
houseAlertRules,
|
||||
houseListings,
|
||||
housePriceHistory,
|
||||
houseSearches,
|
||||
houseWorkerRuns
|
||||
} from '$lib/server/db/schema';
|
||||
import {
|
||||
alertMessage,
|
||||
matchesRule,
|
||||
type AlertMessage,
|
||||
type AlertRule
|
||||
} from '$lib/house-tracker/alerts';
|
||||
import { listingRef } from '$lib/house-tracker/pararius';
|
||||
import type {
|
||||
DetailsRequest,
|
||||
ListingCard,
|
||||
ReportRequest,
|
||||
SyncRequest,
|
||||
SyncResponse
|
||||
} from '$lib/house-tracker/types';
|
||||
|
||||
export function requireWorker(request: Request) {
|
||||
const expected = env.HOUSE_WORKER_TOKEN ?? '';
|
||||
if (expected.length < 32) error(503, 'HOUSE_WORKER_TOKEN is not configured');
|
||||
const given = (request.headers.get('authorization') ?? '').replace(/^Bearer\s+/i, '');
|
||||
const a = Buffer.from(given);
|
||||
const b = Buffer.from(expected);
|
||||
if (a.length !== b.length || !timingSafeEqual(a, b)) error(401, 'Invalid worker token');
|
||||
}
|
||||
|
||||
function citySlug(card: Pick<ListingCard, 'url'>): string | null {
|
||||
return listingRef(card.url)?.citySlug ?? null;
|
||||
}
|
||||
|
||||
type Tx = Parameters<Parameters<typeof db.transaction>[0]>[0];
|
||||
|
||||
async function recordPrice(
|
||||
tx: Tx,
|
||||
listingId: number,
|
||||
previous: number | null,
|
||||
next: number | null
|
||||
) {
|
||||
if (next == null || next === previous) return;
|
||||
await tx.insert(housePriceHistory).values({ listingId, priceEur: next });
|
||||
}
|
||||
|
||||
export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
|
||||
const sourceIds = input.cards.map((card) => card.sourceId);
|
||||
const toAlert: { id: number; message: AlertMessage }[] = [];
|
||||
const newIds: string[] = [];
|
||||
|
||||
const rules = (await db
|
||||
.select()
|
||||
.from(houseAlertRules)
|
||||
.where(eq(houseAlertRules.enabled, true))) as AlertRule[];
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
const [search] = await tx
|
||||
.select({ firstSyncedAt: houseSearches.firstSyncedAt })
|
||||
.from(houseSearches)
|
||||
.where(eq(houseSearches.searchUrl, input.searchUrl));
|
||||
const seeding = !search;
|
||||
await tx
|
||||
.insert(houseSearches)
|
||||
.values({
|
||||
searchUrl: input.searchUrl,
|
||||
source: input.source,
|
||||
lane: input.lane,
|
||||
lastFound: input.cards.length
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: houseSearches.searchUrl,
|
||||
set: { lastSyncedAt: sql`now()`, lastFound: input.cards.length, lane: input.lane }
|
||||
});
|
||||
|
||||
const existing = sourceIds.length
|
||||
? await tx
|
||||
.select({
|
||||
id: houseListings.id,
|
||||
sourceId: houseListings.sourceId,
|
||||
priceEur: houseListings.priceEur
|
||||
})
|
||||
.from(houseListings)
|
||||
.where(
|
||||
and(eq(houseListings.source, input.source), inArray(houseListings.sourceId, sourceIds))
|
||||
)
|
||||
: [];
|
||||
const known = new Map(existing.map((row) => [row.sourceId, row]));
|
||||
|
||||
for (const card of input.cards) {
|
||||
const previous = known.get(card.sourceId);
|
||||
if (previous) {
|
||||
await tx
|
||||
.update(houseListings)
|
||||
.set({
|
||||
lastSeenAt: sql`now()`,
|
||||
removedAt: null,
|
||||
url: card.url,
|
||||
priceEur: card.priceEur ?? previous.priceEur,
|
||||
title: sql`coalesce(${houseListings.title}, ${card.title ?? null})`,
|
||||
postalCode: sql`coalesce(${houseListings.postalCode}, ${card.postalCode ?? null})`,
|
||||
neighbourhood: sql`coalesce(${houseListings.neighbourhood}, ${card.neighbourhood ?? null})`,
|
||||
areaM2: sql`coalesce(${houseListings.areaM2}, ${card.areaM2 ?? null}::int)`,
|
||||
rooms: sql`coalesce(${houseListings.rooms}, ${card.rooms ?? null}::int)`,
|
||||
agent: sql`coalesce(${houseListings.agent}, ${card.agent ?? null})`,
|
||||
imageUrl: sql`coalesce(${houseListings.imageUrl}, ${card.imageUrl ?? null})`
|
||||
})
|
||||
.where(eq(houseListings.id, previous.id));
|
||||
await recordPrice(tx, previous.id, previous.priceEur, card.priceEur ?? null);
|
||||
continue;
|
||||
}
|
||||
|
||||
const [row] = await tx
|
||||
.insert(houseListings)
|
||||
.values({
|
||||
source: input.source,
|
||||
sourceId: card.sourceId,
|
||||
url: card.url,
|
||||
offer: card.offer,
|
||||
propertyType: card.propertyType,
|
||||
title: card.title,
|
||||
postalCode: card.postalCode,
|
||||
city: card.city,
|
||||
citySlug: citySlug(card),
|
||||
neighbourhood: card.neighbourhood,
|
||||
priceEur: card.priceEur,
|
||||
areaM2: card.areaM2,
|
||||
rooms: card.rooms,
|
||||
interior: card.interior,
|
||||
agent: card.agent,
|
||||
imageUrl: card.imageUrl,
|
||||
// Seeded listings were already online; their detection delay means nothing.
|
||||
firstSeenLane: seeding ? 'seed' : input.lane
|
||||
})
|
||||
.onConflictDoNothing()
|
||||
.returning({ id: houseListings.id });
|
||||
if (!row) continue;
|
||||
newIds.push(card.sourceId);
|
||||
await recordPrice(tx, row.id, null, card.priceEur ?? null);
|
||||
if (seeding) continue;
|
||||
const listing = { ...card, citySlug: citySlug(card), source: input.source };
|
||||
const rule = rules.find((candidate) => matchesRule(listing, candidate));
|
||||
if (rule) toAlert.push({ id: row.id, message: alertMessage(listing, rule) });
|
||||
}
|
||||
|
||||
if (input.completedSweep) {
|
||||
await tx
|
||||
.update(houseListings)
|
||||
.set({ removedAt: sql`now()` })
|
||||
.where(
|
||||
and(
|
||||
eq(houseListings.source, input.source),
|
||||
eq(houseListings.offer, input.completedSweep.offer),
|
||||
eq(houseListings.citySlug, input.completedSweep.city),
|
||||
isNull(houseListings.removedAt),
|
||||
lt(houseListings.lastSeenAt, new Date(input.completedSweep.sweepStartedAt))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
await tx.insert(houseWorkerRuns).values({
|
||||
source: input.source,
|
||||
lane: input.lane,
|
||||
url: input.searchUrl,
|
||||
status: 'ok',
|
||||
found: input.cards.length,
|
||||
durationMs: Math.max(0, Date.now() - Date.parse(input.startedAt)),
|
||||
message: seeding ? 'first sync: seeded without alerts' : null
|
||||
});
|
||||
});
|
||||
|
||||
let alerted = 0;
|
||||
for (const { id, message } of toAlert) {
|
||||
if (await sendAlert(message)) {
|
||||
alerted++;
|
||||
await db
|
||||
.update(houseListings)
|
||||
.set({ alertedAt: sql`now()` })
|
||||
.where(eq(houseListings.id, id));
|
||||
}
|
||||
}
|
||||
|
||||
// Fast lane: detail pages for this search first. Otherwise slowly backfill.
|
||||
const pending = await db
|
||||
.select({ sourceId: houseListings.sourceId, url: houseListings.url })
|
||||
.from(houseListings)
|
||||
.where(
|
||||
and(
|
||||
eq(houseListings.source, input.source),
|
||||
isNull(houseListings.detailFetchedAt),
|
||||
isNull(houseListings.removedAt),
|
||||
sourceIds.length && input.lane === 'fast'
|
||||
? inArray(houseListings.sourceId, sourceIds)
|
||||
: undefined
|
||||
)
|
||||
)
|
||||
.orderBy(desc(houseListings.firstSeenAt))
|
||||
.limit(input.lane === 'fast' ? 10 : 5);
|
||||
|
||||
return { newIds, needDetails: pending, alerted };
|
||||
}
|
||||
|
||||
export async function saveDetails(input: DetailsRequest) {
|
||||
let saved = 0;
|
||||
await db.transaction(async (tx) => {
|
||||
for (const detail of input.details) {
|
||||
const [previous] = await tx
|
||||
.select({ id: houseListings.id, priceEur: houseListings.priceEur })
|
||||
.from(houseListings)
|
||||
.where(
|
||||
and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId))
|
||||
);
|
||||
const values = {
|
||||
externalId: detail.externalId,
|
||||
url: detail.url,
|
||||
offer: detail.offer,
|
||||
propertyType: detail.propertyType,
|
||||
title: detail.title,
|
||||
street: detail.street,
|
||||
postalCode: detail.postalCode,
|
||||
city: detail.city,
|
||||
citySlug: citySlug(detail),
|
||||
neighbourhood: detail.neighbourhood,
|
||||
priceEur: detail.priceEur,
|
||||
areaM2: detail.areaM2,
|
||||
rooms: detail.rooms,
|
||||
bedrooms: detail.bedrooms,
|
||||
bathrooms: detail.bathrooms,
|
||||
constructionYear: detail.constructionYear,
|
||||
energyLabel: detail.energyLabel,
|
||||
interior: detail.interior,
|
||||
depositEur: detail.depositEur,
|
||||
availableFrom: detail.availableFrom,
|
||||
contractType: detail.contractType,
|
||||
agent: detail.agent,
|
||||
imageUrl: detail.imageUrl,
|
||||
description: detail.description,
|
||||
features: detail.features ?? {},
|
||||
lat: detail.lat,
|
||||
lng: detail.lng,
|
||||
offeredAt: detail.offeredAt ? new Date(detail.offeredAt) : null,
|
||||
detailFetchedAt: sql`now()`,
|
||||
lastSeenAt: sql`now()`
|
||||
};
|
||||
if (previous) {
|
||||
await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id));
|
||||
await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null);
|
||||
} else {
|
||||
const [row] = await tx
|
||||
.insert(houseListings)
|
||||
.values({ ...values, source: input.source, sourceId: detail.sourceId })
|
||||
.returning({ id: houseListings.id });
|
||||
await recordPrice(tx, row.id, null, detail.priceEur ?? null);
|
||||
}
|
||||
saved++;
|
||||
}
|
||||
});
|
||||
return { saved };
|
||||
}
|
||||
|
||||
export async function recordReport(input: ReportRequest) {
|
||||
// One "challenged" notification per 6 hours is plenty.
|
||||
const [recent] =
|
||||
input.status === 'challenged'
|
||||
? await db
|
||||
.select({ id: houseWorkerRuns.id })
|
||||
.from(houseWorkerRuns)
|
||||
.where(
|
||||
and(
|
||||
eq(houseWorkerRuns.status, 'challenged'),
|
||||
sql`${houseWorkerRuns.createdAt} > now() - interval '6 hours'`
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
: [];
|
||||
await db.insert(houseWorkerRuns).values({
|
||||
source: input.source,
|
||||
lane: input.lane,
|
||||
url: input.url,
|
||||
status: input.status,
|
||||
httpStatus: input.httpStatus,
|
||||
found: input.found,
|
||||
durationMs: input.durationMs,
|
||||
message: input.message
|
||||
});
|
||||
// Keep a month of history; runs are only for spotting trouble.
|
||||
await db
|
||||
.delete(houseWorkerRuns)
|
||||
.where(lt(houseWorkerRuns.createdAt, sql`now() - interval '30 days'`));
|
||||
if (input.status === 'challenged' && !recent) {
|
||||
await sendAlert({
|
||||
title: '⚠️ House worker was challenged',
|
||||
body: `${input.source} answered ${input.httpStatus ?? '?'} on ${input.url}. The worker is backing off.`,
|
||||
url: input.url
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Discord webhook and/or ntfy. Returns true when at least one channel accepted it. */
|
||||
export async function sendAlert(message: AlertMessage): Promise<boolean> {
|
||||
const jobs: Promise<boolean>[] = [];
|
||||
if (env.HOUSE_DISCORD_WEBHOOK_URL) {
|
||||
jobs.push(
|
||||
fetch(env.HOUSE_DISCORD_WEBHOOK_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
username: 'house tracker',
|
||||
embeds: [
|
||||
{
|
||||
title: message.title.slice(0, 256),
|
||||
url: message.url,
|
||||
description: message.body.slice(0, 2000),
|
||||
color: 0x2f9e44
|
||||
}
|
||||
]
|
||||
}),
|
||||
signal: AbortSignal.timeout(8000)
|
||||
}).then(
|
||||
(response) => response.ok,
|
||||
() => false
|
||||
)
|
||||
);
|
||||
}
|
||||
if (env.HOUSE_NTFY_URL) {
|
||||
const url = new URL(env.HOUSE_NTFY_URL);
|
||||
const topic = url.pathname.replace(/^\/+|\/+$/g, '');
|
||||
jobs.push(
|
||||
fetch(url.origin, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(env.HOUSE_NTFY_TOKEN ? { authorization: `Bearer ${env.HOUSE_NTFY_TOKEN}` } : {})
|
||||
},
|
||||
body: JSON.stringify({
|
||||
topic,
|
||||
title: message.title,
|
||||
message: message.body,
|
||||
click: message.url,
|
||||
priority: 4,
|
||||
tags: ['house']
|
||||
}),
|
||||
signal: AbortSignal.timeout(8000)
|
||||
}).then(
|
||||
(response) => response.ok,
|
||||
() => false
|
||||
)
|
||||
);
|
||||
}
|
||||
if (!jobs.length) return false;
|
||||
return (await Promise.all(jobs)).some(Boolean);
|
||||
}
|
||||
235
src/routes/projects/house-tracker/+page.server.ts
Normal file
235
src/routes/projects/house-tracker/+page.server.ts
Normal file
@@ -0,0 +1,235 @@
|
||||
import { fail, redirect } from '@sveltejs/kit';
|
||||
import { and, asc, desc, eq, gte, ilike, isNull, lte, or, sql, type SQL } from 'drizzle-orm';
|
||||
import { db } from '$lib/server/db';
|
||||
import { houseAlertRules, houseListings, houseWorkerRuns } from '$lib/server/db/schema';
|
||||
import { requireAdmin } from '$lib/server/authorization';
|
||||
import { sendAlert } from '$lib/server/house-tracker';
|
||||
import { normalizeCity } from '$lib/house-tracker/alerts';
|
||||
import { listingStatuses, propertyTypes } from '$lib/house-tracker/types';
|
||||
import type { Actions, PageServerLoad } from './$types';
|
||||
|
||||
const int = (value: string | null) => {
|
||||
const n = Number(value);
|
||||
return value && Number.isInteger(n) && n >= 0 && n < 100_000_000 ? n : null;
|
||||
};
|
||||
|
||||
export const load: PageServerLoad = async (event) => {
|
||||
if (!event.locals.user) redirect(303, '/login?returnTo=/projects/house-tracker');
|
||||
requireAdmin(event);
|
||||
const params = event.url.searchParams;
|
||||
const offer = params.get('offer') === 'buy' ? 'buy' : 'rent';
|
||||
const city = (params.get('city') ?? '').toLowerCase().replace(/[^a-z0-9-]/g, '');
|
||||
const min = int(params.get('min'));
|
||||
const max = int(params.get('max'));
|
||||
const showRemoved = params.get('removed') === '1';
|
||||
const status = params.get('status') ?? '';
|
||||
const q = (params.get('q') ?? '').trim().slice(0, 80);
|
||||
const sort = ['new', 'price', 'ppm2', 'area'].includes(params.get('sort') ?? '')
|
||||
? (params.get('sort') as string)
|
||||
: 'new';
|
||||
|
||||
const where: (SQL | undefined)[] = [eq(houseListings.offer, offer)];
|
||||
if (city) where.push(eq(houseListings.citySlug, city));
|
||||
if (min != null) where.push(gte(houseListings.priceEur, min));
|
||||
if (max != null) where.push(lte(houseListings.priceEur, max));
|
||||
if (!showRemoved) where.push(isNull(houseListings.removedAt));
|
||||
if (status === 'any') where.push(sql`${houseListings.status} IS NOT NULL`);
|
||||
else if ((listingStatuses as readonly string[]).includes(status))
|
||||
where.push(eq(houseListings.status, status));
|
||||
if (q) {
|
||||
const pattern = `%${q.replaceAll('\\', '\\\\').replaceAll('%', '\\%').replaceAll('_', '\\_')}%`;
|
||||
where.push(
|
||||
or(
|
||||
ilike(houseListings.title, pattern),
|
||||
ilike(houseListings.neighbourhood, pattern),
|
||||
ilike(houseListings.postalCode, pattern),
|
||||
ilike(houseListings.agent, pattern)
|
||||
)
|
||||
);
|
||||
}
|
||||
const ppm2 = sql<number>`${houseListings.priceEur}::float / nullif(${houseListings.areaM2}, 0)`;
|
||||
const order = {
|
||||
new: [desc(houseListings.firstSeenAt)],
|
||||
price: [asc(houseListings.priceEur), desc(houseListings.firstSeenAt)],
|
||||
ppm2: [sql`${ppm2} ASC NULLS LAST`, desc(houseListings.firstSeenAt)],
|
||||
area: [sql`${houseListings.areaM2} DESC NULLS LAST`, desc(houseListings.firstSeenAt)]
|
||||
}[sort]!;
|
||||
|
||||
const [listings, cities, stats, rules, runs, health] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
id: houseListings.id,
|
||||
source: houseListings.source,
|
||||
url: houseListings.url,
|
||||
propertyType: houseListings.propertyType,
|
||||
title: houseListings.title,
|
||||
city: houseListings.city,
|
||||
neighbourhood: houseListings.neighbourhood,
|
||||
postalCode: houseListings.postalCode,
|
||||
priceEur: houseListings.priceEur,
|
||||
areaM2: houseListings.areaM2,
|
||||
rooms: houseListings.rooms,
|
||||
bedrooms: houseListings.bedrooms,
|
||||
energyLabel: houseListings.energyLabel,
|
||||
interior: houseListings.interior,
|
||||
availableFrom: houseListings.availableFrom,
|
||||
agent: houseListings.agent,
|
||||
imageUrl: houseListings.imageUrl,
|
||||
lat: houseListings.lat,
|
||||
lng: houseListings.lng,
|
||||
offeredAt: houseListings.offeredAt,
|
||||
firstSeenAt: houseListings.firstSeenAt,
|
||||
firstSeenLane: houseListings.firstSeenLane,
|
||||
removedAt: houseListings.removedAt,
|
||||
alertedAt: houseListings.alertedAt,
|
||||
status: houseListings.status,
|
||||
notes: houseListings.notes
|
||||
})
|
||||
.from(houseListings)
|
||||
.where(and(...where))
|
||||
.orderBy(...order)
|
||||
.limit(400),
|
||||
db.execute(sql`SELECT city_slug AS slug, min(city) AS name,
|
||||
count(*) FILTER (WHERE removed_at IS NULL)::int AS active
|
||||
FROM house_listings WHERE offer = ${offer} AND city_slug IS NOT NULL
|
||||
GROUP BY city_slug ORDER BY active DESC, slug LIMIT 200`),
|
||||
db.execute(sql`SELECT coalesce(min(city), city_slug) AS city,
|
||||
count(*) FILTER (WHERE removed_at IS NULL)::int AS active,
|
||||
count(*) FILTER (WHERE first_seen_at > now() - interval '7 days')::int AS new_week,
|
||||
percentile_cont(0.5) WITHIN GROUP (ORDER BY price_eur)
|
||||
FILTER (WHERE removed_at IS NULL AND price_eur > 0) AS median_price,
|
||||
percentile_cont(0.5) WITHIN GROUP (ORDER BY price_eur::float / area_m2)
|
||||
FILTER (WHERE removed_at IS NULL AND price_eur > 0 AND area_m2 > 0) AS median_ppm2,
|
||||
percentile_cont(0.5) WITHIN GROUP (
|
||||
ORDER BY extract(epoch FROM removed_at - coalesce(offered_at, first_seen_at)) / 86400)
|
||||
FILTER (WHERE removed_at > now() - interval '90 days') AS median_days_listed
|
||||
FROM house_listings WHERE offer = ${offer} AND city_slug IS NOT NULL
|
||||
GROUP BY city_slug ORDER BY active DESC LIMIT 30`),
|
||||
db.select().from(houseAlertRules).orderBy(asc(houseAlertRules.id)),
|
||||
db.select().from(houseWorkerRuns).orderBy(desc(houseWorkerRuns.createdAt)).limit(25),
|
||||
db.execute(sql`SELECT
|
||||
(SELECT max(created_at) FROM house_worker_runs WHERE status = 'ok') AS last_ok,
|
||||
(SELECT max(created_at) FROM house_worker_runs WHERE status = 'challenged') AS last_challenge,
|
||||
(SELECT count(*)::int FROM house_listings) AS total,
|
||||
(SELECT count(*)::int FROM house_listings WHERE alerted_at > now() - interval '7 days') AS alerts_week,
|
||||
(SELECT percentile_cont(0.5) WITHIN GROUP (
|
||||
ORDER BY extract(epoch FROM first_seen_at - offered_at) / 60)
|
||||
FROM house_listings WHERE first_seen_lane = 'fast' AND offered_at IS NOT NULL
|
||||
AND first_seen_at >= offered_at AND first_seen_at - offered_at < interval '3 days'
|
||||
AND first_seen_at > now() - interval '30 days')
|
||||
AS median_detect_minutes`)
|
||||
]);
|
||||
|
||||
return {
|
||||
filters: { offer, city, min, max, showRemoved, status, q, sort },
|
||||
listings,
|
||||
cities: cities as unknown as { slug: string; name: string | null; active: number }[],
|
||||
stats: stats as unknown as {
|
||||
city: string;
|
||||
active: number;
|
||||
new_week: number;
|
||||
median_price: number | null;
|
||||
median_ppm2: number | null;
|
||||
median_days_listed: number | null;
|
||||
}[],
|
||||
rules,
|
||||
runs,
|
||||
health: health[0] as {
|
||||
last_ok: Date | string | null;
|
||||
last_challenge: Date | string | null;
|
||||
total: number;
|
||||
alerts_week: number;
|
||||
median_detect_minutes: number | null;
|
||||
},
|
||||
statuses: listingStatuses,
|
||||
propertyTypes
|
||||
};
|
||||
};
|
||||
|
||||
const nullableInt = (value: FormDataEntryValue | null) => {
|
||||
const text = String(value ?? '').trim();
|
||||
if (!text) return null;
|
||||
const n = Number(text);
|
||||
return Number.isInteger(n) && n >= 0 && n < 100_000_000 ? n : undefined;
|
||||
};
|
||||
|
||||
export const actions: Actions = {
|
||||
status: async (event) => {
|
||||
requireAdmin(event);
|
||||
const form = await event.request.formData();
|
||||
const id = Number(form.get('id'));
|
||||
const status = String(form.get('status') ?? '');
|
||||
const notes = String(form.get('notes') ?? '')
|
||||
.trim()
|
||||
.slice(0, 2000);
|
||||
if (!Number.isInteger(id) || id <= 0) return fail(400, { message: 'Invalid listing' });
|
||||
if (status && !(listingStatuses as readonly string[]).includes(status))
|
||||
return fail(400, { message: 'Invalid status' });
|
||||
await db
|
||||
.update(houseListings)
|
||||
.set({ status: status || null, ...(form.has('notes') ? { notes: notes || null } : {}) })
|
||||
.where(eq(houseListings.id, id));
|
||||
return { ok: true };
|
||||
},
|
||||
saveRule: async (event) => {
|
||||
requireAdmin(event);
|
||||
const form = await event.request.formData();
|
||||
const id = Number(form.get('id') || 0);
|
||||
const name = String(form.get('name') ?? '')
|
||||
.trim()
|
||||
.slice(0, 80);
|
||||
const minPrice = nullableInt(form.get('minPrice'));
|
||||
const maxPrice = nullableInt(form.get('maxPrice'));
|
||||
const minArea = nullableInt(form.get('minArea'));
|
||||
const minBedrooms = nullableInt(form.get('minBedrooms'));
|
||||
if (!name) return fail(400, { message: 'Give the rule a name' });
|
||||
if (
|
||||
minPrice === undefined ||
|
||||
maxPrice === undefined ||
|
||||
minArea === undefined ||
|
||||
minBedrooms === undefined
|
||||
)
|
||||
return fail(400, { message: 'Numbers must be whole and positive' });
|
||||
const values = {
|
||||
name,
|
||||
enabled: form.get('enabled') === 'on',
|
||||
offer: form.get('offer') === 'buy' ? 'buy' : 'rent',
|
||||
minPrice,
|
||||
maxPrice,
|
||||
minArea,
|
||||
minBedrooms,
|
||||
cities: String(form.get('cities') ?? '')
|
||||
.split(',')
|
||||
.map(normalizeCity)
|
||||
.filter(Boolean)
|
||||
.slice(0, 60),
|
||||
propertyTypes: form
|
||||
.getAll('propertyTypes')
|
||||
.map(String)
|
||||
.filter((type) => (propertyTypes as readonly string[]).includes(type))
|
||||
};
|
||||
if (id > 0) await db.update(houseAlertRules).set(values).where(eq(houseAlertRules.id, id));
|
||||
else await db.insert(houseAlertRules).values(values);
|
||||
return { ok: true };
|
||||
},
|
||||
deleteRule: async (event) => {
|
||||
requireAdmin(event);
|
||||
const id = Number((await event.request.formData()).get('id'));
|
||||
if (!Number.isInteger(id) || id <= 0) return fail(400, { message: 'Invalid rule' });
|
||||
await db.delete(houseAlertRules).where(eq(houseAlertRules.id, id));
|
||||
return { ok: true };
|
||||
},
|
||||
testAlert: async (event) => {
|
||||
requireAdmin(event);
|
||||
const sent = await sendAlert({
|
||||
title: '🏠 House tracker test',
|
||||
body: 'If you can read this, alerts work.',
|
||||
url: `${event.url.origin}/projects/house-tracker`
|
||||
});
|
||||
return sent
|
||||
? { message: 'Test alert sent' }
|
||||
: fail(400, {
|
||||
message: 'No alert channel accepted it. Set HOUSE_DISCORD_WEBHOOK_URL or HOUSE_NTFY_URL.'
|
||||
});
|
||||
}
|
||||
};
|
||||
412
src/routes/projects/house-tracker/+page.svelte
Normal file
412
src/routes/projects/house-tracker/+page.svelte
Normal file
@@ -0,0 +1,412 @@
|
||||
<script lang="ts">
|
||||
import { enhance } from '$app/forms';
|
||||
import { onMount } from 'svelte';
|
||||
import ExternalLink from 'lucide-svelte/icons/external-link';
|
||||
import BellRing from 'lucide-svelte/icons/bell-ring';
|
||||
import { Button } from '$lib/components/ui/button/index.js';
|
||||
import { Input } from '$lib/components/ui/input/index.js';
|
||||
import { Badge } from '$lib/components/ui/badge/index.js';
|
||||
import { NativeSelect } from '$lib/components/ui/native-select/index.js';
|
||||
import * as Card from '$lib/components/ui/card/index.js';
|
||||
import ListingMap from '$lib/house-tracker/ListingMap.svelte';
|
||||
import { relativeTime } from '$lib/relative-time';
|
||||
import type { PageProps } from './$types';
|
||||
|
||||
let { data, form }: PageProps = $props();
|
||||
|
||||
let now = $state(Date.now());
|
||||
onMount(() => {
|
||||
const timer = setInterval(() => (now = Date.now()), 30_000);
|
||||
return () => clearInterval(timer);
|
||||
});
|
||||
|
||||
let highlighted = $state<number | null>(null);
|
||||
const euro = new Intl.NumberFormat('nl-NL', {
|
||||
style: 'currency',
|
||||
currency: 'EUR',
|
||||
maximumFractionDigits: 0
|
||||
});
|
||||
const clock = new Intl.DateTimeFormat('nl-NL', {
|
||||
timeZone: 'Europe/Amsterdam',
|
||||
weekday: 'short',
|
||||
day: 'numeric',
|
||||
month: 'short',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit'
|
||||
});
|
||||
const suffix = $derived(data.filters.offer === 'rent' ? '/mo' : '');
|
||||
|
||||
function price(value: number | null) {
|
||||
return value == null ? 'on request' : `${euro.format(value)}${suffix}`;
|
||||
}
|
||||
function minutes(from: Date | string | null, to: Date | string | null) {
|
||||
if (!from || !to) return null;
|
||||
return Math.round((new Date(to).getTime() - new Date(from).getTime()) / 60_000);
|
||||
}
|
||||
function delayLabel(value: number | null) {
|
||||
if (value == null || value < 0) return null;
|
||||
if (value < 60) return `+${value} min`;
|
||||
if (value < 48 * 60) return `+${Math.round(value / 60)} h`;
|
||||
return `+${Math.round(value / 1440)} d`;
|
||||
}
|
||||
const workerAge = $derived(
|
||||
data.health.last_ok ? now - new Date(data.health.last_ok).getTime() : Infinity
|
||||
);
|
||||
const workerState = $derived(
|
||||
data.health.last_challenge &&
|
||||
(!data.health.last_ok || new Date(data.health.last_challenge) > new Date(data.health.last_ok))
|
||||
? 'challenged'
|
||||
: workerAge < 75 * 60_000
|
||||
? 'healthy'
|
||||
: 'quiet'
|
||||
);
|
||||
const mappable = $derived(data.listings.filter((row) => row.lat != null).length);
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<title>house tracker</title>
|
||||
<meta name="robots" content="noindex" />
|
||||
</svelte:head>
|
||||
|
||||
<div class="mx-auto max-w-6xl px-4 py-10 sm:px-6">
|
||||
<div class="mb-6 flex flex-wrap items-end justify-between gap-4">
|
||||
<div>
|
||||
<h1 class="text-4xl font-semibold tracking-tight">house tracker</h1>
|
||||
<p class="mt-1 text-sm text-muted-foreground">
|
||||
{data.health.total} listings collected · {data.health.alerts_week}
|
||||
{data.health.alerts_week === 1 ? 'alert' : 'alerts'} this week
|
||||
{#if data.health.median_detect_minutes != null}
|
||||
· the fast lane spots new listings {delayLabel(
|
||||
Math.round(data.health.median_detect_minutes)
|
||||
)?.slice(1)} after they go up (median)
|
||||
{/if}
|
||||
</p>
|
||||
</div>
|
||||
<div class="flex items-center gap-2 text-sm">
|
||||
<span
|
||||
class={[
|
||||
'inline-block size-2.5 rounded-full',
|
||||
workerState === 'healthy' && 'bg-green-600',
|
||||
workerState === 'quiet' && 'bg-amber-500',
|
||||
workerState === 'challenged' && 'bg-red-600'
|
||||
]}
|
||||
></span>
|
||||
{#if workerState === 'challenged'}
|
||||
worker was challenged {relativeTime(data.health.last_challenge!, now)}, backing off
|
||||
{:else if data.health.last_ok}
|
||||
worker last synced {relativeTime(data.health.last_ok, now)}
|
||||
{:else}
|
||||
worker has not synced yet
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<form method="GET" class="mb-4 grid grid-cols-2 gap-2 sm:grid-cols-4 lg:grid-cols-8">
|
||||
<NativeSelect name="offer" value={data.filters.offer} aria-label="Rent or buy">
|
||||
<option value="rent">rent</option>
|
||||
<option value="buy">buy</option>
|
||||
</NativeSelect>
|
||||
<NativeSelect name="city" value={data.filters.city} aria-label="City">
|
||||
<option value="">all cities</option>
|
||||
{#each data.cities as city (city.slug)}
|
||||
<option value={city.slug}>{city.name ?? city.slug} ({city.active})</option>
|
||||
{/each}
|
||||
</NativeSelect>
|
||||
<Input name="min" type="number" min="0" placeholder="min €" value={data.filters.min ?? ''} />
|
||||
<Input name="max" type="number" min="0" placeholder="max €" value={data.filters.max ?? ''} />
|
||||
<Input name="q" placeholder="street, area, agent" value={data.filters.q} class="col-span-2" />
|
||||
<NativeSelect name="sort" value={data.filters.sort} aria-label="Sort">
|
||||
<option value="new">newest</option>
|
||||
<option value="price">cheapest</option>
|
||||
<option value="ppm2">cheapest per m²</option>
|
||||
<option value="area">biggest</option>
|
||||
</NativeSelect>
|
||||
<NativeSelect name="status" value={data.filters.status} aria-label="My status">
|
||||
<option value="">any status</option>
|
||||
<option value="any">tracked only</option>
|
||||
{#each data.statuses as status (status)}<option value={status}>{status}</option>{/each}
|
||||
</NativeSelect>
|
||||
<label class="col-span-2 flex items-center gap-2 text-sm">
|
||||
<input type="checkbox" name="removed" value="1" checked={data.filters.showRemoved} />
|
||||
include listings that are gone
|
||||
</label>
|
||||
<Button type="submit" class="col-span-2 sm:col-span-1">filter</Button>
|
||||
</form>
|
||||
|
||||
<ListingMap
|
||||
listings={data.listings}
|
||||
offer={data.filters.offer}
|
||||
onselect={(id) => (highlighted = id)}
|
||||
/>
|
||||
<p class="mt-1 mb-6 text-xs text-muted-foreground">
|
||||
{mappable} of {data.listings.length} shown listings have coordinates (they arrive with the detail
|
||||
page fetch).
|
||||
</p>
|
||||
|
||||
<ul class="divide-y divide-border rounded-lg border border-border">
|
||||
{#each data.listings as listing (listing.id)}
|
||||
{@const delay = minutes(listing.offeredAt, listing.firstSeenAt)}
|
||||
<li
|
||||
id="listing-{listing.id}"
|
||||
class={[
|
||||
'flex gap-3 p-3 sm:gap-4',
|
||||
highlighted === listing.id && 'bg-muted',
|
||||
listing.removedAt && 'opacity-60'
|
||||
]}
|
||||
>
|
||||
{#if listing.imageUrl}
|
||||
<img
|
||||
src={listing.imageUrl}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
referrerpolicy="no-referrer"
|
||||
class="hidden size-24 shrink-0 rounded-md object-cover sm:block"
|
||||
/>
|
||||
{/if}
|
||||
<div class="min-w-0 flex-1">
|
||||
<div class="flex flex-wrap items-baseline justify-between gap-x-3">
|
||||
<!-- External listing site. -->
|
||||
<!-- eslint-disable svelte/no-navigation-without-resolve -->
|
||||
<a
|
||||
href={listing.url}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
class="inline-flex items-center gap-1 font-medium hover:underline"
|
||||
>
|
||||
{listing.title ?? 'Listing'}
|
||||
<ExternalLink class="size-3.5 text-muted-foreground" />
|
||||
</a>
|
||||
<span class="font-semibold tabular-nums">{price(listing.priceEur)}</span>
|
||||
</div>
|
||||
<div class="mt-0.5 text-sm text-muted-foreground">
|
||||
{[
|
||||
listing.postalCode,
|
||||
[listing.city, listing.neighbourhood].filter(Boolean).join(', '),
|
||||
listing.areaM2 != null && `${listing.areaM2} m²`,
|
||||
listing.priceEur &&
|
||||
listing.areaM2 &&
|
||||
`${euro.format(listing.priceEur / listing.areaM2)}/m²`,
|
||||
listing.rooms != null && `${listing.rooms} rooms`,
|
||||
listing.energyLabel && `label ${listing.energyLabel}`,
|
||||
listing.interior,
|
||||
listing.availableFrom && `available ${listing.availableFrom.toLowerCase()}`
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' · ')}
|
||||
</div>
|
||||
<div
|
||||
class="mt-1 flex flex-wrap items-center gap-x-3 gap-y-1 text-xs text-muted-foreground"
|
||||
>
|
||||
{#if listing.offeredAt}
|
||||
<span title="published on the site"
|
||||
>posted {clock.format(new Date(listing.offeredAt))}</span
|
||||
>
|
||||
{/if}
|
||||
<span title="first seen by the worker">
|
||||
seen {relativeTime(listing.firstSeenAt, now)}
|
||||
{#if listing.firstSeenLane !== 'seed' && delayLabel(delay)}({delayLabel(delay)}, {listing.firstSeenLane}
|
||||
lane){/if}
|
||||
</span>
|
||||
{#if listing.alertedAt}<Badge variant="secondary"
|
||||
><BellRing class="mr-1 size-3" />alerted</Badge
|
||||
>{/if}
|
||||
{#if listing.removedAt}<Badge variant="outline"
|
||||
>gone {relativeTime(listing.removedAt, now)}</Badge
|
||||
>{/if}
|
||||
{#if listing.agent}<span>{listing.agent}</span>{/if}
|
||||
<span>{listing.source}</span>
|
||||
</div>
|
||||
</div>
|
||||
<form
|
||||
method="POST"
|
||||
action="?/status"
|
||||
use:enhance={() =>
|
||||
async ({ update }) =>
|
||||
update({ reset: false, invalidateAll: false })}
|
||||
class="shrink-0"
|
||||
>
|
||||
<input type="hidden" name="id" value={listing.id} />
|
||||
<NativeSelect
|
||||
name="status"
|
||||
value={listing.status ?? ''}
|
||||
aria-label="My status"
|
||||
class="h-8 text-xs"
|
||||
onchange={(event) => (event.currentTarget as HTMLSelectElement).form?.requestSubmit()}
|
||||
>
|
||||
<option value="">—</option>
|
||||
{#each data.statuses as status (status)}<option value={status}>{status}</option>{/each}
|
||||
</NativeSelect>
|
||||
<noscript><Button type="submit" size="sm" variant="outline">save</Button></noscript>
|
||||
</form>
|
||||
</li>
|
||||
{:else}
|
||||
<li class="p-6 text-center text-sm text-muted-foreground">
|
||||
No listings yet. Start the home worker (see docs/house-tracker.md).
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
{#if data.listings.length === 400}
|
||||
<p class="mt-2 text-xs text-muted-foreground">
|
||||
Showing the first 400. Narrow the filters to see more.
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
<h2 class="mt-12 mb-3 text-2xl font-semibold">market ({data.filters.offer})</h2>
|
||||
<div class="overflow-x-auto rounded-lg border border-border">
|
||||
<table class="w-full text-sm">
|
||||
<thead class="bg-muted/50 text-left text-xs text-muted-foreground">
|
||||
<tr>
|
||||
<th class="px-3 py-2">city</th>
|
||||
<th class="px-3 py-2 text-right">active</th>
|
||||
<th class="px-3 py-2 text-right">new (7d)</th>
|
||||
<th class="px-3 py-2 text-right">median price</th>
|
||||
<th class="px-3 py-2 text-right">median €/m²</th>
|
||||
<th class="px-3 py-2 text-right">median days listed</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each data.stats as row (row.city)}
|
||||
<tr class="border-t border-border tabular-nums">
|
||||
<td class="px-3 py-2">{row.city}</td>
|
||||
<td class="px-3 py-2 text-right">{row.active}</td>
|
||||
<td class="px-3 py-2 text-right">{row.new_week}</td>
|
||||
<td class="px-3 py-2 text-right"
|
||||
>{row.median_price != null ? price(Math.round(row.median_price)) : '—'}</td
|
||||
>
|
||||
<td class="px-3 py-2 text-right"
|
||||
>{row.median_ppm2 != null ? euro.format(row.median_ppm2) : '—'}</td
|
||||
>
|
||||
<td class="px-3 py-2 text-right"
|
||||
>{row.median_days_listed != null ? Math.round(row.median_days_listed) : '—'}</td
|
||||
>
|
||||
</tr>
|
||||
{:else}
|
||||
<tr
|
||||
><td colspan="6" class="px-3 py-4 text-center text-muted-foreground">no data yet</td
|
||||
></tr
|
||||
>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p class="mt-1 text-xs text-muted-foreground">
|
||||
"Days listed" needs a few full sweeps before listings can be seen disappearing.
|
||||
</p>
|
||||
|
||||
<div class="mt-12 mb-3 flex flex-wrap items-center justify-between gap-2">
|
||||
<h2 class="text-2xl font-semibold">alerts</h2>
|
||||
<form method="POST" action="?/testAlert" use:enhance>
|
||||
<Button type="submit" variant="outline" size="sm"
|
||||
><BellRing class="mr-1 size-4" />send test alert</Button
|
||||
>
|
||||
</form>
|
||||
</div>
|
||||
{#if form?.message}<p class="mb-3 text-sm">{form.message}</p>{/if}
|
||||
<div class="grid gap-3 md:grid-cols-2">
|
||||
{#each [...data.rules, null] as rule (rule?.id ?? 'new')}
|
||||
<Card.Root class="py-0">
|
||||
<Card.Content class="p-4">
|
||||
<form method="POST" action="?/saveRule" use:enhance class="grid gap-2 text-sm">
|
||||
<input type="hidden" name="id" value={rule?.id ?? ''} />
|
||||
<div class="flex items-center gap-2">
|
||||
<Input name="name" value={rule?.name ?? ''} placeholder="new rule name" required />
|
||||
<label class="flex items-center gap-1 whitespace-nowrap">
|
||||
<input type="checkbox" name="enabled" checked={rule?.enabled ?? true} /> on
|
||||
</label>
|
||||
</div>
|
||||
<div class="grid grid-cols-3 gap-2">
|
||||
<NativeSelect name="offer" value={rule?.offer ?? 'rent'} aria-label="Offer">
|
||||
<option value="rent">rent</option>
|
||||
<option value="buy">buy</option>
|
||||
</NativeSelect>
|
||||
<Input
|
||||
name="minPrice"
|
||||
type="number"
|
||||
min="0"
|
||||
placeholder="min €"
|
||||
value={rule?.minPrice ?? ''}
|
||||
/>
|
||||
<Input
|
||||
name="maxPrice"
|
||||
type="number"
|
||||
min="0"
|
||||
placeholder="max €"
|
||||
value={rule?.maxPrice ?? ''}
|
||||
/>
|
||||
<Input
|
||||
name="minArea"
|
||||
type="number"
|
||||
min="0"
|
||||
placeholder="min m²"
|
||||
value={rule?.minArea ?? ''}
|
||||
/>
|
||||
<Input
|
||||
name="minBedrooms"
|
||||
type="number"
|
||||
min="0"
|
||||
placeholder="min bedrooms"
|
||||
value={rule?.minBedrooms ?? ''}
|
||||
class="col-span-2"
|
||||
/>
|
||||
</div>
|
||||
<Input
|
||||
name="cities"
|
||||
placeholder="cities, comma separated (empty = anywhere)"
|
||||
value={rule?.cities.join(', ') ?? ''}
|
||||
/>
|
||||
<div class="flex flex-wrap gap-3">
|
||||
{#each data.propertyTypes as type (type)}
|
||||
<label class="flex items-center gap-1">
|
||||
<input
|
||||
type="checkbox"
|
||||
name="propertyTypes"
|
||||
value={type}
|
||||
checked={rule?.propertyTypes.includes(type) ?? false}
|
||||
/>
|
||||
{type}
|
||||
</label>
|
||||
{/each}
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<Button type="submit" size="sm">{rule ? 'save' : 'add rule'}</Button>
|
||||
{#if rule}
|
||||
<Button type="submit" size="sm" variant="ghost" formaction="?/deleteRule"
|
||||
>delete</Button
|
||||
>
|
||||
{/if}
|
||||
</div>
|
||||
</form>
|
||||
</Card.Content>
|
||||
</Card.Root>
|
||||
{/each}
|
||||
</div>
|
||||
<p class="mt-1 text-xs text-muted-foreground">
|
||||
Unchecked property types means all types. Listings without a size pass size limits, so you never
|
||||
miss one because a search card left it out.
|
||||
</p>
|
||||
|
||||
<h2 class="mt-12 mb-3 text-2xl font-semibold">worker log</h2>
|
||||
<div class="overflow-x-auto rounded-lg border border-border">
|
||||
<table class="w-full text-xs">
|
||||
<tbody>
|
||||
{#each data.runs as run (run.id)}
|
||||
<tr class="border-t border-border first:border-t-0">
|
||||
<td class="px-3 py-1.5 whitespace-nowrap text-muted-foreground"
|
||||
>{relativeTime(run.createdAt, now)}</td
|
||||
>
|
||||
<td class="px-3 py-1.5">{run.lane}</td>
|
||||
<td class={['px-3 py-1.5', run.status !== 'ok' && 'font-medium text-red-600']}>
|
||||
{run.status}{run.httpStatus ? ` ${run.httpStatus}` : ''}
|
||||
</td>
|
||||
<td class="px-3 py-1.5 tabular-nums">{run.found ?? ''}</td>
|
||||
<td class="max-w-md truncate px-3 py-1.5 text-muted-foreground" title={run.url}>
|
||||
{run.message ?? run.url.replace(/^https:\/\/www\./, '')}
|
||||
</td>
|
||||
</tr>
|
||||
{:else}
|
||||
<tr><td class="px-3 py-4 text-center text-muted-foreground">nothing yet</td></tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
@@ -0,0 +1,11 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { detailsRequestSchema } from '$lib/house-tracker/types';
|
||||
import { requireWorker, saveDetails } from '$lib/server/house-tracker';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
export const POST: RequestHandler = async ({ request }) => {
|
||||
requireWorker(request);
|
||||
const parsed = detailsRequestSchema.safeParse(await request.json().catch(() => null));
|
||||
if (!parsed.success) return json({ error: parsed.error.issues.slice(0, 5) }, { status: 400 });
|
||||
return json(await saveDetails(parsed.data));
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { reportRequestSchema } from '$lib/house-tracker/types';
|
||||
import { recordReport, requireWorker } from '$lib/server/house-tracker';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
export const POST: RequestHandler = async ({ request }) => {
|
||||
requireWorker(request);
|
||||
const parsed = reportRequestSchema.safeParse(await request.json().catch(() => null));
|
||||
if (!parsed.success) return json({ error: parsed.error.issues.slice(0, 5) }, { status: 400 });
|
||||
await recordReport(parsed.data);
|
||||
return json({ ok: true });
|
||||
};
|
||||
11
src/routes/projects/house-tracker/api/worker/sync/+server.ts
Normal file
11
src/routes/projects/house-tracker/api/worker/sync/+server.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { syncRequestSchema } from '$lib/house-tracker/types';
|
||||
import { requireWorker, syncCards } from '$lib/server/house-tracker';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
export const POST: RequestHandler = async ({ request }) => {
|
||||
requireWorker(request);
|
||||
const parsed = syncRequestSchema.safeParse(await request.json().catch(() => null));
|
||||
if (!parsed.success) return json({ error: parsed.error.issues.slice(0, 5) }, { status: 400 });
|
||||
return json(await syncCards(parsed.data));
|
||||
};
|
||||
Reference in New Issue
Block a user