Stan Runge
0846cf7d77
Collects Pararius rent/buy listings into Postgres via a worker that runs on a residential connection, sends Discord/ntfy alerts for new listings matching rules, and shows an admin-only map, market stats and worker log. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013HfF5tnChi8tkCx9svJP6D
436 lines
14 KiB
TypeScript
436 lines
14 KiB
TypeScript
// Pararius HTML parsing. Pure functions, no network: the worker fetches, these parse.
|
|
//
|
|
// Pararius has no public API, so everything here reads markup that can change
|
|
// without notice. Each parser prefers structured data (ld+json, the analytics
|
|
// blob) and falls back to looser text matching, so a redesign degrades fields
|
|
// instead of dropping listings.
|
|
import type { ListingCard, ListingDetail, Offer, PropertyType } from './types';
|
|
|
|
export const PARARIUS_ORIGIN = 'https://www.pararius.nl';
|
|
|
|
const listingPath =
|
|
/\/(appartement|huis|kamer|studio|woning|[a-z]+)-te-(huur|koop)\/([a-z0-9-]+)\/([0-9a-f]{8})\/([a-z0-9-]+)/;
|
|
|
|
const typeWords: Record<string, PropertyType> = {
|
|
appartement: 'apartment',
|
|
apartment: 'apartment',
|
|
huis: 'house',
|
|
house: 'house',
|
|
kamer: 'room',
|
|
room: 'room',
|
|
studio: 'studio'
|
|
};
|
|
|
|
const entities: Record<string, string> = {
|
|
amp: '&',
|
|
lt: '<',
|
|
gt: '>',
|
|
quot: '"',
|
|
apos: "'",
|
|
nbsp: ' ',
|
|
euro: '€',
|
|
sup2: '²',
|
|
sup3: '³'
|
|
};
|
|
|
|
export function decodeEntities(value: string): string {
|
|
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => {
|
|
if (code[0] === '#') {
|
|
const n =
|
|
code[1] === 'x' || code[1] === 'X' ? parseInt(code.slice(2), 16) : Number(code.slice(1));
|
|
return Number.isFinite(n) ? String.fromCodePoint(n) : match;
|
|
}
|
|
return entities[code.toLowerCase()] ?? match;
|
|
});
|
|
}
|
|
|
|
export function textOf(html: string): string {
|
|
return decodeEntities(html.replace(/<[^>]*>/g, ' '))
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
}
|
|
|
|
/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */
|
|
export function isChallenge(html: string, status = 200): boolean {
|
|
if (status === 403 || status === 429 || status === 503) return true;
|
|
const head = html.slice(0, 20_000);
|
|
return (
|
|
/<title>\s*Just a moment/i.test(head) ||
|
|
head.includes('challenges.cloudflare.com/turnstile') ||
|
|
head.includes('cf-chl-') ||
|
|
/<title>\s*Attention Required/i.test(head)
|
|
);
|
|
}
|
|
|
|
export type ListingRef = {
|
|
sourceId: string;
|
|
url: string;
|
|
offer: Offer;
|
|
propertyType: PropertyType;
|
|
citySlug: string;
|
|
};
|
|
|
|
export function listingRef(href: string): ListingRef | null {
|
|
const match = listingPath.exec(href);
|
|
if (!match) return null;
|
|
const [, type, offer, citySlug, shortId, slug] = match;
|
|
return {
|
|
sourceId: shortId,
|
|
url: `${PARARIUS_ORIGIN}/${type}-te-${offer}/${citySlug}/${shortId}/${slug}`,
|
|
offer: offer === 'koop' ? 'buy' : 'rent',
|
|
propertyType: typeWords[type] ?? 'other',
|
|
citySlug
|
|
};
|
|
}
|
|
|
|
/** Whole euros from "€ 1.558 per maand", "€1,558 per month", "€ 425.000 k.k." */
|
|
export function parseEuros(value: string | null | undefined): number | null {
|
|
if (!value) return null;
|
|
const match = /€\s*([\d.,]+)/.exec(value) ?? /([\d][\d.,]*)/.exec(value);
|
|
if (!match) return null;
|
|
const digits = match[1].replace(/[.,]\d{2}$/, '').replace(/[.,]/g, '');
|
|
const n = Number(digits);
|
|
return digits && Number.isSafeInteger(n) ? n : null;
|
|
}
|
|
|
|
export function parseInteger(value: string | null | undefined): number | null {
|
|
const match = value ? /(\d[\d.]*)/.exec(value) : null;
|
|
if (!match) return null;
|
|
const n = Number(match[1].replace(/\./g, ''));
|
|
return Number.isSafeInteger(n) ? n : null;
|
|
}
|
|
|
|
/** "2801 BC Gouda (Centrum)" → postal code, city and neighbourhood. */
|
|
export function parseLocation(value: string | null | undefined) {
|
|
const out = {
|
|
postalCode: null as string | null,
|
|
city: null as string | null,
|
|
neighbourhood: null as string | null
|
|
};
|
|
if (!value) return out;
|
|
let rest = value.trim();
|
|
const hood = /\(([^)]+)\)\s*$/.exec(rest);
|
|
if (hood) {
|
|
out.neighbourhood = hood[1].trim();
|
|
rest = rest.slice(0, hood.index).trim();
|
|
}
|
|
const postal = /^(\d{4})\s?([A-Z]{2})\b/.exec(rest);
|
|
if (postal) {
|
|
out.postalCode = `${postal[1]} ${postal[2]}`;
|
|
rest = rest.slice(postal[0].length).trim();
|
|
}
|
|
if (rest) out.city = rest;
|
|
return out;
|
|
}
|
|
|
|
function classText(html: string, className: string): string | null {
|
|
const re = new RegExp(
|
|
`<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)</\\1>`,
|
|
'i'
|
|
);
|
|
const match = re.exec(html);
|
|
if (!match) return null;
|
|
const value = textOf(match[2]);
|
|
return value || null;
|
|
}
|
|
|
|
function firstImage(html: string): string | null {
|
|
const tag = /<img\b[^>]*>/i.exec(html)?.[0];
|
|
if (!tag) return null;
|
|
const src =
|
|
/\bsrc="(https:[^"]+)"/i.exec(tag)?.[1] ??
|
|
/\bsrcset="(https:[^"\s,]+)/i.exec(tag)?.[1] ??
|
|
/\bdata-src="(https:[^"]+)"/i.exec(tag)?.[1];
|
|
return src ? decodeEntities(src) : null;
|
|
}
|
|
|
|
export type SearchPage = {
|
|
cards: ListingCard[];
|
|
nextPageUrl: string | null;
|
|
totalResults: number | null;
|
|
};
|
|
|
|
/**
|
|
* Parse a search results page (`/huurwoningen/gouda/1300-1700`).
|
|
* Splits on the listing card wrappers when present; otherwise falls back to
|
|
* one chunk per unique listing link so a markup change still yields ids.
|
|
*/
|
|
export function parseSearchPage(html: string, pageUrl: string): SearchPage {
|
|
const chunks: string[] = [];
|
|
const marker =
|
|
/<(section|li|article|div)\b[^>]*class="[^"]*\blisting-search-item\b(?!__)[^"]*"/gi;
|
|
const starts = [...html.matchAll(marker)].map((m) => m.index!);
|
|
if (starts.length) {
|
|
for (let i = 0; i < starts.length; i++) {
|
|
chunks.push(html.slice(starts[i], starts[i + 1] ?? starts[i] + 20_000));
|
|
}
|
|
} else {
|
|
const links = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/gi;
|
|
const seen = new Set<string>();
|
|
for (const m of html.matchAll(links)) {
|
|
const ref = listingRef(m[1]);
|
|
if (!ref || seen.has(ref.sourceId)) continue;
|
|
seen.add(ref.sourceId);
|
|
chunks.push(html.slice(Math.max(0, m.index! - 3000), m.index! + 6000));
|
|
}
|
|
}
|
|
|
|
const cards: ListingCard[] = [];
|
|
const seen = new Set<string>();
|
|
for (const chunk of chunks) {
|
|
const href = /href="([^"]*-te-(?:huur|koop)\/[^"]+)"/i.exec(chunk)?.[1];
|
|
const ref = href ? listingRef(decodeEntities(href)) : null;
|
|
if (!ref || seen.has(ref.sourceId)) continue;
|
|
seen.add(ref.sourceId);
|
|
|
|
const titleLink = /<a\b[^>]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk);
|
|
const title =
|
|
(titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title');
|
|
const location = parseLocation(
|
|
classText(chunk, 'listing-search-item__sub-title') ??
|
|
classText(chunk, 'listing-search-item__location')
|
|
);
|
|
const priceText =
|
|
classText(chunk, 'listing-search-item__price') ?? /€\s*[\d.,]+[^<]{0,30}/.exec(chunk)?.[0];
|
|
const area =
|
|
classText(chunk, 'illustrated-features__item--surface-area') ??
|
|
/(\d+)\s*m(?:²|²|2)/i.exec(chunk)?.[0];
|
|
const rooms = classText(chunk, 'illustrated-features__item--number-of-rooms');
|
|
const agentBlock = classText(chunk, 'listing-search-item__info');
|
|
|
|
cards.push({
|
|
sourceId: ref.sourceId,
|
|
url: ref.url,
|
|
offer: ref.offer,
|
|
propertyType: ref.propertyType,
|
|
title: title?.slice(0, 200) ?? null,
|
|
postalCode: location.postalCode,
|
|
city: location.city ?? cityFromSlug(ref.citySlug),
|
|
neighbourhood: location.neighbourhood,
|
|
priceEur: parseEuros(priceText),
|
|
areaM2: parseInteger(area),
|
|
rooms: parseInteger(rooms),
|
|
interior: classText(chunk, 'illustrated-features__item--interior'),
|
|
agent: agentBlock?.slice(0, 160) ?? null,
|
|
imageUrl: firstImage(chunk)
|
|
});
|
|
}
|
|
|
|
const nextHref =
|
|
/<a\b[^>]*class="[^"]*pagination__link--next[^"]*"[^>]*href="([^"]+)"/i.exec(html)?.[1] ??
|
|
/<a\b[^>]*href="([^"]+)"[^>]*class="[^"]*pagination__link--next[^"]*"/i.exec(html)?.[1] ??
|
|
/<link\b[^>]*rel="next"[^>]*href="([^"]+)"/i.exec(html)?.[1];
|
|
const total =
|
|
classText(html, 'search-list-header__count') ??
|
|
/(\d[\d.]*)\s+(?:huurwoningen|koopwoningen|woningen|results|resultaten)/i.exec(
|
|
textOf(html.slice(0, 200_000))
|
|
)?.[1];
|
|
|
|
return {
|
|
cards,
|
|
nextPageUrl: nextHref ? new URL(decodeEntities(nextHref), pageUrl).toString() : null,
|
|
totalResults: parseInteger(total)
|
|
};
|
|
}
|
|
|
|
/** `/huurwoningen/gouda/1300-1700` → page N url (`.../page-2`). */
|
|
export function searchPageUrl(base: string, page: number): string {
|
|
const url = new URL(base);
|
|
url.pathname = url.pathname.replace(/\/page-\d+\/?$/, '').replace(/\/$/, '');
|
|
if (page > 1) url.pathname += `/page-${page}`;
|
|
return url.toString();
|
|
}
|
|
|
|
export function cityFromSlug(slug: string): string {
|
|
return slug
|
|
.split('-')
|
|
.map((part) =>
|
|
part === 'aan' || part === 'den' || part === 'de'
|
|
? part
|
|
: part[0]?.toUpperCase() + part.slice(1)
|
|
)
|
|
.join(' ')
|
|
.replace(/^\w/, (c) => c.toUpperCase());
|
|
}
|
|
|
|
function jsonBlocks(html: string): unknown[] {
|
|
const out: unknown[] = [];
|
|
for (const m of html.matchAll(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) {
|
|
try {
|
|
out.push(JSON.parse(m[1]));
|
|
} catch {
|
|
// Ignore malformed blocks; other sources may still have the data.
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
type LdListing = {
|
|
'@type'?: string | string[];
|
|
name?: string;
|
|
description?: string;
|
|
image?: string | string[];
|
|
address?: {
|
|
streetAddress?: string;
|
|
addressLocality?: string;
|
|
addressRegion?: string;
|
|
postalCode?: string;
|
|
};
|
|
numberOfRooms?: { value?: number }[] | { value?: number } | number;
|
|
floorSize?: { value?: number };
|
|
yearBuilt?: number;
|
|
numberOfBedrooms?: number;
|
|
numberOfBathroomsTotal?: number;
|
|
datePosted?: string;
|
|
offers?: { price?: number; url?: string };
|
|
};
|
|
|
|
function ldListing(html: string): LdListing | null {
|
|
for (const block of jsonBlocks(html)) {
|
|
const items = Array.isArray(block) ? block : [block];
|
|
for (const item of items) {
|
|
if (!item || typeof item !== 'object') continue;
|
|
const type = (item as LdListing)['@type'];
|
|
const types = Array.isArray(type) ? type : [type];
|
|
if (types.includes('RealEstateListing')) return item as LdListing;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/** The flat `odp.*` analytics object Pararius embeds on listing pages. */
|
|
export function analyticsBlob(html: string): Record<string, string> {
|
|
const candidates = [html, decodeEntities(html)];
|
|
for (const source of candidates) {
|
|
const match = /\{[^{}]*"odp\.listing_id"[^{}]*\}/.exec(source);
|
|
if (!match) continue;
|
|
try {
|
|
const parsed = JSON.parse(match[0]) as Record<string, unknown>;
|
|
const out: Record<string, string> = {};
|
|
for (const [key, value] of Object.entries(parsed)) {
|
|
if (key.startsWith('odp.') && (typeof value === 'string' || typeof value === 'number'))
|
|
out[key.slice(4)] = String(value);
|
|
}
|
|
return out;
|
|
} catch {
|
|
// Fall through to the next candidate.
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
/** The "Kenmerken" definition list: `Borg → € 1.600`, `Interieur → Gestoffeerd`. */
|
|
export function featureList(html: string): Record<string, string> {
|
|
const out: Record<string, string> = {};
|
|
const re =
|
|
/<dt\b[^>]*class="[^"]*listing-features__term[^"]*"[^>]*>([\s\S]*?)<\/dt>\s*<dd\b[^>]*>([\s\S]*?)<\/dd>/gi;
|
|
for (const m of html.matchAll(re)) {
|
|
const key = textOf(m[1]).slice(0, 80);
|
|
const value = textOf(m[2].replace(/<[^>]*class="[^"]*tooltip[^"]*"[\s\S]*?<\/[a-z]+>/gi, ''))
|
|
.replace(/\s*Meer info$/i, '')
|
|
.slice(0, 500);
|
|
if (key && value && !(key in out)) out[key] = value;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
const interiorWords: Record<string, string> = {
|
|
shell: 'kaal',
|
|
upholstered: 'gestoffeerd',
|
|
furnished: 'gemeubileerd'
|
|
};
|
|
|
|
const firstNumber = (value: LdListing['numberOfRooms']): number | null => {
|
|
if (typeof value === 'number') return value;
|
|
const item = Array.isArray(value) ? value[0] : value;
|
|
return typeof item?.value === 'number' ? item.value : null;
|
|
};
|
|
|
|
/** Parse a listing page. Returns null when the page is not a listing. */
|
|
export function parseDetailPage(html: string, fallbackUrl?: string): ListingDetail | null {
|
|
const canonical =
|
|
/<link\b[^>]*rel="canonical"[^>]*href="([^"]+)"/i.exec(html)?.[1] ??
|
|
/<link\b[^>]*href="([^"]+)"[^>]*rel="canonical"/i.exec(html)?.[1];
|
|
const ref = listingRef(canonical ?? '') ?? listingRef(fallbackUrl ?? '');
|
|
if (!ref) return null;
|
|
|
|
const ld = ldListing(html);
|
|
const odp = analyticsBlob(html);
|
|
const features = featureList(html);
|
|
const summaryTitle = classText(html, 'listing-detail-summary__title');
|
|
const location = parseLocation(classText(html, 'listing-detail-summary__location'));
|
|
|
|
const lat = Number(
|
|
/data-latitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"latitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1]
|
|
);
|
|
const lng = Number(
|
|
/data-longitude="(-?[\d.]+)"/.exec(html)?.[1] ?? /"longitude"\s*:\s*(-?[\d.]+)/.exec(html)?.[1]
|
|
);
|
|
|
|
const offeredAt =
|
|
odp.offered_since && !Number.isNaN(Date.parse(odp.offered_since))
|
|
? new Date(odp.offered_since).toISOString()
|
|
: ld?.datePosted
|
|
? new Date(`${ld.datePosted}T00:00:00Z`).toISOString()
|
|
: null;
|
|
|
|
const price =
|
|
(odp.price_pcm ? Number(odp.price_pcm) : null) ??
|
|
(typeof ld?.offers?.price === 'number' ? ld.offers.price : null) ??
|
|
parseEuros(features['Huurprijs'] ?? features['Vraagprijs'] ?? features['Koopprijs']);
|
|
|
|
const image = Array.isArray(ld?.image) ? ld.image[0] : ld?.image;
|
|
const typeFromOdp = odp.property ? typeWords[odp.property] : undefined;
|
|
const interior =
|
|
(odp.interior ? (interiorWords[odp.interior] ?? odp.interior) : null) ??
|
|
features['Interieur']?.toLowerCase() ??
|
|
null;
|
|
|
|
const street = ld?.address?.streetAddress ?? null;
|
|
const title =
|
|
summaryTitle?.replace(/^Te (huur|koop):\s*/i, '') ?? (ld?.name ? `${ld.name}` : null);
|
|
|
|
return {
|
|
sourceId: ref.sourceId,
|
|
url: ref.url,
|
|
offer: odp.offer === 'buy' || ref.offer === 'buy' ? 'buy' : 'rent',
|
|
propertyType: typeFromOdp ?? ref.propertyType,
|
|
externalId: odp.listing_id ?? null,
|
|
title: title?.slice(0, 200) ?? null,
|
|
street,
|
|
postalCode: ld?.address?.postalCode
|
|
? ld.address.postalCode.replace(/^(\d{4})\s?([A-Z]{2})$/, '$1 $2')
|
|
: location.postalCode,
|
|
city: ld?.address?.addressLocality ?? location.city ?? cityFromSlug(ref.citySlug),
|
|
neighbourhood: ld?.address?.addressRegion ?? location.neighbourhood,
|
|
priceEur: Number.isFinite(price) ? price : null,
|
|
areaM2:
|
|
parseInteger(odp.floor_space) ??
|
|
(typeof ld?.floorSize?.value === 'number' ? ld.floorSize.value : null) ??
|
|
parseInteger(features['Woonoppervlakte']),
|
|
rooms:
|
|
parseInteger(odp.rooms) ??
|
|
firstNumber(ld?.numberOfRooms) ??
|
|
parseInteger(features['Aantal kamers']),
|
|
bedrooms:
|
|
parseInteger(odp.bedrooms) ??
|
|
ld?.numberOfBedrooms ??
|
|
parseInteger(features['Aantal slaapkamers']),
|
|
bathrooms: ld?.numberOfBathroomsTotal ?? parseInteger(features['Aantal badkamers']),
|
|
constructionYear:
|
|
parseInteger(odp.construction_year) ?? ld?.yearBuilt ?? parseInteger(features['Bouwjaar']),
|
|
energyLabel: odp.energy_rating ?? features['Energielabel'] ?? null,
|
|
interior,
|
|
depositEur: parseEuros(features['Borg']),
|
|
availableFrom: features['Beschikbaar'] ?? null,
|
|
contractType: features['Contractvorm'] ?? features['Huurperiode'] ?? null,
|
|
agent: classText(html, 'agent-card__title'),
|
|
imageUrl: image ?? null,
|
|
lat: Number.isFinite(lat) && lat !== 0 ? lat : null,
|
|
lng: Number.isFinite(lng) && lng !== 0 ? lng : null,
|
|
offeredAt,
|
|
description: ld?.description?.slice(0, 6000) ?? null,
|
|
features
|
|
};
|
|
}
|