fix: parse real Pararius search titles correctly

This commit is contained in:
2026-10-11 21:38:47 +00:00
parent 4ea0ba2d48
commit ecd49fc410
5 changed files with 472 additions and 17 deletions

View File

@@ -61,11 +61,13 @@ the bearer token.
## Home worker setup
Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with bun.
Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with Bun 1.4 or
newer; older versions cannot read this repository's lockfile. Use `bun upgrade`
if needed.
```sh
git clone https://git.stanrunge.dev/stan/personal-website ~/house-worker
cd ~/house-worker && bun install
cd ~/house-worker && bun install --frozen-lockfile
cat > house.env <<'EOF'
HOUSE_API_URL=https://stanrunge.dev
HOUSE_WORKER_TOKEN=same-value-as-the-server
@@ -130,17 +132,22 @@ red. In order:
1. Do nothing for a day. Backoff handles short blocks.
2. Slow down: raise `HOUSE_MIN_GAP_SECONDS`, lower `HOUSE_DAILY_BUDGET`, or drop
slow searches.
3. `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent profile
(`npx playwright install chromium` once on the worker machine). Heavier, but
passes checks that plain HTTP clients don't.
3. Save a page manually in your normal browser to investigate parsing offline.
Do not switch clients, use proxies or solve challenges to bypass a block.
Commercial VPN exits are usually datacenter IPs and get challenged more, not less.
The optional `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent
profile (`npx playwright install chromium` once on the worker machine). It obeys
the same stop/backoff rules and is not a workaround for a challenged fetch.
## Parsers
`src/lib/house-tracker/pararius.ts`, tested against trimmed real pages in
`src/lib/house-tracker/fixtures/`. Listing pages expose an exact publish
timestamp (`odp.offered_since`), coordinates, deposit, energy label and agent.
The search-page fixture is hand-written; replace it with a real saved page when
the markup is confirmed. If Pararius changes markup, the worker reports
`parse-error` in the worker log on the page.
`pararius-search-gouda.html` is a trimmed real search page saved on 2026-10-11,
with tracking and account data removed. It covers three listings, nested base
and total rent, action attributes and a lazy photo template. The synthetic
fixture still covers pagination and price on request. Rental `priceEur` is the
base/listed rent used by the search filter; a separately shown total can be
higher (Raam: €1,475 base, €1,750 total). If Pararius changes markup, the worker
reports `parse-error` in the worker log on the page.

View File

@@ -0,0 +1,370 @@
<!doctype html>
<!-- Saved from Pararius on 2026-10-11. Trimmed to results and public listing data.
Tracking, account controls, responsive image variants and decorative icons removed.
Original data-action attributes, price nesting and lazy image templates retained. -->
<html lang="nl">
<head>
<title>Huurwoningen Gouda</title>
<link rel="canonical" href="https://www.pararius.nl/huurwoningen/gouda/1300-1700" />
</head>
<body>
<span class="search-list-header__count">3</span>
<wc-search-list
><ul class="search-list">
<li class="search-list__item search-list__item--listing">
<section
class="listing-search-item listing-search-item--with-total-price listing-search-item--list listing-search-item--for-rent"
>
<div class="listing-search-item__depiction">
<a
class="listing-search-item__link listing-search-item__link--depiction"
href="/appartement-te-huur/gouda/d5d8a72c/raam"
aria-label="Appartement Raam"
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
>
<wc-picture
class="picture picture--landscape picture--cover picture--with-total-price picture--list"
>
<picture>
<img
class="picture__image"
alt="Appartement Raam in Gouda"
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/d5d8a72c-e437-5100-af78-e168dc757c21/4c386ef9-f382-51d6-800a-356ee7b43f0f?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F11048%2Ffiles%2Fphotos%2Foriginals%2Fe2da7cd1.jpg&amp;sig=0fdc1e78fd3da5f1fdf46821e3b4413cc97f7090817d1cf9fba290724134403c&amp;width=600&amp;auto=webp"
width="100%"
height="100%"
fetchpriority="high"
/>
</picture>
</wc-picture>
</a>
</div>
<div class="listing-search-item__content">
<h3 class="listing-search-item__title" id="listing-title-1">
<a
class="listing-search-item__link listing-search-item__link--title"
href="/appartement-te-huur/gouda/d5d8a72c/raam"
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
>
Appartement Raam
</a>
</h3>
<div class="listing-search-item__favourite_toggle"></div>
<div class="listing-search-item__sub-title">2801 VJ Gouda (Raam e.o.)</div>
<div class="listing-search-item__price">
<div class="listing-search-item__price-bare">
<span class="listing-search-item__price-bare-label"> Kale huurprijs </span>
<span class="listing-search-item__price-value">
€ 1.475
<span class="listing-search-item__price-suffix">per maand</span>
</span>
</div>
<div class="listing-search-item__price-transparency-badge">
<wc-price-transparency-badge
data-id="d5d8a72c-e437-5100-af78-e168dc757c21"
data-report-url="/contact/gouda/d5d8a72c-e437-5100-af78-e168dc757c21/meld-een-probleem"
>
<button
type="button"
class="price-transparency-badge price-transparency-badge--listing-search-item price-transparency-badge--minimal price-transparency-badge--total-price"
>
<div class="price-transparency-badge__total-price">
<div class="price-transparency-badge__total-price-container">
<span class="price-transparency-badge__total-price-title">
Totale huurprijs
</span>
<img
src="/frontend/assets/Shared/Website/Component/PriceTransparencyBadge/Presentation/Images/Badge-PWUVFLD5.svg"
role="presentation"
class="price-transparency-badge__badge"
alt="Price transparency badge"
/>
<span class="price-transparency-badge__label"> Transparant </span>
</div>
<span class="price-transparency-badge__total-price-value">
€ 1.750
<span class="price-transparency-badge__total-price-suffix"
>per maand</span
>
</span>
</div>
</button>
</wc-price-transparency-badge>
</div>
</div>
<div class="listing-search-item__features">
<ul class="illustrated-features illustrated-features--compact">
<li class="illustrated-features__item illustrated-features__item--surface-area">
<span class="illustrated-features__item-icon"> </span>
66 m²
</li>
<li
class="illustrated-features__item illustrated-features__item--number-of-rooms"
>
<span class="illustrated-features__item-icon"> </span>
3 kamers
</li>
<li class="illustrated-features__item illustrated-features__item--interior">
<span class="illustrated-features__item-icon"> </span>
Kaal
</li>
</ul>
</div>
<div class="listing-search-item__info">
<a class="listing-search-item__link" href="/makelaars/delft/pactum-vastgoed"
>Pactum Vastgoed</a
>
</div>
</div>
</section>
</li>
<li class="search-list__item search-list__item--listing">
<section
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
>
<div class="listing-search-item__depiction">
<a
class="listing-search-item__link listing-search-item__link--depiction"
href="/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat"
aria-label="Appartement Graaf Hendrikstraat 16"
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
>
<wc-picture class="picture picture--landscape picture--cover picture--list">
<picture>
<img
class="picture__image"
alt="Appartement Graaf Hendrikstraat 16 in Gouda"
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/541493b8-5bfc-5351-b59f-ea83578dc0ab/7d19dbd1-eac5-5710-b851-85916e9fb842?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F10133%2Ffiles%2Fphotos%2Foriginals%2F9792972a.jpg&amp;sig=3f86a1fdd2e40aefab4c391adc968abe522088796ab8558eeab07e21d8d6554e&amp;width=600&amp;auto=webp"
width="100%"
height="100%"
fetchpriority="high"
/>
</picture>
</wc-picture>
</a>
</div>
<div class="listing-search-item__content">
<h3 class="listing-search-item__title" id="listing-title-2">
<a
class="listing-search-item__link listing-search-item__link--title"
href="/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat"
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
>
Appartement Graaf Hendrikstraat 16
</a>
</h3>
<div class="listing-search-item__favourite_toggle"></div>
<div class="listing-search-item__sub-title">2805 CK Gouda (Ouwe Gouwe)</div>
<div class="listing-search-item__price">
<span class="listing-search-item__price-main">
€ 1.558
<span class="listing-search-item__price-suffix">per maand</span>
</span>
</div>
<div class="listing-search-item__features">
<ul class="illustrated-features illustrated-features--compact">
<li class="illustrated-features__item illustrated-features__item--surface-area">
<span class="illustrated-features__item-icon"> </span>
65 m²
</li>
<li
class="illustrated-features__item illustrated-features__item--number-of-rooms"
>
<span class="illustrated-features__item-icon"> </span>
2 kamers
</li>
<li class="illustrated-features__item illustrated-features__item--interior">
<span class="illustrated-features__item-icon"> </span>
Gestoffeerd
</li>
</ul>
</div>
<div class="listing-search-item__info">
<a class="listing-search-item__link" href="/makelaars/eindhoven/viadaan">ViaDaan</a>
</div>
</div>
</section>
</li>
<li class="search-list__item search-list__item--listing">
<section
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
>
<div class="listing-search-item__depiction">
<a
class="listing-search-item__link listing-search-item__link--depiction"
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
aria-label="Huis Kongsbergstraat 45"
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
>
<wc-picture class="picture picture--landscape picture--cover picture--list">
<template>
<picture>
<img
class="picture__image"
alt="Huis Kongsbergstraat 45 in Gouda"
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/8f976237-2ce0-5613-adbe-883495268361/d11eeb7b-43c6-5423-93f8-9bb1570101b2?original_uri=https%3A%2F%2Flivresidential.ams3.digitaloceanspaces.com%2Fproduction%2Fproperties%2F51896%2F665a6cc40eca72e83e95774aa08b4ae1f044122a1a25892793cc863e7c17fd45.jpg&amp;sig=c7a6466d5ce0b79a71431c0016133dc004281abcfdfaa1fd12c5d48950ab2df0&amp;width=600&amp;auto=webp"
width="100%"
height="100%"
/>
</picture>
</template>
<picture>
<img
class="picture__image"
alt="Huis Kongsbergstraat 45 in Gouda"
src="data:image/svg+xml;utf8,%3Csvg%20width%3D%22600%22%20height%3D%22400%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%3Cg%20fill%3D%22none%22%20fill-rule%3D%22evenodd%22%3E%3Cpath%20fill%3D%22%23FBFBFB%22%20d%3D%22M0%200h600v400H0z%22%2F%3E%3Cpath%20fill%3D%22%23888%22%20fill-rule%3D%22nonzero%22%20d%3D%22M283.2%20277v-54.353h33.6V277h42v-72.47H384L300%20123l-84%2081.53h25.2V277z%22%2F%3E%3C%2Fg%3E%3C%2Fsvg%3E"
width="100%"
height="100%"
/>
</picture>
</wc-picture>
</a>
</div>
<div class="listing-search-item__content">
<h3 class="listing-search-item__title" id="listing-title-3">
<a
class="listing-search-item__link listing-search-item__link--title"
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
>
Huis Kongsbergstraat 45
</a>
</h3>
<div class="listing-search-item__favourite_toggle"></div>
<div class="listing-search-item__sub-title">2804 XV Gouda (De Mammoet)</div>
<div class="listing-search-item__price">
<span class="listing-search-item__price-main">
€ 1.600
<span class="listing-search-item__price-suffix">per maand</span>
</span>
</div>
<div class="listing-search-item__features">
<ul class="illustrated-features illustrated-features--compact">
<li class="illustrated-features__item illustrated-features__item--surface-area">
<span class="illustrated-features__item-icon"> </span>
113 m²
</li>
<li
class="illustrated-features__item illustrated-features__item--number-of-rooms"
>
<span class="illustrated-features__item-icon"> </span>
5 kamers
</li>
<li
class="illustrated-features__item illustrated-features__item--construction-period"
>
<span class="illustrated-features__item-icon"> </span>
1981
</li>
</ul>
</div>
<div class="listing-search-item__info">
<a class="listing-search-item__link" href="/makelaars/amsterdam/liv-residential"
>LIV Residential</a
>
</div>
</div>
</section>
</li>
</ul></wc-search-list
>
<wc-pagination class="pagination" async="1"> </wc-pagination>
<script type="application/ld+json">
{
"@context": "https://schema.org/",
"@graph": [
{
"@type": ["WebPage", "CollectionPage"],
"@id": "https://www.pararius.nl/huurwoningen/gouda/1300-1700#webpage",
"mainEntity": {
"@type": "ItemList",
"numberOfItems": 3,
"itemListElement": [
{
"@type": "ListItem",
"position": 1,
"item": {
"@type": ["RealEstateListing", "Apartment", "Product"],
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam",
"url": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam",
"name": "Raam",
"description": "Raam",
"offers": {
"@type": "Offer",
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam#offer-1",
"price": 1475,
"priceCurrency": "EUR",
"availability": "https://schema.org/LimitedAvailability"
},
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/d5d8a72c-e437-5100-af78-e168dc757c21/4c386ef9-f382-51d6-800a-356ee7b43f0f?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F11048%2Ffiles%2Fphotos%2Foriginals%2Fe2da7cd1.jpg&sig=0fdc1e78fd3da5f1fdf46821e3b4413cc97f7090817d1cf9fba290724134403c",
"geo": {
"@type": "GeoCoordinates",
"latitude": 52.009681294,
"longitude": 4.706149094
},
"datePosted": "2026-10-07",
"leaseLength": "P12M",
"yearBuilt": 2014,
"petsAllowed": "Niet toegestaan"
}
},
{
"@type": "ListItem",
"position": 2,
"item": {
"@type": ["RealEstateListing", "Apartment", "Product"],
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat",
"url": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat",
"name": "Graaf Hendrikstraat",
"description": "Graaf Hendrikstraat",
"offers": {
"@type": "Offer",
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat#offer-2",
"price": 1558,
"priceCurrency": "EUR",
"availability": "https://schema.org/LimitedAvailability"
},
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/541493b8-5bfc-5351-b59f-ea83578dc0ab/7d19dbd1-eac5-5710-b851-85916e9fb842?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F10133%2Ffiles%2Fphotos%2Foriginals%2F9792972a.jpg&sig=3f86a1fdd2e40aefab4c391adc968abe522088796ab8558eeab07e21d8d6554e",
"geo": {
"@type": "GeoCoordinates",
"latitude": 52.019575948,
"longitude": 4.715390665
},
"datePosted": "2026-10-02",
"yearBuilt": 1954
}
},
{
"@type": "ListItem",
"position": 3,
"item": {
"@type": ["RealEstateListing", "House", "Product"],
"@id": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat",
"url": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat",
"name": "Kongsbergstraat",
"description": "Kongsbergstraat",
"offers": {
"@type": "Offer",
"@id": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat#offer-3",
"price": 1600,
"priceCurrency": "EUR",
"availability": "https://schema.org/LimitedAvailability"
},
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/8f976237-2ce0-5613-adbe-883495268361/d11eeb7b-43c6-5423-93f8-9bb1570101b2?original_uri=https%3A%2F%2Flivresidential.ams3.digitaloceanspaces.com%2Fproduction%2Fproperties%2F51896%2F665a6cc40eca72e83e95774aa08b4ae1f044122a1a25892793cc863e7c17fd45.jpg&sig=c7a6466d5ce0b79a71431c0016133dc004281abcfdfaa1fd12c5d48950ab2df0",
"geo": {
"@type": "GeoCoordinates",
"latitude": 52.028376795,
"longitude": 4.717435004
},
"datePosted": "2026-09-30",
"leaseLength": "P12M",
"yearBuilt": 1981
}
}
]
}
}
]
}
</script>
</body>
</html>

View File

@@ -1,5 +1,5 @@
<!doctype html>
<!-- Hand-written in the shape of Pararius search results. Replace with a real saved page. -->
<!-- Hand-written edge cases for pagination, price on request and older card markup. -->
<html>
<head>
<title>Huurwoningen Gouda</title>

View File

@@ -8,7 +8,8 @@ import {
parseEuros,
parseLocation,
parseSearchPage,
searchPageUrl
searchPageUrl,
textOf
} from './pararius';
import { listingDetailSchema, listingCardSchema } from './types';
@@ -16,6 +17,14 @@ const fixture = (name: string) =>
readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8');
describe('pararius helpers', () => {
it('keeps quoted angle brackets inside tags out of visible text', () => {
expect(
textOf(
`<a data-action="click->listing#open" title='Price > €1,000'><b>Huis</b> A &amp; B</a>`
)
).toBe('Huis A & B');
});
it('reads listing urls', () => {
expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({
sourceId: '541493b8',
@@ -124,6 +133,72 @@ describe('parseDetailPage', () => {
});
describe('parseSearchPage', () => {
it('reads the saved Gouda search with total rent and lazy photos', () => {
const page = parseSearchPage(
fixture('pararius-search-gouda.html'),
'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
);
expect(page.totalResults).toBe(3);
expect(page.nextPageUrl).toBeNull();
expect(page.cards).toHaveLength(3);
expect(page.cards).toMatchObject([
{
sourceId: 'd5d8a72c',
url: 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam',
offer: 'rent',
propertyType: 'apartment',
title: 'Appartement Raam',
postalCode: '2801 VJ',
city: 'Gouda',
neighbourhood: 'Raam e.o.',
// The search filters use base rent; the card also shows €1,750 total.
priceEur: 1475,
areaM2: 66,
rooms: 3,
interior: 'Kaal',
agent: 'Pactum Vastgoed'
},
{
sourceId: '541493b8',
url: 'https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat',
offer: 'rent',
propertyType: 'apartment',
title: 'Appartement Graaf Hendrikstraat 16',
postalCode: '2805 CK',
city: 'Gouda',
neighbourhood: 'Ouwe Gouwe',
priceEur: 1558,
areaM2: 65,
rooms: 2,
interior: 'Gestoffeerd',
agent: 'ViaDaan'
},
{
sourceId: '8f976237',
url: 'https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat',
offer: 'rent',
propertyType: 'house',
title: 'Huis Kongsbergstraat 45',
postalCode: '2804 XV',
city: 'Gouda',
neighbourhood: 'De Mammoet',
priceEur: 1600,
areaM2: 113,
rooms: 5,
interior: null,
agent: 'LIV Residential'
}
]);
for (const card of page.cards) {
expect(listingCardSchema.parse(card)).toBeTruthy();
const image = new URL(card.imageUrl!);
expect(image.origin).toBe('https://casco-media-prod.global.ssl.fastly.net');
expect(image.pathname).toContain(`/listings/api/media/${card.sourceId}-`);
expect(image.searchParams.get('width')).toBe('600');
expect(image.searchParams.get('auto')).toBe('webp');
}
});
it('reads listing cards and pagination', () => {
const page = parseSearchPage(
fixture('pararius-search-synthetic.html'),

View File

@@ -33,6 +33,11 @@ const entities: Record<string, string> = {
sup3: '³'
};
// A literal ">" inside a quoted attribute (e.g. data-action="click->...")
// does not end the opening tag.
const tagAttributes = String.raw`(?:[^>"']|"[^"]*"|'[^']*')*`;
const htmlTag = new RegExp(`<${tagAttributes}>`, 'g');
export function decodeEntities(value: string): string {
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => {
if (code[0] === '#') {
@@ -45,9 +50,7 @@ export function decodeEntities(value: string): string {
}
export function textOf(html: string): string {
return decodeEntities(html.replace(/<[^>]*>/g, ' '))
.replace(/\s+/g, ' ')
.trim();
return decodeEntities(html.replace(htmlTag, ' ')).replace(/\s+/g, ' ').trim();
}
/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */
@@ -125,7 +128,7 @@ export function parseLocation(value: string | null | undefined) {
function classText(html: string, className: string): string | null {
const re = new RegExp(
`<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)</\\1>`,
`<([a-z0-9-]+)\\b${tagAttributes}\\sclass="[^"]*\\b${className}\\b[^"]*"${tagAttributes}>([\\s\\S]*?)</\\1>`,
'i'
);
const match = re.exec(html);
@@ -183,9 +186,9 @@ export function parseSearchPage(html: string, pageUrl: string): SearchPage {
if (!ref || seen.has(ref.sourceId)) continue;
seen.add(ref.sourceId);
const titleLink = /<a\b[^>]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk);
const title =
(titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title');
classText(chunk, 'listing-search-item__link--title') ??
classText(chunk, 'listing-search-item__title');
const location = parseLocation(
classText(chunk, 'listing-search-item__sub-title') ??
classText(chunk, 'listing-search-item__location')