fix: parse real Pararius search titles correctly
This commit is contained in:
@@ -61,11 +61,13 @@ the bearer token.
|
|||||||
|
|
||||||
## Home worker setup
|
## Home worker setup
|
||||||
|
|
||||||
Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with bun.
|
Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with Bun 1.4 or
|
||||||
|
newer; older versions cannot read this repository's lockfile. Use `bun upgrade`
|
||||||
|
if needed.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git clone https://git.stanrunge.dev/stan/personal-website ~/house-worker
|
git clone https://git.stanrunge.dev/stan/personal-website ~/house-worker
|
||||||
cd ~/house-worker && bun install
|
cd ~/house-worker && bun install --frozen-lockfile
|
||||||
cat > house.env <<'EOF'
|
cat > house.env <<'EOF'
|
||||||
HOUSE_API_URL=https://stanrunge.dev
|
HOUSE_API_URL=https://stanrunge.dev
|
||||||
HOUSE_WORKER_TOKEN=same-value-as-the-server
|
HOUSE_WORKER_TOKEN=same-value-as-the-server
|
||||||
@@ -130,17 +132,22 @@ red. In order:
|
|||||||
1. Do nothing for a day. Backoff handles short blocks.
|
1. Do nothing for a day. Backoff handles short blocks.
|
||||||
2. Slow down: raise `HOUSE_MIN_GAP_SECONDS`, lower `HOUSE_DAILY_BUDGET`, or drop
|
2. Slow down: raise `HOUSE_MIN_GAP_SECONDS`, lower `HOUSE_DAILY_BUDGET`, or drop
|
||||||
slow searches.
|
slow searches.
|
||||||
3. `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent profile
|
3. Save a page manually in your normal browser to investigate parsing offline.
|
||||||
(`npx playwright install chromium` once on the worker machine). Heavier, but
|
Do not switch clients, use proxies or solve challenges to bypass a block.
|
||||||
passes checks that plain HTTP clients don't.
|
|
||||||
|
|
||||||
Commercial VPN exits are usually datacenter IPs and get challenged more, not less.
|
The optional `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent
|
||||||
|
profile (`npx playwright install chromium` once on the worker machine). It obeys
|
||||||
|
the same stop/backoff rules and is not a workaround for a challenged fetch.
|
||||||
|
|
||||||
## Parsers
|
## Parsers
|
||||||
|
|
||||||
`src/lib/house-tracker/pararius.ts`, tested against trimmed real pages in
|
`src/lib/house-tracker/pararius.ts`, tested against trimmed real pages in
|
||||||
`src/lib/house-tracker/fixtures/`. Listing pages expose an exact publish
|
`src/lib/house-tracker/fixtures/`. Listing pages expose an exact publish
|
||||||
timestamp (`odp.offered_since`), coordinates, deposit, energy label and agent.
|
timestamp (`odp.offered_since`), coordinates, deposit, energy label and agent.
|
||||||
The search-page fixture is hand-written; replace it with a real saved page when
|
`pararius-search-gouda.html` is a trimmed real search page saved on 2026-10-11,
|
||||||
the markup is confirmed. If Pararius changes markup, the worker reports
|
with tracking and account data removed. It covers three listings, nested base
|
||||||
`parse-error` in the worker log on the page.
|
and total rent, action attributes and a lazy photo template. The synthetic
|
||||||
|
fixture still covers pagination and price on request. Rental `priceEur` is the
|
||||||
|
base/listed rent used by the search filter; a separately shown total can be
|
||||||
|
higher (Raam: €1,475 base, €1,750 total). If Pararius changes markup, the worker
|
||||||
|
reports `parse-error` in the worker log on the page.
|
||||||
|
|||||||
370
src/lib/house-tracker/fixtures/pararius-search-gouda.html
Normal file
370
src/lib/house-tracker/fixtures/pararius-search-gouda.html
Normal file
@@ -0,0 +1,370 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<!-- Saved from Pararius on 2026-10-11. Trimmed to results and public listing data.
|
||||||
|
Tracking, account controls, responsive image variants and decorative icons removed.
|
||||||
|
Original data-action attributes, price nesting and lazy image templates retained. -->
|
||||||
|
<html lang="nl">
|
||||||
|
<head>
|
||||||
|
<title>Huurwoningen Gouda</title>
|
||||||
|
<link rel="canonical" href="https://www.pararius.nl/huurwoningen/gouda/1300-1700" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<span class="search-list-header__count">3</span>
|
||||||
|
<wc-search-list
|
||||||
|
><ul class="search-list">
|
||||||
|
<li class="search-list__item search-list__item--listing">
|
||||||
|
<section
|
||||||
|
class="listing-search-item listing-search-item--with-total-price listing-search-item--list listing-search-item--for-rent"
|
||||||
|
>
|
||||||
|
<div class="listing-search-item__depiction">
|
||||||
|
<a
|
||||||
|
class="listing-search-item__link listing-search-item__link--depiction"
|
||||||
|
href="/appartement-te-huur/gouda/d5d8a72c/raam"
|
||||||
|
aria-label="Appartement Raam"
|
||||||
|
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||||
|
>
|
||||||
|
<wc-picture
|
||||||
|
class="picture picture--landscape picture--cover picture--with-total-price picture--list"
|
||||||
|
>
|
||||||
|
<picture>
|
||||||
|
<img
|
||||||
|
class="picture__image"
|
||||||
|
alt="Appartement Raam in Gouda"
|
||||||
|
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/d5d8a72c-e437-5100-af78-e168dc757c21/4c386ef9-f382-51d6-800a-356ee7b43f0f?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F11048%2Ffiles%2Fphotos%2Foriginals%2Fe2da7cd1.jpg&sig=0fdc1e78fd3da5f1fdf46821e3b4413cc97f7090817d1cf9fba290724134403c&width=600&auto=webp"
|
||||||
|
width="100%"
|
||||||
|
height="100%"
|
||||||
|
fetchpriority="high"
|
||||||
|
/>
|
||||||
|
</picture>
|
||||||
|
</wc-picture>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__content">
|
||||||
|
<h3 class="listing-search-item__title" id="listing-title-1">
|
||||||
|
<a
|
||||||
|
class="listing-search-item__link listing-search-item__link--title"
|
||||||
|
href="/appartement-te-huur/gouda/d5d8a72c/raam"
|
||||||
|
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||||
|
>
|
||||||
|
Appartement Raam
|
||||||
|
</a>
|
||||||
|
</h3>
|
||||||
|
<div class="listing-search-item__favourite_toggle"></div>
|
||||||
|
<div class="listing-search-item__sub-title">2801 VJ Gouda (Raam e.o.)</div>
|
||||||
|
<div class="listing-search-item__price">
|
||||||
|
<div class="listing-search-item__price-bare">
|
||||||
|
<span class="listing-search-item__price-bare-label"> Kale huurprijs </span>
|
||||||
|
<span class="listing-search-item__price-value">
|
||||||
|
€ 1.475
|
||||||
|
<span class="listing-search-item__price-suffix">per maand</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__price-transparency-badge">
|
||||||
|
<wc-price-transparency-badge
|
||||||
|
data-id="d5d8a72c-e437-5100-af78-e168dc757c21"
|
||||||
|
data-report-url="/contact/gouda/d5d8a72c-e437-5100-af78-e168dc757c21/meld-een-probleem"
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="price-transparency-badge price-transparency-badge--listing-search-item price-transparency-badge--minimal price-transparency-badge--total-price"
|
||||||
|
>
|
||||||
|
<div class="price-transparency-badge__total-price">
|
||||||
|
<div class="price-transparency-badge__total-price-container">
|
||||||
|
<span class="price-transparency-badge__total-price-title">
|
||||||
|
Totale huurprijs
|
||||||
|
</span>
|
||||||
|
<img
|
||||||
|
src="/frontend/assets/Shared/Website/Component/PriceTransparencyBadge/Presentation/Images/Badge-PWUVFLD5.svg"
|
||||||
|
role="presentation"
|
||||||
|
class="price-transparency-badge__badge"
|
||||||
|
alt="Price transparency badge"
|
||||||
|
/>
|
||||||
|
<span class="price-transparency-badge__label"> Transparant </span>
|
||||||
|
</div>
|
||||||
|
<span class="price-transparency-badge__total-price-value">
|
||||||
|
€ 1.750
|
||||||
|
<span class="price-transparency-badge__total-price-suffix"
|
||||||
|
>per maand</span
|
||||||
|
>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
|
</wc-price-transparency-badge>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__features">
|
||||||
|
<ul class="illustrated-features illustrated-features--compact">
|
||||||
|
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
66 m²
|
||||||
|
</li>
|
||||||
|
<li
|
||||||
|
class="illustrated-features__item illustrated-features__item--number-of-rooms"
|
||||||
|
>
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
3 kamers
|
||||||
|
</li>
|
||||||
|
<li class="illustrated-features__item illustrated-features__item--interior">
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
Kaal
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__info">
|
||||||
|
<a class="listing-search-item__link" href="/makelaars/delft/pactum-vastgoed"
|
||||||
|
>Pactum Vastgoed</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</li>
|
||||||
|
<li class="search-list__item search-list__item--listing">
|
||||||
|
<section
|
||||||
|
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||||
|
>
|
||||||
|
<div class="listing-search-item__depiction">
|
||||||
|
<a
|
||||||
|
class="listing-search-item__link listing-search-item__link--depiction"
|
||||||
|
href="/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat"
|
||||||
|
aria-label="Appartement Graaf Hendrikstraat 16"
|
||||||
|
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||||
|
>
|
||||||
|
<wc-picture class="picture picture--landscape picture--cover picture--list">
|
||||||
|
<picture>
|
||||||
|
<img
|
||||||
|
class="picture__image"
|
||||||
|
alt="Appartement Graaf Hendrikstraat 16 in Gouda"
|
||||||
|
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/541493b8-5bfc-5351-b59f-ea83578dc0ab/7d19dbd1-eac5-5710-b851-85916e9fb842?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F10133%2Ffiles%2Fphotos%2Foriginals%2F9792972a.jpg&sig=3f86a1fdd2e40aefab4c391adc968abe522088796ab8558eeab07e21d8d6554e&width=600&auto=webp"
|
||||||
|
width="100%"
|
||||||
|
height="100%"
|
||||||
|
fetchpriority="high"
|
||||||
|
/>
|
||||||
|
</picture>
|
||||||
|
</wc-picture>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__content">
|
||||||
|
<h3 class="listing-search-item__title" id="listing-title-2">
|
||||||
|
<a
|
||||||
|
class="listing-search-item__link listing-search-item__link--title"
|
||||||
|
href="/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat"
|
||||||
|
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||||
|
>
|
||||||
|
Appartement Graaf Hendrikstraat 16
|
||||||
|
</a>
|
||||||
|
</h3>
|
||||||
|
<div class="listing-search-item__favourite_toggle"></div>
|
||||||
|
<div class="listing-search-item__sub-title">2805 CK Gouda (Ouwe Gouwe)</div>
|
||||||
|
<div class="listing-search-item__price">
|
||||||
|
<span class="listing-search-item__price-main">
|
||||||
|
€ 1.558
|
||||||
|
<span class="listing-search-item__price-suffix">per maand</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__features">
|
||||||
|
<ul class="illustrated-features illustrated-features--compact">
|
||||||
|
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
65 m²
|
||||||
|
</li>
|
||||||
|
<li
|
||||||
|
class="illustrated-features__item illustrated-features__item--number-of-rooms"
|
||||||
|
>
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
2 kamers
|
||||||
|
</li>
|
||||||
|
<li class="illustrated-features__item illustrated-features__item--interior">
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
Gestoffeerd
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__info">
|
||||||
|
<a class="listing-search-item__link" href="/makelaars/eindhoven/viadaan">ViaDaan</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</li>
|
||||||
|
<li class="search-list__item search-list__item--listing">
|
||||||
|
<section
|
||||||
|
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||||
|
>
|
||||||
|
<div class="listing-search-item__depiction">
|
||||||
|
<a
|
||||||
|
class="listing-search-item__link listing-search-item__link--depiction"
|
||||||
|
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
|
||||||
|
aria-label="Huis Kongsbergstraat 45"
|
||||||
|
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||||
|
>
|
||||||
|
<wc-picture class="picture picture--landscape picture--cover picture--list">
|
||||||
|
<template>
|
||||||
|
<picture>
|
||||||
|
<img
|
||||||
|
class="picture__image"
|
||||||
|
alt="Huis Kongsbergstraat 45 in Gouda"
|
||||||
|
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/8f976237-2ce0-5613-adbe-883495268361/d11eeb7b-43c6-5423-93f8-9bb1570101b2?original_uri=https%3A%2F%2Flivresidential.ams3.digitaloceanspaces.com%2Fproduction%2Fproperties%2F51896%2F665a6cc40eca72e83e95774aa08b4ae1f044122a1a25892793cc863e7c17fd45.jpg&sig=c7a6466d5ce0b79a71431c0016133dc004281abcfdfaa1fd12c5d48950ab2df0&width=600&auto=webp"
|
||||||
|
width="100%"
|
||||||
|
height="100%"
|
||||||
|
/>
|
||||||
|
</picture>
|
||||||
|
</template>
|
||||||
|
<picture>
|
||||||
|
<img
|
||||||
|
class="picture__image"
|
||||||
|
alt="Huis Kongsbergstraat 45 in Gouda"
|
||||||
|
src="data:image/svg+xml;utf8,%3Csvg%20width%3D%22600%22%20height%3D%22400%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%3Cg%20fill%3D%22none%22%20fill-rule%3D%22evenodd%22%3E%3Cpath%20fill%3D%22%23FBFBFB%22%20d%3D%22M0%200h600v400H0z%22%2F%3E%3Cpath%20fill%3D%22%23888%22%20fill-rule%3D%22nonzero%22%20d%3D%22M283.2%20277v-54.353h33.6V277h42v-72.47H384L300%20123l-84%2081.53h25.2V277z%22%2F%3E%3C%2Fg%3E%3C%2Fsvg%3E"
|
||||||
|
width="100%"
|
||||||
|
height="100%"
|
||||||
|
/>
|
||||||
|
</picture>
|
||||||
|
</wc-picture>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__content">
|
||||||
|
<h3 class="listing-search-item__title" id="listing-title-3">
|
||||||
|
<a
|
||||||
|
class="listing-search-item__link listing-search-item__link--title"
|
||||||
|
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
|
||||||
|
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||||
|
>
|
||||||
|
Huis Kongsbergstraat 45
|
||||||
|
</a>
|
||||||
|
</h3>
|
||||||
|
<div class="listing-search-item__favourite_toggle"></div>
|
||||||
|
<div class="listing-search-item__sub-title">2804 XV Gouda (De Mammoet)</div>
|
||||||
|
<div class="listing-search-item__price">
|
||||||
|
<span class="listing-search-item__price-main">
|
||||||
|
€ 1.600
|
||||||
|
<span class="listing-search-item__price-suffix">per maand</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__features">
|
||||||
|
<ul class="illustrated-features illustrated-features--compact">
|
||||||
|
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
113 m²
|
||||||
|
</li>
|
||||||
|
<li
|
||||||
|
class="illustrated-features__item illustrated-features__item--number-of-rooms"
|
||||||
|
>
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
5 kamers
|
||||||
|
</li>
|
||||||
|
<li
|
||||||
|
class="illustrated-features__item illustrated-features__item--construction-period"
|
||||||
|
>
|
||||||
|
<span class="illustrated-features__item-icon"> </span>
|
||||||
|
1981
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="listing-search-item__info">
|
||||||
|
<a class="listing-search-item__link" href="/makelaars/amsterdam/liv-residential"
|
||||||
|
>LIV Residential</a
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</li>
|
||||||
|
</ul></wc-search-list
|
||||||
|
>
|
||||||
|
<wc-pagination class="pagination" async="1"> </wc-pagination>
|
||||||
|
<script type="application/ld+json">
|
||||||
|
{
|
||||||
|
"@context": "https://schema.org/",
|
||||||
|
"@graph": [
|
||||||
|
{
|
||||||
|
"@type": ["WebPage", "CollectionPage"],
|
||||||
|
"@id": "https://www.pararius.nl/huurwoningen/gouda/1300-1700#webpage",
|
||||||
|
"mainEntity": {
|
||||||
|
"@type": "ItemList",
|
||||||
|
"numberOfItems": 3,
|
||||||
|
"itemListElement": [
|
||||||
|
{
|
||||||
|
"@type": "ListItem",
|
||||||
|
"position": 1,
|
||||||
|
"item": {
|
||||||
|
"@type": ["RealEstateListing", "Apartment", "Product"],
|
||||||
|
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam",
|
||||||
|
"url": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam",
|
||||||
|
"name": "Raam",
|
||||||
|
"description": "Raam",
|
||||||
|
"offers": {
|
||||||
|
"@type": "Offer",
|
||||||
|
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam#offer-1",
|
||||||
|
"price": 1475,
|
||||||
|
"priceCurrency": "EUR",
|
||||||
|
"availability": "https://schema.org/LimitedAvailability"
|
||||||
|
},
|
||||||
|
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/d5d8a72c-e437-5100-af78-e168dc757c21/4c386ef9-f382-51d6-800a-356ee7b43f0f?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F11048%2Ffiles%2Fphotos%2Foriginals%2Fe2da7cd1.jpg&sig=0fdc1e78fd3da5f1fdf46821e3b4413cc97f7090817d1cf9fba290724134403c",
|
||||||
|
"geo": {
|
||||||
|
"@type": "GeoCoordinates",
|
||||||
|
"latitude": 52.009681294,
|
||||||
|
"longitude": 4.706149094
|
||||||
|
},
|
||||||
|
"datePosted": "2026-10-07",
|
||||||
|
"leaseLength": "P12M",
|
||||||
|
"yearBuilt": 2014,
|
||||||
|
"petsAllowed": "Niet toegestaan"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"@type": "ListItem",
|
||||||
|
"position": 2,
|
||||||
|
"item": {
|
||||||
|
"@type": ["RealEstateListing", "Apartment", "Product"],
|
||||||
|
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat",
|
||||||
|
"url": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat",
|
||||||
|
"name": "Graaf Hendrikstraat",
|
||||||
|
"description": "Graaf Hendrikstraat",
|
||||||
|
"offers": {
|
||||||
|
"@type": "Offer",
|
||||||
|
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat#offer-2",
|
||||||
|
"price": 1558,
|
||||||
|
"priceCurrency": "EUR",
|
||||||
|
"availability": "https://schema.org/LimitedAvailability"
|
||||||
|
},
|
||||||
|
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/541493b8-5bfc-5351-b59f-ea83578dc0ab/7d19dbd1-eac5-5710-b851-85916e9fb842?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F10133%2Ffiles%2Fphotos%2Foriginals%2F9792972a.jpg&sig=3f86a1fdd2e40aefab4c391adc968abe522088796ab8558eeab07e21d8d6554e",
|
||||||
|
"geo": {
|
||||||
|
"@type": "GeoCoordinates",
|
||||||
|
"latitude": 52.019575948,
|
||||||
|
"longitude": 4.715390665
|
||||||
|
},
|
||||||
|
"datePosted": "2026-10-02",
|
||||||
|
"yearBuilt": 1954
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"@type": "ListItem",
|
||||||
|
"position": 3,
|
||||||
|
"item": {
|
||||||
|
"@type": ["RealEstateListing", "House", "Product"],
|
||||||
|
"@id": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat",
|
||||||
|
"url": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat",
|
||||||
|
"name": "Kongsbergstraat",
|
||||||
|
"description": "Kongsbergstraat",
|
||||||
|
"offers": {
|
||||||
|
"@type": "Offer",
|
||||||
|
"@id": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat#offer-3",
|
||||||
|
"price": 1600,
|
||||||
|
"priceCurrency": "EUR",
|
||||||
|
"availability": "https://schema.org/LimitedAvailability"
|
||||||
|
},
|
||||||
|
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/8f976237-2ce0-5613-adbe-883495268361/d11eeb7b-43c6-5423-93f8-9bb1570101b2?original_uri=https%3A%2F%2Flivresidential.ams3.digitaloceanspaces.com%2Fproduction%2Fproperties%2F51896%2F665a6cc40eca72e83e95774aa08b4ae1f044122a1a25892793cc863e7c17fd45.jpg&sig=c7a6466d5ce0b79a71431c0016133dc004281abcfdfaa1fd12c5d48950ab2df0",
|
||||||
|
"geo": {
|
||||||
|
"@type": "GeoCoordinates",
|
||||||
|
"latitude": 52.028376795,
|
||||||
|
"longitude": 4.717435004
|
||||||
|
},
|
||||||
|
"datePosted": "2026-09-30",
|
||||||
|
"leaseLength": "P12M",
|
||||||
|
"yearBuilt": 1981
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
<!doctype html>
|
<!doctype html>
|
||||||
<!-- Hand-written in the shape of Pararius search results. Replace with a real saved page. -->
|
<!-- Hand-written edge cases for pagination, price on request and older card markup. -->
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
<title>Huurwoningen Gouda</title>
|
<title>Huurwoningen Gouda</title>
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ import {
|
|||||||
parseEuros,
|
parseEuros,
|
||||||
parseLocation,
|
parseLocation,
|
||||||
parseSearchPage,
|
parseSearchPage,
|
||||||
searchPageUrl
|
searchPageUrl,
|
||||||
|
textOf
|
||||||
} from './pararius';
|
} from './pararius';
|
||||||
import { listingDetailSchema, listingCardSchema } from './types';
|
import { listingDetailSchema, listingCardSchema } from './types';
|
||||||
|
|
||||||
@@ -16,6 +17,14 @@ const fixture = (name: string) =>
|
|||||||
readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8');
|
readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8');
|
||||||
|
|
||||||
describe('pararius helpers', () => {
|
describe('pararius helpers', () => {
|
||||||
|
it('keeps quoted angle brackets inside tags out of visible text', () => {
|
||||||
|
expect(
|
||||||
|
textOf(
|
||||||
|
`<a data-action="click->listing#open" title='Price > €1,000'><b>Huis</b> A & B</a>`
|
||||||
|
)
|
||||||
|
).toBe('Huis A & B');
|
||||||
|
});
|
||||||
|
|
||||||
it('reads listing urls', () => {
|
it('reads listing urls', () => {
|
||||||
expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({
|
expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({
|
||||||
sourceId: '541493b8',
|
sourceId: '541493b8',
|
||||||
@@ -124,6 +133,72 @@ describe('parseDetailPage', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('parseSearchPage', () => {
|
describe('parseSearchPage', () => {
|
||||||
|
it('reads the saved Gouda search with total rent and lazy photos', () => {
|
||||||
|
const page = parseSearchPage(
|
||||||
|
fixture('pararius-search-gouda.html'),
|
||||||
|
'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
|
||||||
|
);
|
||||||
|
expect(page.totalResults).toBe(3);
|
||||||
|
expect(page.nextPageUrl).toBeNull();
|
||||||
|
expect(page.cards).toHaveLength(3);
|
||||||
|
expect(page.cards).toMatchObject([
|
||||||
|
{
|
||||||
|
sourceId: 'd5d8a72c',
|
||||||
|
url: 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam',
|
||||||
|
offer: 'rent',
|
||||||
|
propertyType: 'apartment',
|
||||||
|
title: 'Appartement Raam',
|
||||||
|
postalCode: '2801 VJ',
|
||||||
|
city: 'Gouda',
|
||||||
|
neighbourhood: 'Raam e.o.',
|
||||||
|
// The search filters use base rent; the card also shows €1,750 total.
|
||||||
|
priceEur: 1475,
|
||||||
|
areaM2: 66,
|
||||||
|
rooms: 3,
|
||||||
|
interior: 'Kaal',
|
||||||
|
agent: 'Pactum Vastgoed'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
sourceId: '541493b8',
|
||||||
|
url: 'https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat',
|
||||||
|
offer: 'rent',
|
||||||
|
propertyType: 'apartment',
|
||||||
|
title: 'Appartement Graaf Hendrikstraat 16',
|
||||||
|
postalCode: '2805 CK',
|
||||||
|
city: 'Gouda',
|
||||||
|
neighbourhood: 'Ouwe Gouwe',
|
||||||
|
priceEur: 1558,
|
||||||
|
areaM2: 65,
|
||||||
|
rooms: 2,
|
||||||
|
interior: 'Gestoffeerd',
|
||||||
|
agent: 'ViaDaan'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
sourceId: '8f976237',
|
||||||
|
url: 'https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat',
|
||||||
|
offer: 'rent',
|
||||||
|
propertyType: 'house',
|
||||||
|
title: 'Huis Kongsbergstraat 45',
|
||||||
|
postalCode: '2804 XV',
|
||||||
|
city: 'Gouda',
|
||||||
|
neighbourhood: 'De Mammoet',
|
||||||
|
priceEur: 1600,
|
||||||
|
areaM2: 113,
|
||||||
|
rooms: 5,
|
||||||
|
interior: null,
|
||||||
|
agent: 'LIV Residential'
|
||||||
|
}
|
||||||
|
]);
|
||||||
|
for (const card of page.cards) {
|
||||||
|
expect(listingCardSchema.parse(card)).toBeTruthy();
|
||||||
|
const image = new URL(card.imageUrl!);
|
||||||
|
expect(image.origin).toBe('https://casco-media-prod.global.ssl.fastly.net');
|
||||||
|
expect(image.pathname).toContain(`/listings/api/media/${card.sourceId}-`);
|
||||||
|
expect(image.searchParams.get('width')).toBe('600');
|
||||||
|
expect(image.searchParams.get('auto')).toBe('webp');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('reads listing cards and pagination', () => {
|
it('reads listing cards and pagination', () => {
|
||||||
const page = parseSearchPage(
|
const page = parseSearchPage(
|
||||||
fixture('pararius-search-synthetic.html'),
|
fixture('pararius-search-synthetic.html'),
|
||||||
|
|||||||
@@ -33,6 +33,11 @@ const entities: Record<string, string> = {
|
|||||||
sup3: '³'
|
sup3: '³'
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// A literal ">" inside a quoted attribute (e.g. data-action="click->...")
|
||||||
|
// does not end the opening tag.
|
||||||
|
const tagAttributes = String.raw`(?:[^>"']|"[^"]*"|'[^']*')*`;
|
||||||
|
const htmlTag = new RegExp(`<${tagAttributes}>`, 'g');
|
||||||
|
|
||||||
export function decodeEntities(value: string): string {
|
export function decodeEntities(value: string): string {
|
||||||
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => {
|
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => {
|
||||||
if (code[0] === '#') {
|
if (code[0] === '#') {
|
||||||
@@ -45,9 +50,7 @@ export function decodeEntities(value: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function textOf(html: string): string {
|
export function textOf(html: string): string {
|
||||||
return decodeEntities(html.replace(/<[^>]*>/g, ' '))
|
return decodeEntities(html.replace(htmlTag, ' ')).replace(/\s+/g, ' ').trim();
|
||||||
.replace(/\s+/g, ' ')
|
|
||||||
.trim();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */
|
/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */
|
||||||
@@ -125,7 +128,7 @@ export function parseLocation(value: string | null | undefined) {
|
|||||||
|
|
||||||
function classText(html: string, className: string): string | null {
|
function classText(html: string, className: string): string | null {
|
||||||
const re = new RegExp(
|
const re = new RegExp(
|
||||||
`<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)</\\1>`,
|
`<([a-z0-9-]+)\\b${tagAttributes}\\sclass="[^"]*\\b${className}\\b[^"]*"${tagAttributes}>([\\s\\S]*?)</\\1>`,
|
||||||
'i'
|
'i'
|
||||||
);
|
);
|
||||||
const match = re.exec(html);
|
const match = re.exec(html);
|
||||||
@@ -183,9 +186,9 @@ export function parseSearchPage(html: string, pageUrl: string): SearchPage {
|
|||||||
if (!ref || seen.has(ref.sourceId)) continue;
|
if (!ref || seen.has(ref.sourceId)) continue;
|
||||||
seen.add(ref.sourceId);
|
seen.add(ref.sourceId);
|
||||||
|
|
||||||
const titleLink = /<a\b[^>]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk);
|
|
||||||
const title =
|
const title =
|
||||||
(titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title');
|
classText(chunk, 'listing-search-item__link--title') ??
|
||||||
|
classText(chunk, 'listing-search-item__title');
|
||||||
const location = parseLocation(
|
const location = parseLocation(
|
||||||
classText(chunk, 'listing-search-item__sub-title') ??
|
classText(chunk, 'listing-search-item__sub-title') ??
|
||||||
classText(chunk, 'listing-search-item__location')
|
classText(chunk, 'listing-search-item__location')
|
||||||
|
|||||||
Reference in New Issue
Block a user