fix: parse real Pararius search titles correctly
This commit is contained in:
@@ -61,11 +61,13 @@ the bearer token.
|
||||
|
||||
## Home worker setup
|
||||
|
||||
Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with bun.
|
||||
Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with Bun 1.4 or
|
||||
newer; older versions cannot read this repository's lockfile. Use `bun upgrade`
|
||||
if needed.
|
||||
|
||||
```sh
|
||||
git clone https://git.stanrunge.dev/stan/personal-website ~/house-worker
|
||||
cd ~/house-worker && bun install
|
||||
cd ~/house-worker && bun install --frozen-lockfile
|
||||
cat > house.env <<'EOF'
|
||||
HOUSE_API_URL=https://stanrunge.dev
|
||||
HOUSE_WORKER_TOKEN=same-value-as-the-server
|
||||
@@ -130,17 +132,22 @@ red. In order:
|
||||
1. Do nothing for a day. Backoff handles short blocks.
|
||||
2. Slow down: raise `HOUSE_MIN_GAP_SECONDS`, lower `HOUSE_DAILY_BUDGET`, or drop
|
||||
slow searches.
|
||||
3. `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent profile
|
||||
(`npx playwright install chromium` once on the worker machine). Heavier, but
|
||||
passes checks that plain HTTP clients don't.
|
||||
3. Save a page manually in your normal browser to investigate parsing offline.
|
||||
Do not switch clients, use proxies or solve challenges to bypass a block.
|
||||
|
||||
Commercial VPN exits are usually datacenter IPs and get challenged more, not less.
|
||||
The optional `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent
|
||||
profile (`npx playwright install chromium` once on the worker machine). It obeys
|
||||
the same stop/backoff rules and is not a workaround for a challenged fetch.
|
||||
|
||||
## Parsers
|
||||
|
||||
`src/lib/house-tracker/pararius.ts`, tested against trimmed real pages in
|
||||
`src/lib/house-tracker/fixtures/`. Listing pages expose an exact publish
|
||||
timestamp (`odp.offered_since`), coordinates, deposit, energy label and agent.
|
||||
The search-page fixture is hand-written; replace it with a real saved page when
|
||||
the markup is confirmed. If Pararius changes markup, the worker reports
|
||||
`parse-error` in the worker log on the page.
|
||||
`pararius-search-gouda.html` is a trimmed real search page saved on 2026-10-11,
|
||||
with tracking and account data removed. It covers three listings, nested base
|
||||
and total rent, action attributes and a lazy photo template. The synthetic
|
||||
fixture still covers pagination and price on request. Rental `priceEur` is the
|
||||
base/listed rent used by the search filter; a separately shown total can be
|
||||
higher (Raam: €1,475 base, €1,750 total). If Pararius changes markup, the worker
|
||||
reports `parse-error` in the worker log on the page.
|
||||
|
||||
370
src/lib/house-tracker/fixtures/pararius-search-gouda.html
Normal file
370
src/lib/house-tracker/fixtures/pararius-search-gouda.html
Normal file
@@ -0,0 +1,370 @@
|
||||
<!doctype html>
|
||||
<!-- Saved from Pararius on 2026-10-11. Trimmed to results and public listing data.
|
||||
Tracking, account controls, responsive image variants and decorative icons removed.
|
||||
Original data-action attributes, price nesting and lazy image templates retained. -->
|
||||
<html lang="nl">
|
||||
<head>
|
||||
<title>Huurwoningen Gouda</title>
|
||||
<link rel="canonical" href="https://www.pararius.nl/huurwoningen/gouda/1300-1700" />
|
||||
</head>
|
||||
<body>
|
||||
<span class="search-list-header__count">3</span>
|
||||
<wc-search-list
|
||||
><ul class="search-list">
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--with-total-price listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<div class="listing-search-item__depiction">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--depiction"
|
||||
href="/appartement-te-huur/gouda/d5d8a72c/raam"
|
||||
aria-label="Appartement Raam"
|
||||
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||
>
|
||||
<wc-picture
|
||||
class="picture picture--landscape picture--cover picture--with-total-price picture--list"
|
||||
>
|
||||
<picture>
|
||||
<img
|
||||
class="picture__image"
|
||||
alt="Appartement Raam in Gouda"
|
||||
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/d5d8a72c-e437-5100-af78-e168dc757c21/4c386ef9-f382-51d6-800a-356ee7b43f0f?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F11048%2Ffiles%2Fphotos%2Foriginals%2Fe2da7cd1.jpg&sig=0fdc1e78fd3da5f1fdf46821e3b4413cc97f7090817d1cf9fba290724134403c&width=600&auto=webp"
|
||||
width="100%"
|
||||
height="100%"
|
||||
fetchpriority="high"
|
||||
/>
|
||||
</picture>
|
||||
</wc-picture>
|
||||
</a>
|
||||
</div>
|
||||
<div class="listing-search-item__content">
|
||||
<h3 class="listing-search-item__title" id="listing-title-1">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/appartement-te-huur/gouda/d5d8a72c/raam"
|
||||
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||
>
|
||||
Appartement Raam
|
||||
</a>
|
||||
</h3>
|
||||
<div class="listing-search-item__favourite_toggle"></div>
|
||||
<div class="listing-search-item__sub-title">2801 VJ Gouda (Raam e.o.)</div>
|
||||
<div class="listing-search-item__price">
|
||||
<div class="listing-search-item__price-bare">
|
||||
<span class="listing-search-item__price-bare-label"> Kale huurprijs </span>
|
||||
<span class="listing-search-item__price-value">
|
||||
€ 1.475
|
||||
<span class="listing-search-item__price-suffix">per maand</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="listing-search-item__price-transparency-badge">
|
||||
<wc-price-transparency-badge
|
||||
data-id="d5d8a72c-e437-5100-af78-e168dc757c21"
|
||||
data-report-url="/contact/gouda/d5d8a72c-e437-5100-af78-e168dc757c21/meld-een-probleem"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
class="price-transparency-badge price-transparency-badge--listing-search-item price-transparency-badge--minimal price-transparency-badge--total-price"
|
||||
>
|
||||
<div class="price-transparency-badge__total-price">
|
||||
<div class="price-transparency-badge__total-price-container">
|
||||
<span class="price-transparency-badge__total-price-title">
|
||||
Totale huurprijs
|
||||
</span>
|
||||
<img
|
||||
src="/frontend/assets/Shared/Website/Component/PriceTransparencyBadge/Presentation/Images/Badge-PWUVFLD5.svg"
|
||||
role="presentation"
|
||||
class="price-transparency-badge__badge"
|
||||
alt="Price transparency badge"
|
||||
/>
|
||||
<span class="price-transparency-badge__label"> Transparant </span>
|
||||
</div>
|
||||
<span class="price-transparency-badge__total-price-value">
|
||||
€ 1.750
|
||||
<span class="price-transparency-badge__total-price-suffix"
|
||||
>per maand</span
|
||||
>
|
||||
</span>
|
||||
</div>
|
||||
</button>
|
||||
</wc-price-transparency-badge>
|
||||
</div>
|
||||
</div>
|
||||
<div class="listing-search-item__features">
|
||||
<ul class="illustrated-features illustrated-features--compact">
|
||||
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
66 m²
|
||||
</li>
|
||||
<li
|
||||
class="illustrated-features__item illustrated-features__item--number-of-rooms"
|
||||
>
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
3 kamers
|
||||
</li>
|
||||
<li class="illustrated-features__item illustrated-features__item--interior">
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
Kaal
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="listing-search-item__info">
|
||||
<a class="listing-search-item__link" href="/makelaars/delft/pactum-vastgoed"
|
||||
>Pactum Vastgoed</a
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</li>
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<div class="listing-search-item__depiction">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--depiction"
|
||||
href="/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat"
|
||||
aria-label="Appartement Graaf Hendrikstraat 16"
|
||||
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||
>
|
||||
<wc-picture class="picture picture--landscape picture--cover picture--list">
|
||||
<picture>
|
||||
<img
|
||||
class="picture__image"
|
||||
alt="Appartement Graaf Hendrikstraat 16 in Gouda"
|
||||
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/541493b8-5bfc-5351-b59f-ea83578dc0ab/7d19dbd1-eac5-5710-b851-85916e9fb842?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F10133%2Ffiles%2Fphotos%2Foriginals%2F9792972a.jpg&sig=3f86a1fdd2e40aefab4c391adc968abe522088796ab8558eeab07e21d8d6554e&width=600&auto=webp"
|
||||
width="100%"
|
||||
height="100%"
|
||||
fetchpriority="high"
|
||||
/>
|
||||
</picture>
|
||||
</wc-picture>
|
||||
</a>
|
||||
</div>
|
||||
<div class="listing-search-item__content">
|
||||
<h3 class="listing-search-item__title" id="listing-title-2">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat"
|
||||
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||
>
|
||||
Appartement Graaf Hendrikstraat 16
|
||||
</a>
|
||||
</h3>
|
||||
<div class="listing-search-item__favourite_toggle"></div>
|
||||
<div class="listing-search-item__sub-title">2805 CK Gouda (Ouwe Gouwe)</div>
|
||||
<div class="listing-search-item__price">
|
||||
<span class="listing-search-item__price-main">
|
||||
€ 1.558
|
||||
<span class="listing-search-item__price-suffix">per maand</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="listing-search-item__features">
|
||||
<ul class="illustrated-features illustrated-features--compact">
|
||||
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
65 m²
|
||||
</li>
|
||||
<li
|
||||
class="illustrated-features__item illustrated-features__item--number-of-rooms"
|
||||
>
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
2 kamers
|
||||
</li>
|
||||
<li class="illustrated-features__item illustrated-features__item--interior">
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
Gestoffeerd
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="listing-search-item__info">
|
||||
<a class="listing-search-item__link" href="/makelaars/eindhoven/viadaan">ViaDaan</a>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</li>
|
||||
<li class="search-list__item search-list__item--listing">
|
||||
<section
|
||||
class="listing-search-item listing-search-item--list listing-search-item--for-rent"
|
||||
>
|
||||
<div class="listing-search-item__depiction">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--depiction"
|
||||
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
|
||||
aria-label="Huis Kongsbergstraat 45"
|
||||
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||
>
|
||||
<wc-picture class="picture picture--landscape picture--cover picture--list">
|
||||
<template>
|
||||
<picture>
|
||||
<img
|
||||
class="picture__image"
|
||||
alt="Huis Kongsbergstraat 45 in Gouda"
|
||||
src="https://casco-media-prod.global.ssl.fastly.net/listings/api/media/8f976237-2ce0-5613-adbe-883495268361/d11eeb7b-43c6-5423-93f8-9bb1570101b2?original_uri=https%3A%2F%2Flivresidential.ams3.digitaloceanspaces.com%2Fproduction%2Fproperties%2F51896%2F665a6cc40eca72e83e95774aa08b4ae1f044122a1a25892793cc863e7c17fd45.jpg&sig=c7a6466d5ce0b79a71431c0016133dc004281abcfdfaa1fd12c5d48950ab2df0&width=600&auto=webp"
|
||||
width="100%"
|
||||
height="100%"
|
||||
/>
|
||||
</picture>
|
||||
</template>
|
||||
<picture>
|
||||
<img
|
||||
class="picture__image"
|
||||
alt="Huis Kongsbergstraat 45 in Gouda"
|
||||
src="data:image/svg+xml;utf8,%3Csvg%20width%3D%22600%22%20height%3D%22400%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%3Cg%20fill%3D%22none%22%20fill-rule%3D%22evenodd%22%3E%3Cpath%20fill%3D%22%23FBFBFB%22%20d%3D%22M0%200h600v400H0z%22%2F%3E%3Cpath%20fill%3D%22%23888%22%20fill-rule%3D%22nonzero%22%20d%3D%22M283.2%20277v-54.353h33.6V277h42v-72.47H384L300%20123l-84%2081.53h25.2V277z%22%2F%3E%3C%2Fg%3E%3C%2Fsvg%3E"
|
||||
width="100%"
|
||||
height="100%"
|
||||
/>
|
||||
</picture>
|
||||
</wc-picture>
|
||||
</a>
|
||||
</div>
|
||||
<div class="listing-search-item__content">
|
||||
<h3 class="listing-search-item__title" id="listing-title-3">
|
||||
<a
|
||||
class="listing-search-item__link listing-search-item__link--title"
|
||||
href="/huis-te-huur/gouda/8f976237/kongsbergstraat"
|
||||
data-action="click->listing-search-item#onClick auxclick->listing-search-item#rememberSerpOrigin contextmenu->listing-search-item#rememberSerpOrigin"
|
||||
>
|
||||
Huis Kongsbergstraat 45
|
||||
</a>
|
||||
</h3>
|
||||
<div class="listing-search-item__favourite_toggle"></div>
|
||||
<div class="listing-search-item__sub-title">2804 XV Gouda (De Mammoet)</div>
|
||||
<div class="listing-search-item__price">
|
||||
<span class="listing-search-item__price-main">
|
||||
€ 1.600
|
||||
<span class="listing-search-item__price-suffix">per maand</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="listing-search-item__features">
|
||||
<ul class="illustrated-features illustrated-features--compact">
|
||||
<li class="illustrated-features__item illustrated-features__item--surface-area">
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
113 m²
|
||||
</li>
|
||||
<li
|
||||
class="illustrated-features__item illustrated-features__item--number-of-rooms"
|
||||
>
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
5 kamers
|
||||
</li>
|
||||
<li
|
||||
class="illustrated-features__item illustrated-features__item--construction-period"
|
||||
>
|
||||
<span class="illustrated-features__item-icon"> </span>
|
||||
1981
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="listing-search-item__info">
|
||||
<a class="listing-search-item__link" href="/makelaars/amsterdam/liv-residential"
|
||||
>LIV Residential</a
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</li>
|
||||
</ul></wc-search-list
|
||||
>
|
||||
<wc-pagination class="pagination" async="1"> </wc-pagination>
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org/",
|
||||
"@graph": [
|
||||
{
|
||||
"@type": ["WebPage", "CollectionPage"],
|
||||
"@id": "https://www.pararius.nl/huurwoningen/gouda/1300-1700#webpage",
|
||||
"mainEntity": {
|
||||
"@type": "ItemList",
|
||||
"numberOfItems": 3,
|
||||
"itemListElement": [
|
||||
{
|
||||
"@type": "ListItem",
|
||||
"position": 1,
|
||||
"item": {
|
||||
"@type": ["RealEstateListing", "Apartment", "Product"],
|
||||
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam",
|
||||
"url": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam",
|
||||
"name": "Raam",
|
||||
"description": "Raam",
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam#offer-1",
|
||||
"price": 1475,
|
||||
"priceCurrency": "EUR",
|
||||
"availability": "https://schema.org/LimitedAvailability"
|
||||
},
|
||||
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/d5d8a72c-e437-5100-af78-e168dc757c21/4c386ef9-f382-51d6-800a-356ee7b43f0f?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F11048%2Ffiles%2Fphotos%2Foriginals%2Fe2da7cd1.jpg&sig=0fdc1e78fd3da5f1fdf46821e3b4413cc97f7090817d1cf9fba290724134403c",
|
||||
"geo": {
|
||||
"@type": "GeoCoordinates",
|
||||
"latitude": 52.009681294,
|
||||
"longitude": 4.706149094
|
||||
},
|
||||
"datePosted": "2026-10-07",
|
||||
"leaseLength": "P12M",
|
||||
"yearBuilt": 2014,
|
||||
"petsAllowed": "Niet toegestaan"
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "ListItem",
|
||||
"position": 2,
|
||||
"item": {
|
||||
"@type": ["RealEstateListing", "Apartment", "Product"],
|
||||
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat",
|
||||
"url": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat",
|
||||
"name": "Graaf Hendrikstraat",
|
||||
"description": "Graaf Hendrikstraat",
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"@id": "https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat#offer-2",
|
||||
"price": 1558,
|
||||
"priceCurrency": "EUR",
|
||||
"availability": "https://schema.org/LimitedAvailability"
|
||||
},
|
||||
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/541493b8-5bfc-5351-b59f-ea83578dc0ab/7d19dbd1-eac5-5710-b851-85916e9fb842?original_uri=https%3A%2F%2Fpararius-office-prod.global.ssl.fastly.net%2F10133%2Ffiles%2Fphotos%2Foriginals%2F9792972a.jpg&sig=3f86a1fdd2e40aefab4c391adc968abe522088796ab8558eeab07e21d8d6554e",
|
||||
"geo": {
|
||||
"@type": "GeoCoordinates",
|
||||
"latitude": 52.019575948,
|
||||
"longitude": 4.715390665
|
||||
},
|
||||
"datePosted": "2026-10-02",
|
||||
"yearBuilt": 1954
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "ListItem",
|
||||
"position": 3,
|
||||
"item": {
|
||||
"@type": ["RealEstateListing", "House", "Product"],
|
||||
"@id": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat",
|
||||
"url": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat",
|
||||
"name": "Kongsbergstraat",
|
||||
"description": "Kongsbergstraat",
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"@id": "https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat#offer-3",
|
||||
"price": 1600,
|
||||
"priceCurrency": "EUR",
|
||||
"availability": "https://schema.org/LimitedAvailability"
|
||||
},
|
||||
"image": "https://casco-media-prod.global.ssl.fastly.net/listings/api/media/8f976237-2ce0-5613-adbe-883495268361/d11eeb7b-43c6-5423-93f8-9bb1570101b2?original_uri=https%3A%2F%2Flivresidential.ams3.digitaloceanspaces.com%2Fproduction%2Fproperties%2F51896%2F665a6cc40eca72e83e95774aa08b4ae1f044122a1a25892793cc863e7c17fd45.jpg&sig=c7a6466d5ce0b79a71431c0016133dc004281abcfdfaa1fd12c5d48950ab2df0",
|
||||
"geo": {
|
||||
"@type": "GeoCoordinates",
|
||||
"latitude": 52.028376795,
|
||||
"longitude": 4.717435004
|
||||
},
|
||||
"datePosted": "2026-09-30",
|
||||
"leaseLength": "P12M",
|
||||
"yearBuilt": 1981
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,5 +1,5 @@
|
||||
<!doctype html>
|
||||
<!-- Hand-written in the shape of Pararius search results. Replace with a real saved page. -->
|
||||
<!-- Hand-written edge cases for pagination, price on request and older card markup. -->
|
||||
<html>
|
||||
<head>
|
||||
<title>Huurwoningen Gouda</title>
|
||||
|
||||
@@ -8,7 +8,8 @@ import {
|
||||
parseEuros,
|
||||
parseLocation,
|
||||
parseSearchPage,
|
||||
searchPageUrl
|
||||
searchPageUrl,
|
||||
textOf
|
||||
} from './pararius';
|
||||
import { listingDetailSchema, listingCardSchema } from './types';
|
||||
|
||||
@@ -16,6 +17,14 @@ const fixture = (name: string) =>
|
||||
readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8');
|
||||
|
||||
describe('pararius helpers', () => {
|
||||
it('keeps quoted angle brackets inside tags out of visible text', () => {
|
||||
expect(
|
||||
textOf(
|
||||
`<a data-action="click->listing#open" title='Price > €1,000'><b>Huis</b> A & B</a>`
|
||||
)
|
||||
).toBe('Huis A & B');
|
||||
});
|
||||
|
||||
it('reads listing urls', () => {
|
||||
expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({
|
||||
sourceId: '541493b8',
|
||||
@@ -124,6 +133,72 @@ describe('parseDetailPage', () => {
|
||||
});
|
||||
|
||||
describe('parseSearchPage', () => {
|
||||
it('reads the saved Gouda search with total rent and lazy photos', () => {
|
||||
const page = parseSearchPage(
|
||||
fixture('pararius-search-gouda.html'),
|
||||
'https://www.pararius.nl/huurwoningen/gouda/1300-1700'
|
||||
);
|
||||
expect(page.totalResults).toBe(3);
|
||||
expect(page.nextPageUrl).toBeNull();
|
||||
expect(page.cards).toHaveLength(3);
|
||||
expect(page.cards).toMatchObject([
|
||||
{
|
||||
sourceId: 'd5d8a72c',
|
||||
url: 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam',
|
||||
offer: 'rent',
|
||||
propertyType: 'apartment',
|
||||
title: 'Appartement Raam',
|
||||
postalCode: '2801 VJ',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'Raam e.o.',
|
||||
// The search filters use base rent; the card also shows €1,750 total.
|
||||
priceEur: 1475,
|
||||
areaM2: 66,
|
||||
rooms: 3,
|
||||
interior: 'Kaal',
|
||||
agent: 'Pactum Vastgoed'
|
||||
},
|
||||
{
|
||||
sourceId: '541493b8',
|
||||
url: 'https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat',
|
||||
offer: 'rent',
|
||||
propertyType: 'apartment',
|
||||
title: 'Appartement Graaf Hendrikstraat 16',
|
||||
postalCode: '2805 CK',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'Ouwe Gouwe',
|
||||
priceEur: 1558,
|
||||
areaM2: 65,
|
||||
rooms: 2,
|
||||
interior: 'Gestoffeerd',
|
||||
agent: 'ViaDaan'
|
||||
},
|
||||
{
|
||||
sourceId: '8f976237',
|
||||
url: 'https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat',
|
||||
offer: 'rent',
|
||||
propertyType: 'house',
|
||||
title: 'Huis Kongsbergstraat 45',
|
||||
postalCode: '2804 XV',
|
||||
city: 'Gouda',
|
||||
neighbourhood: 'De Mammoet',
|
||||
priceEur: 1600,
|
||||
areaM2: 113,
|
||||
rooms: 5,
|
||||
interior: null,
|
||||
agent: 'LIV Residential'
|
||||
}
|
||||
]);
|
||||
for (const card of page.cards) {
|
||||
expect(listingCardSchema.parse(card)).toBeTruthy();
|
||||
const image = new URL(card.imageUrl!);
|
||||
expect(image.origin).toBe('https://casco-media-prod.global.ssl.fastly.net');
|
||||
expect(image.pathname).toContain(`/listings/api/media/${card.sourceId}-`);
|
||||
expect(image.searchParams.get('width')).toBe('600');
|
||||
expect(image.searchParams.get('auto')).toBe('webp');
|
||||
}
|
||||
});
|
||||
|
||||
it('reads listing cards and pagination', () => {
|
||||
const page = parseSearchPage(
|
||||
fixture('pararius-search-synthetic.html'),
|
||||
|
||||
@@ -33,6 +33,11 @@ const entities: Record<string, string> = {
|
||||
sup3: '³'
|
||||
};
|
||||
|
||||
// A literal ">" inside a quoted attribute (e.g. data-action="click->...")
|
||||
// does not end the opening tag.
|
||||
const tagAttributes = String.raw`(?:[^>"']|"[^"]*"|'[^']*')*`;
|
||||
const htmlTag = new RegExp(`<${tagAttributes}>`, 'g');
|
||||
|
||||
export function decodeEntities(value: string): string {
|
||||
return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => {
|
||||
if (code[0] === '#') {
|
||||
@@ -45,9 +50,7 @@ export function decodeEntities(value: string): string {
|
||||
}
|
||||
|
||||
export function textOf(html: string): string {
|
||||
return decodeEntities(html.replace(/<[^>]*>/g, ' '))
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
return decodeEntities(html.replace(htmlTag, ' ')).replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
/** Bot challenges (Cloudflare "Just a moment...") instead of real content. */
|
||||
@@ -125,7 +128,7 @@ export function parseLocation(value: string | null | undefined) {
|
||||
|
||||
function classText(html: string, className: string): string | null {
|
||||
const re = new RegExp(
|
||||
`<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)</\\1>`,
|
||||
`<([a-z0-9-]+)\\b${tagAttributes}\\sclass="[^"]*\\b${className}\\b[^"]*"${tagAttributes}>([\\s\\S]*?)</\\1>`,
|
||||
'i'
|
||||
);
|
||||
const match = re.exec(html);
|
||||
@@ -183,9 +186,9 @@ export function parseSearchPage(html: string, pageUrl: string): SearchPage {
|
||||
if (!ref || seen.has(ref.sourceId)) continue;
|
||||
seen.add(ref.sourceId);
|
||||
|
||||
const titleLink = /<a\b[^>]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk);
|
||||
const title =
|
||||
(titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title');
|
||||
classText(chunk, 'listing-search-item__link--title') ??
|
||||
classText(chunk, 'listing-search-item__title');
|
||||
const location = parseLocation(
|
||||
classText(chunk, 'listing-search-item__sub-title') ??
|
||||
classText(chunk, 'listing-search-item__location')
|
||||
|
||||
Reference in New Issue
Block a user