diff --git a/docs/house-tracker.md b/docs/house-tracker.md index fe694c4..af8d98a 100644 --- a/docs/house-tracker.md +++ b/docs/house-tracker.md @@ -61,11 +61,13 @@ the bearer token. ## Home worker setup -Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with bun. +Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with Bun 1.4 or +newer; older versions cannot read this repository's lockfile. Use `bun upgrade` +if needed. ```sh git clone https://git.stanrunge.dev/stan/personal-website ~/house-worker -cd ~/house-worker && bun install +cd ~/house-worker && bun install --frozen-lockfile cat > house.env <<'EOF' HOUSE_API_URL=https://stanrunge.dev HOUSE_WORKER_TOKEN=same-value-as-the-server @@ -130,17 +132,22 @@ red. In order: 1. Do nothing for a day. Backoff handles short blocks. 2. Slow down: raise `HOUSE_MIN_GAP_SECONDS`, lower `HOUSE_DAILY_BUDGET`, or drop slow searches. -3. `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent profile - (`npx playwright install chromium` once on the worker machine). Heavier, but - passes checks that plain HTTP clients don't. +3. Save a page manually in your normal browser to investigate parsing offline. + Do not switch clients, use proxies or solve challenges to bypass a block. -Commercial VPN exits are usually datacenter IPs and get challenged more, not less. +The optional `HOUSE_FETCH_MODE=browser` uses headless Chromium with a persistent +profile (`npx playwright install chromium` once on the worker machine). It obeys +the same stop/backoff rules and is not a workaround for a challenged fetch. ## Parsers `src/lib/house-tracker/pararius.ts`, tested against trimmed real pages in `src/lib/house-tracker/fixtures/`. Listing pages expose an exact publish timestamp (`odp.offered_since`), coordinates, deposit, energy label and agent. -The search-page fixture is hand-written; replace it with a real saved page when -the markup is confirmed. If Pararius changes markup, the worker reports -`parse-error` in the worker log on the page. +`pararius-search-gouda.html` is a trimmed real search page saved on 2026-10-11, +with tracking and account data removed. It covers three listings, nested base +and total rent, action attributes and a lazy photo template. The synthetic +fixture still covers pagination and price on request. Rental `priceEur` is the +base/listed rent used by the search filter; a separately shown total can be +higher (Raam: €1,475 base, €1,750 total). If Pararius changes markup, the worker +reports `parse-error` in the worker log on the page. diff --git a/src/lib/house-tracker/fixtures/pararius-search-gouda.html b/src/lib/house-tracker/fixtures/pararius-search-gouda.html new file mode 100644 index 0000000..560a08a --- /dev/null +++ b/src/lib/house-tracker/fixtures/pararius-search-gouda.html @@ -0,0 +1,370 @@ + + + + + Huurwoningen Gouda + + + + 3 + + + + + diff --git a/src/lib/house-tracker/fixtures/pararius-search-synthetic.html b/src/lib/house-tracker/fixtures/pararius-search-synthetic.html index ddfca97..668caf8 100644 --- a/src/lib/house-tracker/fixtures/pararius-search-synthetic.html +++ b/src/lib/house-tracker/fixtures/pararius-search-synthetic.html @@ -1,5 +1,5 @@ - + Huurwoningen Gouda diff --git a/src/lib/house-tracker/pararius.test.ts b/src/lib/house-tracker/pararius.test.ts index 2b14868..2484e70 100644 --- a/src/lib/house-tracker/pararius.test.ts +++ b/src/lib/house-tracker/pararius.test.ts @@ -8,7 +8,8 @@ import { parseEuros, parseLocation, parseSearchPage, - searchPageUrl + searchPageUrl, + textOf } from './pararius'; import { listingDetailSchema, listingCardSchema } from './types'; @@ -16,6 +17,14 @@ const fixture = (name: string) => readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8'); describe('pararius helpers', () => { + it('keeps quoted angle brackets inside tags out of visible text', () => { + expect( + textOf( + `Huis A & B` + ) + ).toBe('Huis A & B'); + }); + it('reads listing urls', () => { expect(listingRef('/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat')).toEqual({ sourceId: '541493b8', @@ -124,6 +133,72 @@ describe('parseDetailPage', () => { }); describe('parseSearchPage', () => { + it('reads the saved Gouda search with total rent and lazy photos', () => { + const page = parseSearchPage( + fixture('pararius-search-gouda.html'), + 'https://www.pararius.nl/huurwoningen/gouda/1300-1700' + ); + expect(page.totalResults).toBe(3); + expect(page.nextPageUrl).toBeNull(); + expect(page.cards).toHaveLength(3); + expect(page.cards).toMatchObject([ + { + sourceId: 'd5d8a72c', + url: 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam', + offer: 'rent', + propertyType: 'apartment', + title: 'Appartement Raam', + postalCode: '2801 VJ', + city: 'Gouda', + neighbourhood: 'Raam e.o.', + // The search filters use base rent; the card also shows €1,750 total. + priceEur: 1475, + areaM2: 66, + rooms: 3, + interior: 'Kaal', + agent: 'Pactum Vastgoed' + }, + { + sourceId: '541493b8', + url: 'https://www.pararius.nl/appartement-te-huur/gouda/541493b8/graaf-hendrikstraat', + offer: 'rent', + propertyType: 'apartment', + title: 'Appartement Graaf Hendrikstraat 16', + postalCode: '2805 CK', + city: 'Gouda', + neighbourhood: 'Ouwe Gouwe', + priceEur: 1558, + areaM2: 65, + rooms: 2, + interior: 'Gestoffeerd', + agent: 'ViaDaan' + }, + { + sourceId: '8f976237', + url: 'https://www.pararius.nl/huis-te-huur/gouda/8f976237/kongsbergstraat', + offer: 'rent', + propertyType: 'house', + title: 'Huis Kongsbergstraat 45', + postalCode: '2804 XV', + city: 'Gouda', + neighbourhood: 'De Mammoet', + priceEur: 1600, + areaM2: 113, + rooms: 5, + interior: null, + agent: 'LIV Residential' + } + ]); + for (const card of page.cards) { + expect(listingCardSchema.parse(card)).toBeTruthy(); + const image = new URL(card.imageUrl!); + expect(image.origin).toBe('https://casco-media-prod.global.ssl.fastly.net'); + expect(image.pathname).toContain(`/listings/api/media/${card.sourceId}-`); + expect(image.searchParams.get('width')).toBe('600'); + expect(image.searchParams.get('auto')).toBe('webp'); + } + }); + it('reads listing cards and pagination', () => { const page = parseSearchPage( fixture('pararius-search-synthetic.html'), diff --git a/src/lib/house-tracker/pararius.ts b/src/lib/house-tracker/pararius.ts index 1417d11..f9b95fe 100644 --- a/src/lib/house-tracker/pararius.ts +++ b/src/lib/house-tracker/pararius.ts @@ -33,6 +33,11 @@ const entities: Record = { sup3: '³' }; +// A literal ">" inside a quoted attribute (e.g. data-action="click->...") +// does not end the opening tag. +const tagAttributes = String.raw`(?:[^>"']|"[^"]*"|'[^']*')*`; +const htmlTag = new RegExp(`<${tagAttributes}>`, 'g'); + export function decodeEntities(value: string): string { return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z0-9]+);/gi, (match, code: string) => { if (code[0] === '#') { @@ -45,9 +50,7 @@ export function decodeEntities(value: string): string { } export function textOf(html: string): string { - return decodeEntities(html.replace(/<[^>]*>/g, ' ')) - .replace(/\s+/g, ' ') - .trim(); + return decodeEntities(html.replace(htmlTag, ' ')).replace(/\s+/g, ' ').trim(); } /** Bot challenges (Cloudflare "Just a moment...") instead of real content. */ @@ -125,7 +128,7 @@ export function parseLocation(value: string | null | undefined) { function classText(html: string, className: string): string | null { const re = new RegExp( - `<([a-z0-9-]+)[^>]*class="[^"]*\\b${className}\\b[^"]*"[^>]*>([\\s\\S]*?)`, + `<([a-z0-9-]+)\\b${tagAttributes}\\sclass="[^"]*\\b${className}\\b[^"]*"${tagAttributes}>([\\s\\S]*?)`, 'i' ); const match = re.exec(html); @@ -183,9 +186,9 @@ export function parseSearchPage(html: string, pageUrl: string): SearchPage { if (!ref || seen.has(ref.sourceId)) continue; seen.add(ref.sourceId); - const titleLink = /]*class="[^"]*link--title[^"]*"[^>]*>([\s\S]*?)<\/a>/i.exec(chunk); const title = - (titleLink ? textOf(titleLink[1]) : null) ?? classText(chunk, 'listing-search-item__title'); + classText(chunk, 'listing-search-item__link--title') ?? + classText(chunk, 'listing-search-item__title'); const location = parseLocation( classText(chunk, 'listing-search-item__sub-title') ?? classText(chunk, 'listing-search-item__location')