diff --git a/scripts/house-scrape.ts b/scripts/house-scrape.ts index a0e11ca..d8d2120 100644 --- a/scripts/house-scrape.ts +++ b/scripts/house-scrape.ts @@ -10,7 +10,7 @@ // (datacenter) IP, so they are not scraped from here. import postgres from 'postgres'; import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts'; -import { citySlug } from '../src/lib/house-tracker/address'; +import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address'; import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest'; import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet'; import { isChallenge } from '../src/lib/house-tracker/pararius'; @@ -94,6 +94,8 @@ type SourceResult = { listings: ScrapedListing[]; completed: boolean }; async function scrapeKamernet(): Promise { const listings: ScrapedListing[] = []; + const seen = new Set(); + const kamernetAllowed = new Set([...GOUDA_REGION, ...KAMERNET_CITIES]); let completed = true; for (const city of KAMERNET_CITIES) { for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) { @@ -121,15 +123,25 @@ async function scrapeKamernet(): Promise { } try { const result = parseKamernetSearch(html); - // Kamernet widens the search to nearby places; keep each city's own listings once. - listings.push(...result.listings); + // Small places get padded with listings from all over the country; keep the + // region only, each listing once, and stop paging once a page has none of ours. + const local = result.listings.filter((l) => kamernetAllowed.has(citySlug(l.city))); + for (const l of local) { + if (seen.has(l.sourceId)) continue; + seen.add(l.sourceId); + listings.push(l); + } await logRun('kamernet', url, 'ok', { httpStatus: 200, - found: result.listings.length, - durationMs: Date.now() - started + found: local.length, + durationMs: Date.now() - started, + message: + local.length < result.listings.length + ? `${result.listings.length - local.length} outside the region ignored` + : undefined }); const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1; - if (page >= pages) break; + if (page >= pages || local.length === 0) break; } catch (error) { await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) }); completed = false; diff --git a/src/lib/house-tracker/address.ts b/src/lib/house-tracker/address.ts index a9ac5c7..4670785 100644 --- a/src/lib/house-tracker/address.ts +++ b/src/lib/house-tracker/address.ts @@ -102,3 +102,19 @@ export function addressKeyFromText( const { street, houseNumber } = splitAddress(text.replace(/,.*$/, '')); return addressKey(street, houseNumber, city); } + +// The area Stan and Juhee are looking in (same list as the "Gouda area rentals" alert rule): +// the page's default filter and the Kamernet scraper's scope. +export const GOUDA_REGION = [ + 'gouda', + 'waddinxveen', + 'reeuwijk', + 'moordrecht', + 'nieuwerkerk-aan-den-ijssel', + 'zevenhuizen', + 'bodegraven', + 'boskoop', + 'oudewater', + 'schoonhoven', + 'haastrecht' +] as const; diff --git a/src/routes/trackers/houses/+page.server.ts b/src/routes/trackers/houses/+page.server.ts index 2fe3d2c..902dc6a 100644 --- a/src/routes/trackers/houses/+page.server.ts +++ b/src/routes/trackers/houses/+page.server.ts @@ -29,22 +29,10 @@ import { normalizeKeys, toColumns } from '#lib/house-tracker/manual.js'; +import { GOUDA_REGION } from '#lib/house-tracker/address.js'; import type { Actions, PageServerLoad } from './$types'; -// The area Stan and Juhee are looking in (same list as the "Gouda area rentals" alert rule). -const REGION = [ - 'gouda', - 'waddinxveen', - 'reeuwijk', - 'moordrecht', - 'nieuwerkerk-aan-den-ijssel', - 'zevenhuizen', - 'bodegraven', - 'boskoop', - 'oudewater', - 'schoonhoven', - 'haastrecht' -] as const; +const REGION = GOUDA_REGION; const DEFAULTS = { city: 'region', max: '1800', area: '35' } as const; const int = (value: string | null) => {