houses: keep Kamernet to the Gouda region (small-city pages are padded with listings from all over NL), share GOUDA_REGION

scripts/house-scrape.ts

@@ -10,7 +10,7 @@
 // (datacenter) IP, so they are not scraped from here.
 import postgres from 'postgres';
 import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
-import { citySlug } from '../src/lib/house-tracker/address';
+import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address';
 import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
 import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
 import { isChallenge } from '../src/lib/house-tracker/pararius';
@@ -94,6 +94,8 @@ type SourceResult = { listings: ScrapedListing[]; completed: boolean };
 
 async function scrapeKamernet(): Promise<SourceResult> {
 	const listings: ScrapedListing[] = [];
+	const seen = new Set<string>();
+	const kamernetAllowed = new Set<string>([...GOUDA_REGION, ...KAMERNET_CITIES]);
 	let completed = true;
 	for (const city of KAMERNET_CITIES) {
 		for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) {
@@ -121,15 +123,25 @@ async function scrapeKamernet(): Promise<SourceResult> {
 			}
 			try {
 				const result = parseKamernetSearch(html);
-				// Kamernet widens the search to nearby places; keep each city's own listings once.
+				// Small places get padded with listings from all over the country; keep the
-				listings.push(...result.listings);
+				// region only, each listing once, and stop paging once a page has none of ours.
+				const local = result.listings.filter((l) => kamernetAllowed.has(citySlug(l.city)));
+				for (const l of local) {
+					if (seen.has(l.sourceId)) continue;
+					seen.add(l.sourceId);
+					listings.push(l);
+				}
 				await logRun('kamernet', url, 'ok', {
 					httpStatus: 200,
-					found: result.listings.length,
+					found: local.length,
-					durationMs: Date.now() - started
+					durationMs: Date.now() - started,
+					message:
+						local.length < result.listings.length
+							? `${result.listings.length - local.length} outside the region ignored`
+							: undefined
 				});
 				const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1;
-				if (page >= pages) break;
+				if (page >= pages || local.length === 0) break;
 			} catch (error) {
 				await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) });
 				completed = false;

src/lib/house-tracker/address.ts

@@ -102,3 +102,19 @@ export function addressKeyFromText(
 	const { street, houseNumber } = splitAddress(text.replace(/,.*$/, ''));
 	return addressKey(street, houseNumber, city);
 }
+
+// The area Stan and Juhee are looking in (same list as the "Gouda area rentals" alert rule):
+// the page's default filter and the Kamernet scraper's scope.
+export const GOUDA_REGION = [
+	'gouda',
+	'waddinxveen',
+	'reeuwijk',
+	'moordrecht',
+	'nieuwerkerk-aan-den-ijssel',
+	'zevenhuizen',
+	'bodegraven',
+	'boskoop',
+	'oudewater',
+	'schoonhoven',
+	'haastrecht'
+] as const;

src/routes/trackers/houses/+page.server.ts

@@ -29,22 +29,10 @@ import {
 	normalizeKeys,
 	toColumns
 } from '#lib/house-tracker/manual.js';
+import { GOUDA_REGION } from '#lib/house-tracker/address.js';
 import type { Actions, PageServerLoad } from './$types';
 
-// The area Stan and Juhee are looking in (same list as the "Gouda area rentals" alert rule).
+const REGION = GOUDA_REGION;
-const REGION = [
-	'gouda',
-	'waddinxveen',
-	'reeuwijk',
-	'moordrecht',
-	'nieuwerkerk-aan-den-ijssel',
-	'zevenhuizen',
-	'bodegraven',
-	'boskoop',
-	'oudewater',
-	'schoonhoven',
-	'haastrecht'
-] as const;
 const DEFAULTS = { city: 'region', max: '1800', area: '35' } as const;
 
 const int = (value: string | null) => {