stan/website
1
0
Fork 0
Code Issues 24 Pull Requests Actions Packages Projects Releases Wiki Activity

houses: keep Kamernet to the Gouda region (small-city pages are padded with listings from all over NL), share GOUDA_REGION
All checks were successful
Deploy production / deploy (push) Successful in 2m42s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-24 14:22:58 +00:00
parent 07d2c70973
commit 8818dde99d
3 changed files with 36 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
scripts/house-scrape.ts
View File

@@ -10,7 +10,7 @@
// (datacenter) IP, so they are not scraped from here.
import postgres from 'postgres';
import { alertMessage, matchesRule, type AlertRule } from '../src/lib/house-tracker/alerts';
import { citySlug } from '../src/lib/house-tracker/address';
import { citySlug, GOUDA_REGION } from '../src/lib/house-tracker/address';
import { dedupe, ingestListings } from '../src/lib/house-tracker/ingest';
import { kamernetSearchUrl, parseKamernetSearch } from '../src/lib/house-tracker/kamernet';
import { isChallenge } from '../src/lib/house-tracker/pararius';
@@ -94,6 +94,8 @@ type SourceResult = { listings: ScrapedListing[]; completed: boolean };
async function scrapeKamernet(): Promise<SourceResult> {
const listings: ScrapedListing[] = [];
const seen = new Set<string>();
const kamernetAllowed = new Set<string>([...GOUDA_REGION, ...KAMERNET_CITIES]);
let completed = true;
for (const city of KAMERNET_CITIES) {
for (let page = 1; page <= KAMERNET_MAX_PAGES; page++) {
@@ -121,15 +123,25 @@ async function scrapeKamernet(): Promise<SourceResult> {
}
try {
const result = parseKamernetSearch(html);
// Kamernet widens the search to nearby places; keep each city's own listings once.
listings.push(...result.listings);
// Small places get padded with listings from all over the country; keep the
// region only, each listing once, and stop paging once a page has none of ours.
const local = result.listings.filter((l) => kamernetAllowed.has(citySlug(l.city)));
for (const l of local) {
if (seen.has(l.sourceId)) continue;
seen.add(l.sourceId);
listings.push(l);
}
await logRun('kamernet', url, 'ok', {
httpStatus: 200,
found: result.listings.length,
durationMs: Date.now() - started
found: local.length,
durationMs: Date.now() - started,
message:
local.length < result.listings.length
? `${result.listings.length - local.length} outside the region ignored`
: undefined
});
const pages = result.total && result.perPage ? Math.ceil(result.total / result.perPage) : 1;
if (page >= pages) break;
if (page >= pages || local.length === 0) break;
} catch (error) {
await logRun('kamernet', url, 'parse-error', { httpStatus: 200, message: String(error) });
completed = false;