houses: nightly Kamernet + Vesteda scraper, dedupe by address, owner-only /trackers/houses
All checks were successful
Deploy production / deploy (push) Successful in 2m44s

- scripts/house-scrape.ts: polite nightly scrape (honest UA, 6-10 s apart, stops a source on 403/challenge), ingest + price history + removals, ntfy alerts on a separate topic (max 5 + summary, first run seeds)
- address_key + duplicate_of: copies on another source link to one listing, manual wins
- /projects/house-tracker -> /trackers/houses (308), owner only (404 otherwise); filters for min m2, rooms, source, Gouda region default; notes, add-by-hand form, total rent, also-on links
- scripts/house-db.sh: run the house-listing CLI as the limited house_cli role
- deploy/systemd/house-scrape.{service,timer}
This commit is contained in:
2026-10-24 14:07:33 +00:00
parent ea7d64a556
commit b51fef14fd
24 changed files with 18142 additions and 35 deletions

View File

@@ -0,0 +1,81 @@
// Kamernet search pages (kamernet.nl/huren/huurwoningen-<city>). The page is server-rendered
// Next.js; the listings sit in the __NEXT_DATA__ JSON, so no HTML scraping is needed.
// /huren/ is allowed in Kamernet's robots.txt.
import { int, str, type ScrapedListing, type SearchPage } from './scraped';
import type { PropertyType } from './types';
export const KAMERNET_ORIGIN = 'https://kamernet.nl';
// listingType -> (URL word, our type). Seen on the site: 1 kamer, 2 appartement, 4 studio.
const types: Record<number, [string, PropertyType]> = {
1: ['kamer', 'room'],
2: ['appartement', 'apartment'],
4: ['studio', 'studio'],
8: ['anti-kraak', 'other']
};
const furnishing: Record<number, string> = { 1: 'kaal', 2: 'gestoffeerd', 4: 'gemeubileerd' };
export function kamernetSearchUrl(citySlug: string, page = 1): string {
const url = `${KAMERNET_ORIGIN}/huren/huurwoningen-${citySlug}`;
return page > 1 ? `${url}?pageNo=${page}` : url;
}
type RawListing = Record<string, unknown>;
export function parseKamernetSearch(html: string): SearchPage {
const match = /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/.exec(html);
if (!match) throw new Error('kamernet: no __NEXT_DATA__ on the page');
const data = JSON.parse(match[1]);
const props = data?.props?.pageProps?.targetPageProps;
const response = props?.findListingsResponse;
if (!response || !Array.isArray(response.listings))
throw new Error('kamernet: findListingsResponse.listings missing');
return {
listings: (response.listings as RawListing[]).map(toListing).filter((l) => l != null),
total: int(response.total),
perPage: int(props?.allFilters?.listingsPerPage)
};
}
function toListing(raw: RawListing): ScrapedListing | null {
const id = int(raw.listingId);
const citySlug = str(raw.citySlug, 80);
const streetSlug = str(raw.streetSlug, 120);
if (!id || !citySlug) return null;
const [word, propertyType] = types[Number(raw.listingType)] ?? ['huurwoning', 'other'];
const url = streetSlug
? `${KAMERNET_ORIGIN}/huren/${word}-${citySlug}/${streetSlug}/${word}-${id}`
: `${KAMERNET_ORIGIN}/huren/huurwoningen-${citySlug}`;
const street = str(raw.street);
const price = int(raw.totalRentalPrice);
const start = str(raw.availabilityStartDate, 40)?.slice(0, 10) ?? null;
const end = str(raw.availabilityEndDate, 40)?.slice(0, 10) ?? null;
const features: Record<string, string> = {};
if (end) features['available until'] = end;
if (raw.utilitiesIncluded === true) features['utilities'] = 'included';
if (raw.isStudentHouseAdvert === true) features['student house'] = 'yes';
return {
sourceId: String(id),
url,
offer: 'rent',
propertyType,
title: [word[0].toUpperCase() + word.slice(1), street].filter(Boolean).join(' '),
street,
houseNumber: null, // Kamernet never shows house numbers on search pages.
postalCode: null,
city: str(raw.city, 80),
priceEur: price,
totalPriceEur: raw.utilitiesIncluded === true ? price : null,
areaM2: int(raw.surfaceArea),
rooms: propertyType === 'room' || propertyType === 'studio' ? 1 : null,
bedrooms: null,
energyLabel: null,
availableFrom: start,
interior: furnishing[Number(raw.furnishingId)] ?? null,
agent: null,
imageUrl: str(raw.resizedFullPreviewImageUrl, 1000) ?? str(raw.thumbnailUrl, 1000),
lat: null,
lng: null,
features
};
}