houses: nightly Kamernet + Vesteda scraper, dedupe by address, owner-only /trackers/houses
All checks were successful
Deploy production / deploy (push) Successful in 2m44s
All checks were successful
Deploy production / deploy (push) Successful in 2m44s
- scripts/house-scrape.ts: polite nightly scrape (honest UA, 6-10 s apart, stops a source on 403/challenge), ingest + price history + removals, ntfy alerts on a separate topic (max 5 + summary, first run seeds)
- address_key + duplicate_of: copies on another source link to one listing, manual wins
- /projects/house-tracker -> /trackers/houses (308), owner only (404 otherwise); filters for min m2, rooms, source, Gouda region default; notes, add-by-hand form, total rent, also-on links
- scripts/house-db.sh: run the house-listing CLI as the limited house_cli role
- deploy/systemd/house-scrape.{service,timer}
This commit is contained in:
81
src/lib/house-tracker/kamernet.ts
Normal file
81
src/lib/house-tracker/kamernet.ts
Normal file
@@ -0,0 +1,81 @@
|
||||
// Kamernet search pages (kamernet.nl/huren/huurwoningen-<city>). The page is server-rendered
|
||||
// Next.js; the listings sit in the __NEXT_DATA__ JSON, so no HTML scraping is needed.
|
||||
// /huren/ is allowed in Kamernet's robots.txt.
|
||||
import { int, str, type ScrapedListing, type SearchPage } from './scraped';
|
||||
import type { PropertyType } from './types';
|
||||
|
||||
export const KAMERNET_ORIGIN = 'https://kamernet.nl';
|
||||
|
||||
// listingType -> (URL word, our type). Seen on the site: 1 kamer, 2 appartement, 4 studio.
|
||||
const types: Record<number, [string, PropertyType]> = {
|
||||
1: ['kamer', 'room'],
|
||||
2: ['appartement', 'apartment'],
|
||||
4: ['studio', 'studio'],
|
||||
8: ['anti-kraak', 'other']
|
||||
};
|
||||
const furnishing: Record<number, string> = { 1: 'kaal', 2: 'gestoffeerd', 4: 'gemeubileerd' };
|
||||
|
||||
export function kamernetSearchUrl(citySlug: string, page = 1): string {
|
||||
const url = `${KAMERNET_ORIGIN}/huren/huurwoningen-${citySlug}`;
|
||||
return page > 1 ? `${url}?pageNo=${page}` : url;
|
||||
}
|
||||
|
||||
type RawListing = Record<string, unknown>;
|
||||
|
||||
export function parseKamernetSearch(html: string): SearchPage {
|
||||
const match = /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/.exec(html);
|
||||
if (!match) throw new Error('kamernet: no __NEXT_DATA__ on the page');
|
||||
const data = JSON.parse(match[1]);
|
||||
const props = data?.props?.pageProps?.targetPageProps;
|
||||
const response = props?.findListingsResponse;
|
||||
if (!response || !Array.isArray(response.listings))
|
||||
throw new Error('kamernet: findListingsResponse.listings missing');
|
||||
return {
|
||||
listings: (response.listings as RawListing[]).map(toListing).filter((l) => l != null),
|
||||
total: int(response.total),
|
||||
perPage: int(props?.allFilters?.listingsPerPage)
|
||||
};
|
||||
}
|
||||
|
||||
function toListing(raw: RawListing): ScrapedListing | null {
|
||||
const id = int(raw.listingId);
|
||||
const citySlug = str(raw.citySlug, 80);
|
||||
const streetSlug = str(raw.streetSlug, 120);
|
||||
if (!id || !citySlug) return null;
|
||||
const [word, propertyType] = types[Number(raw.listingType)] ?? ['huurwoning', 'other'];
|
||||
const url = streetSlug
|
||||
? `${KAMERNET_ORIGIN}/huren/${word}-${citySlug}/${streetSlug}/${word}-${id}`
|
||||
: `${KAMERNET_ORIGIN}/huren/huurwoningen-${citySlug}`;
|
||||
const street = str(raw.street);
|
||||
const price = int(raw.totalRentalPrice);
|
||||
const start = str(raw.availabilityStartDate, 40)?.slice(0, 10) ?? null;
|
||||
const end = str(raw.availabilityEndDate, 40)?.slice(0, 10) ?? null;
|
||||
const features: Record<string, string> = {};
|
||||
if (end) features['available until'] = end;
|
||||
if (raw.utilitiesIncluded === true) features['utilities'] = 'included';
|
||||
if (raw.isStudentHouseAdvert === true) features['student house'] = 'yes';
|
||||
return {
|
||||
sourceId: String(id),
|
||||
url,
|
||||
offer: 'rent',
|
||||
propertyType,
|
||||
title: [word[0].toUpperCase() + word.slice(1), street].filter(Boolean).join(' '),
|
||||
street,
|
||||
houseNumber: null, // Kamernet never shows house numbers on search pages.
|
||||
postalCode: null,
|
||||
city: str(raw.city, 80),
|
||||
priceEur: price,
|
||||
totalPriceEur: raw.utilitiesIncluded === true ? price : null,
|
||||
areaM2: int(raw.surfaceArea),
|
||||
rooms: propertyType === 'room' || propertyType === 'studio' ? 1 : null,
|
||||
bedrooms: null,
|
||||
energyLabel: null,
|
||||
availableFrom: start,
|
||||
interior: furnishing[Number(raw.furnishingId)] ?? null,
|
||||
agent: null,
|
||||
imageUrl: str(raw.resizedFullPreviewImageUrl, 1000) ?? str(raw.thumbnailUrl, 1000),
|
||||
lat: null,
|
||||
lng: null,
|
||||
features
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user