Stan Runge
b51fef14fd
All checks were successful
Deploy production / deploy (push) Successful in 2m44s
- scripts/house-scrape.ts: polite nightly scrape (honest UA, 6-10 s apart, stops a source on 403/challenge), ingest + price history + removals, ntfy alerts on a separate topic (max 5 + summary, first run seeds)
- address_key + duplicate_of: copies on another source link to one listing, manual wins
- /projects/house-tracker -> /trackers/houses (308), owner only (404 otherwise); filters for min m2, rooms, source, Gouda region default; notes, add-by-hand form, total rent, also-on links
- scripts/house-db.sh: run the house-listing CLI as the limited house_cli role
- deploy/systemd/house-scrape.{service,timer}
82 lines
3.3 KiB
TypeScript
82 lines
3.3 KiB
TypeScript
// Kamernet search pages (kamernet.nl/huren/huurwoningen-<city>). The page is server-rendered
|
|
// Next.js; the listings sit in the __NEXT_DATA__ JSON, so no HTML scraping is needed.
|
|
// /huren/ is allowed in Kamernet's robots.txt.
|
|
import { int, str, type ScrapedListing, type SearchPage } from './scraped';
|
|
import type { PropertyType } from './types';
|
|
|
|
export const KAMERNET_ORIGIN = 'https://kamernet.nl';
|
|
|
|
// listingType -> (URL word, our type). Seen on the site: 1 kamer, 2 appartement, 4 studio.
|
|
const types: Record<number, [string, PropertyType]> = {
|
|
1: ['kamer', 'room'],
|
|
2: ['appartement', 'apartment'],
|
|
4: ['studio', 'studio'],
|
|
8: ['anti-kraak', 'other']
|
|
};
|
|
const furnishing: Record<number, string> = { 1: 'kaal', 2: 'gestoffeerd', 4: 'gemeubileerd' };
|
|
|
|
export function kamernetSearchUrl(citySlug: string, page = 1): string {
|
|
const url = `${KAMERNET_ORIGIN}/huren/huurwoningen-${citySlug}`;
|
|
return page > 1 ? `${url}?pageNo=${page}` : url;
|
|
}
|
|
|
|
type RawListing = Record<string, unknown>;
|
|
|
|
export function parseKamernetSearch(html: string): SearchPage {
|
|
const match = /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/.exec(html);
|
|
if (!match) throw new Error('kamernet: no __NEXT_DATA__ on the page');
|
|
const data = JSON.parse(match[1]);
|
|
const props = data?.props?.pageProps?.targetPageProps;
|
|
const response = props?.findListingsResponse;
|
|
if (!response || !Array.isArray(response.listings))
|
|
throw new Error('kamernet: findListingsResponse.listings missing');
|
|
return {
|
|
listings: (response.listings as RawListing[]).map(toListing).filter((l) => l != null),
|
|
total: int(response.total),
|
|
perPage: int(props?.allFilters?.listingsPerPage)
|
|
};
|
|
}
|
|
|
|
function toListing(raw: RawListing): ScrapedListing | null {
|
|
const id = int(raw.listingId);
|
|
const citySlug = str(raw.citySlug, 80);
|
|
const streetSlug = str(raw.streetSlug, 120);
|
|
if (!id || !citySlug) return null;
|
|
const [word, propertyType] = types[Number(raw.listingType)] ?? ['huurwoning', 'other'];
|
|
const url = streetSlug
|
|
? `${KAMERNET_ORIGIN}/huren/${word}-${citySlug}/${streetSlug}/${word}-${id}`
|
|
: `${KAMERNET_ORIGIN}/huren/huurwoningen-${citySlug}`;
|
|
const street = str(raw.street);
|
|
const price = int(raw.totalRentalPrice);
|
|
const start = str(raw.availabilityStartDate, 40)?.slice(0, 10) ?? null;
|
|
const end = str(raw.availabilityEndDate, 40)?.slice(0, 10) ?? null;
|
|
const features: Record<string, string> = {};
|
|
if (end) features['available until'] = end;
|
|
if (raw.utilitiesIncluded === true) features['utilities'] = 'included';
|
|
if (raw.isStudentHouseAdvert === true) features['student house'] = 'yes';
|
|
return {
|
|
sourceId: String(id),
|
|
url,
|
|
offer: 'rent',
|
|
propertyType,
|
|
title: [word[0].toUpperCase() + word.slice(1), street].filter(Boolean).join(' '),
|
|
street,
|
|
houseNumber: null, // Kamernet never shows house numbers on search pages.
|
|
postalCode: null,
|
|
city: str(raw.city, 80),
|
|
priceEur: price,
|
|
totalPriceEur: raw.utilitiesIncluded === true ? price : null,
|
|
areaM2: int(raw.surfaceArea),
|
|
rooms: propertyType === 'room' || propertyType === 'studio' ? 1 : null,
|
|
bedrooms: null,
|
|
energyLabel: null,
|
|
availableFrom: start,
|
|
interior: furnishing[Number(raw.furnishingId)] ?? null,
|
|
agent: null,
|
|
imageUrl: str(raw.resizedFullPreviewImageUrl, 1000) ?? str(raw.thumbnailUrl, 1000),
|
|
lat: null,
|
|
lng: null,
|
|
features
|
|
};
|
|
}
|