house tracker: CLI to add, update, list and remove listings by hand
scripts/house-listing.ts (bun run house add|update|list|remove|restore) upserts house.listings on (source, source_id) derived from the listing URL or address, derives city_slug, records price_history, and accepts flags or JSON.
This commit is contained in:
80
src/lib/house-tracker/manual.test.ts
Normal file
80
src/lib/house-tracker/manual.test.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
citySlugFromName,
|
||||
keyFromAddress,
|
||||
keyFromUrl,
|
||||
manualListingSchema,
|
||||
normalizeKeys,
|
||||
toColumns
|
||||
} from './manual';
|
||||
|
||||
describe('citySlugFromName', () => {
|
||||
it('matches the Pararius slug style', () => {
|
||||
expect(citySlugFromName('Gouda')).toBe('gouda');
|
||||
expect(citySlugFromName('Nieuwerkerk aan den IJssel')).toBe('nieuwerkerk-aan-den-ijssel');
|
||||
expect(citySlugFromName("'s-Gravenhage")).toBe('den-haag');
|
||||
expect(citySlugFromName(' ')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('keyFromUrl', () => {
|
||||
it('uses the worker key for Pararius', () => {
|
||||
expect(
|
||||
keyFromUrl('https://www.pararius.nl/appartement-te-huur/gouda/5179bb6d/raesbergenstraat')
|
||||
).toMatchObject({ source: 'pararius', sourceId: '5179bb6d', citySlug: 'gouda', offer: 'rent' });
|
||||
});
|
||||
|
||||
it('uses the numeric id for Funda, whatever the URL shape', () => {
|
||||
const long = keyFromUrl(
|
||||
'https://www.funda.nl/detail/huur/gouda/appartement-raoul-wallenbergplantsoen-11/43307383/'
|
||||
);
|
||||
const short = keyFromUrl(
|
||||
'https://www.funda.nl/detail/43307383?utm_source=funda&utm_medium=web&utm_campaign=share-listing-modal'
|
||||
);
|
||||
expect(long).toMatchObject({ source: 'funda', sourceId: '43307383' });
|
||||
expect(short).toMatchObject({ source: 'funda', sourceId: '43307383' });
|
||||
expect(short.url).toBe('https://www.funda.nl/detail/43307383');
|
||||
});
|
||||
|
||||
it('uses the domain and path for other sites', () => {
|
||||
expect(keyFromUrl('https://vbtverhuurmakelaars.nl/woning/gouda-vest-7-d/')).toMatchObject({
|
||||
source: 'vbt',
|
||||
sourceId: 'woning/gouda-vest-7-d'
|
||||
});
|
||||
expect(
|
||||
keyFromUrl('https://www.dupreemakelaars.nl/aanbod/gouda-raam-7a/?fbclid=abc&sfnsn=wa')
|
||||
).toEqual({
|
||||
source: 'dupreemakelaars',
|
||||
sourceId: 'aanbod/gouda-raam-7a',
|
||||
url: 'https://www.dupreemakelaars.nl/aanbod/gouda-raam-7a/'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('keyFromAddress', () => {
|
||||
it('is stable and case-insensitive', () => {
|
||||
const a = keyFromAddress({ street: 'Vest 7-D', city: 'Gouda' });
|
||||
const b = keyFromAddress({ street: 'vest 7-d', city: 'GOUDA' });
|
||||
expect(b.sourceId).toBe(a.sourceId);
|
||||
expect(a.source).toBe('manual');
|
||||
expect(a.url).toContain('google.com/maps');
|
||||
});
|
||||
|
||||
it('needs something to go on', () => {
|
||||
expect(() => keyFromAddress({})).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('manualListingSchema', () => {
|
||||
it('is lenient about typed numbers', () => {
|
||||
const parsed = manualListingSchema.parse(
|
||||
normalizeKeys({ price: '€ 1.422,- p.m.', area_m2: '84 m²', bedrooms: '2', lat: '52,01' })
|
||||
);
|
||||
expect(parsed).toMatchObject({ priceEur: 1422, areaM2: 84, bedrooms: 2, lat: 52.01 });
|
||||
});
|
||||
|
||||
it('only maps given fields to columns', () => {
|
||||
const parsed = manualListingSchema.parse({ city: 'Gouda', priceEur: 1350 });
|
||||
expect(toColumns(parsed)).toEqual({ city: 'Gouda', city_slug: 'gouda', price_eur: 1350 });
|
||||
});
|
||||
});
|
||||
255
src/lib/house-tracker/manual.ts
Normal file
255
src/lib/house-tracker/manual.ts
Normal file
@@ -0,0 +1,255 @@
|
||||
// Manually added listings (scripts/house-listing.ts). Pure functions, no DB or
|
||||
// network, so the CLI and tests share them. Keep free of `#lib` / `$app` imports.
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as z from 'zod';
|
||||
import { listingRef, parseEuros, parseInteger } from './pararius';
|
||||
import {
|
||||
listingDetailSchema,
|
||||
listingStatuses,
|
||||
offers,
|
||||
propertyTypes,
|
||||
type Offer,
|
||||
type PropertyType
|
||||
} from './types';
|
||||
|
||||
// Short, stable source names for sites we see often. Anything else uses the
|
||||
// registrable domain name without TLD ("www.dupreemakelaars.nl" → "dupreemakelaars").
|
||||
const sourceAliases: Record<string, string> = {
|
||||
'vbtverhuurmakelaars.nl': 'vbt',
|
||||
'ikwilhuren.nu': 'ikwilhuren',
|
||||
'huurportaal.nl': 'huurportaal',
|
||||
'middenholland.mijndak.nl': 'mijndak'
|
||||
};
|
||||
|
||||
const cityAliases: Record<string, string> = {
|
||||
's-gravenhage': 'den-haag',
|
||||
'the-hague': 'den-haag',
|
||||
's-hertogenbosch': 'den-bosch'
|
||||
};
|
||||
|
||||
/** City name → slug in Pararius' style: "Nieuwerkerk aan den IJssel" → "nieuwerkerk-aan-den-ijssel". */
|
||||
export function citySlugFromName(city: string | null | undefined): string | null {
|
||||
const slug = (city ?? '')
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
if (!slug) return null;
|
||||
return cityAliases[slug] ?? slug;
|
||||
}
|
||||
|
||||
const hash = (value: string, length = 16) =>
|
||||
createHash('sha256').update(value).digest('hex').slice(0, length);
|
||||
|
||||
export type ListingKey = {
|
||||
source: string;
|
||||
sourceId: string;
|
||||
url: string;
|
||||
citySlug?: string;
|
||||
offer?: Offer;
|
||||
propertyType?: PropertyType;
|
||||
};
|
||||
|
||||
/**
|
||||
* Deterministic (source, sourceId) for a listing URL, so adding the same link
|
||||
* twice updates one row. Pararius uses the worker's own key, so a manual add and
|
||||
* a later scrape end up as the same listing.
|
||||
*/
|
||||
export function keyFromUrl(raw: string): ListingKey {
|
||||
const url = new URL(raw.trim());
|
||||
const host = url.hostname
|
||||
.toLowerCase()
|
||||
.replace(/^www\./, '')
|
||||
.replace(/^m\./, '');
|
||||
|
||||
if (host.endsWith('pararius.nl') || host.endsWith('pararius.com')) {
|
||||
const ref = listingRef(url.pathname);
|
||||
if (ref) return { source: 'pararius', ...ref };
|
||||
}
|
||||
if (host.endsWith('funda.nl')) {
|
||||
const ids = url.pathname.match(/\d{6,}/g);
|
||||
if (ids) return { source: 'funda', sourceId: ids.at(-1)!, url: stripTracking(url) };
|
||||
}
|
||||
|
||||
const parts = host.split('.');
|
||||
const source = (sourceAliases[host] ?? (parts.length > 1 ? parts.at(-2)! : host))
|
||||
.replace(/[^a-z0-9-]/g, '-')
|
||||
.slice(0, 40);
|
||||
const path = url.pathname.toLowerCase().replace(/\/+$/, '').replace(/^\/+/, '');
|
||||
const cleaned = stripTracking(url);
|
||||
const sourceId =
|
||||
path && path.length <= 80 && !new URL(cleaned).search ? path : hash(cleaned.toLowerCase());
|
||||
return { source: source || 'manual', sourceId, url: cleaned };
|
||||
}
|
||||
|
||||
const tracking = /^(utm_|fbclid$|gclid$|sfnsn$|mibextid$|unique_share_id$|s$)/;
|
||||
|
||||
function stripTracking(url: URL): string {
|
||||
const copy = new URL(url);
|
||||
for (const name of [...copy.searchParams.keys()])
|
||||
if (tracking.test(name)) copy.searchParams.delete(name);
|
||||
copy.hash = '';
|
||||
return copy.toString();
|
||||
}
|
||||
|
||||
/** Key for a listing without a link (e.g. from a screenshot), from its address. */
|
||||
export function keyFromAddress(parts: {
|
||||
street?: string | null;
|
||||
postalCode?: string | null;
|
||||
city?: string | null;
|
||||
title?: string | null;
|
||||
}): ListingKey {
|
||||
const basis = [parts.street, parts.postalCode, parts.city, parts.title]
|
||||
.map((v) => (v ?? '').toLowerCase().replace(/\s+/g, ' ').trim())
|
||||
.join('|');
|
||||
if (basis === '|||') throw new Error('Need a url, or at least a street/title and city');
|
||||
const query = [parts.street, parts.postalCode, parts.city].filter(Boolean).join(', ');
|
||||
return {
|
||||
source: 'manual',
|
||||
sourceId: `m-${hash(basis, 12)}`,
|
||||
url: `https://www.google.com/maps/search/?api=1&query=${encodeURIComponent(query || parts.title || '')}`
|
||||
};
|
||||
}
|
||||
|
||||
const lenientInt = (parse: (v: string) => number | null) =>
|
||||
z.preprocess(
|
||||
(v) => (typeof v === 'string' ? (v.trim() === '' ? null : parse(v)) : v),
|
||||
z.number().int().min(0).nullish()
|
||||
);
|
||||
const lenientFloat = z.preprocess(
|
||||
(v) => (typeof v === 'string' ? (v.trim() === '' ? null : Number(v.replace(',', '.'))) : v),
|
||||
z.number().finite().nullish()
|
||||
);
|
||||
const lenientDate = z.preprocess(
|
||||
(v) => {
|
||||
if (v == null || v === '') return v;
|
||||
const d = new Date(String(v));
|
||||
return Number.isNaN(d.getTime()) ? v : d.toISOString();
|
||||
},
|
||||
z.iso.datetime({ offset: true }).nullish()
|
||||
);
|
||||
|
||||
// What a person may type. Same limits as the worker contract, but lenient:
|
||||
// "€ 1.422,- p.m." is a price, "84 m²" is an area, everything is optional
|
||||
// except a url or an address.
|
||||
export const manualListingSchema = listingDetailSchema
|
||||
.omit({ sourceId: true, url: true, offer: true, propertyType: true, offeredAt: true })
|
||||
.partial()
|
||||
.extend({
|
||||
url: z.url().max(500).nullish(),
|
||||
source: z
|
||||
.string()
|
||||
.regex(/^[a-z0-9-]{1,40}$/)
|
||||
.nullish(),
|
||||
sourceId: z.string().trim().min(1).max(80).nullish(),
|
||||
offer: z.enum(offers).nullish(),
|
||||
propertyType: z.enum(propertyTypes).nullish(),
|
||||
priceEur: lenientInt(parseEuros),
|
||||
areaM2: lenientInt(parseInteger),
|
||||
rooms: lenientInt(parseInteger),
|
||||
bedrooms: lenientInt(parseInteger),
|
||||
bathrooms: lenientInt(parseInteger),
|
||||
constructionYear: lenientInt(parseInteger),
|
||||
depositEur: lenientInt(parseEuros),
|
||||
lat: lenientFloat,
|
||||
lng: lenientFloat,
|
||||
offeredAt: lenientDate,
|
||||
firstSeenAt: lenientDate,
|
||||
status: z.enum(listingStatuses).nullish(),
|
||||
notes: z.string().trim().max(2000).nullish()
|
||||
});
|
||||
export type ManualListing = z.infer<typeof manualListingSchema>;
|
||||
|
||||
/** Accept camelCase, snake_case and a few short names ("price", "area", "type"). */
|
||||
const fieldAliases: Record<string, string> = {
|
||||
price: 'priceEur',
|
||||
area: 'areaM2',
|
||||
m2: 'areaM2',
|
||||
type: 'propertyType',
|
||||
year: 'constructionYear',
|
||||
deposit: 'depositEur',
|
||||
postcode: 'postalCode',
|
||||
zip: 'postalCode',
|
||||
image: 'imageUrl',
|
||||
label: 'energyLabel',
|
||||
available: 'availableFrom',
|
||||
seen: 'firstSeenAt',
|
||||
firstSeen: 'firstSeenAt',
|
||||
offered: 'offeredAt'
|
||||
};
|
||||
|
||||
export function normalizeKeys(input: Record<string, unknown>): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(input)) {
|
||||
if (value === undefined) continue;
|
||||
const camel = key.replace(/[-_]([a-z0-9])/g, (_, c: string) => c.toUpperCase());
|
||||
out[fieldAliases[camel] ?? camel] = value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Guess rent/buy and the property type from the URL and title when not given. */
|
||||
export function inferOffer(text: string): Offer | null {
|
||||
if (/\b(koop|te-koop|buy|for-sale|kopen)\b/i.test(text)) return 'buy';
|
||||
if (/\b(huur|te-huur|rent|huren|for-rent)\b/i.test(text)) return 'rent';
|
||||
return null;
|
||||
}
|
||||
|
||||
export function inferPropertyType(text: string): PropertyType | null {
|
||||
if (/appartement|apartment|flat|portiek|bovenwoning|benedenwoning/i.test(text))
|
||||
return 'apartment';
|
||||
if (/studio/i.test(text)) return 'studio';
|
||||
if (/\bkamer\b|\broom\b/i.test(text)) return 'room';
|
||||
if (/\bhuis\b|woonhuis|eengezins|tussenwoning|hoekwoning|house|villa/i.test(text)) return 'house';
|
||||
return null;
|
||||
}
|
||||
|
||||
// Listing fields → house.listings columns.
|
||||
export const columns = {
|
||||
url: 'url',
|
||||
offer: 'offer',
|
||||
propertyType: 'property_type',
|
||||
title: 'title',
|
||||
externalId: 'external_id',
|
||||
street: 'street',
|
||||
postalCode: 'postal_code',
|
||||
city: 'city',
|
||||
neighbourhood: 'neighbourhood',
|
||||
priceEur: 'price_eur',
|
||||
areaM2: 'area_m2',
|
||||
rooms: 'rooms',
|
||||
bedrooms: 'bedrooms',
|
||||
bathrooms: 'bathrooms',
|
||||
constructionYear: 'construction_year',
|
||||
energyLabel: 'energy_label',
|
||||
interior: 'interior',
|
||||
depositEur: 'deposit_eur',
|
||||
availableFrom: 'available_from',
|
||||
contractType: 'contract_type',
|
||||
agent: 'agent',
|
||||
imageUrl: 'image_url',
|
||||
description: 'description',
|
||||
features: 'features',
|
||||
lat: 'lat',
|
||||
lng: 'lng',
|
||||
offeredAt: 'offered_at',
|
||||
firstSeenAt: 'first_seen_at',
|
||||
status: 'status',
|
||||
notes: 'notes'
|
||||
} as const satisfies Partial<Record<keyof ManualListing, string>>;
|
||||
|
||||
/** Only the columns the user actually gave; `null` clears a column on update. */
|
||||
export function toColumns(listing: ManualListing): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [field, column] of Object.entries(columns)) {
|
||||
const value = listing[field as keyof ManualListing];
|
||||
if (value === undefined) continue;
|
||||
out[column] =
|
||||
value !== null && (field === 'offeredAt' || field === 'firstSeenAt')
|
||||
? new Date(value as string)
|
||||
: value;
|
||||
}
|
||||
if (listing.city !== undefined) out.city_slug = citySlugFromName(listing.city);
|
||||
return out;
|
||||
}
|
||||
Reference in New Issue
Block a user