houses: one row per listing when a hand-added link is scraped later

The nightly scraper and worker API now take over an existing row (e.g. source=manual)
whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet,
Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has
none. house-listing update/add give manual rows with such links the site key unless
another row has it, update accepts --source/--source-id, and lookups by link ignore
tracking params. scripts/house-rekey.ts backfills existing manual rows.

src/lib/house-tracker/ingest.ts

@@ -2,6 +2,13 @@
 // scraper on stan-vm (scripts/house-scrape.ts). No #lib / $app imports.
 import type postgres from 'postgres';
 import { addressKey, addressKeyFromText, citySlug, DEDUPE_SQL } from './address';
+import {
+	containsPattern,
+	linkHost,
+	pickLinkedListing,
+	withoutEmpty,
+	type LinkCandidate
+} from './link-key';
 import type { ScrapedListing } from './scraped';
 
 type Sql = postgres.Sql;
@@ -10,9 +17,28 @@ export type IngestResult = {
 	seeded: boolean;
 	inserted: { id: number; listing: ScrapedListing }[];
 	updated: number;
+	/** Rows stored under another source (a hand-added link) that this scrape took over. */
+	adopted: number;
 	removed: number;
 };
 
+/**
+ * Rows not keyed under `source` whose link is on the same host as one of `listings`,
+ * for pickLinkedListing. One query per ingest.
+ */
+export async function linkCandidates(
+	sql: Sql | postgres.TransactionSql,
+	source: string,
+	listings: { url: string | null }[]
+): Promise<LinkCandidate[]> {
+	const hosts = [...new Set(listings.map((l) => linkHost(l.url)).filter((h) => h != null))];
+	if (!hosts.length) return [];
+	return sql<LinkCandidate[]>`
+		SELECT id, source, source_id AS "sourceId", url, duplicate_of AS "duplicateOf"
+		FROM house.listings
+		WHERE source <> ${source} AND url ILIKE ANY (${hosts.map(containsPattern)}::text[])`;
+}
+
 export async function ingestListings(
 	sql: Sql,
 	source: string,
@@ -33,6 +59,14 @@ export async function ingestListings(
 		const known = new Map(existing.map((row) => [row.source_id, row]));
 		const inserted: IngestResult['inserted'] = [];
 		let updated = 0;
+		let adopted = 0;
+		// A listing already added by hand (source 'manual' with this link): update that row
+		// and give it this source's key, instead of inserting a copy.
+		let candidates = await linkCandidates(
+			tx,
+			source,
+			unique.filter((l) => !known.has(l.sourceId))
+		);
 
 		for (const l of unique) {
 			const columns = {
@@ -66,6 +100,21 @@ export async function ingestListings(
 				updated++;
 				continue;
 			}
+			const linked = pickLinkedListing(candidates, { source, sourceId: l.sourceId }, l.url);
+			if (linked) {
+				candidates = candidates.filter((c) => c.id !== linked.id);
+				const [before] = await tx<{ price_eur: number | null }[]>`
+					SELECT price_eur FROM house.listings WHERE id = ${linked.id} FOR UPDATE`;
+				await tx`UPDATE house.listings
+					SET ${tx({ ...withoutEmpty(columns), source, source_id: l.sourceId })},
+						last_seen_at = now(), removed_at = NULL
+					WHERE id = ${linked.id}`;
+				if (l.priceEur != null && l.priceEur !== before?.price_eur)
+					await tx`INSERT INTO house.price_history (listing_id, price_eur) VALUES (${linked.id}, ${l.priceEur})`;
+				updated++;
+				adopted++;
+				continue;
+			}
 			const [row] = await tx<{ id: number }[]>`INSERT INTO house.listings ${tx({
 				...columns,
 				source,
@@ -89,7 +138,7 @@ export async function ingestListings(
 				RETURNING id`;
 			removed = gone.length;
 		}
-		return { seeded, inserted, updated, removed };
+		return { seeded, inserted, updated, adopted, removed };
 	});
 }
 

src/lib/house-tracker/link-key.test.ts

@@ -0,0 +1,170 @@
+import { describe, expect, it } from 'vitest';
+import {
+	containsPattern,
+	normalizeListingUrl,
+	pickLinkedListing,
+	siteKeyFromUrl,
+	withoutEmpty,
+	type LinkCandidate
+} from './link-key';
+import { keyFromUrl } from './manual';
+
+describe('siteKeyFromUrl', () => {
+	it('reads the ids the scrapers use', () => {
+		expect(
+			siteKeyFromUrl('https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam')
+		).toEqual({ source: 'pararius', sourceId: 'd5d8a72c' });
+		expect(
+			siteKeyFromUrl(
+				'https://www.funda.nl/detail/huur/gouda/appartement-groenhovenweg-161/44579407/?utm_source=x'
+			)
+		).toEqual({ source: 'funda', sourceId: '44579407' });
+		expect(siteKeyFromUrl('https://www.funda.nl/89967562')).toEqual({
+			source: 'funda',
+			sourceId: '89967562'
+		});
+		expect(
+			siteKeyFromUrl(
+				'https://kamernet.nl/huren/studio-gouda/burgemeester-martenssingel/studio-2413752'
+			)
+		).toEqual({ source: 'kamernet', sourceId: '2413752' });
+		expect(
+			siteKeyFromUrl(
+				'https://www.vesteda.com/nl/huurwoningen-rotterdam/heydnahof/damastroos-27-rotterdam-10885'
+			)
+		).toEqual({ source: 'vesteda', sourceId: '10885' });
+	});
+
+	it('ignores search pages, other sites and junk', () => {
+		expect(siteKeyFromUrl('https://www.pararius.nl/huurwoningen/gouda')).toBeNull();
+		expect(siteKeyFromUrl('https://kamernet.nl/huren/huurwoningen-gouda')).toBeNull();
+		expect(siteKeyFromUrl('https://www.vesteda.com/nl/huurwoningen-gouda')).toBeNull();
+		expect(siteKeyFromUrl('https://notfunda.nl/12345678')).toBeNull();
+		expect(siteKeyFromUrl('https://vbtverhuurmakelaars.nl/woning/gouda-vest-7-d')).toBeNull();
+		expect(siteKeyFromUrl('not a url')).toBeNull();
+		expect(siteKeyFromUrl(null)).toBeNull();
+	});
+
+	it('matches keyFromUrl, so the CLI and the scrapers agree', () => {
+		for (const url of [
+			'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam',
+			'https://www.funda.nl/detail/huur/gouda/appartement-groenhovenweg-161/44579407/',
+			'https://kamernet.nl/huren/kamer-bodegraven/schoutendreef/kamer-2413119',
+			'https://www.vesteda.com/nl/huurwoningen-rotterdam/loreleiflat/clazina-kouwenbergzoom-230-rotterdam-15740'
+		]) {
+			const key = keyFromUrl(url);
+			expect(siteKeyFromUrl(url)).toEqual({ source: key.source, sourceId: key.sourceId });
+		}
+	});
+});
+
+describe('normalizeListingUrl', () => {
+	it('drops scheme, www, tracking, fragment, trailing slash and case', () => {
+		expect(
+			normalizeListingUrl('https://www.DupreeMakelaars.nl/aanbod/Gouda-Raam-7a/?fbclid=1#x')
+		).toBe('dupreemakelaars.nl/aanbod/gouda-raam-7a');
+		expect(normalizeListingUrl('http://dupreemakelaars.nl/aanbod/gouda-raam-7a')).toBe(
+			'dupreemakelaars.nl/aanbod/gouda-raam-7a'
+		);
+		expect(normalizeListingUrl('https://x.nl/a?b=2&a=1&utm_source=z')).toBe('x.nl/a?a=1&b=2');
+		expect(normalizeListingUrl('nope')).toBeNull();
+	});
+});
+
+describe('pickLinkedListing', () => {
+	const rows: LinkCandidate[] = [
+		{
+			id: 1,
+			source: 'manual',
+			sourceId: 'm-1',
+			url: 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam'
+		},
+		{
+			id: 2,
+			source: 'manual',
+			sourceId: 'm-2',
+			url: 'https://www.funda.nl/detail/huur/gouda/appartement-x-1/80993134/'
+		},
+		{ id: 3, source: 'funda', sourceId: '11111111', url: 'https://www.funda.nl/detail/80993134' },
+		{ id: 4, source: 'manual', sourceId: 'm-4', url: 'https://www.makelaar.nl/woning/raam-1/' },
+		{
+			id: 5,
+			source: 'manual',
+			sourceId: 'm-5',
+			url: 'https://www.pararius.nl/appartement-te-huur/gouda/aaaaaaaa/x',
+			duplicateOf: 9
+		},
+		{
+			id: 6,
+			source: 'vbt',
+			sourceId: 'woning/x',
+			url: 'https://www.pararius.nl/appartement-te-huur/gouda/aaaaaaaa/x'
+		}
+	];
+
+	it('finds a manual row by the id in its link, whatever the link shape', () => {
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'pararius', sourceId: 'd5d8a72c' },
+				'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam-12'
+			)?.id
+		).toBe(1);
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'funda', sourceId: '80993134' },
+				'https://www.funda.nl/80993134'
+			)?.id
+		).toBe(2);
+	});
+
+	it('finds other sites by normalized link', () => {
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'makelaar', sourceId: 'woning/raam-1' },
+				'http://makelaar.nl/woning/raam-1?utm_medium=x'
+			)?.id
+		).toBe(4);
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'makelaar', sourceId: 'woning/raam-2' },
+				'https://makelaar.nl/woning/raam-2'
+			)
+		).toBeNull();
+	});
+
+	it('never takes a row already keyed under a linked site, and prefers originals', () => {
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'funda', sourceId: '99999999' },
+				'https://www.funda.nl/99999999'
+			)
+		).toBeNull();
+		// Row 5 (a marked copy) and row 6 (vbt) both link it: the manual row wins despite being a copy.
+		expect(pickLinkedListing(rows, { source: 'pararius', sourceId: 'aaaaaaaa' }, null)?.id).toBe(5);
+		expect(
+			pickLinkedListing(
+				rows.filter((r) => r.id !== 5),
+				{ source: 'pararius', sourceId: 'aaaaaaaa' },
+				null
+			)?.id
+		).toBe(6);
+	});
+});
+
+describe('helpers', () => {
+	it('escapes LIKE wildcards', () => {
+		expect(containsPattern('a_b%c')).toBe('%a\\_b\\%c%');
+	});
+	it('drops empty values only', () => {
+		expect(withoutEmpty({ a: null, b: undefined, c: 0, d: '', e: 'x' })).toEqual({
+			c: 0,
+			d: '',
+			e: 'x'
+		});
+	});
+});

src/lib/house-tracker/link-key.ts

@@ -0,0 +1,126 @@
+// Which listing a link points at, whoever stored it. Pure (no DB, no #lib), shared by
+// the CLI, the nightly scraper and the worker API, so a hand-added row with a Funda or
+// Pararius link and a later scrape of that same listing end up as one row.
+import { listingRef } from './pararius';
+
+export type SiteKey = { source: string; sourceId: string };
+
+/** Sites whose listing id we can read from the link; the same ids the scrapers use. */
+export const LINKED_SOURCES = ['pararius', 'funda', 'kamernet', 'vesteda'] as const;
+
+const tracking = /^(utm_|fbclid$|gclid$|sfnsn$|mibextid$|unique_share_id$|s$)/;
+
+function parse(raw: string | null | undefined): URL | null {
+	if (!raw) return null;
+	try {
+		const url = new URL(raw.trim());
+		return url.protocol === 'http:' || url.protocol === 'https:' ? url : null;
+	} catch {
+		return null;
+	}
+}
+
+const bareHost = (url: URL) =>
+	url.hostname
+		.toLowerCase()
+		.replace(/^www\./, '')
+		.replace(/^m\./, '');
+
+const isSite = (host: string, domain: string) => host === domain || host.endsWith(`.${domain}`);
+
+/**
+ * (source, sourceId) for a Pararius, Funda, Kamernet or Vesteda listing link, the way
+ * their scrapers key it: Pararius' 8-hex short id, Funda's number, Kamernet's and
+ * Vesteda's trailing numeric id. Null for search pages and other sites.
+ */
+export function siteKeyFromUrl(raw: string | null | undefined): SiteKey | null {
+	const url = parse(raw);
+	if (!url) return null;
+	const host = bareHost(url);
+	const path = url.pathname;
+	if (isSite(host, 'pararius.nl') || isSite(host, 'pararius.com')) {
+		const ref = listingRef(path);
+		return ref ? { source: 'pararius', sourceId: ref.sourceId } : null;
+	}
+	if (isSite(host, 'funda.nl')) {
+		const ids = path.match(/\d{6,}/g);
+		return ids ? { source: 'funda', sourceId: ids.at(-1)! } : null;
+	}
+	if (isSite(host, 'kamernet.nl')) {
+		// /huren/studio-gouda/burgemeester-martenssingel/studio-2413752
+		const match = /\/[a-z]+-(\d{4,})\/?$/.exec(path);
+		return match ? { source: 'kamernet', sourceId: match[1] } : null;
+	}
+	if (isSite(host, 'vesteda.com')) {
+		// /nl/huurwoningen-rotterdam/heydnahof/damastroos-27-rotterdam-10885
+		const segments = path.split('/').filter(Boolean);
+		const match = segments.length >= 3 ? /-(\d+)$/.exec(segments.at(-1)!) : null;
+		return match ? { source: 'vesteda', sourceId: match[1] } : null;
+	}
+	return null;
+}
+
+/**
+ * A link reduced to what identifies the page: no scheme, www., tracking parameters,
+ * fragment, trailing slash or case. Null when it is not a web link.
+ */
+export function normalizeListingUrl(raw: string | null | undefined): string | null {
+	const url = parse(raw);
+	if (!url) return null;
+	const params = [...url.searchParams.entries()]
+		.filter(([name]) => !tracking.test(name))
+		.sort(([a], [b]) => a.localeCompare(b));
+	const query = params.length ? `?${new URLSearchParams(params).toString()}` : '';
+	const path = url.pathname.replace(/\/+$/, '');
+	return `${bareHost(url)}${path}${query}`.toLowerCase();
+}
+
+/** (I)LIKE pattern matching `text` literally anywhere. */
+export const containsPattern = (text: string) => `%${text.replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
+
+/** Host to pre-filter candidate rows with (`url ILIKE '%' || host || '%'`). */
+export function linkHost(raw: string | null | undefined): string | null {
+	const url = parse(raw);
+	return url ? bareHost(url) : null;
+}
+
+export type LinkCandidate = {
+	id: number;
+	source: string;
+	sourceId: string;
+	url: string | null;
+	duplicateOf?: number | null;
+};
+
+/**
+ * The existing row (stored under another source, typically 'manual') that is the same
+ * listing as a scraped (source, sourceId, url): same normalized link, or a link whose
+ * site key is that pair. Rows already keyed as another listing of the same site never
+ * match. Prefers manual rows, then originals over marked copies, then the oldest.
+ */
+export function pickLinkedListing(
+	candidates: readonly LinkCandidate[],
+	key: SiteKey,
+	url: string | null | undefined
+): LinkCandidate | null {
+	const wanted = normalizeListingUrl(url);
+	const matches = candidates.filter((row) => {
+		// Already keyed as a listing of the same or another linked site: not ours to take.
+		if (row.source === key.source || (LINKED_SOURCES as readonly string[]).includes(row.source))
+			return false;
+		const site = siteKeyFromUrl(row.url);
+		if (site) return site.source === key.source && site.sourceId === key.sourceId;
+		return wanted != null && normalizeListingUrl(row.url) === wanted;
+	});
+	const rank = (row: LinkCandidate) =>
+		(row.source === 'manual' ? 0 : 2) + (row.duplicateOf != null ? 1 : 0);
+	matches.sort((a, b) => rank(a) - rank(b) || a.id - b.id);
+	return matches[0] ?? null;
+}
+
+/** Drop null/undefined values, so adopting a hand-added row never erases what was typed. */
+export function withoutEmpty<T extends Record<string, unknown>>(values: T): Partial<T> {
+	return Object.fromEntries(
+		Object.entries(values).filter(([, value]) => value !== null && value !== undefined)
+	) as Partial<T>;
+}

src/lib/house-tracker/manual.ts

@@ -2,6 +2,7 @@
 // network, so the CLI and tests share them. Keep free of `#lib` / `$app` imports.
 import { createHash } from 'node:crypto';
 import * as z from 'zod';
+import { siteKeyFromUrl } from './link-key';
 import { listingRef, parseEuros, parseInteger } from './pararius';
 import {
 	listingDetailSchema,
@@ -71,6 +72,11 @@ export function keyFromUrl(raw: string): ListingKey {
 		const ids = url.pathname.match(/\d{6,}/g);
 		if (ids) return { source: 'funda', sourceId: ids.at(-1)!, url: stripTracking(url) };
 	}
+	// Kamernet and Vesteda: the scrapers' numeric id, so a hand-added link and the
+	// nightly scrape are one row.
+	const site = siteKeyFromUrl(url.toString());
+	if (site?.source === 'kamernet' || site?.source === 'vesteda')
+		return { ...site, url: stripTracking(url) };
 
 	const parts = host.split('.');
 	const source = (sourceAliases[host] ?? (parts.length > 1 ? parts.at(-2)! : host))

src/lib/server/house-tracker.ts

@@ -8,7 +8,7 @@ import {
 	HOUSE_NTFY_TOKEN
 } from '$app/env/private';
 import { error } from '@sveltejs/kit';
-import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm';
+import { and, desc, eq, ilike, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm';
 import { db } from '#lib/server/db/index.js';
 import {
 	houseAlertRules,
@@ -23,6 +23,13 @@ import {
 	type AlertMessage,
 	type AlertRule
 } from '#lib/house-tracker/alerts.js';
+import {
+	containsPattern,
+	linkHost,
+	pickLinkedListing,
+	withoutEmpty,
+	type LinkCandidate
+} from '#lib/house-tracker/link-key.js';
 import { listingRef } from '#lib/house-tracker/pararius.js';
 import type {
 	DetailsRequest,
@@ -57,6 +64,31 @@ async function recordPrice(
 	await tx.insert(housePriceHistory).values({ listingId, priceEur: next });
 }
 
+/** Rows stored under another source (e.g. a hand-added Pararius link) on the same host. */
+async function linkCandidates(
+	tx: Tx,
+	source: string,
+	urls: (string | null | undefined)[]
+): Promise<LinkCandidate[]> {
+	const hosts = [...new Set(urls.map(linkHost).filter((h) => h != null))];
+	if (!hosts.length) return [];
+	return tx
+		.select({
+			id: houseListings.id,
+			source: houseListings.source,
+			sourceId: houseListings.sourceId,
+			url: houseListings.url,
+			duplicateOf: houseListings.duplicateOf
+		})
+		.from(houseListings)
+		.where(
+			and(
+				ne(houseListings.source, source),
+				or(...hosts.map((host) => ilike(houseListings.url, containsPattern(host))))
+			)
+		);
+}
+
 export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
 	const sourceIds = input.cards.map((card) => card.sourceId);
 	const toAlert: { id: number; message: AlertMessage }[] = [];
@@ -99,9 +131,35 @@ export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
 					)
 			: [];
 		const known = new Map(existing.map((row) => [row.sourceId, row]));
+		// Listings already added by hand under another source: take those rows over
+		// (new source key, scraped values fill the gaps) instead of inserting copies.
+		let candidates = await linkCandidates(
+			tx,
+			input.source,
+			input.cards.filter((card) => !known.has(card.sourceId)).map((card) => card.url)
+		);
+		const adopt = async (card: ListingCard) => {
+			const linked = pickLinkedListing(
+				candidates,
+				{ source: input.source, sourceId: card.sourceId },
+				card.url
+			);
+			if (!linked) return null;
+			candidates = candidates.filter((c) => c.id !== linked.id);
+			const [row] = await tx
+				.update(houseListings)
+				.set({ source: input.source, sourceId: card.sourceId })
+				.where(eq(houseListings.id, linked.id))
+				.returning({
+					id: houseListings.id,
+					sourceId: houseListings.sourceId,
+					priceEur: houseListings.priceEur
+				});
+			return row ?? null;
+		};
 
 		for (const card of input.cards) {
-			const previous = known.get(card.sourceId);
+			const previous = known.get(card.sourceId) ?? (await adopt(card));
 			if (previous) {
 				await tx
 					.update(houseListings)
@@ -217,12 +275,27 @@ export async function saveDetails(input: DetailsRequest) {
 	let saved = 0;
 	await db.transaction(async (tx) => {
 		for (const detail of input.details) {
-			const [previous] = await tx
+			let [previous] = await tx
 				.select({ id: houseListings.id, priceEur: houseListings.priceEur })
 				.from(houseListings)
 				.where(
 					and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId))
 				);
+			let adopted = false;
+			if (!previous) {
+				const linked = pickLinkedListing(
+					await linkCandidates(tx, input.source, [detail.url]),
+					{ source: input.source, sourceId: detail.sourceId },
+					detail.url
+				);
+				if (linked) {
+					[previous] = await tx
+						.select({ id: houseListings.id, priceEur: houseListings.priceEur })
+						.from(houseListings)
+						.where(eq(houseListings.id, linked.id));
+					adopted = Boolean(previous);
+				}
+			}
 			const values = {
 				externalId: detail.externalId,
 				url: detail.url,
@@ -256,7 +329,11 @@ export async function saveDetails(input: DetailsRequest) {
 				lastSeenAt: sql`now()`
 			};
 			if (previous) {
-				await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id));
+				// A hand-added row keeps what was typed where the page has nothing.
+				const set = adopted
+					? { ...withoutEmpty(values), source: input.source, sourceId: detail.sourceId }
+					: values;
+				await tx.update(houseListings).set(set).where(eq(houseListings.id, previous.id));
 				await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null);
 			} else {
 				const [row] = await tx