houses: one row per listing when a hand-added link is scraped later

The nightly scraper and worker API now take over an existing row (e.g. source=manual)
whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet,
Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has
none. house-listing update/add give manual rows with such links the site key unless
another row has it, update accepts --source/--source-id, and lookups by link ignore
tracking params. scripts/house-rekey.ts backfills existing manual rows.

scripts/house-listing.ts

@@ -23,6 +23,14 @@ import {
 	type ListingKey,
 	type ManualListing
 } from '../src/lib/house-tracker/manual';
+import {
+	containsPattern,
+	linkHost,
+	normalizeListingUrl,
+	pickLinkedListing,
+	siteKeyFromUrl,
+	type LinkCandidate
+} from '../src/lib/house-tracker/link-key';
 
 const usage = `usage: bun run house <add|update|list|remove|restore> [options]
 
@@ -144,8 +152,34 @@ async function findId(ref: string | undefined): Promise<number> {
 	const key = keyFromUrl(ref);
 	const [row] = await sql<{ id: number }[]>`select id from house.listings
 		where (source = ${key.source} and source_id = ${key.sourceId}) or url = ${ref} limit 1`;
-	if (!row) throw new Error(`no listing for ${ref}`);
-	return row.id;
+	if (row) return row.id;
+	// Same page under a slightly different link (tracking params, www., trailing slash).
+	const wanted = normalizeListingUrl(ref);
+	const byLink = (await candidatesFor(sql, ref, '')).find(
+		(c) => normalizeListingUrl(c.url) === wanted
+	);
+	if (!byLink) throw new Error(`no listing for ${ref}`);
+	return byLink.id;
+}
+
+/** Rows on the same host as `url`, except those under `source`, for pickLinkedListing. */
+async function candidatesFor(
+	db: postgres.Sql | Tx,
+	url: string | null | undefined,
+	source: string
+): Promise<LinkCandidate[]> {
+	const host = linkHost(url);
+	if (!host) return [];
+	return db<LinkCandidate[]>`select id, source, source_id as "sourceId", url,
+			duplicate_of as "duplicateOf"
+		from house.listings where source <> ${source} and url ilike ${containsPattern(host)}`;
+}
+
+/** Id of the row already keyed (source, source_id), other than `id`. */
+async function keyOwner(tx: Tx, source: string, sourceId: string, id: number) {
+	const [row] = await tx<{ id: number }[]>`select id from house.listings
+		where source = ${source} and source_id = ${sourceId} and id <> ${id}`;
+	return row?.id ?? null;
 }
 
 // Same convention as the worker: a price_history row whenever the price is new or changes.
@@ -185,11 +219,30 @@ async function add() {
 		}
 
 		const row = await sql.begin(async (tx) => {
-			const [before] = await tx<{ id: number; price_eur: number | null }[]>`
+			let [before] = await tx<{ id: number; price_eur: number | null }[]>`
 				select id, price_eur from house.listings
 				where source = ${key.source} and source_id = ${key.sourceId} for update`;
+			let rekey = false;
+			if (!before && key.source !== 'manual' && listing.url) {
+				// The same link already on a hand-added row: update it and give it the site's key.
+				const linked = pickLinkedListing(
+					await candidatesFor(tx, key.url, key.source),
+					key,
+					key.url
+				);
+				if (linked) {
+					[before] = await tx<{ id: number; price_eur: number | null }[]>`
+						select id, price_eur from house.listings where id = ${linked.id} for update`;
+					rekey = Boolean(before);
+				}
+			}
 			if (before) {
-				const update = { ...given, last_seen_at: new Date(), removed_at: null };
+				const update: Record<string, unknown> = {
+					...given,
+					last_seen_at: new Date(),
+					removed_at: null
+				};
+				if (rekey) Object.assign(update, { source: key.source, source_id: key.sourceId });
 				if (listing.offer) Object.assign(update, { offer: listing.offer });
 				const [updated] = await tx`update house.listings set ${tx(update)}
 					where id = ${before.id} returning ${summary}`;
@@ -211,18 +264,41 @@ async function update() {
 	if (listing.url) given.url = listing.url;
 	if (listing.offer) given.offer = listing.offer;
 	if (listing.propertyType) given.property_type = listing.propertyType;
+	if (listing.source) given.source = listing.source;
+	if (listing.sourceId) given.source_id = listing.sourceId;
 	if (!Object.keys(given).length) throw new Error('nothing to update');
-	if (flag('dry-run')) return console.log({ id, ...given });
 	const [row] = await sql.begin(async (tx) => {
-		const [before] = await tx<{ price_eur: number | null }[]>`
-			select price_eur from house.listings where id = ${id} for update`;
+		const [before] = await tx<
+			{ price_eur: number | null; source: string; source_id: string; url: string }[]
+		>`select price_eur, source, source_id, url from house.listings where id = ${id} for update`;
 		if (!before) throw new Error(`no listing ${id}`);
+		// A manual row with a Funda/Pararius/Kamernet/Vesteda link gets that site's key, so a
+		// later scrape of the listing updates this row instead of adding a copy.
+		const url = (given.url as string | undefined) ?? before.url;
+		const site = siteKeyFromUrl(url);
+		if (before.source === 'manual' && !given.source && !given.source_id && site) {
+			const owner = await keyOwner(tx, site.source, site.sourceId, id);
+			if (owner == null) Object.assign(given, { source: site.source, source_id: site.sourceId });
+			else
+				console.warn(
+					`kept source=manual: ${site.source}/${site.sourceId} is already #${owner} (merge them by hand)`
+				);
+		} else if (given.source || given.source_id) {
+			const source = (given.source as string | undefined) ?? before.source;
+			const sourceId = (given.source_id as string | undefined) ?? before.source_id;
+			const owner = await keyOwner(tx, source, sourceId, id);
+			if (owner != null) throw new Error(`${source}/${sourceId} is already #${owner}`);
+		}
+		if (flag('dry-run')) {
+			console.log({ id, ...given });
+			return [];
+		}
 		const rows =
 			await tx`update house.listings set ${tx(given)} where id = ${id} returning ${summary}`;
 		await recordPrice(tx, id, before.price_eur, given.price_eur);
 		return rows;
 	});
-	print([row]);
+	if (row) print([row]);
 }
 
 async function list() {

scripts/house-rekey.ts

@@ -0,0 +1,41 @@
+// One-off/backfill: give manual house.listings rows with a Funda, Pararius, Kamernet or
+// Vesteda link that site's (source, source_id), so a later scrape updates them instead
+// of adding copies. Rows whose key another row already has are reported, not changed.
+//
+//   DB_URL=... bun scripts/house-rekey.ts [--dry-run]
+//   scripts/house-db.sh is for house-listing.ts; run this with a role that may update listings.
+import postgres from 'postgres';
+import { siteKeyFromUrl } from '../src/lib/house-tracker/link-key';
+
+const dryRun = process.argv.includes('--dry-run');
+const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
+if (!dbUrl) throw new Error('DB_URL is not set');
+const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
+
+try {
+	await sql.begin(async (tx) => {
+		const rows = await tx<{ id: number; source_id: string; url: string }[]>`
+			SELECT id, source_id, url FROM house.listings WHERE source = 'manual' ORDER BY id FOR UPDATE`;
+		const claimed = new Set<string>();
+		for (const row of rows) {
+			const key = siteKeyFromUrl(row.url);
+			if (!key) continue;
+			const pair = `${key.source}/${key.sourceId}`;
+			const [owner] = await tx<{ id: number }[]>`SELECT id FROM house.listings
+				WHERE source = ${key.source} AND source_id = ${key.sourceId}`;
+			if (owner || claimed.has(pair)) {
+				console.log(
+					`#${row.id} conflict: ${pair} is already #${owner?.id ?? '(another manual row)'}, kept manual`
+				);
+				continue;
+			}
+			claimed.add(pair);
+			console.log(`#${row.id} ${row.source_id} -> ${pair}${dryRun ? ' (dry run)' : ''}`);
+			if (!dryRun)
+				await tx`UPDATE house.listings SET source = ${key.source}, source_id = ${key.sourceId}
+					WHERE id = ${row.id}`;
+		}
+	});
+} finally {
+	await sql.end();
+}

scripts/house-scrape.ts

@@ -241,7 +241,7 @@ try {
 		const completed = result.completed && result.listings.length > 0;
 		const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed });
 		console.log(
-			`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated, ${ingest.removed} gone`
+			`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated${ingest.adopted ? ` (${ingest.adopted} hand-added rows taken over)` : ''}, ${ingest.removed} gone`
 		);
 		if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
 	}