houses: one row per listing when a hand-added link is scraped later

The nightly scraper and worker API now take over an existing row (e.g. source=manual)
whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet,
Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has
none. house-listing update/add give manual rows with such links the site key unless
another row has it, update accepts --source/--source-id, and lookups by link ignore
tracking params. scripts/house-rekey.ts backfills existing manual rows.

scripts/house-listing.ts

@@ -23,6 +23,14 @@ import {
 	type ListingKey,
 	type ManualListing
 } from '../src/lib/house-tracker/manual';
+import {
+	containsPattern,
+	linkHost,
+	normalizeListingUrl,
+	pickLinkedListing,
+	siteKeyFromUrl,
+	type LinkCandidate
+} from '../src/lib/house-tracker/link-key';
 
 const usage = `usage: bun run house <add|update|list|remove|restore> [options]
 
@@ -144,8 +152,34 @@ async function findId(ref: string | undefined): Promise<number> {
 	const key = keyFromUrl(ref);
 	const [row] = await sql<{ id: number }[]>`select id from house.listings
 		where (source = ${key.source} and source_id = ${key.sourceId}) or url = ${ref} limit 1`;
-	if (!row) throw new Error(`no listing for ${ref}`);
+	if (row) return row.id;
-	return row.id;
+	// Same page under a slightly different link (tracking params, www., trailing slash).
+	const wanted = normalizeListingUrl(ref);
+	const byLink = (await candidatesFor(sql, ref, '')).find(
+		(c) => normalizeListingUrl(c.url) === wanted
+	);
+	if (!byLink) throw new Error(`no listing for ${ref}`);
+	return byLink.id;
+}
+
+/** Rows on the same host as `url`, except those under `source`, for pickLinkedListing. */
+async function candidatesFor(
+	db: postgres.Sql | Tx,
+	url: string | null | undefined,
+	source: string
+): Promise<LinkCandidate[]> {
+	const host = linkHost(url);
+	if (!host) return [];
+	return db<LinkCandidate[]>`select id, source, source_id as "sourceId", url,
+			duplicate_of as "duplicateOf"
+		from house.listings where source <> ${source} and url ilike ${containsPattern(host)}`;
+}
+
+/** Id of the row already keyed (source, source_id), other than `id`. */
+async function keyOwner(tx: Tx, source: string, sourceId: string, id: number) {
+	const [row] = await tx<{ id: number }[]>`select id from house.listings
+		where source = ${source} and source_id = ${sourceId} and id <> ${id}`;
+	return row?.id ?? null;
 }
 
 // Same convention as the worker: a price_history row whenever the price is new or changes.
@@ -185,11 +219,30 @@ async function add() {
 		}
 
 		const row = await sql.begin(async (tx) => {
-			const [before] = await tx<{ id: number; price_eur: number | null }[]>`
+			let [before] = await tx<{ id: number; price_eur: number | null }[]>`
 				select id, price_eur from house.listings
 				where source = ${key.source} and source_id = ${key.sourceId} for update`;
+			let rekey = false;
+			if (!before && key.source !== 'manual' && listing.url) {
+				// The same link already on a hand-added row: update it and give it the site's key.
+				const linked = pickLinkedListing(
+					await candidatesFor(tx, key.url, key.source),
+					key,
+					key.url
+				);
+				if (linked) {
+					[before] = await tx<{ id: number; price_eur: number | null }[]>`
+						select id, price_eur from house.listings where id = ${linked.id} for update`;
+					rekey = Boolean(before);
+				}
+			}
 			if (before) {
-				const update = { ...given, last_seen_at: new Date(), removed_at: null };
+				const update: Record<string, unknown> = {
+					...given,
+					last_seen_at: new Date(),
+					removed_at: null
+				};
+				if (rekey) Object.assign(update, { source: key.source, source_id: key.sourceId });
 				if (listing.offer) Object.assign(update, { offer: listing.offer });
 				const [updated] = await tx`update house.listings set ${tx(update)}
 					where id = ${before.id} returning ${summary}`;
@@ -211,18 +264,41 @@ async function update() {
 	if (listing.url) given.url = listing.url;
 	if (listing.offer) given.offer = listing.offer;
 	if (listing.propertyType) given.property_type = listing.propertyType;
+	if (listing.source) given.source = listing.source;
+	if (listing.sourceId) given.source_id = listing.sourceId;
 	if (!Object.keys(given).length) throw new Error('nothing to update');
-	if (flag('dry-run')) return console.log({ id, ...given });
 	const [row] = await sql.begin(async (tx) => {
-		const [before] = await tx<{ price_eur: number | null }[]>`
+		const [before] = await tx<
-			select price_eur from house.listings where id = ${id} for update`;
+			{ price_eur: number | null; source: string; source_id: string; url: string }[]
+		>`select price_eur, source, source_id, url from house.listings where id = ${id} for update`;
 		if (!before) throw new Error(`no listing ${id}`);
+		// A manual row with a Funda/Pararius/Kamernet/Vesteda link gets that site's key, so a
+		// later scrape of the listing updates this row instead of adding a copy.
+		const url = (given.url as string | undefined) ?? before.url;
+		const site = siteKeyFromUrl(url);
+		if (before.source === 'manual' && !given.source && !given.source_id && site) {
+			const owner = await keyOwner(tx, site.source, site.sourceId, id);
+			if (owner == null) Object.assign(given, { source: site.source, source_id: site.sourceId });
+			else
+				console.warn(
+					`kept source=manual: ${site.source}/${site.sourceId} is already #${owner} (merge them by hand)`
+				);
+		} else if (given.source || given.source_id) {
+			const source = (given.source as string | undefined) ?? before.source;
+			const sourceId = (given.source_id as string | undefined) ?? before.source_id;
+			const owner = await keyOwner(tx, source, sourceId, id);
+			if (owner != null) throw new Error(`${source}/${sourceId} is already #${owner}`);
+		}
+		if (flag('dry-run')) {
+			console.log({ id, ...given });
+			return [];
+		}
 		const rows =
 			await tx`update house.listings set ${tx(given)} where id = ${id} returning ${summary}`;
 		await recordPrice(tx, id, before.price_eur, given.price_eur);
 		return rows;
 	});
-	print([row]);
+	if (row) print([row]);
 }
 
 async function list() {

scripts/house-rekey.ts

@@ -0,0 +1,41 @@
+// One-off/backfill: give manual house.listings rows with a Funda, Pararius, Kamernet or
+// Vesteda link that site's (source, source_id), so a later scrape updates them instead
+// of adding copies. Rows whose key another row already has are reported, not changed.
+//
+//   DB_URL=... bun scripts/house-rekey.ts [--dry-run]
+//   scripts/house-db.sh is for house-listing.ts; run this with a role that may update listings.
+import postgres from 'postgres';
+import { siteKeyFromUrl } from '../src/lib/house-tracker/link-key';
+
+const dryRun = process.argv.includes('--dry-run');
+const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
+if (!dbUrl) throw new Error('DB_URL is not set');
+const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
+
+try {
+	await sql.begin(async (tx) => {
+		const rows = await tx<{ id: number; source_id: string; url: string }[]>`
+			SELECT id, source_id, url FROM house.listings WHERE source = 'manual' ORDER BY id FOR UPDATE`;
+		const claimed = new Set<string>();
+		for (const row of rows) {
+			const key = siteKeyFromUrl(row.url);
+			if (!key) continue;
+			const pair = `${key.source}/${key.sourceId}`;
+			const [owner] = await tx<{ id: number }[]>`SELECT id FROM house.listings
+				WHERE source = ${key.source} AND source_id = ${key.sourceId}`;
+			if (owner || claimed.has(pair)) {
+				console.log(
+					`#${row.id} conflict: ${pair} is already #${owner?.id ?? '(another manual row)'}, kept manual`
+				);
+				continue;
+			}
+			claimed.add(pair);
+			console.log(`#${row.id} ${row.source_id} -> ${pair}${dryRun ? ' (dry run)' : ''}`);
+			if (!dryRun)
+				await tx`UPDATE house.listings SET source = ${key.source}, source_id = ${key.sourceId}
+					WHERE id = ${row.id}`;
+		}
+	});
+} finally {
+	await sql.end();
+}

scripts/house-scrape.ts

@@ -241,7 +241,7 @@ try {
 		const completed = result.completed && result.listings.length > 0;
 		const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed });
 		console.log(
-			`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated, ${ingest.removed} gone`
+			`${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated${ingest.adopted ? ` (${ingest.adopted} hand-added rows taken over)` : ''}, ${ingest.removed} gone`
 		);
 		if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name });
 	}

src/lib/house-tracker/ingest.ts

@@ -2,6 +2,13 @@
 // scraper on stan-vm (scripts/house-scrape.ts). No #lib / $app imports.
 import type postgres from 'postgres';
 import { addressKey, addressKeyFromText, citySlug, DEDUPE_SQL } from './address';
+import {
+	containsPattern,
+	linkHost,
+	pickLinkedListing,
+	withoutEmpty,
+	type LinkCandidate
+} from './link-key';
 import type { ScrapedListing } from './scraped';
 
 type Sql = postgres.Sql;
@@ -10,9 +17,28 @@ export type IngestResult = {
 	seeded: boolean;
 	inserted: { id: number; listing: ScrapedListing }[];
 	updated: number;
+	/** Rows stored under another source (a hand-added link) that this scrape took over. */
+	adopted: number;
 	removed: number;
 };
 
+/**
+ * Rows not keyed under `source` whose link is on the same host as one of `listings`,
+ * for pickLinkedListing. One query per ingest.
+ */
+export async function linkCandidates(
+	sql: Sql | postgres.TransactionSql,
+	source: string,
+	listings: { url: string | null }[]
+): Promise<LinkCandidate[]> {
+	const hosts = [...new Set(listings.map((l) => linkHost(l.url)).filter((h) => h != null))];
+	if (!hosts.length) return [];
+	return sql<LinkCandidate[]>`
+		SELECT id, source, source_id AS "sourceId", url, duplicate_of AS "duplicateOf"
+		FROM house.listings
+		WHERE source <> ${source} AND url ILIKE ANY (${hosts.map(containsPattern)}::text[])`;
+}
+
 export async function ingestListings(
 	sql: Sql,
 	source: string,
@@ -33,6 +59,14 @@ export async function ingestListings(
 		const known = new Map(existing.map((row) => [row.source_id, row]));
 		const inserted: IngestResult['inserted'] = [];
 		let updated = 0;
+		let adopted = 0;
+		// A listing already added by hand (source 'manual' with this link): update that row
+		// and give it this source's key, instead of inserting a copy.
+		let candidates = await linkCandidates(
+			tx,
+			source,
+			unique.filter((l) => !known.has(l.sourceId))
+		);
 
 		for (const l of unique) {
 			const columns = {
@@ -66,6 +100,21 @@ export async function ingestListings(
 				updated++;
 				continue;
 			}
+			const linked = pickLinkedListing(candidates, { source, sourceId: l.sourceId }, l.url);
+			if (linked) {
+				candidates = candidates.filter((c) => c.id !== linked.id);
+				const [before] = await tx<{ price_eur: number | null }[]>`
+					SELECT price_eur FROM house.listings WHERE id = ${linked.id} FOR UPDATE`;
+				await tx`UPDATE house.listings
+					SET ${tx({ ...withoutEmpty(columns), source, source_id: l.sourceId })},
+						last_seen_at = now(), removed_at = NULL
+					WHERE id = ${linked.id}`;
+				if (l.priceEur != null && l.priceEur !== before?.price_eur)
+					await tx`INSERT INTO house.price_history (listing_id, price_eur) VALUES (${linked.id}, ${l.priceEur})`;
+				updated++;
+				adopted++;
+				continue;
+			}
 			const [row] = await tx<{ id: number }[]>`INSERT INTO house.listings ${tx({
 				...columns,
 				source,
@@ -89,7 +138,7 @@ export async function ingestListings(
 				RETURNING id`;
 			removed = gone.length;
 		}
-		return { seeded, inserted, updated, removed };
+		return { seeded, inserted, updated, adopted, removed };
 	});
 }
 

src/lib/house-tracker/link-key.test.ts

@@ -0,0 +1,170 @@
+import { describe, expect, it } from 'vitest';
+import {
+	containsPattern,
+	normalizeListingUrl,
+	pickLinkedListing,
+	siteKeyFromUrl,
+	withoutEmpty,
+	type LinkCandidate
+} from './link-key';
+import { keyFromUrl } from './manual';
+
+describe('siteKeyFromUrl', () => {
+	it('reads the ids the scrapers use', () => {
+		expect(
+			siteKeyFromUrl('https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam')
+		).toEqual({ source: 'pararius', sourceId: 'd5d8a72c' });
+		expect(
+			siteKeyFromUrl(
+				'https://www.funda.nl/detail/huur/gouda/appartement-groenhovenweg-161/44579407/?utm_source=x'
+			)
+		).toEqual({ source: 'funda', sourceId: '44579407' });
+		expect(siteKeyFromUrl('https://www.funda.nl/89967562')).toEqual({
+			source: 'funda',
+			sourceId: '89967562'
+		});
+		expect(
+			siteKeyFromUrl(
+				'https://kamernet.nl/huren/studio-gouda/burgemeester-martenssingel/studio-2413752'
+			)
+		).toEqual({ source: 'kamernet', sourceId: '2413752' });
+		expect(
+			siteKeyFromUrl(
+				'https://www.vesteda.com/nl/huurwoningen-rotterdam/heydnahof/damastroos-27-rotterdam-10885'
+			)
+		).toEqual({ source: 'vesteda', sourceId: '10885' });
+	});
+
+	it('ignores search pages, other sites and junk', () => {
+		expect(siteKeyFromUrl('https://www.pararius.nl/huurwoningen/gouda')).toBeNull();
+		expect(siteKeyFromUrl('https://kamernet.nl/huren/huurwoningen-gouda')).toBeNull();
+		expect(siteKeyFromUrl('https://www.vesteda.com/nl/huurwoningen-gouda')).toBeNull();
+		expect(siteKeyFromUrl('https://notfunda.nl/12345678')).toBeNull();
+		expect(siteKeyFromUrl('https://vbtverhuurmakelaars.nl/woning/gouda-vest-7-d')).toBeNull();
+		expect(siteKeyFromUrl('not a url')).toBeNull();
+		expect(siteKeyFromUrl(null)).toBeNull();
+	});
+
+	it('matches keyFromUrl, so the CLI and the scrapers agree', () => {
+		for (const url of [
+			'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam',
+			'https://www.funda.nl/detail/huur/gouda/appartement-groenhovenweg-161/44579407/',
+			'https://kamernet.nl/huren/kamer-bodegraven/schoutendreef/kamer-2413119',
+			'https://www.vesteda.com/nl/huurwoningen-rotterdam/loreleiflat/clazina-kouwenbergzoom-230-rotterdam-15740'
+		]) {
+			const key = keyFromUrl(url);
+			expect(siteKeyFromUrl(url)).toEqual({ source: key.source, sourceId: key.sourceId });
+		}
+	});
+});
+
+describe('normalizeListingUrl', () => {
+	it('drops scheme, www, tracking, fragment, trailing slash and case', () => {
+		expect(
+			normalizeListingUrl('https://www.DupreeMakelaars.nl/aanbod/Gouda-Raam-7a/?fbclid=1#x')
+		).toBe('dupreemakelaars.nl/aanbod/gouda-raam-7a');
+		expect(normalizeListingUrl('http://dupreemakelaars.nl/aanbod/gouda-raam-7a')).toBe(
+			'dupreemakelaars.nl/aanbod/gouda-raam-7a'
+		);
+		expect(normalizeListingUrl('https://x.nl/a?b=2&a=1&utm_source=z')).toBe('x.nl/a?a=1&b=2');
+		expect(normalizeListingUrl('nope')).toBeNull();
+	});
+});
+
+describe('pickLinkedListing', () => {
+	const rows: LinkCandidate[] = [
+		{
+			id: 1,
+			source: 'manual',
+			sourceId: 'm-1',
+			url: 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam'
+		},
+		{
+			id: 2,
+			source: 'manual',
+			sourceId: 'm-2',
+			url: 'https://www.funda.nl/detail/huur/gouda/appartement-x-1/80993134/'
+		},
+		{ id: 3, source: 'funda', sourceId: '11111111', url: 'https://www.funda.nl/detail/80993134' },
+		{ id: 4, source: 'manual', sourceId: 'm-4', url: 'https://www.makelaar.nl/woning/raam-1/' },
+		{
+			id: 5,
+			source: 'manual',
+			sourceId: 'm-5',
+			url: 'https://www.pararius.nl/appartement-te-huur/gouda/aaaaaaaa/x',
+			duplicateOf: 9
+		},
+		{
+			id: 6,
+			source: 'vbt',
+			sourceId: 'woning/x',
+			url: 'https://www.pararius.nl/appartement-te-huur/gouda/aaaaaaaa/x'
+		}
+	];
+
+	it('finds a manual row by the id in its link, whatever the link shape', () => {
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'pararius', sourceId: 'd5d8a72c' },
+				'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam-12'
+			)?.id
+		).toBe(1);
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'funda', sourceId: '80993134' },
+				'https://www.funda.nl/80993134'
+			)?.id
+		).toBe(2);
+	});
+
+	it('finds other sites by normalized link', () => {
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'makelaar', sourceId: 'woning/raam-1' },
+				'http://makelaar.nl/woning/raam-1?utm_medium=x'
+			)?.id
+		).toBe(4);
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'makelaar', sourceId: 'woning/raam-2' },
+				'https://makelaar.nl/woning/raam-2'
+			)
+		).toBeNull();
+	});
+
+	it('never takes a row already keyed under a linked site, and prefers originals', () => {
+		expect(
+			pickLinkedListing(
+				rows,
+				{ source: 'funda', sourceId: '99999999' },
+				'https://www.funda.nl/99999999'
+			)
+		).toBeNull();
+		// Row 5 (a marked copy) and row 6 (vbt) both link it: the manual row wins despite being a copy.
+		expect(pickLinkedListing(rows, { source: 'pararius', sourceId: 'aaaaaaaa' }, null)?.id).toBe(5);
+		expect(
+			pickLinkedListing(
+				rows.filter((r) => r.id !== 5),
+				{ source: 'pararius', sourceId: 'aaaaaaaa' },
+				null
+			)?.id
+		).toBe(6);
+	});
+});
+
+describe('helpers', () => {
+	it('escapes LIKE wildcards', () => {
+		expect(containsPattern('a_b%c')).toBe('%a\\_b\\%c%');
+	});
+	it('drops empty values only', () => {
+		expect(withoutEmpty({ a: null, b: undefined, c: 0, d: '', e: 'x' })).toEqual({
+			c: 0,
+			d: '',
+			e: 'x'
+		});
+	});
+});

src/lib/house-tracker/link-key.ts

@@ -0,0 +1,126 @@
+// Which listing a link points at, whoever stored it. Pure (no DB, no #lib), shared by
+// the CLI, the nightly scraper and the worker API, so a hand-added row with a Funda or
+// Pararius link and a later scrape of that same listing end up as one row.
+import { listingRef } from './pararius';
+
+export type SiteKey = { source: string; sourceId: string };
+
+/** Sites whose listing id we can read from the link; the same ids the scrapers use. */
+export const LINKED_SOURCES = ['pararius', 'funda', 'kamernet', 'vesteda'] as const;
+
+const tracking = /^(utm_|fbclid$|gclid$|sfnsn$|mibextid$|unique_share_id$|s$)/;
+
+function parse(raw: string | null | undefined): URL | null {
+	if (!raw) return null;
+	try {
+		const url = new URL(raw.trim());
+		return url.protocol === 'http:' || url.protocol === 'https:' ? url : null;
+	} catch {
+		return null;
+	}
+}
+
+const bareHost = (url: URL) =>
+	url.hostname
+		.toLowerCase()
+		.replace(/^www\./, '')
+		.replace(/^m\./, '');
+
+const isSite = (host: string, domain: string) => host === domain || host.endsWith(`.${domain}`);
+
+/**
+ * (source, sourceId) for a Pararius, Funda, Kamernet or Vesteda listing link, the way
+ * their scrapers key it: Pararius' 8-hex short id, Funda's number, Kamernet's and
+ * Vesteda's trailing numeric id. Null for search pages and other sites.
+ */
+export function siteKeyFromUrl(raw: string | null | undefined): SiteKey | null {
+	const url = parse(raw);
+	if (!url) return null;
+	const host = bareHost(url);
+	const path = url.pathname;
+	if (isSite(host, 'pararius.nl') || isSite(host, 'pararius.com')) {
+		const ref = listingRef(path);
+		return ref ? { source: 'pararius', sourceId: ref.sourceId } : null;
+	}
+	if (isSite(host, 'funda.nl')) {
+		const ids = path.match(/\d{6,}/g);
+		return ids ? { source: 'funda', sourceId: ids.at(-1)! } : null;
+	}
+	if (isSite(host, 'kamernet.nl')) {
+		// /huren/studio-gouda/burgemeester-martenssingel/studio-2413752
+		const match = /\/[a-z]+-(\d{4,})\/?$/.exec(path);
+		return match ? { source: 'kamernet', sourceId: match[1] } : null;
+	}
+	if (isSite(host, 'vesteda.com')) {
+		// /nl/huurwoningen-rotterdam/heydnahof/damastroos-27-rotterdam-10885
+		const segments = path.split('/').filter(Boolean);
+		const match = segments.length >= 3 ? /-(\d+)$/.exec(segments.at(-1)!) : null;
+		return match ? { source: 'vesteda', sourceId: match[1] } : null;
+	}
+	return null;
+}
+
+/**
+ * A link reduced to what identifies the page: no scheme, www., tracking parameters,
+ * fragment, trailing slash or case. Null when it is not a web link.
+ */
+export function normalizeListingUrl(raw: string | null | undefined): string | null {
+	const url = parse(raw);
+	if (!url) return null;
+	const params = [...url.searchParams.entries()]
+		.filter(([name]) => !tracking.test(name))
+		.sort(([a], [b]) => a.localeCompare(b));
+	const query = params.length ? `?${new URLSearchParams(params).toString()}` : '';
+	const path = url.pathname.replace(/\/+$/, '');
+	return `${bareHost(url)}${path}${query}`.toLowerCase();
+}
+
+/** (I)LIKE pattern matching `text` literally anywhere. */
+export const containsPattern = (text: string) => `%${text.replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
+
+/** Host to pre-filter candidate rows with (`url ILIKE '%' || host || '%'`). */
+export function linkHost(raw: string | null | undefined): string | null {
+	const url = parse(raw);
+	return url ? bareHost(url) : null;
+}
+
+export type LinkCandidate = {
+	id: number;
+	source: string;
+	sourceId: string;
+	url: string | null;
+	duplicateOf?: number | null;
+};
+
+/**
+ * The existing row (stored under another source, typically 'manual') that is the same
+ * listing as a scraped (source, sourceId, url): same normalized link, or a link whose
+ * site key is that pair. Rows already keyed as another listing of the same site never
+ * match. Prefers manual rows, then originals over marked copies, then the oldest.
+ */
+export function pickLinkedListing(
+	candidates: readonly LinkCandidate[],
+	key: SiteKey,
+	url: string | null | undefined
+): LinkCandidate | null {
+	const wanted = normalizeListingUrl(url);
+	const matches = candidates.filter((row) => {
+		// Already keyed as a listing of the same or another linked site: not ours to take.
+		if (row.source === key.source || (LINKED_SOURCES as readonly string[]).includes(row.source))
+			return false;
+		const site = siteKeyFromUrl(row.url);
+		if (site) return site.source === key.source && site.sourceId === key.sourceId;
+		return wanted != null && normalizeListingUrl(row.url) === wanted;
+	});
+	const rank = (row: LinkCandidate) =>
+		(row.source === 'manual' ? 0 : 2) + (row.duplicateOf != null ? 1 : 0);
+	matches.sort((a, b) => rank(a) - rank(b) || a.id - b.id);
+	return matches[0] ?? null;
+}
+
+/** Drop null/undefined values, so adopting a hand-added row never erases what was typed. */
+export function withoutEmpty<T extends Record<string, unknown>>(values: T): Partial<T> {
+	return Object.fromEntries(
+		Object.entries(values).filter(([, value]) => value !== null && value !== undefined)
+	) as Partial<T>;
+}

src/lib/house-tracker/manual.ts

@@ -2,6 +2,7 @@
 // network, so the CLI and tests share them. Keep free of `#lib` / `$app` imports.
 import { createHash } from 'node:crypto';
 import * as z from 'zod';
+import { siteKeyFromUrl } from './link-key';
 import { listingRef, parseEuros, parseInteger } from './pararius';
 import {
 	listingDetailSchema,
@@ -71,6 +72,11 @@ export function keyFromUrl(raw: string): ListingKey {
 		const ids = url.pathname.match(/\d{6,}/g);
 		if (ids) return { source: 'funda', sourceId: ids.at(-1)!, url: stripTracking(url) };
 	}
+	// Kamernet and Vesteda: the scrapers' numeric id, so a hand-added link and the
+	// nightly scrape are one row.
+	const site = siteKeyFromUrl(url.toString());
+	if (site?.source === 'kamernet' || site?.source === 'vesteda')
+		return { ...site, url: stripTracking(url) };
 
 	const parts = host.split('.');
 	const source = (sourceAliases[host] ?? (parts.length > 1 ? parts.at(-2)! : host))

src/lib/server/house-tracker.ts

@@ -8,7 +8,7 @@ import {
 	HOUSE_NTFY_TOKEN
 } from '$app/env/private';
 import { error } from '@sveltejs/kit';
-import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm';
+import { and, desc, eq, ilike, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm';
 import { db } from '#lib/server/db/index.js';
 import {
 	houseAlertRules,
@@ -23,6 +23,13 @@ import {
 	type AlertMessage,
 	type AlertRule
 } from '#lib/house-tracker/alerts.js';
+import {
+	containsPattern,
+	linkHost,
+	pickLinkedListing,
+	withoutEmpty,
+	type LinkCandidate
+} from '#lib/house-tracker/link-key.js';
 import { listingRef } from '#lib/house-tracker/pararius.js';
 import type {
 	DetailsRequest,
@@ -57,6 +64,31 @@ async function recordPrice(
 	await tx.insert(housePriceHistory).values({ listingId, priceEur: next });
 }
 
+/** Rows stored under another source (e.g. a hand-added Pararius link) on the same host. */
+async function linkCandidates(
+	tx: Tx,
+	source: string,
+	urls: (string | null | undefined)[]
+): Promise<LinkCandidate[]> {
+	const hosts = [...new Set(urls.map(linkHost).filter((h) => h != null))];
+	if (!hosts.length) return [];
+	return tx
+		.select({
+			id: houseListings.id,
+			source: houseListings.source,
+			sourceId: houseListings.sourceId,
+			url: houseListings.url,
+			duplicateOf: houseListings.duplicateOf
+		})
+		.from(houseListings)
+		.where(
+			and(
+				ne(houseListings.source, source),
+				or(...hosts.map((host) => ilike(houseListings.url, containsPattern(host))))
+			)
+		);
+}
+
 export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
 	const sourceIds = input.cards.map((card) => card.sourceId);
 	const toAlert: { id: number; message: AlertMessage }[] = [];
@@ -99,9 +131,35 @@ export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
 					)
 			: [];
 		const known = new Map(existing.map((row) => [row.sourceId, row]));
+		// Listings already added by hand under another source: take those rows over
+		// (new source key, scraped values fill the gaps) instead of inserting copies.
+		let candidates = await linkCandidates(
+			tx,
+			input.source,
+			input.cards.filter((card) => !known.has(card.sourceId)).map((card) => card.url)
+		);
+		const adopt = async (card: ListingCard) => {
+			const linked = pickLinkedListing(
+				candidates,
+				{ source: input.source, sourceId: card.sourceId },
+				card.url
+			);
+			if (!linked) return null;
+			candidates = candidates.filter((c) => c.id !== linked.id);
+			const [row] = await tx
+				.update(houseListings)
+				.set({ source: input.source, sourceId: card.sourceId })
+				.where(eq(houseListings.id, linked.id))
+				.returning({
+					id: houseListings.id,
+					sourceId: houseListings.sourceId,
+					priceEur: houseListings.priceEur
+				});
+			return row ?? null;
+		};
 
 		for (const card of input.cards) {
-			const previous = known.get(card.sourceId);
+			const previous = known.get(card.sourceId) ?? (await adopt(card));
 			if (previous) {
 				await tx
 					.update(houseListings)
@@ -217,12 +275,27 @@ export async function saveDetails(input: DetailsRequest) {
 	let saved = 0;
 	await db.transaction(async (tx) => {
 		for (const detail of input.details) {
-			const [previous] = await tx
+			let [previous] = await tx
 				.select({ id: houseListings.id, priceEur: houseListings.priceEur })
 				.from(houseListings)
 				.where(
 					and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId))
 				);
+			let adopted = false;
+			if (!previous) {
+				const linked = pickLinkedListing(
+					await linkCandidates(tx, input.source, [detail.url]),
+					{ source: input.source, sourceId: detail.sourceId },
+					detail.url
+				);
+				if (linked) {
+					[previous] = await tx
+						.select({ id: houseListings.id, priceEur: houseListings.priceEur })
+						.from(houseListings)
+						.where(eq(houseListings.id, linked.id));
+					adopted = Boolean(previous);
+				}
+			}
 			const values = {
 				externalId: detail.externalId,
 				url: detail.url,
@@ -256,7 +329,11 @@ export async function saveDetails(input: DetailsRequest) {
 				lastSeenAt: sql`now()`
 			};
 			if (previous) {
-				await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id));
+				// A hand-added row keeps what was typed where the page has nothing.
+				const set = adopted
+					? { ...withoutEmpty(values), source: input.source, sourceId: detail.sourceId }
+					: values;
+				await tx.update(houseListings).set(set).where(eq(houseListings.id, previous.id));
 				await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null);
 			} else {
 				const [row] = await tx