From b5748e5ae58373961498f8c50e272da2b27d57b8 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sat, 24 Oct 2026 14:59:03 +0000 Subject: [PATCH] houses: one row per listing when a hand-added link is scraped later The nightly scraper and worker API now take over an existing row (e.g. source=manual) whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet, Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has none. house-listing update/add give manual rows with such links the site key unless another row has it, update accepts --source/--source-id, and lookups by link ignore tracking params. scripts/house-rekey.ts backfills existing manual rows. --- scripts/house-listing.ts | 92 +++++++++++-- scripts/house-rekey.ts | 41 ++++++ scripts/house-scrape.ts | 2 +- src/lib/house-tracker/ingest.ts | 51 +++++++- src/lib/house-tracker/link-key.test.ts | 170 +++++++++++++++++++++++++ src/lib/house-tracker/link-key.ts | 126 ++++++++++++++++++ src/lib/house-tracker/manual.ts | 6 + src/lib/server/house-tracker.ts | 85 ++++++++++++- 8 files changed, 559 insertions(+), 14 deletions(-) create mode 100644 scripts/house-rekey.ts create mode 100644 src/lib/house-tracker/link-key.test.ts create mode 100644 src/lib/house-tracker/link-key.ts diff --git a/scripts/house-listing.ts b/scripts/house-listing.ts index 9eeb09a..c560b85 100644 --- a/scripts/house-listing.ts +++ b/scripts/house-listing.ts @@ -23,6 +23,14 @@ import { type ListingKey, type ManualListing } from '../src/lib/house-tracker/manual'; +import { + containsPattern, + linkHost, + normalizeListingUrl, + pickLinkedListing, + siteKeyFromUrl, + type LinkCandidate +} from '../src/lib/house-tracker/link-key'; const usage = `usage: bun run house [options] @@ -144,8 +152,34 @@ async function findId(ref: string | undefined): Promise { const key = keyFromUrl(ref); const [row] = await sql<{ id: number }[]>`select id from house.listings where (source = ${key.source} and source_id = ${key.sourceId}) or url = ${ref} limit 1`; - if (!row) throw new Error(`no listing for ${ref}`); - return row.id; + if (row) return row.id; + // Same page under a slightly different link (tracking params, www., trailing slash). + const wanted = normalizeListingUrl(ref); + const byLink = (await candidatesFor(sql, ref, '')).find( + (c) => normalizeListingUrl(c.url) === wanted + ); + if (!byLink) throw new Error(`no listing for ${ref}`); + return byLink.id; +} + +/** Rows on the same host as `url`, except those under `source`, for pickLinkedListing. */ +async function candidatesFor( + db: postgres.Sql | Tx, + url: string | null | undefined, + source: string +): Promise { + const host = linkHost(url); + if (!host) return []; + return db`select id, source, source_id as "sourceId", url, + duplicate_of as "duplicateOf" + from house.listings where source <> ${source} and url ilike ${containsPattern(host)}`; +} + +/** Id of the row already keyed (source, source_id), other than `id`. */ +async function keyOwner(tx: Tx, source: string, sourceId: string, id: number) { + const [row] = await tx<{ id: number }[]>`select id from house.listings + where source = ${source} and source_id = ${sourceId} and id <> ${id}`; + return row?.id ?? null; } // Same convention as the worker: a price_history row whenever the price is new or changes. @@ -185,11 +219,30 @@ async function add() { } const row = await sql.begin(async (tx) => { - const [before] = await tx<{ id: number; price_eur: number | null }[]>` + let [before] = await tx<{ id: number; price_eur: number | null }[]>` select id, price_eur from house.listings where source = ${key.source} and source_id = ${key.sourceId} for update`; + let rekey = false; + if (!before && key.source !== 'manual' && listing.url) { + // The same link already on a hand-added row: update it and give it the site's key. + const linked = pickLinkedListing( + await candidatesFor(tx, key.url, key.source), + key, + key.url + ); + if (linked) { + [before] = await tx<{ id: number; price_eur: number | null }[]>` + select id, price_eur from house.listings where id = ${linked.id} for update`; + rekey = Boolean(before); + } + } if (before) { - const update = { ...given, last_seen_at: new Date(), removed_at: null }; + const update: Record = { + ...given, + last_seen_at: new Date(), + removed_at: null + }; + if (rekey) Object.assign(update, { source: key.source, source_id: key.sourceId }); if (listing.offer) Object.assign(update, { offer: listing.offer }); const [updated] = await tx`update house.listings set ${tx(update)} where id = ${before.id} returning ${summary}`; @@ -211,18 +264,41 @@ async function update() { if (listing.url) given.url = listing.url; if (listing.offer) given.offer = listing.offer; if (listing.propertyType) given.property_type = listing.propertyType; + if (listing.source) given.source = listing.source; + if (listing.sourceId) given.source_id = listing.sourceId; if (!Object.keys(given).length) throw new Error('nothing to update'); - if (flag('dry-run')) return console.log({ id, ...given }); const [row] = await sql.begin(async (tx) => { - const [before] = await tx<{ price_eur: number | null }[]>` - select price_eur from house.listings where id = ${id} for update`; + const [before] = await tx< + { price_eur: number | null; source: string; source_id: string; url: string }[] + >`select price_eur, source, source_id, url from house.listings where id = ${id} for update`; if (!before) throw new Error(`no listing ${id}`); + // A manual row with a Funda/Pararius/Kamernet/Vesteda link gets that site's key, so a + // later scrape of the listing updates this row instead of adding a copy. + const url = (given.url as string | undefined) ?? before.url; + const site = siteKeyFromUrl(url); + if (before.source === 'manual' && !given.source && !given.source_id && site) { + const owner = await keyOwner(tx, site.source, site.sourceId, id); + if (owner == null) Object.assign(given, { source: site.source, source_id: site.sourceId }); + else + console.warn( + `kept source=manual: ${site.source}/${site.sourceId} is already #${owner} (merge them by hand)` + ); + } else if (given.source || given.source_id) { + const source = (given.source as string | undefined) ?? before.source; + const sourceId = (given.source_id as string | undefined) ?? before.source_id; + const owner = await keyOwner(tx, source, sourceId, id); + if (owner != null) throw new Error(`${source}/${sourceId} is already #${owner}`); + } + if (flag('dry-run')) { + console.log({ id, ...given }); + return []; + } const rows = await tx`update house.listings set ${tx(given)} where id = ${id} returning ${summary}`; await recordPrice(tx, id, before.price_eur, given.price_eur); return rows; }); - print([row]); + if (row) print([row]); } async function list() { diff --git a/scripts/house-rekey.ts b/scripts/house-rekey.ts new file mode 100644 index 0000000..8d484af --- /dev/null +++ b/scripts/house-rekey.ts @@ -0,0 +1,41 @@ +// One-off/backfill: give manual house.listings rows with a Funda, Pararius, Kamernet or +// Vesteda link that site's (source, source_id), so a later scrape updates them instead +// of adding copies. Rows whose key another row already has are reported, not changed. +// +// DB_URL=... bun scripts/house-rekey.ts [--dry-run] +// scripts/house-db.sh is for house-listing.ts; run this with a role that may update listings. +import postgres from 'postgres'; +import { siteKeyFromUrl } from '../src/lib/house-tracker/link-key'; + +const dryRun = process.argv.includes('--dry-run'); +const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL; +if (!dbUrl) throw new Error('DB_URL is not set'); +const sql = postgres(dbUrl, { max: 1, onnotice: () => {} }); + +try { + await sql.begin(async (tx) => { + const rows = await tx<{ id: number; source_id: string; url: string }[]>` + SELECT id, source_id, url FROM house.listings WHERE source = 'manual' ORDER BY id FOR UPDATE`; + const claimed = new Set(); + for (const row of rows) { + const key = siteKeyFromUrl(row.url); + if (!key) continue; + const pair = `${key.source}/${key.sourceId}`; + const [owner] = await tx<{ id: number }[]>`SELECT id FROM house.listings + WHERE source = ${key.source} AND source_id = ${key.sourceId}`; + if (owner || claimed.has(pair)) { + console.log( + `#${row.id} conflict: ${pair} is already #${owner?.id ?? '(another manual row)'}, kept manual` + ); + continue; + } + claimed.add(pair); + console.log(`#${row.id} ${row.source_id} -> ${pair}${dryRun ? ' (dry run)' : ''}`); + if (!dryRun) + await tx`UPDATE house.listings SET source = ${key.source}, source_id = ${key.sourceId} + WHERE id = ${row.id}`; + } + }); +} finally { + await sql.end(); +} diff --git a/scripts/house-scrape.ts b/scripts/house-scrape.ts index d8d2120..367fd3e 100644 --- a/scripts/house-scrape.ts +++ b/scripts/house-scrape.ts @@ -241,7 +241,7 @@ try { const completed = result.completed && result.listings.length > 0; const ingest = await ingestListings(sql, name, result.listings, { runStartedAt, completed }); console.log( - `${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated, ${ingest.removed} gone` + `${name}: ${result.listings.length} seen, ${ingest.inserted.length} new${ingest.seeded ? ' (first run: seeded, no alerts)' : ''}, ${ingest.updated} updated${ingest.adopted ? ` (${ingest.adopted} hand-added rows taken over)` : ''}, ${ingest.removed} gone` ); if (!ingest.seeded) for (const row of ingest.inserted) fresh.push({ ...row, source: name }); } diff --git a/src/lib/house-tracker/ingest.ts b/src/lib/house-tracker/ingest.ts index a45f535..8ce2487 100644 --- a/src/lib/house-tracker/ingest.ts +++ b/src/lib/house-tracker/ingest.ts @@ -2,6 +2,13 @@ // scraper on stan-vm (scripts/house-scrape.ts). No #lib / $app imports. import type postgres from 'postgres'; import { addressKey, addressKeyFromText, citySlug, DEDUPE_SQL } from './address'; +import { + containsPattern, + linkHost, + pickLinkedListing, + withoutEmpty, + type LinkCandidate +} from './link-key'; import type { ScrapedListing } from './scraped'; type Sql = postgres.Sql; @@ -10,9 +17,28 @@ export type IngestResult = { seeded: boolean; inserted: { id: number; listing: ScrapedListing }[]; updated: number; + /** Rows stored under another source (a hand-added link) that this scrape took over. */ + adopted: number; removed: number; }; +/** + * Rows not keyed under `source` whose link is on the same host as one of `listings`, + * for pickLinkedListing. One query per ingest. + */ +export async function linkCandidates( + sql: Sql | postgres.TransactionSql, + source: string, + listings: { url: string | null }[] +): Promise { + const hosts = [...new Set(listings.map((l) => linkHost(l.url)).filter((h) => h != null))]; + if (!hosts.length) return []; + return sql` + SELECT id, source, source_id AS "sourceId", url, duplicate_of AS "duplicateOf" + FROM house.listings + WHERE source <> ${source} AND url ILIKE ANY (${hosts.map(containsPattern)}::text[])`; +} + export async function ingestListings( sql: Sql, source: string, @@ -33,6 +59,14 @@ export async function ingestListings( const known = new Map(existing.map((row) => [row.source_id, row])); const inserted: IngestResult['inserted'] = []; let updated = 0; + let adopted = 0; + // A listing already added by hand (source 'manual' with this link): update that row + // and give it this source's key, instead of inserting a copy. + let candidates = await linkCandidates( + tx, + source, + unique.filter((l) => !known.has(l.sourceId)) + ); for (const l of unique) { const columns = { @@ -66,6 +100,21 @@ export async function ingestListings( updated++; continue; } + const linked = pickLinkedListing(candidates, { source, sourceId: l.sourceId }, l.url); + if (linked) { + candidates = candidates.filter((c) => c.id !== linked.id); + const [before] = await tx<{ price_eur: number | null }[]>` + SELECT price_eur FROM house.listings WHERE id = ${linked.id} FOR UPDATE`; + await tx`UPDATE house.listings + SET ${tx({ ...withoutEmpty(columns), source, source_id: l.sourceId })}, + last_seen_at = now(), removed_at = NULL + WHERE id = ${linked.id}`; + if (l.priceEur != null && l.priceEur !== before?.price_eur) + await tx`INSERT INTO house.price_history (listing_id, price_eur) VALUES (${linked.id}, ${l.priceEur})`; + updated++; + adopted++; + continue; + } const [row] = await tx<{ id: number }[]>`INSERT INTO house.listings ${tx({ ...columns, source, @@ -89,7 +138,7 @@ export async function ingestListings( RETURNING id`; removed = gone.length; } - return { seeded, inserted, updated, removed }; + return { seeded, inserted, updated, adopted, removed }; }); } diff --git a/src/lib/house-tracker/link-key.test.ts b/src/lib/house-tracker/link-key.test.ts new file mode 100644 index 0000000..030f113 --- /dev/null +++ b/src/lib/house-tracker/link-key.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from 'vitest'; +import { + containsPattern, + normalizeListingUrl, + pickLinkedListing, + siteKeyFromUrl, + withoutEmpty, + type LinkCandidate +} from './link-key'; +import { keyFromUrl } from './manual'; + +describe('siteKeyFromUrl', () => { + it('reads the ids the scrapers use', () => { + expect( + siteKeyFromUrl('https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam') + ).toEqual({ source: 'pararius', sourceId: 'd5d8a72c' }); + expect( + siteKeyFromUrl( + 'https://www.funda.nl/detail/huur/gouda/appartement-groenhovenweg-161/44579407/?utm_source=x' + ) + ).toEqual({ source: 'funda', sourceId: '44579407' }); + expect(siteKeyFromUrl('https://www.funda.nl/89967562')).toEqual({ + source: 'funda', + sourceId: '89967562' + }); + expect( + siteKeyFromUrl( + 'https://kamernet.nl/huren/studio-gouda/burgemeester-martenssingel/studio-2413752' + ) + ).toEqual({ source: 'kamernet', sourceId: '2413752' }); + expect( + siteKeyFromUrl( + 'https://www.vesteda.com/nl/huurwoningen-rotterdam/heydnahof/damastroos-27-rotterdam-10885' + ) + ).toEqual({ source: 'vesteda', sourceId: '10885' }); + }); + + it('ignores search pages, other sites and junk', () => { + expect(siteKeyFromUrl('https://www.pararius.nl/huurwoningen/gouda')).toBeNull(); + expect(siteKeyFromUrl('https://kamernet.nl/huren/huurwoningen-gouda')).toBeNull(); + expect(siteKeyFromUrl('https://www.vesteda.com/nl/huurwoningen-gouda')).toBeNull(); + expect(siteKeyFromUrl('https://notfunda.nl/12345678')).toBeNull(); + expect(siteKeyFromUrl('https://vbtverhuurmakelaars.nl/woning/gouda-vest-7-d')).toBeNull(); + expect(siteKeyFromUrl('not a url')).toBeNull(); + expect(siteKeyFromUrl(null)).toBeNull(); + }); + + it('matches keyFromUrl, so the CLI and the scrapers agree', () => { + for (const url of [ + 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam', + 'https://www.funda.nl/detail/huur/gouda/appartement-groenhovenweg-161/44579407/', + 'https://kamernet.nl/huren/kamer-bodegraven/schoutendreef/kamer-2413119', + 'https://www.vesteda.com/nl/huurwoningen-rotterdam/loreleiflat/clazina-kouwenbergzoom-230-rotterdam-15740' + ]) { + const key = keyFromUrl(url); + expect(siteKeyFromUrl(url)).toEqual({ source: key.source, sourceId: key.sourceId }); + } + }); +}); + +describe('normalizeListingUrl', () => { + it('drops scheme, www, tracking, fragment, trailing slash and case', () => { + expect( + normalizeListingUrl('https://www.DupreeMakelaars.nl/aanbod/Gouda-Raam-7a/?fbclid=1#x') + ).toBe('dupreemakelaars.nl/aanbod/gouda-raam-7a'); + expect(normalizeListingUrl('http://dupreemakelaars.nl/aanbod/gouda-raam-7a')).toBe( + 'dupreemakelaars.nl/aanbod/gouda-raam-7a' + ); + expect(normalizeListingUrl('https://x.nl/a?b=2&a=1&utm_source=z')).toBe('x.nl/a?a=1&b=2'); + expect(normalizeListingUrl('nope')).toBeNull(); + }); +}); + +describe('pickLinkedListing', () => { + const rows: LinkCandidate[] = [ + { + id: 1, + source: 'manual', + sourceId: 'm-1', + url: 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam' + }, + { + id: 2, + source: 'manual', + sourceId: 'm-2', + url: 'https://www.funda.nl/detail/huur/gouda/appartement-x-1/80993134/' + }, + { id: 3, source: 'funda', sourceId: '11111111', url: 'https://www.funda.nl/detail/80993134' }, + { id: 4, source: 'manual', sourceId: 'm-4', url: 'https://www.makelaar.nl/woning/raam-1/' }, + { + id: 5, + source: 'manual', + sourceId: 'm-5', + url: 'https://www.pararius.nl/appartement-te-huur/gouda/aaaaaaaa/x', + duplicateOf: 9 + }, + { + id: 6, + source: 'vbt', + sourceId: 'woning/x', + url: 'https://www.pararius.nl/appartement-te-huur/gouda/aaaaaaaa/x' + } + ]; + + it('finds a manual row by the id in its link, whatever the link shape', () => { + expect( + pickLinkedListing( + rows, + { source: 'pararius', sourceId: 'd5d8a72c' }, + 'https://www.pararius.nl/appartement-te-huur/gouda/d5d8a72c/raam-12' + )?.id + ).toBe(1); + expect( + pickLinkedListing( + rows, + { source: 'funda', sourceId: '80993134' }, + 'https://www.funda.nl/80993134' + )?.id + ).toBe(2); + }); + + it('finds other sites by normalized link', () => { + expect( + pickLinkedListing( + rows, + { source: 'makelaar', sourceId: 'woning/raam-1' }, + 'http://makelaar.nl/woning/raam-1?utm_medium=x' + )?.id + ).toBe(4); + expect( + pickLinkedListing( + rows, + { source: 'makelaar', sourceId: 'woning/raam-2' }, + 'https://makelaar.nl/woning/raam-2' + ) + ).toBeNull(); + }); + + it('never takes a row already keyed under a linked site, and prefers originals', () => { + expect( + pickLinkedListing( + rows, + { source: 'funda', sourceId: '99999999' }, + 'https://www.funda.nl/99999999' + ) + ).toBeNull(); + // Row 5 (a marked copy) and row 6 (vbt) both link it: the manual row wins despite being a copy. + expect(pickLinkedListing(rows, { source: 'pararius', sourceId: 'aaaaaaaa' }, null)?.id).toBe(5); + expect( + pickLinkedListing( + rows.filter((r) => r.id !== 5), + { source: 'pararius', sourceId: 'aaaaaaaa' }, + null + )?.id + ).toBe(6); + }); +}); + +describe('helpers', () => { + it('escapes LIKE wildcards', () => { + expect(containsPattern('a_b%c')).toBe('%a\\_b\\%c%'); + }); + it('drops empty values only', () => { + expect(withoutEmpty({ a: null, b: undefined, c: 0, d: '', e: 'x' })).toEqual({ + c: 0, + d: '', + e: 'x' + }); + }); +}); diff --git a/src/lib/house-tracker/link-key.ts b/src/lib/house-tracker/link-key.ts new file mode 100644 index 0000000..4d09dce --- /dev/null +++ b/src/lib/house-tracker/link-key.ts @@ -0,0 +1,126 @@ +// Which listing a link points at, whoever stored it. Pure (no DB, no #lib), shared by +// the CLI, the nightly scraper and the worker API, so a hand-added row with a Funda or +// Pararius link and a later scrape of that same listing end up as one row. +import { listingRef } from './pararius'; + +export type SiteKey = { source: string; sourceId: string }; + +/** Sites whose listing id we can read from the link; the same ids the scrapers use. */ +export const LINKED_SOURCES = ['pararius', 'funda', 'kamernet', 'vesteda'] as const; + +const tracking = /^(utm_|fbclid$|gclid$|sfnsn$|mibextid$|unique_share_id$|s$)/; + +function parse(raw: string | null | undefined): URL | null { + if (!raw) return null; + try { + const url = new URL(raw.trim()); + return url.protocol === 'http:' || url.protocol === 'https:' ? url : null; + } catch { + return null; + } +} + +const bareHost = (url: URL) => + url.hostname + .toLowerCase() + .replace(/^www\./, '') + .replace(/^m\./, ''); + +const isSite = (host: string, domain: string) => host === domain || host.endsWith(`.${domain}`); + +/** + * (source, sourceId) for a Pararius, Funda, Kamernet or Vesteda listing link, the way + * their scrapers key it: Pararius' 8-hex short id, Funda's number, Kamernet's and + * Vesteda's trailing numeric id. Null for search pages and other sites. + */ +export function siteKeyFromUrl(raw: string | null | undefined): SiteKey | null { + const url = parse(raw); + if (!url) return null; + const host = bareHost(url); + const path = url.pathname; + if (isSite(host, 'pararius.nl') || isSite(host, 'pararius.com')) { + const ref = listingRef(path); + return ref ? { source: 'pararius', sourceId: ref.sourceId } : null; + } + if (isSite(host, 'funda.nl')) { + const ids = path.match(/\d{6,}/g); + return ids ? { source: 'funda', sourceId: ids.at(-1)! } : null; + } + if (isSite(host, 'kamernet.nl')) { + // /huren/studio-gouda/burgemeester-martenssingel/studio-2413752 + const match = /\/[a-z]+-(\d{4,})\/?$/.exec(path); + return match ? { source: 'kamernet', sourceId: match[1] } : null; + } + if (isSite(host, 'vesteda.com')) { + // /nl/huurwoningen-rotterdam/heydnahof/damastroos-27-rotterdam-10885 + const segments = path.split('/').filter(Boolean); + const match = segments.length >= 3 ? /-(\d+)$/.exec(segments.at(-1)!) : null; + return match ? { source: 'vesteda', sourceId: match[1] } : null; + } + return null; +} + +/** + * A link reduced to what identifies the page: no scheme, www., tracking parameters, + * fragment, trailing slash or case. Null when it is not a web link. + */ +export function normalizeListingUrl(raw: string | null | undefined): string | null { + const url = parse(raw); + if (!url) return null; + const params = [...url.searchParams.entries()] + .filter(([name]) => !tracking.test(name)) + .sort(([a], [b]) => a.localeCompare(b)); + const query = params.length ? `?${new URLSearchParams(params).toString()}` : ''; + const path = url.pathname.replace(/\/+$/, ''); + return `${bareHost(url)}${path}${query}`.toLowerCase(); +} + +/** (I)LIKE pattern matching `text` literally anywhere. */ +export const containsPattern = (text: string) => `%${text.replace(/[\\%_]/g, (c) => `\\${c}`)}%`; + +/** Host to pre-filter candidate rows with (`url ILIKE '%' || host || '%'`). */ +export function linkHost(raw: string | null | undefined): string | null { + const url = parse(raw); + return url ? bareHost(url) : null; +} + +export type LinkCandidate = { + id: number; + source: string; + sourceId: string; + url: string | null; + duplicateOf?: number | null; +}; + +/** + * The existing row (stored under another source, typically 'manual') that is the same + * listing as a scraped (source, sourceId, url): same normalized link, or a link whose + * site key is that pair. Rows already keyed as another listing of the same site never + * match. Prefers manual rows, then originals over marked copies, then the oldest. + */ +export function pickLinkedListing( + candidates: readonly LinkCandidate[], + key: SiteKey, + url: string | null | undefined +): LinkCandidate | null { + const wanted = normalizeListingUrl(url); + const matches = candidates.filter((row) => { + // Already keyed as a listing of the same or another linked site: not ours to take. + if (row.source === key.source || (LINKED_SOURCES as readonly string[]).includes(row.source)) + return false; + const site = siteKeyFromUrl(row.url); + if (site) return site.source === key.source && site.sourceId === key.sourceId; + return wanted != null && normalizeListingUrl(row.url) === wanted; + }); + const rank = (row: LinkCandidate) => + (row.source === 'manual' ? 0 : 2) + (row.duplicateOf != null ? 1 : 0); + matches.sort((a, b) => rank(a) - rank(b) || a.id - b.id); + return matches[0] ?? null; +} + +/** Drop null/undefined values, so adopting a hand-added row never erases what was typed. */ +export function withoutEmpty>(values: T): Partial { + return Object.fromEntries( + Object.entries(values).filter(([, value]) => value !== null && value !== undefined) + ) as Partial; +} diff --git a/src/lib/house-tracker/manual.ts b/src/lib/house-tracker/manual.ts index 93f25b7..796f321 100644 --- a/src/lib/house-tracker/manual.ts +++ b/src/lib/house-tracker/manual.ts @@ -2,6 +2,7 @@ // network, so the CLI and tests share them. Keep free of `#lib` / `$app` imports. import { createHash } from 'node:crypto'; import * as z from 'zod'; +import { siteKeyFromUrl } from './link-key'; import { listingRef, parseEuros, parseInteger } from './pararius'; import { listingDetailSchema, @@ -71,6 +72,11 @@ export function keyFromUrl(raw: string): ListingKey { const ids = url.pathname.match(/\d{6,}/g); if (ids) return { source: 'funda', sourceId: ids.at(-1)!, url: stripTracking(url) }; } + // Kamernet and Vesteda: the scrapers' numeric id, so a hand-added link and the + // nightly scrape are one row. + const site = siteKeyFromUrl(url.toString()); + if (site?.source === 'kamernet' || site?.source === 'vesteda') + return { ...site, url: stripTracking(url) }; const parts = host.split('.'); const source = (sourceAliases[host] ?? (parts.length > 1 ? parts.at(-2)! : host)) diff --git a/src/lib/server/house-tracker.ts b/src/lib/server/house-tracker.ts index 786726f..7c45dcb 100644 --- a/src/lib/server/house-tracker.ts +++ b/src/lib/server/house-tracker.ts @@ -8,7 +8,7 @@ import { HOUSE_NTFY_TOKEN } from '$app/env/private'; import { error } from '@sveltejs/kit'; -import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm'; +import { and, desc, eq, ilike, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm'; import { db } from '#lib/server/db/index.js'; import { houseAlertRules, @@ -23,6 +23,13 @@ import { type AlertMessage, type AlertRule } from '#lib/house-tracker/alerts.js'; +import { + containsPattern, + linkHost, + pickLinkedListing, + withoutEmpty, + type LinkCandidate +} from '#lib/house-tracker/link-key.js'; import { listingRef } from '#lib/house-tracker/pararius.js'; import type { DetailsRequest, @@ -57,6 +64,31 @@ async function recordPrice( await tx.insert(housePriceHistory).values({ listingId, priceEur: next }); } +/** Rows stored under another source (e.g. a hand-added Pararius link) on the same host. */ +async function linkCandidates( + tx: Tx, + source: string, + urls: (string | null | undefined)[] +): Promise { + const hosts = [...new Set(urls.map(linkHost).filter((h) => h != null))]; + if (!hosts.length) return []; + return tx + .select({ + id: houseListings.id, + source: houseListings.source, + sourceId: houseListings.sourceId, + url: houseListings.url, + duplicateOf: houseListings.duplicateOf + }) + .from(houseListings) + .where( + and( + ne(houseListings.source, source), + or(...hosts.map((host) => ilike(houseListings.url, containsPattern(host)))) + ) + ); +} + export async function syncCards(input: SyncRequest): Promise { const sourceIds = input.cards.map((card) => card.sourceId); const toAlert: { id: number; message: AlertMessage }[] = []; @@ -99,9 +131,35 @@ export async function syncCards(input: SyncRequest): Promise { ) : []; const known = new Map(existing.map((row) => [row.sourceId, row])); + // Listings already added by hand under another source: take those rows over + // (new source key, scraped values fill the gaps) instead of inserting copies. + let candidates = await linkCandidates( + tx, + input.source, + input.cards.filter((card) => !known.has(card.sourceId)).map((card) => card.url) + ); + const adopt = async (card: ListingCard) => { + const linked = pickLinkedListing( + candidates, + { source: input.source, sourceId: card.sourceId }, + card.url + ); + if (!linked) return null; + candidates = candidates.filter((c) => c.id !== linked.id); + const [row] = await tx + .update(houseListings) + .set({ source: input.source, sourceId: card.sourceId }) + .where(eq(houseListings.id, linked.id)) + .returning({ + id: houseListings.id, + sourceId: houseListings.sourceId, + priceEur: houseListings.priceEur + }); + return row ?? null; + }; for (const card of input.cards) { - const previous = known.get(card.sourceId); + const previous = known.get(card.sourceId) ?? (await adopt(card)); if (previous) { await tx .update(houseListings) @@ -217,12 +275,27 @@ export async function saveDetails(input: DetailsRequest) { let saved = 0; await db.transaction(async (tx) => { for (const detail of input.details) { - const [previous] = await tx + let [previous] = await tx .select({ id: houseListings.id, priceEur: houseListings.priceEur }) .from(houseListings) .where( and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId)) ); + let adopted = false; + if (!previous) { + const linked = pickLinkedListing( + await linkCandidates(tx, input.source, [detail.url]), + { source: input.source, sourceId: detail.sourceId }, + detail.url + ); + if (linked) { + [previous] = await tx + .select({ id: houseListings.id, priceEur: houseListings.priceEur }) + .from(houseListings) + .where(eq(houseListings.id, linked.id)); + adopted = Boolean(previous); + } + } const values = { externalId: detail.externalId, url: detail.url, @@ -256,7 +329,11 @@ export async function saveDetails(input: DetailsRequest) { lastSeenAt: sql`now()` }; if (previous) { - await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id)); + // A hand-added row keeps what was typed where the page has nothing. + const set = adopted + ? { ...withoutEmpty(values), source: input.source, sourceId: detail.sourceId } + : values; + await tx.update(houseListings).set(set).where(eq(houseListings.id, previous.id)); await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null); } else { const [row] = await tx