stan/website
1
0
Fork 0
Code Issues 21 Pull Requests Actions Packages Projects Releases Wiki Activity

houses: one row per listing when a hand-added link is scraped later
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
Details

Browse Source 
The nightly scraper and worker API now take over an existing row (e.g. source=manual)
whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet,
Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has
none. house-listing update/add give manual rows with such links the site key unless
another row has it, update accepts --source/--source-id, and lookups by link ignore
tracking params. scripts/house-rekey.ts backfills existing manual rows.
This commit is contained in:
Stan Runge
2026-10-24 14:59:03 +00:00
parent c827deb395
commit b5748e5ae5
8 changed files with 559 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/lib/house-tracker/ingest.ts
View File

@@ -2,6 +2,13 @@
// scraper on stan-vm (scripts/house-scrape.ts). No #lib / $app imports.
import type postgres from 'postgres';
import { addressKey, addressKeyFromText, citySlug, DEDUPE_SQL } from './address';
import {
containsPattern,
linkHost,
pickLinkedListing,
withoutEmpty,
type LinkCandidate
} from './link-key';
import type { ScrapedListing } from './scraped';
type Sql = postgres.Sql;
@@ -10,9 +17,28 @@ export type IngestResult = {
seeded: boolean;
inserted: { id: number; listing: ScrapedListing }[];
updated: number;
/** Rows stored under another source (a hand-added link) that this scrape took over. */
adopted: number;
removed: number;
};
/**
* Rows not keyed under `source` whose link is on the same host as one of `listings`,
* for pickLinkedListing. One query per ingest.
*/
export async function linkCandidates(
sql: Sql | postgres.TransactionSql,
source: string,
listings: { url: string | null }[]
): Promise<LinkCandidate[]> {
const hosts = [...new Set(listings.map((l) => linkHost(l.url)).filter((h) => h != null))];
if (!hosts.length) return [];
return sql<LinkCandidate[]>`
SELECT id, source, source_id AS "sourceId", url, duplicate_of AS "duplicateOf"
FROM house.listings
WHERE source <> ${source} AND url ILIKE ANY (${hosts.map(containsPattern)}::text[])`;
}
export async function ingestListings(
sql: Sql,
source: string,
@@ -33,6 +59,14 @@ export async function ingestListings(
const known = new Map(existing.map((row) => [row.source_id, row]));
const inserted: IngestResult['inserted'] = [];
let updated = 0;
let adopted = 0;
// A listing already added by hand (source 'manual' with this link): update that row
// and give it this source's key, instead of inserting a copy.
let candidates = await linkCandidates(
tx,
source,
unique.filter((l) => !known.has(l.sourceId))
);
for (const l of unique) {
const columns = {
@@ -66,6 +100,21 @@ export async function ingestListings(
updated++;
continue;
}
const linked = pickLinkedListing(candidates, { source, sourceId: l.sourceId }, l.url);
if (linked) {
candidates = candidates.filter((c) => c.id !== linked.id);
const [before] = await tx<{ price_eur: number | null }[]>`
SELECT price_eur FROM house.listings WHERE id = ${linked.id} FOR UPDATE`;
await tx`UPDATE house.listings
SET ${tx({ ...withoutEmpty(columns), source, source_id: l.sourceId })},
last_seen_at = now(), removed_at = NULL
WHERE id = ${linked.id}`;
if (l.priceEur != null && l.priceEur !== before?.price_eur)
await tx`INSERT INTO house.price_history (listing_id, price_eur) VALUES (${linked.id}, ${l.priceEur})`;
updated++;
adopted++;
continue;
}
const [row] = await tx<{ id: number }[]>`INSERT INTO house.listings ${tx({
...columns,
source,
@@ -89,7 +138,7 @@ export async function ingestListings(
RETURNING id`;
removed = gone.length;
}
return { seeded, inserted, updated, removed };
return { seeded, inserted, updated, adopted, removed };
});
}