houses: one row per listing when a hand-added link is scraped later
All checks were successful
Deploy production / deploy (push) Successful in 3m0s

The nightly scraper and worker API now take over an existing row (e.g. source=manual)
whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet,
Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has
none. house-listing update/add give manual rows with such links the site key unless
another row has it, update accepts --source/--source-id, and lookups by link ignore
tracking params. scripts/house-rekey.ts backfills existing manual rows.
This commit is contained in:
2026-10-24 14:59:03 +00:00
parent c827deb395
commit b5748e5ae5
8 changed files with 559 additions and 14 deletions

View File

@@ -8,7 +8,7 @@ import {
HOUSE_NTFY_TOKEN
} from '$app/env/private';
import { error } from '@sveltejs/kit';
import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm';
import { and, desc, eq, ilike, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm';
import { db } from '#lib/server/db/index.js';
import {
houseAlertRules,
@@ -23,6 +23,13 @@ import {
type AlertMessage,
type AlertRule
} from '#lib/house-tracker/alerts.js';
import {
containsPattern,
linkHost,
pickLinkedListing,
withoutEmpty,
type LinkCandidate
} from '#lib/house-tracker/link-key.js';
import { listingRef } from '#lib/house-tracker/pararius.js';
import type {
DetailsRequest,
@@ -57,6 +64,31 @@ async function recordPrice(
await tx.insert(housePriceHistory).values({ listingId, priceEur: next });
}
/** Rows stored under another source (e.g. a hand-added Pararius link) on the same host. */
async function linkCandidates(
tx: Tx,
source: string,
urls: (string | null | undefined)[]
): Promise<LinkCandidate[]> {
const hosts = [...new Set(urls.map(linkHost).filter((h) => h != null))];
if (!hosts.length) return [];
return tx
.select({
id: houseListings.id,
source: houseListings.source,
sourceId: houseListings.sourceId,
url: houseListings.url,
duplicateOf: houseListings.duplicateOf
})
.from(houseListings)
.where(
and(
ne(houseListings.source, source),
or(...hosts.map((host) => ilike(houseListings.url, containsPattern(host))))
)
);
}
export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
const sourceIds = input.cards.map((card) => card.sourceId);
const toAlert: { id: number; message: AlertMessage }[] = [];
@@ -99,9 +131,35 @@ export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
)
: [];
const known = new Map(existing.map((row) => [row.sourceId, row]));
// Listings already added by hand under another source: take those rows over
// (new source key, scraped values fill the gaps) instead of inserting copies.
let candidates = await linkCandidates(
tx,
input.source,
input.cards.filter((card) => !known.has(card.sourceId)).map((card) => card.url)
);
const adopt = async (card: ListingCard) => {
const linked = pickLinkedListing(
candidates,
{ source: input.source, sourceId: card.sourceId },
card.url
);
if (!linked) return null;
candidates = candidates.filter((c) => c.id !== linked.id);
const [row] = await tx
.update(houseListings)
.set({ source: input.source, sourceId: card.sourceId })
.where(eq(houseListings.id, linked.id))
.returning({
id: houseListings.id,
sourceId: houseListings.sourceId,
priceEur: houseListings.priceEur
});
return row ?? null;
};
for (const card of input.cards) {
const previous = known.get(card.sourceId);
const previous = known.get(card.sourceId) ?? (await adopt(card));
if (previous) {
await tx
.update(houseListings)
@@ -217,12 +275,27 @@ export async function saveDetails(input: DetailsRequest) {
let saved = 0;
await db.transaction(async (tx) => {
for (const detail of input.details) {
const [previous] = await tx
let [previous] = await tx
.select({ id: houseListings.id, priceEur: houseListings.priceEur })
.from(houseListings)
.where(
and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId))
);
let adopted = false;
if (!previous) {
const linked = pickLinkedListing(
await linkCandidates(tx, input.source, [detail.url]),
{ source: input.source, sourceId: detail.sourceId },
detail.url
);
if (linked) {
[previous] = await tx
.select({ id: houseListings.id, priceEur: houseListings.priceEur })
.from(houseListings)
.where(eq(houseListings.id, linked.id));
adopted = Boolean(previous);
}
}
const values = {
externalId: detail.externalId,
url: detail.url,
@@ -256,7 +329,11 @@ export async function saveDetails(input: DetailsRequest) {
lastSeenAt: sql`now()`
};
if (previous) {
await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id));
// A hand-added row keeps what was typed where the page has nothing.
const set = adopted
? { ...withoutEmpty(values), source: input.source, sourceId: detail.sourceId }
: values;
await tx.update(houseListings).set(set).where(eq(houseListings.id, previous.id));
await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null);
} else {
const [row] = await tx