houses: one row per listing when a hand-added link is scraped later
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
The nightly scraper and worker API now take over an existing row (e.g. source=manual) whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet, Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has none. house-listing update/add give manual rows with such links the site key unless another row has it, update accepts --source/--source-id, and lookups by link ignore tracking params. scripts/house-rekey.ts backfills existing manual rows.
This commit is contained in:
@@ -8,7 +8,7 @@ import {
|
||||
HOUSE_NTFY_TOKEN
|
||||
} from '$app/env/private';
|
||||
import { error } from '@sveltejs/kit';
|
||||
import { and, desc, eq, inArray, isNull, lt, sql } from 'drizzle-orm';
|
||||
import { and, desc, eq, ilike, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm';
|
||||
import { db } from '#lib/server/db/index.js';
|
||||
import {
|
||||
houseAlertRules,
|
||||
@@ -23,6 +23,13 @@ import {
|
||||
type AlertMessage,
|
||||
type AlertRule
|
||||
} from '#lib/house-tracker/alerts.js';
|
||||
import {
|
||||
containsPattern,
|
||||
linkHost,
|
||||
pickLinkedListing,
|
||||
withoutEmpty,
|
||||
type LinkCandidate
|
||||
} from '#lib/house-tracker/link-key.js';
|
||||
import { listingRef } from '#lib/house-tracker/pararius.js';
|
||||
import type {
|
||||
DetailsRequest,
|
||||
@@ -57,6 +64,31 @@ async function recordPrice(
|
||||
await tx.insert(housePriceHistory).values({ listingId, priceEur: next });
|
||||
}
|
||||
|
||||
/** Rows stored under another source (e.g. a hand-added Pararius link) on the same host. */
|
||||
async function linkCandidates(
|
||||
tx: Tx,
|
||||
source: string,
|
||||
urls: (string | null | undefined)[]
|
||||
): Promise<LinkCandidate[]> {
|
||||
const hosts = [...new Set(urls.map(linkHost).filter((h) => h != null))];
|
||||
if (!hosts.length) return [];
|
||||
return tx
|
||||
.select({
|
||||
id: houseListings.id,
|
||||
source: houseListings.source,
|
||||
sourceId: houseListings.sourceId,
|
||||
url: houseListings.url,
|
||||
duplicateOf: houseListings.duplicateOf
|
||||
})
|
||||
.from(houseListings)
|
||||
.where(
|
||||
and(
|
||||
ne(houseListings.source, source),
|
||||
or(...hosts.map((host) => ilike(houseListings.url, containsPattern(host))))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
|
||||
const sourceIds = input.cards.map((card) => card.sourceId);
|
||||
const toAlert: { id: number; message: AlertMessage }[] = [];
|
||||
@@ -99,9 +131,35 @@ export async function syncCards(input: SyncRequest): Promise<SyncResponse> {
|
||||
)
|
||||
: [];
|
||||
const known = new Map(existing.map((row) => [row.sourceId, row]));
|
||||
// Listings already added by hand under another source: take those rows over
|
||||
// (new source key, scraped values fill the gaps) instead of inserting copies.
|
||||
let candidates = await linkCandidates(
|
||||
tx,
|
||||
input.source,
|
||||
input.cards.filter((card) => !known.has(card.sourceId)).map((card) => card.url)
|
||||
);
|
||||
const adopt = async (card: ListingCard) => {
|
||||
const linked = pickLinkedListing(
|
||||
candidates,
|
||||
{ source: input.source, sourceId: card.sourceId },
|
||||
card.url
|
||||
);
|
||||
if (!linked) return null;
|
||||
candidates = candidates.filter((c) => c.id !== linked.id);
|
||||
const [row] = await tx
|
||||
.update(houseListings)
|
||||
.set({ source: input.source, sourceId: card.sourceId })
|
||||
.where(eq(houseListings.id, linked.id))
|
||||
.returning({
|
||||
id: houseListings.id,
|
||||
sourceId: houseListings.sourceId,
|
||||
priceEur: houseListings.priceEur
|
||||
});
|
||||
return row ?? null;
|
||||
};
|
||||
|
||||
for (const card of input.cards) {
|
||||
const previous = known.get(card.sourceId);
|
||||
const previous = known.get(card.sourceId) ?? (await adopt(card));
|
||||
if (previous) {
|
||||
await tx
|
||||
.update(houseListings)
|
||||
@@ -217,12 +275,27 @@ export async function saveDetails(input: DetailsRequest) {
|
||||
let saved = 0;
|
||||
await db.transaction(async (tx) => {
|
||||
for (const detail of input.details) {
|
||||
const [previous] = await tx
|
||||
let [previous] = await tx
|
||||
.select({ id: houseListings.id, priceEur: houseListings.priceEur })
|
||||
.from(houseListings)
|
||||
.where(
|
||||
and(eq(houseListings.source, input.source), eq(houseListings.sourceId, detail.sourceId))
|
||||
);
|
||||
let adopted = false;
|
||||
if (!previous) {
|
||||
const linked = pickLinkedListing(
|
||||
await linkCandidates(tx, input.source, [detail.url]),
|
||||
{ source: input.source, sourceId: detail.sourceId },
|
||||
detail.url
|
||||
);
|
||||
if (linked) {
|
||||
[previous] = await tx
|
||||
.select({ id: houseListings.id, priceEur: houseListings.priceEur })
|
||||
.from(houseListings)
|
||||
.where(eq(houseListings.id, linked.id));
|
||||
adopted = Boolean(previous);
|
||||
}
|
||||
}
|
||||
const values = {
|
||||
externalId: detail.externalId,
|
||||
url: detail.url,
|
||||
@@ -256,7 +329,11 @@ export async function saveDetails(input: DetailsRequest) {
|
||||
lastSeenAt: sql`now()`
|
||||
};
|
||||
if (previous) {
|
||||
await tx.update(houseListings).set(values).where(eq(houseListings.id, previous.id));
|
||||
// A hand-added row keeps what was typed where the page has nothing.
|
||||
const set = adopted
|
||||
? { ...withoutEmpty(values), source: input.source, sourceId: detail.sourceId }
|
||||
: values;
|
||||
await tx.update(houseListings).set(set).where(eq(houseListings.id, previous.id));
|
||||
await recordPrice(tx, previous.id, previous.priceEur, detail.priceEur ?? null);
|
||||
} else {
|
||||
const [row] = await tx
|
||||
|
||||
Reference in New Issue
Block a user