houses: one row per listing when a hand-added link is scraped later
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
The nightly scraper and worker API now take over an existing row (e.g. source=manual) whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet, Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has none. house-listing update/add give manual rows with such links the site key unless another row has it, update accepts --source/--source-id, and lookups by link ignore tracking params. scripts/house-rekey.ts backfills existing manual rows.
This commit is contained in:
41
scripts/house-rekey.ts
Normal file
41
scripts/house-rekey.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
// One-off/backfill: give manual house.listings rows with a Funda, Pararius, Kamernet or
|
||||
// Vesteda link that site's (source, source_id), so a later scrape updates them instead
|
||||
// of adding copies. Rows whose key another row already has are reported, not changed.
|
||||
//
|
||||
// DB_URL=... bun scripts/house-rekey.ts [--dry-run]
|
||||
// scripts/house-db.sh is for house-listing.ts; run this with a role that may update listings.
|
||||
import postgres from 'postgres';
|
||||
import { siteKeyFromUrl } from '../src/lib/house-tracker/link-key';
|
||||
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
|
||||
if (!dbUrl) throw new Error('DB_URL is not set');
|
||||
const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
|
||||
|
||||
try {
|
||||
await sql.begin(async (tx) => {
|
||||
const rows = await tx<{ id: number; source_id: string; url: string }[]>`
|
||||
SELECT id, source_id, url FROM house.listings WHERE source = 'manual' ORDER BY id FOR UPDATE`;
|
||||
const claimed = new Set<string>();
|
||||
for (const row of rows) {
|
||||
const key = siteKeyFromUrl(row.url);
|
||||
if (!key) continue;
|
||||
const pair = `${key.source}/${key.sourceId}`;
|
||||
const [owner] = await tx<{ id: number }[]>`SELECT id FROM house.listings
|
||||
WHERE source = ${key.source} AND source_id = ${key.sourceId}`;
|
||||
if (owner || claimed.has(pair)) {
|
||||
console.log(
|
||||
`#${row.id} conflict: ${pair} is already #${owner?.id ?? '(another manual row)'}, kept manual`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
claimed.add(pair);
|
||||
console.log(`#${row.id} ${row.source_id} -> ${pair}${dryRun ? ' (dry run)' : ''}`);
|
||||
if (!dryRun)
|
||||
await tx`UPDATE house.listings SET source = ${key.source}, source_id = ${key.sourceId}
|
||||
WHERE id = ${row.id}`;
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
Reference in New Issue
Block a user