Stan Runge
b5748e5ae5
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
The nightly scraper and worker API now take over an existing row (e.g. source=manual) whose link is the same listing (normalized URL, or the Pararius key / Funda, Kamernet, Vesteda id in it) instead of inserting a copy; typed values are kept where the scrape has none. house-listing update/add give manual rows with such links the site key unless another row has it, update accepts --source/--source-id, and lookups by link ignore tracking params. scripts/house-rekey.ts backfills existing manual rows.
42 lines
1.7 KiB
TypeScript
42 lines
1.7 KiB
TypeScript
// One-off/backfill: give manual house.listings rows with a Funda, Pararius, Kamernet or
|
|
// Vesteda link that site's (source, source_id), so a later scrape updates them instead
|
|
// of adding copies. Rows whose key another row already has are reported, not changed.
|
|
//
|
|
// DB_URL=... bun scripts/house-rekey.ts [--dry-run]
|
|
// scripts/house-db.sh is for house-listing.ts; run this with a role that may update listings.
|
|
import postgres from 'postgres';
|
|
import { siteKeyFromUrl } from '../src/lib/house-tracker/link-key';
|
|
|
|
const dryRun = process.argv.includes('--dry-run');
|
|
const dbUrl = process.env.DB_URL ?? process.env.DATABASE_URL;
|
|
if (!dbUrl) throw new Error('DB_URL is not set');
|
|
const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
|
|
|
|
try {
|
|
await sql.begin(async (tx) => {
|
|
const rows = await tx<{ id: number; source_id: string; url: string }[]>`
|
|
SELECT id, source_id, url FROM house.listings WHERE source = 'manual' ORDER BY id FOR UPDATE`;
|
|
const claimed = new Set<string>();
|
|
for (const row of rows) {
|
|
const key = siteKeyFromUrl(row.url);
|
|
if (!key) continue;
|
|
const pair = `${key.source}/${key.sourceId}`;
|
|
const [owner] = await tx<{ id: number }[]>`SELECT id FROM house.listings
|
|
WHERE source = ${key.source} AND source_id = ${key.sourceId}`;
|
|
if (owner || claimed.has(pair)) {
|
|
console.log(
|
|
`#${row.id} conflict: ${pair} is already #${owner?.id ?? '(another manual row)'}, kept manual`
|
|
);
|
|
continue;
|
|
}
|
|
claimed.add(pair);
|
|
console.log(`#${row.id} ${row.source_id} -> ${pair}${dryRun ? ' (dry run)' : ''}`);
|
|
if (!dryRun)
|
|
await tx`UPDATE house.listings SET source = ${key.source}, source_id = ${key.sourceId}
|
|
WHERE id = ${row.id}`;
|
|
}
|
|
});
|
|
} finally {
|
|
await sql.end();
|
|
}
|