Stan Runge
113eb54cd9
All checks were successful
Deploy production / deploy (push) Successful in 2m33s
400 lines
14 KiB
TypeScript
400 lines
14 KiB
TypeScript
// Add, update, list and remove house tracker listings by hand (screenshots,
|
|
// links from chats). Writes house.listings directly; no website restart needed.
|
|
//
|
|
// bun run house add --url <link> --city Gouda --price 1422 --area 84 --bedrooms 2
|
|
// bun run house add --json listing.json (or `--json -` for stdin; may be an array)
|
|
// bun run house update <id|url> --status viewing --notes "Sat 14:00"
|
|
// bun run house list [--all] [--city gouda] [--offer buy] [--json]
|
|
// bun run house remove <id|url> [--hard] (soft remove = marked gone)
|
|
// bun run house restore <id|url>
|
|
//
|
|
// DATABASE_URL or DB_URL must point at the database, e.g. `bun --env-file=.env run house list`.
|
|
import { readFileSync } from 'node:fs';
|
|
import { parseArgs } from 'node:util';
|
|
import postgres from 'postgres';
|
|
import { lbmClassLabel, scoreListings } from '../src/lib/house-tracker/liveability';
|
|
import {
|
|
inferOffer,
|
|
inferPropertyType,
|
|
keyFromAddress,
|
|
keyFromUrl,
|
|
manualListingSchema,
|
|
normalizeKeys,
|
|
toColumns,
|
|
type ListingKey,
|
|
type ManualListing
|
|
} from '../src/lib/house-tracker/manual';
|
|
import {
|
|
containsPattern,
|
|
linkHost,
|
|
normalizeListingUrl,
|
|
pickLinkedListing,
|
|
siteKeyFromUrl,
|
|
type LinkCandidate
|
|
} from '../src/lib/house-tracker/link-key';
|
|
|
|
const usage = `usage: bun run house <add|update|list|remove|restore> [options]
|
|
|
|
add [--json <file|->] [field flags] insert, or update when the same link/address exists
|
|
update <id|url> [field flags] change only the given fields (use "" to clear one)
|
|
list [--all] [--city <slug>] [--offer rent|buy] [--status <s>] [--limit n] [--json]
|
|
remove <id|url> [--hard] mark gone (or delete with --hard)
|
|
restore <id|url> undo a soft remove
|
|
|
|
field flags: --url --offer rent|buy --type apartment|house|room|studio|other --title
|
|
--street --postal-code --city --neighbourhood --price --area --rooms --bedrooms
|
|
--bathrooms --year --energy-label --interior --deposit --available-from
|
|
--contract-type --agent --image-url --description --notes --status
|
|
interested|contacted|viewing|applied|rejected --lat --lng --offered-at
|
|
--first-seen (when it was shared) --source --source-id
|
|
other: --dry-run (print, don't write)`;
|
|
|
|
const fieldFlags = [
|
|
'url',
|
|
'offer',
|
|
'type',
|
|
'title',
|
|
'street',
|
|
'postal-code',
|
|
'city',
|
|
'neighbourhood',
|
|
'price',
|
|
'area',
|
|
'rooms',
|
|
'bedrooms',
|
|
'bathrooms',
|
|
'year',
|
|
'energy-label',
|
|
'interior',
|
|
'deposit',
|
|
'available-from',
|
|
'contract-type',
|
|
'agent',
|
|
'image-url',
|
|
'description',
|
|
'notes',
|
|
'status',
|
|
'lat',
|
|
'lng',
|
|
'offered-at',
|
|
'first-seen',
|
|
'source',
|
|
'source-id',
|
|
'external-id'
|
|
] as const;
|
|
|
|
const { values: flags, positionals } = parseArgs({
|
|
allowPositionals: true,
|
|
options: {
|
|
...Object.fromEntries(fieldFlags.map((name) => [name, { type: 'string' as const }])),
|
|
json: { type: 'string' },
|
|
all: { type: 'boolean' },
|
|
hard: { type: 'boolean' },
|
|
'dry-run': { type: 'boolean' },
|
|
limit: { type: 'string' },
|
|
help: { type: 'boolean', short: 'h' }
|
|
}
|
|
});
|
|
const [command, target] = positionals;
|
|
const flag = (name: string) => (flags as Record<string, string | boolean | undefined>)[name];
|
|
|
|
if (!command || flag('help')) {
|
|
console.log(usage);
|
|
process.exit(command ? 0 : 1);
|
|
}
|
|
|
|
function fieldInput(): Record<string, unknown> {
|
|
const raw: Record<string, unknown> = {};
|
|
for (const name of fieldFlags) {
|
|
const value = flag(name);
|
|
// An empty flag clears the column on update.
|
|
if (typeof value === 'string') raw[name] = value === '' ? null : value;
|
|
}
|
|
return normalizeKeys(raw);
|
|
}
|
|
|
|
async function jsonInput(): Promise<Record<string, unknown>[]> {
|
|
const source = flag('json');
|
|
if (typeof source !== 'string') return [];
|
|
const text = source === '-' ? readFileSync(0, 'utf8') : readFileSync(source, 'utf8');
|
|
const parsed = JSON.parse(text) as unknown;
|
|
return (Array.isArray(parsed) ? parsed : [parsed]) as Record<string, unknown>[];
|
|
}
|
|
|
|
function parseListing(raw: Record<string, unknown>): ManualListing {
|
|
const result = manualListingSchema.safeParse(normalizeKeys(raw));
|
|
if (!result.success) {
|
|
const issues = result.error.issues.map((i) => ` ${i.path.join('.')}: ${i.message}`);
|
|
throw new Error(`invalid listing:\n${issues.join('\n')}`);
|
|
}
|
|
return result.data;
|
|
}
|
|
|
|
function keyFor(listing: ManualListing): ListingKey {
|
|
const key = listing.url ? keyFromUrl(listing.url) : keyFromAddress(listing);
|
|
return {
|
|
...key,
|
|
source: listing.source ?? key.source,
|
|
sourceId: listing.sourceId ?? key.sourceId
|
|
};
|
|
}
|
|
|
|
const dbUrl = process.env.DATABASE_URL ?? process.env.DB_URL;
|
|
if (!dbUrl) throw new Error('Set DATABASE_URL (or DB_URL), e.g. bun --env-file=.env run house …');
|
|
const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
|
|
type Tx = postgres.TransactionSql;
|
|
|
|
const summary = sql`id, source, source_id, offer, property_type, title, city, city_slug,
|
|
price_eur, area_m2, rooms, bedrooms, status, removed_at, url`;
|
|
|
|
async function findId(ref: string | undefined): Promise<number> {
|
|
if (!ref) throw new Error('give a listing id or url');
|
|
if (/^\d+$/.test(ref)) return Number(ref);
|
|
const key = keyFromUrl(ref);
|
|
const [row] = await sql<{ id: number }[]>`select id from house.listings
|
|
where (source = ${key.source} and source_id = ${key.sourceId}) or url = ${ref} limit 1`;
|
|
if (row) return row.id;
|
|
// Same page under a slightly different link (tracking params, www., trailing slash).
|
|
const wanted = normalizeListingUrl(ref);
|
|
const byLink = (await candidatesFor(sql, ref, '')).find(
|
|
(c) => normalizeListingUrl(c.url) === wanted
|
|
);
|
|
if (!byLink) throw new Error(`no listing for ${ref}`);
|
|
return byLink.id;
|
|
}
|
|
|
|
/** Rows on the same host as `url`, except those under `source`, for pickLinkedListing. */
|
|
async function candidatesFor(
|
|
db: postgres.Sql | Tx,
|
|
url: string | null | undefined,
|
|
source: string
|
|
): Promise<LinkCandidate[]> {
|
|
const host = linkHost(url);
|
|
if (!host) return [];
|
|
return db<LinkCandidate[]>`select id, source, source_id as "sourceId", url,
|
|
duplicate_of as "duplicateOf"
|
|
from house.listings where source <> ${source} and url ilike ${containsPattern(host)}`;
|
|
}
|
|
|
|
/** Id of the row already keyed (source, source_id), other than `id`. */
|
|
async function keyOwner(tx: Tx, source: string, sourceId: string, id: number) {
|
|
const [row] = await tx<{ id: number }[]>`select id from house.listings
|
|
where source = ${source} and source_id = ${sourceId} and id <> ${id}`;
|
|
return row?.id ?? null;
|
|
}
|
|
|
|
// Same convention as the worker: a price_history row whenever the price is new or changes.
|
|
async function recordPrice(tx: Tx, id: number, previous: number | null, next: unknown) {
|
|
if (typeof next !== 'number' || next === previous) return;
|
|
await tx`insert into house.price_history (listing_id, price_eur) values (${id}, ${next})`;
|
|
}
|
|
|
|
async function add() {
|
|
const fromFlags = fieldInput();
|
|
const inputs = (await jsonInput()).map((item) => ({ ...item, ...fromFlags }));
|
|
if (!inputs.length) inputs.push(fromFlags);
|
|
|
|
for (const raw of inputs) {
|
|
const listing = parseListing(raw);
|
|
const key = keyFor(listing);
|
|
const hint = `${key.url} ${listing.title ?? ''} ${listing.description?.slice(0, 300) ?? ''}`;
|
|
const given = toColumns(listing);
|
|
if (listing.url) given.url = key.url;
|
|
if (!given.city_slug && key.citySlug) given.city_slug = key.citySlug;
|
|
|
|
const insert = {
|
|
source: key.source,
|
|
source_id: key.sourceId,
|
|
url: key.url,
|
|
offer: listing.offer ?? key.offer ?? inferOffer(hint) ?? 'rent',
|
|
property_type: listing.propertyType ?? key.propertyType ?? inferPropertyType(hint) ?? 'other',
|
|
first_seen_lane: 'manual',
|
|
// Nothing for the worker to fetch: we filled in what we know.
|
|
detail_fetched_at: new Date(),
|
|
...given
|
|
};
|
|
|
|
if (flag('dry-run')) {
|
|
console.log(insert);
|
|
continue;
|
|
}
|
|
|
|
const row = await sql.begin(async (tx) => {
|
|
let [before] = await tx<{ id: number; price_eur: number | null }[]>`
|
|
select id, price_eur from house.listings
|
|
where source = ${key.source} and source_id = ${key.sourceId} for update`;
|
|
let rekey = false;
|
|
if (!before && key.source !== 'manual' && listing.url) {
|
|
// The same link already on a hand-added row: update it and give it the site's key.
|
|
const linked = pickLinkedListing(
|
|
await candidatesFor(tx, key.url, key.source),
|
|
key,
|
|
key.url
|
|
);
|
|
if (linked) {
|
|
[before] = await tx<{ id: number; price_eur: number | null }[]>`
|
|
select id, price_eur from house.listings where id = ${linked.id} for update`;
|
|
rekey = Boolean(before);
|
|
}
|
|
}
|
|
if (before) {
|
|
const update: Record<string, unknown> = {
|
|
...given,
|
|
last_seen_at: new Date(),
|
|
removed_at: null
|
|
};
|
|
if (rekey) Object.assign(update, { source: key.source, source_id: key.sourceId });
|
|
if (listing.offer) Object.assign(update, { offer: listing.offer });
|
|
const [updated] = await tx`update house.listings set ${tx(update)}
|
|
where id = ${before.id} returning ${summary}`;
|
|
await recordPrice(tx, before.id, before.price_eur, given.price_eur);
|
|
return { ...updated, action: 'updated' };
|
|
}
|
|
const [inserted] = await tx`insert into house.listings ${tx(insert)} returning ${summary}`;
|
|
await recordPrice(tx, inserted.id, null, given.price_eur);
|
|
return { ...inserted, action: 'added' };
|
|
});
|
|
print([row]);
|
|
if (row?.id) await liveability(row.id as number);
|
|
}
|
|
}
|
|
|
|
/** Leefbaarometer score for a listing that was just added or changed (skips if unchanged). */
|
|
async function liveability(id: number) {
|
|
try {
|
|
const r = await scoreListings(sql, { ids: [id] });
|
|
if (!r.checked) return;
|
|
const [l] = await sql<
|
|
{
|
|
lbm_class: number | null;
|
|
lbm_onv_class: number | null;
|
|
lbm_buurt_name: string | null;
|
|
lbm_precision: string | null;
|
|
}[]
|
|
>`select lbm_class, lbm_onv_class, lbm_buurt_name, lbm_precision from house.listings where id = ${id}`;
|
|
console.log(
|
|
l?.lbm_class != null
|
|
? ` leefbaarometer ${l.lbm_class}/9 ${lbmClassLabel(l.lbm_class)} · overlast ${l.lbm_onv_class}/9 · ${l.lbm_buurt_name}${l.lbm_precision === 'address' ? '' : ` (approx., ${l.lbm_precision})`}`
|
|
: r.failed
|
|
? ' leefbaarometer: lookup failed, the nightly run retries'
|
|
: ' leefbaarometer: no location found for this address'
|
|
);
|
|
} catch (error) {
|
|
console.warn(` leefbaarometer: ${(error as Error).message}`);
|
|
}
|
|
}
|
|
|
|
async function update() {
|
|
const id = await findId(target);
|
|
const listing = parseListing(fieldInput());
|
|
const given = toColumns(listing);
|
|
if (listing.url) given.url = listing.url;
|
|
if (listing.offer) given.offer = listing.offer;
|
|
if (listing.propertyType) given.property_type = listing.propertyType;
|
|
if (listing.source) given.source = listing.source;
|
|
if (listing.sourceId) given.source_id = listing.sourceId;
|
|
if (!Object.keys(given).length) throw new Error('nothing to update');
|
|
const [row] = await sql.begin(async (tx) => {
|
|
const [before] = await tx<
|
|
{ price_eur: number | null; source: string; source_id: string; url: string }[]
|
|
>`select price_eur, source, source_id, url from house.listings where id = ${id} for update`;
|
|
if (!before) throw new Error(`no listing ${id}`);
|
|
// A manual row with a Funda/Pararius/Kamernet/Vesteda link gets that site's key, so a
|
|
// later scrape of the listing updates this row instead of adding a copy.
|
|
const url = (given.url as string | undefined) ?? before.url;
|
|
const site = siteKeyFromUrl(url);
|
|
if (before.source === 'manual' && !given.source && !given.source_id && site) {
|
|
const owner = await keyOwner(tx, site.source, site.sourceId, id);
|
|
if (owner == null) Object.assign(given, { source: site.source, source_id: site.sourceId });
|
|
else
|
|
console.warn(
|
|
`kept source=manual: ${site.source}/${site.sourceId} is already #${owner} (merge them by hand)`
|
|
);
|
|
} else if (given.source || given.source_id) {
|
|
const source = (given.source as string | undefined) ?? before.source;
|
|
const sourceId = (given.source_id as string | undefined) ?? before.source_id;
|
|
const owner = await keyOwner(tx, source, sourceId, id);
|
|
if (owner != null) throw new Error(`${source}/${sourceId} is already #${owner}`);
|
|
}
|
|
if (flag('dry-run')) {
|
|
console.log({ id, ...given });
|
|
return [];
|
|
}
|
|
const rows =
|
|
await tx`update house.listings set ${tx(given)} where id = ${id} returning ${summary}`;
|
|
await recordPrice(tx, id, before.price_eur, given.price_eur);
|
|
return rows;
|
|
});
|
|
if (row) {
|
|
print([row]);
|
|
await liveability(id);
|
|
}
|
|
}
|
|
|
|
async function list() {
|
|
const limit = Number(flag('limit') ?? 100);
|
|
const city = flag('city');
|
|
const offer = flag('offer');
|
|
const status = flag('status');
|
|
const rows = await sql`select ${summary}, first_seen_at, notes from house.listings
|
|
where true
|
|
${flag('all') ? sql`` : sql`and removed_at is null`}
|
|
${typeof city === 'string' ? sql`and city_slug = ${city}` : sql``}
|
|
${typeof offer === 'string' ? sql`and offer = ${offer}` : sql``}
|
|
${typeof status === 'string' ? sql`and status = ${status}` : sql``}
|
|
order by first_seen_at desc limit ${limit}`;
|
|
if (flag('json')) console.log(JSON.stringify(rows, null, 2));
|
|
else print(rows);
|
|
}
|
|
|
|
async function remove(hard: boolean) {
|
|
const id = await findId(target);
|
|
const rows = hard
|
|
? await sql`delete from house.listings where id = ${id} returning ${summary}`
|
|
: await sql`update house.listings set removed_at = now() where id = ${id} returning ${summary}`;
|
|
if (!rows.length) throw new Error(`no listing ${id}`);
|
|
console.log(hard ? 'deleted:' : 'marked gone:');
|
|
print(rows);
|
|
}
|
|
|
|
async function restore() {
|
|
const id = await findId(target);
|
|
const rows =
|
|
await sql`update house.listings set removed_at = null where id = ${id} returning ${summary}`;
|
|
print(rows);
|
|
}
|
|
|
|
function print(rows: readonly Record<string, unknown>[]) {
|
|
if (!rows.length) return console.log('(no listings)');
|
|
for (const r of rows) {
|
|
const bits = [
|
|
r.price_eur != null && `€${r.price_eur}${r.offer === 'rent' ? '/m' : ''}`,
|
|
r.area_m2 != null && `${r.area_m2} m²`,
|
|
r.rooms != null && `${r.rooms} rooms`,
|
|
r.bedrooms != null && `${r.bedrooms} bed`,
|
|
r.status,
|
|
r.removed_at && 'GONE'
|
|
].filter(Boolean);
|
|
console.log(
|
|
`${r.action ? `${r.action} ` : ''}#${r.id} [${r.source}] ${r.title ?? '(no title)'}, ${r.city ?? '?'} · ${bits.join(' · ')}\n ${r.url}`
|
|
);
|
|
}
|
|
}
|
|
|
|
try {
|
|
if (command === 'add') await add();
|
|
else if (command === 'update') await update();
|
|
else if (command === 'list' || command === 'ls') await list();
|
|
else if (command === 'remove' || command === 'rm') await remove(Boolean(flag('hard')));
|
|
else if (command === 'restore') await restore();
|
|
else {
|
|
console.error(usage);
|
|
process.exitCode = 1;
|
|
}
|
|
} catch (error) {
|
|
console.error(error instanceof Error ? error.message : error);
|
|
process.exitCode = 1;
|
|
} finally {
|
|
await sql.end();
|
|
}
|