house tracker: CLI to add, update, list and remove listings by hand
scripts/house-listing.ts (bun run house add|update|list|remove|restore) upserts house.listings on (source, source_id) derived from the listing URL or address, derives city_slug, records price_history, and accepts flags or JSON.
This commit is contained in:
@@ -176,3 +176,29 @@ fixture still covers pagination and price on request. Rental `priceEur` is the
|
||||
base/listed rent used by the search filter; a separately shown total can be
|
||||
higher (Raam: €1,475 base, €1,750 total). If Pararius changes markup, the worker
|
||||
reports `parse-error` in the worker log on the page.
|
||||
|
||||
## Adding listings by hand
|
||||
|
||||
`scripts/house-listing.ts` writes `house.listings` directly (links from chats,
|
||||
screenshots). No restart or deploy is needed for new rows.
|
||||
|
||||
```sh
|
||||
# DATABASE_URL or DB_URL must be set; locally: bun --env-file=.env run house …
|
||||
bun run house add --url https://vbtverhuurmakelaars.nl/woning/gouda-vest-7-d \
|
||||
--city Gouda --price "€ 1.422,- p.m." --area 84 --rooms 3 --bedrooms 2 --energy-label A
|
||||
echo '{"url":"…","city":"Gouda","price_eur":1350,"area_m2":81}' | bun run house add --json -
|
||||
bun run house add --street "Vest 7-D" --city Gouda --price 1422 # no link (screenshot)
|
||||
bun run house update <id|url> --status viewing --notes "za 14:00" # "" clears a field
|
||||
bun run house list [--all] [--city gouda] [--offer buy] [--json]
|
||||
bun run house remove <id|url> [--hard] # soft remove marks it gone; restore undoes it
|
||||
```
|
||||
|
||||
- `source` is the site (`funda`, `pararius`, `vbt`, `vesteda`, … or `manual`
|
||||
without a link); `source_id` comes from the URL (Funda's number, Pararius'
|
||||
worker key, otherwise the path), or a hash of the address. Adding the same
|
||||
link again updates that row and clears `removed_at`.
|
||||
- Fields follow `listingDetailSchema` (camelCase, snake_case or the flag names)
|
||||
and are lenient: "€ 1.422,- p.m." and "84 m²" are parsed. Only given fields are
|
||||
written. `city_slug` is derived from `city` in Pararius' slug style.
|
||||
- `--first-seen` sets when it was shared; price changes go to `price_history`.
|
||||
- Manual rows use `first_seen_lane = 'manual'` and never trigger alerts.
|
||||
|
||||
@@ -23,6 +23,9 @@
|
||||
"db:seed:cod": "bun scripts/seed-cod-catalog.ts",
|
||||
"cs-prices:ingest": "bun scripts/ingest-cs-prices.ts",
|
||||
"house:worker": "bun scripts/house-worker.ts",
|
||||
"house": "bun scripts/house-listing.ts",
|
||||
"house:add": "bun scripts/house-listing.ts add",
|
||||
"house:list": "bun scripts/house-listing.ts list",
|
||||
"auth:role": "bun scripts/set-admin.ts",
|
||||
"catalog:build": "python3 scripts/build-pokemon-catalog.py",
|
||||
"catalog:cod": "python3 scripts/build-cod-catalog.py && prettier --write data/call-of-duty.json",
|
||||
|
||||
293
scripts/house-listing.ts
Normal file
293
scripts/house-listing.ts
Normal file
@@ -0,0 +1,293 @@
|
||||
// Add, update, list and remove house tracker listings by hand (screenshots,
|
||||
// links from chats). Writes house.listings directly; no website restart needed.
|
||||
//
|
||||
// bun run house add --url <link> --city Gouda --price 1422 --area 84 --bedrooms 2
|
||||
// bun run house add --json listing.json (or `--json -` for stdin; may be an array)
|
||||
// bun run house update <id|url> --status viewing --notes "Sat 14:00"
|
||||
// bun run house list [--all] [--city gouda] [--offer buy] [--json]
|
||||
// bun run house remove <id|url> [--hard] (soft remove = marked gone)
|
||||
// bun run house restore <id|url>
|
||||
//
|
||||
// DATABASE_URL or DB_URL must point at the database, e.g. `bun --env-file=.env run house list`.
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { parseArgs } from 'node:util';
|
||||
import postgres from 'postgres';
|
||||
import {
|
||||
inferOffer,
|
||||
inferPropertyType,
|
||||
keyFromAddress,
|
||||
keyFromUrl,
|
||||
manualListingSchema,
|
||||
normalizeKeys,
|
||||
toColumns,
|
||||
type ListingKey,
|
||||
type ManualListing
|
||||
} from '../src/lib/house-tracker/manual';
|
||||
|
||||
const usage = `usage: bun run house <add|update|list|remove|restore> [options]
|
||||
|
||||
add [--json <file|->] [field flags] insert, or update when the same link/address exists
|
||||
update <id|url> [field flags] change only the given fields (use "" to clear one)
|
||||
list [--all] [--city <slug>] [--offer rent|buy] [--status <s>] [--limit n] [--json]
|
||||
remove <id|url> [--hard] mark gone (or delete with --hard)
|
||||
restore <id|url> undo a soft remove
|
||||
|
||||
field flags: --url --offer rent|buy --type apartment|house|room|studio|other --title
|
||||
--street --postal-code --city --neighbourhood --price --area --rooms --bedrooms
|
||||
--bathrooms --year --energy-label --interior --deposit --available-from
|
||||
--contract-type --agent --image-url --description --notes --status
|
||||
interested|contacted|viewing|applied|rejected --lat --lng --offered-at
|
||||
--first-seen (when it was shared) --source --source-id
|
||||
other: --dry-run (print, don't write)`;
|
||||
|
||||
const fieldFlags = [
|
||||
'url',
|
||||
'offer',
|
||||
'type',
|
||||
'title',
|
||||
'street',
|
||||
'postal-code',
|
||||
'city',
|
||||
'neighbourhood',
|
||||
'price',
|
||||
'area',
|
||||
'rooms',
|
||||
'bedrooms',
|
||||
'bathrooms',
|
||||
'year',
|
||||
'energy-label',
|
||||
'interior',
|
||||
'deposit',
|
||||
'available-from',
|
||||
'contract-type',
|
||||
'agent',
|
||||
'image-url',
|
||||
'description',
|
||||
'notes',
|
||||
'status',
|
||||
'lat',
|
||||
'lng',
|
||||
'offered-at',
|
||||
'first-seen',
|
||||
'source',
|
||||
'source-id',
|
||||
'external-id'
|
||||
] as const;
|
||||
|
||||
const { values: flags, positionals } = parseArgs({
|
||||
allowPositionals: true,
|
||||
options: {
|
||||
...Object.fromEntries(fieldFlags.map((name) => [name, { type: 'string' as const }])),
|
||||
json: { type: 'string' },
|
||||
all: { type: 'boolean' },
|
||||
hard: { type: 'boolean' },
|
||||
'dry-run': { type: 'boolean' },
|
||||
limit: { type: 'string' },
|
||||
help: { type: 'boolean', short: 'h' }
|
||||
}
|
||||
});
|
||||
const [command, target] = positionals;
|
||||
const flag = (name: string) => (flags as Record<string, string | boolean | undefined>)[name];
|
||||
|
||||
if (!command || flag('help')) {
|
||||
console.log(usage);
|
||||
process.exit(command ? 0 : 1);
|
||||
}
|
||||
|
||||
function fieldInput(): Record<string, unknown> {
|
||||
const raw: Record<string, unknown> = {};
|
||||
for (const name of fieldFlags) {
|
||||
const value = flag(name);
|
||||
// An empty flag clears the column on update.
|
||||
if (typeof value === 'string') raw[name] = value === '' ? null : value;
|
||||
}
|
||||
return normalizeKeys(raw);
|
||||
}
|
||||
|
||||
async function jsonInput(): Promise<Record<string, unknown>[]> {
|
||||
const source = flag('json');
|
||||
if (typeof source !== 'string') return [];
|
||||
const text = source === '-' ? readFileSync(0, 'utf8') : readFileSync(source, 'utf8');
|
||||
const parsed = JSON.parse(text) as unknown;
|
||||
return (Array.isArray(parsed) ? parsed : [parsed]) as Record<string, unknown>[];
|
||||
}
|
||||
|
||||
function parseListing(raw: Record<string, unknown>): ManualListing {
|
||||
const result = manualListingSchema.safeParse(normalizeKeys(raw));
|
||||
if (!result.success) {
|
||||
const issues = result.error.issues.map((i) => ` ${i.path.join('.')}: ${i.message}`);
|
||||
throw new Error(`invalid listing:\n${issues.join('\n')}`);
|
||||
}
|
||||
return result.data;
|
||||
}
|
||||
|
||||
function keyFor(listing: ManualListing): ListingKey {
|
||||
const key = listing.url ? keyFromUrl(listing.url) : keyFromAddress(listing);
|
||||
return {
|
||||
...key,
|
||||
source: listing.source ?? key.source,
|
||||
sourceId: listing.sourceId ?? key.sourceId
|
||||
};
|
||||
}
|
||||
|
||||
const dbUrl = process.env.DATABASE_URL ?? process.env.DB_URL;
|
||||
if (!dbUrl) throw new Error('Set DATABASE_URL (or DB_URL), e.g. bun --env-file=.env run house …');
|
||||
const sql = postgres(dbUrl, { max: 1, onnotice: () => {} });
|
||||
type Tx = postgres.TransactionSql;
|
||||
|
||||
const summary = sql`id, source, source_id, offer, property_type, title, city, city_slug,
|
||||
price_eur, area_m2, rooms, bedrooms, status, removed_at, url`;
|
||||
|
||||
async function findId(ref: string | undefined): Promise<number> {
|
||||
if (!ref) throw new Error('give a listing id or url');
|
||||
if (/^\d+$/.test(ref)) return Number(ref);
|
||||
const key = keyFromUrl(ref);
|
||||
const [row] = await sql<{ id: number }[]>`select id from house.listings
|
||||
where (source = ${key.source} and source_id = ${key.sourceId}) or url = ${ref} limit 1`;
|
||||
if (!row) throw new Error(`no listing for ${ref}`);
|
||||
return row.id;
|
||||
}
|
||||
|
||||
// Same convention as the worker: a price_history row whenever the price is new or changes.
|
||||
async function recordPrice(tx: Tx, id: number, previous: number | null, next: unknown) {
|
||||
if (typeof next !== 'number' || next === previous) return;
|
||||
await tx`insert into house.price_history (listing_id, price_eur) values (${id}, ${next})`;
|
||||
}
|
||||
|
||||
async function add() {
|
||||
const fromFlags = fieldInput();
|
||||
const inputs = (await jsonInput()).map((item) => ({ ...item, ...fromFlags }));
|
||||
if (!inputs.length) inputs.push(fromFlags);
|
||||
|
||||
for (const raw of inputs) {
|
||||
const listing = parseListing(raw);
|
||||
const key = keyFor(listing);
|
||||
const hint = `${key.url} ${listing.title ?? ''} ${listing.description?.slice(0, 300) ?? ''}`;
|
||||
const given = toColumns(listing);
|
||||
if (listing.url) given.url = key.url;
|
||||
if (!given.city_slug && key.citySlug) given.city_slug = key.citySlug;
|
||||
|
||||
const insert = {
|
||||
source: key.source,
|
||||
source_id: key.sourceId,
|
||||
url: key.url,
|
||||
offer: listing.offer ?? key.offer ?? inferOffer(hint) ?? 'rent',
|
||||
property_type: listing.propertyType ?? key.propertyType ?? inferPropertyType(hint) ?? 'other',
|
||||
first_seen_lane: 'manual',
|
||||
// Nothing for the worker to fetch: we filled in what we know.
|
||||
detail_fetched_at: new Date(),
|
||||
...given
|
||||
};
|
||||
|
||||
if (flag('dry-run')) {
|
||||
console.log(insert);
|
||||
continue;
|
||||
}
|
||||
|
||||
const row = await sql.begin(async (tx) => {
|
||||
const [before] = await tx<{ id: number; price_eur: number | null }[]>`
|
||||
select id, price_eur from house.listings
|
||||
where source = ${key.source} and source_id = ${key.sourceId} for update`;
|
||||
if (before) {
|
||||
const update = { ...given, last_seen_at: new Date(), removed_at: null };
|
||||
if (listing.offer) Object.assign(update, { offer: listing.offer });
|
||||
const [updated] = await tx`update house.listings set ${tx(update)}
|
||||
where id = ${before.id} returning ${summary}`;
|
||||
await recordPrice(tx, before.id, before.price_eur, given.price_eur);
|
||||
return { ...updated, action: 'updated' };
|
||||
}
|
||||
const [inserted] = await tx`insert into house.listings ${tx(insert)} returning ${summary}`;
|
||||
await recordPrice(tx, inserted.id, null, given.price_eur);
|
||||
return { ...inserted, action: 'added' };
|
||||
});
|
||||
print([row]);
|
||||
}
|
||||
}
|
||||
|
||||
async function update() {
|
||||
const id = await findId(target);
|
||||
const listing = parseListing(fieldInput());
|
||||
const given = toColumns(listing);
|
||||
if (listing.url) given.url = listing.url;
|
||||
if (listing.offer) given.offer = listing.offer;
|
||||
if (listing.propertyType) given.property_type = listing.propertyType;
|
||||
if (!Object.keys(given).length) throw new Error('nothing to update');
|
||||
if (flag('dry-run')) return console.log({ id, ...given });
|
||||
const [row] = await sql.begin(async (tx) => {
|
||||
const [before] = await tx<{ price_eur: number | null }[]>`
|
||||
select price_eur from house.listings where id = ${id} for update`;
|
||||
if (!before) throw new Error(`no listing ${id}`);
|
||||
const rows =
|
||||
await tx`update house.listings set ${tx(given)} where id = ${id} returning ${summary}`;
|
||||
await recordPrice(tx, id, before.price_eur, given.price_eur);
|
||||
return rows;
|
||||
});
|
||||
print([row]);
|
||||
}
|
||||
|
||||
async function list() {
|
||||
const limit = Number(flag('limit') ?? 100);
|
||||
const city = flag('city');
|
||||
const offer = flag('offer');
|
||||
const status = flag('status');
|
||||
const rows = await sql`select ${summary}, first_seen_at, notes from house.listings
|
||||
where true
|
||||
${flag('all') ? sql`` : sql`and removed_at is null`}
|
||||
${typeof city === 'string' ? sql`and city_slug = ${city}` : sql``}
|
||||
${typeof offer === 'string' ? sql`and offer = ${offer}` : sql``}
|
||||
${typeof status === 'string' ? sql`and status = ${status}` : sql``}
|
||||
order by first_seen_at desc limit ${limit}`;
|
||||
if (flag('json')) console.log(JSON.stringify(rows, null, 2));
|
||||
else print(rows);
|
||||
}
|
||||
|
||||
async function remove(hard: boolean) {
|
||||
const id = await findId(target);
|
||||
const rows = hard
|
||||
? await sql`delete from house.listings where id = ${id} returning ${summary}`
|
||||
: await sql`update house.listings set removed_at = now() where id = ${id} returning ${summary}`;
|
||||
if (!rows.length) throw new Error(`no listing ${id}`);
|
||||
console.log(hard ? 'deleted:' : 'marked gone:');
|
||||
print(rows);
|
||||
}
|
||||
|
||||
async function restore() {
|
||||
const id = await findId(target);
|
||||
const rows =
|
||||
await sql`update house.listings set removed_at = null where id = ${id} returning ${summary}`;
|
||||
print(rows);
|
||||
}
|
||||
|
||||
function print(rows: readonly Record<string, unknown>[]) {
|
||||
if (!rows.length) return console.log('(no listings)');
|
||||
for (const r of rows) {
|
||||
const bits = [
|
||||
r.price_eur != null && `€${r.price_eur}${r.offer === 'rent' ? '/m' : ''}`,
|
||||
r.area_m2 != null && `${r.area_m2} m²`,
|
||||
r.rooms != null && `${r.rooms} rooms`,
|
||||
r.bedrooms != null && `${r.bedrooms} bed`,
|
||||
r.status,
|
||||
r.removed_at && 'GONE'
|
||||
].filter(Boolean);
|
||||
console.log(
|
||||
`${r.action ? `${r.action} ` : ''}#${r.id} [${r.source}] ${r.title ?? '(no title)'}, ${r.city ?? '?'} · ${bits.join(' · ')}\n ${r.url}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (command === 'add') await add();
|
||||
else if (command === 'update') await update();
|
||||
else if (command === 'list' || command === 'ls') await list();
|
||||
else if (command === 'remove' || command === 'rm') await remove(Boolean(flag('hard')));
|
||||
else if (command === 'restore') await restore();
|
||||
else {
|
||||
console.error(usage);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
80
src/lib/house-tracker/manual.test.ts
Normal file
80
src/lib/house-tracker/manual.test.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
citySlugFromName,
|
||||
keyFromAddress,
|
||||
keyFromUrl,
|
||||
manualListingSchema,
|
||||
normalizeKeys,
|
||||
toColumns
|
||||
} from './manual';
|
||||
|
||||
describe('citySlugFromName', () => {
|
||||
it('matches the Pararius slug style', () => {
|
||||
expect(citySlugFromName('Gouda')).toBe('gouda');
|
||||
expect(citySlugFromName('Nieuwerkerk aan den IJssel')).toBe('nieuwerkerk-aan-den-ijssel');
|
||||
expect(citySlugFromName("'s-Gravenhage")).toBe('den-haag');
|
||||
expect(citySlugFromName(' ')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('keyFromUrl', () => {
|
||||
it('uses the worker key for Pararius', () => {
|
||||
expect(
|
||||
keyFromUrl('https://www.pararius.nl/appartement-te-huur/gouda/5179bb6d/raesbergenstraat')
|
||||
).toMatchObject({ source: 'pararius', sourceId: '5179bb6d', citySlug: 'gouda', offer: 'rent' });
|
||||
});
|
||||
|
||||
it('uses the numeric id for Funda, whatever the URL shape', () => {
|
||||
const long = keyFromUrl(
|
||||
'https://www.funda.nl/detail/huur/gouda/appartement-raoul-wallenbergplantsoen-11/43307383/'
|
||||
);
|
||||
const short = keyFromUrl(
|
||||
'https://www.funda.nl/detail/43307383?utm_source=funda&utm_medium=web&utm_campaign=share-listing-modal'
|
||||
);
|
||||
expect(long).toMatchObject({ source: 'funda', sourceId: '43307383' });
|
||||
expect(short).toMatchObject({ source: 'funda', sourceId: '43307383' });
|
||||
expect(short.url).toBe('https://www.funda.nl/detail/43307383');
|
||||
});
|
||||
|
||||
it('uses the domain and path for other sites', () => {
|
||||
expect(keyFromUrl('https://vbtverhuurmakelaars.nl/woning/gouda-vest-7-d/')).toMatchObject({
|
||||
source: 'vbt',
|
||||
sourceId: 'woning/gouda-vest-7-d'
|
||||
});
|
||||
expect(
|
||||
keyFromUrl('https://www.dupreemakelaars.nl/aanbod/gouda-raam-7a/?fbclid=abc&sfnsn=wa')
|
||||
).toEqual({
|
||||
source: 'dupreemakelaars',
|
||||
sourceId: 'aanbod/gouda-raam-7a',
|
||||
url: 'https://www.dupreemakelaars.nl/aanbod/gouda-raam-7a/'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('keyFromAddress', () => {
|
||||
it('is stable and case-insensitive', () => {
|
||||
const a = keyFromAddress({ street: 'Vest 7-D', city: 'Gouda' });
|
||||
const b = keyFromAddress({ street: 'vest 7-d', city: 'GOUDA' });
|
||||
expect(b.sourceId).toBe(a.sourceId);
|
||||
expect(a.source).toBe('manual');
|
||||
expect(a.url).toContain('google.com/maps');
|
||||
});
|
||||
|
||||
it('needs something to go on', () => {
|
||||
expect(() => keyFromAddress({})).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('manualListingSchema', () => {
|
||||
it('is lenient about typed numbers', () => {
|
||||
const parsed = manualListingSchema.parse(
|
||||
normalizeKeys({ price: '€ 1.422,- p.m.', area_m2: '84 m²', bedrooms: '2', lat: '52,01' })
|
||||
);
|
||||
expect(parsed).toMatchObject({ priceEur: 1422, areaM2: 84, bedrooms: 2, lat: 52.01 });
|
||||
});
|
||||
|
||||
it('only maps given fields to columns', () => {
|
||||
const parsed = manualListingSchema.parse({ city: 'Gouda', priceEur: 1350 });
|
||||
expect(toColumns(parsed)).toEqual({ city: 'Gouda', city_slug: 'gouda', price_eur: 1350 });
|
||||
});
|
||||
});
|
||||
255
src/lib/house-tracker/manual.ts
Normal file
255
src/lib/house-tracker/manual.ts
Normal file
@@ -0,0 +1,255 @@
|
||||
// Manually added listings (scripts/house-listing.ts). Pure functions, no DB or
|
||||
// network, so the CLI and tests share them. Keep free of `#lib` / `$app` imports.
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as z from 'zod';
|
||||
import { listingRef, parseEuros, parseInteger } from './pararius';
|
||||
import {
|
||||
listingDetailSchema,
|
||||
listingStatuses,
|
||||
offers,
|
||||
propertyTypes,
|
||||
type Offer,
|
||||
type PropertyType
|
||||
} from './types';
|
||||
|
||||
// Short, stable source names for sites we see often. Anything else uses the
|
||||
// registrable domain name without TLD ("www.dupreemakelaars.nl" → "dupreemakelaars").
|
||||
const sourceAliases: Record<string, string> = {
|
||||
'vbtverhuurmakelaars.nl': 'vbt',
|
||||
'ikwilhuren.nu': 'ikwilhuren',
|
||||
'huurportaal.nl': 'huurportaal',
|
||||
'middenholland.mijndak.nl': 'mijndak'
|
||||
};
|
||||
|
||||
const cityAliases: Record<string, string> = {
|
||||
's-gravenhage': 'den-haag',
|
||||
'the-hague': 'den-haag',
|
||||
's-hertogenbosch': 'den-bosch'
|
||||
};
|
||||
|
||||
/** City name → slug in Pararius' style: "Nieuwerkerk aan den IJssel" → "nieuwerkerk-aan-den-ijssel". */
|
||||
export function citySlugFromName(city: string | null | undefined): string | null {
|
||||
const slug = (city ?? '')
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
if (!slug) return null;
|
||||
return cityAliases[slug] ?? slug;
|
||||
}
|
||||
|
||||
const hash = (value: string, length = 16) =>
|
||||
createHash('sha256').update(value).digest('hex').slice(0, length);
|
||||
|
||||
export type ListingKey = {
|
||||
source: string;
|
||||
sourceId: string;
|
||||
url: string;
|
||||
citySlug?: string;
|
||||
offer?: Offer;
|
||||
propertyType?: PropertyType;
|
||||
};
|
||||
|
||||
/**
|
||||
* Deterministic (source, sourceId) for a listing URL, so adding the same link
|
||||
* twice updates one row. Pararius uses the worker's own key, so a manual add and
|
||||
* a later scrape end up as the same listing.
|
||||
*/
|
||||
export function keyFromUrl(raw: string): ListingKey {
|
||||
const url = new URL(raw.trim());
|
||||
const host = url.hostname
|
||||
.toLowerCase()
|
||||
.replace(/^www\./, '')
|
||||
.replace(/^m\./, '');
|
||||
|
||||
if (host.endsWith('pararius.nl') || host.endsWith('pararius.com')) {
|
||||
const ref = listingRef(url.pathname);
|
||||
if (ref) return { source: 'pararius', ...ref };
|
||||
}
|
||||
if (host.endsWith('funda.nl')) {
|
||||
const ids = url.pathname.match(/\d{6,}/g);
|
||||
if (ids) return { source: 'funda', sourceId: ids.at(-1)!, url: stripTracking(url) };
|
||||
}
|
||||
|
||||
const parts = host.split('.');
|
||||
const source = (sourceAliases[host] ?? (parts.length > 1 ? parts.at(-2)! : host))
|
||||
.replace(/[^a-z0-9-]/g, '-')
|
||||
.slice(0, 40);
|
||||
const path = url.pathname.toLowerCase().replace(/\/+$/, '').replace(/^\/+/, '');
|
||||
const cleaned = stripTracking(url);
|
||||
const sourceId =
|
||||
path && path.length <= 80 && !new URL(cleaned).search ? path : hash(cleaned.toLowerCase());
|
||||
return { source: source || 'manual', sourceId, url: cleaned };
|
||||
}
|
||||
|
||||
const tracking = /^(utm_|fbclid$|gclid$|sfnsn$|mibextid$|unique_share_id$|s$)/;
|
||||
|
||||
function stripTracking(url: URL): string {
|
||||
const copy = new URL(url);
|
||||
for (const name of [...copy.searchParams.keys()])
|
||||
if (tracking.test(name)) copy.searchParams.delete(name);
|
||||
copy.hash = '';
|
||||
return copy.toString();
|
||||
}
|
||||
|
||||
/** Key for a listing without a link (e.g. from a screenshot), from its address. */
|
||||
export function keyFromAddress(parts: {
|
||||
street?: string | null;
|
||||
postalCode?: string | null;
|
||||
city?: string | null;
|
||||
title?: string | null;
|
||||
}): ListingKey {
|
||||
const basis = [parts.street, parts.postalCode, parts.city, parts.title]
|
||||
.map((v) => (v ?? '').toLowerCase().replace(/\s+/g, ' ').trim())
|
||||
.join('|');
|
||||
if (basis === '|||') throw new Error('Need a url, or at least a street/title and city');
|
||||
const query = [parts.street, parts.postalCode, parts.city].filter(Boolean).join(', ');
|
||||
return {
|
||||
source: 'manual',
|
||||
sourceId: `m-${hash(basis, 12)}`,
|
||||
url: `https://www.google.com/maps/search/?api=1&query=${encodeURIComponent(query || parts.title || '')}`
|
||||
};
|
||||
}
|
||||
|
||||
const lenientInt = (parse: (v: string) => number | null) =>
|
||||
z.preprocess(
|
||||
(v) => (typeof v === 'string' ? (v.trim() === '' ? null : parse(v)) : v),
|
||||
z.number().int().min(0).nullish()
|
||||
);
|
||||
const lenientFloat = z.preprocess(
|
||||
(v) => (typeof v === 'string' ? (v.trim() === '' ? null : Number(v.replace(',', '.'))) : v),
|
||||
z.number().finite().nullish()
|
||||
);
|
||||
const lenientDate = z.preprocess(
|
||||
(v) => {
|
||||
if (v == null || v === '') return v;
|
||||
const d = new Date(String(v));
|
||||
return Number.isNaN(d.getTime()) ? v : d.toISOString();
|
||||
},
|
||||
z.iso.datetime({ offset: true }).nullish()
|
||||
);
|
||||
|
||||
// What a person may type. Same limits as the worker contract, but lenient:
|
||||
// "€ 1.422,- p.m." is a price, "84 m²" is an area, everything is optional
|
||||
// except a url or an address.
|
||||
export const manualListingSchema = listingDetailSchema
|
||||
.omit({ sourceId: true, url: true, offer: true, propertyType: true, offeredAt: true })
|
||||
.partial()
|
||||
.extend({
|
||||
url: z.url().max(500).nullish(),
|
||||
source: z
|
||||
.string()
|
||||
.regex(/^[a-z0-9-]{1,40}$/)
|
||||
.nullish(),
|
||||
sourceId: z.string().trim().min(1).max(80).nullish(),
|
||||
offer: z.enum(offers).nullish(),
|
||||
propertyType: z.enum(propertyTypes).nullish(),
|
||||
priceEur: lenientInt(parseEuros),
|
||||
areaM2: lenientInt(parseInteger),
|
||||
rooms: lenientInt(parseInteger),
|
||||
bedrooms: lenientInt(parseInteger),
|
||||
bathrooms: lenientInt(parseInteger),
|
||||
constructionYear: lenientInt(parseInteger),
|
||||
depositEur: lenientInt(parseEuros),
|
||||
lat: lenientFloat,
|
||||
lng: lenientFloat,
|
||||
offeredAt: lenientDate,
|
||||
firstSeenAt: lenientDate,
|
||||
status: z.enum(listingStatuses).nullish(),
|
||||
notes: z.string().trim().max(2000).nullish()
|
||||
});
|
||||
export type ManualListing = z.infer<typeof manualListingSchema>;
|
||||
|
||||
/** Accept camelCase, snake_case and a few short names ("price", "area", "type"). */
|
||||
const fieldAliases: Record<string, string> = {
|
||||
price: 'priceEur',
|
||||
area: 'areaM2',
|
||||
m2: 'areaM2',
|
||||
type: 'propertyType',
|
||||
year: 'constructionYear',
|
||||
deposit: 'depositEur',
|
||||
postcode: 'postalCode',
|
||||
zip: 'postalCode',
|
||||
image: 'imageUrl',
|
||||
label: 'energyLabel',
|
||||
available: 'availableFrom',
|
||||
seen: 'firstSeenAt',
|
||||
firstSeen: 'firstSeenAt',
|
||||
offered: 'offeredAt'
|
||||
};
|
||||
|
||||
export function normalizeKeys(input: Record<string, unknown>): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(input)) {
|
||||
if (value === undefined) continue;
|
||||
const camel = key.replace(/[-_]([a-z0-9])/g, (_, c: string) => c.toUpperCase());
|
||||
out[fieldAliases[camel] ?? camel] = value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Guess rent/buy and the property type from the URL and title when not given. */
|
||||
export function inferOffer(text: string): Offer | null {
|
||||
if (/\b(koop|te-koop|buy|for-sale|kopen)\b/i.test(text)) return 'buy';
|
||||
if (/\b(huur|te-huur|rent|huren|for-rent)\b/i.test(text)) return 'rent';
|
||||
return null;
|
||||
}
|
||||
|
||||
export function inferPropertyType(text: string): PropertyType | null {
|
||||
if (/appartement|apartment|flat|portiek|bovenwoning|benedenwoning/i.test(text))
|
||||
return 'apartment';
|
||||
if (/studio/i.test(text)) return 'studio';
|
||||
if (/\bkamer\b|\broom\b/i.test(text)) return 'room';
|
||||
if (/\bhuis\b|woonhuis|eengezins|tussenwoning|hoekwoning|house|villa/i.test(text)) return 'house';
|
||||
return null;
|
||||
}
|
||||
|
||||
// Listing fields → house.listings columns.
|
||||
export const columns = {
|
||||
url: 'url',
|
||||
offer: 'offer',
|
||||
propertyType: 'property_type',
|
||||
title: 'title',
|
||||
externalId: 'external_id',
|
||||
street: 'street',
|
||||
postalCode: 'postal_code',
|
||||
city: 'city',
|
||||
neighbourhood: 'neighbourhood',
|
||||
priceEur: 'price_eur',
|
||||
areaM2: 'area_m2',
|
||||
rooms: 'rooms',
|
||||
bedrooms: 'bedrooms',
|
||||
bathrooms: 'bathrooms',
|
||||
constructionYear: 'construction_year',
|
||||
energyLabel: 'energy_label',
|
||||
interior: 'interior',
|
||||
depositEur: 'deposit_eur',
|
||||
availableFrom: 'available_from',
|
||||
contractType: 'contract_type',
|
||||
agent: 'agent',
|
||||
imageUrl: 'image_url',
|
||||
description: 'description',
|
||||
features: 'features',
|
||||
lat: 'lat',
|
||||
lng: 'lng',
|
||||
offeredAt: 'offered_at',
|
||||
firstSeenAt: 'first_seen_at',
|
||||
status: 'status',
|
||||
notes: 'notes'
|
||||
} as const satisfies Partial<Record<keyof ManualListing, string>>;
|
||||
|
||||
/** Only the columns the user actually gave; `null` clears a column on update. */
|
||||
export function toColumns(listing: ManualListing): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [field, column] of Object.entries(columns)) {
|
||||
const value = listing[field as keyof ManualListing];
|
||||
if (value === undefined) continue;
|
||||
out[column] =
|
||||
value !== null && (field === 'offeredAt' || field === 'firstSeenAt')
|
||||
? new Date(value as string)
|
||||
: value;
|
||||
}
|
||||
if (listing.city !== undefined) out.city_slug = citySlugFromName(listing.city);
|
||||
return out;
|
||||
}
|
||||
Reference in New Issue
Block a user