houses: nightly Kamernet + Vesteda scraper, dedupe by address, owner-only /trackers/houses
All checks were successful
Deploy production / deploy (push) Successful in 2m44s
All checks were successful
Deploy production / deploy (push) Successful in 2m44s
- scripts/house-scrape.ts: polite nightly scrape (honest UA, 6-10 s apart, stops a source on 403/challenge), ingest + price history + removals, ntfy alerts on a separate topic (max 5 + summary, first run seeds)
- address_key + duplicate_of: copies on another source link to one listing, manual wins
- /projects/house-tracker -> /trackers/houses (308), owner only (404 otherwise); filters for min m2, rooms, source, Gouda region default; notes, add-by-hand form, total rent, also-on links
- scripts/house-db.sh: run the house-listing CLI as the limited house_cli role
- deploy/systemd/house-scrape.{service,timer}
This commit is contained in:
@@ -10,6 +10,7 @@ import {
|
||||
text,
|
||||
uniqueIndex
|
||||
} from 'drizzle-orm/pg-core';
|
||||
import type { AnyPgColumn } from 'drizzle-orm/pg-core';
|
||||
import { houseSchema } from './schemas';
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { id, timestamptz } from './helpers';
|
||||
@@ -55,7 +56,17 @@ export const houseListings = houseSchema.table(
|
||||
removedAt: timestamptz('removed_at'),
|
||||
alertedAt: timestamptz('alerted_at'),
|
||||
status: text('status'),
|
||||
notes: text('notes')
|
||||
notes: text('notes'),
|
||||
// Normalised street + number + city ("burgemeester van dijkesingel|214b|gouda"), only
|
||||
// when the source gives a house number. Same key on another source = same home.
|
||||
addressKey: text('address_key'),
|
||||
// Set on the newer copy of a home that another source (or a manual entry) already has.
|
||||
// The page shows the original and links the copies; status and notes live there.
|
||||
duplicateOf: integer('duplicate_of').references((): AnyPgColumn => houseListings.id, {
|
||||
onDelete: 'set null'
|
||||
}),
|
||||
// Monthly rent including service costs, when a source shows it separately.
|
||||
totalPriceEur: integer('total_price_eur')
|
||||
},
|
||||
(table) => [
|
||||
check('listings_offer_check', sql`offer IN ('rent', 'buy')`),
|
||||
@@ -67,7 +78,10 @@ export const houseListings = houseSchema.table(
|
||||
index('listings_city_idx').on(table.source, table.offer, table.citySlug),
|
||||
index('listings_details_idx')
|
||||
.on(table.firstSeenAt.desc().nullsFirst())
|
||||
.where(sql`detail_fetched_at IS NULL`)
|
||||
.where(sql`detail_fetched_at IS NULL`),
|
||||
index('listings_address_idx')
|
||||
.on(table.addressKey)
|
||||
.where(sql`address_key IS NOT NULL`)
|
||||
]
|
||||
);
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ describe('owner gate', () => {
|
||||
expect(isOwnerWritePath('/trackers/tasks')).toBe(true);
|
||||
expect(isOwnerOnlyPath('/trackers/groceries')).toBe(true);
|
||||
expect(isOwnerWritePath('/trackers/groceries')).toBe(true);
|
||||
expect(isOwnerOnlyPath('/trackers/houses')).toBe(true);
|
||||
expect(isOwnerWritePath('/trackers/houses')).toBe(true);
|
||||
});
|
||||
|
||||
it('gates the book reader, files and APIs but not the catalogue', () => {
|
||||
|
||||
@@ -23,6 +23,7 @@ export const ownerOnlyPrefixes = [
|
||||
'/trackers/health',
|
||||
'/trackers/tasks',
|
||||
'/trackers/groceries',
|
||||
'/trackers/houses',
|
||||
'/trackers/books/files',
|
||||
'/lists/db'
|
||||
] as const;
|
||||
@@ -41,6 +42,7 @@ export const ownerWritePrefixes = [
|
||||
'/trackers/health',
|
||||
'/trackers/tasks',
|
||||
'/trackers/groceries',
|
||||
'/trackers/houses',
|
||||
'/trackers/books'
|
||||
] as const;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user