Stan Runge
0846cf7d77
Collects Pararius rent/buy listings into Postgres via a worker that runs on a residential connection, sends Discord/ntfy alerts for new listings matching rules, and shows an admin-only map, market stats and worker log. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013HfF5tnChi8tkCx9svJP6D
112 lines
3.9 KiB
SQL
112 lines
3.9 KiB
SQL
-- House tracker: listings from Dutch housing sites, collected by a home worker.
|
|
-- Idempotent; safe to run on every deploy.
|
|
|
|
CREATE TABLE IF NOT EXISTS house_listings (
|
|
id serial PRIMARY KEY,
|
|
source text NOT NULL,
|
|
source_id text NOT NULL,
|
|
external_id text,
|
|
url text NOT NULL,
|
|
offer text NOT NULL CHECK (offer IN ('rent', 'buy')),
|
|
property_type text NOT NULL,
|
|
title text,
|
|
street text,
|
|
postal_code text,
|
|
city text,
|
|
city_slug text,
|
|
neighbourhood text,
|
|
price_eur integer CHECK (price_eur IS NULL OR price_eur >= 0),
|
|
area_m2 integer,
|
|
rooms integer,
|
|
bedrooms integer,
|
|
bathrooms integer,
|
|
construction_year integer,
|
|
energy_label text,
|
|
interior text,
|
|
deposit_eur integer,
|
|
available_from text,
|
|
contract_type text,
|
|
agent text,
|
|
image_url text,
|
|
description text,
|
|
features jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
lat double precision,
|
|
lng double precision,
|
|
-- When the site says it was published. first_seen_at is when we noticed.
|
|
offered_at timestamptz,
|
|
first_seen_at timestamptz NOT NULL DEFAULT now(),
|
|
-- fast, slow, or seed (already listed when its search was first polled).
|
|
first_seen_lane text NOT NULL DEFAULT 'slow',
|
|
last_seen_at timestamptz NOT NULL DEFAULT now(),
|
|
detail_fetched_at timestamptz,
|
|
removed_at timestamptz,
|
|
alerted_at timestamptz,
|
|
-- Personal tracking.
|
|
status text,
|
|
notes text
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS house_listings_source_idx ON house_listings (source, source_id);
|
|
CREATE INDEX IF NOT EXISTS house_listings_active_idx
|
|
ON house_listings (offer, first_seen_at DESC) WHERE removed_at IS NULL;
|
|
CREATE INDEX IF NOT EXISTS house_listings_city_idx ON house_listings (source, offer, city_slug);
|
|
CREATE INDEX IF NOT EXISTS house_listings_details_idx
|
|
ON house_listings (first_seen_at DESC) WHERE detail_fetched_at IS NULL;
|
|
|
|
-- Every price we have seen, so price drops and market trends survive edits.
|
|
CREATE TABLE IF NOT EXISTS house_price_history (
|
|
listing_id integer NOT NULL REFERENCES house_listings(id) ON DELETE CASCADE,
|
|
observed_at timestamptz NOT NULL DEFAULT now(),
|
|
price_eur integer NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS house_price_history_listing_idx
|
|
ON house_price_history (listing_id, observed_at);
|
|
|
|
-- One row per polled search URL. The first sync of a search only seeds the
|
|
-- database: alerting on everything already listed would be noise.
|
|
CREATE TABLE IF NOT EXISTS house_searches (
|
|
search_url text PRIMARY KEY,
|
|
source text NOT NULL,
|
|
lane text NOT NULL,
|
|
first_synced_at timestamptz NOT NULL DEFAULT now(),
|
|
last_synced_at timestamptz NOT NULL DEFAULT now(),
|
|
last_found integer NOT NULL DEFAULT 0
|
|
);
|
|
|
|
-- Worker health: challenges, HTTP errors, parse failures.
|
|
CREATE TABLE IF NOT EXISTS house_worker_runs (
|
|
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
source text NOT NULL,
|
|
lane text NOT NULL,
|
|
url text NOT NULL,
|
|
status text NOT NULL,
|
|
http_status integer,
|
|
found integer,
|
|
duration_ms integer,
|
|
message text,
|
|
created_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
CREATE INDEX IF NOT EXISTS house_worker_runs_created_idx ON house_worker_runs (created_at DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS house_alert_rules (
|
|
id serial PRIMARY KEY,
|
|
name text NOT NULL,
|
|
enabled boolean NOT NULL DEFAULT true,
|
|
offer text NOT NULL DEFAULT 'rent' CHECK (offer IN ('rent', 'buy')),
|
|
min_price integer,
|
|
max_price integer,
|
|
min_area integer,
|
|
min_bedrooms integer,
|
|
-- Lowercase city names; empty means anywhere.
|
|
cities text[] NOT NULL DEFAULT '{}',
|
|
-- Empty means every property type.
|
|
property_types text[] NOT NULL DEFAULT '{}',
|
|
created_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
|
|
INSERT INTO house_alert_rules (name, offer, min_price, max_price, cities)
|
|
SELECT 'Gouda area rentals', 'rent', 1300, 1700, ARRAY[
|
|
'gouda', 'waddinxveen', 'reeuwijk', 'moordrecht', 'nieuwerkerk aan den ijssel',
|
|
'zevenhuizen', 'bodegraven', 'boskoop', 'oudewater', 'schoonhoven', 'haastrecht'
|
|
]
|
|
WHERE NOT EXISTS (SELECT 1 FROM house_alert_rules);
|