house-scrape: re-check huispedia, huislijn, huurportaal and rentola rows and mark clearly gone ones removed
All checks were successful
Deploy production / deploy (push) Successful in 1m15s
All checks were successful
Deploy production / deploy (push) Successful in 1m15s
This commit is contained in:
112
src/lib/house-tracker/recheck.test.ts
Normal file
112
src/lib/house-tracker/recheck.test.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { classifyListingPage, huispediaStatus, recheckSite, removalsLookSane } from './recheck';
|
||||
|
||||
const page = (status: number, finalUrl: string, html = '') => ({ status, finalUrl, html });
|
||||
const hpStatus = (s: string) =>
|
||||
`<li><span class="name">Status</span>\n <span class="value">\n ${s}\n </span></li>`;
|
||||
|
||||
describe('recheckSite', () => {
|
||||
it('knows the four backfill sites, with or without www', () => {
|
||||
expect(recheckSite('https://www.huislijn.nl/huurwoning/x/1')).toBe('huislijn');
|
||||
expect(recheckSite('https://huispedia.nl/gouda/2801gk/kazernestraat/1-a')).toBe('huispedia');
|
||||
expect(recheckSite('https://rentola.nl/listings/a-p1')).toBe('rentola');
|
||||
expect(recheckSite('https://huurportaal.nl/listings/a-p1')).toBe('huurportaal');
|
||||
expect(recheckSite('https://www.pararius.nl/x')).toBeNull();
|
||||
expect(recheckSite(null)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyListingPage', () => {
|
||||
const hp = 'https://huurportaal.nl/listings/mvgm-te-huur-voorzorg-85-gouda-p7d7387';
|
||||
it('huurportaal/rentola: inactive redirect is gone, own page is alive', () => {
|
||||
expect(
|
||||
classifyListingPage(
|
||||
'huurportaal',
|
||||
hp,
|
||||
page(
|
||||
200,
|
||||
'https://huurportaal.nl/huurwoningen/appartement/gouda?inactive_property_id=40072247'
|
||||
)
|
||||
).verdict
|
||||
).toBe('gone');
|
||||
expect(classifyListingPage('huurportaal', hp, page(200, hp)).verdict).toBe('alive');
|
||||
expect(
|
||||
classifyListingPage(
|
||||
'rentola',
|
||||
'https://rentola.nl/listings/x-p1',
|
||||
page(200, 'https://rentola.nl/')
|
||||
).verdict
|
||||
).toBe('unknown');
|
||||
});
|
||||
|
||||
const hl = 'https://www.huislijn.nl/huurwoning/nederland/zuid-holland/4439788/voorzorg-gouda';
|
||||
it('huislijn: redirect to the overview is gone; woningruil is gone; detail is alive', () => {
|
||||
expect(
|
||||
classifyListingPage(
|
||||
'huislijn',
|
||||
hl,
|
||||
page(200, 'https://www.huislijn.nl/huurwoning/nederland/zuid-holland')
|
||||
)
|
||||
).toMatchObject({ verdict: 'gone' });
|
||||
expect(classifyListingPage('huislijn', hl, page(200, hl, '<p>Mooi huis</p>')).verdict).toBe(
|
||||
'alive'
|
||||
);
|
||||
expect(
|
||||
classifyListingPage(
|
||||
'huislijn',
|
||||
hl,
|
||||
page(200, hl, '<p>Belangrijk! Dit geldt uitsluitend voor een woningruil; je moet</p>')
|
||||
)
|
||||
).toMatchObject({ verdict: 'gone', reason: expect.stringContaining('woningruil') });
|
||||
expect(
|
||||
classifyListingPage('huislijn', hl, page(200, 'https://www.huislijn.nl/inloggen')).verdict
|
||||
).toBe('unknown');
|
||||
});
|
||||
|
||||
const hpd = 'https://huispedia.nl/gouda/2801gk/kazernestraat/1-a';
|
||||
it('huispedia: reads the status field', () => {
|
||||
expect(huispediaStatus(hpStatus('Te huur'))).toBe('Te huur');
|
||||
expect(classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Te huur'))).verdict).toBe(
|
||||
'alive'
|
||||
);
|
||||
for (const s of [
|
||||
'Verkocht',
|
||||
'Verhuurd',
|
||||
'Verhuurd onder voorbehoud',
|
||||
'Niet te koop, laatst verkocht in 2019'
|
||||
])
|
||||
expect(classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus(s))).verdict).toBe(
|
||||
'gone'
|
||||
);
|
||||
expect(
|
||||
classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Verkocht onder voorbehoud')))
|
||||
.verdict
|
||||
).toBe('alive');
|
||||
expect(classifyListingPage('huispedia', hpd, page(200, hpd, '<p>nothing</p>')).verdict).toBe(
|
||||
'unknown'
|
||||
);
|
||||
expect(
|
||||
classifyListingPage('huispedia', hpd, page(200, hpd, hpStatus('Iets nieuws'))).verdict
|
||||
).toBe('unknown');
|
||||
});
|
||||
|
||||
it('only a 404/410 on the site itself is gone; errors are unknown', () => {
|
||||
expect(classifyListingPage('huispedia', hpd, page(404, hpd)).verdict).toBe('gone');
|
||||
expect(classifyListingPage('huislijn', hl, page(410, hl)).verdict).toBe('gone');
|
||||
expect(classifyListingPage('huispedia', hpd, page(500, hpd)).verdict).toBe('unknown');
|
||||
expect(classifyListingPage('huispedia', hpd, page(404, 'https://example.com/x')).verdict).toBe(
|
||||
'unknown'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('removalsLookSane', () => {
|
||||
it('refuses a night where most rows look gone', () => {
|
||||
expect(removalsLookSane(60, 6)).toBe(true);
|
||||
expect(removalsLookSane(60, 30)).toBe(true);
|
||||
expect(removalsLookSane(60, 31)).toBe(false);
|
||||
expect(removalsLookSane(5, 3)).toBe(true);
|
||||
expect(removalsLookSane(5, 4)).toBe(false);
|
||||
expect(removalsLookSane(0, 0)).toBe(true);
|
||||
});
|
||||
});
|
||||
128
src/lib/house-tracker/recheck.ts
Normal file
128
src/lib/house-tracker/recheck.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
// Re-checks listings that were added by hand or by a one-off backfill (huispedia, huislijn,
|
||||
// huurportaal, rentola) by fetching their own page, so rows whose ad is gone get marked
|
||||
// removed. Pure helpers, no #lib / $app imports; the fetching lives in scripts/house-scrape.ts.
|
||||
//
|
||||
// Only a clear signal counts as gone: a 404/410 on the listing's own page, the site's own
|
||||
// "no longer available" redirect, or the page's own status saying rented/sold. Anything else
|
||||
// (errors, timeouts, an unfamiliar page) is "unknown" and leaves the row alone.
|
||||
|
||||
export const RECHECK_SITES = ['huispedia', 'huislijn', 'huurportaal', 'rentola'] as const;
|
||||
export type RecheckSite = (typeof RECHECK_SITES)[number];
|
||||
|
||||
export type RecheckVerdict = {
|
||||
verdict: 'alive' | 'gone' | 'unknown';
|
||||
reason: string;
|
||||
};
|
||||
|
||||
const HOSTS: Record<string, RecheckSite> = {
|
||||
'huispedia.nl': 'huispedia',
|
||||
'huislijn.nl': 'huislijn',
|
||||
'huurportaal.nl': 'huurportaal',
|
||||
'rentola.nl': 'rentola'
|
||||
};
|
||||
|
||||
function parse(url: string): URL | null {
|
||||
try {
|
||||
return new URL(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const hostOf = (u: URL) => u.hostname.toLowerCase().replace(/^www\./, '');
|
||||
|
||||
/** Which site a listing link belongs to, or null when it is not one we re-check. */
|
||||
export function recheckSite(url: string | null | undefined): RecheckSite | null {
|
||||
const u = url ? parse(url) : null;
|
||||
return u ? (HOSTS[hostOf(u)] ?? null) : null;
|
||||
}
|
||||
|
||||
/** SQL ILIKE pattern per site for rows whose link is on it (narrowed by recheckSite after). */
|
||||
export const RECHECK_URL_PATTERNS = Object.fromEntries(
|
||||
Object.entries(HOSTS).map(([host, site]) => [site, `%://%${host}/%`])
|
||||
) as Record<RecheckSite, string>;
|
||||
|
||||
const text = (html: string) =>
|
||||
html
|
||||
.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/\s+/g, ' ');
|
||||
|
||||
/** Huispedia's "Status" value from the features list ("Te huur", "Verkocht", ...). */
|
||||
export function huispediaStatus(html: string): string | null {
|
||||
const m = html.match(
|
||||
/<span class="name">\s*Status\s*<\/span>\s*<span class="value">\s*([^<]{1,120}?)\s*<\/span>/i
|
||||
);
|
||||
return m ? m[1].replace(/\s+/g, ' ').trim() : null;
|
||||
}
|
||||
|
||||
const HUISPEDIA_ALIVE = /^(te huur|te koop|onder bod|onder optie|verkocht onder voorbehoud)\b/i;
|
||||
const HUISPEDIA_GONE =
|
||||
/^(verhuurd|verkocht\b(?! onder voorbehoud)|niet te koop|niet te huur|ingetrokken)/i;
|
||||
|
||||
/**
|
||||
* Decide from one fetch of `requestedUrl` whether the ad is still up. `finalUrl` is where
|
||||
* redirects ended. Call only for pages that were not a challenge/block (the caller stops a
|
||||
* site for the night on 403/429/503 or a challenge page).
|
||||
*/
|
||||
export function classifyListingPage(
|
||||
site: RecheckSite,
|
||||
requestedUrl: string,
|
||||
res: { status: number; finalUrl: string; html: string }
|
||||
): RecheckVerdict {
|
||||
const requested = parse(requestedUrl);
|
||||
const final = parse(res.finalUrl) ?? requested;
|
||||
if (!requested || !final) return { verdict: 'unknown', reason: 'bad url' };
|
||||
const sameSite = recheckSite(final.href) === site;
|
||||
|
||||
if ((res.status === 404 || res.status === 410) && sameSite)
|
||||
return { verdict: 'gone', reason: `http ${res.status}` };
|
||||
if (res.status !== 200) return { verdict: 'unknown', reason: `http ${res.status}` };
|
||||
if (!sameSite) return { verdict: 'unknown', reason: `redirected off-site to ${final.host}` };
|
||||
|
||||
switch (site) {
|
||||
case 'huurportaal':
|
||||
case 'rentola': {
|
||||
// Their own "this listing is no longer available" redirect to a search page.
|
||||
if (final.searchParams.has('inactive_property_id'))
|
||||
return { verdict: 'gone', reason: 'redirected as inactive' };
|
||||
if (final.pathname === requested.pathname && final.pathname.startsWith('/listings/'))
|
||||
return { verdict: 'alive', reason: 'listing page' };
|
||||
return { verdict: 'unknown', reason: `redirected to ${final.pathname}` };
|
||||
}
|
||||
case 'huislijn': {
|
||||
const id = requested.pathname.match(/\/(\d{5,})\//)?.[1];
|
||||
if (!id) return { verdict: 'unknown', reason: 'no listing id in link' };
|
||||
// A removed ad redirects to the province/search overview, without the id.
|
||||
if (!final.pathname.includes(`/${id}/`)) {
|
||||
return /\/(huurwoning|koopwoning)\/nederland(\/[a-z-]+)?\/?$/.test(final.pathname)
|
||||
? { verdict: 'gone', reason: 'redirected to the overview' }
|
||||
: { verdict: 'unknown', reason: `redirected to ${final.pathname}` };
|
||||
}
|
||||
if (/geldt uitsluitend voor een woningruil/i.test(text(res.html)))
|
||||
return { verdict: 'gone', reason: 'woningruil only (swap, not a rental)' };
|
||||
return { verdict: 'alive', reason: 'listing page' };
|
||||
}
|
||||
case 'huispedia': {
|
||||
// Huispedia keeps a page for every address; the ad's status is what changes.
|
||||
if (final.pathname.replace(/\/$/, '') !== requested.pathname.replace(/\/$/, ''))
|
||||
return { verdict: 'unknown', reason: `redirected to ${final.pathname}` };
|
||||
const status = huispediaStatus(res.html);
|
||||
if (!status) return { verdict: 'unknown', reason: 'no status on page' };
|
||||
if (HUISPEDIA_ALIVE.test(status)) return { verdict: 'alive', reason: status };
|
||||
if (HUISPEDIA_GONE.test(status)) return { verdict: 'gone', reason: status };
|
||||
return { verdict: 'unknown', reason: `status "${status}"` };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Safety valve: a night where most checked rows of a site look gone is more likely a site
|
||||
* change than a real exodus, so none of that night's removals for the site are applied.
|
||||
*/
|
||||
export function removalsLookSane(checked: number, gone: number): boolean {
|
||||
if (gone === 0) return true;
|
||||
if (checked < 10) return gone <= 3;
|
||||
return gone <= Math.max(5, Math.floor(checked * 0.5));
|
||||
}
|
||||
Reference in New Issue
Block a user