From b52e9188f495fe5ad0a0f08a4a0453b0457d0087 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sun, 11 Oct 2026 22:05:05 +0000 Subject: [PATCH] house worker --- docs/house-tracker.md | 25 +++++ house.env.example | 7 ++ scripts/house-worker.ts | 47 ++++---- src/lib/house-tracker/worker-api.test.ts | 105 ++++++++++++++++++ src/lib/house-tracker/worker-api.ts | 78 +++++++++++++ .../api/worker/health/+server.ts | 9 ++ 6 files changed, 247 insertions(+), 24 deletions(-) create mode 100644 house.env.example create mode 100644 src/lib/house-tracker/worker-api.test.ts create mode 100644 src/lib/house-tracker/worker-api.ts create mode 100644 src/routes/projects/house-tracker/api/worker/health/+server.ts diff --git a/docs/house-tracker.md b/docs/house-tracker.md index af8d98a..14afa65 100644 --- a/docs/house-tracker.md +++ b/docs/house-tracker.md @@ -59,6 +59,31 @@ WAF skip rule for `URI Path starts with /projects/house-tracker/api/worker/`, otherwise the worker's POSTs get challenged too. The endpoints are protected by the bearer token. +### Development behind Cloudflare Access + +For `https://dev.stanrunge.dev`, keep Access enabled. In Cloudflare Zero Trust, +go to **Access controls → Service credentials → Service Tokens** and create a +token named `house-worker`. Save its Client ID and Client Secret. In the Access +application protecting the dev hostname, add a **Service Auth** policy with +**Include → Service Token → house-worker**. Keep the existing browser login +policy. See [Cloudflare's service token documentation](https://developers.cloudflare.com/cloudflare-one/access-controls/service-credentials/service-tokens/). + +Set `HOUSE_WORKER_TOKEN` in the dev website's `.env` and restart its dev server. +On the home machine, copy `house.env.example` to `house.env`, fill in the same +worker token plus the Cloudflare Client ID and Client Secret, then run: + +```sh +chmod 600 house.env +bun --env-file=house.env scripts/house-worker.ts --check-api +``` + +This makes one authenticated GET to the website's worker health endpoint. It +does not contact Pararius, sync listings or send alerts. The expected output is +`API connection OK`. Access credentials are sent only to the website API; the +client refuses redirects and stops immediately on an authentication failure. +An HTTP 503 can mean the website's worker token is missing; an HTTP 404 can mean +the running website does not yet include the health route. + ## Home worker setup Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with Bun 1.4 or diff --git a/house.env.example b/house.env.example new file mode 100644 index 0000000..df563e6 --- /dev/null +++ b/house.env.example @@ -0,0 +1,7 @@ +# Copy to house.env on the HOME machine. That filename is ignored by Git. +HOUSE_API_URL=https://dev.stanrunge.dev +# Same value as HOUSE_WORKER_TOKEN in the website's .env. +HOUSE_WORKER_TOKEN= +# Cloudflare Access service token, used only for requests to HOUSE_API_URL. +HOUSE_CF_ACCESS_CLIENT_ID= +HOUSE_CF_ACCESS_CLIENT_SECRET= diff --git a/scripts/house-worker.ts b/scripts/house-worker.ts index 16f679b..11e3543 100644 --- a/scripts/house-worker.ts +++ b/scripts/house-worker.ts @@ -6,6 +6,7 @@ * HOUSE_API_URL=https://stanrunge.dev HOUSE_WORKER_TOKEN=... bun scripts/house-worker.ts * bun scripts/house-worker.ts --once # one fast-lane poll, then exit * bun scripts/house-worker.ts --dry-run # fetch + parse, print, send nothing + * bun scripts/house-worker.ts --check-api # check website auth only, no Pararius requests * * Politeness rules, in order of importance: * - one request at a time, at least HOUSE_MIN_GAP_SECONDS apart (with jitter) @@ -20,6 +21,7 @@ import { searchPageUrl } from '../src/lib/house-tracker/pararius'; import { backoffMs, nextDelayMs } from '../src/lib/house-tracker/schedule'; +import { createWorkerApi } from '../src/lib/house-tracker/worker-api'; import type { Lane, ListingDetail, @@ -31,6 +33,7 @@ import type { const args = new Set(process.argv.slice(2)); const once = args.has('--once'); +const checkApi = args.has('--check-api'); const dryRun = args.has('--dry-run') || process.env.HOUSE_DRY_RUN === '1'; const list = (value: string | undefined, fallback: string[]) => @@ -66,8 +69,18 @@ const config = { 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36' }; -if (!dryRun && (!config.api || config.token.length < 32)) - throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required'); +if (checkApi && dryRun) throw new Error('--check-api cannot be combined with dry-run mode'); +const requestApi = dryRun + ? null + : createWorkerApi( + { + url: config.api, + token: config.token, + accessClientId: process.env.HOUSE_CF_ACCESS_CLIENT_ID, + accessClientSecret: process.env.HOUSE_CF_ACCESS_CLIENT_SECRET + }, + checkApi ? 1 : 4 + ); const SOURCE = 'pararius'; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -173,28 +186,8 @@ async function getHtml(url: string, referer?: string): Promise<{ html: string; s // --------------------------------------------------------------------------- // Website API -async function api(path: string, body: unknown): Promise { - if (dryRun) return null; - for (let attempt = 0; attempt < 4; attempt++) { - try { - const response = await fetch(`${config.api}/projects/house-tracker/api/worker/${path}`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${config.token}` }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }); - if (response.ok) return (await response.json()) as T; - const text = await response.text(); - if (response.status < 500) - throw new Error(`API ${path} ${response.status}: ${text.slice(0, 300)}`); - log(`API ${path} ${response.status}, retrying`); - } catch (error) { - if (attempt === 3) throw error; - log(`API ${path} failed: ${(error as Error).message}, retrying`); - } - await sleep(5000 * 2 ** attempt); - } - return null; +async function api(path: 'sync' | 'details' | 'report', body: unknown): Promise { + return requestApi ? requestApi(path, body) : null; } async function report( @@ -340,6 +333,12 @@ async function guarded( } async function main() { + if (checkApi) { + const result = await requestApi!<{ ok: boolean }>('health'); + if (result?.ok !== true) throw new Error('API health: unexpected response'); + log('API connection OK. Worker authentication passed; no Pararius requests were made.'); + return; + } log( `house worker: ${config.fastSearches.length} fast, ${config.slowSearches.length} slow searches, mode=${config.fetchMode}${dryRun ? ', dry run' : ''}` ); diff --git a/src/lib/house-tracker/worker-api.test.ts b/src/lib/house-tracker/worker-api.test.ts new file mode 100644 index 0000000..0fef5fb --- /dev/null +++ b/src/lib/house-tracker/worker-api.test.ts @@ -0,0 +1,105 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createWorkerApi } from './worker-api'; + +const config = { + url: 'https://dev.stanrunge.dev', + token: 'test-worker-token-0000000000000000', + accessClientId: 'test-client.access', + accessClientSecret: 'test-access-secret' +}; + +afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); +}); + +describe('worker API', () => { + it('authenticates the health check with both tokens without sending a body', async () => { + const fetch = vi.fn().mockResolvedValue(Response.json({ ok: true })); + vi.stubGlobal('fetch', fetch); + await expect(createWorkerApi(config)('health')).resolves.toEqual({ ok: true }); + expect(fetch).toHaveBeenCalledExactlyOnceWith( + 'https://dev.stanrunge.dev/projects/house-tracker/api/worker/health', + expect.objectContaining({ + method: 'GET', + body: undefined, + redirect: 'manual', + headers: expect.objectContaining({ + authorization: `Bearer ${config.token}`, + 'CF-Access-Client-Id': config.accessClientId, + 'CF-Access-Client-Secret': config.accessClientSecret + }) + }) + ); + }); + + it('can POST to a server without Cloudflare Access', async () => { + const fetch = vi.fn().mockResolvedValue(Response.json({ newIds: [] })); + vi.stubGlobal('fetch', fetch); + await createWorkerApi({ url: config.url, token: config.token })('sync', { cards: [] }); + expect(fetch.mock.calls[0][1]).toMatchObject({ method: 'POST', body: '{"cards":[]}' }); + expect(fetch.mock.calls[0][1].headers).not.toHaveProperty('CF-Access-Client-Secret'); + }); + + it.each([302, 401, 403, 429])( + 'fails immediately on HTTP %i without exposing response text', + async (status) => { + const fetch = vi.fn().mockResolvedValue( + new Response('private login response', { + status, + headers: { location: 'https://login.example.com/' } + }) + ); + vi.stubGlobal('fetch', fetch); + const error = await createWorkerApi(config)('health').catch((error: Error) => error); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain(`API health ${status}`); + expect((error as Error).message).not.toContain('private login response'); + expect((error as Error).message).not.toContain(config.accessClientSecret); + expect(fetch).toHaveBeenCalledTimes(1); + expect(fetch.mock.calls[0][1].redirect).toBe('manual'); + } + ); + + it('rejects a login page returned with HTTP 200', async () => { + const fetch = vi.fn().mockResolvedValue( + new Response('login', { + headers: { 'content-type': 'text/html' } + }) + ); + vi.stubGlobal('fetch', fetch); + await expect(createWorkerApi(config)('health')).rejects.toThrow('expected JSON'); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it('retries transient server failures and preserves sync data', async () => { + vi.useFakeTimers(); + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 502 })) + .mockResolvedValueOnce(Response.json({ newIds: [] })); + vi.stubGlobal('fetch', fetch); + const result = createWorkerApi(config)('sync', { cards: [] }); + await vi.advanceTimersByTimeAsync(5000); + await expect(result).resolves.toEqual({ newIds: [] }); + expect(fetch).toHaveBeenCalledTimes(2); + expect(fetch.mock.calls[1][1].body).toBe('{"cards":[]}'); + }); + + it('does not retry a one-attempt connection check', async () => { + const fetch = vi.fn().mockRejectedValue(new Error('connection failed')); + vi.stubGlobal('fetch', fetch); + await expect(createWorkerApi(config, 1)('health')).rejects.toThrow('could not reach'); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it('requires the complete Access credential pair', () => { + expect(() => createWorkerApi({ ...config, accessClientSecret: undefined })).toThrow('Set both'); + expect(() => createWorkerApi({ ...config, accessClientId: undefined })).toThrow('Set both'); + }); + + it('rejects insecure remote URLs and permits local development', () => { + expect(() => createWorkerApi({ ...config, url: 'http://dev.stanrunge.dev' })).toThrow('HTTPS'); + expect(() => createWorkerApi({ ...config, url: 'http://127.0.0.1:5173' })).not.toThrow(); + }); +}); diff --git a/src/lib/house-tracker/worker-api.ts b/src/lib/house-tracker/worker-api.ts new file mode 100644 index 0000000..558ed8d --- /dev/null +++ b/src/lib/house-tracker/worker-api.ts @@ -0,0 +1,78 @@ +// Only requests to our website use these credentials. Listing fetches stay separate. +type WorkerApiConfig = { + url: string; + token: string; + accessClientId?: string; + accessClientSecret?: string; +}; + +type WorkerApiPath = 'health' | 'sync' | 'details' | 'report'; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export function createWorkerApi(config: WorkerApiConfig, attempts = 4) { + if (!config.url || config.token.length < 32) + throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required'); + if (Boolean(config.accessClientId) !== Boolean(config.accessClientSecret)) + throw new Error('Set both HOUSE_CF_ACCESS_CLIENT_ID and HOUSE_CF_ACCESS_CLIENT_SECRET'); + const base = new URL(config.url); + const local = ['localhost', '127.0.0.1', '[::1]'].includes(base.hostname); + if (base.protocol !== 'https:' && !(base.protocol === 'http:' && local)) + throw new Error('HOUSE_API_URL must use HTTPS (HTTP is allowed for localhost)'); + if (base.username || base.password || base.search || base.hash || base.pathname !== '/') + throw new Error('HOUSE_API_URL must be just the website origin, without a path or credentials'); + + const headers: Record = { + accept: 'application/json', + 'content-type': 'application/json', + authorization: `Bearer ${config.token}` + }; + if (config.accessClientId && config.accessClientSecret) { + headers['CF-Access-Client-Id'] = config.accessClientId; + headers['CF-Access-Client-Secret'] = config.accessClientSecret; + } + + return async function request(path: WorkerApiPath, body?: unknown): Promise { + for (let attempt = 0; attempt < attempts; attempt++) { + if (attempt) await sleep(5000 * 2 ** (attempt - 1)); + let response: Response; + try { + response = await fetch(`${base.origin}/projects/house-tracker/api/worker/${path}`, { + method: body === undefined ? 'GET' : 'POST', + headers, + body: body === undefined ? undefined : JSON.stringify(body), + // Never forward Access credentials to a redirect (including the login page). + redirect: 'manual', + signal: AbortSignal.timeout(30_000) + }); + } catch { + if (attempt + 1 < attempts) continue; + throw new Error(`API ${path}: could not reach the website`); + } + if (response.status >= 500 && attempt + 1 < attempts) { + await response.body?.cancel(); + continue; + } + if (!response.ok) { + await response.body?.cancel(); + const hint = + response.status >= 300 && response.status < 400 + ? 'redirect refused; check the Cloudflare Access service token and Service Auth policy' + : response.status === 401 || response.status === 403 + ? 'check the worker token and Cloudflare Access service token/policy' + : 'check the website configuration'; + throw new Error(`API ${path} ${response.status}: ${hint}`); + } + if (!response.headers.get('content-type')?.toLowerCase().includes('application/json')) { + await response.body?.cancel(); + throw new Error(`API ${path}: expected JSON; check Cloudflare Access and HOUSE_API_URL`); + } + try { + return (await response.json()) as T; + } catch { + throw new Error(`API ${path}: invalid JSON response`); + } + } + throw new Error(`API ${path}: no request attempts configured`); + }; +} diff --git a/src/routes/projects/house-tracker/api/worker/health/+server.ts b/src/routes/projects/house-tracker/api/worker/health/+server.ts new file mode 100644 index 0000000..7090703 --- /dev/null +++ b/src/routes/projects/house-tracker/api/worker/health/+server.ts @@ -0,0 +1,9 @@ +import { json } from '@sveltejs/kit'; +import { requireWorker } from '$lib/server/house-tracker'; +import type { RequestHandler } from './$types'; + +// Authenticated connectivity check; no database writes or listing-site requests. +export const GET: RequestHandler = async ({ request }) => { + requireWorker(request); + return json({ ok: true }, { headers: { 'cache-control': 'no-store' } }); +};