house worker

docs/house-tracker.md

@@ -59,6 +59,31 @@ WAF skip rule for `URI Path starts with /projects/house-tracker/api/worker/`,
 otherwise the worker's POSTs get challenged too. The endpoints are protected by
 the bearer token.
 
+### Development behind Cloudflare Access
+
+For `https://dev.stanrunge.dev`, keep Access enabled. In Cloudflare Zero Trust,
+go to **Access controls → Service credentials → Service Tokens** and create a
+token named `house-worker`. Save its Client ID and Client Secret. In the Access
+application protecting the dev hostname, add a **Service Auth** policy with
+**Include → Service Token → house-worker**. Keep the existing browser login
+policy. See [Cloudflare's service token documentation](https://developers.cloudflare.com/cloudflare-one/access-controls/service-credentials/service-tokens/).
+
+Set `HOUSE_WORKER_TOKEN` in the dev website's `.env` and restart its dev server.
+On the home machine, copy `house.env.example` to `house.env`, fill in the same
+worker token plus the Cloudflare Client ID and Client Secret, then run:
+
+```sh
+chmod 600 house.env
+bun --env-file=house.env scripts/house-worker.ts --check-api
+```
+
+This makes one authenticated GET to the website's worker health endpoint. It
+does not contact Pararius, sync listings or send alerts. The expected output is
+`API connection OK`. Access credentials are sent only to the website API; the
+client refuses redirects and stops immediately on an authentication failure.
+An HTTP 503 can mean the website's worker token is missing; an HTTP 404 can mean
+the running website does not yet include the health route.
+
 ## Home worker setup
 
 Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with Bun 1.4 or

house.env.example

@@ -0,0 +1,7 @@
+# Copy to house.env on the HOME machine. That filename is ignored by Git.
+HOUSE_API_URL=https://dev.stanrunge.dev
+# Same value as HOUSE_WORKER_TOKEN in the website's .env.
+HOUSE_WORKER_TOKEN=
+# Cloudflare Access service token, used only for requests to HOUSE_API_URL.
+HOUSE_CF_ACCESS_CLIENT_ID=
+HOUSE_CF_ACCESS_CLIENT_SECRET=

scripts/house-worker.ts

@@ -6,6 +6,7 @@
  *   HOUSE_API_URL=https://stanrunge.dev HOUSE_WORKER_TOKEN=... bun scripts/house-worker.ts
  *   bun scripts/house-worker.ts --once     # one fast-lane poll, then exit
  *   bun scripts/house-worker.ts --dry-run  # fetch + parse, print, send nothing
+ *   bun scripts/house-worker.ts --check-api # check website auth only, no Pararius requests
  *
  * Politeness rules, in order of importance:
  * - one request at a time, at least HOUSE_MIN_GAP_SECONDS apart (with jitter)
@@ -20,6 +21,7 @@ import {
 	searchPageUrl
 } from '../src/lib/house-tracker/pararius';
 import { backoffMs, nextDelayMs } from '../src/lib/house-tracker/schedule';
+import { createWorkerApi } from '../src/lib/house-tracker/worker-api';
 import type {
 	Lane,
 	ListingDetail,
@@ -31,6 +33,7 @@ import type {
 
 const args = new Set(process.argv.slice(2));
 const once = args.has('--once');
+const checkApi = args.has('--check-api');
 const dryRun = args.has('--dry-run') || process.env.HOUSE_DRY_RUN === '1';
 
 const list = (value: string | undefined, fallback: string[]) =>
@@ -66,8 +69,18 @@ const config = {
 		'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
 };
 
-if (!dryRun && (!config.api || config.token.length < 32))
+if (checkApi && dryRun) throw new Error('--check-api cannot be combined with dry-run mode');
-	throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required');
+const requestApi = dryRun
+	? null
+	: createWorkerApi(
+			{
+				url: config.api,
+				token: config.token,
+				accessClientId: process.env.HOUSE_CF_ACCESS_CLIENT_ID,
+				accessClientSecret: process.env.HOUSE_CF_ACCESS_CLIENT_SECRET
+			},
+			checkApi ? 1 : 4
+		);
 
 const SOURCE = 'pararius';
 const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -173,28 +186,8 @@ async function getHtml(url: string, referer?: string): Promise<{ html: string; s
 // ---------------------------------------------------------------------------
 // Website API
 
-async function api<T>(path: string, body: unknown): Promise<T | null> {
+async function api<T>(path: 'sync' | 'details' | 'report', body: unknown): Promise<T | null> {
-	if (dryRun) return null;
+	return requestApi ? requestApi<T>(path, body) : null;
-	for (let attempt = 0; attempt < 4; attempt++) {
-		try {
-			const response = await fetch(`${config.api}/projects/house-tracker/api/worker/${path}`, {
-				method: 'POST',
-				headers: { 'content-type': 'application/json', authorization: `Bearer ${config.token}` },
-				body: JSON.stringify(body),
-				signal: AbortSignal.timeout(30_000)
-			});
-			if (response.ok) return (await response.json()) as T;
-			const text = await response.text();
-			if (response.status < 500)
-				throw new Error(`API ${path} ${response.status}: ${text.slice(0, 300)}`);
-			log(`API ${path} ${response.status}, retrying`);
-		} catch (error) {
-			if (attempt === 3) throw error;
-			log(`API ${path} failed: ${(error as Error).message}, retrying`);
-		}
-		await sleep(5000 * 2 ** attempt);
-	}
-	return null;
 }
 
 async function report(
@@ -340,6 +333,12 @@ async function guarded(
 }
 
 async function main() {
+	if (checkApi) {
+		const result = await requestApi!<{ ok: boolean }>('health');
+		if (result?.ok !== true) throw new Error('API health: unexpected response');
+		log('API connection OK. Worker authentication passed; no Pararius requests were made.');
+		return;
+	}
 	log(
 		`house worker: ${config.fastSearches.length} fast, ${config.slowSearches.length} slow searches, mode=${config.fetchMode}${dryRun ? ', dry run' : ''}`
 	);

src/lib/house-tracker/worker-api.test.ts

@@ -0,0 +1,105 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { createWorkerApi } from './worker-api';
+
+const config = {
+	url: 'https://dev.stanrunge.dev',
+	token: 'test-worker-token-0000000000000000',
+	accessClientId: 'test-client.access',
+	accessClientSecret: 'test-access-secret'
+};
+
+afterEach(() => {
+	vi.unstubAllGlobals();
+	vi.useRealTimers();
+});
+
+describe('worker API', () => {
+	it('authenticates the health check with both tokens without sending a body', async () => {
+		const fetch = vi.fn().mockResolvedValue(Response.json({ ok: true }));
+		vi.stubGlobal('fetch', fetch);
+		await expect(createWorkerApi(config)('health')).resolves.toEqual({ ok: true });
+		expect(fetch).toHaveBeenCalledExactlyOnceWith(
+			'https://dev.stanrunge.dev/projects/house-tracker/api/worker/health',
+			expect.objectContaining({
+				method: 'GET',
+				body: undefined,
+				redirect: 'manual',
+				headers: expect.objectContaining({
+					authorization: `Bearer ${config.token}`,
+					'CF-Access-Client-Id': config.accessClientId,
+					'CF-Access-Client-Secret': config.accessClientSecret
+				})
+			})
+		);
+	});
+
+	it('can POST to a server without Cloudflare Access', async () => {
+		const fetch = vi.fn().mockResolvedValue(Response.json({ newIds: [] }));
+		vi.stubGlobal('fetch', fetch);
+		await createWorkerApi({ url: config.url, token: config.token })('sync', { cards: [] });
+		expect(fetch.mock.calls[0][1]).toMatchObject({ method: 'POST', body: '{"cards":[]}' });
+		expect(fetch.mock.calls[0][1].headers).not.toHaveProperty('CF-Access-Client-Secret');
+	});
+
+	it.each([302, 401, 403, 429])(
+		'fails immediately on HTTP %i without exposing response text',
+		async (status) => {
+			const fetch = vi.fn().mockResolvedValue(
+				new Response('private login response', {
+					status,
+					headers: { location: 'https://login.example.com/' }
+				})
+			);
+			vi.stubGlobal('fetch', fetch);
+			const error = await createWorkerApi(config)('health').catch((error: Error) => error);
+			expect(error).toBeInstanceOf(Error);
+			expect((error as Error).message).toContain(`API health ${status}`);
+			expect((error as Error).message).not.toContain('private login response');
+			expect((error as Error).message).not.toContain(config.accessClientSecret);
+			expect(fetch).toHaveBeenCalledTimes(1);
+			expect(fetch.mock.calls[0][1].redirect).toBe('manual');
+		}
+	);
+
+	it('rejects a login page returned with HTTP 200', async () => {
+		const fetch = vi.fn().mockResolvedValue(
+			new Response('<html>login</html>', {
+				headers: { 'content-type': 'text/html' }
+			})
+		);
+		vi.stubGlobal('fetch', fetch);
+		await expect(createWorkerApi(config)('health')).rejects.toThrow('expected JSON');
+		expect(fetch).toHaveBeenCalledTimes(1);
+	});
+
+	it('retries transient server failures and preserves sync data', async () => {
+		vi.useFakeTimers();
+		const fetch = vi
+			.fn()
+			.mockResolvedValueOnce(new Response(null, { status: 502 }))
+			.mockResolvedValueOnce(Response.json({ newIds: [] }));
+		vi.stubGlobal('fetch', fetch);
+		const result = createWorkerApi(config)('sync', { cards: [] });
+		await vi.advanceTimersByTimeAsync(5000);
+		await expect(result).resolves.toEqual({ newIds: [] });
+		expect(fetch).toHaveBeenCalledTimes(2);
+		expect(fetch.mock.calls[1][1].body).toBe('{"cards":[]}');
+	});
+
+	it('does not retry a one-attempt connection check', async () => {
+		const fetch = vi.fn().mockRejectedValue(new Error('connection failed'));
+		vi.stubGlobal('fetch', fetch);
+		await expect(createWorkerApi(config, 1)('health')).rejects.toThrow('could not reach');
+		expect(fetch).toHaveBeenCalledTimes(1);
+	});
+
+	it('requires the complete Access credential pair', () => {
+		expect(() => createWorkerApi({ ...config, accessClientSecret: undefined })).toThrow('Set both');
+		expect(() => createWorkerApi({ ...config, accessClientId: undefined })).toThrow('Set both');
+	});
+
+	it('rejects insecure remote URLs and permits local development', () => {
+		expect(() => createWorkerApi({ ...config, url: 'http://dev.stanrunge.dev' })).toThrow('HTTPS');
+		expect(() => createWorkerApi({ ...config, url: 'http://127.0.0.1:5173' })).not.toThrow();
+	});
+});

src/lib/house-tracker/worker-api.ts

@@ -0,0 +1,78 @@
+// Only requests to our website use these credentials. Listing fetches stay separate.
+type WorkerApiConfig = {
+	url: string;
+	token: string;
+	accessClientId?: string;
+	accessClientSecret?: string;
+};
+
+type WorkerApiPath = 'health' | 'sync' | 'details' | 'report';
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export function createWorkerApi(config: WorkerApiConfig, attempts = 4) {
+	if (!config.url || config.token.length < 32)
+		throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required');
+	if (Boolean(config.accessClientId) !== Boolean(config.accessClientSecret))
+		throw new Error('Set both HOUSE_CF_ACCESS_CLIENT_ID and HOUSE_CF_ACCESS_CLIENT_SECRET');
+	const base = new URL(config.url);
+	const local = ['localhost', '127.0.0.1', '[::1]'].includes(base.hostname);
+	if (base.protocol !== 'https:' && !(base.protocol === 'http:' && local))
+		throw new Error('HOUSE_API_URL must use HTTPS (HTTP is allowed for localhost)');
+	if (base.username || base.password || base.search || base.hash || base.pathname !== '/')
+		throw new Error('HOUSE_API_URL must be just the website origin, without a path or credentials');
+
+	const headers: Record<string, string> = {
+		accept: 'application/json',
+		'content-type': 'application/json',
+		authorization: `Bearer ${config.token}`
+	};
+	if (config.accessClientId && config.accessClientSecret) {
+		headers['CF-Access-Client-Id'] = config.accessClientId;
+		headers['CF-Access-Client-Secret'] = config.accessClientSecret;
+	}
+
+	return async function request<T>(path: WorkerApiPath, body?: unknown): Promise<T> {
+		for (let attempt = 0; attempt < attempts; attempt++) {
+			if (attempt) await sleep(5000 * 2 ** (attempt - 1));
+			let response: Response;
+			try {
+				response = await fetch(`${base.origin}/projects/house-tracker/api/worker/${path}`, {
+					method: body === undefined ? 'GET' : 'POST',
+					headers,
+					body: body === undefined ? undefined : JSON.stringify(body),
+					// Never forward Access credentials to a redirect (including the login page).
+					redirect: 'manual',
+					signal: AbortSignal.timeout(30_000)
+				});
+			} catch {
+				if (attempt + 1 < attempts) continue;
+				throw new Error(`API ${path}: could not reach the website`);
+			}
+			if (response.status >= 500 && attempt + 1 < attempts) {
+				await response.body?.cancel();
+				continue;
+			}
+			if (!response.ok) {
+				await response.body?.cancel();
+				const hint =
+					response.status >= 300 && response.status < 400
+						? 'redirect refused; check the Cloudflare Access service token and Service Auth policy'
+						: response.status === 401 || response.status === 403
+							? 'check the worker token and Cloudflare Access service token/policy'
+							: 'check the website configuration';
+				throw new Error(`API ${path} ${response.status}: ${hint}`);
+			}
+			if (!response.headers.get('content-type')?.toLowerCase().includes('application/json')) {
+				await response.body?.cancel();
+				throw new Error(`API ${path}: expected JSON; check Cloudflare Access and HOUSE_API_URL`);
+			}
+			try {
+				return (await response.json()) as T;
+			} catch {
+				throw new Error(`API ${path}: invalid JSON response`);
+			}
+		}
+		throw new Error(`API ${path}: no request attempts configured`);
+	};
+}

src/routes/projects/house-tracker/api/worker/health/+server.ts

@@ -0,0 +1,9 @@
+import { json } from '@sveltejs/kit';
+import { requireWorker } from '$lib/server/house-tracker';
+import type { RequestHandler } from './$types';
+
+// Authenticated connectivity check; no database writes or listing-site requests.
+export const GET: RequestHandler = async ({ request }) => {
+	requireWorker(request);
+	return json({ ok: true }, { headers: { 'cache-control': 'no-store' } });
+};