stan/personal-website
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases Wiki Activity

house worker

Browse Source
This commit is contained in:
Stan Runge
2026-10-11 22:05:05 +00:00
parent ecd49fc410
commit b52e9188f4
6 changed files with 247 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
docs/house-tracker.md
View File

@@ -59,6 +59,31 @@ WAF skip rule for `URI Path starts with /projects/house-tracker/api/worker/`,
otherwise the worker's POSTs get challenged too. The endpoints are protected by
the bearer token.
### Development behind Cloudflare Access
For `https://dev.stanrunge.dev`, keep Access enabled. In Cloudflare Zero Trust,
go to **Access controls → Service credentials → Service Tokens** and create a
token named `house-worker`. Save its Client ID and Client Secret. In the Access
application protecting the dev hostname, add a **Service Auth** policy with
**Include → Service Token → house-worker**. Keep the existing browser login
policy. See [Cloudflare's service token documentation](https://developers.cloudflare.com/cloudflare-one/access-controls/service-credentials/service-tokens/).
Set `HOUSE_WORKER_TOKEN` in the dev website's `.env` and restart its dev server.
On the home machine, copy `house.env.example` to `house.env`, fill in the same
worker token plus the Cloudflare Client ID and Client Secret, then run:
```sh
chmod 600 house.env
bun --env-file=house.env scripts/house-worker.ts --check-api
```
This makes one authenticated GET to the website's worker health endpoint. It
does not contact Pararius, sync listings or send alerts. The expected output is
`API connection OK`. Access credentials are sent only to the website API; the
client refuses redirects and stops immediately on an authentication failure.
An HTTP 503 can mean the website's worker token is missing; an HTTP 404 can mean
the running website does not yet include the health route.
## Home worker setup
Needs a machine that stays on (old laptop, Raspberry Pi, NAS) with Bun 1.4 or

7
house.env.example Normal file
View File

@@ -0,0 +1,7 @@
# Copy to house.env on the HOME machine. That filename is ignored by Git.
HOUSE_API_URL=https://dev.stanrunge.dev
# Same value as HOUSE_WORKER_TOKEN in the website's .env.
HOUSE_WORKER_TOKEN=
# Cloudflare Access service token, used only for requests to HOUSE_API_URL.
HOUSE_CF_ACCESS_CLIENT_ID=
HOUSE_CF_ACCESS_CLIENT_SECRET=

47
scripts/house-worker.ts
View File

@@ -6,6 +6,7 @@
* HOUSE_API_URL=https://stanrunge.dev HOUSE_WORKER_TOKEN=... bun scripts/house-worker.ts
* bun scripts/house-worker.ts --once # one fast-lane poll, then exit
* bun scripts/house-worker.ts --dry-run # fetch + parse, print, send nothing
* bun scripts/house-worker.ts --check-api # check website auth only, no Pararius requests
*
* Politeness rules, in order of importance:
* - one request at a time, at least HOUSE_MIN_GAP_SECONDS apart (with jitter)
@@ -20,6 +21,7 @@ import {
searchPageUrl
} from '../src/lib/house-tracker/pararius';
import { backoffMs, nextDelayMs } from '../src/lib/house-tracker/schedule';
import { createWorkerApi } from '../src/lib/house-tracker/worker-api';
import type {
Lane,
ListingDetail,
@@ -31,6 +33,7 @@ import type {
const args = new Set(process.argv.slice(2));
const once = args.has('--once');
const checkApi = args.has('--check-api');
const dryRun = args.has('--dry-run') || process.env.HOUSE_DRY_RUN === '1';
const list = (value: string | undefined, fallback: string[]) =>
@@ -66,8 +69,18 @@ const config = {
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
};
if (!dryRun && (!config.api || config.token.length < 32))
throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required');
if (checkApi && dryRun) throw new Error('--check-api cannot be combined with dry-run mode');
const requestApi = dryRun
? null
: createWorkerApi(
{
url: config.api,
token: config.token,
accessClientId: process.env.HOUSE_CF_ACCESS_CLIENT_ID,
accessClientSecret: process.env.HOUSE_CF_ACCESS_CLIENT_SECRET
},
checkApi ? 1 : 4
);
const SOURCE = 'pararius';
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -173,28 +186,8 @@ async function getHtml(url: string, referer?: string): Promise<{ html: string; s
// ---------------------------------------------------------------------------
// Website API
async function api<T>(path: string, body: unknown): Promise<T | null> {
if (dryRun) return null;
for (let attempt = 0; attempt < 4; attempt++) {
try {
const response = await fetch(`${config.api}/projects/house-tracker/api/worker/${path}`, {
method: 'POST',
headers: { 'content-type': 'application/json', authorization: `Bearer ${config.token}` },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
});
if (response.ok) return (await response.json()) as T;
const text = await response.text();
if (response.status < 500)
throw new Error(`API ${path} ${response.status}: ${text.slice(0, 300)}`);
log(`API ${path} ${response.status}, retrying`);
} catch (error) {
if (attempt === 3) throw error;
log(`API ${path} failed: ${(error as Error).message}, retrying`);
}
await sleep(5000 * 2 ** attempt);
}
return null;
async function api<T>(path: 'sync' | 'details' | 'report', body: unknown): Promise<T | null> {
return requestApi ? requestApi<T>(path, body) : null;
}
async function report(
@@ -340,6 +333,12 @@ async function guarded(
}
async function main() {
if (checkApi) {
const result = await requestApi!<{ ok: boolean }>('health');
if (result?.ok !== true) throw new Error('API health: unexpected response');
log('API connection OK. Worker authentication passed; no Pararius requests were made.');
return;
}
log(
`house worker: ${config.fastSearches.length} fast, ${config.slowSearches.length} slow searches, mode=${config.fetchMode}${dryRun ? ', dry run' : ''}`
);

105
src/lib/house-tracker/worker-api.test.ts Normal file
View File

@@ -0,0 +1,105 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { createWorkerApi } from './worker-api';
const config = {
url: 'https://dev.stanrunge.dev',
token: 'test-worker-token-0000000000000000',
accessClientId: 'test-client.access',
accessClientSecret: 'test-access-secret'
};
afterEach(() => {
vi.unstubAllGlobals();
vi.useRealTimers();
});
describe('worker API', () => {
it('authenticates the health check with both tokens without sending a body', async () => {
const fetch = vi.fn().mockResolvedValue(Response.json({ ok: true }));
vi.stubGlobal('fetch', fetch);
await expect(createWorkerApi(config)('health')).resolves.toEqual({ ok: true });
expect(fetch).toHaveBeenCalledExactlyOnceWith(
'https://dev.stanrunge.dev/projects/house-tracker/api/worker/health',
expect.objectContaining({
method: 'GET',
body: undefined,
redirect: 'manual',
headers: expect.objectContaining({
authorization: `Bearer ${config.token}`,
'CF-Access-Client-Id': config.accessClientId,
'CF-Access-Client-Secret': config.accessClientSecret
})
})
);
});
it('can POST to a server without Cloudflare Access', async () => {
const fetch = vi.fn().mockResolvedValue(Response.json({ newIds: [] }));
vi.stubGlobal('fetch', fetch);
await createWorkerApi({ url: config.url, token: config.token })('sync', { cards: [] });
expect(fetch.mock.calls[0][1]).toMatchObject({ method: 'POST', body: '{"cards":[]}' });
expect(fetch.mock.calls[0][1].headers).not.toHaveProperty('CF-Access-Client-Secret');
});
it.each([302, 401, 403, 429])(
'fails immediately on HTTP %i without exposing response text',
async (status) => {
const fetch = vi.fn().mockResolvedValue(
new Response('private login response', {
status,
headers: { location: 'https://login.example.com/' }
})
);
vi.stubGlobal('fetch', fetch);
const error = await createWorkerApi(config)('health').catch((error: Error) => error);
expect(error).toBeInstanceOf(Error);
expect((error as Error).message).toContain(`API health ${status}`);
expect((error as Error).message).not.toContain('private login response');
expect((error as Error).message).not.toContain(config.accessClientSecret);
expect(fetch).toHaveBeenCalledTimes(1);
expect(fetch.mock.calls[0][1].redirect).toBe('manual');
}
);
it('rejects a login page returned with HTTP 200', async () => {
const fetch = vi.fn().mockResolvedValue(
new Response('<html>login</html>', {
headers: { 'content-type': 'text/html' }
})
);
vi.stubGlobal('fetch', fetch);
await expect(createWorkerApi(config)('health')).rejects.toThrow('expected JSON');
expect(fetch).toHaveBeenCalledTimes(1);
});
it('retries transient server failures and preserves sync data', async () => {
vi.useFakeTimers();
const fetch = vi
.fn()
.mockResolvedValueOnce(new Response(null, { status: 502 }))
.mockResolvedValueOnce(Response.json({ newIds: [] }));
vi.stubGlobal('fetch', fetch);
const result = createWorkerApi(config)('sync', { cards: [] });
await vi.advanceTimersByTimeAsync(5000);
await expect(result).resolves.toEqual({ newIds: [] });
expect(fetch).toHaveBeenCalledTimes(2);
expect(fetch.mock.calls[1][1].body).toBe('{"cards":[]}');
});
it('does not retry a one-attempt connection check', async () => {
const fetch = vi.fn().mockRejectedValue(new Error('connection failed'));
vi.stubGlobal('fetch', fetch);
await expect(createWorkerApi(config, 1)('health')).rejects.toThrow('could not reach');
expect(fetch).toHaveBeenCalledTimes(1);
});
it('requires the complete Access credential pair', () => {
expect(() => createWorkerApi({ ...config, accessClientSecret: undefined })).toThrow('Set both');
expect(() => createWorkerApi({ ...config, accessClientId: undefined })).toThrow('Set both');
});
it('rejects insecure remote URLs and permits local development', () => {
expect(() => createWorkerApi({ ...config, url: 'http://dev.stanrunge.dev' })).toThrow('HTTPS');
expect(() => createWorkerApi({ ...config, url: 'http://127.0.0.1:5173' })).not.toThrow();
});
});

78
src/lib/house-tracker/worker-api.ts Normal file
View File

@@ -0,0 +1,78 @@
// Only requests to our website use these credentials. Listing fetches stay separate.
type WorkerApiConfig = {
url: string;
token: string;
accessClientId?: string;
accessClientSecret?: string;
};
type WorkerApiPath = 'health' | 'sync' | 'details' | 'report';
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
export function createWorkerApi(config: WorkerApiConfig, attempts = 4) {
if (!config.url || config.token.length < 32)
throw new Error('HOUSE_API_URL and HOUSE_WORKER_TOKEN (32+ chars) are required');
if (Boolean(config.accessClientId) !== Boolean(config.accessClientSecret))
throw new Error('Set both HOUSE_CF_ACCESS_CLIENT_ID and HOUSE_CF_ACCESS_CLIENT_SECRET');
const base = new URL(config.url);
const local = ['localhost', '127.0.0.1', '[::1]'].includes(base.hostname);
if (base.protocol !== 'https:' && !(base.protocol === 'http:' && local))
throw new Error('HOUSE_API_URL must use HTTPS (HTTP is allowed for localhost)');
if (base.username || base.password || base.search || base.hash || base.pathname !== '/')
throw new Error('HOUSE_API_URL must be just the website origin, without a path or credentials');
const headers: Record<string, string> = {
accept: 'application/json',
'content-type': 'application/json',
authorization: `Bearer ${config.token}`
};
if (config.accessClientId && config.accessClientSecret) {
headers['CF-Access-Client-Id'] = config.accessClientId;
headers['CF-Access-Client-Secret'] = config.accessClientSecret;
}
return async function request<T>(path: WorkerApiPath, body?: unknown): Promise<T> {
for (let attempt = 0; attempt < attempts; attempt++) {
if (attempt) await sleep(5000 * 2 ** (attempt - 1));
let response: Response;
try {
response = await fetch(`${base.origin}/projects/house-tracker/api/worker/${path}`, {
method: body === undefined ? 'GET' : 'POST',
headers,
body: body === undefined ? undefined : JSON.stringify(body),
// Never forward Access credentials to a redirect (including the login page).
redirect: 'manual',
signal: AbortSignal.timeout(30_000)
});
} catch {
if (attempt + 1 < attempts) continue;
throw new Error(`API ${path}: could not reach the website`);
}
if (response.status >= 500 && attempt + 1 < attempts) {
await response.body?.cancel();
continue;
}
if (!response.ok) {
await response.body?.cancel();
const hint =
response.status >= 300 && response.status < 400
? 'redirect refused; check the Cloudflare Access service token and Service Auth policy'
: response.status === 401 || response.status === 403
? 'check the worker token and Cloudflare Access service token/policy'
: 'check the website configuration';
throw new Error(`API ${path} ${response.status}: ${hint}`);
}
if (!response.headers.get('content-type')?.toLowerCase().includes('application/json')) {
await response.body?.cancel();
throw new Error(`API ${path}: expected JSON; check Cloudflare Access and HOUSE_API_URL`);
}
try {
return (await response.json()) as T;
} catch {
throw new Error(`API ${path}: invalid JSON response`);
}
}
throw new Error(`API ${path}: no request attempts configured`);
};
}

9
src/routes/projects/house-tracker/api/worker/health/+server.ts Normal file
View File

@@ -0,0 +1,9 @@
import { json } from '@sveltejs/kit';
import { requireWorker } from '$lib/server/house-tracker';
import type { RequestHandler } from './$types';
// Authenticated connectivity check; no database writes or listing-site requests.
export const GET: RequestHandler = async ({ request }) => {
requireWorker(request);
return json({ ok: true }, { headers: { 'cache-control': 'no-store' } });
};