groceries: nightly AH/Jumbo catalogue import, unit-price normalisation, product groups and compare UI

src/lib/server/groceries/http.ts

@@ -0,0 +1,99 @@
+// Polite JSON fetcher for the grocery importers: one request at a time with a minimum delay,
+// exponential backoff on 429/5xx/network errors (honours Retry-After), and a same-day disk
+// cache so re-running an import on the same day does not hit the store again.
+import { mkdirSync, existsSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
+import { gunzipSync, gzipSync } from 'node:zlib';
+import { join } from 'node:path';
+
+export const USER_AGENT =
+	'stanrunge.dev grocery price tracker (personal use, 1 req/2s; +https://stanrunge.dev)';
+
+export type FetcherOptions = {
+	store: string;
+	minDelayMs?: number;
+	maxRetries?: number;
+	cacheDir?: string | null;
+	log?: (msg: string) => void;
+};
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+const today = () => new Date().toLocaleDateString('sv-SE', { timeZone: 'Europe/Amsterdam' });
+
+export class PoliteFetcher {
+	requests = 0;
+	cacheHits = 0;
+	private last = 0;
+	private readonly minDelayMs: number;
+	private readonly maxRetries: number;
+	private readonly dir: string | null;
+	private readonly log: (msg: string) => void;
+
+	constructor(opts: FetcherOptions) {
+		this.minDelayMs = opts.minDelayMs ?? 2000;
+		this.maxRetries = opts.maxRetries ?? 5;
+		this.log = opts.log ?? (() => {});
+		this.dir = opts.cacheDir ? join(opts.cacheDir, today(), opts.store) : null;
+		if (this.dir) mkdirSync(this.dir, { recursive: true });
+	}
+
+	/** Fetch JSON. `cacheKey` (a file-safe string) enables the same-day cache. */
+	async json<T>(url: string, init: RequestInit = {}, cacheKey?: string): Promise<T> {
+		const file = this.dir && cacheKey ? join(this.dir, `${cacheKey}.json.gz`) : null;
+		if (file && existsSync(file)) {
+			this.cacheHits++;
+			return JSON.parse(gunzipSync(readFileSync(file)).toString('utf8')) as T;
+		}
+		for (let attempt = 0; ; attempt++) {
+			const wait = this.last + this.minDelayMs - Date.now();
+			if (wait > 0) await sleep(wait);
+			this.last = Date.now();
+			this.requests++;
+			let retryAfter = 0;
+			let problem: string;
+			try {
+				const res = await fetch(url, { ...init, signal: AbortSignal.timeout(45_000) });
+				if (res.ok) {
+					const text = await res.text();
+					const data = JSON.parse(text) as T;
+					if (file) writeFileSync(file, gzipSync(text));
+					return data;
+				}
+				problem = `HTTP ${res.status}`;
+				if (res.status !== 429 && res.status < 500 && res.status !== 408) {
+					throw new HttpError(
+						res.status,
+						`${problem} for ${url}: ${(await res.text()).slice(0, 200)}`
+					);
+				}
+				retryAfter = Number(res.headers.get('retry-after')) * 1000 || 0;
+			} catch (e) {
+				if (e instanceof HttpError) throw e;
+				problem = e instanceof Error ? e.message : String(e);
+			}
+			if (attempt >= this.maxRetries)
+				throw new Error(`${problem} for ${url} after ${attempt + 1} tries`);
+			const backoff = Math.max(retryAfter, 5000 * 2 ** attempt);
+			this.log(`  ${problem}; retrying in ${Math.round(backoff / 1000)}s`);
+			await sleep(backoff);
+		}
+	}
+}
+
+export class HttpError extends Error {
+	constructor(
+		readonly status: number,
+		message: string
+	) {
+		super(message);
+	}
+}
+
+/** Delete cache days older than `keepDays`. */
+export function pruneCache(cacheDir: string, keepDays = 7) {
+	if (!existsSync(cacheDir)) return;
+	const cutoff = new Date(Date.now() - keepDays * 86_400_000).toLocaleDateString('sv-SE', {
+		timeZone: 'Europe/Amsterdam'
+	});
+	for (const d of readdirSync(cacheDir))
+		if (/^\d{4}-\d{2}-\d{2}$/.test(d) && d < cutoff) rmSync(join(cacheDir, d), { recursive: true });
+}