groceries: add PLUS, Xenos and Amazing Oriental stores, is_food flag, PLUS EAN backfill

src/lib/server/groceries/http.ts

@@ -12,6 +12,8 @@ export type FetcherOptions = {
 	store: string;
 	minDelayMs?: number;
 	maxRetries?: number;
+	/** First retry backoff; doubles each try (default 5 s). */
+	retryBaseMs?: number;
 	cacheDir?: string | null;
 	log?: (msg: string) => void;
 };
@@ -25,12 +27,14 @@ export class PoliteFetcher {
 	private last = 0;
 	private readonly minDelayMs: number;
 	private readonly maxRetries: number;
+	private readonly retryBaseMs: number;
 	private readonly dir: string | null;
 	private readonly log: (msg: string) => void;
 
 	constructor(opts: FetcherOptions) {
 		this.minDelayMs = opts.minDelayMs ?? 2000;
 		this.maxRetries = opts.maxRetries ?? 5;
+		this.retryBaseMs = opts.retryBaseMs ?? 5000;
 		this.log = opts.log ?? (() => {});
 		this.dir = opts.cacheDir ? join(opts.cacheDir, today(), opts.store) : null;
 		if (this.dir) mkdirSync(this.dir, { recursive: true });
@@ -38,10 +42,25 @@ export class PoliteFetcher {
 
 	/** Fetch JSON. `cacheKey` (a file-safe string) enables the same-day cache. */
 	async json<T>(url: string, init: RequestInit = {}, cacheKey?: string): Promise<T> {
-		const file = this.dir && cacheKey ? join(this.dir, `${cacheKey}.json.gz`) : null;
+		return this.request(url, init, cacheKey, 'json', (text) => JSON.parse(text) as T);
+	}
+
+	/** Fetch text (HTML). Same politeness, retries and same-day cache as json(). */
+	async text(url: string, init: RequestInit = {}, cacheKey?: string): Promise<string> {
+		return this.request(url, init, cacheKey, 'html', (text) => text);
+	}
+
+	private async request<T>(
+		url: string,
+		init: RequestInit,
+		cacheKey: string | undefined,
+		ext: string,
+		parse: (text: string) => T
+	): Promise<T> {
+		const file = this.dir && cacheKey ? join(this.dir, `${cacheKey}.${ext}.gz`) : null;
 		if (file && existsSync(file)) {
 			this.cacheHits++;
-			return JSON.parse(gunzipSync(readFileSync(file)).toString('utf8')) as T;
+			return parse(gunzipSync(readFileSync(file)).toString('utf8'));
 		}
 		for (let attempt = 0; ; attempt++) {
 			const wait = this.last + this.minDelayMs - Date.now();
@@ -54,7 +73,7 @@ export class PoliteFetcher {
 				const res = await fetch(url, { ...init, signal: AbortSignal.timeout(45_000) });
 				if (res.ok) {
 					const text = await res.text();
-					const data = JSON.parse(text) as T;
+					const data = parse(text);
 					if (file) writeFileSync(file, gzipSync(text));
 					return data;
 				}
@@ -67,18 +86,21 @@ export class PoliteFetcher {
 				}
 				retryAfter = Number(res.headers.get('retry-after')) * 1000 || 0;
 			} catch (e) {
-				if (e instanceof HttpError) throw e;
+				if (e instanceof HttpError || e instanceof FatalFetchError) throw e;
 				problem = e instanceof Error ? e.message : String(e);
 			}
 			if (attempt >= this.maxRetries)
 				throw new Error(`${problem} for ${url} after ${attempt + 1} tries`);
-			const backoff = Math.max(retryAfter, 5000 * 2 ** attempt);
+			const backoff = Math.max(retryAfter, this.retryBaseMs * 2 ** attempt);
 			this.log(`  ${problem}; retrying in ${Math.round(backoff / 1000)}s`);
 			await sleep(backoff);
 		}
 	}
 }
 
+/** Thrown from a parse callback to stop without retrying (e.g. a bot wall instead of data). */
+export class FatalFetchError extends Error {}
+
 export class HttpError extends Error {
 	constructor(
 		readonly status: number,