131 lines
4.5 KiB
TypeScript
131 lines
4.5 KiB
TypeScript
// Polite JSON fetcher for the grocery importers: one request at a time with a minimum delay,
|
|
// exponential backoff on 429/5xx/network errors (honours Retry-After), and a same-day disk
|
|
// cache so re-running an import on the same day does not hit the store again.
|
|
import { mkdirSync, existsSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
|
|
import { gunzipSync, gzipSync } from 'node:zlib';
|
|
import { join } from 'node:path';
|
|
|
|
export const USER_AGENT =
|
|
'stanrunge.dev grocery price tracker (personal use, 1 req/2s; +https://stanrunge.dev)';
|
|
|
|
export type FetcherOptions = {
|
|
store: string;
|
|
minDelayMs?: number;
|
|
maxRetries?: number;
|
|
/** First retry backoff; doubles each try (default 5 s). */
|
|
retryBaseMs?: number;
|
|
cacheDir?: string | null;
|
|
/** Extra 4xx statuses to retry (e.g. a CDN's transient 401/403). */
|
|
retryStatuses?: number[];
|
|
log?: (msg: string) => void;
|
|
};
|
|
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
const today = () => new Date().toLocaleDateString('sv-SE', { timeZone: 'Europe/Amsterdam' });
|
|
|
|
export class PoliteFetcher {
|
|
requests = 0;
|
|
cacheHits = 0;
|
|
private last = 0;
|
|
private readonly minDelayMs: number;
|
|
private readonly retryStatuses: number[];
|
|
private readonly maxRetries: number;
|
|
private readonly retryBaseMs: number;
|
|
private readonly dir: string | null;
|
|
private readonly log: (msg: string) => void;
|
|
|
|
constructor(opts: FetcherOptions) {
|
|
this.minDelayMs = opts.minDelayMs ?? 2000;
|
|
this.retryStatuses = opts.retryStatuses ?? [];
|
|
this.maxRetries = opts.maxRetries ?? 5;
|
|
this.retryBaseMs = opts.retryBaseMs ?? 5000;
|
|
this.log = opts.log ?? (() => {});
|
|
this.dir = opts.cacheDir ? join(opts.cacheDir, today(), opts.store) : null;
|
|
if (this.dir) mkdirSync(this.dir, { recursive: true });
|
|
}
|
|
|
|
/** Fetch JSON. `cacheKey` (a file-safe string) enables the same-day cache. */
|
|
async json<T>(url: string, init: RequestInit = {}, cacheKey?: string): Promise<T> {
|
|
return this.request(url, init, cacheKey, 'json', (text) => JSON.parse(text) as T);
|
|
}
|
|
|
|
/** Fetch text (HTML). Same politeness, retries and same-day cache as json(). */
|
|
async text(url: string, init: RequestInit = {}, cacheKey?: string): Promise<string> {
|
|
return this.request(url, init, cacheKey, 'html', (text) => text);
|
|
}
|
|
|
|
private async request<T>(
|
|
url: string,
|
|
init: RequestInit,
|
|
cacheKey: string | undefined,
|
|
ext: string,
|
|
parse: (text: string) => T
|
|
): Promise<T> {
|
|
const file = this.dir && cacheKey ? join(this.dir, `${cacheKey}.${ext}.gz`) : null;
|
|
if (file && existsSync(file)) {
|
|
this.cacheHits++;
|
|
return parse(gunzipSync(readFileSync(file)).toString('utf8'));
|
|
}
|
|
for (let attempt = 0; ; attempt++) {
|
|
const wait = this.last + this.minDelayMs - Date.now();
|
|
if (wait > 0) await sleep(wait);
|
|
this.last = Date.now();
|
|
this.requests++;
|
|
let retryAfter = 0;
|
|
let problem: string;
|
|
try {
|
|
const res = await fetch(url, { ...init, signal: AbortSignal.timeout(45_000) });
|
|
if (res.ok) {
|
|
const text = await res.text();
|
|
const data = parse(text);
|
|
if (file) writeFileSync(file, gzipSync(text));
|
|
return data;
|
|
}
|
|
problem = `HTTP ${res.status}`;
|
|
if (
|
|
res.status !== 429 &&
|
|
res.status < 500 &&
|
|
res.status !== 408 &&
|
|
!this.retryStatuses.includes(res.status)
|
|
) {
|
|
throw new HttpError(
|
|
res.status,
|
|
`${problem} for ${url}: ${(await res.text()).slice(0, 200)}`
|
|
);
|
|
}
|
|
retryAfter = Number(res.headers.get('retry-after')) * 1000 || 0;
|
|
} catch (e) {
|
|
if (e instanceof HttpError || e instanceof FatalFetchError) throw e;
|
|
problem = e instanceof Error ? e.message : String(e);
|
|
}
|
|
if (attempt >= this.maxRetries)
|
|
throw new Error(`${problem} for ${url} after ${attempt + 1} tries`);
|
|
const backoff = Math.max(retryAfter, this.retryBaseMs * 2 ** attempt);
|
|
this.log(` ${problem}; retrying in ${Math.round(backoff / 1000)}s`);
|
|
await sleep(backoff);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Thrown from a parse callback to stop without retrying (e.g. a bot wall instead of data). */
|
|
export class FatalFetchError extends Error {}
|
|
|
|
export class HttpError extends Error {
|
|
constructor(
|
|
readonly status: number,
|
|
message: string
|
|
) {
|
|
super(message);
|
|
}
|
|
}
|
|
|
|
/** Delete cache days older than `keepDays`. */
|
|
export function pruneCache(cacheDir: string, keepDays = 7) {
|
|
if (!existsSync(cacheDir)) return;
|
|
const cutoff = new Date(Date.now() - keepDays * 86_400_000).toLocaleDateString('sv-SE', {
|
|
timeZone: 'Europe/Amsterdam'
|
|
});
|
|
for (const d of readdirSync(cacheDir))
|
|
if (/^\d{4}-\d{2}-\d{2}$/.test(d) && d < cutoff) rmSync(join(cacheDir, d), { recursive: true });
|
|
}
|