groceries: nightly AH/Jumbo catalogue import, unit-price normalisation, product groups and compare UI

scripts/groceries-import.ts

@@ -0,0 +1,63 @@
+// Nightly supermarket catalogue import (Albert Heijn + Jumbo) and product regrouping.
+//
+//   bun --env-file=$HOME/.config/groceries-import.env scripts/groceries-import.ts [options]
+//     --store ah|jumbo     only this store (repeatable; default: all)
+//     --skip-fetch         no store requests; only regroup what is in the DB
+//     --gtin-limit N       AH product details to fetch for EANs this run (default 1500, 0 = off)
+//     --delay-ms N         minimum delay between requests (default 2000)
+//
+// Politeness: one request at a time, >= 2 s apart, backoff on 429/5xx, identifying User-Agent,
+// and a same-day cache in $GROCERIES_CACHE_DIR (default ~/.cache/groceries, kept 7 days).
+// Plus is not imported: plus.nl blocks this server's IP range (Incapsula), and we do not evade it.
+import postgres from 'postgres';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
+import { enrichAhGtins, importStore, regroup } from '../src/lib/server/groceries/import';
+import { SOURCES } from '../src/lib/server/groceries/sources';
+
+const args = process.argv.slice(2);
+const values = (name: string) =>
+	args.flatMap((a, i) => (a === name && args[i + 1] ? [args[i + 1]] : []));
+const num = (name: string, d: number) => {
+	const v = values(name)[0];
+	const n = v === undefined ? d : Number(v);
+	if (!Number.isFinite(n) || n < 0) throw new Error(`${name} must be a number`);
+	return n;
+};
+if (!process.env.DB_URL) throw new Error('DB_URL is not set');
+const stores = values('--store').length ? values('--store') : Object.keys(SOURCES);
+for (const s of stores) if (!SOURCES[s]) throw new Error(`unknown store ${s}`);
+const delay = num('--delay-ms', 2000);
+const gtinLimit = num('--gtin-limit', 1500);
+const cacheDir = process.env.GROCERIES_CACHE_DIR || join(homedir(), '.cache', 'groceries');
+const log = (m: string) => console.log(`[${new Date().toISOString().slice(11, 19)}Z] ${m}`);
+
+const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
+let failed = false;
+try {
+	pruneCache(cacheDir);
+	if (!args.includes('--skip-fetch')) {
+		for (const slug of stores) {
+			const f = new PoliteFetcher({ store: slug, minDelayMs: delay, cacheDir, log });
+			try {
+				log(`${slug}: importing`);
+				const r = await importStore(sql, SOURCES[slug], f, log);
+				log(
+					`${slug}: ${r.fetched} products (${r.inserted} new, ${r.priceChanges} price changes, ${r.delisted} delisted); ${r.requests} requests, ${r.cacheHits} from cache`
+				);
+				if (slug === 'ah' && gtinLimit > 0) {
+					const g = await enrichAhGtins(sql, f, gtinLimit, log);
+					log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
+				}
+			} catch (e) {
+				failed = true;
+				log(`${slug}: FAILED ${e instanceof Error ? e.message : e}`);
+			}
+		}
+	}
+	await regroup(sql, log);
+} finally {
+	await sql.end();
+}
+if (failed) process.exitCode = 1;