groceries: Lidl — search API only public for CDN-cached URLs (401 otherwise); later pages best-effort, off the nightly run

src/lib/server/groceries/lidl.ts

@@ -10,6 +10,9 @@ const BASE = 'https://www.lidl.nl';
 const API = `${BASE}/q/api/search`;
 const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
 const PAGE = 60; // the API caps fetchsize at 60
+// The search API is only reliably public for what lidl.nl's CDN has cached (the URLs the site
+// itself requests); other variants (deeper offsets, other sorts) often get 401 from the origin.
+// So: page 0 must load; later pages are best-effort and a 401 there just ends that query.
 const QUERIES: { name: string; params: string }[] = [
 	{ name: 'eten-en-drinken', params: 'category.id=10068374' },
 	{ name: 'in-de-winkel', params: 'q=*&store=1' },
@@ -133,28 +136,43 @@ export function lidlItem(g: LidlGrid): RawItem | null {
 
 export const lidlSource: Source = {
 	slug: 'lidl',
-	minDelayMs: 1000,
+	minDelayMs: 2000,
-	// lidl.nl's CDN sometimes answers 401 for a moment
+	// Off the nightly run until lidl.nl serves the search API reliably (run with --store lidl).
-	fetcher: { maxRetries: 4, retryBaseMs: 20_000, retryStatuses: [401, 403] },
+	nightly: false,
+	// a 401/403 from lidl.nl's CDN is retried a few times, then the run fails loudly
+	fetcher: { maxRetries: 2, retryBaseMs: 30_000, retryStatuses: [401, 403] },
 	async fetchAll(f: PoliteFetcher, log) {
 		const items = new Map<string, RawItem>();
-		for (const q of QUERIES) {
+		const get = (q: (typeof QUERIES)[number], extra: string, key: string) =>
-			const before = items.size;
+			f.json<LidlSearch>(
-			for (let offset = 0; offset < 5000; offset += PAGE) {
+				`${API}?${q.params}${extra}&fetchsize=${PAGE}&locale=nl_NL&assortment=NL&version=2.1.0`,
-				const r = await f.json<LidlSearch>(
-					`${API}?${q.params}&offset=${offset}&fetchsize=${PAGE}&locale=nl_NL&assortment=NL&version=2.1.0`,
 				{ headers: { 'User-Agent': UA, Accept: 'application/json' } },
-					`${q.name}-o${offset}`
+				`${q.name}-${key}`
 			);
-				const rows = r.items ?? [];
+		const add = (q: (typeof QUERIES)[number], r: LidlSearch) => {
-				for (const row of rows) {
+			for (const row of r.items ?? []) {
 				const it = row.gridbox?.data && lidlItem(row.gridbox.data);
 				if (!it || items.has(it.storeProductId)) continue;
 				// In-store deals include lots of tools/clothing: keep food, drugstore, household.
 				if (q.name === 'in-de-winkel' && it.category === 'Overig' && !it.isFood) continue;
 				items.set(it.storeProductId, it);
 			}
-				if (rows.length < PAGE || offset + PAGE >= (r.numFound ?? 0)) break;
+			return r.items?.length ?? 0;
+		};
+		for (const q of QUERIES) {
+			const before = items.size;
+			for (let offset = 0; offset < 5000; offset += PAGE) {
+				let r: LidlSearch;
+				try {
+					r = await get(q, `&offset=${offset}`, `o${offset}`);
+				} catch (e) {
+					if (offset === 0 || !/HTTP 40[13]/.test(String(e))) throw e;
+					log(
+						`  lidl ${q.name}: offset ${offset} not public (${String(e).slice(0, 40)}), stopping`
+					);
+					break;
+				}
+				if (add(q, r) < PAGE || offset + PAGE >= (r.numFound ?? 0)) break;
 			}
 			log(`  lidl ${q.name}: +${items.size - before} (${items.size} total)`);
 		}