From 0e98acd0a864a72f39ed446cdc29ca080697a1ef Mon Sep 17 00:00:00 2001
From: Stan Runge <stan@stanrunge.dev>
Date: Sun, 25 Oct 2026 19:31:39 +0000
Subject: [PATCH] =?UTF-8?q?groceries:=20Lidl=20=E2=80=94=20search=20API=20?=
 =?UTF-8?q?only=20public=20for=20CDN-cached=20URLs=20(401=20otherwise);=20?=
 =?UTF-8?q?later=20pages=20best-effort,=20off=20the=20nightly=20run?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/lib/server/groceries/lidl.ts | 50 ++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/src/lib/server/groceries/lidl.ts b/src/lib/server/groceries/lidl.ts
index 4ea57df..1287f9b 100644
--- a/src/lib/server/groceries/lidl.ts
+++ b/src/lib/server/groceries/lidl.ts
@@ -10,6 +10,9 @@ const BASE = 'https://www.lidl.nl';
 const API = `${BASE}/q/api/search`;
 const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
 const PAGE = 60; // the API caps fetchsize at 60
+// The search API is only reliably public for what lidl.nl's CDN has cached (the URLs the site
+// itself requests); other variants (deeper offsets, other sorts) often get 401 from the origin.
+// So: page 0 must load; later pages are best-effort and a 401 there just ends that query.
 const QUERIES: { name: string; params: string }[] = [
 	{ name: 'eten-en-drinken', params: 'category.id=10068374' },
 	{ name: 'in-de-winkel', params: 'q=*&store=1' },
@@ -133,28 +136,43 @@ export function lidlItem(g: LidlGrid): RawItem | null {
 
 export const lidlSource: Source = {
 	slug: 'lidl',
-	minDelayMs: 1000,
-	// lidl.nl's CDN sometimes answers 401 for a moment
-	fetcher: { maxRetries: 4, retryBaseMs: 20_000, retryStatuses: [401, 403] },
+	minDelayMs: 2000,
+	// Off the nightly run until lidl.nl serves the search API reliably (run with --store lidl).
+	nightly: false,
+	// a 401/403 from lidl.nl's CDN is retried a few times, then the run fails loudly
+	fetcher: { maxRetries: 2, retryBaseMs: 30_000, retryStatuses: [401, 403] },
 	async fetchAll(f: PoliteFetcher, log) {
 		const items = new Map<string, RawItem>();
+		const get = (q: (typeof QUERIES)[number], extra: string, key: string) =>
+			f.json<LidlSearch>(
+				`${API}?${q.params}${extra}&fetchsize=${PAGE}&locale=nl_NL&assortment=NL&version=2.1.0`,
+				{ headers: { 'User-Agent': UA, Accept: 'application/json' } },
+				`${q.name}-${key}`
+			);
+		const add = (q: (typeof QUERIES)[number], r: LidlSearch) => {
+			for (const row of r.items ?? []) {
+				const it = row.gridbox?.data && lidlItem(row.gridbox.data);
+				if (!it || items.has(it.storeProductId)) continue;
+				// In-store deals include lots of tools/clothing: keep food, drugstore, household.
+				if (q.name === 'in-de-winkel' && it.category === 'Overig' && !it.isFood) continue;
+				items.set(it.storeProductId, it);
+			}
+			return r.items?.length ?? 0;
+		};
 		for (const q of QUERIES) {
 			const before = items.size;
 			for (let offset = 0; offset < 5000; offset += PAGE) {
-				const r = await f.json<LidlSearch>(
-					`${API}?${q.params}&offset=${offset}&fetchsize=${PAGE}&locale=nl_NL&assortment=NL&version=2.1.0`,
-					{ headers: { 'User-Agent': UA, Accept: 'application/json' } },
-					`${q.name}-o${offset}`
-				);
-				const rows = r.items ?? [];
-				for (const row of rows) {
-					const it = row.gridbox?.data && lidlItem(row.gridbox.data);
-					if (!it || items.has(it.storeProductId)) continue;
-					// In-store deals include lots of tools/clothing: keep food, drugstore, household.
-					if (q.name === 'in-de-winkel' && it.category === 'Overig' && !it.isFood) continue;
-					items.set(it.storeProductId, it);
+				let r: LidlSearch;
+				try {
+					r = await get(q, `&offset=${offset}`, `o${offset}`);
+				} catch (e) {
+					if (offset === 0 || !/HTTP 40[13]/.test(String(e))) throw e;
+					log(
+						`  lidl ${q.name}: offset ${offset} not public (${String(e).slice(0, 40)}), stopping`
+					);
+					break;
 				}
-				if (rows.length < PAGE || offset + PAGE >= (r.numFound ?? 0)) break;
+				if (add(q, r) < PAGE || offset + PAGE >= (r.numFound ?? 0)) break;
 			}
 			log(`  lidl ${q.name}: +${items.size - before} (${items.size} total)`);
 		}