stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

groceries: Lidl — search API only public for CDN-cached URLs (401 otherwise); later pages best-effort, off the nightly run
All checks were successful
Deploy production / deploy (push) Successful in 1m13s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-25 19:31:39 +00:00
parent e2af941b88
commit 0e98acd0a8
1 changed files with 34 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/lib/server/groceries/lidl.ts
View File

@@ -10,6 +10,9 @@ const BASE = 'https://www.lidl.nl';
const API = `${BASE}/q/api/search`;
const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
const PAGE = 60; // the API caps fetchsize at 60
// The search API is only reliably public for what lidl.nl's CDN has cached (the URLs the site
// itself requests); other variants (deeper offsets, other sorts) often get 401 from the origin.
// So: page 0 must load; later pages are best-effort and a 401 there just ends that query.
const QUERIES: { name: string; params: string }[] = [
{ name: 'eten-en-drinken', params: 'category.id=10068374' },
{ name: 'in-de-winkel', params: 'q=*&store=1' },
@@ -133,28 +136,43 @@ export function lidlItem(g: LidlGrid): RawItem | null {
export const lidlSource: Source = {
slug: 'lidl',
minDelayMs: 1000,
// lidl.nl's CDN sometimes answers 401 for a moment
fetcher: { maxRetries: 4, retryBaseMs: 20_000, retryStatuses: [401, 403] },
minDelayMs: 2000,
// Off the nightly run until lidl.nl serves the search API reliably (run with --store lidl).
nightly: false,
// a 401/403 from lidl.nl's CDN is retried a few times, then the run fails loudly
fetcher: { maxRetries: 2, retryBaseMs: 30_000, retryStatuses: [401, 403] },
async fetchAll(f: PoliteFetcher, log) {
const items = new Map<string, RawItem>();
for (const q of QUERIES) {
const before = items.size;
for (let offset = 0; offset < 5000; offset += PAGE) {
const r = await f.json<LidlSearch>(
`${API}?${q.params}&offset=${offset}&fetchsize=${PAGE}&locale=nl_NL&assortment=NL&version=2.1.0`,
const get = (q: (typeof QUERIES)[number], extra: string, key: string) =>
f.json<LidlSearch>(
`${API}?${q.params}${extra}&fetchsize=${PAGE}&locale=nl_NL&assortment=NL&version=2.1.0`,
{ headers: { 'User-Agent': UA, Accept: 'application/json' } },
`${q.name}-o${offset}`
`${q.name}-${key}`
);
const rows = r.items ?? [];
for (const row of rows) {
const add = (q: (typeof QUERIES)[number], r: LidlSearch) => {
for (const row of r.items ?? []) {
const it = row.gridbox?.data && lidlItem(row.gridbox.data);
if (!it || items.has(it.storeProductId)) continue;
// In-store deals include lots of tools/clothing: keep food, drugstore, household.
if (q.name === 'in-de-winkel' && it.category === 'Overig' && !it.isFood) continue;
items.set(it.storeProductId, it);
}
if (rows.length < PAGE || offset + PAGE >= (r.numFound ?? 0)) break;
return r.items?.length ?? 0;
};
for (const q of QUERIES) {
const before = items.size;
for (let offset = 0; offset < 5000; offset += PAGE) {
let r: LidlSearch;
try {
r = await get(q, `&offset=${offset}`, `o${offset}`);
} catch (e) {
if (offset === 0 || !/HTTP 40[13]/.test(String(e))) throw e;
log(
` lidl ${q.name}: offset ${offset} not public (${String(e).slice(0, 40)}), stopping`
);
break;
}
if (add(q, r) < PAGE || offset + PAGE >= (r.numFound ?? 0)) break;
}
log(` lidl ${q.name}: +${items.size - before} (${items.size} total)`);
}