stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

groceries: Lidl importer (online range), Dirk importer + store row, generic per-store GTIN backfill
All checks were successful
Deploy production / deploy (push) Successful in 1m36s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-25 18:41:12 +00:00
parent 5d3e8d0261
commit ce8b545f60
8 changed files with 29878 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
scripts/groceries-import.ts
View File

@@ -4,9 +4,9 @@
// --store SLUG only this store (repeatable). Default: the nightly stores
// (ah, jumbo, plus, xenos); amazing-oriental runs weekly on its own timer.
// --skip-fetch no store requests; only regroup what is in the DB
// --gtin-limit N AH and PLUS product details to fetch for EANs this run (default 1500, 0 = off)
// --gtin-limit N product details to fetch for EANs per store this run (AH, PLUS, Dirk, …; default 1500, 0 = off)
// --gtin-only skip the catalogue crawl; only the EAN backfill (+ regroup)
// --gtin-parallel N parallel PLUS detail requests for the backfill (default 1, max 4)
// --gtin-parallel N parallel detail requests for the EAN backfill (default 1, max 4)
// --delay-ms N minimum delay between requests (default: per store, else 2000)
//
// Politeness: one request at a time per store (Amazing Oriental: 25 s, its robots.txt
@@ -19,7 +19,7 @@ import { join } from 'node:path';
import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
import {
enrichAhGtins,
enrichPlusGtins,
enrichGtins,
importStore,
regroup
} from '../src/lib/server/groceries/import';
@@ -74,10 +74,10 @@ try {
const g = await enrichAhGtins(sql, f, gtinLimit, log);
log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
}
if (slug === 'plus' && gtinLimit > 0) {
if (SOURCES[slug].gtinLookup && gtinLimit > 0) {
const fs = [f, ...Array.from({ length: gtinParallel - 1 }, fetcher)];
const g = await enrichPlusGtins(sql, fs, gtinLimit, log);
log(`plus: EAN lookup ${g.checked} checked, ${g.found} found`);
const g = await enrichGtins(sql, slug, fs, gtinLimit, log);
log(`${slug}: EAN lookup ${g.checked} checked, ${g.found} found`);
}
} catch (e) {
failed = true;