stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

groceries: add PLUS, Xenos and Amazing Oriental stores, is_food flag, PLUS EAN backfill
All checks were successful
Deploy production / deploy (push) Successful in 1m7s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-25 17:17:53 +00:00
parent b23f7b3475
commit 3f30bb883f
14 changed files with 30444 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
scripts/groceries-import.ts
View File

@@ -1,19 +1,28 @@
// Nightly supermarket catalogue import (Albert Heijn + Jumbo) and product regrouping.
// Supermarket catalogue import and product regrouping.
//
// bun --env-file=$HOME/.config/groceries-import.env scripts/groceries-import.ts [options]
// --store ah|jumbo only this store (repeatable; default: all)
// bun scripts/groceries-import.ts [options]
// --store SLUG only this store (repeatable). Default: the nightly stores
// (ah, jumbo, plus, xenos); amazing-oriental runs weekly on its own timer.
// --skip-fetch no store requests; only regroup what is in the DB
// --gtin-limit N AH product details to fetch for EANs this run (default 1500, 0 = off)
// --delay-ms N minimum delay between requests (default 2000)
// --gtin-limit N AH and PLUS product details to fetch for EANs this run (default 1500, 0 = off)
// --gtin-only skip the catalogue crawl; only the EAN backfill (+ regroup)
// --gtin-parallel N parallel PLUS detail requests for the backfill (default 1, max 4)
// --delay-ms N minimum delay between requests (default: per store, else 2000)
//
// Politeness: one request at a time, >= 2 s apart, backoff on 429/5xx, identifying User-Agent,
// and a same-day cache in $GROCERIES_CACHE_DIR (default ~/.cache/groceries, kept 7 days).
// Plus is not imported: plus.nl blocks this server's IP range (Incapsula), and we do not evade it.
// Politeness: one request at a time per store (Amazing Oriental: 25 s, its robots.txt
// Crawl-delay), backoff on 429/5xx, and a same-day cache in $GROCERIES_CACHE_DIR
// (default ~/.cache/groceries, kept 7 days). The systemd wrapper holds a flock so two imports
// never run at once.
import postgres from 'postgres';
import { homedir } from 'node:os';
import { join } from 'node:path';
import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
import { enrichAhGtins, importStore, regroup } from '../src/lib/server/groceries/import';
import {
enrichAhGtins,
enrichPlusGtins,
importStore,
regroup
} from '../src/lib/server/groceries/import';
import { SOURCES } from '../src/lib/server/groceries/sources';
const args = process.argv.slice(2);
@@ -26,10 +35,14 @@ const num = (name: string, d: number) => {
return n;
};
if (!process.env.DB_URL) throw new Error('DB_URL is not set');
const stores = values('--store').length ? values('--store') : Object.keys(SOURCES);
const stores = values('--store').length
? values('--store')
: Object.keys(SOURCES).filter((s) => SOURCES[s].nightly !== false);
for (const s of stores) if (!SOURCES[s]) throw new Error(`unknown store ${s}`);
const delay = num('--delay-ms', 2000);
const delayArg = values('--delay-ms').length ? num('--delay-ms', 2000) : null;
const gtinLimit = num('--gtin-limit', 1500);
const gtinParallel = Math.min(4, Math.max(1, num('--gtin-parallel', 1)));
const gtinOnly = args.includes('--gtin-only');
const cacheDir = process.env.GROCERIES_CACHE_DIR || join(homedir(), '.cache', 'groceries');
const log = (m: string) => console.log(`[${new Date().toISOString().slice(11, 19)}Z] ${m}`);
@@ -39,17 +52,33 @@ try {
pruneCache(cacheDir);
if (!args.includes('--skip-fetch')) {
for (const slug of stores) {
const f = new PoliteFetcher({ store: slug, minDelayMs: delay, cacheDir, log });
const src = SOURCES[slug];
const fetcher = () =>
new PoliteFetcher({
store: slug,
minDelayMs: delayArg ?? src.minDelayMs ?? 2000,
cacheDir,
log,
...src.fetcher
});
const f = fetcher();
try {
log(`${slug}: importing`);
const r = await importStore(sql, SOURCES[slug], f, log);
log(
`${slug}: ${r.fetched} products (${r.inserted} new, ${r.priceChanges} price changes, ${r.delisted} delisted); ${r.requests} requests, ${r.cacheHits} from cache`
);
if (!gtinOnly) {
log(`${slug}: importing`);
const r = await importStore(sql, src, f, log);
log(
`${slug}: ${r.fetched} products (${r.inserted} new, ${r.priceChanges} price changes, ${r.delisted} delisted); ${r.requests} requests, ${r.cacheHits} from cache`
);
}
if (slug === 'ah' && gtinLimit > 0) {
const g = await enrichAhGtins(sql, f, gtinLimit, log);
log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
}
if (slug === 'plus' && gtinLimit > 0) {
const fs = [f, ...Array.from({ length: gtinParallel - 1 }, fetcher)];
const g = await enrichPlusGtins(sql, fs, gtinLimit, log);
log(`plus: EAN lookup ${g.checked} checked, ${g.found} found`);
}
} catch (e) {
failed = true;
log(`${slug}: FAILED ${e instanceof Error ? e.message : e}`);