Stan Runge
a6ef263c7e
All checks were successful
Deploy production / deploy (push) Successful in 1m19s
102 lines
4.1 KiB
TypeScript
102 lines
4.1 KiB
TypeScript
// Supermarket catalogue import and product regrouping.
|
|
//
|
|
// bun scripts/groceries-import.ts [options]
|
|
// --store SLUG only this store (repeatable). Default: the nightly stores
|
|
// (ah, jumbo, plus, xenos); amazing-oriental runs weekly on its own timer.
|
|
// --skip-fetch no store requests; only regroup what is in the DB
|
|
// --gtin-limit N product details to fetch for EANs per store this run (AH, PLUS, Dirk, …; default 1500, 0 = off)
|
|
// --gtin-only skip the catalogue crawl; only the EAN backfill (+ regroup)
|
|
// --gtin-parallel N parallel detail requests for the EAN backfill (default 1, max 4)
|
|
// --delay-ms N minimum delay between requests (default: per store, else 2000)
|
|
//
|
|
// Politeness: one request at a time per store (Amazing Oriental: 25 s, its robots.txt
|
|
// Crawl-delay), backoff on 429/5xx, and a same-day cache in $GROCERIES_CACHE_DIR
|
|
// (default ~/.cache/groceries, kept 7 days). The systemd wrapper holds a flock so two imports
|
|
// never run at once.
|
|
import postgres from 'postgres';
|
|
import { homedir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
|
|
import {
|
|
enrichAhGtins,
|
|
enrichGtins,
|
|
importStore,
|
|
regroup
|
|
} from '../src/lib/server/groceries/import';
|
|
import { applyTaxonomy } from '../src/lib/server/groceries/taxonomy';
|
|
import { computeStats } from '../src/lib/server/groceries/stats';
|
|
import { SOURCES } from '../src/lib/server/groceries/sources';
|
|
|
|
const args = process.argv.slice(2);
|
|
const values = (name: string) =>
|
|
args.flatMap((a, i) => (a === name && args[i + 1] ? [args[i + 1]] : []));
|
|
const num = (name: string, d: number) => {
|
|
const v = values(name)[0];
|
|
const n = v === undefined ? d : Number(v);
|
|
if (!Number.isFinite(n) || n < 0) throw new Error(`${name} must be a number`);
|
|
return n;
|
|
};
|
|
if (!process.env.DB_URL) throw new Error('DB_URL is not set');
|
|
const stores = values('--store').length
|
|
? values('--store')
|
|
: Object.keys(SOURCES).filter((s) => SOURCES[s].nightly !== false);
|
|
for (const s of stores) if (!SOURCES[s]) throw new Error(`unknown store ${s}`);
|
|
const delayArg = values('--delay-ms').length ? num('--delay-ms', 2000) : null;
|
|
const gtinLimit = num('--gtin-limit', 1500);
|
|
const gtinParallel = Math.min(4, Math.max(1, num('--gtin-parallel', 1)));
|
|
const gtinOnly = args.includes('--gtin-only');
|
|
const cacheDir = process.env.GROCERIES_CACHE_DIR || join(homedir(), '.cache', 'groceries');
|
|
const log = (m: string) => console.log(`[${new Date().toISOString().slice(11, 19)}Z] ${m}`);
|
|
|
|
const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
|
let failed = false;
|
|
try {
|
|
pruneCache(cacheDir);
|
|
if (!args.includes('--skip-fetch')) {
|
|
for (const slug of stores) {
|
|
const src = SOURCES[slug];
|
|
const fetcher = () =>
|
|
new PoliteFetcher({
|
|
store: slug,
|
|
minDelayMs: delayArg ?? src.minDelayMs ?? 2000,
|
|
cacheDir,
|
|
log,
|
|
...src.fetcher
|
|
});
|
|
const f = fetcher();
|
|
try {
|
|
if (!gtinOnly) {
|
|
log(`${slug}: importing`);
|
|
const r = await importStore(sql, src, f, log);
|
|
log(
|
|
`${slug}: ${r.fetched} products (${r.inserted} new, ${r.priceChanges} price changes, ${r.delisted} delisted); ${r.requests} requests, ${r.cacheHits} from cache`
|
|
);
|
|
}
|
|
if (slug === 'ah' && gtinLimit > 0) {
|
|
const g = await enrichAhGtins(sql, f, gtinLimit, log);
|
|
log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
|
|
}
|
|
if (SOURCES[slug].gtinLookup && gtinLimit > 0) {
|
|
const fs = [f, ...Array.from({ length: gtinParallel - 1 }, fetcher)];
|
|
const g = await enrichGtins(sql, slug, fs, gtinLimit, log);
|
|
log(`${slug}: EAN lookup ${g.checked} checked, ${g.found} found`);
|
|
}
|
|
} catch (e) {
|
|
failed = true;
|
|
log(`${slug}: FAILED ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
}
|
|
}
|
|
await regroup(sql, log);
|
|
await applyTaxonomy(sql, log);
|
|
try {
|
|
await computeStats(sql, log);
|
|
} catch (e) {
|
|
failed = true;
|
|
log(`stats: FAILED ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
} finally {
|
|
await sql.end();
|
|
}
|
|
if (failed) process.exitCode = 1;
|