Stan Runge
427f4ed095
All checks were successful
Deploy production / deploy (push) Successful in 2m48s
64 lines
2.9 KiB
TypeScript
64 lines
2.9 KiB
TypeScript
// Nightly supermarket catalogue import (Albert Heijn + Jumbo) and product regrouping.
|
|
//
|
|
// bun --env-file=$HOME/.config/groceries-import.env scripts/groceries-import.ts [options]
|
|
// --store ah|jumbo only this store (repeatable; default: all)
|
|
// --skip-fetch no store requests; only regroup what is in the DB
|
|
// --gtin-limit N AH product details to fetch for EANs this run (default 1500, 0 = off)
|
|
// --delay-ms N minimum delay between requests (default 2000)
|
|
//
|
|
// Politeness: one request at a time, >= 2 s apart, backoff on 429/5xx, identifying User-Agent,
|
|
// and a same-day cache in $GROCERIES_CACHE_DIR (default ~/.cache/groceries, kept 7 days).
|
|
// Plus is not imported: plus.nl blocks this server's IP range (Incapsula), and we do not evade it.
|
|
import postgres from 'postgres';
|
|
import { homedir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
|
|
import { enrichAhGtins, importStore, regroup } from '../src/lib/server/groceries/import';
|
|
import { SOURCES } from '../src/lib/server/groceries/sources';
|
|
|
|
const args = process.argv.slice(2);
|
|
const values = (name: string) =>
|
|
args.flatMap((a, i) => (a === name && args[i + 1] ? [args[i + 1]] : []));
|
|
const num = (name: string, d: number) => {
|
|
const v = values(name)[0];
|
|
const n = v === undefined ? d : Number(v);
|
|
if (!Number.isFinite(n) || n < 0) throw new Error(`${name} must be a number`);
|
|
return n;
|
|
};
|
|
if (!process.env.DB_URL) throw new Error('DB_URL is not set');
|
|
const stores = values('--store').length ? values('--store') : Object.keys(SOURCES);
|
|
for (const s of stores) if (!SOURCES[s]) throw new Error(`unknown store ${s}`);
|
|
const delay = num('--delay-ms', 2000);
|
|
const gtinLimit = num('--gtin-limit', 1500);
|
|
const cacheDir = process.env.GROCERIES_CACHE_DIR || join(homedir(), '.cache', 'groceries');
|
|
const log = (m: string) => console.log(`[${new Date().toISOString().slice(11, 19)}Z] ${m}`);
|
|
|
|
const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
|
let failed = false;
|
|
try {
|
|
pruneCache(cacheDir);
|
|
if (!args.includes('--skip-fetch')) {
|
|
for (const slug of stores) {
|
|
const f = new PoliteFetcher({ store: slug, minDelayMs: delay, cacheDir, log });
|
|
try {
|
|
log(`${slug}: importing`);
|
|
const r = await importStore(sql, SOURCES[slug], f, log);
|
|
log(
|
|
`${slug}: ${r.fetched} products (${r.inserted} new, ${r.priceChanges} price changes, ${r.delisted} delisted); ${r.requests} requests, ${r.cacheHits} from cache`
|
|
);
|
|
if (slug === 'ah' && gtinLimit > 0) {
|
|
const g = await enrichAhGtins(sql, f, gtinLimit, log);
|
|
log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
|
|
}
|
|
} catch (e) {
|
|
failed = true;
|
|
log(`${slug}: FAILED ${e instanceof Error ? e.message : e}`);
|
|
}
|
|
}
|
|
}
|
|
await regroup(sql, log);
|
|
} finally {
|
|
await sql.end();
|
|
}
|
|
if (failed) process.exitCode = 1;
|