groceries: Lidl importer (online range), Dirk importer + store row, generic per-store GTIN backfill

src/lib/server/groceries/import.ts

@@ -12,8 +12,7 @@ import {
 	nameTokens,
 	normaliseUnit
 } from './normalize';
-import { PlusClient } from './plus';
-import { AhClient, itemIsFood, type RawItem, type Source } from './sources';
+import { AhClient, itemIsFood, SOURCES, type RawItem, type Source } from './sources';
 
 type Log = (m: string) => void;
 
@@ -183,42 +182,48 @@ export async function inferBrands(sql: Sql, sid: number, raw: RawItem[]): Promis
 const isHouseBrand = (b: string) => brandClass(b).startsWith('house');
 
 /** Fill in PLUS EANs from the product detail action, `limit` products per run, once per SKU. */
-export async function enrichPlusGtins(
+/**
+ * Fill in EANs from a source's product-detail lookup (Source.gtinLookup), `limit` products per
+ * run, each SKU at most once per 180 days. One lookup per fetcher runs in parallel.
+ */
+export async function enrichGtins(
 	sql: Sql,
+	slug: string,
 	fetchers: PoliteFetcher[],
 	limit: number,
 	log: Log
 ) {
-	if (limit <= 0 || !fetchers.length) return { checked: 0, found: 0 };
-	const sid = await storeId(sql, 'plus');
+	const make = SOURCES[slug]?.gtinLookup;
+	if (limit <= 0 || !fetchers.length || !make) return { checked: 0, found: 0 };
+	const sid = await storeId(sql, slug);
 	const todo = await sql<{ id: number; store_product_id: string; url: string | null }[]>`
 		select id, store_product_id, url from groceries.items
 		where store_id = ${sid} and gtin is null and available
 			and (gtin_checked_at is null or gtin_checked_at < now() - interval '180 days')
 		order by gtin_checked_at asc nulls first, id
 		limit ${limit}`;
-	const clients = fetchers.map((f) => new PlusClient(f));
+	const lookups = fetchers.map((f) => make(f));
 	let found = 0;
 	let errors = 0;
 	let next = 0;
 	let done = 0;
 	await Promise.all(
-		clients.map(async (plus) => {
+		lookups.map(async (lookup) => {
 			while (next < todo.length) {
 				const t = todo[next++];
-				const slug = t.url?.split('/product/')[1] ?? '';
 				let gtin: string | null;
 				try {
-					gtin = await plus.gtin(t.store_product_id, slug);
+					gtin = await lookup(t.store_product_id, t.url);
 				} catch (e) {
 					errors++;
-					log(`  plus gtin ${t.store_product_id}: ${e instanceof Error ? e.message : e}`);
+					log(`  ${slug} gtin ${t.store_product_id}: ${e instanceof Error ? e.message : e}`);
 					if (errors > 25 && found === 0) throw e; // something is wrong; stop hammering
 					continue;
 				}
 				if (gtin) found++;
 				await sql`update groceries.items set gtin = ${gtin}, gtin_checked_at = now() where id = ${t.id}`;
-				if (++done % 500 === 0) log(`  plus gtin: ${done}/${todo.length} checked, ${found} found`);
+				if (++done % 500 === 0)
+					log(`  ${slug} gtin: ${done}/${todo.length} checked, ${found} found`);
 			}
 		})
 	);