groceries: add PLUS, Xenos and Amazing Oriental stores, is_food flag, PLUS EAN backfill

src/lib/server/groceries/import.ts

@@ -7,11 +7,13 @@ import {
 	brandClass,
 	brandLabel,
 	displayName,
+	fold,
 	matchKey,
 	nameTokens,
 	normaliseUnit
 } from './normalize';
-import { AhClient, type RawItem, type Source } from './sources';
+import { PlusClient } from './plus';
+import { AhClient, itemIsFood, type RawItem, type Source } from './sources';
 
 type Log = (m: string) => void;
 
@@ -34,6 +36,10 @@ export async function importStore(sql: Sql, source: Source, f: PoliteFetcher, lo
 		insert into groceries.import_runs (store_id) values (${sid}) returning id, started_at`;
 	try {
 		const raw = await source.fetchAll(f, log);
+		if (BRAND_FROM_NAME.has(source.slug)) {
+			const n = await inferBrands(sql, sid, raw);
+			log(`  ${source.slug}: brand inferred from the name for ${n} products`);
+		}
 		const [prev] = await sql<{ n: number }[]>`
 			select count(*)::int as n from groceries.items where store_id = ${sid} and available`;
 		const res = await upsertItems(sql, sid, raw, run.started_at);
@@ -91,6 +97,7 @@ async function upsertItems(sql: Sql, sid: number, raw: RawItem[], runStart: Date
 				image_url: r.imageUrl,
 				url: r.url,
 				available: r.available,
+				is_food: itemIsFood(r),
 				match_key: keyFor(r.name, r.brand, n.unit).key
 			};
 		});
@@ -104,7 +111,7 @@ async function upsertItems(sql: Sql, sid: number, raw: RawItem[], runStart: Date
 				promo_unit_price_cents = excluded.promo_unit_price_cents, category = excluded.category,
 				category_path = excluded.category_path, gtin = coalesce(excluded.gtin, items.gtin),
 				image_url = excluded.image_url, url = excluded.url, available = excluded.available,
-				match_key = excluded.match_key, last_seen_at = now()
+				is_food = excluded.is_food, match_key = excluded.match_key, last_seen_at = now()
 			returning (xmax = 0) as inserted`;
 		inserted += res.filter((r) => r.inserted).length;
 	}
@@ -132,6 +139,92 @@ async function upsertItems(sql: Sql, sid: number, raw: RawItem[], runStart: Date
 	return { inserted, priceChanges: hist.filter((h) => h.changed).length };
 }
 
+/** Stores whose listings carry no (or little) brand: take it from the start of the name. */
+const BRAND_FROM_NAME = new Set(['xenos', 'amazing-oriental']);
+
+/**
+ * Fill in missing brands when the name starts with a known brand ("Lotus Biscoff …"). Known
+ * brands: those on >= 5 products at other stores, plus the store's own brands in this batch.
+ * House brands are never inferred. Returns how many items got a brand.
+ */
+export async function inferBrands(sql: Sql, sid: number, raw: RawItem[]): Promise<number> {
+	const rows = await sql<{ brand: string }[]>`
+		select brand from groceries.items
+		where brand is not null and store_id <> ${sid} and length(brand) >= 3
+		group by brand having count(*) >= 5`;
+	const known = new Map<string, string>();
+	for (const b of [...rows.map((r) => r.brand), ...raw.map((r) => r.brand)]) {
+		if (!b || b.trim().length < 3 || isHouseBrand(b)) continue;
+		const k = fold(b).replace(/\s+/g, ' ').trim();
+		if (!known.has(k)) known.set(k, b.trim());
+	}
+	const byFirstWord = new Map<string, string[]>();
+	for (const k of known.keys()) {
+		const w = k.split(' ')[0];
+		byFirstWord.set(w, [...(byFirstWord.get(w) ?? []), k]);
+	}
+	for (const list of byFirstWord.values()) list.sort((a, b) => b.length - a.length);
+	let n = 0;
+	for (const r of raw) {
+		if (r.brand) continue;
+		const name = fold(r.name).replace(/\s+/g, ' ').trim();
+		const first = name.split(' ')[0];
+		for (const k of byFirstWord.get(first) ?? []) {
+			// The brand must be followed by more words, so "Mango" alone never becomes a brand.
+			if (name.startsWith(k + ' ') && name.length > k.length + 2) {
+				r.brand = known.get(k)!;
+				n++;
+				break;
+			}
+		}
+	}
+	return n;
+}
+const isHouseBrand = (b: string) => brandClass(b).startsWith('house');
+
+/** Fill in PLUS EANs from the product detail action, `limit` products per run, once per SKU. */
+export async function enrichPlusGtins(
+	sql: Sql,
+	fetchers: PoliteFetcher[],
+	limit: number,
+	log: Log
+) {
+	if (limit <= 0 || !fetchers.length) return { checked: 0, found: 0 };
+	const sid = await storeId(sql, 'plus');
+	const todo = await sql<{ id: number; store_product_id: string; url: string | null }[]>`
+		select id, store_product_id, url from groceries.items
+		where store_id = ${sid} and gtin is null and available
+			and (gtin_checked_at is null or gtin_checked_at < now() - interval '180 days')
+		order by gtin_checked_at asc nulls first, id
+		limit ${limit}`;
+	const clients = fetchers.map((f) => new PlusClient(f));
+	let found = 0;
+	let errors = 0;
+	let next = 0;
+	let done = 0;
+	await Promise.all(
+		clients.map(async (plus) => {
+			while (next < todo.length) {
+				const t = todo[next++];
+				const slug = t.url?.split('/product/')[1] ?? '';
+				let gtin: string | null;
+				try {
+					gtin = await plus.gtin(t.store_product_id, slug);
+				} catch (e) {
+					errors++;
+					log(`  plus gtin ${t.store_product_id}: ${e instanceof Error ? e.message : e}`);
+					if (errors > 25 && found === 0) throw e; // something is wrong; stop hammering
+					continue;
+				}
+				if (gtin) found++;
+				await sql`update groceries.items set gtin = ${gtin}, gtin_checked_at = now() where id = ${t.id}`;
+				if (++done % 500 === 0) log(`  plus gtin: ${done}/${todo.length} checked, ${found} found`);
+			}
+		})
+	);
+	return { checked: done, found };
+}
+
 /** Fill in AH EANs from the product detail endpoint, `limit` products per run (A-brands first). */
 export async function enrichAhGtins(sql: Sql, f: PoliteFetcher, limit: number, log: Log) {
 	if (limit <= 0) return { checked: 0, found: 0 };
@@ -374,7 +467,7 @@ export async function refreshGroupStats(sql: Sql, ids?: number[]) {
 		update groceries.groups g set
 			item_count = s.item_count, store_count = s.store_count, store_ids = s.store_ids,
 			min_unit_price_cents = s.min_unit, min_price_cents = s.min_price, promo_count = s.promo_count,
-			image_url = s.image_url, search_text = s.search_text
+			image_url = s.image_url, search_text = s.search_text, is_food = s.is_food
 		from (
 			select g2.id,
 				count(i.id)::int as item_count,
@@ -384,6 +477,7 @@ export async function refreshGroupStats(sql: Sql, ids?: number[]) {
 				min(coalesce(i.promo_price_cents, i.price_cents)) as min_price,
 				(count(*) filter (where i.promo_price_cents is not null or i.promo_label is not null))::int as promo_count,
 				(array_agg(i.image_url order by i.store_id, i.id) filter (where i.image_url is not null))[1] as image_url,
+				coalesce(bool_or(i.is_food), true) as is_food,
 				translate(lower(concat_ws(' ', g2.name, g2.brand, g2.category,
 					string_agg(distinct concat_ws(' ', i.brand, i.name), ' '))), ${ACCENTS_FROM}, ${ACCENTS_TO}) as search_text
 			from groceries.groups g2
@@ -392,6 +486,6 @@ export async function refreshGroupStats(sql: Sql, ids?: number[]) {
 			group by g2.id
 		) s
 		where g.id = s.id and (g.item_count, g.store_count, g.store_ids, g.min_unit_price_cents, g.min_price_cents,
-			g.promo_count, g.image_url, g.search_text) is distinct from
-			(s.item_count, s.store_count, s.store_ids, s.min_unit, s.min_price, s.promo_count, s.image_url, s.search_text)`;
+			g.promo_count, g.image_url, g.search_text, g.is_food) is distinct from
+			(s.item_count, s.store_count, s.store_ids, s.min_unit, s.min_price, s.promo_count, s.image_url, s.search_text, s.is_food)`;
 }