groceries: nightly AH/Jumbo catalogue import, unit-price normalisation, product groups and compare UI

2026-10-24 14:01:25 +00:00
parent 8dde5aa302
commit 427f4ed095
20 changed files with 34891 additions and 699 deletions
--- a/src/routes/trackers/groceries/+page.server.ts
+++ b/src/routes/trackers/groceries/+page.server.ts
@@ -1,45 +1,198 @@
 import { db } from '#lib/server/db/index.js';
-import { groceryPrices, groceryProducts, groceryStores } from '#lib/server/db/schema/index.js';
 import { requireOwner } from '#lib/server/owner.js';
-import { parseSize, priceInput, productInput, unitPriceCents } from '#lib/server/groceries.js';
-import { and, asc, desc, eq, isNull, sql } from 'drizzle-orm';
+import { refreshGroupStats } from '#lib/server/groceries/import.js';
+import { CATEGORIES, displayName, fold } from '#lib/server/groceries/normalize.js';
 import { fail } from '@sveltejs/kit';
+import type { Sql } from 'postgres';
 import type { Actions, PageServerLoad } from './$types';

+const sql = () => db.$client as unknown as Sql;
+const PAGE_SIZE = 30;
+
+export type GroupRow = {
+	id: number;
+	name: string;
+	brand: string | null;
+	category: string | null;
+	unit: string | null;
+	favorite: boolean;
+	manual: boolean;
+	itemCount: number;
+	storeCount: number;
+	minUnitPriceCents: number | null;
+	imageUrl: string | null;
+	familyKey: string | null;
+};
+export type ItemRow = {
+	id: number;
+	groupId: number;
+	storeId: number;
+	name: string;
+	brand: string | null;
+	sizeText: string | null;
+	unit: string | null;
+	unitAmount: number | null;
+	unitSource: string | null;
+	priceCents: number | null;
+	promoPriceCents: number | null;
+	promoLabel: string | null;
+	unitPriceCents: number | null;
+	promoUnitPriceCents: number | null;
+	gtin: string | null;
+	imageUrl: string | null;
+	url: string | null;
+	available: boolean;
+	matchMethod: string | null;
+	matchConfidence: number | null;
+	matchLocked: boolean;
+	categoryPath: string | null;
+	lastSeenAt: Date;
+};
+
+const ITEM_COLS = (s: Sql) => s`
+	i.id, i.group_id as "groupId", i.store_id as "storeId", i.name, i.brand, i.size_text as "sizeText",
+	i.unit, i.unit_amount::float8 as "unitAmount", i.unit_source as "unitSource", i.price_cents as "priceCents",
+	i.promo_price_cents as "promoPriceCents", i.promo_label as "promoLabel",
+	i.unit_price_cents as "unitPriceCents", i.promo_unit_price_cents as "promoUnitPriceCents", i.gtin,
+	i.image_url as "imageUrl", i.url, i.available, i.match_method as "matchMethod",
+	i.match_confidence as "matchConfidence", i.match_locked as "matchLocked",
+	i.category_path as "categoryPath", i.last_seen_at as "lastSeenAt"`;
+const GROUP_COLS = (s: Sql) => s`
+	g.id, g.name, g.brand, g.category, g.unit, g.favorite, g.manual, g.item_count as "itemCount",
+	g.store_count as "storeCount", g.min_unit_price_cents as "minUnitPriceCents",
+	g.image_url as "imageUrl", g.family_key as "familyKey"`;
+
 export const load: PageServerLoad = async (event) => {
 	requireOwner(event);
-	const [stores, products, prices] = await Promise.all([
-		db
-			.select()
-			.from(groceryStores)
-			.where(isNull(groceryStores.archivedAt))
-			.orderBy(asc(groceryStores.sortOrder), asc(groceryStores.name)),
-		db
-			.select()
-			.from(groceryProducts)
-			.where(isNull(groceryProducts.archivedAt))
-			.orderBy(desc(groceryProducts.favorite), asc(sql`lower(${groceryProducts.name})`)),
-		db
-			.select({
-				id: groceryPrices.id,
-				productId: groceryPrices.productId,
-				storeId: groceryPrices.storeId,
-				priceCents: groceryPrices.priceCents,
-				unitPriceCents: groceryPrices.unitPriceCents,
-				promo: groceryPrices.promo,
-				observedAt: groceryPrices.observedAt,
-				source: groceryPrices.source,
-				url: groceryPrices.url,
-				notes: groceryPrices.notes
-			})
-			.from(groceryPrices)
-			.orderBy(desc(groceryPrices.observedAt), desc(groceryPrices.id))
+	const s = sql();
+	const p = event.url.searchParams;
+	const q = fold(p.get('q')?.trim() ?? '').slice(0, 100);
+	const words = q.split(/\s+/).filter(Boolean).slice(0, 6);
+	const storeSlug = p.get('store') ?? '';
+	const category = p.get('cat') ?? '';
+	const favorites = p.get('fav') === '1';
+	const promo = p.get('promo') === '1';
+	const multi = p.get('multi') === '1';
+	const page = Math.max(1, Math.min(10_000, Number(p.get('page')) || 1));
+	const detailId = Number(p.get('g')) || null;
+
+	const stores = await s<
+		{ id: number; slug: string; name: string; color: string | null; items: number }[]
+	>`
+		select st.id, st.slug, st.name, st.color,
+			(select count(*)::int from groceries.items i where i.store_id = st.id and i.available) as items
+		from groceries.stores st where st.archived_at is null order by st.sort_order, st.name`;
+	const store = stores.find((x) => x.slug === storeSlug) ?? null;
+
+	const conds = [s`g.merged_into_id is null`, s`(g.item_count > 0 or g.favorite)`];
+	for (const w of words)
+		conds.push(s`g.search_text like ${'%' + w.replace(/[%_\\]/g, '\\$&') + '%'}`);
+	if (store) conds.push(s`g.store_ids @> array[${store.id}]::int[]`);
+	if (category) conds.push(s`g.category = ${category}`);
+	if (favorites) conds.push(s`g.favorite`);
+	if (promo) conds.push(s`g.promo_count > 0`);
+	if (multi) conds.push(s`g.store_count >= 2`);
+	const where = conds.reduce((a, c) => s`${a} and ${c}`);
+	const order = q
+		? s`g.favorite desc, (lower(g.name) = ${q}) desc, word_similarity(${q}, lower(g.name)) desc, g.store_count desc, g.item_count desc, g.name`
+		: s`g.favorite desc, g.store_count desc, g.item_count desc, g.name`;
+
+	const [groups, [{ total }]] = await Promise.all([
+		s<GroupRow[]>`
+			select ${GROUP_COLS(s)} from groceries.groups g where ${where}
+			order by ${order}, g.id limit ${PAGE_SIZE} offset ${(page - 1) * PAGE_SIZE}`,
+		s<{ total: number }[]>`select count(*)::int as total from groceries.groups g where ${where}`
 	]);
+	const ids = groups.map((g) => g.id);
+	const items = ids.length
+		? await s<ItemRow[]>`
+			select ${ITEM_COLS(s)} from groceries.items i
+			where i.group_id = any(${ids}::int[]) and i.available
+			order by coalesce(i.promo_unit_price_cents, i.unit_price_cents) nulls last, i.id`
+		: [];
+
+	const [stats] = await s<
+		{
+			groups: number;
+			crossStore: number;
+			items: number;
+			grouped: number;
+			unitPriced: number;
+			promos: number;
+		}[]
+	>`
+		select
+			(select count(*)::int from groceries.groups where merged_into_id is null and item_count > 0) as groups,
+			(select count(*)::int from groceries.groups where merged_into_id is null and store_count >= 2) as "crossStore",
+			count(*)::int as items,
+			(count(*) filter (where g.store_count >= 2))::int as grouped,
+			(count(*) filter (where i.unit_price_cents is not null))::int as "unitPriced",
+			(count(*) filter (where i.promo_price_cents is not null or i.promo_label is not null))::int as promos
+		from groceries.items i left join groceries.groups g on g.id = i.group_id
+		where i.available`;
+	const runs = await s<
+		{ storeId: number; finishedAt: Date | null; status: string; fetched: number }[]
+	>`
+		select distinct on (store_id) store_id as "storeId", finished_at as "finishedAt", status, fetched
+		from groceries.import_runs order by store_id, started_at desc`;
+	const categories = await s<{ category: string }[]>`
+		select distinct category from groceries.groups where category is not null and item_count > 0`;
+	const catList = [
+		...CATEGORIES.filter((c) => categories.some((x) => x.category === c)),
+		...categories
+			.map((x) => x.category)
+			.filter((c) => !(CATEGORIES as readonly string[]).includes(c))
+	];
+
+	let detail = null;
+	if (detailId) {
+		const [group] = await s<(GroupRow & { notes: string | null })[]>`
+			select ${GROUP_COLS(s)}, g.notes from groceries.groups g where g.id = ${detailId}`;
+		if (group) {
+			const [dItems, similar] = await Promise.all([
+				s<ItemRow[]>`
+					select ${ITEM_COLS(s)} from groceries.items i where i.group_id = ${detailId}
+					order by i.available desc, coalesce(i.promo_unit_price_cents, i.unit_price_cents) nulls last, i.id`,
+				group.familyKey
+					? s<GroupRow[]>`
+						select ${GROUP_COLS(s)} from groceries.groups g
+						where g.family_key = ${group.familyKey} and g.id <> ${detailId}
+							and g.merged_into_id is null and g.item_count > 0
+						order by g.min_unit_price_cents nulls last limit 12`
+					: Promise.resolve([] as GroupRow[])
+			]);
+			const history = dItems.length
+				? await s<
+						{
+							itemId: number;
+							observedOn: string;
+							priceCents: number | null;
+							promoPriceCents: number | null;
+							promoLabel: string | null;
+						}[]
+					>`
+					select item_id as "itemId", observed_on::text as "observedOn", price_cents as "priceCents",
+						promo_price_cents as "promoPriceCents", promo_label as "promoLabel"
+					from groceries.item_prices where item_id = any(${dItems.map((i) => i.id)}::int[])
+					order by observed_on desc limit 400`
+				: [];
+			detail = { group, items: dItems, similar, history };
+		}
+	}
+
 	return {
 		stores,
-		products,
-		prices,
-		seo: { title: 'Groceries', description: 'Private grocery and price tracker.', index: false }
+		groups,
+		items,
+		total,
+		page,
+		pageSize: PAGE_SIZE,
+		filters: { q: p.get('q') ?? '', store: store?.slug ?? '', category, favorites, promo, multi },
+		categories: catList,
+		stats,
+		runs,
+		detail,
+		seo: { title: 'Groceries', description: 'Private supermarket price comparison.', index: false }
 	};
 };

@@ -47,132 +200,110 @@ const id = (v: FormDataEntryValue | null) => {
 	const n = Number(v);
 	return Number.isInteger(n) && n > 0 ? n : null;
 };
-const formObject = (form: FormData) => {
-	const o: Record<string, unknown> = {};
-	for (const [k, v] of form) if (typeof v === 'string') o[k] = v;
-	return o;
-};
-const firstIssue = (e: { issues: { path: PropertyKey[]; message: string }[] }) => {
-	const i = e.issues[0];
-	return i ? `${i.path.join('.') || 'input'}: ${i.message}` : 'Invalid input';
-};
-function dbError(e: unknown): string {
-	const cause = (e as { cause?: { message?: string; constraint_name?: string } })?.cause;
-	if (cause?.constraint_name === 'products_name_brand_key')
-		return 'That product (name + brand) already exists';
-	if (cause?.constraint_name) return `Not allowed (${cause.constraint_name})`;
-	return cause?.message || 'Could not save';
-}

 export const actions: Actions = {
-	/** Create or edit a product. */
-	saveProduct: async (event) => {
-		requireOwner(event);
-		const form = await event.request.formData();
-		const productId = id(form.get('id'));
-		const parsed = productInput.safeParse(formObject(form));
-		if (!parsed.success) return fail(400, { message: firstIssue(parsed.error) });
-		const p = parsed.data;
-		const size = parseSize(p.size);
-		const values = {
-			name: p.name,
-			brand: p.brand ?? null,
-			category: p.category ?? null,
-			size: p.size ?? null,
-			unitAmount: size?.amount ?? null,
-			unit: size?.unit ?? null,
-			notes: p.notes ?? null,
-			...(form.has('favorite') ? { favorite: form.get('favorite') === 'on' } : {})
-		};
-		try {
-			if (productId) {
-				await db.transaction(async (tx) => {
-					await tx.update(groceryProducts).set(values).where(eq(groceryProducts.id, productId));
-					// Pack size changed: recompute unit prices of this product's history.
-					await tx
-						.update(groceryPrices)
-						.set({
-							unitPriceCents: values.unitAmount
-								? sql`round(${groceryPrices.priceCents} / ${values.unitAmount}::numeric)::int`
-								: null
-						})
-						.where(eq(groceryPrices.productId, productId));
-				});
-				return { saved: productId };
-			}
-			const [row] = await db
-				.insert(groceryProducts)
-				.values(values)
-				.returning({ id: groceryProducts.id });
-			return { saved: row.id };
-		} catch (e) {
-			return fail(400, { message: dbError(e) });
-		}
-	},
-
 	favorite: async (event) => {
 		requireOwner(event);
 		const form = await event.request.formData();
-		const productId = id(form.get('id'));
-		if (!productId) return fail(400, { message: 'Invalid product' });
-		await db
-			.update(groceryProducts)
-			.set({ favorite: form.get('favorite') === 'true' })
-			.where(eq(groceryProducts.id, productId));
-		return { updated: productId };
+		const gid = id(form.get('id'));
+		if (!gid) return fail(400, { message: 'Invalid group' });
+		await sql()`update groceries.groups set favorite = ${form.get('favorite') === 'true'} where id = ${gid}`;
+		return { updated: gid };
 	},

-	deleteProduct: async (event) => {
-		requireOwner(event);
-		const productId = id((await event.request.formData()).get('id'));
-		if (!productId) return fail(400, { message: 'Invalid product' });
-		await db.delete(groceryProducts).where(eq(groceryProducts.id, productId));
-		return { deleted: productId };
-	},
-
-	/** Record a price observation for a product at a store. */
-	addPrice: async (event) => {
+	/** Rename a group; it becomes manual so the nightly regroup keeps the name. */
+	rename: async (event) => {
 		requireOwner(event);
 		const form = await event.request.formData();
-		const productId = id(form.get('productId'));
-		const storeId = id(form.get('storeId'));
-		if (!productId || !storeId) return fail(400, { message: 'Pick a product and a store' });
-		const parsed = priceInput.safeParse(formObject(form));
-		if (!parsed.success) return fail(400, { message: firstIssue(parsed.error) });
-		const [product] = await db
-			.select({ unitAmount: groceryProducts.unitAmount })
-			.from(groceryProducts)
-			.where(eq(groceryProducts.id, productId));
-		if (!product) return fail(404, { message: 'Product not found' });
-		const p = parsed.data;
-		try {
-			await db.insert(groceryPrices).values({
-				productId,
-				storeId,
-				priceCents: p.price,
-				unitPriceCents: unitPriceCents(p.price, product.unitAmount),
-				promo: form.get('promo') === 'on',
-				// A plain date is stored as noon UTC; the exact time does not matter for prices.
-				observedAt: p.observedOn ? new Date(`${p.observedOn}T12:00:00Z`) : new Date(),
-				source: 'manual',
-				url: p.url ?? null,
-				notes: p.notes ?? null
-			});
-		} catch (e) {
-			return fail(400, { message: dbError(e) });
-		}
-		return { priced: productId };
+		const gid = id(form.get('id'));
+		const name = String(form.get('name') ?? '')
+			.trim()
+			.slice(0, 200);
+		if (!gid || !name) return fail(400, { message: 'Enter a name' });
+		const notes =
+			String(form.get('notes') ?? '')
+				.trim()
+				.slice(0, 2000) || null;
+		await sql()`update groceries.groups set name = ${name}, notes = ${notes}, manual = true where id = ${gid}`;
+		await refreshGroupStats(sql(), [gid]);
+		return { updated: gid };
 	},

-	deletePrice: async (event) => {
+	/** Merge group `id` into `into`: its items move (locked) and future matches follow. */
+	merge: async (event) => {
 		requireOwner(event);
 		const form = await event.request.formData();
-		const priceId = id(form.get('id'));
-		const productId = id(form.get('productId'));
-		if (!priceId || !productId) return fail(400, { message: 'Invalid price' });
-		await db
-			.delete(groceryPrices)
-			.where(and(eq(groceryPrices.id, priceId), eq(groceryPrices.productId, productId)));
-		return { deleted: priceId };
+		const from = id(form.get('id'));
+		const into = id(form.get('into'));
+		if (!from || !into || from === into) return fail(400, { message: 'Pick another group' });
+		const [target] = await sql()`
+			select id from groceries.groups where id = ${into} and merged_into_id is null`;
+		if (!target) return fail(404, { message: 'Group not found' });
+		await sql().begin(async (tx) => {
+			await tx`
+				update groceries.items set group_id = ${into}, match_locked = true, match_method = 'manual',
+					match_confidence = 1 where group_id = ${from}`;
+			await tx`update groceries.groups set merged_into_id = ${into} where merged_into_id = ${from}`;
+			const [src] = await tx<{ favorite: boolean }[]>`
+				select favorite from groceries.groups where id = ${from}`;
+			await tx`update groceries.groups set merged_into_id = ${into}, favorite = false where id = ${from}`;
+			if (src?.favorite) await tx`update groceries.groups set favorite = true where id = ${into}`;
+			await refreshGroupStats(tx as unknown as Sql, [from, into]);
+		});
+		return { merged: into };
+	},
+
+	/** Move one item to another group, or to a new group of its own (split). Locks the item. */
+	moveItem: async (event) => {
+		requireOwner(event);
+		const form = await event.request.formData();
+		const itemId = id(form.get('itemId'));
+		const target = form.get('into') === 'new' ? 'new' : id(form.get('into'));
+		if (!itemId || !target) return fail(400, { message: 'Pick a group' });
+		const moved = await sql().begin(async (tx) => {
+			const [item] = await tx<
+				{
+					group_id: number | null;
+					name: string;
+					brand: string | null;
+					unit: string | null;
+					category: string | null;
+				}[]
+			>`
+				select group_id, name, brand, unit, category from groceries.items where id = ${itemId}`;
+			if (!item) return null;
+			let into: number;
+			if (target === 'new') {
+				const [g] = await tx<{ id: number }[]>`
+					insert into groceries.groups (name, brand, category, unit, manual)
+					values (${displayName(item.name, item.brand).slice(0, 200)}, ${item.brand}, ${item.category}, ${item.unit}, true)
+					returning id`;
+				into = g.id;
+			} else {
+				const [g] =
+					await tx`select id from groceries.groups where id = ${target} and merged_into_id is null`;
+				if (!g) return null;
+				into = target;
+			}
+			await tx`
+				update groceries.items set group_id = ${into}, match_locked = true, match_method = 'manual',
+					match_confidence = 1 where id = ${itemId}`;
+			await refreshGroupStats(tx as unknown as Sql, [
+				into,
+				...(item.group_id ? [item.group_id] : [])
+			]);
+			return into;
+		});
+		if (!moved) return fail(404, { message: 'Not found' });
+		return { moved };
+	},
+
+	/** Hand an item back to the automatic matcher (applies at the next nightly import). */
+	unlockItem: async (event) => {
+		requireOwner(event);
+		const itemId = id((await event.request.formData()).get('itemId'));
+		if (!itemId) return fail(400, { message: 'Invalid item' });
+		await sql()`update groceries.items set match_locked = false where id = ${itemId}`;
+		return { unlocked: itemId };
 	}
 };
--- a/src/routes/trackers/groceries/+page.svelte
+++ b/src/routes/trackers/groceries/+page.svelte
--- a/src/routes/trackers/groceries/search/+server.ts
+++ b/src/routes/trackers/groceries/search/+server.ts
@@ -0,0 +1,23 @@
+import { db } from '#lib/server/db/index.js';
+import { requireOwner } from '#lib/server/owner.js';
+import { fold } from '#lib/server/groceries/normalize.js';
+import { json } from '@sveltejs/kit';
+import type { Sql } from 'postgres';
+import type { RequestHandler } from './$types';
+
+/** Group picker for merge/move: top matches for ?q= (owner only). */
+export const GET: RequestHandler = async (event) => {
+	requireOwner(event);
+	const q = fold(event.url.searchParams.get('q')?.trim() ?? '').slice(0, 100);
+	const words = q.split(/\s+/).filter(Boolean).slice(0, 6);
+	if (!words.length) return json([]);
+	const s = db.$client as unknown as Sql;
+	const conds = words.map((w) => s`search_text like ${'%' + w.replace(/[%_\\]/g, '\\$&') + '%'}`);
+	const rows = await s`
+		select id, name, brand, unit, store_count as "storeCount", item_count as "itemCount"
+		from groceries.groups
+		where merged_into_id is null and ${conds.reduce((a, c) => s`${a} and ${c}`)}
+		order by word_similarity(${q}, lower(name)) desc, store_count desc, item_count desc, id
+		limit 15`;
+	return json(rows, { headers: { 'cache-control': 'private, no-store' } });
+};