groceries: nightly AH/Jumbo catalogue import, unit-price normalisation, product groups and compare UI

drizzle/20261024134401_groceries_drop_manual/migration.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "groceries"."prices" DROP CONSTRAINT "prices_product_id_products_id_fkey";--> statement-breakpoint
+DROP TABLE "groceries"."prices";--> statement-breakpoint
+DROP TABLE "groceries"."products";--> statement-breakpoint
+-- The hand-entered products/prices tables were never used (0 rows); replaced by store items
+-- and groups in the next migration. Unregister them from the /lists database browser.
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    DELETE FROM meta.tables WHERE schema_name = 'groceries' AND table_name IN ('products','prices');
+  END IF;
+END $$;

drizzle/20261024134411_groceries_compare/migration.sql

@@ -0,0 +1,138 @@
+-- Trigram search for group/item names (trusted extension; the database owner may create it).
+CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
+--> statement-breakpoint
+CREATE TABLE "groceries"."groups" (
+	"id" serial PRIMARY KEY,
+	"key" text,
+	"name" text NOT NULL,
+	"brand" text,
+	"category" text,
+	"unit" text,
+	"family_key" text,
+	"favorite" boolean DEFAULT false NOT NULL,
+	"manual" boolean DEFAULT false NOT NULL,
+	"merged_into_id" integer,
+	"notes" text,
+	"item_count" integer DEFAULT 0 NOT NULL,
+	"store_count" integer DEFAULT 0 NOT NULL,
+	"store_ids" integer[] DEFAULT '{}'::integer[] NOT NULL,
+	"min_unit_price_cents" integer,
+	"min_price_cents" integer,
+	"promo_count" integer DEFAULT 0 NOT NULL,
+	"image_url" text,
+	"search_text" text DEFAULT '' NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "groups_name_check" CHECK (length(btrim("name")) > 0),
+	CONSTRAINT "groups_unit_check" CHECK ("unit" IS NULL OR "unit" IN ('piece','kg','l')),
+	CONSTRAINT "groups_not_self_merged" CHECK ("merged_into_id" IS NULL OR "merged_into_id" <> "id")
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."import_runs" (
+	"id" serial PRIMARY KEY,
+	"store_id" integer NOT NULL,
+	"started_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"finished_at" timestamp with time zone,
+	"status" text DEFAULT 'running' NOT NULL,
+	"requests" integer DEFAULT 0 NOT NULL,
+	"fetched" integer DEFAULT 0 NOT NULL,
+	"inserted" integer DEFAULT 0 NOT NULL,
+	"price_changes" integer DEFAULT 0 NOT NULL,
+	"error" text
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."item_prices" (
+	"id" serial PRIMARY KEY,
+	"item_id" integer NOT NULL,
+	"observed_on" date NOT NULL,
+	"price_cents" integer,
+	"promo_price_cents" integer,
+	"promo_label" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."items" (
+	"id" serial PRIMARY KEY,
+	"store_id" integer NOT NULL,
+	"store_product_id" text NOT NULL,
+	"name" text NOT NULL,
+	"brand" text,
+	"size_text" text,
+	"unit" text,
+	"unit_amount" numeric(12,4),
+	"unit_source" text,
+	"price_cents" integer,
+	"promo_price_cents" integer,
+	"promo_label" text,
+	"unit_price_cents" integer,
+	"promo_unit_price_cents" integer,
+	"category" text,
+	"category_path" text,
+	"gtin" text,
+	"gtin_checked_at" timestamp with time zone,
+	"image_url" text,
+	"url" text,
+	"available" boolean DEFAULT true NOT NULL,
+	"first_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"last_seen_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"group_id" integer,
+	"match_key" text,
+	"match_method" text,
+	"match_confidence" real,
+	"match_locked" boolean DEFAULT false NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "items_unit_check" CHECK ("unit" IS NULL OR "unit" IN ('piece','kg','l')),
+	CONSTRAINT "items_price_check" CHECK ("price_cents" IS NULL OR "price_cents" >= 0)
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "groups_key_key" ON "groceries"."groups" ("key");--> statement-breakpoint
+CREATE INDEX "groups_family_idx" ON "groceries"."groups" ("family_key");--> statement-breakpoint
+CREATE INDEX "groups_category_idx" ON "groceries"."groups" ("category");--> statement-breakpoint
+CREATE INDEX "groups_store_ids_idx" ON "groceries"."groups" USING gin ("store_ids");--> statement-breakpoint
+CREATE INDEX "groups_search_trgm_idx" ON "groceries"."groups" USING gin ("search_text" gin_trgm_ops);--> statement-breakpoint
+CREATE INDEX "groups_list_idx" ON "groceries"."groups" ("store_count","name");--> statement-breakpoint
+CREATE INDEX "import_runs_store_idx" ON "groceries"."import_runs" ("store_id","started_at");--> statement-breakpoint
+CREATE UNIQUE INDEX "item_prices_item_day_key" ON "groceries"."item_prices" ("item_id","observed_on");--> statement-breakpoint
+CREATE UNIQUE INDEX "items_store_product_key" ON "groceries"."items" ("store_id","store_product_id");--> statement-breakpoint
+CREATE INDEX "items_group_idx" ON "groceries"."items" ("group_id");--> statement-breakpoint
+CREATE INDEX "items_gtin_idx" ON "groceries"."items" ("gtin");--> statement-breakpoint
+CREATE INDEX "items_name_trgm_idx" ON "groceries"."items" USING gin ("name" gin_trgm_ops);--> statement-breakpoint
+ALTER TABLE "groceries"."groups" ADD CONSTRAINT "groups_merged_into_id_groups_id_fkey" FOREIGN KEY ("merged_into_id") REFERENCES "groceries"."groups"("id") ON DELETE SET NULL;--> statement-breakpoint
+ALTER TABLE "groceries"."import_runs" ADD CONSTRAINT "import_runs_store_id_stores_id_fkey" FOREIGN KEY ("store_id") REFERENCES "groceries"."stores"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "groceries"."item_prices" ADD CONSTRAINT "item_prices_item_id_items_id_fkey" FOREIGN KEY ("item_id") REFERENCES "groceries"."items"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "groceries"."items" ADD CONSTRAINT "items_store_id_stores_id_fkey" FOREIGN KEY ("store_id") REFERENCES "groceries"."stores"("id") ON DELETE CASCADE;--> statement-breakpoint
+ALTER TABLE "groceries"."items" ADD CONSTRAINT "items_group_id_groups_id_fkey" FOREIGN KEY ("group_id") REFERENCES "groceries"."groups"("id") ON DELETE SET NULL;--> statement-breakpoint
+DO $$
+DECLARE t text;
+BEGIN
+  IF to_regprocedure('meta.touch_updated_at()') IS NOT NULL THEN
+    FOREACH t IN ARRAY ARRAY['groups','items'] LOOP
+      EXECUTE format('CREATE TRIGGER touch_updated_at BEFORE UPDATE ON groceries.%I FOR EACH ROW EXECUTE FUNCTION meta.touch_updated_at()', t);
+    END LOOP;
+  END IF;
+END $$;
+--> statement-breakpoint
+-- Private: app role only (default privileges from the first groceries migration also apply).
+DO $$
+BEGIN
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA groceries TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA groceries TO personal_app;
+  END IF;
+END $$;
+--> statement-breakpoint
+DO $$
+BEGIN
+  IF to_regclass('meta.tables') IS NOT NULL THEN
+    INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
+    VALUES ('groceries','groups','groceries-groups','Grocery groups','core',true,false,1100),
+           ('groceries','items','groceries-items','Grocery store items','core',true,false,1100),
+           ('groceries','item_prices','groceries-item-prices','Grocery price history','core',false,false,1100),
+           ('groceries','import_runs','groceries-import-runs','Grocery import runs','core',false,false,1100)
+    ON CONFLICT DO NOTHING;
+    IF has_function_privilege('meta.sync(integer)', 'EXECUTE') THEN
+      PERFORM meta.sync();
+    END IF;
+  END IF;
+END $$;

scripts/deploy-preflight.ts

@@ -167,8 +167,8 @@ try {
 		where n.nspname = 'groceries' and c.relkind = 'r'
 	`;
 	if (
-		Number(groceries.tables) !== 3 ||
+		Number(groceries.tables) !== 5 ||
-		Number(groceries.app_rw) !== 3 ||
+		Number(groceries.app_rw) !== 5 ||
 		Number(groceries.ro_read) !== 0 ||
 		groceries.public_usage
 	)

scripts/groceries-import.ts

@@ -0,0 +1,63 @@
+// Nightly supermarket catalogue import (Albert Heijn + Jumbo) and product regrouping.
+//
+//   bun --env-file=$HOME/.config/groceries-import.env scripts/groceries-import.ts [options]
+//     --store ah|jumbo     only this store (repeatable; default: all)
+//     --skip-fetch         no store requests; only regroup what is in the DB
+//     --gtin-limit N       AH product details to fetch for EANs this run (default 1500, 0 = off)
+//     --delay-ms N         minimum delay between requests (default 2000)
+//
+// Politeness: one request at a time, >= 2 s apart, backoff on 429/5xx, identifying User-Agent,
+// and a same-day cache in $GROCERIES_CACHE_DIR (default ~/.cache/groceries, kept 7 days).
+// Plus is not imported: plus.nl blocks this server's IP range (Incapsula), and we do not evade it.
+import postgres from 'postgres';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
+import { enrichAhGtins, importStore, regroup } from '../src/lib/server/groceries/import';
+import { SOURCES } from '../src/lib/server/groceries/sources';
+
+const args = process.argv.slice(2);
+const values = (name: string) =>
+	args.flatMap((a, i) => (a === name && args[i + 1] ? [args[i + 1]] : []));
+const num = (name: string, d: number) => {
+	const v = values(name)[0];
+	const n = v === undefined ? d : Number(v);
+	if (!Number.isFinite(n) || n < 0) throw new Error(`${name} must be a number`);
+	return n;
+};
+if (!process.env.DB_URL) throw new Error('DB_URL is not set');
+const stores = values('--store').length ? values('--store') : Object.keys(SOURCES);
+for (const s of stores) if (!SOURCES[s]) throw new Error(`unknown store ${s}`);
+const delay = num('--delay-ms', 2000);
+const gtinLimit = num('--gtin-limit', 1500);
+const cacheDir = process.env.GROCERIES_CACHE_DIR || join(homedir(), '.cache', 'groceries');
+const log = (m: string) => console.log(`[${new Date().toISOString().slice(11, 19)}Z] ${m}`);
+
+const sql = postgres(process.env.DB_URL, { max: 2, connect_timeout: 10, onnotice: () => {} });
+let failed = false;
+try {
+	pruneCache(cacheDir);
+	if (!args.includes('--skip-fetch')) {
+		for (const slug of stores) {
+			const f = new PoliteFetcher({ store: slug, minDelayMs: delay, cacheDir, log });
+			try {
+				log(`${slug}: importing`);
+				const r = await importStore(sql, SOURCES[slug], f, log);
+				log(
+					`${slug}: ${r.fetched} products (${r.inserted} new, ${r.priceChanges} price changes, ${r.delisted} delisted); ${r.requests} requests, ${r.cacheHits} from cache`
+				);
+				if (slug === 'ah' && gtinLimit > 0) {
+					const g = await enrichAhGtins(sql, f, gtinLimit, log);
+					log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
+				}
+			} catch (e) {
+				failed = true;
+				log(`${slug}: FAILED ${e instanceof Error ? e.message : e}`);
+			}
+		}
+	}
+	await regroup(sql, log);
+} finally {
+	await sql.end();
+}
+if (failed) process.exitCode = 1;

src/lib/groceries.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from 'vitest';
+import { cheapest, formatAmount, formatUnitPrice, sortByPrice, type PricedItem } from './groceries';
+
+const it_ = (
+	id: number,
+	storeId: number,
+	price: number,
+	unit: number,
+	promo: number | null = null
+): PricedItem => ({
+	id,
+	storeId,
+	priceCents: price,
+	promoPriceCents: promo,
+	unitPriceCents: unit,
+	promoUnitPriceCents: promo == null ? null : Math.round((unit * promo) / price),
+	available: true
+});
+
+describe('grocery price helpers', () => {
+	const items = [
+		it_(1, 1, 119, 119),
+		it_(2, 1, 165, 110),
+		it_(3, 2, 115, 115),
+		it_(4, 2, 200, 200, 100)
+	];
+	it('finds the cheapest by unit price, promos included', () => {
+		const c = cheapest(items, 'unit');
+		expect(c.overall).toBe(4);
+		expect([...c.perStore].sort()).toEqual([2, 4]);
+	});
+	it('finds the cheapest by pack price', () => {
+		expect(cheapest(items, 'pack').overall).toBe(4);
+		expect(cheapest(items.slice(0, 3), 'pack').overall).toBe(3);
+	});
+	it('sorts by unit price', () => {
+		expect(sortByPrice(items, 'unit').map((i) => i.id)).toEqual([4, 2, 3, 1]);
+	});
+	it('formats', () => {
+		expect(formatUnitPrice(250, 'kg')).toMatch(/2,50\/kg$/);
+		expect(formatAmount(1.98, 'l')).toBe('1,98 L');
+		expect(formatUnitPrice(null, 'kg')).toBeNull();
+	});
+});

src/lib/groceries.ts

@@ -1,24 +1,64 @@
-// Shared (client + server) grocery helpers: price comparison and formatting.
+// Shared (client + server) grocery helpers: formatting and cheapest-price picking.
-
-export type LatestPrice = {
-	storeId: number;
-	priceCents: number;
-	unitPriceCents: number | null;
-	promo: boolean;
-	observedAt: Date;
-};
-
-/** Store ids with the lowest comparable price: unit price when every store has one, else pack price. */
-export function cheapestStores(prices: LatestPrice[]): Set<number> {
-	if (prices.length < 2) return new Set();
-	const byUnit = prices.every((p) => p.unitPriceCents !== null);
-	const val = (p: LatestPrice) => (byUnit ? p.unitPriceCents! : p.priceCents);
-	const min = Math.min(...prices.map(val));
-	return new Set(prices.filter((p) => val(p) === min).map((p) => p.storeId));
-}
 
 const eur = new Intl.NumberFormat('nl-NL', { style: 'currency', currency: 'EUR' });
 /** 179 → "€ 1,79". */
 export const formatCents = (cents: number) => eur.format(cents / 100);
 
 export const UNIT_LABEL: Record<string, string> = { kg: 'kg', l: 'L', piece: 'st' };
+
+/** "€ 2,50/kg"; null without a unit price. */
+export function formatUnitPrice(cents: number | null | undefined, unit: string | null | undefined) {
+	if (cents == null || !unit) return null;
+	return `${formatCents(cents)}/${UNIT_LABEL[unit] ?? unit}`;
+}
+
+/** "0,5 kg", "1,98 L", "6 st". */
+export function formatAmount(amount: number | null | undefined, unit: string | null | undefined) {
+	if (amount == null || !unit) return null;
+	const n = amount.toLocaleString('nl-NL', { maximumFractionDigits: 3 });
+	return `${n} ${UNIT_LABEL[unit] ?? unit}`;
+}
+
+export type PricedItem = {
+	id: number;
+	storeId: number;
+	priceCents: number | null;
+	promoPriceCents: number | null;
+	unitPriceCents: number | null;
+	promoUnitPriceCents: number | null;
+	available: boolean;
+};
+
+/** What you pay now (promo if any). */
+export const effectivePrice = (i: PricedItem) => i.promoPriceCents ?? i.priceCents;
+export const effectiveUnitPrice = (i: PricedItem) => i.promoUnitPriceCents ?? i.unitPriceCents;
+
+/**
+ * Cheapest items: overall and per store, by unit price (or pack price). Unavailable items and
+ * items without the compared price are ignored.
+ */
+export function cheapest(items: PricedItem[], mode: 'unit' | 'pack') {
+	const val = (i: PricedItem) => (mode === 'unit' ? effectiveUnitPrice(i) : effectivePrice(i));
+	let overall: number | null = null;
+	let overallVal = Infinity;
+	const perStore = new Map<number, { id: number; v: number }>();
+	for (const i of items) {
+		const v = val(i);
+		if (!i.available || v == null) continue;
+		if (v < overallVal) {
+			overallVal = v;
+			overall = i.id;
+		}
+		const cur = perStore.get(i.storeId);
+		if (!cur || v < cur.v) perStore.set(i.storeId, { id: i.id, v });
+	}
+	return { overall, perStore: new Set([...perStore.values()].map((x) => x.id)) };
+}
+
+/** Sort by unit or pack price ascending; missing prices and unavailable items last. */
+export function sortByPrice<T extends PricedItem>(items: T[], mode: 'unit' | 'pack'): T[] {
+	const val = (i: PricedItem) =>
+		(i.available ? 0 : 1e12) +
+		((mode === 'unit' ? effectiveUnitPrice(i) : effectivePrice(i)) ?? 1e11);
+	return [...items].sort((a, b) => val(a) - val(b) || a.id - b.id);
+}

src/lib/server/db/schema/groceries.ts

@@ -1,12 +1,27 @@
 import { sql } from 'drizzle-orm';
-import { boolean, check, index, integer, numeric, text, uniqueIndex } from 'drizzle-orm/pg-core';
+import {
+	type AnyPgColumn,
+	boolean,
+	check,
+	date,
+	index,
+	integer,
+	numeric,
+	real,
+	text,
+	uniqueIndex
+} from 'drizzle-orm/pg-core';
 import { groceriesSchema } from './schemas';
 import { id, timestamps, timestamptz } from './helpers';
 
-// Private grocery tracker (owner only, see requireOwner): the products Stan buys, which
+// Private grocery price comparison (owner only, see requireOwner).
-// ones are favourites, and what they cost at each supermarket over time.
+//
-// Prices are append-only observations in cents, so price history and "cheapest now" both
+// `items` are store products as the supermarket sells them (one row per store SKU), filled
-// come from the same table. The latest observation per (product, store) is the current price.
+// nightly by scripts/groceries-import.ts. `groups` link equivalent items across stores and pack
+// sizes ("Halfvolle melk" = AH + Jumbo + Plus house brand, any size), so unit prices compare.
+// Grouping is a heuristic (EAN, then normalised name + brand class, then fuzzy name); Stan can
+// fix it in the UI, which locks the item (`match_locked`) so the nightly regroup leaves it alone.
+// `item_prices` keeps history: a row only when an item's price or promo price changes.
 
 export const GROCERY_UNITS = ['piece', 'kg', 'l'] as const;
 export type GroceryUnit = (typeof GROCERY_UNITS)[number];
@@ -32,66 +47,137 @@ export const groceryStores = groceriesSchema.table(
 );
 
 /**
- * A product as Stan thinks of it ("Halfvolle melk 1 L"), not one store's SKU: the same
+ * A comparable product ("Halfvolle melk", house brand) across stores and sizes. Stats columns
- * product can be priced at several stores. `unit_amount` + `unit` (e.g. 1.5 + 'l') make
+ * (item_count … search_text) are denormalised by refreshGroupStats() so the list stays fast.
- * unit prices comparable across pack sizes.
  */
-export const groceryProducts = groceriesSchema.table(
+export const groceryGroups = groceriesSchema.table(
-	'products',
+	'groups',
 	{
 		id: id(),
+		/** Matcher key (brand class | unit | name tokens); null for groups made by hand. */
+		key: text('key'),
 		name: text('name').notNull(),
-		brand: text('brand'),
+		brand: text('brand'), // display brand; 'Huismerk' for AH/Jumbo/Plus own brands
 		category: text('category'),
-		size: text('size'), // as printed, e.g. "6 x 1 L"
-		unitAmount: numeric('unit_amount', { precision: 10, scale: 3, mode: 'number' }),
 		unit: text('unit').$type<GroceryUnit>(),
+		/** Name tokens without brand: groups sharing it are "similar" (e.g. house vs Campina melk). */
+		familyKey: text('family_key'),
 		favorite: boolean('favorite').notNull().default(false),
+		manual: boolean('manual').notNull().default(false),
+		mergedIntoId: integer('merged_into_id').references((): AnyPgColumn => groceryGroups.id, {
+			onDelete: 'set null'
+		}),
 		notes: text('notes'),
-		archivedAt: timestamptz('archived_at'),
+		itemCount: integer('item_count').notNull().default(0),
+		storeCount: integer('store_count').notNull().default(0),
+		storeIds: integer('store_ids')
+			.array()
+			.notNull()
+			.default(sql`'{}'::int[]`),
+		minUnitPriceCents: integer('min_unit_price_cents'),
+		minPriceCents: integer('min_price_cents'),
+		promoCount: integer('promo_count').notNull().default(0),
+		imageUrl: text('image_url'),
+		searchText: text('search_text').notNull().default(''),
 		...timestamps
 	},
 	(t) => [
-		uniqueIndex('products_name_brand_key').on(
+		uniqueIndex('groups_key_key').on(t.key),
-			sql`lower(${t.name})`,
+		index('groups_family_idx').on(t.familyKey),
-			sql`lower(coalesce(${t.brand}, ''))`
+		index('groups_category_idx').on(t.category),
-		),
+		index('groups_store_ids_idx').using('gin', t.storeIds),
-		index('products_category_idx').on(t.category),
+		index('groups_search_trgm_idx').using('gin', sql`${t.searchText} gin_trgm_ops`),
-		check('products_name_check', sql`length(btrim(${t.name})) > 0`),
+		index('groups_list_idx').on(t.storeCount, t.name),
-		check('products_unit_check', sql`${t.unit} IS NULL OR ${t.unit} IN ('piece','kg','l')`),
+		check('groups_name_check', sql`length(btrim(${t.name})) > 0`),
-		check('products_unit_amount_check', sql`${t.unitAmount} IS NULL OR ${t.unitAmount} > 0`)
+		check('groups_unit_check', sql`${t.unit} IS NULL OR ${t.unit} IN ('piece','kg','l')`),
+		check('groups_not_self_merged', sql`${t.mergedIntoId} IS NULL OR ${t.mergedIntoId} <> ${t.id}`)
 	]
 );
 
-/** One observed price of a product at a store (manual entry or a later import). */
+/** One store product (SKU) with its current price. Prices are in cents. */
-export const groceryPrices = groceriesSchema.table(
+export const groceryItems = groceriesSchema.table(
-	'prices',
+	'items',
 	{
 		id: id(),
-		productId: integer('product_id')
-			.notNull()
-			.references(() => groceryProducts.id, { onDelete: 'cascade' }),
 		storeId: integer('store_id')
 			.notNull()
 			.references(() => groceryStores.id, { onDelete: 'cascade' }),
-		priceCents: integer('price_cents').notNull(),
+		storeProductId: text('store_product_id').notNull(), // AH webshopId, Jumbo SKU
-		unitPriceCents: integer('unit_price_cents'), // per product.unit, if known
+		name: text('name').notNull(),
-		promo: boolean('promo').notNull().default(false), // bonus / aanbieding price
+		brand: text('brand'),
-		observedAt: timestamptz('observed_at').notNull().defaultNow(),
+		sizeText: text('size_text'), // as the store prints it, e.g. "6 x 330 ml"
-		source: text('source').notNull().default('manual'), // 'manual', 'ah-api', 'jumbo-api', …
+		unit: text('unit').$type<GroceryUnit>(),
-		storeProductId: text('store_product_id'), // e.g. AH webshopId, Jumbo SKU
+		unitAmount: numeric('unit_amount', { precision: 12, scale: 4, mode: 'number' }),
+		/** 'size' = parsed from the pack size, 'store' = derived from the store's own unit price. */
+		unitSource: text('unit_source'),
+		priceCents: integer('price_cents'),
+		promoPriceCents: integer('promo_price_cents'),
+		promoLabel: text('promo_label'),
+		unitPriceCents: integer('unit_price_cents'),
+		promoUnitPriceCents: integer('promo_unit_price_cents'),
+		category: text('category'), // canonical category (shared across stores)
+		categoryPath: text('category_path'), // the store's own path
+		gtin: text('gtin'),
+		gtinCheckedAt: timestamptz('gtin_checked_at'),
+		imageUrl: text('image_url'),
 		url: text('url'),
-		notes: text('notes'),
+		available: boolean('available').notNull().default(true),
+		firstSeenAt: timestamptz('first_seen_at').notNull().defaultNow(),
+		lastSeenAt: timestamptz('last_seen_at').notNull().defaultNow(),
+		groupId: integer('group_id').references(() => groceryGroups.id, { onDelete: 'set null' }),
+		matchKey: text('match_key'),
+		matchMethod: text('match_method'), // 'ean' | 'key' | 'fuzzy' | 'single' | 'manual'
+		matchConfidence: real('match_confidence'),
+		matchLocked: boolean('match_locked').notNull().default(false),
 		...timestamps
 	},
 	(t) => [
-		index('prices_product_store_idx').on(t.productId, t.storeId, t.observedAt),
+		uniqueIndex('items_store_product_key').on(t.storeId, t.storeProductId),
-		index('prices_store_idx').on(t.storeId),
+		index('items_group_idx').on(t.groupId),
-		check('prices_price_check', sql`${t.priceCents} >= 0`),
+		index('items_gtin_idx').on(t.gtin),
-		check('prices_unit_price_check', sql`${t.unitPriceCents} IS NULL OR ${t.unitPriceCents} >= 0`)
+		index('items_name_trgm_idx').using('gin', sql`${t.name} gin_trgm_ops`),
+		check('items_unit_check', sql`${t.unit} IS NULL OR ${t.unit} IN ('piece','kg','l')`),
+		check('items_price_check', sql`${t.priceCents} IS NULL OR ${t.priceCents} >= 0`)
 	]
 );
 
+/** Price history: one row per item per day on which its price or promo changed. */
+export const groceryItemPrices = groceriesSchema.table(
+	'item_prices',
+	{
+		id: id(),
+		itemId: integer('item_id')
+			.notNull()
+			.references(() => groceryItems.id, { onDelete: 'cascade' }),
+		observedOn: date('observed_on').notNull(),
+		priceCents: integer('price_cents'),
+		promoPriceCents: integer('promo_price_cents'),
+		promoLabel: text('promo_label'),
+		createdAt: timestamptz('created_at').notNull().defaultNow()
+	},
+	(t) => [uniqueIndex('item_prices_item_day_key').on(t.itemId, t.observedOn)]
+);
+
+/** One importer run per store (counts and errors), for the UI footer and debugging. */
+export const groceryImportRuns = groceriesSchema.table(
+	'import_runs',
+	{
+		id: id(),
+		storeId: integer('store_id')
+			.notNull()
+			.references(() => groceryStores.id, { onDelete: 'cascade' }),
+		startedAt: timestamptz('started_at').notNull().defaultNow(),
+		finishedAt: timestamptz('finished_at'),
+		status: text('status').notNull().default('running'), // running | ok | error
+		requests: integer('requests').notNull().default(0),
+		fetched: integer('fetched').notNull().default(0),
+		inserted: integer('inserted').notNull().default(0),
+		priceChanges: integer('price_changes').notNull().default(0),
+		error: text('error')
+	},
+	(t) => [index('import_runs_store_idx').on(t.storeId, t.startedAt)]
+);
+
 export type GroceryStore = typeof groceryStores.$inferSelect;
-export type GroceryProduct = typeof groceryProducts.$inferSelect;
+export type GroceryGroup = typeof groceryGroups.$inferSelect;
-export type GroceryPrice = typeof groceryPrices.$inferSelect;
+export type GroceryItem = typeof groceryItems.$inferSelect;

src/lib/server/groceries.test.ts

@@ -1,45 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { parseCents, parseSize, priceInput, unitPriceCents } from './groceries';
-import { cheapestStores, formatCents } from '../groceries';
-
-describe('groceries helpers', () => {
-	it('parses prices into cents', () => {
-		expect(parseCents('1,79')).toBe(179);
-		expect(parseCents('€ 2.5')).toBe(250);
-		expect(parseCents('3')).toBe(300);
-		expect(parseCents('abc')).toBeNull();
-		expect(parseCents('1,234')).toBeNull();
-	});
-	it('parses pack sizes', () => {
-		expect(parseSize('1 L')).toEqual({ amount: 1, unit: 'l' });
-		expect(parseSize('6 x 1 L')).toEqual({ amount: 6, unit: 'l' });
-		expect(parseSize('500 g')).toEqual({ amount: 0.5, unit: 'kg' });
-		expect(parseSize('750ml')).toEqual({ amount: 0.75, unit: 'l' });
-		expect(parseSize('4 stuks')).toEqual({ amount: 4, unit: 'piece' });
-		expect(parseSize('1,5 liter')).toEqual({ amount: 1.5, unit: 'l' });
-		expect(parseSize('a bunch')).toBeNull();
-	});
-	it('computes unit prices', () => {
-		expect(unitPriceCents(479, 6)).toBe(80);
-		expect(unitPriceCents(100, null)).toBeNull();
-	});
-	it('validates price input', () => {
-		expect(priceInput.safeParse({ price: '1,29' }).data?.price).toBe(129);
-		expect(priceInput.safeParse({ price: 'x' }).success).toBe(false);
-		expect(priceInput.safeParse({ price: '1', url: '' }).success).toBe(true);
-	});
-	it('finds the cheapest store', () => {
-		const p = (storeId: number, priceCents: number, unitPriceCents: number | null) => ({
-			storeId,
-			priceCents,
-			unitPriceCents,
-			promo: false,
-			observedAt: new Date()
-		});
-		expect([...cheapestStores([p(1, 179, 179), p(2, 129, 129)])]).toEqual([2]);
-		expect([...cheapestStores([p(1, 479, 80), p(2, 129, 129)])]).toEqual([1]);
-		expect([...cheapestStores([p(1, 479, null), p(2, 129, 129)])]).toEqual([2]);
-		expect(cheapestStores([p(1, 100, null)]).size).toBe(0);
-		expect(formatCents(179).replace(/\s/g, ' ')).toBe('€ 1,79');
-	});
-});

src/lib/server/groceries.ts

@@ -1,77 +0,0 @@
-import * as z from 'zod';
-import { GROCERY_UNITS } from './db/schema/groceries';
-
-/** "1,79" / "1.79" / "€ 1,79" → 179 cents; null when not a price. */
-export function parseCents(input: unknown): number | null {
-	if (typeof input !== 'string') return null;
-	const s = input.replace(/[€\s]/g, '').replace(',', '.');
-	if (!/^\d+(\.\d{1,2})?$/.test(s)) return null;
-	return Math.round(Number(s) * 100);
-}
-
-/** Price per unit (kg / l / piece) in cents, rounded; null without a usable amount. */
-export function unitPriceCents(priceCents: number, unitAmount: number | null | undefined) {
-	if (!unitAmount || !(unitAmount > 0)) return null;
-	return Math.round(priceCents / unitAmount);
-}
-
-/**
- * Best-effort pack size parse, e.g. "1 L" → 1 l, "6 x 1 L" → 6 l, "500 g" → 0.5 kg,
- * "750ml" → 0.75 l, "4 stuks" → 4 piece. Returns null when unclear.
- */
-export function parseSize(
-	size: string | null | undefined
-): { amount: number; unit: (typeof GROCERY_UNITS)[number] } | null {
-	if (!size) return null;
-	const s = size.toLowerCase().replace(',', '.').trim();
-	const m = s.match(
-		/^(?:(\d+)\s*x\s*)?(\d+(?:\.\d+)?)\s*(kg|kilo|g|gr|gram|l|liter|litre|ml|cl|st|stuk|stuks|pcs|pieces?)$/
-	);
-	if (!m) return null;
-	const n = (m[1] ? Number(m[1]) : 1) * Number(m[2]);
-	const u = m[3];
-	if (['kg', 'kilo'].includes(u)) return { amount: n, unit: 'kg' };
-	if (['g', 'gr', 'gram'].includes(u)) return { amount: n / 1000, unit: 'kg' };
-	if (['l', 'liter', 'litre'].includes(u)) return { amount: n, unit: 'l' };
-	if (u === 'ml') return { amount: n / 1000, unit: 'l' };
-	if (u === 'cl') return { amount: n / 100, unit: 'l' };
-	return { amount: n, unit: 'piece' };
-}
-
-const optText = (max: number) =>
-	z
-		.string()
-		.trim()
-		.max(max)
-		.transform((v) => (v === '' ? null : v))
-		.nullable()
-		.optional();
-
-export const productInput = z.object({
-	name: z.string().trim().min(1, 'Name is required').max(200),
-	brand: optText(100),
-	category: optText(100),
-	size: optText(60),
-	notes: optText(2000)
-});
-
-export const priceInput = z.object({
-	price: z.string().transform((v, ctx) => {
-		const c = parseCents(v);
-		if (c === null) {
-			ctx.addIssue({ code: 'custom', message: 'Enter a price like 1,79' });
-			return z.NEVER;
-		}
-		return c;
-	}),
-	url: optText(2000).refine(
-		(v) => v == null || /^https?:\/\/\S+$/.test(v),
-		'Link must start with http(s)://'
-	),
-	observedOn: z
-		.string()
-		.regex(/^\d{4}-\d{2}-\d{2}$/)
-		.optional()
-		.or(z.literal('').transform(() => undefined)),
-	notes: optText(500)
-});

src/lib/server/groceries/http.ts

@@ -0,0 +1,99 @@
+// Polite JSON fetcher for the grocery importers: one request at a time with a minimum delay,
+// exponential backoff on 429/5xx/network errors (honours Retry-After), and a same-day disk
+// cache so re-running an import on the same day does not hit the store again.
+import { mkdirSync, existsSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
+import { gunzipSync, gzipSync } from 'node:zlib';
+import { join } from 'node:path';
+
+export const USER_AGENT =
+	'stanrunge.dev grocery price tracker (personal use, 1 req/2s; +https://stanrunge.dev)';
+
+export type FetcherOptions = {
+	store: string;
+	minDelayMs?: number;
+	maxRetries?: number;
+	cacheDir?: string | null;
+	log?: (msg: string) => void;
+};
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+const today = () => new Date().toLocaleDateString('sv-SE', { timeZone: 'Europe/Amsterdam' });
+
+export class PoliteFetcher {
+	requests = 0;
+	cacheHits = 0;
+	private last = 0;
+	private readonly minDelayMs: number;
+	private readonly maxRetries: number;
+	private readonly dir: string | null;
+	private readonly log: (msg: string) => void;
+
+	constructor(opts: FetcherOptions) {
+		this.minDelayMs = opts.minDelayMs ?? 2000;
+		this.maxRetries = opts.maxRetries ?? 5;
+		this.log = opts.log ?? (() => {});
+		this.dir = opts.cacheDir ? join(opts.cacheDir, today(), opts.store) : null;
+		if (this.dir) mkdirSync(this.dir, { recursive: true });
+	}
+
+	/** Fetch JSON. `cacheKey` (a file-safe string) enables the same-day cache. */
+	async json<T>(url: string, init: RequestInit = {}, cacheKey?: string): Promise<T> {
+		const file = this.dir && cacheKey ? join(this.dir, `${cacheKey}.json.gz`) : null;
+		if (file && existsSync(file)) {
+			this.cacheHits++;
+			return JSON.parse(gunzipSync(readFileSync(file)).toString('utf8')) as T;
+		}
+		for (let attempt = 0; ; attempt++) {
+			const wait = this.last + this.minDelayMs - Date.now();
+			if (wait > 0) await sleep(wait);
+			this.last = Date.now();
+			this.requests++;
+			let retryAfter = 0;
+			let problem: string;
+			try {
+				const res = await fetch(url, { ...init, signal: AbortSignal.timeout(45_000) });
+				if (res.ok) {
+					const text = await res.text();
+					const data = JSON.parse(text) as T;
+					if (file) writeFileSync(file, gzipSync(text));
+					return data;
+				}
+				problem = `HTTP ${res.status}`;
+				if (res.status !== 429 && res.status < 500 && res.status !== 408) {
+					throw new HttpError(
+						res.status,
+						`${problem} for ${url}: ${(await res.text()).slice(0, 200)}`
+					);
+				}
+				retryAfter = Number(res.headers.get('retry-after')) * 1000 || 0;
+			} catch (e) {
+				if (e instanceof HttpError) throw e;
+				problem = e instanceof Error ? e.message : String(e);
+			}
+			if (attempt >= this.maxRetries)
+				throw new Error(`${problem} for ${url} after ${attempt + 1} tries`);
+			const backoff = Math.max(retryAfter, 5000 * 2 ** attempt);
+			this.log(`  ${problem}; retrying in ${Math.round(backoff / 1000)}s`);
+			await sleep(backoff);
+		}
+	}
+}
+
+export class HttpError extends Error {
+	constructor(
+		readonly status: number,
+		message: string
+	) {
+		super(message);
+	}
+}
+
+/** Delete cache days older than `keepDays`. */
+export function pruneCache(cacheDir: string, keepDays = 7) {
+	if (!existsSync(cacheDir)) return;
+	const cutoff = new Date(Date.now() - keepDays * 86_400_000).toLocaleDateString('sv-SE', {
+		timeZone: 'Europe/Amsterdam'
+	});
+	for (const d of readdirSync(cacheDir))
+		if (/^\d{4}-\d{2}-\d{2}$/.test(d) && d < cutoff) rmSync(join(cacheDir, d), { recursive: true });
+}

src/lib/server/groceries/import.ts

@@ -0,0 +1,397 @@
+// DB side of the grocery importer (used by scripts/groceries-import.ts). Uses a plain postgres.js
+// client so it runs outside SvelteKit. All writes are idempotent; re-running is safe.
+import type { Sql } from 'postgres';
+import type { PoliteFetcher } from './http';
+import { clusterItems, type MatchItem } from './match';
+import {
+	brandClass,
+	brandLabel,
+	displayName,
+	matchKey,
+	nameTokens,
+	normaliseUnit
+} from './normalize';
+import { AhClient, type RawItem, type Source } from './sources';
+
+type Log = (m: string) => void;
+
+export async function storeId(sql: Sql, slug: string): Promise<number> {
+	const [s] = await sql<{ id: number }[]>`select id from groceries.stores where slug = ${slug}`;
+	if (!s) throw new Error(`unknown store ${slug}`);
+	return s.id;
+}
+
+function keyFor(name: string, brand: string | null, unit: string | null) {
+	const cls = brandClass(brand);
+	const tokens = nameTokens(name, brand);
+	return { cls, tokens, key: matchKey(cls, unit, tokens) };
+}
+
+/** Fetch a store's catalogue and upsert it; returns counts. */
+export async function importStore(sql: Sql, source: Source, f: PoliteFetcher, log: Log) {
+	const sid = await storeId(sql, source.slug);
+	const [run] = await sql<{ id: number; started_at: Date }[]>`
+		insert into groceries.import_runs (store_id) values (${sid}) returning id, started_at`;
+	try {
+		const raw = await source.fetchAll(f, log);
+		const [prev] = await sql<{ n: number }[]>`
+			select count(*)::int as n from groceries.items where store_id = ${sid} and available`;
+		const res = await upsertItems(sql, sid, raw, run.started_at);
+		// Only mark unseen items unavailable when the crawl looks complete.
+		let delisted = 0;
+		if (raw.length >= prev.n * 0.8) {
+			const r = await sql`
+				update groceries.items set available = false
+				where store_id = ${sid} and available and last_seen_at < ${run.started_at}`;
+			delisted = r.count;
+		}
+		await sql`
+			update groceries.import_runs set finished_at = now(), status = 'ok', requests = ${f.requests},
+				fetched = ${raw.length}, inserted = ${res.inserted}, price_changes = ${res.priceChanges}
+			where id = ${run.id}`;
+		return { fetched: raw.length, ...res, delisted, requests: f.requests, cacheHits: f.cacheHits };
+	} catch (e) {
+		await sql`
+			update groceries.import_runs set finished_at = now(), status = 'error', requests = ${f.requests},
+				error = ${String(e instanceof Error ? e.message : e).slice(0, 2000)}
+			where id = ${run.id}`;
+		throw e;
+	}
+}
+
+async function upsertItems(sql: Sql, sid: number, raw: RawItem[], runStart: Date) {
+	let inserted = 0;
+	for (let i = 0; i < raw.length; i += 500) {
+		const rows = raw.slice(i, i + 500).map((r) => {
+			const n = normaliseUnit({
+				sizeText: r.sizeText,
+				title: r.name,
+				priceCents: r.priceCents,
+				promoPriceCents: r.promoPriceCents,
+				storeUnitPrice: r.storeUnitPrice,
+				storeUnitBasisCents: r.storeUnitBasisCents
+			});
+			return {
+				store_id: sid,
+				store_product_id: r.storeProductId,
+				name: r.name,
+				brand: r.brand,
+				size_text: r.sizeText,
+				unit: n.unit,
+				unit_amount: n.unitAmount,
+				unit_source: n.unitSource,
+				price_cents: r.priceCents,
+				promo_price_cents: r.promoPriceCents,
+				promo_label: r.promoLabel,
+				unit_price_cents: n.unitPriceCents,
+				promo_unit_price_cents: n.promoUnitPriceCents,
+				category: r.category,
+				category_path: r.categoryPath,
+				gtin: r.gtin,
+				image_url: r.imageUrl,
+				url: r.url,
+				available: r.available,
+				match_key: keyFor(r.name, r.brand, n.unit).key
+			};
+		});
+		const res = await sql<{ inserted: boolean }[]>`
+			insert into groceries.items ${sql(rows)}
+			on conflict (store_id, store_product_id) do update set
+				name = excluded.name, brand = excluded.brand, size_text = excluded.size_text,
+				unit = excluded.unit, unit_amount = excluded.unit_amount, unit_source = excluded.unit_source,
+				price_cents = excluded.price_cents, promo_price_cents = excluded.promo_price_cents,
+				promo_label = excluded.promo_label, unit_price_cents = excluded.unit_price_cents,
+				promo_unit_price_cents = excluded.promo_unit_price_cents, category = excluded.category,
+				category_path = excluded.category_path, gtin = coalesce(excluded.gtin, items.gtin),
+				image_url = excluded.image_url, url = excluded.url, available = excluded.available,
+				match_key = excluded.match_key, last_seen_at = now()
+			returning (xmax = 0) as inserted`;
+		inserted += res.filter((r) => r.inserted).length;
+	}
+	// Price history: a row when the price or promo differs from the item's last recorded row.
+	const hist = await sql<{ changed: boolean }[]>`
+		insert into groceries.item_prices (item_id, observed_on, price_cents, promo_price_cents, promo_label)
+		select i.id, (now() at time zone 'Europe/Amsterdam')::date, i.price_cents, i.promo_price_cents, i.promo_label
+		from groceries.items i
+		left join lateral (
+			select p.price_cents, p.promo_price_cents, p.promo_label from groceries.item_prices p
+			where p.item_id = i.id order by p.observed_on desc limit 1
+		) last on true
+		where i.store_id = ${sid} and i.last_seen_at >= ${runStart}
+			and (last is null
+				or last.price_cents is distinct from i.price_cents
+				or last.promo_price_cents is distinct from i.promo_price_cents
+				or last.promo_label is distinct from i.promo_label)
+		on conflict (item_id, observed_on) do update set
+			price_cents = excluded.price_cents, promo_price_cents = excluded.promo_price_cents,
+			promo_label = excluded.promo_label
+		returning exists (
+			select 1 from groceries.item_prices o
+			where o.item_id = item_prices.item_id and o.observed_on < item_prices.observed_on
+		) as changed`;
+	return { inserted, priceChanges: hist.filter((h) => h.changed).length };
+}
+
+/** Fill in AH EANs from the product detail endpoint, `limit` products per run (A-brands first). */
+export async function enrichAhGtins(sql: Sql, f: PoliteFetcher, limit: number, log: Log) {
+	if (limit <= 0) return { checked: 0, found: 0 };
+	const sid = await storeId(sql, 'ah');
+	const todo = await sql<{ id: number; store_product_id: string }[]>`
+		select id, store_product_id from groceries.items
+		where store_id = ${sid} and gtin is null and available
+			and (gtin_checked_at is null or gtin_checked_at < now() - interval '90 days')
+		order by (lower(coalesce(brand, '')) like 'ah%') asc, gtin_checked_at asc nulls first, id
+		limit ${limit}`;
+	const ah = new AhClient(f);
+	let found = 0;
+	for (const [n, t] of todo.entries()) {
+		let gtin: string | null;
+		try {
+			gtin = await ah.gtin(t.store_product_id);
+		} catch (e) {
+			log(`  ah gtin ${t.store_product_id}: ${e instanceof Error ? e.message : e}`);
+			if (n > 20 && found === 0) throw e; // something is wrong; stop hammering
+			continue;
+		}
+		if (gtin) found++;
+		await sql`update groceries.items set gtin = ${gtin}, gtin_checked_at = now() where id = ${t.id}`;
+		if ((n + 1) % 250 === 0) log(`  ah gtin: ${n + 1}/${todo.length} checked, ${found} found`);
+	}
+	return { checked: todo.length, found };
+}
+
+// ---- regroup --------------------------------------------------------------------------------
+
+type ItemRow = {
+	id: number;
+	store_id: number;
+	store_slug: string;
+	name: string;
+	brand: string | null;
+	unit: string | null;
+	gtin: string | null;
+	category: string | null;
+	group_id: number | null;
+};
+type GroupRow = { id: number; key: string | null; merged_into_id: number | null; manual: boolean };
+
+/**
+ * Recompute groups for all unlocked items: cluster, map each cluster to a stable group (by key,
+ * following manual merges, else the group most of its items already had), create/update/delete
+ * groups, then refresh the denormalised stats. Locked items (moved by hand) are left alone.
+ */
+export async function regroup(sql: Sql, log: Log) {
+	return sql.begin(async (tx) => {
+		const items = await tx<ItemRow[]>`
+			select i.id, i.store_id, s.slug as store_slug, i.name, i.brand, i.unit, i.gtin, i.category, i.group_id
+			from groceries.items i join groceries.stores s on s.id = i.store_id
+			where not i.match_locked
+			order by i.id`;
+		const groups = await tx<
+			GroupRow[]
+		>`select id, key, merged_into_id, manual from groceries.groups`;
+		const byId = new Map(groups.map((g) => [g.id, g]));
+		const byKey = new Map(groups.filter((g) => g.key).map((g) => [g.key!, g]));
+		const resolve = (g: GroupRow | undefined) => {
+			for (let hops = 0; g?.merged_into_id && hops < 20; hops++) g = byId.get(g.merged_into_id);
+			return g;
+		};
+
+		const meta = new Map<number, ItemRow & { cls: string; tokens: string[]; key: string }>();
+		const matchItems: MatchItem[] = items.map((it) => {
+			const k = keyFor(it.name, it.brand, it.unit);
+			meta.set(it.id, { ...it, ...k });
+			return {
+				id: it.id,
+				storeId: it.store_id,
+				gtin: it.gtin,
+				key: k.key,
+				cls: k.cls,
+				unit: it.unit,
+				tokens: k.tokens
+			};
+		});
+		const clusters = clusterItems(matchItems);
+
+		const claimed = new Set<number>();
+		const keysInUse = new Set(clusters.map((c) => c.key));
+		type Plan = { cluster: (typeof clusters)[number]; groupId: number | null; info: GroupInfo };
+		const plans: Plan[] = [];
+		// Pass 1: exact key (or merged redirect).
+		for (const c of clusters) {
+			const g = resolve(byKey.get(c.key));
+			const info = groupInfo(
+				c.itemIds.map((id) => meta.get(id)!),
+				c.key
+			);
+			if (g) claimed.add(g.id);
+			plans.push({ cluster: c, groupId: g?.id ?? null, info });
+		}
+		// Pass 2: the group most items already belong to (keeps ids stable when names change).
+		for (const p of plans) {
+			if (p.groupId) continue;
+			const counts = new Map<number, number>();
+			for (const id of p.cluster.itemIds) {
+				const gid = meta.get(id)!.group_id;
+				if (gid) counts.set(gid, (counts.get(gid) ?? 0) + 1);
+			}
+			const best = [...counts].sort((a, b) => b[1] - a[1] || a[0] - b[0])[0];
+			const g = best ? byId.get(best[0]) : undefined;
+			if (
+				g &&
+				!g.manual &&
+				!g.merged_into_id &&
+				!claimed.has(g.id) &&
+				!(g.key && keysInUse.has(g.key))
+			) {
+				claimed.add(g.id);
+				p.groupId = g.id;
+			}
+		}
+
+		// Create missing groups.
+		const toCreate = plans.filter((p) => !p.groupId);
+		for (let i = 0; i < toCreate.length; i += 1000) {
+			const chunk = toCreate.slice(i, i + 1000);
+			const ids = await tx<{ id: number }[]>`
+				insert into groceries.groups ${tx(
+					chunk.map((p) => ({
+						key: p.cluster.key,
+						name: p.info.name,
+						brand: p.info.brand,
+						category: p.info.category,
+						unit: p.info.unit,
+						family_key: p.info.familyKey
+					}))
+				)} returning id`;
+			chunk.forEach((p, j) => (p.groupId = ids[j].id));
+		}
+		// Update automatic groups' descriptive fields (manual groups keep Stan's name).
+		const toUpdate = plans.filter((p) => !toCreate.includes(p) && !byId.get(p.groupId!)?.manual);
+		const seen = new Set<number>();
+		for (let i = 0; i < toUpdate.length; i += 1000) {
+			const rows = toUpdate
+				.slice(i, i + 1000)
+				.filter((p) => !seen.has(p.groupId!) && seen.add(p.groupId!))
+				.map((p) => {
+					const g = byId.get(p.groupId!)!;
+					// Only take the cluster key when no other group has it.
+					const key = g.key === p.cluster.key || !byKey.has(p.cluster.key) ? p.cluster.key : g.key;
+					return [
+						p.groupId!,
+						key,
+						p.info.name,
+						p.info.brand,
+						p.info.category,
+						p.info.unit,
+						p.info.familyKey
+					];
+				});
+			if (!rows.length) continue;
+			await tx`
+				update groceries.groups g set key = v.key, name = v.name, brand = v.brand,
+					category = v.category, unit = v.unit, family_key = v.family_key
+				from (values ${tx(rows as never)}) as v(id, key, name, brand, category, unit, family_key)
+				where g.id = v.id::int and (g.key is distinct from v.key or g.name is distinct from v.name
+					or g.brand is distinct from v.brand or g.category is distinct from v.category
+					or g.unit is distinct from v.unit or g.family_key is distinct from v.family_key)`;
+		}
+
+		// Assign items.
+		const assignments: [number, number, string, string, number | null][] = [];
+		for (const p of plans)
+			for (const id of p.cluster.itemIds) {
+				const m = p.cluster.method.get(id)!;
+				assignments.push([id, p.groupId!, meta.get(id)!.key, m.method, m.confidence]);
+			}
+		let moved = 0;
+		for (let i = 0; i < assignments.length; i += 2000) {
+			const r = await tx`
+				update groceries.items i set group_id = v.group_id::int, match_key = v.match_key,
+					match_method = v.method, match_confidence = v.confidence::real
+				from (values ${tx(assignments.slice(i, i + 2000) as never)}) as v(id, group_id, match_key, method, confidence)
+				where i.id = v.id::int and (i.group_id is distinct from v.group_id::int
+					or i.match_key is distinct from v.match_key or i.match_method is distinct from v.method
+					or i.match_confidence is distinct from v.confidence::real)`;
+			moved += r.count;
+		}
+
+		// Drop empty automatic groups (favourites, manual groups and merge tombstones stay).
+		const del = await tx`
+			delete from groceries.groups g
+			where not g.favorite and not g.manual and g.merged_into_id is null and g.notes is null
+				and not exists (select 1 from groceries.items i where i.group_id = g.id)
+				and not exists (select 1 from groceries.groups m where m.merged_into_id = g.id)`;
+		await refreshGroupStats(tx as unknown as Sql);
+		log(
+			`regroup: ${items.length} items in ${clusters.length} clusters; ${toCreate.length} groups created, ${moved} items (re)assigned, ${del.count} empty groups removed`
+		);
+		return { clusters: clusters.length, created: toCreate.length, moved, removed: del.count };
+	});
+}
+
+type GroupInfo = {
+	name: string;
+	brand: string | null;
+	category: string | null;
+	unit: string | null;
+	familyKey: string;
+};
+
+function groupInfo(
+	members: (ItemRow & { cls: string; tokens: string[]; key: string })[],
+	key: string
+): GroupInfo {
+	// Representative: an item with the canonical key, preferring AH (cleanest titles), then Jumbo.
+	const order = (s: string) => (s === 'ah' ? 0 : s === 'jumbo' ? 1 : 2);
+	const rep =
+		[...members]
+			.filter((m) => m.key === key)
+			.sort(
+				(a, b) =>
+					order(a.store_slug) - order(b.store_slug) || a.name.length - b.name.length || a.id - b.id
+			)[0] ?? members[0];
+	const cats = new Map<string, number>();
+	for (const m of members) if (m.category) cats.set(m.category, (cats.get(m.category) ?? 0) + 1);
+	const category = [...cats].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))[0]?.[0] ?? null;
+	return {
+		name: displayName(rep.name, rep.brand).slice(0, 200),
+		brand: brandLabel(rep.cls, rep.brand),
+		category,
+		unit: rep.unit,
+		familyKey: `${rep.unit ?? '?'}|${[...rep.tokens].sort().join(' ')}`
+	};
+}
+
+// Accent folding for search_text (no unaccent extension needed); the query side uses fold().
+const ACCENTS_FROM = 'áàäâãéèëêíìïîóòöôõúùüûçñ';
+const ACCENTS_TO = 'aaaaaeeeeiiiiooooouuuucn';
+
+/** Recompute denormalised group stats (all groups, or just `ids`). */
+export async function refreshGroupStats(sql: Sql, ids?: number[]) {
+	const only = ids ? sql`where g2.id = any(${ids}::int[])` : sql``;
+	await sql`
+		update groceries.groups g set
+			item_count = s.item_count, store_count = s.store_count, store_ids = s.store_ids,
+			min_unit_price_cents = s.min_unit, min_price_cents = s.min_price, promo_count = s.promo_count,
+			image_url = s.image_url, search_text = s.search_text
+		from (
+			select g2.id,
+				count(i.id)::int as item_count,
+				count(distinct i.store_id)::int as store_count,
+				coalesce(array_agg(distinct i.store_id) filter (where i.id is not null), '{}')::int[] as store_ids,
+				min(coalesce(i.promo_unit_price_cents, i.unit_price_cents)) as min_unit,
+				min(coalesce(i.promo_price_cents, i.price_cents)) as min_price,
+				(count(*) filter (where i.promo_price_cents is not null or i.promo_label is not null))::int as promo_count,
+				(array_agg(i.image_url order by i.store_id, i.id) filter (where i.image_url is not null))[1] as image_url,
+				translate(lower(concat_ws(' ', g2.name, g2.brand, g2.category,
+					string_agg(distinct concat_ws(' ', i.brand, i.name), ' '))), ${ACCENTS_FROM}, ${ACCENTS_TO}) as search_text
+			from groceries.groups g2
+			left join groceries.items i on i.group_id = g2.id and i.available
+			${only}
+			group by g2.id
+		) s
+		where g.id = s.id and (g.item_count, g.store_count, g.store_ids, g.min_unit_price_cents, g.min_price_cents,
+			g.promo_count, g.image_url, g.search_text) is distinct from
+			(s.item_count, s.store_count, s.store_ids, s.min_unit, s.min_price, s.promo_count, s.image_url, s.search_text)`;
+}

src/lib/server/groceries/match.ts

@@ -0,0 +1,203 @@
+// Heuristic product grouping (no ML). Items are clustered by:
+//   1. the same match key (brand class + unit + name tokens): one product in several pack sizes,
+//      and AH/Jumbo/Plus house brands as one "Huismerk" product;
+//   2. the same EAN/GTIN (an A-brand product at several stores);
+//   3. fuzzy name similarity (token Jaccard >= 0.75) between clusters of different stores with the
+//      same brand class and unit, greedily best-first, never joining two items of one store
+//      that came from different clusters.
+
+export type MatchItem = {
+	id: number;
+	storeId: number;
+	gtin: string | null;
+	key: string;
+	cls: string;
+	unit: string | null;
+	tokens: string[];
+};
+
+export type ClusterResult = {
+	key: string; // canonical key: the key of the cluster's largest exact-key part
+	itemIds: number[];
+	method: Map<number, { method: 'ean' | 'key' | 'fuzzy' | 'single'; confidence: number | null }>;
+};
+
+export const FUZZY_THRESHOLD = 0.75;
+
+class UnionFind {
+	parent: number[];
+	constructor(n: number) {
+		this.parent = Array.from({ length: n }, (_, i) => i);
+	}
+	find(i: number): number {
+		while (this.parent[i] !== i) {
+			this.parent[i] = this.parent[this.parent[i]];
+			i = this.parent[i];
+		}
+		return i;
+	}
+	union(a: number, b: number) {
+		const ra = this.find(a);
+		const rb = this.find(b);
+		if (ra !== rb) this.parent[rb] = ra;
+		return ra;
+	}
+}
+
+export function jaccard(a: string[], b: string[]): number {
+	if (!a.length || !b.length) return 0;
+	const sb = new Set(b);
+	let inter = 0;
+	for (const t of new Set(a)) if (sb.has(t)) inter++;
+	return inter / new Set([...a, ...b]).size;
+}
+
+/** Most common value; ties → lexicographically smallest (deterministic). */
+function mode(values: string[]): string {
+	const counts = new Map<string, number>();
+	for (const v of values) counts.set(v, (counts.get(v) ?? 0) + 1);
+	let best = values[0];
+	for (const [v, c] of counts) {
+		const bc = counts.get(best)!;
+		if (c > bc || (c === bc && v < best)) best = v;
+	}
+	return best;
+}
+
+export function clusterItems(items: MatchItem[]): ClusterResult[] {
+	const n = items.length;
+	const uf = new UnionFind(n);
+	const byKey = new Map<string, number>();
+	const byGtin = new Map<string, number>();
+	const eanLinked = new Set<number>();
+	for (let i = 0; i < n; i++) {
+		const it = items[i];
+		const k = byKey.get(it.key);
+		if (k === undefined) byKey.set(it.key, i);
+		else uf.union(k, i);
+	}
+	for (let i = 0; i < n; i++) {
+		const g = items[i].gtin;
+		if (!g) continue;
+		const j = byGtin.get(g);
+		if (j === undefined) byGtin.set(g, i);
+		else if (uf.find(j) !== uf.find(i) || items[i].storeId !== items[j].storeId) {
+			uf.union(j, i);
+			eanLinked.add(i).add(j);
+		}
+	}
+
+	// Base clusters after exact key + EAN.
+	type Base = {
+		members: number[];
+		stores: Set<number>;
+		key: string;
+		tokens: string[];
+		cls: string;
+		unit: string | null;
+	};
+	const baseOf = new Map<number, Base>();
+	for (let i = 0; i < n; i++) {
+		const r = uf.find(i);
+		let b = baseOf.get(r);
+		if (!b) {
+			b = { members: [], stores: new Set(), key: '', tokens: [], cls: '', unit: null };
+			baseOf.set(r, b);
+		}
+		b.members.push(i);
+		b.stores.add(items[i].storeId);
+	}
+	const bases = [...baseOf.values()];
+	for (const b of bases) {
+		b.key = mode(b.members.map((i) => items[i].key));
+		const rep = items[b.members.find((i) => items[i].key === b.key)!];
+		b.tokens = rep.tokens;
+		b.cls = rep.cls;
+		b.unit = rep.unit;
+	}
+
+	// Fuzzy pass between base clusters with the same brand class + unit.
+	const df = new Map<string, number>();
+	for (const b of bases) for (const t of b.tokens) df.set(t, (df.get(t) ?? 0) + 1);
+	const postings = new Map<string, number[]>();
+	bases.forEach((b, bi) => {
+		for (const t of b.tokens) {
+			const k = `${b.cls}|${b.unit}|${t}`;
+			const list = postings.get(k);
+			if (list) list.push(bi);
+			else postings.set(k, [bi]);
+		}
+	});
+	const pairs: { a: number; b: number; score: number }[] = [];
+	bases.forEach((b, bi) => {
+		if (b.tokens.length < 2) return;
+		const rare = [...b.tokens]
+			.sort((x, y) => df.get(x)! - df.get(y)! || (x < y ? -1 : 1))
+			.slice(0, 2);
+		const seen = new Set<number>();
+		for (const t of rare) {
+			for (const ci of postings.get(`${b.cls}|${b.unit}|${t}`) ?? []) {
+				if (ci <= bi || seen.has(ci)) continue;
+				seen.add(ci);
+				const c = bases[ci];
+				if (c.tokens.length < 2) continue;
+				if ([...b.stores].some((s) => c.stores.has(s))) continue;
+				const score = jaccard(b.tokens, c.tokens);
+				if (score >= FUZZY_THRESHOLD) pairs.push({ a: bi, b: ci, score });
+			}
+		}
+	});
+	pairs.sort((x, y) => y.score - x.score || x.a - y.a || x.b - y.b);
+	const buf = new UnionFind(bases.length);
+	const storesOf = bases.map((b) => new Set(b.stores));
+	const fuzzyScore = new Map<number, number>(); // base index → score it was attached with
+	for (const p of pairs) {
+		const ra = buf.find(p.a);
+		const rb = buf.find(p.b);
+		if (ra === rb) continue;
+		if ([...storesOf[ra]].some((s) => storesOf[rb].has(s))) continue;
+		const root = buf.union(ra, rb);
+		const other = root === ra ? rb : ra;
+		for (const s of storesOf[other]) storesOf[root].add(s);
+		// The smaller side is the one that was "attached".
+		const attached = bases[p.a].members.length >= bases[p.b].members.length ? p.b : p.a;
+		if (!fuzzyScore.has(attached)) fuzzyScore.set(attached, p.score);
+	}
+
+	const finals = new Map<number, number[]>();
+	bases.forEach((_, bi) => {
+		const r = buf.find(bi);
+		const list = finals.get(r);
+		if (list) list.push(bi);
+		else finals.set(r, [bi]);
+	});
+	const out: ClusterResult[] = [];
+	for (const parts of finals.values()) {
+		parts.sort(
+			(x, y) =>
+				bases[y].members.length - bases[x].members.length || (bases[x].key < bases[y].key ? -1 : 1)
+		);
+		const ids: number[] = [];
+		const method: ClusterResult['method'] = new Map();
+		const stores = new Set<number>();
+		for (const bi of parts) for (const i of bases[bi].members) stores.add(items[i].storeId);
+		for (const bi of parts) {
+			const fs = fuzzyScore.get(bi);
+			for (const i of bases[bi].members) {
+				ids.push(items[i].id);
+				method.set(
+					items[i].id,
+					stores.size < 2
+						? { method: 'single', confidence: null }
+						: fs !== undefined
+							? { method: 'fuzzy', confidence: Math.round(fs * 0.9 * 100) / 100 }
+							: eanLinked.has(i)
+								? { method: 'ean', confidence: 1 }
+								: { method: 'key', confidence: 0.9 }
+				);
+			}
+		}
+		out.push({ key: bases[parts[0]].key, itemIds: ids, method });
+	}
+	return out;
+}

src/lib/server/groceries/normalize.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, it } from 'vitest';
+import {
+	brandClass,
+	displayName,
+	matchKey,
+	nameTokens,
+	normaliseGtin,
+	normaliseUnit,
+	parseAhUnitPrice,
+	parseSize
+} from './normalize';
+import { clusterItems, type MatchItem } from './match';
+
+describe('parseSize', () => {
+	it.each([
+		['500 g', { amount: 0.5, unit: 'kg' }],
+		['1 l', { amount: 1, unit: 'l' }],
+		['0,5 l', { amount: 0.5, unit: 'l' }],
+		['6 x 330 ml', { amount: 1.98, unit: 'l' }],
+		['6x1,5L', { amount: 9, unit: 'l' }],
+		['4 stuks', { amount: 4, unit: 'piece' }],
+		['per stuk', { amount: 1, unit: 'piece' }],
+		['per bosje', { amount: 1, unit: 'piece' }],
+		['per kilo', { amount: 1, unit: 'kg' }],
+		['ca. 600 g', { amount: 0.6, unit: 'kg' }],
+		['300 GRAM', { amount: 0.3, unit: 'kg' }],
+		['10 stuks 360 g', { amount: 0.36, unit: 'kg' }],
+		['Jumbo Halfvolle Melk 1,5L', { amount: 1.5, unit: 'l' }],
+		['Jumbo Dole Bananen 5 Stuks', { amount: 5, unit: 'piece' }]
+	])('%s', (input, expected) => {
+		const r = parseSize(input)!;
+		expect(r.unit).toBe(expected.unit);
+		expect(r.amount).toBeCloseTo(expected.amount, 6);
+	});
+	it('returns null when unclear', () => {
+		expect(parseSize('Jumbo Bloemkool')).toBeNull();
+		expect(parseSize(null)).toBeNull();
+	});
+});
+
+describe('unit prices', () => {
+	it('parses AH unit price text', () => {
+		expect(parseAhUnitPrice('normale prijs per kg €12.98')).toEqual({ cents: 1298, unit: 'kg' });
+		expect(parseAhUnitPrice('prijs per liter €1.49')).toEqual({ cents: 149, unit: 'l' });
+		expect(parseAhUnitPrice(null)).toBeNull();
+	});
+	it('uses the parsed size and computes promo unit price', () => {
+		expect(normaliseUnit({ sizeText: '500 g', priceCents: 125, promoPriceCents: 100 })).toEqual({
+			unit: 'kg',
+			unitAmount: 0.5,
+			unitSource: 'size',
+			unitPriceCents: 250,
+			promoUnitPriceCents: 200
+		});
+	});
+	it("derives the amount from the store's unit price when the size is unknown", () => {
+		// Jumbo bases price per unit on the promo price.
+		const n = normaliseUnit({
+			title: 'Jumbo Bloemkool',
+			priceCents: 169,
+			promoPriceCents: 99,
+			storeUnitPrice: { cents: 99, unit: 'piece' },
+			storeUnitBasisCents: 99
+		});
+		expect(n).toMatchObject({
+			unit: 'piece',
+			unitAmount: 1,
+			unitSource: 'store',
+			unitPriceCents: 169
+		});
+	});
+	it('prefers the store unit when the parsed unit disagrees', () => {
+		const n = normaliseUnit({
+			sizeText: '3 stuks',
+			priceCents: 300,
+			storeUnitPrice: { cents: 600, unit: 'kg' }
+		});
+		expect(n).toMatchObject({ unit: 'kg', unitAmount: 0.5, unitPriceCents: 600 });
+	});
+});
+
+describe('names and brands', () => {
+	it('maps store brands to house classes', () => {
+		expect(brandClass('AH')).toBe('house');
+		expect(brandClass('Jumbo')).toBe('house');
+		expect(brandClass('PLUS')).toBe('house');
+		expect(brandClass('AH Biologisch')).toBe('house-bio');
+		expect(brandClass(null)).toBe('house');
+		expect(brandClass('Calvé')).toBe('calve');
+	});
+	it('tokenises without brand, size and packaging', () => {
+		expect(nameTokens('AH Halfvolle melk', 'AH')).toEqual(['halfvolle', 'melk']);
+		expect(nameTokens('Jumbo Halfvolle Melk Voordeelverpakking 1,5L', 'Jumbo')).toEqual([
+			'halfvolle',
+			'melk'
+		]);
+		expect(nameTokens('Calvé Pindakaas 350g', 'Calvé')).toEqual(['pindakaa']);
+	});
+	it('makes display names', () => {
+		expect(displayName('AH Halfvolle melk', 'AH')).toBe('Halfvolle melk');
+		expect(displayName('Jumbo Pitloze Druiven Wit 500 g', 'Jumbo')).toBe('Pitloze Druiven Wit');
+	});
+	it('normalises GTINs', () => {
+		expect(normaliseGtin('08719200252127')).toBe('8719200252127');
+		expect(normaliseGtin('8719200252127')).toBe('8719200252127');
+		expect(normaliseGtin('12')).toBeNull();
+	});
+});
+
+describe('clusterItems', () => {
+	const item = (
+		id: number,
+		storeId: number,
+		name: string,
+		brand: string,
+		unit = 'l',
+		gtin: string | null = null
+	): MatchItem => {
+		const cls = brandClass(brand);
+		const tokens = nameTokens(name, brand);
+		return { id, storeId, gtin, key: matchKey(cls, unit, tokens), cls, unit, tokens };
+	};
+	const groupOf = (r: ReturnType<typeof clusterItems>, id: number) =>
+		r.find((c) => c.itemIds.includes(id))!;
+
+	it('groups house brands and sizes across stores, keeps A-brands apart', () => {
+		const r = clusterItems([
+			item(1, 1, 'AH Halfvolle melk', 'AH'),
+			item(2, 1, 'AH Halfvolle melk 1,5 l', 'AH'),
+			item(3, 2, 'Jumbo Halfvolle Melk 1L', 'Jumbo'),
+			item(4, 2, 'Campina Halfvolle melk', 'Campina'),
+			item(5, 1, 'AH Halfvolle melk lactosevrij', 'AH')
+		]);
+		expect(groupOf(r, 1).itemIds.sort()).toEqual([1, 2, 3]);
+		expect(groupOf(r, 3).method.get(3)).toEqual({ method: 'key', confidence: 0.9 });
+		expect(groupOf(r, 4).itemIds).toEqual([4]);
+		expect(groupOf(r, 5).itemIds).toEqual([5]);
+	});
+	it('links by EAN and fuzzy names across stores', () => {
+		const r = clusterItems([
+			item(1, 1, 'Calvé Pindakaas', 'Calvé', 'kg', '8722700055525'),
+			item(2, 2, 'Calvé Pindakaas Original', 'Calve', 'kg', '8722700055525'),
+			item(3, 1, 'AH Spaghetti volkoren pasta', 'AH', 'kg'),
+			item(4, 2, 'Jumbo Volkoren Spaghetti Pasta Italiaans', 'Jumbo', 'kg')
+		]);
+		expect(groupOf(r, 1).itemIds.sort()).toEqual([1, 2]);
+		expect(groupOf(r, 1).method.get(2)).toEqual({ method: 'ean', confidence: 1 });
+		expect(groupOf(r, 3).itemIds.sort()).toEqual([3, 4]);
+		expect(groupOf(r, 3).method.get(4)?.method).toBe('fuzzy');
+	});
+});

src/lib/server/groceries/normalize.ts

@@ -0,0 +1,361 @@
+// Pure normalisation helpers for the grocery importer and matcher (no DB, no network).
+import type { GroceryUnit } from '../db/schema/groceries';
+
+export type Size = { amount: number; unit: GroceryUnit };
+
+const MASS: Record<string, number> = {
+	kg: 1,
+	kilo: 1,
+	kilogram: 1,
+	g: 0.001,
+	gr: 0.001,
+	gram: 0.001,
+	mg: 0.000001
+};
+const VOLUME: Record<string, number> = {
+	l: 1,
+	ltr: 1,
+	liter: 1,
+	litre: 1,
+	dl: 0.1,
+	cl: 0.01,
+	ml: 0.001
+};
+const COUNT = new Set([
+	'st',
+	'stk',
+	'stuk',
+	'stuks',
+	'stuk(s)',
+	'pcs',
+	'piece',
+	'pieces',
+	'rol',
+	'rollen',
+	'zakjes',
+	'zakje',
+	'tabletten',
+	'tablet',
+	'capsules',
+	'capsule',
+	'cups',
+	'pads',
+	'sachets',
+	'doekjes',
+	'luiers',
+	'wasbeurten',
+	'eieren',
+	'plakken',
+	'porties',
+	'repen',
+	'blikjes',
+	'flesjes',
+	'zakken',
+	'bollen',
+	'tabs',
+	'vellen',
+	'servetten',
+	'theezakjes',
+	'paar'
+]);
+const UNIT_RE =
+	'kilogram|kilo|kg|gram|gr|mg|g|liter|litre|ltr|ml|cl|dl|l|stuk\\(s\\)|stuks|stuk|stk|st|pcs|pieces|piece|rollen|rol|zakjes|zakje|tabletten|tablet|capsules|capsule|cups|pads|sachets|doekjes|luiers|wasbeurten|eieren|plakken|porties|repen|blikjes|flesjes|zakken|bollen|tabs|vellen|servetten|theezakjes|paar';
+const QTY = new RegExp(`(?:(\\d+)\\s*[x×]\\s*)?(\\d+(?:[.,]\\d+)?)\\s*(${UNIT_RE})(?![a-z])`, 'gi');
+
+function toSize(mult: number, n: number, unit: string): Size | null {
+	const u = unit.toLowerCase();
+	const total = mult * n;
+	if (!(total > 0)) return null;
+	if (u in MASS) return { amount: total * MASS[u], unit: 'kg' };
+	if (u in VOLUME) return { amount: total * VOLUME[u], unit: 'l' };
+	if (COUNT.has(u)) return { amount: total, unit: 'piece' };
+	return null;
+}
+
+/**
+ * Pack size → base unit, e.g. "500 g" → 0.5 kg, "6 x 330 ml" → 1.98 l, "4 stuks" → 4 piece,
+ * "per stuk" → 1 piece, "per kilo" → 1 kg. Weight/volume beats a count ("6 stuks 360 g" → kg).
+ * Works on titles too ("Jumbo Halfvolle Melk 1,5L"). Null when nothing usable.
+ */
+export function parseSize(text: string | null | undefined): Size | null {
+	if (!text) return null;
+	const s = text
+		.toLowerCase()
+		.replace(/ca\.?\s*/g, '')
+		.replace(/\s+/g, ' ');
+	let best: Size | null = null;
+	for (const m of s.matchAll(QTY)) {
+		const size = toSize(m[1] ? Number(m[1]) : 1, Number(m[2].replace(',', '.')), m[3]);
+		if (!size) continue;
+		if (!best || (best.unit === 'piece' && size.unit !== 'piece')) best = size;
+		if (best.unit !== 'piece') break;
+	}
+	if (best) return best;
+	if (/\bper\s*(kilo|kg)\b/.test(s)) return { amount: 1, unit: 'kg' };
+	if (/\bper\s*(liter|l)\b/.test(s)) return { amount: 1, unit: 'l' };
+	if (
+		/\bper\s*(stuk|bos|bosje|bundel|krop|doos|pak|tros|net|schaal|bak|pot|zak|paar|set)\b/.test(s)
+	)
+		return { amount: 1, unit: 'piece' };
+	if (/^\s*(\d+)\s*(st|stuks)?\s*$/.test(s))
+		return { amount: Number(s.trim().split(/\s/)[0]), unit: 'piece' };
+	return null;
+}
+
+/** Store unit labels ("kg", "l", "pieces", "stuk", "liter") → our unit. */
+export function storeUnit(label: string | null | undefined): GroceryUnit | null {
+	const u = (label ?? '').toLowerCase().trim();
+	if (['kg', 'kilo', 'kilogram'].includes(u)) return 'kg';
+	if (['l', 'liter', 'litre', 'ltr'].includes(u)) return 'l';
+	if (['piece', 'pieces', 'stuk', 'stuks', 'st', 'pce', 'pc'].includes(u)) return 'piece';
+	return null;
+}
+
+/** AH "prijs per kg €12.98" / "normale prijs per liter €1.49" → { cents, unit }. */
+export function parseAhUnitPrice(
+	desc: string | null | undefined
+): { cents: number; unit: GroceryUnit } | null {
+	if (!desc) return null;
+	const m = desc.toLowerCase().match(/per\s+(kg|kilo|liter|l|stuk|st)\s*€?\s*(\d+(?:[.,]\d+)?)/);
+	if (!m) return null;
+	const unit = storeUnit(m[1]);
+	return unit ? { cents: Math.round(Number(m[2].replace(',', '.')) * 100), unit } : null;
+}
+
+export type Normalised = {
+	unit: GroceryUnit | null;
+	unitAmount: number | null;
+	unitSource: 'size' | 'store' | null;
+	unitPriceCents: number | null;
+	promoUnitPriceCents: number | null;
+};
+
+/**
+ * Pick the unit and pack amount: the parsed size when it agrees with the store's unit price
+ * (or the store gives none); else derive the amount from the store's unit price.
+ */
+export function normaliseUnit(opts: {
+	sizeText?: string | null;
+	title?: string | null;
+	priceCents: number | null;
+	promoPriceCents?: number | null;
+	storeUnitPrice?: { cents: number; unit: GroceryUnit } | null;
+	/** Price the store's unit price was computed from (Jumbo uses the promo price). */
+	storeUnitBasisCents?: number | null;
+}): Normalised {
+	const { priceCents, storeUnitPrice } = opts;
+	let size = parseSize(opts.sizeText) ?? parseSize(opts.title);
+	let source: Normalised['unitSource'] = size ? 'size' : null;
+	const basis = opts.storeUnitBasisCents ?? priceCents;
+	if (storeUnitPrice && storeUnitPrice.cents > 0 && basis) {
+		if (!size || size.unit !== storeUnitPrice.unit) {
+			const derived = basis / storeUnitPrice.cents;
+			const amount =
+				storeUnitPrice.unit === 'piece'
+					? Math.max(1, Math.round(derived))
+					: Math.round(derived * 1000) / 1000;
+			size = amount > 0 ? { amount, unit: storeUnitPrice.unit } : size;
+			source = 'store';
+		}
+	}
+	if (!size || !(size.amount > 0)) {
+		return {
+			unit: null,
+			unitAmount: null,
+			unitSource: null,
+			unitPriceCents: null,
+			promoUnitPriceCents: null
+		};
+	}
+	const per = (c: number | null | undefined) =>
+		c != null && c > 0 ? Math.round(c / size.amount) : null;
+	return {
+		unit: size.unit,
+		unitAmount: Math.round(size.amount * 10000) / 10000,
+		unitSource: source,
+		unitPriceCents: per(priceCents),
+		promoUnitPriceCents: per(opts.promoPriceCents)
+	};
+}
+
+// ---- names, brands, keys -------------------------------------------------------------------
+
+export const fold = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toLowerCase();
+
+/** Store own brands, mapped to a shared class so AH/Jumbo/Plus equivalents group together. */
+const HOUSE: Record<string, string> = {
+	ah: 'house',
+	'albert heijn': 'house',
+	jumbo: 'house',
+	plus: 'house',
+	'ah biologisch': 'house-bio',
+	'jumbo biologisch': 'house-bio',
+	'plus biologisch': 'house-bio',
+	'biologisch plus': 'house-bio',
+	'ah basic': 'house-basic',
+	'jumbo basic': 'house-basic',
+	'plus basic': 'house-basic',
+	"g'woon": 'house-basic',
+	'ah excellent': 'house-premium',
+	'jumbo excellent': 'house-premium',
+	'plus excellent': 'house-premium',
+	'jumbo lekker puur': 'house-premium',
+	'ah terra': 'house-veg',
+	'jumbo veggie': 'house-veg',
+	'ah kids': 'house-kids',
+	'ah puur&eerlijk': 'house-bio',
+	'ah puur & eerlijk': 'house-bio'
+};
+const HOUSE_LABEL: Record<string, string> = {
+	house: 'Huismerk',
+	'house-bio': 'Huismerk biologisch',
+	'house-basic': 'Huismerk basic',
+	'house-premium': 'Huismerk premium',
+	'house-veg': 'Huismerk plantaardig',
+	'house-kids': 'Huismerk kids'
+};
+const STORE_WORDS = ['albert heijn', 'ah', 'jumbo', 'plus'];
+
+export function brandClass(brand: string | null | undefined): string {
+	const b = fold(brand ?? '')
+		.trim()
+		.replace(/\s+/g, ' ');
+	if (!b) return 'house'; // loose produce without a brand
+	return HOUSE[b] ?? b.replace(/[^a-z0-9]+/g, '');
+}
+export const isHouse = (cls: string) => cls.startsWith('house');
+export const brandLabel = (cls: string, brand: string | null | undefined) =>
+	HOUSE_LABEL[cls] ?? brand?.trim() ?? null;
+
+const STOP = new Set([
+	'de',
+	'het',
+	'een',
+	'van',
+	'met',
+	'en',
+	'in',
+	'voor',
+	'per',
+	'x',
+	'ca',
+	'op',
+	'of',
+	'a',
+	'la',
+	'le',
+	'stuks',
+	'stuk',
+	'st',
+	'pak',
+	'fles',
+	'blik',
+	'pot',
+	'zak',
+	'doos',
+	'tray',
+	'bak',
+	'beker',
+	'kuipje',
+	'voordeelverpakking',
+	'voordeelpak',
+	'voordeel',
+	'multipack',
+	'grootverpakking',
+	'xl',
+	'xxl',
+	'navulling',
+	'family',
+	'familiepak',
+	'mega',
+	'pack',
+	'literpak',
+	'petfles',
+	'tube',
+	'tetra',
+	'brik'
+]);
+const SIZE_IN_NAME = new RegExp(
+	`(?:\\d+\\s*[x×]\\s*)?\\d+(?:[.,]\\d+)?\\s*(?:${UNIT_RE})(?![a-z])`,
+	'gi'
+);
+
+const stem = (t: string) =>
+	t.length > 5 && t.endsWith('en')
+		? t.slice(0, -2)
+		: t.length > 4 && t.endsWith('s')
+			? t.slice(0, -1)
+			: t;
+
+/** Meaningful name tokens: no brand, store name, size, packaging or stop words. */
+export function nameTokens(title: string, brand?: string | null): string[] {
+	let s = fold(title).replace(/&/g, ' en ').replace(SIZE_IN_NAME, ' ');
+	const drop = new Set<string>();
+	for (const w of [brand ?? '', ...STORE_WORDS])
+		for (const t of fold(w).split(/[^a-z0-9]+/)) if (t) drop.add(t);
+	s = s.replace(/['’]/g, '').replace(/[^a-z0-9%+]+/g, ' ');
+	const out: string[] = [];
+	for (const raw of s.split(' ')) {
+		if (!raw || drop.has(raw) || STOP.has(raw)) continue;
+		if (/^\d+$/.test(raw) && raw.length > 2) continue; // stray numbers (article codes)
+		const t = stem(raw);
+		if (!out.includes(t)) out.push(t);
+	}
+	return out;
+}
+
+/** Display name without brand/store prefix and size: "AH Halfvolle melk 1 L" → "Halfvolle melk". */
+export function displayName(title: string, brand?: string | null): string {
+	let s = title.replace(SIZE_IN_NAME, ' ').replace(/\s+-\s*$/, '');
+	for (const w of [brand ?? '', ...STORE_WORDS]
+		.filter(Boolean)
+		.sort((a, b) => b.length - a.length)) {
+		const re = new RegExp(`^\\s*${w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b\\s*`, 'i');
+		s = s.replace(re, '');
+	}
+	s = s
+		.replace(/\s+/g, ' ')
+		.replace(/[\s,-]+$/, '')
+		.trim();
+	return s ? s[0].toUpperCase() + s.slice(1) : title.trim();
+}
+
+export function matchKey(cls: string, unit: string | null, tokens: string[]): string {
+	return `${cls}|${unit ?? '?'}|${[...tokens].sort().join(' ')}`;
+}
+
+/** EAN-13 / GTIN-14 → canonical 13-digit EAN (leading zero of GTIN-14 dropped). */
+export function normaliseGtin(v: string | null | undefined): string | null {
+	const d = (v ?? '').replace(/\D/g, '');
+	if (d.length < 8) return null;
+	const t = d.replace(/^0+(?=\d{13}$)/, '');
+	return t.length > 14 ? null : t.padStart(13, '0');
+}
+
+// ---- categories -------------------------------------------------------------------------------
+
+/** Shared category names; each store's top categories map onto these. */
+export const CATEGORIES = [
+	'Groente, fruit & aardappelen',
+	'Maaltijden & gemak',
+	'Vlees, vis & vega',
+	'Vleeswaren, kaas & tapas',
+	'Zuivel & eieren',
+	'Brood & bakkerij',
+	'Ontbijt & beleg',
+	'Pasta, rijst & wereldkeuken',
+	'Soepen, sauzen & conserven',
+	'Snoep, koek & chips',
+	'Diepvries',
+	'Koffie & thee',
+	'Frisdrank & sappen',
+	'Bier & wijn',
+	'Drogisterij',
+	'Huishouden & dieren',
+	'Baby & kind',
+	'Overig'
+] as const;

src/lib/server/groceries/sources.ts

@@ -0,0 +1,352 @@
+// Store catalogue sources. Each returns the full assortment as RawItem[].
+//  - Albert Heijn: the anonymous mobile-app API (api.ah.nl), crawled per top category.
+//  - Jumbo: the public GraphQL search endpoint used by jumbo.com, crawled per top category.
+//  - Plus: not available from the server (plus.nl blocks datacenter IPs via Incapsula).
+import type { GroceryUnit } from '../db/schema/groceries';
+import { HttpError, PoliteFetcher, USER_AGENT } from './http';
+import { normaliseGtin, parseAhUnitPrice, storeUnit } from './normalize';
+
+export type RawItem = {
+	storeProductId: string;
+	name: string;
+	brand: string | null;
+	sizeText: string | null;
+	priceCents: number | null;
+	promoPriceCents: number | null;
+	promoLabel: string | null;
+	storeUnitPrice: { cents: number; unit: GroceryUnit } | null;
+	storeUnitBasisCents: number | null;
+	category: string;
+	categoryPath: string | null;
+	gtin: string | null;
+	imageUrl: string | null;
+	url: string | null;
+	available: boolean;
+};
+
+export type Source = {
+	slug: string;
+	fetchAll(f: PoliteFetcher, log: (m: string) => void): Promise<RawItem[]>;
+};
+
+const cents = (euros: unknown) =>
+	typeof euros === 'number' && Number.isFinite(euros) ? Math.round(euros * 100) : null;
+
+// ---- Albert Heijn ----------------------------------------------------------------------------
+
+const AH_API = 'https://api.ah.nl';
+const AH_HEADERS = {
+	'User-Agent': `Appie/8.22.3 (${USER_AGENT})`,
+	'x-application': 'AHWEBSHOP',
+	'Content-Type': 'application/json',
+	Accept: 'application/json'
+};
+/** AH top category id → shared category. Unknown ids fall back to 'Overig'. */
+const AH_CATEGORIES: Record<number, string> = {
+	6401: 'Groente, fruit & aardappelen',
+	20885: 'Groente, fruit & aardappelen',
+	1301: 'Maaltijden & gemak',
+	9344: 'Vlees, vis & vega',
+	1651: 'Vlees, vis & vega',
+	20128: 'Vlees, vis & vega',
+	5481: 'Vleeswaren, kaas & tapas',
+	1192: 'Vleeswaren, kaas & tapas',
+	1730: 'Zuivel & eieren',
+	1355: 'Brood & bakkerij',
+	20824: 'Snoep, koek & chips',
+	1796: 'Pasta, rijst & wereldkeuken',
+	6409: 'Soepen, sauzen & conserven',
+	20129: 'Snoep, koek & chips',
+	6405: 'Ontbijt & beleg',
+	2457: 'Snoep, koek & chips',
+	5881: 'Diepvries',
+	1043: 'Koffie & thee',
+	20130: 'Frisdrank & sappen',
+	6406: 'Bier & wijn',
+	1045: 'Drogisterij',
+	11717: 'Drogisterij',
+	1165: 'Huishouden & dieren',
+	18521: 'Baby & kind',
+	18519: 'Huishouden & dieren',
+	1057: 'Overig',
+	4246: 'Overig', // Glutenvrij: crawled last so products keep their real category
+	44025: 'Overig' // Bloemenshop
+};
+
+type AhProduct = {
+	webshopId: number;
+	title: string;
+	brand?: string | null;
+	salesUnitSize?: string | null;
+	unitPriceDescription?: string | null;
+	images?: { width: number; url: string }[];
+	priceBeforeBonus?: number | null;
+	currentPrice?: number | null;
+	isBonus?: boolean;
+	bonusMechanism?: string | null;
+	mainCategory?: string | null;
+	subCategory?: string | null;
+	orderAvailabilityStatus?: string | null;
+	availableOnline?: boolean;
+};
+
+export class AhClient {
+	private token: string | null = null;
+	constructor(private f: PoliteFetcher) {}
+
+	private async auth() {
+		const r = await this.f.json<{ access_token: string }>(
+			`${AH_API}/mobile-auth/v1/auth/token/anonymous`,
+			{ method: 'POST', headers: AH_HEADERS, body: JSON.stringify({ clientId: 'appie' }) }
+		);
+		this.token = r.access_token;
+	}
+
+	async get<T>(path: string, cacheKey?: string): Promise<T> {
+		if (!this.token) await this.auth();
+		const run = () =>
+			this.f.json<T>(
+				`${AH_API}${path}`,
+				{ headers: { ...AH_HEADERS, Authorization: `Bearer ${this.token}` } },
+				cacheKey
+			);
+		try {
+			return await run();
+		} catch (e) {
+			if (e instanceof HttpError && e.status === 401) {
+				await this.auth();
+				return run();
+			}
+			throw e;
+		}
+	}
+
+	/** GTIN from the product detail endpoint (search results have none). */
+	async gtin(webshopId: string): Promise<string | null> {
+		const d = await this.get<unknown>(
+			`/mobile-services/product/detail/v4/fir/${encodeURIComponent(webshopId)}`
+		);
+		return normaliseGtin(findKey(d, 'gtin'));
+	}
+}
+
+function findKey(o: unknown, key: string, depth = 0): string | null {
+	if (!o || typeof o !== 'object' || depth > 6) return null;
+	for (const [k, v] of Object.entries(o)) {
+		if (k === key && (typeof v === 'string' || typeof v === 'number')) return String(v);
+		const r = findKey(v, key, depth + 1);
+		if (r) return r;
+	}
+	return null;
+}
+
+function ahItem(p: AhProduct, category: string): RawItem {
+	const before = cents(p.priceBeforeBonus);
+	const now = cents(p.currentPrice);
+	const promoPrice = p.isBonus && now != null && before != null && now < before ? now : null;
+	const img =
+		p.images?.find((i) => i.width === 200) ??
+		p.images?.find((i) => i.width === 400) ??
+		p.images?.[0];
+	return {
+		storeProductId: String(p.webshopId),
+		name: p.title.trim(),
+		brand: p.brand?.trim() || null,
+		sizeText: p.salesUnitSize?.trim() || null,
+		priceCents: before ?? now,
+		promoPriceCents: promoPrice,
+		promoLabel: p.bonusMechanism?.trim() || null,
+		storeUnitPrice: parseAhUnitPrice(p.unitPriceDescription),
+		storeUnitBasisCents: before ?? now, // "normale prijs per kg" is the pre-bonus price
+		category,
+		categoryPath: [p.mainCategory, p.subCategory].filter(Boolean).join(' > ') || null,
+		gtin: null,
+		imageUrl: img?.url ?? null,
+		url: `https://www.ah.nl/producten/product/wi${p.webshopId}`,
+		available: p.orderAvailabilityStatus ? p.orderAvailabilityStatus === 'IN_ASSORTMENT' : true
+	};
+}
+
+/** AH search only pages through the first 3000 hits; bigger categories are crawled per child. */
+const AH_WINDOW = 3000;
+type AhPage = { products?: AhProduct[]; page?: { totalPages: number; totalElements: number } };
+
+async function crawlAhCategory(
+	ah: AhClient,
+	id: number,
+	category: string,
+	items: Map<string, RawItem>,
+	depth = 0
+): Promise<void> {
+	const url = (page: number) =>
+		`/mobile-services/product/search/v2?taxonomyId=${id}&size=1000&page=${page}&sortOn=RELEVANCE`;
+	const first = await ah.get<AhPage>(url(0), `cat-${id}-p0`);
+	const total = first.page?.totalElements ?? 0;
+	if (total > AH_WINDOW && depth < 3) {
+		const sub = await ah.get<{ children?: { id: number }[] }>(
+			`/mobile-services/v1/product-shelves/categories/${id}/sub-categories`,
+			`sub-${id}`
+		);
+		if (sub.children?.length) {
+			for (const c of sub.children) await crawlAhCategory(ah, c.id, category, items, depth + 1);
+			// Products only listed on the parent: whatever the first 3000 hits add.
+		}
+	}
+	const pages = Math.min(first.page?.totalPages ?? 1, Math.floor(AH_WINDOW / 1000));
+	for (let page = 0; page < pages; page++) {
+		const r = page === 0 ? first : await ah.get<AhPage>(url(page), `cat-${id}-p${page}`);
+		for (const p of r.products ?? []) {
+			if (!p.webshopId || !p.title) continue;
+			const key = String(p.webshopId);
+			if (!items.has(key)) items.set(key, ahItem(p, category));
+		}
+		if (!r.products?.length) break;
+	}
+}
+
+export const ahSource: Source = {
+	slug: 'ah',
+	async fetchAll(f, log) {
+		const ah = new AhClient(f);
+		const cats = await ah.get<{ id: number; name: string }[]>(
+			'/mobile-services/v1/product-shelves/categories',
+			'categories'
+		);
+		cats.sort((a, b) => Number(a.id === 4246) - Number(b.id === 4246));
+		const items = new Map<string, RawItem>();
+		for (const c of cats) {
+			const before = items.size;
+			await crawlAhCategory(ah, c.id, AH_CATEGORIES[c.id] ?? 'Overig', items);
+			log(`  ah ${c.name}: +${items.size - before} (${items.size} total)`);
+		}
+		return [...items.values()];
+	}
+};
+
+// ---- Jumbo -----------------------------------------------------------------------------------
+
+const JUMBO_GQL = 'https://www.jumbo.com/api/graphql';
+const JUMBO_HEADERS = {
+	'content-type': 'application/json',
+	'user-agent': USER_AGENT,
+	'apollographql-client-name': 'JUMBO_WEB-search',
+	'apollographql-client-version': 'master-v29.4.0-web',
+	'x-source': 'JUMBO_WEB-search'
+};
+/** Jumbo top category friendlyUrl → shared category (themed/seasonal roots skipped: subsets). */
+const JUMBO_ROOTS: [string, string][] = [
+	['aardappelen,-groente-en-fruit', 'Groente, fruit & aardappelen'],
+	['verse-maaltijden-en-gemak', 'Maaltijden & gemak'],
+	['vlees,-vis-en-vega', 'Vlees, vis & vega'],
+	['vega-en-plantaardig', 'Vlees, vis & vega'],
+	['vleeswaren,-kaas-en-tapas', 'Vleeswaren, kaas & tapas'],
+	['zuivel,-boter-en-eieren', 'Zuivel & eieren'],
+	['brood-en-gebak', 'Brood & bakkerij'],
+	['ontbijt,-broodbeleg-en-bakproducten', 'Ontbijt & beleg'],
+	['wereldkeukens,-kruiden,-pasta-en-rijst', 'Pasta, rijst & wereldkeuken'],
+	['conserven,-soepen,-sauzen,-olien', 'Soepen, sauzen & conserven'],
+	['koek,-snoep,-chocolade-en-chips', 'Snoep, koek & chips'],
+	['tussendoortjes', 'Snoep, koek & chips'],
+	['diepvries', 'Diepvries'],
+	['koffie-en-thee', 'Koffie & thee'],
+	['frisdrank-en-sappen', 'Frisdrank & sappen'],
+	['bier-en-wijn', 'Bier & wijn'],
+	['drogisterij-en-gezondheid', 'Drogisterij'],
+	['huishouden-en-dieren', 'Huishouden & dieren'],
+	['baby-en-kind', 'Baby & kind'],
+	['nonfood-en-servicebalie', 'Overig'],
+	['bewuste-voeding', 'Overig']
+];
+const JUMBO_QUERY = `query SearchProducts($input: ProductSearchInput!) { searchProducts(input: $input) { count products { id ean title brand link image subtitle packSizeDisplay inAssortment availability { isAvailable } categories { name } promotions { tags { text } } price { price promoPrice pricePerUnit { price unit } } } } }`;
+const JUMBO_PAGE = 200;
+
+type JumboProduct = {
+	id: string;
+	ean?: string | null;
+	title: string;
+	brand?: string | null;
+	link?: string | null;
+	image?: string | null;
+	subtitle?: string | null;
+	packSizeDisplay?: string | null;
+	inAssortment?: boolean | null;
+	availability?: { isAvailable?: boolean | null } | null;
+	categories?: { name: string }[] | null;
+	promotions?: { tags?: { text: string }[] | null }[] | null;
+	price?: {
+		price?: number | null;
+		promoPrice?: number | null;
+		pricePerUnit?: { price?: number | null; unit?: string | null } | null;
+	} | null;
+};
+
+const JUMBO_NOT_PROMO = /gratis bezorging|^vanaf\b|bezorg/i;
+
+function jumboItem(p: JumboProduct, category: string): RawItem {
+	const price = p.price?.price ?? null;
+	const promo = p.price?.promoPrice ?? null;
+	const ppu = p.price?.pricePerUnit;
+	const unit = storeUnit(ppu?.unit);
+	const tags = (p.promotions ?? [])
+		.flatMap((x) => x.tags ?? [])
+		.map((t) => t.text.trim())
+		.filter((t) => t && !JUMBO_NOT_PROMO.test(t));
+	return {
+		storeProductId: p.id,
+		name: p.title.trim(),
+		brand: p.brand?.trim() || null,
+		sizeText: p.packSizeDisplay?.trim() || p.subtitle?.trim() || null,
+		priceCents: price,
+		promoPriceCents: promo != null && price != null && promo < price ? promo : null,
+		promoLabel: tags.length ? [...new Set(tags)].join(' · ') : null,
+		storeUnitPrice: ppu?.price && unit ? { cents: ppu.price, unit } : null,
+		storeUnitBasisCents: promo ?? price, // Jumbo's price per unit follows the promo price
+		category,
+		categoryPath: p.categories?.map((c) => c.name).join(' > ') || null,
+		gtin: normaliseGtin(p.ean),
+		imageUrl: p.image || null,
+		url: p.link ? new URL(p.link, 'https://www.jumbo.com').toString() : null,
+		available: (p.inAssortment ?? true) && (p.availability?.isAvailable ?? true)
+	};
+}
+
+export const jumboSource: Source = {
+	slug: 'jumbo',
+	async fetchAll(f, log) {
+		const items = new Map<string, RawItem>();
+		for (const [root, category] of JUMBO_ROOTS) {
+			const before = items.size;
+			for (let offSet = 0; ; offSet += JUMBO_PAGE) {
+				const r = await f.json<{
+					data?: { searchProducts?: { count: number; products: JumboProduct[] } };
+					errors?: { message: string }[];
+				}>(
+					JUMBO_GQL,
+					{
+						method: 'POST',
+						headers: JUMBO_HEADERS,
+						body: JSON.stringify({
+							operationName: 'SearchProducts',
+							variables: {
+								input: { searchType: 'category', friendlyUrl: root, offSet, limit: JUMBO_PAGE }
+							},
+							query: JUMBO_QUERY
+						})
+					},
+					`${root.replace(/[^a-z0-9-]+/g, '_')}-${offSet}`
+				);
+				if (r.errors?.length) throw new Error(`jumbo ${root}: ${r.errors[0].message}`);
+				const sp = r.data?.searchProducts;
+				for (const p of sp?.products ?? []) {
+					if (!p.id || !p.title) continue;
+					if (!items.has(p.id)) items.set(p.id, jumboItem(p, category));
+				}
+				if (!sp?.products?.length || offSet + JUMBO_PAGE >= sp.count) break;
+			}
+			log(`  jumbo ${root}: +${items.size - before} (${items.size} total)`);
+		}
+		return [...items.values()];
+	}
+};
+
+export const SOURCES: Record<string, Source> = { ah: ahSource, jumbo: jumboSource };

src/routes/trackers/groceries/+page.server.ts

@@ -1,45 +1,198 @@
 import { db } from '#lib/server/db/index.js';
-import { groceryPrices, groceryProducts, groceryStores } from '#lib/server/db/schema/index.js';
 import { requireOwner } from '#lib/server/owner.js';
-import { parseSize, priceInput, productInput, unitPriceCents } from '#lib/server/groceries.js';
+import { refreshGroupStats } from '#lib/server/groceries/import.js';
-import { and, asc, desc, eq, isNull, sql } from 'drizzle-orm';
+import { CATEGORIES, displayName, fold } from '#lib/server/groceries/normalize.js';
 import { fail } from '@sveltejs/kit';
+import type { Sql } from 'postgres';
 import type { Actions, PageServerLoad } from './$types';
 
+const sql = () => db.$client as unknown as Sql;
+const PAGE_SIZE = 30;
+
+export type GroupRow = {
+	id: number;
+	name: string;
+	brand: string | null;
+	category: string | null;
+	unit: string | null;
+	favorite: boolean;
+	manual: boolean;
+	itemCount: number;
+	storeCount: number;
+	minUnitPriceCents: number | null;
+	imageUrl: string | null;
+	familyKey: string | null;
+};
+export type ItemRow = {
+	id: number;
+	groupId: number;
+	storeId: number;
+	name: string;
+	brand: string | null;
+	sizeText: string | null;
+	unit: string | null;
+	unitAmount: number | null;
+	unitSource: string | null;
+	priceCents: number | null;
+	promoPriceCents: number | null;
+	promoLabel: string | null;
+	unitPriceCents: number | null;
+	promoUnitPriceCents: number | null;
+	gtin: string | null;
+	imageUrl: string | null;
+	url: string | null;
+	available: boolean;
+	matchMethod: string | null;
+	matchConfidence: number | null;
+	matchLocked: boolean;
+	categoryPath: string | null;
+	lastSeenAt: Date;
+};
+
+const ITEM_COLS = (s: Sql) => s`
+	i.id, i.group_id as "groupId", i.store_id as "storeId", i.name, i.brand, i.size_text as "sizeText",
+	i.unit, i.unit_amount::float8 as "unitAmount", i.unit_source as "unitSource", i.price_cents as "priceCents",
+	i.promo_price_cents as "promoPriceCents", i.promo_label as "promoLabel",
+	i.unit_price_cents as "unitPriceCents", i.promo_unit_price_cents as "promoUnitPriceCents", i.gtin,
+	i.image_url as "imageUrl", i.url, i.available, i.match_method as "matchMethod",
+	i.match_confidence as "matchConfidence", i.match_locked as "matchLocked",
+	i.category_path as "categoryPath", i.last_seen_at as "lastSeenAt"`;
+const GROUP_COLS = (s: Sql) => s`
+	g.id, g.name, g.brand, g.category, g.unit, g.favorite, g.manual, g.item_count as "itemCount",
+	g.store_count as "storeCount", g.min_unit_price_cents as "minUnitPriceCents",
+	g.image_url as "imageUrl", g.family_key as "familyKey"`;
+
 export const load: PageServerLoad = async (event) => {
 	requireOwner(event);
-	const [stores, products, prices] = await Promise.all([
+	const s = sql();
-		db
+	const p = event.url.searchParams;
-			.select()
+	const q = fold(p.get('q')?.trim() ?? '').slice(0, 100);
-			.from(groceryStores)
+	const words = q.split(/\s+/).filter(Boolean).slice(0, 6);
-			.where(isNull(groceryStores.archivedAt))
+	const storeSlug = p.get('store') ?? '';
-			.orderBy(asc(groceryStores.sortOrder), asc(groceryStores.name)),
+	const category = p.get('cat') ?? '';
-		db
+	const favorites = p.get('fav') === '1';
-			.select()
+	const promo = p.get('promo') === '1';
-			.from(groceryProducts)
+	const multi = p.get('multi') === '1';
-			.where(isNull(groceryProducts.archivedAt))
+	const page = Math.max(1, Math.min(10_000, Number(p.get('page')) || 1));
-			.orderBy(desc(groceryProducts.favorite), asc(sql`lower(${groceryProducts.name})`)),
+	const detailId = Number(p.get('g')) || null;
-		db
+
-			.select({
+	const stores = await s<
-				id: groceryPrices.id,
+		{ id: number; slug: string; name: string; color: string | null; items: number }[]
-				productId: groceryPrices.productId,
+	>`
-				storeId: groceryPrices.storeId,
+		select st.id, st.slug, st.name, st.color,
-				priceCents: groceryPrices.priceCents,
+			(select count(*)::int from groceries.items i where i.store_id = st.id and i.available) as items
-				unitPriceCents: groceryPrices.unitPriceCents,
+		from groceries.stores st where st.archived_at is null order by st.sort_order, st.name`;
-				promo: groceryPrices.promo,
+	const store = stores.find((x) => x.slug === storeSlug) ?? null;
-				observedAt: groceryPrices.observedAt,
+
-				source: groceryPrices.source,
+	const conds = [s`g.merged_into_id is null`, s`(g.item_count > 0 or g.favorite)`];
-				url: groceryPrices.url,
+	for (const w of words)
-				notes: groceryPrices.notes
+		conds.push(s`g.search_text like ${'%' + w.replace(/[%_\\]/g, '\\$&') + '%'}`);
-			})
+	if (store) conds.push(s`g.store_ids @> array[${store.id}]::int[]`);
-			.from(groceryPrices)
+	if (category) conds.push(s`g.category = ${category}`);
-			.orderBy(desc(groceryPrices.observedAt), desc(groceryPrices.id))
+	if (favorites) conds.push(s`g.favorite`);
+	if (promo) conds.push(s`g.promo_count > 0`);
+	if (multi) conds.push(s`g.store_count >= 2`);
+	const where = conds.reduce((a, c) => s`${a} and ${c}`);
+	const order = q
+		? s`g.favorite desc, (lower(g.name) = ${q}) desc, word_similarity(${q}, lower(g.name)) desc, g.store_count desc, g.item_count desc, g.name`
+		: s`g.favorite desc, g.store_count desc, g.item_count desc, g.name`;
+
+	const [groups, [{ total }]] = await Promise.all([
+		s<GroupRow[]>`
+			select ${GROUP_COLS(s)} from groceries.groups g where ${where}
+			order by ${order}, g.id limit ${PAGE_SIZE} offset ${(page - 1) * PAGE_SIZE}`,
+		s<{ total: number }[]>`select count(*)::int as total from groceries.groups g where ${where}`
 	]);
+	const ids = groups.map((g) => g.id);
+	const items = ids.length
+		? await s<ItemRow[]>`
+			select ${ITEM_COLS(s)} from groceries.items i
+			where i.group_id = any(${ids}::int[]) and i.available
+			order by coalesce(i.promo_unit_price_cents, i.unit_price_cents) nulls last, i.id`
+		: [];
+
+	const [stats] = await s<
+		{
+			groups: number;
+			crossStore: number;
+			items: number;
+			grouped: number;
+			unitPriced: number;
+			promos: number;
+		}[]
+	>`
+		select
+			(select count(*)::int from groceries.groups where merged_into_id is null and item_count > 0) as groups,
+			(select count(*)::int from groceries.groups where merged_into_id is null and store_count >= 2) as "crossStore",
+			count(*)::int as items,
+			(count(*) filter (where g.store_count >= 2))::int as grouped,
+			(count(*) filter (where i.unit_price_cents is not null))::int as "unitPriced",
+			(count(*) filter (where i.promo_price_cents is not null or i.promo_label is not null))::int as promos
+		from groceries.items i left join groceries.groups g on g.id = i.group_id
+		where i.available`;
+	const runs = await s<
+		{ storeId: number; finishedAt: Date | null; status: string; fetched: number }[]
+	>`
+		select distinct on (store_id) store_id as "storeId", finished_at as "finishedAt", status, fetched
+		from groceries.import_runs order by store_id, started_at desc`;
+	const categories = await s<{ category: string }[]>`
+		select distinct category from groceries.groups where category is not null and item_count > 0`;
+	const catList = [
+		...CATEGORIES.filter((c) => categories.some((x) => x.category === c)),
+		...categories
+			.map((x) => x.category)
+			.filter((c) => !(CATEGORIES as readonly string[]).includes(c))
+	];
+
+	let detail = null;
+	if (detailId) {
+		const [group] = await s<(GroupRow & { notes: string | null })[]>`
+			select ${GROUP_COLS(s)}, g.notes from groceries.groups g where g.id = ${detailId}`;
+		if (group) {
+			const [dItems, similar] = await Promise.all([
+				s<ItemRow[]>`
+					select ${ITEM_COLS(s)} from groceries.items i where i.group_id = ${detailId}
+					order by i.available desc, coalesce(i.promo_unit_price_cents, i.unit_price_cents) nulls last, i.id`,
+				group.familyKey
+					? s<GroupRow[]>`
+						select ${GROUP_COLS(s)} from groceries.groups g
+						where g.family_key = ${group.familyKey} and g.id <> ${detailId}
+							and g.merged_into_id is null and g.item_count > 0
+						order by g.min_unit_price_cents nulls last limit 12`
+					: Promise.resolve([] as GroupRow[])
+			]);
+			const history = dItems.length
+				? await s<
+						{
+							itemId: number;
+							observedOn: string;
+							priceCents: number | null;
+							promoPriceCents: number | null;
+							promoLabel: string | null;
+						}[]
+					>`
+					select item_id as "itemId", observed_on::text as "observedOn", price_cents as "priceCents",
+						promo_price_cents as "promoPriceCents", promo_label as "promoLabel"
+					from groceries.item_prices where item_id = any(${dItems.map((i) => i.id)}::int[])
+					order by observed_on desc limit 400`
+				: [];
+			detail = { group, items: dItems, similar, history };
+		}
+	}
+
 	return {
 		stores,
-		products,
+		groups,
-		prices,
+		items,
-		seo: { title: 'Groceries', description: 'Private grocery and price tracker.', index: false }
+		total,
+		page,
+		pageSize: PAGE_SIZE,
+		filters: { q: p.get('q') ?? '', store: store?.slug ?? '', category, favorites, promo, multi },
+		categories: catList,
+		stats,
+		runs,
+		detail,
+		seo: { title: 'Groceries', description: 'Private supermarket price comparison.', index: false }
 	};
 };
 
@@ -47,132 +200,110 @@ const id = (v: FormDataEntryValue | null) => {
 	const n = Number(v);
 	return Number.isInteger(n) && n > 0 ? n : null;
 };
-const formObject = (form: FormData) => {
-	const o: Record<string, unknown> = {};
-	for (const [k, v] of form) if (typeof v === 'string') o[k] = v;
-	return o;
-};
-const firstIssue = (e: { issues: { path: PropertyKey[]; message: string }[] }) => {
-	const i = e.issues[0];
-	return i ? `${i.path.join('.') || 'input'}: ${i.message}` : 'Invalid input';
-};
-function dbError(e: unknown): string {
-	const cause = (e as { cause?: { message?: string; constraint_name?: string } })?.cause;
-	if (cause?.constraint_name === 'products_name_brand_key')
-		return 'That product (name + brand) already exists';
-	if (cause?.constraint_name) return `Not allowed (${cause.constraint_name})`;
-	return cause?.message || 'Could not save';
-}
 
 export const actions: Actions = {
-	/** Create or edit a product. */
-	saveProduct: async (event) => {
-		requireOwner(event);
-		const form = await event.request.formData();
-		const productId = id(form.get('id'));
-		const parsed = productInput.safeParse(formObject(form));
-		if (!parsed.success) return fail(400, { message: firstIssue(parsed.error) });
-		const p = parsed.data;
-		const size = parseSize(p.size);
-		const values = {
-			name: p.name,
-			brand: p.brand ?? null,
-			category: p.category ?? null,
-			size: p.size ?? null,
-			unitAmount: size?.amount ?? null,
-			unit: size?.unit ?? null,
-			notes: p.notes ?? null,
-			...(form.has('favorite') ? { favorite: form.get('favorite') === 'on' } : {})
-		};
-		try {
-			if (productId) {
-				await db.transaction(async (tx) => {
-					await tx.update(groceryProducts).set(values).where(eq(groceryProducts.id, productId));
-					// Pack size changed: recompute unit prices of this product's history.
-					await tx
-						.update(groceryPrices)
-						.set({
-							unitPriceCents: values.unitAmount
-								? sql`round(${groceryPrices.priceCents} / ${values.unitAmount}::numeric)::int`
-								: null
-						})
-						.where(eq(groceryPrices.productId, productId));
-				});
-				return { saved: productId };
-			}
-			const [row] = await db
-				.insert(groceryProducts)
-				.values(values)
-				.returning({ id: groceryProducts.id });
-			return { saved: row.id };
-		} catch (e) {
-			return fail(400, { message: dbError(e) });
-		}
-	},
-
 	favorite: async (event) => {
 		requireOwner(event);
 		const form = await event.request.formData();
-		const productId = id(form.get('id'));
+		const gid = id(form.get('id'));
-		if (!productId) return fail(400, { message: 'Invalid product' });
+		if (!gid) return fail(400, { message: 'Invalid group' });
-		await db
+		await sql()`update groceries.groups set favorite = ${form.get('favorite') === 'true'} where id = ${gid}`;
-			.update(groceryProducts)
+		return { updated: gid };
-			.set({ favorite: form.get('favorite') === 'true' })
-			.where(eq(groceryProducts.id, productId));
-		return { updated: productId };
 	},
 
-	deleteProduct: async (event) => {
+	/** Rename a group; it becomes manual so the nightly regroup keeps the name. */
-		requireOwner(event);
+	rename: async (event) => {
-		const productId = id((await event.request.formData()).get('id'));
-		if (!productId) return fail(400, { message: 'Invalid product' });
-		await db.delete(groceryProducts).where(eq(groceryProducts.id, productId));
-		return { deleted: productId };
-	},
-
-	/** Record a price observation for a product at a store. */
-	addPrice: async (event) => {
 		requireOwner(event);
 		const form = await event.request.formData();
-		const productId = id(form.get('productId'));
+		const gid = id(form.get('id'));
-		const storeId = id(form.get('storeId'));
+		const name = String(form.get('name') ?? '')
-		if (!productId || !storeId) return fail(400, { message: 'Pick a product and a store' });
+			.trim()
-		const parsed = priceInput.safeParse(formObject(form));
+			.slice(0, 200);
-		if (!parsed.success) return fail(400, { message: firstIssue(parsed.error) });
+		if (!gid || !name) return fail(400, { message: 'Enter a name' });
-		const [product] = await db
+		const notes =
-			.select({ unitAmount: groceryProducts.unitAmount })
+			String(form.get('notes') ?? '')
-			.from(groceryProducts)
+				.trim()
-			.where(eq(groceryProducts.id, productId));
+				.slice(0, 2000) || null;
-		if (!product) return fail(404, { message: 'Product not found' });
+		await sql()`update groceries.groups set name = ${name}, notes = ${notes}, manual = true where id = ${gid}`;
-		const p = parsed.data;
+		await refreshGroupStats(sql(), [gid]);
-		try {
+		return { updated: gid };
-			await db.insert(groceryPrices).values({
-				productId,
-				storeId,
-				priceCents: p.price,
-				unitPriceCents: unitPriceCents(p.price, product.unitAmount),
-				promo: form.get('promo') === 'on',
-				// A plain date is stored as noon UTC; the exact time does not matter for prices.
-				observedAt: p.observedOn ? new Date(`${p.observedOn}T12:00:00Z`) : new Date(),
-				source: 'manual',
-				url: p.url ?? null,
-				notes: p.notes ?? null
-			});
-		} catch (e) {
-			return fail(400, { message: dbError(e) });
-		}
-		return { priced: productId };
 	},
 
-	deletePrice: async (event) => {
+	/** Merge group `id` into `into`: its items move (locked) and future matches follow. */
+	merge: async (event) => {
 		requireOwner(event);
 		const form = await event.request.formData();
-		const priceId = id(form.get('id'));
+		const from = id(form.get('id'));
-		const productId = id(form.get('productId'));
+		const into = id(form.get('into'));
-		if (!priceId || !productId) return fail(400, { message: 'Invalid price' });
+		if (!from || !into || from === into) return fail(400, { message: 'Pick another group' });
-		await db
+		const [target] = await sql()`
-			.delete(groceryPrices)
+			select id from groceries.groups where id = ${into} and merged_into_id is null`;
-			.where(and(eq(groceryPrices.id, priceId), eq(groceryPrices.productId, productId)));
+		if (!target) return fail(404, { message: 'Group not found' });
-		return { deleted: priceId };
+		await sql().begin(async (tx) => {
+			await tx`
+				update groceries.items set group_id = ${into}, match_locked = true, match_method = 'manual',
+					match_confidence = 1 where group_id = ${from}`;
+			await tx`update groceries.groups set merged_into_id = ${into} where merged_into_id = ${from}`;
+			const [src] = await tx<{ favorite: boolean }[]>`
+				select favorite from groceries.groups where id = ${from}`;
+			await tx`update groceries.groups set merged_into_id = ${into}, favorite = false where id = ${from}`;
+			if (src?.favorite) await tx`update groceries.groups set favorite = true where id = ${into}`;
+			await refreshGroupStats(tx as unknown as Sql, [from, into]);
+		});
+		return { merged: into };
+	},
+
+	/** Move one item to another group, or to a new group of its own (split). Locks the item. */
+	moveItem: async (event) => {
+		requireOwner(event);
+		const form = await event.request.formData();
+		const itemId = id(form.get('itemId'));
+		const target = form.get('into') === 'new' ? 'new' : id(form.get('into'));
+		if (!itemId || !target) return fail(400, { message: 'Pick a group' });
+		const moved = await sql().begin(async (tx) => {
+			const [item] = await tx<
+				{
+					group_id: number | null;
+					name: string;
+					brand: string | null;
+					unit: string | null;
+					category: string | null;
+				}[]
+			>`
+				select group_id, name, brand, unit, category from groceries.items where id = ${itemId}`;
+			if (!item) return null;
+			let into: number;
+			if (target === 'new') {
+				const [g] = await tx<{ id: number }[]>`
+					insert into groceries.groups (name, brand, category, unit, manual)
+					values (${displayName(item.name, item.brand).slice(0, 200)}, ${item.brand}, ${item.category}, ${item.unit}, true)
+					returning id`;
+				into = g.id;
+			} else {
+				const [g] =
+					await tx`select id from groceries.groups where id = ${target} and merged_into_id is null`;
+				if (!g) return null;
+				into = target;
+			}
+			await tx`
+				update groceries.items set group_id = ${into}, match_locked = true, match_method = 'manual',
+					match_confidence = 1 where id = ${itemId}`;
+			await refreshGroupStats(tx as unknown as Sql, [
+				into,
+				...(item.group_id ? [item.group_id] : [])
+			]);
+			return into;
+		});
+		if (!moved) return fail(404, { message: 'Not found' });
+		return { moved };
+	},
+
+	/** Hand an item back to the automatic matcher (applies at the next nightly import). */
+	unlockItem: async (event) => {
+		requireOwner(event);
+		const itemId = id((await event.request.formData()).get('itemId'));
+		if (!itemId) return fail(400, { message: 'Invalid item' });
+		await sql()`update groceries.items set match_locked = false where id = ${itemId}`;
+		return { unlocked: itemId };
 	}
 };

src/routes/trackers/groceries/search/+server.ts

@@ -0,0 +1,23 @@
+import { db } from '#lib/server/db/index.js';
+import { requireOwner } from '#lib/server/owner.js';
+import { fold } from '#lib/server/groceries/normalize.js';
+import { json } from '@sveltejs/kit';
+import type { Sql } from 'postgres';
+import type { RequestHandler } from './$types';
+
+/** Group picker for merge/move: top matches for ?q= (owner only). */
+export const GET: RequestHandler = async (event) => {
+	requireOwner(event);
+	const q = fold(event.url.searchParams.get('q')?.trim() ?? '').slice(0, 100);
+	const words = q.split(/\s+/).filter(Boolean).slice(0, 6);
+	if (!words.length) return json([]);
+	const s = db.$client as unknown as Sql;
+	const conds = words.map((w) => s`search_text like ${'%' + w.replace(/[%_\\]/g, '\\$&') + '%'}`);
+	const rows = await s`
+		select id, name, brand, unit, store_count as "storeCount", item_count as "itemCount"
+		from groceries.groups
+		where merged_into_id is null and ${conds.reduce((a, c) => s`${a} and ${c}`)}
+		order by word_similarity(${q}, lower(name)) desc, store_count desc, item_count desc, id
+		limit 15`;
+	return json(rows, { headers: { 'cache-control': 'private, no-store' } });
+};