groceries: stats page with nightly snapshot (battles, basket, promos, CPI, range)

drizzle/20261025203926_groceries_stats/migration.sql

@@ -0,0 +1,32 @@
+CREATE TABLE "groceries"."item_sizes" (
+	"id" serial PRIMARY KEY,
+	"item_id" integer NOT NULL,
+	"observed_on" date NOT NULL,
+	"unit" text,
+	"unit_amount" numeric(12,4),
+	"size_text" text,
+	"price_cents" integer,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "groceries"."stats_snapshots" (
+	"id" serial PRIMARY KEY,
+	"computed_on" date NOT NULL,
+	"scope" text NOT NULL,
+	"computed_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"duration_ms" integer,
+	"data" jsonb NOT NULL,
+	"series" jsonb,
+	CONSTRAINT "stats_snapshots_scope_check" CHECK ("scope" IN ('food','all'))
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "item_sizes_item_day_key" ON "groceries"."item_sizes" ("item_id","observed_on");--> statement-breakpoint
+CREATE UNIQUE INDEX "stats_snapshots_day_scope_key" ON "groceries"."stats_snapshots" ("computed_on","scope");--> statement-breakpoint
+ALTER TABLE "groceries"."item_sizes" ADD CONSTRAINT "item_sizes_item_id_items_id_fkey" FOREIGN KEY ("item_id") REFERENCES "groceries"."items"("id") ON DELETE CASCADE;--> statement-breakpoint
+DO $$
+BEGIN
+  IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA groceries TO personal_app;
+    GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA groceries TO personal_app;
+  END IF;
+END $$;

scripts/deploy-preflight.ts

@@ -167,8 +167,10 @@ try {
 		where n.nspname = 'groceries' and c.relkind = 'r'
 	`;
 	if (
-		Number(groceries.tables) !== 8 ||
-		Number(groceries.app_rw) !== 8 ||
+		// 10 = stores, items, item_prices, groups, import_runs, categories, manual_prices, synonyms,
+		// stats_snapshots, item_sizes (more is fine as long as the app can read/write all of them).
+		Number(groceries.tables) < 10 ||
+		Number(groceries.app_rw) !== Number(groceries.tables) ||
 		Number(groceries.ro_read) !== 0 ||
 		groceries.public_usage
 	)

scripts/groceries-import.ts

@@ -24,6 +24,7 @@ import {
 	regroup
 } from '../src/lib/server/groceries/import';
 import { applyTaxonomy } from '../src/lib/server/groceries/taxonomy';
+import { computeStats } from '../src/lib/server/groceries/stats';
 import { SOURCES } from '../src/lib/server/groceries/sources';
 
 const args = process.argv.slice(2);
@@ -88,6 +89,12 @@ try {
 	}
 	await regroup(sql, log);
 	await applyTaxonomy(sql, log);
+	try {
+		await computeStats(sql, log);
+	} catch (e) {
+		failed = true;
+		log(`stats: FAILED ${e instanceof Error ? e.message : e}`);
+	}
 } finally {
 	await sql.end();
 }

scripts/groceries-stats.ts

@@ -0,0 +1,14 @@
+// Recompute the grocery analytics snapshot (/trackers/groceries/stats) without importing.
+//   bun scripts/groceries-stats.ts
+// The nightly import (scripts/groceries-import.ts) runs this at the end, under the import lock.
+import postgres from 'postgres';
+import { computeStats } from '../src/lib/server/groceries/stats';
+
+if (!process.env.DB_URL) throw new Error('DB_URL is not set');
+const log = (m: string) => console.log(`[${new Date().toISOString().slice(11, 19)}Z] ${m}`);
+const sql = postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
+try {
+	await computeStats(sql, log);
+} finally {
+	await sql.end();
+}

src/lib/groceries-stats.ts

@@ -0,0 +1,254 @@
+// Shared (client + server) types and helpers for /trackers/groceries/stats. The snapshot is
+// computed nightly by computeStats (src/lib/server/groceries/stats.ts); short keys keep it small.
+
+export type StatsScope = 'food' | 'all';
+
+export type StoreRef = { id: number; slug: string; name: string; color: string | null };
+export type CategoryRef = {
+	slug: string;
+	name: string;
+	icon: string | null;
+	color: string | null;
+	isFood: boolean;
+};
+/** A store product referenced from a list (top movers, extremes, gaps…). */
+export type ItemRef = {
+	id: number;
+	s: number; // store id
+	name: string;
+	size: string | null;
+	p: number | null; // pack price, cents
+	up: number | null; // unit price, cents per unit
+	unit: string | null;
+	url: string | null;
+};
+
+/** Price battle result per store: groups (comparable products) sold in 2+ stores. */
+export type BattleRow = {
+	s: number;
+	n: number; // battles the store takes part in
+	w: number; // wins, ties split (1/k each)
+	o: number; // outright wins (sole cheapest)
+	t: number; // tied for cheapest
+	wr: number; // w / n
+	ix: number; // median of store / cheapest unit price × 100 (100 = always cheapest)
+	gx: number; // geometric mean of the same
+	wp: number; // wins incl. promo prices
+	wrp: number;
+	ixp: number;
+	gxp: number;
+};
+export type H2HRow = {
+	a: number;
+	b: number;
+	n: number;
+	aw: number;
+	bw: number;
+	t: number;
+	md: number;
+};
+export type PromoRow = {
+	s: number;
+	items: number;
+	promo: number; // on promo (excl. AH volume voordeel)
+	share: number;
+	volume: number; // AH multipack "volume voordeel"
+	disc: number | null; // average discount 0–1 (explicit or implied by the promo type)
+	cmp: number; // promos in a battle group with a regular price elsewhere
+	beats: number; // … of which the promo is cheaper than any other store's regular price
+	save: number | null; // median promo / cheapest regular elsewhere (below 1 = real saving)
+};
+export type RangeRow = {
+	s: number;
+	items: number;
+	groups: number;
+	excl: number; // groups only this store sells
+	house: number; // own-brand items
+	known: number; // items whose brand class is known (own brand or A-brand)
+};
+export type BrandGapRow = { s: number; n: number; ratio: number; cheaper: number };
+export type GapRow = {
+	gtin: string;
+	cheap: ItemRef;
+	dear: ItemRef;
+	ratio: number;
+	diff: number; // cents
+	stores: number;
+};
+export type MoveRow = ItemRef & { from: number; to: number; pct: number; since: string };
+export type VolRow = ItemRef & { changes: number; min: number; max: number; promoFlips: number };
+export type ShrinkRow = ItemRef & {
+	day: string;
+	fromAmount: number;
+	toAmount: number;
+	fromPrice: number | null;
+	toPrice: number | null;
+};
+export type InflationRow = {
+	s: number; // 0 = all stores
+	days: number;
+	wow: number | null; // regular prices, week over week (fraction)
+	mom: number | null;
+	total: number | null; // since tracking started
+	wowE: number | null; // incl. promos
+	momE: number | null;
+	totalE: number | null;
+	changes: number; // price changes counted in the index
+};
+export type PpkgRow = { s: number; bins: number[]; median: number | null; n: number };
+
+export type Section = {
+	kpi: {
+		items: number;
+		groups: number;
+		battles: number;
+		tied: number;
+		multi: number;
+		ean: number; // EAN-identical products sold by 2+ stores
+		eanSame: number; // … with exactly the same pack price everywhere
+	};
+	battles: BattleRow[];
+	h2h: H2HRow[];
+	promos: PromoRow[];
+	promoTypes: { s: number; type: string; n: number; disc: number | null }[];
+	range: RangeRow[];
+	buckets: { b: string; n: number }[];
+	brandGap: BrandGapRow[]; // s = 0: across stores
+	extremes: { unit: string | null; cheap: ItemRef[]; dear: ItemRef[] };
+	gaps: GapRow[];
+	ppkg: PpkgRow[];
+	moves: { rises: MoveRow[]; drops: MoveRow[] };
+	volatile: VolRow[];
+	inflation: InflationRow[];
+};
+
+export type Staple = { key: string; label: string; qty: number; unit: string; icon: string };
+export type BasketPick = { reg: ItemRef & { cost: number }; eff: ItemRef & { cost: number } };
+
+export type Overview = {
+	/** Store × top category matrices (from the per-category sections). */
+	heat: { cat: string; s: number; n: number; wr: number; ix: number }[];
+	promoHeat: { cat: string; s: number; share: number; items: number }[];
+	rangeHeat: { cat: string; s: number; items: number }[];
+	brandGapByCat: { cat: string; n: number; ratio: number; cheaper: number }[];
+	extremesByCat: {
+		cat: string;
+		unit: string | null;
+		cheap: ItemRef | null;
+		dear: ItemRef | null;
+	}[];
+	basket: { staples: Staple[]; picks: Record<string, Record<string, BasketPick>> };
+	weekday: { s: number; dow: number; n: number }[]; // promo starts seen in the history (1 = Mon)
+	promoEnds: { s: number; dow: number; n: number }[]; // "t/m 27-10" style end dates on labels
+	shrink: ShrinkRow[];
+	shrinkSince: string | null;
+};
+
+export type SnapshotData = {
+	meta: {
+		computedAt: string;
+		scope: StatsScope;
+		stores: StoreRef[];
+		categories: CategoryRef[];
+		firstDay: string | null;
+		lastDay: string | null;
+		historyDays: number;
+		durationMs?: number;
+	};
+	overall: Section;
+	overview: Overview;
+	byCategory: Record<string, Section>;
+};
+
+/** Daily chained price index (Jevons, base 100 on the first day) per category and store. */
+export type SeriesData = {
+	days: string[];
+	cats: Record<string, Record<string, { r: number[]; e: number[]; c: number[] }>>;
+};
+
+/** Basket totals per store and the best 2-store split, from the per-staple picks. */
+export function basketSummary(basket: Overview['basket'], storeIds: number[], mode: 'reg' | 'eff') {
+	const keys = basket.staples.map((s) => s.key);
+	const cost = (s: number, k: string) => basket.picks[k]?.[String(s)]?.[mode]?.cost ?? null;
+	const stores = storeIds.map((s) => {
+		let total = 0;
+		let covered = 0;
+		for (const k of keys) {
+			const c = cost(s, k);
+			if (c != null) {
+				total += c;
+				covered++;
+			}
+		}
+		return { s, total, covered, complete: covered === keys.length };
+	});
+	const complete = stores.filter((r) => r.complete).sort((a, b) => a.total - b.total);
+	let bestPair: { a: number; b: number; total: number; split: Record<string, number> } | null =
+		null;
+	for (let i = 0; i < storeIds.length; i++)
+		for (let j = i + 1; j < storeIds.length; j++) {
+			const a = storeIds[i];
+			const b = storeIds[j];
+			let total = 0;
+			const split: Record<string, number> = {};
+			let ok = true;
+			for (const k of keys) {
+				const ca = cost(a, k);
+				const cb = cost(b, k);
+				if (ca == null && cb == null) {
+					ok = false;
+					break;
+				}
+				if (cb == null || (ca != null && ca <= cb)) {
+					total += ca!;
+					split[k] = a;
+				} else {
+					total += cb;
+					split[k] = b;
+				}
+			}
+			if (ok && (!bestPair || total < bestPair.total)) bestPair = { a, b, total, split };
+		}
+	// Theoretical minimum: every staple at whichever store is cheapest.
+	let cherry = 0;
+	let cherryOk = true;
+	const cherryPick: Record<string, number> = {};
+	for (const k of keys) {
+		let best: number | null = null;
+		for (const s of storeIds) {
+			const c = cost(s, k);
+			if (c != null && (best == null || c < best)) {
+				best = c;
+				cherryPick[k] = s;
+			}
+		}
+		if (best == null) cherryOk = false;
+		else cherry += best;
+	}
+	// Most stores miss a staple or two, so also compare on the staples every "nearly complete"
+	// store (all but 3 at most) carries.
+	const near = stores.filter((r) => r.covered >= keys.length - 3);
+	const commonKeys = keys.filter((k) => near.every((r) => cost(r.s, k) != null));
+	const common = near
+		.map((r) => ({ s: r.s, total: commonKeys.reduce((a, k) => a + (cost(r.s, k) ?? 0), 0) }))
+		.sort((a, b) => a.total - b.total);
+	return {
+		stores,
+		complete,
+		bestPair,
+		cherry: cherryOk ? cherry : null,
+		cherryPick,
+		commonKeys,
+		common
+	};
+}
+
+export const WEEKDAYS = ['ma', 'di', 'wo', 'do', 'vr', 'za', 'zo'];
+
+/** Price per kg histogram edges in euros (width_bucket in the snapshot query). */
+export const PPKG_EDGES = [1, 2, 3, 4, 5, 7.5, 10, 15, 20, 30, 50, 100];
+export const PPKG_LABELS = [
+	'<1',
+	...PPKG_EDGES.slice(0, -1).map((e, i) => `${e}–${PPKG_EDGES[i + 1]}`),
+	'100+'
+];

src/lib/server/db/schema/groceries.ts

@@ -6,6 +6,7 @@ import {
 	date,
 	index,
 	integer,
+	jsonb,
 	numeric,
 	real,
 	text,
@@ -277,6 +278,51 @@ export const grocerySynonyms = groceriesSchema.table(
 	]
 );
 
+/**
+ * Nightly analytics for /trackers/groceries/stats (computeStats in
+ * src/lib/server/groceries/stats.ts, run after import + regroup + taxonomy under the import lock).
+ * One row per day per scope ('food' = edible only, 'all' = incl. drugstore/household). `data` holds
+ * the overall section plus one per top category; `series` (daily chained price index) is only kept
+ * on recent rows, older rows are pruned to the overall section so history stays small.
+ */
+export const groceryStatsSnapshots = groceriesSchema.table(
+	'stats_snapshots',
+	{
+		id: id(),
+		computedOn: date('computed_on').notNull(),
+		scope: text('scope').notNull(),
+		computedAt: timestamptz('computed_at').notNull().defaultNow(),
+		durationMs: integer('duration_ms'),
+		data: jsonb('data').notNull(),
+		series: jsonb('series')
+	},
+	(t) => [
+		uniqueIndex('stats_snapshots_day_scope_key').on(t.computedOn, t.scope),
+		check('stats_snapshots_scope_check', sql`${t.scope} IN ('food','all')`)
+	]
+);
+
+/**
+ * Pack size history (a row only when an item's size changes), written by computeStats so
+ * shrinkflation (smaller pack, same or higher price) can be detected. item_prices has no sizes.
+ */
+export const groceryItemSizes = groceriesSchema.table(
+	'item_sizes',
+	{
+		id: id(),
+		itemId: integer('item_id')
+			.notNull()
+			.references(() => groceryItems.id, { onDelete: 'cascade' }),
+		observedOn: date('observed_on').notNull(),
+		unit: text('unit').$type<GroceryUnit>(),
+		unitAmount: numeric('unit_amount', { precision: 12, scale: 4, mode: 'number' }),
+		sizeText: text('size_text'),
+		priceCents: integer('price_cents'),
+		createdAt: timestamptz('created_at').notNull().defaultNow()
+	},
+	(t) => [uniqueIndex('item_sizes_item_day_key').on(t.itemId, t.observedOn)]
+);
+
 export type GroceryStore = typeof groceryStores.$inferSelect;
 export type GroceryGroup = typeof groceryGroups.$inferSelect;
 export type GroceryItem = typeof groceryItems.$inferSelect;

src/routes/trackers/groceries/+page.svelte

@@ -8,6 +8,7 @@
 	import { Textarea } from '#lib/components/ui/textarea/index.js';
 	import * as Dialog from '#lib/components/ui/dialog/index.js';
 	import Lock from 'lucide-svelte/icons/lock';
+	import ChartColumn from 'lucide-svelte/icons/chart-column-increasing';
 	import Unlock from 'lucide-svelte/icons/lock-open';
 	import Star from 'lucide-svelte/icons/star';
 	import ExternalLink from 'lucide-svelte/icons/external-link';
@@ -486,6 +487,9 @@
 				<dd>{fmtNum(data.stats.promos)}</dd>
 			</div>
 		</dl>
+		<a href="/trackers/groceries/stats" class="stats-link">
+			<ChartColumn size={16} aria-hidden="true" class="text-violet-500" /> stats
+		</a>
 	</header>
 
 	<!-- search + sort -->
@@ -1181,6 +1185,24 @@
 </Dialog.Root>
 
 <style>
+	.stats-link {
+		display: inline-flex;
+		align-items: center;
+		gap: 0.4rem;
+		padding: 0.45rem 0.8rem;
+		border: 1px solid var(--site-border);
+		border-radius: 0.6rem;
+		background: var(--site-surface);
+		font-size: 0.875rem;
+		font-weight: 500;
+		transition:
+			background 0.15s,
+			transform 0.15s;
+	}
+	.stats-link:hover {
+		background: var(--site-hover);
+		transform: translateY(-1px);
+	}
 	.stats {
 		display: flex;
 		gap: 1.25rem;

src/routes/trackers/groceries/stats/+page.server.ts

@@ -0,0 +1,114 @@
+import { db } from '#lib/server/db/index.js';
+import { requireOwner } from '#lib/server/owner.js';
+import { computeBattles } from '#lib/server/groceries/stats.js';
+import type {
+	BattleRow,
+	H2HRow,
+	Overview,
+	PromoRow,
+	Section,
+	SeriesData,
+	SnapshotData,
+	StatsScope
+} from '#lib/groceries-stats.js';
+import type { Sql } from 'postgres';
+import type { PageServerLoad } from './$types';
+
+const sql = () => db.$client as unknown as Sql;
+const DAY = /^\d{4}-\d{2}-\d{2}$/;
+
+type Live = { battles: BattleRow[]; h2h: H2HRow[]; total: number; tied: number };
+// Store-subset battles are recomputed live (~1 s); cache per snapshot so filter clicks are instant.
+const liveCache = new Map<string, Promise<Record<string, Live>>>();
+
+export const load: PageServerLoad = async (event) => {
+	requireOwner(event);
+	const q = event.url.searchParams;
+	const scope: StatsScope = q.get('nonfood') === '1' ? 'all' : 'food';
+	const cat = (q.get('cat') ?? '').trim() || 'all';
+	const from = DAY.test(q.get('from') ?? '') ? q.get('from')! : null;
+	const to = DAY.test(q.get('to') ?? '') ? q.get('to')! : null;
+	const stores = [
+		...new Set(
+			(q.get('stores') ?? '')
+				.split(',')
+				.map(Number)
+				.filter((n) => Number.isInteger(n) && n > 0)
+		)
+	].sort((a, b) => a - b);
+	const s = sql();
+
+	const [snap] = await s`
+		select computed_on::text as day, duration_ms, data->'meta' as meta, data->'overview' as overview,
+			coalesce(data->'byCategory'->${cat}, case when ${cat} = 'all' then data->'overall' end) as section
+		from groceries.stats_snapshots
+		where scope = ${scope} and (${to}::date is null or computed_on <= ${to}::date)
+		order by computed_on desc limit 1`;
+	const days = (
+		await s`select computed_on::text as day from groceries.stats_snapshots where scope = ${scope}
+			order by computed_on`
+	).map((r) => r.day as string);
+	const base = { scope, cat, from, to, stores, days };
+	if (!snap) return { ...base, empty: true as const };
+
+	// Daily index series: the newest snapshot that still carries it, cut to the category/range.
+	const [ser] = await s`
+		select series->'days' as days, series->'cats'->${cat} as cat
+		from groceries.stats_snapshots
+		where scope = ${scope} and series is not null and (${to}::date is null or computed_on <= ${to}::date)
+		order by computed_on desc limit 1`;
+	let series: { days: string[]; rows: SeriesData['cats'][string] } | null = null;
+	if (ser?.days && ser.cat) {
+		const all = ser.days as string[];
+		const keep = all
+			.map((d, i) => [d, i] as const)
+			.filter(([d]) => (!from || d >= from) && (!to || d <= to));
+		const idx = keep.map(([, i]) => i);
+		const rows: SeriesData['cats'][string] = {};
+		for (const [k, v] of Object.entries(ser.cat as SeriesData['cats'][string]))
+			rows[k] = { r: idx.map((i) => v.r[i]), e: idx.map((i) => v.e[i]), c: idx.map((i) => v.c[i]) };
+		series = { days: keep.map(([d]) => d), rows };
+	}
+
+	// Nightly history of the battle and promo tables (older snapshots keep only the overall section).
+	const trend = (await s`
+		select computed_on::text as day,
+			coalesce(data->'byCategory'->${cat}->'battles', case when ${cat} = 'all' then data->'overall'->'battles' end) as battles,
+			coalesce(data->'byCategory'->${cat}->'promos', case when ${cat} = 'all' then data->'overall'->'promos' end) as promos
+		from groceries.stats_snapshots
+		where scope = ${scope} and (${from}::date is null or computed_on >= ${from}::date)
+			and (${to}::date is null or computed_on <= ${to}::date)
+		order by computed_on`) as unknown as {
+		day: string;
+		battles: BattleRow[] | null;
+		promos: PromoRow[] | null;
+	}[];
+
+	const meta = snap.meta as SnapshotData['meta'];
+	let live: Live | null = null;
+	if (stores.length >= 2) {
+		const key = `${scope}|${snap.day}|${meta.computedAt}|${stores.join(',')}`;
+		let job = liveCache.get(key);
+		if (!job) {
+			if (liveCache.size > 50) liveCache.clear();
+			job = computeBattles(s, scope, stores);
+			liveCache.set(key, job);
+			job.catch(() => liveCache.delete(key));
+		}
+		const all = await job;
+		live = all[cat] ?? { battles: [], h2h: [], total: 0, tied: 0 };
+	}
+
+	return {
+		...base,
+		empty: false as const,
+		day: snap.day as string,
+		durationMs: snap.duration_ms as number | null,
+		meta,
+		overview: (snap.overview ?? null) as Overview | null,
+		section: (snap.section ?? null) as Section | null,
+		series,
+		trend,
+		live
+	};
+};