groceries: nightly AH/Jumbo catalogue import, unit-price normalisation, product groups and compare UI
All checks were successful
Deploy production / deploy (push) Successful in 2m48s

This commit is contained in:
2026-10-24 14:01:25 +00:00
parent 8dde5aa302
commit 427f4ed095
20 changed files with 34891 additions and 699 deletions

44
src/lib/groceries.test.ts Normal file
View File

@@ -0,0 +1,44 @@
import { describe, expect, it } from 'vitest';
import { cheapest, formatAmount, formatUnitPrice, sortByPrice, type PricedItem } from './groceries';
const it_ = (
id: number,
storeId: number,
price: number,
unit: number,
promo: number | null = null
): PricedItem => ({
id,
storeId,
priceCents: price,
promoPriceCents: promo,
unitPriceCents: unit,
promoUnitPriceCents: promo == null ? null : Math.round((unit * promo) / price),
available: true
});
describe('grocery price helpers', () => {
const items = [
it_(1, 1, 119, 119),
it_(2, 1, 165, 110),
it_(3, 2, 115, 115),
it_(4, 2, 200, 200, 100)
];
it('finds the cheapest by unit price, promos included', () => {
const c = cheapest(items, 'unit');
expect(c.overall).toBe(4);
expect([...c.perStore].sort()).toEqual([2, 4]);
});
it('finds the cheapest by pack price', () => {
expect(cheapest(items, 'pack').overall).toBe(4);
expect(cheapest(items.slice(0, 3), 'pack').overall).toBe(3);
});
it('sorts by unit price', () => {
expect(sortByPrice(items, 'unit').map((i) => i.id)).toEqual([4, 2, 3, 1]);
});
it('formats', () => {
expect(formatUnitPrice(250, 'kg')).toMatch(/2,50\/kg$/);
expect(formatAmount(1.98, 'l')).toBe('1,98 L');
expect(formatUnitPrice(null, 'kg')).toBeNull();
});
});

View File

@@ -1,24 +1,64 @@
// Shared (client + server) grocery helpers: price comparison and formatting.
export type LatestPrice = {
storeId: number;
priceCents: number;
unitPriceCents: number | null;
promo: boolean;
observedAt: Date;
};
/** Store ids with the lowest comparable price: unit price when every store has one, else pack price. */
export function cheapestStores(prices: LatestPrice[]): Set<number> {
if (prices.length < 2) return new Set();
const byUnit = prices.every((p) => p.unitPriceCents !== null);
const val = (p: LatestPrice) => (byUnit ? p.unitPriceCents! : p.priceCents);
const min = Math.min(...prices.map(val));
return new Set(prices.filter((p) => val(p) === min).map((p) => p.storeId));
}
// Shared (client + server) grocery helpers: formatting and cheapest-price picking.
const eur = new Intl.NumberFormat('nl-NL', { style: 'currency', currency: 'EUR' });
/** 179 → "€ 1,79". */
export const formatCents = (cents: number) => eur.format(cents / 100);
export const UNIT_LABEL: Record<string, string> = { kg: 'kg', l: 'L', piece: 'st' };
/** "€ 2,50/kg"; null without a unit price. */
export function formatUnitPrice(cents: number | null | undefined, unit: string | null | undefined) {
if (cents == null || !unit) return null;
return `${formatCents(cents)}/${UNIT_LABEL[unit] ?? unit}`;
}
/** "0,5 kg", "1,98 L", "6 st". */
export function formatAmount(amount: number | null | undefined, unit: string | null | undefined) {
if (amount == null || !unit) return null;
const n = amount.toLocaleString('nl-NL', { maximumFractionDigits: 3 });
return `${n} ${UNIT_LABEL[unit] ?? unit}`;
}
export type PricedItem = {
id: number;
storeId: number;
priceCents: number | null;
promoPriceCents: number | null;
unitPriceCents: number | null;
promoUnitPriceCents: number | null;
available: boolean;
};
/** What you pay now (promo if any). */
export const effectivePrice = (i: PricedItem) => i.promoPriceCents ?? i.priceCents;
export const effectiveUnitPrice = (i: PricedItem) => i.promoUnitPriceCents ?? i.unitPriceCents;
/**
* Cheapest items: overall and per store, by unit price (or pack price). Unavailable items and
* items without the compared price are ignored.
*/
export function cheapest(items: PricedItem[], mode: 'unit' | 'pack') {
const val = (i: PricedItem) => (mode === 'unit' ? effectiveUnitPrice(i) : effectivePrice(i));
let overall: number | null = null;
let overallVal = Infinity;
const perStore = new Map<number, { id: number; v: number }>();
for (const i of items) {
const v = val(i);
if (!i.available || v == null) continue;
if (v < overallVal) {
overallVal = v;
overall = i.id;
}
const cur = perStore.get(i.storeId);
if (!cur || v < cur.v) perStore.set(i.storeId, { id: i.id, v });
}
return { overall, perStore: new Set([...perStore.values()].map((x) => x.id)) };
}
/** Sort by unit or pack price ascending; missing prices and unavailable items last. */
export function sortByPrice<T extends PricedItem>(items: T[], mode: 'unit' | 'pack'): T[] {
const val = (i: PricedItem) =>
(i.available ? 0 : 1e12) +
((mode === 'unit' ? effectiveUnitPrice(i) : effectivePrice(i)) ?? 1e11);
return [...items].sort((a, b) => val(a) - val(b) || a.id - b.id);
}

View File

@@ -1,12 +1,27 @@
import { sql } from 'drizzle-orm';
import { boolean, check, index, integer, numeric, text, uniqueIndex } from 'drizzle-orm/pg-core';
import {
type AnyPgColumn,
boolean,
check,
date,
index,
integer,
numeric,
real,
text,
uniqueIndex
} from 'drizzle-orm/pg-core';
import { groceriesSchema } from './schemas';
import { id, timestamps, timestamptz } from './helpers';
// Private grocery tracker (owner only, see requireOwner): the products Stan buys, which
// ones are favourites, and what they cost at each supermarket over time.
// Prices are append-only observations in cents, so price history and "cheapest now" both
// come from the same table. The latest observation per (product, store) is the current price.
// Private grocery price comparison (owner only, see requireOwner).
//
// `items` are store products as the supermarket sells them (one row per store SKU), filled
// nightly by scripts/groceries-import.ts. `groups` link equivalent items across stores and pack
// sizes ("Halfvolle melk" = AH + Jumbo + Plus house brand, any size), so unit prices compare.
// Grouping is a heuristic (EAN, then normalised name + brand class, then fuzzy name); Stan can
// fix it in the UI, which locks the item (`match_locked`) so the nightly regroup leaves it alone.
// `item_prices` keeps history: a row only when an item's price or promo price changes.
export const GROCERY_UNITS = ['piece', 'kg', 'l'] as const;
export type GroceryUnit = (typeof GROCERY_UNITS)[number];
@@ -32,66 +47,137 @@ export const groceryStores = groceriesSchema.table(
);
/**
* A product as Stan thinks of it ("Halfvolle melk 1 L"), not one store's SKU: the same
* product can be priced at several stores. `unit_amount` + `unit` (e.g. 1.5 + 'l') make
* unit prices comparable across pack sizes.
* A comparable product ("Halfvolle melk", house brand) across stores and sizes. Stats columns
* (item_count … search_text) are denormalised by refreshGroupStats() so the list stays fast.
*/
export const groceryProducts = groceriesSchema.table(
'products',
export const groceryGroups = groceriesSchema.table(
'groups',
{
id: id(),
/** Matcher key (brand class | unit | name tokens); null for groups made by hand. */
key: text('key'),
name: text('name').notNull(),
brand: text('brand'),
brand: text('brand'), // display brand; 'Huismerk' for AH/Jumbo/Plus own brands
category: text('category'),
size: text('size'), // as printed, e.g. "6 x 1 L"
unitAmount: numeric('unit_amount', { precision: 10, scale: 3, mode: 'number' }),
unit: text('unit').$type<GroceryUnit>(),
/** Name tokens without brand: groups sharing it are "similar" (e.g. house vs Campina melk). */
familyKey: text('family_key'),
favorite: boolean('favorite').notNull().default(false),
manual: boolean('manual').notNull().default(false),
mergedIntoId: integer('merged_into_id').references((): AnyPgColumn => groceryGroups.id, {
onDelete: 'set null'
}),
notes: text('notes'),
archivedAt: timestamptz('archived_at'),
itemCount: integer('item_count').notNull().default(0),
storeCount: integer('store_count').notNull().default(0),
storeIds: integer('store_ids')
.array()
.notNull()
.default(sql`'{}'::int[]`),
minUnitPriceCents: integer('min_unit_price_cents'),
minPriceCents: integer('min_price_cents'),
promoCount: integer('promo_count').notNull().default(0),
imageUrl: text('image_url'),
searchText: text('search_text').notNull().default(''),
...timestamps
},
(t) => [
uniqueIndex('products_name_brand_key').on(
sql`lower(${t.name})`,
sql`lower(coalesce(${t.brand}, ''))`
),
index('products_category_idx').on(t.category),
check('products_name_check', sql`length(btrim(${t.name})) > 0`),
check('products_unit_check', sql`${t.unit} IS NULL OR ${t.unit} IN ('piece','kg','l')`),
check('products_unit_amount_check', sql`${t.unitAmount} IS NULL OR ${t.unitAmount} > 0`)
uniqueIndex('groups_key_key').on(t.key),
index('groups_family_idx').on(t.familyKey),
index('groups_category_idx').on(t.category),
index('groups_store_ids_idx').using('gin', t.storeIds),
index('groups_search_trgm_idx').using('gin', sql`${t.searchText} gin_trgm_ops`),
index('groups_list_idx').on(t.storeCount, t.name),
check('groups_name_check', sql`length(btrim(${t.name})) > 0`),
check('groups_unit_check', sql`${t.unit} IS NULL OR ${t.unit} IN ('piece','kg','l')`),
check('groups_not_self_merged', sql`${t.mergedIntoId} IS NULL OR ${t.mergedIntoId} <> ${t.id}`)
]
);
/** One observed price of a product at a store (manual entry or a later import). */
export const groceryPrices = groceriesSchema.table(
'prices',
/** One store product (SKU) with its current price. Prices are in cents. */
export const groceryItems = groceriesSchema.table(
'items',
{
id: id(),
productId: integer('product_id')
.notNull()
.references(() => groceryProducts.id, { onDelete: 'cascade' }),
storeId: integer('store_id')
.notNull()
.references(() => groceryStores.id, { onDelete: 'cascade' }),
priceCents: integer('price_cents').notNull(),
unitPriceCents: integer('unit_price_cents'), // per product.unit, if known
promo: boolean('promo').notNull().default(false), // bonus / aanbieding price
observedAt: timestamptz('observed_at').notNull().defaultNow(),
source: text('source').notNull().default('manual'), // 'manual', 'ah-api', 'jumbo-api', …
storeProductId: text('store_product_id'), // e.g. AH webshopId, Jumbo SKU
storeProductId: text('store_product_id').notNull(), // AH webshopId, Jumbo SKU
name: text('name').notNull(),
brand: text('brand'),
sizeText: text('size_text'), // as the store prints it, e.g. "6 x 330 ml"
unit: text('unit').$type<GroceryUnit>(),
unitAmount: numeric('unit_amount', { precision: 12, scale: 4, mode: 'number' }),
/** 'size' = parsed from the pack size, 'store' = derived from the store's own unit price. */
unitSource: text('unit_source'),
priceCents: integer('price_cents'),
promoPriceCents: integer('promo_price_cents'),
promoLabel: text('promo_label'),
unitPriceCents: integer('unit_price_cents'),
promoUnitPriceCents: integer('promo_unit_price_cents'),
category: text('category'), // canonical category (shared across stores)
categoryPath: text('category_path'), // the store's own path
gtin: text('gtin'),
gtinCheckedAt: timestamptz('gtin_checked_at'),
imageUrl: text('image_url'),
url: text('url'),
notes: text('notes'),
available: boolean('available').notNull().default(true),
firstSeenAt: timestamptz('first_seen_at').notNull().defaultNow(),
lastSeenAt: timestamptz('last_seen_at').notNull().defaultNow(),
groupId: integer('group_id').references(() => groceryGroups.id, { onDelete: 'set null' }),
matchKey: text('match_key'),
matchMethod: text('match_method'), // 'ean' | 'key' | 'fuzzy' | 'single' | 'manual'
matchConfidence: real('match_confidence'),
matchLocked: boolean('match_locked').notNull().default(false),
...timestamps
},
(t) => [
index('prices_product_store_idx').on(t.productId, t.storeId, t.observedAt),
index('prices_store_idx').on(t.storeId),
check('prices_price_check', sql`${t.priceCents} >= 0`),
check('prices_unit_price_check', sql`${t.unitPriceCents} IS NULL OR ${t.unitPriceCents} >= 0`)
uniqueIndex('items_store_product_key').on(t.storeId, t.storeProductId),
index('items_group_idx').on(t.groupId),
index('items_gtin_idx').on(t.gtin),
index('items_name_trgm_idx').using('gin', sql`${t.name} gin_trgm_ops`),
check('items_unit_check', sql`${t.unit} IS NULL OR ${t.unit} IN ('piece','kg','l')`),
check('items_price_check', sql`${t.priceCents} IS NULL OR ${t.priceCents} >= 0`)
]
);
/** Price history: one row per item per day on which its price or promo changed. */
export const groceryItemPrices = groceriesSchema.table(
'item_prices',
{
id: id(),
itemId: integer('item_id')
.notNull()
.references(() => groceryItems.id, { onDelete: 'cascade' }),
observedOn: date('observed_on').notNull(),
priceCents: integer('price_cents'),
promoPriceCents: integer('promo_price_cents'),
promoLabel: text('promo_label'),
createdAt: timestamptz('created_at').notNull().defaultNow()
},
(t) => [uniqueIndex('item_prices_item_day_key').on(t.itemId, t.observedOn)]
);
/** One importer run per store (counts and errors), for the UI footer and debugging. */
export const groceryImportRuns = groceriesSchema.table(
'import_runs',
{
id: id(),
storeId: integer('store_id')
.notNull()
.references(() => groceryStores.id, { onDelete: 'cascade' }),
startedAt: timestamptz('started_at').notNull().defaultNow(),
finishedAt: timestamptz('finished_at'),
status: text('status').notNull().default('running'), // running | ok | error
requests: integer('requests').notNull().default(0),
fetched: integer('fetched').notNull().default(0),
inserted: integer('inserted').notNull().default(0),
priceChanges: integer('price_changes').notNull().default(0),
error: text('error')
},
(t) => [index('import_runs_store_idx').on(t.storeId, t.startedAt)]
);
export type GroceryStore = typeof groceryStores.$inferSelect;
export type GroceryProduct = typeof groceryProducts.$inferSelect;
export type GroceryPrice = typeof groceryPrices.$inferSelect;
export type GroceryGroup = typeof groceryGroups.$inferSelect;
export type GroceryItem = typeof groceryItems.$inferSelect;

View File

@@ -1,45 +0,0 @@
import { describe, expect, it } from 'vitest';
import { parseCents, parseSize, priceInput, unitPriceCents } from './groceries';
import { cheapestStores, formatCents } from '../groceries';
describe('groceries helpers', () => {
it('parses prices into cents', () => {
expect(parseCents('1,79')).toBe(179);
expect(parseCents('€ 2.5')).toBe(250);
expect(parseCents('3')).toBe(300);
expect(parseCents('abc')).toBeNull();
expect(parseCents('1,234')).toBeNull();
});
it('parses pack sizes', () => {
expect(parseSize('1 L')).toEqual({ amount: 1, unit: 'l' });
expect(parseSize('6 x 1 L')).toEqual({ amount: 6, unit: 'l' });
expect(parseSize('500 g')).toEqual({ amount: 0.5, unit: 'kg' });
expect(parseSize('750ml')).toEqual({ amount: 0.75, unit: 'l' });
expect(parseSize('4 stuks')).toEqual({ amount: 4, unit: 'piece' });
expect(parseSize('1,5 liter')).toEqual({ amount: 1.5, unit: 'l' });
expect(parseSize('a bunch')).toBeNull();
});
it('computes unit prices', () => {
expect(unitPriceCents(479, 6)).toBe(80);
expect(unitPriceCents(100, null)).toBeNull();
});
it('validates price input', () => {
expect(priceInput.safeParse({ price: '1,29' }).data?.price).toBe(129);
expect(priceInput.safeParse({ price: 'x' }).success).toBe(false);
expect(priceInput.safeParse({ price: '1', url: '' }).success).toBe(true);
});
it('finds the cheapest store', () => {
const p = (storeId: number, priceCents: number, unitPriceCents: number | null) => ({
storeId,
priceCents,
unitPriceCents,
promo: false,
observedAt: new Date()
});
expect([...cheapestStores([p(1, 179, 179), p(2, 129, 129)])]).toEqual([2]);
expect([...cheapestStores([p(1, 479, 80), p(2, 129, 129)])]).toEqual([1]);
expect([...cheapestStores([p(1, 479, null), p(2, 129, 129)])]).toEqual([2]);
expect(cheapestStores([p(1, 100, null)]).size).toBe(0);
expect(formatCents(179).replace(/\s/g, ' ')).toBe('€ 1,79');
});
});

View File

@@ -1,77 +0,0 @@
import * as z from 'zod';
import { GROCERY_UNITS } from './db/schema/groceries';
/** "1,79" / "1.79" / "€ 1,79" → 179 cents; null when not a price. */
export function parseCents(input: unknown): number | null {
if (typeof input !== 'string') return null;
const s = input.replace(/[€\s]/g, '').replace(',', '.');
if (!/^\d+(\.\d{1,2})?$/.test(s)) return null;
return Math.round(Number(s) * 100);
}
/** Price per unit (kg / l / piece) in cents, rounded; null without a usable amount. */
export function unitPriceCents(priceCents: number, unitAmount: number | null | undefined) {
if (!unitAmount || !(unitAmount > 0)) return null;
return Math.round(priceCents / unitAmount);
}
/**
* Best-effort pack size parse, e.g. "1 L" → 1 l, "6 x 1 L" → 6 l, "500 g" → 0.5 kg,
* "750ml" → 0.75 l, "4 stuks" → 4 piece. Returns null when unclear.
*/
export function parseSize(
size: string | null | undefined
): { amount: number; unit: (typeof GROCERY_UNITS)[number] } | null {
if (!size) return null;
const s = size.toLowerCase().replace(',', '.').trim();
const m = s.match(
/^(?:(\d+)\s*x\s*)?(\d+(?:\.\d+)?)\s*(kg|kilo|g|gr|gram|l|liter|litre|ml|cl|st|stuk|stuks|pcs|pieces?)$/
);
if (!m) return null;
const n = (m[1] ? Number(m[1]) : 1) * Number(m[2]);
const u = m[3];
if (['kg', 'kilo'].includes(u)) return { amount: n, unit: 'kg' };
if (['g', 'gr', 'gram'].includes(u)) return { amount: n / 1000, unit: 'kg' };
if (['l', 'liter', 'litre'].includes(u)) return { amount: n, unit: 'l' };
if (u === 'ml') return { amount: n / 1000, unit: 'l' };
if (u === 'cl') return { amount: n / 100, unit: 'l' };
return { amount: n, unit: 'piece' };
}
const optText = (max: number) =>
z
.string()
.trim()
.max(max)
.transform((v) => (v === '' ? null : v))
.nullable()
.optional();
export const productInput = z.object({
name: z.string().trim().min(1, 'Name is required').max(200),
brand: optText(100),
category: optText(100),
size: optText(60),
notes: optText(2000)
});
export const priceInput = z.object({
price: z.string().transform((v, ctx) => {
const c = parseCents(v);
if (c === null) {
ctx.addIssue({ code: 'custom', message: 'Enter a price like 1,79' });
return z.NEVER;
}
return c;
}),
url: optText(2000).refine(
(v) => v == null || /^https?:\/\/\S+$/.test(v),
'Link must start with http(s)://'
),
observedOn: z
.string()
.regex(/^\d{4}-\d{2}-\d{2}$/)
.optional()
.or(z.literal('').transform(() => undefined)),
notes: optText(500)
});

View File

@@ -0,0 +1,99 @@
// Polite JSON fetcher for the grocery importers: one request at a time with a minimum delay,
// exponential backoff on 429/5xx/network errors (honours Retry-After), and a same-day disk
// cache so re-running an import on the same day does not hit the store again.
import { mkdirSync, existsSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
import { gunzipSync, gzipSync } from 'node:zlib';
import { join } from 'node:path';
export const USER_AGENT =
'stanrunge.dev grocery price tracker (personal use, 1 req/2s; +https://stanrunge.dev)';
export type FetcherOptions = {
store: string;
minDelayMs?: number;
maxRetries?: number;
cacheDir?: string | null;
log?: (msg: string) => void;
};
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
const today = () => new Date().toLocaleDateString('sv-SE', { timeZone: 'Europe/Amsterdam' });
export class PoliteFetcher {
requests = 0;
cacheHits = 0;
private last = 0;
private readonly minDelayMs: number;
private readonly maxRetries: number;
private readonly dir: string | null;
private readonly log: (msg: string) => void;
constructor(opts: FetcherOptions) {
this.minDelayMs = opts.minDelayMs ?? 2000;
this.maxRetries = opts.maxRetries ?? 5;
this.log = opts.log ?? (() => {});
this.dir = opts.cacheDir ? join(opts.cacheDir, today(), opts.store) : null;
if (this.dir) mkdirSync(this.dir, { recursive: true });
}
/** Fetch JSON. `cacheKey` (a file-safe string) enables the same-day cache. */
async json<T>(url: string, init: RequestInit = {}, cacheKey?: string): Promise<T> {
const file = this.dir && cacheKey ? join(this.dir, `${cacheKey}.json.gz`) : null;
if (file && existsSync(file)) {
this.cacheHits++;
return JSON.parse(gunzipSync(readFileSync(file)).toString('utf8')) as T;
}
for (let attempt = 0; ; attempt++) {
const wait = this.last + this.minDelayMs - Date.now();
if (wait > 0) await sleep(wait);
this.last = Date.now();
this.requests++;
let retryAfter = 0;
let problem: string;
try {
const res = await fetch(url, { ...init, signal: AbortSignal.timeout(45_000) });
if (res.ok) {
const text = await res.text();
const data = JSON.parse(text) as T;
if (file) writeFileSync(file, gzipSync(text));
return data;
}
problem = `HTTP ${res.status}`;
if (res.status !== 429 && res.status < 500 && res.status !== 408) {
throw new HttpError(
res.status,
`${problem} for ${url}: ${(await res.text()).slice(0, 200)}`
);
}
retryAfter = Number(res.headers.get('retry-after')) * 1000 || 0;
} catch (e) {
if (e instanceof HttpError) throw e;
problem = e instanceof Error ? e.message : String(e);
}
if (attempt >= this.maxRetries)
throw new Error(`${problem} for ${url} after ${attempt + 1} tries`);
const backoff = Math.max(retryAfter, 5000 * 2 ** attempt);
this.log(` ${problem}; retrying in ${Math.round(backoff / 1000)}s`);
await sleep(backoff);
}
}
}
export class HttpError extends Error {
constructor(
readonly status: number,
message: string
) {
super(message);
}
}
/** Delete cache days older than `keepDays`. */
export function pruneCache(cacheDir: string, keepDays = 7) {
if (!existsSync(cacheDir)) return;
const cutoff = new Date(Date.now() - keepDays * 86_400_000).toLocaleDateString('sv-SE', {
timeZone: 'Europe/Amsterdam'
});
for (const d of readdirSync(cacheDir))
if (/^\d{4}-\d{2}-\d{2}$/.test(d) && d < cutoff) rmSync(join(cacheDir, d), { recursive: true });
}

View File

@@ -0,0 +1,397 @@
// DB side of the grocery importer (used by scripts/groceries-import.ts). Uses a plain postgres.js
// client so it runs outside SvelteKit. All writes are idempotent; re-running is safe.
import type { Sql } from 'postgres';
import type { PoliteFetcher } from './http';
import { clusterItems, type MatchItem } from './match';
import {
brandClass,
brandLabel,
displayName,
matchKey,
nameTokens,
normaliseUnit
} from './normalize';
import { AhClient, type RawItem, type Source } from './sources';
type Log = (m: string) => void;
export async function storeId(sql: Sql, slug: string): Promise<number> {
const [s] = await sql<{ id: number }[]>`select id from groceries.stores where slug = ${slug}`;
if (!s) throw new Error(`unknown store ${slug}`);
return s.id;
}
function keyFor(name: string, brand: string | null, unit: string | null) {
const cls = brandClass(brand);
const tokens = nameTokens(name, brand);
return { cls, tokens, key: matchKey(cls, unit, tokens) };
}
/** Fetch a store's catalogue and upsert it; returns counts. */
export async function importStore(sql: Sql, source: Source, f: PoliteFetcher, log: Log) {
const sid = await storeId(sql, source.slug);
const [run] = await sql<{ id: number; started_at: Date }[]>`
insert into groceries.import_runs (store_id) values (${sid}) returning id, started_at`;
try {
const raw = await source.fetchAll(f, log);
const [prev] = await sql<{ n: number }[]>`
select count(*)::int as n from groceries.items where store_id = ${sid} and available`;
const res = await upsertItems(sql, sid, raw, run.started_at);
// Only mark unseen items unavailable when the crawl looks complete.
let delisted = 0;
if (raw.length >= prev.n * 0.8) {
const r = await sql`
update groceries.items set available = false
where store_id = ${sid} and available and last_seen_at < ${run.started_at}`;
delisted = r.count;
}
await sql`
update groceries.import_runs set finished_at = now(), status = 'ok', requests = ${f.requests},
fetched = ${raw.length}, inserted = ${res.inserted}, price_changes = ${res.priceChanges}
where id = ${run.id}`;
return { fetched: raw.length, ...res, delisted, requests: f.requests, cacheHits: f.cacheHits };
} catch (e) {
await sql`
update groceries.import_runs set finished_at = now(), status = 'error', requests = ${f.requests},
error = ${String(e instanceof Error ? e.message : e).slice(0, 2000)}
where id = ${run.id}`;
throw e;
}
}
async function upsertItems(sql: Sql, sid: number, raw: RawItem[], runStart: Date) {
let inserted = 0;
for (let i = 0; i < raw.length; i += 500) {
const rows = raw.slice(i, i + 500).map((r) => {
const n = normaliseUnit({
sizeText: r.sizeText,
title: r.name,
priceCents: r.priceCents,
promoPriceCents: r.promoPriceCents,
storeUnitPrice: r.storeUnitPrice,
storeUnitBasisCents: r.storeUnitBasisCents
});
return {
store_id: sid,
store_product_id: r.storeProductId,
name: r.name,
brand: r.brand,
size_text: r.sizeText,
unit: n.unit,
unit_amount: n.unitAmount,
unit_source: n.unitSource,
price_cents: r.priceCents,
promo_price_cents: r.promoPriceCents,
promo_label: r.promoLabel,
unit_price_cents: n.unitPriceCents,
promo_unit_price_cents: n.promoUnitPriceCents,
category: r.category,
category_path: r.categoryPath,
gtin: r.gtin,
image_url: r.imageUrl,
url: r.url,
available: r.available,
match_key: keyFor(r.name, r.brand, n.unit).key
};
});
const res = await sql<{ inserted: boolean }[]>`
insert into groceries.items ${sql(rows)}
on conflict (store_id, store_product_id) do update set
name = excluded.name, brand = excluded.brand, size_text = excluded.size_text,
unit = excluded.unit, unit_amount = excluded.unit_amount, unit_source = excluded.unit_source,
price_cents = excluded.price_cents, promo_price_cents = excluded.promo_price_cents,
promo_label = excluded.promo_label, unit_price_cents = excluded.unit_price_cents,
promo_unit_price_cents = excluded.promo_unit_price_cents, category = excluded.category,
category_path = excluded.category_path, gtin = coalesce(excluded.gtin, items.gtin),
image_url = excluded.image_url, url = excluded.url, available = excluded.available,
match_key = excluded.match_key, last_seen_at = now()
returning (xmax = 0) as inserted`;
inserted += res.filter((r) => r.inserted).length;
}
// Price history: a row when the price or promo differs from the item's last recorded row.
const hist = await sql<{ changed: boolean }[]>`
insert into groceries.item_prices (item_id, observed_on, price_cents, promo_price_cents, promo_label)
select i.id, (now() at time zone 'Europe/Amsterdam')::date, i.price_cents, i.promo_price_cents, i.promo_label
from groceries.items i
left join lateral (
select p.price_cents, p.promo_price_cents, p.promo_label from groceries.item_prices p
where p.item_id = i.id order by p.observed_on desc limit 1
) last on true
where i.store_id = ${sid} and i.last_seen_at >= ${runStart}
and (last is null
or last.price_cents is distinct from i.price_cents
or last.promo_price_cents is distinct from i.promo_price_cents
or last.promo_label is distinct from i.promo_label)
on conflict (item_id, observed_on) do update set
price_cents = excluded.price_cents, promo_price_cents = excluded.promo_price_cents,
promo_label = excluded.promo_label
returning exists (
select 1 from groceries.item_prices o
where o.item_id = item_prices.item_id and o.observed_on < item_prices.observed_on
) as changed`;
return { inserted, priceChanges: hist.filter((h) => h.changed).length };
}
/** Fill in AH EANs from the product detail endpoint, `limit` products per run (A-brands first). */
export async function enrichAhGtins(sql: Sql, f: PoliteFetcher, limit: number, log: Log) {
if (limit <= 0) return { checked: 0, found: 0 };
const sid = await storeId(sql, 'ah');
const todo = await sql<{ id: number; store_product_id: string }[]>`
select id, store_product_id from groceries.items
where store_id = ${sid} and gtin is null and available
and (gtin_checked_at is null or gtin_checked_at < now() - interval '90 days')
order by (lower(coalesce(brand, '')) like 'ah%') asc, gtin_checked_at asc nulls first, id
limit ${limit}`;
const ah = new AhClient(f);
let found = 0;
for (const [n, t] of todo.entries()) {
let gtin: string | null;
try {
gtin = await ah.gtin(t.store_product_id);
} catch (e) {
log(` ah gtin ${t.store_product_id}: ${e instanceof Error ? e.message : e}`);
if (n > 20 && found === 0) throw e; // something is wrong; stop hammering
continue;
}
if (gtin) found++;
await sql`update groceries.items set gtin = ${gtin}, gtin_checked_at = now() where id = ${t.id}`;
if ((n + 1) % 250 === 0) log(` ah gtin: ${n + 1}/${todo.length} checked, ${found} found`);
}
return { checked: todo.length, found };
}
// ---- regroup --------------------------------------------------------------------------------
type ItemRow = {
id: number;
store_id: number;
store_slug: string;
name: string;
brand: string | null;
unit: string | null;
gtin: string | null;
category: string | null;
group_id: number | null;
};
type GroupRow = { id: number; key: string | null; merged_into_id: number | null; manual: boolean };
/**
* Recompute groups for all unlocked items: cluster, map each cluster to a stable group (by key,
* following manual merges, else the group most of its items already had), create/update/delete
* groups, then refresh the denormalised stats. Locked items (moved by hand) are left alone.
*/
export async function regroup(sql: Sql, log: Log) {
return sql.begin(async (tx) => {
const items = await tx<ItemRow[]>`
select i.id, i.store_id, s.slug as store_slug, i.name, i.brand, i.unit, i.gtin, i.category, i.group_id
from groceries.items i join groceries.stores s on s.id = i.store_id
where not i.match_locked
order by i.id`;
const groups = await tx<
GroupRow[]
>`select id, key, merged_into_id, manual from groceries.groups`;
const byId = new Map(groups.map((g) => [g.id, g]));
const byKey = new Map(groups.filter((g) => g.key).map((g) => [g.key!, g]));
const resolve = (g: GroupRow | undefined) => {
for (let hops = 0; g?.merged_into_id && hops < 20; hops++) g = byId.get(g.merged_into_id);
return g;
};
const meta = new Map<number, ItemRow & { cls: string; tokens: string[]; key: string }>();
const matchItems: MatchItem[] = items.map((it) => {
const k = keyFor(it.name, it.brand, it.unit);
meta.set(it.id, { ...it, ...k });
return {
id: it.id,
storeId: it.store_id,
gtin: it.gtin,
key: k.key,
cls: k.cls,
unit: it.unit,
tokens: k.tokens
};
});
const clusters = clusterItems(matchItems);
const claimed = new Set<number>();
const keysInUse = new Set(clusters.map((c) => c.key));
type Plan = { cluster: (typeof clusters)[number]; groupId: number | null; info: GroupInfo };
const plans: Plan[] = [];
// Pass 1: exact key (or merged redirect).
for (const c of clusters) {
const g = resolve(byKey.get(c.key));
const info = groupInfo(
c.itemIds.map((id) => meta.get(id)!),
c.key
);
if (g) claimed.add(g.id);
plans.push({ cluster: c, groupId: g?.id ?? null, info });
}
// Pass 2: the group most items already belong to (keeps ids stable when names change).
for (const p of plans) {
if (p.groupId) continue;
const counts = new Map<number, number>();
for (const id of p.cluster.itemIds) {
const gid = meta.get(id)!.group_id;
if (gid) counts.set(gid, (counts.get(gid) ?? 0) + 1);
}
const best = [...counts].sort((a, b) => b[1] - a[1] || a[0] - b[0])[0];
const g = best ? byId.get(best[0]) : undefined;
if (
g &&
!g.manual &&
!g.merged_into_id &&
!claimed.has(g.id) &&
!(g.key && keysInUse.has(g.key))
) {
claimed.add(g.id);
p.groupId = g.id;
}
}
// Create missing groups.
const toCreate = plans.filter((p) => !p.groupId);
for (let i = 0; i < toCreate.length; i += 1000) {
const chunk = toCreate.slice(i, i + 1000);
const ids = await tx<{ id: number }[]>`
insert into groceries.groups ${tx(
chunk.map((p) => ({
key: p.cluster.key,
name: p.info.name,
brand: p.info.brand,
category: p.info.category,
unit: p.info.unit,
family_key: p.info.familyKey
}))
)} returning id`;
chunk.forEach((p, j) => (p.groupId = ids[j].id));
}
// Update automatic groups' descriptive fields (manual groups keep Stan's name).
const toUpdate = plans.filter((p) => !toCreate.includes(p) && !byId.get(p.groupId!)?.manual);
const seen = new Set<number>();
for (let i = 0; i < toUpdate.length; i += 1000) {
const rows = toUpdate
.slice(i, i + 1000)
.filter((p) => !seen.has(p.groupId!) && seen.add(p.groupId!))
.map((p) => {
const g = byId.get(p.groupId!)!;
// Only take the cluster key when no other group has it.
const key = g.key === p.cluster.key || !byKey.has(p.cluster.key) ? p.cluster.key : g.key;
return [
p.groupId!,
key,
p.info.name,
p.info.brand,
p.info.category,
p.info.unit,
p.info.familyKey
];
});
if (!rows.length) continue;
await tx`
update groceries.groups g set key = v.key, name = v.name, brand = v.brand,
category = v.category, unit = v.unit, family_key = v.family_key
from (values ${tx(rows as never)}) as v(id, key, name, brand, category, unit, family_key)
where g.id = v.id::int and (g.key is distinct from v.key or g.name is distinct from v.name
or g.brand is distinct from v.brand or g.category is distinct from v.category
or g.unit is distinct from v.unit or g.family_key is distinct from v.family_key)`;
}
// Assign items.
const assignments: [number, number, string, string, number | null][] = [];
for (const p of plans)
for (const id of p.cluster.itemIds) {
const m = p.cluster.method.get(id)!;
assignments.push([id, p.groupId!, meta.get(id)!.key, m.method, m.confidence]);
}
let moved = 0;
for (let i = 0; i < assignments.length; i += 2000) {
const r = await tx`
update groceries.items i set group_id = v.group_id::int, match_key = v.match_key,
match_method = v.method, match_confidence = v.confidence::real
from (values ${tx(assignments.slice(i, i + 2000) as never)}) as v(id, group_id, match_key, method, confidence)
where i.id = v.id::int and (i.group_id is distinct from v.group_id::int
or i.match_key is distinct from v.match_key or i.match_method is distinct from v.method
or i.match_confidence is distinct from v.confidence::real)`;
moved += r.count;
}
// Drop empty automatic groups (favourites, manual groups and merge tombstones stay).
const del = await tx`
delete from groceries.groups g
where not g.favorite and not g.manual and g.merged_into_id is null and g.notes is null
and not exists (select 1 from groceries.items i where i.group_id = g.id)
and not exists (select 1 from groceries.groups m where m.merged_into_id = g.id)`;
await refreshGroupStats(tx as unknown as Sql);
log(
`regroup: ${items.length} items in ${clusters.length} clusters; ${toCreate.length} groups created, ${moved} items (re)assigned, ${del.count} empty groups removed`
);
return { clusters: clusters.length, created: toCreate.length, moved, removed: del.count };
});
}
type GroupInfo = {
name: string;
brand: string | null;
category: string | null;
unit: string | null;
familyKey: string;
};
function groupInfo(
members: (ItemRow & { cls: string; tokens: string[]; key: string })[],
key: string
): GroupInfo {
// Representative: an item with the canonical key, preferring AH (cleanest titles), then Jumbo.
const order = (s: string) => (s === 'ah' ? 0 : s === 'jumbo' ? 1 : 2);
const rep =
[...members]
.filter((m) => m.key === key)
.sort(
(a, b) =>
order(a.store_slug) - order(b.store_slug) || a.name.length - b.name.length || a.id - b.id
)[0] ?? members[0];
const cats = new Map<string, number>();
for (const m of members) if (m.category) cats.set(m.category, (cats.get(m.category) ?? 0) + 1);
const category = [...cats].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1))[0]?.[0] ?? null;
return {
name: displayName(rep.name, rep.brand).slice(0, 200),
brand: brandLabel(rep.cls, rep.brand),
category,
unit: rep.unit,
familyKey: `${rep.unit ?? '?'}|${[...rep.tokens].sort().join(' ')}`
};
}
// Accent folding for search_text (no unaccent extension needed); the query side uses fold().
const ACCENTS_FROM = 'áàäâãéèëêíìïîóòöôõúùüûçñ';
const ACCENTS_TO = 'aaaaaeeeeiiiiooooouuuucn';
/** Recompute denormalised group stats (all groups, or just `ids`). */
export async function refreshGroupStats(sql: Sql, ids?: number[]) {
const only = ids ? sql`where g2.id = any(${ids}::int[])` : sql``;
await sql`
update groceries.groups g set
item_count = s.item_count, store_count = s.store_count, store_ids = s.store_ids,
min_unit_price_cents = s.min_unit, min_price_cents = s.min_price, promo_count = s.promo_count,
image_url = s.image_url, search_text = s.search_text
from (
select g2.id,
count(i.id)::int as item_count,
count(distinct i.store_id)::int as store_count,
coalesce(array_agg(distinct i.store_id) filter (where i.id is not null), '{}')::int[] as store_ids,
min(coalesce(i.promo_unit_price_cents, i.unit_price_cents)) as min_unit,
min(coalesce(i.promo_price_cents, i.price_cents)) as min_price,
(count(*) filter (where i.promo_price_cents is not null or i.promo_label is not null))::int as promo_count,
(array_agg(i.image_url order by i.store_id, i.id) filter (where i.image_url is not null))[1] as image_url,
translate(lower(concat_ws(' ', g2.name, g2.brand, g2.category,
string_agg(distinct concat_ws(' ', i.brand, i.name), ' '))), ${ACCENTS_FROM}, ${ACCENTS_TO}) as search_text
from groceries.groups g2
left join groceries.items i on i.group_id = g2.id and i.available
${only}
group by g2.id
) s
where g.id = s.id and (g.item_count, g.store_count, g.store_ids, g.min_unit_price_cents, g.min_price_cents,
g.promo_count, g.image_url, g.search_text) is distinct from
(s.item_count, s.store_count, s.store_ids, s.min_unit, s.min_price, s.promo_count, s.image_url, s.search_text)`;
}

View File

@@ -0,0 +1,203 @@
// Heuristic product grouping (no ML). Items are clustered by:
// 1. the same match key (brand class + unit + name tokens): one product in several pack sizes,
// and AH/Jumbo/Plus house brands as one "Huismerk" product;
// 2. the same EAN/GTIN (an A-brand product at several stores);
// 3. fuzzy name similarity (token Jaccard >= 0.75) between clusters of different stores with the
// same brand class and unit, greedily best-first, never joining two items of one store
// that came from different clusters.
export type MatchItem = {
id: number;
storeId: number;
gtin: string | null;
key: string;
cls: string;
unit: string | null;
tokens: string[];
};
export type ClusterResult = {
key: string; // canonical key: the key of the cluster's largest exact-key part
itemIds: number[];
method: Map<number, { method: 'ean' | 'key' | 'fuzzy' | 'single'; confidence: number | null }>;
};
export const FUZZY_THRESHOLD = 0.75;
class UnionFind {
parent: number[];
constructor(n: number) {
this.parent = Array.from({ length: n }, (_, i) => i);
}
find(i: number): number {
while (this.parent[i] !== i) {
this.parent[i] = this.parent[this.parent[i]];
i = this.parent[i];
}
return i;
}
union(a: number, b: number) {
const ra = this.find(a);
const rb = this.find(b);
if (ra !== rb) this.parent[rb] = ra;
return ra;
}
}
export function jaccard(a: string[], b: string[]): number {
if (!a.length || !b.length) return 0;
const sb = new Set(b);
let inter = 0;
for (const t of new Set(a)) if (sb.has(t)) inter++;
return inter / new Set([...a, ...b]).size;
}
/** Most common value; ties → lexicographically smallest (deterministic). */
function mode(values: string[]): string {
const counts = new Map<string, number>();
for (const v of values) counts.set(v, (counts.get(v) ?? 0) + 1);
let best = values[0];
for (const [v, c] of counts) {
const bc = counts.get(best)!;
if (c > bc || (c === bc && v < best)) best = v;
}
return best;
}
export function clusterItems(items: MatchItem[]): ClusterResult[] {
const n = items.length;
const uf = new UnionFind(n);
const byKey = new Map<string, number>();
const byGtin = new Map<string, number>();
const eanLinked = new Set<number>();
for (let i = 0; i < n; i++) {
const it = items[i];
const k = byKey.get(it.key);
if (k === undefined) byKey.set(it.key, i);
else uf.union(k, i);
}
for (let i = 0; i < n; i++) {
const g = items[i].gtin;
if (!g) continue;
const j = byGtin.get(g);
if (j === undefined) byGtin.set(g, i);
else if (uf.find(j) !== uf.find(i) || items[i].storeId !== items[j].storeId) {
uf.union(j, i);
eanLinked.add(i).add(j);
}
}
// Base clusters after exact key + EAN.
type Base = {
members: number[];
stores: Set<number>;
key: string;
tokens: string[];
cls: string;
unit: string | null;
};
const baseOf = new Map<number, Base>();
for (let i = 0; i < n; i++) {
const r = uf.find(i);
let b = baseOf.get(r);
if (!b) {
b = { members: [], stores: new Set(), key: '', tokens: [], cls: '', unit: null };
baseOf.set(r, b);
}
b.members.push(i);
b.stores.add(items[i].storeId);
}
const bases = [...baseOf.values()];
for (const b of bases) {
b.key = mode(b.members.map((i) => items[i].key));
const rep = items[b.members.find((i) => items[i].key === b.key)!];
b.tokens = rep.tokens;
b.cls = rep.cls;
b.unit = rep.unit;
}
// Fuzzy pass between base clusters with the same brand class + unit.
const df = new Map<string, number>();
for (const b of bases) for (const t of b.tokens) df.set(t, (df.get(t) ?? 0) + 1);
const postings = new Map<string, number[]>();
bases.forEach((b, bi) => {
for (const t of b.tokens) {
const k = `${b.cls}|${b.unit}|${t}`;
const list = postings.get(k);
if (list) list.push(bi);
else postings.set(k, [bi]);
}
});
const pairs: { a: number; b: number; score: number }[] = [];
bases.forEach((b, bi) => {
if (b.tokens.length < 2) return;
const rare = [...b.tokens]
.sort((x, y) => df.get(x)! - df.get(y)! || (x < y ? -1 : 1))
.slice(0, 2);
const seen = new Set<number>();
for (const t of rare) {
for (const ci of postings.get(`${b.cls}|${b.unit}|${t}`) ?? []) {
if (ci <= bi || seen.has(ci)) continue;
seen.add(ci);
const c = bases[ci];
if (c.tokens.length < 2) continue;
if ([...b.stores].some((s) => c.stores.has(s))) continue;
const score = jaccard(b.tokens, c.tokens);
if (score >= FUZZY_THRESHOLD) pairs.push({ a: bi, b: ci, score });
}
}
});
pairs.sort((x, y) => y.score - x.score || x.a - y.a || x.b - y.b);
const buf = new UnionFind(bases.length);
const storesOf = bases.map((b) => new Set(b.stores));
const fuzzyScore = new Map<number, number>(); // base index → score it was attached with
for (const p of pairs) {
const ra = buf.find(p.a);
const rb = buf.find(p.b);
if (ra === rb) continue;
if ([...storesOf[ra]].some((s) => storesOf[rb].has(s))) continue;
const root = buf.union(ra, rb);
const other = root === ra ? rb : ra;
for (const s of storesOf[other]) storesOf[root].add(s);
// The smaller side is the one that was "attached".
const attached = bases[p.a].members.length >= bases[p.b].members.length ? p.b : p.a;
if (!fuzzyScore.has(attached)) fuzzyScore.set(attached, p.score);
}
const finals = new Map<number, number[]>();
bases.forEach((_, bi) => {
const r = buf.find(bi);
const list = finals.get(r);
if (list) list.push(bi);
else finals.set(r, [bi]);
});
const out: ClusterResult[] = [];
for (const parts of finals.values()) {
parts.sort(
(x, y) =>
bases[y].members.length - bases[x].members.length || (bases[x].key < bases[y].key ? -1 : 1)
);
const ids: number[] = [];
const method: ClusterResult['method'] = new Map();
const stores = new Set<number>();
for (const bi of parts) for (const i of bases[bi].members) stores.add(items[i].storeId);
for (const bi of parts) {
const fs = fuzzyScore.get(bi);
for (const i of bases[bi].members) {
ids.push(items[i].id);
method.set(
items[i].id,
stores.size < 2
? { method: 'single', confidence: null }
: fs !== undefined
? { method: 'fuzzy', confidence: Math.round(fs * 0.9 * 100) / 100 }
: eanLinked.has(i)
? { method: 'ean', confidence: 1 }
: { method: 'key', confidence: 0.9 }
);
}
}
out.push({ key: bases[parts[0]].key, itemIds: ids, method });
}
return out;
}

View File

@@ -0,0 +1,151 @@
import { describe, expect, it } from 'vitest';
import {
brandClass,
displayName,
matchKey,
nameTokens,
normaliseGtin,
normaliseUnit,
parseAhUnitPrice,
parseSize
} from './normalize';
import { clusterItems, type MatchItem } from './match';
describe('parseSize', () => {
it.each([
['500 g', { amount: 0.5, unit: 'kg' }],
['1 l', { amount: 1, unit: 'l' }],
['0,5 l', { amount: 0.5, unit: 'l' }],
['6 x 330 ml', { amount: 1.98, unit: 'l' }],
['6x1,5L', { amount: 9, unit: 'l' }],
['4 stuks', { amount: 4, unit: 'piece' }],
['per stuk', { amount: 1, unit: 'piece' }],
['per bosje', { amount: 1, unit: 'piece' }],
['per kilo', { amount: 1, unit: 'kg' }],
['ca. 600 g', { amount: 0.6, unit: 'kg' }],
['300 GRAM', { amount: 0.3, unit: 'kg' }],
['10 stuks 360 g', { amount: 0.36, unit: 'kg' }],
['Jumbo Halfvolle Melk 1,5L', { amount: 1.5, unit: 'l' }],
['Jumbo Dole Bananen 5 Stuks', { amount: 5, unit: 'piece' }]
])('%s', (input, expected) => {
const r = parseSize(input)!;
expect(r.unit).toBe(expected.unit);
expect(r.amount).toBeCloseTo(expected.amount, 6);
});
it('returns null when unclear', () => {
expect(parseSize('Jumbo Bloemkool')).toBeNull();
expect(parseSize(null)).toBeNull();
});
});
describe('unit prices', () => {
it('parses AH unit price text', () => {
expect(parseAhUnitPrice('normale prijs per kg €12.98')).toEqual({ cents: 1298, unit: 'kg' });
expect(parseAhUnitPrice('prijs per liter €1.49')).toEqual({ cents: 149, unit: 'l' });
expect(parseAhUnitPrice(null)).toBeNull();
});
it('uses the parsed size and computes promo unit price', () => {
expect(normaliseUnit({ sizeText: '500 g', priceCents: 125, promoPriceCents: 100 })).toEqual({
unit: 'kg',
unitAmount: 0.5,
unitSource: 'size',
unitPriceCents: 250,
promoUnitPriceCents: 200
});
});
it("derives the amount from the store's unit price when the size is unknown", () => {
// Jumbo bases price per unit on the promo price.
const n = normaliseUnit({
title: 'Jumbo Bloemkool',
priceCents: 169,
promoPriceCents: 99,
storeUnitPrice: { cents: 99, unit: 'piece' },
storeUnitBasisCents: 99
});
expect(n).toMatchObject({
unit: 'piece',
unitAmount: 1,
unitSource: 'store',
unitPriceCents: 169
});
});
it('prefers the store unit when the parsed unit disagrees', () => {
const n = normaliseUnit({
sizeText: '3 stuks',
priceCents: 300,
storeUnitPrice: { cents: 600, unit: 'kg' }
});
expect(n).toMatchObject({ unit: 'kg', unitAmount: 0.5, unitPriceCents: 600 });
});
});
describe('names and brands', () => {
it('maps store brands to house classes', () => {
expect(brandClass('AH')).toBe('house');
expect(brandClass('Jumbo')).toBe('house');
expect(brandClass('PLUS')).toBe('house');
expect(brandClass('AH Biologisch')).toBe('house-bio');
expect(brandClass(null)).toBe('house');
expect(brandClass('Calvé')).toBe('calve');
});
it('tokenises without brand, size and packaging', () => {
expect(nameTokens('AH Halfvolle melk', 'AH')).toEqual(['halfvolle', 'melk']);
expect(nameTokens('Jumbo Halfvolle Melk Voordeelverpakking 1,5L', 'Jumbo')).toEqual([
'halfvolle',
'melk'
]);
expect(nameTokens('Calvé Pindakaas 350g', 'Calvé')).toEqual(['pindakaa']);
});
it('makes display names', () => {
expect(displayName('AH Halfvolle melk', 'AH')).toBe('Halfvolle melk');
expect(displayName('Jumbo Pitloze Druiven Wit 500 g', 'Jumbo')).toBe('Pitloze Druiven Wit');
});
it('normalises GTINs', () => {
expect(normaliseGtin('08719200252127')).toBe('8719200252127');
expect(normaliseGtin('8719200252127')).toBe('8719200252127');
expect(normaliseGtin('12')).toBeNull();
});
});
describe('clusterItems', () => {
const item = (
id: number,
storeId: number,
name: string,
brand: string,
unit = 'l',
gtin: string | null = null
): MatchItem => {
const cls = brandClass(brand);
const tokens = nameTokens(name, brand);
return { id, storeId, gtin, key: matchKey(cls, unit, tokens), cls, unit, tokens };
};
const groupOf = (r: ReturnType<typeof clusterItems>, id: number) =>
r.find((c) => c.itemIds.includes(id))!;
it('groups house brands and sizes across stores, keeps A-brands apart', () => {
const r = clusterItems([
item(1, 1, 'AH Halfvolle melk', 'AH'),
item(2, 1, 'AH Halfvolle melk 1,5 l', 'AH'),
item(3, 2, 'Jumbo Halfvolle Melk 1L', 'Jumbo'),
item(4, 2, 'Campina Halfvolle melk', 'Campina'),
item(5, 1, 'AH Halfvolle melk lactosevrij', 'AH')
]);
expect(groupOf(r, 1).itemIds.sort()).toEqual([1, 2, 3]);
expect(groupOf(r, 3).method.get(3)).toEqual({ method: 'key', confidence: 0.9 });
expect(groupOf(r, 4).itemIds).toEqual([4]);
expect(groupOf(r, 5).itemIds).toEqual([5]);
});
it('links by EAN and fuzzy names across stores', () => {
const r = clusterItems([
item(1, 1, 'Calvé Pindakaas', 'Calvé', 'kg', '8722700055525'),
item(2, 2, 'Calvé Pindakaas Original', 'Calve', 'kg', '8722700055525'),
item(3, 1, 'AH Spaghetti volkoren pasta', 'AH', 'kg'),
item(4, 2, 'Jumbo Volkoren Spaghetti Pasta Italiaans', 'Jumbo', 'kg')
]);
expect(groupOf(r, 1).itemIds.sort()).toEqual([1, 2]);
expect(groupOf(r, 1).method.get(2)).toEqual({ method: 'ean', confidence: 1 });
expect(groupOf(r, 3).itemIds.sort()).toEqual([3, 4]);
expect(groupOf(r, 3).method.get(4)?.method).toBe('fuzzy');
});
});

View File

@@ -0,0 +1,361 @@
// Pure normalisation helpers for the grocery importer and matcher (no DB, no network).
import type { GroceryUnit } from '../db/schema/groceries';
export type Size = { amount: number; unit: GroceryUnit };
const MASS: Record<string, number> = {
kg: 1,
kilo: 1,
kilogram: 1,
g: 0.001,
gr: 0.001,
gram: 0.001,
mg: 0.000001
};
const VOLUME: Record<string, number> = {
l: 1,
ltr: 1,
liter: 1,
litre: 1,
dl: 0.1,
cl: 0.01,
ml: 0.001
};
const COUNT = new Set([
'st',
'stk',
'stuk',
'stuks',
'stuk(s)',
'pcs',
'piece',
'pieces',
'rol',
'rollen',
'zakjes',
'zakje',
'tabletten',
'tablet',
'capsules',
'capsule',
'cups',
'pads',
'sachets',
'doekjes',
'luiers',
'wasbeurten',
'eieren',
'plakken',
'porties',
'repen',
'blikjes',
'flesjes',
'zakken',
'bollen',
'tabs',
'vellen',
'servetten',
'theezakjes',
'paar'
]);
const UNIT_RE =
'kilogram|kilo|kg|gram|gr|mg|g|liter|litre|ltr|ml|cl|dl|l|stuk\\(s\\)|stuks|stuk|stk|st|pcs|pieces|piece|rollen|rol|zakjes|zakje|tabletten|tablet|capsules|capsule|cups|pads|sachets|doekjes|luiers|wasbeurten|eieren|plakken|porties|repen|blikjes|flesjes|zakken|bollen|tabs|vellen|servetten|theezakjes|paar';
const QTY = new RegExp(`(?:(\\d+)\\s*[x×]\\s*)?(\\d+(?:[.,]\\d+)?)\\s*(${UNIT_RE})(?![a-z])`, 'gi');
function toSize(mult: number, n: number, unit: string): Size | null {
const u = unit.toLowerCase();
const total = mult * n;
if (!(total > 0)) return null;
if (u in MASS) return { amount: total * MASS[u], unit: 'kg' };
if (u in VOLUME) return { amount: total * VOLUME[u], unit: 'l' };
if (COUNT.has(u)) return { amount: total, unit: 'piece' };
return null;
}
/**
* Pack size → base unit, e.g. "500 g" → 0.5 kg, "6 x 330 ml" → 1.98 l, "4 stuks" → 4 piece,
* "per stuk" → 1 piece, "per kilo" → 1 kg. Weight/volume beats a count ("6 stuks 360 g" → kg).
* Works on titles too ("Jumbo Halfvolle Melk 1,5L"). Null when nothing usable.
*/
export function parseSize(text: string | null | undefined): Size | null {
if (!text) return null;
const s = text
.toLowerCase()
.replace(/ca\.?\s*/g, '')
.replace(/\s+/g, ' ');
let best: Size | null = null;
for (const m of s.matchAll(QTY)) {
const size = toSize(m[1] ? Number(m[1]) : 1, Number(m[2].replace(',', '.')), m[3]);
if (!size) continue;
if (!best || (best.unit === 'piece' && size.unit !== 'piece')) best = size;
if (best.unit !== 'piece') break;
}
if (best) return best;
if (/\bper\s*(kilo|kg)\b/.test(s)) return { amount: 1, unit: 'kg' };
if (/\bper\s*(liter|l)\b/.test(s)) return { amount: 1, unit: 'l' };
if (
/\bper\s*(stuk|bos|bosje|bundel|krop|doos|pak|tros|net|schaal|bak|pot|zak|paar|set)\b/.test(s)
)
return { amount: 1, unit: 'piece' };
if (/^\s*(\d+)\s*(st|stuks)?\s*$/.test(s))
return { amount: Number(s.trim().split(/\s/)[0]), unit: 'piece' };
return null;
}
/** Store unit labels ("kg", "l", "pieces", "stuk", "liter") → our unit. */
export function storeUnit(label: string | null | undefined): GroceryUnit | null {
const u = (label ?? '').toLowerCase().trim();
if (['kg', 'kilo', 'kilogram'].includes(u)) return 'kg';
if (['l', 'liter', 'litre', 'ltr'].includes(u)) return 'l';
if (['piece', 'pieces', 'stuk', 'stuks', 'st', 'pce', 'pc'].includes(u)) return 'piece';
return null;
}
/** AH "prijs per kg €12.98" / "normale prijs per liter €1.49" → { cents, unit }. */
export function parseAhUnitPrice(
desc: string | null | undefined
): { cents: number; unit: GroceryUnit } | null {
if (!desc) return null;
const m = desc.toLowerCase().match(/per\s+(kg|kilo|liter|l|stuk|st)\s*€?\s*(\d+(?:[.,]\d+)?)/);
if (!m) return null;
const unit = storeUnit(m[1]);
return unit ? { cents: Math.round(Number(m[2].replace(',', '.')) * 100), unit } : null;
}
export type Normalised = {
unit: GroceryUnit | null;
unitAmount: number | null;
unitSource: 'size' | 'store' | null;
unitPriceCents: number | null;
promoUnitPriceCents: number | null;
};
/**
* Pick the unit and pack amount: the parsed size when it agrees with the store's unit price
* (or the store gives none); else derive the amount from the store's unit price.
*/
export function normaliseUnit(opts: {
sizeText?: string | null;
title?: string | null;
priceCents: number | null;
promoPriceCents?: number | null;
storeUnitPrice?: { cents: number; unit: GroceryUnit } | null;
/** Price the store's unit price was computed from (Jumbo uses the promo price). */
storeUnitBasisCents?: number | null;
}): Normalised {
const { priceCents, storeUnitPrice } = opts;
let size = parseSize(opts.sizeText) ?? parseSize(opts.title);
let source: Normalised['unitSource'] = size ? 'size' : null;
const basis = opts.storeUnitBasisCents ?? priceCents;
if (storeUnitPrice && storeUnitPrice.cents > 0 && basis) {
if (!size || size.unit !== storeUnitPrice.unit) {
const derived = basis / storeUnitPrice.cents;
const amount =
storeUnitPrice.unit === 'piece'
? Math.max(1, Math.round(derived))
: Math.round(derived * 1000) / 1000;
size = amount > 0 ? { amount, unit: storeUnitPrice.unit } : size;
source = 'store';
}
}
if (!size || !(size.amount > 0)) {
return {
unit: null,
unitAmount: null,
unitSource: null,
unitPriceCents: null,
promoUnitPriceCents: null
};
}
const per = (c: number | null | undefined) =>
c != null && c > 0 ? Math.round(c / size.amount) : null;
return {
unit: size.unit,
unitAmount: Math.round(size.amount * 10000) / 10000,
unitSource: source,
unitPriceCents: per(priceCents),
promoUnitPriceCents: per(opts.promoPriceCents)
};
}
// ---- names, brands, keys -------------------------------------------------------------------
export const fold = (s: string) =>
s
.normalize('NFD')
.replace(/[\u0300-\u036f]/g, '')
.toLowerCase();
/** Store own brands, mapped to a shared class so AH/Jumbo/Plus equivalents group together. */
const HOUSE: Record<string, string> = {
ah: 'house',
'albert heijn': 'house',
jumbo: 'house',
plus: 'house',
'ah biologisch': 'house-bio',
'jumbo biologisch': 'house-bio',
'plus biologisch': 'house-bio',
'biologisch plus': 'house-bio',
'ah basic': 'house-basic',
'jumbo basic': 'house-basic',
'plus basic': 'house-basic',
"g'woon": 'house-basic',
'ah excellent': 'house-premium',
'jumbo excellent': 'house-premium',
'plus excellent': 'house-premium',
'jumbo lekker puur': 'house-premium',
'ah terra': 'house-veg',
'jumbo veggie': 'house-veg',
'ah kids': 'house-kids',
'ah puur&eerlijk': 'house-bio',
'ah puur & eerlijk': 'house-bio'
};
const HOUSE_LABEL: Record<string, string> = {
house: 'Huismerk',
'house-bio': 'Huismerk biologisch',
'house-basic': 'Huismerk basic',
'house-premium': 'Huismerk premium',
'house-veg': 'Huismerk plantaardig',
'house-kids': 'Huismerk kids'
};
const STORE_WORDS = ['albert heijn', 'ah', 'jumbo', 'plus'];
export function brandClass(brand: string | null | undefined): string {
const b = fold(brand ?? '')
.trim()
.replace(/\s+/g, ' ');
if (!b) return 'house'; // loose produce without a brand
return HOUSE[b] ?? b.replace(/[^a-z0-9]+/g, '');
}
export const isHouse = (cls: string) => cls.startsWith('house');
export const brandLabel = (cls: string, brand: string | null | undefined) =>
HOUSE_LABEL[cls] ?? brand?.trim() ?? null;
const STOP = new Set([
'de',
'het',
'een',
'van',
'met',
'en',
'in',
'voor',
'per',
'x',
'ca',
'op',
'of',
'a',
'la',
'le',
'stuks',
'stuk',
'st',
'pak',
'fles',
'blik',
'pot',
'zak',
'doos',
'tray',
'bak',
'beker',
'kuipje',
'voordeelverpakking',
'voordeelpak',
'voordeel',
'multipack',
'grootverpakking',
'xl',
'xxl',
'navulling',
'family',
'familiepak',
'mega',
'pack',
'literpak',
'petfles',
'tube',
'tetra',
'brik'
]);
const SIZE_IN_NAME = new RegExp(
`(?:\\d+\\s*[x×]\\s*)?\\d+(?:[.,]\\d+)?\\s*(?:${UNIT_RE})(?![a-z])`,
'gi'
);
const stem = (t: string) =>
t.length > 5 && t.endsWith('en')
? t.slice(0, -2)
: t.length > 4 && t.endsWith('s')
? t.slice(0, -1)
: t;
/** Meaningful name tokens: no brand, store name, size, packaging or stop words. */
export function nameTokens(title: string, brand?: string | null): string[] {
let s = fold(title).replace(/&/g, ' en ').replace(SIZE_IN_NAME, ' ');
const drop = new Set<string>();
for (const w of [brand ?? '', ...STORE_WORDS])
for (const t of fold(w).split(/[^a-z0-9]+/)) if (t) drop.add(t);
s = s.replace(/['’]/g, '').replace(/[^a-z0-9%+]+/g, ' ');
const out: string[] = [];
for (const raw of s.split(' ')) {
if (!raw || drop.has(raw) || STOP.has(raw)) continue;
if (/^\d+$/.test(raw) && raw.length > 2) continue; // stray numbers (article codes)
const t = stem(raw);
if (!out.includes(t)) out.push(t);
}
return out;
}
/** Display name without brand/store prefix and size: "AH Halfvolle melk 1 L" → "Halfvolle melk". */
export function displayName(title: string, brand?: string | null): string {
let s = title.replace(SIZE_IN_NAME, ' ').replace(/\s+-\s*$/, '');
for (const w of [brand ?? '', ...STORE_WORDS]
.filter(Boolean)
.sort((a, b) => b.length - a.length)) {
const re = new RegExp(`^\\s*${w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b\\s*`, 'i');
s = s.replace(re, '');
}
s = s
.replace(/\s+/g, ' ')
.replace(/[\s,-]+$/, '')
.trim();
return s ? s[0].toUpperCase() + s.slice(1) : title.trim();
}
export function matchKey(cls: string, unit: string | null, tokens: string[]): string {
return `${cls}|${unit ?? '?'}|${[...tokens].sort().join(' ')}`;
}
/** EAN-13 / GTIN-14 → canonical 13-digit EAN (leading zero of GTIN-14 dropped). */
export function normaliseGtin(v: string | null | undefined): string | null {
const d = (v ?? '').replace(/\D/g, '');
if (d.length < 8) return null;
const t = d.replace(/^0+(?=\d{13}$)/, '');
return t.length > 14 ? null : t.padStart(13, '0');
}
// ---- categories -------------------------------------------------------------------------------
/** Shared category names; each store's top categories map onto these. */
export const CATEGORIES = [
'Groente, fruit & aardappelen',
'Maaltijden & gemak',
'Vlees, vis & vega',
'Vleeswaren, kaas & tapas',
'Zuivel & eieren',
'Brood & bakkerij',
'Ontbijt & beleg',
'Pasta, rijst & wereldkeuken',
'Soepen, sauzen & conserven',
'Snoep, koek & chips',
'Diepvries',
'Koffie & thee',
'Frisdrank & sappen',
'Bier & wijn',
'Drogisterij',
'Huishouden & dieren',
'Baby & kind',
'Overig'
] as const;

View File

@@ -0,0 +1,352 @@
// Store catalogue sources. Each returns the full assortment as RawItem[].
// - Albert Heijn: the anonymous mobile-app API (api.ah.nl), crawled per top category.
// - Jumbo: the public GraphQL search endpoint used by jumbo.com, crawled per top category.
// - Plus: not available from the server (plus.nl blocks datacenter IPs via Incapsula).
import type { GroceryUnit } from '../db/schema/groceries';
import { HttpError, PoliteFetcher, USER_AGENT } from './http';
import { normaliseGtin, parseAhUnitPrice, storeUnit } from './normalize';
export type RawItem = {
storeProductId: string;
name: string;
brand: string | null;
sizeText: string | null;
priceCents: number | null;
promoPriceCents: number | null;
promoLabel: string | null;
storeUnitPrice: { cents: number; unit: GroceryUnit } | null;
storeUnitBasisCents: number | null;
category: string;
categoryPath: string | null;
gtin: string | null;
imageUrl: string | null;
url: string | null;
available: boolean;
};
export type Source = {
slug: string;
fetchAll(f: PoliteFetcher, log: (m: string) => void): Promise<RawItem[]>;
};
const cents = (euros: unknown) =>
typeof euros === 'number' && Number.isFinite(euros) ? Math.round(euros * 100) : null;
// ---- Albert Heijn ----------------------------------------------------------------------------
const AH_API = 'https://api.ah.nl';
const AH_HEADERS = {
'User-Agent': `Appie/8.22.3 (${USER_AGENT})`,
'x-application': 'AHWEBSHOP',
'Content-Type': 'application/json',
Accept: 'application/json'
};
/** AH top category id → shared category. Unknown ids fall back to 'Overig'. */
const AH_CATEGORIES: Record<number, string> = {
6401: 'Groente, fruit & aardappelen',
20885: 'Groente, fruit & aardappelen',
1301: 'Maaltijden & gemak',
9344: 'Vlees, vis & vega',
1651: 'Vlees, vis & vega',
20128: 'Vlees, vis & vega',
5481: 'Vleeswaren, kaas & tapas',
1192: 'Vleeswaren, kaas & tapas',
1730: 'Zuivel & eieren',
1355: 'Brood & bakkerij',
20824: 'Snoep, koek & chips',
1796: 'Pasta, rijst & wereldkeuken',
6409: 'Soepen, sauzen & conserven',
20129: 'Snoep, koek & chips',
6405: 'Ontbijt & beleg',
2457: 'Snoep, koek & chips',
5881: 'Diepvries',
1043: 'Koffie & thee',
20130: 'Frisdrank & sappen',
6406: 'Bier & wijn',
1045: 'Drogisterij',
11717: 'Drogisterij',
1165: 'Huishouden & dieren',
18521: 'Baby & kind',
18519: 'Huishouden & dieren',
1057: 'Overig',
4246: 'Overig', // Glutenvrij: crawled last so products keep their real category
44025: 'Overig' // Bloemenshop
};
type AhProduct = {
webshopId: number;
title: string;
brand?: string | null;
salesUnitSize?: string | null;
unitPriceDescription?: string | null;
images?: { width: number; url: string }[];
priceBeforeBonus?: number | null;
currentPrice?: number | null;
isBonus?: boolean;
bonusMechanism?: string | null;
mainCategory?: string | null;
subCategory?: string | null;
orderAvailabilityStatus?: string | null;
availableOnline?: boolean;
};
export class AhClient {
private token: string | null = null;
constructor(private f: PoliteFetcher) {}
private async auth() {
const r = await this.f.json<{ access_token: string }>(
`${AH_API}/mobile-auth/v1/auth/token/anonymous`,
{ method: 'POST', headers: AH_HEADERS, body: JSON.stringify({ clientId: 'appie' }) }
);
this.token = r.access_token;
}
async get<T>(path: string, cacheKey?: string): Promise<T> {
if (!this.token) await this.auth();
const run = () =>
this.f.json<T>(
`${AH_API}${path}`,
{ headers: { ...AH_HEADERS, Authorization: `Bearer ${this.token}` } },
cacheKey
);
try {
return await run();
} catch (e) {
if (e instanceof HttpError && e.status === 401) {
await this.auth();
return run();
}
throw e;
}
}
/** GTIN from the product detail endpoint (search results have none). */
async gtin(webshopId: string): Promise<string | null> {
const d = await this.get<unknown>(
`/mobile-services/product/detail/v4/fir/${encodeURIComponent(webshopId)}`
);
return normaliseGtin(findKey(d, 'gtin'));
}
}
function findKey(o: unknown, key: string, depth = 0): string | null {
if (!o || typeof o !== 'object' || depth > 6) return null;
for (const [k, v] of Object.entries(o)) {
if (k === key && (typeof v === 'string' || typeof v === 'number')) return String(v);
const r = findKey(v, key, depth + 1);
if (r) return r;
}
return null;
}
function ahItem(p: AhProduct, category: string): RawItem {
const before = cents(p.priceBeforeBonus);
const now = cents(p.currentPrice);
const promoPrice = p.isBonus && now != null && before != null && now < before ? now : null;
const img =
p.images?.find((i) => i.width === 200) ??
p.images?.find((i) => i.width === 400) ??
p.images?.[0];
return {
storeProductId: String(p.webshopId),
name: p.title.trim(),
brand: p.brand?.trim() || null,
sizeText: p.salesUnitSize?.trim() || null,
priceCents: before ?? now,
promoPriceCents: promoPrice,
promoLabel: p.bonusMechanism?.trim() || null,
storeUnitPrice: parseAhUnitPrice(p.unitPriceDescription),
storeUnitBasisCents: before ?? now, // "normale prijs per kg" is the pre-bonus price
category,
categoryPath: [p.mainCategory, p.subCategory].filter(Boolean).join(' > ') || null,
gtin: null,
imageUrl: img?.url ?? null,
url: `https://www.ah.nl/producten/product/wi${p.webshopId}`,
available: p.orderAvailabilityStatus ? p.orderAvailabilityStatus === 'IN_ASSORTMENT' : true
};
}
/** AH search only pages through the first 3000 hits; bigger categories are crawled per child. */
const AH_WINDOW = 3000;
type AhPage = { products?: AhProduct[]; page?: { totalPages: number; totalElements: number } };
async function crawlAhCategory(
ah: AhClient,
id: number,
category: string,
items: Map<string, RawItem>,
depth = 0
): Promise<void> {
const url = (page: number) =>
`/mobile-services/product/search/v2?taxonomyId=${id}&size=1000&page=${page}&sortOn=RELEVANCE`;
const first = await ah.get<AhPage>(url(0), `cat-${id}-p0`);
const total = first.page?.totalElements ?? 0;
if (total > AH_WINDOW && depth < 3) {
const sub = await ah.get<{ children?: { id: number }[] }>(
`/mobile-services/v1/product-shelves/categories/${id}/sub-categories`,
`sub-${id}`
);
if (sub.children?.length) {
for (const c of sub.children) await crawlAhCategory(ah, c.id, category, items, depth + 1);
// Products only listed on the parent: whatever the first 3000 hits add.
}
}
const pages = Math.min(first.page?.totalPages ?? 1, Math.floor(AH_WINDOW / 1000));
for (let page = 0; page < pages; page++) {
const r = page === 0 ? first : await ah.get<AhPage>(url(page), `cat-${id}-p${page}`);
for (const p of r.products ?? []) {
if (!p.webshopId || !p.title) continue;
const key = String(p.webshopId);
if (!items.has(key)) items.set(key, ahItem(p, category));
}
if (!r.products?.length) break;
}
}
export const ahSource: Source = {
slug: 'ah',
async fetchAll(f, log) {
const ah = new AhClient(f);
const cats = await ah.get<{ id: number; name: string }[]>(
'/mobile-services/v1/product-shelves/categories',
'categories'
);
cats.sort((a, b) => Number(a.id === 4246) - Number(b.id === 4246));
const items = new Map<string, RawItem>();
for (const c of cats) {
const before = items.size;
await crawlAhCategory(ah, c.id, AH_CATEGORIES[c.id] ?? 'Overig', items);
log(` ah ${c.name}: +${items.size - before} (${items.size} total)`);
}
return [...items.values()];
}
};
// ---- Jumbo -----------------------------------------------------------------------------------
const JUMBO_GQL = 'https://www.jumbo.com/api/graphql';
const JUMBO_HEADERS = {
'content-type': 'application/json',
'user-agent': USER_AGENT,
'apollographql-client-name': 'JUMBO_WEB-search',
'apollographql-client-version': 'master-v29.4.0-web',
'x-source': 'JUMBO_WEB-search'
};
/** Jumbo top category friendlyUrl → shared category (themed/seasonal roots skipped: subsets). */
const JUMBO_ROOTS: [string, string][] = [
['aardappelen,-groente-en-fruit', 'Groente, fruit & aardappelen'],
['verse-maaltijden-en-gemak', 'Maaltijden & gemak'],
['vlees,-vis-en-vega', 'Vlees, vis & vega'],
['vega-en-plantaardig', 'Vlees, vis & vega'],
['vleeswaren,-kaas-en-tapas', 'Vleeswaren, kaas & tapas'],
['zuivel,-boter-en-eieren', 'Zuivel & eieren'],
['brood-en-gebak', 'Brood & bakkerij'],
['ontbijt,-broodbeleg-en-bakproducten', 'Ontbijt & beleg'],
['wereldkeukens,-kruiden,-pasta-en-rijst', 'Pasta, rijst & wereldkeuken'],
['conserven,-soepen,-sauzen,-olien', 'Soepen, sauzen & conserven'],
['koek,-snoep,-chocolade-en-chips', 'Snoep, koek & chips'],
['tussendoortjes', 'Snoep, koek & chips'],
['diepvries', 'Diepvries'],
['koffie-en-thee', 'Koffie & thee'],
['frisdrank-en-sappen', 'Frisdrank & sappen'],
['bier-en-wijn', 'Bier & wijn'],
['drogisterij-en-gezondheid', 'Drogisterij'],
['huishouden-en-dieren', 'Huishouden & dieren'],
['baby-en-kind', 'Baby & kind'],
['nonfood-en-servicebalie', 'Overig'],
['bewuste-voeding', 'Overig']
];
const JUMBO_QUERY = `query SearchProducts($input: ProductSearchInput!) { searchProducts(input: $input) { count products { id ean title brand link image subtitle packSizeDisplay inAssortment availability { isAvailable } categories { name } promotions { tags { text } } price { price promoPrice pricePerUnit { price unit } } } } }`;
const JUMBO_PAGE = 200;
type JumboProduct = {
id: string;
ean?: string | null;
title: string;
brand?: string | null;
link?: string | null;
image?: string | null;
subtitle?: string | null;
packSizeDisplay?: string | null;
inAssortment?: boolean | null;
availability?: { isAvailable?: boolean | null } | null;
categories?: { name: string }[] | null;
promotions?: { tags?: { text: string }[] | null }[] | null;
price?: {
price?: number | null;
promoPrice?: number | null;
pricePerUnit?: { price?: number | null; unit?: string | null } | null;
} | null;
};
const JUMBO_NOT_PROMO = /gratis bezorging|^vanaf\b|bezorg/i;
function jumboItem(p: JumboProduct, category: string): RawItem {
const price = p.price?.price ?? null;
const promo = p.price?.promoPrice ?? null;
const ppu = p.price?.pricePerUnit;
const unit = storeUnit(ppu?.unit);
const tags = (p.promotions ?? [])
.flatMap((x) => x.tags ?? [])
.map((t) => t.text.trim())
.filter((t) => t && !JUMBO_NOT_PROMO.test(t));
return {
storeProductId: p.id,
name: p.title.trim(),
brand: p.brand?.trim() || null,
sizeText: p.packSizeDisplay?.trim() || p.subtitle?.trim() || null,
priceCents: price,
promoPriceCents: promo != null && price != null && promo < price ? promo : null,
promoLabel: tags.length ? [...new Set(tags)].join(' · ') : null,
storeUnitPrice: ppu?.price && unit ? { cents: ppu.price, unit } : null,
storeUnitBasisCents: promo ?? price, // Jumbo's price per unit follows the promo price
category,
categoryPath: p.categories?.map((c) => c.name).join(' > ') || null,
gtin: normaliseGtin(p.ean),
imageUrl: p.image || null,
url: p.link ? new URL(p.link, 'https://www.jumbo.com').toString() : null,
available: (p.inAssortment ?? true) && (p.availability?.isAvailable ?? true)
};
}
export const jumboSource: Source = {
slug: 'jumbo',
async fetchAll(f, log) {
const items = new Map<string, RawItem>();
for (const [root, category] of JUMBO_ROOTS) {
const before = items.size;
for (let offSet = 0; ; offSet += JUMBO_PAGE) {
const r = await f.json<{
data?: { searchProducts?: { count: number; products: JumboProduct[] } };
errors?: { message: string }[];
}>(
JUMBO_GQL,
{
method: 'POST',
headers: JUMBO_HEADERS,
body: JSON.stringify({
operationName: 'SearchProducts',
variables: {
input: { searchType: 'category', friendlyUrl: root, offSet, limit: JUMBO_PAGE }
},
query: JUMBO_QUERY
})
},
`${root.replace(/[^a-z0-9-]+/g, '_')}-${offSet}`
);
if (r.errors?.length) throw new Error(`jumbo ${root}: ${r.errors[0].message}`);
const sp = r.data?.searchProducts;
for (const p of sp?.products ?? []) {
if (!p.id || !p.title) continue;
if (!items.has(p.id)) items.set(p.id, jumboItem(p, category));
}
if (!sp?.products?.length || offSet + JUMBO_PAGE >= sp.count) break;
}
log(` jumbo ${root}: +${items.size - before} (${items.size} total)`);
}
return [...items.values()];
}
};
export const SOURCES: Record<string, Source> = { ah: ahSource, jumbo: jumboSource };

View File

@@ -1,45 +1,198 @@
import { db } from '#lib/server/db/index.js';
import { groceryPrices, groceryProducts, groceryStores } from '#lib/server/db/schema/index.js';
import { requireOwner } from '#lib/server/owner.js';
import { parseSize, priceInput, productInput, unitPriceCents } from '#lib/server/groceries.js';
import { and, asc, desc, eq, isNull, sql } from 'drizzle-orm';
import { refreshGroupStats } from '#lib/server/groceries/import.js';
import { CATEGORIES, displayName, fold } from '#lib/server/groceries/normalize.js';
import { fail } from '@sveltejs/kit';
import type { Sql } from 'postgres';
import type { Actions, PageServerLoad } from './$types';
const sql = () => db.$client as unknown as Sql;
const PAGE_SIZE = 30;
export type GroupRow = {
id: number;
name: string;
brand: string | null;
category: string | null;
unit: string | null;
favorite: boolean;
manual: boolean;
itemCount: number;
storeCount: number;
minUnitPriceCents: number | null;
imageUrl: string | null;
familyKey: string | null;
};
export type ItemRow = {
id: number;
groupId: number;
storeId: number;
name: string;
brand: string | null;
sizeText: string | null;
unit: string | null;
unitAmount: number | null;
unitSource: string | null;
priceCents: number | null;
promoPriceCents: number | null;
promoLabel: string | null;
unitPriceCents: number | null;
promoUnitPriceCents: number | null;
gtin: string | null;
imageUrl: string | null;
url: string | null;
available: boolean;
matchMethod: string | null;
matchConfidence: number | null;
matchLocked: boolean;
categoryPath: string | null;
lastSeenAt: Date;
};
const ITEM_COLS = (s: Sql) => s`
i.id, i.group_id as "groupId", i.store_id as "storeId", i.name, i.brand, i.size_text as "sizeText",
i.unit, i.unit_amount::float8 as "unitAmount", i.unit_source as "unitSource", i.price_cents as "priceCents",
i.promo_price_cents as "promoPriceCents", i.promo_label as "promoLabel",
i.unit_price_cents as "unitPriceCents", i.promo_unit_price_cents as "promoUnitPriceCents", i.gtin,
i.image_url as "imageUrl", i.url, i.available, i.match_method as "matchMethod",
i.match_confidence as "matchConfidence", i.match_locked as "matchLocked",
i.category_path as "categoryPath", i.last_seen_at as "lastSeenAt"`;
const GROUP_COLS = (s: Sql) => s`
g.id, g.name, g.brand, g.category, g.unit, g.favorite, g.manual, g.item_count as "itemCount",
g.store_count as "storeCount", g.min_unit_price_cents as "minUnitPriceCents",
g.image_url as "imageUrl", g.family_key as "familyKey"`;
export const load: PageServerLoad = async (event) => {
requireOwner(event);
const [stores, products, prices] = await Promise.all([
db
.select()
.from(groceryStores)
.where(isNull(groceryStores.archivedAt))
.orderBy(asc(groceryStores.sortOrder), asc(groceryStores.name)),
db
.select()
.from(groceryProducts)
.where(isNull(groceryProducts.archivedAt))
.orderBy(desc(groceryProducts.favorite), asc(sql`lower(${groceryProducts.name})`)),
db
.select({
id: groceryPrices.id,
productId: groceryPrices.productId,
storeId: groceryPrices.storeId,
priceCents: groceryPrices.priceCents,
unitPriceCents: groceryPrices.unitPriceCents,
promo: groceryPrices.promo,
observedAt: groceryPrices.observedAt,
source: groceryPrices.source,
url: groceryPrices.url,
notes: groceryPrices.notes
})
.from(groceryPrices)
.orderBy(desc(groceryPrices.observedAt), desc(groceryPrices.id))
const s = sql();
const p = event.url.searchParams;
const q = fold(p.get('q')?.trim() ?? '').slice(0, 100);
const words = q.split(/\s+/).filter(Boolean).slice(0, 6);
const storeSlug = p.get('store') ?? '';
const category = p.get('cat') ?? '';
const favorites = p.get('fav') === '1';
const promo = p.get('promo') === '1';
const multi = p.get('multi') === '1';
const page = Math.max(1, Math.min(10_000, Number(p.get('page')) || 1));
const detailId = Number(p.get('g')) || null;
const stores = await s<
{ id: number; slug: string; name: string; color: string | null; items: number }[]
>`
select st.id, st.slug, st.name, st.color,
(select count(*)::int from groceries.items i where i.store_id = st.id and i.available) as items
from groceries.stores st where st.archived_at is null order by st.sort_order, st.name`;
const store = stores.find((x) => x.slug === storeSlug) ?? null;
const conds = [s`g.merged_into_id is null`, s`(g.item_count > 0 or g.favorite)`];
for (const w of words)
conds.push(s`g.search_text like ${'%' + w.replace(/[%_\\]/g, '\\$&') + '%'}`);
if (store) conds.push(s`g.store_ids @> array[${store.id}]::int[]`);
if (category) conds.push(s`g.category = ${category}`);
if (favorites) conds.push(s`g.favorite`);
if (promo) conds.push(s`g.promo_count > 0`);
if (multi) conds.push(s`g.store_count >= 2`);
const where = conds.reduce((a, c) => s`${a} and ${c}`);
const order = q
? s`g.favorite desc, (lower(g.name) = ${q}) desc, word_similarity(${q}, lower(g.name)) desc, g.store_count desc, g.item_count desc, g.name`
: s`g.favorite desc, g.store_count desc, g.item_count desc, g.name`;
const [groups, [{ total }]] = await Promise.all([
s<GroupRow[]>`
select ${GROUP_COLS(s)} from groceries.groups g where ${where}
order by ${order}, g.id limit ${PAGE_SIZE} offset ${(page - 1) * PAGE_SIZE}`,
s<{ total: number }[]>`select count(*)::int as total from groceries.groups g where ${where}`
]);
const ids = groups.map((g) => g.id);
const items = ids.length
? await s<ItemRow[]>`
select ${ITEM_COLS(s)} from groceries.items i
where i.group_id = any(${ids}::int[]) and i.available
order by coalesce(i.promo_unit_price_cents, i.unit_price_cents) nulls last, i.id`
: [];
const [stats] = await s<
{
groups: number;
crossStore: number;
items: number;
grouped: number;
unitPriced: number;
promos: number;
}[]
>`
select
(select count(*)::int from groceries.groups where merged_into_id is null and item_count > 0) as groups,
(select count(*)::int from groceries.groups where merged_into_id is null and store_count >= 2) as "crossStore",
count(*)::int as items,
(count(*) filter (where g.store_count >= 2))::int as grouped,
(count(*) filter (where i.unit_price_cents is not null))::int as "unitPriced",
(count(*) filter (where i.promo_price_cents is not null or i.promo_label is not null))::int as promos
from groceries.items i left join groceries.groups g on g.id = i.group_id
where i.available`;
const runs = await s<
{ storeId: number; finishedAt: Date | null; status: string; fetched: number }[]
>`
select distinct on (store_id) store_id as "storeId", finished_at as "finishedAt", status, fetched
from groceries.import_runs order by store_id, started_at desc`;
const categories = await s<{ category: string }[]>`
select distinct category from groceries.groups where category is not null and item_count > 0`;
const catList = [
...CATEGORIES.filter((c) => categories.some((x) => x.category === c)),
...categories
.map((x) => x.category)
.filter((c) => !(CATEGORIES as readonly string[]).includes(c))
];
let detail = null;
if (detailId) {
const [group] = await s<(GroupRow & { notes: string | null })[]>`
select ${GROUP_COLS(s)}, g.notes from groceries.groups g where g.id = ${detailId}`;
if (group) {
const [dItems, similar] = await Promise.all([
s<ItemRow[]>`
select ${ITEM_COLS(s)} from groceries.items i where i.group_id = ${detailId}
order by i.available desc, coalesce(i.promo_unit_price_cents, i.unit_price_cents) nulls last, i.id`,
group.familyKey
? s<GroupRow[]>`
select ${GROUP_COLS(s)} from groceries.groups g
where g.family_key = ${group.familyKey} and g.id <> ${detailId}
and g.merged_into_id is null and g.item_count > 0
order by g.min_unit_price_cents nulls last limit 12`
: Promise.resolve([] as GroupRow[])
]);
const history = dItems.length
? await s<
{
itemId: number;
observedOn: string;
priceCents: number | null;
promoPriceCents: number | null;
promoLabel: string | null;
}[]
>`
select item_id as "itemId", observed_on::text as "observedOn", price_cents as "priceCents",
promo_price_cents as "promoPriceCents", promo_label as "promoLabel"
from groceries.item_prices where item_id = any(${dItems.map((i) => i.id)}::int[])
order by observed_on desc limit 400`
: [];
detail = { group, items: dItems, similar, history };
}
}
return {
stores,
products,
prices,
seo: { title: 'Groceries', description: 'Private grocery and price tracker.', index: false }
groups,
items,
total,
page,
pageSize: PAGE_SIZE,
filters: { q: p.get('q') ?? '', store: store?.slug ?? '', category, favorites, promo, multi },
categories: catList,
stats,
runs,
detail,
seo: { title: 'Groceries', description: 'Private supermarket price comparison.', index: false }
};
};
@@ -47,132 +200,110 @@ const id = (v: FormDataEntryValue | null) => {
const n = Number(v);
return Number.isInteger(n) && n > 0 ? n : null;
};
const formObject = (form: FormData) => {
const o: Record<string, unknown> = {};
for (const [k, v] of form) if (typeof v === 'string') o[k] = v;
return o;
};
const firstIssue = (e: { issues: { path: PropertyKey[]; message: string }[] }) => {
const i = e.issues[0];
return i ? `${i.path.join('.') || 'input'}: ${i.message}` : 'Invalid input';
};
function dbError(e: unknown): string {
const cause = (e as { cause?: { message?: string; constraint_name?: string } })?.cause;
if (cause?.constraint_name === 'products_name_brand_key')
return 'That product (name + brand) already exists';
if (cause?.constraint_name) return `Not allowed (${cause.constraint_name})`;
return cause?.message || 'Could not save';
}
export const actions: Actions = {
/** Create or edit a product. */
saveProduct: async (event) => {
requireOwner(event);
const form = await event.request.formData();
const productId = id(form.get('id'));
const parsed = productInput.safeParse(formObject(form));
if (!parsed.success) return fail(400, { message: firstIssue(parsed.error) });
const p = parsed.data;
const size = parseSize(p.size);
const values = {
name: p.name,
brand: p.brand ?? null,
category: p.category ?? null,
size: p.size ?? null,
unitAmount: size?.amount ?? null,
unit: size?.unit ?? null,
notes: p.notes ?? null,
...(form.has('favorite') ? { favorite: form.get('favorite') === 'on' } : {})
};
try {
if (productId) {
await db.transaction(async (tx) => {
await tx.update(groceryProducts).set(values).where(eq(groceryProducts.id, productId));
// Pack size changed: recompute unit prices of this product's history.
await tx
.update(groceryPrices)
.set({
unitPriceCents: values.unitAmount
? sql`round(${groceryPrices.priceCents} / ${values.unitAmount}::numeric)::int`
: null
})
.where(eq(groceryPrices.productId, productId));
});
return { saved: productId };
}
const [row] = await db
.insert(groceryProducts)
.values(values)
.returning({ id: groceryProducts.id });
return { saved: row.id };
} catch (e) {
return fail(400, { message: dbError(e) });
}
},
favorite: async (event) => {
requireOwner(event);
const form = await event.request.formData();
const productId = id(form.get('id'));
if (!productId) return fail(400, { message: 'Invalid product' });
await db
.update(groceryProducts)
.set({ favorite: form.get('favorite') === 'true' })
.where(eq(groceryProducts.id, productId));
return { updated: productId };
const gid = id(form.get('id'));
if (!gid) return fail(400, { message: 'Invalid group' });
await sql()`update groceries.groups set favorite = ${form.get('favorite') === 'true'} where id = ${gid}`;
return { updated: gid };
},
deleteProduct: async (event) => {
requireOwner(event);
const productId = id((await event.request.formData()).get('id'));
if (!productId) return fail(400, { message: 'Invalid product' });
await db.delete(groceryProducts).where(eq(groceryProducts.id, productId));
return { deleted: productId };
},
/** Record a price observation for a product at a store. */
addPrice: async (event) => {
/** Rename a group; it becomes manual so the nightly regroup keeps the name. */
rename: async (event) => {
requireOwner(event);
const form = await event.request.formData();
const productId = id(form.get('productId'));
const storeId = id(form.get('storeId'));
if (!productId || !storeId) return fail(400, { message: 'Pick a product and a store' });
const parsed = priceInput.safeParse(formObject(form));
if (!parsed.success) return fail(400, { message: firstIssue(parsed.error) });
const [product] = await db
.select({ unitAmount: groceryProducts.unitAmount })
.from(groceryProducts)
.where(eq(groceryProducts.id, productId));
if (!product) return fail(404, { message: 'Product not found' });
const p = parsed.data;
try {
await db.insert(groceryPrices).values({
productId,
storeId,
priceCents: p.price,
unitPriceCents: unitPriceCents(p.price, product.unitAmount),
promo: form.get('promo') === 'on',
// A plain date is stored as noon UTC; the exact time does not matter for prices.
observedAt: p.observedOn ? new Date(`${p.observedOn}T12:00:00Z`) : new Date(),
source: 'manual',
url: p.url ?? null,
notes: p.notes ?? null
});
} catch (e) {
return fail(400, { message: dbError(e) });
}
return { priced: productId };
const gid = id(form.get('id'));
const name = String(form.get('name') ?? '')
.trim()
.slice(0, 200);
if (!gid || !name) return fail(400, { message: 'Enter a name' });
const notes =
String(form.get('notes') ?? '')
.trim()
.slice(0, 2000) || null;
await sql()`update groceries.groups set name = ${name}, notes = ${notes}, manual = true where id = ${gid}`;
await refreshGroupStats(sql(), [gid]);
return { updated: gid };
},
deletePrice: async (event) => {
/** Merge group `id` into `into`: its items move (locked) and future matches follow. */
merge: async (event) => {
requireOwner(event);
const form = await event.request.formData();
const priceId = id(form.get('id'));
const productId = id(form.get('productId'));
if (!priceId || !productId) return fail(400, { message: 'Invalid price' });
await db
.delete(groceryPrices)
.where(and(eq(groceryPrices.id, priceId), eq(groceryPrices.productId, productId)));
return { deleted: priceId };
const from = id(form.get('id'));
const into = id(form.get('into'));
if (!from || !into || from === into) return fail(400, { message: 'Pick another group' });
const [target] = await sql()`
select id from groceries.groups where id = ${into} and merged_into_id is null`;
if (!target) return fail(404, { message: 'Group not found' });
await sql().begin(async (tx) => {
await tx`
update groceries.items set group_id = ${into}, match_locked = true, match_method = 'manual',
match_confidence = 1 where group_id = ${from}`;
await tx`update groceries.groups set merged_into_id = ${into} where merged_into_id = ${from}`;
const [src] = await tx<{ favorite: boolean }[]>`
select favorite from groceries.groups where id = ${from}`;
await tx`update groceries.groups set merged_into_id = ${into}, favorite = false where id = ${from}`;
if (src?.favorite) await tx`update groceries.groups set favorite = true where id = ${into}`;
await refreshGroupStats(tx as unknown as Sql, [from, into]);
});
return { merged: into };
},
/** Move one item to another group, or to a new group of its own (split). Locks the item. */
moveItem: async (event) => {
requireOwner(event);
const form = await event.request.formData();
const itemId = id(form.get('itemId'));
const target = form.get('into') === 'new' ? 'new' : id(form.get('into'));
if (!itemId || !target) return fail(400, { message: 'Pick a group' });
const moved = await sql().begin(async (tx) => {
const [item] = await tx<
{
group_id: number | null;
name: string;
brand: string | null;
unit: string | null;
category: string | null;
}[]
>`
select group_id, name, brand, unit, category from groceries.items where id = ${itemId}`;
if (!item) return null;
let into: number;
if (target === 'new') {
const [g] = await tx<{ id: number }[]>`
insert into groceries.groups (name, brand, category, unit, manual)
values (${displayName(item.name, item.brand).slice(0, 200)}, ${item.brand}, ${item.category}, ${item.unit}, true)
returning id`;
into = g.id;
} else {
const [g] =
await tx`select id from groceries.groups where id = ${target} and merged_into_id is null`;
if (!g) return null;
into = target;
}
await tx`
update groceries.items set group_id = ${into}, match_locked = true, match_method = 'manual',
match_confidence = 1 where id = ${itemId}`;
await refreshGroupStats(tx as unknown as Sql, [
into,
...(item.group_id ? [item.group_id] : [])
]);
return into;
});
if (!moved) return fail(404, { message: 'Not found' });
return { moved };
},
/** Hand an item back to the automatic matcher (applies at the next nightly import). */
unlockItem: async (event) => {
requireOwner(event);
const itemId = id((await event.request.formData()).get('itemId'));
if (!itemId) return fail(400, { message: 'Invalid item' });
await sql()`update groceries.items set match_locked = false where id = ${itemId}`;
return { unlocked: itemId };
}
};

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,23 @@
import { db } from '#lib/server/db/index.js';
import { requireOwner } from '#lib/server/owner.js';
import { fold } from '#lib/server/groceries/normalize.js';
import { json } from '@sveltejs/kit';
import type { Sql } from 'postgres';
import type { RequestHandler } from './$types';
/** Group picker for merge/move: top matches for ?q= (owner only). */
export const GET: RequestHandler = async (event) => {
requireOwner(event);
const q = fold(event.url.searchParams.get('q')?.trim() ?? '').slice(0, 100);
const words = q.split(/\s+/).filter(Boolean).slice(0, 6);
if (!words.length) return json([]);
const s = db.$client as unknown as Sql;
const conds = words.map((w) => s`search_text like ${'%' + w.replace(/[%_\\]/g, '\\$&') + '%'}`);
const rows = await s`
select id, name, brand, unit, store_count as "storeCount", item_count as "itemCount"
from groceries.groups
where merged_into_id is null and ${conds.reduce((a, c) => s`${a} and ${c}`)}
order by word_similarity(${q}, lower(name)) desc, store_count desc, item_count desc, id
limit 15`;
return json(rows, { headers: { 'cache-control': 'private, no-store' } });
};