groceries: nightly AH/Jumbo catalogue import, unit-price normalisation, product groups and compare UI
All checks were successful
Deploy production / deploy (push) Successful in 2m48s
All checks were successful
Deploy production / deploy (push) Successful in 2m48s
This commit is contained in:
361
src/lib/server/groceries/normalize.ts
Normal file
361
src/lib/server/groceries/normalize.ts
Normal file
@@ -0,0 +1,361 @@
|
||||
// Pure normalisation helpers for the grocery importer and matcher (no DB, no network).
|
||||
import type { GroceryUnit } from '../db/schema/groceries';
|
||||
|
||||
export type Size = { amount: number; unit: GroceryUnit };
|
||||
|
||||
const MASS: Record<string, number> = {
|
||||
kg: 1,
|
||||
kilo: 1,
|
||||
kilogram: 1,
|
||||
g: 0.001,
|
||||
gr: 0.001,
|
||||
gram: 0.001,
|
||||
mg: 0.000001
|
||||
};
|
||||
const VOLUME: Record<string, number> = {
|
||||
l: 1,
|
||||
ltr: 1,
|
||||
liter: 1,
|
||||
litre: 1,
|
||||
dl: 0.1,
|
||||
cl: 0.01,
|
||||
ml: 0.001
|
||||
};
|
||||
const COUNT = new Set([
|
||||
'st',
|
||||
'stk',
|
||||
'stuk',
|
||||
'stuks',
|
||||
'stuk(s)',
|
||||
'pcs',
|
||||
'piece',
|
||||
'pieces',
|
||||
'rol',
|
||||
'rollen',
|
||||
'zakjes',
|
||||
'zakje',
|
||||
'tabletten',
|
||||
'tablet',
|
||||
'capsules',
|
||||
'capsule',
|
||||
'cups',
|
||||
'pads',
|
||||
'sachets',
|
||||
'doekjes',
|
||||
'luiers',
|
||||
'wasbeurten',
|
||||
'eieren',
|
||||
'plakken',
|
||||
'porties',
|
||||
'repen',
|
||||
'blikjes',
|
||||
'flesjes',
|
||||
'zakken',
|
||||
'bollen',
|
||||
'tabs',
|
||||
'vellen',
|
||||
'servetten',
|
||||
'theezakjes',
|
||||
'paar'
|
||||
]);
|
||||
const UNIT_RE =
|
||||
'kilogram|kilo|kg|gram|gr|mg|g|liter|litre|ltr|ml|cl|dl|l|stuk\\(s\\)|stuks|stuk|stk|st|pcs|pieces|piece|rollen|rol|zakjes|zakje|tabletten|tablet|capsules|capsule|cups|pads|sachets|doekjes|luiers|wasbeurten|eieren|plakken|porties|repen|blikjes|flesjes|zakken|bollen|tabs|vellen|servetten|theezakjes|paar';
|
||||
const QTY = new RegExp(`(?:(\\d+)\\s*[x×]\\s*)?(\\d+(?:[.,]\\d+)?)\\s*(${UNIT_RE})(?![a-z])`, 'gi');
|
||||
|
||||
function toSize(mult: number, n: number, unit: string): Size | null {
|
||||
const u = unit.toLowerCase();
|
||||
const total = mult * n;
|
||||
if (!(total > 0)) return null;
|
||||
if (u in MASS) return { amount: total * MASS[u], unit: 'kg' };
|
||||
if (u in VOLUME) return { amount: total * VOLUME[u], unit: 'l' };
|
||||
if (COUNT.has(u)) return { amount: total, unit: 'piece' };
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pack size → base unit, e.g. "500 g" → 0.5 kg, "6 x 330 ml" → 1.98 l, "4 stuks" → 4 piece,
|
||||
* "per stuk" → 1 piece, "per kilo" → 1 kg. Weight/volume beats a count ("6 stuks 360 g" → kg).
|
||||
* Works on titles too ("Jumbo Halfvolle Melk 1,5L"). Null when nothing usable.
|
||||
*/
|
||||
export function parseSize(text: string | null | undefined): Size | null {
|
||||
if (!text) return null;
|
||||
const s = text
|
||||
.toLowerCase()
|
||||
.replace(/ca\.?\s*/g, '')
|
||||
.replace(/\s+/g, ' ');
|
||||
let best: Size | null = null;
|
||||
for (const m of s.matchAll(QTY)) {
|
||||
const size = toSize(m[1] ? Number(m[1]) : 1, Number(m[2].replace(',', '.')), m[3]);
|
||||
if (!size) continue;
|
||||
if (!best || (best.unit === 'piece' && size.unit !== 'piece')) best = size;
|
||||
if (best.unit !== 'piece') break;
|
||||
}
|
||||
if (best) return best;
|
||||
if (/\bper\s*(kilo|kg)\b/.test(s)) return { amount: 1, unit: 'kg' };
|
||||
if (/\bper\s*(liter|l)\b/.test(s)) return { amount: 1, unit: 'l' };
|
||||
if (
|
||||
/\bper\s*(stuk|bos|bosje|bundel|krop|doos|pak|tros|net|schaal|bak|pot|zak|paar|set)\b/.test(s)
|
||||
)
|
||||
return { amount: 1, unit: 'piece' };
|
||||
if (/^\s*(\d+)\s*(st|stuks)?\s*$/.test(s))
|
||||
return { amount: Number(s.trim().split(/\s/)[0]), unit: 'piece' };
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Store unit labels ("kg", "l", "pieces", "stuk", "liter") → our unit. */
|
||||
export function storeUnit(label: string | null | undefined): GroceryUnit | null {
|
||||
const u = (label ?? '').toLowerCase().trim();
|
||||
if (['kg', 'kilo', 'kilogram'].includes(u)) return 'kg';
|
||||
if (['l', 'liter', 'litre', 'ltr'].includes(u)) return 'l';
|
||||
if (['piece', 'pieces', 'stuk', 'stuks', 'st', 'pce', 'pc'].includes(u)) return 'piece';
|
||||
return null;
|
||||
}
|
||||
|
||||
/** AH "prijs per kg €12.98" / "normale prijs per liter €1.49" → { cents, unit }. */
|
||||
export function parseAhUnitPrice(
|
||||
desc: string | null | undefined
|
||||
): { cents: number; unit: GroceryUnit } | null {
|
||||
if (!desc) return null;
|
||||
const m = desc.toLowerCase().match(/per\s+(kg|kilo|liter|l|stuk|st)\s*€?\s*(\d+(?:[.,]\d+)?)/);
|
||||
if (!m) return null;
|
||||
const unit = storeUnit(m[1]);
|
||||
return unit ? { cents: Math.round(Number(m[2].replace(',', '.')) * 100), unit } : null;
|
||||
}
|
||||
|
||||
export type Normalised = {
|
||||
unit: GroceryUnit | null;
|
||||
unitAmount: number | null;
|
||||
unitSource: 'size' | 'store' | null;
|
||||
unitPriceCents: number | null;
|
||||
promoUnitPriceCents: number | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Pick the unit and pack amount: the parsed size when it agrees with the store's unit price
|
||||
* (or the store gives none); else derive the amount from the store's unit price.
|
||||
*/
|
||||
export function normaliseUnit(opts: {
|
||||
sizeText?: string | null;
|
||||
title?: string | null;
|
||||
priceCents: number | null;
|
||||
promoPriceCents?: number | null;
|
||||
storeUnitPrice?: { cents: number; unit: GroceryUnit } | null;
|
||||
/** Price the store's unit price was computed from (Jumbo uses the promo price). */
|
||||
storeUnitBasisCents?: number | null;
|
||||
}): Normalised {
|
||||
const { priceCents, storeUnitPrice } = opts;
|
||||
let size = parseSize(opts.sizeText) ?? parseSize(opts.title);
|
||||
let source: Normalised['unitSource'] = size ? 'size' : null;
|
||||
const basis = opts.storeUnitBasisCents ?? priceCents;
|
||||
if (storeUnitPrice && storeUnitPrice.cents > 0 && basis) {
|
||||
if (!size || size.unit !== storeUnitPrice.unit) {
|
||||
const derived = basis / storeUnitPrice.cents;
|
||||
const amount =
|
||||
storeUnitPrice.unit === 'piece'
|
||||
? Math.max(1, Math.round(derived))
|
||||
: Math.round(derived * 1000) / 1000;
|
||||
size = amount > 0 ? { amount, unit: storeUnitPrice.unit } : size;
|
||||
source = 'store';
|
||||
}
|
||||
}
|
||||
if (!size || !(size.amount > 0)) {
|
||||
return {
|
||||
unit: null,
|
||||
unitAmount: null,
|
||||
unitSource: null,
|
||||
unitPriceCents: null,
|
||||
promoUnitPriceCents: null
|
||||
};
|
||||
}
|
||||
const per = (c: number | null | undefined) =>
|
||||
c != null && c > 0 ? Math.round(c / size.amount) : null;
|
||||
return {
|
||||
unit: size.unit,
|
||||
unitAmount: Math.round(size.amount * 10000) / 10000,
|
||||
unitSource: source,
|
||||
unitPriceCents: per(priceCents),
|
||||
promoUnitPriceCents: per(opts.promoPriceCents)
|
||||
};
|
||||
}
|
||||
|
||||
// ---- names, brands, keys -------------------------------------------------------------------
|
||||
|
||||
export const fold = (s: string) =>
|
||||
s
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase();
|
||||
|
||||
/** Store own brands, mapped to a shared class so AH/Jumbo/Plus equivalents group together. */
|
||||
const HOUSE: Record<string, string> = {
|
||||
ah: 'house',
|
||||
'albert heijn': 'house',
|
||||
jumbo: 'house',
|
||||
plus: 'house',
|
||||
'ah biologisch': 'house-bio',
|
||||
'jumbo biologisch': 'house-bio',
|
||||
'plus biologisch': 'house-bio',
|
||||
'biologisch plus': 'house-bio',
|
||||
'ah basic': 'house-basic',
|
||||
'jumbo basic': 'house-basic',
|
||||
'plus basic': 'house-basic',
|
||||
"g'woon": 'house-basic',
|
||||
'ah excellent': 'house-premium',
|
||||
'jumbo excellent': 'house-premium',
|
||||
'plus excellent': 'house-premium',
|
||||
'jumbo lekker puur': 'house-premium',
|
||||
'ah terra': 'house-veg',
|
||||
'jumbo veggie': 'house-veg',
|
||||
'ah kids': 'house-kids',
|
||||
'ah puur&eerlijk': 'house-bio',
|
||||
'ah puur & eerlijk': 'house-bio'
|
||||
};
|
||||
const HOUSE_LABEL: Record<string, string> = {
|
||||
house: 'Huismerk',
|
||||
'house-bio': 'Huismerk biologisch',
|
||||
'house-basic': 'Huismerk basic',
|
||||
'house-premium': 'Huismerk premium',
|
||||
'house-veg': 'Huismerk plantaardig',
|
||||
'house-kids': 'Huismerk kids'
|
||||
};
|
||||
const STORE_WORDS = ['albert heijn', 'ah', 'jumbo', 'plus'];
|
||||
|
||||
export function brandClass(brand: string | null | undefined): string {
|
||||
const b = fold(brand ?? '')
|
||||
.trim()
|
||||
.replace(/\s+/g, ' ');
|
||||
if (!b) return 'house'; // loose produce without a brand
|
||||
return HOUSE[b] ?? b.replace(/[^a-z0-9]+/g, '');
|
||||
}
|
||||
export const isHouse = (cls: string) => cls.startsWith('house');
|
||||
export const brandLabel = (cls: string, brand: string | null | undefined) =>
|
||||
HOUSE_LABEL[cls] ?? brand?.trim() ?? null;
|
||||
|
||||
const STOP = new Set([
|
||||
'de',
|
||||
'het',
|
||||
'een',
|
||||
'van',
|
||||
'met',
|
||||
'en',
|
||||
'in',
|
||||
'voor',
|
||||
'per',
|
||||
'x',
|
||||
'ca',
|
||||
'op',
|
||||
'of',
|
||||
'a',
|
||||
'la',
|
||||
'le',
|
||||
'stuks',
|
||||
'stuk',
|
||||
'st',
|
||||
'pak',
|
||||
'fles',
|
||||
'blik',
|
||||
'pot',
|
||||
'zak',
|
||||
'doos',
|
||||
'tray',
|
||||
'bak',
|
||||
'beker',
|
||||
'kuipje',
|
||||
'voordeelverpakking',
|
||||
'voordeelpak',
|
||||
'voordeel',
|
||||
'multipack',
|
||||
'grootverpakking',
|
||||
'xl',
|
||||
'xxl',
|
||||
'navulling',
|
||||
'family',
|
||||
'familiepak',
|
||||
'mega',
|
||||
'pack',
|
||||
'literpak',
|
||||
'petfles',
|
||||
'tube',
|
||||
'tetra',
|
||||
'brik'
|
||||
]);
|
||||
const SIZE_IN_NAME = new RegExp(
|
||||
`(?:\\d+\\s*[x×]\\s*)?\\d+(?:[.,]\\d+)?\\s*(?:${UNIT_RE})(?![a-z])`,
|
||||
'gi'
|
||||
);
|
||||
|
||||
const stem = (t: string) =>
|
||||
t.length > 5 && t.endsWith('en')
|
||||
? t.slice(0, -2)
|
||||
: t.length > 4 && t.endsWith('s')
|
||||
? t.slice(0, -1)
|
||||
: t;
|
||||
|
||||
/** Meaningful name tokens: no brand, store name, size, packaging or stop words. */
|
||||
export function nameTokens(title: string, brand?: string | null): string[] {
|
||||
let s = fold(title).replace(/&/g, ' en ').replace(SIZE_IN_NAME, ' ');
|
||||
const drop = new Set<string>();
|
||||
for (const w of [brand ?? '', ...STORE_WORDS])
|
||||
for (const t of fold(w).split(/[^a-z0-9]+/)) if (t) drop.add(t);
|
||||
s = s.replace(/['’]/g, '').replace(/[^a-z0-9%+]+/g, ' ');
|
||||
const out: string[] = [];
|
||||
for (const raw of s.split(' ')) {
|
||||
if (!raw || drop.has(raw) || STOP.has(raw)) continue;
|
||||
if (/^\d+$/.test(raw) && raw.length > 2) continue; // stray numbers (article codes)
|
||||
const t = stem(raw);
|
||||
if (!out.includes(t)) out.push(t);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Display name without brand/store prefix and size: "AH Halfvolle melk 1 L" → "Halfvolle melk". */
|
||||
export function displayName(title: string, brand?: string | null): string {
|
||||
let s = title.replace(SIZE_IN_NAME, ' ').replace(/\s+-\s*$/, '');
|
||||
for (const w of [brand ?? '', ...STORE_WORDS]
|
||||
.filter(Boolean)
|
||||
.sort((a, b) => b.length - a.length)) {
|
||||
const re = new RegExp(`^\\s*${w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b\\s*`, 'i');
|
||||
s = s.replace(re, '');
|
||||
}
|
||||
s = s
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/[\s,-]+$/, '')
|
||||
.trim();
|
||||
return s ? s[0].toUpperCase() + s.slice(1) : title.trim();
|
||||
}
|
||||
|
||||
export function matchKey(cls: string, unit: string | null, tokens: string[]): string {
|
||||
return `${cls}|${unit ?? '?'}|${[...tokens].sort().join(' ')}`;
|
||||
}
|
||||
|
||||
/** EAN-13 / GTIN-14 → canonical 13-digit EAN (leading zero of GTIN-14 dropped). */
|
||||
export function normaliseGtin(v: string | null | undefined): string | null {
|
||||
const d = (v ?? '').replace(/\D/g, '');
|
||||
if (d.length < 8) return null;
|
||||
const t = d.replace(/^0+(?=\d{13}$)/, '');
|
||||
return t.length > 14 ? null : t.padStart(13, '0');
|
||||
}
|
||||
|
||||
// ---- categories -------------------------------------------------------------------------------
|
||||
|
||||
/** Shared category names; each store's top categories map onto these. */
|
||||
export const CATEGORIES = [
|
||||
'Groente, fruit & aardappelen',
|
||||
'Maaltijden & gemak',
|
||||
'Vlees, vis & vega',
|
||||
'Vleeswaren, kaas & tapas',
|
||||
'Zuivel & eieren',
|
||||
'Brood & bakkerij',
|
||||
'Ontbijt & beleg',
|
||||
'Pasta, rijst & wereldkeuken',
|
||||
'Soepen, sauzen & conserven',
|
||||
'Snoep, koek & chips',
|
||||
'Diepvries',
|
||||
'Koffie & thee',
|
||||
'Frisdrank & sappen',
|
||||
'Bier & wijn',
|
||||
'Drogisterij',
|
||||
'Huishouden & dieren',
|
||||
'Baby & kind',
|
||||
'Overig'
|
||||
] as const;
|
||||
Reference in New Issue
Block a user