Stan Runge
3f30bb883f
All checks were successful
Deploy production / deploy (push) Successful in 1m7s
369 lines
10 KiB
TypeScript
369 lines
10 KiB
TypeScript
// Pure normalisation helpers for the grocery importer and matcher (no DB, no network).
|
||
import type { GroceryUnit } from '../db/schema/groceries';
|
||
|
||
export type Size = { amount: number; unit: GroceryUnit };
|
||
|
||
const MASS: Record<string, number> = {
|
||
kg: 1,
|
||
kilo: 1,
|
||
kilogram: 1,
|
||
g: 0.001,
|
||
gr: 0.001,
|
||
gram: 0.001,
|
||
mg: 0.000001
|
||
};
|
||
const VOLUME: Record<string, number> = {
|
||
l: 1,
|
||
ltr: 1,
|
||
liter: 1,
|
||
litre: 1,
|
||
dl: 0.1,
|
||
cl: 0.01,
|
||
ml: 0.001
|
||
};
|
||
const COUNT = new Set([
|
||
'st',
|
||
'stk',
|
||
'stuk',
|
||
'stuks',
|
||
'stuk(s)',
|
||
'pcs',
|
||
'piece',
|
||
'pieces',
|
||
'rol',
|
||
'rollen',
|
||
'zakjes',
|
||
'zakje',
|
||
'tabletten',
|
||
'tablet',
|
||
'capsules',
|
||
'capsule',
|
||
'cups',
|
||
'pads',
|
||
'sachets',
|
||
'doekjes',
|
||
'luiers',
|
||
'wasbeurten',
|
||
'eieren',
|
||
'plakken',
|
||
'porties',
|
||
'repen',
|
||
'blikjes',
|
||
'flesjes',
|
||
'zakken',
|
||
'bollen',
|
||
'tabs',
|
||
'vellen',
|
||
'servetten',
|
||
'theezakjes',
|
||
'paar'
|
||
]);
|
||
const UNIT_RE =
|
||
'kilogram|kilo|kg|gram|gr|mg|g|liter|litre|ltr|ml|cl|dl|l|stuk\\(s\\)|stuks|stuk|stk|st|pcs|pieces|piece|rollen|rol|zakjes|zakje|tabletten|tablet|capsules|capsule|cups|pads|sachets|doekjes|luiers|wasbeurten|eieren|plakken|porties|repen|blikjes|flesjes|zakken|bollen|tabs|vellen|servetten|theezakjes|paar';
|
||
const QTY = new RegExp(`(?:(\\d+)\\s*[x×]\\s*)?(\\d+(?:[.,]\\d+)?)\\s*(${UNIT_RE})(?![a-z])`, 'gi');
|
||
|
||
function toSize(mult: number, n: number, unit: string): Size | null {
|
||
const u = unit.toLowerCase();
|
||
const total = mult * n;
|
||
if (!(total > 0)) return null;
|
||
if (u in MASS) return { amount: total * MASS[u], unit: 'kg' };
|
||
if (u in VOLUME) return { amount: total * VOLUME[u], unit: 'l' };
|
||
if (COUNT.has(u)) return { amount: total, unit: 'piece' };
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* Pack size → base unit, e.g. "500 g" → 0.5 kg, "6 x 330 ml" → 1.98 l, "4 stuks" → 4 piece,
|
||
* "per stuk" → 1 piece, "per kilo" → 1 kg. Weight/volume beats a count ("6 stuks 360 g" → kg).
|
||
* Works on titles too ("Jumbo Halfvolle Melk 1,5L"). Null when nothing usable.
|
||
*/
|
||
export function parseSize(text: string | null | undefined): Size | null {
|
||
if (!text) return null;
|
||
const s = text
|
||
.toLowerCase()
|
||
.replace(/ca\.?\s*/g, '')
|
||
.replace(/\s+/g, ' ');
|
||
let best: Size | null = null;
|
||
for (const m of s.matchAll(QTY)) {
|
||
const size = toSize(m[1] ? Number(m[1]) : 1, Number(m[2].replace(',', '.')), m[3]);
|
||
if (!size) continue;
|
||
if (!best || (best.unit === 'piece' && size.unit !== 'piece')) best = size;
|
||
if (best.unit !== 'piece') break;
|
||
}
|
||
if (best) return best;
|
||
if (/\bper\s*(kilo|kg)\b/.test(s)) return { amount: 1, unit: 'kg' };
|
||
if (/\bper\s*(liter|l)\b/.test(s)) return { amount: 1, unit: 'l' };
|
||
if (
|
||
/\bper\s*(stuk|bos|bosje|bundel|krop|doos|pak|tros|net|schaal|bak|pot|zak|paar|set)\b/.test(s)
|
||
)
|
||
return { amount: 1, unit: 'piece' };
|
||
if (/^\s*(\d+)\s*(st|stuks)?\s*$/.test(s))
|
||
return { amount: Number(s.trim().split(/\s/)[0]), unit: 'piece' };
|
||
return null;
|
||
}
|
||
|
||
/** Store unit labels ("kg", "l", "pieces", "stuk", "liter") → our unit. */
|
||
export function storeUnit(label: string | null | undefined): GroceryUnit | null {
|
||
const u = (label ?? '').toLowerCase().trim();
|
||
if (['kg', 'kilo', 'kilogram'].includes(u)) return 'kg';
|
||
if (['l', 'liter', 'litre', 'ltr'].includes(u)) return 'l';
|
||
if (['piece', 'pieces', 'stuk', 'stuks', 'st', 'pce', 'pc'].includes(u)) return 'piece';
|
||
return null;
|
||
}
|
||
|
||
/** AH "prijs per kg €12.98" / "normale prijs per liter €1.49" → { cents, unit }. */
|
||
export function parseAhUnitPrice(
|
||
desc: string | null | undefined
|
||
): { cents: number; unit: GroceryUnit } | null {
|
||
if (!desc) return null;
|
||
const m = desc.toLowerCase().match(/per\s+(kg|kilo|liter|l|stuk|st)\s*€?\s*(\d+(?:[.,]\d+)?)/);
|
||
if (!m) return null;
|
||
const unit = storeUnit(m[1]);
|
||
return unit ? { cents: Math.round(Number(m[2].replace(',', '.')) * 100), unit } : null;
|
||
}
|
||
|
||
export type Normalised = {
|
||
unit: GroceryUnit | null;
|
||
unitAmount: number | null;
|
||
unitSource: 'size' | 'store' | null;
|
||
unitPriceCents: number | null;
|
||
promoUnitPriceCents: number | null;
|
||
};
|
||
|
||
/**
|
||
* Pick the unit and pack amount: the parsed size when it agrees with the store's unit price
|
||
* (or the store gives none); else derive the amount from the store's unit price.
|
||
*/
|
||
export function normaliseUnit(opts: {
|
||
sizeText?: string | null;
|
||
title?: string | null;
|
||
priceCents: number | null;
|
||
promoPriceCents?: number | null;
|
||
storeUnitPrice?: { cents: number; unit: GroceryUnit } | null;
|
||
/** Price the store's unit price was computed from (Jumbo uses the promo price). */
|
||
storeUnitBasisCents?: number | null;
|
||
}): Normalised {
|
||
const { priceCents, storeUnitPrice } = opts;
|
||
let size = parseSize(opts.sizeText) ?? parseSize(opts.title);
|
||
let source: Normalised['unitSource'] = size ? 'size' : null;
|
||
const basis = opts.storeUnitBasisCents ?? priceCents;
|
||
if (storeUnitPrice && storeUnitPrice.cents > 0 && basis) {
|
||
if (!size || size.unit !== storeUnitPrice.unit) {
|
||
const derived = basis / storeUnitPrice.cents;
|
||
const amount =
|
||
storeUnitPrice.unit === 'piece'
|
||
? Math.max(1, Math.round(derived))
|
||
: Math.round(derived * 1000) / 1000;
|
||
size = amount > 0 ? { amount, unit: storeUnitPrice.unit } : size;
|
||
source = 'store';
|
||
}
|
||
}
|
||
if (!size || !(size.amount > 0)) {
|
||
return {
|
||
unit: null,
|
||
unitAmount: null,
|
||
unitSource: null,
|
||
unitPriceCents: null,
|
||
promoUnitPriceCents: null
|
||
};
|
||
}
|
||
const per = (c: number | null | undefined) =>
|
||
c != null && c > 0 ? Math.round(c / size.amount) : null;
|
||
return {
|
||
unit: size.unit,
|
||
unitAmount: Math.round(size.amount * 10000) / 10000,
|
||
unitSource: source,
|
||
unitPriceCents: per(priceCents),
|
||
promoUnitPriceCents: per(opts.promoPriceCents)
|
||
};
|
||
}
|
||
|
||
// ---- names, brands, keys -------------------------------------------------------------------
|
||
|
||
export const fold = (s: string) =>
|
||
s
|
||
.normalize('NFD')
|
||
.replace(/[\u0300-\u036f]/g, '')
|
||
.toLowerCase();
|
||
|
||
/** Store own brands, mapped to a shared class so AH/Jumbo/Plus equivalents group together. */
|
||
const HOUSE: Record<string, string> = {
|
||
ah: 'house',
|
||
'albert heijn': 'house',
|
||
jumbo: 'house',
|
||
plus: 'house',
|
||
'ah biologisch': 'house-bio',
|
||
'jumbo biologisch': 'house-bio',
|
||
'plus biologisch': 'house-bio',
|
||
'biologisch plus': 'house-bio',
|
||
'ah basic': 'house-basic',
|
||
'jumbo basic': 'house-basic',
|
||
'plus basic': 'house-basic',
|
||
"g'woon": 'house-basic',
|
||
'ah excellent': 'house-premium',
|
||
'jumbo excellent': 'house-premium',
|
||
'plus excellent': 'house-premium',
|
||
'jumbo lekker puur': 'house-premium',
|
||
'ah terra': 'house-veg',
|
||
'jumbo veggie': 'house-veg',
|
||
'ah kids': 'house-kids',
|
||
'ah puur&eerlijk': 'house-bio',
|
||
'ah puur & eerlijk': 'house-bio'
|
||
};
|
||
const HOUSE_LABEL: Record<string, string> = {
|
||
house: 'Huismerk',
|
||
'house-bio': 'Huismerk biologisch',
|
||
'house-basic': 'Huismerk basic',
|
||
'house-premium': 'Huismerk premium',
|
||
'house-veg': 'Huismerk plantaardig',
|
||
'house-kids': 'Huismerk kids'
|
||
};
|
||
const STORE_WORDS = ['albert heijn', 'ah', 'jumbo', 'plus'];
|
||
|
||
export function brandClass(brand: string | null | undefined): string {
|
||
const b = fold(brand ?? '')
|
||
.trim()
|
||
.replace(/\s+/g, ' ');
|
||
if (!b) return 'house'; // loose produce without a brand
|
||
return HOUSE[b] ?? b.replace(/[^a-z0-9]+/g, '');
|
||
}
|
||
export const isHouse = (cls: string) => cls.startsWith('house');
|
||
export const brandLabel = (cls: string, brand: string | null | undefined) =>
|
||
HOUSE_LABEL[cls] ?? brand?.trim() ?? null;
|
||
|
||
const STOP = new Set([
|
||
'de',
|
||
'het',
|
||
'een',
|
||
'van',
|
||
'met',
|
||
'en',
|
||
'in',
|
||
'voor',
|
||
'per',
|
||
'x',
|
||
'ca',
|
||
'op',
|
||
'of',
|
||
'a',
|
||
'la',
|
||
'le',
|
||
'stuks',
|
||
'stuk',
|
||
'st',
|
||
'pak',
|
||
'fles',
|
||
'blik',
|
||
'pot',
|
||
'zak',
|
||
'doos',
|
||
'tray',
|
||
'bak',
|
||
'beker',
|
||
'kuipje',
|
||
'voordeelverpakking',
|
||
'voordeelpak',
|
||
'voordeel',
|
||
'multipack',
|
||
'grootverpakking',
|
||
'xl',
|
||
'xxl',
|
||
'navulling',
|
||
'family',
|
||
'familiepak',
|
||
'mega',
|
||
'pack',
|
||
'literpak',
|
||
'petfles',
|
||
'tube',
|
||
'tetra',
|
||
'brik'
|
||
]);
|
||
const SIZE_IN_NAME = new RegExp(
|
||
`(?:\\d+\\s*[x×]\\s*)?\\d+(?:[.,]\\d+)?\\s*(?:${UNIT_RE})(?![a-z])`,
|
||
'gi'
|
||
);
|
||
|
||
const stem = (t: string) =>
|
||
t.length > 5 && t.endsWith('en')
|
||
? t.slice(0, -2)
|
||
: t.length > 4 && t.endsWith('s')
|
||
? t.slice(0, -1)
|
||
: t;
|
||
|
||
/** Meaningful name tokens: no brand, store name, size, packaging or stop words. */
|
||
export function nameTokens(title: string, brand?: string | null): string[] {
|
||
let s = fold(title)
|
||
.replace(/&/g, ' en ')
|
||
.replace(SIZE_IN_NAME, ' ')
|
||
.replace(/\bdurum\s*-?\s*(?:tarwe)?\b/g, ' '); // "Penne durum tarwe" is plain penne
|
||
const drop = new Set<string>();
|
||
for (const w of [brand ?? '', ...STORE_WORDS])
|
||
for (const t of fold(w)
|
||
.replace(/['’]/g, '')
|
||
.split(/[^a-z0-9]+/))
|
||
if (t) drop.add(t);
|
||
s = s.replace(/['’]/g, '').replace(/[^a-z0-9%+]+/g, ' ');
|
||
const out: string[] = [];
|
||
for (const raw of s.split(' ')) {
|
||
if (!raw || drop.has(raw) || STOP.has(raw)) continue;
|
||
if (/^(?:ah|jumbo|plus)s$/.test(raw)) continue; // possessive store lines: Jumbo's
|
||
if (/^\d+$/.test(raw) && raw.length > 2) continue; // stray numbers (article codes)
|
||
const t = stem(raw);
|
||
if (!out.includes(t)) out.push(t);
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/** Display name without brand/store prefix and size: "AH Halfvolle melk 1 L" → "Halfvolle melk". */
|
||
export function displayName(title: string, brand?: string | null): string {
|
||
let s = title.replace(SIZE_IN_NAME, ' ').replace(/\s+-\s*$/, '');
|
||
for (const w of [brand ?? '', ...STORE_WORDS]
|
||
.filter(Boolean)
|
||
.sort((a, b) => b.length - a.length)) {
|
||
const re = new RegExp(`^\\s*${w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b\\s*`, 'i');
|
||
s = s.replace(re, '');
|
||
}
|
||
s = s
|
||
.replace(/\s+/g, ' ')
|
||
.replace(/[\s,-]+$/, '')
|
||
.trim();
|
||
return s ? s[0].toUpperCase() + s.slice(1) : title.trim();
|
||
}
|
||
|
||
export function matchKey(cls: string, unit: string | null, tokens: string[]): string {
|
||
return `${cls}|${unit ?? '?'}|${[...tokens].sort().join(' ')}`;
|
||
}
|
||
|
||
/** EAN-13 / GTIN-14 → canonical 13-digit EAN (leading zero of GTIN-14 dropped). */
|
||
export function normaliseGtin(v: string | null | undefined): string | null {
|
||
const d = (v ?? '').replace(/\D/g, '');
|
||
if (d.length < 8) return null;
|
||
const t = d.replace(/^0+(?=\d{13}$)/, '');
|
||
return t.length > 14 ? null : t.padStart(13, '0');
|
||
}
|
||
|
||
// ---- categories -------------------------------------------------------------------------------
|
||
|
||
/** Shared category names; each store's top categories map onto these. */
|
||
export const CATEGORIES = [
|
||
'Groente, fruit & aardappelen',
|
||
'Maaltijden & gemak',
|
||
'Vlees, vis & vega',
|
||
'Vleeswaren, kaas & tapas',
|
||
'Zuivel & eieren',
|
||
'Brood & bakkerij',
|
||
'Ontbijt & beleg',
|
||
'Pasta, rijst & wereldkeuken',
|
||
'Soepen, sauzen & conserven',
|
||
'Snoep, koek & chips',
|
||
'Diepvries',
|
||
'Koffie & thee',
|
||
'Frisdrank & sappen',
|
||
'Bier & wijn',
|
||
'Drogisterij',
|
||
'Huishouden & dieren',
|
||
'Baby & kind',
|
||
'Overig'
|
||
] as const;
|