groceries: relevance-ranked search (plain/basic products first)
All checks were successful
Deploy production / deploy (push) Successful in 1m13s

- search.ts: score on the core name (brand/size stripped): exact or query+base suffix
  (kipdij+filet), head-final compounds, prefix, inside; penalties per extra word and per
  flavour/processing word (search-words.ts, editable); bonuses for store brands, the
  query's dominant category, store count, favourites; trigram tiebreak
- groceries.synonyms table (seeded: kipdij/kippendij, pasta → spaghetti, …) expands queries
- list rows show all pack sizes of a group; group picker uses the same ranking
- scripts/groceries-search-eval.ts prints top results + latency; preflight now 8 tables
This commit is contained in:
2026-10-25 20:20:34 +00:00
parent 5028524537
commit 60370d4572
14 changed files with 31221 additions and 16 deletions

View File

@@ -0,0 +1,127 @@
CREATE TABLE "groceries"."synonyms" (
"id" serial PRIMARY KEY,
"term" text NOT NULL,
"synonym" text NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
CONSTRAINT "synonyms_term_check" CHECK ("term" ~ '^[a-z0-9][a-z0-9 -]{0,40}$'),
CONSTRAINT "synonyms_synonym_check" CHECK ("synonym" ~ '^[a-z0-9][a-z0-9 -]{0,40}$'),
CONSTRAINT "synonyms_not_self" CHECK ("term" <> "synonym")
);
--> statement-breakpoint
CREATE UNIQUE INDEX "synonyms_term_synonym_key" ON "groceries"."synonyms" ("term","synonym");
--> statement-breakpoint
-- Starter synonyms for search ranking (search.ts). Edit freely; the app caches them for 5 minutes.
INSERT INTO "groceries"."synonyms" ("term", "synonym") VALUES
('afwasmiddel', 'vaatwasmiddel'),
('bacon', 'ontbijtspek'),
('bacon', 'spek'),
('champignon', 'champignons'),
('champignons', 'champignon'),
('chocoladepasta', 'chocopasta'),
('chocopasta', 'chocoladepasta'),
('dijfilet', 'kipdij'),
('dijfilet', 'kippedij'),
('dijfilet', 'kippendij'),
('drumstick', 'kipdrumstick'),
('drumstick', 'kippenpoot'),
('drumstick', 'kippenpoten'),
('eieren', 'eitjes'),
('eitjes', 'eieren'),
('friet', 'frites'),
('friet', 'patat'),
('frites', 'friet'),
('frites', 'patat'),
('kaas', 'belegen'),
('kaas', 'edammer'),
('kaas', 'goudse'),
('kipdij', 'dijfilet'),
('kipdij', 'kippedij'),
('kipdij', 'kippendij'),
('kipdrumstick', 'drumstick'),
('kipdrumstick', 'kippenpoot'),
('kipdrumstick', 'kippenpoten'),
('kipfilet', 'kippefilet'),
('kipfilet', 'kippenfilet'),
('kippedij', 'dijfilet'),
('kippedij', 'kipdij'),
('kippedij', 'kippendij'),
('kippefilet', 'kipfilet'),
('kippefilet', 'kippenfilet'),
('kippendij', 'dijfilet'),
('kippendij', 'kipdij'),
('kippendij', 'kippedij'),
('kippenfilet', 'kipfilet'),
('kippenfilet', 'kippefilet'),
('kippenpoot', 'drumstick'),
('kippenpoot', 'kipdrumstick'),
('kippenpoot', 'kippenpoten'),
('kippenpoten', 'drumstick'),
('kippenpoten', 'kipdrumstick'),
('kippenpoten', 'kippenpoot'),
('komkommer', 'komkommers'),
('komkommers', 'komkommer'),
('mayo', 'mayonaise'),
('mayonaise', 'mayo'),
('mie', 'noedels'),
('mie', 'noodles'),
('noedels', 'mie'),
('noedels', 'noodles'),
('noodles', 'mie'),
('noodles', 'noedels'),
('ontbijtspek', 'bacon'),
('ontbijtspek', 'spek'),
('paprika', 'paprikas'),
('paprikas', 'paprika'),
('pasta', 'farfalle'),
('pasta', 'fettuccine'),
('pasta', 'fusilli'),
('pasta', 'linguine'),
('pasta', 'macaroni'),
('pasta', 'penne'),
('pasta', 'rigatoni'),
('pasta', 'spaghetti'),
('pasta', 'tagliatelle'),
('patat', 'friet'),
('patat', 'frites'),
('pindakaas', 'pindapasta'),
('pindapasta', 'pindakaas'),
('rijst', 'basmati'),
('rijst', 'pandan'),
('rijst', 'zilvervlies'),
('rundergehakt', 'rundgehakt'),
('rundgehakt', 'rundergehakt'),
('sojasaus', 'soy'),
('sojasaus', 'soyasaus'),
('soy', 'sojasaus'),
('soy', 'soyasaus'),
('soyasaus', 'sojasaus'),
('soyasaus', 'soy'),
('spek', 'bacon'),
('spek', 'ontbijtspek'),
('toiletpapier', 'wc'),
('tomaat', 'tomaten'),
('tomaten', 'tomaat'),
('tortilla', 'tortillas'),
('tortilla', 'wraps'),
('tortillas', 'tortilla'),
('tortillas', 'wraps'),
('vaatwasmiddel', 'afwasmiddel'),
('wc', 'toiletpapier'),
('wraps', 'tortilla'),
('wraps', 'tortillas'),
('yoghourt', 'yoghurt'),
('yoghourt', 'yogurt'),
('yoghurt', 'yoghourt'),
('yoghurt', 'yogurt'),
('yogurt', 'yoghourt'),
('yogurt', 'yoghurt')
ON CONFLICT DO NOTHING;
--> statement-breakpoint
DO $$
BEGIN
IF current_user = 'personal' AND EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
GRANT SELECT, INSERT, UPDATE, DELETE ON "groceries"."synonyms" TO personal_app;
GRANT USAGE, SELECT ON SEQUENCE "groceries"."synonyms_id_seq" TO personal_app;
END IF;
END $$;

File diff suppressed because it is too large Load Diff

View File

@@ -167,8 +167,8 @@ try {
where n.nspname = 'groceries' and c.relkind = 'r' where n.nspname = 'groceries' and c.relkind = 'r'
`; `;
if ( if (
Number(groceries.tables) !== 7 || Number(groceries.tables) !== 8 ||
Number(groceries.app_rw) !== 7 || Number(groceries.app_rw) !== 8 ||
Number(groceries.ro_read) !== 0 || Number(groceries.ro_read) !== 0 ||
groceries.public_usage groceries.public_usage
) )

View File

@@ -0,0 +1,64 @@
// Print the top results for some grocery searches with the relevance ranking (search.ts).
// Read-only. Usage: DB_URL=… bun scripts/groceries-search-eval.ts [query …]
// Uses the synonyms table, or the seed in the migration when the table doesn't exist yet.
import { readdirSync, readFileSync } from 'node:fs';
import postgres from 'postgres';
import {
expand,
queryWords,
rankedGroups,
searchConds,
synonymMap,
type Synonyms
} from '../src/lib/server/groceries/search';
const sql = postgres(process.env.DB_URL!, { max: 2 });
const queries = process.argv.slice(2).length
? process.argv.slice(2)
: ['kipdij', 'melk', 'rijst', 'tofu', 'kaas', 'eieren', 'pasta'];
let synonyms: Synonyms;
try {
synonyms = synonymMap(await sql`select term, synonym from groceries.synonyms`);
} catch {
const dir = readdirSync('drizzle').find((d) => d.endsWith('_groceries_search_synonyms'))!;
const seed = readFileSync(`drizzle/${dir}/migration.sql`, 'utf8');
const rows = [...seed.matchAll(/\('([a-z0-9 -]+)', '([a-z0-9 -]+)'\)/g)].map((m) => ({
term: m[1],
synonym: m[2]
}));
synonyms = synonymMap(rows);
console.log(`(synonyms table missing; using the ${rows.length} from the migration seed)`);
}
const cats = await sql<
{ id: number; parent_id: number | null }[]
>`select id, parent_id from groceries.categories`;
const ctx = { at: Date.now(), synonyms, parent: new Map(cats.map((c) => [c.id, c.parent_id])) };
for (const q of queries) {
const words = expand(queryWords(q), synonyms);
const where = [
sql`g.merged_into_id is null`,
sql`(g.item_count > 0 or g.favorite)`,
...searchConds(sql, words)
].reduce((a, c) => sql`${a} and ${c}`);
const t = performance.now();
const ranked = await rankedGroups(sql, where, q, words, { ctx });
const ids = ranked.map((r) => r.id);
const ms = performance.now() - t;
const top = ids.slice(0, Number(process.env.TOP) || 5);
const rows = await sql<{ id: number; name: string; brand: string | null; store_count: number }[]>`
select id, name, brand, store_count from groceries.groups where id = any(${top}::int[])`;
console.log(`\n${q}: ${ids.length} candidates, ${ms.toFixed(0)} ms`);
for (const id of top) {
const r = rows.find((x) => x.id === id)!;
const sc = ranked.find((x) => x.id === id)!;
console.log(
` ${r.name} | ${r.brand ?? '-'} | ${r.store_count} stores | #${r.id}` +
(process.env.DEBUG
? ` | score ${sc.score} text ${sc.text} pen ${sc.penalty} cat ${sc.categoryId}`
: '')
);
}
}
await sql.end();

View File

@@ -4,6 +4,8 @@ import {
cheapest, cheapest,
formatAmount, formatAmount,
formatUnitPrice, formatUnitPrice,
packLabel,
packSizes,
parseEuros, parseEuros,
sortByPrice, sortByPrice,
toOffers, toOffers,
@@ -117,3 +119,21 @@ describe('bestPerStore', () => {
expect(bestPerStore(toOffers([item(1, 1, 100, 100)], []), 'pack').spread).toBeNull(); expect(bestPerStore(toOffers([item(1, 1, 100, 100)], []), 'pack').spread).toBeNull();
}); });
}); });
describe('packSizes', () => {
it('lists distinct pack sizes smallest first with friendly units', () => {
expect(
packSizes([
{ unitAmount: 0.8, unit: 'kg' },
{ unitAmount: 0.22, unit: 'kg' },
{ unitAmount: 0.45, unit: 'kg' },
{ unitAmount: 0.22, unit: 'kg' },
{ unitAmount: 1.5, unit: 'kg', available: false },
{ unitAmount: null, unit: null }
])
).toEqual(['220 g', '450 g', '800 g']);
expect(packLabel(1.5, 'l')).toBe('1,5 L');
expect(packLabel(0.33, 'l')).toBe('330 ml');
expect(packLabel(6, 'piece')).toBe('6 st');
});
});

View File

@@ -178,3 +178,25 @@ export function bestPerStore(offers: Offer[], mode: 'unit' | 'pack') {
: null; : null;
return { best: stores[0] ?? null, stores, spread }; return { best: stores[0] ?? null, stores, spread };
} }
/** Pack size label: 0.5 kg → "500 g", 1.5 l → "1,5 L", 0.33 l → "330 ml", 6 piece → "6 st". */
export function packLabel(amount: number | null | undefined, unit: string | null | undefined) {
if (amount == null || !(amount > 0) || !unit) return null;
const nl = (n: number) => n.toLocaleString('nl-NL', { maximumFractionDigits: 2 });
if (unit === 'kg' && amount < 1) return `${nl(Math.round(amount * 1000))} g`;
if (unit === 'l' && amount < 1) return `${nl(Math.round(amount * 1000))} ml`;
return `${nl(amount)} ${UNIT_LABEL[unit] ?? unit}`;
}
/** Distinct pack sizes of a group's products, smallest first ("220 g", "450 g", "800 g"). */
export function packSizes(
items: { unitAmount: number | null; unit: string | null; available?: boolean }[]
): string[] {
const seen = new Map<string, number>();
for (const i of items) {
if (i.available === false) continue;
const label = packLabel(i.unitAmount, i.unit);
if (label && !seen.has(label)) seen.set(label, i.unitAmount!);
}
return [...seen].sort((a, b) => a[1] - b[1]).map(([l]) => l);
}

View File

@@ -256,8 +256,30 @@ export const groceryImportRuns = groceriesSchema.table(
(t) => [index('import_runs_store_idx').on(t.storeId, t.startedAt)] (t) => [index('import_runs_store_idx').on(t.storeId, t.startedAt)]
); );
/**
* Search synonyms (search.ts): a query word `term` also finds products with `synonym`.
* One-directional (pasta → spaghetti); spelling variants are stored both ways. Folded,
* lower-case words; edits apply within 5 minutes (cached).
*/
export const grocerySynonyms = groceriesSchema.table(
'synonyms',
{
id: id(),
term: text('term').notNull(),
synonym: text('synonym').notNull(),
...timestamps
},
(t) => [
uniqueIndex('synonyms_term_synonym_key').on(t.term, t.synonym),
check('synonyms_term_check', sql`${t.term} ~ '^[a-z0-9][a-z0-9 -]{0,40}$'`),
check('synonyms_synonym_check', sql`${t.synonym} ~ '^[a-z0-9][a-z0-9 -]{0,40}$'`),
check('synonyms_not_self', sql`${t.term} <> ${t.synonym}`)
]
);
export type GroceryStore = typeof groceryStores.$inferSelect; export type GroceryStore = typeof groceryStores.$inferSelect;
export type GroceryGroup = typeof groceryGroups.$inferSelect; export type GroceryGroup = typeof groceryGroups.$inferSelect;
export type GroceryItem = typeof groceryItems.$inferSelect; export type GroceryItem = typeof groceryItems.$inferSelect;
export type GroceryCategory = typeof groceryCategories.$inferSelect; export type GroceryCategory = typeof groceryCategories.$inferSelect;
export type GroceryManualPrice = typeof groceryManualPrices.$inferSelect; export type GroceryManualPrice = typeof groceryManualPrices.$inferSelect;
export type GrocerySynonym = typeof grocerySynonyms.$inferSelect;

View File

@@ -284,7 +284,7 @@ const SIZE_IN_NAME = new RegExp(
'gi' 'gi'
); );
const stem = (t: string) => export const stem = (t: string) =>
t.length > 5 && t.endsWith('en') t.length > 5 && t.endsWith('en')
? t.slice(0, -2) ? t.slice(0, -2)
: t.length > 4 && t.endsWith('s') : t.length > 4 && t.endsWith('s')

View File

@@ -0,0 +1,215 @@
// Editable word lists for grocery search ranking (see search.ts). Lower-case, accents allowed
// (they are folded). Synonyms live in the groceries.synonyms table so they can change without
// a deploy; these lists change with the code.
/**
* Base forms that still mean "the plain product" when glued to the query or next to it:
* kipdij + filet, kipdijfilet + blokjes, gehakt, … "query+suffix" counts as an exact match;
* the value is a small penalty so the plainest cut comes first.
*/
export const BASE_SUFFIXES: Record<string, number> = {
// value = small penalty: kipdijfilet is the plainest, blokjes/reepjes are a cut further.
filet: 0,
filets: 0,
vlees: 0,
naturel: 0,
puur: 0,
plakken: 1,
plakjes: 1,
gehakt: 2,
stuk: 2,
stukken: 2,
stukjes: 4,
schijven: 3,
schijfjes: 3,
lapjes: 3,
lapje: 3,
blokjes: 5,
blokje: 5,
reepjes: 5,
reepje: 5
};
/**
* Words that make a product "more than the basic thing", with a penalty weight (1 = one strong
* flavour/processing word; 0.1 = a harmless qualifier like halfvolle or scharrel). Words not in
* this list that aren't part of the query cost a fixed extra-word penalty instead.
*/
export const MODIFIERS: Record<string, number> = {
// flavoured, marinated, processed
gekruid: 1,
gekruide: 1,
gemarineerd: 1,
gemarineerde: 1,
madras: 1,
shoarma: 1,
doner: 1,
kebab: 1,
gyros: 1,
spies: 1,
spiesjes: 1,
spiesje: 1,
sate: 1,
satay: 1,
gegrild: 1,
gegrilde: 1,
grill: 1,
tandoori: 1,
tikka: 1,
honing: 1,
pittig: 1,
pittige: 1,
piri: 1,
cajun: 1,
teriyaki: 1,
bbq: 1,
barbecue: 1,
knoflook: 0.8,
paprika: 0.8,
kerrie: 0.8,
curry: 0.8,
tuinkruiden: 0.8,
pesto: 0.8,
krokant: 1,
krokante: 1,
gepaneerd: 1,
gepaneerde: 1,
gerookt: 0.8,
gerookte: 0.8,
gebakken: 0.8,
gefrituurd: 1,
burger: 1,
burgers: 1,
worst: 1,
worstjes: 1,
nuggets: 1,
schnitzel: 1,
cordon: 1,
rollade: 1,
salade: 1,
saus: 1,
spread: 1,
snack: 1,
snacks: 1,
chips: 1,
soep: 1,
maaltijd: 1,
chocolade: 1,
choco: 1,
aardbei: 0.8,
vanille: 0.8,
banaan: 0.8,
drink: 0.8,
drank: 0.8,
dessert: 1,
toetje: 1,
ijs: 1,
wafels: 1,
koek: 1,
koekjes: 1,
reep: 1,
repen: 1,
carbonara: 1,
bolognese: 1,
pomodoro: 1,
kaassaus: 1,
kip: 0.5,
mix: 0.6,
smaak: 0.6,
rooksmaak: 1,
rook: 0.8,
style: 0.6,
// pets and non-food that share food words (kattensnoepjes "kaas", …)
kat: 1,
katten: 1,
hond: 1,
honden: 1,
snoepjes: 1,
// harmless qualifiers: still the basic product
halfvolle: 0.1,
halfvol: 0.1,
volle: 0.1,
vol: 0.1,
magere: 0.1,
mager: 0.1,
scharrel: 0.1,
vrije: 0.1,
uitloop: 0.1,
biologisch: 0.2,
biologische: 0.2,
bio: 0.2,
verse: 0.1,
vers: 0.1,
jong: 0.15,
jonge: 0.15,
belegen: 0.15,
oud: 0.2,
oude: 0.2,
extra: 0.2,
goudse: 0.15,
gouda: 0.15,
basmati: 0.2,
pandan: 0.2,
zilvervlies: 0.2,
witte: 0.15,
wit: 0.15,
bruine: 0.2,
volkoren: 0.2,
zachte: 0.15,
stevige: 0.15,
naturel: 0,
ongezouten: 0.1,
halal: 0.2,
groot: 0.1,
grote: 0.1,
klein: 0.1,
kleine: 0.1,
lang: 0.1,
houdbaar: 0.1,
langlekker: 0.2,
plakken: 0.1,
geraspt: 0.2,
geraspte: 0.2,
durum: 0.1,
tarwe: 0.1
};
/**
* Private-label / store brands (folded brand label, matched exactly or as a prefix). "Huismerk…"
* labels come from the grouping (AH/Jumbo/PLUS house brands share a group), see normalize.ts.
*/
export const STORE_BRANDS = [
'huismerk',
'ah',
'albert heijn',
'jumbo',
"jumbo's",
'plus',
'1 de beste',
"g'woon",
'vomar',
'hoogvliet',
'dirk',
'aldi',
'milsani',
'lidl',
'boni',
'spar',
'coop',
'deen',
'poiesz',
'dekamarkt',
'jan linders',
'nettorama',
'merkloos'
];
/** Brand labels that are a store's own line but not its plain one (smaller bonus). */
export const STORE_BRAND_PREMIUM = [
'huismerk premium',
'huismerk kids',
'huismerk plantaardig',
'ah excellent',
'jumbo excellent',
'plus excellent'
];

View File

@@ -0,0 +1,151 @@
import { describe, expect, it } from 'vitest';
import {
brandBonus,
expand,
queryWords,
rank,
synonymMap,
textScore,
wordPenalty,
type Candidate
} from './search';
let nextId = 1;
const c = (
name: string,
brand: string | null,
categoryId: number | null,
storeCount = 1,
extra: Partial<Candidate> = {}
): Candidate => ({
id: nextId++,
name,
brand,
categoryId,
storeCount,
itemCount: storeCount,
favorite: false,
...extra
});
// Categories: 1 kip, 2 tapas (both top 10), 3 vis (top 10), 4 melk, 5 chocolade (top 20), 6 pasta, 7 sauzen
const TOP: Record<number, number> = { 1: 10, 2: 11, 3: 10, 4: 12, 5: 20, 6: 13, 7: 14 };
const synonyms = synonymMap([
{ term: 'kipdij', synonym: 'kippendij' },
{ term: 'kippendij', synonym: 'kipdij' },
{ term: 'pasta', synonym: 'spaghetti' },
{ term: 'pasta', synonym: 'penne' }
]);
const names = (q: string, cands: Candidate[]) =>
rank(cands, {
words: expand(queryWords(q), synonyms),
topOf: (id) => (id == null ? null : (TOP[id] ?? id))
}).map((r) => `${r.name} (${r.brand})`);
describe('grocery search ranking', () => {
it('puts plain store-brand kipdijfilet above doner, spiesjes and madras', () => {
const out = names('kipdij', [
c('Kipdij doner', 'Wahid', 1, 4),
c('Kipdijspiesjes Madras', 'Kips', 2, 3),
c('Kipdij-spies', 'Bits & Bites', 2),
c('Kipdijspiesjes', 'Kips', 2, 2),
c('Kipdijfiletreepjes', 'Huismerk', 1),
c('Kipdijfilet', 'Huismerk biologisch', 1, 2),
c('Kipdijfilet', 'Hoogvliet', 3),
c('Kipdijshoarma', 'Huismerk', 1),
c('Scharrel kipdijfilet', 'Huismerk', 1)
]);
expect(out[0]).toBe('Kipdijfilet (Huismerk biologisch)');
expect(out.slice(0, 4).sort()).toEqual([
'Kipdijfilet (Hoogvliet)',
'Kipdijfilet (Huismerk biologisch)',
'Kipdijfiletreepjes (Huismerk)',
'Scharrel kipdijfilet (Huismerk)'
]);
expect(out.indexOf('Kipdijfiletreepjes (Huismerk)')).toBeLessThan(
out.indexOf('Kipdij doner (Wahid)')
);
for (const flavoured of [
'Kipdij doner (Wahid)',
'Kipdijspiesjes Madras (Kips)',
'Kipdijshoarma (Huismerk)'
])
expect(out.indexOf(flavoured)).toBeGreaterThan(
out.indexOf('Scharrel kipdijfilet (Huismerk)')
);
expect(out.at(-1)).toBe('Kipdijspiesjes Madras (Kips)');
});
it('finds spelling variants through synonyms', () => {
const out = names('kippendij', [
c('Kipdijfilet', 'Huismerk', 1),
c('Kippenpoten', 'Huismerk', 1)
]);
expect(out[0]).toBe('Kipdijfilet (Huismerk)');
});
it('ranks milk above chocolate with melk in the name', () => {
const out = names('melk', [
c('Melk chocoladereep', 'Verkade', 5, 5),
c('Chocoladereep melk', "Côte d'Or", 5, 6),
c('Melk', "Tony's Chocolonely", 5, 2),
c('Halfvolle melk', 'Huismerk', 4, 3),
c('Volle melk', 'Huismerk', 4, 3),
c('Halfvolle melk', 'Campina', 4, 5)
]);
expect(out.slice(0, 2).sort()).toEqual(['Halfvolle melk (Huismerk)', 'Volle melk (Huismerk)']);
expect(out[2]).toBe('Halfvolle melk (Campina)');
});
it('treats head-final compounds as the product (zilvervliesrijst is rijst)', () => {
const rijst = textScore({ name: 'Zilvervliesrijst', brand: 'Huismerk' }, [['rijst']]);
const wafel = textScore({ name: 'Rijstwafels', brand: 'Huismerk' }, [['rijst']]);
expect(rijst.text - rijst.penalty).toBeGreaterThan(wafel.text - wafel.penalty);
});
it('expands pasta to pasta shapes and prefers them over ready meals', () => {
const out = names('pasta', [
c('Good pasta carbonara', 'Knorr', 7, 6),
c('Pasta Frutta F!ZZ', 'Haribo', 5, 5),
c('Spaghetti', 'Huismerk', 6, 3),
c('Penne', 'Huismerk', 6, 2)
]);
expect(out.slice(0, 2)).toEqual(['Spaghetti (Huismerk)', 'Penne (Huismerk)']);
});
it('keeps the plain product first for tofu', () => {
const out = names('tofu', [
c('Gerookte Tofu', 'Vivera', 1, 5),
c('Tofu met Rooksmaak', 'Huismerk', 1),
c('Tofu', 'Vivera', 1, 4),
c('Tofu Naturel', 'Huismerk', 1)
]);
expect(out.slice(0, 2).sort()).toEqual(['Tofu (Vivera)', 'Tofu Naturel (Huismerk)']);
});
it('scores words, brands and multi-word queries sensibly', () => {
expect(wordPenalty('madras')).toBeGreaterThan(wordPenalty('scharrel'));
expect(wordPenalty('filet')).toBe(0);
expect(wordPenalty('48+')).toBe(0);
expect(wordPenalty('chocoladereep')).toBeGreaterThan(10);
expect(brandBonus('Huismerk')).toBeGreaterThan(brandBonus('Campina'));
expect(brandBonus('1 de Beste')).toBeGreaterThan(0);
expect(brandBonus(null)).toBeGreaterThan(0);
const two = textScore({ name: 'Halfvolle melk', brand: 'Huismerk' }, [['halfvolle'], ['melk']]);
const one = textScore({ name: 'Halfvolle melk', brand: 'Huismerk' }, [['melk']]);
expect(two.penalty).toBeLessThan(one.penalty + 0.01);
expect(queryWords('Kip-dij Filet')).toEqual(['kip', 'dij', 'filet']);
});
it('can put promotions first without losing relevance order', () => {
const out = rank(
[
c('Kipdijshoarma', 'Huismerk', 1, 1, { promoCount: 1 }),
c('Kipdijfilet', 'Huismerk', 1),
c('Kipdij doner', 'Wahid', 1, 1, { promoCount: 2 })
],
{ words: [['kipdij']], topOf: (id) => id },
{ promoFirst: true }
).map((r) => r.name);
expect(out).toEqual(['Kipdij doner', 'Kipdijshoarma', 'Kipdijfilet']);
});
});

View File

@@ -0,0 +1,274 @@
// Relevance ranking for grocery search: plain/basic products first, flavoured and specialised
// ones further down. Pure scoring (tested in search.test.ts) plus a small DB layer that
// fetches candidates and the synonyms table.
//
// Score = text match on the core name (brand and size stripped)
// - complexity (extra words, flavour/processing words; lists in search-words.ts)
// + bonuses (store brand, the query's dominant category, sold in more stores, favourite)
// + trigram similarity as a tiebreak.
import type { PendingQuery, Row, Sql } from 'postgres';
import { fold, nameTokens, stem } from './normalize.js';
import { BASE_SUFFIXES, MODIFIERS, STORE_BRAND_PREMIUM, STORE_BRANDS } from './search-words.js';
export type Candidate = {
id: number;
name: string;
brand: string | null;
categoryId: number | null;
storeCount: number;
itemCount: number;
favorite: boolean;
promoCount?: number;
/** pg_trgm word similarity of the query to the name, 0..1 */
sim?: number;
};
export type Scored = Candidate & { score: number; text: number; penalty: number };
/** term → extra search terms (folded). One-directional; seed both ways for spelling variants. */
export type Synonyms = Map<string, string[]>;
const SUFFIXES = new Map(Object.entries(BASE_SUFFIXES).map(([w, v]) => [stem(fold(w)), v]));
const MODS = new Map(Object.entries(MODIFIERS).map(([w, v]) => [stem(fold(w)), v]));
const EXTRA_WORD = 10;
const MOD_WORD = 22;
const SYN_FACTOR = 0.92;
/** Query → folded words (max 6). */
export function queryWords(q: string): string[] {
return fold(q)
.replace(/['’]/g, '')
.split(/[^a-z0-9%+]+/)
.filter(Boolean)
.slice(0, 6);
}
/** Each query word with its synonyms: [[word, syn, …], …] (folded, deduped). */
export function expand(words: string[], synonyms: Synonyms): string[][] {
return words.map((w) => [...new Set([w, ...(synonyms.get(w) ?? [])])]);
}
/** Cost of a glued run of base suffixes ("filet" 0, "filetblokje" 5, "" 0); null if not one. */
function suffixCost(r: string, depth = 0): number | null {
if (!r) return 0;
if (depth > 3) return null;
let best: number | null = null;
for (const [s, v] of SUFFIXES)
if (r.startsWith(s)) {
const rest = suffixCost(r.slice(s.length), depth + 1);
if (rest != null && (best == null || v + rest < best)) best = v + rest;
}
return best;
}
/** Penalty for a leftover word or compound part (small or 0 for base suffixes). */
export function wordPenalty(w: string): number {
if (!w || w.length < 2 || /^\d+[%+]?$/.test(w)) return 0;
const s = stem(w);
const sc = suffixCost(s) ?? suffixCost(w);
if (sc != null) return sc;
const m = MODS.get(s) ?? MODS.get(w);
if (m != null) return m * MOD_WORD;
// Compounds containing a flavour/processing word (madrasfilet, chocoladereep, grillworst).
let m2 = 0;
for (const [k, v] of MODS) if (v >= 0.5 && k.length > 3 && s.includes(k)) m2 = Math.max(m2, v);
return m2 ? Math.max(m2 * MOD_WORD, EXTRA_WORD) : EXTRA_WORD;
}
type Hit = { level: number; token: number; leftover: string; cost: number };
/** Best match of one term against the core tokens (stemmed). */
function matchTerm(term: string, tokens: string[]): Hit | null {
let best: Hit | null = null;
const take = (h: Hit) => {
if (!best || h.level - h.cost > best.level - best.cost) best = h;
};
tokens.forEach((t, i) => {
for (const e of new Set([term, stem(term)])) {
if (e.length < 2) continue;
const at = t.indexOf(e);
if (at < 0) continue;
const after = t.slice(at + e.length);
const cost = suffixCost(after);
if (at === 0 && cost != null) take({ level: 100, token: i, leftover: '', cost });
else if (at > 0 && cost != null)
// Dutch compounds are head-final: zilvervliesrijst is rijst, scharrelkipdijfilet is kipdij.
take({ level: 80, token: i, leftover: t.slice(0, at), cost });
else if (at === 0)
// kipdijspiesjes: starts with the query but is something else.
take({ level: 60, token: i, leftover: after, cost: 0 });
else take({ level: 40, token: i, leftover: t.slice(0, at) + ' ' + after, cost: 0 });
}
});
return best;
}
/** Text score (0..106: 100 exact or query+suffix, 80 compound end, 60 prefix, 40 inside, +6 first word) and complexity penalty. */
export function textScore(c: Pick<Candidate, 'name' | 'brand'>, words: string[][]) {
const tokens = nameTokens(c.name, c.brand);
const brandToks = new Set(
fold(c.brand ?? '')
.split(/[^a-z0-9]+/)
.filter(Boolean)
);
const used = new Set<number>();
let total = 0;
let penalty = 0;
let first = false;
for (const alts of words) {
let best: (Hit & { score: number }) | null = null;
alts.forEach((term, k) => {
const h = matchTerm(term, tokens);
if (!h) return;
const score = h.level * (k === 0 ? 1 : SYN_FACTOR);
if (!best || score - h.cost > best.score - best.cost) best = { ...h, score };
});
if (best) {
const b = best as Hit & { score: number };
total += b.score;
used.add(b.token);
if (b.token === 0) first = true;
penalty += b.cost;
for (const part of b.leftover.split(' ')) penalty += wordPenalty(part);
} else if (alts.some((a) => brandToks.has(a))) total += 70;
}
tokens.forEach((t, i) => {
if (!used.has(i)) penalty += wordPenalty(t);
});
const text = words.length ? total / words.length : 0;
return { text: text + (first && text > 0 ? 6 : 0), penalty: Math.min(penalty, 70) };
}
/** Store/private-label bonus from the group's brand label. */
export function brandBonus(brand: string | null): number {
const b = fold(brand ?? '').trim();
if (!b) return 12; // loose produce
if (STORE_BRAND_PREMIUM.includes(b)) return 6;
if (b === 'huismerk' || b === 'huismerk basic') return 15;
if (b.startsWith('huismerk')) return 10;
if (STORE_BRANDS.some((p) => b === p || b.startsWith(p + ' '))) return 12;
return 0;
}
export type Ctx = {
words: string[][];
/** category id → its top-level id */
topOf: (id: number | null) => number | null;
};
/** Score and sort candidates (best first). */
export function rank(cands: Candidate[], ctx: Ctx, opts: { promoFirst?: boolean } = {}): Scored[] {
const pre = cands.map((c) => ({ c, ...textScore(c, ctx.words) }));
// Dominant category: the most common one among strong name matches; plain products (few
// extra words) and products sold in more stores decide what the query is about.
const dominant = (min: number) => {
const w = new Map<number, number>();
for (const p of pre)
if (p.text >= min && p.c.categoryId != null)
w.set(
p.c.categoryId,
(w.get(p.c.categoryId) ?? 0) + (1 + p.c.storeCount) / (1 + p.penalty / 5) ** 2
);
return [...w].sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
};
const dom = dominant(90) ?? dominant(75) ?? dominant(55);
const domTop = ctx.topOf(dom);
const out = pre.map(({ c, text, penalty }) => {
let score = text - penalty;
if (text === 0) score = (c.sim ?? 0) * 30 - 40; // matched only via brand/category/other names
score += brandBonus(c.brand);
if (dom != null) {
if (c.categoryId === dom) score += 12;
else if (domTop != null && ctx.topOf(c.categoryId) === domTop) score += 5;
else score -= 10;
}
score += Math.min(Math.max(c.storeCount - 1, 0), 5) * 4;
score += Math.min(Math.log2(1 + c.itemCount), 4);
if (c.favorite) score += 10;
score += (c.sim ?? 0) * 5;
return {
...c,
score: Math.round(score * 100) / 100,
text,
penalty: Math.round(penalty * 10) / 10
};
});
return out.sort(
(a, b) =>
(opts.promoFirst ? Number((b.promoCount ?? 0) > 0) - Number((a.promoCount ?? 0) > 0) : 0) ||
b.score - a.score ||
b.storeCount - a.storeCount ||
b.itemCount - a.itemCount ||
a.name.length - b.name.length ||
a.id - b.id
);
}
// ---- DB --------------------------------------------------------------------------------------
type SearchCtx = { at: number; synonyms: Synonyms; parent: Map<number, number | null> };
let cache: SearchCtx | null = null;
const TTL = 5 * 60_000;
/** Synonyms table + category parents, cached for 5 minutes. */
export async function searchContext(s: Sql): Promise<SearchCtx> {
if (cache && Date.now() - cache.at < TTL) return cache;
const [syn, cats] = await Promise.all([
s<{ term: string; synonym: string }[]>`select term, synonym from groceries.synonyms`,
s<{ id: number; parent_id: number | null }[]>`select id, parent_id from groceries.categories`
]);
cache = {
at: Date.now(),
synonyms: synonymMap(syn),
parent: new Map(cats.map((c) => [c.id, c.parent_id]))
};
return cache;
}
export function synonymMap(rows: { term: string; synonym: string }[]): Synonyms {
const m: Synonyms = new Map();
for (const r of rows) {
const t = fold(r.term).trim();
m.set(t, [...(m.get(t) ?? []), fold(r.synonym).trim()]);
}
return m;
}
const like = (w: string) => '%' + w.replace(/[%_\\]/g, '\\$&') + '%';
/** WHERE fragments: every query word (or one of its synonyms) appears in g.search_text. */
export function searchConds(s: Sql, words: string[][]) {
return words.map((alts) =>
alts.length === 1
? s`g.search_text like ${like(alts[0])}`
: s`g.search_text like any(${alts.map(like)}::text[])`
);
}
const MAX_CANDIDATES = 4000;
/**
* Rank the groups matching `where` (alias g) for query `q`, best first. Up to MAX_CANDIDATES;
* groups whose name contains a query word are fetched first so broad queries keep them.
*/
export async function rankedGroups(
s: Sql,
where: PendingQuery<Row[]>,
q: string,
words: string[][],
opts: { promoFirst?: boolean; ctx?: SearchCtx } = {}
): Promise<Scored[]> {
const ctx = opts.ctx ?? (await searchContext(s));
const nameLike = words.flat().map(like);
const cands = await s<Candidate[]>`
select g.id, g.name, g.brand, g.category_id as "categoryId", g.store_count as "storeCount",
g.item_count as "itemCount", g.favorite, g.promo_count as "promoCount",
public.word_similarity(${q}::text, lower(g.name))::float8 as sim
from groceries.groups g where ${where}
order by (lower(g.name) like any(${nameLike}::text[])) desc, g.store_count desc, g.item_count desc
limit ${MAX_CANDIDATES}`;
const topOf = (id: number | null) => (id == null ? null : (ctx.parent.get(id) ?? id));
return rank(cands, { words, topOf }, opts);
}
/** rankedGroups(), ids only. */
export const rankedIds = async (...a: Parameters<typeof rankedGroups>) =>
(await rankedGroups(...a)).map((c) => c.id);

View File

@@ -4,6 +4,13 @@ import { refreshGroupStats } from '#lib/server/groceries/import.js';
import { parseEuros } from '#lib/groceries.js'; import { parseEuros } from '#lib/groceries.js';
import { displayName, fold, parseSize } from '#lib/server/groceries/normalize.js'; import { displayName, fold, parseSize } from '#lib/server/groceries/normalize.js';
import { HOMEWARE_TOPS } from '#lib/server/groceries/taxonomy.js'; import { HOMEWARE_TOPS } from '#lib/server/groceries/taxonomy.js';
import {
expand,
queryWords,
rankedIds,
searchConds,
searchContext
} from '#lib/server/groceries/search.js';
import { deleteObject, groceryImageKey, uploadImage } from '#lib/server/r2.js'; import { deleteObject, groceryImageKey, uploadImage } from '#lib/server/r2.js';
import { fail } from '@sveltejs/kit'; import { fail } from '@sveltejs/kit';
import type { Sql } from 'postgres'; import type { Sql } from 'postgres';
@@ -112,7 +119,7 @@ export const load: PageServerLoad = async (event) => {
const s = sql(); const s = sql();
const p = event.url.searchParams; const p = event.url.searchParams;
const q = fold(p.get('q')?.trim() ?? '').slice(0, 100); const q = fold(p.get('q')?.trim() ?? '').slice(0, 100);
const words = q.split(/\s+/).filter(Boolean).slice(0, 6); const words = q ? expand(queryWords(q), (await searchContext(s)).synonyms) : [];
const storeSlug = p.get('store') ?? ''; const storeSlug = p.get('store') ?? '';
const catSlug = p.get('cat') ?? ''; const catSlug = p.get('cat') ?? '';
const sortParam = p.get('sort') ?? ''; const sortParam = p.get('sort') ?? '';
@@ -148,8 +155,7 @@ export const load: PageServerLoad = async (event) => {
const showNonfood = nonfood || (cat ? homewareIds.includes(cat.id) : false); const showNonfood = nonfood || (cat ? homewareIds.includes(cat.id) : false);
const conds = [s`g.merged_into_id is null`, s`(g.item_count > 0 or g.favorite)`]; const conds = [s`g.merged_into_id is null`, s`(g.item_count > 0 or g.favorite)`];
for (const w of words) conds.push(...searchConds(s, words));
conds.push(s`g.search_text like ${'%' + w.replace(/[%_\\]/g, '\\$&') + '%'}`);
if (store) if (store)
conds.push(s`(g.store_ids @> array[${store.id}]::int[] or exists ( conds.push(s`(g.store_ids @> array[${store.id}]::int[] or exists (
select 1 from groceries.manual_prices m where m.group_id = g.id and m.store_id = ${store.id}))`); select 1 from groceries.manual_prices m where m.group_id = g.id and m.store_id = ${store.id}))`);
@@ -178,8 +184,20 @@ export const load: PageServerLoad = async (event) => {
? s`g.store_count desc, g.item_count desc` ? s`g.store_count desc, g.item_count desc`
: s`g.favorite desc, ${relevance} g.store_count desc, g.item_count desc`; : s`g.favorite desc, ${relevance} g.store_count desc, g.item_count desc`;
// With a search query the default and promo sorts use relevance ranking (search.ts):
// plain/basic products first, flavoured and specialised ones further down.
const relevant = words.length > 0 && (sort === '' || sort === 'promo');
const [groups, [{ total }]] = await Promise.all([ const [groups, [{ total }]] = await Promise.all([
s<GroupRow[]>` relevant
? rankedIds(s, where, q, words, { promoFirst: sort === 'promo' }).then(async (all) => {
const ids = all.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE);
if (!ids.length) return [] as GroupRow[];
const rows = await s<GroupRow[]>`
select ${GROUP_COLS(s)} from groceries.groups g where g.id = any(${ids}::int[])`;
const pos = new Map(ids.map((id, i) => [id, i]));
return rows.sort((a, b) => pos.get(a.id)! - pos.get(b.id)!);
})
: s<GroupRow[]>`
select ${GROUP_COLS(s)} from groceries.groups g where ${where} select ${GROUP_COLS(s)} from groceries.groups g where ${where}
order by ${order}, g.name, g.id limit ${PAGE_SIZE} offset ${(page - 1) * PAGE_SIZE}`, order by ${order}, g.name, g.id limit ${PAGE_SIZE} offset ${(page - 1) * PAGE_SIZE}`,
s<{ total: number }[]>`select count(*)::int as total from groceries.groups g where ${where}` s<{ total: number }[]>`select count(*)::int as total from groceries.groups g where ${where}`

View File

@@ -54,6 +54,7 @@
formatAmount, formatAmount,
formatCents, formatCents,
formatUnitPrice, formatUnitPrice,
packSizes,
parseEuros, parseEuros,
sortByPrice, sortByPrice,
toOffers, toOffers,
@@ -178,6 +179,7 @@
); );
return { return {
g, g,
sizes: packSizes(itemsByGroup.get(g.id) ?? []),
...bestPerStore(offers, mode), ...bestPerStore(offers, mode),
anyPromo: offers.some((o) => o.promoLabel || o.regularCents) anyPromo: offers.some((o) => o.promoLabel || o.regularCents)
}; };
@@ -657,7 +659,11 @@
<div class="row-title">{g.name}</div> <div class="row-title">{g.name}</div>
<div class="meta flex flex-wrap items-center gap-x-1.5"> <div class="meta flex flex-wrap items-center gap-x-1.5">
{#if g.brand}<span>{g.brand}</span>{/if} {#if g.brand}<span>{g.brand}</span>{/if}
{#if r.best?.sizeText}<span>· {r.best.sizeText}</span>{/if} {#if r.sizes.length > 1}<span class="sizes" title="Pack sizes"
>· {r.sizes.slice(0, 4).join(' / ')}{r.sizes.length > 4
? ` +${r.sizes.length - 4}`
: ''}</span
>{:else if r.best?.sizeText}<span>· {r.best.sizeText}</span>{/if}
{#if r.stores.length > 1}<span>· {r.stores.length} stores</span>{/if} {#if r.stores.length > 1}<span>· {r.stores.length} stores</span>{/if}
{#if r.spread && r.spread.pct > 0}<span {#if r.spread && r.spread.pct > 0}<span
class="spread" class="spread"

View File

@@ -1,6 +1,13 @@
import { db } from '#lib/server/db/index.js'; import { db } from '#lib/server/db/index.js';
import { requireOwner } from '#lib/server/owner.js'; import { requireOwner } from '#lib/server/owner.js';
import { fold } from '#lib/server/groceries/normalize.js'; import { fold } from '#lib/server/groceries/normalize.js';
import {
expand,
queryWords,
rankedIds,
searchConds,
searchContext
} from '#lib/server/groceries/search.js';
import { json } from '@sveltejs/kit'; import { json } from '@sveltejs/kit';
import type { Sql } from 'postgres'; import type { Sql } from 'postgres';
import type { RequestHandler } from './$types'; import type { RequestHandler } from './$types';
@@ -9,16 +16,20 @@ import type { RequestHandler } from './$types';
export const GET: RequestHandler = async (event) => { export const GET: RequestHandler = async (event) => {
requireOwner(event); requireOwner(event);
const q = fold(event.url.searchParams.get('q')?.trim() ?? '').slice(0, 100); const q = fold(event.url.searchParams.get('q')?.trim() ?? '').slice(0, 100);
const words = q.split(/\s+/).filter(Boolean).slice(0, 6);
if (!words.length) return json([]);
const s = db.$client as unknown as Sql; const s = db.$client as unknown as Sql;
const conds = words.map((w) => s`search_text like ${'%' + w.replace(/[%_\\]/g, '\\$&') + '%'}`); const words = expand(queryWords(q), (await searchContext(s)).synonyms);
const rows = await s` if (!words.length) return json([]);
const where = [s`merged_into_id is null`, ...searchConds(s, words)].reduce(
(a, c) => s`${a} and ${c}`
);
const ids = (await rankedIds(s, where, q, words)).slice(0, 15);
const rows = ids.length
? await s<{ id: number }[]>`
select id, name, brand, unit, store_count as "storeCount", item_count as "itemCount", select id, name, brand, unit, store_count as "storeCount", item_count as "itemCount",
image_url as "imageUrl", min_price_cents as "minPriceCents" image_url as "imageUrl", min_price_cents as "minPriceCents"
from groceries.groups from groceries.groups g where id = any(${ids}::int[])`
where merged_into_id is null and ${conds.reduce((a, c) => s`${a} and ${c}`)} : [];
order by public.word_similarity(${q}::text, lower(name)) desc, store_count desc, item_count desc, id const pos = new Map(ids.map((id, i) => [id, i]));
limit 15`; rows.sort((a, b) => pos.get(a.id)! - pos.get(b.id)!);
return json(rows, { headers: { 'cache-control': 'private, no-store' } }); return json(rows, { headers: { 'cache-control': 'private, no-store' } });
}; };