Stan Runge
60370d4572
All checks were successful
Deploy production / deploy (push) Successful in 1m13s
- search.ts: score on the core name (brand/size stripped): exact or query+base suffix (kipdij+filet), head-final compounds, prefix, inside; penalties per extra word and per flavour/processing word (search-words.ts, editable); bonuses for store brands, the query's dominant category, store count, favourites; trigram tiebreak - groceries.synonyms table (seeded: kipdij/kippendij, pasta → spaghetti, …) expands queries - list rows show all pack sizes of a group; group picker uses the same ranking - scripts/groceries-search-eval.ts prints top results + latency; preflight now 8 tables
275 lines
10 KiB
TypeScript
275 lines
10 KiB
TypeScript
// Relevance ranking for grocery search: plain/basic products first, flavoured and specialised
|
||
// ones further down. Pure scoring (tested in search.test.ts) plus a small DB layer that
|
||
// fetches candidates and the synonyms table.
|
||
//
|
||
// Score = text match on the core name (brand and size stripped)
|
||
// - complexity (extra words, flavour/processing words; lists in search-words.ts)
|
||
// + bonuses (store brand, the query's dominant category, sold in more stores, favourite)
|
||
// + trigram similarity as a tiebreak.
|
||
import type { PendingQuery, Row, Sql } from 'postgres';
|
||
import { fold, nameTokens, stem } from './normalize.js';
|
||
import { BASE_SUFFIXES, MODIFIERS, STORE_BRAND_PREMIUM, STORE_BRANDS } from './search-words.js';
|
||
|
||
export type Candidate = {
|
||
id: number;
|
||
name: string;
|
||
brand: string | null;
|
||
categoryId: number | null;
|
||
storeCount: number;
|
||
itemCount: number;
|
||
favorite: boolean;
|
||
promoCount?: number;
|
||
/** pg_trgm word similarity of the query to the name, 0..1 */
|
||
sim?: number;
|
||
};
|
||
export type Scored = Candidate & { score: number; text: number; penalty: number };
|
||
/** term → extra search terms (folded). One-directional; seed both ways for spelling variants. */
|
||
export type Synonyms = Map<string, string[]>;
|
||
|
||
const SUFFIXES = new Map(Object.entries(BASE_SUFFIXES).map(([w, v]) => [stem(fold(w)), v]));
|
||
const MODS = new Map(Object.entries(MODIFIERS).map(([w, v]) => [stem(fold(w)), v]));
|
||
const EXTRA_WORD = 10;
|
||
const MOD_WORD = 22;
|
||
const SYN_FACTOR = 0.92;
|
||
|
||
/** Query → folded words (max 6). */
|
||
export function queryWords(q: string): string[] {
|
||
return fold(q)
|
||
.replace(/['’]/g, '')
|
||
.split(/[^a-z0-9%+]+/)
|
||
.filter(Boolean)
|
||
.slice(0, 6);
|
||
}
|
||
|
||
/** Each query word with its synonyms: [[word, syn, …], …] (folded, deduped). */
|
||
export function expand(words: string[], synonyms: Synonyms): string[][] {
|
||
return words.map((w) => [...new Set([w, ...(synonyms.get(w) ?? [])])]);
|
||
}
|
||
|
||
/** Cost of a glued run of base suffixes ("filet" 0, "filetblokje" 5, "" 0); null if not one. */
|
||
function suffixCost(r: string, depth = 0): number | null {
|
||
if (!r) return 0;
|
||
if (depth > 3) return null;
|
||
let best: number | null = null;
|
||
for (const [s, v] of SUFFIXES)
|
||
if (r.startsWith(s)) {
|
||
const rest = suffixCost(r.slice(s.length), depth + 1);
|
||
if (rest != null && (best == null || v + rest < best)) best = v + rest;
|
||
}
|
||
return best;
|
||
}
|
||
|
||
/** Penalty for a leftover word or compound part (small or 0 for base suffixes). */
|
||
export function wordPenalty(w: string): number {
|
||
if (!w || w.length < 2 || /^\d+[%+]?$/.test(w)) return 0;
|
||
const s = stem(w);
|
||
const sc = suffixCost(s) ?? suffixCost(w);
|
||
if (sc != null) return sc;
|
||
const m = MODS.get(s) ?? MODS.get(w);
|
||
if (m != null) return m * MOD_WORD;
|
||
// Compounds containing a flavour/processing word (madrasfilet, chocoladereep, grillworst).
|
||
let m2 = 0;
|
||
for (const [k, v] of MODS) if (v >= 0.5 && k.length > 3 && s.includes(k)) m2 = Math.max(m2, v);
|
||
return m2 ? Math.max(m2 * MOD_WORD, EXTRA_WORD) : EXTRA_WORD;
|
||
}
|
||
|
||
type Hit = { level: number; token: number; leftover: string; cost: number };
|
||
|
||
/** Best match of one term against the core tokens (stemmed). */
|
||
function matchTerm(term: string, tokens: string[]): Hit | null {
|
||
let best: Hit | null = null;
|
||
const take = (h: Hit) => {
|
||
if (!best || h.level - h.cost > best.level - best.cost) best = h;
|
||
};
|
||
tokens.forEach((t, i) => {
|
||
for (const e of new Set([term, stem(term)])) {
|
||
if (e.length < 2) continue;
|
||
const at = t.indexOf(e);
|
||
if (at < 0) continue;
|
||
const after = t.slice(at + e.length);
|
||
const cost = suffixCost(after);
|
||
if (at === 0 && cost != null) take({ level: 100, token: i, leftover: '', cost });
|
||
else if (at > 0 && cost != null)
|
||
// Dutch compounds are head-final: zilvervliesrijst is rijst, scharrelkipdijfilet is kipdij.
|
||
take({ level: 80, token: i, leftover: t.slice(0, at), cost });
|
||
else if (at === 0)
|
||
// kipdijspiesjes: starts with the query but is something else.
|
||
take({ level: 60, token: i, leftover: after, cost: 0 });
|
||
else take({ level: 40, token: i, leftover: t.slice(0, at) + ' ' + after, cost: 0 });
|
||
}
|
||
});
|
||
return best;
|
||
}
|
||
|
||
/** Text score (0..106: 100 exact or query+suffix, 80 compound end, 60 prefix, 40 inside, +6 first word) and complexity penalty. */
|
||
export function textScore(c: Pick<Candidate, 'name' | 'brand'>, words: string[][]) {
|
||
const tokens = nameTokens(c.name, c.brand);
|
||
const brandToks = new Set(
|
||
fold(c.brand ?? '')
|
||
.split(/[^a-z0-9]+/)
|
||
.filter(Boolean)
|
||
);
|
||
const used = new Set<number>();
|
||
let total = 0;
|
||
let penalty = 0;
|
||
let first = false;
|
||
for (const alts of words) {
|
||
let best: (Hit & { score: number }) | null = null;
|
||
alts.forEach((term, k) => {
|
||
const h = matchTerm(term, tokens);
|
||
if (!h) return;
|
||
const score = h.level * (k === 0 ? 1 : SYN_FACTOR);
|
||
if (!best || score - h.cost > best.score - best.cost) best = { ...h, score };
|
||
});
|
||
if (best) {
|
||
const b = best as Hit & { score: number };
|
||
total += b.score;
|
||
used.add(b.token);
|
||
if (b.token === 0) first = true;
|
||
penalty += b.cost;
|
||
for (const part of b.leftover.split(' ')) penalty += wordPenalty(part);
|
||
} else if (alts.some((a) => brandToks.has(a))) total += 70;
|
||
}
|
||
tokens.forEach((t, i) => {
|
||
if (!used.has(i)) penalty += wordPenalty(t);
|
||
});
|
||
const text = words.length ? total / words.length : 0;
|
||
return { text: text + (first && text > 0 ? 6 : 0), penalty: Math.min(penalty, 70) };
|
||
}
|
||
|
||
/** Store/private-label bonus from the group's brand label. */
|
||
export function brandBonus(brand: string | null): number {
|
||
const b = fold(brand ?? '').trim();
|
||
if (!b) return 12; // loose produce
|
||
if (STORE_BRAND_PREMIUM.includes(b)) return 6;
|
||
if (b === 'huismerk' || b === 'huismerk basic') return 15;
|
||
if (b.startsWith('huismerk')) return 10;
|
||
if (STORE_BRANDS.some((p) => b === p || b.startsWith(p + ' '))) return 12;
|
||
return 0;
|
||
}
|
||
|
||
export type Ctx = {
|
||
words: string[][];
|
||
/** category id → its top-level id */
|
||
topOf: (id: number | null) => number | null;
|
||
};
|
||
|
||
/** Score and sort candidates (best first). */
|
||
export function rank(cands: Candidate[], ctx: Ctx, opts: { promoFirst?: boolean } = {}): Scored[] {
|
||
const pre = cands.map((c) => ({ c, ...textScore(c, ctx.words) }));
|
||
// Dominant category: the most common one among strong name matches; plain products (few
|
||
// extra words) and products sold in more stores decide what the query is about.
|
||
const dominant = (min: number) => {
|
||
const w = new Map<number, number>();
|
||
for (const p of pre)
|
||
if (p.text >= min && p.c.categoryId != null)
|
||
w.set(
|
||
p.c.categoryId,
|
||
(w.get(p.c.categoryId) ?? 0) + (1 + p.c.storeCount) / (1 + p.penalty / 5) ** 2
|
||
);
|
||
return [...w].sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
|
||
};
|
||
const dom = dominant(90) ?? dominant(75) ?? dominant(55);
|
||
const domTop = ctx.topOf(dom);
|
||
const out = pre.map(({ c, text, penalty }) => {
|
||
let score = text - penalty;
|
||
if (text === 0) score = (c.sim ?? 0) * 30 - 40; // matched only via brand/category/other names
|
||
score += brandBonus(c.brand);
|
||
if (dom != null) {
|
||
if (c.categoryId === dom) score += 12;
|
||
else if (domTop != null && ctx.topOf(c.categoryId) === domTop) score += 5;
|
||
else score -= 10;
|
||
}
|
||
score += Math.min(Math.max(c.storeCount - 1, 0), 5) * 4;
|
||
score += Math.min(Math.log2(1 + c.itemCount), 4);
|
||
if (c.favorite) score += 10;
|
||
score += (c.sim ?? 0) * 5;
|
||
return {
|
||
...c,
|
||
score: Math.round(score * 100) / 100,
|
||
text,
|
||
penalty: Math.round(penalty * 10) / 10
|
||
};
|
||
});
|
||
return out.sort(
|
||
(a, b) =>
|
||
(opts.promoFirst ? Number((b.promoCount ?? 0) > 0) - Number((a.promoCount ?? 0) > 0) : 0) ||
|
||
b.score - a.score ||
|
||
b.storeCount - a.storeCount ||
|
||
b.itemCount - a.itemCount ||
|
||
a.name.length - b.name.length ||
|
||
a.id - b.id
|
||
);
|
||
}
|
||
|
||
// ---- DB --------------------------------------------------------------------------------------
|
||
|
||
type SearchCtx = { at: number; synonyms: Synonyms; parent: Map<number, number | null> };
|
||
let cache: SearchCtx | null = null;
|
||
const TTL = 5 * 60_000;
|
||
|
||
/** Synonyms table + category parents, cached for 5 minutes. */
|
||
export async function searchContext(s: Sql): Promise<SearchCtx> {
|
||
if (cache && Date.now() - cache.at < TTL) return cache;
|
||
const [syn, cats] = await Promise.all([
|
||
s<{ term: string; synonym: string }[]>`select term, synonym from groceries.synonyms`,
|
||
s<{ id: number; parent_id: number | null }[]>`select id, parent_id from groceries.categories`
|
||
]);
|
||
cache = {
|
||
at: Date.now(),
|
||
synonyms: synonymMap(syn),
|
||
parent: new Map(cats.map((c) => [c.id, c.parent_id]))
|
||
};
|
||
return cache;
|
||
}
|
||
|
||
export function synonymMap(rows: { term: string; synonym: string }[]): Synonyms {
|
||
const m: Synonyms = new Map();
|
||
for (const r of rows) {
|
||
const t = fold(r.term).trim();
|
||
m.set(t, [...(m.get(t) ?? []), fold(r.synonym).trim()]);
|
||
}
|
||
return m;
|
||
}
|
||
|
||
const like = (w: string) => '%' + w.replace(/[%_\\]/g, '\\$&') + '%';
|
||
|
||
/** WHERE fragments: every query word (or one of its synonyms) appears in g.search_text. */
|
||
export function searchConds(s: Sql, words: string[][]) {
|
||
return words.map((alts) =>
|
||
alts.length === 1
|
||
? s`g.search_text like ${like(alts[0])}`
|
||
: s`g.search_text like any(${alts.map(like)}::text[])`
|
||
);
|
||
}
|
||
|
||
const MAX_CANDIDATES = 4000;
|
||
|
||
/**
|
||
* Rank the groups matching `where` (alias g) for query `q`, best first. Up to MAX_CANDIDATES;
|
||
* groups whose name contains a query word are fetched first so broad queries keep them.
|
||
*/
|
||
export async function rankedGroups(
|
||
s: Sql,
|
||
where: PendingQuery<Row[]>,
|
||
q: string,
|
||
words: string[][],
|
||
opts: { promoFirst?: boolean; ctx?: SearchCtx } = {}
|
||
): Promise<Scored[]> {
|
||
const ctx = opts.ctx ?? (await searchContext(s));
|
||
const nameLike = words.flat().map(like);
|
||
const cands = await s<Candidate[]>`
|
||
select g.id, g.name, g.brand, g.category_id as "categoryId", g.store_count as "storeCount",
|
||
g.item_count as "itemCount", g.favorite, g.promo_count as "promoCount",
|
||
public.word_similarity(${q}::text, lower(g.name))::float8 as sim
|
||
from groceries.groups g where ${where}
|
||
order by (lower(g.name) like any(${nameLike}::text[])) desc, g.store_count desc, g.item_count desc
|
||
limit ${MAX_CANDIDATES}`;
|
||
const topOf = (id: number | null) => (id == null ? null : (ctx.parent.get(id) ?? id));
|
||
return rank(cands, { words, topOf }, opts);
|
||
}
|
||
|
||
/** rankedGroups(), ids only. */
|
||
export const rankedIds = async (...a: Parameters<typeof rankedGroups>) =>
|
||
(await rankedGroups(...a)).map((c) => c.id);
|