Stan Runge
a6ef263c7e
All checks were successful
Deploy production / deploy (push) Successful in 1m19s
1205 lines
46 KiB
TypeScript
1205 lines
46 KiB
TypeScript
// Analytics for /trackers/groceries/stats. computeStats() runs nightly after the import, regroup
|
||
// and taxonomy (scripts/groceries-import.ts, under the import lock) and stores one snapshot per
|
||
// scope in groceries.stats_snapshots; the page only reads snapshots. computeBattles() is the
|
||
// same battle maths for a store subset, used live when the page filters stores.
|
||
//
|
||
// Rules (shared by every section):
|
||
// - an "offer" is an available store product with a price, a unit (kg / l / piece) and a unit
|
||
// price inside sane bounds; 'food' scope keeps edible items only.
|
||
// - outlier guard: offers more than 8× above or below their group's median unit price are
|
||
// dropped (almost always a wrongly parsed pack size).
|
||
// - a battle is a group (comparable product, any pack size) sold in 2+ stores, compared on each
|
||
// store's cheapest unit price. Ties split the win (1/k each), so shared Superunie prices
|
||
// (Dirk, Vomar, Hoogvliet) don't all count as full wins.
|
||
import type { Sql, TransactionSql } from 'postgres';
|
||
import type {
|
||
BasketPick,
|
||
BattleRow,
|
||
BrandGapRow,
|
||
CategoryRef,
|
||
GapRow,
|
||
H2HRow,
|
||
InflationRow,
|
||
ItemRef,
|
||
MoveRow,
|
||
Overview,
|
||
PpkgRow,
|
||
PromoRow,
|
||
RangeRow,
|
||
Section,
|
||
SeriesData,
|
||
ShrinkRow,
|
||
SnapshotData,
|
||
Staple,
|
||
StatsScope,
|
||
StoreRef,
|
||
VolRow
|
||
} from '../../groceries-stats';
|
||
|
||
type Log = (m: string) => void;
|
||
type Tx = TransactionSql;
|
||
const ALL = 'all';
|
||
// Leaves under food tops that are not edible (the taxonomy flags them food via their top).
|
||
const NON_FOOD_LEAVES = ['baby-kind--luiers', 'baby-kind--verzorging', 'baby-kind--accessoires'];
|
||
|
||
// Own (private-label) brands per store, as a regex on lower(brand). Superunie labels are shared
|
||
// by its members (Dirk, Hoogvliet, Vomar, PLUS). `unbranded`: items without a brand are the
|
||
// store's own (loose produce, bakery) rather than unknown. Stores not listed (Xenos, Action,
|
||
// Amazing Oriental) mostly have no brand data, so their own-brand share is unknown.
|
||
const SUPERUNIE = '1 de beste|g.woon|bio\\+|daily chef|melkan|bonbebe|smaakt|kanis';
|
||
const ALDI =
|
||
'milsani|daylicious|trader joe|all seasons|mucci|choceur|gourmet|rio d.oro|barissimo|golden seafood|river|lyttos|cucina nobile|moser roth|park street deli|nature.s|westminster|sweet valley|lacura|tandil|bramwells|deluxe|mamia|happy harvest|biocentury|simply v|asia green garden|ombra|my vay|enerbio|cool|lomee|snack day|gut bio|wonnemeyer|knusperone|cantinella|almare|cafe d.or|bellasan|ofterdinger|nur nur natur|ja!';
|
||
export const OWN_BRANDS: { slug: string; rx: string; unbranded: boolean }[] = [
|
||
{ slug: 'ah', rx: '^(ah|albert heijn)($|\\s)', unbranded: true },
|
||
{ slug: 'jumbo', rx: '^jumbo', unbranded: true },
|
||
{ slug: 'plus', rx: `(^|\\s)plus($|\\s)|^(${SUPERUNIE})`, unbranded: true },
|
||
{ slug: 'dirk', rx: `^(dirk|vleeschmeesters|bits & bites|${SUPERUNIE})`, unbranded: true },
|
||
{ slug: 'hoogvliet', rx: `^(hoogvliet|${SUPERUNIE})`, unbranded: true },
|
||
{ slug: 'vomar', rx: `^(vomar|${SUPERUNIE})`, unbranded: true },
|
||
{ slug: 'aldi', rx: `^(aldi|${ALDI})`, unbranded: true },
|
||
{
|
||
slug: 'lidl',
|
||
rx: '^(lidl|milbona|pilos|vitasia|chef select|deluxe|solevita|freeway)',
|
||
unbranded: true
|
||
}
|
||
];
|
||
// A group counts as own brand when its display brand is a house class or any store's own label.
|
||
const GROUP_HOUSE_RX = `^(huismerk|ah|albert heijn|jumbo|plus|dirk|hoogvliet|vomar|aldi|${SUPERUNIE}|${ALDI})`;
|
||
|
||
/** Staples basket ("Gouda shopping route"). rx/nrx match lower(name); qty in kg, l or pieces. */
|
||
const STAPLES: (Staple & {
|
||
rx: string;
|
||
nrx: string | null;
|
||
units: string;
|
||
cats: string | null;
|
||
min: number;
|
||
max: number;
|
||
pieceKg?: number;
|
||
})[] = [
|
||
{
|
||
key: 'melk',
|
||
label: 'Halfvolle melk',
|
||
qty: 1,
|
||
unit: 'l',
|
||
icon: 'milk',
|
||
units: 'l',
|
||
rx: 'halfvolle (verse |houdbare )?melk',
|
||
nrx: 'lactose|choco|koffie|poeder|drink|yoghurt|vla|kwark|karne|geit|vitamine|pakjes|mini|kids|proteine|eiwit',
|
||
cats: 'zuivel-eieren',
|
||
min: 0.5,
|
||
max: 2
|
||
},
|
||
{
|
||
key: 'eieren',
|
||
label: 'Eieren',
|
||
qty: 10,
|
||
unit: 'piece',
|
||
icon: 'egg',
|
||
units: 'piece',
|
||
rx: 'eieren\\M',
|
||
nrx: 'pack|choco|paas|salade|gekookt|spiegel|omelet|kwartel|noedel|mie\\M|pasta|vervanger|plantaardig|poeder|wafel|koek|spread|smeer|vegan|salad|sandwich',
|
||
cats: 'zuivel-eieren',
|
||
min: 0.6,
|
||
max: 3
|
||
},
|
||
{
|
||
key: 'brood',
|
||
label: 'Volkorenbrood',
|
||
qty: 0.8,
|
||
unit: 'kg',
|
||
icon: 'croissant',
|
||
units: 'kg',
|
||
rx: '\\mvolkoren',
|
||
nrx: 'koopmans|bakmix|pita|bagel|tortilla|bake|beschuit|cracker|toast|rozijn|mini|pistolet|bol\\M|bolletje|wrap|krenten|stokbrood|meel|mix|kruimel|paneer|vlok|glutenvrij|afbak|diepvries|sandwich|rogge|spelt|tijger|zaden|pitten|haver|lijnzaad|eiwit|keto|knack|half',
|
||
cats: 'brood-bakkerij',
|
||
min: 0.6,
|
||
max: 1.0,
|
||
pieceKg: 0.8
|
||
},
|
||
{
|
||
key: 'rijst',
|
||
label: 'Rijst',
|
||
qty: 1,
|
||
unit: 'kg',
|
||
icon: 'cooking-pot',
|
||
units: 'kg',
|
||
rx: '\\mrijst\\M',
|
||
nrx: 'wafel|melk|pudding|noedel|mie\\M|cracker|meel|azijn|drink|maaltijd|kip|salade|pap|snack|chips|vel|papier|wijn|sushi|risotto|paella|nasi|gerecht|mix|kant|magnetron|pouch|zakje|kookzak|builtje|cups|express|snel|bladeren|vermicelli|bloem|crisps|ontbijt|baby|olie|koek|cake|bal|stick|wilde|rood|zwart|kleef|glutinous|bruin|zilvervlies|risotto|arborio|sake|mochi|kroepoek|krupuk',
|
||
cats: 'pasta-rijst-wereld',
|
||
min: 0.4,
|
||
max: 5
|
||
},
|
||
{
|
||
key: 'pasta',
|
||
label: 'Spaghetti',
|
||
qty: 0.5,
|
||
unit: 'kg',
|
||
icon: 'cooking-pot',
|
||
units: 'kg',
|
||
rx: 'spaghetti',
|
||
nrx: 'saus|maaltijd|kant|vers|glutenvrij|bolognese|carbonara|pot|squash|courgette|groente|kids|soep|ijs|snack|blik|eiwit|linzen|kikkererwt|konjac|volkoren|spelt|bio|lijn|tomaten|kruiden|mix|hollandse|verse',
|
||
cats: 'pasta-rijst-wereld',
|
||
min: 0.4,
|
||
max: 2
|
||
},
|
||
{
|
||
key: 'kipdij',
|
||
label: 'Kipdijfilet',
|
||
qty: 1,
|
||
unit: 'kg',
|
||
icon: 'beef',
|
||
units: 'kg',
|
||
rx: 'dijfilet|dij filet|dijfilets',
|
||
nrx: 'gekruid|gemarineerd|madras|shoarma|doner|döner|spies|sat[eé]|gegrild|tandoori|honing|pittig|ketjap|ketjab|bbq|piri|kebab|gyros|teriyaki|gerookt|krokant|burger|worst|paneer|marinade|knoflook|kruiden|tikka|gestoofd|stoof|pulled|curry|salade|wrap|soep|maaltijd|oven|airfryer|smaak|zoet|grill|chili|mexica|thai|oriental|hot|roast|yakitori|katsu|tacos?|fajita|tex|crispy|nugget|schnitzel|cordon|gehakt|rollade|vega|vegan|plantaardig',
|
||
cats: 'vlees-vis-vega',
|
||
min: 0.25,
|
||
max: 3
|
||
},
|
||
{
|
||
key: 'kaas',
|
||
label: 'Jong belegen kaas',
|
||
qty: 0.5,
|
||
unit: 'kg',
|
||
icon: 'ham',
|
||
units: 'kg',
|
||
rx: 'jong belegen',
|
||
nrx: 'smeer|light|20\\+|30\\+|35\\+|geit|lactosevrij|stengel|tosti|snack|sandwich|kruiden|komijn|plantaardig|vegan|mini|hapjes|blokjes|kids|minder|zout|rasp|gerasp',
|
||
cats: 'kaas-vleeswaren',
|
||
min: 0.15,
|
||
max: 2
|
||
},
|
||
{
|
||
key: 'bananen',
|
||
label: 'Bananen',
|
||
qty: 1,
|
||
unit: 'kg',
|
||
icon: 'apple',
|
||
units: 'kg',
|
||
rx: '\\mbananen\\M',
|
||
nrx: 'chips|gedroogd|mini|baby|smoothie|yoghurt|brood|snoep|spekjes|pap|drink|knijp|cake|muffin|ijs|schuim|melk|mix|chocola',
|
||
cats: 'groente-fruit',
|
||
min: 0.4,
|
||
max: 3,
|
||
pieceKg: 0.18
|
||
},
|
||
{
|
||
key: 'uien',
|
||
label: 'Uien',
|
||
qty: 1,
|
||
unit: 'kg',
|
||
icon: 'carrot',
|
||
units: 'kg',
|
||
rx: '\\m(gele |rode )?uien\\M',
|
||
nrx: 'gebakken|soep|ringen|poeder|zilver|sjalot|lente|chips|gedroogd|saus|cr[eè]me|gesneden|gesnipperd|snipper|diepvries|ingelegd|chutney|confit|knoflook|mix|pakket|kruiden|stoof|rode',
|
||
cats: 'groente-fruit',
|
||
min: 0.4,
|
||
max: 5
|
||
},
|
||
{
|
||
key: 'tofu',
|
||
label: 'Tofu',
|
||
qty: 0.4,
|
||
unit: 'kg',
|
||
icon: 'salad',
|
||
units: 'kg',
|
||
rx: '\\mtofu\\M',
|
||
nrx: 'gemarineerd|gerookt|gekruid|saus|zijde|silken|puff|noedel|soep|maaltijd|gebakken|gefrituurd|kimchi|curry|teriyaki|spread|burger|smaak|pittig|basilicum|kruiden|rook|tamari|stukjes|blokjes|knoflook|sesam|gember|chili|bbq|crispy|vegan worst|dessert|pudding|snack',
|
||
cats: null,
|
||
min: 0.25,
|
||
max: 2
|
||
},
|
||
{
|
||
key: 'pindakaas',
|
||
label: 'Pindakaas',
|
||
qty: 0.35,
|
||
unit: 'kg',
|
||
icon: 'egg',
|
||
units: 'kg',
|
||
rx: '\\mpindakaas\\M',
|
||
nrx: 'reep|koek|saus|ijs|choco|snack|cup|bar\\M|repen|poeder|vulling|bonbon|cookie|muesli|granola|hond|kat|vogel|bites|wafel|crunch bar|pop|knijp',
|
||
cats: 'ontbijt-beleg',
|
||
min: 0.2,
|
||
max: 2
|
||
},
|
||
{
|
||
key: 'roomboter',
|
||
label: 'Roomboter',
|
||
qty: 0.25,
|
||
unit: 'kg',
|
||
icon: 'milk',
|
||
units: 'kg',
|
||
rx: '\\mroomboter\\M',
|
||
nrx: 'koek|croissant|cake|bladerdeeg|kruiden|knoflook|cookie|spritz|sprits|saus|bonbon|kruimel|wafel|amandel|appel|taart|brioche|gebak|stroop|bitter|karamel|ijs|toffee|frites|mini|kuipje|cups|cupjes|bakje|halfvolle|light|smeerbaar|plant|bladerdeeg|deeg|spek|speculaas|hopjes|tompouce|bol|broodje|kerst|letter|stol|zout(je|jes)',
|
||
cats: 'zuivel-eieren',
|
||
min: 0.2,
|
||
max: 1
|
||
},
|
||
{
|
||
key: 'appels',
|
||
label: 'Appels',
|
||
qty: 1,
|
||
unit: 'kg',
|
||
icon: 'apple',
|
||
units: 'kg',
|
||
rx: '\\mappels\\M|\\mhandappels\\M|\\melstar\\M|\\mjonagold\\M|\\mgoudrenet\\M',
|
||
nrx: 'moes|sap|chips|taart|flap|gedroogd|stroop|azijn|siroop|beignet|crumble|kaneel|knijp|parten|partjes|schijf|ringen|baby|snack|compote|wafel|ijs|pap|yoghurt|smoothie|cider|drink|cake|koek|gebak|muffin|reep|mix|bites',
|
||
cats: 'groente-fruit',
|
||
min: 0.4,
|
||
max: 3
|
||
},
|
||
{
|
||
key: 'aardappelen',
|
||
label: 'Aardappelen',
|
||
qty: 2,
|
||
unit: 'kg',
|
||
icon: 'carrot',
|
||
units: 'kg',
|
||
rx: '\\maardappel(en|s)?\\M|kruimige|vastkokende',
|
||
nrx: 'puree|chips|frites|kroket|schijf|partjes|rosti|rösti|gratin|wedges|salade|zetmeel|meel|soep|gebakken|kriel|zoete|parten|blokjes|kruiden|spek|ovenschotel|stamppot|nest|poffers|gesneden|geschild|voorgekookt|roosjes|duchesse|baby|smash|mix|krokant|schotel|pan|maaltijd|truffel|knoflook|room|kaas|ui\\M|dauphine|hash|pommes|crisp|garnituur|aardappelpuree',
|
||
cats: 'groente-fruit',
|
||
min: 0.5,
|
||
max: 5
|
||
},
|
||
{
|
||
key: 'yoghurt',
|
||
label: 'Volle yoghurt',
|
||
qty: 1,
|
||
unit: 'l',
|
||
icon: 'milk',
|
||
units: 'l,kg',
|
||
rx: '\\mvolle yoghurt',
|
||
nrx: 'drink|fruit|aardbei|vanille|perzik|honing|kers|mango|griek|bosvrucht|framboos|limoen|citroen|stracciatella|muesli|granola|cups|kids|mini|knijp|abrikoos|ananas|kokos|bramen|bessen|appel|banaan|rabarber|passie|mix|lactosevrij|proteine|eiwit|skyr|vla|kwark|0%|mager',
|
||
cats: 'zuivel-eieren',
|
||
min: 0.4,
|
||
max: 2
|
||
},
|
||
{
|
||
key: 'koffie',
|
||
label: 'Filterkoffie',
|
||
qty: 0.5,
|
||
unit: 'kg',
|
||
icon: 'coffee',
|
||
units: 'kg',
|
||
rx: 'snelfilter|filterkoffie|filtermaling|filter maling',
|
||
nrx: 'pads|cups|capsules|bonen|zakjes|apparaat|papier|filters\\M|oploskoffie|instant|decaf|cafe[iï]nevrij|caf[eé]ine vrij|machine|zakje|houder|kan\\M|kannen|dosis|bags',
|
||
cats: 'koffie-thee',
|
||
min: 0.2,
|
||
max: 1.5
|
||
},
|
||
{
|
||
key: 'suiker',
|
||
label: 'Kristalsuiker',
|
||
qty: 1,
|
||
unit: 'kg',
|
||
icon: 'candy',
|
||
units: 'kg',
|
||
rx: 'kristalsuiker|\\mwitte suiker\\M|fijne kristal|\\msuiker\\M',
|
||
nrx: 'palm|gula|java|vrij|riet|klont|poeder|parel|stroop|wafel|brood|vanille|kaneel|spin|snoep|basterd|kandij|gelei|ahorn|kokos|zoetstof|stevia|stick|zakje|cake|koek|lolly|spek|hagel|ijs|siroop|candy|donut|glazuur|chocola|sinterklaas|fondant|kleur|decor|confetti|vermicelli|minder|ontbijt|muesli|granola|jam|drop|kauwgom|pinda|noten|amandel|thee|limonade|cola|frisdrank|sap|druiven|dadel|gember|geleisuiker|gelei',
|
||
cats: null,
|
||
min: 0.5,
|
||
max: 3
|
||
}
|
||
];
|
||
|
||
const num = (v: unknown): number => (v == null ? 0 : Number(v));
|
||
const numOrNull = (v: unknown): number | null => (v == null ? null : Number(v));
|
||
const r4 = (v: unknown): number => Math.round(Number(v) * 10000) / 10000;
|
||
const r4n = (v: unknown): number | null => (v == null ? null : r4(v));
|
||
|
||
/** Prepares temp tables o (offers), sp (best price per group per store) and b (battles). */
|
||
async function prepare(tx: Tx, scope: StatsScope, stores: number[] | null) {
|
||
const storeArr = stores ? `{${stores.map((s) => Math.trunc(s)).join(',')}}` : null;
|
||
await tx`
|
||
create temp table o on commit drop as
|
||
select i.id item_id, i.store_id, coalesce(g.merged_into_id, i.group_id) gid,
|
||
g.family_key, g.brand gbrand, i.name, i.brand, i.size_text, i.unit,
|
||
i.unit_amount::float8 ua, i.price_cents p, i.promo_price_cents pp, i.promo_label pl,
|
||
i.unit_price_cents up,
|
||
least(i.unit_price_cents, coalesce(i.promo_unit_price_cents, i.unit_price_cents)) eup,
|
||
coalesce(tc.slug, 'overig') top_slug, i.category_id leaf_id, i.is_food, i.gtin, i.url,
|
||
case
|
||
when ob.slug is null then null
|
||
when coalesce(btrim(i.brand), '') = '' then case when ob.unbranded then true end
|
||
else lower(i.brand) ~ ob.rx
|
||
end is_house
|
||
from groceries.items i
|
||
join groceries.stores s on s.id = i.store_id and s.archived_at is null
|
||
left join groceries.groups g on g.id = i.group_id
|
||
left join groceries.categories c on c.id = i.category_id
|
||
left join groceries.categories tc on tc.id = coalesce(c.parent_id, c.id)
|
||
left join json_to_recordset(${tx.json(OWN_BRANDS as never)}::json)
|
||
as ob(slug text, rx text, unbranded boolean) on ob.slug = s.slug
|
||
where i.available and i.price_cents > 0 and i.unit_price_cents > 0 and i.unit is not null
|
||
and (${scope} = 'all' or (i.is_food and coalesce(c.slug, '') <> all(${NON_FOOD_LEAVES})
|
||
and i.name !~* 'e-?gift|cadeaukaart|giftcard|gift card'))
|
||
and (${storeArr}::int[] is null or i.store_id = any(${storeArr}::int[]))
|
||
and i.unit_price_cents >= case i.unit when 'piece' then 1 else 10 end
|
||
and i.unit_price_cents <= case i.unit when 'kg' then 1500000 else 500000 end`;
|
||
await tx`create index on o (gid)`;
|
||
await tx`
|
||
with m as (
|
||
select gid, percentile_cont(0.5) within group (order by up) med
|
||
from o where gid is not null group by gid
|
||
)
|
||
delete from o using m where o.gid = m.gid and (o.up > m.med * 8 or o.up < m.med / 8)`;
|
||
// Category guard: 400× the leaf median per unit is a parse error (e.g. "3.330 g" read as
|
||
// 3.33 g), while saffron at ~€10k/kg still passes against the spices median.
|
||
await tx`
|
||
with m as (
|
||
select leaf_id, unit, percentile_cont(0.5) within group (order by up) med, count(*) n
|
||
from o where leaf_id is not null group by 1, 2
|
||
)
|
||
delete from o using m where o.leaf_id = m.leaf_id and o.unit = m.unit and m.n >= 20
|
||
and (o.up > m.med * 400 or o.up < m.med / 100)`;
|
||
await tx`
|
||
create temp table sp on commit drop as
|
||
select gid, store_id, min(up) up, min(eup) eup, mode() within group (order by top_slug) top_slug
|
||
from o where gid is not null group by gid, store_id`;
|
||
await tx`create index on sp (gid)`;
|
||
await tx`
|
||
create temp table b on commit drop as
|
||
with gc as (select gid, mode() within group (order by top_slug) top_slug from sp group by gid)
|
||
select sp.gid, sp.store_id, sp.up, sp.eup, gc.top_slug,
|
||
min(sp.up) over w mn, min(sp.eup) over w emn, count(*) over w n
|
||
from sp join gc using (gid) window w as (partition by sp.gid)`;
|
||
await tx`delete from b where n < 2`;
|
||
await tx`
|
||
alter table b add column nw int, add column enw int`;
|
||
await tx`
|
||
update b set nw = x.nw, enw = x.enw from (
|
||
select gid, count(*) filter (where up = mn) nw, count(*) filter (where eup = emn) enw
|
||
from b group by gid
|
||
) x where x.gid = b.gid`;
|
||
}
|
||
|
||
type ByCat<T> = Map<string, T[]>;
|
||
function byCat<T>(rows: (T & { cat: string })[]): ByCat<T> {
|
||
const m: ByCat<T> = new Map();
|
||
for (const r of rows) {
|
||
const { cat, ...rest } = r;
|
||
if (!m.has(cat)) m.set(cat, []);
|
||
m.get(cat)!.push(rest as T);
|
||
}
|
||
return m;
|
||
}
|
||
|
||
async function battles(tx: Tx) {
|
||
const rows = await tx`
|
||
select k.cat, b.store_id s, count(*)::int n,
|
||
sum(case when up = mn then 1.0 / nw else 0 end) w,
|
||
count(*) filter (where up = mn and nw = 1)::int o,
|
||
count(*) filter (where up = mn and nw > 1)::int t,
|
||
percentile_cont(0.5) within group (order by up::float8 / mn) ix,
|
||
exp(avg(ln(up::float8 / mn))) gx,
|
||
sum(case when eup = emn then 1.0 / enw else 0 end) wp,
|
||
percentile_cont(0.5) within group (order by eup::float8 / emn) ixp,
|
||
exp(avg(ln(eup::float8 / emn))) gxp
|
||
from b cross join lateral (values (${ALL}), (b.top_slug)) k(cat)
|
||
group by k.cat, b.store_id`;
|
||
const totals = await tx`
|
||
select k.cat, count(distinct gid)::int battles,
|
||
count(distinct gid) filter (where nw > 1)::int tied
|
||
from b cross join lateral (values (${ALL}), (b.top_slug)) k(cat) group by k.cat`;
|
||
const h2h = await tx`
|
||
select k.cat, x.store_id a, y.store_id b, count(*)::int n,
|
||
count(*) filter (where x.up < y.up)::int aw, count(*) filter (where x.up > y.up)::int bw,
|
||
count(*) filter (where x.up = y.up)::int t,
|
||
percentile_cont(0.5) within group (order by x.up::float8 / y.up - 1) md
|
||
from b x join b y on y.gid = x.gid and y.store_id <> x.store_id
|
||
cross join lateral (values (${ALL}), (x.top_slug)) k(cat)
|
||
group by k.cat, x.store_id, y.store_id`;
|
||
const bRows = byCat<BattleRow>(
|
||
rows.map((r) => ({
|
||
cat: r.cat,
|
||
s: r.s,
|
||
n: r.n,
|
||
w: r4(r.w),
|
||
o: r.o,
|
||
t: r.t,
|
||
wr: r4(num(r.w) / r.n),
|
||
ix: Math.round(num(r.ix) * 1000) / 10,
|
||
gx: Math.round(num(r.gx) * 1000) / 10,
|
||
wp: r4(r.wp),
|
||
wrp: r4(num(r.wp) / r.n),
|
||
ixp: Math.round(num(r.ixp) * 1000) / 10,
|
||
gxp: Math.round(num(r.gxp) * 1000) / 10
|
||
}))
|
||
);
|
||
const hRows = byCat<H2HRow>(
|
||
h2h.map((r) => ({
|
||
cat: r.cat,
|
||
a: r.a,
|
||
b: r.b,
|
||
n: r.n,
|
||
aw: r.aw,
|
||
bw: r.bw,
|
||
t: r.t,
|
||
md: r4(r.md)
|
||
}))
|
||
);
|
||
const tot = new Map(totals.map((t) => [t.cat as string, { battles: t.battles, tied: t.tied }]));
|
||
return { bRows, hRows, tot };
|
||
}
|
||
|
||
const PROMO_TYPE = `
|
||
case
|
||
when pl is null then 'Prijsverlaging'
|
||
when pl ~* 'volume\\s*voordeel' then 'Volume voordeel'
|
||
when pl ~* 'kaartkorting' then 'Kaartkorting'
|
||
when pl ~* '^\\s*1\\s*\\+\\s*1|2e\\s*gratis' then '1+1 gratis'
|
||
when pl ~* '\\d\\s*\\+\\s*\\d' then 'X+Y gratis'
|
||
when pl ~* '2e\\s*halve' then '2e halve prijs'
|
||
when pl ~* '\\d+\\s*voor\\s*€?\\s*\\d' then 'X voor €'
|
||
when pl ~* '%' then '% korting'
|
||
when pl ~* 'bonus' then 'Bonus'
|
||
when pl ~* 'actie' then 'Actie'
|
||
else 'Overig'
|
||
end`;
|
||
|
||
async function promos(tx: Tx) {
|
||
// Discount: explicit promo price, else implied by the promo type (1+1 = 50 %, …).
|
||
await tx.unsafe(`
|
||
create temp table pr on commit drop as
|
||
with x as (
|
||
select o.*, ${PROMO_TYPE} ptype,
|
||
regexp_match(pl, '(\\d+)\\s*\\+\\s*(\\d+)') xy,
|
||
regexp_match(pl, '(\\d+)\\s*%') pct,
|
||
regexp_match(pl, '(\\d+)\\s*voor\\s*€?\\s*(\\d+)[.,]?(\\d{0,2})') xv
|
||
from o where pl is not null or pp is not null
|
||
)
|
||
select x.*, least(0.9, greatest(0, coalesce(
|
||
case when pp is not null and pp < p then 1 - pp::float8 / p end,
|
||
case ptype
|
||
when '1+1 gratis' then 0.5
|
||
when '2e halve prijs' then 0.25
|
||
when 'X+Y gratis' then xy[2]::float8 / nullif(xy[1]::int + xy[2]::int, 0)
|
||
when '% korting' then pct[1]::float8 / 100
|
||
when 'Kaartkorting' then pct[1]::float8 / 100
|
||
when 'X voor €' then 1 - ((xv[2] || '.' || coalesce(nullif(xv[3], ''), '0'))::float8 * 100
|
||
/ nullif(xv[1]::int, 0)) / p
|
||
end))) disc
|
||
from x`);
|
||
const per = await tx`
|
||
with base as (
|
||
select k.cat, o.store_id, count(*)::int items from o
|
||
cross join lateral (values (${ALL}), (o.top_slug)) k(cat) group by 1, 2
|
||
),
|
||
p as (
|
||
select k.cat, pr.store_id,
|
||
count(*) filter (where ptype <> 'Volume voordeel')::int promo,
|
||
count(*) filter (where ptype = 'Volume voordeel')::int volume,
|
||
avg(nullif(disc, 0)) filter (where ptype <> 'Volume voordeel') disc
|
||
from pr cross join lateral (values (${ALL}), (pr.top_slug)) k(cat) group by 1, 2
|
||
),
|
||
c as (
|
||
select k.cat, pr.store_id, count(*)::int cmp,
|
||
count(*) filter (where pr.up * (1 - coalesce(pr.disc, 0)) < x.other)::int beats,
|
||
percentile_cont(0.5) within group (order by pr.up * (1 - coalesce(pr.disc, 0)) / x.other) save
|
||
from pr
|
||
join lateral (
|
||
select min(sp.up)::float8 other from sp where sp.gid = pr.gid and sp.store_id <> pr.store_id
|
||
) x on x.other is not null
|
||
cross join lateral (values (${ALL}), (pr.top_slug)) k(cat)
|
||
where pr.ptype <> 'Volume voordeel' and pr.gid is not null
|
||
group by 1, 2
|
||
)
|
||
select base.cat, base.store_id s, base.items, coalesce(p.promo, 0) promo,
|
||
coalesce(p.volume, 0) volume, p.disc, coalesce(c.cmp, 0) cmp, coalesce(c.beats, 0) beats, c.save
|
||
from base left join p using (cat, store_id) left join c using (cat, store_id)`;
|
||
const types = await tx`
|
||
select k.cat, store_id s, ptype, count(*)::int n, avg(nullif(disc, 0)) disc
|
||
from pr cross join lateral (values (${ALL}), (pr.top_slug)) k(cat) group by 1, 2, 3`;
|
||
return {
|
||
pRows: byCat<PromoRow>(
|
||
per.map((r) => ({
|
||
cat: r.cat,
|
||
s: r.s,
|
||
items: r.items,
|
||
promo: r.promo,
|
||
share: r4(r.promo / r.items),
|
||
volume: r.volume,
|
||
disc: r4n(r.disc),
|
||
cmp: r.cmp,
|
||
beats: r.beats,
|
||
save: r4n(r.save)
|
||
}))
|
||
),
|
||
tRows: byCat<{ s: number; type: string; n: number; disc: number | null }>(
|
||
types.map((r) => ({ cat: r.cat, s: r.s, type: r.ptype, n: r.n, disc: r4n(r.disc) }))
|
||
)
|
||
};
|
||
}
|
||
|
||
async function range(tx: Tx) {
|
||
const rows = await tx`
|
||
with gs as (select gid, count(distinct store_id) n from o where gid is not null group by gid)
|
||
select k.cat, o.store_id s, count(*)::int items, count(distinct o.gid)::int groups,
|
||
count(distinct o.gid) filter (where gs.n = 1)::int excl,
|
||
count(*) filter (where o.is_house)::int house,
|
||
count(*) filter (where o.is_house is not null)::int known
|
||
from o left join gs using (gid)
|
||
cross join lateral (values (${ALL}), (o.top_slug)) k(cat)
|
||
group by 1, 2`;
|
||
const buckets = await tx`
|
||
with g as (
|
||
select gid, count(distinct store_id) n, mode() within group (order by top_slug) top_slug
|
||
from o where gid is not null group by gid
|
||
)
|
||
select k.cat, case when n >= 4 then '4+' else n::text end b, count(*)::int n
|
||
from g cross join lateral (values (${ALL}), (g.top_slug)) k(cat) group by 1, 2 order by 2`;
|
||
// Own brand vs A-brand for the same family (name without brand) and unit: within each store
|
||
// (item level), and across stores (s = 0, group level). ratio = A-brand / own brand.
|
||
const gap = await tx`
|
||
with st as (
|
||
select store_id, family_key, unit, mode() within group (order by top_slug) top_slug,
|
||
min(up) filter (where is_house) h,
|
||
percentile_cont(0.5) within group (order by up) filter (where is_house = false) a
|
||
from o where family_key is not null and is_house is not null
|
||
group by 1, 2, 3
|
||
),
|
||
gl as (
|
||
select gid, family_key, unit, min(up) up, mode() within group (order by top_slug) top_slug,
|
||
bool_or(coalesce(gbrand, '') ~* ${GROUP_HOUSE_RX} or coalesce(gbrand, '') = '') house
|
||
from o where family_key is not null and gid is not null group by 1, 2, 3
|
||
),
|
||
x as (
|
||
select 0 store_id, family_key, unit, mode() within group (order by top_slug) top_slug,
|
||
min(up) filter (where house) h,
|
||
percentile_cont(0.5) within group (order by up) filter (where not house) a
|
||
from gl group by 1, 2, 3
|
||
union all select * from st
|
||
)
|
||
select k.cat, store_id s, count(*)::int n,
|
||
percentile_cont(0.5) within group (order by a / h) ratio,
|
||
avg((h < a)::int) cheaper
|
||
from x cross join lateral (values (${ALL}), (x.top_slug)) k(cat)
|
||
where h is not null and a is not null and h > 0
|
||
and a / h between 0.1 and 10
|
||
group by 1, 2`;
|
||
return {
|
||
rRows: byCat<RangeRow>(rows.map((r) => ({ ...r }) as RangeRow & { cat: string })),
|
||
bkRows: byCat<{ b: string; n: number }>(buckets.map((r) => ({ cat: r.cat, b: r.b, n: r.n }))),
|
||
gRows: byCat<BrandGapRow>(
|
||
gap.map((r) => ({ cat: r.cat, s: r.s, n: r.n, ratio: r4(r.ratio), cheaper: r4(r.cheaper) }))
|
||
)
|
||
};
|
||
}
|
||
|
||
const ref = (r: Record<string, unknown>, prefix = ''): ItemRef => ({
|
||
id: r[`${prefix}id`] as number,
|
||
s: r[`${prefix}s`] as number,
|
||
name: r[`${prefix}name`] as string,
|
||
size: (r[`${prefix}size`] as string) ?? null,
|
||
p: numOrNull(r[`${prefix}p`]),
|
||
up: numOrNull(r[`${prefix}up`]),
|
||
unit: (r[`${prefix}unit`] as string) ?? null,
|
||
url: (r[`${prefix}url`] as string) ?? null
|
||
});
|
||
|
||
async function extras(tx: Tx) {
|
||
const ext = await tx`
|
||
with u as (
|
||
select k.cat, mode() within group (order by unit) unit
|
||
from o cross join lateral (values (${ALL}), (o.top_slug)) k(cat) group by 1
|
||
),
|
||
r as (
|
||
select k.cat, o.item_id id, o.store_id s, o.name, o.size_text size, o.p, o.up, o.unit, o.url,
|
||
row_number() over (partition by k.cat order by o.up, o.item_id) rc,
|
||
row_number() over (partition by k.cat order by o.up desc, o.item_id) rd
|
||
from o cross join lateral (values (${ALL}), (o.top_slug)) k(cat)
|
||
join u on u.cat = k.cat and u.unit = o.unit
|
||
)
|
||
select * from r where rc <= 5 or rd <= 5`;
|
||
// Same EAN in several stores: pack prices compared directly (same pack size checked).
|
||
const ean = await tx`
|
||
with e as (
|
||
select gtin, store_id, min(p) p, min(ua) ua, mode() within group (order by top_slug) top_slug
|
||
from o where gtin is not null and gtin <> '' group by 1, 2
|
||
),
|
||
g as (
|
||
select gtin, count(*) n, min(p) mn, max(p) mx, min(ua) mnu, max(ua) mxu,
|
||
mode() within group (order by top_slug) top_slug
|
||
from e group by gtin having count(*) >= 2
|
||
)
|
||
select k.cat, count(*)::int multi, count(*) filter (where mn = mx)::int same
|
||
from g cross join lateral (values (${ALL}), (g.top_slug)) k(cat)
|
||
where mxu <= mnu * 1.05 group by 1`;
|
||
const gaps = await tx`
|
||
with e as (
|
||
select distinct on (gtin, store_id) gtin, store_id, item_id, p, ua, top_slug
|
||
from o where gtin is not null and gtin <> '' order by gtin, store_id, p
|
||
),
|
||
g as (
|
||
select gtin, count(*)::int stores, min(p) mn, max(p) mx, min(ua) mnu, max(ua) mxu,
|
||
mode() within group (order by top_slug) top_slug,
|
||
(array_agg(item_id order by p, store_id))[1] cheap_id,
|
||
(array_agg(item_id order by p desc, store_id))[1] dear_id
|
||
from e group by gtin having count(*) >= 2
|
||
),
|
||
r as (
|
||
select k.cat, g.*, mx::float8 / mn ratio,
|
||
row_number() over (partition by k.cat order by mx::float8 / mn desc, mx - mn desc) rn
|
||
from g cross join lateral (values (${ALL}), (g.top_slug)) k(cat)
|
||
where mxu <= mnu * 1.05 and mx - mn >= 50 and mx::float8 / mn <= 4
|
||
)
|
||
select r.cat, r.gtin, r.stores, r.ratio, r.mx - r.mn diff,
|
||
c.item_id c_id, c.store_id c_s, c.name c_name, c.size_text c_size, c.p c_p, c.up c_up, c.unit c_unit, c.url c_url,
|
||
d.item_id d_id, d.store_id d_s, d.name d_name, d.size_text d_size, d.p d_p, d.up d_up, d.unit d_unit, d.url d_url
|
||
from r join o c on c.item_id = r.cheap_id join o d on d.item_id = r.dear_id
|
||
where r.rn <= 20 order by r.cat, r.rn`;
|
||
// Price per kg distribution (euros, log-ish bins).
|
||
const ppkg = await tx`
|
||
select k.cat, o.store_id s,
|
||
width_bucket(o.up / 100.0, array[1, 2, 3, 4, 5, 7.5, 10, 15, 20, 30, 50, 100]::numeric[]) bin,
|
||
count(*)::int n
|
||
from o cross join lateral (values (${ALL}), (o.top_slug)) k(cat)
|
||
where o.unit = 'kg' group by 1, 2, 3`;
|
||
const ppkgMed = await tx`
|
||
select k.cat, o.store_id s, percentile_cont(0.5) within group (order by o.up) med, count(*)::int n
|
||
from o cross join lateral (values (${ALL}), (o.top_slug)) k(cat)
|
||
where o.unit = 'kg' group by 1, 2`;
|
||
const extMap = new Map<string, Section['extremes']>();
|
||
for (const r of ext) {
|
||
const e = extMap.get(r.cat) ?? { unit: r.unit, cheap: [], dear: [] };
|
||
if (Number(r.rc) <= 5) e.cheap[Number(r.rc) - 1] = ref(r);
|
||
if (Number(r.rd) <= 5) e.dear[Number(r.rd) - 1] = ref(r);
|
||
extMap.set(r.cat, e);
|
||
}
|
||
for (const e of extMap.values()) {
|
||
e.cheap = e.cheap.filter(Boolean);
|
||
e.dear = e.dear.filter(Boolean);
|
||
}
|
||
const gapRows = byCat<GapRow>(
|
||
gaps.map((r) => ({
|
||
cat: r.cat,
|
||
gtin: r.gtin,
|
||
stores: r.stores,
|
||
ratio: r4(r.ratio),
|
||
diff: num(r.diff),
|
||
cheap: ref(r, 'c_'),
|
||
dear: ref(r, 'd_')
|
||
}))
|
||
);
|
||
const ppkgMap = new Map<string, PpkgRow[]>();
|
||
const med = new Map(ppkgMed.map((r) => [`${r.cat}|${r.s}`, r]));
|
||
for (const r of ppkg) {
|
||
const list = ppkgMap.get(r.cat) ?? [];
|
||
let row = list.find((x) => x.s === r.s);
|
||
if (!row) {
|
||
const m = med.get(`${r.cat}|${r.s}`);
|
||
row = {
|
||
s: r.s,
|
||
bins: Array(13).fill(0),
|
||
median: m ? Math.round(num(m.med)) : null,
|
||
n: m ? m.n : 0
|
||
};
|
||
list.push(row);
|
||
}
|
||
row.bins[Number(r.bin)] = r.n;
|
||
ppkgMap.set(r.cat, list);
|
||
}
|
||
return {
|
||
extMap,
|
||
eanMap: new Map(ean.map((r) => [r.cat as string, { multi: r.multi, same: r.same }])),
|
||
gapRows,
|
||
ppkgMap
|
||
};
|
||
}
|
||
|
||
/** Price history: chained index, this week's movers, volatility, promo weekdays. */
|
||
async function history(tx: Tx, scope: StatsScope, stores: number[] | null) {
|
||
const storeArr = stores ? `{${stores.map((s) => Math.trunc(s)).join(',')}}` : null;
|
||
await tx`
|
||
create temp table it on commit drop as
|
||
select i.id, i.store_id, coalesce(tc.slug, 'overig') top_slug, i.name, i.size_text, i.unit,
|
||
i.unit_price_cents up, i.url, i.available
|
||
from groceries.items i
|
||
join groceries.stores s on s.id = i.store_id and s.archived_at is null
|
||
left join groceries.categories c on c.id = i.category_id
|
||
left join groceries.categories tc on tc.id = coalesce(c.parent_id, c.id)
|
||
where (${scope} = 'all' or (i.is_food and coalesce(c.slug, '') <> all(${NON_FOOD_LEAVES})
|
||
and i.name !~* 'e-?gift|cadeaukaart|giftcard|gift card'))
|
||
and (${storeArr}::int[] is null or i.store_id = any(${storeArr}::int[]))`;
|
||
await tx`create unique index on it (id)`;
|
||
await tx`
|
||
create temp table h on commit drop as
|
||
select ip.item_id, ip.observed_on d, ip.price_cents p,
|
||
least(ip.price_cents, coalesce(ip.promo_price_cents, ip.price_cents)) ep,
|
||
ip.promo_price_cents is not null or (ip.promo_label is not null and ip.promo_label !~* 'volume\\s*voordeel') promo,
|
||
lag(ip.price_cents) over w pp,
|
||
lag(least(ip.price_cents, coalesce(ip.promo_price_cents, ip.price_cents))) over w pep,
|
||
lag(ip.promo_price_cents is not null or (ip.promo_label is not null and ip.promo_label !~* 'volume\\s*voordeel')) over w ppromo,
|
||
min(ip.observed_on) over (partition by ip.item_id) first_d,
|
||
row_number() over (partition by ip.item_id order by ip.observed_on desc) rn_desc,
|
||
it.store_id, it.top_slug
|
||
from groceries.item_prices ip join it on it.id = ip.item_id
|
||
where ip.price_cents > 0
|
||
window w as (partition by ip.item_id order by ip.observed_on)`;
|
||
const [span] = await tx`select min(d)::text first, max(d)::text last from h`;
|
||
const present = await tx`
|
||
select k.cat, st.s, first_d::text d, count(*)::int n
|
||
from h cross join lateral (values (${ALL}), (h.top_slug)) k(cat)
|
||
cross join lateral (values (h.store_id), (0)) st(s)
|
||
where d = first_d group by 1, 2, 3`;
|
||
const changes = await tx`
|
||
select k.cat, st.s, d::text d,
|
||
sum(ln(p::float8 / pp)) filter (where p::float8 / pp between 0.2 and 5) lr,
|
||
sum(ln(ep::float8 / pep)) filter (where pep > 0 and ep > 0 and ep::float8 / pep between 0.2 and 5) le,
|
||
count(*) filter (where p <> pp)::int c
|
||
from h cross join lateral (values (${ALL}), (h.top_slug)) k(cat)
|
||
cross join lateral (values (h.store_id), (0)) st(s)
|
||
where pp is not null and pp > 0 group by 1, 2, 3`;
|
||
// Days from first to last observation (inclusive).
|
||
const days: string[] = [];
|
||
if (span?.first) {
|
||
const end = new Date(`${span.last}T00:00:00Z`);
|
||
for (let t = new Date(`${span.first}T00:00:00Z`); t <= end; t.setUTCDate(t.getUTCDate() + 1))
|
||
days.push(t.toISOString().slice(0, 10));
|
||
}
|
||
const dayIx = new Map(days.map((d, i) => [d, i]));
|
||
const series: SeriesData = { days, cats: {} };
|
||
const firstCounts = new Map<string, number[]>();
|
||
for (const r of present) {
|
||
const key = `${r.cat}|${r.s}`;
|
||
const arr = firstCounts.get(key) ?? Array(days.length).fill(0);
|
||
const i = dayIx.get(r.d);
|
||
if (i != null) arr[i] += r.n;
|
||
firstCounts.set(key, arr);
|
||
}
|
||
const chg = new Map<string, { lr: number; le: number; c: number }[]>();
|
||
for (const r of changes) {
|
||
const key = `${r.cat}|${r.s}`;
|
||
const arr = chg.get(key) ?? days.map(() => ({ lr: 0, le: 0, c: 0 }));
|
||
const i = dayIx.get(r.d);
|
||
if (i != null) arr[i] = { lr: num(r.lr), le: num(r.le), c: r.c };
|
||
chg.set(key, arr);
|
||
}
|
||
for (const [key, firsts] of firstCounts) {
|
||
const [cat, s] = key.split('|');
|
||
const ch = chg.get(key);
|
||
let n = 0;
|
||
let ir = 100;
|
||
let ie = 100;
|
||
const out = { r: [] as number[], e: [] as number[], c: [] as number[] };
|
||
days.forEach((_, i) => {
|
||
// Items present before day i (first observed on an earlier day) form the basket.
|
||
if (i > 0 && n > 0 && ch) {
|
||
ir *= Math.exp(ch[i].lr / n);
|
||
ie *= Math.exp(ch[i].le / n);
|
||
}
|
||
out.r.push(Math.round(ir * 1000) / 1000);
|
||
out.e.push(Math.round(ie * 1000) / 1000);
|
||
out.c.push(ch?.[i].c ?? 0);
|
||
n += firsts[i];
|
||
});
|
||
(series.cats[cat] ??= {})[s] = out;
|
||
}
|
||
const inflation = new Map<string, InflationRow[]>();
|
||
for (const [cat, perStore] of Object.entries(series.cats)) {
|
||
const rows: InflationRow[] = [];
|
||
for (const [s, v] of Object.entries(perStore)) {
|
||
const L = days.length - 1;
|
||
const ch = (arr: number[], back: number) =>
|
||
L - back >= 0 && L >= back && back > 0 ? r4(arr[L] / arr[L - back] - 1) : null;
|
||
rows.push({
|
||
s: Number(s),
|
||
days: days.length,
|
||
wow: L >= 7 ? ch(v.r, 7) : null,
|
||
mom: L >= 30 ? ch(v.r, 30) : null,
|
||
total: L >= 1 ? r4(v.r[L] / 100 - 1) : null,
|
||
wowE: L >= 7 ? ch(v.e, 7) : null,
|
||
momE: L >= 30 ? ch(v.e, 30) : null,
|
||
totalE: L >= 1 ? r4(v.e[L] / 100 - 1) : null,
|
||
changes: v.c.reduce((a, b) => a + b, 0)
|
||
});
|
||
}
|
||
inflation.set(cat, rows);
|
||
}
|
||
// This week's biggest moves: current price vs the price 7 days ago (or the first one seen).
|
||
const moves = await tx`
|
||
with cur as (select * from h where rn_desc = 1),
|
||
old as (
|
||
select distinct on (h.item_id) h.item_id, h.p, h.d
|
||
from h join cur on cur.item_id = h.item_id
|
||
where h.d < cur.d and (h.d <= cur.d - 7 or h.d = h.first_d)
|
||
order by h.item_id, (h.d <= cur.d - 7) desc, h.d desc
|
||
),
|
||
m as (
|
||
select k.cat, it.id, it.store_id s, it.name, it.size_text size, cur.p, it.up, it.unit, it.url,
|
||
old.p "from", cur.p "to", cur.p::float8 / old.p - 1 pct, old.d::text since
|
||
from cur join old using (item_id) join it on it.id = cur.item_id
|
||
cross join lateral (values (${ALL}), (it.top_slug)) k(cat)
|
||
where it.available and old.p > 0 and abs(cur.p - old.p) >= 10
|
||
and cur.p::float8 / old.p between 0.2 and 5
|
||
),
|
||
r as (
|
||
select m.*, row_number() over (partition by cat order by pct desc) ru,
|
||
row_number() over (partition by cat order by pct) rd from m
|
||
)
|
||
select * from r where ru <= 15 or rd <= 15`;
|
||
const vol = await tx`
|
||
with v as (
|
||
select item_id,
|
||
count(*) filter (where pp is not null and (p <> pp or ep <> pep))::int changes,
|
||
min(p) mn, max(p) mx,
|
||
count(*) filter (where ppromo is not null and promo <> ppromo)::int flips
|
||
from h where d >= current_date - 90 group by item_id
|
||
),
|
||
r as (
|
||
select k.cat, it.id, it.store_id s, it.name, it.size_text size, it.up, it.unit, it.url,
|
||
v.changes, v.mn, v.mx, v.flips,
|
||
row_number() over (partition by k.cat order by v.changes + v.flips desc, v.mx::float8 / nullif(v.mn, 0) desc, it.id) rn
|
||
from v join it on it.id = v.item_id
|
||
cross join lateral (values (${ALL}), (it.top_slug)) k(cat)
|
||
where v.changes + v.flips >= 1 and it.available
|
||
)
|
||
select * from r where rn <= 15`;
|
||
const weekday = await tx`
|
||
select store_id s, extract(isodow from d)::int dow, count(*)::int n
|
||
from h where promo and ppromo = false group by 1, 2`;
|
||
const movesMap = new Map<string, { rises: MoveRow[]; drops: MoveRow[] }>();
|
||
for (const r of moves) {
|
||
const m = movesMap.get(r.cat) ?? { rises: [], drops: [] };
|
||
const row: MoveRow = {
|
||
...ref(r),
|
||
from: num(r.from),
|
||
to: num(r.to),
|
||
pct: r4(r.pct),
|
||
since: r.since
|
||
};
|
||
if (Number(r.ru) <= 15 && row.pct > 0) m.rises.push(row);
|
||
if (Number(r.rd) <= 15 && row.pct < 0) m.drops.push(row);
|
||
movesMap.set(r.cat, m);
|
||
}
|
||
for (const m of movesMap.values()) {
|
||
m.rises.sort((a, b) => b.pct - a.pct);
|
||
m.drops.sort((a, b) => a.pct - b.pct);
|
||
}
|
||
const volMap = new Map<string, VolRow[]>();
|
||
for (const r of vol) {
|
||
const list = volMap.get(r.cat) ?? [];
|
||
list[Number(r.rn) - 1] = {
|
||
...ref(r),
|
||
p: null,
|
||
changes: r.changes,
|
||
min: num(r.mn),
|
||
max: num(r.mx),
|
||
promoFlips: r.flips
|
||
};
|
||
volMap.set(r.cat, list);
|
||
}
|
||
for (const [k, v] of volMap) volMap.set(k, v.filter(Boolean));
|
||
return {
|
||
series,
|
||
inflation,
|
||
movesMap,
|
||
volMap,
|
||
weekday: weekday.map((r) => ({ s: r.s, dow: r.dow, n: r.n })),
|
||
first: (span?.first as string) ?? null,
|
||
last: (span?.last as string) ?? null
|
||
};
|
||
}
|
||
|
||
/** Staples basket: cheapest matching product per staple per store (regular and with promo). */
|
||
async function basket(tx: Tx) {
|
||
const rows = await tx`
|
||
with st as (
|
||
select * from json_to_recordset(${tx.json(STAPLES as never)}::json)
|
||
as x(key text, rx text, nrx text, units text, qty float8, cats text, min float8, max float8,
|
||
"pieceKg" float8)
|
||
),
|
||
m as (
|
||
-- Normalised to the staple's unit; loose fruit sold per piece uses pieceKg.
|
||
select st.key, st.qty, o.*,
|
||
case when o.unit = 'piece' and position('piece' in st.units) = 0 then o.up / st."pieceKg" else o.up end uq,
|
||
case when o.unit = 'piece' and position('piece' in st.units) = 0 then o.eup / st."pieceKg" else o.eup end euq,
|
||
case when o.unit = 'piece' and position('piece' in st.units) = 0 then o.ua * st."pieceKg" else o.ua end amt,
|
||
st.min, st.max
|
||
from st join o on (o.unit = any(string_to_array(st.units, ','))
|
||
or (o.unit = 'piece' and st."pieceKg" is not null))
|
||
and lower(o.name) ~ st.rx and (st.nrx is null or lower(o.name) !~ st.nrx)
|
||
and (st.cats is null or o.top_slug = any(string_to_array(st.cats, ',')))
|
||
and o.is_food
|
||
),
|
||
c as (
|
||
select key, qty, item_id id, store_id s, name, size_text size, p, uq up, euq eup, unit, url,
|
||
row_number() over (partition by key, store_id order by uq, item_id) rr,
|
||
row_number() over (partition by key, store_id order by euq, item_id) re
|
||
from m where amt between qty * min and qty * max
|
||
)
|
||
select * from c where rr = 1 or re = 1`;
|
||
const picks: Record<string, Record<string, BasketPick>> = {};
|
||
for (const r of rows) {
|
||
const p = ((picks[r.key] ??= {})[String(r.s)] ??= {} as BasketPick);
|
||
const base = ref(r);
|
||
base.up = Math.round(num(r.up));
|
||
if (Number(r.rr) === 1) p.reg = { ...base, cost: Math.round(num(r.up) * num(r.qty)) };
|
||
if (Number(r.re) === 1)
|
||
p.eff = { ...base, up: Math.round(num(r.eup)), cost: Math.round(num(r.eup) * num(r.qty)) };
|
||
}
|
||
return {
|
||
staples: STAPLES.map(({ key, label, qty, unit, icon }) => ({ key, label, qty, unit, icon })),
|
||
picks
|
||
};
|
||
}
|
||
|
||
/** Records pack size changes (item_sizes) and lists shrinkflation: smaller pack, price not lower. */
|
||
async function sizes(tx: Tx, scope: StatsScope, record: boolean) {
|
||
const [t] = await tx`select to_regclass('groceries.item_sizes') is not null ok`;
|
||
if (!t.ok) return { since: null, shrink: [] as ShrinkRow[] };
|
||
if (record)
|
||
await tx`
|
||
insert into groceries.item_sizes (item_id, observed_on, unit, unit_amount, size_text, price_cents)
|
||
select i.id, current_date, i.unit, i.unit_amount, i.size_text, i.price_cents
|
||
from groceries.items i
|
||
left join lateral (
|
||
select s.unit, s.unit_amount, true found from groceries.item_sizes s
|
||
where s.item_id = i.id order by s.observed_on desc limit 1
|
||
) l on true
|
||
where i.available and i.unit_amount is not null
|
||
and (l.found is null or l.unit is distinct from i.unit or l.unit_amount is distinct from i.unit_amount)
|
||
on conflict (item_id, observed_on) do update
|
||
set unit = excluded.unit, unit_amount = excluded.unit_amount,
|
||
size_text = excluded.size_text, price_cents = excluded.price_cents`;
|
||
const [since] = await tx`select min(observed_on)::text d from groceries.item_sizes`;
|
||
const rows = await tx`
|
||
with s as (
|
||
select s.*, lag(s.unit) over w pu, lag(s.unit_amount) over w pa, lag(s.price_cents) over w pp,
|
||
lag(s.size_text) over w ps
|
||
from groceries.item_sizes s window w as (partition by s.item_id order by s.observed_on)
|
||
)
|
||
select i.id, i.store_id s, i.name, s.size_text size, i.price_cents p, i.unit_price_cents up,
|
||
i.unit, i.url, s.observed_on::text day, s.pa::float8 from_amount, s.unit_amount::float8 to_amount,
|
||
s.pp from_price, s.price_cents to_price
|
||
from s join groceries.items i on i.id = s.item_id
|
||
where s.pu = s.unit and s.unit_amount < s.pa * 0.99 and s.price_cents >= s.pp * 0.99
|
||
and (${scope} = 'all' or i.is_food)
|
||
order by s.observed_on desc, s.unit_amount / s.pa limit 50`;
|
||
return {
|
||
since: (since?.d as string) ?? null,
|
||
shrink: rows.map(
|
||
(r) =>
|
||
({
|
||
...ref(r),
|
||
day: r.day,
|
||
fromAmount: num(r.from_amount),
|
||
toAmount: num(r.to_amount),
|
||
fromPrice: numOrNull(r.from_price),
|
||
toPrice: numOrNull(r.to_price)
|
||
}) as ShrinkRow
|
||
)
|
||
};
|
||
}
|
||
|
||
async function promoEnds(tx: Tx) {
|
||
const rows = await tx`
|
||
with x as (
|
||
select store_id, regexp_match(pl, 't/m\\s*(\\d{1,2})[-/.](\\d{1,2})') m from o
|
||
where pl ~* 't/m\\s*\\d'
|
||
),
|
||
d as (
|
||
select store_id, make_date(
|
||
extract(year from current_date)::int
|
||
+ case when m[2]::int < extract(month from current_date) - 6 then 1 else 0 end,
|
||
m[2]::int, 1) + (m[1]::int - 1) d
|
||
from x where m[2]::int between 1 and 12 and m[1]::int between 1 and 31
|
||
)
|
||
select store_id s, extract(isodow from d)::int dow, count(*)::int n from d group by 1, 2`;
|
||
return rows.map((r) => ({ s: r.s as number, dow: r.dow as number, n: r.n as number }));
|
||
}
|
||
|
||
async function refs(sql: Sql | Tx) {
|
||
const stores = await sql<StoreRef[]>`
|
||
select s.id, s.slug, s.name, s.color from groceries.stores s
|
||
where s.archived_at is null and exists (select 1 from groceries.items i where i.store_id = s.id)
|
||
order by s.sort_order, s.id`;
|
||
const categories = await sql<CategoryRef[]>`
|
||
select slug, name, icon, color, is_food "isFood" from groceries.categories
|
||
where parent_id is null order by sort_order, id`;
|
||
return { stores: [...stores], categories: [...categories] };
|
||
}
|
||
|
||
function emptySection(): Section {
|
||
return {
|
||
kpi: { items: 0, groups: 0, battles: 0, tied: 0, multi: 0, ean: 0, eanSame: 0 },
|
||
battles: [],
|
||
h2h: [],
|
||
promos: [],
|
||
promoTypes: [],
|
||
range: [],
|
||
buckets: [],
|
||
brandGap: [],
|
||
extremes: { unit: null, cheap: [], dear: [] },
|
||
gaps: [],
|
||
ppkg: [],
|
||
moves: { rises: [], drops: [] },
|
||
volatile: [],
|
||
inflation: []
|
||
};
|
||
}
|
||
|
||
/** Computes everything for one scope (inside one transaction; temp tables are dropped after). */
|
||
export async function computeSnapshot(
|
||
sql: Sql,
|
||
scope: StatsScope,
|
||
opts: { record?: boolean; log?: Log } = {}
|
||
): Promise<{ data: SnapshotData; series: SeriesData }> {
|
||
const log = opts.log ?? (() => {});
|
||
const t0 = Date.now();
|
||
return (await sql.begin(async (tx) => {
|
||
await tx`set local work_mem = '128MB'`;
|
||
const { stores, categories } = await refs(tx);
|
||
await prepare(tx, scope, null);
|
||
const lap = (m: string) => log(`stats ${scope}: ${m} (${Date.now() - t0} ms)`);
|
||
lap('offers ready');
|
||
const bt = await battles(tx);
|
||
lap('battles');
|
||
const pm = await promos(tx);
|
||
lap('promos');
|
||
const rg = await range(tx);
|
||
lap('range');
|
||
const ex = await extras(tx);
|
||
lap('extras');
|
||
const hs = await history(tx, scope, null);
|
||
lap('history');
|
||
const bk = await basket(tx);
|
||
const sz = await sizes(tx, scope, opts.record ?? false);
|
||
const ends = await promoEnds(tx);
|
||
const counts = await tx`
|
||
select k.cat, count(*)::int items, count(distinct gid)::int groups
|
||
from o cross join lateral (values (${ALL}), (o.top_slug)) k(cat) group by 1`;
|
||
const multi = new Map(
|
||
rg.bkRows
|
||
? [...rg.bkRows].map(([c, rows]) => [
|
||
c,
|
||
rows.filter((r) => r.b !== '1').reduce((a, r) => a + r.n, 0)
|
||
])
|
||
: []
|
||
);
|
||
lap('basket, sizes');
|
||
const section = (cat: string): Section => {
|
||
const s = emptySection();
|
||
const c = counts.find((r) => r.cat === cat);
|
||
const tot = bt.tot.get(cat);
|
||
const e = ex.eanMap.get(cat);
|
||
s.kpi = {
|
||
items: c?.items ?? 0,
|
||
groups: c?.groups ?? 0,
|
||
battles: tot?.battles ?? 0,
|
||
tied: tot?.tied ?? 0,
|
||
multi: multi.get(cat) ?? 0,
|
||
ean: e?.multi ?? 0,
|
||
eanSame: e?.same ?? 0
|
||
};
|
||
s.battles = bt.bRows.get(cat) ?? [];
|
||
s.h2h = bt.hRows.get(cat) ?? [];
|
||
s.promos = pm.pRows.get(cat) ?? [];
|
||
s.promoTypes = pm.tRows.get(cat) ?? [];
|
||
s.range = rg.rRows.get(cat) ?? [];
|
||
s.buckets = rg.bkRows.get(cat) ?? [];
|
||
s.brandGap = rg.gRows.get(cat) ?? [];
|
||
s.extremes = ex.extMap.get(cat) ?? s.extremes;
|
||
s.gaps = ex.gapRows.get(cat) ?? [];
|
||
s.ppkg = ex.ppkgMap.get(cat) ?? [];
|
||
s.moves = hs.movesMap.get(cat) ?? s.moves;
|
||
s.volatile = hs.volMap.get(cat) ?? [];
|
||
s.inflation = hs.inflation.get(cat) ?? [];
|
||
return s;
|
||
};
|
||
const byCategory: Record<string, Section> = {};
|
||
const cats = categories.filter((c) => scope === 'all' || c.isFood).map((c) => c.slug);
|
||
for (const cat of cats) if (counts.some((r) => r.cat === cat)) byCategory[cat] = section(cat);
|
||
const overview: Overview = {
|
||
heat: [],
|
||
promoHeat: [],
|
||
rangeHeat: [],
|
||
brandGapByCat: [],
|
||
extremesByCat: [],
|
||
basket: bk,
|
||
weekday: hs.weekday,
|
||
promoEnds: ends,
|
||
shrink: sz.shrink,
|
||
shrinkSince: sz.since
|
||
};
|
||
for (const [cat, s] of Object.entries(byCategory)) {
|
||
for (const b of s.battles) overview.heat.push({ cat, s: b.s, n: b.n, wr: b.wr, ix: b.ix });
|
||
for (const p of s.promos)
|
||
overview.promoHeat.push({ cat, s: p.s, share: p.share, items: p.items });
|
||
for (const r of s.range) overview.rangeHeat.push({ cat, s: r.s, items: r.items });
|
||
const g = s.brandGap.find((x) => x.s === 0);
|
||
if (g) overview.brandGapByCat.push({ cat, n: g.n, ratio: g.ratio, cheaper: g.cheaper });
|
||
overview.extremesByCat.push({
|
||
cat,
|
||
unit: s.extremes.unit,
|
||
cheap: s.extremes.cheap[0] ?? null,
|
||
dear: s.extremes.dear[0] ?? null
|
||
});
|
||
}
|
||
const durationMs = Date.now() - t0;
|
||
const data: SnapshotData = {
|
||
meta: {
|
||
computedAt: new Date().toISOString(),
|
||
scope,
|
||
stores,
|
||
categories,
|
||
firstDay: hs.first,
|
||
lastDay: hs.last,
|
||
historyDays: hs.series.days.length,
|
||
durationMs
|
||
},
|
||
overall: section(ALL),
|
||
overview,
|
||
byCategory
|
||
};
|
||
lap('done');
|
||
return { data, series: hs.series };
|
||
})) as { data: SnapshotData; series: SeriesData };
|
||
}
|
||
|
||
/** Nightly: both scopes, stored per day (re-runs on the same day overwrite), then prune. */
|
||
export async function computeStats(sql: Sql, log: Log = () => {}) {
|
||
for (const scope of ['food', 'all'] as const) {
|
||
const t0 = Date.now();
|
||
const { data, series } = await computeSnapshot(sql, scope, { record: scope === 'food', log });
|
||
const ms = Date.now() - t0;
|
||
await sql`
|
||
insert into groceries.stats_snapshots (computed_on, scope, computed_at, duration_ms, data, series)
|
||
values ((now() at time zone 'Europe/Amsterdam')::date, ${scope}, now(), ${ms},
|
||
${sql.json(data as never)}, ${sql.json(series as never)})
|
||
on conflict (computed_on, scope) do update set computed_at = excluded.computed_at,
|
||
duration_ms = excluded.duration_ms, data = excluded.data, series = excluded.series`;
|
||
log(`stats ${scope}: snapshot saved in ${ms} ms`);
|
||
}
|
||
// Keep the history small: the time series only on the last 7 days, full per-category detail
|
||
// for 60 days; older snapshots keep the overall section (for trend charts).
|
||
await sql`
|
||
update groceries.stats_snapshots set series = null
|
||
where series is not null and computed_on < current_date - 7`;
|
||
await sql`
|
||
update groceries.stats_snapshots
|
||
set data = data - 'byCategory' - 'overview'
|
||
where computed_on < current_date - 60 and data ? 'byCategory'`;
|
||
}
|
||
|
||
/** Live battles for a store subset (page filter); same rules as the snapshot. */
|
||
export async function computeBattles(sql: Sql, scope: StatsScope, stores: number[]) {
|
||
return (await sql.begin(async (tx) => {
|
||
await prepare(tx, scope, stores);
|
||
const bt = await battles(tx);
|
||
const out: Record<
|
||
string,
|
||
{ battles: BattleRow[]; h2h: H2HRow[]; total: number; tied: number }
|
||
> = {};
|
||
for (const [cat, rows] of bt.bRows) {
|
||
const t = bt.tot.get(cat);
|
||
out[cat] = {
|
||
battles: rows,
|
||
h2h: bt.hRows.get(cat) ?? [],
|
||
total: t?.battles ?? 0,
|
||
tied: t?.tied ?? 0
|
||
};
|
||
}
|
||
return out;
|
||
})) as Record<string, { battles: BattleRow[]; h2h: H2HRow[]; total: number; tied: number }>;
|
||
}
|