groceries: add Picnic store (picnic.app storefront API, full range via category pages)
Deploy production / deploy (push) Successful in 1m24s
Deploy production / deploy (push) Successful in 1m24s
- picnic.ts: crawls the category tree, subcategory and overview pages (RSC), maps Picnic categories to the shared list; no EANs, so matching is by name/brand/size - picnic-auth.ts + scripts/picnic-login.ts: one-shot login from /etc/grocery-tracker/picnic.env, token kept in ~/.config/groceries/picnic.env; 2FA is never requested unattended (run picnic-login.ts by hand) - migration: store row picnic; Picnic as own brand in stats
This commit is contained in:
stan committed
1 parent
079399635d
commit
c094754c5b
11 files changed
+46305
-2
No files matched your search
@@ -0,0 +1,262 @@
|
||||
// Picnic (picnic.app) catalogue source. Picnic has no public web shop: the app's storefront API
|
||||
// needs an account session (see picnic-auth.ts; read-only use, nothing touches the cart). The app's
|
||||
// category pages are React Server Components payloads (picnic-rsc.ts):
|
||||
// category-tree-root → the top categories (subcategory-page links)
|
||||
// subcategory-page?category_id=… → the sub-categories (category-product-overview links)
|
||||
// category-product-overview?… → every product tile of the sub-category, in one response
|
||||
// One request at a time (~1/s); "Custom…" theme categories (glutenvrij, lactosevrij, …) are
|
||||
// crawled last so a product keeps its regular category. Tiles carry no EAN (nor does the product
|
||||
// page), so Picnic is matched on name/brand/size.
|
||||
import { HttpError, type PoliteFetcher } from './http';
|
||||
import {
|
||||
PICNIC_API,
|
||||
PicnicAuthError,
|
||||
checkAuth,
|
||||
loadState,
|
||||
login,
|
||||
picnicHeaders,
|
||||
type PicnicState
|
||||
} from './picnic-auth';
|
||||
import { decodeRscPage, parseRscPayload, walkObjects } from './picnic-rsc';
|
||||
import type { RawItem, Source } from './sources';
|
||||
|
||||
const IMAGES = 'https://storefront-prod.nl.picnicinternational.com/static/images';
|
||||
const PRODUCT_LINK = 'https://picnic.app/nl/link/store/page;id=product-details-page-root,id=';
|
||||
|
||||
type Cat = { category: string; isFood?: boolean };
|
||||
const c = (category: string, isFood?: boolean): Cat => ({ category, isFood });
|
||||
/** Picnic top category id → shared category (names as of 2026-10). */
|
||||
export const TOP: Record<string, Cat> = {
|
||||
'21724': c('Groente, fruit & aardappelen'), // Fruit
|
||||
'21725': c('Groente, fruit & aardappelen'), // Aardappelen & groente
|
||||
'21726': c('Vlees, vis & vega'), // Vlees & vis
|
||||
'32811': c('Vlees, vis & vega'), // Vega & plantaardig
|
||||
'21727': c('Maaltijden & gemak'),
|
||||
'21728': c('Brood & bakkerij'), // Bakkerij
|
||||
'21730': c('Zuivel & eieren'),
|
||||
'26714': c('Vleeswaren, kaas & tapas'), // Kaas
|
||||
'21731': c('Vleeswaren, kaas & tapas'), // Vleeswaren & broodsalades
|
||||
'36362': c('Vleeswaren, kaas & tapas'), // Borrel, tapas & spreads
|
||||
'21732': c('Ontbijt & beleg'), // Ontbijt & zoet beleg
|
||||
'21733': c('Pasta, rijst & wereldkeuken'),
|
||||
'21734': c('Soepen, sauzen & conserven'), // Voorraadkast
|
||||
'21735': c('Snoep, koek & chips'), // Koek, snoep & snacks
|
||||
'21740': c('Diepvries'),
|
||||
'21738': c('Koffie & thee'),
|
||||
'21736': c('Frisdrank & sappen'), // Drinken
|
||||
'33825': c('Bier & wijn'), // Bier & aperitieven
|
||||
'33826': c('Bier & wijn'), // Wijn & bubbels
|
||||
'38115': c('Baby & kind'),
|
||||
'37727': c('Drogisterij', false), // Gezichtsverzorging
|
||||
'38177': c('Drogisterij', false), // Gezondheid & vitamines
|
||||
'37655': c('Drogisterij', false), // Haarverzorging
|
||||
'37784': c('Drogisterij', false), // Lichaamsverzorging
|
||||
'37627': c('Drogisterij', false), // Mondverzorging
|
||||
'37611': c('Drogisterij', false), // Zonbescherming
|
||||
'21741': c('Huishouden & dieren', false), // Huishouden
|
||||
'21744': c('Huishouden & dieren', false), // Dier
|
||||
'34134': c('Overig', false), // Koken, tafelen & vrije tijd
|
||||
'38263': c('Overig') // Sport (sports nutrition)
|
||||
};
|
||||
/** Sub-category name overrides inside a top category ("top id/regex"). */
|
||||
const SUB: [string, RegExp, Cat][] = [
|
||||
['21734', /ontbijt|beleg|broodsmeer/i, c('Ontbijt & beleg')],
|
||||
['21734', /bak(ken|producten)|meel|suiker/i, c('Ontbijt & beleg')],
|
||||
['21734', /pasta|rijst|noedel|wereld/i, c('Pasta, rijst & wereldkeuken')],
|
||||
['21738', /sap|frisdrank/i, c('Frisdrank & sappen')],
|
||||
['21736', /koffie|thee/i, c('Koffie & thee')],
|
||||
['21736', /bier|wijn/i, c('Bier & wijn')],
|
||||
['38115', /verzorging|luiers|billendoek/i, c('Baby & kind', false)]
|
||||
];
|
||||
|
||||
export function picnicCategory(topId: string, subName: string): Cat {
|
||||
for (const [t, re, cat] of SUB) if (t === topId && re.test(subName)) return cat;
|
||||
return TOP[topId] ?? c('Overig');
|
||||
}
|
||||
|
||||
export type PicnicTile = {
|
||||
id?: string;
|
||||
type?: string;
|
||||
title?: { title?: string; prefix?: string };
|
||||
subtitle?: { text?: string };
|
||||
images?: { primaryImage?: { imageId?: string }; sellingUnitImage?: { imageId?: string } };
|
||||
price?: {
|
||||
priceValue?: number;
|
||||
referencePriceValue?: number;
|
||||
basePrice?: number;
|
||||
priceRanges?: { fromQuantity?: number; price?: number }[];
|
||||
};
|
||||
discountInfo?: { label?: string } | null;
|
||||
labels?: Record<string, { type?: string; labelText?: string }[]>;
|
||||
suffixes?: { type?: string; text?: string }[];
|
||||
unavailability?: unknown;
|
||||
};
|
||||
|
||||
const cents = (euros: unknown) =>
|
||||
typeof euros === 'number' && Number.isFinite(euros) && euros > 0 ? Math.round(euros * 100) : null;
|
||||
|
||||
export function picnicItem(
|
||||
t: PicnicTile,
|
||||
cat: Cat,
|
||||
path: string,
|
||||
unavailable = false
|
||||
): RawItem | null {
|
||||
const title = t.title?.title?.trim();
|
||||
if (!t.id || !title) return null;
|
||||
const name = `${t.title?.prefix?.trim() ? `${t.title.prefix.trim()} ` : ''}${title}`;
|
||||
const tier1 = t.price?.priceRanges?.find((r) => r.fromQuantity === 1)?.price;
|
||||
const now =
|
||||
cents(t.price?.priceValue) ??
|
||||
(typeof tier1 === 'number' && tier1 > 0 ? Math.round(tier1) : null);
|
||||
const ref = cents(t.price?.referencePriceValue);
|
||||
const promo = now !== null && ref !== null && now < ref ? now : null;
|
||||
const promoText =
|
||||
Object.values(t.labels ?? {})
|
||||
.flat()
|
||||
.find((l) => l?.type === 'PROMO' && l.labelText)?.labelText ?? t.discountInfo?.label;
|
||||
const texts = (t.suffixes ?? []).filter((s) => s.type === 'TEXT' && s.text).map((s) => s.text!);
|
||||
const imageId = t.images?.primaryImage?.imageId ?? t.images?.sellingUnitImage?.imageId;
|
||||
return {
|
||||
storeProductId: t.id,
|
||||
name,
|
||||
brand: t.subtitle?.text?.trim() || null,
|
||||
sizeText: texts[0]?.trim() || null,
|
||||
priceCents: promo ? ref : now,
|
||||
promoPriceCents: promo,
|
||||
promoLabel: promo ? promoText?.trim() || 'Actie' : null,
|
||||
storeUnitPrice: null,
|
||||
storeUnitBasisCents: null,
|
||||
category: cat.category,
|
||||
categoryPath: path,
|
||||
gtin: null,
|
||||
imageUrl: imageId ? `${IMAGES}/${imageId}/medium.png` : null,
|
||||
url: `${PRODUCT_LINK}${t.id}`,
|
||||
available: !unavailable && !t.unavailability && now !== null,
|
||||
...(cat.isFood !== undefined ? { isFood: cat.isFood } : {})
|
||||
};
|
||||
}
|
||||
|
||||
type Link = { title: string; page: string; categoryId: string };
|
||||
/** Category links (row items) of a decoded page: deeplinks like `…;id=subcategory-page,category_id=21724`. */
|
||||
export function pageLinks(decoded: unknown): Link[] {
|
||||
const out = new Map<string, Link>();
|
||||
for (const o of walkObjects(decoded)) {
|
||||
if (o.type !== 'row-item' || typeof o.deeplink !== 'string' || typeof o.title !== 'string')
|
||||
continue;
|
||||
const [page, ...params] = (o.deeplink.split(';id=')[1] ?? '').split(',');
|
||||
const id = params.find((p) => p.startsWith('category_id='))?.slice('category_id='.length);
|
||||
if (!id) continue;
|
||||
const categoryId = decodeURIComponent(id);
|
||||
out.set(`${page}:${categoryId}`, { title: o.title.trim(), page, categoryId });
|
||||
}
|
||||
return [...out.values()];
|
||||
}
|
||||
|
||||
/** Product tiles of a decoded overview page, with the ids Picnic lists as not available. */
|
||||
export function pageTiles(decoded: unknown): {
|
||||
tiles: PicnicTile[];
|
||||
unavailable: Set<string>;
|
||||
relevant: number | null;
|
||||
} {
|
||||
const tiles = new Map<string, PicnicTile>();
|
||||
const unavailable = new Set<string>();
|
||||
let relevant: number | null = null;
|
||||
for (const o of walkObjects(decoded)) {
|
||||
if (o.type === 'sellable' && typeof o.id === 'string' && !tiles.has(o.id))
|
||||
tiles.set(o.id, o as PicnicTile);
|
||||
const facets = o.entityIdsByFacetId as Record<string, unknown> | undefined;
|
||||
if (facets && typeof facets === 'object') {
|
||||
for (const [k, ids] of Object.entries(facets)) {
|
||||
if (!Array.isArray(ids)) continue;
|
||||
if (k === 'relevancy.relevant') relevant = Math.max(relevant ?? 0, ids.length);
|
||||
if (k.startsWith('availability.') && k !== 'availability.AVAILABLE')
|
||||
for (const id of ids) if (typeof id === 'string') unavailable.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { tiles: [...tiles.values()], unavailable, relevant };
|
||||
}
|
||||
|
||||
const isTheme = (id: string) => /^Custom/i.test(id);
|
||||
|
||||
/** A logged-in session: the saved one, else one password login (never a 2FA prompt unattended). */
|
||||
async function session(log: (m: string) => void): Promise<PicnicState> {
|
||||
const s = loadState();
|
||||
if (await checkAuth(s)) return s;
|
||||
log(' picnic: saved session missing or expired; logging in');
|
||||
if ((await login(s)) === '2fa-required')
|
||||
throw new PicnicAuthError(
|
||||
'picnic: login needs a 2FA code; run `bun scripts/picnic-login.ts` on stan-ovh (then `verify <code>`)'
|
||||
);
|
||||
return s;
|
||||
}
|
||||
|
||||
export const picnicSource: Source = {
|
||||
slug: 'picnic',
|
||||
minDelayMs: 1200,
|
||||
async fetchAll(f: PoliteFetcher, log) {
|
||||
const s = await session(log);
|
||||
const only = process.env.PICNIC_ONLY_CATEGORY; // test runs: one top category id
|
||||
const get = async (page: string, key: string) => {
|
||||
try {
|
||||
const text = await f.text(
|
||||
`${PICNIC_API}/pages/${page}`,
|
||||
{ headers: picnicHeaders(s) },
|
||||
key.replace(/[^\w.-]+/g, '_')
|
||||
);
|
||||
return decodeRscPage(parseRscPayload(text));
|
||||
} catch (e) {
|
||||
if (e instanceof HttpError && (e.status === 401 || e.status === 403))
|
||||
throw new PicnicAuthError(`picnic: session rejected (HTTP ${e.status})`);
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
const tops = pageLinks(await get('category-tree-root', 'tree')).filter(
|
||||
(l) => l.page === 'subcategory-page' && (!only || l.categoryId === only)
|
||||
);
|
||||
if (!only && tops.length < 20) throw new Error(`picnic: only ${tops.length} top categories`);
|
||||
for (const t of tops)
|
||||
if (!TOP[t.categoryId] && !isTheme(t.categoryId))
|
||||
log(` picnic: unmapped top category ${t.categoryId} "${t.title}" → Overig`);
|
||||
tops.sort((a, b) => Number(isTheme(a.categoryId)) - Number(isTheme(b.categoryId)));
|
||||
|
||||
// Sub-categories of every top category; theme ones (Custom…) go last overall.
|
||||
const subs: (Link & { top: Link })[] = [];
|
||||
for (const top of tops) {
|
||||
const links = pageLinks(
|
||||
await get(
|
||||
`subcategory-page?category_id=${encodeURIComponent(top.categoryId)}`,
|
||||
`sub-${top.categoryId}`
|
||||
)
|
||||
);
|
||||
for (const l of links) if (l.page === 'category-product-overview') subs.push({ ...l, top });
|
||||
}
|
||||
subs.sort(
|
||||
(a, b) =>
|
||||
Number(isTheme(a.top.categoryId) || isTheme(a.categoryId)) -
|
||||
Number(isTheme(b.top.categoryId) || isTheme(b.categoryId))
|
||||
);
|
||||
log(` picnic: ${tops.length} categories, ${subs.length} sub-categories`);
|
||||
|
||||
const items = new Map<string, RawItem>();
|
||||
const seen = new Set<string>();
|
||||
for (const sub of subs) {
|
||||
if (seen.has(sub.categoryId)) continue;
|
||||
seen.add(sub.categoryId);
|
||||
const d = await get(
|
||||
`category-product-overview?category_id=${encodeURIComponent(sub.categoryId)}`,
|
||||
`ov-${sub.categoryId}`
|
||||
);
|
||||
const { tiles, unavailable, relevant } = pageTiles(d);
|
||||
if (relevant !== null && relevant > tiles.length)
|
||||
log(` picnic ${sub.top.title} / ${sub.title}: ${relevant} listed, ${tiles.length} tiles`);
|
||||
const cat = picnicCategory(sub.top.categoryId, sub.title);
|
||||
for (const t of tiles) {
|
||||
const it = picnicItem(t, cat, `${sub.top.title} / ${sub.title}`, unavailable.has(t.id!));
|
||||
if (it && !items.has(it.storeProductId)) items.set(it.storeProductId, it);
|
||||
}
|
||||
}
|
||||
log(` picnic: ${items.size} products from ${seen.size} sub-categories`);
|
||||
return [...items.values()];
|
||||
}
|
||||
};
|
||||
Reference in new issue
Block a user