268 lines
11 KiB
TypeScript
268 lines
11 KiB
TypeScript
// Picnic (picnic.app) catalogue source. Picnic has no public web shop: the app's storefront API
|
|
// needs an account session (see picnic-auth.ts; read-only use, nothing touches the cart). The app's
|
|
// category pages are React Server Components payloads (picnic-rsc.ts):
|
|
// category-tree-root → the top categories (subcategory-page links)
|
|
// subcategory-page?category_id=… → the sub-categories (category-product-overview links)
|
|
// category-product-overview?… → every product tile of the sub-category, in one response
|
|
// One request at a time (~1/s); "Custom…" theme categories (glutenvrij, lactosevrij, …) are
|
|
// crawled last so a product keeps its regular category. Tiles carry no EAN (nor does the product
|
|
// page), so Picnic is matched on name/brand/size.
|
|
import { HttpError, type PoliteFetcher } from './http';
|
|
import {
|
|
PICNIC_API,
|
|
PicnicAuthError,
|
|
checkAuth,
|
|
loadState,
|
|
login,
|
|
picnicHeaders,
|
|
type PicnicState
|
|
} from './picnic-auth';
|
|
import { decodeRscPage, parseRscPayload, walkObjects } from './picnic-rsc';
|
|
import type { RawItem, Source } from './sources';
|
|
|
|
const IMAGES = 'https://storefront-prod.nl.picnicinternational.com/static/images';
|
|
const PRODUCT_LINK = 'https://picnic.app/nl/link/store/page;id=product-details-page-root,id=';
|
|
|
|
type Cat = { category: string; isFood?: boolean };
|
|
const c = (category: string, isFood?: boolean): Cat => ({ category, isFood });
|
|
/** Picnic top category id → shared category (names as of 2026-10). */
|
|
export const TOP: Record<string, Cat> = {
|
|
'21724': c('Groente, fruit & aardappelen'), // Fruit
|
|
'21725': c('Groente, fruit & aardappelen'), // Aardappelen & groente
|
|
'21726': c('Vlees, vis & vega'), // Vlees & vis
|
|
'32811': c('Vlees, vis & vega'), // Vega & plantaardig
|
|
'21727': c('Maaltijden & gemak'),
|
|
'21728': c('Brood & bakkerij'), // Bakkerij
|
|
'21730': c('Zuivel & eieren'),
|
|
'26714': c('Vleeswaren, kaas & tapas'), // Kaas
|
|
'21731': c('Vleeswaren, kaas & tapas'), // Vleeswaren & broodsalades
|
|
'36362': c('Vleeswaren, kaas & tapas'), // Borrel, tapas & spreads
|
|
'21732': c('Ontbijt & beleg'), // Ontbijt & zoet beleg
|
|
'21733': c('Pasta, rijst & wereldkeuken'),
|
|
'21734': c('Soepen, sauzen & conserven'), // Voorraadkast
|
|
'21735': c('Snoep, koek & chips'), // Koek, snoep & snacks
|
|
'21740': c('Diepvries'),
|
|
'21738': c('Koffie & thee'),
|
|
'21736': c('Frisdrank & sappen'), // Drinken
|
|
'33825': c('Bier & wijn'), // Bier & aperitieven
|
|
'33826': c('Bier & wijn'), // Wijn & bubbels
|
|
'38115': c('Baby & kind'),
|
|
'37727': c('Drogisterij', false), // Gezichtsverzorging
|
|
'38177': c('Drogisterij', false), // Gezondheid & vitamines
|
|
'37655': c('Drogisterij', false), // Haarverzorging
|
|
'37784': c('Drogisterij', false), // Lichaamsverzorging
|
|
'37627': c('Drogisterij', false), // Mondverzorging
|
|
'37611': c('Drogisterij', false), // Zonbescherming
|
|
'21741': c('Huishouden & dieren', false), // Huishouden
|
|
'21744': c('Huishouden & dieren', false), // Dier
|
|
'34134': c('Overig', false), // Koken, tafelen & vrije tijd
|
|
'38263': c('Overig') // Sport (sports nutrition)
|
|
};
|
|
/** Sub-category name overrides inside a top category ("top id/regex"). */
|
|
const SUB: [string, RegExp, Cat][] = [
|
|
['21734', /ontbijt|beleg|broodsmeer/i, c('Ontbijt & beleg')],
|
|
['21734', /bak(ken|producten)|meel|suiker/i, c('Ontbijt & beleg')],
|
|
['21734', /pasta|rijst|noedel|wereld/i, c('Pasta, rijst & wereldkeuken')],
|
|
['21738', /sap|frisdrank/i, c('Frisdrank & sappen')],
|
|
['21736', /koffie|thee/i, c('Koffie & thee')],
|
|
['21736', /bier|wijn/i, c('Bier & wijn')],
|
|
['38115', /verzorging|luiers|billendoek/i, c('Baby & kind', false)]
|
|
];
|
|
|
|
export function picnicCategory(topId: string, subName: string): Cat {
|
|
for (const [t, re, cat] of SUB) if (t === topId && re.test(subName)) return cat;
|
|
return TOP[topId] ?? c('Overig');
|
|
}
|
|
|
|
export type PicnicTile = {
|
|
id?: string;
|
|
type?: string;
|
|
title?: { title?: string; prefix?: string };
|
|
subtitle?: { text?: string };
|
|
images?: { primaryImage?: { imageId?: string }; sellingUnitImage?: { imageId?: string } };
|
|
price?: {
|
|
priceValue?: number;
|
|
referencePriceValue?: number;
|
|
basePrice?: number;
|
|
priceRanges?: { fromQuantity?: number; price?: number }[];
|
|
};
|
|
discountInfo?: { label?: string } | null;
|
|
labels?: Record<string, { type?: string; labelText?: string }[]>;
|
|
suffixes?: { type?: string; text?: string }[];
|
|
unavailability?: unknown;
|
|
};
|
|
|
|
const cents = (euros: unknown) =>
|
|
typeof euros === 'number' && Number.isFinite(euros) && euros > 0 ? Math.round(euros * 100) : null;
|
|
|
|
export function picnicItem(
|
|
t: PicnicTile,
|
|
cat: Cat,
|
|
path: string,
|
|
unavailable = false
|
|
): RawItem | null {
|
|
const title = t.title?.title?.trim();
|
|
if (!t.id || !title) return null;
|
|
const name = `${t.title?.prefix?.trim() ? `${t.title.prefix.trim()} ` : ''}${title}`;
|
|
const tier1 = t.price?.priceRanges?.find((r) => r.fromQuantity === 1)?.price;
|
|
const now =
|
|
cents(t.price?.priceValue) ??
|
|
(typeof tier1 === 'number' && tier1 > 0 ? Math.round(tier1) : null);
|
|
const ref = cents(t.price?.referencePriceValue);
|
|
const promo = now !== null && ref !== null && now < ref ? now : null;
|
|
const promoText =
|
|
Object.values(t.labels ?? {})
|
|
.flat()
|
|
.find((l) => l?.type === 'PROMO' && l.labelText)?.labelText ?? t.discountInfo?.label;
|
|
const texts = (t.suffixes ?? []).filter((s) => s.type === 'TEXT' && s.text).map((s) => s.text!);
|
|
const imageId = t.images?.primaryImage?.imageId ?? t.images?.sellingUnitImage?.imageId;
|
|
return {
|
|
storeProductId: t.id,
|
|
name,
|
|
brand: t.subtitle?.text?.trim() || null,
|
|
sizeText: texts[0]?.trim() || null,
|
|
priceCents: promo ? ref : now,
|
|
promoPriceCents: promo,
|
|
// "Family" is Picnic's member discount (Picnic Family), not a store-wide promo.
|
|
promoLabel: promo
|
|
? promoText?.trim() === 'Family'
|
|
? 'Picnic Family-korting'
|
|
: promoText?.trim() || 'Actie'
|
|
: null,
|
|
storeUnitPrice: null,
|
|
storeUnitBasisCents: null,
|
|
category: cat.category,
|
|
categoryPath: path,
|
|
gtin: null,
|
|
imageUrl: imageId ? `${IMAGES}/${imageId}/medium.png` : null,
|
|
url: `${PRODUCT_LINK}${t.id}`,
|
|
available: !unavailable && !t.unavailability && now !== null,
|
|
...(cat.isFood !== undefined ? { isFood: cat.isFood } : {})
|
|
};
|
|
}
|
|
|
|
type Link = { title: string; page: string; categoryId: string };
|
|
/** Category links (row items) of a decoded page: deeplinks like `…;id=subcategory-page,category_id=21724`. */
|
|
export function pageLinks(decoded: unknown): Link[] {
|
|
const out = new Map<string, Link>();
|
|
for (const o of walkObjects(decoded)) {
|
|
if (o.type !== 'row-item' || typeof o.deeplink !== 'string' || typeof o.title !== 'string')
|
|
continue;
|
|
const [page, ...params] = (o.deeplink.split(';id=')[1] ?? '').split(',');
|
|
const id = params.find((p) => p.startsWith('category_id='))?.slice('category_id='.length);
|
|
if (!id) continue;
|
|
const categoryId = decodeURIComponent(id);
|
|
out.set(`${page}:${categoryId}`, { title: o.title.trim(), page, categoryId });
|
|
}
|
|
return [...out.values()];
|
|
}
|
|
|
|
/** Product tiles of a decoded overview page, with the ids Picnic lists as not available. */
|
|
export function pageTiles(decoded: unknown): {
|
|
tiles: PicnicTile[];
|
|
unavailable: Set<string>;
|
|
relevant: number | null;
|
|
} {
|
|
const tiles = new Map<string, PicnicTile>();
|
|
const unavailable = new Set<string>();
|
|
let relevant: number | null = null;
|
|
for (const o of walkObjects(decoded)) {
|
|
if (o.type === 'sellable' && typeof o.id === 'string' && !tiles.has(o.id))
|
|
tiles.set(o.id, o as PicnicTile);
|
|
const facets = o.entityIdsByFacetId as Record<string, unknown> | undefined;
|
|
if (facets && typeof facets === 'object') {
|
|
for (const [k, ids] of Object.entries(facets)) {
|
|
if (!Array.isArray(ids)) continue;
|
|
if (k === 'relevancy.relevant') relevant = Math.max(relevant ?? 0, ids.length);
|
|
if (k.startsWith('availability.') && k !== 'availability.AVAILABLE')
|
|
for (const id of ids) if (typeof id === 'string') unavailable.add(id);
|
|
}
|
|
}
|
|
}
|
|
return { tiles: [...tiles.values()], unavailable, relevant };
|
|
}
|
|
|
|
const isTheme = (id: string) => /^Custom/i.test(id);
|
|
|
|
/** A logged-in session: the saved one, else one password login (never a 2FA prompt unattended). */
|
|
async function session(log: (m: string) => void): Promise<PicnicState> {
|
|
const s = loadState();
|
|
if (await checkAuth(s)) return s;
|
|
log(' picnic: saved session missing or expired; logging in');
|
|
if ((await login(s)) === '2fa-required')
|
|
throw new PicnicAuthError(
|
|
'picnic: login needs a 2FA code; run `bun scripts/picnic-login.ts` on stan-ovh (then `verify <code>`)'
|
|
);
|
|
return s;
|
|
}
|
|
|
|
export const picnicSource: Source = {
|
|
slug: 'picnic',
|
|
minDelayMs: 1200,
|
|
async fetchAll(f: PoliteFetcher, log) {
|
|
const s = await session(log);
|
|
const only = process.env.PICNIC_ONLY_CATEGORY; // test runs: one top category id
|
|
const get = async (page: string, key: string) => {
|
|
try {
|
|
const text = await f.text(
|
|
`${PICNIC_API}/pages/${page}`,
|
|
{ headers: picnicHeaders(s) },
|
|
key.replace(/[^\w.-]+/g, '_')
|
|
);
|
|
return decodeRscPage(parseRscPayload(text));
|
|
} catch (e) {
|
|
if (e instanceof HttpError && (e.status === 401 || e.status === 403))
|
|
throw new PicnicAuthError(`picnic: session rejected (HTTP ${e.status})`);
|
|
throw e;
|
|
}
|
|
};
|
|
const tops = pageLinks(await get('category-tree-root', 'tree')).filter(
|
|
(l) => l.page === 'subcategory-page' && (!only || l.categoryId === only)
|
|
);
|
|
if (!only && tops.length < 20) throw new Error(`picnic: only ${tops.length} top categories`);
|
|
for (const t of tops)
|
|
if (!TOP[t.categoryId] && !isTheme(t.categoryId))
|
|
log(` picnic: unmapped top category ${t.categoryId} "${t.title}" → Overig`);
|
|
tops.sort((a, b) => Number(isTheme(a.categoryId)) - Number(isTheme(b.categoryId)));
|
|
|
|
// Sub-categories of every top category; theme ones (Custom…) go last overall.
|
|
const subs: (Link & { top: Link })[] = [];
|
|
for (const top of tops) {
|
|
const links = pageLinks(
|
|
await get(
|
|
`subcategory-page?category_id=${encodeURIComponent(top.categoryId)}`,
|
|
`sub-${top.categoryId}`
|
|
)
|
|
);
|
|
for (const l of links) if (l.page === 'category-product-overview') subs.push({ ...l, top });
|
|
}
|
|
subs.sort(
|
|
(a, b) =>
|
|
Number(isTheme(a.top.categoryId) || isTheme(a.categoryId)) -
|
|
Number(isTheme(b.top.categoryId) || isTheme(b.categoryId))
|
|
);
|
|
log(` picnic: ${tops.length} categories, ${subs.length} sub-categories`);
|
|
|
|
const items = new Map<string, RawItem>();
|
|
const seen = new Set<string>();
|
|
for (const sub of subs) {
|
|
if (seen.has(sub.categoryId)) continue;
|
|
seen.add(sub.categoryId);
|
|
const d = await get(
|
|
`category-product-overview?category_id=${encodeURIComponent(sub.categoryId)}`,
|
|
`ov-${sub.categoryId}`
|
|
);
|
|
const { tiles, unavailable, relevant } = pageTiles(d);
|
|
if (relevant !== null && relevant > tiles.length)
|
|
log(` picnic ${sub.top.title} / ${sub.title}: ${relevant} listed, ${tiles.length} tiles`);
|
|
const cat = picnicCategory(sub.top.categoryId, sub.title);
|
|
for (const t of tiles) {
|
|
const it = picnicItem(t, cat, `${sub.top.title} / ${sub.title}`, unavailable.has(t.id!));
|
|
if (it && !items.has(it.storeProductId)) items.set(it.storeProductId, it);
|
|
}
|
|
}
|
|
log(` picnic: ${items.size} products from ${seen.size} sub-categories`);
|
|
return [...items.values()];
|
|
}
|
|
};
|