website/src/lib/server/groceries/action.ts

// Action (action.com/nl-nl) source. Product pages (/p/) are disallowed in robots.txt, but the
// category listings are allowed and server-render 24 product cards per page (?page=N): id, name,
// size, price and unit price, and the product image file is named "<id>_<EAN>-…", which gives
// the barcode. Action has fixed prices (no promo prices on the cards). We crawl the grocery-ish
// parts: food & drink (per sub-category, for our categories), personal care, pets, and the
// consumable household sub-categories.
import { FatalFetchError, type PoliteFetcher } from './http';
import { normaliseGtin, storeUnit } from './normalize';
import type { RawItem, Source } from './sources';

const BASE = 'https://www.action.com';
const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';

type Cat = { category: string; isFood: boolean };
const food = (category: string): Cat => ({ category, isFood: true });
const non = (category: string): Cat => ({ category, isFood: false });
/** Listing path (under /nl-nl/c/) → shared category. Crawled in this order; first seen wins. */
const LISTINGS: [string, Cat][] = [
	['eten--drinken/koek--bakproducten', food('Snoep, koek & chips')],
	['eten--drinken/drop--snoep', food('Snoep, koek & chips')],
	['eten--drinken/chocolade', food('Snoep, koek & chips')],
	['eten--drinken/noten--snacks', food('Snoep, koek & chips')],
	['eten--drinken/chips', food('Snoep, koek & chips')],
	['eten--drinken/drinken', food('Frisdrank & sappen')],
	['eten--drinken/voeding', food('Overig')],
	['eten--drinken', food('Overig')],
	['persoonlijke-verzorging/babyverzorging', non('Baby & kind')],
	['persoonlijke-verzorging', non('Drogisterij')],
	['huisdieren', non('Huishouden & dieren')],
	['huishouden/afwas---wasmiddelen', non('Huishouden & dieren')],
	['huishouden/schoonmaakmiddelen', non('Huishouden & dieren')],
	['huishouden/toilet---keukenpapier', non('Huishouden & dieren')],
	['keuken/bakpapier-aluminium-huishoudfolie', non('Huishouden & dieren')]
];

const decode = (s: string) =>
	s
		.replace(/&amp;/g, '&')
		.replace(/&#x27;|&#0?39;|&apos;/g, "'")
		.replace(/&quot;/g, '"')
		.replace(/&lt;/g, '<')
		.replace(/&gt;/g, '>')
		.replace(/\s+/g, ' ')
		.trim();
/** "€ 1,99" → 199 */
const euroCents = (s: string | undefined) => {
	const m = s?.match(/€\s*(\d+)(?:[.,](\d{1,2}))?/);
	return m ? Number(m[1]) * 100 + Number((m[2] ?? '0').padEnd(2, '0')) : null;
};
const UNIT: Record<string, string> = { kg: 'kg', l: 'l', st: 'stuk', stuk: 'stuk' };

/** Parse a listing page's product cards (exported for tests). */
export function parseActionListing(html: string, cat: Cat, path: string): RawItem[] {
	const out: RawItem[] = [];
	const parts = html.split('data-testid="product-card"').slice(1);
	for (const card of parts) {
		const href = card.match(/href="(\/nl-nl\/p\/(\d+)\/[^"]*)"/);
		const name = card.match(/data-testid="product-card-title"[^>]*>([^<]*)</)?.[1];
		if (!href || !name) continue;
		const id = href[2];
		const aria = card.match(/aria-label="(€[^"]*)"/)?.[1]?.replace(/\u00a0/g, ' ');
		const price = euroCents(aria);
		const per = card
			.match(/data-testid="product-card-price-description"[^>]*>([^<]*)</)?.[1]
			?.replace(/\u00a0/g, ' ');
		const pm = per?.match(/(€\s*\d+(?:[.,]\d{1,2})?)\s*\/\s*([a-z]+)/i);
		const unit = pm ? storeUnit(UNIT[pm[2].toLowerCase()] ?? pm[2]) : null;
		const unitCents = pm ? euroCents(pm[1]) : null;
		const img = card.match(/<img[^>]*\ssrc="([^"]+)"/)?.[1] ?? null;
		const ean = img?.match(new RegExp(`/${id}_(\\d{8,14})-`))?.[1];
		out.push({
			storeProductId: id,
			name: decode(name),
			brand: null,
			sizeText:
				decode(card.match(/data-testid="product-card-description"[^>]*>([^<]*)</)?.[1] ?? '') ||
				null,
			priceCents: price,
			promoPriceCents: null,
			promoLabel: null,
			storeUnitPrice: unit && unitCents ? { cents: unitCents, unit } : null,
			storeUnitBasisCents: null,
			category: cat.category,
			categoryPath: path.replace(/--/g, ' & ').replace(/\//g, ' > ').replace(/-/g, ' '),
			gtin: ean ? normaliseGtin(ean) : null,
			imageUrl: img ? decode(img) : null,
			url: `${BASE}${decode(href[1])}`,
			available: price !== null,
			isFood: cat.isFood
		});
	}
	return out;
}

export const actionSource: Source = {
	slug: 'action',
	minDelayMs: 3000,
	async fetchAll(f: PoliteFetcher, log) {
		const items = new Map<string, RawItem>();
		const get = (path: string, page: number) =>
			f.text(
				`${BASE}/nl-nl/c/${path}/${page > 1 ? `?page=${page}` : ''}`,
				{ headers: { 'User-Agent': UA, Accept: 'text/html' } },
				`${path.replace(/\//g, '__')}-p${page}`
			);
		for (const [path, cat] of LISTINGS) {
			const before = items.size;
			const first = await get(path, 1);
			if (!first.includes('data-testid="product-card"'))
				throw new FatalFetchError(`action: no product cards on ${path} (layout or bot wall?)`);
			const last = Math.max(1, ...[...first.matchAll(/\?page=(\d+)/g)].map((m) => Number(m[1])));
			for (let page = 1; page <= Math.min(last, 200); page++) {
				const html = page === 1 ? first : await get(path, page);
				const cards = parseActionListing(html, cat, path);
				if (!cards.length) break;
				for (const it of cards) if (!items.has(it.storeProductId)) items.set(it.storeProductId, it);
			}
			log(`  action ${path}: ${last} pages, +${items.size - before} (${items.size} total)`);
		}
		if (items.size < 300) throw new Error(`action: only ${items.size} products, layout changed?`);
		return [...items.values()];
	}
};