groceries: Action importer (allowed category listings: price, unit price, EAN from image name; food/drinks, personal care, pets, household consumables) + store row

drizzle/20261025193533_groceries_store_action/migration.sql

@@ -0,0 +1 @@
+INSERT INTO "groceries"."stores" ("slug","name","website","color","sort_order") VALUES ('action','Action','https://www.action.com/nl-nl','#0058a3',110) ON CONFLICT ("slug") DO NOTHING;

src/lib/server/groceries/action.ts

@@ -0,0 +1,123 @@
+// Action (action.com/nl-nl) source. Product pages (/p/) are disallowed in robots.txt, but the
+// category listings are allowed and server-render 24 product cards per page (?page=N): id, name,
+// size, price and unit price, and the product image file is named "<id>_<EAN>-…", which gives
+// the barcode. Action has fixed prices (no promo prices on the cards). We crawl the grocery-ish
+// parts: food & drink (per sub-category, for our categories), personal care, pets, and the
+// consumable household sub-categories.
+import { FatalFetchError, type PoliteFetcher } from './http';
+import { normaliseGtin, storeUnit } from './normalize';
+import type { RawItem, Source } from './sources';
+
+const BASE = 'https://www.action.com';
+const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
+
+type Cat = { category: string; isFood: boolean };
+const food = (category: string): Cat => ({ category, isFood: true });
+const non = (category: string): Cat => ({ category, isFood: false });
+/** Listing path (under /nl-nl/c/) → shared category. Crawled in this order; first seen wins. */
+const LISTINGS: [string, Cat][] = [
+	['eten--drinken/koek--bakproducten', food('Snoep, koek & chips')],
+	['eten--drinken/drop--snoep', food('Snoep, koek & chips')],
+	['eten--drinken/chocolade', food('Snoep, koek & chips')],
+	['eten--drinken/noten--snacks', food('Snoep, koek & chips')],
+	['eten--drinken/chips', food('Snoep, koek & chips')],
+	['eten--drinken/drinken', food('Frisdrank & sappen')],
+	['eten--drinken/voeding', food('Overig')],
+	['eten--drinken', food('Overig')],
+	['persoonlijke-verzorging/babyverzorging', non('Baby & kind')],
+	['persoonlijke-verzorging', non('Drogisterij')],
+	['huisdieren', non('Huishouden & dieren')],
+	['huishouden/afwas---wasmiddelen', non('Huishouden & dieren')],
+	['huishouden/schoonmaakmiddelen', non('Huishouden & dieren')],
+	['huishouden/toilet---keukenpapier', non('Huishouden & dieren')],
+	['keuken/bakpapier-aluminium-huishoudfolie', non('Huishouden & dieren')]
+];
+
+const decode = (s: string) =>
+	s
+		.replace(/&amp;/g, '&')
+		.replace(/&#x27;|&#0?39;|&apos;/g, "'")
+		.replace(/&quot;/g, '"')
+		.replace(/&lt;/g, '<')
+		.replace(/&gt;/g, '>')
+		.replace(/\s+/g, ' ')
+		.trim();
+/** "€ 1,99" → 199 */
+const euroCents = (s: string | undefined) => {
+	const m = s?.match(/€\s*(\d+)(?:[.,](\d{1,2}))?/);
+	return m ? Number(m[1]) * 100 + Number((m[2] ?? '0').padEnd(2, '0')) : null;
+};
+const UNIT: Record<string, string> = { kg: 'kg', l: 'l', st: 'stuk', stuk: 'stuk' };
+
+/** Parse a listing page's product cards (exported for tests). */
+export function parseActionListing(html: string, cat: Cat, path: string): RawItem[] {
+	const out: RawItem[] = [];
+	const parts = html.split('data-testid="product-card"').slice(1);
+	for (const card of parts) {
+		const href = card.match(/href="(\/nl-nl\/p\/(\d+)\/[^"]*)"/);
+		const name = card.match(/data-testid="product-card-title"[^>]*>([^<]*)</)?.[1];
+		if (!href || !name) continue;
+		const id = href[2];
+		const aria = card.match(/aria-label="(€[^"]*)"/)?.[1]?.replace(/\u00a0/g, ' ');
+		const price = euroCents(aria);
+		const per = card
+			.match(/data-testid="product-card-price-description"[^>]*>([^<]*)</)?.[1]
+			?.replace(/\u00a0/g, ' ');
+		const pm = per?.match(/(€\s*\d+(?:[.,]\d{1,2})?)\s*\/\s*([a-z]+)/i);
+		const unit = pm ? storeUnit(UNIT[pm[2].toLowerCase()] ?? pm[2]) : null;
+		const unitCents = pm ? euroCents(pm[1]) : null;
+		const img = card.match(/<img[^>]*\ssrc="([^"]+)"/)?.[1] ?? null;
+		const ean = img?.match(new RegExp(`/${id}_(\\d{8,14})-`))?.[1];
+		out.push({
+			storeProductId: id,
+			name: decode(name),
+			brand: null,
+			sizeText:
+				decode(card.match(/data-testid="product-card-description"[^>]*>([^<]*)</)?.[1] ?? '') ||
+				null,
+			priceCents: price,
+			promoPriceCents: null,
+			promoLabel: null,
+			storeUnitPrice: unit && unitCents ? { cents: unitCents, unit } : null,
+			storeUnitBasisCents: null,
+			category: cat.category,
+			categoryPath: path.replace(/--/g, ' & ').replace(/\//g, ' > ').replace(/-/g, ' '),
+			gtin: ean ? normaliseGtin(ean) : null,
+			imageUrl: img ? decode(img) : null,
+			url: `${BASE}${decode(href[1])}`,
+			available: price !== null,
+			isFood: cat.isFood
+		});
+	}
+	return out;
+}
+
+export const actionSource: Source = {
+	slug: 'action',
+	minDelayMs: 3000,
+	async fetchAll(f: PoliteFetcher, log) {
+		const items = new Map<string, RawItem>();
+		const get = (path: string, page: number) =>
+			f.text(
+				`${BASE}/nl-nl/c/${path}/${page > 1 ? `?page=${page}` : ''}`,
+				{ headers: { 'User-Agent': UA, Accept: 'text/html' } },
+				`${path.replace(/\//g, '__')}-p${page}`
+			);
+		for (const [path, cat] of LISTINGS) {
+			const before = items.size;
+			const first = await get(path, 1);
+			if (!first.includes('data-testid="product-card"'))
+				throw new FatalFetchError(`action: no product cards on ${path} (layout or bot wall?)`);
+			const last = Math.max(1, ...[...first.matchAll(/\?page=(\d+)/g)].map((m) => Number(m[1])));
+			for (let page = 1; page <= Math.min(last, 200); page++) {
+				const html = page === 1 ? first : await get(path, page);
+				const cards = parseActionListing(html, cat, path);
+				if (!cards.length) break;
+				for (const it of cards) if (!items.has(it.storeProductId)) items.set(it.storeProductId, it);
+			}
+			log(`  action ${path}: ${last} pages, +${items.size - before} (${items.size} total)`);
+		}
+		if (items.size < 300) throw new Error(`action: only ${items.size} products, layout changed?`);
+		return [...items.values()];
+	}
+};

src/lib/server/groceries/sources.ts

@@ -5,6 +5,7 @@
 import type { GroceryUnit } from '../db/schema/groceries';
 import { HttpError, PoliteFetcher, USER_AGENT } from './http';
 import { normaliseGtin, parseAhUnitPrice, storeUnit } from './normalize';
+import { actionSource } from './action';
 import { aldiSource } from './aldi';
 import { dirkSource } from './dirk';
 import { hoogvlietSource } from './hoogvliet';
@@ -393,5 +394,6 @@ export const SOURCES: Record<string, Source> = {
 	dirk: dirkSource,
 	hoogvliet: hoogvlietSource,
 	vomar: vomarSource,
-	aldi: aldiSource
+	aldi: aldiSource,
+	action: actionSource
 };

src/lib/server/groceries/stores2.test.ts

@@ -1,3 +1,4 @@
+import { parseActionListing } from './action';
 import { describe, expect, it } from 'vitest';
 import { dirkItem } from './dirk';
 import { hoogvlietCategory, hoogvlietItem } from './hoogvliet';
@@ -66,3 +67,25 @@ describe('hoogvliet', () => {
 		).toBeNull();
 	});
 });
+
+describe('action', () => {
+	const card = `<div data-testid="product-card" data-content-king="product-card"><a class="g" href="/nl-nl/p/3223916/natural-happiness-pittenmix/" data-testid="product-card-link"><span><img data-testid="product-card-image" alt="" srcSet="x 128w" src="https://asset.action.com/image/upload/t_digital_product_image/w_1920/3223916_8711299010862-111_01_h7ywzh.webp"/></span><span><span data-testid="product-card-title">Natural Happiness pittenmix &amp; zo</span><span class="x" data-testid="product-card-description">300 gram</span></span><span aria-label="€\u00a01,99 €\u00a06,63/kg" class="flex"><span data-testid="product-card-price-description" class="m">€\u00a06,63/kg</span></span></a></div>`;
+	it('parses a listing card', () => {
+		const [it] = parseActionListing(
+			card + card,
+			{ category: 'Snoep, koek & chips', isFood: true },
+			'eten--drinken/noten--snacks'
+		);
+		expect(it).toMatchObject({
+			storeProductId: '3223916',
+			name: 'Natural Happiness pittenmix & zo',
+			sizeText: '300 gram',
+			priceCents: 199,
+			storeUnitPrice: { cents: 663, unit: 'kg' },
+			gtin: '8711299010862',
+			url: 'https://www.action.com/nl-nl/p/3223916/natural-happiness-pittenmix/',
+			categoryPath: 'eten & drinken > noten & snacks',
+			isFood: true
+		});
+	});
+});