groceries: ALDI importer (category pages SSR Algolia hits, offers; no EANs) + store row

drizzle/20261025190428_groceries_store_aldi/migration.sql

@@ -0,0 +1,4 @@
+-- ALDI (aldi.nl).
+INSERT INTO "groceries"."stores" ("slug", "name", "website", "color", "sort_order") VALUES
+	('aldi', 'ALDI', 'https://www.aldi.nl', '#00005f', 100)
+ON CONFLICT ("slug") DO NOTHING;

src/lib/server/groceries/aldi.ts

@@ -0,0 +1,180 @@
+// ALDI (aldi.nl) catalogue source: each category page server-renders its whole Algolia result
+// (up to 1000 hits) in __NEXT_DATA__, so one polite page fetch per sub-category gives the range
+// with current prices and weekly offers (strike price). ALDI publishes no EANs. Items that are
+// "not available" (seasonal, sold out) have no price and are kept as unavailable.
+import { FatalFetchError, type PoliteFetcher } from './http';
+import type { RawItem, Source } from './sources';
+
+const BASE = 'https://www.aldi.nl';
+const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
+const INDEX = 'an_prd_nl_nl_products2';
+
+type Cat = { category: string; isFood?: boolean };
+const c = (category: string, isFood?: boolean): Cat => ({ category, isFood });
+/** top slug → shared category; sub-slug overrides as "top/sub". */
+const TOP: Record<string, Cat> = {
+	'aardappels-groente-fruit': c('Groente, fruit & aardappelen'),
+	'baby-persoonlijke-verzorging': c('Drogisterij', false),
+	'bier-en-likeuren': c('Bier & wijn'),
+	wijn: c('Bier & wijn'),
+	'brood-bakkerij-bakken': c('Brood & bakkerij'),
+	'chips--noten': c('Snoep, koek & chips'),
+	'snoep-koeken-chocolade': c('Snoep, koek & chips'),
+	diepvries: c('Diepvries'),
+	huisdieren: c('Huishouden & dieren', false),
+	huishouden: c('Huishouden & dieren', false),
+	'kaas-vleeswaren': c('Vleeswaren, kaas & tapas'),
+	tapas: c('Vleeswaren, kaas & tapas'),
+	'maaltijden-salades': c('Maaltijden & gemak'),
+	'ontbijtgranen-broodbeleg-en-tussendoortjes': c('Ontbijt & beleg'),
+	'pasta-rijst-internationale-keuken': c('Pasta, rijst & wereldkeuken'),
+	'sappen-frisdrank': c('Frisdrank & sappen'),
+	'soepen-sauzen-smaakmakers-conserven': c('Soepen, sauzen & conserven'),
+	'thee-koffie': c('Koffie & thee'),
+	'vlees-vis-vega': c('Vlees, vis & vega'),
+	'zuivel-eieren-boter': c('Zuivel & eieren'),
+	// themes: crawled last, only add items not seen in a regular category
+	feestassortiment: c('Overig'),
+	'speciaal-assortiment': c('Overig'),
+	winnaarsproducten: c('Overig'),
+	sintassortiment: c('Overig', false)
+};
+const SUB: Record<string, Cat> = {
+	'baby-persoonlijke-verzorging/baby': c('Baby & kind'),
+	'brood-bakkerij-bakken/beschuit-toast-crackers': c('Ontbijt & beleg'),
+	'brood-bakkerij-bakken/zelf-bakken': c('Ontbijt & beleg'),
+	'ontbijtgranen-broodbeleg-en-tussendoortjes/tussendoortjes': c('Snoep, koek & chips'),
+	'zuivel-eieren-boter/houdbare-zuviel': c('Zuivel & eieren'),
+	'speciaal-assortiment/cadeaukaarten': c('Overig', false),
+	'winnaarsproducten/verzorging': c('Drogisterij', false),
+	'sintassortiment/snoepgoed': c('Snoep, koek & chips')
+};
+const THEMES = new Set([
+	'feestassortiment',
+	'speciaal-assortiment',
+	'winnaarsproducten',
+	'sintassortiment'
+]);
+
+type Price = {
+	priceValue?: number;
+	strikePrice?: { strikePriceValue?: number };
+	basePrice?: { basePriceValue?: number; basePriceScale?: string }[];
+	priceTagLabels?: { promoText1?: string; promoText2?: string };
+	validUntil?: number;
+	validUntilLocalDate?: string;
+};
+export type AldiHit = {
+	objectID: string;
+	name: string;
+	brandName?: string;
+	salesUnit?: string;
+	isAvailable?: boolean;
+	productSlug?: string;
+	currentPrice?: Price;
+	promotionPrices?: Price[];
+	assets?: { type: string; url: string }[];
+	hierarchicalCategories?: { lvl0?: string[]; lvl1?: string[] };
+};
+
+const cents = (n: unknown) =>
+	typeof n === 'number' && Number.isFinite(n) && n > 0 ? Math.round(n * 100) : null;
+
+export function aldiItem(h: AldiHit, page: string): RawItem | null {
+	const name = h.name?.trim();
+	if (!h.objectID || !name) return null;
+	const cat = SUB[page] ?? TOP[page.split('/')[0]] ?? c('Overig');
+	const p = h.currentPrice;
+	const now = cents(p?.priceValue);
+	const strike = cents(p?.strikePrice?.strikePriceValue);
+	const promo = now && strike && now < strike ? now : null;
+	const until =
+		h.promotionPrices?.find((x) => cents(x.priceValue) === promo)?.validUntilLocalDate ??
+		(p?.validUntil ? new Date(p.validUntil * 1000).toISOString().slice(0, 10) : undefined);
+	const tag = [p?.priceTagLabels?.promoText1, p?.priceTagLabels?.promoText2]
+		.filter(Boolean)
+		.join(' ');
+	const brand = h.brandName?.trim();
+	const fullName =
+		brand && !name.toLowerCase().startsWith(brand.toLowerCase())
+			? `${brand.charAt(0) + brand.slice(1).toLowerCase()} ${name}`
+			: name;
+	return {
+		storeProductId: h.objectID,
+		name: fullName,
+		brand: brand || null,
+		sizeText: h.salesUnit?.replace(/\s+\./g, '.').trim() || null,
+		priceCents: promo ? strike : now,
+		promoPriceCents: promo,
+		promoLabel: promo
+			? [tag || 'Actie', until && `t/m ${until.slice(8, 10)}-${until.slice(5, 7)}`]
+					.filter(Boolean)
+					.join(' · ')
+			: null,
+		storeUnitPrice: null,
+		storeUnitBasisCents: null,
+		category: cat.category,
+		categoryPath:
+			h.hierarchicalCategories?.lvl1?.[0] ?? h.hierarchicalCategories?.lvl0?.[0] ?? null,
+		gtin: null,
+		imageUrl: h.assets?.find((a) => a.type === 'primary')?.url ?? h.assets?.[0]?.url ?? null,
+		url: h.productSlug ? `${BASE}/product/${h.productSlug}.html` : null,
+		available: h.isAvailable !== false && now !== null,
+		...(cat.isFood !== undefined ? { isFood: cat.isFood } : {})
+	};
+}
+
+function hitsOf(html: string): { hits: AldiHit[]; nbHits: number } | null {
+	const m = html.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/);
+	if (!m) throw new FatalFetchError('aldi: no __NEXT_DATA__ (bot wall?)');
+	const d = JSON.parse(m[1]) as {
+		props?: {
+			pageProps?: {
+				algoliaState?: {
+					initialResults?: Record<string, { results?: { hits: AldiHit[]; nbHits: number }[] }>;
+				};
+			};
+		};
+	};
+	return d.props?.pageProps?.algoliaState?.initialResults?.[INDEX]?.results?.[0] ?? null;
+}
+
+export const aldiSource: Source = {
+	slug: 'aldi',
+	minDelayMs: 2500,
+	async fetchAll(f: PoliteFetcher, log) {
+		const index = await f.text(
+			`${BASE}/producten.html`,
+			{ headers: { 'User-Agent': UA } },
+			'index'
+		);
+		const pages = [
+			...new Set(
+				[
+					...index.matchAll(/https:\/\/www\.aldi\.nl\/producten\/([a-z0-9-]+\/[a-z0-9-]+)\.html/g)
+				].map((m) => m[1])
+			)
+		]
+			.filter((p) => TOP[p.split('/')[0]])
+			.sort((a, b) => Number(THEMES.has(a.split('/')[0])) - Number(THEMES.has(b.split('/')[0])));
+		if (pages.length < 50) throw new Error(`aldi: only ${pages.length} category pages found`);
+		const items = new Map<string, RawItem>();
+		for (const page of pages) {
+			const html = await f.text(
+				`${BASE}/producten/${page}.html`,
+				{ headers: { 'User-Agent': UA } },
+				page.replace('/', '__')
+			);
+			const r = hitsOf(html);
+			if (!r) continue;
+			if (r.nbHits > r.hits.length)
+				log(`  aldi ${page}: ${r.nbHits} hits, only ${r.hits.length} on the page`);
+			for (const h of r.hits) {
+				const it = aldiItem(h, page);
+				if (it && !items.has(it.storeProductId)) items.set(it.storeProductId, it);
+			}
+		}
+		log(`  aldi: ${items.size} products from ${pages.length} pages`);
+		return [...items.values()];
+	}
+};

src/lib/server/groceries/sources.ts

@@ -5,6 +5,7 @@
 import type { GroceryUnit } from '../db/schema/groceries';
 import { HttpError, PoliteFetcher, USER_AGENT } from './http';
 import { normaliseGtin, parseAhUnitPrice, storeUnit } from './normalize';
+import { aldiSource } from './aldi';
 import { dirkSource } from './dirk';
 import { hoogvlietSource } from './hoogvliet';
 import { lidlSource } from './lidl';
@@ -391,5 +392,6 @@ export const SOURCES: Record<string, Source> = {
 	lidl: lidlSource,
 	dirk: dirkSource,
 	hoogvliet: hoogvlietSource,
-	vomar: vomarSource
+	vomar: vomarSource,
+	aldi: aldiSource
 };