groceries: Hoogvliet importer (sitemap SKUs + storefront GraphQL, EANs, promo labels) + store row; Lidl retries transient CDN 401s

drizzle/20261025185055_groceries_store_hoogvliet/migration.sql

@@ -0,0 +1,4 @@
+-- Hoogvliet (hoogvliet.nl).
+INSERT INTO "groceries"."stores" ("slug", "name", "website", "color", "sort_order") VALUES
+	('hoogvliet', 'Hoogvliet', 'https://hoogvliet.nl', '#0060a9', 80)
+ON CONFLICT ("slug") DO NOTHING;

src/lib/server/groceries/hoogvliet.ts

@@ -0,0 +1,216 @@
+// Hoogvliet (hoogvliet.nl) catalogue source. The product SKUs come from the public sitemap
+// (sitemap-catalog.xml); prices, EANs and categories from the site's own storefront GraphQL
+// (hoogvliet.nl/api/graphql, Magento-style) in batches of SKUs. Weekly promotions are separate
+// "promotion group" products that list their member SKUs; they give the promo label.
+import type { PoliteFetcher } from './http';
+import { normaliseGtin } from './normalize';
+import type { RawItem, Source } from './sources';
+
+const BASE = 'https://hoogvliet.nl';
+const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
+const HEADERS = {
+	'User-Agent': UA,
+	'content-type': 'application/json',
+	'Accept-Language': 'nl',
+	xp_env: 'production'
+};
+const BATCH = 50;
+
+type Cat = { category: string; isFood?: boolean };
+const TOP: Record<string, Cat> = {
+	'aardappelen-groente-fruit': { category: 'Groente, fruit & aardappelen' },
+	'verse-maaltijden-salades': { category: 'Maaltijden & gemak' },
+	'vlees-vis-vegetarisch': { category: 'Vlees, vis & vega' },
+	'kaas-vleeswaren-tapas': { category: 'Vleeswaren, kaas & tapas' },
+	'zuivel-eieren-boter': { category: 'Zuivel & eieren' },
+	bakkerij: { category: 'Brood & bakkerij' },
+	'ontbijt-broodbeleg': { category: 'Ontbijt & beleg' },
+	'pasta-rijst-olien-wereldkeuken': { category: 'Pasta, rijst & wereldkeuken' },
+	'soepen-conserven-sauzen-kruiden': { category: 'Soepen, sauzen & conserven' },
+	'koek-snoep-chocolade': { category: 'Snoep, koek & chips' },
+	'chips-zoutjes-noten': { category: 'Snoep, koek & chips' },
+	diepvries: { category: 'Diepvries' },
+	'koffie-thee': { category: 'Koffie & thee' },
+	'frisdrank-sappen': { category: 'Frisdrank & sappen' },
+	'bier-wijn-sterke-drank': { category: 'Bier & wijn' },
+	'gezondheid-drogisterij': { category: 'Drogisterij', isFood: false },
+	'huishouden-non-food': { category: 'Huishouden & dieren', isFood: false },
+	huisdier: { category: 'Huishouden & dieren', isFood: false },
+	'baby-kind': { category: 'Baby & kind' }
+};
+const SUB: Record<string, Cat> = {
+	'baby-kind/verzorging-en-benodigdheden': { category: 'Baby & kind', isFood: false },
+	'baby-kind/luiers-en-doekjes': { category: 'Baby & kind', isFood: false },
+	'kaas-vleeswaren-tapas/salades': { category: 'Maaltijden & gemak' },
+	'bakkerij/bakproducten': { category: 'Ontbijt & beleg' }
+};
+// Theme/landing trees (BBQ, holidays, offers, diets) only count when nothing else matches.
+const THEMES = new Set([
+	'bewuste-voeding',
+	'alles-voor-de-barbecue',
+	'feestdagen-kerst-en-pasen',
+	'themashops',
+	'aanbiedingen'
+]);
+
+const UNITS: Record<string, string> = {
+	gram: 'g',
+	kilogram: 'kg',
+	milliliter: 'ml',
+	centiliter: 'cl',
+	liter: 'l',
+	stuk: 'stuks',
+	stuks: 'stuks',
+	meter: 'm'
+};
+
+export type HoogvlietProduct = {
+	sku: string;
+	name: string;
+	stock_status?: string | null;
+	url_key?: string | null;
+	metadata?: string | null;
+	price_range?: {
+		minimum_price?: {
+			regular_price?: { value?: number | null } | null;
+			final_price?: { value?: number | null } | null;
+		} | null;
+	} | null;
+	categories?: { level: number; url_path: string }[] | null;
+	small_image?: { url?: string | null } | null;
+};
+type Meta = Record<string, string | number | undefined>;
+
+const cents = (n: unknown) =>
+	typeof n === 'number' && Number.isFinite(n) && n > 0 ? Math.round(n * 100) : null;
+const meta = (p: HoogvlietProduct): Meta => {
+	try {
+		return p.metadata ? (JSON.parse(p.metadata) as Meta) : {};
+	} catch {
+		return {};
+	}
+};
+
+export function hoogvlietCategory(paths: string[]): Cat & { path: string | null } {
+	const top = paths.map((p) => p.split('/')[0]).find((t) => TOP[t]);
+	if (top) {
+		// the deepest path in that tree, for the breadcrumb
+		const path = paths
+			.filter((p) => p.split('/')[0] === top)
+			.sort((a, b) => b.split('/').length - a.split('/').length)[0];
+		return { ...(SUB[path.split('/').slice(0, 2).join('/')] ?? TOP[top]), path };
+	}
+	const theme = paths.find((p) => THEMES.has(p.split('/')[0]));
+	return { category: 'Overig', path: theme ?? paths[0] ?? null };
+}
+
+/** member SKU → promo label, from the promotion-group products of the current folder. */
+export function promoLabels(items: HoogvlietProduct[]): Map<string, string> {
+	const out = new Map<string, string>();
+	for (const p of items) {
+		const m = meta(p);
+		if (typeof m.promo_products !== 'string') continue;
+		let skus: string[];
+		try {
+			skus = JSON.parse(m.promo_products) as string[];
+		} catch {
+			continue;
+		}
+		const until =
+			typeof m.valid_to === 'string'
+				? m.valid_to.slice(0, 10).split('-').reverse().slice(0, 2).join('-')
+				: null;
+		const kind = String(m.korting_label ?? '').trim();
+		const label = [kind && !/^per\b/i.test(kind) ? kind : 'Actie', until && `t/m ${until}`]
+			.filter(Boolean)
+			.join(' · ');
+		for (const s of skus) if (!out.has(s)) out.set(s, label);
+	}
+	return out;
+}
+
+export const isPromotionGroup = (p: HoogvlietProduct) => typeof meta(p).promo_products === 'string';
+
+export function hoogvlietItem(p: HoogvlietProduct, labels: Map<string, string>): RawItem | null {
+	if (!p.sku || !p.name || isPromotionGroup(p)) return null;
+	const m = meta(p);
+	const brand = typeof m.merk === 'string' && m.merk.trim() ? m.merk.trim() : null;
+	const name =
+		brand && !p.name.toLowerCase().startsWith(brand.toLowerCase())
+			? `${brand} ${p.name.trim()}`
+			: p.name.trim();
+	const price = cents(p.price_range?.minimum_price?.regular_price?.value);
+	const final = cents(p.price_range?.minimum_price?.final_price?.value);
+	const promo = final && price && final < price ? final : null;
+	const unit = UNITS[String(m.eenheid ?? '').toLowerCase()];
+	const qty = Number(m.inhoud_ce);
+	const cat = hoogvlietCategory((p.categories ?? []).map((c) => c.url_path));
+	const path = (p.url_key ?? '').replace(/^product\//, '');
+	// 2xxxxxxxxxxxx = in-store codes (bakery, weighed goods): not a real EAN to group on.
+	const ean = m.ean === undefined || /^0*2\d{12}$/.test(String(m.ean)) ? null : String(m.ean);
+	return {
+		storeProductId: p.sku,
+		name,
+		brand,
+		sizeText: unit && qty > 0 ? `${qty} ${unit}` : null,
+		priceCents: price ?? final,
+		promoPriceCents: promo,
+		promoLabel: promo ? (labels.get(p.sku) ?? 'Actie') : null,
+		storeUnitPrice: null,
+		storeUnitBasisCents: null,
+		category: cat.category,
+		categoryPath: cat.path,
+		gtin: normaliseGtin(ean),
+		imageUrl: p.small_image?.url ?? null,
+		url: path ? `${BASE}/product/${path}` : null,
+		available: p.stock_status !== 'OUT_OF_STOCK' && (price ?? final) !== null,
+		...(cat.isFood !== undefined ? { isFood: cat.isFood } : {})
+	};
+}
+
+const FIELDS = `sku name stock_status url_key metadata
+	price_range { minimum_price { regular_price { value } final_price { value } } }
+	categories { level url_path } small_image { url }`;
+
+export const hoogvlietSource: Source = {
+	slug: 'hoogvliet',
+	minDelayMs: 1500,
+	async fetchAll(f: PoliteFetcher, log) {
+		const xml = await f.text(
+			`${BASE}/sitemap-catalog.xml`,
+			{ headers: { 'User-Agent': UA } },
+			'sitemap'
+		);
+		const skus = [
+			...new Set(
+				[...xml.matchAll(/<loc>https:\/\/hoogvliet\.nl\/[^<]*-(\d{9})<\/loc>/g)].map((m) => m[1])
+			)
+		];
+		log(`  hoogvliet: ${skus.length} SKUs in sitemap`);
+		if (skus.length < 1000) throw new Error(`hoogvliet: only ${skus.length} SKUs in the sitemap`);
+		const all: HoogvlietProduct[] = [];
+		for (let i = 0; i < skus.length; i += BATCH) {
+			const batch = skus.slice(i, i + BATCH);
+			const query = `{ products(filter: { sku: { in: ${JSON.stringify(batch)} } }, pageSize: ${BATCH}) { items { ${FIELDS} } } }`;
+			const r = await f.json<{
+				data?: { products?: { items?: HoogvlietProduct[] | null } | null };
+				errors?: { message: string }[];
+			}>(
+				`${BASE}/api/graphql`,
+				{ method: 'POST', headers: HEADERS, body: JSON.stringify({ query }) },
+				`b${i / BATCH}`
+			);
+			if (!r.data?.products)
+				throw new Error(`hoogvliet graphql: ${r.errors?.[0]?.message ?? 'no data'}`);
+			all.push(...(r.data.products.items ?? []));
+			if ((i / BATCH) % 40 === 0) log(`  hoogvliet: ${all.length}/${skus.length}`);
+		}
+		const labels = promoLabels(all);
+		const items = new Map<string, RawItem>();
+		for (const p of all) {
+			const it = hoogvlietItem(p, labels);
+			if (it) items.set(it.storeProductId, it);
+		}
+		return [...items.values()];
+	}
+};

src/lib/server/groceries/http.ts

@@ -15,6 +15,8 @@ export type FetcherOptions = {
 	/** First retry backoff; doubles each try (default 5 s). */
 	retryBaseMs?: number;
 	cacheDir?: string | null;
+	/** Extra 4xx statuses to retry (e.g. a CDN's transient 401/403). */
+	retryStatuses?: number[];
 	log?: (msg: string) => void;
 };
 
@@ -26,6 +28,7 @@ export class PoliteFetcher {
 	cacheHits = 0;
 	private last = 0;
 	private readonly minDelayMs: number;
+	private readonly retryStatuses: number[];
 	private readonly maxRetries: number;
 	private readonly retryBaseMs: number;
 	private readonly dir: string | null;
@@ -33,6 +36,7 @@ export class PoliteFetcher {
 
 	constructor(opts: FetcherOptions) {
 		this.minDelayMs = opts.minDelayMs ?? 2000;
+		this.retryStatuses = opts.retryStatuses ?? [];
 		this.maxRetries = opts.maxRetries ?? 5;
 		this.retryBaseMs = opts.retryBaseMs ?? 5000;
 		this.log = opts.log ?? (() => {});
@@ -78,7 +82,12 @@ export class PoliteFetcher {
 					return data;
 				}
 				problem = `HTTP ${res.status}`;
-				if (res.status !== 429 && res.status < 500 && res.status !== 408) {
+				if (
+					res.status !== 429 &&
+					res.status < 500 &&
+					res.status !== 408 &&
+					!this.retryStatuses.includes(res.status)
+				) {
 					throw new HttpError(
 						res.status,
 						`${problem} for ${url}: ${(await res.text()).slice(0, 200)}`

src/lib/server/groceries/lidl.ts

@@ -134,6 +134,8 @@ export function lidlItem(g: LidlGrid): RawItem | null {
 export const lidlSource: Source = {
 	slug: 'lidl',
 	minDelayMs: 1000,
+	// lidl.nl's CDN sometimes answers 401 for a moment
+	fetcher: { maxRetries: 4, retryBaseMs: 20_000, retryStatuses: [401, 403] },
 	async fetchAll(f: PoliteFetcher, log) {
 		const items = new Map<string, RawItem>();
 		for (const q of QUERIES) {

src/lib/server/groceries/sources.ts

@@ -6,6 +6,7 @@ import type { GroceryUnit } from '../db/schema/groceries';
 import { HttpError, PoliteFetcher, USER_AGENT } from './http';
 import { normaliseGtin, parseAhUnitPrice, storeUnit } from './normalize';
 import { dirkSource } from './dirk';
+import { hoogvlietSource } from './hoogvliet';
 import { lidlSource } from './lidl';
 import { orientalSource } from './oriental';
 import { plusSource } from './plus';
@@ -38,7 +39,7 @@ export type Source = {
 	/** Part of the default nightly run (default true); false = only with --store. */
 	nightly?: boolean;
 	/** PoliteFetcher overrides (retries/backoff). */
-	fetcher?: { maxRetries?: number; retryBaseMs?: number };
+	fetcher?: { maxRetries?: number; retryBaseMs?: number; retryStatuses?: number[] };
 	fetchAll(f: PoliteFetcher, log: (m: string) => void): Promise<RawItem[]>;
 	/** EAN from a product-detail request, for stores whose listing has none (backfilled once per SKU). */
 	gtinLookup?: (
@@ -387,5 +388,6 @@ export const SOURCES: Record<string, Source> = {
 	xenos: xenosSource,
 	'amazing-oriental': orientalSource,
 	lidl: lidlSource,
-	dirk: dirkSource
+	dirk: dirkSource,
+	hoogvliet: hoogvlietSource
 };

src/lib/server/groceries/stores2.test.ts

@@ -0,0 +1,68 @@
+import { describe, expect, it } from 'vitest';
+import { dirkItem } from './dirk';
+import { hoogvlietCategory, hoogvlietItem } from './hoogvliet';
+
+describe('dirk', () => {
+	it('maps a product with an offer', () => {
+		const it = dirkItem(
+			{
+				productId: 1,
+				normalPrice: 2.85,
+				offerPrice: 1.39,
+				endDate: '2026-10-27T00:00:00',
+				productInformation: {
+					headerText: 'Alpro Vanille',
+					packaging: '1 l',
+					brand: 'Alpro',
+					department: 'Zuivel & kaas',
+					webgroup: 'Melk'
+				}
+			},
+			4,
+			60
+		);
+		expect(it).toMatchObject({
+			priceCents: 285,
+			promoPriceCents: 139,
+			category: 'Zuivel & eieren',
+			promoLabel: 'Actie · t/m 27-10'
+		});
+	});
+	it('flags drugstore as non-food but baby food as food', () => {
+		const base = { productId: 2, normalPrice: 1, productInformation: { headerText: 'x' } };
+		expect(dirkItem(base, 10, 120)?.isFood).toBe(false);
+		expect(dirkItem(base, 10, 100)?.isFood).toBe(true);
+	});
+});
+
+describe('hoogvliet', () => {
+	it('prefers a real tree over theme trees and keeps the deepest path', () => {
+		expect(
+			hoogvlietCategory(['aanbiedingen', 'huishouden-non-food', 'huishouden-non-food/schoonmaken'])
+		).toEqual({
+			category: 'Huishouden & dieren',
+			isFood: false,
+			path: 'huishouden-non-food/schoonmaken'
+		});
+	});
+	it('drops in-store EANs and skips promotion groups', () => {
+		const p = {
+			sku: '980498000',
+			name: 'Casino wit',
+			url_key: 'hoogvliet-casino-wit-980498000',
+			metadata: JSON.stringify({ ean: '2140040000000', merk: 'Hoogvliet' }),
+			price_range: {
+				minimum_price: { regular_price: { value: 1.48 }, final_price: { value: 1.48 } }
+			},
+			categories: [{ level: 2, url_path: 'bakkerij' }]
+		};
+		expect(hoogvlietItem(p, new Map())).toMatchObject({
+			gtin: null,
+			name: 'Hoogvliet Casino wit',
+			url: 'https://hoogvliet.nl/product/hoogvliet-casino-wit-980498000'
+		});
+		expect(
+			hoogvlietItem({ ...p, metadata: JSON.stringify({ promo_products: '["1"]' }) }, new Map())
+		).toBeNull();
+	});
+});