groceries: Lidl importer (online range), Dirk importer + store row, generic per-store GTIN backfill

src/lib/server/groceries/dirk.ts

@@ -0,0 +1,185 @@
+// Dirk (dirk.nl) catalogue source: the public GraphQL gateway behind dirk.nl
+// (web-gateway.dirk.nl/graphql, with the site's public gateway key from its Nuxt config).
+// Departments → web groups → products with the prices of the site's default store; EANs via
+// the product detail query (gtinLookup, backfilled once per SKU).
+import type { PoliteFetcher } from './http';
+import { normaliseGtin } from './normalize';
+import type { RawItem, Source } from './sources';
+
+const BASE = 'https://www.dirk.nl';
+const GATEWAY = 'https://web-gateway.dirk.nl/graphql';
+const API_KEY = '6d3a42a3-6d93-4f98-838d-bcc0ab2307fd'; // public, in dirk.nl's page config
+const STORE_ID = 66; // the store dirk.nl shows prices for
+const IMG = 'https://web-fileserver.dirk.nl/';
+const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
+const HEADERS = { 'User-Agent': UA, api_key: API_KEY, 'content-type': 'application/json' };
+
+type Cat = { category: string; isFood?: boolean };
+const DEPT: Record<number, Cat> = {
+	1: { category: 'Groente, fruit & aardappelen' },
+	2: { category: 'Vlees, vis & vega' },
+	3: { category: 'Brood & bakkerij' },
+	4: { category: 'Zuivel & eieren' },
+	5: { category: 'Frisdrank & sappen' },
+	6: { category: 'Pasta, rijst & wereldkeuken' },
+	7: { category: 'Maaltijden & gemak' },
+	8: { category: 'Diepvries' },
+	9: { category: 'Huishouden & dieren', isFood: false },
+	10: { category: 'Drogisterij', isFood: false },
+	11: { category: 'Overig', isFood: false },
+	12: { category: 'Overig', isFood: false },
+	13: { category: 'Bier & wijn' },
+	14: { category: 'Brood & bakkerij' },
+	15: { category: 'Overig' },
+	16: { category: 'Overig', isFood: false },
+	17: { category: 'Snoep, koek & chips' }
+};
+const conserven = { category: 'Soepen, sauzen & conserven' };
+const WEBGROUP: Record<number, Cat> = {
+	3: conserven,
+	4: { category: 'Snoep, koek & chips' },
+	6: conserven,
+	17: conserven,
+	18: conserven,
+	19: { category: 'Vleeswaren, kaas & tapas' },
+	27: { category: 'Ontbijt & beleg' },
+	29: { category: 'Ontbijt & beleg' },
+	31: { category: 'Snoep, koek & chips' },
+	32: { category: 'Snoep, koek & chips' },
+	37: { category: 'Vleeswaren, kaas & tapas' },
+	38: { category: 'Vleeswaren, kaas & tapas' },
+	46: { category: 'Vleeswaren, kaas & tapas' },
+	48: { category: 'Bier & wijn' },
+	49: { category: 'Bier & wijn' },
+	51: { category: 'Koffie & thee' },
+	52: { category: 'Koffie & thee' },
+	55: { category: 'Bier & wijn' },
+	56: { category: 'Koffie & thee' },
+	58: { category: 'Bier & wijn' },
+	60: { category: 'Zuivel & eieren' },
+	61: { category: 'Ontbijt & beleg' },
+	63: conserven,
+	64: { category: 'Ontbijt & beleg' },
+	65: conserven,
+	71: conserven,
+	73: { category: 'Vleeswaren, kaas & tapas' },
+	78: { category: 'Huishouden & dieren', isFood: false },
+	107: { category: 'Baby & kind', isFood: false },
+	113: { category: 'Baby & kind', isFood: false }
+};
+for (let id = 99; id <= 106; id++) WEBGROUP[id] = { category: 'Baby & kind', isFood: true };
+
+export type DirkAssortment = {
+	productId: number;
+	normalPrice?: number | null;
+	offerPrice?: number | null;
+	endDate?: string | null;
+	productOffer?: { textPriceSign?: string | null } | null;
+	productInformation?: {
+		headerText?: string | null;
+		subText?: string | null;
+		packaging?: string | null;
+		image?: string | null;
+		department?: string | null;
+		webgroup?: string | null;
+		brand?: string | null;
+	} | null;
+};
+
+const cents = (n: unknown) =>
+	typeof n === 'number' && Number.isFinite(n) && n > 0 ? Math.round(n * 100) : null;
+const slug = (s: string) =>
+	s
+		.toLowerCase()
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-|-$/g, '');
+
+export function dirkItem(a: DirkAssortment, deptId: number, webGroupId: number): RawItem | null {
+	const p = a.productInformation;
+	const name = p?.headerText?.trim();
+	if (!a.productId || !name) return null;
+	const price = cents(a.normalPrice);
+	const offer = cents(a.offerPrice);
+	const promo = offer && (!price || offer < price) ? offer : null;
+	const cat = WEBGROUP[webGroupId] ?? DEPT[deptId] ?? { category: 'Overig' };
+	const sign = a.productOffer?.textPriceSign?.replace(/_/g, ' ').replace(/\s+/g, ' ').trim();
+	const until = a.endDate?.slice(0, 10).split('-').reverse().slice(0, 2).join('-');
+	return {
+		storeProductId: String(a.productId),
+		name,
+		brand: p?.brand?.trim() || null,
+		sizeText: p?.packaging?.trim() || null,
+		priceCents: price ?? promo,
+		promoPriceCents: price ? promo : null,
+		promoLabel: promo
+			? [sign && sign.toUpperCase() !== 'ACTIE' ? sign : 'Actie', until && `t/m ${until}`]
+					.filter(Boolean)
+					.join(' · ')
+			: null,
+		storeUnitPrice: null,
+		storeUnitBasisCents: null,
+		category: cat.category,
+		categoryPath: [p?.department, p?.webgroup].filter(Boolean).join(' > ') || null,
+		gtin: null, // via gtinLookup
+		imageUrl: p?.image ? IMG + p.image.replace('/', '%2F') : null,
+		url:
+			p?.department && p.webgroup
+				? `${BASE}/boodschappen/${slug(p.department)}/${slug(p.webgroup)}/${encodeURIComponent(name.toLowerCase())}/${a.productId}`
+				: null,
+		available: (price ?? promo) !== null,
+		...(cat.isFood !== undefined ? { isFood: cat.isFood } : {})
+	};
+}
+
+const ASSORTMENT = `productId normalPrice offerPrice endDate productOffer { textPriceSign }
+	productInformation { headerText subText packaging image department webgroup brand }`;
+
+async function gql<T>(f: PoliteFetcher, query: string, cacheKey?: string): Promise<T> {
+	const r = await f.json<{ data?: T; errors?: { message: string }[] }>(
+		GATEWAY,
+		{ method: 'POST', headers: HEADERS, body: JSON.stringify({ query }) },
+		cacheKey
+	);
+	if (!r.data) throw new Error(`dirk graphql: ${r.errors?.[0]?.message ?? 'no data'}`);
+	return r.data;
+}
+
+export const dirkSource: Source = {
+	slug: 'dirk',
+	minDelayMs: 500,
+	gtinLookup: (f) => async (id) => {
+		const d = await gql<{ product: { barcode?: string | null } | null }>(
+			f,
+			`{ product(productId: ${Number(id)}) { barcode } }`
+		);
+		return normaliseGtin(d.product?.barcode ?? undefined);
+	},
+	async fetchAll(f, log) {
+		const { listDepartments } = await gql<{
+			listDepartments: {
+				departments: { id: number; description: string; webGroups: { webGroupId: number }[] }[];
+			};
+		}>(f, '{ listDepartments { departments { id description webGroups { webGroupId } } } }');
+		const items = new Map<string, RawItem>();
+		for (const d of listDepartments.departments) {
+			const before = items.size;
+			for (const { webGroupId } of d.webGroups) {
+				const r = await gql<{
+					listWebGroupProducts: { productAssortment: (DirkAssortment | null)[] | null } | null;
+				}>(
+					f,
+					`{ listWebGroupProducts(webGroupId: ${webGroupId}) { productAssortment(storeId: ${STORE_ID}) { ${ASSORTMENT} } } }`,
+					`wg-${webGroupId}`
+				);
+				for (const a of r.listWebGroupProducts?.productAssortment ?? []) {
+					const it = a && dirkItem(a, d.id, webGroupId);
+					if (it && !items.has(it.storeProductId)) items.set(it.storeProductId, it);
+				}
+			}
+			log(`  dirk ${d.description}: +${items.size - before} (${items.size} total)`);
+		}
+		return [...items.values()];
+	}
+};