groceries: Lidl importer (online range), Dirk importer + store row, generic per-store GTIN backfill

drizzle/20261025183905_groceries_store_dirk/migration.sql

@@ -0,0 +1,4 @@
+-- Dirk van den Broek (dirk.nl). Lidl's row already exists.
+INSERT INTO "groceries"."stores" ("slug", "name", "website", "color", "sort_order") VALUES
+	('dirk', 'Dirk', 'https://www.dirk.nl', '#d6001c', 70)
+ON CONFLICT ("slug") DO NOTHING;

scripts/groceries-import.ts

@@ -4,9 +4,9 @@
 //     --store SLUG         only this store (repeatable). Default: the nightly stores
 //                          (ah, jumbo, plus, xenos); amazing-oriental runs weekly on its own timer.
 //     --skip-fetch         no store requests; only regroup what is in the DB
-//     --gtin-limit N       AH and PLUS product details to fetch for EANs this run (default 1500, 0 = off)
+//     --gtin-limit N       product details to fetch for EANs per store this run (AH, PLUS, Dirk, …; default 1500, 0 = off)
 //     --gtin-only          skip the catalogue crawl; only the EAN backfill (+ regroup)
-//     --gtin-parallel N    parallel PLUS detail requests for the backfill (default 1, max 4)
+//     --gtin-parallel N    parallel detail requests for the EAN backfill (default 1, max 4)
 //     --delay-ms N         minimum delay between requests (default: per store, else 2000)
 //
 // Politeness: one request at a time per store (Amazing Oriental: 25 s, its robots.txt
@@ -19,7 +19,7 @@ import { join } from 'node:path';
 import { PoliteFetcher, pruneCache } from '../src/lib/server/groceries/http';
 import {
 	enrichAhGtins,
-	enrichPlusGtins,
+	enrichGtins,
 	importStore,
 	regroup
 } from '../src/lib/server/groceries/import';
@@ -74,10 +74,10 @@ try {
 					const g = await enrichAhGtins(sql, f, gtinLimit, log);
 					log(`ah: EAN lookup ${g.checked} checked, ${g.found} found`);
 				}
-				if (slug === 'plus' && gtinLimit > 0) {
+				if (SOURCES[slug].gtinLookup && gtinLimit > 0) {
 					const fs = [f, ...Array.from({ length: gtinParallel - 1 }, fetcher)];
-					const g = await enrichPlusGtins(sql, fs, gtinLimit, log);
-					log(`plus: EAN lookup ${g.checked} checked, ${g.found} found`);
+					const g = await enrichGtins(sql, slug, fs, gtinLimit, log);
+					log(`${slug}: EAN lookup ${g.checked} checked, ${g.found} found`);
 				}
 			} catch (e) {
 				failed = true;

src/lib/server/groceries/dirk.ts

@@ -0,0 +1,185 @@
+// Dirk (dirk.nl) catalogue source: the public GraphQL gateway behind dirk.nl
+// (web-gateway.dirk.nl/graphql, with the site's public gateway key from its Nuxt config).
+// Departments → web groups → products with the prices of the site's default store; EANs via
+// the product detail query (gtinLookup, backfilled once per SKU).
+import type { PoliteFetcher } from './http';
+import { normaliseGtin } from './normalize';
+import type { RawItem, Source } from './sources';
+
+const BASE = 'https://www.dirk.nl';
+const GATEWAY = 'https://web-gateway.dirk.nl/graphql';
+const API_KEY = '6d3a42a3-6d93-4f98-838d-bcc0ab2307fd'; // public, in dirk.nl's page config
+const STORE_ID = 66; // the store dirk.nl shows prices for
+const IMG = 'https://web-fileserver.dirk.nl/';
+const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
+const HEADERS = { 'User-Agent': UA, api_key: API_KEY, 'content-type': 'application/json' };
+
+type Cat = { category: string; isFood?: boolean };
+const DEPT: Record<number, Cat> = {
+	1: { category: 'Groente, fruit & aardappelen' },
+	2: { category: 'Vlees, vis & vega' },
+	3: { category: 'Brood & bakkerij' },
+	4: { category: 'Zuivel & eieren' },
+	5: { category: 'Frisdrank & sappen' },
+	6: { category: 'Pasta, rijst & wereldkeuken' },
+	7: { category: 'Maaltijden & gemak' },
+	8: { category: 'Diepvries' },
+	9: { category: 'Huishouden & dieren', isFood: false },
+	10: { category: 'Drogisterij', isFood: false },
+	11: { category: 'Overig', isFood: false },
+	12: { category: 'Overig', isFood: false },
+	13: { category: 'Bier & wijn' },
+	14: { category: 'Brood & bakkerij' },
+	15: { category: 'Overig' },
+	16: { category: 'Overig', isFood: false },
+	17: { category: 'Snoep, koek & chips' }
+};
+const conserven = { category: 'Soepen, sauzen & conserven' };
+const WEBGROUP: Record<number, Cat> = {
+	3: conserven,
+	4: { category: 'Snoep, koek & chips' },
+	6: conserven,
+	17: conserven,
+	18: conserven,
+	19: { category: 'Vleeswaren, kaas & tapas' },
+	27: { category: 'Ontbijt & beleg' },
+	29: { category: 'Ontbijt & beleg' },
+	31: { category: 'Snoep, koek & chips' },
+	32: { category: 'Snoep, koek & chips' },
+	37: { category: 'Vleeswaren, kaas & tapas' },
+	38: { category: 'Vleeswaren, kaas & tapas' },
+	46: { category: 'Vleeswaren, kaas & tapas' },
+	48: { category: 'Bier & wijn' },
+	49: { category: 'Bier & wijn' },
+	51: { category: 'Koffie & thee' },
+	52: { category: 'Koffie & thee' },
+	55: { category: 'Bier & wijn' },
+	56: { category: 'Koffie & thee' },
+	58: { category: 'Bier & wijn' },
+	60: { category: 'Zuivel & eieren' },
+	61: { category: 'Ontbijt & beleg' },
+	63: conserven,
+	64: { category: 'Ontbijt & beleg' },
+	65: conserven,
+	71: conserven,
+	73: { category: 'Vleeswaren, kaas & tapas' },
+	78: { category: 'Huishouden & dieren', isFood: false },
+	107: { category: 'Baby & kind', isFood: false },
+	113: { category: 'Baby & kind', isFood: false }
+};
+for (let id = 99; id <= 106; id++) WEBGROUP[id] = { category: 'Baby & kind', isFood: true };
+
+export type DirkAssortment = {
+	productId: number;
+	normalPrice?: number | null;
+	offerPrice?: number | null;
+	endDate?: string | null;
+	productOffer?: { textPriceSign?: string | null } | null;
+	productInformation?: {
+		headerText?: string | null;
+		subText?: string | null;
+		packaging?: string | null;
+		image?: string | null;
+		department?: string | null;
+		webgroup?: string | null;
+		brand?: string | null;
+	} | null;
+};
+
+const cents = (n: unknown) =>
+	typeof n === 'number' && Number.isFinite(n) && n > 0 ? Math.round(n * 100) : null;
+const slug = (s: string) =>
+	s
+		.toLowerCase()
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-|-$/g, '');
+
+export function dirkItem(a: DirkAssortment, deptId: number, webGroupId: number): RawItem | null {
+	const p = a.productInformation;
+	const name = p?.headerText?.trim();
+	if (!a.productId || !name) return null;
+	const price = cents(a.normalPrice);
+	const offer = cents(a.offerPrice);
+	const promo = offer && (!price || offer < price) ? offer : null;
+	const cat = WEBGROUP[webGroupId] ?? DEPT[deptId] ?? { category: 'Overig' };
+	const sign = a.productOffer?.textPriceSign?.replace(/_/g, ' ').replace(/\s+/g, ' ').trim();
+	const until = a.endDate?.slice(0, 10).split('-').reverse().slice(0, 2).join('-');
+	return {
+		storeProductId: String(a.productId),
+		name,
+		brand: p?.brand?.trim() || null,
+		sizeText: p?.packaging?.trim() || null,
+		priceCents: price ?? promo,
+		promoPriceCents: price ? promo : null,
+		promoLabel: promo
+			? [sign && sign.toUpperCase() !== 'ACTIE' ? sign : 'Actie', until && `t/m ${until}`]
+					.filter(Boolean)
+					.join(' · ')
+			: null,
+		storeUnitPrice: null,
+		storeUnitBasisCents: null,
+		category: cat.category,
+		categoryPath: [p?.department, p?.webgroup].filter(Boolean).join(' > ') || null,
+		gtin: null, // via gtinLookup
+		imageUrl: p?.image ? IMG + p.image.replace('/', '%2F') : null,
+		url:
+			p?.department && p.webgroup
+				? `${BASE}/boodschappen/${slug(p.department)}/${slug(p.webgroup)}/${encodeURIComponent(name.toLowerCase())}/${a.productId}`
+				: null,
+		available: (price ?? promo) !== null,
+		...(cat.isFood !== undefined ? { isFood: cat.isFood } : {})
+	};
+}
+
+const ASSORTMENT = `productId normalPrice offerPrice endDate productOffer { textPriceSign }
+	productInformation { headerText subText packaging image department webgroup brand }`;
+
+async function gql<T>(f: PoliteFetcher, query: string, cacheKey?: string): Promise<T> {
+	const r = await f.json<{ data?: T; errors?: { message: string }[] }>(
+		GATEWAY,
+		{ method: 'POST', headers: HEADERS, body: JSON.stringify({ query }) },
+		cacheKey
+	);
+	if (!r.data) throw new Error(`dirk graphql: ${r.errors?.[0]?.message ?? 'no data'}`);
+	return r.data;
+}
+
+export const dirkSource: Source = {
+	slug: 'dirk',
+	minDelayMs: 500,
+	gtinLookup: (f) => async (id) => {
+		const d = await gql<{ product: { barcode?: string | null } | null }>(
+			f,
+			`{ product(productId: ${Number(id)}) { barcode } }`
+		);
+		return normaliseGtin(d.product?.barcode ?? undefined);
+	},
+	async fetchAll(f, log) {
+		const { listDepartments } = await gql<{
+			listDepartments: {
+				departments: { id: number; description: string; webGroups: { webGroupId: number }[] }[];
+			};
+		}>(f, '{ listDepartments { departments { id description webGroups { webGroupId } } } }');
+		const items = new Map<string, RawItem>();
+		for (const d of listDepartments.departments) {
+			const before = items.size;
+			for (const { webGroupId } of d.webGroups) {
+				const r = await gql<{
+					listWebGroupProducts: { productAssortment: (DirkAssortment | null)[] | null } | null;
+				}>(
+					f,
+					`{ listWebGroupProducts(webGroupId: ${webGroupId}) { productAssortment(storeId: ${STORE_ID}) { ${ASSORTMENT} } } }`,
+					`wg-${webGroupId}`
+				);
+				for (const a of r.listWebGroupProducts?.productAssortment ?? []) {
+					const it = a && dirkItem(a, d.id, webGroupId);
+					if (it && !items.has(it.storeProductId)) items.set(it.storeProductId, it);
+				}
+			}
+			log(`  dirk ${d.description}: +${items.size - before} (${items.size} total)`);
+		}
+		return [...items.values()];
+	}
+};

src/lib/server/groceries/import.ts

@@ -12,8 +12,7 @@ import {
 	nameTokens,
 	normaliseUnit
 } from './normalize';
-import { PlusClient } from './plus';
-import { AhClient, itemIsFood, type RawItem, type Source } from './sources';
+import { AhClient, itemIsFood, SOURCES, type RawItem, type Source } from './sources';
 
 type Log = (m: string) => void;
 
@@ -183,42 +182,48 @@ export async function inferBrands(sql: Sql, sid: number, raw: RawItem[]): Promis
 const isHouseBrand = (b: string) => brandClass(b).startsWith('house');
 
 /** Fill in PLUS EANs from the product detail action, `limit` products per run, once per SKU. */
-export async function enrichPlusGtins(
+/**
+ * Fill in EANs from a source's product-detail lookup (Source.gtinLookup), `limit` products per
+ * run, each SKU at most once per 180 days. One lookup per fetcher runs in parallel.
+ */
+export async function enrichGtins(
 	sql: Sql,
+	slug: string,
 	fetchers: PoliteFetcher[],
 	limit: number,
 	log: Log
 ) {
-	if (limit <= 0 || !fetchers.length) return { checked: 0, found: 0 };
-	const sid = await storeId(sql, 'plus');
+	const make = SOURCES[slug]?.gtinLookup;
+	if (limit <= 0 || !fetchers.length || !make) return { checked: 0, found: 0 };
+	const sid = await storeId(sql, slug);
 	const todo = await sql<{ id: number; store_product_id: string; url: string | null }[]>`
 		select id, store_product_id, url from groceries.items
 		where store_id = ${sid} and gtin is null and available
 			and (gtin_checked_at is null or gtin_checked_at < now() - interval '180 days')
 		order by gtin_checked_at asc nulls first, id
 		limit ${limit}`;
-	const clients = fetchers.map((f) => new PlusClient(f));
+	const lookups = fetchers.map((f) => make(f));
 	let found = 0;
 	let errors = 0;
 	let next = 0;
 	let done = 0;
 	await Promise.all(
-		clients.map(async (plus) => {
+		lookups.map(async (lookup) => {
 			while (next < todo.length) {
 				const t = todo[next++];
-				const slug = t.url?.split('/product/')[1] ?? '';
 				let gtin: string | null;
 				try {
-					gtin = await plus.gtin(t.store_product_id, slug);
+					gtin = await lookup(t.store_product_id, t.url);
 				} catch (e) {
 					errors++;
-					log(`  plus gtin ${t.store_product_id}: ${e instanceof Error ? e.message : e}`);
+					log(`  ${slug} gtin ${t.store_product_id}: ${e instanceof Error ? e.message : e}`);
 					if (errors > 25 && found === 0) throw e; // something is wrong; stop hammering
 					continue;
 				}
 				if (gtin) found++;
 				await sql`update groceries.items set gtin = ${gtin}, gtin_checked_at = now() where id = ${t.id}`;
-				if (++done % 500 === 0) log(`  plus gtin: ${done}/${todo.length} checked, ${found} found`);
+				if (++done % 500 === 0)
+					log(`  ${slug} gtin: ${done}/${todo.length} checked, ${found} found`);
 			}
 		})
 	);

src/lib/server/groceries/lidl.ts

@@ -0,0 +1,161 @@
+// Lidl (lidl.nl) catalogue source: the public search API behind lidl.nl (/q/api/search).
+// Lidl NL does not publish its regular supermarket range with prices online: the API only
+// has the webshop (≈9.5k non-food items), "Eten & drinken" (wine, meat packs, current food
+// deals) and the in-store weekly deals. We import food + in-store deals; the webshop's
+// clothing/tools/furniture is skipped (not groceries).
+import type { PoliteFetcher } from './http';
+import type { RawItem, Source } from './sources';
+
+const BASE = 'https://www.lidl.nl';
+const API = `${BASE}/q/api/search`;
+const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
+const PAGE = 60; // the API caps fetchsize at 60
+const QUERIES: { name: string; params: string }[] = [
+	{ name: 'eten-en-drinken', params: 'category.id=10068374' },
+	{ name: 'in-de-winkel', params: 'q=*&store=1' },
+	{ name: 'dieren', params: 'category.id=10100091' }
+];
+
+/** Lidl "wonCategoryPrimary" level 3 (Werelden van nood/Eten en …/<this>) → shared category. */
+const WON: [RegExp, string][] = [
+	[/wijn|champagne|bier|sterke drank|alcohol/i, 'Bier & wijn'],
+	[/vlees|gevogelte|vis|zeevruchten|vega/i, 'Vlees, vis & vega'],
+	[/kaas|zuivel|eieren/i, 'Zuivel & eieren'],
+	[/bakkerij|brood/i, 'Brood & bakkerij'],
+	[/kant-en-klaar|maaltijd/i, 'Maaltijden & gemak'],
+	[/diepvries/i, 'Diepvries'],
+	[/snoep|snack|koek|chips/i, 'Snoep, koek & chips'],
+	[/koffie|thee|cacao/i, 'Koffie & thee'],
+	[/dranken|sap|frisdrank|water/i, 'Frisdrank & sappen'],
+	[/groente|fruit|aardappel/i, 'Groente, fruit & aardappelen'],
+	[/ontbijt|beleg|broodbeleg/i, 'Ontbijt & beleg'],
+	[/voedselvoorraad|conserven|sauzen|soep/i, 'Soepen, sauzen & conserven'],
+	[/pasta|rijst|wereld/i, 'Pasta, rijst & wereldkeuken'],
+	[/dier/i, 'Huishouden & dieren'],
+	[/huishoud|schoonmaak/i, 'Huishouden & dieren'],
+	[/verzorging|drogist|mooi & gezond|gezondheid/i, 'Drogisterij'],
+	[/baby/i, 'Baby & kind']
+];
+
+type LidlPrice = {
+	price?: number;
+	oldPrice?: number;
+	packaging?: { text?: string };
+	basePrice?: { text?: string };
+	discount?: { deletedPrice?: number; percentageDiscount?: number; showDiscount?: boolean };
+};
+export type LidlGrid = {
+	erpNumber?: string;
+	itemId?: number;
+	fullTitle?: string;
+	title?: string;
+	category?: string;
+	canonicalUrl?: string;
+	image?: string;
+	brand?: { name?: string; showBrand?: boolean };
+	keyfacts?: { wonCategoryPrimary?: string; description?: string };
+	price?: LidlPrice;
+	lidlPlus?: { price?: LidlPrice; highlightText?: string }[];
+	stockAvailability?: { badgeInfo?: { badges?: { text?: string }[] } };
+};
+type LidlSearch = { numFound?: number; items?: { gridbox?: { data?: LidlGrid } }[] };
+
+const cents = (n: unknown) =>
+	typeof n === 'number' && Number.isFinite(n) && n > 0 ? Math.round(n * 100) : null;
+const strip = (s: string) =>
+	s
+		.replace(/<[^>]+>/g, ' ')
+		.replace(/\s+/g, ' ')
+		.trim();
+
+export function lidlCategory(g: LidlGrid): { category: string; isFood: boolean } {
+	const won = g.keyfacts?.wonCategoryPrimary ?? '';
+	const parts = won.split('/');
+	const food = g.category === 'Food' || /^Werelden van nood\/Eten/.test(won);
+	const leaf = parts.slice(2).join(' ');
+	if (/bloemen|planten/i.test(leaf)) return { category: 'Overig', isFood: false };
+	for (const [re, cat] of WON)
+		if (re.test(leaf || won)) {
+			const nonFood = cat === 'Huishouden & dieren' || cat === 'Drogisterij';
+			return { category: cat, isFood: !nonFood && (food || cat !== 'Baby & kind') };
+		}
+	return { category: 'Overig', isFood: food };
+}
+
+export function lidlItem(g: LidlGrid): RawItem | null {
+	const id = g.erpNumber ?? (g.itemId ? String(g.itemId) : null);
+	const name = (g.fullTitle ?? g.title ?? '').replace(/®/g, '').trim();
+	if (!id || !name) return null;
+	const p = g.price ?? {};
+	const plus = g.lidlPlus?.[0];
+	let price = cents(p.discount?.deletedPrice) ?? cents(p.oldPrice) ?? cents(p.price);
+	let promo = price && cents(p.price) && cents(p.price)! < price ? cents(p.price) : null;
+	let promoLabel = promo
+		? p.discount?.percentageDiscount
+			? `${p.discount.percentageDiscount}% korting`
+			: 'Actie'
+		: null;
+	const plusPrice = cents(plus?.price?.price);
+	if (!price && plusPrice) {
+		// Lidl Plus-only deal: the regular price isn't shown.
+		price = plusPrice;
+		promoLabel = `Lidl Plus${plus?.highlightText ? ' ' + plus.highlightText : ''}`;
+	} else if (plusPrice && price && plusPrice < (promo ?? price)) {
+		promo = plusPrice;
+		promoLabel = `Lidl Plus${plus?.highlightText ? ' ' + plus.highlightText : ''}`;
+	}
+	const size =
+		p.packaging?.text ??
+		plus?.price?.packaging?.text ??
+		(g.keyfacts?.description ? strip(g.keyfacts.description).split(/\s{2,}|·/)[0] : null);
+	const badge = g.stockAvailability?.badgeInfo?.badges?.[0]?.text;
+	const { category, isFood } = lidlCategory(g);
+	const brand = g.brand?.showBrand && g.brand.name && g.brand.name !== '-' ? g.brand.name : null;
+	return {
+		storeProductId: id,
+		name,
+		brand: brand ? brand.replace(/®/g, '').trim() : null,
+		sizeText: size && size.length <= 40 ? size : null,
+		priceCents: price,
+		promoPriceCents: promo,
+		promoLabel: promoLabel && badge ? `${promoLabel} · ${badge}` : promoLabel,
+		storeUnitPrice: null,
+		storeUnitBasisCents: null,
+		category,
+		categoryPath: g.keyfacts?.wonCategoryPrimary?.split('/').slice(1).join(' > ') || null,
+		gtin: null, // not published
+		imageUrl: g.image ?? null,
+		url: g.canonicalUrl ? BASE + g.canonicalUrl : null,
+		available: price !== null,
+		isFood
+	};
+}
+
+export const lidlSource: Source = {
+	slug: 'lidl',
+	minDelayMs: 1000,
+	async fetchAll(f: PoliteFetcher, log) {
+		const items = new Map<string, RawItem>();
+		for (const q of QUERIES) {
+			const before = items.size;
+			for (let offset = 0; offset < 5000; offset += PAGE) {
+				const r = await f.json<LidlSearch>(
+					`${API}?${q.params}&offset=${offset}&fetchsize=${PAGE}&locale=nl_NL&assortment=NL&version=2.1.0`,
+					{ headers: { 'User-Agent': UA, Accept: 'application/json' } },
+					`${q.name}-o${offset}`
+				);
+				const rows = r.items ?? [];
+				for (const row of rows) {
+					const it = row.gridbox?.data && lidlItem(row.gridbox.data);
+					if (!it || items.has(it.storeProductId)) continue;
+					// In-store deals include lots of tools/clothing: keep food, drugstore, household.
+					if (q.name === 'in-de-winkel' && it.category === 'Overig' && !it.isFood) continue;
+					items.set(it.storeProductId, it);
+				}
+				if (rows.length < PAGE || offset + PAGE >= (r.numFound ?? 0)) break;
+			}
+			log(`  lidl ${q.name}: +${items.size - before} (${items.size} total)`);
+		}
+		return [...items.values()];
+	}
+};

src/lib/server/groceries/plus.ts

@@ -280,6 +280,10 @@ export class PlusClient {
 export const plusSource: Source = {
 	slug: 'plus',
 	minDelayMs: 300,
+	gtinLookup: (f) => {
+		const plus = new PlusClient(f);
+		return (sku, url) => plus.gtin(sku, url?.split('/product/')[1] ?? '');
+	},
 	async fetchAll(f, log) {
 		const plus = new PlusClient(f);
 		const items = new Map<string, RawItem>();

src/lib/server/groceries/sources.ts

@@ -1,10 +1,12 @@
 // Store catalogue sources. Each returns the full assortment as RawItem[].
 //  - Albert Heijn: the anonymous mobile-app API (api.ah.nl), crawled per top category.
 //  - Jumbo: the public GraphQL search endpoint used by jumbo.com, crawled per top category.
-//  - Plus, Xenos, Amazing Oriental: see plus.ts, xenos.ts, oriental.ts.
+//  - Plus, Xenos, Amazing Oriental, Lidl, …: see plus.ts, xenos.ts, oriental.ts, lidl.ts, ….
 import type { GroceryUnit } from '../db/schema/groceries';
 import { HttpError, PoliteFetcher, USER_AGENT } from './http';
 import { normaliseGtin, parseAhUnitPrice, storeUnit } from './normalize';
+import { dirkSource } from './dirk';
+import { lidlSource } from './lidl';
 import { orientalSource } from './oriental';
 import { plusSource } from './plus';
 import { xenosSource } from './xenos';
@@ -38,6 +40,10 @@ export type Source = {
 	/** PoliteFetcher overrides (retries/backoff). */
 	fetcher?: { maxRetries?: number; retryBaseMs?: number };
 	fetchAll(f: PoliteFetcher, log: (m: string) => void): Promise<RawItem[]>;
+	/** EAN from a product-detail request, for stores whose listing has none (backfilled once per SKU). */
+	gtinLookup?: (
+		f: PoliteFetcher
+	) => (storeProductId: string, url: string | null) => Promise<string | null>;
 };
 
 const NON_FOOD_CATEGORIES = new Set(['Drogisterij', 'Huishouden & dieren']);
@@ -379,5 +385,7 @@ export const SOURCES: Record<string, Source> = {
 	jumbo: jumboSource,
 	plus: plusSource,
 	xenos: xenosSource,
-	'amazing-oriental': orientalSource
+	'amazing-oriental': orientalSource,
+	lidl: lidlSource,
+	dirk: dirkSource
 };