website/src/lib/server/groceries/oriental.ts

// Amazing Oriental webshop (orientalwebshop.nl, Magento 2) catalogue source. There is no JSON
// API (graphql/rest are closed), so we parse the category listing HTML. The site's bot wall
// (Anubis) lets honest non-browser User-Agents through and expects them to follow robots.txt,
// which asks for Crawl-delay: 25, so this source makes ONE request per 25 s (~45 min a crawl)
// and runs weekly on its own timer. A challenge page stops the crawl; we never work around it.
import { FatalFetchError, HttpError, type PoliteFetcher } from './http';
import type { RawItem, Source } from './sources';

const BASE = 'https://www.orientalwebshop.nl/nl';
const UA = 'stan-grocery-tracker/0.1 (personal price tracker; +https://stanrunge.dev)';
export const AO_CRAWL_DELAY_MS = 25_000; // robots.txt Crawl-delay: 25
const LIMIT = 95; // the largest product_list_limit the shop offers

/** Top category url key → shared category. Crawled in this order; outlet last. */
const TOPS: [string, string][] = [
	['verse-producten', 'Groente, fruit & aardappelen'],
	['diepvriesproducten', 'Diepvries'],
	['dranken-alcohol', 'Frisdrank & sappen'],
	['rijst', 'Pasta, rijst & wereldkeuken'],
	['noedels', 'Pasta, rijst & wereldkeuken'],
	['sushi-producten', 'Pasta, rijst & wereldkeuken'],
	['hot-pot-fondue', 'Pasta, rijst & wereldkeuken'],
	['sauzen', 'Soepen, sauzen & conserven'],
	['kruiden', 'Soepen, sauzen & conserven'],
	['geconserveerd', 'Soepen, sauzen & conserven'],
	['bakproducten', 'Ontbijt & beleg'],
	['aziatische-snacks', 'Snoep, koek & chips'],
	['mooncakes', 'Snoep, koek & chips'], // seasonal; 404 out of season
	['huis-beauty-welzijn', 'Overig'],
	['outlet', 'Overig'] // short-dated cross-cut; items keep their real category when seen first
];
const TOP_MAP = new Map(TOPS);
/** Level-2 (data-category segment 2, slugified) overrides. */
const SUB: Record<string, string> = {
	'zeevruchten-vlees-eieren': 'Vlees, vis & vega',
	'tofu-tempeh': 'Vlees, vis & vega',
	'cake-dessert': 'Brood & bakkerij',
	thee: 'Koffie & thee',
	'instant-drank': 'Koffie & thee',
	koffie: 'Koffie & thee',
	alcohol: 'Bier & wijn',
	bier: 'Bier & wijn',
	wijn: 'Bier & wijn'
};

const decode = (s: string) =>
	s
		.replace(/&amp;/g, '&')
		.replace(/&apos;|&#0?39;/g, "'")
		.replace(/&quot;/g, '"')
		.replace(/&lt;/g, '<')
		.replace(/&gt;/g, '>')
		.replace(/\s+/g, ' ')
		.trim();
const attr = (blk: string, name: string) => {
	const m = blk.match(new RegExp(`${name}="([^"]*)"`));
	return m ? decode(m[1]) : null;
};
const slug = (s: string) =>
	s
		.toLowerCase()
		.replace(/&/g, '')
		.replace(/[^a-z0-9]+/g, '-')
		.replace(/^-|-$/g, '');
const cents = (v: string | undefined) => {
	const n = v ? parseFloat(v) : NaN;
	return Number.isFinite(n) && n > 0 ? Math.round(n * 100) : null;
};

/** Parse one listing page into items (exported for tests). */
export function parseOrientalListing(html: string, top: string): RawItem[] {
	const arts = html.match(/<article class="item product product-item"[\s\S]*?<\/article>/g) ?? [];
	const out: RawItem[] = [];
	for (const a of arts) {
		const id = attr(a, 'data-sku');
		const link =
			a.match(/class="product-item-link"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/) ??
			a.match(/href="([^"]+)"[^>]*class="product-item-link"[^>]*>([\s\S]*?)<\/a>/);
		const name = decode(link?.[2]?.replace(/<[^>]+>/g, '') ?? attr(a, 'data-name') ?? '');
		if (!id || !name) continue;
		const fin = cents(a.match(/data-price-amount="([\d.]+)" data-price-type="finalPrice"/)?.[1]);
		const old = cents(a.match(/data-price-amount="([\d.]+)" data-price-type="oldPrice"/)?.[1]);
		const promo = old && fin && fin < old ? fin : null;
		const labels = [
			...a.matchAll(/class="product-label (label-[a-z_-]+)"[^>]*><span>([^<]*)</g)
		].map((m) => [m[1], decode(m[2])] as const);
		const sale = labels.find(([c]) => c === 'label-sale')?.[1] || null;
		const exp = labels.find(([c]) => c.startsWith('label-expiration'))?.[1] || null;
		const catAttr = attr(a, 'data-category'); // "Verse Producten/Tofu & Tempeh" or "OUTLET"
		const real = catAttr && catAttr.toUpperCase() !== 'OUTLET' ? catAttr : null;
		const category = (real && SUB[slug(real.split('/')[1] ?? '')]) ?? TOP_MAP.get(top) ?? 'Overig';
		out.push({
			storeProductId: id,
			name,
			brand: attr(a, 'data-brand') || null,
			sizeText:
				name.match(
					/(\d+\s*[x×]\s*)?\d+(?:[.,]\d+)?\s*(?:kg|g|gr|ml|cl|l|ltr|liter|st|stuks)\b/i
				)?.[0] ?? null,
			priceCents: old ?? fin,
			promoPriceCents: promo,
			promoLabel: promo
				? [sale ?? 'Aanbieding', exp ? `THT ${exp.replace(/^THT\s*/i, '')}` : null]
						.filter(Boolean)
						.join(' · ')
				: null,
			storeUnitPrice: null,
			storeUnitBasisCents: null,
			category,
			categoryPath: real ? real.replace(/\//g, ' > ') : top === 'outlet' ? 'Outlet' : top,
			gtin: null,
			imageUrl: a.match(/class="product-image-photo"[^>]*src="([^"]+)"/)?.[1] ?? null,
			url: link?.[1] ?? null,
			available: attr(a, 'data-dimension10') !== 'Niet op voorraad',
			isFood: top !== 'huis-beauty-welzijn'
		});
	}
	return out;
}

export const orientalSource: Source = {
	slug: 'amazing-oriental',
	minDelayMs: AO_CRAWL_DELAY_MS,
	nightly: false, // weekly, own timer (groceries-import-oriental.timer)
	// On 429 back off hard (minutes), never hammer.
	fetcher: { maxRetries: 3, retryBaseMs: 120_000 },
	async fetchAll(f: PoliteFetcher, log) {
		const items = new Map<string, RawItem>();
		const get = (top: string, p: number) =>
			f.text(
				`${BASE}/${top}?product_list_limit=${LIMIT}&p=${p}`,
				{ headers: { 'User-Agent': UA, Accept: 'text/html' } },
				`${top}-p${p}`
			);
		for (const [top] of TOPS) {
			const before = items.size;
			let prevFirst = '';
			for (let p = 1; p <= 200; p++) {
				let html: string;
				try {
					html = await get(top, p);
				} catch (e) {
					if (e instanceof HttpError && e.status === 404) {
						log(`  ao ${top}: 404, skipped (seasonal?)`);
						break;
					}
					throw e;
				}
				if (html.includes('anubis_challenge'))
					throw new FatalFetchError('amazing-oriental: bot challenge page (policy changed)');
				const page = parseOrientalListing(html, top);
				if (!page.length || page[0].storeProductId === prevFirst) break; // Magento clamps p
				prevFirst = page[0].storeProductId;
				for (const it of page) {
					const prev = items.get(it.storeProductId);
					if (!prev) items.set(it.storeProductId, it);
					else if (top === 'outlet' && it.promoPriceCents && !prev.promoPriceCents)
						items.set(it.storeProductId, {
							...prev,
							promoPriceCents: it.promoPriceCents,
							promoLabel: it.promoLabel,
							priceCents: it.priceCents
						});
				}
				if (page.length < LIMIT) break;
			}
			log(`  ao ${top}: +${items.size - before} (${items.size} total, ${f.requests} requests)`);
		}
		return [...items.values()];
	}
};