groceries: add PLUS, Xenos and Amazing Oriental stores, is_food flag, PLUS EAN backfill
Deploy production / deploy (push) Successful in 1m7s
Deploy production / deploy (push) Successful in 1m7s
This commit is contained in:
stan committed
1 parent
b23f7b3475
commit
3f30bb883f
14 files changed
+30444
-38
No files matched your search
@@ -0,0 +1,171 @@
|
||||
// Amazing Oriental webshop (orientalwebshop.nl, Magento 2) catalogue source. There is no JSON
|
||||
// API (graphql/rest are closed), so we parse the category listing HTML. The site's bot wall
|
||||
// (Anubis) lets honest non-browser User-Agents through and expects them to follow robots.txt,
|
||||
// which asks for Crawl-delay: 25, so this source makes ONE request per 25 s (~45 min a crawl)
|
||||
// and runs weekly on its own timer. A challenge page stops the crawl; we never work around it.
|
||||
import { FatalFetchError, HttpError, type PoliteFetcher } from './http';
|
||||
import type { RawItem, Source } from './sources';
|
||||
|
||||
const BASE = 'https://www.orientalwebshop.nl/nl';
|
||||
const UA = 'stan-grocery-tracker/0.1 (personal price tracker; +https://stanrunge.dev)';
|
||||
export const AO_CRAWL_DELAY_MS = 25_000; // robots.txt Crawl-delay: 25
|
||||
const LIMIT = 95; // the largest product_list_limit the shop offers
|
||||
|
||||
/** Top category url key → shared category. Crawled in this order; outlet last. */
|
||||
const TOPS: [string, string][] = [
|
||||
['verse-producten', 'Groente, fruit & aardappelen'],
|
||||
['diepvriesproducten', 'Diepvries'],
|
||||
['dranken-alcohol', 'Frisdrank & sappen'],
|
||||
['rijst', 'Pasta, rijst & wereldkeuken'],
|
||||
['noedels', 'Pasta, rijst & wereldkeuken'],
|
||||
['sushi-producten', 'Pasta, rijst & wereldkeuken'],
|
||||
['hot-pot-fondue', 'Pasta, rijst & wereldkeuken'],
|
||||
['sauzen', 'Soepen, sauzen & conserven'],
|
||||
['kruiden', 'Soepen, sauzen & conserven'],
|
||||
['geconserveerd', 'Soepen, sauzen & conserven'],
|
||||
['bakproducten', 'Ontbijt & beleg'],
|
||||
['aziatische-snacks', 'Snoep, koek & chips'],
|
||||
['mooncakes', 'Snoep, koek & chips'], // seasonal; 404 out of season
|
||||
['huis-beauty-welzijn', 'Overig'],
|
||||
['outlet', 'Overig'] // short-dated cross-cut; items keep their real category when seen first
|
||||
];
|
||||
const TOP_MAP = new Map(TOPS);
|
||||
/** Level-2 (data-category segment 2, slugified) overrides. */
|
||||
const SUB: Record<string, string> = {
|
||||
'zeevruchten-vlees-eieren': 'Vlees, vis & vega',
|
||||
'tofu-tempeh': 'Vlees, vis & vega',
|
||||
'cake-dessert': 'Brood & bakkerij',
|
||||
thee: 'Koffie & thee',
|
||||
'instant-drank': 'Koffie & thee',
|
||||
koffie: 'Koffie & thee',
|
||||
alcohol: 'Bier & wijn',
|
||||
bier: 'Bier & wijn',
|
||||
wijn: 'Bier & wijn'
|
||||
};
|
||||
|
||||
const decode = (s: string) =>
|
||||
s
|
||||
.replace(/&/g, '&')
|
||||
.replace(/'|�?39;/g, "'")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
const attr = (blk: string, name: string) => {
|
||||
const m = blk.match(new RegExp(`${name}="([^"]*)"`));
|
||||
return m ? decode(m[1]) : null;
|
||||
};
|
||||
const slug = (s: string) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/&/g, '')
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
const cents = (v: string | undefined) => {
|
||||
const n = v ? parseFloat(v) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? Math.round(n * 100) : null;
|
||||
};
|
||||
|
||||
/** Parse one listing page into items (exported for tests). */
|
||||
export function parseOrientalListing(html: string, top: string): RawItem[] {
|
||||
const arts = html.match(/<article class="item product product-item"[\s\S]*?<\/article>/g) ?? [];
|
||||
const out: RawItem[] = [];
|
||||
for (const a of arts) {
|
||||
const id = attr(a, 'data-sku');
|
||||
const link =
|
||||
a.match(/class="product-item-link"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/) ??
|
||||
a.match(/href="([^"]+)"[^>]*class="product-item-link"[^>]*>([\s\S]*?)<\/a>/);
|
||||
const name = decode(link?.[2]?.replace(/<[^>]+>/g, '') ?? attr(a, 'data-name') ?? '');
|
||||
if (!id || !name) continue;
|
||||
const fin = cents(a.match(/data-price-amount="([\d.]+)" data-price-type="finalPrice"/)?.[1]);
|
||||
const old = cents(a.match(/data-price-amount="([\d.]+)" data-price-type="oldPrice"/)?.[1]);
|
||||
const promo = old && fin && fin < old ? fin : null;
|
||||
const labels = [
|
||||
...a.matchAll(/class="product-label (label-[a-z_-]+)"[^>]*><span>([^<]*)</g)
|
||||
].map((m) => [m[1], decode(m[2])] as const);
|
||||
const sale = labels.find(([c]) => c === 'label-sale')?.[1] || null;
|
||||
const exp = labels.find(([c]) => c.startsWith('label-expiration'))?.[1] || null;
|
||||
const catAttr = attr(a, 'data-category'); // "Verse Producten/Tofu & Tempeh" or "OUTLET"
|
||||
const real = catAttr && catAttr.toUpperCase() !== 'OUTLET' ? catAttr : null;
|
||||
const category = (real && SUB[slug(real.split('/')[1] ?? '')]) ?? TOP_MAP.get(top) ?? 'Overig';
|
||||
out.push({
|
||||
storeProductId: id,
|
||||
name,
|
||||
brand: attr(a, 'data-brand') || null,
|
||||
sizeText:
|
||||
name.match(
|
||||
/(\d+\s*[x×]\s*)?\d+(?:[.,]\d+)?\s*(?:kg|g|gr|ml|cl|l|ltr|liter|st|stuks)\b/i
|
||||
)?.[0] ?? null,
|
||||
priceCents: old ?? fin,
|
||||
promoPriceCents: promo,
|
||||
promoLabel: promo
|
||||
? [sale ?? 'Aanbieding', exp ? `THT ${exp.replace(/^THT\s*/i, '')}` : null]
|
||||
.filter(Boolean)
|
||||
.join(' · ')
|
||||
: null,
|
||||
storeUnitPrice: null,
|
||||
storeUnitBasisCents: null,
|
||||
category,
|
||||
categoryPath: real ? real.replace(/\//g, ' > ') : top === 'outlet' ? 'Outlet' : top,
|
||||
gtin: null,
|
||||
imageUrl: a.match(/class="product-image-photo"[^>]*src="([^"]+)"/)?.[1] ?? null,
|
||||
url: link?.[1] ?? null,
|
||||
available: attr(a, 'data-dimension10') !== 'Niet op voorraad',
|
||||
isFood: top !== 'huis-beauty-welzijn'
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export const orientalSource: Source = {
|
||||
slug: 'amazing-oriental',
|
||||
minDelayMs: AO_CRAWL_DELAY_MS,
|
||||
nightly: false, // weekly, own timer (groceries-import-oriental.timer)
|
||||
// On 429 back off hard (minutes), never hammer.
|
||||
fetcher: { maxRetries: 3, retryBaseMs: 120_000 },
|
||||
async fetchAll(f: PoliteFetcher, log) {
|
||||
const items = new Map<string, RawItem>();
|
||||
const get = (top: string, p: number) =>
|
||||
f.text(
|
||||
`${BASE}/${top}?product_list_limit=${LIMIT}&p=${p}`,
|
||||
{ headers: { 'User-Agent': UA, Accept: 'text/html' } },
|
||||
`${top}-p${p}`
|
||||
);
|
||||
for (const [top] of TOPS) {
|
||||
const before = items.size;
|
||||
let prevFirst = '';
|
||||
for (let p = 1; p <= 200; p++) {
|
||||
let html: string;
|
||||
try {
|
||||
html = await get(top, p);
|
||||
} catch (e) {
|
||||
if (e instanceof HttpError && e.status === 404) {
|
||||
log(` ao ${top}: 404, skipped (seasonal?)`);
|
||||
break;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
if (html.includes('anubis_challenge'))
|
||||
throw new FatalFetchError('amazing-oriental: bot challenge page (policy changed)');
|
||||
const page = parseOrientalListing(html, top);
|
||||
if (!page.length || page[0].storeProductId === prevFirst) break; // Magento clamps p
|
||||
prevFirst = page[0].storeProductId;
|
||||
for (const it of page) {
|
||||
const prev = items.get(it.storeProductId);
|
||||
if (!prev) items.set(it.storeProductId, it);
|
||||
else if (top === 'outlet' && it.promoPriceCents && !prev.promoPriceCents)
|
||||
items.set(it.storeProductId, {
|
||||
...prev,
|
||||
promoPriceCents: it.promoPriceCents,
|
||||
promoLabel: it.promoLabel,
|
||||
priceCents: it.priceCents
|
||||
});
|
||||
}
|
||||
if (page.length < LIMIT) break;
|
||||
}
|
||||
log(` ao ${top}: +${items.size - before} (${items.size} total, ${f.requests} requests)`);
|
||||
}
|
||||
return [...items.values()];
|
||||
}
|
||||
};
|
||||
Reference in new issue
Block a user