Stan Runge
942c436d03
All checks were successful
Deploy production / deploy (push) Successful in 1m24s
217 lines
8.1 KiB
TypeScript
217 lines
8.1 KiB
TypeScript
// Hoogvliet (hoogvliet.nl) catalogue source. The product SKUs come from the public sitemap
|
|
// (sitemap-catalog.xml); prices, EANs and categories from the site's own storefront GraphQL
|
|
// (hoogvliet.nl/api/graphql, Magento-style) in batches of SKUs. Weekly promotions are separate
|
|
// "promotion group" products that list their member SKUs; they give the promo label.
|
|
import type { PoliteFetcher } from './http';
|
|
import { normaliseGtin } from './normalize';
|
|
import type { RawItem, Source } from './sources';
|
|
|
|
const BASE = 'https://hoogvliet.nl';
|
|
const UA = 'Mozilla/5.0 (compatible; stan-grocery-tracker/0.1; +https://stanrunge.dev)';
|
|
const HEADERS = {
|
|
'User-Agent': UA,
|
|
'content-type': 'application/json',
|
|
'Accept-Language': 'nl',
|
|
xp_env: 'production'
|
|
};
|
|
const BATCH = 50;
|
|
|
|
type Cat = { category: string; isFood?: boolean };
|
|
const TOP: Record<string, Cat> = {
|
|
'aardappelen-groente-fruit': { category: 'Groente, fruit & aardappelen' },
|
|
'verse-maaltijden-salades': { category: 'Maaltijden & gemak' },
|
|
'vlees-vis-vegetarisch': { category: 'Vlees, vis & vega' },
|
|
'kaas-vleeswaren-tapas': { category: 'Vleeswaren, kaas & tapas' },
|
|
'zuivel-eieren-boter': { category: 'Zuivel & eieren' },
|
|
bakkerij: { category: 'Brood & bakkerij' },
|
|
'ontbijt-broodbeleg': { category: 'Ontbijt & beleg' },
|
|
'pasta-rijst-olien-wereldkeuken': { category: 'Pasta, rijst & wereldkeuken' },
|
|
'soepen-conserven-sauzen-kruiden': { category: 'Soepen, sauzen & conserven' },
|
|
'koek-snoep-chocolade': { category: 'Snoep, koek & chips' },
|
|
'chips-zoutjes-noten': { category: 'Snoep, koek & chips' },
|
|
diepvries: { category: 'Diepvries' },
|
|
'koffie-thee': { category: 'Koffie & thee' },
|
|
'frisdrank-sappen': { category: 'Frisdrank & sappen' },
|
|
'bier-wijn-sterke-drank': { category: 'Bier & wijn' },
|
|
'gezondheid-drogisterij': { category: 'Drogisterij', isFood: false },
|
|
'huishouden-non-food': { category: 'Huishouden & dieren', isFood: false },
|
|
huisdier: { category: 'Huishouden & dieren', isFood: false },
|
|
'baby-kind': { category: 'Baby & kind' }
|
|
};
|
|
const SUB: Record<string, Cat> = {
|
|
'baby-kind/verzorging-en-benodigdheden': { category: 'Baby & kind', isFood: false },
|
|
'baby-kind/luiers-en-doekjes': { category: 'Baby & kind', isFood: false },
|
|
'kaas-vleeswaren-tapas/salades': { category: 'Maaltijden & gemak' },
|
|
'bakkerij/bakproducten': { category: 'Ontbijt & beleg' }
|
|
};
|
|
// Theme/landing trees (BBQ, holidays, offers, diets) only count when nothing else matches.
|
|
const THEMES = new Set([
|
|
'bewuste-voeding',
|
|
'alles-voor-de-barbecue',
|
|
'feestdagen-kerst-en-pasen',
|
|
'themashops',
|
|
'aanbiedingen'
|
|
]);
|
|
|
|
const UNITS: Record<string, string> = {
|
|
gram: 'g',
|
|
kilogram: 'kg',
|
|
milliliter: 'ml',
|
|
centiliter: 'cl',
|
|
liter: 'l',
|
|
stuk: 'stuks',
|
|
stuks: 'stuks',
|
|
meter: 'm'
|
|
};
|
|
|
|
export type HoogvlietProduct = {
|
|
sku: string;
|
|
name: string;
|
|
stock_status?: string | null;
|
|
url_key?: string | null;
|
|
metadata?: string | null;
|
|
price_range?: {
|
|
minimum_price?: {
|
|
regular_price?: { value?: number | null } | null;
|
|
final_price?: { value?: number | null } | null;
|
|
} | null;
|
|
} | null;
|
|
categories?: { level: number; url_path: string }[] | null;
|
|
small_image?: { url?: string | null } | null;
|
|
};
|
|
type Meta = Record<string, string | number | undefined>;
|
|
|
|
const cents = (n: unknown) =>
|
|
typeof n === 'number' && Number.isFinite(n) && n > 0 ? Math.round(n * 100) : null;
|
|
const meta = (p: HoogvlietProduct): Meta => {
|
|
try {
|
|
return p.metadata ? (JSON.parse(p.metadata) as Meta) : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
};
|
|
|
|
export function hoogvlietCategory(paths: string[]): Cat & { path: string | null } {
|
|
const top = paths.map((p) => p.split('/')[0]).find((t) => TOP[t]);
|
|
if (top) {
|
|
// the deepest path in that tree, for the breadcrumb
|
|
const path = paths
|
|
.filter((p) => p.split('/')[0] === top)
|
|
.sort((a, b) => b.split('/').length - a.split('/').length)[0];
|
|
return { ...(SUB[path.split('/').slice(0, 2).join('/')] ?? TOP[top]), path };
|
|
}
|
|
const theme = paths.find((p) => THEMES.has(p.split('/')[0]));
|
|
return { category: 'Overig', path: theme ?? paths[0] ?? null };
|
|
}
|
|
|
|
/** member SKU → promo label, from the promotion-group products of the current folder. */
|
|
export function promoLabels(items: HoogvlietProduct[]): Map<string, string> {
|
|
const out = new Map<string, string>();
|
|
for (const p of items) {
|
|
const m = meta(p);
|
|
if (typeof m.promo_products !== 'string') continue;
|
|
let skus: string[];
|
|
try {
|
|
skus = JSON.parse(m.promo_products) as string[];
|
|
} catch {
|
|
continue;
|
|
}
|
|
const until =
|
|
typeof m.valid_to === 'string'
|
|
? m.valid_to.slice(0, 10).split('-').reverse().slice(0, 2).join('-')
|
|
: null;
|
|
const kind = String(m.korting_label ?? '').trim();
|
|
const label = [kind && !/^per\b/i.test(kind) ? kind : 'Actie', until && `t/m ${until}`]
|
|
.filter(Boolean)
|
|
.join(' · ');
|
|
for (const s of skus) if (!out.has(s)) out.set(s, label);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
export const isPromotionGroup = (p: HoogvlietProduct) => typeof meta(p).promo_products === 'string';
|
|
|
|
export function hoogvlietItem(p: HoogvlietProduct, labels: Map<string, string>): RawItem | null {
|
|
if (!p.sku || !p.name || isPromotionGroup(p)) return null;
|
|
const m = meta(p);
|
|
const brand = typeof m.merk === 'string' && m.merk.trim() ? m.merk.trim() : null;
|
|
const name =
|
|
brand && !p.name.toLowerCase().startsWith(brand.toLowerCase())
|
|
? `${brand} ${p.name.trim()}`
|
|
: p.name.trim();
|
|
const price = cents(p.price_range?.minimum_price?.regular_price?.value);
|
|
const final = cents(p.price_range?.minimum_price?.final_price?.value);
|
|
const promo = final && price && final < price ? final : null;
|
|
const unit = UNITS[String(m.eenheid ?? '').toLowerCase()];
|
|
const qty = Number(m.inhoud_ce);
|
|
const cat = hoogvlietCategory((p.categories ?? []).map((c) => c.url_path));
|
|
const path = (p.url_key ?? '').replace(/^product\//, '');
|
|
// 2xxxxxxxxxxxx = in-store codes (bakery, weighed goods): not a real EAN to group on.
|
|
const ean = m.ean === undefined || /^0*2\d{12}$/.test(String(m.ean)) ? null : String(m.ean);
|
|
return {
|
|
storeProductId: p.sku,
|
|
name,
|
|
brand,
|
|
sizeText: unit && qty > 0 ? `${qty} ${unit}` : null,
|
|
priceCents: price ?? final,
|
|
promoPriceCents: promo,
|
|
promoLabel: promo ? (labels.get(p.sku) ?? 'Actie') : null,
|
|
storeUnitPrice: null,
|
|
storeUnitBasisCents: null,
|
|
category: cat.category,
|
|
categoryPath: cat.path,
|
|
gtin: normaliseGtin(ean),
|
|
imageUrl: p.small_image?.url ?? null,
|
|
url: path ? `${BASE}/product/${path}` : null,
|
|
available: p.stock_status !== 'OUT_OF_STOCK' && (price ?? final) !== null,
|
|
...(cat.isFood !== undefined ? { isFood: cat.isFood } : {})
|
|
};
|
|
}
|
|
|
|
const FIELDS = `sku name stock_status url_key metadata
|
|
price_range { minimum_price { regular_price { value } final_price { value } } }
|
|
categories { level url_path } small_image { url }`;
|
|
|
|
export const hoogvlietSource: Source = {
|
|
slug: 'hoogvliet',
|
|
minDelayMs: 1500,
|
|
async fetchAll(f: PoliteFetcher, log) {
|
|
const xml = await f.text(
|
|
`${BASE}/sitemap-catalog.xml`,
|
|
{ headers: { 'User-Agent': UA } },
|
|
'sitemap'
|
|
);
|
|
const skus = [
|
|
...new Set(
|
|
[...xml.matchAll(/<loc>https:\/\/hoogvliet\.nl\/[^<]*-(\d{9})<\/loc>/g)].map((m) => m[1])
|
|
)
|
|
];
|
|
log(` hoogvliet: ${skus.length} SKUs in sitemap`);
|
|
if (skus.length < 1000) throw new Error(`hoogvliet: only ${skus.length} SKUs in the sitemap`);
|
|
const all: HoogvlietProduct[] = [];
|
|
for (let i = 0; i < skus.length; i += BATCH) {
|
|
const batch = skus.slice(i, i + BATCH);
|
|
const query = `{ products(filter: { sku: { in: ${JSON.stringify(batch)} } }, pageSize: ${BATCH}) { items { ${FIELDS} } } }`;
|
|
const r = await f.json<{
|
|
data?: { products?: { items?: HoogvlietProduct[] | null } | null };
|
|
errors?: { message: string }[];
|
|
}>(
|
|
`${BASE}/api/graphql`,
|
|
{ method: 'POST', headers: HEADERS, body: JSON.stringify({ query }) },
|
|
`b${i / BATCH}`
|
|
);
|
|
if (!r.data?.products)
|
|
throw new Error(`hoogvliet graphql: ${r.errors?.[0]?.message ?? 'no data'}`);
|
|
all.push(...(r.data.products.items ?? []));
|
|
if ((i / BATCH) % 40 === 0) log(` hoogvliet: ${all.length}/${skus.length}`);
|
|
}
|
|
const labels = promoLabels(all);
|
|
const items = new Map<string, RawItem>();
|
|
for (const p of all) {
|
|
const it = hoogvlietItem(p, labels);
|
|
if (it) items.set(it.storeProductId, it);
|
|
}
|
|
return [...items.values()];
|
|
}
|
|
};
|