wildlife: iNaturalist/Wikidata/GBIF import, CC photos and refresh script

scripts/wildlife-import.ts

@@ -0,0 +1,603 @@
+// Builds the public wildlife pokedex (/trackers/wildlife, schema `wildlife`) from open data.
+//
+//   DB_URL=... bun scripts/wildlife-import.ts               fetch sources and write to the database
+//   bun scripts/wildlife-import.ts --dry-run                fetch + build only, print a summary
+//   bun scripts/wildlife-import.ts --cache /tmp/wildlife    reuse API responses from that folder
+//   bun scripts/wildlife-import.ts --report-new             JSON of species new since the last run on stdout
+//
+// Which species (wild animals only, no pets/livestock, no captive observations):
+//  - the most observed species per group in the Netherlands (iNaturalist research grade),
+//  - the most observed species per group in Europe,
+//  - the most observed species per group worldwide, plus a short list of famous animals.
+// Species already in the dex are always kept (refreshed, never deleted), so Dex numbers stay
+// stable and new species get the next number.
+// Per species: names (iNaturalist nl/en locale, Wikidata nl label), taxonomy, IUCN Red List
+// (iNaturalist, Wikidata P141 fallback), Dutch red list status, establishment in NL, iNat
+// observation counts (NL, Europe, world), Wikipedia intros (en + nl via iNaturalist),
+// Wikidata mass/length/lifespan/name year+author and GBIF occurrence records in NL.
+import postgres from 'postgres';
+import {
+	chunks,
+	log as logErr,
+	saveCache,
+	slugify,
+	sparql,
+	stripHtml,
+	useCache,
+	getJson
+} from './planes-wiki';
+import {
+	groupOf,
+	GROUPS,
+	inat,
+	IUCN_BY_NUMBER,
+	PLACE_EUROPE,
+	PLACE_NL,
+	UA,
+	type InatList,
+	type InatTaxon
+} from './wildlife-inat';
+
+const args = process.argv.slice(2);
+const dryRun = args.includes('--dry-run');
+const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
+const cacheDir = opt('--cache');
+const reportNew = args.includes('--report-new');
+const SITE = 'https://stanrunge.dev/trackers/wildlife';
+const log = (...a: unknown[]) => (reportNew ? logErr(...a) : console.log(...a));
+
+// iconic taxon filter -> how many species to take (NL, Europe extra, world extra)
+const ICONIC: { iconic: string; taxonId: number; nl: number; eu: number; world: number }[] = [
+	{ iconic: 'Aves', taxonId: 3, nl: 450, eu: 250, world: 300 },
+	{ iconic: 'Mammalia', taxonId: 40151, nl: 120, eu: 120, world: 350 },
+	{ iconic: 'Reptilia', taxonId: 26036, nl: 40, eu: 100, world: 250 },
+	{ iconic: 'Amphibia', taxonId: 20978, nl: 30, eu: 50, world: 100 },
+	{ iconic: 'Actinopterygii', taxonId: 47178, nl: 150, eu: 80, world: 150 },
+	{ iconic: 'Insecta', taxonId: 47158, nl: 1300, eu: 300, world: 200 },
+	{ iconic: 'Arachnida', taxonId: 47119, nl: 150, eu: 50, world: 50 },
+	{ iconic: 'Mollusca', taxonId: 47115, nl: 180, eu: 50, world: 50 },
+	{ iconic: 'Animalia', taxonId: 1, nl: 250, eu: 80, world: 0 }
+];
+// Extra worldwide lists for groups without their own iconic taxon.
+const WORLD_EXTRA = [
+	{ taxonId: 47273, n: 60 }, // Elasmobranchii (sharks & rays)
+	{ taxonId: 47534, n: 30 }, // Cnidaria
+	{ taxonId: 47549, n: 30 }, // Echinodermata
+	{ taxonId: 85493, n: 40 } // Malacostraca (crabs, lobsters, ...)
+];
+const MIN_NL = 5;
+const FAMOUS =
+	`Panthera tigris, Panthera leo, Panthera pardus, Panthera uncia, Panthera onca, Acinonyx jubatus,
+ Ailuropoda melanoleuca, Ailurus fulgens, Ursus maritimus, Ursus arctos, Canis lupus, Gulo gulo, Lynx lynx,
+ Bison bonasus, Loxodonta africana, Elephas maximus, Giraffa camelopardalis, Okapia johnstoni, Equus quagga,
+ Hippopotamus amphibius, Ceratotherium simum, Diceros bicornis, Rhinoceros unicornis, Gorilla gorilla,
+ Gorilla beringei, Pan troglodytes, Pan paniscus, Pongo pygmaeus, Lemur catta, Bradypus variegatus,
+ Myrmecophaga tridactyla, Hydrochoerus hydrochaeris, Phascolarctos cinereus, Macropus giganteus,
+ Ornithorhynchus anatinus, Tachyglossus aculeatus, Sarcophilus harrisii, Balaenoptera musculus,
+ Megaptera novaeangliae, Physeter macrocephalus, Orcinus orca, Tursiops truncatus, Monodon monoceros,
+ Odobenus rosmarus, Enhydra lutris, Aptenodytes forsteri, Spheniscus demersus, Struthio camelus,
+ Dromaius novaehollandiae, Aquila chrysaetos, Haliaeetus leucocephalus, Harpia harpyja, Vultur gryphus,
+ Diomedea exulans, Fratercula arctica, Pavo cristatus, Ramphastos toco, Phoenicopterus roseus,
+ Crocodylus niloticus, Crocodylus porosus, Alligator mississippiensis, Varanus komodoensis, Chelonia mydas,
+ Dermochelys coriacea, Chelonoidis niger, Ophiophagus hannah, Iguana iguana, Chamaeleo calyptratus,
+ Ambystoma mexicanum, Andrias japonicus, Dendrobates tinctorius, Phyllobates terribilis,
+ Carcharodon carcharias, Rhincodon typus, Mobula birostris, Latimeria chalumnae, Hippocampus hippocampus,
+ Amphiprion ocellaris, Octopus vulgaris, Architeuthis dux, Danaus plexippus, Dynastes hercules,
+ Apis mellifera, Lucanus cervus, Aurelia aurita`
+		.split(',')
+		.map((s) => s.trim())
+		.filter(Boolean);
+// Pets, livestock and us.
+const EXCLUDE = new Set(
+	`Homo sapiens, Canis familiaris, Felis catus, Bos taurus, Equus caballus, Ovis aries, Capra hircus,
+ Sus domesticus, Gallus gallus, Equus asinus, Cavia porcellus, Bombyx mori, Lama glama, Vicugna pacos,
+ Bubalus bubalis, Camelus dromedarius, Bos indicus, Mesocricetus auratus, Meleagris gallopavo domesticus`
+		.split(',')
+		.map((s) => s.trim())
+);
+
+type Cand = { id: number; scope: 'nl' | 'europe' | 'world'; name: string };
+
+async function speciesCounts(
+	place: number,
+	iconic: string,
+	n: number
+): Promise<InatList['results']> {
+	const out: InatList['results'] = [];
+	for (let page = 1; out.length < n && page <= 10; page++) {
+		const res = await inat(
+			`/observations/species_counts?place_id=${place}&iconic_taxa=${iconic}&captive=false&quality_grade=research&per_page=500&page=${page}&locale=en`
+		);
+		out.push(...res.results);
+		if (res.results.length < 500) break;
+	}
+	return out;
+}
+async function topWorld(taxonId: number, n: number): Promise<InatTaxon[]> {
+	const out: InatTaxon[] = [];
+	for (let page = 1; out.length < n && page <= 10; page++) {
+		const res = await inat<{ results: InatTaxon[] }>(
+			`/taxa?rank=species&taxon_id=${taxonId}&is_active=true&order_by=observations_count&order=desc&per_page=200&page=${page}`
+		);
+		out.push(...res.results);
+		if (res.results.length < 200) break;
+	}
+	return out;
+}
+
+async function collect(existing: number[]): Promise<Map<number, Cand>> {
+	const cands = new Map<number, Cand>();
+	const add = (t: InatTaxon, scope: Cand['scope']) => {
+		if (t.rank !== 'species' || EXCLUDE.has(t.name) || cands.has(t.id)) return false;
+		cands.set(t.id, { id: t.id, scope, name: t.name });
+		return true;
+	};
+	for (const g of ICONIC) {
+		const nl = await speciesCounts(PLACE_NL, g.iconic, g.nl * 2);
+		let k = 0;
+		for (const r of nl) if (k < g.nl && r.count >= MIN_NL && add(r.taxon, 'nl')) k++;
+		const eu = await speciesCounts(PLACE_EUROPE, g.iconic, g.eu * 2 + g.nl);
+		let e = 0;
+		for (const r of eu) if (e < g.eu && add(r.taxon, 'europe')) e++;
+		let w = 0;
+		if (g.world)
+			for (const t of await topWorld(g.taxonId, g.world * 2))
+				if (w < g.world && add(t, 'world')) w++;
+		log(`  ${g.iconic}: ${k} NL, +${e} Europe, +${w} world`);
+	}
+	for (const x of WORLD_EXTRA) {
+		let w = 0;
+		for (const t of await topWorld(x.taxonId, x.n * 2)) if (w < x.n && add(t, 'world')) w++;
+	}
+	let famous = 0;
+	for (const name of FAMOUS) {
+		const res = await inat<{ results: InatTaxon[] }>(
+			`/taxa?q=${encodeURIComponent(name)}&rank=species&is_active=true&per_page=5`
+		);
+		const t = res.results.find((r) => r.name.toLowerCase() === name.toLowerCase());
+		if (t && add(t, 'world')) famous++;
+		if (!t) log(`  famous species not found: ${name}`);
+	}
+	log(`  +${famous} famous species`);
+	for (const id of existing) if (!cands.has(id)) cands.set(id, { id, scope: 'world', name: '' });
+	return cands;
+}
+
+async function details(ids: number[], locale: 'en' | 'nl') {
+	const out = new Map<number, InatTaxon>();
+	for (const [i, batch] of chunks(ids, 30).entries()) {
+		const res = await inat<{ results: InatTaxon[] }>(
+			`/taxa/${batch.join(',')}?locale=${locale}&preferred_place_id=${PLACE_NL}`
+		);
+		for (const t of res.results) out.set(t.id, t);
+		if (i % 20 === 0) log(`  taxa (${locale}) ${Math.min((i + 1) * 30, ids.length)}/${ids.length}`);
+	}
+	return out;
+}
+async function placeCounts(ids: number[], place: number) {
+	const out = new Map<number, number>();
+	for (const batch of chunks(ids, 200)) {
+		const res = await inat(
+			`/observations/species_counts?place_id=${place}&taxon_id=${batch.join(',')}&captive=false&quality_grade=research&per_page=500`
+		);
+		for (const r of res.results) out.set(r.taxon.id, r.count);
+	}
+	return out;
+}
+
+type Wd = {
+	item: string;
+	nl: string | null;
+	iucn: string | null;
+	gbif: number | null;
+	nlwiki: string | null;
+	mass: number | null;
+	len: number | null;
+	life: number | null;
+	year: number | null;
+	author: string | null;
+};
+const WD_IUCN: Record<string, string> = {
+	Q211005: 'LC',
+	Q719675: 'NT',
+	Q278113: 'VU',
+	Q11394: 'EN',
+	Q219127: 'CR',
+	Q239509: 'EW',
+	Q237350: 'EX',
+	Q3245245: 'DD'
+};
+async function wikidata(ids: number[]) {
+	const out = new Map<number, Wd>();
+	for (const batch of chunks(ids, 250)) {
+		const rows =
+			await sparql(`SELECT ?inat ?item ?nl ?iucn ?gbif ?nlwiki ?mass ?len ?life ?year ?authorLabel WHERE {
+			VALUES ?inat { ${batch.map((id) => `"${id}"`).join(' ')} }
+			?item wdt:P3151 ?inat .
+			OPTIONAL { ?item rdfs:label ?nl FILTER(lang(?nl) = "nl") }
+			OPTIONAL { ?item wdt:P141 ?iucn }
+			OPTIONAL { ?item wdt:P846 ?gbif }
+			OPTIONAL { ?nlwiki schema:about ?item ; schema:isPartOf <https://nl.wikipedia.org/> }
+			OPTIONAL { ?item p:P2067/psn:P2067/wikibase:quantityAmount ?mass }
+			OPTIONAL { ?item p:P2043/psn:P2043/wikibase:quantityAmount ?len }
+			OPTIONAL { ?item p:P2250/psn:P2250/wikibase:quantityAmount ?life }
+			OPTIONAL { ?item p:P225 ?st . ?st pq:P574 ?year .
+				OPTIONAL { ?st pq:P405 ?author . ?author rdfs:label ?authorLabel FILTER(lang(?authorLabel) = "en") } }
+		}`);
+		for (const r of rows) {
+			const id = Number(r.inat);
+			const prev = out.get(id);
+			const num = (v: string | undefined) =>
+				v != null && Number.isFinite(Number(v)) ? Number(v) : null;
+			const max = (a: number | null | undefined, b: number | null) =>
+				a == null ? b : b == null ? a : Math.max(a, b);
+			const yr = r.year ? Number(r.year.slice(0, 4)) : null;
+			out.set(id, {
+				item: r.item.split('/').at(-1)!,
+				nl: prev?.nl ?? r.nl ?? null,
+				iucn: prev?.iucn ?? (r.iucn ? (WD_IUCN[r.iucn.split('/').at(-1)!] ?? null) : null),
+				gbif: prev?.gbif ?? num(r.gbif),
+				nlwiki:
+					prev?.nlwiki ??
+					(r.nlwiki ? decodeURIComponent(r.nlwiki.split('/wiki/')[1]).replace(/_/g, ' ') : null),
+				mass: max(prev?.mass, num(r.mass)),
+				len: max(prev?.len, num(r.len)),
+				life: max(prev?.life, num(r.life)),
+				year: prev?.year ?? (yr && yr > 1700 && yr <= new Date().getFullYear() ? yr : null),
+				author: prev?.author ?? r.authorLabel ?? null
+			});
+		}
+	}
+	return out;
+}
+
+async function gbifNl(): Promise<Map<number, number> | null> {
+	const out = new Map<number, number>();
+	try {
+		for (let offset = 0; offset < 100_000; offset += 5000) {
+			const res = await getJson<{ facets: { counts: { name: string; count: number }[] }[] }>(
+				`https://api.gbif.org/v1/occurrence/search?country=NL&taxonKey=1&limit=0&facet=speciesKey&facetLimit=5000&facetOffset=${offset}`,
+				{ headers: { 'user-agent': UA } }
+			);
+			const counts = res.facets[0]?.counts ?? [];
+			for (const c of counts) out.set(Number(c.name), c.count);
+			if (counts.length < 5000) return out;
+		}
+		return out;
+	} catch (e) {
+		log(`  GBIF facet failed: ${(e as Error).message}`);
+		return null;
+	}
+}
+
+const fmt = (n: number) => n.toLocaleString('en-GB');
+const trimSummary = (html: string | null | undefined) => {
+	if (!html) return null;
+	const text = stripHtml(html).replace(/\s*\[\d+\]/g, '');
+	const sentences = text.match(/[^.!?]+[.!?]+(\s|$)/g) ?? [text];
+	let out = '';
+	for (const s of sentences) {
+		if (out && (out + s).length > 650) break;
+		out += s;
+	}
+	return out.trim() || null;
+};
+const IUCN_TEXT: Record<string, string> = {
+	LC: 'Least Concern',
+	NT: 'Near Threatened',
+	VU: 'Vulnerable',
+	EN: 'Endangered',
+	CR: 'Critically Endangered',
+	EW: 'Extinct in the Wild',
+	EX: 'Extinct',
+	DD: 'Data Deficient'
+};
+
+function funFacts(s: Row, rankInGroup: number | null, groupName: string): string[] {
+	const f: string[] = [];
+	if (s.nl_obs > 0)
+		f.push(
+			`👀 Spotted ${fmt(s.nl_obs)} times in the Netherlands on iNaturalist${rankInGroup && rankInGroup <= 50 ? `: the #${rankInGroup} most seen of the ${groupName.toLowerCase()} in NL` : ''}.`
+		);
+	else
+		f.push(
+			`✈️ No confirmed wild sightings in the Netherlands on iNaturalist: you'd have to travel for this one.`
+		);
+	if (s.world_obs > 0) f.push(`🌍 ${fmt(s.world_obs)} observations worldwide on iNaturalist.`);
+	if (s.mass_kg != null) {
+		const kg = s.mass_kg;
+		if (kg >= 1000)
+			f.push(
+				`🐘 Can weigh about ${fmt(Math.round(kg / 1000))} tonnes: as much as ${fmt(Math.max(1, Math.round(kg / 1400)))} cars.`
+			);
+		else if (kg >= 1) f.push(`⚖️ Weighs up to about ${fmt(Math.round(kg * 10) / 10)} kg.`);
+		else if (kg >= 0.001) f.push(`🪶 Weighs only about ${fmt(Math.round(kg * 1000))} g.`);
+	}
+	if (s.length_m != null) {
+		const m = s.length_m;
+		if (m >= 2)
+			f.push(
+				`📏 Up to ${fmt(Math.round(m * 10) / 10)} m long: ${(m / 1.8).toFixed(1)}× a tall person.`
+			);
+		else if (m >= 0.01) f.push(`📏 About ${fmt(Math.round(m * 100))} cm long.`);
+		else f.push(`🔬 Only about ${fmt(Math.round(m * 1000 * 10) / 10)} mm long.`);
+	}
+	if (s.lifespan_years != null && s.lifespan_years >= 1)
+		f.push(`⏳ Can live up to ${fmt(Math.round(s.lifespan_years))} years.`);
+	if (s.named_year)
+		f.push(
+			`📜 Scientifically named ${s.named_by ? `by ${s.named_by} ` : ''}in ${s.named_year} (${new Date().getFullYear() - s.named_year} years ago).`
+		);
+	if (s.nl_establishment === 'introduced')
+		f.push(`🧳 Not originally Dutch: it was introduced to the Netherlands.`);
+	if (s.iucn && ['VU', 'EN', 'CR', 'EW'].includes(s.iucn))
+		f.push(`⚠️ Listed as ${IUCN_TEXT[s.iucn]} on the IUCN Red List.`);
+	if (s.extinct) f.push(`🪦 This species is extinct.`);
+	if (s.nl_status) f.push(`🇳🇱 Dutch status: ${s.nl_status}.`);
+	return f;
+}
+
+type Row = {
+	inat_id: number;
+	group: string;
+	slug: string;
+	scientific_name: string;
+	name_en: string | null;
+	name_nl: string | null;
+	search: string;
+	gbif_key: number | null;
+	wikidata: string | null;
+	wikipedia_en: string | null;
+	wikipedia_nl: string | null;
+	class_name: string | null;
+	order_name: string | null;
+	family_name: string | null;
+	iucn: string | null;
+	nl_status: string | null;
+	nl_establishment: string | null;
+	extinct: boolean;
+	nl_obs: number;
+	europe_obs: number;
+	world_obs: number;
+	gbif_nl: number | null;
+	scope: string;
+	summary: string | null;
+	summary_nl: string | null;
+	facts: string[];
+	mass_kg: number | null;
+	length_m: number | null;
+	lifespan_years: number | null;
+	named_year: number | null;
+	named_by: string | null;
+	sources: string[];
+};
+const wikiTitle = (url: string | null | undefined, host: string) => {
+	if (!url || !url.includes(host)) return null;
+	try {
+		return decodeURIComponent(url.split('/wiki/')[1] ?? '').replace(/_/g, ' ') || null;
+	} catch {
+		return null;
+	}
+};
+
+async function main() {
+	await useCache(cacheDir, 'wildlife-api.json');
+	const url = process.env.DB_URL;
+	if (!dryRun && !url) throw new Error('Set DB_URL (or use --dry-run)');
+	const sql = url ? postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
+	try {
+		const existing = sql
+			? await sql<{ inat_id: number; dex: number }[]>`select inat_id, dex from wildlife.species`
+			: [];
+		log('collecting species lists…');
+		const cands = await collect(existing.map((e) => e.inat_id));
+		const ids = [...cands.keys()];
+		log(`${ids.length} candidate species; fetching details…`);
+		const en = await details(ids, 'en');
+		const nl = await details(ids, 'nl');
+		await saveCache(true);
+		log('NL / Europe counts…');
+		const nlCounts = await placeCounts(ids, PLACE_NL);
+		const euCounts = await placeCounts(ids, PLACE_EUROPE);
+		await saveCache(true);
+		log('Wikidata…');
+		const wd = await wikidata(ids);
+		log('GBIF NL occurrence counts…');
+		const gbif = await gbifNl();
+
+		const rows: Row[] = [];
+		for (const id of ids) {
+			const t = en.get(id);
+			const tn = nl.get(id);
+			if (!t || t.rank !== 'species' || EXCLUDE.has(t.name)) continue;
+			const anc = t.ancestors ?? [];
+			if (!anc.some((a) => a.name === 'Animalia')) continue;
+			const g = groupOf(anc.map((a) => a.name));
+			const w = wd.get(id);
+			const nameEn = t.preferred_common_name || t.english_common_name || null;
+			let nameNl: string | null = tn?.preferred_common_name || null;
+			if (nameNl && nameEn && nameNl === nameEn && w?.nl && w.nl !== t.name && w.nl !== nameEn)
+				nameNl = w.nl;
+			if (!nameNl && w?.nl && w.nl.toLowerCase() !== t.name.toLowerCase()) nameNl = w.nl;
+			if (nameNl) nameNl = nameNl.charAt(0).toUpperCase() + nameNl.slice(1);
+			const statuses = t.conservation_statuses ?? [];
+			const global = statuses.find((s) => !s.place && s.authority === 'IUCN Red List');
+			let iucn = global?.iucn != null ? (IUCN_BY_NUMBER[global.iucn] ?? null) : null;
+			if (!iucn && global?.status && /^(LC|NT|VU|EN|CR|EW|EX|DD)$/i.test(global.status))
+				iucn = global.status.toUpperCase();
+			if (!iucn) iucn = w?.iucn ?? null;
+			if (t.extinct && !iucn) iucn = 'EX';
+			const nlS = statuses.find((s) => s.place?.id === PLACE_NL);
+			const em = (tn ?? t).establishment_means;
+			const nlEst = em?.place?.id === PLACE_NL ? (em.establishment_means ?? null) : null;
+			const nlObs = nlCounts.get(id) ?? 0;
+			const cand = cands.get(id)!;
+			const rank = (r: string) => anc.find((a) => a.rank === r)?.name ?? null;
+			const wikipediaEn = wikiTitle(t.wikipedia_url, 'en.wikipedia.org');
+			const wikipediaNl = w?.nlwiki ?? wikiTitle(tn?.wikipedia_url, 'nl.wikipedia.org');
+			const summaryNl = tn?.wikipedia_url?.includes('nl.wikipedia.org')
+				? trimSummary(tn.wikipedia_summary)
+				: null;
+			const row: Row = {
+				inat_id: id,
+				group: g.slug,
+				slug: slugify(t.name),
+				scientific_name: t.name,
+				name_en: nameEn,
+				name_nl: nameNl,
+				search: [nameNl, nameEn, t.name, rank('order'), rank('family'), rank('genus')]
+					.filter(Boolean)
+					.join(' ')
+					.toLowerCase(),
+				gbif_key: w?.gbif ?? null,
+				wikidata: w?.item ?? null,
+				wikipedia_en: wikipediaEn,
+				wikipedia_nl: wikipediaNl,
+				class_name: rank('class'),
+				order_name: rank('order'),
+				family_name: rank('family'),
+				iucn,
+				nl_status: nlS ? nlS.description?.trim() || nlS.status : null,
+				nl_establishment: nlEst,
+				extinct: !!t.extinct,
+				nl_obs: nlObs,
+				europe_obs: euCounts.get(id) ?? 0,
+				world_obs: t.observations_count ?? 0,
+				gbif_nl: w?.gbif != null && gbif ? (gbif.get(w.gbif) ?? 0) : null,
+				scope:
+					nlObs >= MIN_NL
+						? 'nl'
+						: (euCounts.get(id) ?? 0) > 0
+							? 'europe'
+							: cand.scope === 'nl'
+								? 'europe'
+								: 'world',
+				summary: trimSummary(t.wikipedia_summary),
+				summary_nl: summaryNl,
+				facts: [],
+				mass_kg: w?.mass != null && w.mass > 0 && w.mass < 200_000 ? w.mass : null,
+				length_m: w?.len != null && w.len > 0 && w.len < 40 ? w.len : null,
+				lifespan_years:
+					w?.life != null && w.life > 0 ? Math.round((w.life / 31_557_600) * 10) / 10 : null,
+				named_year: w?.year ?? null,
+				named_by: w?.author ?? null,
+				sources: [
+					'iNaturalist',
+					...(w ? ['Wikidata'] : []),
+					...(w?.gbif != null && gbif ? ['GBIF'] : [])
+				]
+			};
+			if (row.lifespan_years != null && (row.lifespan_years < 0.01 || row.lifespan_years > 500))
+				row.lifespan_years = null;
+			rows.push(row);
+		}
+		// Unique slugs (scientific names are unique on iNaturalist, but be safe).
+		const seen = new Set<string>();
+		for (const r of rows) {
+			let s = r.slug;
+			for (let n = 2; seen.has(s); n++) s = `${r.slug}-${n}`;
+			r.slug = s;
+			seen.add(s);
+		}
+		// Fun facts with the rank among the group in NL.
+		const byGroup = new Map<string, Row[]>();
+		for (const r of rows) byGroup.set(r.group, [...(byGroup.get(r.group) ?? []), r]);
+		for (const [slug, list] of byGroup) {
+			const g = GROUPS.find((x) => x.slug === slug)!;
+			const ranked = list.filter((r) => r.nl_obs > 0).toSorted((a, b) => b.nl_obs - a.nl_obs);
+			for (const r of list) {
+				const i = ranked.indexOf(r);
+				r.facts = funFacts(r, i >= 0 ? i + 1 : null, g.name);
+			}
+		}
+		const summary = GROUPS.map((g) => `${g.slug}: ${byGroup.get(g.slug)?.length ?? 0}`).join(', ');
+		log(`${rows.length} species (${summary})`);
+		if (dryRun || !sql) {
+			log(JSON.stringify(rows.slice(0, 3), null, 1));
+			return;
+		}
+
+		const known = new Map(existing.map((e) => [e.inat_id, e.dex]));
+		let nextDex = Math.max(0, ...existing.map((e) => e.dex)) + 1;
+		const order = (r: Row) => GROUPS.findIndex((g) => g.slug === r.group);
+		const scopeOrder = { nl: 0, europe: 1, world: 2 } as Record<string, number>;
+		const sorted = rows.toSorted(
+			(a, b) =>
+				order(a) - order(b) ||
+				scopeOrder[a.scope] - scopeOrder[b.scope] ||
+				b.nl_obs - a.nl_obs ||
+				b.world_obs - a.world_obs ||
+				a.scientific_name.localeCompare(b.scientific_name)
+		);
+		const added: Row[] = [];
+		const dexOf = new Map<number, number>();
+		for (const r of sorted) {
+			const d = known.get(r.inat_id);
+			if (d != null) dexOf.set(r.inat_id, d);
+			else {
+				dexOf.set(r.inat_id, nextDex++);
+				added.push(r);
+			}
+		}
+		await sql.begin(async (tx) => {
+			const groupIds = new Map<string, number>();
+			for (const [i, g] of GROUPS.entries()) {
+				const [row] = await tx<{ id: number }[]>`
+					insert into wildlife.groups (slug, name, name_nl, emoji, sort_order)
+					values (${g.slug}, ${g.name}, ${g.nameNl}, ${g.emoji}, ${i})
+					on conflict (slug) do update set name = excluded.name, name_nl = excluded.name_nl,
+						emoji = excluded.emoji, sort_order = excluded.sort_order
+					returning id`;
+				groupIds.set(g.slug, row.id);
+			}
+			// Avoid unique-slug clashes while rows move around.
+			await tx`update wildlife.species set slug = slug || '~' || id where slug not like '%~%'`;
+			for (const r of sorted) {
+				const { group, ...rest } = r;
+				const row = { ...rest, dex: dexOf.get(r.inat_id)!, group_id: groupIds.get(group)! };
+				await tx`
+					insert into wildlife.species ${tx(row)}
+					on conflict (inat_id) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'inat_id' && k !== 'dex') as (keyof typeof row)[])}`;
+			}
+			// Restore slugs of species that were not part of this run (kept, not deleted).
+			await tx`update wildlife.species set slug = split_part(slug, '~', 1) where slug like '%~%'`;
+			await tx`
+				update wildlife.groups g set
+					species_count = (select count(*) from wildlife.species s where s.group_id = g.id),
+					nl_count = (select count(*) from wildlife.species s where s.group_id = g.id and s.nl_obs > 0)`;
+		});
+		log(`written: ${rows.length} species, ${added.length} new`);
+		if (reportNew) {
+			const [totals] = await sql`
+				select (select count(*)::int from wildlife.species) as species,
+					(select count(*)::int from wildlife.groups where species_count > 0) as groups,
+					(select count(*)::int from wildlife.photos) as photos`;
+			console.log(
+				JSON.stringify(
+					{
+						generatedAt: new Date().toISOString(),
+						totals,
+						newSpecies: (existing.length ? added : added.slice(0, 0)).map((r) => ({
+							dex: dexOf.get(r.inat_id),
+							inatId: r.inat_id,
+							name: r.name_en ?? r.scientific_name,
+							nameNl: r.name_nl,
+							scientificName: r.scientific_name,
+							group: r.group,
+							iucn: r.iucn,
+							nlObs: r.nl_obs,
+							url: `${SITE}/${r.group}/${r.slug}`
+						}))
+					},
+					null,
+					1
+				)
+			);
+		}
+	} finally {
+		await saveCache(true);
+		await sql?.end();
+	}
+}
+
+await main();