wildlife: iNaturalist/Wikidata/GBIF import, CC photos and refresh script

scripts/wildlife-import.ts

@@ -0,0 +1,603 @@
+// Builds the public wildlife pokedex (/trackers/wildlife, schema `wildlife`) from open data.
+//
+//   DB_URL=... bun scripts/wildlife-import.ts               fetch sources and write to the database
+//   bun scripts/wildlife-import.ts --dry-run                fetch + build only, print a summary
+//   bun scripts/wildlife-import.ts --cache /tmp/wildlife    reuse API responses from that folder
+//   bun scripts/wildlife-import.ts --report-new             JSON of species new since the last run on stdout
+//
+// Which species (wild animals only, no pets/livestock, no captive observations):
+//  - the most observed species per group in the Netherlands (iNaturalist research grade),
+//  - the most observed species per group in Europe,
+//  - the most observed species per group worldwide, plus a short list of famous animals.
+// Species already in the dex are always kept (refreshed, never deleted), so Dex numbers stay
+// stable and new species get the next number.
+// Per species: names (iNaturalist nl/en locale, Wikidata nl label), taxonomy, IUCN Red List
+// (iNaturalist, Wikidata P141 fallback), Dutch red list status, establishment in NL, iNat
+// observation counts (NL, Europe, world), Wikipedia intros (en + nl via iNaturalist),
+// Wikidata mass/length/lifespan/name year+author and GBIF occurrence records in NL.
+import postgres from 'postgres';
+import {
+	chunks,
+	log as logErr,
+	saveCache,
+	slugify,
+	sparql,
+	stripHtml,
+	useCache,
+	getJson
+} from './planes-wiki';
+import {
+	groupOf,
+	GROUPS,
+	inat,
+	IUCN_BY_NUMBER,
+	PLACE_EUROPE,
+	PLACE_NL,
+	UA,
+	type InatList,
+	type InatTaxon
+} from './wildlife-inat';
+
+const args = process.argv.slice(2);
+const dryRun = args.includes('--dry-run');
+const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
+const cacheDir = opt('--cache');
+const reportNew = args.includes('--report-new');
+const SITE = 'https://stanrunge.dev/trackers/wildlife';
+const log = (...a: unknown[]) => (reportNew ? logErr(...a) : console.log(...a));
+
+// iconic taxon filter -> how many species to take (NL, Europe extra, world extra)
+const ICONIC: { iconic: string; taxonId: number; nl: number; eu: number; world: number }[] = [
+	{ iconic: 'Aves', taxonId: 3, nl: 450, eu: 250, world: 300 },
+	{ iconic: 'Mammalia', taxonId: 40151, nl: 120, eu: 120, world: 350 },
+	{ iconic: 'Reptilia', taxonId: 26036, nl: 40, eu: 100, world: 250 },
+	{ iconic: 'Amphibia', taxonId: 20978, nl: 30, eu: 50, world: 100 },
+	{ iconic: 'Actinopterygii', taxonId: 47178, nl: 150, eu: 80, world: 150 },
+	{ iconic: 'Insecta', taxonId: 47158, nl: 1300, eu: 300, world: 200 },
+	{ iconic: 'Arachnida', taxonId: 47119, nl: 150, eu: 50, world: 50 },
+	{ iconic: 'Mollusca', taxonId: 47115, nl: 180, eu: 50, world: 50 },
+	{ iconic: 'Animalia', taxonId: 1, nl: 250, eu: 80, world: 0 }
+];
+// Extra worldwide lists for groups without their own iconic taxon.
+const WORLD_EXTRA = [
+	{ taxonId: 47273, n: 60 }, // Elasmobranchii (sharks & rays)
+	{ taxonId: 47534, n: 30 }, // Cnidaria
+	{ taxonId: 47549, n: 30 }, // Echinodermata
+	{ taxonId: 85493, n: 40 } // Malacostraca (crabs, lobsters, ...)
+];
+const MIN_NL = 5;
+const FAMOUS =
+	`Panthera tigris, Panthera leo, Panthera pardus, Panthera uncia, Panthera onca, Acinonyx jubatus,
+ Ailuropoda melanoleuca, Ailurus fulgens, Ursus maritimus, Ursus arctos, Canis lupus, Gulo gulo, Lynx lynx,
+ Bison bonasus, Loxodonta africana, Elephas maximus, Giraffa camelopardalis, Okapia johnstoni, Equus quagga,
+ Hippopotamus amphibius, Ceratotherium simum, Diceros bicornis, Rhinoceros unicornis, Gorilla gorilla,
+ Gorilla beringei, Pan troglodytes, Pan paniscus, Pongo pygmaeus, Lemur catta, Bradypus variegatus,
+ Myrmecophaga tridactyla, Hydrochoerus hydrochaeris, Phascolarctos cinereus, Macropus giganteus,
+ Ornithorhynchus anatinus, Tachyglossus aculeatus, Sarcophilus harrisii, Balaenoptera musculus,
+ Megaptera novaeangliae, Physeter macrocephalus, Orcinus orca, Tursiops truncatus, Monodon monoceros,
+ Odobenus rosmarus, Enhydra lutris, Aptenodytes forsteri, Spheniscus demersus, Struthio camelus,
+ Dromaius novaehollandiae, Aquila chrysaetos, Haliaeetus leucocephalus, Harpia harpyja, Vultur gryphus,
+ Diomedea exulans, Fratercula arctica, Pavo cristatus, Ramphastos toco, Phoenicopterus roseus,
+ Crocodylus niloticus, Crocodylus porosus, Alligator mississippiensis, Varanus komodoensis, Chelonia mydas,
+ Dermochelys coriacea, Chelonoidis niger, Ophiophagus hannah, Iguana iguana, Chamaeleo calyptratus,
+ Ambystoma mexicanum, Andrias japonicus, Dendrobates tinctorius, Phyllobates terribilis,
+ Carcharodon carcharias, Rhincodon typus, Mobula birostris, Latimeria chalumnae, Hippocampus hippocampus,
+ Amphiprion ocellaris, Octopus vulgaris, Architeuthis dux, Danaus plexippus, Dynastes hercules,
+ Apis mellifera, Lucanus cervus, Aurelia aurita`
+		.split(',')
+		.map((s) => s.trim())
+		.filter(Boolean);
+// Pets, livestock and us.
+const EXCLUDE = new Set(
+	`Homo sapiens, Canis familiaris, Felis catus, Bos taurus, Equus caballus, Ovis aries, Capra hircus,
+ Sus domesticus, Gallus gallus, Equus asinus, Cavia porcellus, Bombyx mori, Lama glama, Vicugna pacos,
+ Bubalus bubalis, Camelus dromedarius, Bos indicus, Mesocricetus auratus, Meleagris gallopavo domesticus`
+		.split(',')
+		.map((s) => s.trim())
+);
+
+type Cand = { id: number; scope: 'nl' | 'europe' | 'world'; name: string };
+
+async function speciesCounts(
+	place: number,
+	iconic: string,
+	n: number
+): Promise<InatList['results']> {
+	const out: InatList['results'] = [];
+	for (let page = 1; out.length < n && page <= 10; page++) {
+		const res = await inat(
+			`/observations/species_counts?place_id=${place}&iconic_taxa=${iconic}&captive=false&quality_grade=research&per_page=500&page=${page}&locale=en`
+		);
+		out.push(...res.results);
+		if (res.results.length < 500) break;
+	}
+	return out;
+}
+async function topWorld(taxonId: number, n: number): Promise<InatTaxon[]> {
+	const out: InatTaxon[] = [];
+	for (let page = 1; out.length < n && page <= 10; page++) {
+		const res = await inat<{ results: InatTaxon[] }>(
+			`/taxa?rank=species&taxon_id=${taxonId}&is_active=true&order_by=observations_count&order=desc&per_page=200&page=${page}`
+		);
+		out.push(...res.results);
+		if (res.results.length < 200) break;
+	}
+	return out;
+}
+
+async function collect(existing: number[]): Promise<Map<number, Cand>> {
+	const cands = new Map<number, Cand>();
+	const add = (t: InatTaxon, scope: Cand['scope']) => {
+		if (t.rank !== 'species' || EXCLUDE.has(t.name) || cands.has(t.id)) return false;
+		cands.set(t.id, { id: t.id, scope, name: t.name });
+		return true;
+	};
+	for (const g of ICONIC) {
+		const nl = await speciesCounts(PLACE_NL, g.iconic, g.nl * 2);
+		let k = 0;
+		for (const r of nl) if (k < g.nl && r.count >= MIN_NL && add(r.taxon, 'nl')) k++;
+		const eu = await speciesCounts(PLACE_EUROPE, g.iconic, g.eu * 2 + g.nl);
+		let e = 0;
+		for (const r of eu) if (e < g.eu && add(r.taxon, 'europe')) e++;
+		let w = 0;
+		if (g.world)
+			for (const t of await topWorld(g.taxonId, g.world * 2))
+				if (w < g.world && add(t, 'world')) w++;
+		log(`  ${g.iconic}: ${k} NL, +${e} Europe, +${w} world`);
+	}
+	for (const x of WORLD_EXTRA) {
+		let w = 0;
+		for (const t of await topWorld(x.taxonId, x.n * 2)) if (w < x.n && add(t, 'world')) w++;
+	}
+	let famous = 0;
+	for (const name of FAMOUS) {
+		const res = await inat<{ results: InatTaxon[] }>(
+			`/taxa?q=${encodeURIComponent(name)}&rank=species&is_active=true&per_page=5`
+		);
+		const t = res.results.find((r) => r.name.toLowerCase() === name.toLowerCase());
+		if (t && add(t, 'world')) famous++;
+		if (!t) log(`  famous species not found: ${name}`);
+	}
+	log(`  +${famous} famous species`);
+	for (const id of existing) if (!cands.has(id)) cands.set(id, { id, scope: 'world', name: '' });
+	return cands;
+}
+
+async function details(ids: number[], locale: 'en' | 'nl') {
+	const out = new Map<number, InatTaxon>();
+	for (const [i, batch] of chunks(ids, 30).entries()) {
+		const res = await inat<{ results: InatTaxon[] }>(
+			`/taxa/${batch.join(',')}?locale=${locale}&preferred_place_id=${PLACE_NL}`
+		);
+		for (const t of res.results) out.set(t.id, t);
+		if (i % 20 === 0) log(`  taxa (${locale}) ${Math.min((i + 1) * 30, ids.length)}/${ids.length}`);
+	}
+	return out;
+}
+async function placeCounts(ids: number[], place: number) {
+	const out = new Map<number, number>();
+	for (const batch of chunks(ids, 200)) {
+		const res = await inat(
+			`/observations/species_counts?place_id=${place}&taxon_id=${batch.join(',')}&captive=false&quality_grade=research&per_page=500`
+		);
+		for (const r of res.results) out.set(r.taxon.id, r.count);
+	}
+	return out;
+}
+
+type Wd = {
+	item: string;
+	nl: string | null;
+	iucn: string | null;
+	gbif: number | null;
+	nlwiki: string | null;
+	mass: number | null;
+	len: number | null;
+	life: number | null;
+	year: number | null;
+	author: string | null;
+};
+const WD_IUCN: Record<string, string> = {
+	Q211005: 'LC',
+	Q719675: 'NT',
+	Q278113: 'VU',
+	Q11394: 'EN',
+	Q219127: 'CR',
+	Q239509: 'EW',
+	Q237350: 'EX',
+	Q3245245: 'DD'
+};
+async function wikidata(ids: number[]) {
+	const out = new Map<number, Wd>();
+	for (const batch of chunks(ids, 250)) {
+		const rows =
+			await sparql(`SELECT ?inat ?item ?nl ?iucn ?gbif ?nlwiki ?mass ?len ?life ?year ?authorLabel WHERE {
+			VALUES ?inat { ${batch.map((id) => `"${id}"`).join(' ')} }
+			?item wdt:P3151 ?inat .
+			OPTIONAL { ?item rdfs:label ?nl FILTER(lang(?nl) = "nl") }
+			OPTIONAL { ?item wdt:P141 ?iucn }
+			OPTIONAL { ?item wdt:P846 ?gbif }
+			OPTIONAL { ?nlwiki schema:about ?item ; schema:isPartOf <https://nl.wikipedia.org/> }
+			OPTIONAL { ?item p:P2067/psn:P2067/wikibase:quantityAmount ?mass }
+			OPTIONAL { ?item p:P2043/psn:P2043/wikibase:quantityAmount ?len }
+			OPTIONAL { ?item p:P2250/psn:P2250/wikibase:quantityAmount ?life }
+			OPTIONAL { ?item p:P225 ?st . ?st pq:P574 ?year .
+				OPTIONAL { ?st pq:P405 ?author . ?author rdfs:label ?authorLabel FILTER(lang(?authorLabel) = "en") } }
+		}`);
+		for (const r of rows) {
+			const id = Number(r.inat);
+			const prev = out.get(id);
+			const num = (v: string | undefined) =>
+				v != null && Number.isFinite(Number(v)) ? Number(v) : null;
+			const max = (a: number | null | undefined, b: number | null) =>
+				a == null ? b : b == null ? a : Math.max(a, b);
+			const yr = r.year ? Number(r.year.slice(0, 4)) : null;
+			out.set(id, {
+				item: r.item.split('/').at(-1)!,
+				nl: prev?.nl ?? r.nl ?? null,
+				iucn: prev?.iucn ?? (r.iucn ? (WD_IUCN[r.iucn.split('/').at(-1)!] ?? null) : null),
+				gbif: prev?.gbif ?? num(r.gbif),
+				nlwiki:
+					prev?.nlwiki ??
+					(r.nlwiki ? decodeURIComponent(r.nlwiki.split('/wiki/')[1]).replace(/_/g, ' ') : null),
+				mass: max(prev?.mass, num(r.mass)),
+				len: max(prev?.len, num(r.len)),
+				life: max(prev?.life, num(r.life)),
+				year: prev?.year ?? (yr && yr > 1700 && yr <= new Date().getFullYear() ? yr : null),
+				author: prev?.author ?? r.authorLabel ?? null
+			});
+		}
+	}
+	return out;
+}
+
+async function gbifNl(): Promise<Map<number, number> | null> {
+	const out = new Map<number, number>();
+	try {
+		for (let offset = 0; offset < 100_000; offset += 5000) {
+			const res = await getJson<{ facets: { counts: { name: string; count: number }[] }[] }>(
+				`https://api.gbif.org/v1/occurrence/search?country=NL&taxonKey=1&limit=0&facet=speciesKey&facetLimit=5000&facetOffset=${offset}`,
+				{ headers: { 'user-agent': UA } }
+			);
+			const counts = res.facets[0]?.counts ?? [];
+			for (const c of counts) out.set(Number(c.name), c.count);
+			if (counts.length < 5000) return out;
+		}
+		return out;
+	} catch (e) {
+		log(`  GBIF facet failed: ${(e as Error).message}`);
+		return null;
+	}
+}
+
+const fmt = (n: number) => n.toLocaleString('en-GB');
+const trimSummary = (html: string | null | undefined) => {
+	if (!html) return null;
+	const text = stripHtml(html).replace(/\s*\[\d+\]/g, '');
+	const sentences = text.match(/[^.!?]+[.!?]+(\s|$)/g) ?? [text];
+	let out = '';
+	for (const s of sentences) {
+		if (out && (out + s).length > 650) break;
+		out += s;
+	}
+	return out.trim() || null;
+};
+const IUCN_TEXT: Record<string, string> = {
+	LC: 'Least Concern',
+	NT: 'Near Threatened',
+	VU: 'Vulnerable',
+	EN: 'Endangered',
+	CR: 'Critically Endangered',
+	EW: 'Extinct in the Wild',
+	EX: 'Extinct',
+	DD: 'Data Deficient'
+};
+
+function funFacts(s: Row, rankInGroup: number | null, groupName: string): string[] {
+	const f: string[] = [];
+	if (s.nl_obs > 0)
+		f.push(
+			`👀 Spotted ${fmt(s.nl_obs)} times in the Netherlands on iNaturalist${rankInGroup && rankInGroup <= 50 ? `: the #${rankInGroup} most seen of the ${groupName.toLowerCase()} in NL` : ''}.`
+		);
+	else
+		f.push(
+			`✈️ No confirmed wild sightings in the Netherlands on iNaturalist: you'd have to travel for this one.`
+		);
+	if (s.world_obs > 0) f.push(`🌍 ${fmt(s.world_obs)} observations worldwide on iNaturalist.`);
+	if (s.mass_kg != null) {
+		const kg = s.mass_kg;
+		if (kg >= 1000)
+			f.push(
+				`🐘 Can weigh about ${fmt(Math.round(kg / 1000))} tonnes: as much as ${fmt(Math.max(1, Math.round(kg / 1400)))} cars.`
+			);
+		else if (kg >= 1) f.push(`⚖️ Weighs up to about ${fmt(Math.round(kg * 10) / 10)} kg.`);
+		else if (kg >= 0.001) f.push(`🪶 Weighs only about ${fmt(Math.round(kg * 1000))} g.`);
+	}
+	if (s.length_m != null) {
+		const m = s.length_m;
+		if (m >= 2)
+			f.push(
+				`📏 Up to ${fmt(Math.round(m * 10) / 10)} m long: ${(m / 1.8).toFixed(1)}× a tall person.`
+			);
+		else if (m >= 0.01) f.push(`📏 About ${fmt(Math.round(m * 100))} cm long.`);
+		else f.push(`🔬 Only about ${fmt(Math.round(m * 1000 * 10) / 10)} mm long.`);
+	}
+	if (s.lifespan_years != null && s.lifespan_years >= 1)
+		f.push(`⏳ Can live up to ${fmt(Math.round(s.lifespan_years))} years.`);
+	if (s.named_year)
+		f.push(
+			`📜 Scientifically named ${s.named_by ? `by ${s.named_by} ` : ''}in ${s.named_year} (${new Date().getFullYear() - s.named_year} years ago).`
+		);
+	if (s.nl_establishment === 'introduced')
+		f.push(`🧳 Not originally Dutch: it was introduced to the Netherlands.`);
+	if (s.iucn && ['VU', 'EN', 'CR', 'EW'].includes(s.iucn))
+		f.push(`⚠️ Listed as ${IUCN_TEXT[s.iucn]} on the IUCN Red List.`);
+	if (s.extinct) f.push(`🪦 This species is extinct.`);
+	if (s.nl_status) f.push(`🇳🇱 Dutch status: ${s.nl_status}.`);
+	return f;
+}
+
+type Row = {
+	inat_id: number;
+	group: string;
+	slug: string;
+	scientific_name: string;
+	name_en: string | null;
+	name_nl: string | null;
+	search: string;
+	gbif_key: number | null;
+	wikidata: string | null;
+	wikipedia_en: string | null;
+	wikipedia_nl: string | null;
+	class_name: string | null;
+	order_name: string | null;
+	family_name: string | null;
+	iucn: string | null;
+	nl_status: string | null;
+	nl_establishment: string | null;
+	extinct: boolean;
+	nl_obs: number;
+	europe_obs: number;
+	world_obs: number;
+	gbif_nl: number | null;
+	scope: string;
+	summary: string | null;
+	summary_nl: string | null;
+	facts: string[];
+	mass_kg: number | null;
+	length_m: number | null;
+	lifespan_years: number | null;
+	named_year: number | null;
+	named_by: string | null;
+	sources: string[];
+};
+const wikiTitle = (url: string | null | undefined, host: string) => {
+	if (!url || !url.includes(host)) return null;
+	try {
+		return decodeURIComponent(url.split('/wiki/')[1] ?? '').replace(/_/g, ' ') || null;
+	} catch {
+		return null;
+	}
+};
+
+async function main() {
+	await useCache(cacheDir, 'wildlife-api.json');
+	const url = process.env.DB_URL;
+	if (!dryRun && !url) throw new Error('Set DB_URL (or use --dry-run)');
+	const sql = url ? postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
+	try {
+		const existing = sql
+			? await sql<{ inat_id: number; dex: number }[]>`select inat_id, dex from wildlife.species`
+			: [];
+		log('collecting species lists…');
+		const cands = await collect(existing.map((e) => e.inat_id));
+		const ids = [...cands.keys()];
+		log(`${ids.length} candidate species; fetching details…`);
+		const en = await details(ids, 'en');
+		const nl = await details(ids, 'nl');
+		await saveCache(true);
+		log('NL / Europe counts…');
+		const nlCounts = await placeCounts(ids, PLACE_NL);
+		const euCounts = await placeCounts(ids, PLACE_EUROPE);
+		await saveCache(true);
+		log('Wikidata…');
+		const wd = await wikidata(ids);
+		log('GBIF NL occurrence counts…');
+		const gbif = await gbifNl();
+
+		const rows: Row[] = [];
+		for (const id of ids) {
+			const t = en.get(id);
+			const tn = nl.get(id);
+			if (!t || t.rank !== 'species' || EXCLUDE.has(t.name)) continue;
+			const anc = t.ancestors ?? [];
+			if (!anc.some((a) => a.name === 'Animalia')) continue;
+			const g = groupOf(anc.map((a) => a.name));
+			const w = wd.get(id);
+			const nameEn = t.preferred_common_name || t.english_common_name || null;
+			let nameNl: string | null = tn?.preferred_common_name || null;
+			if (nameNl && nameEn && nameNl === nameEn && w?.nl && w.nl !== t.name && w.nl !== nameEn)
+				nameNl = w.nl;
+			if (!nameNl && w?.nl && w.nl.toLowerCase() !== t.name.toLowerCase()) nameNl = w.nl;
+			if (nameNl) nameNl = nameNl.charAt(0).toUpperCase() + nameNl.slice(1);
+			const statuses = t.conservation_statuses ?? [];
+			const global = statuses.find((s) => !s.place && s.authority === 'IUCN Red List');
+			let iucn = global?.iucn != null ? (IUCN_BY_NUMBER[global.iucn] ?? null) : null;
+			if (!iucn && global?.status && /^(LC|NT|VU|EN|CR|EW|EX|DD)$/i.test(global.status))
+				iucn = global.status.toUpperCase();
+			if (!iucn) iucn = w?.iucn ?? null;
+			if (t.extinct && !iucn) iucn = 'EX';
+			const nlS = statuses.find((s) => s.place?.id === PLACE_NL);
+			const em = (tn ?? t).establishment_means;
+			const nlEst = em?.place?.id === PLACE_NL ? (em.establishment_means ?? null) : null;
+			const nlObs = nlCounts.get(id) ?? 0;
+			const cand = cands.get(id)!;
+			const rank = (r: string) => anc.find((a) => a.rank === r)?.name ?? null;
+			const wikipediaEn = wikiTitle(t.wikipedia_url, 'en.wikipedia.org');
+			const wikipediaNl = w?.nlwiki ?? wikiTitle(tn?.wikipedia_url, 'nl.wikipedia.org');
+			const summaryNl = tn?.wikipedia_url?.includes('nl.wikipedia.org')
+				? trimSummary(tn.wikipedia_summary)
+				: null;
+			const row: Row = {
+				inat_id: id,
+				group: g.slug,
+				slug: slugify(t.name),
+				scientific_name: t.name,
+				name_en: nameEn,
+				name_nl: nameNl,
+				search: [nameNl, nameEn, t.name, rank('order'), rank('family'), rank('genus')]
+					.filter(Boolean)
+					.join(' ')
+					.toLowerCase(),
+				gbif_key: w?.gbif ?? null,
+				wikidata: w?.item ?? null,
+				wikipedia_en: wikipediaEn,
+				wikipedia_nl: wikipediaNl,
+				class_name: rank('class'),
+				order_name: rank('order'),
+				family_name: rank('family'),
+				iucn,
+				nl_status: nlS ? nlS.description?.trim() || nlS.status : null,
+				nl_establishment: nlEst,
+				extinct: !!t.extinct,
+				nl_obs: nlObs,
+				europe_obs: euCounts.get(id) ?? 0,
+				world_obs: t.observations_count ?? 0,
+				gbif_nl: w?.gbif != null && gbif ? (gbif.get(w.gbif) ?? 0) : null,
+				scope:
+					nlObs >= MIN_NL
+						? 'nl'
+						: (euCounts.get(id) ?? 0) > 0
+							? 'europe'
+							: cand.scope === 'nl'
+								? 'europe'
+								: 'world',
+				summary: trimSummary(t.wikipedia_summary),
+				summary_nl: summaryNl,
+				facts: [],
+				mass_kg: w?.mass != null && w.mass > 0 && w.mass < 200_000 ? w.mass : null,
+				length_m: w?.len != null && w.len > 0 && w.len < 40 ? w.len : null,
+				lifespan_years:
+					w?.life != null && w.life > 0 ? Math.round((w.life / 31_557_600) * 10) / 10 : null,
+				named_year: w?.year ?? null,
+				named_by: w?.author ?? null,
+				sources: [
+					'iNaturalist',
+					...(w ? ['Wikidata'] : []),
+					...(w?.gbif != null && gbif ? ['GBIF'] : [])
+				]
+			};
+			if (row.lifespan_years != null && (row.lifespan_years < 0.01 || row.lifespan_years > 500))
+				row.lifespan_years = null;
+			rows.push(row);
+		}
+		// Unique slugs (scientific names are unique on iNaturalist, but be safe).
+		const seen = new Set<string>();
+		for (const r of rows) {
+			let s = r.slug;
+			for (let n = 2; seen.has(s); n++) s = `${r.slug}-${n}`;
+			r.slug = s;
+			seen.add(s);
+		}
+		// Fun facts with the rank among the group in NL.
+		const byGroup = new Map<string, Row[]>();
+		for (const r of rows) byGroup.set(r.group, [...(byGroup.get(r.group) ?? []), r]);
+		for (const [slug, list] of byGroup) {
+			const g = GROUPS.find((x) => x.slug === slug)!;
+			const ranked = list.filter((r) => r.nl_obs > 0).toSorted((a, b) => b.nl_obs - a.nl_obs);
+			for (const r of list) {
+				const i = ranked.indexOf(r);
+				r.facts = funFacts(r, i >= 0 ? i + 1 : null, g.name);
+			}
+		}
+		const summary = GROUPS.map((g) => `${g.slug}: ${byGroup.get(g.slug)?.length ?? 0}`).join(', ');
+		log(`${rows.length} species (${summary})`);
+		if (dryRun || !sql) {
+			log(JSON.stringify(rows.slice(0, 3), null, 1));
+			return;
+		}
+
+		const known = new Map(existing.map((e) => [e.inat_id, e.dex]));
+		let nextDex = Math.max(0, ...existing.map((e) => e.dex)) + 1;
+		const order = (r: Row) => GROUPS.findIndex((g) => g.slug === r.group);
+		const scopeOrder = { nl: 0, europe: 1, world: 2 } as Record<string, number>;
+		const sorted = rows.toSorted(
+			(a, b) =>
+				order(a) - order(b) ||
+				scopeOrder[a.scope] - scopeOrder[b.scope] ||
+				b.nl_obs - a.nl_obs ||
+				b.world_obs - a.world_obs ||
+				a.scientific_name.localeCompare(b.scientific_name)
+		);
+		const added: Row[] = [];
+		const dexOf = new Map<number, number>();
+		for (const r of sorted) {
+			const d = known.get(r.inat_id);
+			if (d != null) dexOf.set(r.inat_id, d);
+			else {
+				dexOf.set(r.inat_id, nextDex++);
+				added.push(r);
+			}
+		}
+		await sql.begin(async (tx) => {
+			const groupIds = new Map<string, number>();
+			for (const [i, g] of GROUPS.entries()) {
+				const [row] = await tx<{ id: number }[]>`
+					insert into wildlife.groups (slug, name, name_nl, emoji, sort_order)
+					values (${g.slug}, ${g.name}, ${g.nameNl}, ${g.emoji}, ${i})
+					on conflict (slug) do update set name = excluded.name, name_nl = excluded.name_nl,
+						emoji = excluded.emoji, sort_order = excluded.sort_order
+					returning id`;
+				groupIds.set(g.slug, row.id);
+			}
+			// Avoid unique-slug clashes while rows move around.
+			await tx`update wildlife.species set slug = slug || '~' || id where slug not like '%~%'`;
+			for (const r of sorted) {
+				const { group, ...rest } = r;
+				const row = { ...rest, dex: dexOf.get(r.inat_id)!, group_id: groupIds.get(group)! };
+				await tx`
+					insert into wildlife.species ${tx(row)}
+					on conflict (inat_id) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'inat_id' && k !== 'dex') as (keyof typeof row)[])}`;
+			}
+			// Restore slugs of species that were not part of this run (kept, not deleted).
+			await tx`update wildlife.species set slug = split_part(slug, '~', 1) where slug like '%~%'`;
+			await tx`
+				update wildlife.groups g set
+					species_count = (select count(*) from wildlife.species s where s.group_id = g.id),
+					nl_count = (select count(*) from wildlife.species s where s.group_id = g.id and s.nl_obs > 0)`;
+		});
+		log(`written: ${rows.length} species, ${added.length} new`);
+		if (reportNew) {
+			const [totals] = await sql`
+				select (select count(*)::int from wildlife.species) as species,
+					(select count(*)::int from wildlife.groups where species_count > 0) as groups,
+					(select count(*)::int from wildlife.photos) as photos`;
+			console.log(
+				JSON.stringify(
+					{
+						generatedAt: new Date().toISOString(),
+						totals,
+						newSpecies: (existing.length ? added : added.slice(0, 0)).map((r) => ({
+							dex: dexOf.get(r.inat_id),
+							inatId: r.inat_id,
+							name: r.name_en ?? r.scientific_name,
+							nameNl: r.name_nl,
+							scientificName: r.scientific_name,
+							group: r.group,
+							iucn: r.iucn,
+							nlObs: r.nl_obs,
+							url: `${SITE}/${r.group}/${r.slug}`
+						}))
+					},
+					null,
+					1
+				)
+			);
+		}
+	} finally {
+		await saveCache(true);
+		await sql?.end();
+	}
+}
+
+await main();

scripts/wildlife-inat.ts

@@ -0,0 +1,211 @@
+// Shared helpers for the wildlife pokedex scripts (scripts/wildlife-import.ts,
+// scripts/wildlife-photos.ts): polite iNaturalist API access (about 1 request per second,
+// as the API asks), the group definitions and photo licence handling.
+import { getJson, sleep } from './planes-wiki';
+
+export const UA =
+	'StanRungeWildlifeDex/1.0 (https://stanrunge.dev/trackers/wildlife; wildlife pokedex importer) bun';
+export const INAT = 'https://api.inaturalist.org/v1';
+export const PLACE_NL = 7506;
+export const PLACE_EUROPE = 97391;
+
+let last = 0;
+/** GET an iNaturalist API path, at most one call per ~1.1 s (cached when --cache is set). */
+export async function inat<T = InatList>(path: string, useCached = true): Promise<T> {
+	const url = `${INAT}${path}`;
+	const wait = last + 1100 - Date.now();
+	if (wait > 0) await sleep(wait);
+	last = Date.now();
+	return getJson<T>(url, { headers: { 'user-agent': UA } }, useCached);
+}
+
+export type InatPhoto = {
+	id: number;
+	license_code: string | null;
+	attribution: string;
+	attribution_name?: string | null;
+	url: string;
+	original_dimensions?: { width: number; height: number } | null;
+};
+export type InatTaxon = {
+	id: number;
+	name: string;
+	rank: string;
+	is_active?: boolean;
+	extinct?: boolean;
+	iconic_taxon_name?: string | null;
+	preferred_common_name?: string | null;
+	english_common_name?: string | null;
+	observations_count?: number;
+	wikipedia_url?: string | null;
+	wikipedia_summary?: string | null;
+	default_photo?: InatPhoto | null;
+	taxon_photos?: { photo: InatPhoto }[];
+	ancestors?: { id: number; rank: string; name: string }[];
+	ancestor_ids?: number[];
+	conservation_status?: { status?: string; iucn?: number; authority?: string } | null;
+	conservation_statuses?: {
+		status: string;
+		iucn: number | null;
+		authority: string | null;
+		description?: string | null;
+		place: { id: number } | null;
+	}[];
+	establishment_means?: { establishment_means?: string; place?: { id: number } } | null;
+};
+export type InatList = {
+	total_results: number;
+	page: number;
+	per_page: number;
+	results: { count: number; taxon: InatTaxon }[];
+};
+
+export const CC = [
+	'cc0',
+	'cc-by',
+	'cc-by-sa',
+	'cc-by-nc',
+	'cc-by-nc-sa',
+	'cc-by-nd',
+	'cc-by-nc-nd'
+];
+export const isCc = (code: string | null | undefined) => !!code && CC.includes(code.toLowerCase());
+export const licenseLabel = (code: string) =>
+	code.toLowerCase() === 'cc0' ? 'CC0' : `${code.toUpperCase().replace(/^CC-/, 'CC ')} 4.0`;
+export const licenseUrl = (code: string) =>
+	code.toLowerCase() === 'cc0'
+		? 'https://creativecommons.org/publicdomain/zero/1.0/'
+		: `https://creativecommons.org/licenses/${code.toLowerCase().replace(/^cc-/, '')}/4.0/`;
+/** "(c) Jane Doe, some rights reserved (CC BY)" -> "Jane Doe". */
+export const author = (p: InatPhoto) =>
+	p.attribution_name?.trim() ||
+	p.attribution
+		.replace(/^\(c\)\s*/i, '')
+		.replace(/,\s*(some|no) rights reserved.*$/i, '')
+		.replace(/^no known copyright restrictions.*$/i, '')
+		.trim() ||
+	null;
+/** Any size URL -> the "medium" (500px) one. */
+export const mediumUrl = (url: string) =>
+	url.replace(/\/(square|small|thumb|medium|large|original)\.(jpe?g|png|gif|webp)/i, '/medium.$2');
+
+// Groups shown in the dex, first match wins (by ancestor names on iNaturalist).
+export type GroupDef = {
+	slug: string;
+	name: string;
+	nameNl: string;
+	emoji: string;
+	match: string[];
+};
+export const GROUPS: GroupDef[] = [
+	{ slug: 'mammals', name: 'Mammals', nameNl: 'Zoogdieren', emoji: '🦊', match: ['Mammalia'] },
+	{ slug: 'birds', name: 'Birds', nameNl: 'Vogels', emoji: '🐦', match: ['Aves'] },
+	{
+		slug: 'reptiles',
+		name: 'Reptiles',
+		nameNl: 'Reptielen',
+		emoji: '🦎',
+		match: ['Reptilia', 'Testudines', 'Squamata', 'Crocodylia']
+	},
+	{ slug: 'amphibians', name: 'Amphibians', nameNl: 'Amfibieën', emoji: '🐸', match: ['Amphibia'] },
+	{
+		slug: 'sharks-and-rays',
+		name: 'Sharks & rays',
+		nameNl: 'Haaien en roggen',
+		emoji: '🦈',
+		match: ['Elasmobranchii', 'Chondrichthyes']
+	},
+	{
+		slug: 'fish',
+		name: 'Fish',
+		nameNl: 'Vissen',
+		emoji: '🐟',
+		match: ['Actinopterygii', 'Sarcopterygii', 'Petromyzonti', 'Myxini', 'Agnatha']
+	},
+	{
+		slug: 'butterflies-and-moths',
+		name: 'Butterflies & moths',
+		nameNl: 'Vlinders',
+		emoji: '🦋',
+		match: ['Lepidoptera']
+	},
+	{ slug: 'beetles', name: 'Beetles', nameNl: 'Kevers', emoji: '🪲', match: ['Coleoptera'] },
+	{
+		slug: 'bees-wasps-and-ants',
+		name: 'Bees, wasps & ants',
+		nameNl: 'Bijen, wespen en mieren',
+		emoji: '🐝',
+		match: ['Hymenoptera']
+	},
+	{
+		slug: 'dragonflies',
+		name: 'Dragonflies & damselflies',
+		nameNl: 'Libellen',
+		emoji: '🪽',
+		match: ['Odonata']
+	},
+	{
+		slug: 'flies',
+		name: 'Flies & mosquitoes',
+		nameNl: 'Vliegen en muggen',
+		emoji: '🪰',
+		match: ['Diptera']
+	},
+	{
+		slug: 'bugs-and-other-insects',
+		name: 'Bugs, crickets & other insects',
+		nameNl: 'Wantsen, sprinkhanen en andere insecten',
+		emoji: '🦗',
+		match: ['Insecta']
+	},
+	{
+		slug: 'spiders',
+		name: 'Spiders & kin',
+		nameNl: 'Spinnen en verwanten',
+		emoji: '🕷️',
+		match: ['Arachnida']
+	},
+	{
+		slug: 'crustaceans',
+		name: 'Crabs & other crustaceans',
+		nameNl: 'Kreeftachtigen',
+		emoji: '🦀',
+		match: [
+			'Crustacea',
+			'Malacostraca',
+			'Maxillopoda',
+			'Thecostraca',
+			'Branchiopoda',
+			'Hexanauplia'
+		]
+	},
+	{
+		slug: 'molluscs',
+		name: 'Snails, slugs & other molluscs',
+		nameNl: 'Weekdieren',
+		emoji: '🐌',
+		match: ['Mollusca']
+	},
+	{
+		slug: 'other-invertebrates',
+		name: 'Jellyfish, starfish, worms & co',
+		nameNl: 'Overige ongewervelden',
+		emoji: '🪼',
+		match: []
+	}
+];
+export function groupOf(ancestorNames: string[]): GroupDef {
+	for (const g of GROUPS) if (g.match.some((m) => ancestorNames.includes(m))) return g;
+	return GROUPS.at(-1)!;
+}
+
+export const IUCN_BY_NUMBER: Record<number, string> = {
+	5: 'DD',
+	10: 'LC',
+	20: 'NT',
+	30: 'VU',
+	40: 'EN',
+	50: 'CR',
+	60: 'EW',
+	70: 'EX'
+};

scripts/wildlife-photos.ts

@@ -0,0 +1,172 @@
+// Photos for the wildlife pokedex (/trackers/wildlife). Only Creative Commons licensed photos
+// from iNaturalist are used; they are hotlinked (inaturalist-open-data S3), never downloaded,
+// and every photo keeps its author, licence and a link back to iNaturalist.
+//
+//   DB_URL=... bun scripts/wildlife-photos.ts             (re)do photos for every species
+//   bun scripts/wildlife-photos.ts --missing              only species never checked / without photos
+//   bun scripts/wildlife-photos.ts --limit 100            at most 100 species this run
+//   bun scripts/wildlife-photos.ts --cache /tmp/wildlife  reuse API responses
+//   bun scripts/wildlife-photos.ts --missing --enrich report.json   print the import report with photos
+//
+// Per species: the curated iNaturalist taxon photos with a CC licence, topped up with the most
+// faved research-grade observation photos (CC licence only) until there are MAX photos.
+import postgres from 'postgres';
+import { readFile } from 'node:fs/promises';
+import { chunks, log, saveCache, useCache } from './planes-wiki';
+import {
+	author,
+	inat,
+	isCc,
+	mediumUrl,
+	PLACE_NL,
+	type InatPhoto,
+	type InatTaxon
+} from './wildlife-inat';
+
+const args = process.argv.slice(2);
+const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
+const missing = args.includes('--missing');
+const limit = Number(opt('--limit')) || 0;
+const enrich = opt('--enrich');
+const MAX = 10;
+const MIN_TAXON = 7; // fewer CC taxon photos than this -> top up from observations
+const LICENSES = 'cc0,cc-by,cc-by-sa,cc-by-nc,cc-by-nc-sa,cc-by-nd,cc-by-nc-nd';
+
+type Photo = {
+	kind: string;
+	photo_id: number;
+	url: string;
+	width: number | null;
+	height: number | null;
+	author: string | null;
+	license: string;
+	page_url: string;
+	place: string | null;
+};
+const toPhoto = (p: InatPhoto, kind: string, page: string, place: string | null): Photo | null => {
+	if (!isCc(p.license_code) || !p.url) return null;
+	const dims = p.original_dimensions;
+	const scale = dims ? Math.min(1, 500 / Math.max(dims.width, dims.height)) : 1;
+	return {
+		kind,
+		photo_id: p.id,
+		url: mediumUrl(p.url),
+		width: dims ? Math.round(dims.width * scale) : null,
+		height: dims ? Math.round(dims.height * scale) : null,
+		author: author(p),
+		license: p.license_code!.toLowerCase(),
+		page_url: page,
+		place
+	};
+};
+
+type Obs = { id: number; uri?: string; place_guess?: string | null; photos: InatPhoto[] };
+
+async function main() {
+	await useCache(opt('--cache'), 'wildlife-photos-api.json');
+	const url = process.env.DB_URL;
+	if (!url) throw new Error('Set DB_URL');
+	const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} });
+	try {
+		let todo = await sql<
+			{ id: number; inat_id: number; scientific_name: string; nl_obs: number }[]
+		>`
+			select id, inat_id, scientific_name, nl_obs from wildlife.species
+			${missing ? sql`where photos_checked_at is null or photo_count = 0` : sql``}
+			order by nl_obs desc, world_obs desc`;
+		if (limit) todo = todo.slice(0, limit);
+		log(`photos for ${todo.length} species`);
+		let done = 0;
+		let total = 0;
+		for (const batch of chunks(todo, 30)) {
+			const res = await inat<{ results: InatTaxon[] }>(
+				`/taxa/${batch.map((b) => b.inat_id).join(',')}?locale=en`,
+				false
+			);
+			const taxa = new Map(res.results.map((t) => [t.id, t]));
+			for (const s of batch) {
+				const t = taxa.get(s.inat_id);
+				const photos: Photo[] = [];
+				const seen = new Set<number>();
+				const push = (p: Photo | null) => {
+					if (p && !seen.has(p.photo_id) && photos.length < MAX) {
+						seen.add(p.photo_id);
+						photos.push(p);
+					}
+				};
+				for (const tp of t?.taxon_photos ?? [])
+					push(
+						toPhoto(tp.photo, 'taxon', `https://www.inaturalist.org/photos/${tp.photo.id}`, null)
+					);
+				if (photos.length < MIN_TAXON) {
+					// Prefer Dutch observations for species seen here, then anywhere.
+					const places = s.nl_obs >= 20 ? [`&place_id=${PLACE_NL}`, ''] : [''];
+					for (const place of places) {
+						if (photos.length >= MAX) break;
+						const obs = await inat<{ results: Obs[] }>(
+							`/observations?taxon_id=${s.inat_id}${place}&quality_grade=research&captive=false&photo_license=${LICENSES}&order_by=votes&order=desc&per_page=20&locale=en`
+						);
+						for (const o of obs.results) {
+							const page = `https://www.inaturalist.org/observations/${o.id}`;
+							// One photo per observation first, for variety.
+							push(toPhoto(o.photos[0], 'observation', page, o.place_guess ?? null));
+						}
+						for (const o of obs.results)
+							for (const p of o.photos.slice(1, 2))
+								push(
+									toPhoto(
+										p,
+										'observation',
+										`https://www.inaturalist.org/observations/${o.id}`,
+										o.place_guess ?? null
+									)
+								);
+					}
+				}
+				await sql.begin(async (tx) => {
+					await tx`delete from wildlife.photos where species_id = ${s.id}`;
+					for (const [i, p] of photos.entries())
+						await tx`insert into wildlife.photos ${tx({ ...p, species_id: s.id, position: i })}`;
+					await tx`update wildlife.species set photo = ${photos[0]?.url ?? null},
+						photo_count = ${photos.length}, photos_checked_at = now() where id = ${s.id}`;
+				});
+				total += photos.length;
+				done++;
+			}
+			await saveCache();
+			if (done % 150 < 30) log(`  ${done}/${todo.length} species, ${total} photos`);
+		}
+		await sql`
+			update wildlife.groups g set photo = (
+				select s.photo from wildlife.species s where s.group_id = g.id and s.photo is not null
+				order by s.nl_obs desc, s.world_obs desc limit 1)`;
+		log(`done: ${done} species, ${total} photos`);
+		if (enrich) {
+			const report = JSON.parse(await readFile(enrich, 'utf8'));
+			const ids = (report.newSpecies ?? []).map((n: { inatId: number }) => n.inatId);
+			const rows = ids.length
+				? await sql<{ inat_id: number; photo: string | null; photo_count: number }[]>`
+					select inat_id, photo, photo_count from wildlife.species where inat_id in ${sql(ids)}`
+				: [];
+			const by = new Map(rows.map((r) => [r.inat_id, r]));
+			const [totals] = await sql`
+				select (select count(*)::int from wildlife.species) as species,
+					(select count(*)::int from wildlife.groups where species_count > 0) as groups,
+					(select count(*)::int from wildlife.photos) as photos`;
+			report.totals = totals;
+			report.newSpecies = (report.newSpecies ?? [])
+				.map((n: { inatId: number }) => ({
+					...n,
+					photo: by.get(n.inatId)?.photo?.replace('/medium.', '/large.') ?? null,
+					photos: by.get(n.inatId)?.photo_count ?? 0
+				}))
+				.toSorted((a: { nlObs: number }, b: { nlObs: number }) => b.nlObs - a.nlObs);
+			console.log(JSON.stringify(report, null, 1));
+		}
+	} finally {
+		await saveCache(true);
+		await sql.end();
+	}
+}
+
+await main();

scripts/wildlife-refresh.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Refresh of the wildlife pokedex (/trackers/wildlife) on stan-ovh.
+#
+#   ssh stan@37.187.157.191 'cd ~/dev/website-wildlife && git fetch -q && git checkout -q --detach origin/main \
+#     && bun install --frozen-lockfile >/dev/null && scripts/wildlife-refresh.sh --report-new'
+#
+# 1. bun scripts/wildlife-import.ts   re-reads the species lists (NL, Europe, world) from
+#    iNaturalist + Wikidata + GBIF, refreshes counts/status of every species and adds species
+#    that entered the lists (existing ones are kept, Dex numbers are stable).
+# 2. bun scripts/wildlife-photos.ts --missing   CC-licensed iNaturalist photos for species
+#    without photos (new ones included).
+# --report-new: stdout is one JSON report {generatedAt, totals, newSpecies[] (with photo)}.
+# Without it, progress only. Progress goes to stderr. DB_URL is read from the running app
+# container and never printed.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [ -z "${DB_URL:-}" ]; then
+	DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' |
+		sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
+	export DB_URL
+fi
+
+if [ "${1:-}" = "--report-new" ]; then
+	shift
+	report=$(mktemp /tmp/wildlife-report.XXXXXX.json)
+	trap 'rm -f "$report"' EXIT
+	bun scripts/wildlife-import.ts --report-new "$@" >"$report"
+	bun scripts/wildlife-photos.ts --missing --enrich "$report"
+else
+	bun scripts/wildlife-import.ts "$@" >&2
+	bun scripts/wildlife-photos.ts --missing >&2
+fi