website/scripts/wildlife-import.ts

// Builds the public wildlife pokedex (/trackers/wildlife, schema `wildlife`) from open data.
//
//   DB_URL=... bun scripts/wildlife-import.ts               fetch sources and write to the database
//   bun scripts/wildlife-import.ts --dry-run                fetch + build only, print a summary
//   bun scripts/wildlife-import.ts --cache /tmp/wildlife    reuse API responses from that folder
//   bun scripts/wildlife-import.ts --report-new             JSON of species new since the last run on stdout
//
// Which species (wild animals only, no pets/livestock, no captive observations):
//  - the most observed species per group in the Netherlands (iNaturalist research grade),
//  - the most observed species per group in Europe,
//  - the most observed species per group worldwide, plus a short list of famous animals.
// Species already in the dex are always kept (refreshed, never deleted), so Dex numbers stay
// stable and new species get the next number.
// Per species: names (iNaturalist nl/en locale, Wikidata nl label), taxonomy, IUCN Red List
// (iNaturalist, Wikidata P141 fallback), Dutch red list status, establishment in NL, iNat
// observation counts (NL, Europe, world), Wikipedia intros (en + nl via iNaturalist),
// Wikidata mass/length/lifespan/name year+author and GBIF occurrence records in NL.
import postgres from 'postgres';
import {
	chunks,
	log as logErr,
	saveCache,
	slugify,
	sparql,
	stripHtml,
	useCache,
	getJson
} from './planes-wiki';
import {
	groupOf,
	GROUPS,
	inat,
	IUCN_BY_NUMBER,
	PLACE_EUROPE,
	PLACE_NL,
	UA,
	type InatList,
	type InatTaxon
} from './wildlife-inat';

const args = process.argv.slice(2);
const dryRun = args.includes('--dry-run');
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
const cacheDir = opt('--cache');
const reportNew = args.includes('--report-new');
const SITE = 'https://stanrunge.dev/trackers/wildlife';
const log = (...a: unknown[]) => (reportNew ? logErr(...a) : console.log(...a));

// iconic taxon filter -> how many species to take (NL, Europe extra, world extra)
const ICONIC: { iconic: string; taxonId: number; nl: number; eu: number; world: number }[] = [
	{ iconic: 'Aves', taxonId: 3, nl: 450, eu: 250, world: 300 },
	{ iconic: 'Mammalia', taxonId: 40151, nl: 120, eu: 120, world: 350 },
	{ iconic: 'Reptilia', taxonId: 26036, nl: 40, eu: 100, world: 250 },
	{ iconic: 'Amphibia', taxonId: 20978, nl: 30, eu: 50, world: 100 },
	{ iconic: 'Actinopterygii', taxonId: 47178, nl: 150, eu: 80, world: 150 },
	{ iconic: 'Insecta', taxonId: 47158, nl: 1300, eu: 300, world: 200 },
	{ iconic: 'Arachnida', taxonId: 47119, nl: 150, eu: 50, world: 50 },
	{ iconic: 'Mollusca', taxonId: 47115, nl: 180, eu: 50, world: 50 },
	{ iconic: 'Animalia', taxonId: 1, nl: 250, eu: 80, world: 0 }
];
// Extra worldwide lists for groups without their own iconic taxon.
const WORLD_EXTRA = [
	{ taxonId: 47273, n: 60 }, // Elasmobranchii (sharks & rays)
	{ taxonId: 47534, n: 30 }, // Cnidaria
	{ taxonId: 47549, n: 30 }, // Echinodermata
	{ taxonId: 85493, n: 40 } // Malacostraca (crabs, lobsters, ...)
];
const MIN_NL = 5;
const FAMOUS =
	`Panthera tigris, Panthera leo, Panthera pardus, Panthera uncia, Panthera onca, Acinonyx jubatus,
 Ailuropoda melanoleuca, Ailurus fulgens, Ursus maritimus, Ursus arctos, Canis lupus, Gulo gulo, Lynx lynx,
 Bison bonasus, Loxodonta africana, Elephas maximus, Giraffa camelopardalis, Okapia johnstoni, Equus quagga,
 Hippopotamus amphibius, Ceratotherium simum, Diceros bicornis, Rhinoceros unicornis, Gorilla gorilla,
 Gorilla beringei, Pan troglodytes, Pan paniscus, Pongo pygmaeus, Lemur catta, Bradypus variegatus,
 Myrmecophaga tridactyla, Hydrochoerus hydrochaeris, Phascolarctos cinereus, Macropus giganteus,
 Ornithorhynchus anatinus, Tachyglossus aculeatus, Sarcophilus harrisii, Balaenoptera musculus,
 Megaptera novaeangliae, Physeter macrocephalus, Orcinus orca, Tursiops truncatus, Monodon monoceros,
 Odobenus rosmarus, Enhydra lutris, Aptenodytes forsteri, Spheniscus demersus, Struthio camelus,
 Dromaius novaehollandiae, Aquila chrysaetos, Haliaeetus leucocephalus, Harpia harpyja, Vultur gryphus,
 Diomedea exulans, Fratercula arctica, Pavo cristatus, Ramphastos toco, Phoenicopterus roseus,
 Crocodylus niloticus, Crocodylus porosus, Alligator mississippiensis, Varanus komodoensis, Chelonia mydas,
 Dermochelys coriacea, Chelonoidis niger, Ophiophagus hannah, Iguana iguana, Chamaeleo calyptratus,
 Ambystoma mexicanum, Andrias japonicus, Dendrobates tinctorius, Phyllobates terribilis,
 Carcharodon carcharias, Rhincodon typus, Mobula birostris, Latimeria chalumnae, Hippocampus hippocampus,
 Amphiprion ocellaris, Octopus vulgaris, Architeuthis dux, Danaus plexippus, Dynastes hercules,
 Apis mellifera, Lucanus cervus, Aurelia aurita`
		.split(',')
		.map((s) => s.trim())
		.filter(Boolean);
// Pets, livestock and us.
const EXCLUDE = new Set(
	`Homo sapiens, Canis familiaris, Felis catus, Bos taurus, Equus caballus, Ovis aries, Capra hircus,
 Sus domesticus, Gallus gallus, Equus asinus, Cavia porcellus, Bombyx mori, Lama glama, Vicugna pacos,
 Bubalus bubalis, Camelus dromedarius, Bos indicus, Mesocricetus auratus, Meleagris gallopavo domesticus`
		.split(',')
		.map((s) => s.trim())
);

type Cand = { id: number; scope: 'nl' | 'europe' | 'world'; name: string };

async function speciesCounts(
	place: number,
	iconic: string,
	n: number
): Promise<InatList['results']> {
	const out: InatList['results'] = [];
	for (let page = 1; out.length < n && page <= 10; page++) {
		const res = await inat(
			`/observations/species_counts?place_id=${place}&iconic_taxa=${iconic}&captive=false&quality_grade=research&per_page=500&page=${page}&locale=en`
		);
		out.push(...res.results);
		if (res.results.length < 500) break;
	}
	return out;
}
async function topWorld(taxonId: number, n: number): Promise<InatTaxon[]> {
	const out: InatTaxon[] = [];
	for (let page = 1; out.length < n && page <= 10; page++) {
		const res = await inat<{ results: InatTaxon[] }>(
			`/taxa?rank=species&taxon_id=${taxonId}&is_active=true&order_by=observations_count&order=desc&per_page=200&page=${page}`
		);
		out.push(...res.results);
		if (res.results.length < 200) break;
	}
	return out;
}

async function collect(existing: number[]): Promise<Map<number, Cand>> {
	const cands = new Map<number, Cand>();
	const add = (t: InatTaxon, scope: Cand['scope']) => {
		if (t.rank !== 'species' || EXCLUDE.has(t.name) || cands.has(t.id)) return false;
		cands.set(t.id, { id: t.id, scope, name: t.name });
		return true;
	};
	for (const g of ICONIC) {
		const nl = await speciesCounts(PLACE_NL, g.iconic, g.nl * 2);
		let k = 0;
		for (const r of nl) if (k < g.nl && r.count >= MIN_NL && add(r.taxon, 'nl')) k++;
		const eu = await speciesCounts(PLACE_EUROPE, g.iconic, g.eu * 2 + g.nl);
		let e = 0;
		for (const r of eu) if (e < g.eu && add(r.taxon, 'europe')) e++;
		let w = 0;
		if (g.world)
			for (const t of await topWorld(g.taxonId, g.world * 2))
				if (w < g.world && add(t, 'world')) w++;
		log(`  ${g.iconic}: ${k} NL, +${e} Europe, +${w} world`);
	}
	for (const x of WORLD_EXTRA) {
		let w = 0;
		for (const t of await topWorld(x.taxonId, x.n * 2)) if (w < x.n && add(t, 'world')) w++;
	}
	let famous = 0;
	for (const name of FAMOUS) {
		const res = await inat<{ results: InatTaxon[] }>(
			`/taxa?q=${encodeURIComponent(name)}&rank=species&is_active=true&per_page=5`
		);
		const t = res.results.find((r) => r.name.toLowerCase() === name.toLowerCase());
		if (t && add(t, 'world')) famous++;
		if (!t) log(`  famous species not found: ${name}`);
	}
	log(`  +${famous} famous species`);
	for (const id of existing) if (!cands.has(id)) cands.set(id, { id, scope: 'world', name: '' });
	return cands;
}

async function details(ids: number[], locale: 'en' | 'nl') {
	const out = new Map<number, InatTaxon>();
	for (const [i, batch] of chunks(ids, 30).entries()) {
		const res = await inat<{ results: InatTaxon[] }>(
			// English names must not be localised to the NL place (that returns Dutch ones).
			`/taxa/${batch.join(',')}?locale=${locale}${locale === 'nl' ? `&preferred_place_id=${PLACE_NL}` : ''}`
		);
		for (const t of res.results) out.set(t.id, t);
		if (i % 20 === 0) log(`  taxa (${locale}) ${Math.min((i + 1) * 30, ids.length)}/${ids.length}`);
	}
	return out;
}
async function placeCounts(ids: number[], place: number) {
	const out = new Map<number, number>();
	for (const batch of chunks(ids, 200)) {
		const res = await inat(
			`/observations/species_counts?place_id=${place}&taxon_id=${batch.join(',')}&captive=false&quality_grade=research&per_page=500`
		);
		for (const r of res.results) out.set(r.taxon.id, r.count);
	}
	return out;
}

type Wd = {
	item: string;
	nl: string | null;
	iucn: string | null;
	gbif: number | null;
	nlwiki: string | null;
	mass: number | null;
	len: number | null;
	life: number | null;
	year: number | null;
	author: string | null;
};
const WD_IUCN: Record<string, string> = {
	Q211005: 'LC',
	Q719675: 'NT',
	Q278113: 'VU',
	Q11394: 'EN',
	Q219127: 'CR',
	Q239509: 'EW',
	Q237350: 'EX',
	Q3245245: 'DD'
};
async function wikidata(ids: number[]) {
	const out = new Map<number, Wd>();
	for (const batch of chunks(ids, 250)) {
		const rows =
			await sparql(`SELECT ?inat ?item ?nl ?iucn ?gbif ?nlwiki ?mass ?len ?life ?year ?authorLabel WHERE {
			VALUES ?inat { ${batch.map((id) => `"${id}"`).join(' ')} }
			?item wdt:P3151 ?inat .
			OPTIONAL { ?item rdfs:label ?nl FILTER(lang(?nl) = "nl") }
			OPTIONAL { ?item wdt:P141 ?iucn }
			OPTIONAL { ?item wdt:P846 ?gbif }
			OPTIONAL { ?nlwiki schema:about ?item ; schema:isPartOf <https://nl.wikipedia.org/> }
			OPTIONAL { ?item p:P2067/psn:P2067/wikibase:quantityAmount ?mass }
			OPTIONAL { ?item p:P2043/psn:P2043/wikibase:quantityAmount ?len }
			OPTIONAL { ?item p:P2250/psn:P2250/wikibase:quantityAmount ?life }
			OPTIONAL { ?item p:P225 ?st . ?st pq:P574 ?year .
				OPTIONAL { ?st pq:P405 ?author . ?author rdfs:label ?authorLabel FILTER(lang(?authorLabel) = "en") } }
		}`);
		for (const r of rows) {
			const id = Number(r.inat);
			const prev = out.get(id);
			const num = (v: string | undefined) =>
				v != null && Number.isFinite(Number(v)) ? Number(v) : null;
			const max = (a: number | null | undefined, b: number | null) =>
				a == null ? b : b == null ? a : Math.max(a, b);
			const yr = r.year ? Number(r.year.slice(0, 4)) : null;
			out.set(id, {
				item: r.item.split('/').at(-1)!,
				nl: prev?.nl ?? r.nl ?? null,
				iucn: prev?.iucn ?? (r.iucn ? (WD_IUCN[r.iucn.split('/').at(-1)!] ?? null) : null),
				gbif: prev?.gbif ?? num(r.gbif),
				nlwiki:
					prev?.nlwiki ??
					(r.nlwiki ? decodeURIComponent(r.nlwiki.split('/wiki/')[1]).replace(/_/g, ' ') : null),
				mass: max(prev?.mass, num(r.mass)),
				len: max(prev?.len, num(r.len)),
				life: max(prev?.life, num(r.life)),
				year: prev?.year ?? (yr && yr > 1700 && yr <= new Date().getFullYear() ? yr : null),
				author: prev?.author ?? r.authorLabel ?? null
			});
		}
	}
	return out;
}

async function gbifNl(): Promise<Map<number, number> | null> {
	const out = new Map<number, number>();
	try {
		for (let offset = 0; offset < 100_000; offset += 5000) {
			const res = await getJson<{ facets: { counts: { name: string; count: number }[] }[] }>(
				`https://api.gbif.org/v1/occurrence/search?country=NL&taxonKey=1&limit=0&facet=speciesKey&facetLimit=5000&facetOffset=${offset}`,
				{ headers: { 'user-agent': UA } }
			);
			const counts = res.facets[0]?.counts ?? [];
			for (const c of counts) out.set(Number(c.name), c.count);
			if (counts.length < 5000) return out;
		}
		return out;
	} catch (e) {
		log(`  GBIF facet failed: ${(e as Error).message}`);
		return null;
	}
}

const fmt = (n: number) => n.toLocaleString('en-GB');
const trimSummary = (html: string | null | undefined) => {
	if (!html) return null;
	const text = stripHtml(html).replace(/\s*\[\d+\]/g, '');
	const sentences = text.match(/[^.!?]+[.!?]+(\s|$)/g) ?? [text];
	let out = '';
	for (const s of sentences) {
		if (out && (out + s).length > 650) break;
		out += s;
	}
	return out.trim() || null;
};
const IUCN_TEXT: Record<string, string> = {
	LC: 'Least Concern',
	NT: 'Near Threatened',
	VU: 'Vulnerable',
	EN: 'Endangered',
	CR: 'Critically Endangered',
	EW: 'Extinct in the Wild',
	EX: 'Extinct',
	DD: 'Data Deficient'
};

function funFacts(s: Row, rankInGroup: number | null, groupName: string): string[] {
	const f: string[] = [];
	if (s.nl_obs > 0)
		f.push(
			`👀 Spotted ${fmt(s.nl_obs)} times in the Netherlands on iNaturalist${rankInGroup && rankInGroup <= 50 ? `: the #${rankInGroup} most seen of the ${groupName.toLowerCase()} in NL` : ''}.`
		);
	else
		f.push(
			`✈️ No confirmed wild sightings in the Netherlands on iNaturalist: you'd have to travel for this one.`
		);
	if (s.world_obs > 0) f.push(`🌍 ${fmt(s.world_obs)} observations worldwide on iNaturalist.`);
	if (s.mass_kg != null) {
		const kg = s.mass_kg;
		if (kg >= 1000)
			f.push(
				`🐘 Can weigh about ${fmt(Math.round(kg / 1000))} tonnes: as much as ${fmt(Math.max(1, Math.round(kg / 1400)))} cars.`
			);
		else if (kg >= 1) f.push(`⚖️ Weighs up to about ${fmt(Math.round(kg * 10) / 10)} kg.`);
		else if (kg >= 0.001) f.push(`🪶 Weighs only about ${fmt(Math.round(kg * 1000))} g.`);
	}
	if (s.length_m != null) {
		const m = s.length_m;
		if (m >= 2)
			f.push(
				`📏 Up to ${fmt(Math.round(m * 10) / 10)} m long: ${(m / 1.8).toFixed(1)}× a tall person.`
			);
		else if (m >= 0.01) f.push(`📏 About ${fmt(Math.round(m * 100))} cm long.`);
		else f.push(`🔬 Only about ${fmt(Math.round(m * 1000 * 10) / 10)} mm long.`);
	}
	if (s.lifespan_years != null && s.lifespan_years >= 1)
		f.push(`⏳ Can live up to ${fmt(Math.round(s.lifespan_years))} years.`);
	if (s.named_year)
		f.push(
			`📜 Scientifically named ${s.named_by ? `by ${s.named_by} ` : ''}in ${s.named_year} (${new Date().getFullYear() - s.named_year} years ago).`
		);
	if (s.nl_establishment === 'introduced')
		f.push(`🧳 Not originally Dutch: it was introduced to the Netherlands.`);
	if (s.iucn && ['VU', 'EN', 'CR', 'EW'].includes(s.iucn))
		f.push(`⚠️ Listed as ${IUCN_TEXT[s.iucn]} on the IUCN Red List.`);
	if (s.extinct) f.push(`🪦 This species is extinct.`);
	if (s.nl_status) f.push(`🇳🇱 Dutch status: ${s.nl_status}.`);
	return f;
}

type Row = {
	inat_id: number;
	group: string;
	slug: string;
	scientific_name: string;
	name_en: string | null;
	name_nl: string | null;
	search: string;
	gbif_key: number | null;
	wikidata: string | null;
	wikipedia_en: string | null;
	wikipedia_nl: string | null;
	class_name: string | null;
	order_name: string | null;
	family_name: string | null;
	iucn: string | null;
	nl_status: string | null;
	nl_establishment: string | null;
	extinct: boolean;
	nl_obs: number;
	europe_obs: number;
	world_obs: number;
	gbif_nl: number | null;
	scope: string;
	summary: string | null;
	summary_nl: string | null;
	facts: string[];
	mass_kg: number | null;
	length_m: number | null;
	lifespan_years: number | null;
	named_year: number | null;
	named_by: string | null;
	sources: string[];
};
const wikiTitle = (url: string | null | undefined, host: string) => {
	if (!url || !url.includes(host)) return null;
	try {
		return decodeURIComponent(url.split('/wiki/')[1] ?? '').replace(/_/g, ' ') || null;
	} catch {
		return null;
	}
};

async function main() {
	await useCache(cacheDir, 'wildlife-api.json');
	const url = process.env.DB_URL;
	if (!dryRun && !url) throw new Error('Set DB_URL (or use --dry-run)');
	const sql = url ? postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
	try {
		const existing = sql
			? await sql<{ inat_id: number; dex: number }[]>`select inat_id, dex from wildlife.species`
			: [];
		log('collecting species lists…');
		const cands = await collect(existing.map((e) => e.inat_id));
		const ids = [...cands.keys()];
		log(`${ids.length} candidate species; fetching details…`);
		const en = await details(ids, 'en');
		const nl = await details(ids, 'nl');
		await saveCache(true);
		log('NL / Europe counts…');
		const nlCounts = await placeCounts(ids, PLACE_NL);
		const euCounts = await placeCounts(ids, PLACE_EUROPE);
		await saveCache(true);
		log('Wikidata…');
		const wd = await wikidata(ids);
		log('GBIF NL occurrence counts…');
		const gbif = await gbifNl();

		const rows: Row[] = [];
		for (const id of ids) {
			const t = en.get(id);
			const tn = nl.get(id);
			if (!t || t.rank !== 'species' || EXCLUDE.has(t.name)) continue;
			const anc = t.ancestors ?? [];
			if (!anc.some((a) => a.name === 'Animalia')) continue;
			const g = groupOf(anc.map((a) => a.name));
			const w = wd.get(id);
			const nameEn = t.preferred_common_name || t.english_common_name || null;
			let nameNl: string | null = tn?.preferred_common_name || null;
			if (nameNl && nameEn && nameNl === nameEn && w?.nl && w.nl !== t.name && w.nl !== nameEn)
				nameNl = w.nl;
			if (!nameNl && w?.nl && w.nl.toLowerCase() !== t.name.toLowerCase()) nameNl = w.nl;
			if (nameNl) nameNl = nameNl.charAt(0).toUpperCase() + nameNl.slice(1);
			const statuses = [...(tn?.conservation_statuses ?? []), ...(t.conservation_statuses ?? [])];
			const global = statuses.find((s) => !s.place && s.authority === 'IUCN Red List');
			let iucn = global?.iucn != null ? (IUCN_BY_NUMBER[global.iucn] ?? null) : null;
			if (!iucn && global?.status && /^(LC|NT|VU|EN|CR|EW|EX|DD)$/i.test(global.status))
				iucn = global.status.toUpperCase();
			if (!iucn) iucn = w?.iucn ?? null;
			if (t.extinct && !iucn) iucn = 'EX';
			const nlS = statuses.find((s) => s.place?.id === PLACE_NL);
			const em = (tn ?? t).establishment_means;
			const nlEst = em?.place?.id === PLACE_NL ? (em.establishment_means ?? null) : null;
			const nlObs = nlCounts.get(id) ?? 0;
			const cand = cands.get(id)!;
			const rank = (r: string) => anc.find((a) => a.rank === r)?.name ?? null;
			const wikipediaEn = wikiTitle(t.wikipedia_url, 'en.wikipedia.org');
			const wikipediaNl = w?.nlwiki ?? wikiTitle(tn?.wikipedia_url, 'nl.wikipedia.org');
			const summaryNl = tn?.wikipedia_url?.includes('nl.wikipedia.org')
				? trimSummary(tn.wikipedia_summary)
				: null;
			const row: Row = {
				inat_id: id,
				group: g.slug,
				slug: slugify(t.name),
				scientific_name: t.name,
				name_en: nameEn,
				name_nl: nameNl,
				search: [nameNl, nameEn, t.name, rank('order'), rank('family'), rank('genus')]
					.filter(Boolean)
					.join(' ')
					.toLowerCase(),
				gbif_key: w?.gbif ?? null,
				wikidata: w?.item ?? null,
				wikipedia_en: wikipediaEn,
				wikipedia_nl: wikipediaNl,
				class_name: rank('class'),
				order_name: rank('order'),
				family_name: rank('family'),
				iucn,
				nl_status: nlS ? nlS.description?.trim() || nlS.status : null,
				nl_establishment: nlEst,
				extinct: !!t.extinct,
				nl_obs: nlObs,
				europe_obs: euCounts.get(id) ?? 0,
				world_obs: t.observations_count ?? 0,
				gbif_nl: w?.gbif != null && gbif ? (gbif.get(w.gbif) ?? 0) : null,
				scope:
					nlObs >= MIN_NL
						? 'nl'
						: (euCounts.get(id) ?? 0) > 0
							? 'europe'
							: cand.scope === 'nl'
								? 'europe'
								: 'world',
				summary: trimSummary(t.wikipedia_summary),
				summary_nl: summaryNl,
				facts: [],
				mass_kg: w?.mass != null && w.mass > 0 && w.mass < 200_000 ? w.mass : null,
				length_m: w?.len != null && w.len > 0 && w.len < 40 ? w.len : null,
				lifespan_years:
					w?.life != null && w.life > 0 ? Math.round((w.life / 31_557_600) * 10) / 10 : null,
				named_year: w?.year ?? null,
				named_by: w?.author ?? null,
				sources: [
					'iNaturalist',
					...(w ? ['Wikidata'] : []),
					...(w?.gbif != null && gbif ? ['GBIF'] : [])
				]
			};
			if (row.lifespan_years != null && (row.lifespan_years < 0.01 || row.lifespan_years > 500))
				row.lifespan_years = null;
			rows.push(row);
		}
		// Unique slugs (scientific names are unique on iNaturalist, but be safe).
		const seen = new Set<string>();
		for (const r of rows) {
			let s = r.slug;
			for (let n = 2; seen.has(s); n++) s = `${r.slug}-${n}`;
			r.slug = s;
			seen.add(s);
		}
		// Fun facts with the rank among the group in NL.
		const byGroup = new Map<string, Row[]>();
		for (const r of rows) byGroup.set(r.group, [...(byGroup.get(r.group) ?? []), r]);
		for (const [slug, list] of byGroup) {
			const g = GROUPS.find((x) => x.slug === slug)!;
			const ranked = list.filter((r) => r.nl_obs > 0).toSorted((a, b) => b.nl_obs - a.nl_obs);
			for (const r of list) {
				const i = ranked.indexOf(r);
				r.facts = funFacts(r, i >= 0 ? i + 1 : null, g.name);
			}
		}
		const summary = GROUPS.map((g) => `${g.slug}: ${byGroup.get(g.slug)?.length ?? 0}`).join(', ');
		log(`${rows.length} species (${summary})`);
		if (dryRun || !sql) {
			log(JSON.stringify(rows.slice(0, 3), null, 1));
			return;
		}

		const known = new Map(existing.map((e) => [e.inat_id, e.dex]));
		let nextDex = Math.max(0, ...existing.map((e) => e.dex)) + 1;
		const order = (r: Row) => GROUPS.findIndex((g) => g.slug === r.group);
		const scopeOrder = { nl: 0, europe: 1, world: 2 } as Record<string, number>;
		const sorted = rows.toSorted(
			(a, b) =>
				order(a) - order(b) ||
				scopeOrder[a.scope] - scopeOrder[b.scope] ||
				b.nl_obs - a.nl_obs ||
				b.world_obs - a.world_obs ||
				a.scientific_name.localeCompare(b.scientific_name)
		);
		const added: Row[] = [];
		const dexOf = new Map<number, number>();
		for (const r of sorted) {
			const d = known.get(r.inat_id);
			if (d != null) dexOf.set(r.inat_id, d);
			else {
				dexOf.set(r.inat_id, nextDex++);
				added.push(r);
			}
		}
		await sql.begin(async (tx) => {
			const groupIds = new Map<string, number>();
			for (const [i, g] of GROUPS.entries()) {
				const [row] = await tx<{ id: number }[]>`
					insert into wildlife.groups (slug, name, name_nl, emoji, sort_order)
					values (${g.slug}, ${g.name}, ${g.nameNl}, ${g.emoji}, ${i})
					on conflict (slug) do update set name = excluded.name, name_nl = excluded.name_nl,
						emoji = excluded.emoji, sort_order = excluded.sort_order
					returning id`;
				groupIds.set(g.slug, row.id);
			}
			// Avoid unique-slug clashes while rows move around.
			await tx`update wildlife.species set slug = slug || '~' || id where slug not like '%~%'`;
			for (const r of sorted) {
				const { group, ...rest } = r;
				const row = { ...rest, dex: dexOf.get(r.inat_id)!, group_id: groupIds.get(group)! };
				await tx`
					insert into wildlife.species ${tx(row)}
					on conflict (inat_id) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'inat_id' && k !== 'dex') as (keyof typeof row)[])}`;
			}
			// Restore slugs of species that were not part of this run (kept, not deleted).
			await tx`update wildlife.species set slug = split_part(slug, '~', 1) where slug like '%~%'`;
			await tx`
				update wildlife.groups g set
					species_count = (select count(*) from wildlife.species s where s.group_id = g.id),
					nl_count = (select count(*) from wildlife.species s where s.group_id = g.id and s.nl_obs > 0)`;
		});
		log(`written: ${rows.length} species, ${added.length} new`);
		if (reportNew) {
			const [totals] = await sql`
				select (select count(*)::int from wildlife.species) as species,
					(select count(*)::int from wildlife.groups where species_count > 0) as groups,
					(select count(*)::int from wildlife.photos) as photos`;
			console.log(
				JSON.stringify(
					{
						generatedAt: new Date().toISOString(),
						totals,
						newSpecies: (existing.length ? added : added.slice(0, 0)).map((r) => ({
							dex: dexOf.get(r.inat_id),
							inatId: r.inat_id,
							name: r.name_en ?? r.scientific_name,
							nameNl: r.name_nl,
							scientificName: r.scientific_name,
							group: r.group,
							iucn: r.iucn,
							nlObs: r.nl_obs,
							url: `${SITE}/${r.group}/${r.slug}`
						}))
					},
					null,
					1
				)
			);
		}
	} finally {
		await saveCache(true);
		await sql?.end();
	}
}

await main();