From ba44365c7199dd2adad1d93376bba2e00833ebc9 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sat, 24 Oct 2026 21:58:35 +0000 Subject: [PATCH] wildlife: iNaturalist/Wikidata/GBIF import, CC photos and refresh script --- package.json | 3 + scripts/wildlife-import.ts | 603 ++++++++++++++++++++++++++++++++++++ scripts/wildlife-inat.ts | 211 +++++++++++++ scripts/wildlife-photos.ts | 172 ++++++++++ scripts/wildlife-refresh.sh | 33 ++ 5 files changed, 1022 insertions(+) create mode 100644 scripts/wildlife-import.ts create mode 100644 scripts/wildlife-inat.ts create mode 100644 scripts/wildlife-photos.ts create mode 100755 scripts/wildlife-refresh.sh diff --git a/package.json b/package.json index 70487e8..62b0db4 100755 --- a/package.json +++ b/package.json @@ -26,6 +26,9 @@ "planes:import": "bun scripts/planes-import.ts", "planes:photos": "bun scripts/planes-photos.ts", "planes:refresh": "scripts/planes-refresh.sh", + "wildlife:import": "bun scripts/wildlife-import.ts", + "wildlife:photos": "bun scripts/wildlife-photos.ts", + "wildlife:refresh": "scripts/wildlife-refresh.sh", "db:seed:cod": "bun scripts/seed-cod-catalog.ts", "cs-prices:ingest": "bun scripts/ingest-cs-prices.ts", "house:worker": "bun scripts/house-worker.ts", diff --git a/scripts/wildlife-import.ts b/scripts/wildlife-import.ts new file mode 100644 index 0000000..9cd7d32 --- /dev/null +++ b/scripts/wildlife-import.ts @@ -0,0 +1,603 @@ +// Builds the public wildlife pokedex (/trackers/wildlife, schema `wildlife`) from open data. +// +// DB_URL=... bun scripts/wildlife-import.ts fetch sources and write to the database +// bun scripts/wildlife-import.ts --dry-run fetch + build only, print a summary +// bun scripts/wildlife-import.ts --cache /tmp/wildlife reuse API responses from that folder +// bun scripts/wildlife-import.ts --report-new JSON of species new since the last run on stdout +// +// Which species (wild animals only, no pets/livestock, no captive observations): +// - the most observed species per group in the Netherlands (iNaturalist research grade), +// - the most observed species per group in Europe, +// - the most observed species per group worldwide, plus a short list of famous animals. +// Species already in the dex are always kept (refreshed, never deleted), so Dex numbers stay +// stable and new species get the next number. +// Per species: names (iNaturalist nl/en locale, Wikidata nl label), taxonomy, IUCN Red List +// (iNaturalist, Wikidata P141 fallback), Dutch red list status, establishment in NL, iNat +// observation counts (NL, Europe, world), Wikipedia intros (en + nl via iNaturalist), +// Wikidata mass/length/lifespan/name year+author and GBIF occurrence records in NL. +import postgres from 'postgres'; +import { + chunks, + log as logErr, + saveCache, + slugify, + sparql, + stripHtml, + useCache, + getJson +} from './planes-wiki'; +import { + groupOf, + GROUPS, + inat, + IUCN_BY_NUMBER, + PLACE_EUROPE, + PLACE_NL, + UA, + type InatList, + type InatTaxon +} from './wildlife-inat'; + +const args = process.argv.slice(2); +const dryRun = args.includes('--dry-run'); +const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null); +const cacheDir = opt('--cache'); +const reportNew = args.includes('--report-new'); +const SITE = 'https://stanrunge.dev/trackers/wildlife'; +const log = (...a: unknown[]) => (reportNew ? logErr(...a) : console.log(...a)); + +// iconic taxon filter -> how many species to take (NL, Europe extra, world extra) +const ICONIC: { iconic: string; taxonId: number; nl: number; eu: number; world: number }[] = [ + { iconic: 'Aves', taxonId: 3, nl: 450, eu: 250, world: 300 }, + { iconic: 'Mammalia', taxonId: 40151, nl: 120, eu: 120, world: 350 }, + { iconic: 'Reptilia', taxonId: 26036, nl: 40, eu: 100, world: 250 }, + { iconic: 'Amphibia', taxonId: 20978, nl: 30, eu: 50, world: 100 }, + { iconic: 'Actinopterygii', taxonId: 47178, nl: 150, eu: 80, world: 150 }, + { iconic: 'Insecta', taxonId: 47158, nl: 1300, eu: 300, world: 200 }, + { iconic: 'Arachnida', taxonId: 47119, nl: 150, eu: 50, world: 50 }, + { iconic: 'Mollusca', taxonId: 47115, nl: 180, eu: 50, world: 50 }, + { iconic: 'Animalia', taxonId: 1, nl: 250, eu: 80, world: 0 } +]; +// Extra worldwide lists for groups without their own iconic taxon. +const WORLD_EXTRA = [ + { taxonId: 47273, n: 60 }, // Elasmobranchii (sharks & rays) + { taxonId: 47534, n: 30 }, // Cnidaria + { taxonId: 47549, n: 30 }, // Echinodermata + { taxonId: 85493, n: 40 } // Malacostraca (crabs, lobsters, ...) +]; +const MIN_NL = 5; +const FAMOUS = + `Panthera tigris, Panthera leo, Panthera pardus, Panthera uncia, Panthera onca, Acinonyx jubatus, + Ailuropoda melanoleuca, Ailurus fulgens, Ursus maritimus, Ursus arctos, Canis lupus, Gulo gulo, Lynx lynx, + Bison bonasus, Loxodonta africana, Elephas maximus, Giraffa camelopardalis, Okapia johnstoni, Equus quagga, + Hippopotamus amphibius, Ceratotherium simum, Diceros bicornis, Rhinoceros unicornis, Gorilla gorilla, + Gorilla beringei, Pan troglodytes, Pan paniscus, Pongo pygmaeus, Lemur catta, Bradypus variegatus, + Myrmecophaga tridactyla, Hydrochoerus hydrochaeris, Phascolarctos cinereus, Macropus giganteus, + Ornithorhynchus anatinus, Tachyglossus aculeatus, Sarcophilus harrisii, Balaenoptera musculus, + Megaptera novaeangliae, Physeter macrocephalus, Orcinus orca, Tursiops truncatus, Monodon monoceros, + Odobenus rosmarus, Enhydra lutris, Aptenodytes forsteri, Spheniscus demersus, Struthio camelus, + Dromaius novaehollandiae, Aquila chrysaetos, Haliaeetus leucocephalus, Harpia harpyja, Vultur gryphus, + Diomedea exulans, Fratercula arctica, Pavo cristatus, Ramphastos toco, Phoenicopterus roseus, + Crocodylus niloticus, Crocodylus porosus, Alligator mississippiensis, Varanus komodoensis, Chelonia mydas, + Dermochelys coriacea, Chelonoidis niger, Ophiophagus hannah, Iguana iguana, Chamaeleo calyptratus, + Ambystoma mexicanum, Andrias japonicus, Dendrobates tinctorius, Phyllobates terribilis, + Carcharodon carcharias, Rhincodon typus, Mobula birostris, Latimeria chalumnae, Hippocampus hippocampus, + Amphiprion ocellaris, Octopus vulgaris, Architeuthis dux, Danaus plexippus, Dynastes hercules, + Apis mellifera, Lucanus cervus, Aurelia aurita` + .split(',') + .map((s) => s.trim()) + .filter(Boolean); +// Pets, livestock and us. +const EXCLUDE = new Set( + `Homo sapiens, Canis familiaris, Felis catus, Bos taurus, Equus caballus, Ovis aries, Capra hircus, + Sus domesticus, Gallus gallus, Equus asinus, Cavia porcellus, Bombyx mori, Lama glama, Vicugna pacos, + Bubalus bubalis, Camelus dromedarius, Bos indicus, Mesocricetus auratus, Meleagris gallopavo domesticus` + .split(',') + .map((s) => s.trim()) +); + +type Cand = { id: number; scope: 'nl' | 'europe' | 'world'; name: string }; + +async function speciesCounts( + place: number, + iconic: string, + n: number +): Promise { + const out: InatList['results'] = []; + for (let page = 1; out.length < n && page <= 10; page++) { + const res = await inat( + `/observations/species_counts?place_id=${place}&iconic_taxa=${iconic}&captive=false&quality_grade=research&per_page=500&page=${page}&locale=en` + ); + out.push(...res.results); + if (res.results.length < 500) break; + } + return out; +} +async function topWorld(taxonId: number, n: number): Promise { + const out: InatTaxon[] = []; + for (let page = 1; out.length < n && page <= 10; page++) { + const res = await inat<{ results: InatTaxon[] }>( + `/taxa?rank=species&taxon_id=${taxonId}&is_active=true&order_by=observations_count&order=desc&per_page=200&page=${page}` + ); + out.push(...res.results); + if (res.results.length < 200) break; + } + return out; +} + +async function collect(existing: number[]): Promise> { + const cands = new Map(); + const add = (t: InatTaxon, scope: Cand['scope']) => { + if (t.rank !== 'species' || EXCLUDE.has(t.name) || cands.has(t.id)) return false; + cands.set(t.id, { id: t.id, scope, name: t.name }); + return true; + }; + for (const g of ICONIC) { + const nl = await speciesCounts(PLACE_NL, g.iconic, g.nl * 2); + let k = 0; + for (const r of nl) if (k < g.nl && r.count >= MIN_NL && add(r.taxon, 'nl')) k++; + const eu = await speciesCounts(PLACE_EUROPE, g.iconic, g.eu * 2 + g.nl); + let e = 0; + for (const r of eu) if (e < g.eu && add(r.taxon, 'europe')) e++; + let w = 0; + if (g.world) + for (const t of await topWorld(g.taxonId, g.world * 2)) + if (w < g.world && add(t, 'world')) w++; + log(` ${g.iconic}: ${k} NL, +${e} Europe, +${w} world`); + } + for (const x of WORLD_EXTRA) { + let w = 0; + for (const t of await topWorld(x.taxonId, x.n * 2)) if (w < x.n && add(t, 'world')) w++; + } + let famous = 0; + for (const name of FAMOUS) { + const res = await inat<{ results: InatTaxon[] }>( + `/taxa?q=${encodeURIComponent(name)}&rank=species&is_active=true&per_page=5` + ); + const t = res.results.find((r) => r.name.toLowerCase() === name.toLowerCase()); + if (t && add(t, 'world')) famous++; + if (!t) log(` famous species not found: ${name}`); + } + log(` +${famous} famous species`); + for (const id of existing) if (!cands.has(id)) cands.set(id, { id, scope: 'world', name: '' }); + return cands; +} + +async function details(ids: number[], locale: 'en' | 'nl') { + const out = new Map(); + for (const [i, batch] of chunks(ids, 30).entries()) { + const res = await inat<{ results: InatTaxon[] }>( + `/taxa/${batch.join(',')}?locale=${locale}&preferred_place_id=${PLACE_NL}` + ); + for (const t of res.results) out.set(t.id, t); + if (i % 20 === 0) log(` taxa (${locale}) ${Math.min((i + 1) * 30, ids.length)}/${ids.length}`); + } + return out; +} +async function placeCounts(ids: number[], place: number) { + const out = new Map(); + for (const batch of chunks(ids, 200)) { + const res = await inat( + `/observations/species_counts?place_id=${place}&taxon_id=${batch.join(',')}&captive=false&quality_grade=research&per_page=500` + ); + for (const r of res.results) out.set(r.taxon.id, r.count); + } + return out; +} + +type Wd = { + item: string; + nl: string | null; + iucn: string | null; + gbif: number | null; + nlwiki: string | null; + mass: number | null; + len: number | null; + life: number | null; + year: number | null; + author: string | null; +}; +const WD_IUCN: Record = { + Q211005: 'LC', + Q719675: 'NT', + Q278113: 'VU', + Q11394: 'EN', + Q219127: 'CR', + Q239509: 'EW', + Q237350: 'EX', + Q3245245: 'DD' +}; +async function wikidata(ids: number[]) { + const out = new Map(); + for (const batch of chunks(ids, 250)) { + const rows = + await sparql(`SELECT ?inat ?item ?nl ?iucn ?gbif ?nlwiki ?mass ?len ?life ?year ?authorLabel WHERE { + VALUES ?inat { ${batch.map((id) => `"${id}"`).join(' ')} } + ?item wdt:P3151 ?inat . + OPTIONAL { ?item rdfs:label ?nl FILTER(lang(?nl) = "nl") } + OPTIONAL { ?item wdt:P141 ?iucn } + OPTIONAL { ?item wdt:P846 ?gbif } + OPTIONAL { ?nlwiki schema:about ?item ; schema:isPartOf } + OPTIONAL { ?item p:P2067/psn:P2067/wikibase:quantityAmount ?mass } + OPTIONAL { ?item p:P2043/psn:P2043/wikibase:quantityAmount ?len } + OPTIONAL { ?item p:P2250/psn:P2250/wikibase:quantityAmount ?life } + OPTIONAL { ?item p:P225 ?st . ?st pq:P574 ?year . + OPTIONAL { ?st pq:P405 ?author . ?author rdfs:label ?authorLabel FILTER(lang(?authorLabel) = "en") } } + }`); + for (const r of rows) { + const id = Number(r.inat); + const prev = out.get(id); + const num = (v: string | undefined) => + v != null && Number.isFinite(Number(v)) ? Number(v) : null; + const max = (a: number | null | undefined, b: number | null) => + a == null ? b : b == null ? a : Math.max(a, b); + const yr = r.year ? Number(r.year.slice(0, 4)) : null; + out.set(id, { + item: r.item.split('/').at(-1)!, + nl: prev?.nl ?? r.nl ?? null, + iucn: prev?.iucn ?? (r.iucn ? (WD_IUCN[r.iucn.split('/').at(-1)!] ?? null) : null), + gbif: prev?.gbif ?? num(r.gbif), + nlwiki: + prev?.nlwiki ?? + (r.nlwiki ? decodeURIComponent(r.nlwiki.split('/wiki/')[1]).replace(/_/g, ' ') : null), + mass: max(prev?.mass, num(r.mass)), + len: max(prev?.len, num(r.len)), + life: max(prev?.life, num(r.life)), + year: prev?.year ?? (yr && yr > 1700 && yr <= new Date().getFullYear() ? yr : null), + author: prev?.author ?? r.authorLabel ?? null + }); + } + } + return out; +} + +async function gbifNl(): Promise | null> { + const out = new Map(); + try { + for (let offset = 0; offset < 100_000; offset += 5000) { + const res = await getJson<{ facets: { counts: { name: string; count: number }[] }[] }>( + `https://api.gbif.org/v1/occurrence/search?country=NL&taxonKey=1&limit=0&facet=speciesKey&facetLimit=5000&facetOffset=${offset}`, + { headers: { 'user-agent': UA } } + ); + const counts = res.facets[0]?.counts ?? []; + for (const c of counts) out.set(Number(c.name), c.count); + if (counts.length < 5000) return out; + } + return out; + } catch (e) { + log(` GBIF facet failed: ${(e as Error).message}`); + return null; + } +} + +const fmt = (n: number) => n.toLocaleString('en-GB'); +const trimSummary = (html: string | null | undefined) => { + if (!html) return null; + const text = stripHtml(html).replace(/\s*\[\d+\]/g, ''); + const sentences = text.match(/[^.!?]+[.!?]+(\s|$)/g) ?? [text]; + let out = ''; + for (const s of sentences) { + if (out && (out + s).length > 650) break; + out += s; + } + return out.trim() || null; +}; +const IUCN_TEXT: Record = { + LC: 'Least Concern', + NT: 'Near Threatened', + VU: 'Vulnerable', + EN: 'Endangered', + CR: 'Critically Endangered', + EW: 'Extinct in the Wild', + EX: 'Extinct', + DD: 'Data Deficient' +}; + +function funFacts(s: Row, rankInGroup: number | null, groupName: string): string[] { + const f: string[] = []; + if (s.nl_obs > 0) + f.push( + `👀 Spotted ${fmt(s.nl_obs)} times in the Netherlands on iNaturalist${rankInGroup && rankInGroup <= 50 ? `: the #${rankInGroup} most seen of the ${groupName.toLowerCase()} in NL` : ''}.` + ); + else + f.push( + `✈️ No confirmed wild sightings in the Netherlands on iNaturalist: you'd have to travel for this one.` + ); + if (s.world_obs > 0) f.push(`🌍 ${fmt(s.world_obs)} observations worldwide on iNaturalist.`); + if (s.mass_kg != null) { + const kg = s.mass_kg; + if (kg >= 1000) + f.push( + `🐘 Can weigh about ${fmt(Math.round(kg / 1000))} tonnes: as much as ${fmt(Math.max(1, Math.round(kg / 1400)))} cars.` + ); + else if (kg >= 1) f.push(`⚖️ Weighs up to about ${fmt(Math.round(kg * 10) / 10)} kg.`); + else if (kg >= 0.001) f.push(`🪶 Weighs only about ${fmt(Math.round(kg * 1000))} g.`); + } + if (s.length_m != null) { + const m = s.length_m; + if (m >= 2) + f.push( + `📏 Up to ${fmt(Math.round(m * 10) / 10)} m long: ${(m / 1.8).toFixed(1)}× a tall person.` + ); + else if (m >= 0.01) f.push(`📏 About ${fmt(Math.round(m * 100))} cm long.`); + else f.push(`🔬 Only about ${fmt(Math.round(m * 1000 * 10) / 10)} mm long.`); + } + if (s.lifespan_years != null && s.lifespan_years >= 1) + f.push(`⏳ Can live up to ${fmt(Math.round(s.lifespan_years))} years.`); + if (s.named_year) + f.push( + `📜 Scientifically named ${s.named_by ? `by ${s.named_by} ` : ''}in ${s.named_year} (${new Date().getFullYear() - s.named_year} years ago).` + ); + if (s.nl_establishment === 'introduced') + f.push(`🧳 Not originally Dutch: it was introduced to the Netherlands.`); + if (s.iucn && ['VU', 'EN', 'CR', 'EW'].includes(s.iucn)) + f.push(`⚠️ Listed as ${IUCN_TEXT[s.iucn]} on the IUCN Red List.`); + if (s.extinct) f.push(`🪦 This species is extinct.`); + if (s.nl_status) f.push(`🇳🇱 Dutch status: ${s.nl_status}.`); + return f; +} + +type Row = { + inat_id: number; + group: string; + slug: string; + scientific_name: string; + name_en: string | null; + name_nl: string | null; + search: string; + gbif_key: number | null; + wikidata: string | null; + wikipedia_en: string | null; + wikipedia_nl: string | null; + class_name: string | null; + order_name: string | null; + family_name: string | null; + iucn: string | null; + nl_status: string | null; + nl_establishment: string | null; + extinct: boolean; + nl_obs: number; + europe_obs: number; + world_obs: number; + gbif_nl: number | null; + scope: string; + summary: string | null; + summary_nl: string | null; + facts: string[]; + mass_kg: number | null; + length_m: number | null; + lifespan_years: number | null; + named_year: number | null; + named_by: string | null; + sources: string[]; +}; +const wikiTitle = (url: string | null | undefined, host: string) => { + if (!url || !url.includes(host)) return null; + try { + return decodeURIComponent(url.split('/wiki/')[1] ?? '').replace(/_/g, ' ') || null; + } catch { + return null; + } +}; + +async function main() { + await useCache(cacheDir, 'wildlife-api.json'); + const url = process.env.DB_URL; + if (!dryRun && !url) throw new Error('Set DB_URL (or use --dry-run)'); + const sql = url ? postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null; + try { + const existing = sql + ? await sql<{ inat_id: number; dex: number }[]>`select inat_id, dex from wildlife.species` + : []; + log('collecting species lists…'); + const cands = await collect(existing.map((e) => e.inat_id)); + const ids = [...cands.keys()]; + log(`${ids.length} candidate species; fetching details…`); + const en = await details(ids, 'en'); + const nl = await details(ids, 'nl'); + await saveCache(true); + log('NL / Europe counts…'); + const nlCounts = await placeCounts(ids, PLACE_NL); + const euCounts = await placeCounts(ids, PLACE_EUROPE); + await saveCache(true); + log('Wikidata…'); + const wd = await wikidata(ids); + log('GBIF NL occurrence counts…'); + const gbif = await gbifNl(); + + const rows: Row[] = []; + for (const id of ids) { + const t = en.get(id); + const tn = nl.get(id); + if (!t || t.rank !== 'species' || EXCLUDE.has(t.name)) continue; + const anc = t.ancestors ?? []; + if (!anc.some((a) => a.name === 'Animalia')) continue; + const g = groupOf(anc.map((a) => a.name)); + const w = wd.get(id); + const nameEn = t.preferred_common_name || t.english_common_name || null; + let nameNl: string | null = tn?.preferred_common_name || null; + if (nameNl && nameEn && nameNl === nameEn && w?.nl && w.nl !== t.name && w.nl !== nameEn) + nameNl = w.nl; + if (!nameNl && w?.nl && w.nl.toLowerCase() !== t.name.toLowerCase()) nameNl = w.nl; + if (nameNl) nameNl = nameNl.charAt(0).toUpperCase() + nameNl.slice(1); + const statuses = t.conservation_statuses ?? []; + const global = statuses.find((s) => !s.place && s.authority === 'IUCN Red List'); + let iucn = global?.iucn != null ? (IUCN_BY_NUMBER[global.iucn] ?? null) : null; + if (!iucn && global?.status && /^(LC|NT|VU|EN|CR|EW|EX|DD)$/i.test(global.status)) + iucn = global.status.toUpperCase(); + if (!iucn) iucn = w?.iucn ?? null; + if (t.extinct && !iucn) iucn = 'EX'; + const nlS = statuses.find((s) => s.place?.id === PLACE_NL); + const em = (tn ?? t).establishment_means; + const nlEst = em?.place?.id === PLACE_NL ? (em.establishment_means ?? null) : null; + const nlObs = nlCounts.get(id) ?? 0; + const cand = cands.get(id)!; + const rank = (r: string) => anc.find((a) => a.rank === r)?.name ?? null; + const wikipediaEn = wikiTitle(t.wikipedia_url, 'en.wikipedia.org'); + const wikipediaNl = w?.nlwiki ?? wikiTitle(tn?.wikipedia_url, 'nl.wikipedia.org'); + const summaryNl = tn?.wikipedia_url?.includes('nl.wikipedia.org') + ? trimSummary(tn.wikipedia_summary) + : null; + const row: Row = { + inat_id: id, + group: g.slug, + slug: slugify(t.name), + scientific_name: t.name, + name_en: nameEn, + name_nl: nameNl, + search: [nameNl, nameEn, t.name, rank('order'), rank('family'), rank('genus')] + .filter(Boolean) + .join(' ') + .toLowerCase(), + gbif_key: w?.gbif ?? null, + wikidata: w?.item ?? null, + wikipedia_en: wikipediaEn, + wikipedia_nl: wikipediaNl, + class_name: rank('class'), + order_name: rank('order'), + family_name: rank('family'), + iucn, + nl_status: nlS ? nlS.description?.trim() || nlS.status : null, + nl_establishment: nlEst, + extinct: !!t.extinct, + nl_obs: nlObs, + europe_obs: euCounts.get(id) ?? 0, + world_obs: t.observations_count ?? 0, + gbif_nl: w?.gbif != null && gbif ? (gbif.get(w.gbif) ?? 0) : null, + scope: + nlObs >= MIN_NL + ? 'nl' + : (euCounts.get(id) ?? 0) > 0 + ? 'europe' + : cand.scope === 'nl' + ? 'europe' + : 'world', + summary: trimSummary(t.wikipedia_summary), + summary_nl: summaryNl, + facts: [], + mass_kg: w?.mass != null && w.mass > 0 && w.mass < 200_000 ? w.mass : null, + length_m: w?.len != null && w.len > 0 && w.len < 40 ? w.len : null, + lifespan_years: + w?.life != null && w.life > 0 ? Math.round((w.life / 31_557_600) * 10) / 10 : null, + named_year: w?.year ?? null, + named_by: w?.author ?? null, + sources: [ + 'iNaturalist', + ...(w ? ['Wikidata'] : []), + ...(w?.gbif != null && gbif ? ['GBIF'] : []) + ] + }; + if (row.lifespan_years != null && (row.lifespan_years < 0.01 || row.lifespan_years > 500)) + row.lifespan_years = null; + rows.push(row); + } + // Unique slugs (scientific names are unique on iNaturalist, but be safe). + const seen = new Set(); + for (const r of rows) { + let s = r.slug; + for (let n = 2; seen.has(s); n++) s = `${r.slug}-${n}`; + r.slug = s; + seen.add(s); + } + // Fun facts with the rank among the group in NL. + const byGroup = new Map(); + for (const r of rows) byGroup.set(r.group, [...(byGroup.get(r.group) ?? []), r]); + for (const [slug, list] of byGroup) { + const g = GROUPS.find((x) => x.slug === slug)!; + const ranked = list.filter((r) => r.nl_obs > 0).toSorted((a, b) => b.nl_obs - a.nl_obs); + for (const r of list) { + const i = ranked.indexOf(r); + r.facts = funFacts(r, i >= 0 ? i + 1 : null, g.name); + } + } + const summary = GROUPS.map((g) => `${g.slug}: ${byGroup.get(g.slug)?.length ?? 0}`).join(', '); + log(`${rows.length} species (${summary})`); + if (dryRun || !sql) { + log(JSON.stringify(rows.slice(0, 3), null, 1)); + return; + } + + const known = new Map(existing.map((e) => [e.inat_id, e.dex])); + let nextDex = Math.max(0, ...existing.map((e) => e.dex)) + 1; + const order = (r: Row) => GROUPS.findIndex((g) => g.slug === r.group); + const scopeOrder = { nl: 0, europe: 1, world: 2 } as Record; + const sorted = rows.toSorted( + (a, b) => + order(a) - order(b) || + scopeOrder[a.scope] - scopeOrder[b.scope] || + b.nl_obs - a.nl_obs || + b.world_obs - a.world_obs || + a.scientific_name.localeCompare(b.scientific_name) + ); + const added: Row[] = []; + const dexOf = new Map(); + for (const r of sorted) { + const d = known.get(r.inat_id); + if (d != null) dexOf.set(r.inat_id, d); + else { + dexOf.set(r.inat_id, nextDex++); + added.push(r); + } + } + await sql.begin(async (tx) => { + const groupIds = new Map(); + for (const [i, g] of GROUPS.entries()) { + const [row] = await tx<{ id: number }[]>` + insert into wildlife.groups (slug, name, name_nl, emoji, sort_order) + values (${g.slug}, ${g.name}, ${g.nameNl}, ${g.emoji}, ${i}) + on conflict (slug) do update set name = excluded.name, name_nl = excluded.name_nl, + emoji = excluded.emoji, sort_order = excluded.sort_order + returning id`; + groupIds.set(g.slug, row.id); + } + // Avoid unique-slug clashes while rows move around. + await tx`update wildlife.species set slug = slug || '~' || id where slug not like '%~%'`; + for (const r of sorted) { + const { group, ...rest } = r; + const row = { ...rest, dex: dexOf.get(r.inat_id)!, group_id: groupIds.get(group)! }; + await tx` + insert into wildlife.species ${tx(row)} + on conflict (inat_id) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'inat_id' && k !== 'dex') as (keyof typeof row)[])}`; + } + // Restore slugs of species that were not part of this run (kept, not deleted). + await tx`update wildlife.species set slug = split_part(slug, '~', 1) where slug like '%~%'`; + await tx` + update wildlife.groups g set + species_count = (select count(*) from wildlife.species s where s.group_id = g.id), + nl_count = (select count(*) from wildlife.species s where s.group_id = g.id and s.nl_obs > 0)`; + }); + log(`written: ${rows.length} species, ${added.length} new`); + if (reportNew) { + const [totals] = await sql` + select (select count(*)::int from wildlife.species) as species, + (select count(*)::int from wildlife.groups where species_count > 0) as groups, + (select count(*)::int from wildlife.photos) as photos`; + console.log( + JSON.stringify( + { + generatedAt: new Date().toISOString(), + totals, + newSpecies: (existing.length ? added : added.slice(0, 0)).map((r) => ({ + dex: dexOf.get(r.inat_id), + inatId: r.inat_id, + name: r.name_en ?? r.scientific_name, + nameNl: r.name_nl, + scientificName: r.scientific_name, + group: r.group, + iucn: r.iucn, + nlObs: r.nl_obs, + url: `${SITE}/${r.group}/${r.slug}` + })) + }, + null, + 1 + ) + ); + } + } finally { + await saveCache(true); + await sql?.end(); + } +} + +await main(); diff --git a/scripts/wildlife-inat.ts b/scripts/wildlife-inat.ts new file mode 100644 index 0000000..89b0436 --- /dev/null +++ b/scripts/wildlife-inat.ts @@ -0,0 +1,211 @@ +// Shared helpers for the wildlife pokedex scripts (scripts/wildlife-import.ts, +// scripts/wildlife-photos.ts): polite iNaturalist API access (about 1 request per second, +// as the API asks), the group definitions and photo licence handling. +import { getJson, sleep } from './planes-wiki'; + +export const UA = + 'StanRungeWildlifeDex/1.0 (https://stanrunge.dev/trackers/wildlife; wildlife pokedex importer) bun'; +export const INAT = 'https://api.inaturalist.org/v1'; +export const PLACE_NL = 7506; +export const PLACE_EUROPE = 97391; + +let last = 0; +/** GET an iNaturalist API path, at most one call per ~1.1 s (cached when --cache is set). */ +export async function inat(path: string, useCached = true): Promise { + const url = `${INAT}${path}`; + const wait = last + 1100 - Date.now(); + if (wait > 0) await sleep(wait); + last = Date.now(); + return getJson(url, { headers: { 'user-agent': UA } }, useCached); +} + +export type InatPhoto = { + id: number; + license_code: string | null; + attribution: string; + attribution_name?: string | null; + url: string; + original_dimensions?: { width: number; height: number } | null; +}; +export type InatTaxon = { + id: number; + name: string; + rank: string; + is_active?: boolean; + extinct?: boolean; + iconic_taxon_name?: string | null; + preferred_common_name?: string | null; + english_common_name?: string | null; + observations_count?: number; + wikipedia_url?: string | null; + wikipedia_summary?: string | null; + default_photo?: InatPhoto | null; + taxon_photos?: { photo: InatPhoto }[]; + ancestors?: { id: number; rank: string; name: string }[]; + ancestor_ids?: number[]; + conservation_status?: { status?: string; iucn?: number; authority?: string } | null; + conservation_statuses?: { + status: string; + iucn: number | null; + authority: string | null; + description?: string | null; + place: { id: number } | null; + }[]; + establishment_means?: { establishment_means?: string; place?: { id: number } } | null; +}; +export type InatList = { + total_results: number; + page: number; + per_page: number; + results: { count: number; taxon: InatTaxon }[]; +}; + +export const CC = [ + 'cc0', + 'cc-by', + 'cc-by-sa', + 'cc-by-nc', + 'cc-by-nc-sa', + 'cc-by-nd', + 'cc-by-nc-nd' +]; +export const isCc = (code: string | null | undefined) => !!code && CC.includes(code.toLowerCase()); +export const licenseLabel = (code: string) => + code.toLowerCase() === 'cc0' ? 'CC0' : `${code.toUpperCase().replace(/^CC-/, 'CC ')} 4.0`; +export const licenseUrl = (code: string) => + code.toLowerCase() === 'cc0' + ? 'https://creativecommons.org/publicdomain/zero/1.0/' + : `https://creativecommons.org/licenses/${code.toLowerCase().replace(/^cc-/, '')}/4.0/`; +/** "(c) Jane Doe, some rights reserved (CC BY)" -> "Jane Doe". */ +export const author = (p: InatPhoto) => + p.attribution_name?.trim() || + p.attribution + .replace(/^\(c\)\s*/i, '') + .replace(/,\s*(some|no) rights reserved.*$/i, '') + .replace(/^no known copyright restrictions.*$/i, '') + .trim() || + null; +/** Any size URL -> the "medium" (500px) one. */ +export const mediumUrl = (url: string) => + url.replace(/\/(square|small|thumb|medium|large|original)\.(jpe?g|png|gif|webp)/i, '/medium.$2'); + +// Groups shown in the dex, first match wins (by ancestor names on iNaturalist). +export type GroupDef = { + slug: string; + name: string; + nameNl: string; + emoji: string; + match: string[]; +}; +export const GROUPS: GroupDef[] = [ + { slug: 'mammals', name: 'Mammals', nameNl: 'Zoogdieren', emoji: '🦊', match: ['Mammalia'] }, + { slug: 'birds', name: 'Birds', nameNl: 'Vogels', emoji: '🐦', match: ['Aves'] }, + { + slug: 'reptiles', + name: 'Reptiles', + nameNl: 'Reptielen', + emoji: '🦎', + match: ['Reptilia', 'Testudines', 'Squamata', 'Crocodylia'] + }, + { slug: 'amphibians', name: 'Amphibians', nameNl: 'Amfibieën', emoji: '🐸', match: ['Amphibia'] }, + { + slug: 'sharks-and-rays', + name: 'Sharks & rays', + nameNl: 'Haaien en roggen', + emoji: '🦈', + match: ['Elasmobranchii', 'Chondrichthyes'] + }, + { + slug: 'fish', + name: 'Fish', + nameNl: 'Vissen', + emoji: '🐟', + match: ['Actinopterygii', 'Sarcopterygii', 'Petromyzonti', 'Myxini', 'Agnatha'] + }, + { + slug: 'butterflies-and-moths', + name: 'Butterflies & moths', + nameNl: 'Vlinders', + emoji: '🦋', + match: ['Lepidoptera'] + }, + { slug: 'beetles', name: 'Beetles', nameNl: 'Kevers', emoji: '🪲', match: ['Coleoptera'] }, + { + slug: 'bees-wasps-and-ants', + name: 'Bees, wasps & ants', + nameNl: 'Bijen, wespen en mieren', + emoji: '🐝', + match: ['Hymenoptera'] + }, + { + slug: 'dragonflies', + name: 'Dragonflies & damselflies', + nameNl: 'Libellen', + emoji: '🪽', + match: ['Odonata'] + }, + { + slug: 'flies', + name: 'Flies & mosquitoes', + nameNl: 'Vliegen en muggen', + emoji: '🪰', + match: ['Diptera'] + }, + { + slug: 'bugs-and-other-insects', + name: 'Bugs, crickets & other insects', + nameNl: 'Wantsen, sprinkhanen en andere insecten', + emoji: '🦗', + match: ['Insecta'] + }, + { + slug: 'spiders', + name: 'Spiders & kin', + nameNl: 'Spinnen en verwanten', + emoji: '🕷️', + match: ['Arachnida'] + }, + { + slug: 'crustaceans', + name: 'Crabs & other crustaceans', + nameNl: 'Kreeftachtigen', + emoji: '🦀', + match: [ + 'Crustacea', + 'Malacostraca', + 'Maxillopoda', + 'Thecostraca', + 'Branchiopoda', + 'Hexanauplia' + ] + }, + { + slug: 'molluscs', + name: 'Snails, slugs & other molluscs', + nameNl: 'Weekdieren', + emoji: '🐌', + match: ['Mollusca'] + }, + { + slug: 'other-invertebrates', + name: 'Jellyfish, starfish, worms & co', + nameNl: 'Overige ongewervelden', + emoji: '🪼', + match: [] + } +]; +export function groupOf(ancestorNames: string[]): GroupDef { + for (const g of GROUPS) if (g.match.some((m) => ancestorNames.includes(m))) return g; + return GROUPS.at(-1)!; +} + +export const IUCN_BY_NUMBER: Record = { + 5: 'DD', + 10: 'LC', + 20: 'NT', + 30: 'VU', + 40: 'EN', + 50: 'CR', + 60: 'EW', + 70: 'EX' +}; diff --git a/scripts/wildlife-photos.ts b/scripts/wildlife-photos.ts new file mode 100644 index 0000000..8fc1474 --- /dev/null +++ b/scripts/wildlife-photos.ts @@ -0,0 +1,172 @@ +// Photos for the wildlife pokedex (/trackers/wildlife). Only Creative Commons licensed photos +// from iNaturalist are used; they are hotlinked (inaturalist-open-data S3), never downloaded, +// and every photo keeps its author, licence and a link back to iNaturalist. +// +// DB_URL=... bun scripts/wildlife-photos.ts (re)do photos for every species +// bun scripts/wildlife-photos.ts --missing only species never checked / without photos +// bun scripts/wildlife-photos.ts --limit 100 at most 100 species this run +// bun scripts/wildlife-photos.ts --cache /tmp/wildlife reuse API responses +// bun scripts/wildlife-photos.ts --missing --enrich report.json print the import report with photos +// +// Per species: the curated iNaturalist taxon photos with a CC licence, topped up with the most +// faved research-grade observation photos (CC licence only) until there are MAX photos. +import postgres from 'postgres'; +import { readFile } from 'node:fs/promises'; +import { chunks, log, saveCache, useCache } from './planes-wiki'; +import { + author, + inat, + isCc, + mediumUrl, + PLACE_NL, + type InatPhoto, + type InatTaxon +} from './wildlife-inat'; + +const args = process.argv.slice(2); +const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null); +const missing = args.includes('--missing'); +const limit = Number(opt('--limit')) || 0; +const enrich = opt('--enrich'); +const MAX = 10; +const MIN_TAXON = 7; // fewer CC taxon photos than this -> top up from observations +const LICENSES = 'cc0,cc-by,cc-by-sa,cc-by-nc,cc-by-nc-sa,cc-by-nd,cc-by-nc-nd'; + +type Photo = { + kind: string; + photo_id: number; + url: string; + width: number | null; + height: number | null; + author: string | null; + license: string; + page_url: string; + place: string | null; +}; +const toPhoto = (p: InatPhoto, kind: string, page: string, place: string | null): Photo | null => { + if (!isCc(p.license_code) || !p.url) return null; + const dims = p.original_dimensions; + const scale = dims ? Math.min(1, 500 / Math.max(dims.width, dims.height)) : 1; + return { + kind, + photo_id: p.id, + url: mediumUrl(p.url), + width: dims ? Math.round(dims.width * scale) : null, + height: dims ? Math.round(dims.height * scale) : null, + author: author(p), + license: p.license_code!.toLowerCase(), + page_url: page, + place + }; +}; + +type Obs = { id: number; uri?: string; place_guess?: string | null; photos: InatPhoto[] }; + +async function main() { + await useCache(opt('--cache'), 'wildlife-photos-api.json'); + const url = process.env.DB_URL; + if (!url) throw new Error('Set DB_URL'); + const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} }); + try { + let todo = await sql< + { id: number; inat_id: number; scientific_name: string; nl_obs: number }[] + >` + select id, inat_id, scientific_name, nl_obs from wildlife.species + ${missing ? sql`where photos_checked_at is null or photo_count = 0` : sql``} + order by nl_obs desc, world_obs desc`; + if (limit) todo = todo.slice(0, limit); + log(`photos for ${todo.length} species`); + let done = 0; + let total = 0; + for (const batch of chunks(todo, 30)) { + const res = await inat<{ results: InatTaxon[] }>( + `/taxa/${batch.map((b) => b.inat_id).join(',')}?locale=en`, + false + ); + const taxa = new Map(res.results.map((t) => [t.id, t])); + for (const s of batch) { + const t = taxa.get(s.inat_id); + const photos: Photo[] = []; + const seen = new Set(); + const push = (p: Photo | null) => { + if (p && !seen.has(p.photo_id) && photos.length < MAX) { + seen.add(p.photo_id); + photos.push(p); + } + }; + for (const tp of t?.taxon_photos ?? []) + push( + toPhoto(tp.photo, 'taxon', `https://www.inaturalist.org/photos/${tp.photo.id}`, null) + ); + if (photos.length < MIN_TAXON) { + // Prefer Dutch observations for species seen here, then anywhere. + const places = s.nl_obs >= 20 ? [`&place_id=${PLACE_NL}`, ''] : ['']; + for (const place of places) { + if (photos.length >= MAX) break; + const obs = await inat<{ results: Obs[] }>( + `/observations?taxon_id=${s.inat_id}${place}&quality_grade=research&captive=false&photo_license=${LICENSES}&order_by=votes&order=desc&per_page=20&locale=en` + ); + for (const o of obs.results) { + const page = `https://www.inaturalist.org/observations/${o.id}`; + // One photo per observation first, for variety. + push(toPhoto(o.photos[0], 'observation', page, o.place_guess ?? null)); + } + for (const o of obs.results) + for (const p of o.photos.slice(1, 2)) + push( + toPhoto( + p, + 'observation', + `https://www.inaturalist.org/observations/${o.id}`, + o.place_guess ?? null + ) + ); + } + } + await sql.begin(async (tx) => { + await tx`delete from wildlife.photos where species_id = ${s.id}`; + for (const [i, p] of photos.entries()) + await tx`insert into wildlife.photos ${tx({ ...p, species_id: s.id, position: i })}`; + await tx`update wildlife.species set photo = ${photos[0]?.url ?? null}, + photo_count = ${photos.length}, photos_checked_at = now() where id = ${s.id}`; + }); + total += photos.length; + done++; + } + await saveCache(); + if (done % 150 < 30) log(` ${done}/${todo.length} species, ${total} photos`); + } + await sql` + update wildlife.groups g set photo = ( + select s.photo from wildlife.species s where s.group_id = g.id and s.photo is not null + order by s.nl_obs desc, s.world_obs desc limit 1)`; + log(`done: ${done} species, ${total} photos`); + if (enrich) { + const report = JSON.parse(await readFile(enrich, 'utf8')); + const ids = (report.newSpecies ?? []).map((n: { inatId: number }) => n.inatId); + const rows = ids.length + ? await sql<{ inat_id: number; photo: string | null; photo_count: number }[]>` + select inat_id, photo, photo_count from wildlife.species where inat_id in ${sql(ids)}` + : []; + const by = new Map(rows.map((r) => [r.inat_id, r])); + const [totals] = await sql` + select (select count(*)::int from wildlife.species) as species, + (select count(*)::int from wildlife.groups where species_count > 0) as groups, + (select count(*)::int from wildlife.photos) as photos`; + report.totals = totals; + report.newSpecies = (report.newSpecies ?? []) + .map((n: { inatId: number }) => ({ + ...n, + photo: by.get(n.inatId)?.photo?.replace('/medium.', '/large.') ?? null, + photos: by.get(n.inatId)?.photo_count ?? 0 + })) + .toSorted((a: { nlObs: number }, b: { nlObs: number }) => b.nlObs - a.nlObs); + console.log(JSON.stringify(report, null, 1)); + } + } finally { + await saveCache(true); + await sql.end(); + } +} + +await main(); diff --git a/scripts/wildlife-refresh.sh b/scripts/wildlife-refresh.sh new file mode 100755 index 0000000..6b38732 --- /dev/null +++ b/scripts/wildlife-refresh.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Refresh of the wildlife pokedex (/trackers/wildlife) on stan-ovh. +# +# ssh stan@37.187.157.191 'cd ~/dev/website-wildlife && git fetch -q && git checkout -q --detach origin/main \ +# && bun install --frozen-lockfile >/dev/null && scripts/wildlife-refresh.sh --report-new' +# +# 1. bun scripts/wildlife-import.ts re-reads the species lists (NL, Europe, world) from +# iNaturalist + Wikidata + GBIF, refreshes counts/status of every species and adds species +# that entered the lists (existing ones are kept, Dex numbers are stable). +# 2. bun scripts/wildlife-photos.ts --missing CC-licensed iNaturalist photos for species +# without photos (new ones included). +# --report-new: stdout is one JSON report {generatedAt, totals, newSpecies[] (with photo)}. +# Without it, progress only. Progress goes to stderr. DB_URL is read from the running app +# container and never printed. +set -euo pipefail +cd "$(dirname "$0")/.." + +if [ -z "${DB_URL:-}" ]; then + DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' | + sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/') + export DB_URL +fi + +if [ "${1:-}" = "--report-new" ]; then + shift + report=$(mktemp /tmp/wildlife-report.XXXXXX.json) + trap 'rm -f "$report"' EXIT + bun scripts/wildlife-import.ts --report-new "$@" >"$report" + bun scripts/wildlife-photos.ts --missing --enrich "$report" +else + bun scripts/wildlife-import.ts "$@" >&2 + bun scripts/wildlife-photos.ts --missing >&2 +fi