wildlife: iNaturalist/Wikidata/GBIF import, CC photos and refresh script
All checks were successful
Deploy production / deploy (push) Successful in 1m8s
All checks were successful
Deploy production / deploy (push) Successful in 1m8s
This commit is contained in:
@@ -26,6 +26,9 @@
|
|||||||
"planes:import": "bun scripts/planes-import.ts",
|
"planes:import": "bun scripts/planes-import.ts",
|
||||||
"planes:photos": "bun scripts/planes-photos.ts",
|
"planes:photos": "bun scripts/planes-photos.ts",
|
||||||
"planes:refresh": "scripts/planes-refresh.sh",
|
"planes:refresh": "scripts/planes-refresh.sh",
|
||||||
|
"wildlife:import": "bun scripts/wildlife-import.ts",
|
||||||
|
"wildlife:photos": "bun scripts/wildlife-photos.ts",
|
||||||
|
"wildlife:refresh": "scripts/wildlife-refresh.sh",
|
||||||
"db:seed:cod": "bun scripts/seed-cod-catalog.ts",
|
"db:seed:cod": "bun scripts/seed-cod-catalog.ts",
|
||||||
"cs-prices:ingest": "bun scripts/ingest-cs-prices.ts",
|
"cs-prices:ingest": "bun scripts/ingest-cs-prices.ts",
|
||||||
"house:worker": "bun scripts/house-worker.ts",
|
"house:worker": "bun scripts/house-worker.ts",
|
||||||
|
|||||||
603
scripts/wildlife-import.ts
Normal file
603
scripts/wildlife-import.ts
Normal file
@@ -0,0 +1,603 @@
|
|||||||
|
// Builds the public wildlife pokedex (/trackers/wildlife, schema `wildlife`) from open data.
|
||||||
|
//
|
||||||
|
// DB_URL=... bun scripts/wildlife-import.ts fetch sources and write to the database
|
||||||
|
// bun scripts/wildlife-import.ts --dry-run fetch + build only, print a summary
|
||||||
|
// bun scripts/wildlife-import.ts --cache /tmp/wildlife reuse API responses from that folder
|
||||||
|
// bun scripts/wildlife-import.ts --report-new JSON of species new since the last run on stdout
|
||||||
|
//
|
||||||
|
// Which species (wild animals only, no pets/livestock, no captive observations):
|
||||||
|
// - the most observed species per group in the Netherlands (iNaturalist research grade),
|
||||||
|
// - the most observed species per group in Europe,
|
||||||
|
// - the most observed species per group worldwide, plus a short list of famous animals.
|
||||||
|
// Species already in the dex are always kept (refreshed, never deleted), so Dex numbers stay
|
||||||
|
// stable and new species get the next number.
|
||||||
|
// Per species: names (iNaturalist nl/en locale, Wikidata nl label), taxonomy, IUCN Red List
|
||||||
|
// (iNaturalist, Wikidata P141 fallback), Dutch red list status, establishment in NL, iNat
|
||||||
|
// observation counts (NL, Europe, world), Wikipedia intros (en + nl via iNaturalist),
|
||||||
|
// Wikidata mass/length/lifespan/name year+author and GBIF occurrence records in NL.
|
||||||
|
import postgres from 'postgres';
|
||||||
|
import {
|
||||||
|
chunks,
|
||||||
|
log as logErr,
|
||||||
|
saveCache,
|
||||||
|
slugify,
|
||||||
|
sparql,
|
||||||
|
stripHtml,
|
||||||
|
useCache,
|
||||||
|
getJson
|
||||||
|
} from './planes-wiki';
|
||||||
|
import {
|
||||||
|
groupOf,
|
||||||
|
GROUPS,
|
||||||
|
inat,
|
||||||
|
IUCN_BY_NUMBER,
|
||||||
|
PLACE_EUROPE,
|
||||||
|
PLACE_NL,
|
||||||
|
UA,
|
||||||
|
type InatList,
|
||||||
|
type InatTaxon
|
||||||
|
} from './wildlife-inat';
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const dryRun = args.includes('--dry-run');
|
||||||
|
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
||||||
|
const cacheDir = opt('--cache');
|
||||||
|
const reportNew = args.includes('--report-new');
|
||||||
|
const SITE = 'https://stanrunge.dev/trackers/wildlife';
|
||||||
|
const log = (...a: unknown[]) => (reportNew ? logErr(...a) : console.log(...a));
|
||||||
|
|
||||||
|
// iconic taxon filter -> how many species to take (NL, Europe extra, world extra)
|
||||||
|
const ICONIC: { iconic: string; taxonId: number; nl: number; eu: number; world: number }[] = [
|
||||||
|
{ iconic: 'Aves', taxonId: 3, nl: 450, eu: 250, world: 300 },
|
||||||
|
{ iconic: 'Mammalia', taxonId: 40151, nl: 120, eu: 120, world: 350 },
|
||||||
|
{ iconic: 'Reptilia', taxonId: 26036, nl: 40, eu: 100, world: 250 },
|
||||||
|
{ iconic: 'Amphibia', taxonId: 20978, nl: 30, eu: 50, world: 100 },
|
||||||
|
{ iconic: 'Actinopterygii', taxonId: 47178, nl: 150, eu: 80, world: 150 },
|
||||||
|
{ iconic: 'Insecta', taxonId: 47158, nl: 1300, eu: 300, world: 200 },
|
||||||
|
{ iconic: 'Arachnida', taxonId: 47119, nl: 150, eu: 50, world: 50 },
|
||||||
|
{ iconic: 'Mollusca', taxonId: 47115, nl: 180, eu: 50, world: 50 },
|
||||||
|
{ iconic: 'Animalia', taxonId: 1, nl: 250, eu: 80, world: 0 }
|
||||||
|
];
|
||||||
|
// Extra worldwide lists for groups without their own iconic taxon.
|
||||||
|
const WORLD_EXTRA = [
|
||||||
|
{ taxonId: 47273, n: 60 }, // Elasmobranchii (sharks & rays)
|
||||||
|
{ taxonId: 47534, n: 30 }, // Cnidaria
|
||||||
|
{ taxonId: 47549, n: 30 }, // Echinodermata
|
||||||
|
{ taxonId: 85493, n: 40 } // Malacostraca (crabs, lobsters, ...)
|
||||||
|
];
|
||||||
|
const MIN_NL = 5;
|
||||||
|
const FAMOUS =
|
||||||
|
`Panthera tigris, Panthera leo, Panthera pardus, Panthera uncia, Panthera onca, Acinonyx jubatus,
|
||||||
|
Ailuropoda melanoleuca, Ailurus fulgens, Ursus maritimus, Ursus arctos, Canis lupus, Gulo gulo, Lynx lynx,
|
||||||
|
Bison bonasus, Loxodonta africana, Elephas maximus, Giraffa camelopardalis, Okapia johnstoni, Equus quagga,
|
||||||
|
Hippopotamus amphibius, Ceratotherium simum, Diceros bicornis, Rhinoceros unicornis, Gorilla gorilla,
|
||||||
|
Gorilla beringei, Pan troglodytes, Pan paniscus, Pongo pygmaeus, Lemur catta, Bradypus variegatus,
|
||||||
|
Myrmecophaga tridactyla, Hydrochoerus hydrochaeris, Phascolarctos cinereus, Macropus giganteus,
|
||||||
|
Ornithorhynchus anatinus, Tachyglossus aculeatus, Sarcophilus harrisii, Balaenoptera musculus,
|
||||||
|
Megaptera novaeangliae, Physeter macrocephalus, Orcinus orca, Tursiops truncatus, Monodon monoceros,
|
||||||
|
Odobenus rosmarus, Enhydra lutris, Aptenodytes forsteri, Spheniscus demersus, Struthio camelus,
|
||||||
|
Dromaius novaehollandiae, Aquila chrysaetos, Haliaeetus leucocephalus, Harpia harpyja, Vultur gryphus,
|
||||||
|
Diomedea exulans, Fratercula arctica, Pavo cristatus, Ramphastos toco, Phoenicopterus roseus,
|
||||||
|
Crocodylus niloticus, Crocodylus porosus, Alligator mississippiensis, Varanus komodoensis, Chelonia mydas,
|
||||||
|
Dermochelys coriacea, Chelonoidis niger, Ophiophagus hannah, Iguana iguana, Chamaeleo calyptratus,
|
||||||
|
Ambystoma mexicanum, Andrias japonicus, Dendrobates tinctorius, Phyllobates terribilis,
|
||||||
|
Carcharodon carcharias, Rhincodon typus, Mobula birostris, Latimeria chalumnae, Hippocampus hippocampus,
|
||||||
|
Amphiprion ocellaris, Octopus vulgaris, Architeuthis dux, Danaus plexippus, Dynastes hercules,
|
||||||
|
Apis mellifera, Lucanus cervus, Aurelia aurita`
|
||||||
|
.split(',')
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
// Pets, livestock and us.
|
||||||
|
const EXCLUDE = new Set(
|
||||||
|
`Homo sapiens, Canis familiaris, Felis catus, Bos taurus, Equus caballus, Ovis aries, Capra hircus,
|
||||||
|
Sus domesticus, Gallus gallus, Equus asinus, Cavia porcellus, Bombyx mori, Lama glama, Vicugna pacos,
|
||||||
|
Bubalus bubalis, Camelus dromedarius, Bos indicus, Mesocricetus auratus, Meleagris gallopavo domesticus`
|
||||||
|
.split(',')
|
||||||
|
.map((s) => s.trim())
|
||||||
|
);
|
||||||
|
|
||||||
|
type Cand = { id: number; scope: 'nl' | 'europe' | 'world'; name: string };
|
||||||
|
|
||||||
|
async function speciesCounts(
|
||||||
|
place: number,
|
||||||
|
iconic: string,
|
||||||
|
n: number
|
||||||
|
): Promise<InatList['results']> {
|
||||||
|
const out: InatList['results'] = [];
|
||||||
|
for (let page = 1; out.length < n && page <= 10; page++) {
|
||||||
|
const res = await inat(
|
||||||
|
`/observations/species_counts?place_id=${place}&iconic_taxa=${iconic}&captive=false&quality_grade=research&per_page=500&page=${page}&locale=en`
|
||||||
|
);
|
||||||
|
out.push(...res.results);
|
||||||
|
if (res.results.length < 500) break;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
async function topWorld(taxonId: number, n: number): Promise<InatTaxon[]> {
|
||||||
|
const out: InatTaxon[] = [];
|
||||||
|
for (let page = 1; out.length < n && page <= 10; page++) {
|
||||||
|
const res = await inat<{ results: InatTaxon[] }>(
|
||||||
|
`/taxa?rank=species&taxon_id=${taxonId}&is_active=true&order_by=observations_count&order=desc&per_page=200&page=${page}`
|
||||||
|
);
|
||||||
|
out.push(...res.results);
|
||||||
|
if (res.results.length < 200) break;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function collect(existing: number[]): Promise<Map<number, Cand>> {
|
||||||
|
const cands = new Map<number, Cand>();
|
||||||
|
const add = (t: InatTaxon, scope: Cand['scope']) => {
|
||||||
|
if (t.rank !== 'species' || EXCLUDE.has(t.name) || cands.has(t.id)) return false;
|
||||||
|
cands.set(t.id, { id: t.id, scope, name: t.name });
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
for (const g of ICONIC) {
|
||||||
|
const nl = await speciesCounts(PLACE_NL, g.iconic, g.nl * 2);
|
||||||
|
let k = 0;
|
||||||
|
for (const r of nl) if (k < g.nl && r.count >= MIN_NL && add(r.taxon, 'nl')) k++;
|
||||||
|
const eu = await speciesCounts(PLACE_EUROPE, g.iconic, g.eu * 2 + g.nl);
|
||||||
|
let e = 0;
|
||||||
|
for (const r of eu) if (e < g.eu && add(r.taxon, 'europe')) e++;
|
||||||
|
let w = 0;
|
||||||
|
if (g.world)
|
||||||
|
for (const t of await topWorld(g.taxonId, g.world * 2))
|
||||||
|
if (w < g.world && add(t, 'world')) w++;
|
||||||
|
log(` ${g.iconic}: ${k} NL, +${e} Europe, +${w} world`);
|
||||||
|
}
|
||||||
|
for (const x of WORLD_EXTRA) {
|
||||||
|
let w = 0;
|
||||||
|
for (const t of await topWorld(x.taxonId, x.n * 2)) if (w < x.n && add(t, 'world')) w++;
|
||||||
|
}
|
||||||
|
let famous = 0;
|
||||||
|
for (const name of FAMOUS) {
|
||||||
|
const res = await inat<{ results: InatTaxon[] }>(
|
||||||
|
`/taxa?q=${encodeURIComponent(name)}&rank=species&is_active=true&per_page=5`
|
||||||
|
);
|
||||||
|
const t = res.results.find((r) => r.name.toLowerCase() === name.toLowerCase());
|
||||||
|
if (t && add(t, 'world')) famous++;
|
||||||
|
if (!t) log(` famous species not found: ${name}`);
|
||||||
|
}
|
||||||
|
log(` +${famous} famous species`);
|
||||||
|
for (const id of existing) if (!cands.has(id)) cands.set(id, { id, scope: 'world', name: '' });
|
||||||
|
return cands;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function details(ids: number[], locale: 'en' | 'nl') {
|
||||||
|
const out = new Map<number, InatTaxon>();
|
||||||
|
for (const [i, batch] of chunks(ids, 30).entries()) {
|
||||||
|
const res = await inat<{ results: InatTaxon[] }>(
|
||||||
|
`/taxa/${batch.join(',')}?locale=${locale}&preferred_place_id=${PLACE_NL}`
|
||||||
|
);
|
||||||
|
for (const t of res.results) out.set(t.id, t);
|
||||||
|
if (i % 20 === 0) log(` taxa (${locale}) ${Math.min((i + 1) * 30, ids.length)}/${ids.length}`);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
async function placeCounts(ids: number[], place: number) {
|
||||||
|
const out = new Map<number, number>();
|
||||||
|
for (const batch of chunks(ids, 200)) {
|
||||||
|
const res = await inat(
|
||||||
|
`/observations/species_counts?place_id=${place}&taxon_id=${batch.join(',')}&captive=false&quality_grade=research&per_page=500`
|
||||||
|
);
|
||||||
|
for (const r of res.results) out.set(r.taxon.id, r.count);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
type Wd = {
|
||||||
|
item: string;
|
||||||
|
nl: string | null;
|
||||||
|
iucn: string | null;
|
||||||
|
gbif: number | null;
|
||||||
|
nlwiki: string | null;
|
||||||
|
mass: number | null;
|
||||||
|
len: number | null;
|
||||||
|
life: number | null;
|
||||||
|
year: number | null;
|
||||||
|
author: string | null;
|
||||||
|
};
|
||||||
|
const WD_IUCN: Record<string, string> = {
|
||||||
|
Q211005: 'LC',
|
||||||
|
Q719675: 'NT',
|
||||||
|
Q278113: 'VU',
|
||||||
|
Q11394: 'EN',
|
||||||
|
Q219127: 'CR',
|
||||||
|
Q239509: 'EW',
|
||||||
|
Q237350: 'EX',
|
||||||
|
Q3245245: 'DD'
|
||||||
|
};
|
||||||
|
async function wikidata(ids: number[]) {
|
||||||
|
const out = new Map<number, Wd>();
|
||||||
|
for (const batch of chunks(ids, 250)) {
|
||||||
|
const rows =
|
||||||
|
await sparql(`SELECT ?inat ?item ?nl ?iucn ?gbif ?nlwiki ?mass ?len ?life ?year ?authorLabel WHERE {
|
||||||
|
VALUES ?inat { ${batch.map((id) => `"${id}"`).join(' ')} }
|
||||||
|
?item wdt:P3151 ?inat .
|
||||||
|
OPTIONAL { ?item rdfs:label ?nl FILTER(lang(?nl) = "nl") }
|
||||||
|
OPTIONAL { ?item wdt:P141 ?iucn }
|
||||||
|
OPTIONAL { ?item wdt:P846 ?gbif }
|
||||||
|
OPTIONAL { ?nlwiki schema:about ?item ; schema:isPartOf <https://nl.wikipedia.org/> }
|
||||||
|
OPTIONAL { ?item p:P2067/psn:P2067/wikibase:quantityAmount ?mass }
|
||||||
|
OPTIONAL { ?item p:P2043/psn:P2043/wikibase:quantityAmount ?len }
|
||||||
|
OPTIONAL { ?item p:P2250/psn:P2250/wikibase:quantityAmount ?life }
|
||||||
|
OPTIONAL { ?item p:P225 ?st . ?st pq:P574 ?year .
|
||||||
|
OPTIONAL { ?st pq:P405 ?author . ?author rdfs:label ?authorLabel FILTER(lang(?authorLabel) = "en") } }
|
||||||
|
}`);
|
||||||
|
for (const r of rows) {
|
||||||
|
const id = Number(r.inat);
|
||||||
|
const prev = out.get(id);
|
||||||
|
const num = (v: string | undefined) =>
|
||||||
|
v != null && Number.isFinite(Number(v)) ? Number(v) : null;
|
||||||
|
const max = (a: number | null | undefined, b: number | null) =>
|
||||||
|
a == null ? b : b == null ? a : Math.max(a, b);
|
||||||
|
const yr = r.year ? Number(r.year.slice(0, 4)) : null;
|
||||||
|
out.set(id, {
|
||||||
|
item: r.item.split('/').at(-1)!,
|
||||||
|
nl: prev?.nl ?? r.nl ?? null,
|
||||||
|
iucn: prev?.iucn ?? (r.iucn ? (WD_IUCN[r.iucn.split('/').at(-1)!] ?? null) : null),
|
||||||
|
gbif: prev?.gbif ?? num(r.gbif),
|
||||||
|
nlwiki:
|
||||||
|
prev?.nlwiki ??
|
||||||
|
(r.nlwiki ? decodeURIComponent(r.nlwiki.split('/wiki/')[1]).replace(/_/g, ' ') : null),
|
||||||
|
mass: max(prev?.mass, num(r.mass)),
|
||||||
|
len: max(prev?.len, num(r.len)),
|
||||||
|
life: max(prev?.life, num(r.life)),
|
||||||
|
year: prev?.year ?? (yr && yr > 1700 && yr <= new Date().getFullYear() ? yr : null),
|
||||||
|
author: prev?.author ?? r.authorLabel ?? null
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function gbifNl(): Promise<Map<number, number> | null> {
|
||||||
|
const out = new Map<number, number>();
|
||||||
|
try {
|
||||||
|
for (let offset = 0; offset < 100_000; offset += 5000) {
|
||||||
|
const res = await getJson<{ facets: { counts: { name: string; count: number }[] }[] }>(
|
||||||
|
`https://api.gbif.org/v1/occurrence/search?country=NL&taxonKey=1&limit=0&facet=speciesKey&facetLimit=5000&facetOffset=${offset}`,
|
||||||
|
{ headers: { 'user-agent': UA } }
|
||||||
|
);
|
||||||
|
const counts = res.facets[0]?.counts ?? [];
|
||||||
|
for (const c of counts) out.set(Number(c.name), c.count);
|
||||||
|
if (counts.length < 5000) return out;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
} catch (e) {
|
||||||
|
log(` GBIF facet failed: ${(e as Error).message}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fmt = (n: number) => n.toLocaleString('en-GB');
|
||||||
|
const trimSummary = (html: string | null | undefined) => {
|
||||||
|
if (!html) return null;
|
||||||
|
const text = stripHtml(html).replace(/\s*\[\d+\]/g, '');
|
||||||
|
const sentences = text.match(/[^.!?]+[.!?]+(\s|$)/g) ?? [text];
|
||||||
|
let out = '';
|
||||||
|
for (const s of sentences) {
|
||||||
|
if (out && (out + s).length > 650) break;
|
||||||
|
out += s;
|
||||||
|
}
|
||||||
|
return out.trim() || null;
|
||||||
|
};
|
||||||
|
const IUCN_TEXT: Record<string, string> = {
|
||||||
|
LC: 'Least Concern',
|
||||||
|
NT: 'Near Threatened',
|
||||||
|
VU: 'Vulnerable',
|
||||||
|
EN: 'Endangered',
|
||||||
|
CR: 'Critically Endangered',
|
||||||
|
EW: 'Extinct in the Wild',
|
||||||
|
EX: 'Extinct',
|
||||||
|
DD: 'Data Deficient'
|
||||||
|
};
|
||||||
|
|
||||||
|
function funFacts(s: Row, rankInGroup: number | null, groupName: string): string[] {
|
||||||
|
const f: string[] = [];
|
||||||
|
if (s.nl_obs > 0)
|
||||||
|
f.push(
|
||||||
|
`👀 Spotted ${fmt(s.nl_obs)} times in the Netherlands on iNaturalist${rankInGroup && rankInGroup <= 50 ? `: the #${rankInGroup} most seen of the ${groupName.toLowerCase()} in NL` : ''}.`
|
||||||
|
);
|
||||||
|
else
|
||||||
|
f.push(
|
||||||
|
`✈️ No confirmed wild sightings in the Netherlands on iNaturalist: you'd have to travel for this one.`
|
||||||
|
);
|
||||||
|
if (s.world_obs > 0) f.push(`🌍 ${fmt(s.world_obs)} observations worldwide on iNaturalist.`);
|
||||||
|
if (s.mass_kg != null) {
|
||||||
|
const kg = s.mass_kg;
|
||||||
|
if (kg >= 1000)
|
||||||
|
f.push(
|
||||||
|
`🐘 Can weigh about ${fmt(Math.round(kg / 1000))} tonnes: as much as ${fmt(Math.max(1, Math.round(kg / 1400)))} cars.`
|
||||||
|
);
|
||||||
|
else if (kg >= 1) f.push(`⚖️ Weighs up to about ${fmt(Math.round(kg * 10) / 10)} kg.`);
|
||||||
|
else if (kg >= 0.001) f.push(`🪶 Weighs only about ${fmt(Math.round(kg * 1000))} g.`);
|
||||||
|
}
|
||||||
|
if (s.length_m != null) {
|
||||||
|
const m = s.length_m;
|
||||||
|
if (m >= 2)
|
||||||
|
f.push(
|
||||||
|
`📏 Up to ${fmt(Math.round(m * 10) / 10)} m long: ${(m / 1.8).toFixed(1)}× a tall person.`
|
||||||
|
);
|
||||||
|
else if (m >= 0.01) f.push(`📏 About ${fmt(Math.round(m * 100))} cm long.`);
|
||||||
|
else f.push(`🔬 Only about ${fmt(Math.round(m * 1000 * 10) / 10)} mm long.`);
|
||||||
|
}
|
||||||
|
if (s.lifespan_years != null && s.lifespan_years >= 1)
|
||||||
|
f.push(`⏳ Can live up to ${fmt(Math.round(s.lifespan_years))} years.`);
|
||||||
|
if (s.named_year)
|
||||||
|
f.push(
|
||||||
|
`📜 Scientifically named ${s.named_by ? `by ${s.named_by} ` : ''}in ${s.named_year} (${new Date().getFullYear() - s.named_year} years ago).`
|
||||||
|
);
|
||||||
|
if (s.nl_establishment === 'introduced')
|
||||||
|
f.push(`🧳 Not originally Dutch: it was introduced to the Netherlands.`);
|
||||||
|
if (s.iucn && ['VU', 'EN', 'CR', 'EW'].includes(s.iucn))
|
||||||
|
f.push(`⚠️ Listed as ${IUCN_TEXT[s.iucn]} on the IUCN Red List.`);
|
||||||
|
if (s.extinct) f.push(`🪦 This species is extinct.`);
|
||||||
|
if (s.nl_status) f.push(`🇳🇱 Dutch status: ${s.nl_status}.`);
|
||||||
|
return f;
|
||||||
|
}
|
||||||
|
|
||||||
|
type Row = {
|
||||||
|
inat_id: number;
|
||||||
|
group: string;
|
||||||
|
slug: string;
|
||||||
|
scientific_name: string;
|
||||||
|
name_en: string | null;
|
||||||
|
name_nl: string | null;
|
||||||
|
search: string;
|
||||||
|
gbif_key: number | null;
|
||||||
|
wikidata: string | null;
|
||||||
|
wikipedia_en: string | null;
|
||||||
|
wikipedia_nl: string | null;
|
||||||
|
class_name: string | null;
|
||||||
|
order_name: string | null;
|
||||||
|
family_name: string | null;
|
||||||
|
iucn: string | null;
|
||||||
|
nl_status: string | null;
|
||||||
|
nl_establishment: string | null;
|
||||||
|
extinct: boolean;
|
||||||
|
nl_obs: number;
|
||||||
|
europe_obs: number;
|
||||||
|
world_obs: number;
|
||||||
|
gbif_nl: number | null;
|
||||||
|
scope: string;
|
||||||
|
summary: string | null;
|
||||||
|
summary_nl: string | null;
|
||||||
|
facts: string[];
|
||||||
|
mass_kg: number | null;
|
||||||
|
length_m: number | null;
|
||||||
|
lifespan_years: number | null;
|
||||||
|
named_year: number | null;
|
||||||
|
named_by: string | null;
|
||||||
|
sources: string[];
|
||||||
|
};
|
||||||
|
const wikiTitle = (url: string | null | undefined, host: string) => {
|
||||||
|
if (!url || !url.includes(host)) return null;
|
||||||
|
try {
|
||||||
|
return decodeURIComponent(url.split('/wiki/')[1] ?? '').replace(/_/g, ' ') || null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
await useCache(cacheDir, 'wildlife-api.json');
|
||||||
|
const url = process.env.DB_URL;
|
||||||
|
if (!dryRun && !url) throw new Error('Set DB_URL (or use --dry-run)');
|
||||||
|
const sql = url ? postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} }) : null;
|
||||||
|
try {
|
||||||
|
const existing = sql
|
||||||
|
? await sql<{ inat_id: number; dex: number }[]>`select inat_id, dex from wildlife.species`
|
||||||
|
: [];
|
||||||
|
log('collecting species lists…');
|
||||||
|
const cands = await collect(existing.map((e) => e.inat_id));
|
||||||
|
const ids = [...cands.keys()];
|
||||||
|
log(`${ids.length} candidate species; fetching details…`);
|
||||||
|
const en = await details(ids, 'en');
|
||||||
|
const nl = await details(ids, 'nl');
|
||||||
|
await saveCache(true);
|
||||||
|
log('NL / Europe counts…');
|
||||||
|
const nlCounts = await placeCounts(ids, PLACE_NL);
|
||||||
|
const euCounts = await placeCounts(ids, PLACE_EUROPE);
|
||||||
|
await saveCache(true);
|
||||||
|
log('Wikidata…');
|
||||||
|
const wd = await wikidata(ids);
|
||||||
|
log('GBIF NL occurrence counts…');
|
||||||
|
const gbif = await gbifNl();
|
||||||
|
|
||||||
|
const rows: Row[] = [];
|
||||||
|
for (const id of ids) {
|
||||||
|
const t = en.get(id);
|
||||||
|
const tn = nl.get(id);
|
||||||
|
if (!t || t.rank !== 'species' || EXCLUDE.has(t.name)) continue;
|
||||||
|
const anc = t.ancestors ?? [];
|
||||||
|
if (!anc.some((a) => a.name === 'Animalia')) continue;
|
||||||
|
const g = groupOf(anc.map((a) => a.name));
|
||||||
|
const w = wd.get(id);
|
||||||
|
const nameEn = t.preferred_common_name || t.english_common_name || null;
|
||||||
|
let nameNl: string | null = tn?.preferred_common_name || null;
|
||||||
|
if (nameNl && nameEn && nameNl === nameEn && w?.nl && w.nl !== t.name && w.nl !== nameEn)
|
||||||
|
nameNl = w.nl;
|
||||||
|
if (!nameNl && w?.nl && w.nl.toLowerCase() !== t.name.toLowerCase()) nameNl = w.nl;
|
||||||
|
if (nameNl) nameNl = nameNl.charAt(0).toUpperCase() + nameNl.slice(1);
|
||||||
|
const statuses = t.conservation_statuses ?? [];
|
||||||
|
const global = statuses.find((s) => !s.place && s.authority === 'IUCN Red List');
|
||||||
|
let iucn = global?.iucn != null ? (IUCN_BY_NUMBER[global.iucn] ?? null) : null;
|
||||||
|
if (!iucn && global?.status && /^(LC|NT|VU|EN|CR|EW|EX|DD)$/i.test(global.status))
|
||||||
|
iucn = global.status.toUpperCase();
|
||||||
|
if (!iucn) iucn = w?.iucn ?? null;
|
||||||
|
if (t.extinct && !iucn) iucn = 'EX';
|
||||||
|
const nlS = statuses.find((s) => s.place?.id === PLACE_NL);
|
||||||
|
const em = (tn ?? t).establishment_means;
|
||||||
|
const nlEst = em?.place?.id === PLACE_NL ? (em.establishment_means ?? null) : null;
|
||||||
|
const nlObs = nlCounts.get(id) ?? 0;
|
||||||
|
const cand = cands.get(id)!;
|
||||||
|
const rank = (r: string) => anc.find((a) => a.rank === r)?.name ?? null;
|
||||||
|
const wikipediaEn = wikiTitle(t.wikipedia_url, 'en.wikipedia.org');
|
||||||
|
const wikipediaNl = w?.nlwiki ?? wikiTitle(tn?.wikipedia_url, 'nl.wikipedia.org');
|
||||||
|
const summaryNl = tn?.wikipedia_url?.includes('nl.wikipedia.org')
|
||||||
|
? trimSummary(tn.wikipedia_summary)
|
||||||
|
: null;
|
||||||
|
const row: Row = {
|
||||||
|
inat_id: id,
|
||||||
|
group: g.slug,
|
||||||
|
slug: slugify(t.name),
|
||||||
|
scientific_name: t.name,
|
||||||
|
name_en: nameEn,
|
||||||
|
name_nl: nameNl,
|
||||||
|
search: [nameNl, nameEn, t.name, rank('order'), rank('family'), rank('genus')]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(' ')
|
||||||
|
.toLowerCase(),
|
||||||
|
gbif_key: w?.gbif ?? null,
|
||||||
|
wikidata: w?.item ?? null,
|
||||||
|
wikipedia_en: wikipediaEn,
|
||||||
|
wikipedia_nl: wikipediaNl,
|
||||||
|
class_name: rank('class'),
|
||||||
|
order_name: rank('order'),
|
||||||
|
family_name: rank('family'),
|
||||||
|
iucn,
|
||||||
|
nl_status: nlS ? nlS.description?.trim() || nlS.status : null,
|
||||||
|
nl_establishment: nlEst,
|
||||||
|
extinct: !!t.extinct,
|
||||||
|
nl_obs: nlObs,
|
||||||
|
europe_obs: euCounts.get(id) ?? 0,
|
||||||
|
world_obs: t.observations_count ?? 0,
|
||||||
|
gbif_nl: w?.gbif != null && gbif ? (gbif.get(w.gbif) ?? 0) : null,
|
||||||
|
scope:
|
||||||
|
nlObs >= MIN_NL
|
||||||
|
? 'nl'
|
||||||
|
: (euCounts.get(id) ?? 0) > 0
|
||||||
|
? 'europe'
|
||||||
|
: cand.scope === 'nl'
|
||||||
|
? 'europe'
|
||||||
|
: 'world',
|
||||||
|
summary: trimSummary(t.wikipedia_summary),
|
||||||
|
summary_nl: summaryNl,
|
||||||
|
facts: [],
|
||||||
|
mass_kg: w?.mass != null && w.mass > 0 && w.mass < 200_000 ? w.mass : null,
|
||||||
|
length_m: w?.len != null && w.len > 0 && w.len < 40 ? w.len : null,
|
||||||
|
lifespan_years:
|
||||||
|
w?.life != null && w.life > 0 ? Math.round((w.life / 31_557_600) * 10) / 10 : null,
|
||||||
|
named_year: w?.year ?? null,
|
||||||
|
named_by: w?.author ?? null,
|
||||||
|
sources: [
|
||||||
|
'iNaturalist',
|
||||||
|
...(w ? ['Wikidata'] : []),
|
||||||
|
...(w?.gbif != null && gbif ? ['GBIF'] : [])
|
||||||
|
]
|
||||||
|
};
|
||||||
|
if (row.lifespan_years != null && (row.lifespan_years < 0.01 || row.lifespan_years > 500))
|
||||||
|
row.lifespan_years = null;
|
||||||
|
rows.push(row);
|
||||||
|
}
|
||||||
|
// Unique slugs (scientific names are unique on iNaturalist, but be safe).
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const r of rows) {
|
||||||
|
let s = r.slug;
|
||||||
|
for (let n = 2; seen.has(s); n++) s = `${r.slug}-${n}`;
|
||||||
|
r.slug = s;
|
||||||
|
seen.add(s);
|
||||||
|
}
|
||||||
|
// Fun facts with the rank among the group in NL.
|
||||||
|
const byGroup = new Map<string, Row[]>();
|
||||||
|
for (const r of rows) byGroup.set(r.group, [...(byGroup.get(r.group) ?? []), r]);
|
||||||
|
for (const [slug, list] of byGroup) {
|
||||||
|
const g = GROUPS.find((x) => x.slug === slug)!;
|
||||||
|
const ranked = list.filter((r) => r.nl_obs > 0).toSorted((a, b) => b.nl_obs - a.nl_obs);
|
||||||
|
for (const r of list) {
|
||||||
|
const i = ranked.indexOf(r);
|
||||||
|
r.facts = funFacts(r, i >= 0 ? i + 1 : null, g.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const summary = GROUPS.map((g) => `${g.slug}: ${byGroup.get(g.slug)?.length ?? 0}`).join(', ');
|
||||||
|
log(`${rows.length} species (${summary})`);
|
||||||
|
if (dryRun || !sql) {
|
||||||
|
log(JSON.stringify(rows.slice(0, 3), null, 1));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const known = new Map(existing.map((e) => [e.inat_id, e.dex]));
|
||||||
|
let nextDex = Math.max(0, ...existing.map((e) => e.dex)) + 1;
|
||||||
|
const order = (r: Row) => GROUPS.findIndex((g) => g.slug === r.group);
|
||||||
|
const scopeOrder = { nl: 0, europe: 1, world: 2 } as Record<string, number>;
|
||||||
|
const sorted = rows.toSorted(
|
||||||
|
(a, b) =>
|
||||||
|
order(a) - order(b) ||
|
||||||
|
scopeOrder[a.scope] - scopeOrder[b.scope] ||
|
||||||
|
b.nl_obs - a.nl_obs ||
|
||||||
|
b.world_obs - a.world_obs ||
|
||||||
|
a.scientific_name.localeCompare(b.scientific_name)
|
||||||
|
);
|
||||||
|
const added: Row[] = [];
|
||||||
|
const dexOf = new Map<number, number>();
|
||||||
|
for (const r of sorted) {
|
||||||
|
const d = known.get(r.inat_id);
|
||||||
|
if (d != null) dexOf.set(r.inat_id, d);
|
||||||
|
else {
|
||||||
|
dexOf.set(r.inat_id, nextDex++);
|
||||||
|
added.push(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await sql.begin(async (tx) => {
|
||||||
|
const groupIds = new Map<string, number>();
|
||||||
|
for (const [i, g] of GROUPS.entries()) {
|
||||||
|
const [row] = await tx<{ id: number }[]>`
|
||||||
|
insert into wildlife.groups (slug, name, name_nl, emoji, sort_order)
|
||||||
|
values (${g.slug}, ${g.name}, ${g.nameNl}, ${g.emoji}, ${i})
|
||||||
|
on conflict (slug) do update set name = excluded.name, name_nl = excluded.name_nl,
|
||||||
|
emoji = excluded.emoji, sort_order = excluded.sort_order
|
||||||
|
returning id`;
|
||||||
|
groupIds.set(g.slug, row.id);
|
||||||
|
}
|
||||||
|
// Avoid unique-slug clashes while rows move around.
|
||||||
|
await tx`update wildlife.species set slug = slug || '~' || id where slug not like '%~%'`;
|
||||||
|
for (const r of sorted) {
|
||||||
|
const { group, ...rest } = r;
|
||||||
|
const row = { ...rest, dex: dexOf.get(r.inat_id)!, group_id: groupIds.get(group)! };
|
||||||
|
await tx`
|
||||||
|
insert into wildlife.species ${tx(row)}
|
||||||
|
on conflict (inat_id) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'inat_id' && k !== 'dex') as (keyof typeof row)[])}`;
|
||||||
|
}
|
||||||
|
// Restore slugs of species that were not part of this run (kept, not deleted).
|
||||||
|
await tx`update wildlife.species set slug = split_part(slug, '~', 1) where slug like '%~%'`;
|
||||||
|
await tx`
|
||||||
|
update wildlife.groups g set
|
||||||
|
species_count = (select count(*) from wildlife.species s where s.group_id = g.id),
|
||||||
|
nl_count = (select count(*) from wildlife.species s where s.group_id = g.id and s.nl_obs > 0)`;
|
||||||
|
});
|
||||||
|
log(`written: ${rows.length} species, ${added.length} new`);
|
||||||
|
if (reportNew) {
|
||||||
|
const [totals] = await sql`
|
||||||
|
select (select count(*)::int from wildlife.species) as species,
|
||||||
|
(select count(*)::int from wildlife.groups where species_count > 0) as groups,
|
||||||
|
(select count(*)::int from wildlife.photos) as photos`;
|
||||||
|
console.log(
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
totals,
|
||||||
|
newSpecies: (existing.length ? added : added.slice(0, 0)).map((r) => ({
|
||||||
|
dex: dexOf.get(r.inat_id),
|
||||||
|
inatId: r.inat_id,
|
||||||
|
name: r.name_en ?? r.scientific_name,
|
||||||
|
nameNl: r.name_nl,
|
||||||
|
scientificName: r.scientific_name,
|
||||||
|
group: r.group,
|
||||||
|
iucn: r.iucn,
|
||||||
|
nlObs: r.nl_obs,
|
||||||
|
url: `${SITE}/${r.group}/${r.slug}`
|
||||||
|
}))
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
1
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await saveCache(true);
|
||||||
|
await sql?.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await main();
|
||||||
211
scripts/wildlife-inat.ts
Normal file
211
scripts/wildlife-inat.ts
Normal file
@@ -0,0 +1,211 @@
|
|||||||
|
// Shared helpers for the wildlife pokedex scripts (scripts/wildlife-import.ts,
|
||||||
|
// scripts/wildlife-photos.ts): polite iNaturalist API access (about 1 request per second,
|
||||||
|
// as the API asks), the group definitions and photo licence handling.
|
||||||
|
import { getJson, sleep } from './planes-wiki';
|
||||||
|
|
||||||
|
export const UA =
|
||||||
|
'StanRungeWildlifeDex/1.0 (https://stanrunge.dev/trackers/wildlife; wildlife pokedex importer) bun';
|
||||||
|
export const INAT = 'https://api.inaturalist.org/v1';
|
||||||
|
export const PLACE_NL = 7506;
|
||||||
|
export const PLACE_EUROPE = 97391;
|
||||||
|
|
||||||
|
let last = 0;
|
||||||
|
/** GET an iNaturalist API path, at most one call per ~1.1 s (cached when --cache is set). */
|
||||||
|
export async function inat<T = InatList>(path: string, useCached = true): Promise<T> {
|
||||||
|
const url = `${INAT}${path}`;
|
||||||
|
const wait = last + 1100 - Date.now();
|
||||||
|
if (wait > 0) await sleep(wait);
|
||||||
|
last = Date.now();
|
||||||
|
return getJson<T>(url, { headers: { 'user-agent': UA } }, useCached);
|
||||||
|
}
|
||||||
|
|
||||||
|
export type InatPhoto = {
|
||||||
|
id: number;
|
||||||
|
license_code: string | null;
|
||||||
|
attribution: string;
|
||||||
|
attribution_name?: string | null;
|
||||||
|
url: string;
|
||||||
|
original_dimensions?: { width: number; height: number } | null;
|
||||||
|
};
|
||||||
|
export type InatTaxon = {
|
||||||
|
id: number;
|
||||||
|
name: string;
|
||||||
|
rank: string;
|
||||||
|
is_active?: boolean;
|
||||||
|
extinct?: boolean;
|
||||||
|
iconic_taxon_name?: string | null;
|
||||||
|
preferred_common_name?: string | null;
|
||||||
|
english_common_name?: string | null;
|
||||||
|
observations_count?: number;
|
||||||
|
wikipedia_url?: string | null;
|
||||||
|
wikipedia_summary?: string | null;
|
||||||
|
default_photo?: InatPhoto | null;
|
||||||
|
taxon_photos?: { photo: InatPhoto }[];
|
||||||
|
ancestors?: { id: number; rank: string; name: string }[];
|
||||||
|
ancestor_ids?: number[];
|
||||||
|
conservation_status?: { status?: string; iucn?: number; authority?: string } | null;
|
||||||
|
conservation_statuses?: {
|
||||||
|
status: string;
|
||||||
|
iucn: number | null;
|
||||||
|
authority: string | null;
|
||||||
|
description?: string | null;
|
||||||
|
place: { id: number } | null;
|
||||||
|
}[];
|
||||||
|
establishment_means?: { establishment_means?: string; place?: { id: number } } | null;
|
||||||
|
};
|
||||||
|
export type InatList = {
|
||||||
|
total_results: number;
|
||||||
|
page: number;
|
||||||
|
per_page: number;
|
||||||
|
results: { count: number; taxon: InatTaxon }[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export const CC = [
|
||||||
|
'cc0',
|
||||||
|
'cc-by',
|
||||||
|
'cc-by-sa',
|
||||||
|
'cc-by-nc',
|
||||||
|
'cc-by-nc-sa',
|
||||||
|
'cc-by-nd',
|
||||||
|
'cc-by-nc-nd'
|
||||||
|
];
|
||||||
|
export const isCc = (code: string | null | undefined) => !!code && CC.includes(code.toLowerCase());
|
||||||
|
export const licenseLabel = (code: string) =>
|
||||||
|
code.toLowerCase() === 'cc0' ? 'CC0' : `${code.toUpperCase().replace(/^CC-/, 'CC ')} 4.0`;
|
||||||
|
export const licenseUrl = (code: string) =>
|
||||||
|
code.toLowerCase() === 'cc0'
|
||||||
|
? 'https://creativecommons.org/publicdomain/zero/1.0/'
|
||||||
|
: `https://creativecommons.org/licenses/${code.toLowerCase().replace(/^cc-/, '')}/4.0/`;
|
||||||
|
/** "(c) Jane Doe, some rights reserved (CC BY)" -> "Jane Doe". */
|
||||||
|
export const author = (p: InatPhoto) =>
|
||||||
|
p.attribution_name?.trim() ||
|
||||||
|
p.attribution
|
||||||
|
.replace(/^\(c\)\s*/i, '')
|
||||||
|
.replace(/,\s*(some|no) rights reserved.*$/i, '')
|
||||||
|
.replace(/^no known copyright restrictions.*$/i, '')
|
||||||
|
.trim() ||
|
||||||
|
null;
|
||||||
|
/** Any size URL -> the "medium" (500px) one. */
|
||||||
|
export const mediumUrl = (url: string) =>
|
||||||
|
url.replace(/\/(square|small|thumb|medium|large|original)\.(jpe?g|png|gif|webp)/i, '/medium.$2');
|
||||||
|
|
||||||
|
// Groups shown in the dex, first match wins (by ancestor names on iNaturalist).
|
||||||
|
export type GroupDef = {
|
||||||
|
slug: string;
|
||||||
|
name: string;
|
||||||
|
nameNl: string;
|
||||||
|
emoji: string;
|
||||||
|
match: string[];
|
||||||
|
};
|
||||||
|
export const GROUPS: GroupDef[] = [
|
||||||
|
{ slug: 'mammals', name: 'Mammals', nameNl: 'Zoogdieren', emoji: '🦊', match: ['Mammalia'] },
|
||||||
|
{ slug: 'birds', name: 'Birds', nameNl: 'Vogels', emoji: '🐦', match: ['Aves'] },
|
||||||
|
{
|
||||||
|
slug: 'reptiles',
|
||||||
|
name: 'Reptiles',
|
||||||
|
nameNl: 'Reptielen',
|
||||||
|
emoji: '🦎',
|
||||||
|
match: ['Reptilia', 'Testudines', 'Squamata', 'Crocodylia']
|
||||||
|
},
|
||||||
|
{ slug: 'amphibians', name: 'Amphibians', nameNl: 'Amfibieën', emoji: '🐸', match: ['Amphibia'] },
|
||||||
|
{
|
||||||
|
slug: 'sharks-and-rays',
|
||||||
|
name: 'Sharks & rays',
|
||||||
|
nameNl: 'Haaien en roggen',
|
||||||
|
emoji: '🦈',
|
||||||
|
match: ['Elasmobranchii', 'Chondrichthyes']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'fish',
|
||||||
|
name: 'Fish',
|
||||||
|
nameNl: 'Vissen',
|
||||||
|
emoji: '🐟',
|
||||||
|
match: ['Actinopterygii', 'Sarcopterygii', 'Petromyzonti', 'Myxini', 'Agnatha']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'butterflies-and-moths',
|
||||||
|
name: 'Butterflies & moths',
|
||||||
|
nameNl: 'Vlinders',
|
||||||
|
emoji: '🦋',
|
||||||
|
match: ['Lepidoptera']
|
||||||
|
},
|
||||||
|
{ slug: 'beetles', name: 'Beetles', nameNl: 'Kevers', emoji: '🪲', match: ['Coleoptera'] },
|
||||||
|
{
|
||||||
|
slug: 'bees-wasps-and-ants',
|
||||||
|
name: 'Bees, wasps & ants',
|
||||||
|
nameNl: 'Bijen, wespen en mieren',
|
||||||
|
emoji: '🐝',
|
||||||
|
match: ['Hymenoptera']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'dragonflies',
|
||||||
|
name: 'Dragonflies & damselflies',
|
||||||
|
nameNl: 'Libellen',
|
||||||
|
emoji: '🪽',
|
||||||
|
match: ['Odonata']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'flies',
|
||||||
|
name: 'Flies & mosquitoes',
|
||||||
|
nameNl: 'Vliegen en muggen',
|
||||||
|
emoji: '🪰',
|
||||||
|
match: ['Diptera']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'bugs-and-other-insects',
|
||||||
|
name: 'Bugs, crickets & other insects',
|
||||||
|
nameNl: 'Wantsen, sprinkhanen en andere insecten',
|
||||||
|
emoji: '🦗',
|
||||||
|
match: ['Insecta']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'spiders',
|
||||||
|
name: 'Spiders & kin',
|
||||||
|
nameNl: 'Spinnen en verwanten',
|
||||||
|
emoji: '🕷️',
|
||||||
|
match: ['Arachnida']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'crustaceans',
|
||||||
|
name: 'Crabs & other crustaceans',
|
||||||
|
nameNl: 'Kreeftachtigen',
|
||||||
|
emoji: '🦀',
|
||||||
|
match: [
|
||||||
|
'Crustacea',
|
||||||
|
'Malacostraca',
|
||||||
|
'Maxillopoda',
|
||||||
|
'Thecostraca',
|
||||||
|
'Branchiopoda',
|
||||||
|
'Hexanauplia'
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'molluscs',
|
||||||
|
name: 'Snails, slugs & other molluscs',
|
||||||
|
nameNl: 'Weekdieren',
|
||||||
|
emoji: '🐌',
|
||||||
|
match: ['Mollusca']
|
||||||
|
},
|
||||||
|
{
|
||||||
|
slug: 'other-invertebrates',
|
||||||
|
name: 'Jellyfish, starfish, worms & co',
|
||||||
|
nameNl: 'Overige ongewervelden',
|
||||||
|
emoji: '🪼',
|
||||||
|
match: []
|
||||||
|
}
|
||||||
|
];
|
||||||
|
export function groupOf(ancestorNames: string[]): GroupDef {
|
||||||
|
for (const g of GROUPS) if (g.match.some((m) => ancestorNames.includes(m))) return g;
|
||||||
|
return GROUPS.at(-1)!;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const IUCN_BY_NUMBER: Record<number, string> = {
|
||||||
|
5: 'DD',
|
||||||
|
10: 'LC',
|
||||||
|
20: 'NT',
|
||||||
|
30: 'VU',
|
||||||
|
40: 'EN',
|
||||||
|
50: 'CR',
|
||||||
|
60: 'EW',
|
||||||
|
70: 'EX'
|
||||||
|
};
|
||||||
172
scripts/wildlife-photos.ts
Normal file
172
scripts/wildlife-photos.ts
Normal file
@@ -0,0 +1,172 @@
|
|||||||
|
// Photos for the wildlife pokedex (/trackers/wildlife). Only Creative Commons licensed photos
|
||||||
|
// from iNaturalist are used; they are hotlinked (inaturalist-open-data S3), never downloaded,
|
||||||
|
// and every photo keeps its author, licence and a link back to iNaturalist.
|
||||||
|
//
|
||||||
|
// DB_URL=... bun scripts/wildlife-photos.ts (re)do photos for every species
|
||||||
|
// bun scripts/wildlife-photos.ts --missing only species never checked / without photos
|
||||||
|
// bun scripts/wildlife-photos.ts --limit 100 at most 100 species this run
|
||||||
|
// bun scripts/wildlife-photos.ts --cache /tmp/wildlife reuse API responses
|
||||||
|
// bun scripts/wildlife-photos.ts --missing --enrich report.json print the import report with photos
|
||||||
|
//
|
||||||
|
// Per species: the curated iNaturalist taxon photos with a CC licence, topped up with the most
|
||||||
|
// faved research-grade observation photos (CC licence only) until there are MAX photos.
|
||||||
|
import postgres from 'postgres';
|
||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import { chunks, log, saveCache, useCache } from './planes-wiki';
|
||||||
|
import {
|
||||||
|
author,
|
||||||
|
inat,
|
||||||
|
isCc,
|
||||||
|
mediumUrl,
|
||||||
|
PLACE_NL,
|
||||||
|
type InatPhoto,
|
||||||
|
type InatTaxon
|
||||||
|
} from './wildlife-inat';
|
||||||
|
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
||||||
|
const missing = args.includes('--missing');
|
||||||
|
const limit = Number(opt('--limit')) || 0;
|
||||||
|
const enrich = opt('--enrich');
|
||||||
|
const MAX = 10;
|
||||||
|
const MIN_TAXON = 7; // fewer CC taxon photos than this -> top up from observations
|
||||||
|
const LICENSES = 'cc0,cc-by,cc-by-sa,cc-by-nc,cc-by-nc-sa,cc-by-nd,cc-by-nc-nd';
|
||||||
|
|
||||||
|
type Photo = {
|
||||||
|
kind: string;
|
||||||
|
photo_id: number;
|
||||||
|
url: string;
|
||||||
|
width: number | null;
|
||||||
|
height: number | null;
|
||||||
|
author: string | null;
|
||||||
|
license: string;
|
||||||
|
page_url: string;
|
||||||
|
place: string | null;
|
||||||
|
};
|
||||||
|
const toPhoto = (p: InatPhoto, kind: string, page: string, place: string | null): Photo | null => {
|
||||||
|
if (!isCc(p.license_code) || !p.url) return null;
|
||||||
|
const dims = p.original_dimensions;
|
||||||
|
const scale = dims ? Math.min(1, 500 / Math.max(dims.width, dims.height)) : 1;
|
||||||
|
return {
|
||||||
|
kind,
|
||||||
|
photo_id: p.id,
|
||||||
|
url: mediumUrl(p.url),
|
||||||
|
width: dims ? Math.round(dims.width * scale) : null,
|
||||||
|
height: dims ? Math.round(dims.height * scale) : null,
|
||||||
|
author: author(p),
|
||||||
|
license: p.license_code!.toLowerCase(),
|
||||||
|
page_url: page,
|
||||||
|
place
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
type Obs = { id: number; uri?: string; place_guess?: string | null; photos: InatPhoto[] };
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
await useCache(opt('--cache'), 'wildlife-photos-api.json');
|
||||||
|
const url = process.env.DB_URL;
|
||||||
|
if (!url) throw new Error('Set DB_URL');
|
||||||
|
const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
||||||
|
try {
|
||||||
|
let todo = await sql<
|
||||||
|
{ id: number; inat_id: number; scientific_name: string; nl_obs: number }[]
|
||||||
|
>`
|
||||||
|
select id, inat_id, scientific_name, nl_obs from wildlife.species
|
||||||
|
${missing ? sql`where photos_checked_at is null or photo_count = 0` : sql``}
|
||||||
|
order by nl_obs desc, world_obs desc`;
|
||||||
|
if (limit) todo = todo.slice(0, limit);
|
||||||
|
log(`photos for ${todo.length} species`);
|
||||||
|
let done = 0;
|
||||||
|
let total = 0;
|
||||||
|
for (const batch of chunks(todo, 30)) {
|
||||||
|
const res = await inat<{ results: InatTaxon[] }>(
|
||||||
|
`/taxa/${batch.map((b) => b.inat_id).join(',')}?locale=en`,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
const taxa = new Map(res.results.map((t) => [t.id, t]));
|
||||||
|
for (const s of batch) {
|
||||||
|
const t = taxa.get(s.inat_id);
|
||||||
|
const photos: Photo[] = [];
|
||||||
|
const seen = new Set<number>();
|
||||||
|
const push = (p: Photo | null) => {
|
||||||
|
if (p && !seen.has(p.photo_id) && photos.length < MAX) {
|
||||||
|
seen.add(p.photo_id);
|
||||||
|
photos.push(p);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
for (const tp of t?.taxon_photos ?? [])
|
||||||
|
push(
|
||||||
|
toPhoto(tp.photo, 'taxon', `https://www.inaturalist.org/photos/${tp.photo.id}`, null)
|
||||||
|
);
|
||||||
|
if (photos.length < MIN_TAXON) {
|
||||||
|
// Prefer Dutch observations for species seen here, then anywhere.
|
||||||
|
const places = s.nl_obs >= 20 ? [`&place_id=${PLACE_NL}`, ''] : [''];
|
||||||
|
for (const place of places) {
|
||||||
|
if (photos.length >= MAX) break;
|
||||||
|
const obs = await inat<{ results: Obs[] }>(
|
||||||
|
`/observations?taxon_id=${s.inat_id}${place}&quality_grade=research&captive=false&photo_license=${LICENSES}&order_by=votes&order=desc&per_page=20&locale=en`
|
||||||
|
);
|
||||||
|
for (const o of obs.results) {
|
||||||
|
const page = `https://www.inaturalist.org/observations/${o.id}`;
|
||||||
|
// One photo per observation first, for variety.
|
||||||
|
push(toPhoto(o.photos[0], 'observation', page, o.place_guess ?? null));
|
||||||
|
}
|
||||||
|
for (const o of obs.results)
|
||||||
|
for (const p of o.photos.slice(1, 2))
|
||||||
|
push(
|
||||||
|
toPhoto(
|
||||||
|
p,
|
||||||
|
'observation',
|
||||||
|
`https://www.inaturalist.org/observations/${o.id}`,
|
||||||
|
o.place_guess ?? null
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await sql.begin(async (tx) => {
|
||||||
|
await tx`delete from wildlife.photos where species_id = ${s.id}`;
|
||||||
|
for (const [i, p] of photos.entries())
|
||||||
|
await tx`insert into wildlife.photos ${tx({ ...p, species_id: s.id, position: i })}`;
|
||||||
|
await tx`update wildlife.species set photo = ${photos[0]?.url ?? null},
|
||||||
|
photo_count = ${photos.length}, photos_checked_at = now() where id = ${s.id}`;
|
||||||
|
});
|
||||||
|
total += photos.length;
|
||||||
|
done++;
|
||||||
|
}
|
||||||
|
await saveCache();
|
||||||
|
if (done % 150 < 30) log(` ${done}/${todo.length} species, ${total} photos`);
|
||||||
|
}
|
||||||
|
await sql`
|
||||||
|
update wildlife.groups g set photo = (
|
||||||
|
select s.photo from wildlife.species s where s.group_id = g.id and s.photo is not null
|
||||||
|
order by s.nl_obs desc, s.world_obs desc limit 1)`;
|
||||||
|
log(`done: ${done} species, ${total} photos`);
|
||||||
|
if (enrich) {
|
||||||
|
const report = JSON.parse(await readFile(enrich, 'utf8'));
|
||||||
|
const ids = (report.newSpecies ?? []).map((n: { inatId: number }) => n.inatId);
|
||||||
|
const rows = ids.length
|
||||||
|
? await sql<{ inat_id: number; photo: string | null; photo_count: number }[]>`
|
||||||
|
select inat_id, photo, photo_count from wildlife.species where inat_id in ${sql(ids)}`
|
||||||
|
: [];
|
||||||
|
const by = new Map(rows.map((r) => [r.inat_id, r]));
|
||||||
|
const [totals] = await sql`
|
||||||
|
select (select count(*)::int from wildlife.species) as species,
|
||||||
|
(select count(*)::int from wildlife.groups where species_count > 0) as groups,
|
||||||
|
(select count(*)::int from wildlife.photos) as photos`;
|
||||||
|
report.totals = totals;
|
||||||
|
report.newSpecies = (report.newSpecies ?? [])
|
||||||
|
.map((n: { inatId: number }) => ({
|
||||||
|
...n,
|
||||||
|
photo: by.get(n.inatId)?.photo?.replace('/medium.', '/large.') ?? null,
|
||||||
|
photos: by.get(n.inatId)?.photo_count ?? 0
|
||||||
|
}))
|
||||||
|
.toSorted((a: { nlObs: number }, b: { nlObs: number }) => b.nlObs - a.nlObs);
|
||||||
|
console.log(JSON.stringify(report, null, 1));
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await saveCache(true);
|
||||||
|
await sql.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await main();
|
||||||
33
scripts/wildlife-refresh.sh
Executable file
33
scripts/wildlife-refresh.sh
Executable file
@@ -0,0 +1,33 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Refresh of the wildlife pokedex (/trackers/wildlife) on stan-ovh.
|
||||||
|
#
|
||||||
|
# ssh stan@37.187.157.191 'cd ~/dev/website-wildlife && git fetch -q && git checkout -q --detach origin/main \
|
||||||
|
# && bun install --frozen-lockfile >/dev/null && scripts/wildlife-refresh.sh --report-new'
|
||||||
|
#
|
||||||
|
# 1. bun scripts/wildlife-import.ts re-reads the species lists (NL, Europe, world) from
|
||||||
|
# iNaturalist + Wikidata + GBIF, refreshes counts/status of every species and adds species
|
||||||
|
# that entered the lists (existing ones are kept, Dex numbers are stable).
|
||||||
|
# 2. bun scripts/wildlife-photos.ts --missing CC-licensed iNaturalist photos for species
|
||||||
|
# without photos (new ones included).
|
||||||
|
# --report-new: stdout is one JSON report {generatedAt, totals, newSpecies[] (with photo)}.
|
||||||
|
# Without it, progress only. Progress goes to stderr. DB_URL is read from the running app
|
||||||
|
# container and never printed.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
if [ -z "${DB_URL:-}" ]; then
|
||||||
|
DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' |
|
||||||
|
sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
|
||||||
|
export DB_URL
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${1:-}" = "--report-new" ]; then
|
||||||
|
shift
|
||||||
|
report=$(mktemp /tmp/wildlife-report.XXXXXX.json)
|
||||||
|
trap 'rm -f "$report"' EXIT
|
||||||
|
bun scripts/wildlife-import.ts --report-new "$@" >"$report"
|
||||||
|
bun scripts/wildlife-photos.ts --missing --enrich "$report"
|
||||||
|
else
|
||||||
|
bun scripts/wildlife-import.ts "$@" >&2
|
||||||
|
bun scripts/wildlife-photos.ts --missing >&2
|
||||||
|
fi
|
||||||
Reference in New Issue
Block a user