wildlife: iNaturalist/Wikidata/GBIF import, CC photos and refresh script

2026-10-24 21:58:35 +00:00
parent 3f251145be
commit ba44365c71
5 changed files with 1022 additions and 0 deletions
--- a/scripts/wildlife-photos.ts
+++ b/scripts/wildlife-photos.ts
@@ -0,0 +1,172 @@
+// Photos for the wildlife pokedex (/trackers/wildlife). Only Creative Commons licensed photos
+// from iNaturalist are used; they are hotlinked (inaturalist-open-data S3), never downloaded,
+// and every photo keeps its author, licence and a link back to iNaturalist.
+//
+//   DB_URL=... bun scripts/wildlife-photos.ts             (re)do photos for every species
+//   bun scripts/wildlife-photos.ts --missing              only species never checked / without photos
+//   bun scripts/wildlife-photos.ts --limit 100            at most 100 species this run
+//   bun scripts/wildlife-photos.ts --cache /tmp/wildlife  reuse API responses
+//   bun scripts/wildlife-photos.ts --missing --enrich report.json   print the import report with photos
+//
+// Per species: the curated iNaturalist taxon photos with a CC licence, topped up with the most
+// faved research-grade observation photos (CC licence only) until there are MAX photos.
+import postgres from 'postgres';
+import { readFile } from 'node:fs/promises';
+import { chunks, log, saveCache, useCache } from './planes-wiki';
+import {
+	author,
+	inat,
+	isCc,
+	mediumUrl,
+	PLACE_NL,
+	type InatPhoto,
+	type InatTaxon
+} from './wildlife-inat';
+
+const args = process.argv.slice(2);
+const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
+const missing = args.includes('--missing');
+const limit = Number(opt('--limit')) || 0;
+const enrich = opt('--enrich');
+const MAX = 10;
+const MIN_TAXON = 7; // fewer CC taxon photos than this -> top up from observations
+const LICENSES = 'cc0,cc-by,cc-by-sa,cc-by-nc,cc-by-nc-sa,cc-by-nd,cc-by-nc-nd';
+
+type Photo = {
+	kind: string;
+	photo_id: number;
+	url: string;
+	width: number | null;
+	height: number | null;
+	author: string | null;
+	license: string;
+	page_url: string;
+	place: string | null;
+};
+const toPhoto = (p: InatPhoto, kind: string, page: string, place: string | null): Photo | null => {
+	if (!isCc(p.license_code) || !p.url) return null;
+	const dims = p.original_dimensions;
+	const scale = dims ? Math.min(1, 500 / Math.max(dims.width, dims.height)) : 1;
+	return {
+		kind,
+		photo_id: p.id,
+		url: mediumUrl(p.url),
+		width: dims ? Math.round(dims.width * scale) : null,
+		height: dims ? Math.round(dims.height * scale) : null,
+		author: author(p),
+		license: p.license_code!.toLowerCase(),
+		page_url: page,
+		place
+	};
+};
+
+type Obs = { id: number; uri?: string; place_guess?: string | null; photos: InatPhoto[] };
+
+async function main() {
+	await useCache(opt('--cache'), 'wildlife-photos-api.json');
+	const url = process.env.DB_URL;
+	if (!url) throw new Error('Set DB_URL');
+	const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} });
+	try {
+		let todo = await sql<
+			{ id: number; inat_id: number; scientific_name: string; nl_obs: number }[]
+		>`
+			select id, inat_id, scientific_name, nl_obs from wildlife.species
+			${missing ? sql`where photos_checked_at is null or photo_count = 0` : sql``}
+			order by nl_obs desc, world_obs desc`;
+		if (limit) todo = todo.slice(0, limit);
+		log(`photos for ${todo.length} species`);
+		let done = 0;
+		let total = 0;
+		for (const batch of chunks(todo, 30)) {
+			const res = await inat<{ results: InatTaxon[] }>(
+				`/taxa/${batch.map((b) => b.inat_id).join(',')}?locale=en`,
+				false
+			);
+			const taxa = new Map(res.results.map((t) => [t.id, t]));
+			for (const s of batch) {
+				const t = taxa.get(s.inat_id);
+				const photos: Photo[] = [];
+				const seen = new Set<number>();
+				const push = (p: Photo | null) => {
+					if (p && !seen.has(p.photo_id) && photos.length < MAX) {
+						seen.add(p.photo_id);
+						photos.push(p);
+					}
+				};
+				for (const tp of t?.taxon_photos ?? [])
+					push(
+						toPhoto(tp.photo, 'taxon', `https://www.inaturalist.org/photos/${tp.photo.id}`, null)
+					);
+				if (photos.length < MIN_TAXON) {
+					// Prefer Dutch observations for species seen here, then anywhere.
+					const places = s.nl_obs >= 20 ? [`&place_id=${PLACE_NL}`, ''] : [''];
+					for (const place of places) {
+						if (photos.length >= MAX) break;
+						const obs = await inat<{ results: Obs[] }>(
+							`/observations?taxon_id=${s.inat_id}${place}&quality_grade=research&captive=false&photo_license=${LICENSES}&order_by=votes&order=desc&per_page=20&locale=en`
+						);
+						for (const o of obs.results) {
+							const page = `https://www.inaturalist.org/observations/${o.id}`;
+							// One photo per observation first, for variety.
+							push(toPhoto(o.photos[0], 'observation', page, o.place_guess ?? null));
+						}
+						for (const o of obs.results)
+							for (const p of o.photos.slice(1, 2))
+								push(
+									toPhoto(
+										p,
+										'observation',
+										`https://www.inaturalist.org/observations/${o.id}`,
+										o.place_guess ?? null
+									)
+								);
+					}
+				}
+				await sql.begin(async (tx) => {
+					await tx`delete from wildlife.photos where species_id = ${s.id}`;
+					for (const [i, p] of photos.entries())
+						await tx`insert into wildlife.photos ${tx({ ...p, species_id: s.id, position: i })}`;
+					await tx`update wildlife.species set photo = ${photos[0]?.url ?? null},
+						photo_count = ${photos.length}, photos_checked_at = now() where id = ${s.id}`;
+				});
+				total += photos.length;
+				done++;
+			}
+			await saveCache();
+			if (done % 150 < 30) log(`  ${done}/${todo.length} species, ${total} photos`);
+		}
+		await sql`
+			update wildlife.groups g set photo = (
+				select s.photo from wildlife.species s where s.group_id = g.id and s.photo is not null
+				order by s.nl_obs desc, s.world_obs desc limit 1)`;
+		log(`done: ${done} species, ${total} photos`);
+		if (enrich) {
+			const report = JSON.parse(await readFile(enrich, 'utf8'));
+			const ids = (report.newSpecies ?? []).map((n: { inatId: number }) => n.inatId);
+			const rows = ids.length
+				? await sql<{ inat_id: number; photo: string | null; photo_count: number }[]>`
+					select inat_id, photo, photo_count from wildlife.species where inat_id in ${sql(ids)}`
+				: [];
+			const by = new Map(rows.map((r) => [r.inat_id, r]));
+			const [totals] = await sql`
+				select (select count(*)::int from wildlife.species) as species,
+					(select count(*)::int from wildlife.groups where species_count > 0) as groups,
+					(select count(*)::int from wildlife.photos) as photos`;
+			report.totals = totals;
+			report.newSpecies = (report.newSpecies ?? [])
+				.map((n: { inatId: number }) => ({
+					...n,
+					photo: by.get(n.inatId)?.photo?.replace('/medium.', '/large.') ?? null,
+					photos: by.get(n.inatId)?.photo_count ?? 0
+				}))
+				.toSorted((a: { nlObs: number }, b: { nlObs: number }) => b.nlObs - a.nlObs);
+			console.log(JSON.stringify(report, null, 1));
+		}
+	} finally {
+		await saveCache(true);
+		await sql.end();
+	}
+}
+
+await main();