website/scripts/wildlife-photos.ts

// Photos for the wildlife pokedex (/trackers/wildlife). Only Creative Commons licensed photos
// from iNaturalist are used; they are hotlinked (inaturalist-open-data S3), never downloaded,
// and every photo keeps its author, licence and a link back to iNaturalist.
//
//   DB_URL=... bun scripts/wildlife-photos.ts             (re)do photos for every species
//   bun scripts/wildlife-photos.ts --missing              only species never checked / without photos
//   bun scripts/wildlife-photos.ts --limit 100            at most 100 species this run
//   bun scripts/wildlife-photos.ts --missing --enrich report.json   print the import report with photos
//
// Per species: the curated iNaturalist taxon photos with a CC licence, topped up with the most
// faved research-grade observation photos (CC licence only) until there are MAX photos.
import postgres from 'postgres';
import { readFile } from 'node:fs/promises';
import { chunks, log } from './planes-wiki';
import {
	author,
	inat,
	isCc,
	mediumUrl,
	PLACE_NL,
	type InatPhoto,
	type InatTaxon
} from './wildlife-inat';

const args = process.argv.slice(2);
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
const missing = args.includes('--missing');
const limit = Number(opt('--limit')) || 0;
const enrich = opt('--enrich');
const MAX = 10;
const MIN_TAXON = 7; // fewer CC taxon photos than this -> top up from observations
const LICENSES = 'cc0,cc-by,cc-by-sa,cc-by-nc,cc-by-nc-sa,cc-by-nd,cc-by-nc-nd';

type Photo = {
	kind: string;
	photo_id: number;
	url: string;
	width: number | null;
	height: number | null;
	author: string | null;
	license: string;
	page_url: string;
	place: string | null;
};
const toPhoto = (p: InatPhoto, kind: string, page: string, place: string | null): Photo | null => {
	if (!isCc(p.license_code) || !p.url) return null;
	const dims = p.original_dimensions;
	const scale = dims ? Math.min(1, 500 / Math.max(dims.width, dims.height)) : 1;
	return {
		kind,
		photo_id: p.id,
		url: mediumUrl(p.url),
		width: dims ? Math.round(dims.width * scale) : null,
		height: dims ? Math.round(dims.height * scale) : null,
		author: author(p),
		license: p.license_code!.toLowerCase(),
		page_url: page,
		place
	};
};

type Obs = { id: number; uri?: string; place_guess?: string | null; photos: InatPhoto[] };

async function main() {
	// No response cache: --missing already resumes, and caching every observation page grew
	// past what JSON.stringify can write (RangeError: Out of memory).
	const url = process.env.DB_URL;
	if (!url) throw new Error('Set DB_URL');
	const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} });
	try {
		let todo = await sql<
			{ id: number; inat_id: number; scientific_name: string; nl_obs: number }[]
		>`
			select id, inat_id, scientific_name, nl_obs from wildlife.species
			${missing ? sql`where photos_checked_at is null or photo_count = 0` : sql``}
			order by nl_obs desc, world_obs desc`;
		if (limit) todo = todo.slice(0, limit);
		log(`photos for ${todo.length} species`);
		let done = 0;
		let total = 0;
		for (const batch of chunks(todo, 30)) {
			const res = await inat<{ results: InatTaxon[] }>(
				`/taxa/${batch.map((b) => b.inat_id).join(',')}?locale=en`,
				false
			);
			const taxa = new Map(res.results.map((t) => [t.id, t]));
			for (const s of batch) {
				const t = taxa.get(s.inat_id);
				const photos: Photo[] = [];
				const seen = new Set<number>();
				const push = (p: Photo | null) => {
					if (p && !seen.has(p.photo_id) && photos.length < MAX) {
						seen.add(p.photo_id);
						photos.push(p);
					}
				};
				for (const tp of t?.taxon_photos ?? [])
					push(
						toPhoto(tp.photo, 'taxon', `https://www.inaturalist.org/photos/${tp.photo.id}`, null)
					);
				if (photos.length < MIN_TAXON) {
					// Prefer Dutch observations for species seen here, then anywhere.
					const places = s.nl_obs >= 20 ? [`&place_id=${PLACE_NL}`, ''] : [''];
					for (const place of places) {
						if (photos.length >= MAX) break;
						const obs = await inat<{ results: Obs[] }>(
							`/observations?taxon_id=${s.inat_id}${place}&quality_grade=research&captive=false&photo_license=${LICENSES}&order_by=votes&order=desc&per_page=20&locale=en`
						);
						for (const o of obs.results) {
							const page = `https://www.inaturalist.org/observations/${o.id}`;
							// One photo per observation first, for variety.
							push(toPhoto(o.photos[0], 'observation', page, o.place_guess ?? null));
						}
						for (const o of obs.results)
							for (const p of o.photos.slice(1, 2))
								push(
									toPhoto(
										p,
										'observation',
										`https://www.inaturalist.org/observations/${o.id}`,
										o.place_guess ?? null
									)
								);
					}
				}
				await sql.begin(async (tx) => {
					await tx`delete from wildlife.photos where species_id = ${s.id}`;
					for (const [i, p] of photos.entries())
						await tx`insert into wildlife.photos ${tx({ ...p, species_id: s.id, position: i })}`;
					await tx`update wildlife.species set photo = ${photos[0]?.url ?? null},
						photo_count = ${photos.length}, photos_checked_at = now() where id = ${s.id}`;
				});
				total += photos.length;
				done++;
			}
			if (done % 150 < 30) log(`  ${done}/${todo.length} species, ${total} photos`);
		}
		await sql`
			update wildlife.groups g set photo = (
				select s.photo from wildlife.species s where s.group_id = g.id and s.photo is not null
				order by s.nl_obs desc, s.world_obs desc limit 1)`;
		log(`done: ${done} species, ${total} photos`);
		if (enrich) {
			const report = JSON.parse(await readFile(enrich, 'utf8'));
			const ids = (report.newSpecies ?? []).map((n: { inatId: number }) => n.inatId);
			const rows = ids.length
				? await sql<{ inat_id: number; photo: string | null; photo_count: number }[]>`
					select inat_id, photo, photo_count from wildlife.species where inat_id in ${sql(ids)}`
				: [];
			const by = new Map(rows.map((r) => [r.inat_id, r]));
			const [totals] = await sql`
				select (select count(*)::int from wildlife.species) as species,
					(select count(*)::int from wildlife.groups where species_count > 0) as groups,
					(select count(*)::int from wildlife.photos) as photos`;
			report.totals = totals;
			report.newSpecies = (report.newSpecies ?? [])
				.map((n: { inatId: number }) => ({
					...n,
					photo: by.get(n.inatId)?.photo?.replace('/medium.', '/large.') ?? null,
					photos: by.get(n.inatId)?.photo_count ?? 0
				}))
				.toSorted((a: { nlObs: number }, b: { nlObs: number }) => b.nlObs - a.nlObs);
			console.log(JSON.stringify(report, null, 1));
		}
	} finally {
		await sql.end();
	}
}

await main();