spotters: most-active-in sort with country/region selector, collection tracking (seen marks, progress, filters), refresh scripts score activity and report new NL entries

scripts/spotters-activity.ts

@@ -0,0 +1,397 @@
+// Precomputes "how active is this entry in country X" for the four spotting pokedexes
+// (/trackers/cars, planes, trains, wildlife) into <schema>.activity (item, area, score),
+// so the dex can sort by most active in NL / Germany / Europe ... with one indexed join.
+//
+//   DB_URL=... bun scripts/spotters-activity.ts <cars|planes|trains|wildlife|all> [--dry-run]
+//
+// Scores are only comparable within one area (higher = more active there):
+//  - cars:     NL = cars registered in NL (RDW). US = EPA-tested model years and variants.
+//              The brand's home country = overall popularity (NL regs + 100 x US variants).
+//              Other countries have no registration data (gap).
+//  - planes:   aircraft registered per country with that type (OpenSky aircraft database,
+//              matched by maker + model name, then by ICAO type code), plus 25 per
+//              operator from that country on Wikidata (P137 -> P17; air forces, airlines).
+//              Variants roll up into their family.
+//  - trains:   100 if an operator from that country runs it (+25 per extra operator), +300
+//              if still in service (+100 unknown), 20 for the country of origin only, small
+//              tie-break for number built.
+//  - wildlife: research-grade iNaturalist observations per country (same query as the NL
+//              count in scripts/wildlife-import.ts); Europe / world use the stored counts.
+// Regions (src/lib/spotting/areas.ts) are the sum of their countries; `world` = all.
+// External calls: OpenSky CSV (1 download), Wikidata SPARQL (~10 queries) and the
+// iNaturalist API (~1 call/s, ~450 calls). Progress on stderr.
+import postgres from 'postgres';
+import { existsSync } from 'node:fs';
+import { REGIONS, COUNTRIES } from '../src/lib/spotting/areas';
+import { countryCode } from './spotters-countries';
+import { chunks, log, sparql } from './planes-wiki';
+import { inat } from './wildlife-inat';
+
+type Scores = Map<number, Map<string, number>>;
+const args = process.argv.slice(2);
+const which = args.find((a) => !a.startsWith('--')) ?? 'all';
+const dryRun = args.includes('--dry-run');
+const flag = (name: string) => {
+	const i = args.indexOf(name);
+	return i >= 0 ? args[i + 1] : undefined;
+};
+
+const url = process.env.DB_URL;
+if (!url) throw new Error('Set DB_URL');
+const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} });
+
+function add(scores: Scores, id: number, area: string | null, value: number) {
+	if (!area || !(value > 0)) return;
+	let m = scores.get(id);
+	if (!m) scores.set(id, (m = new Map()));
+	m.set(area, (m.get(area) ?? 0) + value);
+}
+
+/** Adds region rows (sum of countries) unless already present, then replaces the table. */
+async function write(schema: string, column: string, scores: Scores, keepRegions = false) {
+	const rows: { id: number; area: string; score: number }[] = [];
+	for (const [id, m] of scores) {
+		const out = new Map(m);
+		for (const r of REGIONS) {
+			if (keepRegions && m.has(r.code)) continue;
+			let sum = 0;
+			for (const [area, v] of m)
+				if (/^[a-z]{2}$/.test(area) && (r.code === 'world' || r.countries.includes(area))) sum += v;
+			if (sum > 0) out.set(r.code, sum);
+		}
+		for (const [area, score] of out) rows.push({ id, area, score: Math.round(score * 100) / 100 });
+	}
+	const areas = new Map<string, number>();
+	for (const r of rows) areas.set(r.area, (areas.get(r.area) ?? 0) + 1);
+	log(
+		`${schema}: ${rows.length} activity rows for ${scores.size} entries; nl ${areas.get('nl') ?? 0}, de ${areas.get('de') ?? 0}, us ${areas.get('us') ?? 0}, europe ${areas.get('europe') ?? 0}, ${areas.size} areas`
+	);
+	if (dryRun) return;
+	await sql.begin(async (tx) => {
+		await tx.unsafe(`delete from ${schema}.activity`);
+		for (const c of chunks(rows, 5000))
+			await tx.unsafe(
+				`insert into ${schema}.activity (${column}, area, score)
+				 select * from unnest($1::int[], $2::text[], $3::real[])`,
+				[c.map((r) => r.id), c.map((r) => r.area), c.map((r) => r.score)] as never
+			);
+	});
+}
+
+// ------------------------------------------------------------------ cars
+
+async function cars() {
+	const models = await sql<
+		{ id: number; nl: number; variants: number; years: number; country: string | null }[]
+	>`select m.id, m.nl_count as nl, m.variant_count as variants,
+		coalesce(cardinality(m.us_years), 0) as years, b.country
+		from cars.models m join cars.brands b on b.id = m.brand_id`;
+	const scores: Scores = new Map();
+	for (const m of models) {
+		add(scores, m.id, 'nl', m.nl);
+		add(scores, m.id, 'us', m.years * 10 + m.variants);
+		const home = countryCode(m.country);
+		if (home && home !== 'nl' && home !== 'us')
+			add(scores, m.id, home, 1 + m.nl + 100 * m.variants);
+	}
+	await write('cars', 'model_id', scores);
+}
+
+// ------------------------------------------------------------------ trains
+
+async function trains() {
+	const types = await sql<
+		{
+			id: number;
+			country: string | null;
+			built: number | null;
+			in_service: boolean | null;
+			retired: number | null;
+			ops: (string | null)[];
+		}[]
+	>`select t.id, t.country, t.number_built as built, t.in_service, t.retired_year as retired,
+		coalesce(array_agg(o.country) filter (where o.id is not null), '{}') as ops
+		from trains.types t
+		left join trains.type_operators x on x.type_id = t.id
+		left join trains.operators o on o.id = x.operator_id
+		group by t.id`;
+	const scores: Scores = new Map();
+	for (const t of types) {
+		// Running there now matters most ("does it run on Dutch rails?"): in service +300,
+		// unknown status +100, retired +0 (+50 if retired after 2000).
+		const tie = Math.min(t.built ?? 0, 1000) / 100;
+		const status =
+			t.in_service === true ? 300 : t.in_service == null ? 100 : (t.retired ?? 0) >= 2000 ? 50 : 0;
+		const opCountries = new Map<string, number>();
+		for (const name of t.ops) {
+			const cc = countryCode(name);
+			if (cc) opCountries.set(cc, (opCountries.get(cc) ?? 0) + 1);
+		}
+		for (const [cc, n] of opCountries)
+			add(scores, t.id, cc, 100 + 25 * (Math.min(n, 4) - 1) + status + tie);
+		const origin = countryCode(t.country);
+		if (origin && !opCountries.has(origin)) add(scores, t.id, origin, 20 + tie);
+	}
+	await write('trains', 'type_id', scores);
+}
+
+// ------------------------------------------------------------------ planes
+
+/** Minimal CSV reader for OpenSky's single-quoted dump ('' escapes a quote). */
+function* csvRows(text: string, quote = "'") {
+	let row: string[] = [];
+	let field = '';
+	let inQ = false;
+	for (let i = 0; i < text.length; i++) {
+		const ch = text[i];
+		if (inQ) {
+			if (ch === quote) {
+				if (text[i + 1] === quote) {
+					field += quote;
+					i++;
+				} else inQ = false;
+			} else field += ch;
+		} else if (ch === quote) inQ = true;
+		else if (ch === ',') {
+			row.push(field);
+			field = '';
+		} else if (ch === '\n') {
+			row.push(field);
+			yield row;
+			row = [];
+			field = '';
+		} else if (ch !== '\r') field += ch;
+	}
+	if (field || row.length) yield [...row, field];
+}
+
+const norm = (s: string) =>
+	s
+		.toLowerCase()
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.replace(/[^a-z0-9]+/g, ' ')
+		.trim();
+const GENERIC = /\b(family|series|aircraft|airliner|helicopter|classic|next generation)\b/g;
+
+async function planes() {
+	const csvPath = flag('--opensky') ?? '/tmp/spotters/opensky.csv';
+	if (!existsSync(csvPath)) {
+		log(`planes: downloading the OpenSky aircraft database to ${csvPath}`);
+		const res = await fetch(
+			'https://s3.opensky-network.org/data-samples/metadata/aircraft-database-complete-2025-08.csv'
+		);
+		if (!res.ok) throw new Error(`OpenSky download: HTTP ${res.status}`);
+		await Bun.write(csvPath, await res.arrayBuffer());
+	}
+	const types = await sql<
+		{
+			id: number;
+			family: number | null;
+			name: string;
+			maker: string;
+			makers: string[];
+			icao: string[];
+			wikidata: string;
+		}[]
+	>`select t.id, t.family_id as family, t.name, m.name as maker, t.makers, t.icao, t.wikidata
+		from planes.types t join planes.makers m on m.id = t.maker_id`;
+
+	// Candidate keys per maker word: the type's name, without generic words, and its
+	// leading code part ("PA-28 Cherokee" -> "pa 28", "F-16 Fighting Falcon" -> "f 16").
+	type Cand = { id: number; key: string; exact: boolean; family: number | null };
+	const byMakerWord = new Map<string, Cand[]>();
+	for (const t of types) {
+		const keys = new Set<string>();
+		const n = norm(t.name);
+		keys.add(n);
+		keys.add(n.replace(GENERIC, ' ').replace(/\s+/g, ' ').trim());
+		const toks = n.split(' ');
+		const codeEnd = toks.findIndex((w, i) => i > 0 && /^[a-z]{3,}$/.test(w));
+		if (codeEnd > 0 && toks.slice(0, codeEnd).some((w) => /\d/.test(w)))
+			keys.add(toks.slice(0, codeEnd).join(' '));
+		const words = new Set([t.maker, ...t.makers].map((m) => norm(m).split(' ')[0]).filter(Boolean));
+		for (const key of keys) {
+			if (key.length < 2 || (!/\d/.test(key) && key.length < 4)) continue;
+			for (const w of words) {
+				if (!byMakerWord.has(w)) byMakerWord.set(w, []);
+				byMakerWord.get(w)!.push({ id: t.id, key, exact: key === n, family: t.family });
+			}
+		}
+	}
+	const matchModel = (manufacturer: string, model: string): number | null => {
+		const cands = byMakerWord.get(norm(manufacturer).split(' ')[0]);
+		if (!cands) return null;
+		const m = norm(model);
+		let best: Cand | null = null;
+		for (const c of cands) {
+			if (!(m === c.key || (m.startsWith(c.key) && !/\d/.test(m[c.key.length] ?? '')))) continue;
+			if (
+				!best ||
+				c.key.length > best.key.length ||
+				(c.key.length === best.key.length && best.family !== null && c.family === null)
+			)
+				best = c;
+		}
+		return best?.id ?? null;
+	};
+
+	log('planes: reading OpenSky aircraft');
+	const text = await Bun.file(csvPath).text();
+	const rows = csvRows(text);
+	const head = rows.next().value as string[];
+	const col = (n: string) => head.indexOf(n);
+	const [iCountry, iMan, iModel, iType] = [
+		col('country'),
+		col('manufacturerName'),
+		col('model'),
+		col('typecode')
+	];
+	const direct = new Map<string, Map<string, number>>(); // "id" -> country -> n
+	const byCode = new Map<string, Map<string, number>>(); // typecode -> country -> n (unmatched)
+	const codeVotes = new Map<string, Map<number, number>>(); // typecode -> type id -> votes
+	let total = 0;
+	let matched = 0;
+	for (const r of rows) {
+		if (r.length < head.length) continue;
+		const cc = countryCode(r[iCountry]);
+		if (!cc) continue;
+		total++;
+		const code = r[iType]?.trim().toUpperCase();
+		const id = r[iMan] && r[iModel] ? matchModel(r[iMan], r[iModel]) : null;
+		if (id) {
+			matched++;
+			const key = String(id);
+			if (!direct.has(key)) direct.set(key, new Map());
+			direct.get(key)!.set(cc, (direct.get(key)!.get(cc) ?? 0) + 1);
+			if (code) {
+				if (!codeVotes.has(code)) codeVotes.set(code, new Map());
+				codeVotes.get(code)!.set(id, (codeVotes.get(code)!.get(id) ?? 0) + 1);
+			}
+		} else if (code) {
+			if (!byCode.has(code)) byCode.set(code, new Map());
+			byCode.get(code)!.set(cc, (byCode.get(code)!.get(cc) ?? 0) + 1);
+		}
+	}
+	// Unmatched aircraft follow their ICAO type code: the type's own ICAO list first, else
+	// the type most matched aircraft with that code went to (needs a clear majority).
+	const icaoOwner = new Map<string, number>();
+	for (const t of types) for (const c of t.icao) if (!icaoOwner.has(c)) icaoOwner.set(c, t.id);
+	let viaCode = 0;
+	for (const [code, counts] of byCode) {
+		let owner = icaoOwner.get(code) ?? null;
+		const votes = codeVotes.get(code);
+		if (!owner && votes) {
+			const sorted = [...votes].sort((a, b) => b[1] - a[1]);
+			const sum = sorted.reduce((s, [, n]) => s + n, 0);
+			if (sorted[0][1] >= 3 && sorted[0][1] / sum >= 0.6) owner = sorted[0][0];
+		}
+		if (!owner) continue;
+		const key = String(owner);
+		if (!direct.has(key)) direct.set(key, new Map());
+		for (const [cc, n] of counts) {
+			direct.get(key)!.set(cc, (direct.get(key)!.get(cc) ?? 0) + n);
+			viaCode += n;
+		}
+	}
+	log(
+		`planes: ${total} aircraft with a country, ${matched} matched by model, ${viaCode} by type code`
+	);
+
+	const scores: Scores = new Map();
+	// Roll registrations up the family tree (737 MAX 8 -> 737 MAX -> 737).
+	const parent = new Map(types.map((t) => [t.id, t.family]));
+	for (const [key, m] of direct) {
+		let id: number | null = Number(key);
+		for (let depth = 0; id && depth < 6; depth++, id = parent.get(id) ?? null)
+			for (const [cc, n] of m) add(scores, id, cc, n);
+	}
+	// Wikidata operators (P137) and their countries.
+	const byQid = new Map(types.map((t) => [t.wikidata, t.id]));
+	let ops = 0;
+	let failed = 0;
+	for (const c of chunks(
+		[...byQid.keys()].filter((q): q is string => !!q),
+		100
+	)) {
+		// A failing chunk (query timeout) only loses its operator bonus; registrations stay.
+		const res = await sparql<{ item: string; iso: string; n: string }>(`
+			SELECT ?item ?iso (COUNT(DISTINCT ?op) AS ?n) WHERE {
+				VALUES ?item { ${c.map((q) => `wd:${q}`).join(' ')} }
+				?item wdt:P137 ?op . ?op wdt:P17 ?country . ?country wdt:P297 ?iso .
+			} GROUP BY ?item ?iso`).catch((e: unknown) => {
+			failed++;
+			log(`planes: operator chunk failed (${String(e).slice(0, 80)})`);
+			return [];
+		});
+		for (const r of res) {
+			const id = byQid.get(r.item.split('/').at(-1)!);
+			if (!id) continue;
+			add(scores, id, r.iso.toLowerCase(), 25 * Math.min(Number(r.n) || 1, 6));
+			ops++;
+		}
+	}
+	log(`planes: ${ops} type x country operator links from Wikidata (${failed} chunks failed)`);
+	await write('planes', 'type_id', scores);
+}
+
+// ------------------------------------------------------------------ wildlife
+
+async function wildlife() {
+	const species = await sql<{ id: number; inat: number; nl: number; eu: number; world: number }[]>`
+		select id, inat_id as inat, nl_obs as nl, europe_obs as eu, world_obs as world from wildlife.species`;
+	const scores: Scores = new Map();
+	for (const s of species) {
+		add(scores, s.id, 'nl', s.nl);
+		add(scores, s.id, 'europe', s.eu);
+		add(scores, s.id, 'world', s.world);
+	}
+	const byInat = new Map(species.map((s) => [s.inat, s.id]));
+	const countries = [...new Set([...COUNTRIES, 'lu'])].filter((c) => c !== 'nl');
+	const dn = new Intl.DisplayNames(['en'], { type: 'region' });
+	for (const cc of countries) {
+		const name =
+			{
+				us: 'United States',
+				gb: 'United Kingdom',
+				cz: 'Czech Republic',
+				tr: 'Turkey',
+				kr: 'South Korea'
+			}[cc] ?? dn.of(cc.toUpperCase())!;
+		const found = await inat<{
+			results: { id: number; name: string; admin_level: number | null }[];
+		}>(`/places/autocomplete?q=${encodeURIComponent(name)}&per_page=20`);
+		const place =
+			found.results.find((p) => p.admin_level === 0 && norm(p.name) === norm(name)) ??
+			found.results.find((p) => p.admin_level === 0);
+		if (!place) {
+			log(`wildlife: no iNaturalist place for ${cc} (${name})`);
+			continue;
+		}
+		let n = 0;
+		for (const batch of chunks([...byInat.keys()], 400)) {
+			const res = await inat(
+				`/observations/species_counts?place_id=${place.id}&taxon_id=${batch.join(',')}&captive=false&quality_grade=research&per_page=500`
+			);
+			for (const r of res.results) {
+				const id = byInat.get(r.taxon.id);
+				if (id && r.count > 0) {
+					add(scores, id, cc, r.count);
+					n++;
+				}
+			}
+		}
+		log(`wildlife: ${cc} (${place.name}, place ${place.id}): ${n} species`);
+	}
+	await write('wildlife', 'species_id', scores, true);
+}
+
+const jobs: Record<string, () => Promise<void>> = { cars, trains, planes, wildlife };
+try {
+	for (const name of which === 'all' ? Object.keys(jobs) : which.split(',')) {
+		if (!jobs[name]) throw new Error(`unknown tracker ${name}`);
+		await jobs[name]();
+	}
+} finally {
+	await sql.end();
+}