Stan Runge
7aeb31c46a
All checks were successful
Deploy production / deploy (push) Successful in 1m9s
430 lines
16 KiB
TypeScript
430 lines
16 KiB
TypeScript
// Precomputes "how active is this entry in country X" for the four spotting pokedexes
|
|
// (/trackers/cars, planes, trains, wildlife) into <schema>.activity (item, area, score),
|
|
// so the dex can sort by most active in NL / Germany / Europe ... with one indexed join.
|
|
//
|
|
// DB_URL=... bun scripts/spotters-activity.ts <cars|planes|trains|wildlife|all> [--dry-run]
|
|
//
|
|
// Scores are only comparable within one area (higher = more active there):
|
|
// - cars: NL = cars registered in NL (RDW). US = EPA-tested model years and variants.
|
|
// The brand's home country = overall popularity (NL regs + 100 x US variants).
|
|
// Other countries have no registration data (gap).
|
|
// - planes: aircraft registered per country with that type (OpenSky aircraft database,
|
|
// matched by maker + model name, then by ICAO type code), plus 25 per
|
|
// operator from that country on Wikidata (P137 -> P17; air forces, airlines).
|
|
// Variants roll up into their family.
|
|
// - trains: 100 if an operator from that country runs it (+25 per extra operator), +300
|
|
// if still in service (+100 unknown), 20 for the country of origin only, small
|
|
// tie-break for number built.
|
|
// - wildlife: research-grade iNaturalist observations per country (same query as the NL
|
|
// count in scripts/wildlife-import.ts); Europe / world use the stored counts.
|
|
// Regions (src/lib/spotting/areas.ts) are the sum of their countries; `world` = all.
|
|
// External calls: OpenSky CSV (1 download), Wikidata SPARQL (~10 queries) and the
|
|
// iNaturalist API (~1 call/s, ~450 calls). Progress on stderr.
|
|
import postgres from 'postgres';
|
|
import { existsSync } from 'node:fs';
|
|
import { REGIONS, COUNTRIES } from '../src/lib/spotting/areas';
|
|
import { countryCode } from './spotters-countries';
|
|
import { chunks, log, sparql } from './planes-wiki';
|
|
import { inat } from './wildlife-inat';
|
|
|
|
type Scores = Map<number, Map<string, number>>;
|
|
const args = process.argv.slice(2);
|
|
const which = args.find((a) => !a.startsWith('--')) ?? 'all';
|
|
const dryRun = args.includes('--dry-run');
|
|
const flag = (name: string) => {
|
|
const i = args.indexOf(name);
|
|
return i >= 0 ? args[i + 1] : undefined;
|
|
};
|
|
|
|
const url = process.env.DB_URL;
|
|
if (!url) throw new Error('Set DB_URL');
|
|
const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} });
|
|
|
|
function add(scores: Scores, id: number, area: string | null, value: number) {
|
|
if (!area || !(value > 0)) return;
|
|
let m = scores.get(id);
|
|
if (!m) scores.set(id, (m = new Map()));
|
|
m.set(area, (m.get(area) ?? 0) + value);
|
|
}
|
|
|
|
/** Adds region rows (sum of countries) unless already present, then replaces the table. */
|
|
async function write(schema: string, column: string, scores: Scores, keepRegions = false) {
|
|
const rows: { id: number; area: string; score: number }[] = [];
|
|
for (const [id, m] of scores) {
|
|
const out = new Map(m);
|
|
for (const r of REGIONS) {
|
|
if (keepRegions && m.has(r.code)) continue;
|
|
let sum = 0;
|
|
for (const [area, v] of m)
|
|
if (/^[a-z]{2}$/.test(area) && (r.code === 'world' || r.countries.includes(area))) sum += v;
|
|
if (sum > 0) out.set(r.code, sum);
|
|
}
|
|
for (const [area, score] of out) rows.push({ id, area, score: Math.round(score * 100) / 100 });
|
|
}
|
|
const areas = new Map<string, number>();
|
|
for (const r of rows) areas.set(r.area, (areas.get(r.area) ?? 0) + 1);
|
|
log(
|
|
`${schema}: ${rows.length} activity rows for ${scores.size} entries; nl ${areas.get('nl') ?? 0}, de ${areas.get('de') ?? 0}, us ${areas.get('us') ?? 0}, europe ${areas.get('europe') ?? 0}, ${areas.size} areas`
|
|
);
|
|
if (dryRun) return;
|
|
await sql.begin(async (tx) => {
|
|
await tx.unsafe(`delete from ${schema}.activity`);
|
|
for (const c of chunks(rows, 5000))
|
|
await tx.unsafe(
|
|
`insert into ${schema}.activity (${column}, area, score)
|
|
select * from unnest($1::int[], $2::text[], $3::real[])`,
|
|
[c.map((r) => r.id), c.map((r) => r.area), c.map((r) => r.score)] as never
|
|
);
|
|
});
|
|
}
|
|
|
|
// ------------------------------------------------------------------ cars
|
|
|
|
async function cars() {
|
|
const models = await sql<
|
|
{ id: number; nl: number; variants: number; years: number; country: string | null }[]
|
|
>`select m.id, m.nl_count as nl, m.variant_count as variants,
|
|
coalesce(cardinality(m.us_years), 0) as years, b.country
|
|
from cars.models m join cars.brands b on b.id = m.brand_id`;
|
|
const scores: Scores = new Map();
|
|
for (const m of models) {
|
|
add(scores, m.id, 'nl', m.nl);
|
|
add(scores, m.id, 'us', m.years * 10 + m.variants);
|
|
const home = countryCode(m.country);
|
|
if (home && home !== 'nl' && home !== 'us')
|
|
add(scores, m.id, home, 1 + m.nl + 100 * m.variants);
|
|
}
|
|
await write('cars', 'model_id', scores);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ trains
|
|
|
|
async function trains() {
|
|
const types = await sql<
|
|
{
|
|
id: number;
|
|
country: string | null;
|
|
built: number | null;
|
|
in_service: boolean | null;
|
|
retired: number | null;
|
|
ops: (string | null)[];
|
|
}[]
|
|
>`select t.id, t.country, t.number_built as built, t.in_service, t.retired_year as retired,
|
|
coalesce(array_agg(o.country) filter (where o.id is not null), '{}') as ops
|
|
from trains.types t
|
|
left join trains.type_operators x on x.type_id = t.id
|
|
left join trains.operators o on o.id = x.operator_id
|
|
group by t.id`;
|
|
const scores: Scores = new Map();
|
|
for (const t of types) {
|
|
// Running there now matters most ("does it run on Dutch rails?"): in service +300,
|
|
// unknown status +100, retired +0 (+50 if retired after 2000).
|
|
const tie = Math.min(t.built ?? 0, 1000) / 100;
|
|
const status =
|
|
t.in_service === true ? 300 : t.in_service == null ? 100 : (t.retired ?? 0) >= 2000 ? 50 : 0;
|
|
const opCountries = new Map<string, number>();
|
|
for (const name of t.ops) {
|
|
const cc = countryCode(name);
|
|
if (cc) opCountries.set(cc, (opCountries.get(cc) ?? 0) + 1);
|
|
}
|
|
for (const [cc, n] of opCountries)
|
|
add(scores, t.id, cc, 100 + 25 * (Math.min(n, 4) - 1) + status + tie);
|
|
const origin = countryCode(t.country);
|
|
if (origin && !opCountries.has(origin)) add(scores, t.id, origin, 20 + tie);
|
|
}
|
|
await write('trains', 'type_id', scores);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ planes
|
|
|
|
/** Minimal CSV reader for OpenSky's single-quoted dump ('' escapes a quote). */
|
|
function* csvRows(text: string, quote = "'") {
|
|
let row: string[] = [];
|
|
let field = '';
|
|
let inQ = false;
|
|
for (let i = 0; i < text.length; i++) {
|
|
const ch = text[i];
|
|
if (inQ) {
|
|
if (ch === quote) {
|
|
if (text[i + 1] === quote) {
|
|
field += quote;
|
|
i++;
|
|
} else inQ = false;
|
|
} else field += ch;
|
|
} else if (ch === quote) inQ = true;
|
|
else if (ch === ',') {
|
|
row.push(field);
|
|
field = '';
|
|
} else if (ch === '\n') {
|
|
row.push(field);
|
|
yield row;
|
|
row = [];
|
|
field = '';
|
|
} else if (ch !== '\r') field += ch;
|
|
}
|
|
if (field || row.length) yield [...row, field];
|
|
}
|
|
|
|
const norm = (s: string) =>
|
|
s
|
|
.toLowerCase()
|
|
.normalize('NFD')
|
|
.replace(/[\u0300-\u036f]/g, '')
|
|
.replace(/[^a-z0-9]+/g, ' ')
|
|
.trim();
|
|
const GENERIC = /\b(family|series|aircraft|airliner|helicopter|classic|next generation)\b/g;
|
|
|
|
async function planes() {
|
|
const csvPath = flag('--opensky') ?? '/tmp/spotters/opensky.csv';
|
|
if (!existsSync(csvPath)) {
|
|
log(`planes: downloading the OpenSky aircraft database to ${csvPath}`);
|
|
const res = await fetch(
|
|
'https://s3.opensky-network.org/data-samples/metadata/aircraft-database-complete-2025-08.csv'
|
|
);
|
|
if (!res.ok) throw new Error(`OpenSky download: HTTP ${res.status}`);
|
|
await Bun.write(csvPath, await res.arrayBuffer());
|
|
}
|
|
const types = await sql<
|
|
{
|
|
id: number;
|
|
family: number | null;
|
|
name: string;
|
|
maker: string;
|
|
makers: string[];
|
|
icao: string[];
|
|
wikidata: string;
|
|
current: boolean;
|
|
}[]
|
|
>`select t.id, t.family_id as family, t.name, m.name as maker, t.makers, t.icao, t.wikidata,
|
|
(t.status in ('in service', 'in production') or (t.retired_year is null and coalesce(t.introduced_year, t.first_flight_year, 0) >= 1960)) as current
|
|
from planes.types t join planes.makers m on m.id = t.maker_id`;
|
|
|
|
// Candidate keys per maker word: the type's name, without generic words, and its
|
|
// leading code part ("PA-28 Cherokee" -> "pa 28", "F-16 Fighting Falcon" -> "f 16").
|
|
type Cand = { id: number; key: string; exact: boolean; family: number | null };
|
|
const byMakerWord = new Map<string, Cand[]>();
|
|
for (const t of types) {
|
|
const keys = new Set<string>();
|
|
const n = norm(t.name);
|
|
keys.add(n);
|
|
keys.add(n.replace(GENERIC, ' ').replace(/\s+/g, ' ').trim());
|
|
const toks = n.split(' ');
|
|
const codeEnd = toks.findIndex((w, i) => i > 0 && /^[a-z]{3,}$/.test(w));
|
|
if (codeEnd > 0 && toks.slice(0, codeEnd).some((w) => /\d/.test(w)))
|
|
keys.add(toks.slice(0, codeEnd).join(' '));
|
|
const words = new Set([t.maker, ...t.makers].map((m) => norm(m).split(' ')[0]).filter(Boolean));
|
|
for (const key of keys) {
|
|
if (key.length < 2 || (!/\d/.test(key) && key.length < 4)) continue;
|
|
for (const w of words) {
|
|
if (!byMakerWord.has(w)) byMakerWord.set(w, []);
|
|
byMakerWord.get(w)!.push({ id: t.id, key, exact: key === n, family: t.family });
|
|
}
|
|
}
|
|
}
|
|
const matchModel = (manufacturer: string, model: string): number | null => {
|
|
const cands = byMakerWord.get(norm(manufacturer).split(' ')[0]);
|
|
if (!cands) return null;
|
|
const m = norm(model);
|
|
let best: Cand | null = null;
|
|
for (const c of cands) {
|
|
if (!(m === c.key || (m.startsWith(c.key) && !/\d/.test(m[c.key.length] ?? '')))) continue;
|
|
if (
|
|
!best ||
|
|
c.key.length > best.key.length ||
|
|
(c.key.length === best.key.length && best.family !== null && c.family === null)
|
|
)
|
|
best = c;
|
|
}
|
|
return best?.id ?? null;
|
|
};
|
|
|
|
log('planes: reading OpenSky aircraft');
|
|
const text = await Bun.file(csvPath).text();
|
|
const rows = csvRows(text);
|
|
const head = rows.next().value as string[];
|
|
const col = (n: string) => head.indexOf(n);
|
|
const [iCountry, iMan, iModel, iType] = [
|
|
col('country'),
|
|
col('manufacturerName'),
|
|
col('model'),
|
|
col('typecode')
|
|
];
|
|
const direct = new Map<string, Map<string, number>>(); // "id" -> country -> n
|
|
const byCode = new Map<string, Map<string, number>>(); // typecode -> country -> n (unmatched)
|
|
const codeVotes = new Map<string, Map<number, number>>(); // typecode -> type id -> votes
|
|
let total = 0;
|
|
let matched = 0;
|
|
for (const r of rows) {
|
|
if (r.length < head.length) continue;
|
|
const cc = countryCode(r[iCountry]);
|
|
if (!cc) continue;
|
|
total++;
|
|
const code = r[iType]?.trim().toUpperCase();
|
|
const id = r[iMan] && r[iModel] ? matchModel(r[iMan], r[iModel]) : null;
|
|
if (id) {
|
|
matched++;
|
|
const key = String(id);
|
|
if (!direct.has(key)) direct.set(key, new Map());
|
|
direct.get(key)!.set(cc, (direct.get(key)!.get(cc) ?? 0) + 1);
|
|
if (code) {
|
|
if (!codeVotes.has(code)) codeVotes.set(code, new Map());
|
|
codeVotes.get(code)!.set(id, (codeVotes.get(code)!.get(id) ?? 0) + 1);
|
|
}
|
|
} else if (code) {
|
|
if (!byCode.has(code)) byCode.set(code, new Map());
|
|
byCode.get(code)!.set(cc, (byCode.get(code)!.get(cc) ?? 0) + 1);
|
|
}
|
|
}
|
|
// Unmatched aircraft follow their ICAO type code: the type's own ICAO list first, else
|
|
// the type most matched aircraft with that code went to (needs a clear majority).
|
|
const icaoOwner = new Map<string, number>();
|
|
for (const t of types) for (const c of t.icao) if (!icaoOwner.has(c)) icaoOwner.set(c, t.id);
|
|
let viaCode = 0;
|
|
for (const [code, counts] of byCode) {
|
|
let owner = icaoOwner.get(code) ?? null;
|
|
const votes = codeVotes.get(code);
|
|
if (!owner && votes) {
|
|
const sorted = [...votes].sort((a, b) => b[1] - a[1]);
|
|
const sum = sorted.reduce((s, [, n]) => s + n, 0);
|
|
if (sorted[0][1] >= 3 && sorted[0][1] / sum >= 0.6) owner = sorted[0][0];
|
|
}
|
|
if (!owner) continue;
|
|
const key = String(owner);
|
|
if (!direct.has(key)) direct.set(key, new Map());
|
|
for (const [cc, n] of counts) {
|
|
direct.get(key)!.set(cc, (direct.get(key)!.get(cc) ?? 0) + n);
|
|
viaCode += n;
|
|
}
|
|
}
|
|
log(
|
|
`planes: ${total} aircraft with a country, ${matched} matched by model, ${viaCode} by type code`
|
|
);
|
|
|
|
const scores: Scores = new Map();
|
|
// Roll registrations up the family tree (737 MAX 8 -> 737 MAX -> 737).
|
|
const parent = new Map(types.map((t) => [t.id, t.family]));
|
|
for (const [key, m] of direct) {
|
|
let id: number | null = Number(key);
|
|
for (let depth = 0; id && depth < 6; depth++, id = parent.get(id) ?? null)
|
|
for (const [cc, n] of m) add(scores, id, cc, n);
|
|
}
|
|
// Wikidata operators (P137) and their countries, as two cheap queries (the joined
|
|
// version times out on the public endpoint).
|
|
const byQid = new Map(types.map((t) => [t.wikidata, t.id]));
|
|
let ops = 0;
|
|
let failed = 0;
|
|
const opsOf = new Map<string, Set<string>>();
|
|
const fail = (e: unknown) => {
|
|
failed++;
|
|
log(`planes: operator chunk failed (${String(e).slice(0, 80)})`);
|
|
return [];
|
|
};
|
|
for (const c of chunks(
|
|
[...byQid.keys()].filter((q): q is string => !!q),
|
|
200
|
|
)) {
|
|
// A failing chunk (query timeout) only loses its operator bonus; registrations stay.
|
|
const res = await sparql<{ item: string; op: string }>(`
|
|
SELECT ?item ?op WHERE { VALUES ?item { ${c.map((q) => `wd:${q}`).join(' ')} } ?item wdt:P137 ?op . }`).catch(
|
|
fail
|
|
);
|
|
for (const r of res) {
|
|
const item = r.item.split('/').at(-1)!;
|
|
const set = opsOf.get(item) ?? new Set<string>();
|
|
set.add(r.op.split('/').at(-1)!);
|
|
opsOf.set(item, set);
|
|
}
|
|
}
|
|
const isoOf = new Map<string, Set<string>>();
|
|
for (const c of chunks([...new Set([...opsOf.values()].flatMap((s) => [...s]))], 300)) {
|
|
const res = await sparql<{ op: string; iso: string }>(`
|
|
SELECT ?op ?iso WHERE { VALUES ?op { ${c.map((q) => `wd:${q}`).join(' ')} } ?op wdt:P17 ?country . ?country wdt:P297 ?iso . }`).catch(
|
|
fail
|
|
);
|
|
for (const r of res) {
|
|
const op = r.op.split('/').at(-1)!;
|
|
const set = isoOf.get(op) ?? new Set<string>();
|
|
set.add(r.iso.toLowerCase());
|
|
isoOf.set(op, set);
|
|
}
|
|
}
|
|
// Historic operators (a 1918 air force flying the DH.9) say little about what you can
|
|
// spot today: types that are retired or older than 1960 get a fifth of the bonus.
|
|
const current = new Map(types.map((t) => [t.id, t.current]));
|
|
for (const [item, opSet] of opsOf) {
|
|
const id = byQid.get(item);
|
|
if (!id) continue;
|
|
const weight = current.get(id) ? 25 : 5;
|
|
const perCountry = new Map<string, number>();
|
|
for (const op of opSet)
|
|
for (const cc of isoOf.get(op) ?? []) perCountry.set(cc, (perCountry.get(cc) ?? 0) + 1);
|
|
for (const [cc, n] of perCountry) {
|
|
add(scores, id, cc, weight * Math.min(n, 6));
|
|
ops++;
|
|
}
|
|
}
|
|
log(`planes: ${ops} type x country operator links from Wikidata (${failed} chunks failed)`);
|
|
await write('planes', 'type_id', scores);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ wildlife
|
|
|
|
async function wildlife() {
|
|
const species = await sql<{ id: number; inat: number; nl: number; eu: number; world: number }[]>`
|
|
select id, inat_id as inat, nl_obs as nl, europe_obs as eu, world_obs as world from wildlife.species`;
|
|
const scores: Scores = new Map();
|
|
for (const s of species) {
|
|
add(scores, s.id, 'nl', s.nl);
|
|
add(scores, s.id, 'europe', s.eu);
|
|
add(scores, s.id, 'world', s.world);
|
|
}
|
|
const byInat = new Map(species.map((s) => [s.inat, s.id]));
|
|
const countries = [...new Set([...COUNTRIES, 'lu'])].filter((c) => c !== 'nl');
|
|
const dn = new Intl.DisplayNames(['en'], { type: 'region' });
|
|
for (const cc of countries) {
|
|
const name =
|
|
{
|
|
us: 'United States',
|
|
gb: 'United Kingdom',
|
|
cz: 'Czech Republic',
|
|
tr: 'Türkiye',
|
|
kr: 'South Korea'
|
|
}[cc] ?? dn.of(cc.toUpperCase())!;
|
|
const found = await inat<{
|
|
results: { id: number; name: string; admin_level: number | null }[];
|
|
}>(`/places/autocomplete?q=${encodeURIComponent(name)}&per_page=20`);
|
|
const place =
|
|
found.results.find((p) => p.admin_level === 0 && norm(p.name) === norm(name)) ??
|
|
found.results.find((p) => p.admin_level === 0);
|
|
if (!place) {
|
|
log(`wildlife: no iNaturalist place for ${cc} (${name})`);
|
|
continue;
|
|
}
|
|
let n = 0;
|
|
for (const batch of chunks([...byInat.keys()], 400)) {
|
|
const res = await inat(
|
|
`/observations/species_counts?place_id=${place.id}&taxon_id=${batch.join(',')}&captive=false&quality_grade=research&per_page=500`
|
|
);
|
|
for (const r of res.results) {
|
|
const id = byInat.get(r.taxon.id);
|
|
if (id && r.count > 0) {
|
|
add(scores, id, cc, r.count);
|
|
n++;
|
|
}
|
|
}
|
|
}
|
|
log(`wildlife: ${cc} (${place.name}, place ${place.id}): ${n} species`);
|
|
}
|
|
await write('wildlife', 'species_id', scores, true);
|
|
}
|
|
|
|
const jobs: Record<string, () => Promise<void>> = { cars, trains, planes, wildlife };
|
|
try {
|
|
for (const name of which === 'all' ? Object.keys(jobs) : which.split(',')) {
|
|
if (!jobs[name]) throw new Error(`unknown tracker ${name}`);
|
|
await jobs[name]();
|
|
}
|
|
} finally {
|
|
await sql.end();
|
|
}
|