diff --git a/scripts/cars-refresh.sh b/scripts/cars-refresh.sh index 7819b05..5f63cd1 100755 --- a/scripts/cars-refresh.sh +++ b/scripts/cars-refresh.sh @@ -1,14 +1,18 @@ #!/usr/bin/env bash -# Weekly-ish refresh of the car pokedex (/trackers/cars) on stan-vm. +# Weekly-ish refresh of the car pokedex (/trackers/cars) on stan-ovh. # -# ssh stan-vm 'cd ~/dev/website-cars && git fetch -q && git checkout -q --detach origin/main \ +# ssh stan@37.187.157.191 'cd ~/dev/website-cars && git fetch -q && git checkout -q --detach origin/main \ # && bun install --frozen-lockfile >/dev/null && scripts/cars-refresh.sh' # # 1. bun scripts/cars-import.ts --report-new rebuilds brands/models/variants from RDW + # fueleconomy.gov and notes which brands/models did not exist before. # 2. bun scripts/cars-photos.ts --missing finds Wikimedia Commons photos for models # that have none yet (new ones included). -# stdout: one JSON report {generatedAt, totals, newBrands, newModels[] (with photo)}. +# 3. bun scripts/spotters-activity.ts cars refreshes the per-country activity scores +# ("most active in NL" sort; see that script). +# stdout: one JSON report {generatedAt, totals, newBrands, newModels[] (with photo)}, plus `nl` per new +# entry and `newInNl` (new entries that are active in the Netherlands, see +# scripts/spotters-report-nl.ts). --report-new is accepted (the report is always printed). # Progress goes to stderr. DB_URL is read from the running app container and never printed. set -euo pipefail cd "$(dirname "$0")/.." @@ -19,8 +23,12 @@ if [ -z "${DB_URL:-}" ]; then export DB_URL fi +args=() +for a in "$@"; do [ "$a" = "--report-new" ] || args+=("$a"); done + report=$(mktemp /tmp/cars-report.XXXXXX.json) trap 'rm -f "$report"' EXIT -bun scripts/cars-import.ts --report-new "$@" >"$report" -bun scripts/cars-photos.ts --missing --enrich "$report" --fetch +bun scripts/cars-import.ts --report-new "${args[@]}" >"$report" +bun scripts/spotters-activity.ts cars >&2 +bun scripts/cars-photos.ts --missing --enrich "$report" --fetch | bun scripts/spotters-report-nl.ts cars diff --git a/scripts/planes-refresh.sh b/scripts/planes-refresh.sh index db33c75..56dbb4d 100755 --- a/scripts/planes-refresh.sh +++ b/scripts/planes-refresh.sh @@ -9,7 +9,11 @@ # are stable (types are matched by Wikidata id). # 2. bun scripts/planes-photos.ts --missing finds Wikimedia Commons photos for types that # have none yet (new ones included) and maker logos that are missing. -# stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}. +# 3. bun scripts/spotters-activity.ts planes refreshes the per-country activity scores +# ("most active in NL" sort; OpenSky registrations, see that script). +# stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}, plus `nl` per new +# entry and `newInNl` (new entries that are active in the Netherlands, see +# scripts/spotters-report-nl.ts). --report-new is accepted (the report is always printed). # Progress goes to stderr. DB_URL is read from the running app container and never printed. set -euo pipefail cd "$(dirname "$0")/.." @@ -20,8 +24,12 @@ if [ -z "${DB_URL:-}" ]; then export DB_URL fi +args=() +for a in "$@"; do [ "$a" = "--report-new" ] || args+=("$a"); done + report=$(mktemp /tmp/planes-report.XXXXXX.json) trap 'rm -f "$report"' EXIT -bun scripts/planes-import.ts --report-new "$@" >"$report" -bun scripts/planes-photos.ts --missing --enrich "$report" --fetch +bun scripts/planes-import.ts --report-new "${args[@]}" >"$report" +bun scripts/spotters-activity.ts planes >&2 +bun scripts/planes-photos.ts --missing --enrich "$report" --fetch | bun scripts/spotters-report-nl.ts planes diff --git a/scripts/spotters-activity.ts b/scripts/spotters-activity.ts new file mode 100644 index 0000000..4df7032 --- /dev/null +++ b/scripts/spotters-activity.ts @@ -0,0 +1,397 @@ +// Precomputes "how active is this entry in country X" for the four spotting pokedexes +// (/trackers/cars, planes, trains, wildlife) into .activity (item, area, score), +// so the dex can sort by most active in NL / Germany / Europe ... with one indexed join. +// +// DB_URL=... bun scripts/spotters-activity.ts [--dry-run] +// +// Scores are only comparable within one area (higher = more active there): +// - cars: NL = cars registered in NL (RDW). US = EPA-tested model years and variants. +// The brand's home country = overall popularity (NL regs + 100 x US variants). +// Other countries have no registration data (gap). +// - planes: aircraft registered per country with that type (OpenSky aircraft database, +// matched by maker + model name, then by ICAO type code), plus 25 per +// operator from that country on Wikidata (P137 -> P17; air forces, airlines). +// Variants roll up into their family. +// - trains: 100 if an operator from that country runs it (+25 per extra operator), +300 +// if still in service (+100 unknown), 20 for the country of origin only, small +// tie-break for number built. +// - wildlife: research-grade iNaturalist observations per country (same query as the NL +// count in scripts/wildlife-import.ts); Europe / world use the stored counts. +// Regions (src/lib/spotting/areas.ts) are the sum of their countries; `world` = all. +// External calls: OpenSky CSV (1 download), Wikidata SPARQL (~10 queries) and the +// iNaturalist API (~1 call/s, ~450 calls). Progress on stderr. +import postgres from 'postgres'; +import { existsSync } from 'node:fs'; +import { REGIONS, COUNTRIES } from '../src/lib/spotting/areas'; +import { countryCode } from './spotters-countries'; +import { chunks, log, sparql } from './planes-wiki'; +import { inat } from './wildlife-inat'; + +type Scores = Map>; +const args = process.argv.slice(2); +const which = args.find((a) => !a.startsWith('--')) ?? 'all'; +const dryRun = args.includes('--dry-run'); +const flag = (name: string) => { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : undefined; +}; + +const url = process.env.DB_URL; +if (!url) throw new Error('Set DB_URL'); +const sql = postgres(url, { max: 2, connect_timeout: 10, onnotice: () => {} }); + +function add(scores: Scores, id: number, area: string | null, value: number) { + if (!area || !(value > 0)) return; + let m = scores.get(id); + if (!m) scores.set(id, (m = new Map())); + m.set(area, (m.get(area) ?? 0) + value); +} + +/** Adds region rows (sum of countries) unless already present, then replaces the table. */ +async function write(schema: string, column: string, scores: Scores, keepRegions = false) { + const rows: { id: number; area: string; score: number }[] = []; + for (const [id, m] of scores) { + const out = new Map(m); + for (const r of REGIONS) { + if (keepRegions && m.has(r.code)) continue; + let sum = 0; + for (const [area, v] of m) + if (/^[a-z]{2}$/.test(area) && (r.code === 'world' || r.countries.includes(area))) sum += v; + if (sum > 0) out.set(r.code, sum); + } + for (const [area, score] of out) rows.push({ id, area, score: Math.round(score * 100) / 100 }); + } + const areas = new Map(); + for (const r of rows) areas.set(r.area, (areas.get(r.area) ?? 0) + 1); + log( + `${schema}: ${rows.length} activity rows for ${scores.size} entries; nl ${areas.get('nl') ?? 0}, de ${areas.get('de') ?? 0}, us ${areas.get('us') ?? 0}, europe ${areas.get('europe') ?? 0}, ${areas.size} areas` + ); + if (dryRun) return; + await sql.begin(async (tx) => { + await tx.unsafe(`delete from ${schema}.activity`); + for (const c of chunks(rows, 5000)) + await tx.unsafe( + `insert into ${schema}.activity (${column}, area, score) + select * from unnest($1::int[], $2::text[], $3::real[])`, + [c.map((r) => r.id), c.map((r) => r.area), c.map((r) => r.score)] as never + ); + }); +} + +// ------------------------------------------------------------------ cars + +async function cars() { + const models = await sql< + { id: number; nl: number; variants: number; years: number; country: string | null }[] + >`select m.id, m.nl_count as nl, m.variant_count as variants, + coalesce(cardinality(m.us_years), 0) as years, b.country + from cars.models m join cars.brands b on b.id = m.brand_id`; + const scores: Scores = new Map(); + for (const m of models) { + add(scores, m.id, 'nl', m.nl); + add(scores, m.id, 'us', m.years * 10 + m.variants); + const home = countryCode(m.country); + if (home && home !== 'nl' && home !== 'us') + add(scores, m.id, home, 1 + m.nl + 100 * m.variants); + } + await write('cars', 'model_id', scores); +} + +// ------------------------------------------------------------------ trains + +async function trains() { + const types = await sql< + { + id: number; + country: string | null; + built: number | null; + in_service: boolean | null; + retired: number | null; + ops: (string | null)[]; + }[] + >`select t.id, t.country, t.number_built as built, t.in_service, t.retired_year as retired, + coalesce(array_agg(o.country) filter (where o.id is not null), '{}') as ops + from trains.types t + left join trains.type_operators x on x.type_id = t.id + left join trains.operators o on o.id = x.operator_id + group by t.id`; + const scores: Scores = new Map(); + for (const t of types) { + // Running there now matters most ("does it run on Dutch rails?"): in service +300, + // unknown status +100, retired +0 (+50 if retired after 2000). + const tie = Math.min(t.built ?? 0, 1000) / 100; + const status = + t.in_service === true ? 300 : t.in_service == null ? 100 : (t.retired ?? 0) >= 2000 ? 50 : 0; + const opCountries = new Map(); + for (const name of t.ops) { + const cc = countryCode(name); + if (cc) opCountries.set(cc, (opCountries.get(cc) ?? 0) + 1); + } + for (const [cc, n] of opCountries) + add(scores, t.id, cc, 100 + 25 * (Math.min(n, 4) - 1) + status + tie); + const origin = countryCode(t.country); + if (origin && !opCountries.has(origin)) add(scores, t.id, origin, 20 + tie); + } + await write('trains', 'type_id', scores); +} + +// ------------------------------------------------------------------ planes + +/** Minimal CSV reader for OpenSky's single-quoted dump ('' escapes a quote). */ +function* csvRows(text: string, quote = "'") { + let row: string[] = []; + let field = ''; + let inQ = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inQ) { + if (ch === quote) { + if (text[i + 1] === quote) { + field += quote; + i++; + } else inQ = false; + } else field += ch; + } else if (ch === quote) inQ = true; + else if (ch === ',') { + row.push(field); + field = ''; + } else if (ch === '\n') { + row.push(field); + yield row; + row = []; + field = ''; + } else if (ch !== '\r') field += ch; + } + if (field || row.length) yield [...row, field]; +} + +const norm = (s: string) => + s + .toLowerCase() + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .replace(/[^a-z0-9]+/g, ' ') + .trim(); +const GENERIC = /\b(family|series|aircraft|airliner|helicopter|classic|next generation)\b/g; + +async function planes() { + const csvPath = flag('--opensky') ?? '/tmp/spotters/opensky.csv'; + if (!existsSync(csvPath)) { + log(`planes: downloading the OpenSky aircraft database to ${csvPath}`); + const res = await fetch( + 'https://s3.opensky-network.org/data-samples/metadata/aircraft-database-complete-2025-08.csv' + ); + if (!res.ok) throw new Error(`OpenSky download: HTTP ${res.status}`); + await Bun.write(csvPath, await res.arrayBuffer()); + } + const types = await sql< + { + id: number; + family: number | null; + name: string; + maker: string; + makers: string[]; + icao: string[]; + wikidata: string; + }[] + >`select t.id, t.family_id as family, t.name, m.name as maker, t.makers, t.icao, t.wikidata + from planes.types t join planes.makers m on m.id = t.maker_id`; + + // Candidate keys per maker word: the type's name, without generic words, and its + // leading code part ("PA-28 Cherokee" -> "pa 28", "F-16 Fighting Falcon" -> "f 16"). + type Cand = { id: number; key: string; exact: boolean; family: number | null }; + const byMakerWord = new Map(); + for (const t of types) { + const keys = new Set(); + const n = norm(t.name); + keys.add(n); + keys.add(n.replace(GENERIC, ' ').replace(/\s+/g, ' ').trim()); + const toks = n.split(' '); + const codeEnd = toks.findIndex((w, i) => i > 0 && /^[a-z]{3,}$/.test(w)); + if (codeEnd > 0 && toks.slice(0, codeEnd).some((w) => /\d/.test(w))) + keys.add(toks.slice(0, codeEnd).join(' ')); + const words = new Set([t.maker, ...t.makers].map((m) => norm(m).split(' ')[0]).filter(Boolean)); + for (const key of keys) { + if (key.length < 2 || (!/\d/.test(key) && key.length < 4)) continue; + for (const w of words) { + if (!byMakerWord.has(w)) byMakerWord.set(w, []); + byMakerWord.get(w)!.push({ id: t.id, key, exact: key === n, family: t.family }); + } + } + } + const matchModel = (manufacturer: string, model: string): number | null => { + const cands = byMakerWord.get(norm(manufacturer).split(' ')[0]); + if (!cands) return null; + const m = norm(model); + let best: Cand | null = null; + for (const c of cands) { + if (!(m === c.key || (m.startsWith(c.key) && !/\d/.test(m[c.key.length] ?? '')))) continue; + if ( + !best || + c.key.length > best.key.length || + (c.key.length === best.key.length && best.family !== null && c.family === null) + ) + best = c; + } + return best?.id ?? null; + }; + + log('planes: reading OpenSky aircraft'); + const text = await Bun.file(csvPath).text(); + const rows = csvRows(text); + const head = rows.next().value as string[]; + const col = (n: string) => head.indexOf(n); + const [iCountry, iMan, iModel, iType] = [ + col('country'), + col('manufacturerName'), + col('model'), + col('typecode') + ]; + const direct = new Map>(); // "id" -> country -> n + const byCode = new Map>(); // typecode -> country -> n (unmatched) + const codeVotes = new Map>(); // typecode -> type id -> votes + let total = 0; + let matched = 0; + for (const r of rows) { + if (r.length < head.length) continue; + const cc = countryCode(r[iCountry]); + if (!cc) continue; + total++; + const code = r[iType]?.trim().toUpperCase(); + const id = r[iMan] && r[iModel] ? matchModel(r[iMan], r[iModel]) : null; + if (id) { + matched++; + const key = String(id); + if (!direct.has(key)) direct.set(key, new Map()); + direct.get(key)!.set(cc, (direct.get(key)!.get(cc) ?? 0) + 1); + if (code) { + if (!codeVotes.has(code)) codeVotes.set(code, new Map()); + codeVotes.get(code)!.set(id, (codeVotes.get(code)!.get(id) ?? 0) + 1); + } + } else if (code) { + if (!byCode.has(code)) byCode.set(code, new Map()); + byCode.get(code)!.set(cc, (byCode.get(code)!.get(cc) ?? 0) + 1); + } + } + // Unmatched aircraft follow their ICAO type code: the type's own ICAO list first, else + // the type most matched aircraft with that code went to (needs a clear majority). + const icaoOwner = new Map(); + for (const t of types) for (const c of t.icao) if (!icaoOwner.has(c)) icaoOwner.set(c, t.id); + let viaCode = 0; + for (const [code, counts] of byCode) { + let owner = icaoOwner.get(code) ?? null; + const votes = codeVotes.get(code); + if (!owner && votes) { + const sorted = [...votes].sort((a, b) => b[1] - a[1]); + const sum = sorted.reduce((s, [, n]) => s + n, 0); + if (sorted[0][1] >= 3 && sorted[0][1] / sum >= 0.6) owner = sorted[0][0]; + } + if (!owner) continue; + const key = String(owner); + if (!direct.has(key)) direct.set(key, new Map()); + for (const [cc, n] of counts) { + direct.get(key)!.set(cc, (direct.get(key)!.get(cc) ?? 0) + n); + viaCode += n; + } + } + log( + `planes: ${total} aircraft with a country, ${matched} matched by model, ${viaCode} by type code` + ); + + const scores: Scores = new Map(); + // Roll registrations up the family tree (737 MAX 8 -> 737 MAX -> 737). + const parent = new Map(types.map((t) => [t.id, t.family])); + for (const [key, m] of direct) { + let id: number | null = Number(key); + for (let depth = 0; id && depth < 6; depth++, id = parent.get(id) ?? null) + for (const [cc, n] of m) add(scores, id, cc, n); + } + // Wikidata operators (P137) and their countries. + const byQid = new Map(types.map((t) => [t.wikidata, t.id])); + let ops = 0; + let failed = 0; + for (const c of chunks( + [...byQid.keys()].filter((q): q is string => !!q), + 100 + )) { + // A failing chunk (query timeout) only loses its operator bonus; registrations stay. + const res = await sparql<{ item: string; iso: string; n: string }>(` + SELECT ?item ?iso (COUNT(DISTINCT ?op) AS ?n) WHERE { + VALUES ?item { ${c.map((q) => `wd:${q}`).join(' ')} } + ?item wdt:P137 ?op . ?op wdt:P17 ?country . ?country wdt:P297 ?iso . + } GROUP BY ?item ?iso`).catch((e: unknown) => { + failed++; + log(`planes: operator chunk failed (${String(e).slice(0, 80)})`); + return []; + }); + for (const r of res) { + const id = byQid.get(r.item.split('/').at(-1)!); + if (!id) continue; + add(scores, id, r.iso.toLowerCase(), 25 * Math.min(Number(r.n) || 1, 6)); + ops++; + } + } + log(`planes: ${ops} type x country operator links from Wikidata (${failed} chunks failed)`); + await write('planes', 'type_id', scores); +} + +// ------------------------------------------------------------------ wildlife + +async function wildlife() { + const species = await sql<{ id: number; inat: number; nl: number; eu: number; world: number }[]>` + select id, inat_id as inat, nl_obs as nl, europe_obs as eu, world_obs as world from wildlife.species`; + const scores: Scores = new Map(); + for (const s of species) { + add(scores, s.id, 'nl', s.nl); + add(scores, s.id, 'europe', s.eu); + add(scores, s.id, 'world', s.world); + } + const byInat = new Map(species.map((s) => [s.inat, s.id])); + const countries = [...new Set([...COUNTRIES, 'lu'])].filter((c) => c !== 'nl'); + const dn = new Intl.DisplayNames(['en'], { type: 'region' }); + for (const cc of countries) { + const name = + { + us: 'United States', + gb: 'United Kingdom', + cz: 'Czech Republic', + tr: 'Turkey', + kr: 'South Korea' + }[cc] ?? dn.of(cc.toUpperCase())!; + const found = await inat<{ + results: { id: number; name: string; admin_level: number | null }[]; + }>(`/places/autocomplete?q=${encodeURIComponent(name)}&per_page=20`); + const place = + found.results.find((p) => p.admin_level === 0 && norm(p.name) === norm(name)) ?? + found.results.find((p) => p.admin_level === 0); + if (!place) { + log(`wildlife: no iNaturalist place for ${cc} (${name})`); + continue; + } + let n = 0; + for (const batch of chunks([...byInat.keys()], 400)) { + const res = await inat( + `/observations/species_counts?place_id=${place.id}&taxon_id=${batch.join(',')}&captive=false&quality_grade=research&per_page=500` + ); + for (const r of res.results) { + const id = byInat.get(r.taxon.id); + if (id && r.count > 0) { + add(scores, id, cc, r.count); + n++; + } + } + } + log(`wildlife: ${cc} (${place.name}, place ${place.id}): ${n} species`); + } + await write('wildlife', 'species_id', scores, true); +} + +const jobs: Record Promise> = { cars, trains, planes, wildlife }; +try { + for (const name of which === 'all' ? Object.keys(jobs) : which.split(',')) { + if (!jobs[name]) throw new Error(`unknown tracker ${name}`); + await jobs[name](); + } +} finally { + await sql.end(); +} diff --git a/scripts/spotters-countries.ts b/scripts/spotters-countries.ts new file mode 100644 index 0000000..d2bfeeb --- /dev/null +++ b/scripts/spotters-countries.ts @@ -0,0 +1,172 @@ +// Country name (as found in Wikidata labels, OpenSky, the train/plane/car data) -> lowercase +// ISO 3166-1 alpha-2 code, for scripts/spotters-activity.ts. Historic states map to their +// main successor (Deutsches Reich -> de, Soviet Union -> ru, Yugoslavia -> rs). + +const ALIASES: Record = { + 'united states of america': 'us', + usa: 'us', + 'u.s.': 'us', + 'united states': 'us', + "people's republic of china": 'cn', + 'republic of china': 'tw', + 'taiwan (republic of china)': 'tw', + 'taiwan under japanese rule': 'tw', + manchukuo: 'cn', + 'hong kong': 'hk', + macau: 'mo', + macao: 'mo', + 'russian federation': 'ru', + 'soviet union': 'ru', + ussr: 'ru', + 'soviet union (russia since 1991)': 'ru', + 'russian empire': 'ru', + 'russian sfsr': 'ru', + 'deutsches reich': 'de', + 'german reich': 'de', + 'german empire': 'de', + 'nazi germany': 'de', + 'east germany': 'de', + 'west germany': 'de', + 'weimar republic': 'de', + 'weimarer republik': 'de', + 'deutsches kaiserreich': 'de', + 'allied-occupied germany': 'de', + 'grand duchy of baden': 'de', + 'territory of the saar basin': 'de', + 'saar protectorate': 'de', + 'kingdom of bavaria': 'de', + 'kingdom of prussia': 'de', + 'kingdom of saxony': 'de', + 'kingdom of württemberg': 'de', + 'german reich (nazi period) deutsches reich': 'de', + germa: 'de', + czechoslovakia: 'cz', + 'first czechoslovak republic': 'cz', + 'czech republic': 'cz', + 'protectorate of bohemia and moravia': 'cz', + 'czechoslovak socialist republic': 'cz', + 'korea under japanese rule': 'kr', + 'korean empire': 'kr', + 'south korea': 'kr', + 'north korea': 'kp', + 'republic of korea': 'kr', + "democratic people's republic of korea": 'kp', + 'irish free state': 'ie', + 'republic of ireland': 'ie', + yugoslavia: 'rs', + 'kingdom of yugoslavia': 'rs', + 'socialist federal republic of yugoslavia': 'rs', + 'federal republic of yugoslavia': 'rs', + 'serbia and montenegro': 'rs', + 'empire of japan': 'jp', + 'kingdom of italy': 'it', + 'austria–hungary': 'at', + 'austria-hungary': 'at', + cisleithania: 'at', + 'first republic of austria': 'at', + 'austrian empire': 'at', + 'colony of natal': 'za', + 'south african republic': 'za', + 'orange free state': 'za', + 'cape colony': 'za', + 'union of south africa': 'za', + 'dutch east indies': 'id', + 'kingdom of the netherlands': 'nl', + netherlands: 'nl', + holland: 'nl', + england: 'gb', + scotland: 'gb', + wales: 'gb', + 'northern ireland': 'gb', + 'great britain': 'gb', + 'united kingdom of great britain and ireland': 'gb', + 'united kingdom': 'gb', + uk: 'gb', + tanganyika: 'tz', + 'mandatory palestine': 'il', + 'second polish republic': 'pl', + "polish people's republic": 'pl', + 'polska rzeczpospolita ludowa': 'pl', + 'second spanish republic': 'es', + 'francoist spain': 'es', + 'kingdom of romania': 'ro', + "ukrainian people's republic": 'ua', + 'ukrainian ssr': 'ua', + turkey: 'tr', + türkiye: 'tr', + 'ottoman empire': 'tr', + 'kingdom of hungary': 'hu', + 'kingdom of greece': 'gr', + 'kingdom of bulgaria': 'bg', + 'british raj': 'in', + 'dominion of india': 'in', + ceylon: 'lk', + burma: 'mm', + 'french third republic': 'fr', + 'vichy france': 'fr', + 'french fourth republic': 'fr', + 'kingdom of egypt': 'eg', + rhodesia: 'zw', + 'southern rhodesia': 'zw', + 'northern rhodesia': 'zm', + 'belgian congo': 'cd', + zaire: 'cd', + swaziland: 'sz', + eswatini: 'sz', + persia: 'ir', + iran: 'ir', + syria: 'sy', + vietnam: 'vn', + laos: 'la', + bolivia: 'bo', + venezuela: 've', + tanzania: 'tz', + moldova: 'md', + macedonia: 'mk', + 'north macedonia': 'mk', + 'bosnia and herzegovina': 'ba', + 'ivory coast': 'ci', + "côte d'ivoire": 'ci', + 'cape verde': 'cv', + 'east timor': 'tl', + kosovo: 'xk', + 'isle of man': 'im', + jersey: 'je', + guernsey: 'gg', + 'state of palestine': 'ps', + brunei: 'bn', + 'republic of the congo': 'cg', + 'democratic republic of the congo': 'cd', + 'the gambia': 'gm', + 'the bahamas': 'bs', + 'federated states of micronesia': 'fm', + 'vatican city': 'va', + czechia: 'cz' +}; + +let byName: Map | null = null; + +function table() { + if (byName) return byName; + byName = new Map(); + const dn = new Intl.DisplayNames(['en'], { type: 'region' }); + for (let a = 65; a <= 90; a++) + for (let b = 65; b <= 90; b++) { + const code = String.fromCharCode(a, b); + try { + const name = dn.of(code); + if (name && name !== code) byName.set(name.toLowerCase(), code.toLowerCase()); + } catch { + /* not a region */ + } + } + for (const [k, v] of Object.entries(ALIASES)) byName.set(k, v); + return byName; +} + +/** Country label -> ISO2 (lowercase), or null when unknown / multinational. */ +export function countryCode(name: string | null | undefined): string | null { + if (!name) return null; + const key = name.trim().toLowerCase().replace(/\s+/g, ' '); + return table().get(key) ?? table().get(key.replace(/^the /, '')) ?? null; +} diff --git a/scripts/spotters-report-nl.ts b/scripts/spotters-report-nl.ts new file mode 100644 index 0000000..db1bbd4 --- /dev/null +++ b/scripts/spotters-report-nl.ts @@ -0,0 +1,56 @@ +// Adds Netherlands relevance to a refresh report (stdin JSON -> stdout JSON), so a "new +// entries" ping can be limited to things that actually show up in NL. +// +// bun scripts/cars-import.ts --report-new | bun scripts/spotters-report-nl.ts cars +// +// Every new item (newModels / newTypes / newSpecies, matched by the /trackers/... url) gets +// `nl` (its activity score in NL from .activity, 0 when none) and the report gets +// `newInNl`: the same items with nl > 0, most active first. Needs DB_URL. Run after +// scripts/spotters-activity.ts so new entries already have scores. +import postgres from 'postgres'; + +const tracker = process.argv[2]; +const CFG = { + cars: { + key: 'newModels', + sql: "select a.score from cars.activity a join cars.models m on m.id = a.model_id join cars.brands b on b.id = m.brand_id where a.area = 'nl' and b.slug = $1 and m.slug = $2" + }, + planes: { + key: 'newTypes', + sql: "select a.score from planes.activity a join planes.types t on t.id = a.type_id join planes.makers m on m.id = t.maker_id where a.area = 'nl' and m.slug = $1 and t.slug = $2" + }, + trains: { + key: 'newTypes', + sql: "select a.score from trains.activity a join trains.types t on t.id = a.type_id join trains.makers m on m.id = t.maker_id where a.area = 'nl' and m.slug = $1 and t.slug = $2" + }, + wildlife: { + key: 'newSpecies', + sql: "select a.score from wildlife.activity a join wildlife.species s on s.id = a.species_id join wildlife.groups g on g.id = s.group_id where a.area = 'nl' and g.slug = $1 and s.slug = $2" + } +} as const; +if (!(tracker in CFG)) { + console.error( + 'usage: bun scripts/spotters-report-nl.ts < report.json' + ); + process.exit(2); +} +const cfg = CFG[tracker as keyof typeof CFG]; +const input = await new Response(Bun.stdin.stream()).text(); +if (!input.trim()) process.exit(0); +const report = JSON.parse(input) as Record; +const items = (report[cfg.key] ?? []) as { url?: string; nl?: number }[]; +const sql = postgres(process.env.DB_URL!, { max: 1 }); +try { + for (const item of items) { + const path = item.url ? new URL(item.url, 'https://x').pathname : ''; + const [a, b] = path.replace(/^\/trackers\/[a-z]+\//, '').split('/'); + const [row] = a && b ? await sql.unsafe<{ score: number }[]>(cfg.sql, [a, b]) : []; + item.nl = row ? Math.round(row.score * 10) / 10 : 0; + } + report.newInNl = items + .filter((x) => (x.nl ?? 0) > 0) + .toSorted((x, y) => (y.nl ?? 0) - (x.nl ?? 0)); +} finally { + await sql.end(); +} +console.log(JSON.stringify(report, null, 1)); diff --git a/scripts/trains-refresh.sh b/scripts/trains-refresh.sh index a671be8..a0d9eaf 100755 --- a/scripts/trains-refresh.sh +++ b/scripts/trains-refresh.sh @@ -9,7 +9,11 @@ # are stable (types are matched by Wikidata id). # 2. bun scripts/trains-photos.ts --missing finds Wikimedia Commons photos for types that # have none yet (new ones included) and maker and operator logos that are missing. -# stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}. +# 3. bun scripts/spotters-activity.ts trains refreshes the per-country activity scores +# ("most active in NL" sort; see that script). +# stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}, plus `nl` per new +# entry and `newInNl` (new entries that are active in the Netherlands, see +# scripts/spotters-report-nl.ts). --report-new is accepted (the report is always printed). # Progress goes to stderr. DB_URL is read from the running app container and never printed. set -euo pipefail cd "$(dirname "$0")/.." @@ -20,8 +24,12 @@ if [ -z "${DB_URL:-}" ]; then export DB_URL fi +args=() +for a in "$@"; do [ "$a" = "--report-new" ] || args+=("$a"); done + report=$(mktemp /tmp/trains-report.XXXXXX.json) trap 'rm -f "$report"' EXIT -bun scripts/trains-import.ts --report-new "$@" >"$report" -bun scripts/trains-photos.ts --missing --enrich "$report" --fetch +bun scripts/trains-import.ts --report-new "${args[@]}" >"$report" +bun scripts/spotters-activity.ts trains >&2 +bun scripts/trains-photos.ts --missing --enrich "$report" --fetch | bun scripts/spotters-report-nl.ts trains diff --git a/scripts/wildlife-refresh.sh b/scripts/wildlife-refresh.sh index 6b38732..e1d25f1 100755 --- a/scripts/wildlife-refresh.sh +++ b/scripts/wildlife-refresh.sh @@ -9,7 +9,10 @@ # that entered the lists (existing ones are kept, Dex numbers are stable). # 2. bun scripts/wildlife-photos.ts --missing CC-licensed iNaturalist photos for species # without photos (new ones included). -# --report-new: stdout is one JSON report {generatedAt, totals, newSpecies[] (with photo)}. +# 3. bun scripts/spotters-activity.ts wildlife per-country activity scores (iNaturalist +# research-grade observations per country) for the "most active in …" sort. +# --report-new: stdout is one JSON report {generatedAt, totals, newSpecies[] (with photo)}, +# plus `nl` per new species and `newInNl` (see scripts/spotters-report-nl.ts). # Without it, progress only. Progress goes to stderr. DB_URL is read from the running app # container and never printed. set -euo pipefail @@ -26,8 +29,10 @@ if [ "${1:-}" = "--report-new" ]; then report=$(mktemp /tmp/wildlife-report.XXXXXX.json) trap 'rm -f "$report"' EXIT bun scripts/wildlife-import.ts --report-new "$@" >"$report" - bun scripts/wildlife-photos.ts --missing --enrich "$report" + bun scripts/spotters-activity.ts wildlife >&2 + bun scripts/wildlife-photos.ts --missing --enrich "$report" | bun scripts/spotters-report-nl.ts wildlife else bun scripts/wildlife-import.ts "$@" >&2 bun scripts/wildlife-photos.ts --missing >&2 + bun scripts/spotters-activity.ts wildlife >&2 fi diff --git a/src/lib/cars/BrandTile.svelte b/src/lib/cars/BrandTile.svelte index e99f16e..6077165 100644 --- a/src/lib/cars/BrandTile.svelte +++ b/src/lib/cars/BrandTile.svelte @@ -6,6 +6,7 @@ + {#if brand.seen}✓ {brand.seen}{/if} {brand.name} {#if !compact} @@ -18,7 +19,20 @@ diff --git a/src/lib/spotting/SpotItem.svelte b/src/lib/spotting/SpotItem.svelte new file mode 100644 index 0000000..4ba78e0 --- /dev/null +++ b/src/lib/spotting/SpotItem.svelte @@ -0,0 +1,88 @@ + + +
+ {@render children()} + {#if owner} + + {:else if on} + ✓collected + {/if} +
+ + diff --git a/src/lib/spotting/SpotProgress.svelte b/src/lib/spotting/SpotProgress.svelte new file mode 100644 index 0000000..f4206f3 --- /dev/null +++ b/src/lib/spotting/SpotProgress.svelte @@ -0,0 +1,98 @@ + + +
+
+ ✅ {seen.toLocaleString('en-GB')} + / {total.toLocaleString('en-GB')} {label} + {pct}%{#if href} + · to go →{/if} +
+ + {#if got !== null} +
+ {#each Object.entries(GOT) as [key, text] (key)} + {text} + {/each} +
+ {/if} +
+ + diff --git a/src/lib/spotting/areas.ts b/src/lib/spotting/areas.ts new file mode 100644 index 0000000..cda4adf --- /dev/null +++ b/src/lib/spotting/areas.ts @@ -0,0 +1,277 @@ +// Countries and regions for the spotting pokedexes' "most active in ..." sort +// (/trackers/cars, planes, trains, wildlife). Codes are lowercase ISO 3166-1 alpha-2 +// ("nl") or a region code ("europe"). Scores per area are precomputed into each +// tracker's `activity` table by scripts/spotters-activity.ts; a region's score is the sum +// of its countries' scores (wildlife: iNaturalist's own Europe / world counts). + +export const DEFAULT_AREA = 'nl'; + +export type Region = { code: string; name: string; emoji: string; countries: readonly string[] }; + +const EUROPE = [ + 'nl', + 'be', + 'lu', + 'de', + 'fr', + 'gb', + 'ie', + 'es', + 'pt', + 'it', + 'ch', + 'at', + 'dk', + 'se', + 'no', + 'fi', + 'is', + 'pl', + 'cz', + 'sk', + 'hu', + 'si', + 'hr', + 'ba', + 'rs', + 'me', + 'mk', + 'al', + 'gr', + 'bg', + 'ro', + 'md', + 'ua', + 'by', + 'lt', + 'lv', + 'ee', + 'ru', + 'mt', + 'cy', + 'li', + 'mc', + 'sm', + 'ad', + 'im', + 'je', + 'gg', + 'xk', + 'tr', + 'ge', + 'am', + 'az' +] as const; + +export const REGIONS: readonly Region[] = [ + { code: 'benelux', name: 'Benelux', emoji: '🧇', countries: ['nl', 'be', 'lu'] }, + { code: 'europe', name: 'Europe', emoji: '🇪🇺', countries: EUROPE }, + { + code: 'north-america', + name: 'North America', + emoji: '🌎', + countries: ['us', 'ca', 'mx', 'cu', 'gt', 'cr', 'pa', 'jm', 'do', 'hn', 'sv', 'ni', 'bs', 'pr'] + }, + { + code: 'south-america', + name: 'South America', + emoji: '🌎', + countries: ['br', 'ar', 'cl', 'co', 'pe', 've', 'ec', 'bo', 'py', 'uy', 'gy', 'sr'] + }, + { + code: 'asia', + name: 'Asia', + emoji: '🌏', + countries: [ + 'jp', + 'cn', + 'kr', + 'kp', + 'tw', + 'hk', + 'in', + 'id', + 'th', + 'vn', + 'my', + 'sg', + 'ph', + 'pk', + 'bd', + 'lk', + 'mm', + 'kh', + 'la', + 'np', + 'mn', + 'kz', + 'uz', + 'tm', + 'kg', + 'tj', + 'ir', + 'iq', + 'sa', + 'ae', + 'il', + 'jo', + 'sy', + 'lb', + 'qa', + 'kw', + 'om', + 'bh', + 'ye', + 'af', + 'mo', + 'bt', + 'bn' + ] + }, + { + code: 'africa', + name: 'Africa', + emoji: '🌍', + countries: [ + 'za', + 'eg', + 'ma', + 'dz', + 'tn', + 'ly', + 'ng', + 'ke', + 'tz', + 'ug', + 'et', + 'gh', + 'zw', + 'zm', + 'mz', + 'na', + 'bw', + 'ao', + 'cd', + 'cg', + 'cm', + 'sn', + 'ci', + 'sd', + 'mg', + 'mw', + 'rw', + 'sz', + 'ls', + 'er', + 'sl', + 'ga', + 'ml', + 'ne', + 'bf', + 'td', + 'so', + 'mu' + ] + }, + { + code: 'oceania', + name: 'Oceania', + emoji: '🦘', + countries: ['au', 'nz', 'pg', 'fj', 'nc', 'pf', 'sb', 'vu', 'ws', 'to'] + }, + { code: 'world', name: 'the whole world', emoji: '🌐', countries: [] } +]; + +/** Countries offered in the selector (any ISO code with data also works via ?in=xx). */ +export const COUNTRIES = [ + 'nl', + 'be', + 'de', + 'fr', + 'gb', + 'ie', + 'es', + 'pt', + 'it', + 'ch', + 'at', + 'dk', + 'se', + 'no', + 'fi', + 'pl', + 'cz', + 'hu', + 'gr', + 'tr', + 'ru', + 'ua', + 'us', + 'ca', + 'mx', + 'br', + 'ar', + 'jp', + 'cn', + 'kr', + 'tw', + 'in', + 'id', + 'th', + 'au', + 'nz', + 'za' +] as const; + +const regionByCode = new Map(REGIONS.map((r) => [r.code, r])); +export const isRegion = (code: string) => regionByCode.has(code); +export const regionOf = (code: string) => regionByCode.get(code) ?? null; + +let names: Intl.DisplayNames | null = null; +/** "nl" -> "the Netherlands", "europe" -> "Europe". */ +export function areaName(code: string): string { + const region = regionByCode.get(code); + if (region) return region.name; + if (code === 'nl') return 'the Netherlands'; + if (code === 'us') return 'the USA'; + if (code === 'gb') return 'the UK'; + try { + names ??= new Intl.DisplayNames(['en'], { type: 'region' }); + return names.of(code.toUpperCase()) ?? code.toUpperCase(); + } catch { + return code.toUpperCase(); + } +} + +/** Short label for chips and selects: "🇳🇱 Netherlands", "🇪🇺 Europe". */ +export function areaLabel(code: string): string { + const region = regionByCode.get(code); + if (region) return `${region.emoji} ${code === 'world' ? 'World' : region.name}`; + return `${flagOf(code)} ${areaName(code).replace(/^the /, '')}`; +} + +export function flagOf(code: string): string { + if (!/^[a-z]{2}$/.test(code)) return '🏳️'; + return String.fromCodePoint(...[...code.toUpperCase()].map((c) => 0x1f1a5 + c.charCodeAt(0))); +} + +/** ?in= value -> a valid area code (default NL). */ +export function parseArea(value: string | null | undefined): string { + const v = (value ?? '').trim().toLowerCase(); + if (regionByCode.has(v)) return v; + if (/^[a-z]{2}$/.test(v)) return v; + return DEFAULT_AREA; +} + +/** Selector options, grouped: countries then regions. */ +export const AREA_OPTIONS = { + countries: COUNTRIES.map((code) => ({ code, label: areaLabel(code) })).sort((a, b) => + a.code === 'nl' ? -1 : b.code === 'nl' ? 1 : a.label.slice(5).localeCompare(b.label.slice(5)) + ), + regions: REGIONS.map((r) => ({ code: r.code, label: areaLabel(r.code) })) +}; + +/** Collection filter (?got=). */ +export const GOT = { '': 'all', yes: '✅ collected', no: '⬜ still to go' } as const; +export type Got = keyof typeof GOT; +export const parseGot = (v: string | null): Got => (v === 'yes' || v === 'no' ? v : ''); diff --git a/src/lib/spotting/seen.svelte.ts b/src/lib/spotting/seen.svelte.ts new file mode 100644 index 0000000..1bfac9f --- /dev/null +++ b/src/lib/spotting/seen.svelte.ts @@ -0,0 +1,50 @@ +// Client-side collection state for the spotting pokedexes: optimistic overrides on top of +// what the server sent, so a toggle updates every card/panel for that entry at once. +import { invalidateAll } from '$app/navigation'; + +export type SpotTracker = 'cars' | 'planes' | 'trains' | 'wildlife'; + +const overrides = $state>({}); +const key = (tracker: SpotTracker, id: number) => `${tracker}:${id}`; + +export function isSeen(tracker: SpotTracker, id: number, fromServer: boolean): boolean { + const k = key(tracker, id); + return k in overrides ? overrides[k] : fromServer; +} + +/** Owner-only POST; reverts the optimistic state when the server refuses. */ +export async function saveSeen( + tracker: SpotTracker, + id: number, + seen: boolean, + extra: { seenOn?: string | null; note?: string | null } = {} +): Promise { + const k = key(tracker, id); + const before = k in overrides ? overrides[k] : undefined; + overrides[k] = seen; + try { + const res = await fetch(`/trackers/${tracker}/seen`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id, seen, ...extra }) + }); + if (!res.ok) throw new Error(String(res.status)); + await invalidateAll(); + delete overrides[k]; + return true; + } catch { + if (before === undefined) delete overrides[k]; + else overrides[k] = before; + return false; + } +} + +type HasSeen = { id: number; seen: boolean }; +/** How many of these entries are collected (respecting optimistic toggles). */ +export const countSeen = (tracker: SpotTracker, items: HasSeen[]) => + items.reduce((n, x) => n + (isSeen(tracker, x.id, x.seen) ? 1 : 0), 0); +/** Client-side ?got= filter for group pages that list every entry at once. */ +export function filterGot(tracker: SpotTracker, items: T[], got: string): T[] { + if (got !== 'yes' && got !== 'no') return items; + return items.filter((x) => isSeen(tracker, x.id, x.seen) === (got === 'yes')); +} diff --git a/src/lib/trains/MakerTile.svelte b/src/lib/trains/MakerTile.svelte index f9787ac..fcd5107 100644 --- a/src/lib/trains/MakerTile.svelte +++ b/src/lib/trains/MakerTile.svelte @@ -10,6 +10,7 @@ class={['tile', { compact }]} title={maker.name} > + {#if maker.seen}✓ {maker.seen}{/if} {maker.name} {#if !compact} @@ -22,7 +23,20 @@