From 2259965a285593509185295a20fa6ec1c332e2f7 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sat, 24 Oct 2026 20:10:36 +0000 Subject: [PATCH] planes: wikidata/wikipedia import, commons photos and refresh script --- package.json | 3 + scripts/planes-import.ts | 1464 +++++++++++++++++++++++++++++++++++++ scripts/planes-photos.ts | 668 +++++++++++++++++ scripts/planes-refresh.sh | 27 + scripts/planes-wiki.ts | 184 +++++ 5 files changed, 2346 insertions(+) create mode 100644 scripts/planes-import.ts create mode 100644 scripts/planes-photos.ts create mode 100755 scripts/planes-refresh.sh create mode 100644 scripts/planes-wiki.ts diff --git a/package.json b/package.json index 05a1fa5..70487e8 100755 --- a/package.json +++ b/package.json @@ -23,6 +23,9 @@ "cars:import": "bun scripts/cars-import.ts", "cars:photos": "bun scripts/cars-photos.ts", "cars:refresh": "scripts/cars-refresh.sh", + "planes:import": "bun scripts/planes-import.ts", + "planes:photos": "bun scripts/planes-photos.ts", + "planes:refresh": "scripts/planes-refresh.sh", "db:seed:cod": "bun scripts/seed-cod-catalog.ts", "cs-prices:ingest": "bun scripts/ingest-cs-prices.ts", "house:worker": "bun scripts/house-worker.ts", diff --git a/scripts/planes-import.ts b/scripts/planes-import.ts new file mode 100644 index 0000000..aed9d7f --- /dev/null +++ b/scripts/planes-import.ts @@ -0,0 +1,1464 @@ +// Builds the public plane pokedex (/trackers/planes, schema `planes`) from open data. +// +// DB_URL=... bun scripts/planes-import.ts fetch sources and write to the database +// bun scripts/planes-import.ts --dry-run fetch + build only, print a summary +// bun scripts/planes-import.ts --cache /tmp/planes reuse API responses from that folder +// bun scripts/planes-import.ts --dry-run --out x.json also dump the built catalogue +// bun scripts/planes-import.ts --renumber reassign Dex numbers from #1 (oldest first) +// bun scripts/planes-import.ts --report-new JSON of makers/types new since the last run on stdout +// +// Sources (no API keys): +// - Wikidata (CC0): every item that is an "aircraft family" (Q15056993), "aircraft model" +// (Q15056995) or "aircraft type" (Q45296117) and has an English Wikipedia article: +// manufacturer (P176/P178), country (P495), first flight (P606), service entry/retirement +// (P729/P730), number built (P1092), capacity (P1083), engines (P516), wing configuration +// (P1654), undercarriage (P1637), dimensions/mass/speed/range, classes (P279) and uses (P366). +// - English Wikipedia (CC BY-SA): the article's {{Infobox aircraft}} (type, status, number +// built) and {{Aircraft specs}} / {{Aircraft specifications}} block (crew, capacity, +// engines, dimensions, MTOW, speed, range, ceiling), plus a 3-sentence intro extract. +// - "List of aircraft type designators" on Wikipedia for ICAO type codes. +// Only production types: prototypes, cancelled projects, drones, missiles, spacecraft and +// types with fewer than 5 built are skipped. Unknown values stay null; nothing is estimated. +// Re-runnable: types are upserted by Wikidata id (Dex numbers stay stable, new types get the +// next number), types that vanished are deleted, makers are upserted by slug. +import postgres from 'postgres'; +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { + chunks, + cmp, + fileName, + log as logErr, + mw, + qid, + saveCache, + slugify, + sparql, + tokens, + useCache +} from './planes-wiki'; + +const args = process.argv.slice(2); +const dryRun = args.includes('--dry-run'); +const renumber = args.includes('--renumber'); +const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null); +const cacheDir = opt('--cache'); +const outFile = opt('--out'); +const reportNew = args.includes('--report-new'); +const SITE = 'https://stanrunge.dev/trackers/planes'; +const log = (...a: unknown[]) => (reportNew || outFile === '-' ? logErr(...a) : console.log(...a)); +const SRC_WD = 'Wikidata'; +const SRC_WP = 'Wikipedia'; +const KINDS: Record = { + Q15056993: 'family', + Q15056995: 'model', + Q45296117: 'type' +}; +const MIN_BUILT = 5; +const year = (s: string | undefined | null) => { + const m = s?.match(/\b(1[89]\d\d|20\d\d)\b/); + return m ? Number(m[1]) : null; +}; + +// ---------------------------------------------------------------- wikidata + +type Item = { + id: string; + kind: string; + article: string; + label: string; + description: string | null; + aliases: string[]; + links: number; + image: string | null; + category: string | null; + firstFlight: string | null; // ISO date, trimmed to its precision + introduced: number | null; + retired: number | null; + built: number | null; + capacity: number | null; + countries: string[]; + makers: { id: string; label: string }[]; + developers: string[]; + classes: string[]; + uses: string[]; + engines: { id: string; label: string; qty: number | null }[]; + wing: string | null; + gear: string | null; + wingspan: number | null; + length: number | null; + height: number | null; + mass: number | null; + range: number | null; // m + speed: number | null; // m/s + cruise: number | null; + ceiling: number | null; +}; + +/** Too generic to walk past (aircraft, vehicle, machine, engine, ...). */ +const STOP = new Set([ + 'Q11436', + 'Q42889', + 'Q11019', + 'Q44167', + 'Q16686448', + 'Q223557', + 'Q35120', + 'Q488383', + 'Q7397', + 'Q1183543', + 'Q2424752', + 'Q39546', + 'Q17537576' +]); +/** Labels and parent classes of `ids` and their ancestors up to `depth` levels (P279; P31 too for the first level when `instance`). */ +async function walkUp(ids: string[], depth: number, instance = false) { + const nodes = new Map(); + let frontier = [...new Set(ids)]; + for (let level = 0; level <= depth && frontier.length; level++) { + const next = new Set(); + for (const c of chunks(frontier, 300)) { + const path = instance && level === 0 ? 'wdt:P279|wdt:P31' : 'wdt:P279'; + const rows = + await sparql(`SELECT ?c ?l ?p WHERE { VALUES ?c { ${c.map((x) => `wd:${x}`).join(' ')} } + OPTIONAL { ?c rdfs:label ?l FILTER(LANG(?l) = "en") } OPTIONAL { ?c ${path} ?p } }`); + for (const r of rows) { + const id = qid(r.c); + const node = nodes.get(id) ?? { label: null, parents: [] }; + if (r.l) node.label = r.l.toLowerCase(); + if (r.p && !STOP.has(id)) { + const p = qid(r.p); + if (!node.parents.includes(p)) node.parents.push(p); + if (!nodes.has(p)) next.add(p); + } + nodes.set(id, node); + } + } + frontier = [...next].filter((x) => !nodes.has(x)); + } + return nodes; +} + +async function values( + ids: string[], + body: string, + select = '?v' +): Promise[]> { + return sparql( + `SELECT ?i ${select} WHERE { VALUES ?i { ${ids.map((i) => `wd:${i}`).join(' ')} } ${body} }` + ); +} +const en = (v: string, l: string) => `${v} rdfs:label ${l} FILTER(LANG(${l}) = "en")`; + +async function loadWikidata() { + log('wikidata: listing aircraft families/models with an English article'); + const list = await sparql(` +SELECT ?i ?kind ?article WHERE { + VALUES ?kind { ${Object.keys(KINDS) + .map((k) => `wd:${k}`) + .join(' ')} } + ?i wdt:P31 ?kind . + ?article schema:about ?i ; schema:isPartOf . +}`); + const items = new Map(); + for (const r of list) { + const id = qid(r.i); + if (items.has(id)) continue; + items.set(id, { + id, + kind: KINDS[qid(r.kind)], + article: decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' '), + label: id, + description: null, + aliases: [], + links: 0, + image: null, + category: null, + firstFlight: null, + introduced: null, + retired: null, + built: null, + capacity: null, + countries: [], + makers: [], + developers: [], + classes: [], + uses: [], + engines: [], + wing: null, + gear: null, + wingspan: null, + length: null, + height: null, + mass: null, + range: null, + speed: null, + cruise: null, + ceiling: null + }); + } + log(`wikidata: ${items.size} items`); + const ids = [...items.keys()]; + let n = 0; + for (const c of chunks(ids, 350)) { + const get = (r: Record) => items.get(qid(r.i))!; + for (const r of await values( + c, + `OPTIONAL { ${en('?i', '?l')} } OPTIONAL { ?i schema:description ?d FILTER(LANG(?d) = "en") } + OPTIONAL { ?i wikibase:sitelinks ?sl }`, + '?l ?d ?sl' + )) { + const it = get(r); + if (r.l) it.label = r.l; + if (r.d) it.description = r.d; + it.links = Number(r.sl ?? 0); + } + for (const r of await values(c, `?i skos:altLabel ?v FILTER(LANG(?v) = "en")`)) + get(r).aliases.push(r.v); + for (const r of await values(c, `?i wdt:P18 ?v`)) get(r).image ??= fileName(r.v); + for (const r of await values(c, `?i wdt:P373 ?v`)) get(r).category ??= r.v; + for (const r of await values( + c, + `?i p:P606/psv:P606 [ wikibase:timeValue ?v ; wikibase:timePrecision ?p ]`, + '?v ?p' + )) { + const it = get(r); + const p = Number(r.p); + const iso = r.v.slice(0, p >= 11 ? 10 : p === 10 ? 7 : 4); + if (!/^\d{4}/.test(iso)) continue; + if (!it.firstFlight || iso.slice(0, 4) < it.firstFlight.slice(0, 4)) it.firstFlight = iso; + } + for (const r of await values(c, `?i wdt:P729 ?v`)) { + const it = get(r); + const y = year(r.v); + if (y && (!it.introduced || y < it.introduced)) it.introduced = y; + } + for (const r of await values(c, `?i wdt:P730 ?v`)) { + const it = get(r); + const y = year(r.v); + if (y && (!it.retired || y > it.retired)) it.retired = y; + } + for (const r of await values(c, `?i wdt:P1092 ?v`)) { + const it = get(r); + const v = Math.round(Number(r.v)); + if (Number.isFinite(v) && v >= 0 && (it.built == null || v > it.built)) it.built = v; + } + for (const r of await values(c, `?i wdt:P1083 ?v`)) { + const it = get(r); + const v = Math.round(Number(r.v)); + if (Number.isFinite(v) && v > 0 && (it.capacity == null || v > it.capacity)) it.capacity = v; + } + for (const r of await values(c, `?i wdt:P495 ?c . ${en('?c', '?v')}`)) + get(r).countries.push(r.v); + for (const r of await values(c, `?i wdt:P176 ?m . ${en('?m', '?v')}`, '?m ?v')) { + const it = get(r); + if (!it.makers.some((m) => m.id === qid(r.m))) it.makers.push({ id: qid(r.m), label: r.v }); + } + for (const r of await values(c, `?i wdt:P178 ?v`)) get(r).developers.push(qid(r.v)); + for (const r of await values(c, `?i wdt:P279 ?v`)) get(r).classes.push(qid(r.v)); + for (const r of await values(c, `?i wdt:P366 ?v`)) get(r).uses.push(qid(r.v)); + for (const r of await values( + c, + `?i p:P516 ?st . ?st ps:P516 ?e . OPTIONAL { ?st pq:P1114 ?q } OPTIONAL { ${en('?e', '?el')} }`, + '?e ?el ?q' + )) { + const it = get(r); + if (!it.engines.some((e) => e.id === qid(r.e))) + it.engines.push({ id: qid(r.e), label: r.el ?? '', qty: r.q ? Number(r.q) : null }); + } + for (const r of await values(c, `?i wdt:P1654 ?w . ${en('?w', '?v')}`)) get(r).wing ??= r.v; + for (const r of await values(c, `?i wdt:P1637 ?w . ${en('?w', '?v')}`)) get(r).gear ??= r.v; + const quantity = async (prop: string, key: keyof Item, pick: 'max' | 'min' = 'max') => { + for (const r of await values(c, `?i p:${prop}/psn:${prop}/wikibase:quantityAmount ?v`)) { + const it = get(r); + const v = Number(r.v); + if (!Number.isFinite(v) || v <= 0) continue; + const cur = it[key] as number | null; + if (cur == null || (pick === 'max' ? v > cur : v < cur)) (it[key] as number) = v; + } + }; + await quantity('P2050', 'wingspan'); + await quantity('P2043', 'length'); + await quantity('P2048', 'height'); + await quantity('P2067', 'mass'); + await quantity('P2073', 'range'); + await quantity('P2052', 'speed'); + await quantity('P2217', 'cruise'); + await quantity('P2254', 'ceiling'); + n += c.length; + log(`wikidata: details ${n}/${ids.length}`); + await saveCache(true); + } + + // Class/use ancestors with English labels (for roles, engine counts and exclusions). + // Walked breadth-first a few levels up (P279* over the whole ontology is too slow), stopping + // at very generic classes. + const classIds = [...new Set([...items.values()].flatMap((i) => [...i.classes, ...i.uses]))]; + const parents = await walkUp(classIds, 5); + const ancestors = new Map>(); // class id -> lowercase labels of itself + ancestors + for (const c of classIds) { + const seen = new Set([c]); + const queue = [c]; + const labels = new Set(); + while (queue.length) { + const x = queue.shift()!; + const node = parents.get(x); + if (!node) continue; + if (node.label) labels.add(node.label); + for (const p of node.parents) + if (!seen.has(p)) { + seen.add(p); + queue.push(p); + } + } + ancestors.set(c, labels); + } + // Own labels of the direct classes (to read "with 2 engines" only from the closest class). + const classLabels = new Map(); + for (const c of chunks(classIds, 300)) + for (const r of await sparql( + `SELECT ?c ?l WHERE { VALUES ?c { ${c.map((x) => `wd:${x}`).join(' ')} } ${en('?c', '?l')} }` + )) + classLabels.set(qid(r.c), r.l.toLowerCase()); + // Engine item classes ("turbofan", "radial engine", ...). + const engineIds = [...new Set([...items.values()].flatMap((i) => i.engines.map((e) => e.id)))]; + const engineUp = await walkUp(engineIds, 4, true); + const engineKinds = new Map>(); + for (const e of engineIds) { + const seen = new Set([e]); + const queue = [e]; + const labels = new Set(); + while (queue.length) { + const node = engineUp.get(queue.shift()!); + if (!node) continue; + if (node.label) labels.add(node.label); + for (const p of node.parents) + if (!seen.has(p)) { + seen.add(p); + queue.push(p); + } + } + engineKinds.set(e, labels); + } + // Maker countries. + const makerIds = [...new Set([...items.values()].flatMap((i) => i.makers.map((m) => m.id)))]; + const makerCountry = new Map(); + for (const c of chunks(makerIds, 300)) + for (const r of await sparql( + `SELECT ?m ?v WHERE { VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } ?m wdt:P17 ?c . ${en('?c', '?v')} }` + )) + makerCountry.set(qid(r.m), r.v); + // English aliases of makers ("Eurocopter" for Airbus Helicopters), for matching article titles. + const makerAliases = new Map(); + for (const c of chunks(makerIds, 300)) + for (const r of await sparql( + `SELECT ?m ?a WHERE { VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } ?m skos:altLabel ?a FILTER(LANG(?a) = "en") }` + )) + makerAliases.set(qid(r.m), [...(makerAliases.get(qid(r.m)) ?? []), r.a]); + await saveCache(true); + return { items, ancestors, classLabels, engineKinds, makerCountry, makerAliases }; +} + +// ---------------------------------------------------------------- wikipedia + +type Wiki = { + title: string; // resolved article title + extract: string | null; + infobox: Record; + specs: Record; + legacySpecs: boolean; +}; + +/** The `{{name ...}}` template (first match), with nested braces balanced. */ +function findTemplate(text: string, name: RegExp): string | null { + const re = /\{\{\s*([^|}\n]+)/g; + let m: RegExpExecArray | null; + while ((m = re.exec(text))) { + if (!name.test(m[1].trim())) continue; + let depth = 0; + for (let i = m.index; i < text.length - 1; i++) { + if (text[i] === '{' && text[i + 1] === '{') { + depth++; + i++; + } else if (text[i] === '}' && text[i + 1] === '}') { + depth--; + i++; + if (depth === 0) return text.slice(m.index + 2, i - 1); + } + } + return null; + } + return null; +} +function templateParams(tpl: string): Record { + const out: Record = {}; + const parts: string[] = []; + let depth = 0; + let link = 0; + let cur = ''; + for (let i = 0; i < tpl.length; i++) { + const two = tpl.slice(i, i + 2); + if (two === '{{') depth++; + else if (two === '}}') depth--; + else if (two === '[[') link++; + else if (two === ']]') link--; + if (tpl[i] === '|' && depth === 0 && link === 0) { + parts.push(cur); + cur = ''; + continue; + } + cur += tpl[i]; + } + parts.push(cur); + for (const p of parts.slice(1)) { + const eq = p.indexOf('='); + if (eq < 0) continue; + const key = p.slice(0, eq).trim().toLowerCase().replace(/_/g, ' '); + const value = p.slice(eq + 1).trim(); + if (key && value) out[key] = value; + } + return out; +} +/** Wikitext to plain text: no refs, comments, markup; templates keep their unnamed arguments. */ +export function plain(v: string): string { + let s = v + .replace(//g, '') + .replace(/]*\/>/gi, '') + .replace(/]*>[\s\S]*?<\/ref>/gi, '') + .replace(//gi, ', ') + .replace(/<[^>]+>/g, ''); + for (let guard = 0; guard < 10 && /\{\{/.test(s); guard++) + s = s.replace(/\{\{([^{}]*)\}\}/g, (_, inner: string) => { + const [name, ...rest] = inner.split('|'); + const n = name.trim().toLowerCase(); + if (/^(cvt|convert)$/.test(n)) return rest.slice(0, 2).join(' '); + if (/^(efn|sfn|refn|citation needed|cn|when|as of|dubious|clarify|webarchive|nbsp)/.test(n)) + return /^as of/.test(n) ? `as of ${rest.join(' ')}` : ''; + return rest + .filter((r) => !/^\s*[a-z_ ]+\s*=/.test(r)) + .map((r) => r.trim()) + .filter(Boolean) + .join(', '); + }); + s = s + .replace(/\[\[(?:[^|\]]*\|)?([^\]]*)\]\]/g, '$1') + .replace(/\[https?:[^\s\]]+\s*([^\]]*)\]/g, '$1') + .replace(/'{2,}/g, '') + .replace(/ /g, ' ') + .replace(/\s*\n\s*\*\s*/g, ', ') + .replace(/^\s*\*\s*/, '') + .replace(/\s+/g, ' ') + .replace(/^[,\s]+|[,\s]+$/g, '') + .trim(); + return s; +} +const num = (v: string | undefined | null): number | null => { + if (!v) return null; + const m = plain(v) + .replace(/(\d),(\d{3})/g, '$1$2') + .replace(/(\d),(\d{3})/g, '$1$2') + .match(/\d+(?:\.\d+)?/); + return m ? Number(m[0]) : null; +}; + +async function loadWikipedia(titles: string[]) { + const out = new Map(); // keyed by the title we asked for + const todo = [...new Set(titles)]; + let done = 0; + for (const c of chunks(todo, 20)) { + const res = await mw('en.wikipedia.org', { + action: 'query', + prop: 'revisions|extracts', + rvprop: 'content', + rvslots: 'main', + exintro: '1', + explaintext: '1', + exsentences: '3', + exlimit: '20', + redirects: '1', + titles: c.join('|') + }); + const back = new Map(); + for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])]) + back.set(n.to, back.get(n.from) ?? n.from); + for (const p of res.query?.pages ?? []) { + const asked = back.get(p.title) ?? p.title; + const text = p.revisions?.[0]?.slots?.main?.content ?? ''; + const box = + findTemplate(text, /^infobox aircraft type$/i) ?? + findTemplate(text, /^infobox aircraft$/i) ?? + findTemplate(text, /^infobox (aircraft|helicopter|military aircraft|glider)\b/i); + const begin = findTemplate(text, /^infobox aircraft begin$/i); + let specsTpl = findTemplate(text, /^aircraft specs$/i); + let legacy = false; + if (!specsTpl) { + specsTpl = findTemplate(text, /^aircraft specifications$|^aerospecs$/i); + legacy = !!specsTpl; + } + out.set(asked, { + title: p.title, + extract: p.extract?.trim() || null, + infobox: { ...(begin ? templateParams(begin) : {}), ...(box ? templateParams(box) : {}) }, + specs: specsTpl ? templateParams(specsTpl) : {}, + legacySpecs: legacy + }); + } + done += c.length; + if (done % 400 === 0 || done === todo.length) { + log(`wikipedia: ${done}/${todo.length} articles`); + await saveCache(true); + } + } + return out; +} + +/** ICAO type designators from Wikipedia's list, keyed by the (redirect-resolved) article. */ +async function loadIcao() { + const res = await mw('en.wikipedia.org', { + action: 'query', + prop: 'revisions', + rvprop: 'content', + rvslots: 'main', + redirects: '1', + titles: 'List of aircraft type designators' + }); + const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? ''; + const rows: { code: string; link: string }[] = []; + for (const line of text.split('\n')) { + const m = line.match(/^\|\s*([A-Z0-9]{2,4})\s*\|\|[^|]*\|\|\s*\[\[([^\]|#]+)/); + if (m) rows.push({ code: m[1], link: m[2].trim() }); + } + const resolved = new Map(); + for (const c of chunks([...new Set(rows.map((r) => r.link))], 50)) { + const r = await mw('en.wikipedia.org', { + action: 'query', + redirects: '1', + titles: c.join('|') + }); + const map = new Map(); + for (const n of [...(r.query?.normalized ?? []), ...(r.query?.redirects ?? [])]) + map.set(n.from, n.to); + for (const t of c) { + let cur = t; + for (let g = 0; g < 3 && map.has(cur); g++) cur = map.get(cur)!; + resolved.set(t, cur); + } + } + const byArticle = new Map>(); + for (const r of rows) { + const a = resolved.get(r.link) ?? r.link; + if (!byArticle.has(a)) byArticle.set(a, new Set()); + byArticle.get(a)!.add(r.code); + } + log(`icao: ${rows.length} designators for ${byArticle.size} articles`); + return byArticle; +} + +// ---------------------------------------------------------------- classification + +export const ROLES = [ + 'airliner', + 'regional', + 'cargo', + 'business', + 'general aviation', + 'glider', + 'helicopter', + 'military', + 'airship' +] as const; +const ROLE_RULES: [string, RegExp][] = [ + [ + 'helicopter', + /helicopter|rotorcraft|gyroplane|autogyro|gyrocopter|tiltrotor|compound helicopter/ + ], + ['regional', /regional|commuter|feederliner|feeder liner|turboprop airliner/], + [ + 'airliner', + /airliner|jetliner|passenger aircraft|passenger plane|airbus a3|wide-body|narrow-body/ + ], + ['business', /business jet|business aircraft|bizjet|executive|very light jet|corporate jet/], + ['cargo', /cargo|freighter|airlifter|transport aircraft|transport plane|tactical transport/], + [ + 'military', + /military|fighter|bomber|attack aircraft|interceptor|reconnaissance|maritime patrol|patrol aircraft|warplane|combat|gunship|torpedo|dive bomber|early warning|electronic warfare|air superiority|tanker|anti-submarine|ground-attack|strike aircraft|observation aircraft|army co-operation|naval aircraft|carrier-based|night fighter|attack helicopter|trainer.*military|military trainer|jet trainer/ + ], + ['glider', /glider|sailplane/], + ['airship', /airship|zeppelin|blimp|dirigible|balloon/], + [ + 'general aviation', + /light aircraft|utility aircraft|sport aircraft|sports aircraft|sportplane|aerobatic|trainer|agricultural|homebuilt|ultralight|microlight|general aviation|touring|bush plane|floatplane|seaplane|amphibi|flying boat|light-sport|civil utility|kit aircraft|air taxi|liaison aircraft|private aircraft|civil aircraft/ + ] +]; +const PRIMARY = [ + 'helicopter', + 'regional', + 'airliner', + 'business', + 'military', + 'cargo', + 'glider', + 'airship', + 'general aviation' +]; +const EXCLUDE = + /unmanned|\buav\b|\bdrone|missile|spacecraft|spaceplane|space plane|launch vehicle|\brocket\b(?!-powered| plane| aircraft)|experimental aircraft|research aircraft|prototype|concept aircraft|x-plane|target aircraft|cruise missile|space shuttle|technology demonstrator|aircraft project/; + +const ENGINE_RULES: [string, RegExp][] = [ + ['turboprop', /turboprop|propfan/], + ['turboshaft', /turboshaft/], + ['jet', /turbofan|turbojet|jet engine|\bjets?\b|afterburn|ramjet/], + [ + 'piston', + /piston|radial|inline|in-line|flat-(?:four|six|twin)|boxer|opposed|v-?(?:8|12|16)\b|v engine|rotary engine|reciprocating|two-stroke|four-stroke|diesel|cylinder|liquid-cooled|air-cooled|internal combustion/ + ], + ['electric', /electric motor|electric/], + ['rocket', /rocket/] +]; +export const engineTypeOf = (text: string): string | null => + ENGINE_RULES.find(([, re]) => re.test(text.toLowerCase()))?.[0] ?? null; + +const WORD_COUNT: Record = { + single: 1, + one: 1, + twin: 2, + two: 2, + tri: 3, + three: 3, + four: 4, + quad: 4, + six: 6, + eight: 8 +}; +function engineCountFromLabel(label: string): number | null { + const m = label.match(/with (\d+) engines?/); + if (m) return Number(m[1]); + if (/\btwinjet\b/.test(label)) return 2; + if (/\btrijet\b/.test(label)) return 3; + if (/\bquadjet\b/.test(label)) return 4; + const w = label.match( + /\b(single|one|twin|two|tri|three|four|quad|six|eight)[- ]?(?:engine|engined|jet|motor)/ + ); + return w ? WORD_COUNT[w[1]] : null; +} + +function passengers(capacity: string | undefined): number | null { + if (!capacity) return null; + const s = plain(capacity).replace(/(\d),(\d{3})/g, '$1$2'); + let best: number | null = null; + for (const m of s.matchAll( + /(? 0 && v < 1000 && (best == null || v > best)) best = v; + } + return best; +} + +type Unit = 'len' | 'mass' | 'speed' | 'range' | 'alt'; +const UNIT_RE = + /(\d+(?:\.\d+)?)\s*(km\/h|kmh|km|mph|mi\b|nmi|kn\b|kts|knots|kg|lb|lbs|m\b|ft|in\b|mach)/gi; +/** Value in metric (m, kg, km/h, km, m) from a legacy "63 ft 9 in (19.4 m)" text. */ +function measure(v: string | undefined, unit: Unit): number | null { + if (!v) return null; + const s = plain(v) + .replace(/(\d),(\d{3})/g, '$1$2') + .replace(/(\d),(\d{3})/g, '$1$2'); + const hits = [...s.matchAll(UNIT_RE)].map((m) => ({ n: Number(m[1]), u: m[2].toLowerCase() })); + for (let i = 0; i < hits.length; i++) { + const { n, u } = hits[i]; + if (unit === 'len' || unit === 'alt') { + if (u === 'm') return n; + if (u === 'ft') { + const inch = hits[i + 1]?.u === 'in' ? hits[i + 1].n : 0; + return n * 0.3048 + inch * 0.0254; + } + } else if (unit === 'mass') { + if (u === 'kg') return n; + if (u === 'lb' || u === 'lbs') return n * 0.4536; + } else if (unit === 'speed') { + if (u === 'km/h' || u === 'kmh') return n; + if (u === 'mph') return n * 1.609; + if (u === 'kn' || u === 'kts' || u === 'knots') return n * 1.852; + } else if (unit === 'range') { + if (u === 'km') return n; + if (u === 'mi') return n * 1.609; + if (u === 'nmi') return n * 1.852; + } + } + return null; +} + +type Specs = { + crew: number | null; + passengers: number | null; + length: number | null; + span: number | null; + height: number | null; + rotor: number | null; + mtow: number | null; + maxSpeed: number | null; + cruise: number | null; + range: number | null; + ceiling: number | null; + engineCount: number | null; + engineType: string | null; + engineNames: string[]; +}; +function parseSpecs(p: Record, legacy: boolean): Specs { + const pick = (base: string, units: [string, number][]) => { + for (const [u, f] of units) { + const n = num(p[`${base} ${u}`]); + if (n != null && n > 0) { + if (u === 'ft') return n * f + (num(p[`${base} in`]) ?? 0) * 0.0254; + return n * f; + } + } + return null; + }; + const L: [string, number][] = [ + ['m', 1], + ['ft', 0.3048] + ]; + const engines: string[] = []; + let count: number | null = null; + let typeText = ''; + if (!legacy) { + for (const k of ['eng1', 'eng2', 'eng3']) { + const n = num(p[`${k} number`]); + if (n != null && n > 0 && n <= 16) count = (count ?? 0) + Math.round(n); + if (p[`${k} name`]) engines.push(plain(p[`${k} name`])); + if (p[`${k} type`]) typeText += ` ${plain(p[`${k} type`])}`; + } + } else { + const n = num(p['number of jets']) ?? num(p['number of props']) ?? num(p['number of engines']); + if (n != null && n > 0 && n <= 16) count = Math.round(n); + for (const k of ['engine (jet)', 'engine (prop)', 'engine']) + if (p[k]) engines.push(plain(p[k])); + typeText = `${plain(p['type of jet'] ?? '')} ${plain(p['type of prop'] ?? '')} ${ + /jet/i.test(p['jet or prop?'] ?? '') && !p['type of jet'] ? 'jet' : '' + }`; + } + const s: Specs = { + crew: num(p.crew), + passengers: passengers(p.capacity), + length: legacy ? measure(p['length main'], 'len') : pick('length', L), + span: legacy ? measure(p['span main'], 'len') : pick('span', L), + height: legacy ? measure(p['height main'], 'len') : pick('height', L), + rotor: legacy ? measure(p['rotor diameter main'], 'len') : pick('rot dia', L), + mtow: legacy + ? measure(p['max takeoff weight main'] ?? p['loaded weight main'], 'mass') + : (pick('max takeoff weight', [ + ['kg', 1], + ['lb', 0.4536], + ['t', 1000] + ]) ?? + pick('gross weight', [ + ['kg', 1], + ['lb', 0.4536] + ])), + maxSpeed: legacy + ? measure(p['max speed main'], 'speed') + : pick('max speed', [ + ['kmh', 1], + ['mph', 1.609], + ['kts', 1.852], + ['kn', 1.852] + ]), + cruise: legacy + ? measure(p['cruise speed main'], 'speed') + : pick('cruise speed', [ + ['kmh', 1], + ['mph', 1.609], + ['kts', 1.852], + ['kn', 1.852] + ]), + range: legacy + ? measure(p['range main'], 'range') + : pick('range', [ + ['km', 1], + ['mi', 1.609], + ['nmi', 1.852] + ]), + ceiling: legacy + ? measure(p['ceiling main'], 'alt') + : pick('ceiling', [ + ['m', 1], + ['ft', 0.3048] + ]), + engineCount: count, + engineType: engineTypeOf(typeText), + engineNames: engines.filter((e) => e && e.length < 80) + }; + if (s.crew != null && (s.crew < 0 || s.crew > 30)) s.crew = null; + return s; +} + +function status(text: string | undefined): { label: string | null; exclude: boolean } { + const s = plain(text ?? '').toLowerCase(); + if (!s) return { label: null, exclude: false }; + if (/in production/.test(s)) return { label: 'in production', exclude: false }; + if (/in service|active|operational|in use|in limited service/.test(s)) + return { label: 'in service', exclude: false }; + if ( + /retired|out of service|withdrawn|inactive|out of production|no longer|historic|museum|preserved|display/.test( + s + ) + ) + return { label: 'retired', exclude: false }; + if (/development|testing|flight test|certification/.test(s)) + return { label: 'in development', exclude: false }; + if ( + /prototype|cancel|design study|proposal|proposed|paper|study|experimental|project|concept|abandoned|demonstrator|research|one-off|unbuilt|never built|destroyed|crashed/.test( + s + ) + ) + return { label: null, exclude: true }; + return { label: null, exclude: false }; +} + +// ---------------------------------------------------------------- build + +const CORP = new Set( + 'COMPANY CORPORATION CORP INC INCORPORATED LTD LIMITED GMBH AG SA SAS SPA PLC CO LLC AIRCRAFT AIRCRAFTS AIRPLANE AIRPLANES AEROPLANE AEROPLANES AVIATION INDUSTRIES INDUSTRY WORKS MANUFACTURING GROUP THE OF FLUGZEUGBAU FLUGZEUGWERKE AVIONS AERONAUTICS AERONAUTICAL ENGINEERING MOTORS'.split( + ' ' + ) +); +/** Words that are not a maker on their own ("British" Aircraft Corporation). */ +const GENERIC = new Set( + 'BRITISH AMERICAN GENERAL NATIONAL UNITED INTERNATIONAL CONSOLIDATED CANADIAN AUSTRALIAN ROYAL IMPERIAL STATE FEDERAL CENTRAL NORTHERN SOUTHERN EASTERN WESTERN NORTH SOUTH EAST WEST AERO AIR SKY'.split( + ' ' + ) +); +const makerKey = (label: string) => { + const t = tokens(label.replace(/\(.*?\)/g, '')); + const k = t.filter((x) => !CORP.has(x)); + return (k.length && !(k.length === 1 && GENERIC.has(k[0])) ? k : t).join(' '); +}; +const startsWith = (a: string[], b: string[]) => b.length > 0 && b.every((x, i) => a[i] === x); +const cleanTitle = (t: string) => + t + .replace(/\s*\((?:aircraft|airplane|aeroplane|helicopter|glider|airship|plane)[^)]*\)$/i, '') + .trim(); + +export type BuiltType = { + wikidata: string; + kind: string; + slug: string; + name: string; + fullName: string; + search: string; + wikipedia: string; + description: string | null; + summary: string | null; + role: string | null; + roles: string[]; + roleLabel: string | null; + country: string | null; + firstFlight: string | null; + firstFlightYear: number | null; + introducedYear: number | null; + retiredYear: number | null; + status: string | null; + inService: boolean | null; + numberBuilt: number | null; + makers: string[]; + engineType: string | null; + engineCount: number | null; + engines: string[]; + crew: number | null; + passengers: number | null; + lengthM: number | null; + wingspanM: number | null; + heightM: number | null; + rotorM: number | null; + mtowKg: number | null; + maxSpeedKmh: number | null; + cruiseKmh: number | null; + rangeKm: number | null; + ceilingM: number | null; + wing: string | null; + gear: string | null; + icao: string[]; + recognize: string | null; + parent: string | null; + sources: string[]; + links: number; +}; +export type BuiltMaker = { + slug: string; + name: string; + fullName: string; + wikidata: string; + country: string | null; + types: BuiltType[]; +}; + +const round = (n: number | null, d = 0) => (n == null ? null : Math.round(n * 10 ** d) / 10 ** d); +const sane = (n: number | null, min: number, max: number) => + n != null && n >= min && n <= max ? n : null; + +function recognizeLine(t: BuiltType): string | null { + const parts: string[] = []; + const engineWord: Record = { + jet: 'jet engine', + turboprop: 'turboprop', + piston: 'piston engine', + turboshaft: 'turboshaft engine', + electric: 'electric motor', + rocket: 'rocket engine' + }; + if (t.engineType && t.engineType !== 'none') { + const w = engineWord[t.engineType]; + const prop = t.engineType === 'piston' && t.role !== 'helicopter' ? ' with propeller' : ''; + parts.push( + t.engineCount + ? `${t.engineCount} ${w}${t.engineCount > 1 ? 's' : ''}${prop ? `${prop}${t.engineCount > 1 ? 's' : ''}` : ''}` + : `${w}s`.replace(/ss$/, 's') + ); + } else if (t.engineCount && t.engineCount > 0) + parts.push(`${t.engineCount} engine${t.engineCount > 1 ? 's' : ''}`); + if (t.role === 'glider' && !t.engineType) parts.push('no engine, long thin wings'); + if (t.wing) { + const w = t.wing.toLowerCase(); + if (/biplane|triplane|monoplane|wing/.test(w) && !/^monoplane$/.test(w)) parts.push(w); + } + if (t.gear) { + const g = t.gear.toLowerCase(); + if (/tricycle|tail|conventional|float|skid|retractable|fixed|ski|hull/.test(g)) + parts.push( + g + .replace('conventional landing gear', 'tailwheel (taildragger)') + .replace(/ undercarriage$/, ' gear') + ); + } + if (t.role === 'helicopter' && t.rotorM) parts.push(`${round(t.rotorM, 1)} m main rotor`); + if (t.wingspanM && t.role !== 'helicopter') parts.push(`${round(t.wingspanM, 1)} m wingspan`); + if (parts.length < 2) return null; + return parts.slice(0, 4).join(' · '); +} + +function build( + wd: Awaited>, + wiki: Map, + icao: Map> +) { + const { items, ancestors, classLabels, engineKinds, makerCountry, makerAliases } = wd; + const skipped: Record = {}; + const skip = (why: string, n = 1) => (skipped[why] = (skipped[why] ?? 0) + n); + // How many items each maker has, to break ties and to find "brand" makers. + const makerUse = new Map(); + const makerLabel = new Map(); + for (const it of items.values()) + for (const m of it.makers) { + makerUse.set(m.id, (makerUse.get(m.id) ?? 0) + 1); + makerLabel.set(m.id, m.label); + } + const companyNames = new Set([...makerLabel.values()].map((l) => l.toLowerCase())); + const byKey = new Map(); // makerKey -> most used maker id + for (const [id, label] of makerLabel) { + const k = makerKey(label); + const cur = byKey.get(k); + if (!cur || (makerUse.get(id) ?? 0) > (makerUse.get(cur) ?? 0)) byKey.set(k, id); + } + + type Draft = BuiltType & { makerId: string; prefix: string }; + const drafts: Draft[] = []; + for (const it of items.values()) { + const w = wiki.get(it.article); + const anc = new Set([...it.classes, ...it.uses].flatMap((c) => [...(ancestors.get(c) ?? [])])); + const ancText = [...anc].join(' | '); + const box = w?.infobox ?? {}; + const typeText = plain(box['aircraft type'] ?? box.type ?? '').toLowerCase(); + const st = status(box.status); + const builtBox = num(box['number built']); + const built = it.built ?? (builtBox != null ? Math.round(builtBox) : null); + if (EXCLUDE.test(ancText) || EXCLUDE.test(typeText)) { + skip('drone/prototype/experimental class'); + continue; + } + if (st.exclude) { + skip('status prototype/cancelled'); + continue; + } + if (built != null && built < MIN_BUILT) { + skip(`fewer than ${MIN_BUILT} built`); + continue; + } + if (!w) { + skip('no article'); + continue; + } + if (/\bvariants\b|^list of|operators$|\bin (?:.+ )?service$/i.test(w.title)) { + skip('variants/list article'); + continue; + } + if (companyNames.has(w.title.toLowerCase())) { + skip('article is about the company'); + continue; + } + const title = cleanTitle(w.title); + const tt = tokens(title); + // Maker: the manufacturer the article title starts with, else a known maker named like the + // title's first words, else the developer, else the most prolific manufacturer. + let makerId: string | null = null; + let prefix = ''; + const cands = it.makers.length ? it.makers : []; + const full = cands + .flatMap((m) => + [m.label, ...(makerAliases.get(m.id) ?? [])].map((l) => ({ + m, + t: tokens(l.replace(/\(.*?\)/g, '')) + })) + ) + .filter((x) => !(x.t.length === 1 && GENERIC.has(x.t[0]))) + .filter((x) => startsWith(tt, x.t)) + .sort((a, b) => b.t.length - a.t.length)[0]; + if (full) { + makerId = full.m.id; + prefix = full.t.length ? title.split(/\s+/).slice(0, full.t.length).join(' ') : ''; + } + if (!makerId) + for (let k = Math.min(3, tt.length - 1); k >= 1 && !makerId; k--) { + const key = tt.slice(0, k).join(' '); + const global = byKey.get(key); + const cand = cands.find( + (m) => makerKey(m.label) === key || makerKey(m.label).startsWith(`${key} `) + ); + if (cand || (global && cands.some((m) => m.id === global))) { + makerId = global && (cands.some((m) => m.id === global) || !cand) ? global : cand!.id; + prefix = title.split(/\s+/).slice(0, k).join(' '); + } else if (global && !cands.length) { + makerId = global; + prefix = title.split(/\s+/).slice(0, k).join(' '); + } + } + if (!makerId && cands.length) { + const dev = cands.find((m) => it.developers.includes(m.id)); + makerId = ( + dev ?? [...cands].sort((a, b) => (makerUse.get(b.id) ?? 0) - (makerUse.get(a.id) ?? 0))[0] + ).id; + } + if (!makerId) { + skip('no manufacturer'); + continue; + } + // Roles. + // Roles come from the infobox type and the direct classes/uses; far ancestors ("civil + // aircraft", "transport aircraft") only decide when nothing closer does. + const nearText = `${[...it.classes, ...it.uses].map((c) => classLabels.get(c) ?? '').join(' | ')} | ${typeText}`; + let roleText = nearText; + let roles = ROLE_RULES.filter(([, re]) => re.test(roleText)).map(([r]) => r); + if (!roles.length) { + roleText = `${ancText} | ${typeText}`; + roles = ROLE_RULES.filter(([, re]) => re.test(roleText)).map(([r]) => r); + } + if (roles.includes('regional') && !/airliner|jet|aircraft|plane/.test(roleText)) + roles.splice(roles.indexOf('regional'), 1); + if (roles.includes('helicopter')) { + for (const r of ['airliner', 'regional', 'general aviation', 'glider']) + if (roles.includes(r)) roles.splice(roles.indexOf(r), 1); + } + if ( + roles.includes('military') && + roles.includes('general aviation') && + !/civil|light aircraft|utility|sport|aerobatic|agricultural/.test(roleText) + ) + roles.splice(roles.indexOf('general aviation'), 1); + const role = PRIMARY.find((r) => roles.includes(r)) ?? null; + const roleLabel = + (typeText && typeText.length < 60 ? typeText : null) ?? + it.classes + .map((c) => classLabels.get(c)) + .find((l) => l && !/aircraft family|aircraft model/.test(l)) ?? + null; + // Engines. + const specs = parseSpecs(w.specs, w.legacySpecs); + const engineText = it.engines + .map((e) => `${e.label} ${[...(engineKinds.get(e.id) ?? [])].join(' ')}`) + .join(' '); + let engineType = specs.engineType ?? engineTypeOf(engineText); + if (!engineType) { + const direct = it.classes.map((c) => classLabels.get(c) ?? '').join(' '); + engineType = + engineTypeOf(direct) ?? + (/\bjet\b|jet airliner|jet aircraft|jet fighter|jet trainer|business jet/.test(roleText) + ? 'jet' + : null); + } + if (engineType === 'jet' && role === 'helicopter') engineType = 'turboshaft'; + if ( + !engineType && + (role === 'glider' || /balloon/.test(roleText)) && + !/motor glider|powered/.test(roleText) + ) + engineType = 'none'; + const qty = it.engines.length === 1 && it.engines[0].qty ? it.engines[0].qty : null; + const fromClass = [ + ...new Set( + it.classes + .map((c) => engineCountFromLabel(classLabels.get(c) ?? '')) + .filter((x): x is number => x != null) + ) + ]; + const engineCount = specs.engineCount ?? qty ?? (fromClass.length === 1 ? fromClass[0] : null); + // Years and status. + const firstFlight = + it.firstFlight ?? + (year(plain(box['first flight'] ?? '')) ? String(year(plain(box['first flight']!))) : null); + const firstFlightYear = firstFlight ? Number(firstFlight.slice(0, 4)) : null; + const introducedYear = it.introduced ?? year(plain(box.introduction ?? box.introduced ?? '')); + const retiredYear = it.retired ?? year(plain(box.retired ?? '')); + let statusLabel = st.label; + if (!statusLabel && retiredYear) statusLabel = 'retired'; + const inService = + statusLabel === 'in service' || statusLabel === 'in production' + ? true + : statusLabel === 'retired' + ? false + : null; + const country = + it.countries[0] ?? + (plain(box['national origin'] ?? '') + .split(/,|\//)[0] + .trim() || + null); + const fullName = title; + let name = + prefix && tt.length > tokens(prefix).length ? title.slice(prefix.length).trim() : title; + if (!name) name = title; + const codes = [...(icao.get(w.title) ?? icao.get(it.article) ?? [])].sort(); + const makerNames = it.makers.map((m) => m.label); + const t: Draft = { + wikidata: it.id, + kind: it.kind, + slug: slugify(name), + name, + fullName, + search: '', + wikipedia: w.title, + description: it.description, + summary: w.extract && w.extract.length > 40 ? w.extract.slice(0, 900) : null, + role, + roles, + roleLabel: roleLabel ? roleLabel.replace(/^an? /, '') : null, + country, + firstFlight, + firstFlightYear: sane(firstFlightYear, 1890, 2100), + introducedYear: sane(introducedYear, 1890, 2100), + retiredYear: sane(retiredYear, 1890, 2100), + status: statusLabel, + inService, + numberBuilt: built, + makers: makerNames, + engineType, + engineCount: sane(engineCount, 0, 16), + engines: [ + ...new Set([...specs.engineNames, ...it.engines.map((e) => e.label).filter(Boolean)]) + ].slice(0, 6), + crew: specs.crew != null ? Math.round(specs.crew) : null, + passengers: specs.passengers ?? sane(it.capacity, 1, 1000), + lengthM: round(sane(specs.length ?? it.length, 1, 400), 2), + wingspanM: round(sane(specs.span ?? it.wingspan, 1, 400), 2), + heightM: round(sane(specs.height ?? it.height, 0.5, 100), 2), + rotorM: round(sane(specs.rotor, 1, 60), 2), + mtowKg: round(sane(specs.mtow ?? it.mass, 50, 1_000_000)), + maxSpeedKmh: round(sane(specs.maxSpeed ?? (it.speed ? it.speed * 3.6 : null), 30, 8000)), + cruiseKmh: round(sane(specs.cruise ?? (it.cruise ? it.cruise * 3.6 : null), 30, 4000)), + rangeKm: round(sane(specs.range ?? (it.range ? it.range / 1000 : null), 10, 30000)), + ceilingM: round(sane(specs.ceiling ?? it.ceiling, 100, 40000)), + wing: it.wing, + gear: it.gear, + icao: codes, + recognize: null, + parent: null, + sources: [SRC_WD, SRC_WP], + links: it.links, + makerId, + prefix + }; + t.recognize = recognizeLine(t); + drafts.push(t); + } + // Family links: a type that is a subclass of another type in the dex. + // Several items can link (via redirects) to one article ("Boeing 777-200" -> "Boeing 777"): + // keep the item whose sitelink is the article itself, else the best-linked one. + const byArticle = new Map(); + for (const d of drafts) { + const key = d.wikipedia ?? d.wikidata; + const cur = byArticle.get(key); + const exact = (x: Draft) => items.get(x.wikidata)!.article === x.wikipedia; + if (!cur || (exact(d) && !exact(cur)) || (exact(d) === exact(cur) && d.links > cur.links)) + byArticle.set(key, d); + } + if (byArticle.size < drafts.length) + skip('same article as another item', drafts.length - byArticle.size); + drafts.splice(0, drafts.length, ...byArticle.values()); + const kept = new Set(drafts.map((d) => d.wikidata)); + for (const d of drafts) { + const p = items.get(d.wikidata)!.classes.find((c) => kept.has(c) && c !== d.wikidata); + d.parent = p ?? null; + } + // Makers: grouped by item, then merged by display name. + const groups = new Map(); + for (const d of drafts) groups.set(d.makerId, [...(groups.get(d.makerId) ?? []), d]); + const makers = new Map(); // slug -> maker + for (const [id, list] of [...groups].sort((a, b) => b[1].length - a[1].length)) { + const prefixes = new Map(); + for (const d of list) if (d.prefix) prefixes.set(d.prefix, (prefixes.get(d.prefix) ?? 0) + 1); + const label = makerLabel.get(id) ?? id; + const plainLabel = label.replace(/\s*\(.*?\)/g, '').trim(); + const short = + prefixes.has(plainLabel) && plainLabel.length <= 26 + ? plainLabel + : ([...prefixes] + .sort((a, b) => b[1] - a[1]) + .find(([p]) => !GENERIC.has(p.toUpperCase()))?.[0] ?? label); + const display = short.length <= label.length ? short : label; + const slug = slugify(display); + const existing = makers.get(slug); + if (existing) existing.types.push(...list); + else + makers.set(slug, { + slug, + name: display, + fullName: label, + wikidata: id, + country: + makerCountry.get(id) ?? + [ + ...list.reduce( + (m, t) => (t.country ? m.set(t.country, (m.get(t.country) ?? 0) + 1) : m), + new Map() + ) + ].sort((a, b) => b[1] - a[1])[0]?.[0] ?? + null, + types: list + }); + } + // Unique type slugs per maker; the best-known type keeps the plain slug. + for (const m of makers.values()) { + const seen = new Set(); + m.types.sort((a, b) => b.links - a.links); + for (const t of m.types) { + let s = t.slug; + if (seen.has(s)) s = `${s}-${t.wikidata.toLowerCase()}`; + seen.add(s); + t.slug = s; + const ancestorsNames = t.icao.join(' '); + t.search = [ + m.name, + m.fullName, + t.fullName, + t.name, + items.get(t.wikidata)!.label, + ...items.get(t.wikidata)!.aliases.slice(0, 12), + ancestorsNames, + cmp(t.fullName).toLowerCase(), + t.roleLabel ?? '' + ] + .join(' ') + .toLowerCase() + .slice(0, 2000); + } + } + log(`skipped: ${JSON.stringify(skipped)}`); + return [...makers.values()]; +} + +// ---------------------------------------------------------------- write + +const added = { + makers: [] as { name: string; url: string }[], + types: [] as { + dex: number; + name: string; + maker: string; + role: string | null; + firstFlight: string | null; + numberBuilt: number | null; + url: string; + photo?: string | null; + }[] +}; + +async function write(makers: BuiltMaker[]) { + const url = process.env.DB_URL; + if (!url) throw new Error('Set DB_URL (or use --dry-run)'); + const sql = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} }); + try { + await sql.begin(async (tx) => { + const oldTypes = new Map( + ( + await tx< + { id: number; wikidata: string; dex: number }[] + >`select id, wikidata, dex from planes.types` + ).map((r) => [r.wikidata, r]) + ); + const oldMakers = new Set( + (await tx<{ slug: string }[]>`select slug from planes.makers`).map((r) => r.slug) + ); + const firstRun = oldTypes.size === 0; + if (renumber) await tx`update planes.types set dex = -id`; + const all = makers.flatMap((m) => m.types.map((t) => ({ m, t }))); + let nextDex = renumber ? 1 : Math.max(0, ...[...oldTypes.values()].map((r) => r.dex)) + 1; + const order = all.toSorted( + (a, b) => + (a.t.firstFlightYear ?? 9999) - (b.t.firstFlightYear ?? 9999) || + (a.t.firstFlight ?? '').localeCompare(b.t.firstFlight ?? '') || + a.t.fullName.localeCompare(b.t.fullName) + ); + const dex = new Map(); + for (const { t } of order) { + const old = oldTypes.get(t.wikidata); + dex.set(t.wikidata, !renumber && old ? old.dex : nextDex++); + } + const makerIds = new Map(); + for (const m of makers) { + const years = m.types.map((t) => t.firstFlightYear).filter((y): y is number => y != null); + const built = m.types.reduce((a, t) => a + (t.parent ? 0 : (t.numberBuilt ?? 0)), 0); + const row = { + slug: m.slug, + name: m.name, + full_name: m.fullName, + country: m.country, + wikidata: m.wikidata, + type_count: m.types.length, + first_year: years.length ? Math.min(...years) : null, + last_year: years.length ? Math.max(...years) : null, + built_total: built, + sources: [SRC_WD, SRC_WP] + }; + const [r] = await tx<{ id: number }[]>` + insert into planes.makers ${tx(row)} + on conflict (slug) do update set name = excluded.name, full_name = excluded.full_name, + country = excluded.country, wikidata = excluded.wikidata, type_count = excluded.type_count, + first_year = excluded.first_year, last_year = excluded.last_year, + built_total = excluded.built_total, sources = excluded.sources + returning id`; + makerIds.set(m.slug, r.id); + if (!firstRun && !oldMakers.has(m.slug)) + added.makers.push({ name: m.name, url: `${SITE}/${m.slug}` }); + } + // Slugs can move between rows (renames); clear them first to dodge the unique index. + await tx`update planes.types set slug = 'tmp-' || id`; + const ids = new Map(); + for (const { m, t } of all) { + const row = { + dex: dex.get(t.wikidata)!, + maker_id: makerIds.get(m.slug)!, + slug: t.slug, + name: t.name, + full_name: t.fullName, + search: t.search, + wikidata: t.wikidata, + wikipedia: t.wikipedia, + kind: t.kind, + description: t.description, + summary: t.summary, + role: t.role, + roles: t.roles, + role_label: t.roleLabel, + country: t.country, + first_flight: t.firstFlight, + first_flight_year: t.firstFlightYear, + introduced_year: t.introducedYear, + retired_year: t.retiredYear, + status: t.status, + in_service: t.inService, + number_built: t.numberBuilt, + makers: t.makers, + engine_type: t.engineType, + engine_count: t.engineCount, + engines: t.engines, + crew: t.crew, + passengers: t.passengers, + length_m: t.lengthM, + wingspan_m: t.wingspanM, + height_m: t.heightM, + rotor_m: t.rotorM, + mtow_kg: t.mtowKg, + max_speed_kmh: t.maxSpeedKmh, + cruise_kmh: t.cruiseKmh, + range_km: t.rangeKm, + ceiling_m: t.ceilingM, + wing: t.wing, + gear: t.gear, + icao: t.icao, + recognize: t.recognize, + sources: t.sources + }; + const [r] = await tx<{ id: number }[]>` + insert into planes.types ${tx(row)} + on conflict (wikidata) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'wikidata') as (keyof typeof row)[])} + returning id`; + ids.set(t.wikidata, r.id); + if (!firstRun && !oldTypes.has(t.wikidata)) + added.types.push({ + dex: row.dex, + name: t.fullName, + maker: m.name, + role: t.role, + firstFlight: t.firstFlight, + numberBuilt: t.numberBuilt, + url: `${SITE}/${m.slug}/${t.slug}` + }); + } + for (const { t } of all) + await tx`update planes.types set family_id = ${t.parent ? (ids.get(t.parent) ?? null) : null} where id = ${ids.get(t.wikidata)!}`; + const keep = [...ids.values()]; + const gone = await tx`delete from planes.types where not (id = any(${keep})) returning id`; + const goneMakers = + await tx`delete from planes.makers where not (slug = any(${makers.map((m) => m.slug)})) returning id`; + log( + `written: ${keep.length} types, ${makers.length} makers; removed ${gone.length} types, ${goneMakers.length} makers` + ); + }); + } finally { + await sql.end(); + } +} + +// ---------------------------------------------------------------- main + +await useCache(cacheDir, 'import.json'); +const wd = await loadWikidata(); +const wiki = await loadWikipedia([...wd.items.values()].map((i) => i.article)); +const icao = await loadIcao(); +await saveCache(true); +const makers = build(wd, wiki, icao); +const types = makers.flatMap((m) => m.types); +log( + `built ${makers.length} makers, ${types.length} types (${types.filter((t) => t.role).length} with a role, ${types.filter((t) => t.engineType).length} engine type, ${types.filter((t) => t.engineCount).length} engine count, ${types.filter((t) => t.maxSpeedKmh).length} speed, ${types.filter((t) => t.icao.length).length} ICAO, ${types.filter((t) => t.recognize).length} recognise)` +); +if (outFile) { + const json = JSON.stringify(makers.map((m) => ({ ...m, types: m.types }))); + if (outFile === '-') console.log(json); + else await writeFile(outFile, json); +} +if (dryRun) { + for (const t of types + .toSorted((a, b) => (b.numberBuilt ?? 0) - (a.numberBuilt ?? 0)) + .slice(0, 20)) + log( + ` ${t.fullName} built=${t.numberBuilt} ${t.role} ${t.engineCount}x${t.engineType} ${t.firstFlight}` + ); +} else { + await write(makers); + if (reportNew) + console.log( + JSON.stringify( + { + generatedAt: new Date().toISOString(), + totals: { makers: makers.length, types: types.length }, + newMakers: added.makers, + newTypes: added.types.toSorted((a, b) => (b.numberBuilt ?? 0) - (a.numberBuilt ?? 0)) + }, + null, + 2 + ) + ); +} +void readFile; +void join; diff --git a/scripts/planes-photos.ts b/scripts/planes-photos.ts new file mode 100644 index 0000000..702c52e --- /dev/null +++ b/scripts/planes-photos.ts @@ -0,0 +1,668 @@ +// Finds freely licensed photos (Wikimedia Commons) for the plane pokedex (/trackers/planes) +// and logos for its makers. +// +// DB_URL=... bun scripts/planes-photos.ts every type, replace all photos + logos +// DB_URL=... bun scripts/planes-photos.ts --missing only types without photos / makers without a logo +// DB_URL=... bun scripts/planes-photos.ts --logos maker logos only +// DB_URL=... bun scripts/planes-photos.ts --enrich r.json [--fetch] +// add lead photos to a planes-import --report-new file (stdout); +// with --fetch, first match photos for types that have none +// bun scripts/planes-photos.ts --dry-run --types built.json [--cache dir] [--out x.json] +// +// Every type already is a Wikidata item (from planes-import), so there is no fuzzy item +// matching. Photos, in order (max 10, low-confidence files are skipped, never guessed): +// 1. The item's own image (P18). +// 2. Images of its variants (items that are "subclass of" it), captioned with the variant. +// 3. Images used in its English Wikipedia article and files in its Commons category (P373). +// These must name the type in the file name: its designator ("AH-64", "A320", "Boeing 737" +// / "B737" for number-only designators) or, for names without one, a distinctive word +// ("Spitfire") that no other type in the dex uses. Files naming a longer sibling type +// of the same maker that is not a variant ("737 MAX" on a "737 Classic") are rejected, +// as are cockpits, cabins, engines, crashes, drawings, maps, logos and small files. +// Logos: the maker item's logo (P154, current/preferred first), else the infobox logo of its +// article (Commons files only), else a plain " logo" file in " logos". +// Nothing is downloaded: thumbnail URLs (standard widths) plus author and licence are stored. +import postgres from 'postgres'; +import { readFile, writeFile } from 'node:fs/promises'; +import { + chunks, + cmp, + fileName, + log, + mw, + qid, + saveCache, + sparql, + stripHtml, + tokens, + useCache, + type ImageInfo +} from './planes-wiki'; + +const args = process.argv.slice(2); +const flag = (name: string) => args.includes(name); +const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null); +const dryRun = flag('--dry-run'); +const missingOnly = flag('--missing'); +const logosOnly = flag('--logos'); +const enrichFile = opt('--enrich'); +const typesFile = opt('--types'); +/** Only types whose full name matches this regex (for testing). */ +const only = opt('--only'); +const cacheDir = opt('--cache'); +const outFile = opt('--out'); +const MAX_PHOTOS = 10; +const LOGO = 250; +const THUMB = 960; + +type Photo = { + file: string; + kind: 'lead' | 'variant' | 'article' | 'commons' | 'logo'; + caption: string | null; + thumb: string; + width: number; + height: number; + author: string | null; + license: string | null; + licenseUrl: string | null; +}; + +// ---------------------------------------------------------------- commons + +const BAD_FILE = + /\.(svg|gif|tiff?|pdf|djvu|webm|ogv|stl)$|logo|emblem|badge|insignia|roundel|patch\b|wordmark|map\b|karte|locator|diagram|chart|graph|schema|schematic|drawing|sketch|blueprint|patent|3[- ]?view|three[- ]view|icon|flag|coat of arms|wappen|signature|symbol|cutaway|silhouette|commons-|wiki|stub|poster|advert|brochure|stamp|timeline|cockpit|flight deck|instrument|panel|cabin|interior|seats?\b|galley|lavatory|engine\b|nacelle|propeller\b|landing gear|undercarriage|wheel|tyre|tire\b|crash|wreck|accident|debris|incident|fire\b|burnt|scrap|graveyard|boneyard|\btoys?\b|scale model|model aircraft|lego|diecast|plastic model|kit\b|simulator|mock-?up|ticket|menu|uniform|crew\b|pilot\b|portrait|ceremony|memorial|grave|plaque|statue|stamp/i; + +const infoCache = new Map(); +async function imageInfo(files: string[], width = THUMB) { + const todo = [...new Set(files)].filter((f) => !infoCache.has(`${width}|${f}`)); + for (const chunk of chunks(todo, 50)) { + const res = await mw('commons.wikimedia.org', { + action: 'query', + prop: 'imageinfo', + iiprop: 'url|size|mime|extmetadata', + iiurlwidth: String(width), + iiextmetadatafilter: `Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree${ + width === LOGO ? '|Restrictions' : '' + }`, + titles: chunk.map((f) => `File:${f}`).join('|') + }); + const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from])); + for (const p of res.query?.pages ?? []) { + const original = (norm.get(p.title) ?? p.title).replace(/^File:/, ''); + infoCache.set(`${width}|${original}`, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]); + } + for (const f of chunk) + if (!infoCache.has(`${width}|${f}`)) infoCache.set(`${width}|${f}`, null); + } +} +function toPhoto( + file: string, + kind: Photo['kind'], + caption: string | null, + width = THUMB +): Photo | null { + const info = infoCache.get(`${width}|${file}`); + if (!info?.thumburl) return null; + const meta = info.extmetadata ?? {}; + if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null; + let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null; + if (!license || /fair use|non-free/i.test(license)) return null; + if (kind === 'logo' && /trademark/i.test(meta.Restrictions?.value ?? '')) + license += ' · trademark'; + if (kind !== 'logo') { + if (!/^image\/(jpeg|png|webp)$/.test(info.mime)) return null; + if (info.width < THUMB || info.height < 400) return null; + if (info.width / info.height > 3 || info.height / info.width > 1.6) return null; + } + const author = meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null; + return { + file, + kind, + caption, + thumb: info.thumburl + .replace(/\?.*$/, '') + .replace('//thumb.wikimedia.org/', '//upload.wikimedia.org/'), + width: info.width, + height: info.height, + author, + license, + licenseUrl: meta.LicenseUrl?.value ?? null + }; +} + +async function articleImages(titles: string[]) { + const out = new Map(); + for (const chunk of chunks([...new Set(titles)], 50)) { + let cont: Record = {}; + for (let guard = 0; guard < 30; guard++) { + const res = await mw('en.wikipedia.org', { + action: 'query', + prop: 'images', + imlimit: 'max', + redirects: '1', + titles: chunk.join('|'), + ...cont + }); + const norm = new Map(); + for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])]) + norm.set(n.to, norm.get(n.from) ?? n.from); + for (const p of res.query?.pages ?? []) { + const key = norm.get(p.title) ?? p.title; + out.set(key, [ + ...(out.get(key) ?? []), + ...(p.images ?? []).map((im) => im.title.replace(/^File:/, '')) + ]); + } + if (!res.continue) break; + cont = res.continue; + } + } + return out; +} +async function categoryFiles(category: string, limit = 200) { + const files: string[] = []; + let cont: Record = {}; + for (let guard = 0; guard < 4 && files.length < limit; guard++) { + const res = await mw('commons.wikimedia.org', { + action: 'query', + list: 'categorymembers', + cmtitle: `Category:${category}`, + cmtype: 'file', + cmlimit: '100', + ...cont + }); + files.push(...(res.query?.categorymembers ?? []).map((m) => m.title.replace(/^File:/, ''))); + if (!res.continue) break; + cont = res.continue; + } + return files; +} + +// ---------------------------------------------------------------- matching + +type Target = { + id: number | null; + wikidata: string; + wikipedia: string | null; + name: string; + fullName: string; + maker: string; + makerSlug: string; + numberBuilt: number; + photos: number; +}; + +/** File name in a comparable form: uppercase, "AH-64D" -> "AH64D", "Boeing 737" -> "BOEING737". */ +const norm = (s: string) => + ` ${s + .replace(/\.[a-z0-9]+$/i, '') + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toUpperCase() + .replace(/[_]+/g, ' ') + .replace(/([A-Z])[\s.-]+(?=\d)/g, '$1') + .replace(/(\d)[\s-]+(?=\d)/g, '$1 ') + .replace(/[^A-Z0-9]+/g, ' ')} `; +const GENERIC = new Set( + 'FAMILY SERIES AIRCRAFT AIRPLANE AEROPLANE HELICOPTER GLIDER BOMBER FIGHTER TRAINER TRANSPORT CARGO SUPER MODEL TYPE CLASS CLASSIC NEXT GENERATION VARIANT VARIANTS COMMANDER STANDARD ROYAL AMERICAN BRITISH FLYING SPORT SPORTS SPECIAL MARK LIGHT HEAVY MEDIUM'.split( + ' ' + ) +); +const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +type Rule = { test: (f: string) => boolean; designators: string[]; required: string[] }; +/** Name-based tests for one type: designators like AH64 / A320 / 737 (with maker) and rare words. */ +function ruleFor(t: Target, wordUse: Map): Rule { + const makerWords = tokens(t.maker).filter((w) => w.length >= 3 && !GENERIC.has(w)); + const prefixes = [...new Set([...makerWords, ...makerWords.map((w) => w[0])])]; + const T = norm(t.name).trim().split(' ').filter(Boolean); + const designators = T.filter((x) => /^[A-Z]{0,5}\d{1,5}[A-Z]{0,4}$/.test(x)); + // Short extra words that tell a sub-type apart ("737 MAX", "A320 NEO"). + const required = T.filter( + (x) => /^[A-Z]{2,4}$/.test(x) && !GENERIC.has(x) && !prefixes.includes(x) + ); + const words = T.filter((x) => /^[A-Z]{5,}$/.test(x) && !GENERIC.has(x)); + const tests: RegExp[] = []; + for (const d of designators) { + if (/^\d+$/.test(d)) { + // Number-only designator: needs the maker ("Boeing 737", "B737", "Cessna 172"). + if (prefixes.length) + tests.push(new RegExp(` (?:${prefixes.map(esc).join('|')}) ?${d}(?![0-9])`)); + } else if (d.length >= 3) + tests.push(new RegExp(` (?:${prefixes.map(esc).join('|') || 'X'})?${esc(d)}(?![0-9])`)); + else if (makerWords.length) + tests.push(new RegExp(` (?:${makerWords.map(esc).join('|')}) ${esc(d)}(?![0-9])`)); + } + for (const w of words) { + if ((wordUse.get(w) ?? 0) <= 1 && w.length >= 6) tests.push(new RegExp(` ${esc(w)} `)); + else if (makerWords.length) + tests.push(new RegExp(`(?=.* (?:${makerWords.map(esc).join('|')}) ).* ${esc(w)} `)); + } + return { + designators, + required, + test: (f) => { + const F = norm(f); + return ( + tests.some((re) => re.test(F)) && + (!designators.length || required.every((r) => F.includes(r))) + ); + } + }; +} + +async function loadWikidata(targets: Target[]) { + const ids = targets.map((t) => t.wikidata); + const own = new Map(); + const children = new Map(); + for (const c of chunks(ids, 300)) { + for (const r of await sparql(`SELECT ?i ?img ?cat WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} } + OPTIONAL { ?i wdt:P18 ?img } OPTIONAL { ?i wdt:P373 ?cat } }`)) { + const id = qid(r.i); + const cur = own.get(id) ?? { image: null, category: null }; + cur.image ??= r.img ? fileName(r.img) : null; + cur.category ??= r.cat ?? null; + own.set(id, cur); + } + for (const r of await sparql(`SELECT ?i ?c ?l ?img WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} } + ?c wdt:P279 ?i ; wdt:P18 ?img . ?c rdfs:label ?l FILTER(LANG(?l) = "en") }`)) { + const id = qid(r.i); + const list = children.get(id) ?? []; + if (!list.some((x) => x.label === r.l)) list.push({ label: r.l, image: fileName(r.img) }); + children.set(id, list); + } + } + await saveCache(true); + return { own, children }; +} + +async function photosFor(targets: Target[], all: Target[]) { + const wd = await loadWikidata(targets); + const wordUse = new Map(); + for (const t of all) + for (const w of new Set(tokens(t.name))) wordUse.set(w, (wordUse.get(w) ?? 0) + 1); + const byMaker = new Map(); + for (const t of all) byMaker.set(t.makerSlug, [...(byMaker.get(t.makerSlug) ?? []), t]); + const childLabels = (t: Target) => (wd.children.get(t.wikidata) ?? []).map((c) => cmp(c.label)); + const articles = await articleImages( + targets.map((t) => t.wikipedia).filter((a): a is string => !!a) + ); + const result = new Map(); + let done = 0; + for (const t of targets) { + const rule = ruleFor(t, wordUse); + const ownCmp = cmp(t.name); + const variants = new Set(childLabels(t)); + // Longer-named types of the same maker that are not variants of this one. + const siblings = (byMaker.get(t.makerSlug) ?? []) + .filter( + (o) => o !== t && cmp(o.name).length > ownCmp.length && cmp(o.name).startsWith(ownCmp) + ) + .filter((o) => ![...variants].some((v) => v.includes(cmp(o.name)))) + .map((o) => ruleFor(o, wordUse)) + // Only siblings with an extra telling word ("MAX") can claim a file from us. + .filter( + (o) => + o.required.some((r) => !rule.required.includes(r)) || + o.designators.some((d) => !rule.designators.includes(d)) + ); + const ok = (f: string) => !BAD_FILE.test(f) && rule.test(f) && !siblings.some((s) => s.test(f)); + const own = wd.own.get(t.wikidata); + const picks: { file: string; kind: Photo['kind']; caption: string | null }[] = []; + if (own?.image && !/\.(svg|gif)$/i.test(own.image)) + picks.push({ file: own.image, kind: 'lead', caption: null }); + for (const c of (wd.children.get(t.wikidata) ?? []).slice(0, 12)) + if (!BAD_FILE.test(c.image)) picks.push({ file: c.image, kind: 'variant', caption: c.label }); + for (const f of (t.wikipedia ? (articles.get(t.wikipedia) ?? []) : []).filter(ok)) + picks.push({ file: f, kind: 'article', caption: null }); + const unique = () => [...new Map(picks.map((p) => [p.file, p])).values()]; + await imageInfo(unique().map((p) => p.file)); + let photos = unique() + .map((p) => toPhoto(p.file, p.kind, p.caption)) + .filter((p): p is Photo => !!p); + if (photos.length < MAX_PHOTOS && own?.category) { + const files = (await categoryFiles(own.category)).filter(ok); + const want = MAX_PHOTOS * 2; + const step = Math.max(1, files.length / want); + const spread = Array.from( + { length: Math.min(want, files.length) }, + (_, i) => files[Math.floor(i * step)] + ); + await imageInfo(spread); + const seen = new Set(photos.map((p) => p.file)); + for (const f of spread) { + if (seen.has(f)) continue; + const p = toPhoto(f, 'commons', null); + if (p) photos.push(p); + seen.add(f); + } + } + photos = photos.slice(0, MAX_PHOTOS); + if (photos.length) result.set(t, photos); + if (++done % 200 === 0) { + log(`photos: ${done}/${targets.length} types`); + await saveCache(true); + } + } + return result; +} + +// ---------------------------------------------------------------- logos + +type Maker = { + slug: string; + name: string; + fullName: string | null; + wikidata: string | null; + hasLogo: boolean; +}; +const LOGO_WORD = /logo|wordmark|crest|emblem|marque|brand|logotip|insignia/i; +const FREE_PD = /public domain|^pd\b|cc0/i; +function logoFile(file: string, keys: string[]): Photo | null { + const photo = toPhoto(file, 'logo', null, LOGO); + if (!photo) return null; + const named = keys.some((k) => cmp(file).includes(k)); + if (/\.svg$/i.test(file)) return LOGO_WORD.test(file) || named ? photo : null; + if (/\.(png|gif)$/i.test(file)) + return LOGO_WORD.test(file) || (FREE_PD.test(photo.license ?? '') && named) ? photo : null; + if (/\.(jpe?g|webp)$/i.test(file)) + return LOGO_WORD.test(file) && FREE_PD.test(photo.license ?? '') ? photo : null; + return null; +} +const PLAIN_WORDS = new Set( + 'LOGO LOGOS LOGOTYPE WORDMARK EMBLEM BRAND MARQUE SYMBOL ICON BADGE CREST NEW OLD CURRENT SIMPLE FLAT COLOR COLOUR COLOURED COLORED BLACK WHITE GREY GRAY RED BLUE SILVER VERTICAL HORIZONTAL STACKED TRANSPARENT AIRCRAFT AVIATION AEROSPACE COMPANY CORPORATION INC SVG PNG'.split( + ' ' + ) +); +function plainLogoName(file: string, names: string[]) { + const t = tokens(file.replace(/\.[a-z]+$/i, '')); + for (const n of names) { + const nt = tokens(n); + const at = t.findIndex((_, i) => nt.every((x, j) => t[i + j] === x)); + if (at < 0) continue; + const rest = [...t.slice(0, at), ...t.slice(at + nt.length)]; + if ( + rest.some((x) => x.startsWith('LOGO')) && + rest.every((x) => PLAIN_WORDS.has(x) || /^\d+$/.test(x)) + ) + return true; + } + return false; +} +async function infoboxLogo(article: string) { + const res = await mw('en.wikipedia.org', { + action: 'query', + prop: 'revisions', + rvprop: 'content', + rvslots: 'main', + rvsection: '0', + redirects: '1', + titles: article + }); + const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? ''; + const m = text.match( + /\|\s*logo\s*=\s*(?:\[\[)?(?:File:|Image:)?\s*([^|\]\n{}<]+?\.(?:svg|png|gif))/i + ); + return m ? m[1].trim().replace(/_/g, ' ') : null; +} + +async function findLogos(makers: Maker[]) { + const ids = makers.map((m) => m.wikidata).filter((x): x is string => !!x); + const info = new Map< + string, + { article: string | null; category: string | null; logos: string[] } + >(); + for (const c of chunks(ids, 200)) { + const rows = await sparql(` +SELECT ?m (SAMPLE(?a) AS ?article) (SAMPLE(?cat) AS ?category) + (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(YEAR(?start)), ""), ";", COALESCE(STR(YEAR(?end)), "")); separator="|") AS ?logos) +WHERE { + VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } + OPTIONAL { ?a schema:about ?m ; schema:isPartOf } + OPTIONAL { ?m wdt:P373 ?cat } + OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) + OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end } } +} GROUP BY ?m`); + for (const r of rows) { + const logos = (r.logos ? r.logos.split('|').filter(Boolean) : []) + .map((x) => { + const [uri, rank, start, end] = x.split(';'); + return { + file: fileName(uri), + current: !end, + score: + (rank.endsWith('PreferredRank') ? 10000 : 0) + (end ? 0 : 5000) + (Number(start) || 0) + }; + }) + .sort((a, b) => b.score - a.score); + const current = logos.filter((x) => x.current); + info.set(qid(r.m), { + article: r.article + ? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ') + : null, + category: r.category ?? null, + logos: [...new Set((current.length ? current : logos).map((x) => x.file))] + }); + } + } + await imageInfo( + [...info.values()].flatMap((i) => i.logos), + LOGO + ); + const out = new Map(); + for (const m of makers) { + const i = m.wikidata ? info.get(m.wikidata) : undefined; + const names = [...new Set([m.name, m.fullName ?? m.name])]; + const keys = names.map(cmp); + let photo: Photo | null = null; + for (const f of i?.logos ?? []) if ((photo = logoFile(f, keys))) break; + if (!photo && i?.article) { + const f = await infoboxLogo(i.article); + if (f) { + await imageInfo([f], LOGO); + const p = logoFile(f, keys); + if (p && (LOGO_WORD.test(f) || keys.some((k) => cmp(f).includes(k)))) photo = p; + } + } + if (!photo && i?.category) { + const files = (await categoryFiles(`${i.category} logos`, 100)) + .filter((f) => /\.(svg|png)$/i.test(f) && /logo/i.test(f) && plainLogoName(f, names)) + .sort( + (x, y) => + Math.max(0, ...(y.match(/(?:19|20)\d\d/g) ?? []).map(Number)) - + Math.max(0, ...(x.match(/(?:19|20)\d\d/g) ?? []).map(Number)) || + Number(/\.svg$/i.test(y)) - Number(/\.svg$/i.test(x)) || + x.length - y.length + ) + .slice(0, 5); + await imageInfo(files, LOGO); + for (const f of files) if ((photo = logoFile(f, keys))) break; + } + out.set(m.slug, { photo, article: i?.article ?? null }); + } + await saveCache(true); + log(`logos: ${[...out.values()].filter((x) => x.photo).length}/${makers.length} makers`); + return out; +} + +// ---------------------------------------------------------------- main + +let sqlClient: ReturnType | null = null; +function db() { + if (!process.env.DB_URL) throw new Error('Set DB_URL (or use --dry-run --types )'); + sqlClient ??= postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} }); + return sqlClient; +} + +async function loadTargets(): Promise<{ targets: Target[]; makers: Maker[] }> { + if (typesFile) { + const built = JSON.parse(await readFile(typesFile, 'utf8')) as { + slug: string; + name: string; + fullName: string; + wikidata: string; + types: { + wikidata: string; + wikipedia: string; + name: string; + fullName: string; + numberBuilt: number | null; + }[]; + }[]; + return { + targets: built.flatMap((m) => + m.types.map((t) => ({ + id: null, + wikidata: t.wikidata, + wikipedia: t.wikipedia, + name: t.name, + fullName: t.fullName, + maker: m.name, + makerSlug: m.slug, + numberBuilt: t.numberBuilt ?? 0, + photos: 0 + })) + ), + makers: built.map((m) => ({ + slug: m.slug, + name: m.name, + fullName: m.fullName, + wikidata: m.wikidata, + hasLogo: false + })) + }; + } + const sql = db(); + const rows = await sql< + { + id: number; + wikidata: string; + wikipedia: string | null; + name: string; + full_name: string; + maker: string; + maker_slug: string; + built: number | null; + photos: number; + }[] + >`select t.id, t.wikidata, t.wikipedia, t.name, t.full_name, m.name as maker, m.slug as maker_slug, + t.number_built as built, t.photo_count as photos + from planes.types t join planes.makers m on m.id = t.maker_id`; + const makers = await sql< + { + slug: string; + name: string; + full_name: string | null; + wikidata: string | null; + logo: string | null; + }[] + >` + select slug, name, full_name, wikidata, logo from planes.makers`; + return { + targets: rows.map((r) => ({ + id: r.id, + wikidata: r.wikidata, + wikipedia: r.wikipedia, + name: r.name, + fullName: r.full_name, + maker: r.maker, + makerSlug: r.maker_slug, + numberBuilt: r.built ?? 0, + photos: r.photos + })), + makers: makers.map((m) => ({ + slug: m.slug, + name: m.name, + fullName: m.full_name, + wikidata: m.wikidata, + hasLogo: !!m.logo + })) + }; +} + +async function enrich(file: string) { + const report = JSON.parse(await readFile(file, 'utf8')) as { + newTypes?: { url: string; photo?: string | null }[]; + }; + const sql = db(); + for (const t of report.newTypes ?? []) { + const [, maker, type] = + new URL(t.url).pathname.match(/\/trackers\/planes\/([^/]+)\/([^/]+)/) ?? []; + const [row] = await sql<{ photo: string | null }[]>` + select t.photo from planes.types t join planes.makers m on m.id = t.maker_id + where m.slug = ${maker} and t.slug = ${type}`; + t.photo = row?.photo ?? null; + } + console.log(JSON.stringify(report, null, 2)); +} + +async function main() { + await useCache(cacheDir, 'photos.json'); + if (enrichFile && !flag('--fetch')) return enrich(enrichFile); + const loaded = await loadTargets(); + const targets = loaded.targets + .filter((t) => !missingOnly || t.photos === 0) + .filter((t) => !only || new RegExp(only, 'i').test(t.fullName)) + .sort((a, b) => b.numberBuilt - a.numberBuilt); + const photos = logosOnly ? new Map() : await photosFor(targets, loaded.targets); + const makers = loaded.makers.filter( + (m) => (!missingOnly || !m.hasLogo) && (!only || targets.some((t) => t.makerSlug === m.slug)) + ); + const logos = await findLogos(makers); + await saveCache(true); + const total = [...photos.values()].reduce((a, p) => a + p.length, 0); + log(`photos: ${photos.size}/${targets.length} types with photos, ${total} photos`); + if (outFile) + await writeFile( + outFile, + JSON.stringify({ + types: targets.map((t) => ({ + type: t.fullName, + photos: (photos.get(t) ?? []).map((p) => `${p.kind}: ${p.file}`) + })), + logos: makers.map((m) => ({ maker: m.name, logo: logos.get(m.slug)?.photo?.file ?? null })) + }) + ); + if (dryRun) return; + const sql = db(); + await sql.begin(async (tx) => { + const ids = logosOnly ? [] : targets.map((t) => t.id!).filter(Boolean); + await tx`delete from planes.photos where type_id = any(${ids})`; + await tx`update planes.types set photo = null, photo_count = 0, photos_checked_at = now() where id = any(${ids})`; + for (const t of logosOnly ? [] : targets) { + const list = photos.get(t) ?? []; + if (!t.id || !list.length) continue; + await tx`update planes.types set photo = ${list[0].thumb}, photo_count = ${list.length} where id = ${t.id}`; + await tx`insert into planes.photos ${tx( + list.map((p, i) => ({ + type_id: t.id, + position: i, + kind: p.kind, + file: p.file, + caption: p.caption, + thumb: p.thumb, + width: p.width, + height: p.height, + author: p.author, + license: p.license, + license_url: p.licenseUrl + })) + )}`; + } + for (const m of makers) { + const l = logos.get(m.slug); + await tx`update planes.makers set wikipedia = ${l?.article ?? null}, + logo = ${l?.photo?.thumb ?? null}, logo_file = ${l?.photo?.file ?? null}, + logo_license = ${l?.photo?.license ?? null}, logo_author = ${l?.photo?.author ?? null} + where slug = ${m.slug}`; + } + await tx`update planes.makers k set photo = ( + select t.photo from planes.types t where t.maker_id = k.id and t.photo is not null + order by t.family_id is null desc, t.number_built desc nulls last limit 1)`; + }); + log('written'); + if (enrichFile) await enrich(enrichFile); +} + +await main(); +await sqlClient?.end(); diff --git a/scripts/planes-refresh.sh b/scripts/planes-refresh.sh new file mode 100755 index 0000000..db33c75 --- /dev/null +++ b/scripts/planes-refresh.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Refresh of the plane pokedex (/trackers/planes) on stan-ovh. +# +# ssh stan@37.187.157.191 'cd ~/dev/website-planes && git fetch -q && git checkout -q --detach origin/main \ +# && bun install --frozen-lockfile >/dev/null && scripts/planes-refresh.sh' +# +# 1. bun scripts/planes-import.ts --report-new rebuilds makers/types from Wikidata + English +# Wikipedia (infobox + specs) and notes which makers/types did not exist before. Dex numbers +# are stable (types are matched by Wikidata id). +# 2. bun scripts/planes-photos.ts --missing finds Wikimedia Commons photos for types that +# have none yet (new ones included) and maker logos that are missing. +# stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}. +# Progress goes to stderr. DB_URL is read from the running app container and never printed. +set -euo pipefail +cd "$(dirname "$0")/.." + +if [ -z "${DB_URL:-}" ]; then + DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' | + sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/') + export DB_URL +fi + +report=$(mktemp /tmp/planes-report.XXXXXX.json) +trap 'rm -f "$report"' EXIT + +bun scripts/planes-import.ts --report-new "$@" >"$report" +bun scripts/planes-photos.ts --missing --enrich "$report" --fetch diff --git a/scripts/planes-wiki.ts b/scripts/planes-wiki.ts new file mode 100644 index 0000000..6c50027 --- /dev/null +++ b/scripts/planes-wiki.ts @@ -0,0 +1,184 @@ +// Shared HTTP helpers for the plane pokedex scripts (scripts/planes-import.ts, +// scripts/planes-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries, +// maxlag handling and an optional on-disk cache (--cache ). +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +export const UA = + 'StanRungePlaneDex/1.0 (https://stanrunge.dev/trackers/planes; plane pokedex importer) bun'; +export const log = (...a: unknown[]) => console.error(...a); +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +let cacheDir: string | null = null; +let cache: Record = {}; +let dirty = 0; +let cacheName = 'api.json'; + +/** Use `/` as a persistent response cache (only for dev runs and refreshes). */ +export async function useCache(dir: string | null, name = 'api.json') { + cacheDir = dir; + cacheName = name; + if (!dir) return; + await mkdir(dir, { recursive: true }); + try { + cache = JSON.parse(await readFile(join(dir, name), 'utf8')); + } catch { + cache = {}; + } +} +export async function saveCache(force = false) { + if (!cacheDir || (!force && dirty < 200)) return; + dirty = 0; + await writeFile(join(cacheDir, cacheName), JSON.stringify(cache)); +} + +export async function getJson(url: string, init?: RequestInit, useCached = true): Promise { + const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`; + if (useCached && key in cache) return cache[key] as T; + for (let attempt = 0; ; attempt++) { + let res: Response; + try { + res = await fetch(url, { + ...init, + headers: { + 'user-agent': UA, + 'api-user-agent': UA, + accept: 'application/json', + ...init?.headers + }, + signal: AbortSignal.timeout(120_000) + }); + } catch (e) { + if (attempt < 6) { + await sleep(2000 * 2 ** attempt); + continue; + } + throw e; + } + const retryAfter = Number(res.headers.get('retry-after')) || 0; + if (res.ok) { + const body = (await res.json()) as T & { error?: { code?: string } }; + if (body?.error?.code === 'maxlag' && attempt < 8) { + await sleep(Math.max(retryAfter, 5) * 1000); + continue; + } + if (useCached && cacheDir) { + cache[key] = body; + dirty++; + await saveCache(); + } + return body; + } + if ((res.status === 429 || res.status >= 500) && attempt < 6) { + await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt)); + continue; + } + throw new Error(`${res.status} ${url.slice(0, 200)}`); + } +} + +let lastCall = 0; +/** MediaWiki action API, one request at a time with a small gap (API etiquette). */ +export async function mw( + host: string, + params: Record, + useCached = true +): Promise { + const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params }); + const key = `https://${host}/w/api.php?${body}`; + if (!useCached || !(key in cache)) { + const wait = lastCall + 150 - Date.now(); + if (wait > 0) await sleep(wait); + lastCall = Date.now(); + } + return getJson(key, undefined, useCached); +} +export type MwPage = { + pageid?: number; + title: string; + missing?: boolean; + lastrevid?: number; + extract?: string; + images?: { title: string }[]; + imageinfo?: ImageInfo[]; + revisions?: { revid?: number; slots?: { main?: { content?: string } } }[]; + categorymembers?: { title: string }[]; +}; +export type ImageInfo = { + width: number; + height: number; + mime: string; + thumburl?: string; + descriptionurl?: string; + extmetadata?: Record; +}; +export type MwResponse = { + query?: { + pages?: MwPage[]; + normalized?: { from: string; to: string }[]; + redirects?: { from: string; to: string }[]; + categorymembers?: { title: string }[]; + }; + continue?: Record; +}; + +let lastSparql = 0; +export async function sparql>(query: string): Promise { + const wait = lastSparql + 400 - Date.now(); + if (wait > 0) await sleep(wait); + lastSparql = Date.now(); + const res = await getJson<{ results: { bindings: Record[] } }>( + 'https://query.wikidata.org/sparql', + { + method: 'POST', + headers: { + 'content-type': 'application/x-www-form-urlencoded', + accept: 'application/sparql-results+json' + }, + body: new URLSearchParams({ query }).toString() + } + ); + return res.results.bindings.map( + (b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T + ); +} + +export const qid = (uri: string) => uri.split('/').at(-1)!; +export const fileName = (uri: string) => + decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' '); +/** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */ +export const cmp = (s: string) => + s + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toUpperCase() + .replace(/&/g, ' AND ') + .replace(/[^A-Z0-9]/g, ''); +export const tokens = (s: string) => + s + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toUpperCase() + .replace(/&/g, ' ') + .split(/[^A-Z0-9]+/) + .filter(Boolean); +export const slugify = (s: string) => + s + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .replace(/&/g, ' and ') + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80) || 'x'; +export const chunks = (xs: T[], n: number) => + Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n)); +export const stripHtml = (s: string) => + s + .replace(/<[^>]+>/g, ' ') + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/�?39;/g, "'") + .replace(/ /g, ' ') + .replace(/\s+/g, ' ') + .trim();