1465 lines
51 KiB
TypeScript
1465 lines
51 KiB
TypeScript
// Builds the public plane pokedex (/trackers/planes, schema `planes`) from open data.
|
||
//
|
||
// DB_URL=... bun scripts/planes-import.ts fetch sources and write to the database
|
||
// bun scripts/planes-import.ts --dry-run fetch + build only, print a summary
|
||
// bun scripts/planes-import.ts --cache /tmp/planes reuse API responses from that folder
|
||
// bun scripts/planes-import.ts --dry-run --out x.json also dump the built catalogue
|
||
// bun scripts/planes-import.ts --renumber reassign Dex numbers from #1 (oldest first)
|
||
// bun scripts/planes-import.ts --report-new JSON of makers/types new since the last run on stdout
|
||
//
|
||
// Sources (no API keys):
|
||
// - Wikidata (CC0): every item that is an "aircraft family" (Q15056993), "aircraft model"
|
||
// (Q15056995) or "aircraft type" (Q45296117) and has an English Wikipedia article:
|
||
// manufacturer (P176/P178), country (P495), first flight (P606), service entry/retirement
|
||
// (P729/P730), number built (P1092), capacity (P1083), engines (P516), wing configuration
|
||
// (P1654), undercarriage (P1637), dimensions/mass/speed/range, classes (P279) and uses (P366).
|
||
// - English Wikipedia (CC BY-SA): the article's {{Infobox aircraft}} (type, status, number
|
||
// built) and {{Aircraft specs}} / {{Aircraft specifications}} block (crew, capacity,
|
||
// engines, dimensions, MTOW, speed, range, ceiling), plus a 3-sentence intro extract.
|
||
// - "List of aircraft type designators" on Wikipedia for ICAO type codes.
|
||
// Only production types: prototypes, cancelled projects, drones, missiles, spacecraft and
|
||
// types with fewer than 5 built are skipped. Unknown values stay null; nothing is estimated.
|
||
// Re-runnable: types are upserted by Wikidata id (Dex numbers stay stable, new types get the
|
||
// next number), types that vanished are deleted, makers are upserted by slug.
|
||
import postgres from 'postgres';
|
||
import { readFile, writeFile } from 'node:fs/promises';
|
||
import { join } from 'node:path';
|
||
import {
|
||
chunks,
|
||
cmp,
|
||
fileName,
|
||
log as logErr,
|
||
mw,
|
||
qid,
|
||
saveCache,
|
||
slugify,
|
||
sparql,
|
||
tokens,
|
||
useCache
|
||
} from './planes-wiki';
|
||
|
||
const args = process.argv.slice(2);
|
||
const dryRun = args.includes('--dry-run');
|
||
const renumber = args.includes('--renumber');
|
||
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
||
const cacheDir = opt('--cache');
|
||
const outFile = opt('--out');
|
||
const reportNew = args.includes('--report-new');
|
||
const SITE = 'https://stanrunge.dev/trackers/planes';
|
||
const log = (...a: unknown[]) => (reportNew || outFile === '-' ? logErr(...a) : console.log(...a));
|
||
const SRC_WD = 'Wikidata';
|
||
const SRC_WP = 'Wikipedia';
|
||
const KINDS: Record<string, string> = {
|
||
Q15056993: 'family',
|
||
Q15056995: 'model',
|
||
Q45296117: 'type'
|
||
};
|
||
const MIN_BUILT = 5;
|
||
const year = (s: string | undefined | null) => {
|
||
const m = s?.match(/\b(1[89]\d\d|20\d\d)\b/);
|
||
return m ? Number(m[1]) : null;
|
||
};
|
||
|
||
// ---------------------------------------------------------------- wikidata
|
||
|
||
type Item = {
|
||
id: string;
|
||
kind: string;
|
||
article: string;
|
||
label: string;
|
||
description: string | null;
|
||
aliases: string[];
|
||
links: number;
|
||
image: string | null;
|
||
category: string | null;
|
||
firstFlight: string | null; // ISO date, trimmed to its precision
|
||
introduced: number | null;
|
||
retired: number | null;
|
||
built: number | null;
|
||
capacity: number | null;
|
||
countries: string[];
|
||
makers: { id: string; label: string }[];
|
||
developers: string[];
|
||
classes: string[];
|
||
uses: string[];
|
||
engines: { id: string; label: string; qty: number | null }[];
|
||
wing: string | null;
|
||
gear: string | null;
|
||
wingspan: number | null;
|
||
length: number | null;
|
||
height: number | null;
|
||
mass: number | null;
|
||
range: number | null; // m
|
||
speed: number | null; // m/s
|
||
cruise: number | null;
|
||
ceiling: number | null;
|
||
};
|
||
|
||
/** Too generic to walk past (aircraft, vehicle, machine, engine, ...). */
|
||
const STOP = new Set([
|
||
'Q11436',
|
||
'Q42889',
|
||
'Q11019',
|
||
'Q44167',
|
||
'Q16686448',
|
||
'Q223557',
|
||
'Q35120',
|
||
'Q488383',
|
||
'Q7397',
|
||
'Q1183543',
|
||
'Q2424752',
|
||
'Q39546',
|
||
'Q17537576'
|
||
]);
|
||
/** Labels and parent classes of `ids` and their ancestors up to `depth` levels (P279; P31 too for the first level when `instance`). */
|
||
async function walkUp(ids: string[], depth: number, instance = false) {
|
||
const nodes = new Map<string, { label: string | null; parents: string[] }>();
|
||
let frontier = [...new Set(ids)];
|
||
for (let level = 0; level <= depth && frontier.length; level++) {
|
||
const next = new Set<string>();
|
||
for (const c of chunks(frontier, 300)) {
|
||
const path = instance && level === 0 ? 'wdt:P279|wdt:P31' : 'wdt:P279';
|
||
const rows =
|
||
await sparql(`SELECT ?c ?l ?p WHERE { VALUES ?c { ${c.map((x) => `wd:${x}`).join(' ')} }
|
||
OPTIONAL { ?c rdfs:label ?l FILTER(LANG(?l) = "en") } OPTIONAL { ?c ${path} ?p } }`);
|
||
for (const r of rows) {
|
||
const id = qid(r.c);
|
||
const node = nodes.get(id) ?? { label: null, parents: [] };
|
||
if (r.l) node.label = r.l.toLowerCase();
|
||
if (r.p && !STOP.has(id)) {
|
||
const p = qid(r.p);
|
||
if (!node.parents.includes(p)) node.parents.push(p);
|
||
if (!nodes.has(p)) next.add(p);
|
||
}
|
||
nodes.set(id, node);
|
||
}
|
||
}
|
||
frontier = [...next].filter((x) => !nodes.has(x));
|
||
}
|
||
return nodes;
|
||
}
|
||
|
||
async function values(
|
||
ids: string[],
|
||
body: string,
|
||
select = '?v'
|
||
): Promise<Record<string, string>[]> {
|
||
return sparql(
|
||
`SELECT ?i ${select} WHERE { VALUES ?i { ${ids.map((i) => `wd:${i}`).join(' ')} } ${body} }`
|
||
);
|
||
}
|
||
const en = (v: string, l: string) => `${v} rdfs:label ${l} FILTER(LANG(${l}) = "en")`;
|
||
|
||
async function loadWikidata() {
|
||
log('wikidata: listing aircraft families/models with an English article');
|
||
const list = await sparql(`
|
||
SELECT ?i ?kind ?article WHERE {
|
||
VALUES ?kind { ${Object.keys(KINDS)
|
||
.map((k) => `wd:${k}`)
|
||
.join(' ')} }
|
||
?i wdt:P31 ?kind .
|
||
?article schema:about ?i ; schema:isPartOf <https://en.wikipedia.org/> .
|
||
}`);
|
||
const items = new Map<string, Item>();
|
||
for (const r of list) {
|
||
const id = qid(r.i);
|
||
if (items.has(id)) continue;
|
||
items.set(id, {
|
||
id,
|
||
kind: KINDS[qid(r.kind)],
|
||
article: decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' '),
|
||
label: id,
|
||
description: null,
|
||
aliases: [],
|
||
links: 0,
|
||
image: null,
|
||
category: null,
|
||
firstFlight: null,
|
||
introduced: null,
|
||
retired: null,
|
||
built: null,
|
||
capacity: null,
|
||
countries: [],
|
||
makers: [],
|
||
developers: [],
|
||
classes: [],
|
||
uses: [],
|
||
engines: [],
|
||
wing: null,
|
||
gear: null,
|
||
wingspan: null,
|
||
length: null,
|
||
height: null,
|
||
mass: null,
|
||
range: null,
|
||
speed: null,
|
||
cruise: null,
|
||
ceiling: null
|
||
});
|
||
}
|
||
log(`wikidata: ${items.size} items`);
|
||
const ids = [...items.keys()];
|
||
let n = 0;
|
||
for (const c of chunks(ids, 350)) {
|
||
const get = (r: Record<string, string>) => items.get(qid(r.i))!;
|
||
for (const r of await values(
|
||
c,
|
||
`OPTIONAL { ${en('?i', '?l')} } OPTIONAL { ?i schema:description ?d FILTER(LANG(?d) = "en") }
|
||
OPTIONAL { ?i wikibase:sitelinks ?sl }`,
|
||
'?l ?d ?sl'
|
||
)) {
|
||
const it = get(r);
|
||
if (r.l) it.label = r.l;
|
||
if (r.d) it.description = r.d;
|
||
it.links = Number(r.sl ?? 0);
|
||
}
|
||
for (const r of await values(c, `?i skos:altLabel ?v FILTER(LANG(?v) = "en")`))
|
||
get(r).aliases.push(r.v);
|
||
for (const r of await values(c, `?i wdt:P18 ?v`)) get(r).image ??= fileName(r.v);
|
||
for (const r of await values(c, `?i wdt:P373 ?v`)) get(r).category ??= r.v;
|
||
for (const r of await values(
|
||
c,
|
||
`?i p:P606/psv:P606 [ wikibase:timeValue ?v ; wikibase:timePrecision ?p ]`,
|
||
'?v ?p'
|
||
)) {
|
||
const it = get(r);
|
||
const p = Number(r.p);
|
||
const iso = r.v.slice(0, p >= 11 ? 10 : p === 10 ? 7 : 4);
|
||
if (!/^\d{4}/.test(iso)) continue;
|
||
if (!it.firstFlight || iso.slice(0, 4) < it.firstFlight.slice(0, 4)) it.firstFlight = iso;
|
||
}
|
||
for (const r of await values(c, `?i wdt:P729 ?v`)) {
|
||
const it = get(r);
|
||
const y = year(r.v);
|
||
if (y && (!it.introduced || y < it.introduced)) it.introduced = y;
|
||
}
|
||
for (const r of await values(c, `?i wdt:P730 ?v`)) {
|
||
const it = get(r);
|
||
const y = year(r.v);
|
||
if (y && (!it.retired || y > it.retired)) it.retired = y;
|
||
}
|
||
for (const r of await values(c, `?i wdt:P1092 ?v`)) {
|
||
const it = get(r);
|
||
const v = Math.round(Number(r.v));
|
||
if (Number.isFinite(v) && v >= 0 && (it.built == null || v > it.built)) it.built = v;
|
||
}
|
||
for (const r of await values(c, `?i wdt:P1083 ?v`)) {
|
||
const it = get(r);
|
||
const v = Math.round(Number(r.v));
|
||
if (Number.isFinite(v) && v > 0 && (it.capacity == null || v > it.capacity)) it.capacity = v;
|
||
}
|
||
for (const r of await values(c, `?i wdt:P495 ?c . ${en('?c', '?v')}`))
|
||
get(r).countries.push(r.v);
|
||
for (const r of await values(c, `?i wdt:P176 ?m . ${en('?m', '?v')}`, '?m ?v')) {
|
||
const it = get(r);
|
||
if (!it.makers.some((m) => m.id === qid(r.m))) it.makers.push({ id: qid(r.m), label: r.v });
|
||
}
|
||
for (const r of await values(c, `?i wdt:P178 ?v`)) get(r).developers.push(qid(r.v));
|
||
for (const r of await values(c, `?i wdt:P279 ?v`)) get(r).classes.push(qid(r.v));
|
||
for (const r of await values(c, `?i wdt:P366 ?v`)) get(r).uses.push(qid(r.v));
|
||
for (const r of await values(
|
||
c,
|
||
`?i p:P516 ?st . ?st ps:P516 ?e . OPTIONAL { ?st pq:P1114 ?q } OPTIONAL { ${en('?e', '?el')} }`,
|
||
'?e ?el ?q'
|
||
)) {
|
||
const it = get(r);
|
||
if (!it.engines.some((e) => e.id === qid(r.e)))
|
||
it.engines.push({ id: qid(r.e), label: r.el ?? '', qty: r.q ? Number(r.q) : null });
|
||
}
|
||
for (const r of await values(c, `?i wdt:P1654 ?w . ${en('?w', '?v')}`)) get(r).wing ??= r.v;
|
||
for (const r of await values(c, `?i wdt:P1637 ?w . ${en('?w', '?v')}`)) get(r).gear ??= r.v;
|
||
const quantity = async (prop: string, key: keyof Item, pick: 'max' | 'min' = 'max') => {
|
||
for (const r of await values(c, `?i p:${prop}/psn:${prop}/wikibase:quantityAmount ?v`)) {
|
||
const it = get(r);
|
||
const v = Number(r.v);
|
||
if (!Number.isFinite(v) || v <= 0) continue;
|
||
const cur = it[key] as number | null;
|
||
if (cur == null || (pick === 'max' ? v > cur : v < cur)) (it[key] as number) = v;
|
||
}
|
||
};
|
||
await quantity('P2050', 'wingspan');
|
||
await quantity('P2043', 'length');
|
||
await quantity('P2048', 'height');
|
||
await quantity('P2067', 'mass');
|
||
await quantity('P2073', 'range');
|
||
await quantity('P2052', 'speed');
|
||
await quantity('P2217', 'cruise');
|
||
await quantity('P2254', 'ceiling');
|
||
n += c.length;
|
||
log(`wikidata: details ${n}/${ids.length}`);
|
||
await saveCache(true);
|
||
}
|
||
|
||
// Class/use ancestors with English labels (for roles, engine counts and exclusions).
|
||
// Walked breadth-first a few levels up (P279* over the whole ontology is too slow), stopping
|
||
// at very generic classes.
|
||
const classIds = [...new Set([...items.values()].flatMap((i) => [...i.classes, ...i.uses]))];
|
||
const parents = await walkUp(classIds, 5);
|
||
const ancestors = new Map<string, Set<string>>(); // class id -> lowercase labels of itself + ancestors
|
||
for (const c of classIds) {
|
||
const seen = new Set<string>([c]);
|
||
const queue = [c];
|
||
const labels = new Set<string>();
|
||
while (queue.length) {
|
||
const x = queue.shift()!;
|
||
const node = parents.get(x);
|
||
if (!node) continue;
|
||
if (node.label) labels.add(node.label);
|
||
for (const p of node.parents)
|
||
if (!seen.has(p)) {
|
||
seen.add(p);
|
||
queue.push(p);
|
||
}
|
||
}
|
||
ancestors.set(c, labels);
|
||
}
|
||
// Own labels of the direct classes (to read "with 2 engines" only from the closest class).
|
||
const classLabels = new Map<string, string>();
|
||
for (const c of chunks(classIds, 300))
|
||
for (const r of await sparql(
|
||
`SELECT ?c ?l WHERE { VALUES ?c { ${c.map((x) => `wd:${x}`).join(' ')} } ${en('?c', '?l')} }`
|
||
))
|
||
classLabels.set(qid(r.c), r.l.toLowerCase());
|
||
// Engine item classes ("turbofan", "radial engine", ...).
|
||
const engineIds = [...new Set([...items.values()].flatMap((i) => i.engines.map((e) => e.id)))];
|
||
const engineUp = await walkUp(engineIds, 4, true);
|
||
const engineKinds = new Map<string, Set<string>>();
|
||
for (const e of engineIds) {
|
||
const seen = new Set<string>([e]);
|
||
const queue = [e];
|
||
const labels = new Set<string>();
|
||
while (queue.length) {
|
||
const node = engineUp.get(queue.shift()!);
|
||
if (!node) continue;
|
||
if (node.label) labels.add(node.label);
|
||
for (const p of node.parents)
|
||
if (!seen.has(p)) {
|
||
seen.add(p);
|
||
queue.push(p);
|
||
}
|
||
}
|
||
engineKinds.set(e, labels);
|
||
}
|
||
// Maker countries.
|
||
const makerIds = [...new Set([...items.values()].flatMap((i) => i.makers.map((m) => m.id)))];
|
||
const makerCountry = new Map<string, string>();
|
||
for (const c of chunks(makerIds, 300))
|
||
for (const r of await sparql(
|
||
`SELECT ?m ?v WHERE { VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } ?m wdt:P17 ?c . ${en('?c', '?v')} }`
|
||
))
|
||
makerCountry.set(qid(r.m), r.v);
|
||
// English aliases of makers ("Eurocopter" for Airbus Helicopters), for matching article titles.
|
||
const makerAliases = new Map<string, string[]>();
|
||
for (const c of chunks(makerIds, 300))
|
||
for (const r of await sparql(
|
||
`SELECT ?m ?a WHERE { VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } ?m skos:altLabel ?a FILTER(LANG(?a) = "en") }`
|
||
))
|
||
makerAliases.set(qid(r.m), [...(makerAliases.get(qid(r.m)) ?? []), r.a]);
|
||
await saveCache(true);
|
||
return { items, ancestors, classLabels, engineKinds, makerCountry, makerAliases };
|
||
}
|
||
|
||
// ---------------------------------------------------------------- wikipedia
|
||
|
||
type Wiki = {
|
||
title: string; // resolved article title
|
||
extract: string | null;
|
||
infobox: Record<string, string>;
|
||
specs: Record<string, string>;
|
||
legacySpecs: boolean;
|
||
};
|
||
|
||
/** The `{{name ...}}` template (first match), with nested braces balanced. */
|
||
function findTemplate(text: string, name: RegExp): string | null {
|
||
const re = /\{\{\s*([^|}\n]+)/g;
|
||
let m: RegExpExecArray | null;
|
||
while ((m = re.exec(text))) {
|
||
if (!name.test(m[1].trim())) continue;
|
||
let depth = 0;
|
||
for (let i = m.index; i < text.length - 1; i++) {
|
||
if (text[i] === '{' && text[i + 1] === '{') {
|
||
depth++;
|
||
i++;
|
||
} else if (text[i] === '}' && text[i + 1] === '}') {
|
||
depth--;
|
||
i++;
|
||
if (depth === 0) return text.slice(m.index + 2, i - 1);
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
return null;
|
||
}
|
||
function templateParams(tpl: string): Record<string, string> {
|
||
const out: Record<string, string> = {};
|
||
const parts: string[] = [];
|
||
let depth = 0;
|
||
let link = 0;
|
||
let cur = '';
|
||
for (let i = 0; i < tpl.length; i++) {
|
||
const two = tpl.slice(i, i + 2);
|
||
if (two === '{{') depth++;
|
||
else if (two === '}}') depth--;
|
||
else if (two === '[[') link++;
|
||
else if (two === ']]') link--;
|
||
if (tpl[i] === '|' && depth === 0 && link === 0) {
|
||
parts.push(cur);
|
||
cur = '';
|
||
continue;
|
||
}
|
||
cur += tpl[i];
|
||
}
|
||
parts.push(cur);
|
||
for (const p of parts.slice(1)) {
|
||
const eq = p.indexOf('=');
|
||
if (eq < 0) continue;
|
||
const key = p.slice(0, eq).trim().toLowerCase().replace(/_/g, ' ');
|
||
const value = p.slice(eq + 1).trim();
|
||
if (key && value) out[key] = value;
|
||
}
|
||
return out;
|
||
}
|
||
/** Wikitext to plain text: no refs, comments, markup; templates keep their unnamed arguments. */
|
||
export function plain(v: string): string {
|
||
let s = v
|
||
.replace(/<!--[\s\S]*?-->/g, '')
|
||
.replace(/<ref[^>]*\/>/gi, '')
|
||
.replace(/<ref[^>]*>[\s\S]*?<\/ref>/gi, '')
|
||
.replace(/<br\s*\/?>/gi, ', ')
|
||
.replace(/<[^>]+>/g, '');
|
||
for (let guard = 0; guard < 10 && /\{\{/.test(s); guard++)
|
||
s = s.replace(/\{\{([^{}]*)\}\}/g, (_, inner: string) => {
|
||
const [name, ...rest] = inner.split('|');
|
||
const n = name.trim().toLowerCase();
|
||
if (/^(cvt|convert)$/.test(n)) return rest.slice(0, 2).join(' ');
|
||
if (/^(efn|sfn|refn|citation needed|cn|when|as of|dubious|clarify|webarchive|nbsp)/.test(n))
|
||
return /^as of/.test(n) ? `as of ${rest.join(' ')}` : '';
|
||
return rest
|
||
.filter((r) => !/^\s*[a-z_ ]+\s*=/.test(r))
|
||
.map((r) => r.trim())
|
||
.filter(Boolean)
|
||
.join(', ');
|
||
});
|
||
s = s
|
||
.replace(/\[\[(?:[^|\]]*\|)?([^\]]*)\]\]/g, '$1')
|
||
.replace(/\[https?:[^\s\]]+\s*([^\]]*)\]/g, '$1')
|
||
.replace(/'{2,}/g, '')
|
||
.replace(/ /g, ' ')
|
||
.replace(/\s*\n\s*\*\s*/g, ', ')
|
||
.replace(/^\s*\*\s*/, '')
|
||
.replace(/\s+/g, ' ')
|
||
.replace(/^[,\s]+|[,\s]+$/g, '')
|
||
.trim();
|
||
return s;
|
||
}
|
||
const num = (v: string | undefined | null): number | null => {
|
||
if (!v) return null;
|
||
const m = plain(v)
|
||
.replace(/(\d),(\d{3})/g, '$1$2')
|
||
.replace(/(\d),(\d{3})/g, '$1$2')
|
||
.match(/\d+(?:\.\d+)?/);
|
||
return m ? Number(m[0]) : null;
|
||
};
|
||
|
||
async function loadWikipedia(titles: string[]) {
|
||
const out = new Map<string, Wiki>(); // keyed by the title we asked for
|
||
const todo = [...new Set(titles)];
|
||
let done = 0;
|
||
for (const c of chunks(todo, 20)) {
|
||
const res = await mw('en.wikipedia.org', {
|
||
action: 'query',
|
||
prop: 'revisions|extracts',
|
||
rvprop: 'content',
|
||
rvslots: 'main',
|
||
exintro: '1',
|
||
explaintext: '1',
|
||
exsentences: '3',
|
||
exlimit: '20',
|
||
redirects: '1',
|
||
titles: c.join('|')
|
||
});
|
||
const back = new Map<string, string>();
|
||
for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])])
|
||
back.set(n.to, back.get(n.from) ?? n.from);
|
||
for (const p of res.query?.pages ?? []) {
|
||
const asked = back.get(p.title) ?? p.title;
|
||
const text = p.revisions?.[0]?.slots?.main?.content ?? '';
|
||
const box =
|
||
findTemplate(text, /^infobox aircraft type$/i) ??
|
||
findTemplate(text, /^infobox aircraft$/i) ??
|
||
findTemplate(text, /^infobox (aircraft|helicopter|military aircraft|glider)\b/i);
|
||
const begin = findTemplate(text, /^infobox aircraft begin$/i);
|
||
let specsTpl = findTemplate(text, /^aircraft specs$/i);
|
||
let legacy = false;
|
||
if (!specsTpl) {
|
||
specsTpl = findTemplate(text, /^aircraft specifications$|^aerospecs$/i);
|
||
legacy = !!specsTpl;
|
||
}
|
||
out.set(asked, {
|
||
title: p.title,
|
||
extract: p.extract?.trim() || null,
|
||
infobox: { ...(begin ? templateParams(begin) : {}), ...(box ? templateParams(box) : {}) },
|
||
specs: specsTpl ? templateParams(specsTpl) : {},
|
||
legacySpecs: legacy
|
||
});
|
||
}
|
||
done += c.length;
|
||
if (done % 400 === 0 || done === todo.length) {
|
||
log(`wikipedia: ${done}/${todo.length} articles`);
|
||
await saveCache(true);
|
||
}
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/** ICAO type designators from Wikipedia's list, keyed by the (redirect-resolved) article. */
|
||
async function loadIcao() {
|
||
const res = await mw('en.wikipedia.org', {
|
||
action: 'query',
|
||
prop: 'revisions',
|
||
rvprop: 'content',
|
||
rvslots: 'main',
|
||
redirects: '1',
|
||
titles: 'List of aircraft type designators'
|
||
});
|
||
const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? '';
|
||
const rows: { code: string; link: string }[] = [];
|
||
for (const line of text.split('\n')) {
|
||
const m = line.match(/^\|\s*([A-Z0-9]{2,4})\s*\|\|[^|]*\|\|\s*\[\[([^\]|#]+)/);
|
||
if (m) rows.push({ code: m[1], link: m[2].trim() });
|
||
}
|
||
const resolved = new Map<string, string>();
|
||
for (const c of chunks([...new Set(rows.map((r) => r.link))], 50)) {
|
||
const r = await mw('en.wikipedia.org', {
|
||
action: 'query',
|
||
redirects: '1',
|
||
titles: c.join('|')
|
||
});
|
||
const map = new Map<string, string>();
|
||
for (const n of [...(r.query?.normalized ?? []), ...(r.query?.redirects ?? [])])
|
||
map.set(n.from, n.to);
|
||
for (const t of c) {
|
||
let cur = t;
|
||
for (let g = 0; g < 3 && map.has(cur); g++) cur = map.get(cur)!;
|
||
resolved.set(t, cur);
|
||
}
|
||
}
|
||
const byArticle = new Map<string, Set<string>>();
|
||
for (const r of rows) {
|
||
const a = resolved.get(r.link) ?? r.link;
|
||
if (!byArticle.has(a)) byArticle.set(a, new Set());
|
||
byArticle.get(a)!.add(r.code);
|
||
}
|
||
log(`icao: ${rows.length} designators for ${byArticle.size} articles`);
|
||
return byArticle;
|
||
}
|
||
|
||
// ---------------------------------------------------------------- classification
|
||
|
||
export const ROLES = [
|
||
'airliner',
|
||
'regional',
|
||
'cargo',
|
||
'business',
|
||
'general aviation',
|
||
'glider',
|
||
'helicopter',
|
||
'military',
|
||
'airship'
|
||
] as const;
|
||
const ROLE_RULES: [string, RegExp][] = [
|
||
[
|
||
'helicopter',
|
||
/helicopter|rotorcraft|gyroplane|autogyro|gyrocopter|tiltrotor|compound helicopter/
|
||
],
|
||
['regional', /regional|commuter|feederliner|feeder liner|turboprop airliner/],
|
||
[
|
||
'airliner',
|
||
/airliner|jetliner|passenger aircraft|passenger plane|airbus a3|wide-body|narrow-body/
|
||
],
|
||
['business', /business jet|business aircraft|bizjet|executive|very light jet|corporate jet/],
|
||
['cargo', /cargo|freighter|airlifter|transport aircraft|transport plane|tactical transport/],
|
||
[
|
||
'military',
|
||
/military|fighter|bomber|attack aircraft|interceptor|reconnaissance|maritime patrol|patrol aircraft|warplane|combat|gunship|torpedo|dive bomber|early warning|electronic warfare|air superiority|tanker|anti-submarine|ground-attack|strike aircraft|observation aircraft|army co-operation|naval aircraft|carrier-based|night fighter|attack helicopter|trainer.*military|military trainer|jet trainer/
|
||
],
|
||
['glider', /glider|sailplane/],
|
||
['airship', /airship|zeppelin|blimp|dirigible|balloon/],
|
||
[
|
||
'general aviation',
|
||
/light aircraft|utility aircraft|sport aircraft|sports aircraft|sportplane|aerobatic|trainer|agricultural|homebuilt|ultralight|microlight|general aviation|touring|bush plane|floatplane|seaplane|amphibi|flying boat|light-sport|civil utility|kit aircraft|air taxi|liaison aircraft|private aircraft|civil aircraft/
|
||
]
|
||
];
|
||
const PRIMARY = [
|
||
'helicopter',
|
||
'regional',
|
||
'airliner',
|
||
'business',
|
||
'military',
|
||
'cargo',
|
||
'glider',
|
||
'airship',
|
||
'general aviation'
|
||
];
|
||
const EXCLUDE =
|
||
/unmanned|\buav\b|\bdrone|missile|spacecraft|spaceplane|space plane|launch vehicle|\brocket\b(?!-powered| plane| aircraft)|experimental aircraft|research aircraft|prototype|concept aircraft|x-plane|target aircraft|cruise missile|space shuttle|technology demonstrator|aircraft project/;
|
||
|
||
const ENGINE_RULES: [string, RegExp][] = [
|
||
['turboprop', /turboprop|propfan/],
|
||
['turboshaft', /turboshaft/],
|
||
['jet', /turbofan|turbojet|jet engine|\bjets?\b|afterburn|ramjet/],
|
||
[
|
||
'piston',
|
||
/piston|radial|inline|in-line|flat-(?:four|six|twin)|boxer|opposed|v-?(?:8|12|16)\b|v engine|rotary engine|reciprocating|two-stroke|four-stroke|diesel|cylinder|liquid-cooled|air-cooled|internal combustion/
|
||
],
|
||
['electric', /electric motor|electric/],
|
||
['rocket', /rocket/]
|
||
];
|
||
export const engineTypeOf = (text: string): string | null =>
|
||
ENGINE_RULES.find(([, re]) => re.test(text.toLowerCase()))?.[0] ?? null;
|
||
|
||
const WORD_COUNT: Record<string, number> = {
|
||
single: 1,
|
||
one: 1,
|
||
twin: 2,
|
||
two: 2,
|
||
tri: 3,
|
||
three: 3,
|
||
four: 4,
|
||
quad: 4,
|
||
six: 6,
|
||
eight: 8
|
||
};
|
||
function engineCountFromLabel(label: string): number | null {
|
||
const m = label.match(/with (\d+) engines?/);
|
||
if (m) return Number(m[1]);
|
||
if (/\btwinjet\b/.test(label)) return 2;
|
||
if (/\btrijet\b/.test(label)) return 3;
|
||
if (/\bquadjet\b/.test(label)) return 4;
|
||
const w = label.match(
|
||
/\b(single|one|twin|two|tri|three|four|quad|six|eight)[- ]?(?:engine|engined|jet|motor)/
|
||
);
|
||
return w ? WORD_COUNT[w[1]] : null;
|
||
}
|
||
|
||
function passengers(capacity: string | undefined): number | null {
|
||
if (!capacity) return null;
|
||
const s = plain(capacity).replace(/(\d),(\d{3})/g, '$1$2');
|
||
let best: number | null = null;
|
||
for (const m of s.matchAll(
|
||
/(?<![\d/.])(\d+)(?:\s*(?:–|-|to)\s*(\d+))?\s*(?:\(?[a-z -]{0,24}\)?\s*)?(?:passengers?|pax|seats?|people|persons|troops)\b/gi
|
||
)) {
|
||
if (/\b(?:kg|lbs?|tonnes?|t|ft|m|km|cu)\b/i.test(m[0])) continue; // "900 kg pax" = payload
|
||
const v = Number(m[2] ?? m[1]);
|
||
if (v > 0 && v < 1000 && (best == null || v > best)) best = v;
|
||
}
|
||
return best;
|
||
}
|
||
|
||
type Unit = 'len' | 'mass' | 'speed' | 'range' | 'alt';
|
||
const UNIT_RE =
|
||
/(\d+(?:\.\d+)?)\s*(km\/h|kmh|km|mph|mi\b|nmi|kn\b|kts|knots|kg|lb|lbs|m\b|ft|in\b|mach)/gi;
|
||
/** Value in metric (m, kg, km/h, km, m) from a legacy "63 ft 9 in (19.4 m)" text. */
|
||
function measure(v: string | undefined, unit: Unit): number | null {
|
||
if (!v) return null;
|
||
const s = plain(v)
|
||
.replace(/(\d),(\d{3})/g, '$1$2')
|
||
.replace(/(\d),(\d{3})/g, '$1$2');
|
||
const hits = [...s.matchAll(UNIT_RE)].map((m) => ({ n: Number(m[1]), u: m[2].toLowerCase() }));
|
||
for (let i = 0; i < hits.length; i++) {
|
||
const { n, u } = hits[i];
|
||
if (unit === 'len' || unit === 'alt') {
|
||
if (u === 'm') return n;
|
||
if (u === 'ft') {
|
||
const inch = hits[i + 1]?.u === 'in' ? hits[i + 1].n : 0;
|
||
return n * 0.3048 + inch * 0.0254;
|
||
}
|
||
} else if (unit === 'mass') {
|
||
if (u === 'kg') return n;
|
||
if (u === 'lb' || u === 'lbs') return n * 0.4536;
|
||
} else if (unit === 'speed') {
|
||
if (u === 'km/h' || u === 'kmh') return n;
|
||
if (u === 'mph') return n * 1.609;
|
||
if (u === 'kn' || u === 'kts' || u === 'knots') return n * 1.852;
|
||
} else if (unit === 'range') {
|
||
if (u === 'km') return n;
|
||
if (u === 'mi') return n * 1.609;
|
||
if (u === 'nmi') return n * 1.852;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
type Specs = {
|
||
crew: number | null;
|
||
passengers: number | null;
|
||
length: number | null;
|
||
span: number | null;
|
||
height: number | null;
|
||
rotor: number | null;
|
||
mtow: number | null;
|
||
maxSpeed: number | null;
|
||
cruise: number | null;
|
||
range: number | null;
|
||
ceiling: number | null;
|
||
engineCount: number | null;
|
||
engineType: string | null;
|
||
engineNames: string[];
|
||
};
|
||
function parseSpecs(p: Record<string, string>, legacy: boolean): Specs {
|
||
const pick = (base: string, units: [string, number][]) => {
|
||
for (const [u, f] of units) {
|
||
const n = num(p[`${base} ${u}`]);
|
||
if (n != null && n > 0) {
|
||
if (u === 'ft') return n * f + (num(p[`${base} in`]) ?? 0) * 0.0254;
|
||
return n * f;
|
||
}
|
||
}
|
||
return null;
|
||
};
|
||
const L: [string, number][] = [
|
||
['m', 1],
|
||
['ft', 0.3048]
|
||
];
|
||
const engines: string[] = [];
|
||
let count: number | null = null;
|
||
let typeText = '';
|
||
if (!legacy) {
|
||
for (const k of ['eng1', 'eng2', 'eng3']) {
|
||
const n = num(p[`${k} number`]);
|
||
if (n != null && n > 0 && n <= 16) count = (count ?? 0) + Math.round(n);
|
||
if (p[`${k} name`]) engines.push(plain(p[`${k} name`]));
|
||
if (p[`${k} type`]) typeText += ` ${plain(p[`${k} type`])}`;
|
||
}
|
||
} else {
|
||
const n = num(p['number of jets']) ?? num(p['number of props']) ?? num(p['number of engines']);
|
||
if (n != null && n > 0 && n <= 16) count = Math.round(n);
|
||
for (const k of ['engine (jet)', 'engine (prop)', 'engine'])
|
||
if (p[k]) engines.push(plain(p[k]));
|
||
typeText = `${plain(p['type of jet'] ?? '')} ${plain(p['type of prop'] ?? '')} ${
|
||
/jet/i.test(p['jet or prop?'] ?? '') && !p['type of jet'] ? 'jet' : ''
|
||
}`;
|
||
}
|
||
const s: Specs = {
|
||
crew: num(p.crew),
|
||
passengers: passengers(p.capacity),
|
||
length: legacy ? measure(p['length main'], 'len') : pick('length', L),
|
||
span: legacy ? measure(p['span main'], 'len') : pick('span', L),
|
||
height: legacy ? measure(p['height main'], 'len') : pick('height', L),
|
||
rotor: legacy ? measure(p['rotor diameter main'], 'len') : pick('rot dia', L),
|
||
mtow: legacy
|
||
? measure(p['max takeoff weight main'] ?? p['loaded weight main'], 'mass')
|
||
: (pick('max takeoff weight', [
|
||
['kg', 1],
|
||
['lb', 0.4536],
|
||
['t', 1000]
|
||
]) ??
|
||
pick('gross weight', [
|
||
['kg', 1],
|
||
['lb', 0.4536]
|
||
])),
|
||
maxSpeed: legacy
|
||
? measure(p['max speed main'], 'speed')
|
||
: pick('max speed', [
|
||
['kmh', 1],
|
||
['mph', 1.609],
|
||
['kts', 1.852],
|
||
['kn', 1.852]
|
||
]),
|
||
cruise: legacy
|
||
? measure(p['cruise speed main'], 'speed')
|
||
: pick('cruise speed', [
|
||
['kmh', 1],
|
||
['mph', 1.609],
|
||
['kts', 1.852],
|
||
['kn', 1.852]
|
||
]),
|
||
range: legacy
|
||
? measure(p['range main'], 'range')
|
||
: pick('range', [
|
||
['km', 1],
|
||
['mi', 1.609],
|
||
['nmi', 1.852]
|
||
]),
|
||
ceiling: legacy
|
||
? measure(p['ceiling main'], 'alt')
|
||
: pick('ceiling', [
|
||
['m', 1],
|
||
['ft', 0.3048]
|
||
]),
|
||
engineCount: count,
|
||
engineType: engineTypeOf(typeText),
|
||
engineNames: engines.filter((e) => e && e.length < 80)
|
||
};
|
||
if (s.crew != null && (s.crew < 0 || s.crew > 30)) s.crew = null;
|
||
return s;
|
||
}
|
||
|
||
function status(text: string | undefined): { label: string | null; exclude: boolean } {
|
||
const s = plain(text ?? '').toLowerCase();
|
||
if (!s) return { label: null, exclude: false };
|
||
if (/in production/.test(s)) return { label: 'in production', exclude: false };
|
||
if (/in service|active|operational|in use|in limited service/.test(s))
|
||
return { label: 'in service', exclude: false };
|
||
if (
|
||
/retired|out of service|withdrawn|inactive|out of production|no longer|historic|museum|preserved|display/.test(
|
||
s
|
||
)
|
||
)
|
||
return { label: 'retired', exclude: false };
|
||
if (/development|testing|flight test|certification/.test(s))
|
||
return { label: 'in development', exclude: false };
|
||
if (
|
||
/prototype|cancel|design study|proposal|proposed|paper|study|experimental|project|concept|abandoned|demonstrator|research|one-off|unbuilt|never built|destroyed|crashed/.test(
|
||
s
|
||
)
|
||
)
|
||
return { label: null, exclude: true };
|
||
return { label: null, exclude: false };
|
||
}
|
||
|
||
// ---------------------------------------------------------------- build
|
||
|
||
const CORP = new Set(
|
||
'COMPANY CORPORATION CORP INC INCORPORATED LTD LIMITED GMBH AG SA SAS SPA PLC CO LLC AIRCRAFT AIRCRAFTS AIRPLANE AIRPLANES AEROPLANE AEROPLANES AVIATION INDUSTRIES INDUSTRY WORKS MANUFACTURING GROUP THE OF FLUGZEUGBAU FLUGZEUGWERKE AVIONS AERONAUTICS AERONAUTICAL ENGINEERING MOTORS'.split(
|
||
' '
|
||
)
|
||
);
|
||
/** Words that are not a maker on their own ("British" Aircraft Corporation). */
|
||
const GENERIC = new Set(
|
||
'BRITISH AMERICAN GENERAL NATIONAL UNITED INTERNATIONAL CONSOLIDATED CANADIAN AUSTRALIAN ROYAL IMPERIAL STATE FEDERAL CENTRAL NORTHERN SOUTHERN EASTERN WESTERN NORTH SOUTH EAST WEST AERO AIR SKY'.split(
|
||
' '
|
||
)
|
||
);
|
||
const makerKey = (label: string) => {
|
||
const t = tokens(label.replace(/\(.*?\)/g, ''));
|
||
const k = t.filter((x) => !CORP.has(x));
|
||
return (k.length && !(k.length === 1 && GENERIC.has(k[0])) ? k : t).join(' ');
|
||
};
|
||
const startsWith = (a: string[], b: string[]) => b.length > 0 && b.every((x, i) => a[i] === x);
|
||
const cleanTitle = (t: string) =>
|
||
t
|
||
.replace(/\s*\((?:aircraft|airplane|aeroplane|helicopter|glider|airship|plane)[^)]*\)$/i, '')
|
||
.trim();
|
||
|
||
export type BuiltType = {
|
||
wikidata: string;
|
||
kind: string;
|
||
slug: string;
|
||
name: string;
|
||
fullName: string;
|
||
search: string;
|
||
wikipedia: string;
|
||
description: string | null;
|
||
summary: string | null;
|
||
role: string | null;
|
||
roles: string[];
|
||
roleLabel: string | null;
|
||
country: string | null;
|
||
firstFlight: string | null;
|
||
firstFlightYear: number | null;
|
||
introducedYear: number | null;
|
||
retiredYear: number | null;
|
||
status: string | null;
|
||
inService: boolean | null;
|
||
numberBuilt: number | null;
|
||
makers: string[];
|
||
engineType: string | null;
|
||
engineCount: number | null;
|
||
engines: string[];
|
||
crew: number | null;
|
||
passengers: number | null;
|
||
lengthM: number | null;
|
||
wingspanM: number | null;
|
||
heightM: number | null;
|
||
rotorM: number | null;
|
||
mtowKg: number | null;
|
||
maxSpeedKmh: number | null;
|
||
cruiseKmh: number | null;
|
||
rangeKm: number | null;
|
||
ceilingM: number | null;
|
||
wing: string | null;
|
||
gear: string | null;
|
||
icao: string[];
|
||
recognize: string | null;
|
||
parent: string | null;
|
||
sources: string[];
|
||
links: number;
|
||
};
|
||
export type BuiltMaker = {
|
||
slug: string;
|
||
name: string;
|
||
fullName: string;
|
||
wikidata: string;
|
||
country: string | null;
|
||
types: BuiltType[];
|
||
};
|
||
|
||
const round = (n: number | null, d = 0) => (n == null ? null : Math.round(n * 10 ** d) / 10 ** d);
|
||
const sane = (n: number | null, min: number, max: number) =>
|
||
n != null && n >= min && n <= max ? n : null;
|
||
|
||
function recognizeLine(t: BuiltType): string | null {
|
||
const parts: string[] = [];
|
||
const engineWord: Record<string, string> = {
|
||
jet: 'jet engine',
|
||
turboprop: 'turboprop',
|
||
piston: 'piston engine',
|
||
turboshaft: 'turboshaft engine',
|
||
electric: 'electric motor',
|
||
rocket: 'rocket engine'
|
||
};
|
||
if (t.engineType && t.engineType !== 'none') {
|
||
const w = engineWord[t.engineType];
|
||
const prop = t.engineType === 'piston' && t.role !== 'helicopter' ? ' with propeller' : '';
|
||
parts.push(
|
||
t.engineCount
|
||
? `${t.engineCount} ${w}${t.engineCount > 1 ? 's' : ''}${prop ? `${prop}${t.engineCount > 1 ? 's' : ''}` : ''}`
|
||
: `${w}s`.replace(/ss$/, 's')
|
||
);
|
||
} else if (t.engineCount && t.engineCount > 0)
|
||
parts.push(`${t.engineCount} engine${t.engineCount > 1 ? 's' : ''}`);
|
||
if (t.role === 'glider' && !t.engineType) parts.push('no engine, long thin wings');
|
||
if (t.wing) {
|
||
const w = t.wing.toLowerCase();
|
||
if (/biplane|triplane|monoplane|wing/.test(w) && !/^monoplane$/.test(w)) parts.push(w);
|
||
}
|
||
if (t.gear) {
|
||
const g = t.gear.toLowerCase();
|
||
if (/tricycle|tail|conventional|float|skid|retractable|fixed|ski|hull/.test(g))
|
||
parts.push(
|
||
g
|
||
.replace('conventional landing gear', 'tailwheel (taildragger)')
|
||
.replace(/ undercarriage$/, ' gear')
|
||
);
|
||
}
|
||
if (t.role === 'helicopter' && t.rotorM) parts.push(`${round(t.rotorM, 1)} m main rotor`);
|
||
if (t.wingspanM && t.role !== 'helicopter') parts.push(`${round(t.wingspanM, 1)} m wingspan`);
|
||
if (parts.length < 2) return null;
|
||
return parts.slice(0, 4).join(' · ');
|
||
}
|
||
|
||
function build(
|
||
wd: Awaited<ReturnType<typeof loadWikidata>>,
|
||
wiki: Map<string, Wiki>,
|
||
icao: Map<string, Set<string>>
|
||
) {
|
||
const { items, ancestors, classLabels, engineKinds, makerCountry, makerAliases } = wd;
|
||
const skipped: Record<string, number> = {};
|
||
const skip = (why: string, n = 1) => (skipped[why] = (skipped[why] ?? 0) + n);
|
||
// How many items each maker has, to break ties and to find "brand" makers.
|
||
const makerUse = new Map<string, number>();
|
||
const makerLabel = new Map<string, string>();
|
||
for (const it of items.values())
|
||
for (const m of it.makers) {
|
||
makerUse.set(m.id, (makerUse.get(m.id) ?? 0) + 1);
|
||
makerLabel.set(m.id, m.label);
|
||
}
|
||
const companyNames = new Set([...makerLabel.values()].map((l) => l.toLowerCase()));
|
||
const byKey = new Map<string, string>(); // makerKey -> most used maker id
|
||
for (const [id, label] of makerLabel) {
|
||
const k = makerKey(label);
|
||
const cur = byKey.get(k);
|
||
if (!cur || (makerUse.get(id) ?? 0) > (makerUse.get(cur) ?? 0)) byKey.set(k, id);
|
||
}
|
||
|
||
type Draft = BuiltType & { makerId: string; prefix: string };
|
||
const drafts: Draft[] = [];
|
||
for (const it of items.values()) {
|
||
const w = wiki.get(it.article);
|
||
const anc = new Set([...it.classes, ...it.uses].flatMap((c) => [...(ancestors.get(c) ?? [])]));
|
||
const ancText = [...anc].join(' | ');
|
||
const box = w?.infobox ?? {};
|
||
const typeText = plain(box['aircraft type'] ?? box.type ?? '').toLowerCase();
|
||
const st = status(box.status);
|
||
const builtBox = num(box['number built']);
|
||
const built = it.built ?? (builtBox != null ? Math.round(builtBox) : null);
|
||
if (EXCLUDE.test(ancText) || EXCLUDE.test(typeText)) {
|
||
skip('drone/prototype/experimental class');
|
||
continue;
|
||
}
|
||
if (st.exclude) {
|
||
skip('status prototype/cancelled');
|
||
continue;
|
||
}
|
||
if (built != null && built < MIN_BUILT) {
|
||
skip(`fewer than ${MIN_BUILT} built`);
|
||
continue;
|
||
}
|
||
if (!w) {
|
||
skip('no article');
|
||
continue;
|
||
}
|
||
if (/\bvariants\b|^list of|operators$|\bin (?:.+ )?service$/i.test(w.title)) {
|
||
skip('variants/list article');
|
||
continue;
|
||
}
|
||
if (companyNames.has(w.title.toLowerCase())) {
|
||
skip('article is about the company');
|
||
continue;
|
||
}
|
||
const title = cleanTitle(w.title);
|
||
const tt = tokens(title);
|
||
// Maker: the manufacturer the article title starts with, else a known maker named like the
|
||
// title's first words, else the developer, else the most prolific manufacturer.
|
||
let makerId: string | null = null;
|
||
let prefix = '';
|
||
const cands = it.makers.length ? it.makers : [];
|
||
const full = cands
|
||
.flatMap((m) =>
|
||
[m.label, ...(makerAliases.get(m.id) ?? [])].map((l) => ({
|
||
m,
|
||
t: tokens(l.replace(/\(.*?\)/g, ''))
|
||
}))
|
||
)
|
||
.filter((x) => !(x.t.length === 1 && GENERIC.has(x.t[0])))
|
||
.filter((x) => startsWith(tt, x.t))
|
||
.sort((a, b) => b.t.length - a.t.length)[0];
|
||
if (full) {
|
||
makerId = full.m.id;
|
||
prefix = full.t.length ? title.split(/\s+/).slice(0, full.t.length).join(' ') : '';
|
||
}
|
||
if (!makerId)
|
||
for (let k = Math.min(3, tt.length - 1); k >= 1 && !makerId; k--) {
|
||
const key = tt.slice(0, k).join(' ');
|
||
const global = byKey.get(key);
|
||
const cand = cands.find(
|
||
(m) => makerKey(m.label) === key || makerKey(m.label).startsWith(`${key} `)
|
||
);
|
||
if (cand || (global && cands.some((m) => m.id === global))) {
|
||
makerId = global && (cands.some((m) => m.id === global) || !cand) ? global : cand!.id;
|
||
prefix = title.split(/\s+/).slice(0, k).join(' ');
|
||
} else if (global && !cands.length) {
|
||
makerId = global;
|
||
prefix = title.split(/\s+/).slice(0, k).join(' ');
|
||
}
|
||
}
|
||
if (!makerId && cands.length) {
|
||
const dev = cands.find((m) => it.developers.includes(m.id));
|
||
makerId = (
|
||
dev ?? [...cands].sort((a, b) => (makerUse.get(b.id) ?? 0) - (makerUse.get(a.id) ?? 0))[0]
|
||
).id;
|
||
}
|
||
if (!makerId) {
|
||
skip('no manufacturer');
|
||
continue;
|
||
}
|
||
// Roles.
|
||
// Roles come from the infobox type and the direct classes/uses; far ancestors ("civil
|
||
// aircraft", "transport aircraft") only decide when nothing closer does.
|
||
const nearText = `${[...it.classes, ...it.uses].map((c) => classLabels.get(c) ?? '').join(' | ')} | ${typeText}`;
|
||
let roleText = nearText;
|
||
let roles = ROLE_RULES.filter(([, re]) => re.test(roleText)).map(([r]) => r);
|
||
if (!roles.length) {
|
||
roleText = `${ancText} | ${typeText}`;
|
||
roles = ROLE_RULES.filter(([, re]) => re.test(roleText)).map(([r]) => r);
|
||
}
|
||
if (roles.includes('regional') && !/airliner|jet|aircraft|plane/.test(roleText))
|
||
roles.splice(roles.indexOf('regional'), 1);
|
||
if (roles.includes('helicopter')) {
|
||
for (const r of ['airliner', 'regional', 'general aviation', 'glider'])
|
||
if (roles.includes(r)) roles.splice(roles.indexOf(r), 1);
|
||
}
|
||
if (
|
||
roles.includes('military') &&
|
||
roles.includes('general aviation') &&
|
||
!/civil|light aircraft|utility|sport|aerobatic|agricultural/.test(roleText)
|
||
)
|
||
roles.splice(roles.indexOf('general aviation'), 1);
|
||
const role = PRIMARY.find((r) => roles.includes(r)) ?? null;
|
||
const roleLabel =
|
||
(typeText && typeText.length < 60 ? typeText : null) ??
|
||
it.classes
|
||
.map((c) => classLabels.get(c))
|
||
.find((l) => l && !/aircraft family|aircraft model/.test(l)) ??
|
||
null;
|
||
// Engines.
|
||
const specs = parseSpecs(w.specs, w.legacySpecs);
|
||
const engineText = it.engines
|
||
.map((e) => `${e.label} ${[...(engineKinds.get(e.id) ?? [])].join(' ')}`)
|
||
.join(' ');
|
||
let engineType = specs.engineType ?? engineTypeOf(engineText);
|
||
if (!engineType) {
|
||
const direct = it.classes.map((c) => classLabels.get(c) ?? '').join(' ');
|
||
engineType =
|
||
engineTypeOf(direct) ??
|
||
(/\bjet\b|jet airliner|jet aircraft|jet fighter|jet trainer|business jet/.test(roleText)
|
||
? 'jet'
|
||
: null);
|
||
}
|
||
if (engineType === 'jet' && role === 'helicopter') engineType = 'turboshaft';
|
||
if (
|
||
!engineType &&
|
||
(role === 'glider' || /balloon/.test(roleText)) &&
|
||
!/motor glider|powered/.test(roleText)
|
||
)
|
||
engineType = 'none';
|
||
const qty = it.engines.length === 1 && it.engines[0].qty ? it.engines[0].qty : null;
|
||
const fromClass = [
|
||
...new Set(
|
||
it.classes
|
||
.map((c) => engineCountFromLabel(classLabels.get(c) ?? ''))
|
||
.filter((x): x is number => x != null)
|
||
)
|
||
];
|
||
const engineCount = specs.engineCount ?? qty ?? (fromClass.length === 1 ? fromClass[0] : null);
|
||
// Years and status.
|
||
const firstFlight =
|
||
it.firstFlight ??
|
||
(year(plain(box['first flight'] ?? '')) ? String(year(plain(box['first flight']!))) : null);
|
||
const firstFlightYear = firstFlight ? Number(firstFlight.slice(0, 4)) : null;
|
||
const introducedYear = it.introduced ?? year(plain(box.introduction ?? box.introduced ?? ''));
|
||
const retiredYear = it.retired ?? year(plain(box.retired ?? ''));
|
||
let statusLabel = st.label;
|
||
if (!statusLabel && retiredYear) statusLabel = 'retired';
|
||
const inService =
|
||
statusLabel === 'in service' || statusLabel === 'in production'
|
||
? true
|
||
: statusLabel === 'retired'
|
||
? false
|
||
: null;
|
||
const country =
|
||
it.countries[0] ??
|
||
(plain(box['national origin'] ?? '')
|
||
.split(/,|\//)[0]
|
||
.trim() ||
|
||
null);
|
||
const fullName = title;
|
||
let name =
|
||
prefix && tt.length > tokens(prefix).length ? title.slice(prefix.length).trim() : title;
|
||
if (!name) name = title;
|
||
const codes = [...(icao.get(w.title) ?? icao.get(it.article) ?? [])].sort();
|
||
const makerNames = it.makers.map((m) => m.label);
|
||
const t: Draft = {
|
||
wikidata: it.id,
|
||
kind: it.kind,
|
||
slug: slugify(name),
|
||
name,
|
||
fullName,
|
||
search: '',
|
||
wikipedia: w.title,
|
||
description: it.description,
|
||
summary: w.extract && w.extract.length > 40 ? w.extract.slice(0, 900) : null,
|
||
role,
|
||
roles,
|
||
roleLabel: roleLabel ? roleLabel.replace(/^an? /, '') : null,
|
||
country,
|
||
firstFlight,
|
||
firstFlightYear: sane(firstFlightYear, 1890, 2100),
|
||
introducedYear: sane(introducedYear, 1890, 2100),
|
||
retiredYear: sane(retiredYear, 1890, 2100),
|
||
status: statusLabel,
|
||
inService,
|
||
numberBuilt: built,
|
||
makers: makerNames,
|
||
engineType,
|
||
engineCount: sane(engineCount, 0, 16),
|
||
engines: [
|
||
...new Set([...specs.engineNames, ...it.engines.map((e) => e.label).filter(Boolean)])
|
||
].slice(0, 6),
|
||
crew: specs.crew != null ? Math.round(specs.crew) : null,
|
||
passengers: specs.passengers ?? sane(it.capacity, 1, 1000),
|
||
lengthM: round(sane(specs.length ?? it.length, 1, 400), 2),
|
||
wingspanM: round(sane(specs.span ?? it.wingspan, 1, 400), 2),
|
||
heightM: round(sane(specs.height ?? it.height, 0.5, 100), 2),
|
||
rotorM: round(sane(specs.rotor, 1, 60), 2),
|
||
mtowKg: round(sane(specs.mtow ?? it.mass, 50, 1_000_000)),
|
||
maxSpeedKmh: round(sane(specs.maxSpeed ?? (it.speed ? it.speed * 3.6 : null), 30, 8000)),
|
||
cruiseKmh: round(sane(specs.cruise ?? (it.cruise ? it.cruise * 3.6 : null), 30, 4000)),
|
||
rangeKm: round(sane(specs.range ?? (it.range ? it.range / 1000 : null), 10, 30000)),
|
||
ceilingM: round(sane(specs.ceiling ?? it.ceiling, 100, 40000)),
|
||
wing: it.wing,
|
||
gear: it.gear,
|
||
icao: codes,
|
||
recognize: null,
|
||
parent: null,
|
||
sources: [SRC_WD, SRC_WP],
|
||
links: it.links,
|
||
makerId,
|
||
prefix
|
||
};
|
||
t.recognize = recognizeLine(t);
|
||
drafts.push(t);
|
||
}
|
||
// Family links: a type that is a subclass of another type in the dex.
|
||
// Several items can link (via redirects) to one article ("Boeing 777-200" -> "Boeing 777"):
|
||
// keep the item whose sitelink is the article itself, else the best-linked one.
|
||
const byArticle = new Map<string, Draft>();
|
||
for (const d of drafts) {
|
||
const key = d.wikipedia ?? d.wikidata;
|
||
const cur = byArticle.get(key);
|
||
const exact = (x: Draft) => items.get(x.wikidata)!.article === x.wikipedia;
|
||
if (!cur || (exact(d) && !exact(cur)) || (exact(d) === exact(cur) && d.links > cur.links))
|
||
byArticle.set(key, d);
|
||
}
|
||
if (byArticle.size < drafts.length)
|
||
skip('same article as another item', drafts.length - byArticle.size);
|
||
drafts.splice(0, drafts.length, ...byArticle.values());
|
||
const kept = new Set(drafts.map((d) => d.wikidata));
|
||
for (const d of drafts) {
|
||
const p = items.get(d.wikidata)!.classes.find((c) => kept.has(c) && c !== d.wikidata);
|
||
d.parent = p ?? null;
|
||
}
|
||
// Makers: grouped by item, then merged by display name.
|
||
const groups = new Map<string, Draft[]>();
|
||
for (const d of drafts) groups.set(d.makerId, [...(groups.get(d.makerId) ?? []), d]);
|
||
const makers = new Map<string, BuiltMaker>(); // slug -> maker
|
||
for (const [id, list] of [...groups].sort((a, b) => b[1].length - a[1].length)) {
|
||
const prefixes = new Map<string, number>();
|
||
for (const d of list) if (d.prefix) prefixes.set(d.prefix, (prefixes.get(d.prefix) ?? 0) + 1);
|
||
const label = makerLabel.get(id) ?? id;
|
||
const plainLabel = label.replace(/\s*\(.*?\)/g, '').trim();
|
||
const short =
|
||
prefixes.has(plainLabel) && plainLabel.length <= 26
|
||
? plainLabel
|
||
: ([...prefixes]
|
||
.sort((a, b) => b[1] - a[1])
|
||
.find(([p]) => !GENERIC.has(p.toUpperCase()))?.[0] ?? label);
|
||
const display = short.length <= label.length ? short : label;
|
||
const slug = slugify(display);
|
||
const existing = makers.get(slug);
|
||
if (existing) existing.types.push(...list);
|
||
else
|
||
makers.set(slug, {
|
||
slug,
|
||
name: display,
|
||
fullName: label,
|
||
wikidata: id,
|
||
country:
|
||
makerCountry.get(id) ??
|
||
[
|
||
...list.reduce(
|
||
(m, t) => (t.country ? m.set(t.country, (m.get(t.country) ?? 0) + 1) : m),
|
||
new Map<string, number>()
|
||
)
|
||
].sort((a, b) => b[1] - a[1])[0]?.[0] ??
|
||
null,
|
||
types: list
|
||
});
|
||
}
|
||
// Unique type slugs per maker; the best-known type keeps the plain slug.
|
||
for (const m of makers.values()) {
|
||
const seen = new Set<string>();
|
||
m.types.sort((a, b) => b.links - a.links);
|
||
for (const t of m.types) {
|
||
let s = t.slug;
|
||
if (seen.has(s)) s = `${s}-${t.wikidata.toLowerCase()}`;
|
||
seen.add(s);
|
||
t.slug = s;
|
||
const ancestorsNames = t.icao.join(' ');
|
||
t.search = [
|
||
m.name,
|
||
m.fullName,
|
||
t.fullName,
|
||
t.name,
|
||
items.get(t.wikidata)!.label,
|
||
...items.get(t.wikidata)!.aliases.slice(0, 12),
|
||
ancestorsNames,
|
||
cmp(t.fullName).toLowerCase(),
|
||
t.roleLabel ?? ''
|
||
]
|
||
.join(' ')
|
||
.toLowerCase()
|
||
.slice(0, 2000);
|
||
}
|
||
}
|
||
log(`skipped: ${JSON.stringify(skipped)}`);
|
||
return [...makers.values()];
|
||
}
|
||
|
||
// ---------------------------------------------------------------- write
|
||
|
||
const added = {
|
||
makers: [] as { name: string; url: string }[],
|
||
types: [] as {
|
||
dex: number;
|
||
name: string;
|
||
maker: string;
|
||
role: string | null;
|
||
firstFlight: string | null;
|
||
numberBuilt: number | null;
|
||
url: string;
|
||
photo?: string | null;
|
||
}[]
|
||
};
|
||
|
||
async function write(makers: BuiltMaker[]) {
|
||
const url = process.env.DB_URL;
|
||
if (!url) throw new Error('Set DB_URL (or use --dry-run)');
|
||
const sql = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
||
try {
|
||
await sql.begin(async (tx) => {
|
||
const oldTypes = new Map(
|
||
(
|
||
await tx<
|
||
{ id: number; wikidata: string; dex: number }[]
|
||
>`select id, wikidata, dex from planes.types`
|
||
).map((r) => [r.wikidata, r])
|
||
);
|
||
const oldMakers = new Set(
|
||
(await tx<{ slug: string }[]>`select slug from planes.makers`).map((r) => r.slug)
|
||
);
|
||
const firstRun = oldTypes.size === 0;
|
||
if (renumber) await tx`update planes.types set dex = -id`;
|
||
const all = makers.flatMap((m) => m.types.map((t) => ({ m, t })));
|
||
let nextDex = renumber ? 1 : Math.max(0, ...[...oldTypes.values()].map((r) => r.dex)) + 1;
|
||
const order = all.toSorted(
|
||
(a, b) =>
|
||
(a.t.firstFlightYear ?? 9999) - (b.t.firstFlightYear ?? 9999) ||
|
||
(a.t.firstFlight ?? '').localeCompare(b.t.firstFlight ?? '') ||
|
||
a.t.fullName.localeCompare(b.t.fullName)
|
||
);
|
||
const dex = new Map<string, number>();
|
||
for (const { t } of order) {
|
||
const old = oldTypes.get(t.wikidata);
|
||
dex.set(t.wikidata, !renumber && old ? old.dex : nextDex++);
|
||
}
|
||
const makerIds = new Map<string, number>();
|
||
for (const m of makers) {
|
||
const years = m.types.map((t) => t.firstFlightYear).filter((y): y is number => y != null);
|
||
const built = m.types.reduce((a, t) => a + (t.parent ? 0 : (t.numberBuilt ?? 0)), 0);
|
||
const row = {
|
||
slug: m.slug,
|
||
name: m.name,
|
||
full_name: m.fullName,
|
||
country: m.country,
|
||
wikidata: m.wikidata,
|
||
type_count: m.types.length,
|
||
first_year: years.length ? Math.min(...years) : null,
|
||
last_year: years.length ? Math.max(...years) : null,
|
||
built_total: built,
|
||
sources: [SRC_WD, SRC_WP]
|
||
};
|
||
const [r] = await tx<{ id: number }[]>`
|
||
insert into planes.makers ${tx(row)}
|
||
on conflict (slug) do update set name = excluded.name, full_name = excluded.full_name,
|
||
country = excluded.country, wikidata = excluded.wikidata, type_count = excluded.type_count,
|
||
first_year = excluded.first_year, last_year = excluded.last_year,
|
||
built_total = excluded.built_total, sources = excluded.sources
|
||
returning id`;
|
||
makerIds.set(m.slug, r.id);
|
||
if (!firstRun && !oldMakers.has(m.slug))
|
||
added.makers.push({ name: m.name, url: `${SITE}/${m.slug}` });
|
||
}
|
||
// Slugs can move between rows (renames); clear them first to dodge the unique index.
|
||
await tx`update planes.types set slug = 'tmp-' || id`;
|
||
const ids = new Map<string, number>();
|
||
for (const { m, t } of all) {
|
||
const row = {
|
||
dex: dex.get(t.wikidata)!,
|
||
maker_id: makerIds.get(m.slug)!,
|
||
slug: t.slug,
|
||
name: t.name,
|
||
full_name: t.fullName,
|
||
search: t.search,
|
||
wikidata: t.wikidata,
|
||
wikipedia: t.wikipedia,
|
||
kind: t.kind,
|
||
description: t.description,
|
||
summary: t.summary,
|
||
role: t.role,
|
||
roles: t.roles,
|
||
role_label: t.roleLabel,
|
||
country: t.country,
|
||
first_flight: t.firstFlight,
|
||
first_flight_year: t.firstFlightYear,
|
||
introduced_year: t.introducedYear,
|
||
retired_year: t.retiredYear,
|
||
status: t.status,
|
||
in_service: t.inService,
|
||
number_built: t.numberBuilt,
|
||
makers: t.makers,
|
||
engine_type: t.engineType,
|
||
engine_count: t.engineCount,
|
||
engines: t.engines,
|
||
crew: t.crew,
|
||
passengers: t.passengers,
|
||
length_m: t.lengthM,
|
||
wingspan_m: t.wingspanM,
|
||
height_m: t.heightM,
|
||
rotor_m: t.rotorM,
|
||
mtow_kg: t.mtowKg,
|
||
max_speed_kmh: t.maxSpeedKmh,
|
||
cruise_kmh: t.cruiseKmh,
|
||
range_km: t.rangeKm,
|
||
ceiling_m: t.ceilingM,
|
||
wing: t.wing,
|
||
gear: t.gear,
|
||
icao: t.icao,
|
||
recognize: t.recognize,
|
||
sources: t.sources
|
||
};
|
||
const [r] = await tx<{ id: number }[]>`
|
||
insert into planes.types ${tx(row)}
|
||
on conflict (wikidata) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'wikidata') as (keyof typeof row)[])}
|
||
returning id`;
|
||
ids.set(t.wikidata, r.id);
|
||
if (!firstRun && !oldTypes.has(t.wikidata))
|
||
added.types.push({
|
||
dex: row.dex,
|
||
name: t.fullName,
|
||
maker: m.name,
|
||
role: t.role,
|
||
firstFlight: t.firstFlight,
|
||
numberBuilt: t.numberBuilt,
|
||
url: `${SITE}/${m.slug}/${t.slug}`
|
||
});
|
||
}
|
||
for (const { t } of all)
|
||
await tx`update planes.types set family_id = ${t.parent ? (ids.get(t.parent) ?? null) : null} where id = ${ids.get(t.wikidata)!}`;
|
||
const keep = [...ids.values()];
|
||
const gone = await tx`delete from planes.types where not (id = any(${keep})) returning id`;
|
||
const goneMakers =
|
||
await tx`delete from planes.makers where not (slug = any(${makers.map((m) => m.slug)})) returning id`;
|
||
log(
|
||
`written: ${keep.length} types, ${makers.length} makers; removed ${gone.length} types, ${goneMakers.length} makers`
|
||
);
|
||
});
|
||
} finally {
|
||
await sql.end();
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------- main
|
||
|
||
await useCache(cacheDir, 'import.json');
|
||
const wd = await loadWikidata();
|
||
const wiki = await loadWikipedia([...wd.items.values()].map((i) => i.article));
|
||
const icao = await loadIcao();
|
||
await saveCache(true);
|
||
const makers = build(wd, wiki, icao);
|
||
const types = makers.flatMap((m) => m.types);
|
||
log(
|
||
`built ${makers.length} makers, ${types.length} types (${types.filter((t) => t.role).length} with a role, ${types.filter((t) => t.engineType).length} engine type, ${types.filter((t) => t.engineCount).length} engine count, ${types.filter((t) => t.maxSpeedKmh).length} speed, ${types.filter((t) => t.icao.length).length} ICAO, ${types.filter((t) => t.recognize).length} recognise)`
|
||
);
|
||
if (outFile) {
|
||
const json = JSON.stringify(makers.map((m) => ({ ...m, types: m.types })));
|
||
if (outFile === '-') console.log(json);
|
||
else await writeFile(outFile, json);
|
||
}
|
||
if (dryRun) {
|
||
for (const t of types
|
||
.toSorted((a, b) => (b.numberBuilt ?? 0) - (a.numberBuilt ?? 0))
|
||
.slice(0, 20))
|
||
log(
|
||
` ${t.fullName} built=${t.numberBuilt} ${t.role} ${t.engineCount}x${t.engineType} ${t.firstFlight}`
|
||
);
|
||
} else {
|
||
await write(makers);
|
||
if (reportNew)
|
||
console.log(
|
||
JSON.stringify(
|
||
{
|
||
generatedAt: new Date().toISOString(),
|
||
totals: { makers: makers.length, types: types.length },
|
||
newMakers: added.makers,
|
||
newTypes: added.types.toSorted((a, b) => (b.numberBuilt ?? 0) - (a.numberBuilt ?? 0))
|
||
},
|
||
null,
|
||
2
|
||
)
|
||
);
|
||
}
|
||
void readFile;
|
||
void join;
|