Stan Runge
8c0fc74283
All checks were successful
Deploy production / deploy (push) Successful in 1m13s
1657 lines
56 KiB
TypeScript
1657 lines
56 KiB
TypeScript
// Builds the public train pokedex (/trackers/trains, schema `trains`) from open data.
|
||
//
|
||
// DB_URL=... bun scripts/trains-import.ts fetch sources and write to the database
|
||
// bun scripts/trains-import.ts --dry-run fetch + build only, print a summary
|
||
// bun scripts/trains-import.ts --cache /tmp/trains reuse API responses from that folder
|
||
// (one small file per request, resumable)
|
||
// bun scripts/trains-import.ts --dry-run --out x.json also dump the built catalogue
|
||
// bun scripts/trains-import.ts --renumber reassign Dex numbers from #1 (oldest first)
|
||
// bun scripts/trains-import.ts --report-new JSON of makers/types new since the last run on stdout
|
||
//
|
||
// Sources (no API keys):
|
||
// - English Wikipedia (CC BY-SA): every article that uses {{Infobox locomotive}}, {{Infobox
|
||
// train}} or {{Infobox tram}} (and their redirects such as {{Infobox EMU}}): builder, designer,
|
||
// operators, build dates, number built, service dates, top speed, power, gauge, electric
|
||
// system, wheel arrangement, length, weight, capacity, plus a 3-sentence intro extract.
|
||
// - Wikidata (CC0) for the same items: manufacturer (P176), operator (P137), country (P495/P17),
|
||
// number built (P1092), service entry/retirement (P729/P730), speed, power, gauge, English and
|
||
// Dutch labels, and "subclass of" for families. Linked builders/operators are resolved to their
|
||
// Wikidata items (label, country) so makers and operators group cleanly.
|
||
// Only classes/types: individual locomotives, list articles and unbuilt projects are skipped.
|
||
// Unknown values stay null; nothing is estimated.
|
||
// Re-runnable: types are upserted by Wikidata id (Dex numbers stay stable, new types get the
|
||
// next number), types that vanished are deleted, makers/operators are upserted by slug.
|
||
import postgres from 'postgres';
|
||
import { writeFile } from 'node:fs/promises';
|
||
import {
|
||
chunks,
|
||
cmp,
|
||
log as logErr,
|
||
mw,
|
||
qid,
|
||
slugify,
|
||
sparql,
|
||
tokens,
|
||
useCache
|
||
} from './trains-wiki';
|
||
|
||
const args = process.argv.slice(2);
|
||
const dryRun = args.includes('--dry-run');
|
||
const renumber = args.includes('--renumber');
|
||
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
||
const cacheDir = opt('--cache');
|
||
const outFile = opt('--out');
|
||
const reportNew = args.includes('--report-new');
|
||
const SITE = 'https://stanrunge.dev/trackers/trains';
|
||
const log = (...a: unknown[]) => (reportNew || outFile === '-' ? logErr(...a) : console.log(...a));
|
||
const SRC_WD = 'Wikidata';
|
||
const SRC_WP = 'Wikipedia';
|
||
const TEMPLATES = ['Infobox locomotive', 'Infobox train', 'Infobox tram'];
|
||
const thisYear = new Date().getFullYear();
|
||
const years = (s: string | undefined | null) =>
|
||
[...(s ?? '').matchAll(/\b(1[89]\d\d|20[0-4]\d)\b/g)]
|
||
.map((m) => Number(m[1]))
|
||
.filter((y) => y >= 1800 && y <= thisYear + 10);
|
||
const year = (s: string | undefined | null) => years(s)[0] ?? null;
|
||
|
||
// ---------------------------------------------------------------- wikitext
|
||
|
||
/** The `{{name ...}}` template (first match), with nested braces balanced. */
|
||
function findTemplate(text: string, name: RegExp): { name: string; body: string } | null {
|
||
const re = /\{\{\s*([^|}\n]+)/g;
|
||
let m: RegExpExecArray | null;
|
||
while ((m = re.exec(text))) {
|
||
if (!name.test(m[1].trim())) continue;
|
||
let depth = 0;
|
||
for (let i = m.index; i < text.length - 1; i++) {
|
||
if (text[i] === '{' && text[i + 1] === '{') {
|
||
depth++;
|
||
i++;
|
||
} else if (text[i] === '}' && text[i + 1] === '}') {
|
||
depth--;
|
||
i++;
|
||
if (depth === 0) return { name: m[1].trim(), body: text.slice(m.index + 2, i - 1) };
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
return null;
|
||
}
|
||
function templateParams(tpl: string): Record<string, string> {
|
||
const out: Record<string, string> = {};
|
||
const parts: string[] = [];
|
||
let depth = 0;
|
||
let link = 0;
|
||
let cur = '';
|
||
for (let i = 0; i < tpl.length; i++) {
|
||
const two = tpl.slice(i, i + 2);
|
||
if (two === '{{') depth++;
|
||
else if (two === '}}') depth--;
|
||
else if (two === '[[') link++;
|
||
else if (two === ']]') link--;
|
||
if (tpl[i] === '|' && depth === 0 && link === 0) {
|
||
parts.push(cur);
|
||
cur = '';
|
||
continue;
|
||
}
|
||
cur += tpl[i];
|
||
}
|
||
parts.push(cur);
|
||
for (const p of parts.slice(1)) {
|
||
const eq = p.indexOf('=');
|
||
if (eq < 0) continue;
|
||
const key = p
|
||
.slice(0, eq)
|
||
.trim()
|
||
.toLowerCase()
|
||
.replace(/[_\s]+/g, '');
|
||
const value = p.slice(eq + 1).trim();
|
||
if (key && value) out[key] = value;
|
||
}
|
||
return out;
|
||
}
|
||
const noRefs = (v: string) =>
|
||
v
|
||
.replace(/<!--[\s\S]*?-->/g, '')
|
||
.replace(/<ref[^>]*\/>/gi, '')
|
||
.replace(/<ref[^>]*>[\s\S]*?<\/ref>/gi, '');
|
||
/** Wikitext to plain text: no refs, comments, markup; templates keep their unnamed arguments. */
|
||
export function plain(v: string): string {
|
||
let s = noRefs(v)
|
||
.replace(/\{\{\s*(long|short) ton\s*\|\s*([\d.,]+)[^{}]*\}\}/gi, '$2 $1 tons')
|
||
.replace(/<br\s*\/?>/gi, ', ')
|
||
.replace(/<[^>]+>/g, '');
|
||
for (let guard = 0; guard < 10 && /\{\{/.test(s); guard++)
|
||
s = s.replace(/\{\{([^{}]*)\}\}/g, (_, inner: string) => {
|
||
const [name, ...rest] = inner.split('|');
|
||
const n = name.trim().toLowerCase();
|
||
if (/^(cvt|convert)$/.test(n)) return rest.slice(0, 2).join(' ');
|
||
if (
|
||
/^(efn|sfn|refn|rp|r|page needed|citation needed|cn|when|dubious|clarify|webarchive|nbsp|flagicon|flag)$|^(efn|sfn|refn|citation|webarchive|flagicon)/.test(
|
||
n
|
||
)
|
||
)
|
||
return '';
|
||
if (/^as of/.test(n)) return `as of ${rest.join(' ')}`;
|
||
if (/^(ubl|unbulleted list|plainlist|plain list|flatlist|hlist|bulleted list|ublist)/.test(n))
|
||
return rest
|
||
.filter((r) => !/^\s*[a-z_ ]+\s*=/.test(r))
|
||
.map((r) => r.trim())
|
||
.filter(Boolean)
|
||
.join(', ');
|
||
return rest
|
||
.filter((r) => !/^\s*[a-z_ ]+\s*=/.test(r))
|
||
.map((r) => r.trim())
|
||
.filter(Boolean)
|
||
.join(', ');
|
||
});
|
||
return s
|
||
.replace(/\[\[(?:[^|\]]*\|)?([^\]]*)\]\]/g, '$1')
|
||
.replace(/\[https?:[^\s\]]+\s*([^\]]*)\]/g, '$1')
|
||
.replace(/'{2,}/g, '')
|
||
.replace(/ | /g, ' ')
|
||
.replace(/–|—/g, '–')
|
||
.replace(/\s*\n\s*\*\s*/g, ', ')
|
||
.replace(/^\s*\*\s*/, '')
|
||
.replace(/\s+/g, ' ')
|
||
.replace(/^[,\s]+|[,\s]+$/g, '')
|
||
.trim();
|
||
}
|
||
/** Wikilink targets in a field, in order ("[[Alstom|GEC-Alsthom]]" -> "Alstom"). */
|
||
function linkTargets(v: string | undefined): string[] {
|
||
if (!v) return [];
|
||
const out: string[] = [];
|
||
for (const m of noRefs(v).matchAll(/\[\[([^|\]#]+)(?:#[^|\]]*)?(?:\|[^\]]*)?\]\]/g)) {
|
||
const t = m[1].trim().replace(/_/g, ' ');
|
||
if (!t || /^(file|image|category|wikt|wiktionary|w|commons):/i.test(t)) continue;
|
||
const title = t.charAt(0).toUpperCase() + t.slice(1);
|
||
if (!out.includes(title)) out.push(title);
|
||
}
|
||
return out;
|
||
}
|
||
const field = (box: Record<string, string>, ...keys: string[]) => {
|
||
for (const k of keys) if (box[k] && plain(box[k])) return box[k];
|
||
return undefined;
|
||
};
|
||
|
||
// ---------------------------------------------------------------- number parsing
|
||
|
||
function numberBuilt(v: string | undefined): number | null {
|
||
if (!v) return null;
|
||
const s = plain(v).replace(/(\d)[,\s](\d{3})\b/g, '$1$2');
|
||
const parts = s.split(/,\s|;\s|\n/).filter(Boolean);
|
||
// "ICE 3 neo: 65, Velaro D: 17, ..." -> sum of the counts after each colon.
|
||
const labelled = parts
|
||
.map((p) => p.match(/:\s*(\d+)/))
|
||
.filter((m): m is RegExpMatchArray => !!m)
|
||
.map((m) => Number(m[1]));
|
||
if (labelled.length >= 2 && labelled.length >= parts.length / 2)
|
||
return labelled.reduce((a, b) => a + b, 0);
|
||
const m = s.match(/(?<![\d.])(\d+)(?![\d.]*\s*(?:km|mm|kW|hp|t\b|%))/);
|
||
if (!m) return null;
|
||
const n = Number(m[1]);
|
||
return n >= 1800 && n <= 2100 && /\b(?:in|from|since|by|until)\s+\d{4}/.test(s) ? null : n;
|
||
}
|
||
function speedKmh(v: string | undefined): number | null {
|
||
if (!v) return null;
|
||
const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2');
|
||
let best: number | null = null;
|
||
for (const m of s.matchAll(/(\d+(?:\.\d+)?)\s*(km\/h|kmh|kph|km\/hr|mph|mi\/h)/gi)) {
|
||
const n = Number(m[1]) * (/^m/i.test(m[2]) ? 1.609 : 1);
|
||
if (n > 3 && n < 700 && (best == null || n > best)) best = n;
|
||
}
|
||
return best == null ? null : Math.round(best);
|
||
}
|
||
function powerKw(v: string | undefined): number | null {
|
||
if (!v) return null;
|
||
const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2');
|
||
let best: number | null = null;
|
||
for (const m of s.matchAll(/(\d+(?:\.\d+)?)\s*(MW|kW|bhp|hp|horsepower|PS|ihp|shp)\b/gi)) {
|
||
const u = m[2].toLowerCase();
|
||
const n = Number(m[1]) * (u === 'mw' ? 1000 : u === 'kw' ? 1 : u === 'ps' ? 0.7355 : 0.7457);
|
||
if (n > 1 && n < 40000 && (best == null || n > best)) best = n;
|
||
}
|
||
return best == null ? null : Math.round(best);
|
||
}
|
||
function lengthM(v: string | undefined): number | null {
|
||
if (!v) return null;
|
||
const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2');
|
||
const hits = [...s.matchAll(/(\d+(?:\.\d+)?)\s*(mm|m|ft|in)\b/gi)].map((m) => ({
|
||
n: Number(m[1]),
|
||
u: m[2].toLowerCase()
|
||
}));
|
||
for (let i = 0; i < hits.length; i++) {
|
||
const { n, u } = hits[i];
|
||
if (u === 'mm') return n / 1000;
|
||
if (u === 'm') return n;
|
||
if (u === 'ft') return n * 0.3048 + (hits[i + 1]?.u === 'in' ? hits[i + 1].n * 0.0254 : 0);
|
||
}
|
||
return null;
|
||
}
|
||
function weightT(v: string | undefined): number | null {
|
||
if (!v) return null;
|
||
const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2');
|
||
const m = s.match(/(\d+(?:\.\d+)?)\s*(long tons?|short tons?|tonnes?|tons?|t\b|kg|lb|lbs)/i);
|
||
if (!m) return null;
|
||
const n = Number(m[1]);
|
||
const u = m[2].toLowerCase();
|
||
if (u.startsWith('long')) return n * 1.016;
|
||
if (u.startsWith('short')) return n * 0.907;
|
||
if (u === 'kg') return n / 1000;
|
||
if (u.startsWith('lb')) return (n * 0.4536) / 1000;
|
||
return n;
|
||
}
|
||
function capacity(v: string | undefined): number | null {
|
||
if (!v) return null;
|
||
const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2');
|
||
// "129 (1st), 442 (2nd)": first + second class seats of the first variant.
|
||
const cls = [...s.matchAll(/(\d+)\s*\(?\s*(?:1st|2nd|first|second|standard)/gi)].map((m) =>
|
||
Number(m[1])
|
||
);
|
||
if (cls.length >= 2) return cls[0] + cls[1];
|
||
let best: number | null = null;
|
||
for (const m of s.matchAll(
|
||
/(\d+)(?:\s*(?:–|-|to)\s*(\d+))?\s*(?:\(?[a-z ]{0,20}\)?\s*)?(?:seat|passenger|standing|people|pax|total)/gi
|
||
)) {
|
||
const n = Number(m[2] ?? m[1]);
|
||
if (n > 0 && n < 4000 && (best == null || n > best)) best = n;
|
||
}
|
||
if (best == null) {
|
||
const only = s.match(/^(\d+)(?:\s|$)/);
|
||
if (only && Number(only[1]) < 4000) best = Number(only[1]);
|
||
}
|
||
return best;
|
||
}
|
||
|
||
const GAUGE_CODES: Record<string, number> = {
|
||
sg: 1435,
|
||
ussg: 1435,
|
||
standard: 1435,
|
||
ib: 1668,
|
||
iberian: 1668,
|
||
rus: 1520,
|
||
russian: 1520,
|
||
ig: 1676,
|
||
indian: 1676,
|
||
irish: 1600,
|
||
cape: 1067,
|
||
mm: 1000,
|
||
metre: 1000,
|
||
meter: 1000,
|
||
'3ft': 914,
|
||
'3ft6in': 1067,
|
||
'2ft': 610,
|
||
'2ft6in': 762,
|
||
'1ft11.5in': 597,
|
||
'5ft': 1524,
|
||
'5ft3in': 1600,
|
||
'5ft6in': 1676,
|
||
brunel: 2140,
|
||
'7ft': 2134,
|
||
'4ft8.5in': 1435,
|
||
'4ft6in': 1372,
|
||
'3ft6': 1067,
|
||
'15in': 381
|
||
};
|
||
const GAUGE_NAME: Record<number, string> = {
|
||
1435: 'standard gauge',
|
||
1668: 'Iberian broad gauge',
|
||
1520: 'Russian broad gauge',
|
||
1524: 'Russian broad gauge',
|
||
1676: 'Indian broad gauge',
|
||
1600: 'Irish broad gauge',
|
||
1067: 'Cape gauge',
|
||
1000: 'metre gauge',
|
||
914: '3 ft narrow gauge',
|
||
762: 'narrow gauge',
|
||
750: 'narrow gauge',
|
||
610: '2 ft narrow gauge',
|
||
600: 'narrow gauge',
|
||
597: 'narrow gauge',
|
||
2140: 'Brunel broad gauge',
|
||
1372: 'Scotch gauge'
|
||
};
|
||
function gauge(
|
||
v: string | undefined,
|
||
wdLabel: string | null
|
||
): { mm: number | null; label: string | null } {
|
||
let mm: number | null = null;
|
||
if (v) {
|
||
const tpl = noRefs(v).match(/\{\{\s*(?:track ?gauge|rail ?gauge|gauge)\s*\|\s*([^|}]+)/i);
|
||
if (tpl) {
|
||
const code = tpl[1].trim().toLowerCase().replace(/\s+/g, '');
|
||
mm =
|
||
GAUGE_CODES[code] ?? (/^\d{3,4}(mm)?$/.test(code) ? Number(code.replace('mm', '')) : null);
|
||
if (mm == null) {
|
||
const ft = code.match(/^(\d)ft(?:(\d+(?:\.\d+)?)in)?$/);
|
||
if (ft) mm = Math.round(Number(ft[1]) * 304.8 + Number(ft[2] ?? 0) * 25.4);
|
||
}
|
||
}
|
||
if (mm == null) {
|
||
const m = plain(v)
|
||
.replace(/(\d),(\d{3})/g, '$1$2')
|
||
.match(/(\d{3,4})\s*mm/);
|
||
if (m) mm = Number(m[1]);
|
||
}
|
||
}
|
||
if (mm == null && wdLabel) {
|
||
const m = wdLabel.replace(/(\d),(\d{3})/g, '$1$2').match(/(\d{3,4})\s*mm/);
|
||
if (m) mm = Number(m[1]);
|
||
else if (/standard gauge/i.test(wdLabel)) mm = 1435;
|
||
else if (/metre gauge/i.test(wdLabel)) mm = 1000;
|
||
else if (/cape gauge/i.test(wdLabel)) mm = 1067;
|
||
else if (/russian gauge|1520/i.test(wdLabel)) mm = 1520;
|
||
}
|
||
if (mm != null && (mm < 200 || mm > 3000)) mm = null;
|
||
if (mm == null) return { mm: null, label: null };
|
||
const name = GAUGE_NAME[mm] ?? (mm < 1435 ? 'narrow gauge' : 'broad gauge');
|
||
return { mm, label: `${mm.toLocaleString('en-GB')} mm ${name}` };
|
||
}
|
||
|
||
// ---------------------------------------------------------------- wikipedia
|
||
|
||
/** "... built by [[British Rail Engineering Limited]] ..." in the first paragraphs. */
|
||
function leadBuilders(text: string): string[] {
|
||
const start = text.search(/^'''/m);
|
||
const lead = noRefs(text.slice(start >= 0 ? start : 0, (start >= 0 ? start : 0) + 2500));
|
||
const out: string[] = [];
|
||
for (const m of lead.matchAll(
|
||
/\b(?:built|manufactured|constructed|produced|developed and built|designed and built)\s+(?:(?:jointly|primarily|mainly|initially|originally|in \d{4}(?:\s*[–-]\s*\d{4})?|between \d{4} and \d{4}|from \d{4})\s+)*by\s+(?:the\s+)?\[\[([^\]|#]+)/gi
|
||
)) {
|
||
const t = m[1].trim().replace(/_/g, ' ');
|
||
const title = t.charAt(0).toUpperCase() + t.slice(1);
|
||
if (!out.includes(title)) out.push(title);
|
||
if (out.length >= 2) break;
|
||
}
|
||
return out;
|
||
}
|
||
|
||
type Article = {
|
||
title: string;
|
||
template: string;
|
||
qid: string | null;
|
||
extract: string | null;
|
||
box: Record<string, string>;
|
||
boxName: string;
|
||
/** Builders linked in the lead ("built by [[X]]"), for infoboxes without a builder field. */
|
||
leadBuilders: string[];
|
||
};
|
||
|
||
async function listArticles() {
|
||
const titles = new Map<string, string>(); // title -> template
|
||
for (const t of TEMPLATES) {
|
||
let cont: Record<string, string> = {};
|
||
for (let guard = 0; guard < 100; guard++) {
|
||
const res = await mw<{
|
||
query?: { embeddedin?: { title: string }[] };
|
||
continue?: Record<string, string>;
|
||
}>(
|
||
'en.wikipedia.org',
|
||
{
|
||
action: 'query',
|
||
list: 'embeddedin',
|
||
eititle: `Template:${t}`,
|
||
einamespace: '0',
|
||
eilimit: '500',
|
||
...cont
|
||
},
|
||
false
|
||
);
|
||
for (const p of res.query?.embeddedin ?? []) if (!titles.has(p.title)) titles.set(p.title, t);
|
||
if (!res.continue) break;
|
||
cont = res.continue;
|
||
}
|
||
}
|
||
log(`wikipedia: ${titles.size} articles with a locomotive/train/tram infobox`);
|
||
return titles;
|
||
}
|
||
|
||
const INFOBOX =
|
||
/^infobox\s*(locomotive|train|tram|emu|dmu|multiple unit|rolling stock|railcar|.*(?:locomotive|train|tram|railcar|carriage|coach))\b/i;
|
||
|
||
async function loadArticles(titles: Map<string, string>) {
|
||
const out: Article[] = [];
|
||
const todo = [...titles.keys()];
|
||
let done = 0;
|
||
for (const c of chunks(todo, 20)) {
|
||
const res = await mw('en.wikipedia.org', {
|
||
action: 'query',
|
||
prop: 'revisions|extracts|pageprops',
|
||
rvprop: 'content|ids',
|
||
rvslots: 'main',
|
||
exintro: '1',
|
||
explaintext: '1',
|
||
exsentences: '3',
|
||
exlimit: '20',
|
||
ppprop: 'wikibase_item',
|
||
titles: c.join('|')
|
||
});
|
||
for (const p of (res.query?.pages ?? []) as unknown[]) {
|
||
const page = p as {
|
||
title: string;
|
||
missing?: boolean;
|
||
extract?: string;
|
||
pageprops?: { wikibase_item?: string };
|
||
revisions?: { slots?: { main?: { content?: string } } }[];
|
||
};
|
||
if (page.missing) continue;
|
||
const text = page.revisions?.[0]?.slots?.main?.content ?? '';
|
||
const box = findTemplate(text, INFOBOX);
|
||
out.push({
|
||
title: page.title,
|
||
template: titles.get(page.title) ?? '',
|
||
qid: page.pageprops?.wikibase_item ?? null,
|
||
extract: page.extract?.trim() || null,
|
||
box: box ? templateParams(box.body) : {},
|
||
boxName: box?.name ?? '',
|
||
leadBuilders: leadBuilders(text)
|
||
});
|
||
}
|
||
done += c.length;
|
||
if (done % 1000 < 20 || done === todo.length) log(`wikipedia: ${done}/${todo.length} articles`);
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/** Resolves article titles (redirects followed) to Wikidata ids. */
|
||
async function resolveTitles(titles: string[]) {
|
||
const out = new Map<string, string>(); // asked title -> qid
|
||
for (const c of chunks([...new Set(titles)], 50)) {
|
||
const res = (await mw('en.wikipedia.org', {
|
||
action: 'query',
|
||
prop: 'pageprops',
|
||
ppprop: 'wikibase_item',
|
||
redirects: '1',
|
||
titles: c.join('|')
|
||
})) as {
|
||
query?: {
|
||
pages?: { title: string; pageprops?: { wikibase_item?: string } }[];
|
||
normalized?: { from: string; to: string }[];
|
||
redirects?: { from: string; to: string }[];
|
||
};
|
||
};
|
||
const fwd = new Map<string, string>();
|
||
for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])])
|
||
fwd.set(n.from, n.to);
|
||
const byTitle = new Map(
|
||
(res.query?.pages ?? []).map((p) => [p.title, p.pageprops?.wikibase_item ?? null])
|
||
);
|
||
for (const t of c) {
|
||
let cur = t;
|
||
for (let g = 0; g < 4 && fwd.has(cur); g++) cur = fwd.get(cur)!;
|
||
const q = byTitle.get(cur);
|
||
if (q) out.set(t, q);
|
||
}
|
||
}
|
||
return out;
|
||
}
|
||
|
||
// ---------------------------------------------------------------- wikidata
|
||
|
||
type Item = {
|
||
id: string;
|
||
label: string | null;
|
||
nl: string | null;
|
||
description: string | null;
|
||
aliases: string[];
|
||
links: number;
|
||
instanceOf: string[]; // lowercase labels
|
||
classes: string[]; // P279 ids
|
||
classLabels: string[];
|
||
makers: { id: string; label: string }[];
|
||
operators: { id: string; label: string }[];
|
||
origin: string[];
|
||
country: string[];
|
||
built: number | null;
|
||
inception: number | null;
|
||
introduced: number | null;
|
||
retired: number | null;
|
||
speed: number | null; // km/h
|
||
power: number | null; // kW
|
||
gauge: string | null;
|
||
length: number | null;
|
||
mass: number | null; // kg
|
||
};
|
||
|
||
const values = (ids: string[], body: string, select = '?v') =>
|
||
sparql(
|
||
`SELECT ?i ${select} WHERE { VALUES ?i { ${ids.map((i) => `wd:${i}`).join(' ')} } ${body} }`
|
||
);
|
||
// English label, or the language-neutral "mul" label many company items now carry instead.
|
||
const en = (v: string, l: string) => `${v} rdfs:label ${l} FILTER(LANG(${l}) IN ("en", "mul"))`;
|
||
|
||
async function loadItems(ids: string[]) {
|
||
const items = new Map<string, Item>();
|
||
for (const id of ids)
|
||
items.set(id, {
|
||
id,
|
||
label: null,
|
||
nl: null,
|
||
description: null,
|
||
aliases: [],
|
||
links: 0,
|
||
instanceOf: [],
|
||
classes: [],
|
||
classLabels: [],
|
||
makers: [],
|
||
operators: [],
|
||
origin: [],
|
||
country: [],
|
||
built: null,
|
||
inception: null,
|
||
introduced: null,
|
||
retired: null,
|
||
speed: null,
|
||
power: null,
|
||
gauge: null,
|
||
length: null,
|
||
mass: null
|
||
});
|
||
let n = 0;
|
||
for (const c of chunks(ids, 300)) {
|
||
const get = (r: Record<string, string>) => items.get(qid(r.i))!;
|
||
for (const r of await values(
|
||
c,
|
||
`OPTIONAL { ${en('?i', '?l')} } OPTIONAL { ?i rdfs:label ?nl FILTER(LANG(?nl) = "nl") }
|
||
OPTIONAL { ?i schema:description ?d FILTER(LANG(?d) = "en") } OPTIONAL { ?i wikibase:sitelinks ?sl }`,
|
||
'?l ?nl ?d ?sl'
|
||
)) {
|
||
const it = get(r);
|
||
if (r.l) it.label = r.l;
|
||
if (r.nl) it.nl = r.nl;
|
||
if (r.d) it.description = r.d;
|
||
it.links = Number(r.sl ?? 0);
|
||
}
|
||
for (const r of await values(
|
||
c,
|
||
`?i skos:altLabel ?v FILTER(LANG(?v) = "en" || LANG(?v) = "nl")`
|
||
))
|
||
if (get(r).aliases.length < 16) get(r).aliases.push(r.v);
|
||
for (const r of await values(c, `?i wdt:P31 ?c . ${en('?c', '?v')}`))
|
||
get(r).instanceOf.push(r.v.toLowerCase());
|
||
for (const r of await values(c, `?i wdt:P279 ?c . OPTIONAL { ${en('?c', '?v')} }`, '?c ?v')) {
|
||
get(r).classes.push(qid(r.c));
|
||
if (r.v) get(r).classLabels.push(r.v.toLowerCase());
|
||
}
|
||
for (const r of await values(c, `?i wdt:P176 ?m . ${en('?m', '?v')}`, '?m ?v')) {
|
||
const it = get(r);
|
||
if (!it.makers.some((m) => m.id === qid(r.m))) it.makers.push({ id: qid(r.m), label: r.v });
|
||
}
|
||
for (const r of await values(c, `?i wdt:P137 ?m . ${en('?m', '?v')}`, '?m ?v')) {
|
||
const it = get(r);
|
||
if (!it.operators.some((m) => m.id === qid(r.m)))
|
||
it.operators.push({ id: qid(r.m), label: r.v });
|
||
}
|
||
for (const r of await values(c, `?i wdt:P495 ?c . ${en('?c', '?v')}`)) get(r).origin.push(r.v);
|
||
for (const r of await values(c, `?i wdt:P17 ?c . ${en('?c', '?v')}`)) get(r).country.push(r.v);
|
||
for (const r of await values(c, `?i wdt:P1092 ?v`)) {
|
||
const it = get(r);
|
||
const v = Math.round(Number(r.v));
|
||
if (Number.isFinite(v) && v >= 0 && (it.built == null || v > it.built)) it.built = v;
|
||
}
|
||
const yearProp = async (
|
||
prop: string,
|
||
key: 'inception' | 'introduced' | 'retired',
|
||
max = false
|
||
) => {
|
||
for (const r of await values(c, `?i wdt:${prop} ?v`)) {
|
||
const it = get(r);
|
||
const y = year(r.v);
|
||
if (y && (it[key] == null || (max ? y > it[key]! : y < it[key]!))) it[key] = y;
|
||
}
|
||
};
|
||
await yearProp('P571', 'inception');
|
||
await yearProp('P729', 'introduced');
|
||
await yearProp('P730', 'retired', true);
|
||
const quantity = async (
|
||
prop: string,
|
||
key: 'speed' | 'power' | 'length' | 'mass',
|
||
f: number
|
||
) => {
|
||
for (const r of await values(c, `?i p:${prop}/psn:${prop}/wikibase:quantityAmount ?v`)) {
|
||
const it = get(r);
|
||
const v = Number(r.v) * f;
|
||
if (Number.isFinite(v) && v > 0 && (it[key] == null || v > it[key]!)) it[key] = v;
|
||
}
|
||
};
|
||
await quantity('P2052', 'speed', 3.6);
|
||
await quantity('P2109', 'power', 0.001);
|
||
await quantity('P2043', 'length', 1);
|
||
await quantity('P2067', 'mass', 1);
|
||
for (const r of await values(c, `?i wdt:P1064 ?g . ${en('?g', '?v')}`)) get(r).gauge ??= r.v;
|
||
n += c.length;
|
||
log(`wikidata: details ${n}/${ids.length}`);
|
||
}
|
||
return items;
|
||
}
|
||
|
||
type Entity = {
|
||
id: string;
|
||
label: string;
|
||
country: string | null;
|
||
aliases: string[];
|
||
human: boolean;
|
||
place: boolean;
|
||
};
|
||
/** Labels, countries and short aliases of makers/operators. */
|
||
async function loadEntities(ids: string[]) {
|
||
const out = new Map<string, Entity>();
|
||
for (const c of chunks([...new Set(ids)], 300)) {
|
||
for (const r of await values(
|
||
c,
|
||
`OPTIONAL { ${en('?i', '?l')} } OPTIONAL { ?i wdt:P17 ?c . ${en('?c', '?cl')} }
|
||
OPTIONAL { ?i wdt:P31 wd:Q5 . BIND(1 AS ?h) }
|
||
OPTIONAL { ?i wdt:P625 ?coord . BIND(1 AS ?p) }`,
|
||
'?l ?cl ?h ?p'
|
||
)) {
|
||
const id = qid(r.i);
|
||
const cur = out.get(id) ?? {
|
||
id,
|
||
label: id,
|
||
country: null,
|
||
aliases: [],
|
||
human: false,
|
||
place: false
|
||
};
|
||
if (r.l) cur.label = r.l;
|
||
cur.country ??= r.cl ?? null;
|
||
if (r.h) cur.human = true;
|
||
if (r.p) cur.place = true;
|
||
out.set(id, cur);
|
||
}
|
||
for (const r of await values(c, `?i skos:altLabel ?v FILTER(LANG(?v) = "en")`)) {
|
||
const e = out.get(qid(r.i));
|
||
if (e && e.aliases.length < 8) e.aliases.push(r.v);
|
||
}
|
||
}
|
||
return out;
|
||
}
|
||
|
||
// ---------------------------------------------------------------- classification
|
||
|
||
export const CATEGORIES = [
|
||
'steam',
|
||
'diesel',
|
||
'electric',
|
||
'high-speed',
|
||
'multiple unit',
|
||
'metro',
|
||
'tram',
|
||
'carriage',
|
||
'other'
|
||
] as const;
|
||
|
||
function tractionOf(text: string, hasElectricSystem: boolean): string | null {
|
||
const s = text.toLowerCase();
|
||
if (/hydrogen|fuel[- ]cell/.test(s)) return 'hydrogen';
|
||
if (/battery[- ]electric|battery[- ]powered|accumulator|\bbemu\b|akku/.test(s)) return 'battery';
|
||
if (/\bsteam\b|fireless|geared steam/.test(s)) return 'steam';
|
||
if (
|
||
/electro-diesel|electro diesel|bi-mode|bimode|dual-mode|dual mode|tri-mode|diesel[- ]hybrid|hybrid (?:diesel|locomotive|train|multiple unit|railcar|drive|traction)/.test(
|
||
s
|
||
)
|
||
)
|
||
return 'bi-mode';
|
||
if (/gas turbine|turbotrain|turbine/.test(s)) return 'gas turbine';
|
||
if (
|
||
/diesel|petrol|gasoline|internal combustion|paraffin|benzene|\bdmu\b|\bdemu\b|railbus/.test(s)
|
||
)
|
||
return 'diesel';
|
||
if (
|
||
/electric|\bkv\b|overhead|third rail|fourth rail|catenary|pantograph|\bemu\b|trolley|volt|\bdc\b|\bac\b/.test(
|
||
s
|
||
) ||
|
||
hasElectricSystem
|
||
)
|
||
return 'electric';
|
||
if (/horse|cable car|cable-hauled|funicular/.test(s)) return 'none';
|
||
return null;
|
||
}
|
||
|
||
function categoryOf(o: {
|
||
template: string;
|
||
text: string; // stocktype, powertype, P31/P279 labels, description, title
|
||
traction: string | null;
|
||
speed: number | null;
|
||
}): string {
|
||
const s = o.text.toLowerCase();
|
||
const locoTemplate = /locomotive/i.test(o.template);
|
||
if (
|
||
/tram-train|\btram|streetcar|street car|light rail|trolley(?!bus)|stadtbahn|\blrv\b/.test(s) ||
|
||
/tram/i.test(o.template)
|
||
)
|
||
return 'tram';
|
||
if (
|
||
/\bmetro\b|subway|underground|rapid transit|\bmrt\b|monorail|people mover|automated guideway|rubber-tyred|light metro|\bu-bahn/.test(
|
||
s
|
||
)
|
||
)
|
||
return 'metro';
|
||
if ((o.speed ?? 0) >= 250) return 'high-speed';
|
||
if (
|
||
/high-speed|high speed|shinkansen|bullet train|maglev|\btgv\b|velaro|frecciarossa|\bcrh\d/.test(
|
||
s
|
||
) &&
|
||
(o.speed == null || o.speed >= 190)
|
||
)
|
||
return 'high-speed';
|
||
const mu =
|
||
/multiple unit|multiple-unit|\bemu\b|\bdmu\b|\bdemu\b|\bbemu\b|railcar|railbus|rail motor|motor coach|trainset|train set|power car|sprinter|\bunit\b/.test(
|
||
s
|
||
);
|
||
const loco = /locomotive|shunter|switcher/.test(s);
|
||
const locoKind = () =>
|
||
o.traction === 'steam'
|
||
? 'steam'
|
||
: o.traction === 'electric' || o.traction === 'battery' || o.traction === 'bi-mode'
|
||
? 'electric'
|
||
: o.traction
|
||
? 'diesel'
|
||
: null;
|
||
if (locoTemplate && !mu) return locoKind() ?? (loco ? 'other' : 'other');
|
||
if (mu) return 'multiple unit';
|
||
if (
|
||
/coach|carriage|passenger car|sleeping car|dining car|restaurant car|buffet car|\bvans?\b|wagon|freight car|tank car|hopper|boxcar|caboose|observation car|parlor|parlour|sleeper|brake van|goods|rolling stock/.test(
|
||
s
|
||
)
|
||
)
|
||
return 'carriage';
|
||
if (loco) return locoKind() ?? 'other';
|
||
if (o.traction && o.traction !== 'none' && o.traction !== 'steam') return 'multiple unit';
|
||
return 'other';
|
||
}
|
||
|
||
const INDIVIDUAL =
|
||
/^(?:(?:steam|diesel|electric|diesel-electric|diesel-hydraulic|tank|tender|preserved|narrow-gauge|rack|shunting)\s+)*(?:locomotive|tram|railcar|train|multiple unit|railway vehicle|rail vehicle|individual locomotive|preserved locomotive|steam engine|rolling stock|electric multiple unit|diesel multiple unit|passenger car|railroad car)$/;
|
||
const CLASSLIKE = /class|type|model|series|family|product|design|platform/;
|
||
/** Named services (Palace on Wheels, the Capricornian): a timetable, not a kind of train. */
|
||
const SERVICE =
|
||
/luxury train|named (?:passenger )?train|train service|rail service|passenger train service|tourist train|excursion train|train route/;
|
||
|
||
// ---------------------------------------------------------------- build
|
||
|
||
const CORP = new Set(
|
||
'COMPANY CORPORATION CORP INC INCORPORATED LTD LIMITED GMBH AG SA SAS SPA PLC CO LLC KG NV BV AB AS OY OYJ SRO KK JSC PJSC OAO ZAO GROUP THE OF AND MANUFACTURING WORKS WORKSHOP WORKSHOPS LOCOMOTIVE LOCOMOTIVES TRANSPORTATION TRANSPORT MOBILITY RAIL ROLLING STOCK CAR CARS DIVISION INDUSTRIES INDUSTRY ENGINEERING HOLDINGS INTERNATIONAL DIESEL'.split(
|
||
' '
|
||
)
|
||
);
|
||
const GENERIC = new Set(
|
||
'BRITISH AMERICAN GENERAL NATIONAL UNITED STATE FEDERAL CENTRAL NORTHERN SOUTHERN EASTERN WESTERN NORTH SOUTH EAST WEST GREAT NEW ROYAL IMPERIAL'.split(
|
||
' '
|
||
)
|
||
);
|
||
const entityKey = (label: string) => {
|
||
const t = tokens(label.replace(/\(.*?\)/g, ''));
|
||
const k = t.filter((x) => !CORP.has(x));
|
||
return (k.length && !(k.length === 1 && GENERIC.has(k[0])) ? k : t).join(' ');
|
||
};
|
||
const COUNTRY_ALIAS: Record<string, string> = {
|
||
'United States of America': 'United States',
|
||
"People's Republic of China": 'China',
|
||
'Kingdom of the Netherlands': 'Netherlands',
|
||
'West Germany': 'Germany',
|
||
'German Reich': 'Germany',
|
||
'German Empire': 'Germany',
|
||
'Nazi Germany': 'Germany',
|
||
'Weimar Republic': 'Germany',
|
||
'Russian Empire': 'Russia',
|
||
'Kingdom of Italy': 'Italy',
|
||
'Empire of Japan': 'Japan',
|
||
'United Kingdom of Great Britain and Ireland': 'United Kingdom',
|
||
'Kingdom of Great Britain': 'United Kingdom',
|
||
'British Raj': 'India',
|
||
'Kingdom of Prussia': 'Germany',
|
||
'Kingdom of Bavaria': 'Germany',
|
||
'Kingdom of Saxony': 'Germany',
|
||
'Kingdom of Württemberg': 'Germany',
|
||
'French Third Republic': 'France',
|
||
'Second French Empire': 'France',
|
||
'Kingdom of Spain': 'Spain',
|
||
'Kingdom of Belgium': 'Belgium',
|
||
'Kingdom of Hungary': 'Hungary',
|
||
'Kingdom of Romania': 'Romania',
|
||
'Kingdom of Yugoslavia': 'Yugoslavia',
|
||
'Socialist Federal Republic of Yugoslavia': 'Yugoslavia',
|
||
'Republic of China (1912–1949)': 'China',
|
||
'Union of South Africa': 'South Africa',
|
||
'Dominion of Canada': 'Canada',
|
||
'Dominion of New Zealand': 'New Zealand',
|
||
'Colony of Victoria': 'Australia',
|
||
'Colony of New South Wales': 'Australia',
|
||
'Colony of Queensland': 'Australia',
|
||
Taiwan: 'Taiwan',
|
||
'Republic of China': 'Taiwan',
|
||
Korea: 'South Korea'
|
||
};
|
||
const country = (c: string | null | undefined) => (c ? (COUNTRY_ALIAS[c] ?? c) : null);
|
||
const round = (n: number | null, d = 0) => (n == null ? null : Math.round(n * 10 ** d) / 10 ** d);
|
||
const sane = (n: number | null, min: number, max: number) =>
|
||
n != null && n >= min && n <= max ? n : null;
|
||
const cleanTitle = (t: string) =>
|
||
t
|
||
.replace(
|
||
/\s*\((?:locomotive|steam locomotive|diesel locomotive|electric locomotive|train|trainset|tram|tramcar|railcar|multiple unit|electric multiple unit|diesel multiple unit|rolling stock|railway|railroad|rail vehicle|\d{4}|[^)]*locomotive[^)]*|[^)]*multiple unit[^)]*)\)$/i,
|
||
''
|
||
)
|
||
.trim();
|
||
|
||
export type BuiltType = {
|
||
wikidata: string;
|
||
slug: string;
|
||
name: string;
|
||
fullName: string;
|
||
nlName: string | null;
|
||
search: string;
|
||
wikipedia: string;
|
||
description: string | null;
|
||
summary: string | null;
|
||
category: string;
|
||
categoryLabel: string | null;
|
||
traction: string | null;
|
||
country: string | null;
|
||
countries: string[];
|
||
makers: string[];
|
||
operators: { key: string; name: string }[];
|
||
firstBuilt: number | null;
|
||
lastBuilt: number | null;
|
||
introducedYear: number | null;
|
||
retiredYear: number | null;
|
||
status: string | null;
|
||
inService: boolean | null;
|
||
numberBuilt: number | null;
|
||
maxSpeedKmh: number | null;
|
||
powerKw: number | null;
|
||
gauge: string | null;
|
||
gaugeMm: number | null;
|
||
electricSystem: string | null;
|
||
wheels: string | null;
|
||
lengthM: number | null;
|
||
weightT: number | null;
|
||
capacity: number | null;
|
||
formation: string | null;
|
||
designer: string | null;
|
||
nicknames: string | null;
|
||
parent: string | null;
|
||
sources: string[];
|
||
links: number;
|
||
};
|
||
export type BuiltMaker = {
|
||
slug: string;
|
||
name: string;
|
||
fullName: string;
|
||
wikidata: string | null;
|
||
country: string | null;
|
||
types: BuiltType[];
|
||
};
|
||
export type BuiltOperator = {
|
||
key: string;
|
||
slug: string;
|
||
name: string;
|
||
fullName: string;
|
||
wikidata: string | null;
|
||
country: string | null;
|
||
count: number;
|
||
};
|
||
|
||
const short = (s: string | null | undefined, n: number) => {
|
||
if (!s) return null;
|
||
const t = s.trim();
|
||
return t ? (t.length > n ? `${t.slice(0, n - 1).trimEnd()}…` : t) : null;
|
||
};
|
||
|
||
async function build(
|
||
articles: Article[],
|
||
items: Map<string, Item>,
|
||
linkQ: Map<string, string>,
|
||
entities: Map<string, Entity>
|
||
) {
|
||
const skipped: Record<string, number> = {};
|
||
const skippedTitles: Record<string, string[]> = {};
|
||
let current = '';
|
||
const skip = (why: string, n = 1) => {
|
||
skipped[why] = (skipped[why] ?? 0) + n;
|
||
if (n === 1 && current) (skippedTitles[why] ??= []).push(current);
|
||
};
|
||
type Draft = BuiltType & { makerKey: string; makerLabel: string; makerId: string | null };
|
||
const drafts: Draft[] = [];
|
||
const operatorInfo = new Map<
|
||
string,
|
||
{ name: string; wikidata: string | null; country: string | null }
|
||
>();
|
||
const isCompany = (e: Entity | undefined) => !!e && !e.human;
|
||
|
||
for (const a of articles) {
|
||
current = a.title;
|
||
if (!a.qid) {
|
||
skip('no wikidata item');
|
||
continue;
|
||
}
|
||
if (
|
||
/^lists? of|rolling stock$|rolling stock of|locomotives of|\bfleet\b|^history of|^\d{4} in rail/i.test(
|
||
a.title
|
||
)
|
||
) {
|
||
skip('list article');
|
||
continue;
|
||
}
|
||
if (!Object.keys(a.box).length) {
|
||
skip('no infobox parsed');
|
||
continue;
|
||
}
|
||
const it = items.get(a.qid);
|
||
if (!it) {
|
||
skip('no wikidata details');
|
||
continue;
|
||
}
|
||
const box = a.box;
|
||
const builtBoxEarly = numberBuilt(
|
||
field(box, 'totalproduction', 'numberbuilt', 'numinclass', 'quantity')
|
||
);
|
||
// Wikidata often files a class under "electric multiple unit" too, so a count above one wins.
|
||
const individual =
|
||
it.instanceOf.some((l) => INDIVIDUAL.test(l)) &&
|
||
!it.instanceOf.some((l) => CLASSLIKE.test(l)) &&
|
||
!((it.built ?? builtBoxEarly ?? 0) > 1);
|
||
// "Santa Fe 2926", "GWR 6800 Class 6880 Betton Grange": one preserved engine, whose
|
||
// infobox "numinclass" counts its class, not itself.
|
||
const countBuilt = it.built ?? numberBuilt(field(box, 'totalproduction', 'numberbuilt'));
|
||
const title = cleanTitle(a.title);
|
||
const oneEngine =
|
||
/\bclass\s+\S+\s+(?:No\.\s*)?\d{2,5}[A-Z]?\s+[A-Z][a-z]/.test(title) ||
|
||
(!/class|series|type|shinkansen|stock|railcar|unit|model|tram|car\b/i.test(title) &&
|
||
/^\D+?\s(?:No\.\s*)?\d{1,5}(?:-[A-Z])?(?:\s+[A-Z][\w'.]*)*$/.test(title) &&
|
||
!!box.serialnumber &&
|
||
!((countBuilt ?? 0) > 1));
|
||
const builtBox = numberBuilt(
|
||
field(box, 'totalproduction', 'numberbuilt', 'numinclass', 'quantity')
|
||
);
|
||
const built = it.built ?? builtBox;
|
||
if (individual || oneEngine || (built === 1 && !it.instanceOf.some((l) => CLASSLIKE.test(l)))) {
|
||
skip('individual vehicle');
|
||
continue;
|
||
}
|
||
const desc = it.description ?? '';
|
||
if (
|
||
built === 0 ||
|
||
(/\b(proposed|cancelled|canceled|planned|concept|unbuilt)\b/i.test(desc) && !built)
|
||
) {
|
||
skip('never built');
|
||
continue;
|
||
}
|
||
|
||
// Makers: Wikidata manufacturer, else the linked builder in the infobox, else its plain text.
|
||
const builderField = field(
|
||
box,
|
||
'builder',
|
||
'manufacturer',
|
||
'builtby',
|
||
'manufacturers',
|
||
'builders'
|
||
);
|
||
const builderLinks = (builderField ? linkTargets(builderField) : a.leadBuilders)
|
||
.map((t) => linkQ.get(t))
|
||
.filter((q): q is string => !!q && isCompany(entities.get(q)) && !entities.get(q)!.place);
|
||
const makerIds = [...new Set([...builderLinks, ...it.makers.map((m) => m.id)])];
|
||
const builderPlain = builderField ? plain(builderField) : '';
|
||
const builderText = builderPlain.toLowerCase();
|
||
// Primary maker: the first linked builder, or the Wikidata maker named first in the infobox.
|
||
const makerId =
|
||
builderLinks[0] ??
|
||
[...it.makers].sort(
|
||
(x, y) =>
|
||
(builderText.indexOf(x.label.toLowerCase()) + 1 || 999) -
|
||
(builderText.indexOf(y.label.toLowerCase()) + 1 || 999)
|
||
)[0]?.id ??
|
||
null;
|
||
let makerLabel = makerId
|
||
? (entities.get(makerId)?.label ?? it.makers.find((m) => m.id === makerId)?.label ?? makerId)
|
||
: '';
|
||
if (!makerId) {
|
||
const first = builderPlain
|
||
.split(/,|;|\band\b|\(|\//)[0]
|
||
.replace(/^(?:built by|by)\s+/i, '')
|
||
.trim();
|
||
if (first && first.length >= 2 && first.length <= 60 && !/^\d/.test(first))
|
||
makerLabel = first;
|
||
}
|
||
if (!makerLabel) {
|
||
makerLabel = 'Unknown builder';
|
||
skip('maker unknown (kept)');
|
||
}
|
||
const makerNames = [
|
||
...new Set([
|
||
makerLabel,
|
||
...makerIds
|
||
.map((q) => entities.get(q)?.label ?? it.makers.find((m) => m.id === q)?.label ?? '')
|
||
.filter(Boolean)
|
||
])
|
||
].slice(0, 8);
|
||
|
||
// Operators: Wikidata operators + linked operators in the infobox.
|
||
const opField = field(box, 'operator', 'operators', 'currentowner', 'owner', 'owners');
|
||
const opLinks = linkTargets(opField)
|
||
.map((t) => linkQ.get(t))
|
||
.filter((q): q is string => !!q && isCompany(entities.get(q)) && !entities.get(q)!.place);
|
||
const opIds = [...new Set([...opLinks, ...it.operators.map((o) => o.id)])].slice(0, 14);
|
||
const operators: { key: string; name: string }[] = [];
|
||
for (const q of opIds) {
|
||
const e = entities.get(q);
|
||
const name = e?.label ?? it.operators.find((o) => o.id === q)?.label;
|
||
if (!name || /^Q\d+$/.test(name)) continue;
|
||
operators.push({ key: q, name });
|
||
if (!operatorInfo.has(q))
|
||
operatorInfo.set(q, { name, wikidata: q, country: country(e?.country) });
|
||
}
|
||
if (!operators.length && opField && !linkTargets(opField).length) {
|
||
for (const name of plain(opField)
|
||
.split(/,|;|\n/)
|
||
.map((x) => x.replace(/\(.*?\)/g, '').trim())
|
||
.filter((x) => x.length >= 2 && x.length <= 60 && !/^\d/.test(x))
|
||
.slice(0, 6)) {
|
||
const key = `name:${slugify(name)}`;
|
||
operators.push({ key, name });
|
||
if (!operatorInfo.has(key)) operatorInfo.set(key, { name, wikidata: null, country: null });
|
||
}
|
||
}
|
||
|
||
const stock = plain(field(box, 'stocktype', 'type', 'powertype', 'power') ?? '');
|
||
const elec = field(
|
||
box,
|
||
'electricsystem',
|
||
'electricsystems',
|
||
'electricsystem(s)',
|
||
'currentcollection',
|
||
'collectionmethod'
|
||
);
|
||
const tractionText = [
|
||
plain(
|
||
field(
|
||
box,
|
||
'powertype',
|
||
'power',
|
||
'traction',
|
||
'fueltype',
|
||
'primemover',
|
||
'stocktype',
|
||
'type'
|
||
) ?? ''
|
||
),
|
||
it.instanceOf.join(' '),
|
||
it.classLabels.join(' '),
|
||
desc,
|
||
a.title
|
||
].join(' | ');
|
||
// Steam-era infoboxes rarely say "steam": boiler, firebox, cylinder or tender fields give it away,
|
||
// as do German steam class codes like "1'C h2" or "C n2t".
|
||
const steamBox =
|
||
Object.keys(box).some((k) =>
|
||
/^(boiler|firebox|firegrate|tubearea|superheater|cylinder|tender|tubesandflues|watercap|valvegear)/.test(
|
||
k
|
||
)
|
||
) ||
|
||
/(?:^|[\s'′])[nh][2-4](?:v|t|vt)?\b/.test(
|
||
plain(field(box, 'uicclass', 'wheelarrangement', 'whytetype') ?? '')
|
||
);
|
||
const traction =
|
||
tractionOf(tractionText, !!(elec && plain(elec))) ?? (steamBox ? 'steam' : null);
|
||
const speedBox = speedKmh(
|
||
field(box, 'maxspeed', 'topspeed', 'speed', 'operatingspeed', 'designspeed')
|
||
);
|
||
const speed = round(sane(speedBox ?? it.speed, 5, 650));
|
||
const catText = [
|
||
stock,
|
||
it.instanceOf.join(' '),
|
||
it.classLabels.join(' '),
|
||
desc,
|
||
cleanTitle(a.title)
|
||
].join(' | ');
|
||
const category = categoryOf({
|
||
template: `${a.template} ${a.boxName}`,
|
||
text: catText,
|
||
traction,
|
||
speed
|
||
});
|
||
if (
|
||
it.instanceOf.some((l) => SERVICE.test(l)) ||
|
||
(category === 'other' && makerLabel === 'Unknown builder')
|
||
) {
|
||
skip('train service, not rolling stock');
|
||
continue;
|
||
}
|
||
|
||
// Years.
|
||
const buildYears = years(
|
||
plain(
|
||
field(
|
||
box,
|
||
'builddate',
|
||
'yearconstructed',
|
||
'constructed',
|
||
'yearsbuilt',
|
||
'built',
|
||
'buildyear',
|
||
'builddates'
|
||
) ?? ''
|
||
)
|
||
);
|
||
const serviceText = plain(
|
||
field(
|
||
box,
|
||
'service',
|
||
'inservice',
|
||
'yearservice',
|
||
'enteredservice',
|
||
'introduced',
|
||
'firstrundate',
|
||
'deliverydate'
|
||
) ?? ''
|
||
);
|
||
const serviceYears = years(serviceText);
|
||
const retiredText = plain(
|
||
field(
|
||
box,
|
||
'retiredate',
|
||
'retired',
|
||
'withdrawndate',
|
||
'withdrawn',
|
||
'yearscrapped',
|
||
'scrapped',
|
||
'scrapdate',
|
||
'lastrundate'
|
||
) ?? ''
|
||
);
|
||
const firstBuilt = buildYears.length
|
||
? Math.min(...buildYears)
|
||
: (it.inception ?? serviceYears[0] ?? it.introduced ?? null);
|
||
const lastBuilt =
|
||
buildYears.length > 1 ? Math.max(...buildYears) : buildYears.length ? buildYears[0] : null;
|
||
const introducedYear =
|
||
it.introduced ?? (serviceYears.length ? Math.min(...serviceYears) : null);
|
||
const present = /present|current|in service|today/i.test(serviceText);
|
||
let retiredYear: number | null =
|
||
it.retired ?? (years(retiredText).length ? Math.max(...years(retiredText)) : null);
|
||
if (!retiredYear && !present && serviceYears.length >= 2)
|
||
retiredYear = Math.max(...serviceYears);
|
||
if (present) retiredYear = null;
|
||
const disposition = plain(
|
||
field(box, 'disposition', 'status', 'preserved', 'numberpreserved') ?? ''
|
||
).toLowerCase();
|
||
let status: string | null = null;
|
||
if (present || /in service|in use|operational|active/.test(disposition)) status = 'in service';
|
||
else if (retiredYear || /withdrawn|scrapped|retired/.test(disposition))
|
||
status = /preserved|museum|heritage/.test(disposition) ? 'preserved' : 'retired';
|
||
else if (/preserved|museum|heritage/.test(disposition)) status = 'preserved';
|
||
const inService = status === 'in service' ? true : status ? false : null;
|
||
|
||
const g = gauge(field(box, 'gauge', 'trackgauge'), it.gauge);
|
||
const origin = country(it.origin[0]) ?? country(it.country[0]);
|
||
const makerCountry = makerId ? country(entities.get(makerId)?.country) : null;
|
||
const opCountries = operators
|
||
.map((o) => operatorInfo.get(o.key)?.country)
|
||
.filter((c): c is string => !!c);
|
||
const mainCountry = origin ?? opCountries[0] ?? makerCountry ?? null;
|
||
const countries = [
|
||
...new Set([mainCountry, ...opCountries].filter((c): c is string => !!c))
|
||
].slice(0, 12);
|
||
|
||
const mk = entityKey(makerLabel);
|
||
const mt = tokens(makerLabel.replace(/\(.*?\)/g, ''));
|
||
const tt = tokens(title);
|
||
// "Siemens Velaro" -> "Velaro" (maker words dropped only when the title starts with them).
|
||
const pre = mt.filter((x) => !CORP.has(x));
|
||
let name = title;
|
||
if (pre.length && tt.length > pre.length && pre.every((x, i) => tt[i] === x))
|
||
name = title.split(/\s+/).slice(pre.length).join(' ') || title;
|
||
const wheels = plain(
|
||
field(box, 'whytetype', 'uicclass', 'aarwheels', 'wheelarrangement', 'axlearrangement') ?? ''
|
||
)
|
||
.replace(/\s+/g, ' ')
|
||
.split(/,\s/)[0];
|
||
const lengthVal = lengthM(
|
||
field(box, 'trainlength', 'length', 'overalllength', 'carlength', 'lengthovercouplers')
|
||
);
|
||
const weightVal = weightT(
|
||
field(
|
||
box,
|
||
'locoweight',
|
||
'trainweight',
|
||
'weight',
|
||
'emptyweight',
|
||
'carweight',
|
||
'locotenderweight'
|
||
)
|
||
);
|
||
const t: Draft = {
|
||
wikidata: a.qid,
|
||
slug: slugify(name),
|
||
name,
|
||
fullName: title,
|
||
nlName:
|
||
it.nl && cmp(it.nl) !== cmp(title) && cmp(it.nl) !== cmp(it.label ?? '') ? it.nl : null,
|
||
search: '',
|
||
wikipedia: a.title,
|
||
description: short(it.description, 300),
|
||
summary: a.extract && a.extract.length > 40 ? a.extract.slice(0, 900) : null,
|
||
category,
|
||
categoryLabel: short(stock.replace(/^an? /i, ''), 70),
|
||
traction,
|
||
country: mainCountry,
|
||
countries,
|
||
makers: makerNames,
|
||
operators,
|
||
firstBuilt: sane(firstBuilt, 1800, thisYear + 10),
|
||
lastBuilt: sane(lastBuilt, 1800, thisYear + 10),
|
||
introducedYear: sane(introducedYear, 1800, thisYear + 10),
|
||
retiredYear: sane(retiredYear, 1800, thisYear + 1),
|
||
status,
|
||
inService,
|
||
numberBuilt: sane(built, 1, 200_000),
|
||
maxSpeedKmh: speed,
|
||
powerKw: round(
|
||
sane(
|
||
powerKw(
|
||
field(box, 'poweroutput', 'power', 'powerout', 'enginepower', 'output', 'tractionpower')
|
||
) ?? it.power,
|
||
1,
|
||
30_000
|
||
)
|
||
),
|
||
gauge: g.label,
|
||
gaugeMm: g.mm,
|
||
electricSystem: short(
|
||
elec
|
||
? plain(field(box, 'electricsystem', 'electricsystems', 'electricsystem(s)') ?? '')
|
||
: null,
|
||
140
|
||
),
|
||
wheels: wheels && wheels.length <= 40 ? wheels : null,
|
||
lengthM: round(sane(lengthVal ?? it.length, 2, 500), 2),
|
||
weightT: round(sane(weightVal ?? (it.mass ? it.mass / 1000 : null), 0.3, 3000), 1),
|
||
capacity: sane(capacity(field(box, 'capacity', 'seating', 'seats', 'passengers')), 1, 4000),
|
||
formation: short(plain(field(box, 'formation', 'carspertrainset', 'cars') ?? ''), 120),
|
||
designer: short(plain(field(box, 'designer', 'designers') ?? ''), 100),
|
||
nicknames: short(plain(field(box, 'nicknames', 'nickname') ?? ''), 120),
|
||
parent: null,
|
||
sources: [SRC_WP, SRC_WD],
|
||
links: it.links,
|
||
makerKey: mk,
|
||
makerLabel,
|
||
makerId
|
||
};
|
||
drafts.push(t);
|
||
}
|
||
|
||
// One entry per Wikidata item (several articles can share one via redirects).
|
||
const byQ = new Map<string, Draft>();
|
||
for (const d of drafts) {
|
||
const cur = byQ.get(d.wikidata);
|
||
if (!cur || d.links > cur.links) byQ.set(d.wikidata, d);
|
||
}
|
||
if (byQ.size < drafts.length) skip('same item as another article', drafts.length - byQ.size);
|
||
drafts.splice(0, drafts.length, ...byQ.values());
|
||
const kept = new Set(drafts.map((d) => d.wikidata));
|
||
for (const d of drafts) {
|
||
const p = items.get(d.wikidata)!.classes.find((c) => kept.has(c) && c !== d.wikidata);
|
||
d.parent = p ?? null;
|
||
}
|
||
|
||
// Makers: grouped by normalised name; the most used label (and its item) names the group.
|
||
const groups = new Map<string, Draft[]>();
|
||
for (const d of drafts) groups.set(d.makerKey, [...(groups.get(d.makerKey) ?? []), d]);
|
||
const makers = new Map<string, BuiltMaker>();
|
||
for (const list of [...groups.values()].sort((a, b) => b.length - a.length)) {
|
||
const labels = new Map<string, { n: number; id: string | null }>();
|
||
for (const d of list) {
|
||
const cur = labels.get(d.makerLabel) ?? { n: 0, id: d.makerId };
|
||
cur.n++;
|
||
cur.id ??= d.makerId;
|
||
labels.set(d.makerLabel, cur);
|
||
}
|
||
const [label, best] = [...labels].sort(
|
||
(a, b) => Number(!!b[1].id) - Number(!!a[1].id) || b[1].n - a[1].n
|
||
)[0];
|
||
const display = label
|
||
.replace(/\s*\((?:company|manufacturer|firm|railway|locomotive manufacturer)\)$/i, '')
|
||
.trim();
|
||
let slug = slugify(display);
|
||
if (['makers', 'operators', 'countries', 'random'].includes(slug)) slug = `${slug}-maker`;
|
||
const existing = makers.get(slug);
|
||
if (existing) existing.types.push(...list);
|
||
else
|
||
makers.set(slug, {
|
||
slug,
|
||
name: display,
|
||
fullName: label,
|
||
wikidata: best.id,
|
||
country:
|
||
label === 'Unknown builder'
|
||
? null
|
||
: ((best.id ? country(entities.get(best.id)?.country) : null) ??
|
||
[
|
||
...list.reduce(
|
||
(m, t) => (t.country ? m.set(t.country, (m.get(t.country) ?? 0) + 1) : m),
|
||
new Map<string, number>()
|
||
)
|
||
].sort((a, b) => b[1] - a[1])[0]?.[0] ??
|
||
null),
|
||
types: list
|
||
});
|
||
}
|
||
// Operators: by Wikidata item (or name), slug from the label.
|
||
const operatorCount = new Map<string, number>();
|
||
for (const d of drafts)
|
||
for (const o of d.operators) operatorCount.set(o.key, (operatorCount.get(o.key) ?? 0) + 1);
|
||
const operators = new Map<string, BuiltOperator>(); // key -> operator
|
||
const slugTaken = new Map<string, string>(); // slug -> key
|
||
for (const [key, count] of [...operatorCount].sort((a, b) => b[1] - a[1])) {
|
||
const info = operatorInfo.get(key)!;
|
||
const display = info.name
|
||
.replace(/\s*\((?:company|railway|operator|train operating company)\)$/i, '')
|
||
.trim();
|
||
let slug = slugify(display);
|
||
if (slugTaken.has(slug)) {
|
||
// Same name, different item: merge plain-name operators into the item, else disambiguate.
|
||
const other = slugTaken.get(slug)!;
|
||
if (key.startsWith('name:') || other.startsWith('name:')) {
|
||
operators.set(key, { ...operators.get(other)! });
|
||
operators.get(other)!.count += count;
|
||
continue;
|
||
}
|
||
slug = `${slug}-${key.toLowerCase()}`;
|
||
}
|
||
slugTaken.set(slug, key);
|
||
operators.set(key, {
|
||
key,
|
||
slug,
|
||
name: display,
|
||
fullName: info.name,
|
||
wikidata: info.wikidata,
|
||
country: info.country,
|
||
count
|
||
});
|
||
}
|
||
// Unique type slugs per maker; the best-known type keeps the plain slug.
|
||
for (const m of makers.values()) {
|
||
const seen = new Set<string>();
|
||
m.types.sort((a, b) => b.links - a.links);
|
||
for (const t of m.types) {
|
||
let s = t.slug;
|
||
if (seen.has(s)) s = `${s}-${t.wikidata.toLowerCase()}`;
|
||
seen.add(s);
|
||
t.slug = s;
|
||
const it = items.get(t.wikidata)!;
|
||
t.search = [
|
||
m.name,
|
||
t.fullName,
|
||
t.name,
|
||
t.nlName ?? '',
|
||
it.label ?? '',
|
||
...it.aliases.slice(0, 12),
|
||
...t.makers,
|
||
...t.operators.map((o) => o.name),
|
||
...t.operators.map((o) => operators.get(o.key)?.name ?? ''),
|
||
cmp(t.fullName).toLowerCase(),
|
||
cmp(t.name).toLowerCase(),
|
||
t.categoryLabel ?? '',
|
||
t.wheels ?? '',
|
||
t.nicknames ?? ''
|
||
]
|
||
.join(' ')
|
||
.toLowerCase()
|
||
.slice(0, 2500);
|
||
}
|
||
}
|
||
log(`skipped: ${JSON.stringify(skipped)}`);
|
||
if (outFile && outFile !== '-')
|
||
await writeFile(`${outFile}.skipped.json`, JSON.stringify(skippedTitles, null, 1));
|
||
return { makers: [...makers.values()], operators };
|
||
}
|
||
|
||
// ---------------------------------------------------------------- write
|
||
|
||
const added = {
|
||
makers: [] as { name: string; url: string }[],
|
||
types: [] as {
|
||
dex: number;
|
||
name: string;
|
||
maker: string;
|
||
category: string;
|
||
firstBuilt: number | null;
|
||
numberBuilt: number | null;
|
||
url: string;
|
||
photo?: string | null;
|
||
}[]
|
||
};
|
||
|
||
async function write(makers: BuiltMaker[], operators: Map<string, BuiltOperator>) {
|
||
const url = process.env.DB_URL;
|
||
if (!url) throw new Error('Set DB_URL (or use --dry-run)');
|
||
const sql = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
||
try {
|
||
await sql.begin(async (tx) => {
|
||
const oldTypes = new Map(
|
||
(
|
||
await tx<
|
||
{ id: number; wikidata: string; dex: number }[]
|
||
>`select id, wikidata, dex from trains.types`
|
||
).map((r) => [r.wikidata, r])
|
||
);
|
||
const oldMakers = new Set(
|
||
(await tx<{ slug: string }[]>`select slug from trains.makers`).map((r) => r.slug)
|
||
);
|
||
const firstRun = oldTypes.size === 0;
|
||
if (renumber) await tx`update trains.types set dex = -id`;
|
||
const all = makers.flatMap((m) => m.types.map((t) => ({ m, t })));
|
||
let nextDex = renumber ? 1 : Math.max(0, ...[...oldTypes.values()].map((r) => r.dex)) + 1;
|
||
const order = all.toSorted(
|
||
(a, b) =>
|
||
(a.t.firstBuilt ?? 9999) - (b.t.firstBuilt ?? 9999) ||
|
||
a.t.fullName.localeCompare(b.t.fullName)
|
||
);
|
||
const dex = new Map<string, number>();
|
||
for (const { t } of order) {
|
||
const old = oldTypes.get(t.wikidata);
|
||
dex.set(t.wikidata, !renumber && old ? old.dex : nextDex++);
|
||
}
|
||
const makerIds = new Map<string, number>();
|
||
for (const m of makers) {
|
||
const ys = m.types.map((t) => t.firstBuilt).filter((y): y is number => y != null);
|
||
const built = m.types.reduce((a, t) => a + (t.parent ? 0 : (t.numberBuilt ?? 0)), 0);
|
||
const row = {
|
||
slug: m.slug,
|
||
name: m.name,
|
||
full_name: m.fullName,
|
||
country: m.country,
|
||
wikidata: m.wikidata,
|
||
type_count: m.types.length,
|
||
first_year: ys.length ? Math.min(...ys) : null,
|
||
last_year: ys.length ? Math.max(...ys) : null,
|
||
built_total: Math.min(built, 2_000_000_000),
|
||
sources: [SRC_WP, SRC_WD]
|
||
};
|
||
const [r] = await tx<{ id: number }[]>`
|
||
insert into trains.makers ${tx(row)}
|
||
on conflict (slug) do update set name = excluded.name, full_name = excluded.full_name,
|
||
country = excluded.country, wikidata = excluded.wikidata, type_count = excluded.type_count,
|
||
first_year = excluded.first_year, last_year = excluded.last_year,
|
||
built_total = excluded.built_total, sources = excluded.sources
|
||
returning id`;
|
||
makerIds.set(m.slug, r.id);
|
||
if (!firstRun && !oldMakers.has(m.slug))
|
||
added.makers.push({ name: m.name, url: `${SITE}/${m.slug}` });
|
||
}
|
||
const opIds = new Map<string, number>(); // slug -> id
|
||
for (const o of new Map([...operators.values()].map((o) => [o.slug, o])).values()) {
|
||
const row = {
|
||
slug: o.slug,
|
||
name: o.name,
|
||
full_name: o.fullName,
|
||
country: o.country,
|
||
wikidata: o.wikidata,
|
||
type_count: o.count
|
||
};
|
||
const [r] = await tx<{ id: number }[]>`
|
||
insert into trains.operators ${tx(row)}
|
||
on conflict (slug) do update set name = excluded.name, full_name = excluded.full_name,
|
||
country = excluded.country, wikidata = excluded.wikidata, type_count = excluded.type_count
|
||
returning id`;
|
||
opIds.set(o.slug, r.id);
|
||
}
|
||
await tx`update trains.types set slug = 'tmp-' || id`;
|
||
const ids = new Map<string, number>();
|
||
for (const { m, t } of all) {
|
||
const row = {
|
||
dex: dex.get(t.wikidata)!,
|
||
maker_id: makerIds.get(m.slug)!,
|
||
slug: t.slug,
|
||
name: t.name,
|
||
full_name: t.fullName,
|
||
nl_name: t.nlName,
|
||
search: t.search,
|
||
wikidata: t.wikidata,
|
||
wikipedia: t.wikipedia,
|
||
description: t.description,
|
||
summary: t.summary,
|
||
category: t.category,
|
||
category_label: t.categoryLabel,
|
||
traction: t.traction,
|
||
country: t.country,
|
||
countries: t.countries,
|
||
makers: t.makers,
|
||
operators: [...new Set(t.operators.map((o) => operators.get(o.key)?.name ?? o.name))],
|
||
first_built: t.firstBuilt,
|
||
last_built: t.lastBuilt,
|
||
introduced_year: t.introducedYear,
|
||
retired_year: t.retiredYear,
|
||
status: t.status,
|
||
in_service: t.inService,
|
||
number_built: t.numberBuilt,
|
||
max_speed_kmh: t.maxSpeedKmh,
|
||
power_kw: t.powerKw,
|
||
gauge: t.gauge,
|
||
gauge_mm: t.gaugeMm,
|
||
electric_system: t.electricSystem,
|
||
wheels: t.wheels,
|
||
length_m: t.lengthM,
|
||
weight_t: t.weightT,
|
||
capacity: t.capacity,
|
||
formation: t.formation,
|
||
designer: t.designer,
|
||
nicknames: t.nicknames,
|
||
sources: t.sources
|
||
};
|
||
const [r] = await tx<{ id: number }[]>`
|
||
insert into trains.types ${tx(row)}
|
||
on conflict (wikidata) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'wikidata') as (keyof typeof row)[])}
|
||
returning id`;
|
||
ids.set(t.wikidata, r.id);
|
||
if (!firstRun && !oldTypes.has(t.wikidata))
|
||
added.types.push({
|
||
dex: row.dex,
|
||
name: t.fullName,
|
||
maker: m.name,
|
||
category: t.category,
|
||
firstBuilt: t.firstBuilt,
|
||
numberBuilt: t.numberBuilt,
|
||
url: `${SITE}/${m.slug}/${t.slug}`
|
||
});
|
||
}
|
||
for (const { t } of all)
|
||
if (t.parent && ids.has(t.parent))
|
||
await tx`update trains.types set family_id = ${ids.get(t.parent)!} where id = ${ids.get(t.wikidata)!}`;
|
||
else
|
||
await tx`update trains.types set family_id = null where id = ${ids.get(t.wikidata)!} and family_id is not null`;
|
||
const keep = [...ids.values()];
|
||
await tx`delete from trains.type_operators where type_id = any(${keep})`;
|
||
const links: { type_id: number; operator_id: number; position: number }[] = [];
|
||
for (const { t } of all) {
|
||
const seen = new Set<number>();
|
||
t.operators.forEach((o) => {
|
||
const op = operators.get(o.key);
|
||
const id = op ? opIds.get(op.slug) : undefined;
|
||
if (id && !seen.has(id)) {
|
||
seen.add(id);
|
||
links.push({ type_id: ids.get(t.wikidata)!, operator_id: id, position: seen.size - 1 });
|
||
}
|
||
});
|
||
}
|
||
for (const c of chunks(links, 2000)) await tx`insert into trains.type_operators ${tx(c)}`;
|
||
await tx`update trains.operators o set type_count = (select count(*) from trains.type_operators x where x.operator_id = o.id)`;
|
||
const gone = await tx`delete from trains.types where not (id = any(${keep})) returning id`;
|
||
const goneMakers =
|
||
await tx`delete from trains.makers where not (slug = any(${makers.map((m) => m.slug)})) returning id`;
|
||
const goneOps =
|
||
await tx`delete from trains.operators where not (id = any(${[...opIds.values()]})) returning id`;
|
||
log(
|
||
`written: ${keep.length} types, ${makers.length} makers, ${opIds.size} operators, ${links.length} links; removed ${gone.length} types, ${goneMakers.length} makers, ${goneOps.length} operators`
|
||
);
|
||
});
|
||
} finally {
|
||
await sql.end();
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------- main
|
||
|
||
await useCache(cacheDir);
|
||
const titles = await listArticles();
|
||
const articles = await loadArticles(titles);
|
||
const qids = [...new Set(articles.map((a) => a.qid).filter((q): q is string => !!q))];
|
||
const items = await loadItems(qids);
|
||
const linkTitles = articles.flatMap((a) => [
|
||
...linkTargets(
|
||
field(a.box, 'builder', 'manufacturer', 'builtby', 'manufacturers', 'builders')
|
||
).slice(0, 6),
|
||
...a.leadBuilders,
|
||
...linkTargets(field(a.box, 'operator', 'operators', 'currentowner', 'owner', 'owners')).slice(
|
||
0,
|
||
14
|
||
)
|
||
]);
|
||
log(`wikipedia: resolving ${new Set(linkTitles).size} linked builders/operators`);
|
||
const linkQ = await resolveTitles(linkTitles);
|
||
const entityIds = [
|
||
...new Set([
|
||
...linkQ.values(),
|
||
...[...items.values()].flatMap((i) => [...i.makers, ...i.operators].map((m) => m.id))
|
||
])
|
||
];
|
||
const entities = await loadEntities(entityIds);
|
||
const { makers, operators } = await build(articles, items, linkQ, entities);
|
||
const types = makers.flatMap((m) => m.types);
|
||
const byCat = types.reduce(
|
||
(m, t) => m.set(t.category, (m.get(t.category) ?? 0) + 1),
|
||
new Map<string, number>()
|
||
);
|
||
log(
|
||
`built ${makers.length} makers, ${new Set([...operators.values()].map((o) => o.slug)).size} operators, ${types.length} types (${JSON.stringify(Object.fromEntries(byCat))}; ${types.filter((t) => t.numberBuilt).length} built, ${types.filter((t) => t.maxSpeedKmh).length} speed, ${types.filter((t) => t.powerKw).length} power, ${types.filter((t) => t.gaugeMm).length} gauge, ${types.filter((t) => t.operators.length).length} operators, ${types.filter((t) => t.firstBuilt).length} year)`
|
||
);
|
||
if (outFile) {
|
||
const json = JSON.stringify(makers);
|
||
if (outFile === '-') console.log(json);
|
||
else await writeFile(outFile, json);
|
||
}
|
||
if (dryRun) {
|
||
for (const t of types
|
||
.toSorted((a, b) => (b.numberBuilt ?? 0) - (a.numberBuilt ?? 0))
|
||
.slice(0, 15))
|
||
log(
|
||
` ${t.fullName} [${t.makers[0]}] built=${t.numberBuilt} ${t.category}/${t.traction} ${t.firstBuilt} ${t.maxSpeedKmh}km/h ${t.powerKw}kW ${t.gauge}`
|
||
);
|
||
} else {
|
||
await write(makers, operators);
|
||
if (reportNew)
|
||
console.log(
|
||
JSON.stringify(
|
||
{
|
||
generatedAt: new Date().toISOString(),
|
||
totals: { makers: makers.length, types: types.length },
|
||
newMakers: added.makers,
|
||
newTypes: added.types.toSorted((a, b) => (b.numberBuilt ?? 0) - (a.numberBuilt ?? 0))
|
||
},
|
||
null,
|
||
2
|
||
)
|
||
);
|
||
}
|