From 8c0fc74283cd258ba811872a23b22387722262d0 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sun, 25 Oct 2026 00:09:07 +0000 Subject: [PATCH] trains: Wikidata/Wikipedia import, Commons photos and refresh scripts --- package.json | 3 + scripts/trains-import.ts | 1656 +++++++++++++++++++++++++++++++++++++ scripts/trains-photos.ts | 759 +++++++++++++++++ scripts/trains-refresh.sh | 27 + scripts/trains-wiki.ts | 191 +++++ 5 files changed, 2636 insertions(+) create mode 100644 scripts/trains-import.ts create mode 100644 scripts/trains-photos.ts create mode 100755 scripts/trains-refresh.sh create mode 100644 scripts/trains-wiki.ts diff --git a/package.json b/package.json index 62b0db4..17e0627 100755 --- a/package.json +++ b/package.json @@ -26,6 +26,9 @@ "planes:import": "bun scripts/planes-import.ts", "planes:photos": "bun scripts/planes-photos.ts", "planes:refresh": "scripts/planes-refresh.sh", + "trains:import": "bun scripts/trains-import.ts", + "trains:photos": "bun scripts/trains-photos.ts", + "trains:refresh": "scripts/trains-refresh.sh", "wildlife:import": "bun scripts/wildlife-import.ts", "wildlife:photos": "bun scripts/wildlife-photos.ts", "wildlife:refresh": "scripts/wildlife-refresh.sh", diff --git a/scripts/trains-import.ts b/scripts/trains-import.ts new file mode 100644 index 0000000..75d7e45 --- /dev/null +++ b/scripts/trains-import.ts @@ -0,0 +1,1656 @@ +// Builds the public train pokedex (/trackers/trains, schema `trains`) from open data. +// +// DB_URL=... bun scripts/trains-import.ts fetch sources and write to the database +// bun scripts/trains-import.ts --dry-run fetch + build only, print a summary +// bun scripts/trains-import.ts --cache /tmp/trains reuse API responses from that folder +// (one small file per request, resumable) +// bun scripts/trains-import.ts --dry-run --out x.json also dump the built catalogue +// bun scripts/trains-import.ts --renumber reassign Dex numbers from #1 (oldest first) +// bun scripts/trains-import.ts --report-new JSON of makers/types new since the last run on stdout +// +// Sources (no API keys): +// - English Wikipedia (CC BY-SA): every article that uses {{Infobox locomotive}}, {{Infobox +// train}} or {{Infobox tram}} (and their redirects such as {{Infobox EMU}}): builder, designer, +// operators, build dates, number built, service dates, top speed, power, gauge, electric +// system, wheel arrangement, length, weight, capacity, plus a 3-sentence intro extract. +// - Wikidata (CC0) for the same items: manufacturer (P176), operator (P137), country (P495/P17), +// number built (P1092), service entry/retirement (P729/P730), speed, power, gauge, English and +// Dutch labels, and "subclass of" for families. Linked builders/operators are resolved to their +// Wikidata items (label, country) so makers and operators group cleanly. +// Only classes/types: individual locomotives, list articles and unbuilt projects are skipped. +// Unknown values stay null; nothing is estimated. +// Re-runnable: types are upserted by Wikidata id (Dex numbers stay stable, new types get the +// next number), types that vanished are deleted, makers/operators are upserted by slug. +import postgres from 'postgres'; +import { writeFile } from 'node:fs/promises'; +import { + chunks, + cmp, + log as logErr, + mw, + qid, + slugify, + sparql, + tokens, + useCache +} from './trains-wiki'; + +const args = process.argv.slice(2); +const dryRun = args.includes('--dry-run'); +const renumber = args.includes('--renumber'); +const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null); +const cacheDir = opt('--cache'); +const outFile = opt('--out'); +const reportNew = args.includes('--report-new'); +const SITE = 'https://stanrunge.dev/trackers/trains'; +const log = (...a: unknown[]) => (reportNew || outFile === '-' ? logErr(...a) : console.log(...a)); +const SRC_WD = 'Wikidata'; +const SRC_WP = 'Wikipedia'; +const TEMPLATES = ['Infobox locomotive', 'Infobox train', 'Infobox tram']; +const thisYear = new Date().getFullYear(); +const years = (s: string | undefined | null) => + [...(s ?? '').matchAll(/\b(1[89]\d\d|20[0-4]\d)\b/g)] + .map((m) => Number(m[1])) + .filter((y) => y >= 1800 && y <= thisYear + 10); +const year = (s: string | undefined | null) => years(s)[0] ?? null; + +// ---------------------------------------------------------------- wikitext + +/** The `{{name ...}}` template (first match), with nested braces balanced. */ +function findTemplate(text: string, name: RegExp): { name: string; body: string } | null { + const re = /\{\{\s*([^|}\n]+)/g; + let m: RegExpExecArray | null; + while ((m = re.exec(text))) { + if (!name.test(m[1].trim())) continue; + let depth = 0; + for (let i = m.index; i < text.length - 1; i++) { + if (text[i] === '{' && text[i + 1] === '{') { + depth++; + i++; + } else if (text[i] === '}' && text[i + 1] === '}') { + depth--; + i++; + if (depth === 0) return { name: m[1].trim(), body: text.slice(m.index + 2, i - 1) }; + } + } + return null; + } + return null; +} +function templateParams(tpl: string): Record { + const out: Record = {}; + const parts: string[] = []; + let depth = 0; + let link = 0; + let cur = ''; + for (let i = 0; i < tpl.length; i++) { + const two = tpl.slice(i, i + 2); + if (two === '{{') depth++; + else if (two === '}}') depth--; + else if (two === '[[') link++; + else if (two === ']]') link--; + if (tpl[i] === '|' && depth === 0 && link === 0) { + parts.push(cur); + cur = ''; + continue; + } + cur += tpl[i]; + } + parts.push(cur); + for (const p of parts.slice(1)) { + const eq = p.indexOf('='); + if (eq < 0) continue; + const key = p + .slice(0, eq) + .trim() + .toLowerCase() + .replace(/[_\s]+/g, ''); + const value = p.slice(eq + 1).trim(); + if (key && value) out[key] = value; + } + return out; +} +const noRefs = (v: string) => + v + .replace(//g, '') + .replace(/]*\/>/gi, '') + .replace(/]*>[\s\S]*?<\/ref>/gi, ''); +/** Wikitext to plain text: no refs, comments, markup; templates keep their unnamed arguments. */ +export function plain(v: string): string { + let s = noRefs(v) + .replace(/\{\{\s*(long|short) ton\s*\|\s*([\d.,]+)[^{}]*\}\}/gi, '$2 $1 tons') + .replace(//gi, ', ') + .replace(/<[^>]+>/g, ''); + for (let guard = 0; guard < 10 && /\{\{/.test(s); guard++) + s = s.replace(/\{\{([^{}]*)\}\}/g, (_, inner: string) => { + const [name, ...rest] = inner.split('|'); + const n = name.trim().toLowerCase(); + if (/^(cvt|convert)$/.test(n)) return rest.slice(0, 2).join(' '); + if ( + /^(efn|sfn|refn|rp|r|page needed|citation needed|cn|when|dubious|clarify|webarchive|nbsp|flagicon|flag)$|^(efn|sfn|refn|citation|webarchive|flagicon)/.test( + n + ) + ) + return ''; + if (/^as of/.test(n)) return `as of ${rest.join(' ')}`; + if (/^(ubl|unbulleted list|plainlist|plain list|flatlist|hlist|bulleted list|ublist)/.test(n)) + return rest + .filter((r) => !/^\s*[a-z_ ]+\s*=/.test(r)) + .map((r) => r.trim()) + .filter(Boolean) + .join(', '); + return rest + .filter((r) => !/^\s*[a-z_ ]+\s*=/.test(r)) + .map((r) => r.trim()) + .filter(Boolean) + .join(', '); + }); + return s + .replace(/\[\[(?:[^|\]]*\|)?([^\]]*)\]\]/g, '$1') + .replace(/\[https?:[^\s\]]+\s*([^\]]*)\]/g, '$1') + .replace(/'{2,}/g, '') + .replace(/ | /g, ' ') + .replace(/–|—/g, '–') + .replace(/\s*\n\s*\*\s*/g, ', ') + .replace(/^\s*\*\s*/, '') + .replace(/\s+/g, ' ') + .replace(/^[,\s]+|[,\s]+$/g, '') + .trim(); +} +/** Wikilink targets in a field, in order ("[[Alstom|GEC-Alsthom]]" -> "Alstom"). */ +function linkTargets(v: string | undefined): string[] { + if (!v) return []; + const out: string[] = []; + for (const m of noRefs(v).matchAll(/\[\[([^|\]#]+)(?:#[^|\]]*)?(?:\|[^\]]*)?\]\]/g)) { + const t = m[1].trim().replace(/_/g, ' '); + if (!t || /^(file|image|category|wikt|wiktionary|w|commons):/i.test(t)) continue; + const title = t.charAt(0).toUpperCase() + t.slice(1); + if (!out.includes(title)) out.push(title); + } + return out; +} +const field = (box: Record, ...keys: string[]) => { + for (const k of keys) if (box[k] && plain(box[k])) return box[k]; + return undefined; +}; + +// ---------------------------------------------------------------- number parsing + +function numberBuilt(v: string | undefined): number | null { + if (!v) return null; + const s = plain(v).replace(/(\d)[,\s](\d{3})\b/g, '$1$2'); + const parts = s.split(/,\s|;\s|\n/).filter(Boolean); + // "ICE 3 neo: 65, Velaro D: 17, ..." -> sum of the counts after each colon. + const labelled = parts + .map((p) => p.match(/:\s*(\d+)/)) + .filter((m): m is RegExpMatchArray => !!m) + .map((m) => Number(m[1])); + if (labelled.length >= 2 && labelled.length >= parts.length / 2) + return labelled.reduce((a, b) => a + b, 0); + const m = s.match(/(?= 1800 && n <= 2100 && /\b(?:in|from|since|by|until)\s+\d{4}/.test(s) ? null : n; +} +function speedKmh(v: string | undefined): number | null { + if (!v) return null; + const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2'); + let best: number | null = null; + for (const m of s.matchAll(/(\d+(?:\.\d+)?)\s*(km\/h|kmh|kph|km\/hr|mph|mi\/h)/gi)) { + const n = Number(m[1]) * (/^m/i.test(m[2]) ? 1.609 : 1); + if (n > 3 && n < 700 && (best == null || n > best)) best = n; + } + return best == null ? null : Math.round(best); +} +function powerKw(v: string | undefined): number | null { + if (!v) return null; + const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2'); + let best: number | null = null; + for (const m of s.matchAll(/(\d+(?:\.\d+)?)\s*(MW|kW|bhp|hp|horsepower|PS|ihp|shp)\b/gi)) { + const u = m[2].toLowerCase(); + const n = Number(m[1]) * (u === 'mw' ? 1000 : u === 'kw' ? 1 : u === 'ps' ? 0.7355 : 0.7457); + if (n > 1 && n < 40000 && (best == null || n > best)) best = n; + } + return best == null ? null : Math.round(best); +} +function lengthM(v: string | undefined): number | null { + if (!v) return null; + const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2'); + const hits = [...s.matchAll(/(\d+(?:\.\d+)?)\s*(mm|m|ft|in)\b/gi)].map((m) => ({ + n: Number(m[1]), + u: m[2].toLowerCase() + })); + for (let i = 0; i < hits.length; i++) { + const { n, u } = hits[i]; + if (u === 'mm') return n / 1000; + if (u === 'm') return n; + if (u === 'ft') return n * 0.3048 + (hits[i + 1]?.u === 'in' ? hits[i + 1].n * 0.0254 : 0); + } + return null; +} +function weightT(v: string | undefined): number | null { + if (!v) return null; + const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2'); + const m = s.match(/(\d+(?:\.\d+)?)\s*(long tons?|short tons?|tonnes?|tons?|t\b|kg|lb|lbs)/i); + if (!m) return null; + const n = Number(m[1]); + const u = m[2].toLowerCase(); + if (u.startsWith('long')) return n * 1.016; + if (u.startsWith('short')) return n * 0.907; + if (u === 'kg') return n / 1000; + if (u.startsWith('lb')) return (n * 0.4536) / 1000; + return n; +} +function capacity(v: string | undefined): number | null { + if (!v) return null; + const s = plain(v).replace(/(\d),(\d{3})/g, '$1$2'); + // "129 (1st), 442 (2nd)": first + second class seats of the first variant. + const cls = [...s.matchAll(/(\d+)\s*\(?\s*(?:1st|2nd|first|second|standard)/gi)].map((m) => + Number(m[1]) + ); + if (cls.length >= 2) return cls[0] + cls[1]; + let best: number | null = null; + for (const m of s.matchAll( + /(\d+)(?:\s*(?:–|-|to)\s*(\d+))?\s*(?:\(?[a-z ]{0,20}\)?\s*)?(?:seat|passenger|standing|people|pax|total)/gi + )) { + const n = Number(m[2] ?? m[1]); + if (n > 0 && n < 4000 && (best == null || n > best)) best = n; + } + if (best == null) { + const only = s.match(/^(\d+)(?:\s|$)/); + if (only && Number(only[1]) < 4000) best = Number(only[1]); + } + return best; +} + +const GAUGE_CODES: Record = { + sg: 1435, + ussg: 1435, + standard: 1435, + ib: 1668, + iberian: 1668, + rus: 1520, + russian: 1520, + ig: 1676, + indian: 1676, + irish: 1600, + cape: 1067, + mm: 1000, + metre: 1000, + meter: 1000, + '3ft': 914, + '3ft6in': 1067, + '2ft': 610, + '2ft6in': 762, + '1ft11.5in': 597, + '5ft': 1524, + '5ft3in': 1600, + '5ft6in': 1676, + brunel: 2140, + '7ft': 2134, + '4ft8.5in': 1435, + '4ft6in': 1372, + '3ft6': 1067, + '15in': 381 +}; +const GAUGE_NAME: Record = { + 1435: 'standard gauge', + 1668: 'Iberian broad gauge', + 1520: 'Russian broad gauge', + 1524: 'Russian broad gauge', + 1676: 'Indian broad gauge', + 1600: 'Irish broad gauge', + 1067: 'Cape gauge', + 1000: 'metre gauge', + 914: '3 ft narrow gauge', + 762: 'narrow gauge', + 750: 'narrow gauge', + 610: '2 ft narrow gauge', + 600: 'narrow gauge', + 597: 'narrow gauge', + 2140: 'Brunel broad gauge', + 1372: 'Scotch gauge' +}; +function gauge( + v: string | undefined, + wdLabel: string | null +): { mm: number | null; label: string | null } { + let mm: number | null = null; + if (v) { + const tpl = noRefs(v).match(/\{\{\s*(?:track ?gauge|rail ?gauge|gauge)\s*\|\s*([^|}]+)/i); + if (tpl) { + const code = tpl[1].trim().toLowerCase().replace(/\s+/g, ''); + mm = + GAUGE_CODES[code] ?? (/^\d{3,4}(mm)?$/.test(code) ? Number(code.replace('mm', '')) : null); + if (mm == null) { + const ft = code.match(/^(\d)ft(?:(\d+(?:\.\d+)?)in)?$/); + if (ft) mm = Math.round(Number(ft[1]) * 304.8 + Number(ft[2] ?? 0) * 25.4); + } + } + if (mm == null) { + const m = plain(v) + .replace(/(\d),(\d{3})/g, '$1$2') + .match(/(\d{3,4})\s*mm/); + if (m) mm = Number(m[1]); + } + } + if (mm == null && wdLabel) { + const m = wdLabel.replace(/(\d),(\d{3})/g, '$1$2').match(/(\d{3,4})\s*mm/); + if (m) mm = Number(m[1]); + else if (/standard gauge/i.test(wdLabel)) mm = 1435; + else if (/metre gauge/i.test(wdLabel)) mm = 1000; + else if (/cape gauge/i.test(wdLabel)) mm = 1067; + else if (/russian gauge|1520/i.test(wdLabel)) mm = 1520; + } + if (mm != null && (mm < 200 || mm > 3000)) mm = null; + if (mm == null) return { mm: null, label: null }; + const name = GAUGE_NAME[mm] ?? (mm < 1435 ? 'narrow gauge' : 'broad gauge'); + return { mm, label: `${mm.toLocaleString('en-GB')} mm ${name}` }; +} + +// ---------------------------------------------------------------- wikipedia + +/** "... built by [[British Rail Engineering Limited]] ..." in the first paragraphs. */ +function leadBuilders(text: string): string[] { + const start = text.search(/^'''/m); + const lead = noRefs(text.slice(start >= 0 ? start : 0, (start >= 0 ? start : 0) + 2500)); + const out: string[] = []; + for (const m of lead.matchAll( + /\b(?:built|manufactured|constructed|produced|developed and built|designed and built)\s+(?:(?:jointly|primarily|mainly|initially|originally|in \d{4}(?:\s*[–-]\s*\d{4})?|between \d{4} and \d{4}|from \d{4})\s+)*by\s+(?:the\s+)?\[\[([^\]|#]+)/gi + )) { + const t = m[1].trim().replace(/_/g, ' '); + const title = t.charAt(0).toUpperCase() + t.slice(1); + if (!out.includes(title)) out.push(title); + if (out.length >= 2) break; + } + return out; +} + +type Article = { + title: string; + template: string; + qid: string | null; + extract: string | null; + box: Record; + boxName: string; + /** Builders linked in the lead ("built by [[X]]"), for infoboxes without a builder field. */ + leadBuilders: string[]; +}; + +async function listArticles() { + const titles = new Map(); // title -> template + for (const t of TEMPLATES) { + let cont: Record = {}; + for (let guard = 0; guard < 100; guard++) { + const res = await mw<{ + query?: { embeddedin?: { title: string }[] }; + continue?: Record; + }>( + 'en.wikipedia.org', + { + action: 'query', + list: 'embeddedin', + eititle: `Template:${t}`, + einamespace: '0', + eilimit: '500', + ...cont + }, + false + ); + for (const p of res.query?.embeddedin ?? []) if (!titles.has(p.title)) titles.set(p.title, t); + if (!res.continue) break; + cont = res.continue; + } + } + log(`wikipedia: ${titles.size} articles with a locomotive/train/tram infobox`); + return titles; +} + +const INFOBOX = + /^infobox\s*(locomotive|train|tram|emu|dmu|multiple unit|rolling stock|railcar|.*(?:locomotive|train|tram|railcar|carriage|coach))\b/i; + +async function loadArticles(titles: Map) { + const out: Article[] = []; + const todo = [...titles.keys()]; + let done = 0; + for (const c of chunks(todo, 20)) { + const res = await mw('en.wikipedia.org', { + action: 'query', + prop: 'revisions|extracts|pageprops', + rvprop: 'content|ids', + rvslots: 'main', + exintro: '1', + explaintext: '1', + exsentences: '3', + exlimit: '20', + ppprop: 'wikibase_item', + titles: c.join('|') + }); + for (const p of (res.query?.pages ?? []) as unknown[]) { + const page = p as { + title: string; + missing?: boolean; + extract?: string; + pageprops?: { wikibase_item?: string }; + revisions?: { slots?: { main?: { content?: string } } }[]; + }; + if (page.missing) continue; + const text = page.revisions?.[0]?.slots?.main?.content ?? ''; + const box = findTemplate(text, INFOBOX); + out.push({ + title: page.title, + template: titles.get(page.title) ?? '', + qid: page.pageprops?.wikibase_item ?? null, + extract: page.extract?.trim() || null, + box: box ? templateParams(box.body) : {}, + boxName: box?.name ?? '', + leadBuilders: leadBuilders(text) + }); + } + done += c.length; + if (done % 1000 < 20 || done === todo.length) log(`wikipedia: ${done}/${todo.length} articles`); + } + return out; +} + +/** Resolves article titles (redirects followed) to Wikidata ids. */ +async function resolveTitles(titles: string[]) { + const out = new Map(); // asked title -> qid + for (const c of chunks([...new Set(titles)], 50)) { + const res = (await mw('en.wikipedia.org', { + action: 'query', + prop: 'pageprops', + ppprop: 'wikibase_item', + redirects: '1', + titles: c.join('|') + })) as { + query?: { + pages?: { title: string; pageprops?: { wikibase_item?: string } }[]; + normalized?: { from: string; to: string }[]; + redirects?: { from: string; to: string }[]; + }; + }; + const fwd = new Map(); + for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])]) + fwd.set(n.from, n.to); + const byTitle = new Map( + (res.query?.pages ?? []).map((p) => [p.title, p.pageprops?.wikibase_item ?? null]) + ); + for (const t of c) { + let cur = t; + for (let g = 0; g < 4 && fwd.has(cur); g++) cur = fwd.get(cur)!; + const q = byTitle.get(cur); + if (q) out.set(t, q); + } + } + return out; +} + +// ---------------------------------------------------------------- wikidata + +type Item = { + id: string; + label: string | null; + nl: string | null; + description: string | null; + aliases: string[]; + links: number; + instanceOf: string[]; // lowercase labels + classes: string[]; // P279 ids + classLabels: string[]; + makers: { id: string; label: string }[]; + operators: { id: string; label: string }[]; + origin: string[]; + country: string[]; + built: number | null; + inception: number | null; + introduced: number | null; + retired: number | null; + speed: number | null; // km/h + power: number | null; // kW + gauge: string | null; + length: number | null; + mass: number | null; // kg +}; + +const values = (ids: string[], body: string, select = '?v') => + sparql( + `SELECT ?i ${select} WHERE { VALUES ?i { ${ids.map((i) => `wd:${i}`).join(' ')} } ${body} }` + ); +// English label, or the language-neutral "mul" label many company items now carry instead. +const en = (v: string, l: string) => `${v} rdfs:label ${l} FILTER(LANG(${l}) IN ("en", "mul"))`; + +async function loadItems(ids: string[]) { + const items = new Map(); + for (const id of ids) + items.set(id, { + id, + label: null, + nl: null, + description: null, + aliases: [], + links: 0, + instanceOf: [], + classes: [], + classLabels: [], + makers: [], + operators: [], + origin: [], + country: [], + built: null, + inception: null, + introduced: null, + retired: null, + speed: null, + power: null, + gauge: null, + length: null, + mass: null + }); + let n = 0; + for (const c of chunks(ids, 300)) { + const get = (r: Record) => items.get(qid(r.i))!; + for (const r of await values( + c, + `OPTIONAL { ${en('?i', '?l')} } OPTIONAL { ?i rdfs:label ?nl FILTER(LANG(?nl) = "nl") } + OPTIONAL { ?i schema:description ?d FILTER(LANG(?d) = "en") } OPTIONAL { ?i wikibase:sitelinks ?sl }`, + '?l ?nl ?d ?sl' + )) { + const it = get(r); + if (r.l) it.label = r.l; + if (r.nl) it.nl = r.nl; + if (r.d) it.description = r.d; + it.links = Number(r.sl ?? 0); + } + for (const r of await values( + c, + `?i skos:altLabel ?v FILTER(LANG(?v) = "en" || LANG(?v) = "nl")` + )) + if (get(r).aliases.length < 16) get(r).aliases.push(r.v); + for (const r of await values(c, `?i wdt:P31 ?c . ${en('?c', '?v')}`)) + get(r).instanceOf.push(r.v.toLowerCase()); + for (const r of await values(c, `?i wdt:P279 ?c . OPTIONAL { ${en('?c', '?v')} }`, '?c ?v')) { + get(r).classes.push(qid(r.c)); + if (r.v) get(r).classLabels.push(r.v.toLowerCase()); + } + for (const r of await values(c, `?i wdt:P176 ?m . ${en('?m', '?v')}`, '?m ?v')) { + const it = get(r); + if (!it.makers.some((m) => m.id === qid(r.m))) it.makers.push({ id: qid(r.m), label: r.v }); + } + for (const r of await values(c, `?i wdt:P137 ?m . ${en('?m', '?v')}`, '?m ?v')) { + const it = get(r); + if (!it.operators.some((m) => m.id === qid(r.m))) + it.operators.push({ id: qid(r.m), label: r.v }); + } + for (const r of await values(c, `?i wdt:P495 ?c . ${en('?c', '?v')}`)) get(r).origin.push(r.v); + for (const r of await values(c, `?i wdt:P17 ?c . ${en('?c', '?v')}`)) get(r).country.push(r.v); + for (const r of await values(c, `?i wdt:P1092 ?v`)) { + const it = get(r); + const v = Math.round(Number(r.v)); + if (Number.isFinite(v) && v >= 0 && (it.built == null || v > it.built)) it.built = v; + } + const yearProp = async ( + prop: string, + key: 'inception' | 'introduced' | 'retired', + max = false + ) => { + for (const r of await values(c, `?i wdt:${prop} ?v`)) { + const it = get(r); + const y = year(r.v); + if (y && (it[key] == null || (max ? y > it[key]! : y < it[key]!))) it[key] = y; + } + }; + await yearProp('P571', 'inception'); + await yearProp('P729', 'introduced'); + await yearProp('P730', 'retired', true); + const quantity = async ( + prop: string, + key: 'speed' | 'power' | 'length' | 'mass', + f: number + ) => { + for (const r of await values(c, `?i p:${prop}/psn:${prop}/wikibase:quantityAmount ?v`)) { + const it = get(r); + const v = Number(r.v) * f; + if (Number.isFinite(v) && v > 0 && (it[key] == null || v > it[key]!)) it[key] = v; + } + }; + await quantity('P2052', 'speed', 3.6); + await quantity('P2109', 'power', 0.001); + await quantity('P2043', 'length', 1); + await quantity('P2067', 'mass', 1); + for (const r of await values(c, `?i wdt:P1064 ?g . ${en('?g', '?v')}`)) get(r).gauge ??= r.v; + n += c.length; + log(`wikidata: details ${n}/${ids.length}`); + } + return items; +} + +type Entity = { + id: string; + label: string; + country: string | null; + aliases: string[]; + human: boolean; + place: boolean; +}; +/** Labels, countries and short aliases of makers/operators. */ +async function loadEntities(ids: string[]) { + const out = new Map(); + for (const c of chunks([...new Set(ids)], 300)) { + for (const r of await values( + c, + `OPTIONAL { ${en('?i', '?l')} } OPTIONAL { ?i wdt:P17 ?c . ${en('?c', '?cl')} } + OPTIONAL { ?i wdt:P31 wd:Q5 . BIND(1 AS ?h) } + OPTIONAL { ?i wdt:P625 ?coord . BIND(1 AS ?p) }`, + '?l ?cl ?h ?p' + )) { + const id = qid(r.i); + const cur = out.get(id) ?? { + id, + label: id, + country: null, + aliases: [], + human: false, + place: false + }; + if (r.l) cur.label = r.l; + cur.country ??= r.cl ?? null; + if (r.h) cur.human = true; + if (r.p) cur.place = true; + out.set(id, cur); + } + for (const r of await values(c, `?i skos:altLabel ?v FILTER(LANG(?v) = "en")`)) { + const e = out.get(qid(r.i)); + if (e && e.aliases.length < 8) e.aliases.push(r.v); + } + } + return out; +} + +// ---------------------------------------------------------------- classification + +export const CATEGORIES = [ + 'steam', + 'diesel', + 'electric', + 'high-speed', + 'multiple unit', + 'metro', + 'tram', + 'carriage', + 'other' +] as const; + +function tractionOf(text: string, hasElectricSystem: boolean): string | null { + const s = text.toLowerCase(); + if (/hydrogen|fuel[- ]cell/.test(s)) return 'hydrogen'; + if (/battery[- ]electric|battery[- ]powered|accumulator|\bbemu\b|akku/.test(s)) return 'battery'; + if (/\bsteam\b|fireless|geared steam/.test(s)) return 'steam'; + if ( + /electro-diesel|electro diesel|bi-mode|bimode|dual-mode|dual mode|tri-mode|diesel[- ]hybrid|hybrid (?:diesel|locomotive|train|multiple unit|railcar|drive|traction)/.test( + s + ) + ) + return 'bi-mode'; + if (/gas turbine|turbotrain|turbine/.test(s)) return 'gas turbine'; + if ( + /diesel|petrol|gasoline|internal combustion|paraffin|benzene|\bdmu\b|\bdemu\b|railbus/.test(s) + ) + return 'diesel'; + if ( + /electric|\bkv\b|overhead|third rail|fourth rail|catenary|pantograph|\bemu\b|trolley|volt|\bdc\b|\bac\b/.test( + s + ) || + hasElectricSystem + ) + return 'electric'; + if (/horse|cable car|cable-hauled|funicular/.test(s)) return 'none'; + return null; +} + +function categoryOf(o: { + template: string; + text: string; // stocktype, powertype, P31/P279 labels, description, title + traction: string | null; + speed: number | null; +}): string { + const s = o.text.toLowerCase(); + const locoTemplate = /locomotive/i.test(o.template); + if ( + /tram-train|\btram|streetcar|street car|light rail|trolley(?!bus)|stadtbahn|\blrv\b/.test(s) || + /tram/i.test(o.template) + ) + return 'tram'; + if ( + /\bmetro\b|subway|underground|rapid transit|\bmrt\b|monorail|people mover|automated guideway|rubber-tyred|light metro|\bu-bahn/.test( + s + ) + ) + return 'metro'; + if ((o.speed ?? 0) >= 250) return 'high-speed'; + if ( + /high-speed|high speed|shinkansen|bullet train|maglev|\btgv\b|velaro|frecciarossa|\bcrh\d/.test( + s + ) && + (o.speed == null || o.speed >= 190) + ) + return 'high-speed'; + const mu = + /multiple unit|multiple-unit|\bemu\b|\bdmu\b|\bdemu\b|\bbemu\b|railcar|railbus|rail motor|motor coach|trainset|train set|power car|sprinter|\bunit\b/.test( + s + ); + const loco = /locomotive|shunter|switcher/.test(s); + const locoKind = () => + o.traction === 'steam' + ? 'steam' + : o.traction === 'electric' || o.traction === 'battery' || o.traction === 'bi-mode' + ? 'electric' + : o.traction + ? 'diesel' + : null; + if (locoTemplate && !mu) return locoKind() ?? (loco ? 'other' : 'other'); + if (mu) return 'multiple unit'; + if ( + /coach|carriage|passenger car|sleeping car|dining car|restaurant car|buffet car|\bvans?\b|wagon|freight car|tank car|hopper|boxcar|caboose|observation car|parlor|parlour|sleeper|brake van|goods|rolling stock/.test( + s + ) + ) + return 'carriage'; + if (loco) return locoKind() ?? 'other'; + if (o.traction && o.traction !== 'none' && o.traction !== 'steam') return 'multiple unit'; + return 'other'; +} + +const INDIVIDUAL = + /^(?:(?:steam|diesel|electric|diesel-electric|diesel-hydraulic|tank|tender|preserved|narrow-gauge|rack|shunting)\s+)*(?:locomotive|tram|railcar|train|multiple unit|railway vehicle|rail vehicle|individual locomotive|preserved locomotive|steam engine|rolling stock|electric multiple unit|diesel multiple unit|passenger car|railroad car)$/; +const CLASSLIKE = /class|type|model|series|family|product|design|platform/; +/** Named services (Palace on Wheels, the Capricornian): a timetable, not a kind of train. */ +const SERVICE = + /luxury train|named (?:passenger )?train|train service|rail service|passenger train service|tourist train|excursion train|train route/; + +// ---------------------------------------------------------------- build + +const CORP = new Set( + 'COMPANY CORPORATION CORP INC INCORPORATED LTD LIMITED GMBH AG SA SAS SPA PLC CO LLC KG NV BV AB AS OY OYJ SRO KK JSC PJSC OAO ZAO GROUP THE OF AND MANUFACTURING WORKS WORKSHOP WORKSHOPS LOCOMOTIVE LOCOMOTIVES TRANSPORTATION TRANSPORT MOBILITY RAIL ROLLING STOCK CAR CARS DIVISION INDUSTRIES INDUSTRY ENGINEERING HOLDINGS INTERNATIONAL DIESEL'.split( + ' ' + ) +); +const GENERIC = new Set( + 'BRITISH AMERICAN GENERAL NATIONAL UNITED STATE FEDERAL CENTRAL NORTHERN SOUTHERN EASTERN WESTERN NORTH SOUTH EAST WEST GREAT NEW ROYAL IMPERIAL'.split( + ' ' + ) +); +const entityKey = (label: string) => { + const t = tokens(label.replace(/\(.*?\)/g, '')); + const k = t.filter((x) => !CORP.has(x)); + return (k.length && !(k.length === 1 && GENERIC.has(k[0])) ? k : t).join(' '); +}; +const COUNTRY_ALIAS: Record = { + 'United States of America': 'United States', + "People's Republic of China": 'China', + 'Kingdom of the Netherlands': 'Netherlands', + 'West Germany': 'Germany', + 'German Reich': 'Germany', + 'German Empire': 'Germany', + 'Nazi Germany': 'Germany', + 'Weimar Republic': 'Germany', + 'Russian Empire': 'Russia', + 'Kingdom of Italy': 'Italy', + 'Empire of Japan': 'Japan', + 'United Kingdom of Great Britain and Ireland': 'United Kingdom', + 'Kingdom of Great Britain': 'United Kingdom', + 'British Raj': 'India', + 'Kingdom of Prussia': 'Germany', + 'Kingdom of Bavaria': 'Germany', + 'Kingdom of Saxony': 'Germany', + 'Kingdom of Württemberg': 'Germany', + 'French Third Republic': 'France', + 'Second French Empire': 'France', + 'Kingdom of Spain': 'Spain', + 'Kingdom of Belgium': 'Belgium', + 'Kingdom of Hungary': 'Hungary', + 'Kingdom of Romania': 'Romania', + 'Kingdom of Yugoslavia': 'Yugoslavia', + 'Socialist Federal Republic of Yugoslavia': 'Yugoslavia', + 'Republic of China (1912–1949)': 'China', + 'Union of South Africa': 'South Africa', + 'Dominion of Canada': 'Canada', + 'Dominion of New Zealand': 'New Zealand', + 'Colony of Victoria': 'Australia', + 'Colony of New South Wales': 'Australia', + 'Colony of Queensland': 'Australia', + Taiwan: 'Taiwan', + 'Republic of China': 'Taiwan', + Korea: 'South Korea' +}; +const country = (c: string | null | undefined) => (c ? (COUNTRY_ALIAS[c] ?? c) : null); +const round = (n: number | null, d = 0) => (n == null ? null : Math.round(n * 10 ** d) / 10 ** d); +const sane = (n: number | null, min: number, max: number) => + n != null && n >= min && n <= max ? n : null; +const cleanTitle = (t: string) => + t + .replace( + /\s*\((?:locomotive|steam locomotive|diesel locomotive|electric locomotive|train|trainset|tram|tramcar|railcar|multiple unit|electric multiple unit|diesel multiple unit|rolling stock|railway|railroad|rail vehicle|\d{4}|[^)]*locomotive[^)]*|[^)]*multiple unit[^)]*)\)$/i, + '' + ) + .trim(); + +export type BuiltType = { + wikidata: string; + slug: string; + name: string; + fullName: string; + nlName: string | null; + search: string; + wikipedia: string; + description: string | null; + summary: string | null; + category: string; + categoryLabel: string | null; + traction: string | null; + country: string | null; + countries: string[]; + makers: string[]; + operators: { key: string; name: string }[]; + firstBuilt: number | null; + lastBuilt: number | null; + introducedYear: number | null; + retiredYear: number | null; + status: string | null; + inService: boolean | null; + numberBuilt: number | null; + maxSpeedKmh: number | null; + powerKw: number | null; + gauge: string | null; + gaugeMm: number | null; + electricSystem: string | null; + wheels: string | null; + lengthM: number | null; + weightT: number | null; + capacity: number | null; + formation: string | null; + designer: string | null; + nicknames: string | null; + parent: string | null; + sources: string[]; + links: number; +}; +export type BuiltMaker = { + slug: string; + name: string; + fullName: string; + wikidata: string | null; + country: string | null; + types: BuiltType[]; +}; +export type BuiltOperator = { + key: string; + slug: string; + name: string; + fullName: string; + wikidata: string | null; + country: string | null; + count: number; +}; + +const short = (s: string | null | undefined, n: number) => { + if (!s) return null; + const t = s.trim(); + return t ? (t.length > n ? `${t.slice(0, n - 1).trimEnd()}…` : t) : null; +}; + +async function build( + articles: Article[], + items: Map, + linkQ: Map, + entities: Map +) { + const skipped: Record = {}; + const skippedTitles: Record = {}; + let current = ''; + const skip = (why: string, n = 1) => { + skipped[why] = (skipped[why] ?? 0) + n; + if (n === 1 && current) (skippedTitles[why] ??= []).push(current); + }; + type Draft = BuiltType & { makerKey: string; makerLabel: string; makerId: string | null }; + const drafts: Draft[] = []; + const operatorInfo = new Map< + string, + { name: string; wikidata: string | null; country: string | null } + >(); + const isCompany = (e: Entity | undefined) => !!e && !e.human; + + for (const a of articles) { + current = a.title; + if (!a.qid) { + skip('no wikidata item'); + continue; + } + if ( + /^lists? of|rolling stock$|rolling stock of|locomotives of|\bfleet\b|^history of|^\d{4} in rail/i.test( + a.title + ) + ) { + skip('list article'); + continue; + } + if (!Object.keys(a.box).length) { + skip('no infobox parsed'); + continue; + } + const it = items.get(a.qid); + if (!it) { + skip('no wikidata details'); + continue; + } + const box = a.box; + const builtBoxEarly = numberBuilt( + field(box, 'totalproduction', 'numberbuilt', 'numinclass', 'quantity') + ); + // Wikidata often files a class under "electric multiple unit" too, so a count above one wins. + const individual = + it.instanceOf.some((l) => INDIVIDUAL.test(l)) && + !it.instanceOf.some((l) => CLASSLIKE.test(l)) && + !((it.built ?? builtBoxEarly ?? 0) > 1); + // "Santa Fe 2926", "GWR 6800 Class 6880 Betton Grange": one preserved engine, whose + // infobox "numinclass" counts its class, not itself. + const countBuilt = it.built ?? numberBuilt(field(box, 'totalproduction', 'numberbuilt')); + const title = cleanTitle(a.title); + const oneEngine = + /\bclass\s+\S+\s+(?:No\.\s*)?\d{2,5}[A-Z]?\s+[A-Z][a-z]/.test(title) || + (!/class|series|type|shinkansen|stock|railcar|unit|model|tram|car\b/i.test(title) && + /^\D+?\s(?:No\.\s*)?\d{1,5}(?:-[A-Z])?(?:\s+[A-Z][\w'.]*)*$/.test(title) && + !!box.serialnumber && + !((countBuilt ?? 0) > 1)); + const builtBox = numberBuilt( + field(box, 'totalproduction', 'numberbuilt', 'numinclass', 'quantity') + ); + const built = it.built ?? builtBox; + if (individual || oneEngine || (built === 1 && !it.instanceOf.some((l) => CLASSLIKE.test(l)))) { + skip('individual vehicle'); + continue; + } + const desc = it.description ?? ''; + if ( + built === 0 || + (/\b(proposed|cancelled|canceled|planned|concept|unbuilt)\b/i.test(desc) && !built) + ) { + skip('never built'); + continue; + } + + // Makers: Wikidata manufacturer, else the linked builder in the infobox, else its plain text. + const builderField = field( + box, + 'builder', + 'manufacturer', + 'builtby', + 'manufacturers', + 'builders' + ); + const builderLinks = (builderField ? linkTargets(builderField) : a.leadBuilders) + .map((t) => linkQ.get(t)) + .filter((q): q is string => !!q && isCompany(entities.get(q)) && !entities.get(q)!.place); + const makerIds = [...new Set([...builderLinks, ...it.makers.map((m) => m.id)])]; + const builderPlain = builderField ? plain(builderField) : ''; + const builderText = builderPlain.toLowerCase(); + // Primary maker: the first linked builder, or the Wikidata maker named first in the infobox. + const makerId = + builderLinks[0] ?? + [...it.makers].sort( + (x, y) => + (builderText.indexOf(x.label.toLowerCase()) + 1 || 999) - + (builderText.indexOf(y.label.toLowerCase()) + 1 || 999) + )[0]?.id ?? + null; + let makerLabel = makerId + ? (entities.get(makerId)?.label ?? it.makers.find((m) => m.id === makerId)?.label ?? makerId) + : ''; + if (!makerId) { + const first = builderPlain + .split(/,|;|\band\b|\(|\//)[0] + .replace(/^(?:built by|by)\s+/i, '') + .trim(); + if (first && first.length >= 2 && first.length <= 60 && !/^\d/.test(first)) + makerLabel = first; + } + if (!makerLabel) { + makerLabel = 'Unknown builder'; + skip('maker unknown (kept)'); + } + const makerNames = [ + ...new Set([ + makerLabel, + ...makerIds + .map((q) => entities.get(q)?.label ?? it.makers.find((m) => m.id === q)?.label ?? '') + .filter(Boolean) + ]) + ].slice(0, 8); + + // Operators: Wikidata operators + linked operators in the infobox. + const opField = field(box, 'operator', 'operators', 'currentowner', 'owner', 'owners'); + const opLinks = linkTargets(opField) + .map((t) => linkQ.get(t)) + .filter((q): q is string => !!q && isCompany(entities.get(q)) && !entities.get(q)!.place); + const opIds = [...new Set([...opLinks, ...it.operators.map((o) => o.id)])].slice(0, 14); + const operators: { key: string; name: string }[] = []; + for (const q of opIds) { + const e = entities.get(q); + const name = e?.label ?? it.operators.find((o) => o.id === q)?.label; + if (!name || /^Q\d+$/.test(name)) continue; + operators.push({ key: q, name }); + if (!operatorInfo.has(q)) + operatorInfo.set(q, { name, wikidata: q, country: country(e?.country) }); + } + if (!operators.length && opField && !linkTargets(opField).length) { + for (const name of plain(opField) + .split(/,|;|\n/) + .map((x) => x.replace(/\(.*?\)/g, '').trim()) + .filter((x) => x.length >= 2 && x.length <= 60 && !/^\d/.test(x)) + .slice(0, 6)) { + const key = `name:${slugify(name)}`; + operators.push({ key, name }); + if (!operatorInfo.has(key)) operatorInfo.set(key, { name, wikidata: null, country: null }); + } + } + + const stock = plain(field(box, 'stocktype', 'type', 'powertype', 'power') ?? ''); + const elec = field( + box, + 'electricsystem', + 'electricsystems', + 'electricsystem(s)', + 'currentcollection', + 'collectionmethod' + ); + const tractionText = [ + plain( + field( + box, + 'powertype', + 'power', + 'traction', + 'fueltype', + 'primemover', + 'stocktype', + 'type' + ) ?? '' + ), + it.instanceOf.join(' '), + it.classLabels.join(' '), + desc, + a.title + ].join(' | '); + // Steam-era infoboxes rarely say "steam": boiler, firebox, cylinder or tender fields give it away, + // as do German steam class codes like "1'C h2" or "C n2t". + const steamBox = + Object.keys(box).some((k) => + /^(boiler|firebox|firegrate|tubearea|superheater|cylinder|tender|tubesandflues|watercap|valvegear)/.test( + k + ) + ) || + /(?:^|[\s'′])[nh][2-4](?:v|t|vt)?\b/.test( + plain(field(box, 'uicclass', 'wheelarrangement', 'whytetype') ?? '') + ); + const traction = + tractionOf(tractionText, !!(elec && plain(elec))) ?? (steamBox ? 'steam' : null); + const speedBox = speedKmh( + field(box, 'maxspeed', 'topspeed', 'speed', 'operatingspeed', 'designspeed') + ); + const speed = round(sane(speedBox ?? it.speed, 5, 650)); + const catText = [ + stock, + it.instanceOf.join(' '), + it.classLabels.join(' '), + desc, + cleanTitle(a.title) + ].join(' | '); + const category = categoryOf({ + template: `${a.template} ${a.boxName}`, + text: catText, + traction, + speed + }); + if ( + it.instanceOf.some((l) => SERVICE.test(l)) || + (category === 'other' && makerLabel === 'Unknown builder') + ) { + skip('train service, not rolling stock'); + continue; + } + + // Years. + const buildYears = years( + plain( + field( + box, + 'builddate', + 'yearconstructed', + 'constructed', + 'yearsbuilt', + 'built', + 'buildyear', + 'builddates' + ) ?? '' + ) + ); + const serviceText = plain( + field( + box, + 'service', + 'inservice', + 'yearservice', + 'enteredservice', + 'introduced', + 'firstrundate', + 'deliverydate' + ) ?? '' + ); + const serviceYears = years(serviceText); + const retiredText = plain( + field( + box, + 'retiredate', + 'retired', + 'withdrawndate', + 'withdrawn', + 'yearscrapped', + 'scrapped', + 'scrapdate', + 'lastrundate' + ) ?? '' + ); + const firstBuilt = buildYears.length + ? Math.min(...buildYears) + : (it.inception ?? serviceYears[0] ?? it.introduced ?? null); + const lastBuilt = + buildYears.length > 1 ? Math.max(...buildYears) : buildYears.length ? buildYears[0] : null; + const introducedYear = + it.introduced ?? (serviceYears.length ? Math.min(...serviceYears) : null); + const present = /present|current|in service|today/i.test(serviceText); + let retiredYear: number | null = + it.retired ?? (years(retiredText).length ? Math.max(...years(retiredText)) : null); + if (!retiredYear && !present && serviceYears.length >= 2) + retiredYear = Math.max(...serviceYears); + if (present) retiredYear = null; + const disposition = plain( + field(box, 'disposition', 'status', 'preserved', 'numberpreserved') ?? '' + ).toLowerCase(); + let status: string | null = null; + if (present || /in service|in use|operational|active/.test(disposition)) status = 'in service'; + else if (retiredYear || /withdrawn|scrapped|retired/.test(disposition)) + status = /preserved|museum|heritage/.test(disposition) ? 'preserved' : 'retired'; + else if (/preserved|museum|heritage/.test(disposition)) status = 'preserved'; + const inService = status === 'in service' ? true : status ? false : null; + + const g = gauge(field(box, 'gauge', 'trackgauge'), it.gauge); + const origin = country(it.origin[0]) ?? country(it.country[0]); + const makerCountry = makerId ? country(entities.get(makerId)?.country) : null; + const opCountries = operators + .map((o) => operatorInfo.get(o.key)?.country) + .filter((c): c is string => !!c); + const mainCountry = origin ?? opCountries[0] ?? makerCountry ?? null; + const countries = [ + ...new Set([mainCountry, ...opCountries].filter((c): c is string => !!c)) + ].slice(0, 12); + + const mk = entityKey(makerLabel); + const mt = tokens(makerLabel.replace(/\(.*?\)/g, '')); + const tt = tokens(title); + // "Siemens Velaro" -> "Velaro" (maker words dropped only when the title starts with them). + const pre = mt.filter((x) => !CORP.has(x)); + let name = title; + if (pre.length && tt.length > pre.length && pre.every((x, i) => tt[i] === x)) + name = title.split(/\s+/).slice(pre.length).join(' ') || title; + const wheels = plain( + field(box, 'whytetype', 'uicclass', 'aarwheels', 'wheelarrangement', 'axlearrangement') ?? '' + ) + .replace(/\s+/g, ' ') + .split(/,\s/)[0]; + const lengthVal = lengthM( + field(box, 'trainlength', 'length', 'overalllength', 'carlength', 'lengthovercouplers') + ); + const weightVal = weightT( + field( + box, + 'locoweight', + 'trainweight', + 'weight', + 'emptyweight', + 'carweight', + 'locotenderweight' + ) + ); + const t: Draft = { + wikidata: a.qid, + slug: slugify(name), + name, + fullName: title, + nlName: + it.nl && cmp(it.nl) !== cmp(title) && cmp(it.nl) !== cmp(it.label ?? '') ? it.nl : null, + search: '', + wikipedia: a.title, + description: short(it.description, 300), + summary: a.extract && a.extract.length > 40 ? a.extract.slice(0, 900) : null, + category, + categoryLabel: short(stock.replace(/^an? /i, ''), 70), + traction, + country: mainCountry, + countries, + makers: makerNames, + operators, + firstBuilt: sane(firstBuilt, 1800, thisYear + 10), + lastBuilt: sane(lastBuilt, 1800, thisYear + 10), + introducedYear: sane(introducedYear, 1800, thisYear + 10), + retiredYear: sane(retiredYear, 1800, thisYear + 1), + status, + inService, + numberBuilt: sane(built, 1, 200_000), + maxSpeedKmh: speed, + powerKw: round( + sane( + powerKw( + field(box, 'poweroutput', 'power', 'powerout', 'enginepower', 'output', 'tractionpower') + ) ?? it.power, + 1, + 30_000 + ) + ), + gauge: g.label, + gaugeMm: g.mm, + electricSystem: short( + elec + ? plain(field(box, 'electricsystem', 'electricsystems', 'electricsystem(s)') ?? '') + : null, + 140 + ), + wheels: wheels && wheels.length <= 40 ? wheels : null, + lengthM: round(sane(lengthVal ?? it.length, 2, 500), 2), + weightT: round(sane(weightVal ?? (it.mass ? it.mass / 1000 : null), 0.3, 3000), 1), + capacity: sane(capacity(field(box, 'capacity', 'seating', 'seats', 'passengers')), 1, 4000), + formation: short(plain(field(box, 'formation', 'carspertrainset', 'cars') ?? ''), 120), + designer: short(plain(field(box, 'designer', 'designers') ?? ''), 100), + nicknames: short(plain(field(box, 'nicknames', 'nickname') ?? ''), 120), + parent: null, + sources: [SRC_WP, SRC_WD], + links: it.links, + makerKey: mk, + makerLabel, + makerId + }; + drafts.push(t); + } + + // One entry per Wikidata item (several articles can share one via redirects). + const byQ = new Map(); + for (const d of drafts) { + const cur = byQ.get(d.wikidata); + if (!cur || d.links > cur.links) byQ.set(d.wikidata, d); + } + if (byQ.size < drafts.length) skip('same item as another article', drafts.length - byQ.size); + drafts.splice(0, drafts.length, ...byQ.values()); + const kept = new Set(drafts.map((d) => d.wikidata)); + for (const d of drafts) { + const p = items.get(d.wikidata)!.classes.find((c) => kept.has(c) && c !== d.wikidata); + d.parent = p ?? null; + } + + // Makers: grouped by normalised name; the most used label (and its item) names the group. + const groups = new Map(); + for (const d of drafts) groups.set(d.makerKey, [...(groups.get(d.makerKey) ?? []), d]); + const makers = new Map(); + for (const list of [...groups.values()].sort((a, b) => b.length - a.length)) { + const labels = new Map(); + for (const d of list) { + const cur = labels.get(d.makerLabel) ?? { n: 0, id: d.makerId }; + cur.n++; + cur.id ??= d.makerId; + labels.set(d.makerLabel, cur); + } + const [label, best] = [...labels].sort( + (a, b) => Number(!!b[1].id) - Number(!!a[1].id) || b[1].n - a[1].n + )[0]; + const display = label + .replace(/\s*\((?:company|manufacturer|firm|railway|locomotive manufacturer)\)$/i, '') + .trim(); + let slug = slugify(display); + if (['makers', 'operators', 'countries', 'random'].includes(slug)) slug = `${slug}-maker`; + const existing = makers.get(slug); + if (existing) existing.types.push(...list); + else + makers.set(slug, { + slug, + name: display, + fullName: label, + wikidata: best.id, + country: + label === 'Unknown builder' + ? null + : ((best.id ? country(entities.get(best.id)?.country) : null) ?? + [ + ...list.reduce( + (m, t) => (t.country ? m.set(t.country, (m.get(t.country) ?? 0) + 1) : m), + new Map() + ) + ].sort((a, b) => b[1] - a[1])[0]?.[0] ?? + null), + types: list + }); + } + // Operators: by Wikidata item (or name), slug from the label. + const operatorCount = new Map(); + for (const d of drafts) + for (const o of d.operators) operatorCount.set(o.key, (operatorCount.get(o.key) ?? 0) + 1); + const operators = new Map(); // key -> operator + const slugTaken = new Map(); // slug -> key + for (const [key, count] of [...operatorCount].sort((a, b) => b[1] - a[1])) { + const info = operatorInfo.get(key)!; + const display = info.name + .replace(/\s*\((?:company|railway|operator|train operating company)\)$/i, '') + .trim(); + let slug = slugify(display); + if (slugTaken.has(slug)) { + // Same name, different item: merge plain-name operators into the item, else disambiguate. + const other = slugTaken.get(slug)!; + if (key.startsWith('name:') || other.startsWith('name:')) { + operators.set(key, { ...operators.get(other)! }); + operators.get(other)!.count += count; + continue; + } + slug = `${slug}-${key.toLowerCase()}`; + } + slugTaken.set(slug, key); + operators.set(key, { + key, + slug, + name: display, + fullName: info.name, + wikidata: info.wikidata, + country: info.country, + count + }); + } + // Unique type slugs per maker; the best-known type keeps the plain slug. + for (const m of makers.values()) { + const seen = new Set(); + m.types.sort((a, b) => b.links - a.links); + for (const t of m.types) { + let s = t.slug; + if (seen.has(s)) s = `${s}-${t.wikidata.toLowerCase()}`; + seen.add(s); + t.slug = s; + const it = items.get(t.wikidata)!; + t.search = [ + m.name, + t.fullName, + t.name, + t.nlName ?? '', + it.label ?? '', + ...it.aliases.slice(0, 12), + ...t.makers, + ...t.operators.map((o) => o.name), + ...t.operators.map((o) => operators.get(o.key)?.name ?? ''), + cmp(t.fullName).toLowerCase(), + cmp(t.name).toLowerCase(), + t.categoryLabel ?? '', + t.wheels ?? '', + t.nicknames ?? '' + ] + .join(' ') + .toLowerCase() + .slice(0, 2500); + } + } + log(`skipped: ${JSON.stringify(skipped)}`); + if (outFile && outFile !== '-') + await writeFile(`${outFile}.skipped.json`, JSON.stringify(skippedTitles, null, 1)); + return { makers: [...makers.values()], operators }; +} + +// ---------------------------------------------------------------- write + +const added = { + makers: [] as { name: string; url: string }[], + types: [] as { + dex: number; + name: string; + maker: string; + category: string; + firstBuilt: number | null; + numberBuilt: number | null; + url: string; + photo?: string | null; + }[] +}; + +async function write(makers: BuiltMaker[], operators: Map) { + const url = process.env.DB_URL; + if (!url) throw new Error('Set DB_URL (or use --dry-run)'); + const sql = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} }); + try { + await sql.begin(async (tx) => { + const oldTypes = new Map( + ( + await tx< + { id: number; wikidata: string; dex: number }[] + >`select id, wikidata, dex from trains.types` + ).map((r) => [r.wikidata, r]) + ); + const oldMakers = new Set( + (await tx<{ slug: string }[]>`select slug from trains.makers`).map((r) => r.slug) + ); + const firstRun = oldTypes.size === 0; + if (renumber) await tx`update trains.types set dex = -id`; + const all = makers.flatMap((m) => m.types.map((t) => ({ m, t }))); + let nextDex = renumber ? 1 : Math.max(0, ...[...oldTypes.values()].map((r) => r.dex)) + 1; + const order = all.toSorted( + (a, b) => + (a.t.firstBuilt ?? 9999) - (b.t.firstBuilt ?? 9999) || + a.t.fullName.localeCompare(b.t.fullName) + ); + const dex = new Map(); + for (const { t } of order) { + const old = oldTypes.get(t.wikidata); + dex.set(t.wikidata, !renumber && old ? old.dex : nextDex++); + } + const makerIds = new Map(); + for (const m of makers) { + const ys = m.types.map((t) => t.firstBuilt).filter((y): y is number => y != null); + const built = m.types.reduce((a, t) => a + (t.parent ? 0 : (t.numberBuilt ?? 0)), 0); + const row = { + slug: m.slug, + name: m.name, + full_name: m.fullName, + country: m.country, + wikidata: m.wikidata, + type_count: m.types.length, + first_year: ys.length ? Math.min(...ys) : null, + last_year: ys.length ? Math.max(...ys) : null, + built_total: Math.min(built, 2_000_000_000), + sources: [SRC_WP, SRC_WD] + }; + const [r] = await tx<{ id: number }[]>` + insert into trains.makers ${tx(row)} + on conflict (slug) do update set name = excluded.name, full_name = excluded.full_name, + country = excluded.country, wikidata = excluded.wikidata, type_count = excluded.type_count, + first_year = excluded.first_year, last_year = excluded.last_year, + built_total = excluded.built_total, sources = excluded.sources + returning id`; + makerIds.set(m.slug, r.id); + if (!firstRun && !oldMakers.has(m.slug)) + added.makers.push({ name: m.name, url: `${SITE}/${m.slug}` }); + } + const opIds = new Map(); // slug -> id + for (const o of new Map([...operators.values()].map((o) => [o.slug, o])).values()) { + const row = { + slug: o.slug, + name: o.name, + full_name: o.fullName, + country: o.country, + wikidata: o.wikidata, + type_count: o.count + }; + const [r] = await tx<{ id: number }[]>` + insert into trains.operators ${tx(row)} + on conflict (slug) do update set name = excluded.name, full_name = excluded.full_name, + country = excluded.country, wikidata = excluded.wikidata, type_count = excluded.type_count + returning id`; + opIds.set(o.slug, r.id); + } + await tx`update trains.types set slug = 'tmp-' || id`; + const ids = new Map(); + for (const { m, t } of all) { + const row = { + dex: dex.get(t.wikidata)!, + maker_id: makerIds.get(m.slug)!, + slug: t.slug, + name: t.name, + full_name: t.fullName, + nl_name: t.nlName, + search: t.search, + wikidata: t.wikidata, + wikipedia: t.wikipedia, + description: t.description, + summary: t.summary, + category: t.category, + category_label: t.categoryLabel, + traction: t.traction, + country: t.country, + countries: t.countries, + makers: t.makers, + operators: [...new Set(t.operators.map((o) => operators.get(o.key)?.name ?? o.name))], + first_built: t.firstBuilt, + last_built: t.lastBuilt, + introduced_year: t.introducedYear, + retired_year: t.retiredYear, + status: t.status, + in_service: t.inService, + number_built: t.numberBuilt, + max_speed_kmh: t.maxSpeedKmh, + power_kw: t.powerKw, + gauge: t.gauge, + gauge_mm: t.gaugeMm, + electric_system: t.electricSystem, + wheels: t.wheels, + length_m: t.lengthM, + weight_t: t.weightT, + capacity: t.capacity, + formation: t.formation, + designer: t.designer, + nicknames: t.nicknames, + sources: t.sources + }; + const [r] = await tx<{ id: number }[]>` + insert into trains.types ${tx(row)} + on conflict (wikidata) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'wikidata') as (keyof typeof row)[])} + returning id`; + ids.set(t.wikidata, r.id); + if (!firstRun && !oldTypes.has(t.wikidata)) + added.types.push({ + dex: row.dex, + name: t.fullName, + maker: m.name, + category: t.category, + firstBuilt: t.firstBuilt, + numberBuilt: t.numberBuilt, + url: `${SITE}/${m.slug}/${t.slug}` + }); + } + for (const { t } of all) + if (t.parent && ids.has(t.parent)) + await tx`update trains.types set family_id = ${ids.get(t.parent)!} where id = ${ids.get(t.wikidata)!}`; + else + await tx`update trains.types set family_id = null where id = ${ids.get(t.wikidata)!} and family_id is not null`; + const keep = [...ids.values()]; + await tx`delete from trains.type_operators where type_id = any(${keep})`; + const links: { type_id: number; operator_id: number; position: number }[] = []; + for (const { t } of all) { + const seen = new Set(); + t.operators.forEach((o) => { + const op = operators.get(o.key); + const id = op ? opIds.get(op.slug) : undefined; + if (id && !seen.has(id)) { + seen.add(id); + links.push({ type_id: ids.get(t.wikidata)!, operator_id: id, position: seen.size - 1 }); + } + }); + } + for (const c of chunks(links, 2000)) await tx`insert into trains.type_operators ${tx(c)}`; + await tx`update trains.operators o set type_count = (select count(*) from trains.type_operators x where x.operator_id = o.id)`; + const gone = await tx`delete from trains.types where not (id = any(${keep})) returning id`; + const goneMakers = + await tx`delete from trains.makers where not (slug = any(${makers.map((m) => m.slug)})) returning id`; + const goneOps = + await tx`delete from trains.operators where not (id = any(${[...opIds.values()]})) returning id`; + log( + `written: ${keep.length} types, ${makers.length} makers, ${opIds.size} operators, ${links.length} links; removed ${gone.length} types, ${goneMakers.length} makers, ${goneOps.length} operators` + ); + }); + } finally { + await sql.end(); + } +} + +// ---------------------------------------------------------------- main + +await useCache(cacheDir); +const titles = await listArticles(); +const articles = await loadArticles(titles); +const qids = [...new Set(articles.map((a) => a.qid).filter((q): q is string => !!q))]; +const items = await loadItems(qids); +const linkTitles = articles.flatMap((a) => [ + ...linkTargets( + field(a.box, 'builder', 'manufacturer', 'builtby', 'manufacturers', 'builders') + ).slice(0, 6), + ...a.leadBuilders, + ...linkTargets(field(a.box, 'operator', 'operators', 'currentowner', 'owner', 'owners')).slice( + 0, + 14 + ) +]); +log(`wikipedia: resolving ${new Set(linkTitles).size} linked builders/operators`); +const linkQ = await resolveTitles(linkTitles); +const entityIds = [ + ...new Set([ + ...linkQ.values(), + ...[...items.values()].flatMap((i) => [...i.makers, ...i.operators].map((m) => m.id)) + ]) +]; +const entities = await loadEntities(entityIds); +const { makers, operators } = await build(articles, items, linkQ, entities); +const types = makers.flatMap((m) => m.types); +const byCat = types.reduce( + (m, t) => m.set(t.category, (m.get(t.category) ?? 0) + 1), + new Map() +); +log( + `built ${makers.length} makers, ${new Set([...operators.values()].map((o) => o.slug)).size} operators, ${types.length} types (${JSON.stringify(Object.fromEntries(byCat))}; ${types.filter((t) => t.numberBuilt).length} built, ${types.filter((t) => t.maxSpeedKmh).length} speed, ${types.filter((t) => t.powerKw).length} power, ${types.filter((t) => t.gaugeMm).length} gauge, ${types.filter((t) => t.operators.length).length} operators, ${types.filter((t) => t.firstBuilt).length} year)` +); +if (outFile) { + const json = JSON.stringify(makers); + if (outFile === '-') console.log(json); + else await writeFile(outFile, json); +} +if (dryRun) { + for (const t of types + .toSorted((a, b) => (b.numberBuilt ?? 0) - (a.numberBuilt ?? 0)) + .slice(0, 15)) + log( + ` ${t.fullName} [${t.makers[0]}] built=${t.numberBuilt} ${t.category}/${t.traction} ${t.firstBuilt} ${t.maxSpeedKmh}km/h ${t.powerKw}kW ${t.gauge}` + ); +} else { + await write(makers, operators); + if (reportNew) + console.log( + JSON.stringify( + { + generatedAt: new Date().toISOString(), + totals: { makers: makers.length, types: types.length }, + newMakers: added.makers, + newTypes: added.types.toSorted((a, b) => (b.numberBuilt ?? 0) - (a.numberBuilt ?? 0)) + }, + null, + 2 + ) + ); +} diff --git a/scripts/trains-photos.ts b/scripts/trains-photos.ts new file mode 100644 index 0000000..7d214c5 --- /dev/null +++ b/scripts/trains-photos.ts @@ -0,0 +1,759 @@ +// Finds freely licensed photos (Wikimedia Commons) for the train pokedex (/trackers/trains) +// and logos for its makers and operators. +// +// DB_URL=... bun scripts/trains-photos.ts every type, replace all photos + logos +// DB_URL=... bun scripts/trains-photos.ts --missing only types without photos (or never checked), +// makers/operators without a logo +// DB_URL=... bun scripts/trains-photos.ts --logos logos only +// DB_URL=... bun scripts/trains-photos.ts --enrich r.json [--fetch] +// add lead photos to a trains-import --report-new file (stdout); +// with --fetch, first match photos for types that have none +// bun scripts/trains-photos.ts --dry-run --types built.json [--cache dir] [--out x.json] [--only re] +// +// Every type already is a Wikidata item (from trains-import), so there is no fuzzy item +// matching. Photos, in order (max 10; low-confidence files are skipped, never guessed): +// 1. The item's own image (P18), then the lead image of its English Wikipedia article. +// 2. Images of its variants (items that are "subclass of" it), captioned with the variant. +// 3. Files in its own Commons category (P373), only when the category name itself names the +// type (so company, station or line categories never count), plus unit subcategories +// ("NS VIRM 8642") when the category is small. +// 4. Other images of its Wikipedia article whose file name names the type: its designator with +// the class/operator prefix ("NS 1700", "Class 373", "LNER A4"), a telling acronym ("VIRM", +// "ICNG") or the full name. Files that name a longer sibling type ("ICE 3" on "ICE") are not +// taken, nor are interiors, cabs, details, maps, drawings, models, accidents or small files. +// Logos: the maker's/operator's logo (P154, current/preferred first), else the infobox logo of +// its article (Commons files only), else a plain " logo" file in " logos". +// Nothing is downloaded: thumbnail URLs plus author and licence are stored. Each type is written +// as soon as it is done, so an interrupted run resumes with --missing. +import postgres from 'postgres'; +import { readFile, writeFile } from 'node:fs/promises'; +import { + chunks, + cmp, + fileName, + log, + mw, + qid, + sparql, + stripHtml, + tokens, + useCache, + type ImageInfo +} from './trains-wiki'; + +const args = process.argv.slice(2); +const flag = (name: string) => args.includes(name); +const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null); +const dryRun = flag('--dry-run'); +const missingOnly = flag('--missing'); +const logosOnly = flag('--logos'); +const enrichFile = opt('--enrich'); +const typesFile = opt('--types'); +const only = opt('--only'); +const cacheDir = opt('--cache'); +const outFile = opt('--out'); +const MAX_PHOTOS = 10; +const LOGO = 250; +const THUMB = 960; + +type Photo = { + file: string; + kind: 'lead' | 'variant' | 'article' | 'commons' | 'logo'; + caption: string | null; + thumb: string; + width: number; + height: number; + author: string | null; + license: string | null; + licenseUrl: string | null; +}; + +// ---------------------------------------------------------------- commons + +const BAD_FILE = + /\.(svg|gif|tiff?|pdf|djvu|webm|ogv|ogg|stl|png)$|logo|emblem|badge|insignia|wordmark|\bmaps?\b|karte|locator|diagram|chart|graph|schema|schematic|drawing|sketch|blueprint|patent|zeichnung|tekening|plan\b|icon|\bflag\b|coat of arms|wappen|signature|symbol|cutaway|silhouette|commons-|wiki|stub|poster|advert|brochure|stamp|timeline|timetable|dienstregeling|ticket|interior|innenraum|interieur|inside|cab\b|cabin|cockpit|f[uü]hrerstand|machinistenhok|driver'?s|desk|control|console|dashboard|display|screen|seats?\b|seating|toilet|lavatory|wc\b|bistro|restaurant interior|luggage|bogie|draaistel|drehgestell|wheelset|wheel|pantograph|stroomafnemer|coupler|coupling|koppeling|buffer|headlight|lamp|detail|close-?up|closeup|builder'?s plate|works plate|number ?plate|nameplate|plaque|\bplate\b|sign\b|signage|crash|wreck|accident|ongeval|unfall|derail|collision|\bfire\b|burnt|brand\b|scrap|sloop|\btoys?\b|model|modell|scale|märklin|marklin|hornby|bachmann|roco|fleischmann|piko|lionel|\bh0\b|\bho\b|\boo gauge|n gauge|n scale|lego|diecast|layout|miniatur|simulator|mock-?up|render|cgi\b|uniform|portrait|ceremony|memorial|grave|statue|\bvs\b|versus|comparison|livery diagram|motor\b|engine room|transformer|cylinder|valve|boiler|firebox|smokebox|chimney|tender interior|gauge glass|whistle|bell\b/i; + +const infoCache = new Map(); +async function imageInfo(files: string[], width = THUMB) { + const todo = [...new Set(files)].filter((f) => !infoCache.has(`${width}|${f}`)); + for (const chunk of chunks(todo, 50)) { + const res = await mw('commons.wikimedia.org', { + action: 'query', + prop: 'imageinfo', + iiprop: 'url|size|mime|extmetadata', + iiurlwidth: String(width), + iiextmetadatafilter: `Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree${ + width === LOGO ? '|Restrictions' : '' + }`, + titles: chunk.map((f) => `File:${f}`).join('|') + }); + const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from])); + for (const p of res.query?.pages ?? []) { + const original = (norm.get(p.title) ?? p.title).replace(/^File:/, ''); + infoCache.set(`${width}|${original}`, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]); + } + for (const f of chunk) + if (!infoCache.has(`${width}|${f}`)) infoCache.set(`${width}|${f}`, null); + } + // Keep memory flat on long runs: photo lookups are only needed for the current type. + if (infoCache.size > 20_000) + for (const k of [...infoCache.keys()].slice(0, 10_000)) infoCache.delete(k); +} +function toPhoto( + file: string, + kind: Photo['kind'], + caption: string | null, + width = THUMB +): Photo | null { + const info = infoCache.get(`${width}|${file}`); + if (!info?.thumburl) return null; + const meta = info.extmetadata ?? {}; + if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null; + let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null; + if (!license || /fair use|non-free/i.test(license)) return null; + if (kind === 'logo' && /trademark/i.test(meta.Restrictions?.value ?? '')) + license += ' · trademark'; + if (kind !== 'logo') { + if (!/^image\/(jpeg|webp)$/.test(info.mime)) return null; + if (info.width < 640 || info.height < 360) return null; + if (info.width / info.height > 3 || info.height / info.width > 1.6) return null; + } + const author = meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null; + return { + file, + kind, + caption, + thumb: info.thumburl + .replace(/\?.*$/, '') + .replace('//thumb.wikimedia.org/', '//upload.wikimedia.org/'), + width: info.width, + height: info.height, + author, + license, + licenseUrl: meta.LicenseUrl?.value ?? null + }; +} + +async function articleImages(titles: string[]) { + const out = new Map(); + for (const chunk of chunks([...new Set(titles)], 50)) { + let cont: Record = {}; + for (let guard = 0; guard < 30; guard++) { + const res = (await mw('en.wikipedia.org', { + action: 'query', + prop: 'images|pageimages', + piprop: 'name', + imlimit: 'max', + redirects: '1', + titles: chunk.join('|'), + ...cont + })) as { + query?: { + pages?: { title: string; pageimage?: string; images?: { title: string }[] }[]; + normalized?: { from: string; to: string }[]; + redirects?: { from: string; to: string }[]; + }; + continue?: Record; + }; + const norm = new Map(); + for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])]) + norm.set(n.to, norm.get(n.from) ?? n.from); + for (const p of res.query?.pages ?? []) { + const key = norm.get(p.title) ?? p.title; + const cur = out.get(key) ?? { lead: null, files: [] }; + if (p.pageimage) cur.lead = p.pageimage.replace(/_/g, ' '); + cur.files.push(...(p.images ?? []).map((im) => im.title.replace(/^File:/, ''))); + out.set(key, cur); + } + if (!res.continue) break; + cont = res.continue; + } + } + return out; +} +async function categoryMembers(category: string, type: 'file' | 'subcat', limit = 200) { + const out: string[] = []; + let cont: Record = {}; + for (let guard = 0; guard < 4 && out.length < limit; guard++) { + const res = await mw('commons.wikimedia.org', { + action: 'query', + list: 'categorymembers', + cmtitle: `Category:${category}`, + cmtype: type, + cmlimit: '100', + ...cont + }); + out.push( + ...(res.query?.categorymembers ?? []).map((m) => m.title.replace(/^(File|Category):/, '')) + ); + if (!res.continue) break; + cont = res.continue; + } + return out; +} + +// ---------------------------------------------------------------- matching + +type Target = { + id: number | null; + wikidata: string; + wikipedia: string | null; + name: string; + fullName: string; + nlName: string | null; + maker: string; + makerSlug: string; + numberBuilt: number; + photos: number; + checked: boolean; +}; + +/** File name in a comparable form: uppercase words, separated by single spaces. */ +const norm = (s: string) => + ` ${s + .replace(/\.[a-z0-9]+$/i, '') + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toUpperCase() + .replace(/[^A-Z0-9]+/g, ' ') + .trim()} `; +const GENERIC = new Set( + 'CLASS SERIES TYPE TRAIN TRAINS LOCOMOTIVE LOCOMOTIVES ENGINE STEAM DIESEL ELECTRIC HYDRAULIC MULTIPLE UNIT UNITS RAILCAR RAILCARS TRAM TRAMS TRAMCAR STOCK ROLLING CAR CARS COACH COACHES CARRIAGE WAGON METRO SUBWAY RAILWAY RAILWAYS RAILROAD RAIL LINE EXPRESS HIGH SPEED THE AND OF FOR NEW OLD STANDARD GAUGE NARROW TANK TENDER FAMILY GENERATION NATIONAL BRITISH AMERICAN GREAT NORTHERN SOUTHERN EASTERN WESTERN CENTRAL PACIFIC UNION STATE STATES INTERCITY REGIONAL'.split( + ' ' + ) +); +const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +type Rule = { test: (f: string) => boolean; keys: string[] }; +/** Name-based tests for one type: "NS 1700", "Class 373", "LNER A4", "VIRM", "Sprinter Lighttrain". */ +function ruleFor(t: Target, wordUse: Map): Rule { + const makerWords = tokens(t.maker).filter((w) => w.length >= 3 && !GENERIC.has(w)); + const tests: RegExp[] = []; + const keys: string[] = []; + for (const n of [...new Set([t.fullName, t.name, t.nlName].filter((x): x is string => !!x))]) { + const T = norm(n).trim().split(' ').filter(Boolean); + const lead = T.length > 1 && /^[A-Z]{1,6}$/.test(T[0]) ? [T[0]] : []; + const prefixes = [...new Set([...lead, ...makerWords])]; + T.forEach((d, i) => { + if (!/\d/.test(d)) return; + const before = [ + ...new Set([...(i > 0 && /^[A-Z]+$/.test(T[i - 1]) ? [T[i - 1]] : []), ...prefixes]) + ]; + if (T.includes('CLASS')) before.push('CLASS', 'BR'); + if (before.length) + tests.push(new RegExp(` (?:${before.map(esc).join('|')}) ?${esc(d)}(?![0-9])`)); + if (/^(SERIES|CLASS|TYPE|KEI)$/.test(T[i + 1] ?? '')) + tests.push(new RegExp(` ${esc(d)} ?${T[i + 1]} `)); + if (/[A-Z]/.test(d) && d.length >= 3) tests.push(new RegExp(` ${esc(d)} `)); + keys.push(d); + }); + // Telling acronyms ("VIRM", "ICNG", "SLT") and rare words ("Velaro", "Pendolino"). + for (const w of T) { + if (/\d/.test(w) || GENERIC.has(w) || makerWords.includes(w) || lead.includes(w)) continue; + if ( + /^[A-Z]{3,5}$/.test(w) && + (wordUse.get(w) ?? 0) <= 3 && + n === t.fullName && + !/[AEIOU]{2}/.test(w) && + w === n.split(/\s+/).find((x) => x.toUpperCase() === w) + ) { + tests.push(new RegExp(` ${esc(w)} `)); + keys.push(w); + } else if (w.length >= 6 && (wordUse.get(w) ?? 0) <= 1) { + tests.push(new RegExp(` ${esc(w)} `)); + keys.push(w); + } + } + // The whole name as a phrase ("Union Pacific Big Boy", "Sprinter Lighttrain"). + const words = T.filter((x) => !GENERIC.has(x)); + if (T.length >= 2 && words.length >= 1 && T.join('').length >= 6) + tests.push(new RegExp(` ${T.map(esc).join(' ?')} `)); + } + return { keys, test: (f) => tests.some((re) => re.test(norm(f))) }; +} + +async function loadWikidata(targets: Target[]) { + const ids = targets.map((t) => t.wikidata); + const own = new Map(); + const children = new Map(); + for (const c of chunks(ids, 300)) { + for (const r of await sparql(`SELECT ?i ?img ?cat WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} } + OPTIONAL { ?i wdt:P18 ?img } OPTIONAL { ?i wdt:P373 ?cat } }`)) { + const id = qid(r.i); + const cur = own.get(id) ?? { image: null, category: null }; + cur.image ??= r.img ? fileName(r.img) : null; + cur.category ??= r.cat ?? null; + own.set(id, cur); + } + for (const r of await sparql(`SELECT ?i ?c ?l ?img WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} } + ?c wdt:P279 ?i ; wdt:P18 ?img . ?c rdfs:label ?l FILTER(LANG(?l) = "en") }`)) { + const id = qid(r.i); + const list = children.get(id) ?? []; + if (!list.some((x) => x.label === r.l)) list.push({ label: r.l, image: fileName(r.img) }); + children.set(id, list); + } + } + return { own, children }; +} + +/** Category names that are about something else than the vehicles themselves. */ +const BAD_CATEGORY = + /station|bahnhof|depot|\bworks\b|company|companies|logos|\bline\b|lines\b|museum|people|interiors?\b|models?\b|accidents?|maps|diagrams|drawings|liveries|by country|by year|in art/i; + +async function photosFor( + t: Target, + rule: Rule, + wd: Awaited>, + art: { lead: string | null; files: string[] } | undefined, + siblings: Rule[] +) { + const ok = (f: string) => !BAD_FILE.test(f) && rule.test(f) && !siblings.some((s) => s.test(f)); + const own = wd.own.get(t.wikidata); + const picks: { file: string; kind: Photo['kind']; caption: string | null }[] = []; + if (own?.image && !BAD_FILE.test(own.image.replace(/\.png$/i, '.jpg'))) + picks.push({ file: own.image, kind: 'lead', caption: null }); + if (art?.lead && !BAD_FILE.test(art.lead)) + picks.push({ file: art.lead, kind: 'lead', caption: null }); + for (const c of (wd.children.get(t.wikidata) ?? []).slice(0, 8)) + if (!BAD_FILE.test(c.image)) picks.push({ file: c.image, kind: 'variant', caption: c.label }); + for (const f of (art?.files ?? []).filter(ok)) + picks.push({ file: f, kind: 'article', caption: null }); + const unique = () => [...new Map(picks.map((p) => [p.file, p])).values()]; + await imageInfo(unique().map((p) => p.file)); + let photos = unique() + .map((p) => toPhoto(p.file, p.kind, p.caption)) + .filter((p): p is Photo => !!p); + const cat = own?.category; + const catOk = + !!cat && + !BAD_CATEGORY.test(cat) && + (rule.test(cat) || + [t.fullName, t.name, t.nlName].some( + (n) => n && cmp(n).length >= 4 && cmp(cat).includes(cmp(n)) + )); + if (photos.length < MAX_PHOTOS && cat && catOk) { + let files = (await categoryMembers(cat, 'file')).filter( + (f) => !BAD_FILE.test(f) && !siblings.some((s) => s.test(f)) + ); + if (files.length < MAX_PHOTOS) { + // Small category: look into unit subcategories ("NS VIRM 8642", "4468 Mallard"). + const subs = (await categoryMembers(cat, 'subcat', 100)) + .filter( + (s) => + !BAD_CATEGORY.test(s) && /\d/.test(s) && (cmp(s).includes(cmp(cat)) || rule.test(s)) + ) + .slice(0, 6); + for (const s of subs) + files.push( + ...(await categoryMembers(s, 'file', 30)).filter((f) => !BAD_FILE.test(f)).slice(0, 6) + ); + } + files = [...new Set(files)]; + const want = MAX_PHOTOS * 2; + const step = Math.max(1, files.length / want); + const spread = Array.from( + { length: Math.min(want, files.length) }, + (_, i) => files[Math.floor(i * step)] + ); + await imageInfo(spread); + const seen = new Set(photos.map((p) => p.file)); + for (const f of spread) { + if (seen.has(f)) continue; + const p = toPhoto(f, 'commons', null); + if (p) photos.push(p); + seen.add(f); + } + } + // The lead first, then the rest; at most one Wikidata/article lead duplicate. + photos = photos.slice(0, MAX_PHOTOS); + return photos; +} + +// ---------------------------------------------------------------- logos + +type Org = { + table: 'makers' | 'operators'; + slug: string; + name: string; + fullName: string | null; + wikidata: string | null; + hasLogo: boolean; +}; +const LOGO_WORD = /logo|wordmark|crest|emblem|marque|brand|logotip|insignia/i; +const FREE_PD = /public domain|^pd\b|cc0/i; +function logoFile(file: string, keys: string[]): Photo | null { + const photo = toPhoto(file, 'logo', null, LOGO); + if (!photo) return null; + const named = keys.some((k) => k.length >= 2 && cmp(file).includes(k)); + if (/\.svg$/i.test(file)) return LOGO_WORD.test(file) || named ? photo : null; + if (/\.(png|gif)$/i.test(file)) + return LOGO_WORD.test(file) || (FREE_PD.test(photo.license ?? '') && named) ? photo : null; + if (/\.(jpe?g|webp)$/i.test(file)) + return LOGO_WORD.test(file) && FREE_PD.test(photo.license ?? '') ? photo : null; + return null; +} +const PLAIN_WORDS = new Set( + 'LOGO LOGOS LOGOTYPE WORDMARK EMBLEM BRAND MARQUE SYMBOL ICON BADGE CREST NEW OLD CURRENT SIMPLE FLAT COLOR COLOUR COLOURED COLORED BLACK WHITE GREY GRAY RED BLUE YELLOW SILVER VERTICAL HORIZONTAL STACKED TRANSPARENT RAIL RAILWAY RAILWAYS COMPANY CORPORATION INC SVG PNG'.split( + ' ' + ) +); +function plainLogoName(file: string, names: string[]) { + const t = tokens(file.replace(/\.[a-z]+$/i, '')); + for (const n of names) { + const nt = tokens(n); + const at = t.findIndex((_, i) => nt.every((x, j) => t[i + j] === x)); + if (at < 0) continue; + const rest = [...t.slice(0, at), ...t.slice(at + nt.length)]; + if ( + rest.some((x) => x.startsWith('LOGO')) && + rest.every((x) => PLAIN_WORDS.has(x) || /^\d+$/.test(x)) + ) + return true; + } + return false; +} +async function infoboxLogo(article: string) { + const res = await mw('en.wikipedia.org', { + action: 'query', + prop: 'revisions', + rvprop: 'content', + rvslots: 'main', + rvsection: '0', + redirects: '1', + titles: article + }); + const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? ''; + const m = text.match( + /\|\s*logo\s*=\s*(?:\[\[)?(?:File:|Image:)?\s*([^|\]\n{}<]+?\.(?:svg|png|gif))/i + ); + return m ? m[1].trim().replace(/_/g, ' ') : null; +} + +async function findLogos(orgs: Org[]) { + const ids = [...new Set(orgs.map((m) => m.wikidata).filter((x): x is string => !!x))]; + const info = new Map< + string, + { article: string | null; category: string | null; logos: string[]; short: string[] } + >(); + for (const c of chunks(ids, 200)) { + const rows = await sparql(` +SELECT ?m (SAMPLE(?a) AS ?article) (SAMPLE(?cat) AS ?category) (GROUP_CONCAT(DISTINCT ?sn; separator="|") AS ?short) + (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(YEAR(?start)), ""), ";", COALESCE(STR(YEAR(?end)), "")); separator="|") AS ?logos) +WHERE { + VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } + OPTIONAL { ?a schema:about ?m ; schema:isPartOf } + OPTIONAL { ?m wdt:P373 ?cat } + OPTIONAL { ?m wdt:P1813 ?sn } + OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) + OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end } } +} GROUP BY ?m`); + for (const r of rows) { + const logos = (r.logos ? r.logos.split('|').filter(Boolean) : []) + .map((x) => { + const [uri, rank, start, end] = x.split(';'); + return { + file: fileName(uri), + current: !end, + score: + (rank.endsWith('PreferredRank') ? 10000 : 0) + (end ? 0 : 5000) + (Number(start) || 0) + }; + }) + .sort((a, b) => b.score - a.score); + const current = logos.filter((x) => x.current); + info.set(qid(r.m), { + article: r.article + ? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ') + : null, + category: r.category ?? null, + logos: [...new Set((current.length ? current : logos).map((x) => x.file))], + short: r.short ? r.short.split('|').filter((s) => s.length >= 2 && s.length <= 12) : [] + }); + } + } + await imageInfo( + [...info.values()].flatMap((i) => i.logos), + LOGO + ); + const out = new Map(); + let n = 0; + for (const m of orgs) { + const i = m.wikidata ? info.get(m.wikidata) : undefined; + const names = [...new Set([m.name, m.fullName ?? m.name, ...(i?.short ?? [])])]; + const keys = names.map(cmp); + let photo: Photo | null = null; + for (const f of i?.logos ?? []) if ((photo = logoFile(f, keys))) break; + if (!photo && i?.article) { + const f = await infoboxLogo(i.article); + if (f) { + await imageInfo([f], LOGO); + const p = logoFile(f, keys); + if (p && (LOGO_WORD.test(f) || keys.some((k) => k.length >= 3 && cmp(f).includes(k)))) + photo = p; + } + } + if (!photo && i?.category) { + const files = (await categoryMembers(`${i.category} logos`, 'file', 100)) + .filter((f) => /\.(svg|png)$/i.test(f) && /logo/i.test(f) && plainLogoName(f, names)) + .sort( + (x, y) => + Math.max(0, ...(y.match(/(?:19|20)\d\d/g) ?? []).map(Number)) - + Math.max(0, ...(x.match(/(?:19|20)\d\d/g) ?? []).map(Number)) || + Number(/\.svg$/i.test(y)) - Number(/\.svg$/i.test(x)) || + x.length - y.length + ) + .slice(0, 5); + await imageInfo(files, LOGO); + for (const f of files) if ((photo = logoFile(f, keys))) break; + } + out.set(`${m.table}/${m.slug}`, { photo, article: i?.article ?? null }); + if (++n % 200 === 0) log(`logos: ${n}/${orgs.length}`); + } + log(`logos: ${[...out.values()].filter((x) => x.photo).length}/${orgs.length} makers/operators`); + return out; +} + +// ---------------------------------------------------------------- main + +let sqlClient: ReturnType | null = null; +function db() { + if (!process.env.DB_URL) throw new Error('Set DB_URL (or use --dry-run --types )'); + sqlClient ??= postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} }); + return sqlClient; +} + +async function loadTargets(): Promise<{ targets: Target[]; orgs: Org[] }> { + if (typesFile) { + const built = JSON.parse(await readFile(typesFile, 'utf8')) as { + slug: string; + name: string; + fullName: string; + wikidata: string | null; + types: { + wikidata: string; + wikipedia: string; + name: string; + fullName: string; + nlName: string | null; + numberBuilt: number | null; + }[]; + }[]; + return { + targets: built.flatMap((m) => + m.types.map((t) => ({ + id: null, + wikidata: t.wikidata, + wikipedia: t.wikipedia, + name: t.name, + fullName: t.fullName, + nlName: t.nlName, + maker: m.name, + makerSlug: m.slug, + numberBuilt: t.numberBuilt ?? 0, + photos: 0, + checked: false + })) + ), + orgs: built.map((m) => ({ + table: 'makers' as const, + slug: m.slug, + name: m.name, + fullName: m.fullName, + wikidata: m.wikidata, + hasLogo: false + })) + }; + } + const sql = db(); + const rows = await sql< + { + id: number; + wikidata: string; + wikipedia: string | null; + name: string; + full_name: string; + nl_name: string | null; + maker: string; + maker_slug: string; + built: number | null; + photos: number; + checked: boolean; + }[] + >`select t.id, t.wikidata, t.wikipedia, t.name, t.full_name, t.nl_name, m.name as maker, m.slug as maker_slug, + t.number_built as built, t.photo_count as photos, t.photos_checked_at is not null as checked + from trains.types t join trains.makers m on m.id = t.maker_id`; + const orgs = await sql< + { + table: 'makers' | 'operators'; + slug: string; + name: string; + full_name: string | null; + wikidata: string | null; + logo: string | null; + }[] + >` + select 'makers' as table, slug, name, full_name, wikidata, logo from trains.makers + union all select 'operators', slug, name, full_name, wikidata, logo from trains.operators`; + return { + targets: rows.map((r) => ({ + id: r.id, + wikidata: r.wikidata, + wikipedia: r.wikipedia, + name: r.name, + fullName: r.full_name, + nlName: r.nl_name, + maker: r.maker, + makerSlug: r.maker_slug, + numberBuilt: r.built ?? 0, + photos: r.photos, + checked: r.checked + })), + orgs: orgs.map((m) => ({ + table: m.table, + slug: m.slug, + name: m.name, + fullName: m.full_name, + wikidata: m.wikidata, + hasLogo: !!m.logo + })) + }; +} + +async function enrich(file: string) { + const report = JSON.parse(await readFile(file, 'utf8')) as { + newTypes?: { url: string; photo?: string | null }[]; + }; + const sql = db(); + for (const t of report.newTypes ?? []) { + const [, maker, type] = + new URL(t.url).pathname.match(/\/trackers\/trains\/([^/]+)\/([^/]+)/) ?? []; + const [row] = await sql<{ photo: string | null }[]>` + select t.photo from trains.types t join trains.makers m on m.id = t.maker_id + where m.slug = ${maker} and t.slug = ${type}`; + t.photo = row?.photo ?? null; + } + console.log(JSON.stringify(report, null, 2)); +} + +async function saveType(t: Target, list: Photo[]) { + if (dryRun || !t.id) return; + await db().begin(async (tx) => { + await tx`delete from trains.photos where type_id = ${t.id}`; + await tx`update trains.types set photo = ${list[0]?.thumb ?? null}, photo_count = ${list.length}, photos_checked_at = now() where id = ${t.id}`; + if (list.length) + await tx`insert into trains.photos ${tx( + list.map((p, i) => ({ + type_id: t.id, + position: i, + kind: p.kind, + file: p.file, + caption: p.caption, + thumb: p.thumb, + width: p.width, + height: p.height, + author: p.author, + license: p.license, + license_url: p.licenseUrl + })) + )}`; + }); +} + +async function main() { + await useCache(cacheDir); + if (enrichFile && !flag('--fetch')) return enrich(enrichFile); + const loaded = await loadTargets(); + const targets = loaded.targets + .filter((t) => !missingOnly || t.photos === 0 || !t.checked) + .filter((t) => !only || new RegExp(only, 'i').test(t.fullName)) + .sort((a, b) => b.numberBuilt - a.numberBuilt); + const report: { type: string; photos: string[] }[] = []; + let withPhotos = 0; + let total = 0; + if (!logosOnly && targets.length) { + const wd = await loadWikidata(targets); + const wordUse = new Map(); + for (const t of loaded.targets) + for (const w of new Set(tokens(t.fullName))) wordUse.set(w, (wordUse.get(w) ?? 0) + 1); + const byLead = new Map(); + const leadOf = (t: Target) => tokens(t.fullName)[0] ?? ''; + for (const t of loaded.targets) byLead.set(leadOf(t), [...(byLead.get(leadOf(t)) ?? []), t]); + const childLabels = (t: Target) => (wd.children.get(t.wikidata) ?? []).map((c) => cmp(c.label)); + let done = 0; + for (const c of chunks(targets, 100)) { + const articles = await articleImages( + c.map((t) => t.wikipedia).filter((a): a is string => !!a) + ); + for (const t of c) { + const rule = ruleFor(t, wordUse); + const own = cmp(t.fullName); + const variants = childLabels(t); + // Longer-named types that start with this one's name and are not its variants ("ICE 3" vs "ICE"). + const siblings = (byLead.get(leadOf(t)) ?? []) + .filter( + (o) => o !== t && cmp(o.fullName).length > own.length && cmp(o.fullName).startsWith(own) + ) + .filter((o) => !variants.some((v) => v.includes(cmp(o.name)))) + .map((o) => ruleFor(o, wordUse)) + .filter((o) => o.keys.some((k) => !rule.keys.includes(k))); + const list = await photosFor( + t, + rule, + wd, + t.wikipedia ? articles.get(t.wikipedia) : undefined, + siblings + ); + await saveType(t, list); + if (list.length) withPhotos++; + total += list.length; + if (outFile) + report.push({ type: t.fullName, photos: list.map((p) => `${p.kind}: ${p.file}`) }); + if (++done % 100 === 0) + log( + `photos: ${done}/${targets.length} types, ${withPhotos} with photos, ${total} photos` + ); + } + } + } + log(`photos: ${withPhotos}/${targets.length} types with photos, ${total} photos`); + const orgs = loaded.orgs.filter( + (m) => (!missingOnly || !m.hasLogo) && (!only || targets.some((t) => t.makerSlug === m.slug)) + ); + const logos = await findLogos(orgs); + if (outFile) + await writeFile( + outFile, + JSON.stringify({ + types: report, + logos: orgs.map((m) => ({ + org: `${m.table}/${m.name}`, + logo: logos.get(`${m.table}/${m.slug}`)?.photo?.file ?? null + })) + }) + ); + if (dryRun) return; + const sql = db(); + await sql.begin(async (tx) => { + for (const m of orgs) { + const l = logos.get(`${m.table}/${m.slug}`); + const p = l?.photo; + if (m.table === 'makers') + await tx`update trains.makers set wikipedia = ${l?.article ?? null}, logo = ${p?.thumb ?? null}, logo_file = ${p?.file ?? null}, + logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug}`; + else + await tx`update trains.operators set wikipedia = ${l?.article ?? null}, logo = ${p?.thumb ?? null}, logo_file = ${p?.file ?? null}, + logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug}`; + } + await tx`update trains.makers k set photo = ( + select t.photo from trains.types t where t.maker_id = k.id and t.photo is not null + order by t.family_id is null desc, t.number_built desc nulls last limit 1)`; + await tx`update trains.operators o set photo = ( + select t.photo from trains.types t join trains.type_operators x on x.type_id = t.id + where x.operator_id = o.id and t.photo is not null + order by x.position, t.number_built desc nulls last limit 1)`; + }); + log('written'); + if (enrichFile) await enrich(enrichFile); +} + +await main(); +await sqlClient?.end(); diff --git a/scripts/trains-refresh.sh b/scripts/trains-refresh.sh new file mode 100755 index 0000000..a671be8 --- /dev/null +++ b/scripts/trains-refresh.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Refresh of the train pokedex (/trackers/trains) on stan-ovh. +# +# ssh stan@37.187.157.191 'cd ~/dev/website-trains && git fetch -q && git checkout -q --detach origin/main \ +# && bun install --frozen-lockfile >/dev/null && scripts/trains-refresh.sh' +# +# 1. bun scripts/trains-import.ts --report-new rebuilds makers, operators and types from Wikidata + English +# Wikipedia (infobox) and notes which makers/types did not exist before. Dex numbers +# are stable (types are matched by Wikidata id). +# 2. bun scripts/trains-photos.ts --missing finds Wikimedia Commons photos for types that +# have none yet (new ones included) and maker and operator logos that are missing. +# stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}. +# Progress goes to stderr. DB_URL is read from the running app container and never printed. +set -euo pipefail +cd "$(dirname "$0")/.." + +if [ -z "${DB_URL:-}" ]; then + DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' | + sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/') + export DB_URL +fi + +report=$(mktemp /tmp/trains-report.XXXXXX.json) +trap 'rm -f "$report"' EXIT + +bun scripts/trains-import.ts --report-new "$@" >"$report" +bun scripts/trains-photos.ts --missing --enrich "$report" --fetch diff --git a/scripts/trains-wiki.ts b/scripts/trains-wiki.ts new file mode 100644 index 0000000..d5150a3 --- /dev/null +++ b/scripts/trains-wiki.ts @@ -0,0 +1,191 @@ +// Shared HTTP helpers for the train pokedex scripts (scripts/trains-import.ts, +// scripts/trains-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries, +// maxlag handling and an optional on-disk cache (--cache , one file per request). +import { createHash } from 'node:crypto'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +export const UA = + 'StanRungeTrainDex/1.0 (https://stanrunge.dev/trackers/trains; train pokedex importer) bun'; +export const log = (...a: unknown[]) => console.error(...a); +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +let cacheDir: string | null = null; + +/** + * Use `` as a persistent response cache: one small file per request (keyed by a hash), so + * there is never one giant JSON in memory and an interrupted run resumes where it stopped. + */ +export async function useCache(dir: string | null) { + cacheDir = dir; + if (dir) await mkdir(dir, { recursive: true }); +} +const cachePath = (key: string) => + join(cacheDir!, `${createHash('sha1').update(key).digest('hex')}.json`); +async function readCached(key: string): Promise { + if (!cacheDir) return undefined; + try { + return JSON.parse(await readFile(cachePath(key), 'utf8')) as T; + } catch { + return undefined; + } +} +async function writeCached(key: string, body: unknown) { + if (cacheDir) await writeFile(cachePath(key), JSON.stringify(body)); +} +export async function hasCached(key: string) { + return (await readCached(key)) !== undefined; +} + +export async function getJson(url: string, init?: RequestInit, useCached = true): Promise { + const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`; + if (useCached) { + const hit = await readCached(key); + if (hit !== undefined) return hit; + } + for (let attempt = 0; ; attempt++) { + let res: Response; + try { + res = await fetch(url, { + ...init, + headers: { + 'user-agent': UA, + 'api-user-agent': UA, + accept: 'application/json', + ...init?.headers + }, + signal: AbortSignal.timeout(120_000) + }); + } catch (e) { + if (attempt < 6) { + await sleep(2000 * 2 ** attempt); + continue; + } + throw e; + } + const retryAfter = Number(res.headers.get('retry-after')) || 0; + if (res.ok) { + const body = (await res.json()) as T & { error?: { code?: string } }; + if (body?.error?.code === 'maxlag' && attempt < 8) { + await sleep(Math.max(retryAfter, 5) * 1000); + continue; + } + if (useCached) await writeCached(key, body); + return body; + } + if ((res.status === 429 || res.status >= 500) && attempt < 6) { + await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt)); + continue; + } + throw new Error(`${res.status} ${url.slice(0, 200)}`); + } +} + +let lastCall = 0; +/** MediaWiki action API, one request at a time with a small gap (API etiquette). */ +export async function mw( + host: string, + params: Record, + useCached = true +): Promise { + const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params }); + const key = `https://${host}/w/api.php?${body}`; + if (!useCached || !(await hasCached(key))) { + const wait = lastCall + 150 - Date.now(); + if (wait > 0) await sleep(wait); + lastCall = Date.now(); + } + return getJson(key, undefined, useCached); +} +export type MwPage = { + pageid?: number; + title: string; + missing?: boolean; + lastrevid?: number; + extract?: string; + images?: { title: string }[]; + imageinfo?: ImageInfo[]; + revisions?: { revid?: number; slots?: { main?: { content?: string } } }[]; + categorymembers?: { title: string }[]; +}; +export type ImageInfo = { + width: number; + height: number; + mime: string; + thumburl?: string; + descriptionurl?: string; + extmetadata?: Record; +}; +export type MwResponse = { + query?: { + pages?: MwPage[]; + normalized?: { from: string; to: string }[]; + redirects?: { from: string; to: string }[]; + categorymembers?: { title: string }[]; + }; + continue?: Record; +}; + +let lastSparql = 0; +export async function sparql>(query: string): Promise { + const body = new URLSearchParams({ query }).toString(); + if (!(await hasCached(`https://query.wikidata.org/sparql#${body}`))) { + const wait = lastSparql + 400 - Date.now(); + if (wait > 0) await sleep(wait); + lastSparql = Date.now(); + } + const res = await getJson<{ results: { bindings: Record[] } }>( + 'https://query.wikidata.org/sparql', + { + method: 'POST', + headers: { + 'content-type': 'application/x-www-form-urlencoded', + accept: 'application/sparql-results+json' + }, + body + } + ); + return res.results.bindings.map( + (b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T + ); +} + +export const qid = (uri: string) => uri.split('/').at(-1)!; +export const fileName = (uri: string) => + decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' '); +/** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */ +export const cmp = (s: string) => + s + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toUpperCase() + .replace(/&/g, ' AND ') + .replace(/[^A-Z0-9]/g, ''); +export const tokens = (s: string) => + s + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toUpperCase() + .replace(/&/g, ' ') + .split(/[^A-Z0-9]+/) + .filter(Boolean); +export const slugify = (s: string) => + s + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .replace(/&/g, ' and ') + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80) || 'x'; +export const chunks = (xs: T[], n: number) => + Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n)); +export const stripHtml = (s: string) => + s + .replace(/<[^>]+>/g, ' ') + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/�?39;/g, "'") + .replace(/ /g, ' ') + .replace(/\s+/g, ' ') + .trim();