trains: Wikidata/Wikipedia import, Commons photos and refresh scripts

scripts/trains-photos.ts

@@ -0,0 +1,759 @@
+// Finds freely licensed photos (Wikimedia Commons) for the train pokedex (/trackers/trains)
+// and logos for its makers and operators.
+//
+//   DB_URL=... bun scripts/trains-photos.ts              every type, replace all photos + logos
+//   DB_URL=... bun scripts/trains-photos.ts --missing    only types without photos (or never checked),
+//                                                        makers/operators without a logo
+//   DB_URL=... bun scripts/trains-photos.ts --logos      logos only
+//   DB_URL=... bun scripts/trains-photos.ts --enrich r.json [--fetch]
+//                                    add lead photos to a trains-import --report-new file (stdout);
+//                                    with --fetch, first match photos for types that have none
+//   bun scripts/trains-photos.ts --dry-run --types built.json [--cache dir] [--out x.json] [--only re]
+//
+// Every type already is a Wikidata item (from trains-import), so there is no fuzzy item
+// matching. Photos, in order (max 10; low-confidence files are skipped, never guessed):
+//  1. The item's own image (P18), then the lead image of its English Wikipedia article.
+//  2. Images of its variants (items that are "subclass of" it), captioned with the variant.
+//  3. Files in its own Commons category (P373), only when the category name itself names the
+//     type (so company, station or line categories never count), plus unit subcategories
+//     ("NS VIRM 8642") when the category is small.
+//  4. Other images of its Wikipedia article whose file name names the type: its designator with
+//     the class/operator prefix ("NS 1700", "Class 373", "LNER A4"), a telling acronym ("VIRM",
+//     "ICNG") or the full name. Files that name a longer sibling type ("ICE 3" on "ICE") are not
+//     taken, nor are interiors, cabs, details, maps, drawings, models, accidents or small files.
+// Logos: the maker's/operator's logo (P154, current/preferred first), else the infobox logo of
+// its article (Commons files only), else a plain "<Name> logo" file in "<Name> logos".
+// Nothing is downloaded: thumbnail URLs plus author and licence are stored. Each type is written
+// as soon as it is done, so an interrupted run resumes with --missing.
+import postgres from 'postgres';
+import { readFile, writeFile } from 'node:fs/promises';
+import {
+	chunks,
+	cmp,
+	fileName,
+	log,
+	mw,
+	qid,
+	sparql,
+	stripHtml,
+	tokens,
+	useCache,
+	type ImageInfo
+} from './trains-wiki';
+
+const args = process.argv.slice(2);
+const flag = (name: string) => args.includes(name);
+const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
+const dryRun = flag('--dry-run');
+const missingOnly = flag('--missing');
+const logosOnly = flag('--logos');
+const enrichFile = opt('--enrich');
+const typesFile = opt('--types');
+const only = opt('--only');
+const cacheDir = opt('--cache');
+const outFile = opt('--out');
+const MAX_PHOTOS = 10;
+const LOGO = 250;
+const THUMB = 960;
+
+type Photo = {
+	file: string;
+	kind: 'lead' | 'variant' | 'article' | 'commons' | 'logo';
+	caption: string | null;
+	thumb: string;
+	width: number;
+	height: number;
+	author: string | null;
+	license: string | null;
+	licenseUrl: string | null;
+};
+
+// ---------------------------------------------------------------- commons
+
+const BAD_FILE =
+	/\.(svg|gif|tiff?|pdf|djvu|webm|ogv|ogg|stl|png)$|logo|emblem|badge|insignia|wordmark|\bmaps?\b|karte|locator|diagram|chart|graph|schema|schematic|drawing|sketch|blueprint|patent|zeichnung|tekening|plan\b|icon|\bflag\b|coat of arms|wappen|signature|symbol|cutaway|silhouette|commons-|wiki|stub|poster|advert|brochure|stamp|timeline|timetable|dienstregeling|ticket|interior|innenraum|interieur|inside|cab\b|cabin|cockpit|f[uü]hrerstand|machinistenhok|driver'?s|desk|control|console|dashboard|display|screen|seats?\b|seating|toilet|lavatory|wc\b|bistro|restaurant interior|luggage|bogie|draaistel|drehgestell|wheelset|wheel|pantograph|stroomafnemer|coupler|coupling|koppeling|buffer|headlight|lamp|detail|close-?up|closeup|builder'?s plate|works plate|number ?plate|nameplate|plaque|\bplate\b|sign\b|signage|crash|wreck|accident|ongeval|unfall|derail|collision|\bfire\b|burnt|brand\b|scrap|sloop|\btoys?\b|model|modell|scale|märklin|marklin|hornby|bachmann|roco|fleischmann|piko|lionel|\bh0\b|\bho\b|\boo gauge|n gauge|n scale|lego|diecast|layout|miniatur|simulator|mock-?up|render|cgi\b|uniform|portrait|ceremony|memorial|grave|statue|\bvs\b|versus|comparison|livery diagram|motor\b|engine room|transformer|cylinder|valve|boiler|firebox|smokebox|chimney|tender interior|gauge glass|whistle|bell\b/i;
+
+const infoCache = new Map<string, ImageInfo | null>();
+async function imageInfo(files: string[], width = THUMB) {
+	const todo = [...new Set(files)].filter((f) => !infoCache.has(`${width}|${f}`));
+	for (const chunk of chunks(todo, 50)) {
+		const res = await mw('commons.wikimedia.org', {
+			action: 'query',
+			prop: 'imageinfo',
+			iiprop: 'url|size|mime|extmetadata',
+			iiurlwidth: String(width),
+			iiextmetadatafilter: `Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree${
+				width === LOGO ? '|Restrictions' : ''
+			}`,
+			titles: chunk.map((f) => `File:${f}`).join('|')
+		});
+		const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
+		for (const p of res.query?.pages ?? []) {
+			const original = (norm.get(p.title) ?? p.title).replace(/^File:/, '');
+			infoCache.set(`${width}|${original}`, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]);
+		}
+		for (const f of chunk)
+			if (!infoCache.has(`${width}|${f}`)) infoCache.set(`${width}|${f}`, null);
+	}
+	// Keep memory flat on long runs: photo lookups are only needed for the current type.
+	if (infoCache.size > 20_000)
+		for (const k of [...infoCache.keys()].slice(0, 10_000)) infoCache.delete(k);
+}
+function toPhoto(
+	file: string,
+	kind: Photo['kind'],
+	caption: string | null,
+	width = THUMB
+): Photo | null {
+	const info = infoCache.get(`${width}|${file}`);
+	if (!info?.thumburl) return null;
+	const meta = info.extmetadata ?? {};
+	if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
+	let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
+	if (!license || /fair use|non-free/i.test(license)) return null;
+	if (kind === 'logo' && /trademark/i.test(meta.Restrictions?.value ?? ''))
+		license += ' · trademark';
+	if (kind !== 'logo') {
+		if (!/^image\/(jpeg|webp)$/.test(info.mime)) return null;
+		if (info.width < 640 || info.height < 360) return null;
+		if (info.width / info.height > 3 || info.height / info.width > 1.6) return null;
+	}
+	const author = meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null;
+	return {
+		file,
+		kind,
+		caption,
+		thumb: info.thumburl
+			.replace(/\?.*$/, '')
+			.replace('//thumb.wikimedia.org/', '//upload.wikimedia.org/'),
+		width: info.width,
+		height: info.height,
+		author,
+		license,
+		licenseUrl: meta.LicenseUrl?.value ?? null
+	};
+}
+
+async function articleImages(titles: string[]) {
+	const out = new Map<string, { lead: string | null; files: string[] }>();
+	for (const chunk of chunks([...new Set(titles)], 50)) {
+		let cont: Record<string, string> = {};
+		for (let guard = 0; guard < 30; guard++) {
+			const res = (await mw('en.wikipedia.org', {
+				action: 'query',
+				prop: 'images|pageimages',
+				piprop: 'name',
+				imlimit: 'max',
+				redirects: '1',
+				titles: chunk.join('|'),
+				...cont
+			})) as {
+				query?: {
+					pages?: { title: string; pageimage?: string; images?: { title: string }[] }[];
+					normalized?: { from: string; to: string }[];
+					redirects?: { from: string; to: string }[];
+				};
+				continue?: Record<string, string>;
+			};
+			const norm = new Map<string, string>();
+			for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])])
+				norm.set(n.to, norm.get(n.from) ?? n.from);
+			for (const p of res.query?.pages ?? []) {
+				const key = norm.get(p.title) ?? p.title;
+				const cur = out.get(key) ?? { lead: null, files: [] };
+				if (p.pageimage) cur.lead = p.pageimage.replace(/_/g, ' ');
+				cur.files.push(...(p.images ?? []).map((im) => im.title.replace(/^File:/, '')));
+				out.set(key, cur);
+			}
+			if (!res.continue) break;
+			cont = res.continue;
+		}
+	}
+	return out;
+}
+async function categoryMembers(category: string, type: 'file' | 'subcat', limit = 200) {
+	const out: string[] = [];
+	let cont: Record<string, string> = {};
+	for (let guard = 0; guard < 4 && out.length < limit; guard++) {
+		const res = await mw('commons.wikimedia.org', {
+			action: 'query',
+			list: 'categorymembers',
+			cmtitle: `Category:${category}`,
+			cmtype: type,
+			cmlimit: '100',
+			...cont
+		});
+		out.push(
+			...(res.query?.categorymembers ?? []).map((m) => m.title.replace(/^(File|Category):/, ''))
+		);
+		if (!res.continue) break;
+		cont = res.continue;
+	}
+	return out;
+}
+
+// ---------------------------------------------------------------- matching
+
+type Target = {
+	id: number | null;
+	wikidata: string;
+	wikipedia: string | null;
+	name: string;
+	fullName: string;
+	nlName: string | null;
+	maker: string;
+	makerSlug: string;
+	numberBuilt: number;
+	photos: number;
+	checked: boolean;
+};
+
+/** File name in a comparable form: uppercase words, separated by single spaces. */
+const norm = (s: string) =>
+	` ${s
+		.replace(/\.[a-z0-9]+$/i, '')
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/[^A-Z0-9]+/g, ' ')
+		.trim()} `;
+const GENERIC = new Set(
+	'CLASS SERIES TYPE TRAIN TRAINS LOCOMOTIVE LOCOMOTIVES ENGINE STEAM DIESEL ELECTRIC HYDRAULIC MULTIPLE UNIT UNITS RAILCAR RAILCARS TRAM TRAMS TRAMCAR STOCK ROLLING CAR CARS COACH COACHES CARRIAGE WAGON METRO SUBWAY RAILWAY RAILWAYS RAILROAD RAIL LINE EXPRESS HIGH SPEED THE AND OF FOR NEW OLD STANDARD GAUGE NARROW TANK TENDER FAMILY GENERATION NATIONAL BRITISH AMERICAN GREAT NORTHERN SOUTHERN EASTERN WESTERN CENTRAL PACIFIC UNION STATE STATES INTERCITY REGIONAL'.split(
+		' '
+	)
+);
+const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+type Rule = { test: (f: string) => boolean; keys: string[] };
+/** Name-based tests for one type: "NS 1700", "Class 373", "LNER A4", "VIRM", "Sprinter Lighttrain". */
+function ruleFor(t: Target, wordUse: Map<string, number>): Rule {
+	const makerWords = tokens(t.maker).filter((w) => w.length >= 3 && !GENERIC.has(w));
+	const tests: RegExp[] = [];
+	const keys: string[] = [];
+	for (const n of [...new Set([t.fullName, t.name, t.nlName].filter((x): x is string => !!x))]) {
+		const T = norm(n).trim().split(' ').filter(Boolean);
+		const lead = T.length > 1 && /^[A-Z]{1,6}$/.test(T[0]) ? [T[0]] : [];
+		const prefixes = [...new Set([...lead, ...makerWords])];
+		T.forEach((d, i) => {
+			if (!/\d/.test(d)) return;
+			const before = [
+				...new Set([...(i > 0 && /^[A-Z]+$/.test(T[i - 1]) ? [T[i - 1]] : []), ...prefixes])
+			];
+			if (T.includes('CLASS')) before.push('CLASS', 'BR');
+			if (before.length)
+				tests.push(new RegExp(` (?:${before.map(esc).join('|')}) ?${esc(d)}(?![0-9])`));
+			if (/^(SERIES|CLASS|TYPE|KEI)$/.test(T[i + 1] ?? ''))
+				tests.push(new RegExp(` ${esc(d)} ?${T[i + 1]} `));
+			if (/[A-Z]/.test(d) && d.length >= 3) tests.push(new RegExp(` ${esc(d)} `));
+			keys.push(d);
+		});
+		// Telling acronyms ("VIRM", "ICNG", "SLT") and rare words ("Velaro", "Pendolino").
+		for (const w of T) {
+			if (/\d/.test(w) || GENERIC.has(w) || makerWords.includes(w) || lead.includes(w)) continue;
+			if (
+				/^[A-Z]{3,5}$/.test(w) &&
+				(wordUse.get(w) ?? 0) <= 3 &&
+				n === t.fullName &&
+				!/[AEIOU]{2}/.test(w) &&
+				w === n.split(/\s+/).find((x) => x.toUpperCase() === w)
+			) {
+				tests.push(new RegExp(` ${esc(w)} `));
+				keys.push(w);
+			} else if (w.length >= 6 && (wordUse.get(w) ?? 0) <= 1) {
+				tests.push(new RegExp(` ${esc(w)} `));
+				keys.push(w);
+			}
+		}
+		// The whole name as a phrase ("Union Pacific Big Boy", "Sprinter Lighttrain").
+		const words = T.filter((x) => !GENERIC.has(x));
+		if (T.length >= 2 && words.length >= 1 && T.join('').length >= 6)
+			tests.push(new RegExp(` ${T.map(esc).join(' ?')} `));
+	}
+	return { keys, test: (f) => tests.some((re) => re.test(norm(f))) };
+}
+
+async function loadWikidata(targets: Target[]) {
+	const ids = targets.map((t) => t.wikidata);
+	const own = new Map<string, { image: string | null; category: string | null }>();
+	const children = new Map<string, { label: string; image: string }[]>();
+	for (const c of chunks(ids, 300)) {
+		for (const r of await sparql(`SELECT ?i ?img ?cat WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} }
+			OPTIONAL { ?i wdt:P18 ?img } OPTIONAL { ?i wdt:P373 ?cat } }`)) {
+			const id = qid(r.i);
+			const cur = own.get(id) ?? { image: null, category: null };
+			cur.image ??= r.img ? fileName(r.img) : null;
+			cur.category ??= r.cat ?? null;
+			own.set(id, cur);
+		}
+		for (const r of await sparql(`SELECT ?i ?c ?l ?img WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} }
+			?c wdt:P279 ?i ; wdt:P18 ?img . ?c rdfs:label ?l FILTER(LANG(?l) = "en") }`)) {
+			const id = qid(r.i);
+			const list = children.get(id) ?? [];
+			if (!list.some((x) => x.label === r.l)) list.push({ label: r.l, image: fileName(r.img) });
+			children.set(id, list);
+		}
+	}
+	return { own, children };
+}
+
+/** Category names that are about something else than the vehicles themselves. */
+const BAD_CATEGORY =
+	/station|bahnhof|depot|\bworks\b|company|companies|logos|\bline\b|lines\b|museum|people|interiors?\b|models?\b|accidents?|maps|diagrams|drawings|liveries|by country|by year|in art/i;
+
+async function photosFor(
+	t: Target,
+	rule: Rule,
+	wd: Awaited<ReturnType<typeof loadWikidata>>,
+	art: { lead: string | null; files: string[] } | undefined,
+	siblings: Rule[]
+) {
+	const ok = (f: string) => !BAD_FILE.test(f) && rule.test(f) && !siblings.some((s) => s.test(f));
+	const own = wd.own.get(t.wikidata);
+	const picks: { file: string; kind: Photo['kind']; caption: string | null }[] = [];
+	if (own?.image && !BAD_FILE.test(own.image.replace(/\.png$/i, '.jpg')))
+		picks.push({ file: own.image, kind: 'lead', caption: null });
+	if (art?.lead && !BAD_FILE.test(art.lead))
+		picks.push({ file: art.lead, kind: 'lead', caption: null });
+	for (const c of (wd.children.get(t.wikidata) ?? []).slice(0, 8))
+		if (!BAD_FILE.test(c.image)) picks.push({ file: c.image, kind: 'variant', caption: c.label });
+	for (const f of (art?.files ?? []).filter(ok))
+		picks.push({ file: f, kind: 'article', caption: null });
+	const unique = () => [...new Map(picks.map((p) => [p.file, p])).values()];
+	await imageInfo(unique().map((p) => p.file));
+	let photos = unique()
+		.map((p) => toPhoto(p.file, p.kind, p.caption))
+		.filter((p): p is Photo => !!p);
+	const cat = own?.category;
+	const catOk =
+		!!cat &&
+		!BAD_CATEGORY.test(cat) &&
+		(rule.test(cat) ||
+			[t.fullName, t.name, t.nlName].some(
+				(n) => n && cmp(n).length >= 4 && cmp(cat).includes(cmp(n))
+			));
+	if (photos.length < MAX_PHOTOS && cat && catOk) {
+		let files = (await categoryMembers(cat, 'file')).filter(
+			(f) => !BAD_FILE.test(f) && !siblings.some((s) => s.test(f))
+		);
+		if (files.length < MAX_PHOTOS) {
+			// Small category: look into unit subcategories ("NS VIRM 8642", "4468 Mallard").
+			const subs = (await categoryMembers(cat, 'subcat', 100))
+				.filter(
+					(s) =>
+						!BAD_CATEGORY.test(s) && /\d/.test(s) && (cmp(s).includes(cmp(cat)) || rule.test(s))
+				)
+				.slice(0, 6);
+			for (const s of subs)
+				files.push(
+					...(await categoryMembers(s, 'file', 30)).filter((f) => !BAD_FILE.test(f)).slice(0, 6)
+				);
+		}
+		files = [...new Set(files)];
+		const want = MAX_PHOTOS * 2;
+		const step = Math.max(1, files.length / want);
+		const spread = Array.from(
+			{ length: Math.min(want, files.length) },
+			(_, i) => files[Math.floor(i * step)]
+		);
+		await imageInfo(spread);
+		const seen = new Set(photos.map((p) => p.file));
+		for (const f of spread) {
+			if (seen.has(f)) continue;
+			const p = toPhoto(f, 'commons', null);
+			if (p) photos.push(p);
+			seen.add(f);
+		}
+	}
+	// The lead first, then the rest; at most one Wikidata/article lead duplicate.
+	photos = photos.slice(0, MAX_PHOTOS);
+	return photos;
+}
+
+// ---------------------------------------------------------------- logos
+
+type Org = {
+	table: 'makers' | 'operators';
+	slug: string;
+	name: string;
+	fullName: string | null;
+	wikidata: string | null;
+	hasLogo: boolean;
+};
+const LOGO_WORD = /logo|wordmark|crest|emblem|marque|brand|logotip|insignia/i;
+const FREE_PD = /public domain|^pd\b|cc0/i;
+function logoFile(file: string, keys: string[]): Photo | null {
+	const photo = toPhoto(file, 'logo', null, LOGO);
+	if (!photo) return null;
+	const named = keys.some((k) => k.length >= 2 && cmp(file).includes(k));
+	if (/\.svg$/i.test(file)) return LOGO_WORD.test(file) || named ? photo : null;
+	if (/\.(png|gif)$/i.test(file))
+		return LOGO_WORD.test(file) || (FREE_PD.test(photo.license ?? '') && named) ? photo : null;
+	if (/\.(jpe?g|webp)$/i.test(file))
+		return LOGO_WORD.test(file) && FREE_PD.test(photo.license ?? '') ? photo : null;
+	return null;
+}
+const PLAIN_WORDS = new Set(
+	'LOGO LOGOS LOGOTYPE WORDMARK EMBLEM BRAND MARQUE SYMBOL ICON BADGE CREST NEW OLD CURRENT SIMPLE FLAT COLOR COLOUR COLOURED COLORED BLACK WHITE GREY GRAY RED BLUE YELLOW SILVER VERTICAL HORIZONTAL STACKED TRANSPARENT RAIL RAILWAY RAILWAYS COMPANY CORPORATION INC SVG PNG'.split(
+		' '
+	)
+);
+function plainLogoName(file: string, names: string[]) {
+	const t = tokens(file.replace(/\.[a-z]+$/i, ''));
+	for (const n of names) {
+		const nt = tokens(n);
+		const at = t.findIndex((_, i) => nt.every((x, j) => t[i + j] === x));
+		if (at < 0) continue;
+		const rest = [...t.slice(0, at), ...t.slice(at + nt.length)];
+		if (
+			rest.some((x) => x.startsWith('LOGO')) &&
+			rest.every((x) => PLAIN_WORDS.has(x) || /^\d+$/.test(x))
+		)
+			return true;
+	}
+	return false;
+}
+async function infoboxLogo(article: string) {
+	const res = await mw('en.wikipedia.org', {
+		action: 'query',
+		prop: 'revisions',
+		rvprop: 'content',
+		rvslots: 'main',
+		rvsection: '0',
+		redirects: '1',
+		titles: article
+	});
+	const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? '';
+	const m = text.match(
+		/\|\s*logo\s*=\s*(?:\[\[)?(?:File:|Image:)?\s*([^|\]\n{}<]+?\.(?:svg|png|gif))/i
+	);
+	return m ? m[1].trim().replace(/_/g, ' ') : null;
+}
+
+async function findLogos(orgs: Org[]) {
+	const ids = [...new Set(orgs.map((m) => m.wikidata).filter((x): x is string => !!x))];
+	const info = new Map<
+		string,
+		{ article: string | null; category: string | null; logos: string[]; short: string[] }
+	>();
+	for (const c of chunks(ids, 200)) {
+		const rows = await sparql(`
+SELECT ?m (SAMPLE(?a) AS ?article) (SAMPLE(?cat) AS ?category) (GROUP_CONCAT(DISTINCT ?sn; separator="|") AS ?short)
+  (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(YEAR(?start)), ""), ";", COALESCE(STR(YEAR(?end)), "")); separator="|") AS ?logos)
+WHERE {
+  VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} }
+  OPTIONAL { ?a schema:about ?m ; schema:isPartOf <https://en.wikipedia.org/> }
+  OPTIONAL { ?m wdt:P373 ?cat }
+  OPTIONAL { ?m wdt:P1813 ?sn }
+  OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank)
+    OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end } }
+} GROUP BY ?m`);
+		for (const r of rows) {
+			const logos = (r.logos ? r.logos.split('|').filter(Boolean) : [])
+				.map((x) => {
+					const [uri, rank, start, end] = x.split(';');
+					return {
+						file: fileName(uri),
+						current: !end,
+						score:
+							(rank.endsWith('PreferredRank') ? 10000 : 0) + (end ? 0 : 5000) + (Number(start) || 0)
+					};
+				})
+				.sort((a, b) => b.score - a.score);
+			const current = logos.filter((x) => x.current);
+			info.set(qid(r.m), {
+				article: r.article
+					? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
+					: null,
+				category: r.category ?? null,
+				logos: [...new Set((current.length ? current : logos).map((x) => x.file))],
+				short: r.short ? r.short.split('|').filter((s) => s.length >= 2 && s.length <= 12) : []
+			});
+		}
+	}
+	await imageInfo(
+		[...info.values()].flatMap((i) => i.logos),
+		LOGO
+	);
+	const out = new Map<string, { photo: Photo | null; article: string | null }>();
+	let n = 0;
+	for (const m of orgs) {
+		const i = m.wikidata ? info.get(m.wikidata) : undefined;
+		const names = [...new Set([m.name, m.fullName ?? m.name, ...(i?.short ?? [])])];
+		const keys = names.map(cmp);
+		let photo: Photo | null = null;
+		for (const f of i?.logos ?? []) if ((photo = logoFile(f, keys))) break;
+		if (!photo && i?.article) {
+			const f = await infoboxLogo(i.article);
+			if (f) {
+				await imageInfo([f], LOGO);
+				const p = logoFile(f, keys);
+				if (p && (LOGO_WORD.test(f) || keys.some((k) => k.length >= 3 && cmp(f).includes(k))))
+					photo = p;
+			}
+		}
+		if (!photo && i?.category) {
+			const files = (await categoryMembers(`${i.category} logos`, 'file', 100))
+				.filter((f) => /\.(svg|png)$/i.test(f) && /logo/i.test(f) && plainLogoName(f, names))
+				.sort(
+					(x, y) =>
+						Math.max(0, ...(y.match(/(?:19|20)\d\d/g) ?? []).map(Number)) -
+							Math.max(0, ...(x.match(/(?:19|20)\d\d/g) ?? []).map(Number)) ||
+						Number(/\.svg$/i.test(y)) - Number(/\.svg$/i.test(x)) ||
+						x.length - y.length
+				)
+				.slice(0, 5);
+			await imageInfo(files, LOGO);
+			for (const f of files) if ((photo = logoFile(f, keys))) break;
+		}
+		out.set(`${m.table}/${m.slug}`, { photo, article: i?.article ?? null });
+		if (++n % 200 === 0) log(`logos: ${n}/${orgs.length}`);
+	}
+	log(`logos: ${[...out.values()].filter((x) => x.photo).length}/${orgs.length} makers/operators`);
+	return out;
+}
+
+// ---------------------------------------------------------------- main
+
+let sqlClient: ReturnType<typeof postgres> | null = null;
+function db() {
+	if (!process.env.DB_URL) throw new Error('Set DB_URL (or use --dry-run --types <built.json>)');
+	sqlClient ??= postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
+	return sqlClient;
+}
+
+async function loadTargets(): Promise<{ targets: Target[]; orgs: Org[] }> {
+	if (typesFile) {
+		const built = JSON.parse(await readFile(typesFile, 'utf8')) as {
+			slug: string;
+			name: string;
+			fullName: string;
+			wikidata: string | null;
+			types: {
+				wikidata: string;
+				wikipedia: string;
+				name: string;
+				fullName: string;
+				nlName: string | null;
+				numberBuilt: number | null;
+			}[];
+		}[];
+		return {
+			targets: built.flatMap((m) =>
+				m.types.map((t) => ({
+					id: null,
+					wikidata: t.wikidata,
+					wikipedia: t.wikipedia,
+					name: t.name,
+					fullName: t.fullName,
+					nlName: t.nlName,
+					maker: m.name,
+					makerSlug: m.slug,
+					numberBuilt: t.numberBuilt ?? 0,
+					photos: 0,
+					checked: false
+				}))
+			),
+			orgs: built.map((m) => ({
+				table: 'makers' as const,
+				slug: m.slug,
+				name: m.name,
+				fullName: m.fullName,
+				wikidata: m.wikidata,
+				hasLogo: false
+			}))
+		};
+	}
+	const sql = db();
+	const rows = await sql<
+		{
+			id: number;
+			wikidata: string;
+			wikipedia: string | null;
+			name: string;
+			full_name: string;
+			nl_name: string | null;
+			maker: string;
+			maker_slug: string;
+			built: number | null;
+			photos: number;
+			checked: boolean;
+		}[]
+	>`select t.id, t.wikidata, t.wikipedia, t.name, t.full_name, t.nl_name, m.name as maker, m.slug as maker_slug,
+		t.number_built as built, t.photo_count as photos, t.photos_checked_at is not null as checked
+		from trains.types t join trains.makers m on m.id = t.maker_id`;
+	const orgs = await sql<
+		{
+			table: 'makers' | 'operators';
+			slug: string;
+			name: string;
+			full_name: string | null;
+			wikidata: string | null;
+			logo: string | null;
+		}[]
+	>`
+		select 'makers' as table, slug, name, full_name, wikidata, logo from trains.makers
+		union all select 'operators', slug, name, full_name, wikidata, logo from trains.operators`;
+	return {
+		targets: rows.map((r) => ({
+			id: r.id,
+			wikidata: r.wikidata,
+			wikipedia: r.wikipedia,
+			name: r.name,
+			fullName: r.full_name,
+			nlName: r.nl_name,
+			maker: r.maker,
+			makerSlug: r.maker_slug,
+			numberBuilt: r.built ?? 0,
+			photos: r.photos,
+			checked: r.checked
+		})),
+		orgs: orgs.map((m) => ({
+			table: m.table,
+			slug: m.slug,
+			name: m.name,
+			fullName: m.full_name,
+			wikidata: m.wikidata,
+			hasLogo: !!m.logo
+		}))
+	};
+}
+
+async function enrich(file: string) {
+	const report = JSON.parse(await readFile(file, 'utf8')) as {
+		newTypes?: { url: string; photo?: string | null }[];
+	};
+	const sql = db();
+	for (const t of report.newTypes ?? []) {
+		const [, maker, type] =
+			new URL(t.url).pathname.match(/\/trackers\/trains\/([^/]+)\/([^/]+)/) ?? [];
+		const [row] = await sql<{ photo: string | null }[]>`
+			select t.photo from trains.types t join trains.makers m on m.id = t.maker_id
+			where m.slug = ${maker} and t.slug = ${type}`;
+		t.photo = row?.photo ?? null;
+	}
+	console.log(JSON.stringify(report, null, 2));
+}
+
+async function saveType(t: Target, list: Photo[]) {
+	if (dryRun || !t.id) return;
+	await db().begin(async (tx) => {
+		await tx`delete from trains.photos where type_id = ${t.id}`;
+		await tx`update trains.types set photo = ${list[0]?.thumb ?? null}, photo_count = ${list.length}, photos_checked_at = now() where id = ${t.id}`;
+		if (list.length)
+			await tx`insert into trains.photos ${tx(
+				list.map((p, i) => ({
+					type_id: t.id,
+					position: i,
+					kind: p.kind,
+					file: p.file,
+					caption: p.caption,
+					thumb: p.thumb,
+					width: p.width,
+					height: p.height,
+					author: p.author,
+					license: p.license,
+					license_url: p.licenseUrl
+				}))
+			)}`;
+	});
+}
+
+async function main() {
+	await useCache(cacheDir);
+	if (enrichFile && !flag('--fetch')) return enrich(enrichFile);
+	const loaded = await loadTargets();
+	const targets = loaded.targets
+		.filter((t) => !missingOnly || t.photos === 0 || !t.checked)
+		.filter((t) => !only || new RegExp(only, 'i').test(t.fullName))
+		.sort((a, b) => b.numberBuilt - a.numberBuilt);
+	const report: { type: string; photos: string[] }[] = [];
+	let withPhotos = 0;
+	let total = 0;
+	if (!logosOnly && targets.length) {
+		const wd = await loadWikidata(targets);
+		const wordUse = new Map<string, number>();
+		for (const t of loaded.targets)
+			for (const w of new Set(tokens(t.fullName))) wordUse.set(w, (wordUse.get(w) ?? 0) + 1);
+		const byLead = new Map<string, Target[]>();
+		const leadOf = (t: Target) => tokens(t.fullName)[0] ?? '';
+		for (const t of loaded.targets) byLead.set(leadOf(t), [...(byLead.get(leadOf(t)) ?? []), t]);
+		const childLabels = (t: Target) => (wd.children.get(t.wikidata) ?? []).map((c) => cmp(c.label));
+		let done = 0;
+		for (const c of chunks(targets, 100)) {
+			const articles = await articleImages(
+				c.map((t) => t.wikipedia).filter((a): a is string => !!a)
+			);
+			for (const t of c) {
+				const rule = ruleFor(t, wordUse);
+				const own = cmp(t.fullName);
+				const variants = childLabels(t);
+				// Longer-named types that start with this one's name and are not its variants ("ICE 3" vs "ICE").
+				const siblings = (byLead.get(leadOf(t)) ?? [])
+					.filter(
+						(o) => o !== t && cmp(o.fullName).length > own.length && cmp(o.fullName).startsWith(own)
+					)
+					.filter((o) => !variants.some((v) => v.includes(cmp(o.name))))
+					.map((o) => ruleFor(o, wordUse))
+					.filter((o) => o.keys.some((k) => !rule.keys.includes(k)));
+				const list = await photosFor(
+					t,
+					rule,
+					wd,
+					t.wikipedia ? articles.get(t.wikipedia) : undefined,
+					siblings
+				);
+				await saveType(t, list);
+				if (list.length) withPhotos++;
+				total += list.length;
+				if (outFile)
+					report.push({ type: t.fullName, photos: list.map((p) => `${p.kind}: ${p.file}`) });
+				if (++done % 100 === 0)
+					log(
+						`photos: ${done}/${targets.length} types, ${withPhotos} with photos, ${total} photos`
+					);
+			}
+		}
+	}
+	log(`photos: ${withPhotos}/${targets.length} types with photos, ${total} photos`);
+	const orgs = loaded.orgs.filter(
+		(m) => (!missingOnly || !m.hasLogo) && (!only || targets.some((t) => t.makerSlug === m.slug))
+	);
+	const logos = await findLogos(orgs);
+	if (outFile)
+		await writeFile(
+			outFile,
+			JSON.stringify({
+				types: report,
+				logos: orgs.map((m) => ({
+					org: `${m.table}/${m.name}`,
+					logo: logos.get(`${m.table}/${m.slug}`)?.photo?.file ?? null
+				}))
+			})
+		);
+	if (dryRun) return;
+	const sql = db();
+	await sql.begin(async (tx) => {
+		for (const m of orgs) {
+			const l = logos.get(`${m.table}/${m.slug}`);
+			const p = l?.photo;
+			if (m.table === 'makers')
+				await tx`update trains.makers set wikipedia = ${l?.article ?? null}, logo = ${p?.thumb ?? null}, logo_file = ${p?.file ?? null},
+					logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug}`;
+			else
+				await tx`update trains.operators set wikipedia = ${l?.article ?? null}, logo = ${p?.thumb ?? null}, logo_file = ${p?.file ?? null},
+					logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug}`;
+		}
+		await tx`update trains.makers k set photo = (
+			select t.photo from trains.types t where t.maker_id = k.id and t.photo is not null
+			order by t.family_id is null desc, t.number_built desc nulls last limit 1)`;
+		await tx`update trains.operators o set photo = (
+			select t.photo from trains.types t join trains.type_operators x on x.type_id = t.id
+			where x.operator_id = o.id and t.photo is not null
+			order by x.position, t.number_built desc nulls last limit 1)`;
+	});
+	log('written');
+	if (enrichFile) await enrich(enrichFile);
+}
+
+await main();
+await sqlClient?.end();

scripts/trains-refresh.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Refresh of the train pokedex (/trackers/trains) on stan-ovh.
+#
+#   ssh stan@37.187.157.191 'cd ~/dev/website-trains && git fetch -q && git checkout -q --detach origin/main \
+#     && bun install --frozen-lockfile >/dev/null && scripts/trains-refresh.sh'
+#
+# 1. bun scripts/trains-import.ts --report-new   rebuilds makers, operators and types from Wikidata + English
+#    Wikipedia (infobox) and notes which makers/types did not exist before. Dex numbers
+#    are stable (types are matched by Wikidata id).
+# 2. bun scripts/trains-photos.ts --missing       finds Wikimedia Commons photos for types that
+#    have none yet (new ones included) and maker and operator logos that are missing.
+# stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}.
+# Progress goes to stderr. DB_URL is read from the running app container and never printed.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [ -z "${DB_URL:-}" ]; then
+	DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' |
+		sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
+	export DB_URL
+fi
+
+report=$(mktemp /tmp/trains-report.XXXXXX.json)
+trap 'rm -f "$report"' EXIT
+
+bun scripts/trains-import.ts --report-new "$@" >"$report"
+bun scripts/trains-photos.ts --missing --enrich "$report" --fetch

scripts/trains-wiki.ts

@@ -0,0 +1,191 @@
+// Shared HTTP helpers for the train pokedex scripts (scripts/trains-import.ts,
+// scripts/trains-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries,
+// maxlag handling and an optional on-disk cache (--cache <dir>, one file per request).
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export const UA =
+	'StanRungeTrainDex/1.0 (https://stanrunge.dev/trackers/trains; train pokedex importer) bun';
+export const log = (...a: unknown[]) => console.error(...a);
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+let cacheDir: string | null = null;
+
+/**
+ * Use `<dir>` as a persistent response cache: one small file per request (keyed by a hash), so
+ * there is never one giant JSON in memory and an interrupted run resumes where it stopped.
+ */
+export async function useCache(dir: string | null) {
+	cacheDir = dir;
+	if (dir) await mkdir(dir, { recursive: true });
+}
+const cachePath = (key: string) =>
+	join(cacheDir!, `${createHash('sha1').update(key).digest('hex')}.json`);
+async function readCached<T>(key: string): Promise<T | undefined> {
+	if (!cacheDir) return undefined;
+	try {
+		return JSON.parse(await readFile(cachePath(key), 'utf8')) as T;
+	} catch {
+		return undefined;
+	}
+}
+async function writeCached(key: string, body: unknown) {
+	if (cacheDir) await writeFile(cachePath(key), JSON.stringify(body));
+}
+export async function hasCached(key: string) {
+	return (await readCached(key)) !== undefined;
+}
+
+export async function getJson<T>(url: string, init?: RequestInit, useCached = true): Promise<T> {
+	const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
+	if (useCached) {
+		const hit = await readCached<T>(key);
+		if (hit !== undefined) return hit;
+	}
+	for (let attempt = 0; ; attempt++) {
+		let res: Response;
+		try {
+			res = await fetch(url, {
+				...init,
+				headers: {
+					'user-agent': UA,
+					'api-user-agent': UA,
+					accept: 'application/json',
+					...init?.headers
+				},
+				signal: AbortSignal.timeout(120_000)
+			});
+		} catch (e) {
+			if (attempt < 6) {
+				await sleep(2000 * 2 ** attempt);
+				continue;
+			}
+			throw e;
+		}
+		const retryAfter = Number(res.headers.get('retry-after')) || 0;
+		if (res.ok) {
+			const body = (await res.json()) as T & { error?: { code?: string } };
+			if (body?.error?.code === 'maxlag' && attempt < 8) {
+				await sleep(Math.max(retryAfter, 5) * 1000);
+				continue;
+			}
+			if (useCached) await writeCached(key, body);
+			return body;
+		}
+		if ((res.status === 429 || res.status >= 500) && attempt < 6) {
+			await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt));
+			continue;
+		}
+		throw new Error(`${res.status} ${url.slice(0, 200)}`);
+	}
+}
+
+let lastCall = 0;
+/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
+export async function mw<T = MwResponse>(
+	host: string,
+	params: Record<string, string>,
+	useCached = true
+): Promise<T> {
+	const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
+	const key = `https://${host}/w/api.php?${body}`;
+	if (!useCached || !(await hasCached(key))) {
+		const wait = lastCall + 150 - Date.now();
+		if (wait > 0) await sleep(wait);
+		lastCall = Date.now();
+	}
+	return getJson<T>(key, undefined, useCached);
+}
+export type MwPage = {
+	pageid?: number;
+	title: string;
+	missing?: boolean;
+	lastrevid?: number;
+	extract?: string;
+	images?: { title: string }[];
+	imageinfo?: ImageInfo[];
+	revisions?: { revid?: number; slots?: { main?: { content?: string } } }[];
+	categorymembers?: { title: string }[];
+};
+export type ImageInfo = {
+	width: number;
+	height: number;
+	mime: string;
+	thumburl?: string;
+	descriptionurl?: string;
+	extmetadata?: Record<string, { value: string }>;
+};
+export type MwResponse = {
+	query?: {
+		pages?: MwPage[];
+		normalized?: { from: string; to: string }[];
+		redirects?: { from: string; to: string }[];
+		categorymembers?: { title: string }[];
+	};
+	continue?: Record<string, string>;
+};
+
+let lastSparql = 0;
+export async function sparql<T = Record<string, string>>(query: string): Promise<T[]> {
+	const body = new URLSearchParams({ query }).toString();
+	if (!(await hasCached(`https://query.wikidata.org/sparql#${body}`))) {
+		const wait = lastSparql + 400 - Date.now();
+		if (wait > 0) await sleep(wait);
+		lastSparql = Date.now();
+	}
+	const res = await getJson<{ results: { bindings: Record<string, { value: string }>[] } }>(
+		'https://query.wikidata.org/sparql',
+		{
+			method: 'POST',
+			headers: {
+				'content-type': 'application/x-www-form-urlencoded',
+				accept: 'application/sparql-results+json'
+			},
+			body
+		}
+	);
+	return res.results.bindings.map(
+		(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
+	);
+}
+
+export const qid = (uri: string) => uri.split('/').at(-1)!;
+export const fileName = (uri: string) =>
+	decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
+/** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */
+export const cmp = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/&/g, ' AND ')
+		.replace(/[^A-Z0-9]/g, '');
+export const tokens = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/&/g, ' ')
+		.split(/[^A-Z0-9]+/)
+		.filter(Boolean);
+export const slugify = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toLowerCase()
+		.replace(/&/g, ' and ')
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-+|-+$/g, '')
+		.slice(0, 80) || 'x';
+export const chunks = <T>(xs: T[], n: number) =>
+	Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n));
+export const stripHtml = (s: string) =>
+	s
+		.replace(/<[^>]+>/g, ' ')
+		.replace(/&amp;/g, '&')
+		.replace(/&quot;/g, '"')
+		.replace(/&#0?39;/g, "'")
+		.replace(/&nbsp;/g, ' ')
+		.replace(/\s+/g, ' ')
+		.trim();