Stan Runge
704880dc98
All checks were successful
Deploy production / deploy (push) Successful in 1m9s
760 lines
29 KiB
TypeScript
760 lines
29 KiB
TypeScript
// Finds freely licensed photos (Wikimedia Commons) for the train pokedex (/trackers/trains)
|
|
// and logos for its makers and operators.
|
|
//
|
|
// DB_URL=... bun scripts/trains-photos.ts every type, replace all photos + logos
|
|
// DB_URL=... bun scripts/trains-photos.ts --missing only types without photos (or never checked),
|
|
// makers/operators without a logo
|
|
// DB_URL=... bun scripts/trains-photos.ts --logos logos only
|
|
// DB_URL=... bun scripts/trains-photos.ts --enrich r.json [--fetch]
|
|
// add lead photos to a trains-import --report-new file (stdout);
|
|
// with --fetch, first match photos for types that have none
|
|
// bun scripts/trains-photos.ts --dry-run --types built.json [--cache dir] [--out x.json] [--only re]
|
|
//
|
|
// Every type already is a Wikidata item (from trains-import), so there is no fuzzy item
|
|
// matching. Photos, in order (max 10; low-confidence files are skipped, never guessed):
|
|
// 1. The item's own image (P18), then the lead image of its English Wikipedia article.
|
|
// 2. Images of its variants (items that are "subclass of" it), captioned with the variant.
|
|
// 3. Files in its own Commons category (P373), only when the category name itself names the
|
|
// type (so company, station or line categories never count), plus unit subcategories
|
|
// ("NS VIRM 8642") when the category is small.
|
|
// 4. Other images of its Wikipedia article whose file name names the type: its designator with
|
|
// the class/operator prefix ("NS 1700", "Class 373", "LNER A4"), a telling acronym ("VIRM",
|
|
// "ICNG") or the full name. Files that name a longer sibling type ("ICE 3" on "ICE") are not
|
|
// taken, nor are interiors, cabs, details, maps, drawings, models, accidents or small files.
|
|
// Logos: the maker's/operator's logo (P154, current/preferred first), else the infobox logo of
|
|
// its article (Commons files only), else a plain "<Name> logo" file in "<Name> logos".
|
|
// Nothing is downloaded: thumbnail URLs plus author and licence are stored. Each type is written
|
|
// as soon as it is done, so an interrupted run resumes with --missing.
|
|
import postgres from 'postgres';
|
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
import {
|
|
chunks,
|
|
cmp,
|
|
fileName,
|
|
log,
|
|
mw,
|
|
qid,
|
|
sparql,
|
|
stripHtml,
|
|
tokens,
|
|
useCache,
|
|
type ImageInfo
|
|
} from './trains-wiki';
|
|
|
|
const args = process.argv.slice(2);
|
|
const flag = (name: string) => args.includes(name);
|
|
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
|
const dryRun = flag('--dry-run');
|
|
const missingOnly = flag('--missing');
|
|
const logosOnly = flag('--logos');
|
|
const enrichFile = opt('--enrich');
|
|
const typesFile = opt('--types');
|
|
const only = opt('--only');
|
|
const cacheDir = opt('--cache');
|
|
const outFile = opt('--out');
|
|
const MAX_PHOTOS = 10;
|
|
const LOGO = 250;
|
|
const THUMB = 960;
|
|
|
|
type Photo = {
|
|
file: string;
|
|
kind: 'lead' | 'variant' | 'article' | 'commons' | 'logo';
|
|
caption: string | null;
|
|
thumb: string;
|
|
width: number;
|
|
height: number;
|
|
author: string | null;
|
|
license: string | null;
|
|
licenseUrl: string | null;
|
|
};
|
|
|
|
// ---------------------------------------------------------------- commons
|
|
|
|
const BAD_FILE =
|
|
/\.(svg|gif|tiff?|pdf|djvu|webm|ogv|ogg|stl|png)$|logo|emblem|badge|insignia|wordmark|\bmaps?\b|karte|locator|diagram|chart|graph|schema|schematic|drawing|sketch|blueprint|patent|zeichnung|tekening|plan\b|icon|\bflag\b|coat of arms|wappen|signature|symbol|cutaway|silhouette|commons-|wiki|stub|poster|advert|brochure|stamp|timeline|timetable|dienstregeling|ticket|interior|innenraum|\d(?:e|st|nd|rd)? ?klas(?:se)?\b|\bklas\b|first class|second class|1st class|2nd class|balkon|operating area|service area|network|netwerk|lijnennet|route ?map|interieur|inside|cab\b|cabin|cockpit|f[uü]hrerstand|machinistenhok|driver'?s|desk|control|console|dashboard|display|screen|seats?\b|seating|toilet|lavatory|wc\b|bistro|restaurant interior|luggage|bogie|\btrucks?\b|draaistel|drehgestell|wheelset|wheel|pantograph|stroomafnemer|coupler|coupling|koppeling|buffer|headlight|lamp|detail|close-?up|closeup|builder'?s plate|works plate|number ?plate|nameplate|plaque|\bplate\b|sign\b|signage|crash|wreck|accident|ongeval|unfall|derail|collision|\bfire\b|burnt|brand\b|scrap|sloop|\btoys?\b|model|modell|scale|märklin|marklin|hornby|bachmann|roco|fleischmann|piko|lionel|\bh0\b|\bho\b|\boo gauge|n gauge|n scale|lego|diecast|layout|miniatur|simulator|mock-?up|render|cgi\b|uniform|portrait|ceremony|memorial|grave|statue|\bvs\b|versus|comparison|livery diagram|motor\b|engine room|transformer|cylinder|valve|boiler|firebox|smokebox|chimney|tender interior|gauge glass|whistle|bell\b/i;
|
|
|
|
const infoCache = new Map<string, ImageInfo | null>();
|
|
async function imageInfo(files: string[], width = THUMB) {
|
|
const todo = [...new Set(files)].filter((f) => !infoCache.has(`${width}|${f}`));
|
|
for (const chunk of chunks(todo, 50)) {
|
|
const res = await mw('commons.wikimedia.org', {
|
|
action: 'query',
|
|
prop: 'imageinfo',
|
|
iiprop: 'url|size|mime|extmetadata',
|
|
iiurlwidth: String(width),
|
|
iiextmetadatafilter: `Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree${
|
|
width === LOGO ? '|Restrictions' : ''
|
|
}`,
|
|
titles: chunk.map((f) => `File:${f}`).join('|')
|
|
});
|
|
const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
|
|
for (const p of res.query?.pages ?? []) {
|
|
const original = (norm.get(p.title) ?? p.title).replace(/^File:/, '');
|
|
infoCache.set(`${width}|${original}`, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]);
|
|
}
|
|
for (const f of chunk)
|
|
if (!infoCache.has(`${width}|${f}`)) infoCache.set(`${width}|${f}`, null);
|
|
}
|
|
// Keep memory flat on long runs: photo lookups are only needed for the current type.
|
|
if (infoCache.size > 20_000)
|
|
for (const k of [...infoCache.keys()].slice(0, 10_000)) infoCache.delete(k);
|
|
}
|
|
function toPhoto(
|
|
file: string,
|
|
kind: Photo['kind'],
|
|
caption: string | null,
|
|
width = THUMB
|
|
): Photo | null {
|
|
const info = infoCache.get(`${width}|${file}`);
|
|
if (!info?.thumburl) return null;
|
|
const meta = info.extmetadata ?? {};
|
|
if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
|
|
let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
|
|
if (!license || /fair use|non-free/i.test(license)) return null;
|
|
if (kind === 'logo' && /trademark/i.test(meta.Restrictions?.value ?? ''))
|
|
license += ' · trademark';
|
|
if (kind !== 'logo') {
|
|
if (!/^image\/(jpeg|webp)$/.test(info.mime)) return null;
|
|
if (info.width < 640 || info.height < 360) return null;
|
|
if (info.width / info.height > 3 || info.height / info.width > 1.6) return null;
|
|
}
|
|
const author = meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null;
|
|
return {
|
|
file,
|
|
kind,
|
|
caption,
|
|
thumb: info.thumburl
|
|
.replace(/\?.*$/, '')
|
|
.replace('//thumb.wikimedia.org/', '//upload.wikimedia.org/'),
|
|
width: info.width,
|
|
height: info.height,
|
|
author,
|
|
license,
|
|
licenseUrl: meta.LicenseUrl?.value ?? null
|
|
};
|
|
}
|
|
|
|
async function articleImages(titles: string[]) {
|
|
const out = new Map<string, { lead: string | null; files: string[] }>();
|
|
for (const chunk of chunks([...new Set(titles)], 50)) {
|
|
let cont: Record<string, string> = {};
|
|
for (let guard = 0; guard < 30; guard++) {
|
|
const res = (await mw('en.wikipedia.org', {
|
|
action: 'query',
|
|
prop: 'images|pageimages',
|
|
piprop: 'name',
|
|
imlimit: 'max',
|
|
redirects: '1',
|
|
titles: chunk.join('|'),
|
|
...cont
|
|
})) as {
|
|
query?: {
|
|
pages?: { title: string; pageimage?: string; images?: { title: string }[] }[];
|
|
normalized?: { from: string; to: string }[];
|
|
redirects?: { from: string; to: string }[];
|
|
};
|
|
continue?: Record<string, string>;
|
|
};
|
|
const norm = new Map<string, string>();
|
|
for (const n of [...(res.query?.normalized ?? []), ...(res.query?.redirects ?? [])])
|
|
norm.set(n.to, norm.get(n.from) ?? n.from);
|
|
for (const p of res.query?.pages ?? []) {
|
|
const key = norm.get(p.title) ?? p.title;
|
|
const cur = out.get(key) ?? { lead: null, files: [] };
|
|
if (p.pageimage) cur.lead = p.pageimage.replace(/_/g, ' ');
|
|
cur.files.push(...(p.images ?? []).map((im) => im.title.replace(/^File:/, '')));
|
|
out.set(key, cur);
|
|
}
|
|
if (!res.continue) break;
|
|
cont = res.continue;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
async function categoryMembers(category: string, type: 'file' | 'subcat', limit = 200) {
|
|
const out: string[] = [];
|
|
let cont: Record<string, string> = {};
|
|
for (let guard = 0; guard < 4 && out.length < limit; guard++) {
|
|
const res = await mw('commons.wikimedia.org', {
|
|
action: 'query',
|
|
list: 'categorymembers',
|
|
cmtitle: `Category:${category}`,
|
|
cmtype: type,
|
|
cmlimit: '100',
|
|
...cont
|
|
});
|
|
out.push(
|
|
...(res.query?.categorymembers ?? []).map((m) => m.title.replace(/^(File|Category):/, ''))
|
|
);
|
|
if (!res.continue) break;
|
|
cont = res.continue;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- matching
|
|
|
|
type Target = {
|
|
id: number | null;
|
|
wikidata: string;
|
|
wikipedia: string | null;
|
|
name: string;
|
|
fullName: string;
|
|
nlName: string | null;
|
|
maker: string;
|
|
makerSlug: string;
|
|
numberBuilt: number;
|
|
photos: number;
|
|
checked: boolean;
|
|
};
|
|
|
|
/** File name in a comparable form: uppercase words, separated by single spaces. */
|
|
const norm = (s: string) =>
|
|
` ${s
|
|
.replace(/\.[a-z0-9]+$/i, '')
|
|
.normalize('NFD')
|
|
.replace(/[\u0300-\u036f]/g, '')
|
|
.toUpperCase()
|
|
.replace(/[^A-Z0-9]+/g, ' ')
|
|
.trim()} `;
|
|
const GENERIC = new Set(
|
|
'CLASS SERIES TYPE TRAIN TRAINS LOCOMOTIVE LOCOMOTIVES ENGINE STEAM DIESEL ELECTRIC HYDRAULIC MULTIPLE UNIT UNITS RAILCAR RAILCARS TRAM TRAMS TRAMCAR STOCK ROLLING CAR CARS COACH COACHES CARRIAGE WAGON METRO SUBWAY RAILWAY RAILWAYS RAILROAD RAIL LINE EXPRESS HIGH SPEED THE AND OF FOR NEW OLD STANDARD GAUGE NARROW TANK TENDER FAMILY GENERATION NATIONAL BRITISH AMERICAN GREAT NORTHERN SOUTHERN EASTERN WESTERN CENTRAL PACIFIC UNION STATE STATES INTERCITY REGIONAL'.split(
|
|
' '
|
|
)
|
|
);
|
|
const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
|
|
type Rule = { test: (f: string) => boolean; keys: string[] };
|
|
/** Name-based tests for one type: "NS 1700", "Class 373", "LNER A4", "VIRM", "Sprinter Lighttrain". */
|
|
function ruleFor(t: Target, wordUse: Map<string, number>): Rule {
|
|
const makerWords = tokens(t.maker).filter((w) => w.length >= 3 && !GENERIC.has(w));
|
|
const tests: RegExp[] = [];
|
|
const keys: string[] = [];
|
|
for (const n of [...new Set([t.fullName, t.name, t.nlName].filter((x): x is string => !!x))]) {
|
|
const T = norm(n).trim().split(' ').filter(Boolean);
|
|
const lead = T.length > 1 && /^[A-Z]{1,6}$/.test(T[0]) ? [T[0]] : [];
|
|
const prefixes = [...new Set([...lead, ...makerWords])];
|
|
T.forEach((d, i) => {
|
|
if (!/\d/.test(d)) return;
|
|
const before = [
|
|
...new Set([...(i > 0 && /^[A-Z]+$/.test(T[i - 1]) ? [T[i - 1]] : []), ...prefixes])
|
|
];
|
|
if (T.includes('CLASS')) before.push('CLASS', 'BR');
|
|
if (before.length)
|
|
tests.push(new RegExp(` (?:${before.map(esc).join('|')}) ?${esc(d)}(?![0-9])`));
|
|
if (/^(SERIES|CLASS|TYPE|KEI)$/.test(T[i + 1] ?? ''))
|
|
tests.push(new RegExp(` ${esc(d)} ?${T[i + 1]} `));
|
|
if (/[A-Z]/.test(d) && d.length >= 3) tests.push(new RegExp(` ${esc(d)} `));
|
|
keys.push(d);
|
|
});
|
|
// Telling acronyms ("VIRM", "ICNG", "SLT") and rare words ("Velaro", "Pendolino").
|
|
for (const w of T) {
|
|
if (/\d/.test(w) || GENERIC.has(w) || makerWords.includes(w) || lead.includes(w)) continue;
|
|
if (
|
|
/^[A-Z]{3,5}$/.test(w) &&
|
|
(wordUse.get(w) ?? 0) <= 3 &&
|
|
n === t.fullName &&
|
|
!/[AEIOU]{2}/.test(w) &&
|
|
w === n.split(/\s+/).find((x) => x.toUpperCase() === w)
|
|
) {
|
|
tests.push(new RegExp(` ${esc(w)} `));
|
|
keys.push(w);
|
|
} else if (w.length >= 6 && (wordUse.get(w) ?? 0) <= 1) {
|
|
tests.push(new RegExp(` ${esc(w)} `));
|
|
keys.push(w);
|
|
}
|
|
}
|
|
// The whole name as a phrase ("Union Pacific Big Boy", "Sprinter Lighttrain").
|
|
const words = T.filter((x) => !GENERIC.has(x));
|
|
if (T.length >= 2 && words.length >= 1 && T.join('').length >= 6)
|
|
tests.push(new RegExp(` ${T.map(esc).join(' ?')} `));
|
|
}
|
|
return { keys, test: (f) => tests.some((re) => re.test(norm(f))) };
|
|
}
|
|
|
|
async function loadWikidata(targets: Target[]) {
|
|
const ids = targets.map((t) => t.wikidata);
|
|
const own = new Map<string, { image: string | null; category: string | null }>();
|
|
const children = new Map<string, { label: string; image: string }[]>();
|
|
for (const c of chunks(ids, 300)) {
|
|
for (const r of await sparql(`SELECT ?i ?img ?cat WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} }
|
|
OPTIONAL { ?i wdt:P18 ?img } OPTIONAL { ?i wdt:P373 ?cat } }`)) {
|
|
const id = qid(r.i);
|
|
const cur = own.get(id) ?? { image: null, category: null };
|
|
cur.image ??= r.img ? fileName(r.img) : null;
|
|
cur.category ??= r.cat ?? null;
|
|
own.set(id, cur);
|
|
}
|
|
for (const r of await sparql(`SELECT ?i ?c ?l ?img WHERE { VALUES ?i { ${c.map((x) => `wd:${x}`).join(' ')} }
|
|
?c wdt:P279 ?i ; wdt:P18 ?img . ?c rdfs:label ?l FILTER(LANG(?l) = "en") }`)) {
|
|
const id = qid(r.i);
|
|
const list = children.get(id) ?? [];
|
|
if (!list.some((x) => x.label === r.l)) list.push({ label: r.l, image: fileName(r.img) });
|
|
children.set(id, list);
|
|
}
|
|
}
|
|
return { own, children };
|
|
}
|
|
|
|
/** Category names that are about something else than the vehicles themselves. */
|
|
const BAD_CATEGORY =
|
|
/station|bahnhof|depot|\bworks\b|company|companies|logos|\bline\b|lines\b|museum|people|interiors?\b|models?\b|accidents?|maps|diagrams|drawings|liveries|by country|by year|in art/i;
|
|
|
|
async function photosFor(
|
|
t: Target,
|
|
rule: Rule,
|
|
wd: Awaited<ReturnType<typeof loadWikidata>>,
|
|
art: { lead: string | null; files: string[] } | undefined,
|
|
siblings: Rule[]
|
|
) {
|
|
const ok = (f: string) => !BAD_FILE.test(f) && rule.test(f) && !siblings.some((s) => s.test(f));
|
|
const own = wd.own.get(t.wikidata);
|
|
const picks: { file: string; kind: Photo['kind']; caption: string | null }[] = [];
|
|
if (own?.image && !BAD_FILE.test(own.image.replace(/\.png$/i, '.jpg')))
|
|
picks.push({ file: own.image, kind: 'lead', caption: null });
|
|
if (art?.lead && !BAD_FILE.test(art.lead))
|
|
picks.push({ file: art.lead, kind: 'lead', caption: null });
|
|
for (const c of (wd.children.get(t.wikidata) ?? []).slice(0, 8))
|
|
if (!BAD_FILE.test(c.image)) picks.push({ file: c.image, kind: 'variant', caption: c.label });
|
|
for (const f of (art?.files ?? []).filter(ok))
|
|
picks.push({ file: f, kind: 'article', caption: null });
|
|
const unique = () => [...new Map(picks.map((p) => [p.file, p])).values()];
|
|
await imageInfo(unique().map((p) => p.file));
|
|
let photos = unique()
|
|
.map((p) => toPhoto(p.file, p.kind, p.caption))
|
|
.filter((p): p is Photo => !!p);
|
|
const cat = own?.category;
|
|
const catOk =
|
|
!!cat &&
|
|
!BAD_CATEGORY.test(cat) &&
|
|
(rule.test(cat) ||
|
|
[t.fullName, t.name, t.nlName].some(
|
|
(n) => n && cmp(n).length >= 4 && cmp(cat).includes(cmp(n))
|
|
));
|
|
if (photos.length < MAX_PHOTOS && cat && catOk) {
|
|
// Strict: a category file must name the type itself ("NS 1700 trains Amsterdam 2019"),
|
|
// so platform, station and skyline shots from the same category stay out.
|
|
let files = (await categoryMembers(cat, 'file')).filter((f) => ok(f));
|
|
if (files.length < MAX_PHOTOS) {
|
|
// Small category: look into unit subcategories ("NS VIRM 8642", "4468 Mallard").
|
|
const subs = (await categoryMembers(cat, 'subcat', 100))
|
|
.filter(
|
|
(s) =>
|
|
!BAD_CATEGORY.test(s) && /\d/.test(s) && (cmp(s).includes(cmp(cat)) || rule.test(s))
|
|
)
|
|
.slice(0, 6);
|
|
for (const s of subs)
|
|
files.push(
|
|
...(await categoryMembers(s, 'file', 30)).filter((f) => !BAD_FILE.test(f)).slice(0, 6)
|
|
);
|
|
}
|
|
files = [...new Set(files)];
|
|
const want = MAX_PHOTOS * 2;
|
|
const step = Math.max(1, files.length / want);
|
|
const spread = Array.from(
|
|
{ length: Math.min(want, files.length) },
|
|
(_, i) => files[Math.floor(i * step)]
|
|
);
|
|
await imageInfo(spread);
|
|
const seen = new Set(photos.map((p) => p.file));
|
|
for (const f of spread) {
|
|
if (seen.has(f)) continue;
|
|
const p = toPhoto(f, 'commons', null);
|
|
if (p) photos.push(p);
|
|
seen.add(f);
|
|
}
|
|
}
|
|
// The lead first, then the rest; at most one Wikidata/article lead duplicate.
|
|
photos = photos.slice(0, MAX_PHOTOS);
|
|
return photos;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- logos
|
|
|
|
type Org = {
|
|
table: 'makers' | 'operators';
|
|
slug: string;
|
|
name: string;
|
|
fullName: string | null;
|
|
wikidata: string | null;
|
|
hasLogo: boolean;
|
|
};
|
|
const LOGO_WORD = /logo|wordmark|crest|emblem|marque|brand|logotip|insignia/i;
|
|
const FREE_PD = /public domain|^pd\b|cc0/i;
|
|
function logoFile(file: string, keys: string[]): Photo | null {
|
|
const photo = toPhoto(file, 'logo', null, LOGO);
|
|
if (!photo) return null;
|
|
const named = keys.some((k) => k.length >= 2 && cmp(file).includes(k));
|
|
if (/\.svg$/i.test(file)) return LOGO_WORD.test(file) || named ? photo : null;
|
|
if (/\.(png|gif)$/i.test(file))
|
|
return LOGO_WORD.test(file) || (FREE_PD.test(photo.license ?? '') && named) ? photo : null;
|
|
if (/\.(jpe?g|webp)$/i.test(file))
|
|
return LOGO_WORD.test(file) && FREE_PD.test(photo.license ?? '') ? photo : null;
|
|
return null;
|
|
}
|
|
const PLAIN_WORDS = new Set(
|
|
'LOGO LOGOS LOGOTYPE WORDMARK EMBLEM BRAND MARQUE SYMBOL ICON BADGE CREST NEW OLD CURRENT SIMPLE FLAT COLOR COLOUR COLOURED COLORED BLACK WHITE GREY GRAY RED BLUE YELLOW SILVER VERTICAL HORIZONTAL STACKED TRANSPARENT RAIL RAILWAY RAILWAYS COMPANY CORPORATION INC SVG PNG'.split(
|
|
' '
|
|
)
|
|
);
|
|
function plainLogoName(file: string, names: string[]) {
|
|
const t = tokens(file.replace(/\.[a-z]+$/i, ''));
|
|
for (const n of names) {
|
|
const nt = tokens(n);
|
|
const at = t.findIndex((_, i) => nt.every((x, j) => t[i + j] === x));
|
|
if (at < 0) continue;
|
|
const rest = [...t.slice(0, at), ...t.slice(at + nt.length)];
|
|
if (
|
|
rest.some((x) => x.startsWith('LOGO')) &&
|
|
rest.every((x) => PLAIN_WORDS.has(x) || /^\d+$/.test(x))
|
|
)
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
async function infoboxLogo(article: string) {
|
|
const res = await mw('en.wikipedia.org', {
|
|
action: 'query',
|
|
prop: 'revisions',
|
|
rvprop: 'content',
|
|
rvslots: 'main',
|
|
rvsection: '0',
|
|
redirects: '1',
|
|
titles: article
|
|
});
|
|
const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? '';
|
|
const m = text.match(
|
|
/\|\s*logo\s*=\s*(?:\[\[)?(?:File:|Image:)?\s*([^|\]\n{}<]+?\.(?:svg|png|gif))/i
|
|
);
|
|
return m ? m[1].trim().replace(/_/g, ' ') : null;
|
|
}
|
|
|
|
async function findLogos(orgs: Org[]) {
|
|
const ids = [...new Set(orgs.map((m) => m.wikidata).filter((x): x is string => !!x))];
|
|
const info = new Map<
|
|
string,
|
|
{ article: string | null; category: string | null; logos: string[]; short: string[] }
|
|
>();
|
|
for (const c of chunks(ids, 200)) {
|
|
const rows = await sparql(`
|
|
SELECT ?m (SAMPLE(?a) AS ?article) (SAMPLE(?cat) AS ?category) (GROUP_CONCAT(DISTINCT ?sn; separator="|") AS ?short)
|
|
(GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(YEAR(?start)), ""), ";", COALESCE(STR(YEAR(?end)), "")); separator="|") AS ?logos)
|
|
WHERE {
|
|
VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} }
|
|
OPTIONAL { ?a schema:about ?m ; schema:isPartOf <https://en.wikipedia.org/> }
|
|
OPTIONAL { ?m wdt:P373 ?cat }
|
|
OPTIONAL { ?m wdt:P1813 ?sn }
|
|
OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank)
|
|
OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end } }
|
|
} GROUP BY ?m`);
|
|
for (const r of rows) {
|
|
const logos = (r.logos ? r.logos.split('|').filter(Boolean) : [])
|
|
.map((x) => {
|
|
const [uri, rank, start, end] = x.split(';');
|
|
return {
|
|
file: fileName(uri),
|
|
current: !end,
|
|
score:
|
|
(rank.endsWith('PreferredRank') ? 10000 : 0) + (end ? 0 : 5000) + (Number(start) || 0)
|
|
};
|
|
})
|
|
.sort((a, b) => b.score - a.score);
|
|
const current = logos.filter((x) => x.current);
|
|
info.set(qid(r.m), {
|
|
article: r.article
|
|
? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
|
|
: null,
|
|
category: r.category ?? null,
|
|
logos: [...new Set((current.length ? current : logos).map((x) => x.file))],
|
|
short: r.short ? r.short.split('|').filter((s) => s.length >= 2 && s.length <= 12) : []
|
|
});
|
|
}
|
|
}
|
|
await imageInfo(
|
|
[...info.values()].flatMap((i) => i.logos),
|
|
LOGO
|
|
);
|
|
const out = new Map<string, { photo: Photo | null; article: string | null }>();
|
|
let n = 0;
|
|
for (const m of orgs) {
|
|
const i = m.wikidata ? info.get(m.wikidata) : undefined;
|
|
const names = [...new Set([m.name, m.fullName ?? m.name, ...(i?.short ?? [])])];
|
|
const keys = names.map(cmp);
|
|
let photo: Photo | null = null;
|
|
for (const f of i?.logos ?? []) if ((photo = logoFile(f, keys))) break;
|
|
if (!photo && i?.article) {
|
|
const f = await infoboxLogo(i.article);
|
|
if (f) {
|
|
await imageInfo([f], LOGO);
|
|
const p = logoFile(f, keys);
|
|
if (p && (LOGO_WORD.test(f) || keys.some((k) => k.length >= 3 && cmp(f).includes(k))))
|
|
photo = p;
|
|
}
|
|
}
|
|
if (!photo && i?.category) {
|
|
const files = (await categoryMembers(`${i.category} logos`, 'file', 100))
|
|
.filter((f) => /\.(svg|png)$/i.test(f) && /logo/i.test(f) && plainLogoName(f, names))
|
|
.sort(
|
|
(x, y) =>
|
|
Math.max(0, ...(y.match(/(?:19|20)\d\d/g) ?? []).map(Number)) -
|
|
Math.max(0, ...(x.match(/(?:19|20)\d\d/g) ?? []).map(Number)) ||
|
|
Number(/\.svg$/i.test(y)) - Number(/\.svg$/i.test(x)) ||
|
|
x.length - y.length
|
|
)
|
|
.slice(0, 5);
|
|
await imageInfo(files, LOGO);
|
|
for (const f of files) if ((photo = logoFile(f, keys))) break;
|
|
}
|
|
out.set(`${m.table}/${m.slug}`, { photo, article: i?.article ?? null });
|
|
if (++n % 200 === 0) log(`logos: ${n}/${orgs.length}`);
|
|
}
|
|
log(`logos: ${[...out.values()].filter((x) => x.photo).length}/${orgs.length} makers/operators`);
|
|
return out;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- main
|
|
|
|
let sqlClient: ReturnType<typeof postgres> | null = null;
|
|
function db() {
|
|
if (!process.env.DB_URL) throw new Error('Set DB_URL (or use --dry-run --types <built.json>)');
|
|
sqlClient ??= postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
|
return sqlClient;
|
|
}
|
|
|
|
async function loadTargets(): Promise<{ targets: Target[]; orgs: Org[] }> {
|
|
if (typesFile) {
|
|
const built = JSON.parse(await readFile(typesFile, 'utf8')) as {
|
|
slug: string;
|
|
name: string;
|
|
fullName: string;
|
|
wikidata: string | null;
|
|
types: {
|
|
wikidata: string;
|
|
wikipedia: string;
|
|
name: string;
|
|
fullName: string;
|
|
nlName: string | null;
|
|
numberBuilt: number | null;
|
|
}[];
|
|
}[];
|
|
return {
|
|
targets: built.flatMap((m) =>
|
|
m.types.map((t) => ({
|
|
id: null,
|
|
wikidata: t.wikidata,
|
|
wikipedia: t.wikipedia,
|
|
name: t.name,
|
|
fullName: t.fullName,
|
|
nlName: t.nlName,
|
|
maker: m.name,
|
|
makerSlug: m.slug,
|
|
numberBuilt: t.numberBuilt ?? 0,
|
|
photos: 0,
|
|
checked: false
|
|
}))
|
|
),
|
|
orgs: built.map((m) => ({
|
|
table: 'makers' as const,
|
|
slug: m.slug,
|
|
name: m.name,
|
|
fullName: m.fullName,
|
|
wikidata: m.wikidata,
|
|
hasLogo: false
|
|
}))
|
|
};
|
|
}
|
|
const sql = db();
|
|
const rows = await sql<
|
|
{
|
|
id: number;
|
|
wikidata: string;
|
|
wikipedia: string | null;
|
|
name: string;
|
|
full_name: string;
|
|
nl_name: string | null;
|
|
maker: string;
|
|
maker_slug: string;
|
|
built: number | null;
|
|
photos: number;
|
|
checked: boolean;
|
|
}[]
|
|
>`select t.id, t.wikidata, t.wikipedia, t.name, t.full_name, t.nl_name, m.name as maker, m.slug as maker_slug,
|
|
t.number_built as built, t.photo_count as photos, t.photos_checked_at is not null as checked
|
|
from trains.types t join trains.makers m on m.id = t.maker_id`;
|
|
const orgs = await sql<
|
|
{
|
|
table: 'makers' | 'operators';
|
|
slug: string;
|
|
name: string;
|
|
full_name: string | null;
|
|
wikidata: string | null;
|
|
logo: string | null;
|
|
}[]
|
|
>`
|
|
select 'makers' as table, slug, name, full_name, wikidata, logo from trains.makers
|
|
union all select 'operators', slug, name, full_name, wikidata, logo from trains.operators`;
|
|
return {
|
|
targets: rows.map((r) => ({
|
|
id: r.id,
|
|
wikidata: r.wikidata,
|
|
wikipedia: r.wikipedia,
|
|
name: r.name,
|
|
fullName: r.full_name,
|
|
nlName: r.nl_name,
|
|
maker: r.maker,
|
|
makerSlug: r.maker_slug,
|
|
numberBuilt: r.built ?? 0,
|
|
photos: r.photos,
|
|
checked: r.checked
|
|
})),
|
|
orgs: orgs.map((m) => ({
|
|
table: m.table,
|
|
slug: m.slug,
|
|
name: m.name,
|
|
fullName: m.full_name,
|
|
wikidata: m.wikidata,
|
|
hasLogo: !!m.logo
|
|
}))
|
|
};
|
|
}
|
|
|
|
async function enrich(file: string) {
|
|
const report = JSON.parse(await readFile(file, 'utf8')) as {
|
|
newTypes?: { url: string; photo?: string | null }[];
|
|
};
|
|
const sql = db();
|
|
for (const t of report.newTypes ?? []) {
|
|
const [, maker, type] =
|
|
new URL(t.url).pathname.match(/\/trackers\/trains\/([^/]+)\/([^/]+)/) ?? [];
|
|
const [row] = await sql<{ photo: string | null }[]>`
|
|
select t.photo from trains.types t join trains.makers m on m.id = t.maker_id
|
|
where m.slug = ${maker} and t.slug = ${type}`;
|
|
t.photo = row?.photo ?? null;
|
|
}
|
|
console.log(JSON.stringify(report, null, 2));
|
|
}
|
|
|
|
async function saveType(t: Target, list: Photo[]) {
|
|
if (dryRun || !t.id) return;
|
|
await db().begin(async (tx) => {
|
|
await tx`delete from trains.photos where type_id = ${t.id}`;
|
|
await tx`update trains.types set photo = ${list[0]?.thumb ?? null}, photo_count = ${list.length}, photos_checked_at = now() where id = ${t.id}`;
|
|
if (list.length)
|
|
await tx`insert into trains.photos ${tx(
|
|
list.map((p, i) => ({
|
|
type_id: t.id,
|
|
position: i,
|
|
kind: p.kind,
|
|
file: p.file,
|
|
caption: p.caption,
|
|
thumb: p.thumb,
|
|
width: p.width,
|
|
height: p.height,
|
|
author: p.author,
|
|
license: p.license,
|
|
license_url: p.licenseUrl
|
|
}))
|
|
)}`;
|
|
});
|
|
}
|
|
|
|
async function main() {
|
|
await useCache(cacheDir);
|
|
if (enrichFile && !flag('--fetch')) return enrich(enrichFile);
|
|
const loaded = await loadTargets();
|
|
const targets = loaded.targets
|
|
.filter((t) => !missingOnly || t.photos === 0 || !t.checked)
|
|
.filter((t) => !only || new RegExp(only, 'i').test(t.fullName))
|
|
.sort((a, b) => b.numberBuilt - a.numberBuilt);
|
|
const report: { type: string; photos: string[] }[] = [];
|
|
let withPhotos = 0;
|
|
let total = 0;
|
|
if (!logosOnly && targets.length) {
|
|
const wd = await loadWikidata(targets);
|
|
const wordUse = new Map<string, number>();
|
|
for (const t of loaded.targets)
|
|
for (const w of new Set(tokens(t.fullName))) wordUse.set(w, (wordUse.get(w) ?? 0) + 1);
|
|
const byLead = new Map<string, Target[]>();
|
|
const leadOf = (t: Target) => tokens(t.fullName)[0] ?? '';
|
|
for (const t of loaded.targets) byLead.set(leadOf(t), [...(byLead.get(leadOf(t)) ?? []), t]);
|
|
const childLabels = (t: Target) => (wd.children.get(t.wikidata) ?? []).map((c) => cmp(c.label));
|
|
let done = 0;
|
|
for (const c of chunks(targets, 100)) {
|
|
const articles = await articleImages(
|
|
c.map((t) => t.wikipedia).filter((a): a is string => !!a)
|
|
);
|
|
for (const t of c) {
|
|
const rule = ruleFor(t, wordUse);
|
|
const own = cmp(t.fullName);
|
|
const variants = childLabels(t);
|
|
// Longer-named types that start with this one's name and are not its variants ("ICE 3" vs "ICE").
|
|
const siblings = (byLead.get(leadOf(t)) ?? [])
|
|
.filter(
|
|
(o) => o !== t && cmp(o.fullName).length > own.length && cmp(o.fullName).startsWith(own)
|
|
)
|
|
.filter((o) => !variants.some((v) => v.includes(cmp(o.name))))
|
|
.map((o) => ruleFor(o, wordUse))
|
|
.filter((o) => o.keys.some((k) => !rule.keys.includes(k)));
|
|
const list = await photosFor(
|
|
t,
|
|
rule,
|
|
wd,
|
|
t.wikipedia ? articles.get(t.wikipedia) : undefined,
|
|
siblings
|
|
);
|
|
await saveType(t, list);
|
|
if (list.length) withPhotos++;
|
|
total += list.length;
|
|
if (outFile)
|
|
report.push({ type: t.fullName, photos: list.map((p) => `${p.kind}: ${p.file}`) });
|
|
if (++done % 100 === 0)
|
|
log(
|
|
`photos: ${done}/${targets.length} types, ${withPhotos} with photos, ${total} photos`
|
|
);
|
|
}
|
|
}
|
|
}
|
|
log(`photos: ${withPhotos}/${targets.length} types with photos, ${total} photos`);
|
|
const orgs = loaded.orgs.filter(
|
|
(m) => (!missingOnly || !m.hasLogo) && (!only || targets.some((t) => t.makerSlug === m.slug))
|
|
);
|
|
const logos = await findLogos(orgs);
|
|
if (outFile)
|
|
await writeFile(
|
|
outFile,
|
|
JSON.stringify({
|
|
types: report,
|
|
logos: orgs.map((m) => ({
|
|
org: `${m.table}/${m.name}`,
|
|
logo: logos.get(`${m.table}/${m.slug}`)?.photo?.file ?? null
|
|
}))
|
|
})
|
|
);
|
|
if (dryRun) return;
|
|
const sql = db();
|
|
await sql.begin(async (tx) => {
|
|
for (const m of orgs) {
|
|
const l = logos.get(`${m.table}/${m.slug}`);
|
|
const p = l?.photo;
|
|
if (m.table === 'makers')
|
|
await tx`update trains.makers set wikipedia = ${l?.article ?? null}, logo = ${p?.thumb ?? null}, logo_file = ${p?.file ?? null},
|
|
logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug}`;
|
|
else
|
|
await tx`update trains.operators set wikipedia = ${l?.article ?? null}, logo = ${p?.thumb ?? null}, logo_file = ${p?.file ?? null},
|
|
logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug}`;
|
|
}
|
|
await tx`update trains.makers k set photo = (
|
|
select t.photo from trains.types t where t.maker_id = k.id and t.photo is not null
|
|
order by t.family_id is null desc, t.number_built desc nulls last limit 1)`;
|
|
await tx`update trains.operators o set photo = (
|
|
select t.photo from trains.types t join trains.type_operators x on x.type_id = t.id
|
|
where x.operator_id = o.id and t.photo is not null
|
|
order by x.position, t.number_built desc nulls last limit 1)`;
|
|
});
|
|
log('written');
|
|
if (enrichFile) await enrich(enrichFile);
|
|
}
|
|
|
|
await main();
|
|
await sqlClient?.end();
|