cars: brand logos everywhere (search cards, brand grid, model headers), stricter logo matching + curated fixes, logo refresh in cars-photos

scripts/cars-photos.ts

@@ -27,11 +27,13 @@ const flag = (name: string) => args.includes(name);
 const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
 const dryRun = flag('--dry-run');
 const missingOnly = flag('--missing');
+const logosOnly = flag('--logos'); // brand logos only, models untouched
 const enrichFile = opt('--enrich');
 const modelsFile = opt('--models');
 const cacheDir = opt('--cache');
 const outFile = opt('--out');
 const MAX_PHOTOS = 12;
+const LOGO = 250; // logo render width (SVGs come back as PNG thumbnails)
 const THUMB = 960; // Wikimedia standard thumbnail widths: 120 250 330 500 960 1280 1920
 const UA =
 	'StanRungeCarDex/1.0 (https://stanrunge.dev/trackers/cars; car pokedex photo matcher) bun';
@@ -256,6 +258,352 @@ SELECT ?m (SAMPLE(?l) AS ?label) (SAMPLE(?logo) AS ?logo) (SAMPLE(?a) AS ?articl
 	return { items, makers, children };
 }
 
+// ---------------------------------------------------------------- brand logos
+
+type Brand = { slug: string; name: string; wikidata: string | null; hasLogo: boolean };
+type LogoItem = {
+	id: string;
+	label: string;
+	article: string | null;
+	category: string | null;
+	links: number;
+	classes: Set<string>;
+	industries: Set<string>;
+	logos: string[]; // best first
+};
+
+/** Registration names whose brand goes by another name on Wikidata/Wikipedia. */
+const BRAND_NAMES: Record<string, string[]> = {
+	'BMW Alpina': ['Alpina'],
+	'Auto Bianchi': ['Autobianchi'],
+	'VW-Porsche': ['Volkswagen-Porsche'],
+	KGM: ['KG Mobility'],
+	SsangYong: ['SsangYong Motor'],
+	Dfsk: ['DFSK'],
+	Faw: ['FAW', 'FAW Group'],
+	Austin: ['Austin Motor Company'],
+	Daimler: ['Daimler Company'],
+	Nekaf: ['NEKAF'],
+	INEOS: ['Ineos Automotive', 'Ineos Grenadier'],
+	STI: ['Subaru Tecnica International'],
+	'Lynk & Co': ['Lynk & Co'],
+	Smart: ['Smart (marque)'],
+	'Aston Martin': ['Aston Martin Lagonda'],
+	Ram: ['Ram Trucks'],
+	Think: ['Think Global'],
+	Shelby: ['Shelby American', 'Carroll Shelby International']
+};
+const AUTO_INDUSTRY = new Set(['Q190117', 'Q1420', 'Q3266850']); // automotive industry, car, ...
+const AUTO_CLASS = new Set([
+	'Q786820', // automobile manufacturer
+	'Q2001319', // automobile marque
+	'Q16889133', // vehicle brand? (checked below against the data)
+	'Q1137109', // car brand
+	'Q19866001' // motor vehicle manufacturer
+]);
+const CORP_SUFFIX =
+	/[\s,]+(motors?|motor cars?|motor car company|motor company|motor corporation|automobiles?|automobili|automobil|automotive|auto|cars?|company|corporation|corp\.?|inc\.?|ltd\.?|limited|llc|gmbh|kg|ag|co\.?|s\.?a\.?|s\.?p\.?a\.?|marque|brand)$/i;
+/** "Bentley Motors Limited", "Mini (marque)" and "Škoda Auto" all reduce to the bare brand. */
+const brandKey = (s: string) => {
+	let t = s.replace(/\s*\([^)]*\)\s*$/, '').trim();
+	for (let prev = ''; prev !== t;) {
+		prev = t;
+		t = t.replace(CORP_SUFFIX, '').trim();
+	}
+	return cmp(t);
+};
+const LOGO_WORD = /logo|wordmark|crest|emblem|marque|brand|logotip/i;
+const FREE_PD = /public domain|^pd\b|cc0/i;
+/** A file is only used when it is clearly a drawn logo, never a photo of a badge on a car. */
+const BAD_LOGOS = new Set(['Morris motor logo.jpg']); // photos of badges despite the name
+function logoFile(file: string, brand: string[]): Photo | null {
+	if (BAD_LOGOS.has(file)) return null;
+	const photo = toPhoto(file, 'logo', null, LOGO);
+	if (!photo) return null;
+	const named = brand.some((k) => cmp(file).includes(k));
+	if (/\.svg$/i.test(file)) return photo;
+	if (/\.(png|gif)$/i.test(file))
+		return LOGO_WORD.test(file) || (FREE_PD.test(photo.license ?? '') && named) ? photo : null;
+	if (/\.(jpe?g|webp)$/i.test(file))
+		return LOGO_WORD.test(file) && FREE_PD.test(photo.license ?? '') ? photo : null;
+	return null;
+}
+
+/** Category files must be "<Brand> logo 2021.svg"-like: no F1 team, motorcycle or parent names. */
+const PLAIN_WORDS = new Set(
+	'LOGO LOGOS LOGOTYPE WORDMARK EMBLEM BRAND MARQUE SYMBOL ICON BADGE CREST NEW OLD CURRENT SIMPLE FLAT COLOR COLOUR COLOURED COLORED BLACK WHITE GREY GRAY RED BLUE SILVER VERTICAL HORIZONTAL STACKED TRANSPARENT AUTOMOBILE AUTOMOBILES AUTOMOBILI CARS CAR MOTOR MOTORS COMPANY SVG PNG'.split(
+		' '
+	)
+);
+function plainLogoName(file: string, names: string[]) {
+	const t = tokens(file.replace(/\.[a-z]+$/i, ''));
+	for (const n of names) {
+		const nt = tokens(n);
+		const at = t.findIndex((_, i) => nt.every((x, j) => t[i + j] === x));
+		if (at < 0) continue;
+		const rest = [...t.slice(0, at), ...t.slice(at + nt.length)];
+		if (
+			rest.some((x) => x.startsWith('LOGO')) &&
+			rest.every((x) => PLAIN_WORDS.has(x) || /^\d+$/.test(x))
+		)
+			return true;
+	}
+	return false;
+}
+/** Hand-checked Commons logos for brands whose Wikidata item has none (or only photos). */
+const LOGO_FILES: Record<string, string> = {
+	// Wikidata prefers the group/corporate logo for these; the car brand has its own.
+	BMW: 'BMW logo (gray).svg',
+	Renault: 'Renault 2021 Text.svg',
+	Seat: 'SEAT Logo from 2017.svg',
+	SsangYong: 'Ssangyong company logo.svg',
+	Bentley: 'Bentley (Logo 1919).png',
+	Caterham: 'Logo of Caterham Cars.png',
+	NSU: 'NSU 1960 Logo.svg',
+	Scion: 'Scion logo.png',
+	Yugo: 'Yugo-logo.svg',
+	Mahindra: 'Mahindra Automotive new logo.png'
+};
+/** Registration names that are not the company Wikidata knows under that name. */
+const BRAND_SKIP = new Set(['Iav']);
+
+async function logoItems(ids: string[]) {
+	type Row = Record<string, string | undefined>;
+	const out = new Map<string, LogoItem>();
+	for (let i = 0; i < ids.length; i += 200) {
+		const chunk = ids.slice(i, i + 200);
+		const rows = await sparql<Row>(`
+SELECT ?item (SAMPLE(?l) AS ?label) (SAMPLE(?a) AS ?article) (SAMPLE(?c) AS ?cat) (SAMPLE(?sl) AS ?links)
+  (GROUP_CONCAT(DISTINCT STRAFTER(STR(?cls), "entity/"); separator="|") AS ?classes)
+  (GROUP_CONCAT(DISTINCT STRAFTER(STR(?ind), "entity/"); separator="|") AS ?industries)
+  (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(YEAR(?start)), ""), ";", IF(BOUND(?end), "end", "")); separator="|") AS ?logos)
+WHERE {
+  VALUES ?item { ${chunk.map((q) => `wd:${q}`).join(' ')} }
+  OPTIONAL { ?item rdfs:label ?l FILTER(LANG(?l) = "en") }
+  OPTIONAL { ?a schema:about ?item; schema:isPartOf <https://en.wikipedia.org/> }
+  OPTIONAL { ?item wdt:P373 ?c }
+  OPTIONAL { ?item wikibase:sitelinks ?sl }
+  OPTIONAL { ?item wdt:P31 ?cls }
+  OPTIONAL { ?item wdt:P452 ?ind }
+  OPTIONAL {
+    ?item p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank .
+    FILTER(?rank != wikibase:DeprecatedRank)
+    OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end }
+  }
+} GROUP BY ?item`);
+		for (const r of rows) {
+			const id = qid(r.item!);
+			const logos = split(r.logos)
+				.map((x) => {
+					const [uri, rank, start, end] = x.split(';');
+					return {
+						file: fileName(uri),
+						current: !end,
+						score:
+							(rank.endsWith('PreferredRank') ? 10000 : 0) + (end ? 0 : 5000) + (Number(start) || 0)
+					};
+				})
+				.sort((a, b) => b.score - a.score);
+			// Logos with an end date are history; only used when nothing current is known.
+			const current = logos.filter((x) => x.current);
+			out.set(id, {
+				id,
+				label: r.label ?? id,
+				article: r.article
+					? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
+					: null,
+				category: r.cat ?? null,
+				links: Number(r.links ?? 0),
+				classes: new Set(split(r.classes)),
+				industries: new Set(split(r.industries)),
+				logos: [...new Set((current.length ? current : logos).map((x) => x.file))]
+			});
+		}
+	}
+	return out;
+}
+
+/** The logo= file of an enwiki infobox, only if it lives on Commons (local files are fair use). */
+async function infoboxLogo(article: string) {
+	const res = (await mw('en.wikipedia.org', {
+		action: 'query',
+		prop: 'revisions',
+		rvprop: 'content',
+		rvslots: 'main',
+		rvsection: '0',
+		redirects: '1',
+		titles: article
+	})) as { query?: { pages?: { revisions?: { slots?: { main?: { content?: string } } }[] }[] } };
+	const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? '';
+	const m = text.match(
+		/\|\s*logo\s*=\s*(?:\[\[)?(?:File:|Image:)?\s*([^|\]\n{}<]+?\.(?:svg|png|gif))/i
+	);
+	return m ? m[1].trim().replace(/_/g, ' ') : null;
+}
+
+async function findLogos(
+	brands: Brand[],
+	makers: Map<string, { id: string; label: string }>
+): Promise<{
+	logos: Map<
+		string,
+		{ photo: Photo; wikidata: string | null; article: string | null; via: string }
+	>;
+	brandPick: Map<string, { id: string; label: string; article: string | null }>;
+}> {
+	brands = brands.filter((b) => !BRAND_SKIP.has(b.name));
+	log(`logos: looking up ${brands.length} brands`);
+	const names = new Map(brands.map((b) => [b.slug, [b.name, ...(BRAND_NAMES[b.name] ?? [])]]));
+	const keys = new Map(brands.map((b) => [b.slug, [...new Set(names.get(b.slug)!.map(brandKey))]]));
+	// Candidates: Wikidata search hits for every name, the stored item and car-model makers.
+	const cands = new Map<string, Set<string>>();
+	for (const b of brands) {
+		const set = new Set<string>();
+		if (b.wikidata) set.add(b.wikidata);
+		for (const m of makers.values())
+			if (keys.get(b.slug)!.includes(brandKey(m.label))) set.add(m.id);
+		for (const n of names.get(b.slug)!) {
+			const res = (await mw('www.wikidata.org', {
+				action: 'wbsearchentities',
+				search: n,
+				language: 'en',
+				type: 'item',
+				limit: '20'
+			})) as { search?: { id: string }[] };
+			for (const hit of res.search ?? []) set.add(hit.id);
+		}
+		cands.set(b.slug, set);
+	}
+	const items = await logoItems([...new Set([...cands.values()].flatMap((s) => [...s]))]);
+	await saveApiCache(true);
+
+	const logos = new Map<
+		string,
+		{ photo: Photo; wikidata: string | null; article: string | null; via: string }
+	>();
+	const brandPick = new Map<string, { id: string; label: string; article: string | null }>();
+	const ranked = new Map<string, LogoItem[]>();
+	for (const b of brands) {
+		const k = keys.get(b.slug)!;
+		const ok = [...cands.get(b.slug)!]
+			.map((id) => items.get(id))
+			.filter((i): i is LogoItem => !!i)
+			// Named like the brand (the article only counts for items without an English label).
+			.filter((i) => k.includes(brandKey(i.label === i.id && i.article ? i.article : i.label)))
+			.filter(
+				(i) =>
+					makers.has(i.id) ||
+					[...i.industries].some((x) => AUTO_INDUSTRY.has(x)) ||
+					[...i.classes].some((x) => AUTO_CLASS.has(x))
+			);
+		const score = (i: LogoItem) =>
+			(makers.has(i.id) ? 4 : 0) +
+			([...i.classes].some((x) => AUTO_CLASS.has(x)) ? 2 : 0) +
+			([...i.industries].some((x) => AUTO_INDUSTRY.has(x)) ? 2 : 0) +
+			(k.includes(cmp(i.label)) ? 1 : 0) +
+			Math.min(i.links, 400) / 1000;
+		ok.sort((a, b) => score(b) - score(a));
+		ranked.set(b.slug, ok);
+		if (ok[0]) brandPick.set(b.slug, ok[0]);
+	}
+	// 1. P154 of the brand's own item (never a parent company's).
+	await imageInfo(
+		[...ranked.values()].flatMap((l) => l.flatMap((i) => i.logos)),
+		LOGO
+	);
+	// 0. Hand-checked files.
+	await imageInfo(Object.values(LOGO_FILES), LOGO);
+	for (const b of brands) {
+		const file = LOGO_FILES[b.name];
+		const photo = file ? toPhoto(file, 'logo', null, LOGO) : null;
+		const i = brandPick.get(b.slug);
+		if (photo)
+			logos.set(b.slug, {
+				photo,
+				wikidata: i?.id ?? null,
+				article: i?.article ?? null,
+				via: 'curated'
+			});
+	}
+	// A file named after the brand wins over e.g. a parent company's logo on a shared item.
+	for (const b of brands)
+		for (const named of [true, false]) {
+			if (logos.has(b.slug)) break;
+			for (const i of ranked.get(b.slug)!) {
+				const photo = i.logos
+					.filter((f) => !named || keys.get(b.slug)!.some((k) => cmp(f).includes(k)))
+					.map((f) => logoFile(f, keys.get(b.slug)!))
+					.find(Boolean);
+				if (photo) {
+					logos.set(b.slug, { photo, wikidata: i.id, article: i.article, via: `P154 ${i.id}` });
+					break;
+				}
+			}
+		}
+	// 2. The logo in the brand article's infobox, when that file is on Commons.
+	for (const b of brands) {
+		if (logos.has(b.slug)) continue;
+		for (const i of ranked
+			.get(b.slug)!
+			.filter((i) => i.article)
+			.slice(0, 2)) {
+			const file = await infoboxLogo(i.article!);
+			if (!file) continue;
+			await imageInfo([file], LOGO);
+			const photo = logoFile(file, keys.get(b.slug)!);
+			if (photo && (LOGO_WORD.test(file) || keys.get(b.slug)!.some((x) => cmp(file).includes(x)))) {
+				logos.set(b.slug, {
+					photo,
+					wikidata: i.id,
+					article: i.article,
+					via: `infobox ${i.article}`
+				});
+				break;
+			}
+		}
+	}
+	// 3. "<Brand> logos" on Commons: a logo file named after the brand, newest year first.
+	for (const b of brands) {
+		if (logos.has(b.slug)) continue;
+		const best = ranked.get(b.slug)![0];
+		if (!best) continue; // without an automotive item the category could be anyone's
+		// The item's own Commons category is trusted; a category merely named like the brand
+		// (shared with e.g. a motorcycle maker) only if the brand's article uses the file.
+		const used = best.article
+			? new Set((await articleImages([best.article])).get(best.article) ?? [])
+			: new Set<string>();
+		const cats = [...new Set([best.category, ...names.get(b.slug)!].filter(Boolean))] as string[];
+		for (const c of cats) {
+			const files = (await categoryFiles(`${c} logos`))
+				.filter((f) => /\.(svg|png)$/i.test(f) && /logo/i.test(f))
+				.filter((f) => plainLogoName(f, names.get(b.slug)!))
+				.filter((f) => c === best.category || used.has(f))
+				.sort(
+					(x, y) =>
+						Math.max(0, ...(y.match(/(?:19|20)\d\d/g) ?? []).map(Number)) -
+							Math.max(0, ...(x.match(/(?:19|20)\d\d/g) ?? []).map(Number)) ||
+						Number(/\.svg$/i.test(y)) - Number(/\.svg$/i.test(x)) ||
+						x.length - y.length
+				);
+			await imageInfo(files.slice(0, 5), LOGO);
+			const file = files.slice(0, 5).find((f) => logoFile(f, keys.get(b.slug)!));
+			if (file) {
+				const photo = logoFile(file, keys.get(b.slug)!)!;
+				logos.set(b.slug, {
+					photo,
+					wikidata: best.id,
+					article: best.article,
+					via: `Category:${c} logos`
+				});
+				break;
+			}
+		}
+	}
+	await saveApiCache(true);
+	log(`logos: ${logos.size}/${brands.length} brands`);
+	return { logos, brandPick };
+}
+
 // ---------------------------------------------------------------- matching
 
 type Target = {
@@ -371,7 +719,9 @@ async function imageInfo(files: string[], width = THUMB) {
 			prop: 'imageinfo',
 			iiprop: 'url|size|mime|extmetadata',
 			iiurlwidth: String(width),
-			iiextmetadatafilter: 'Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree',
+			iiextmetadatafilter: `Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree${
+				width === LOGO ? '|Restrictions' : ''
+			}`,
 			titles: chunk.map((f) => `File:${f}`).join('|')
 		});
 		const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
@@ -393,8 +743,11 @@ function toPhoto(
 	if (!info?.thumburl) return null;
 	const meta = info.extmetadata ?? {};
 	if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
-	const license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
+	let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
 	if (!license || /fair use|non-free/i.test(license)) return null;
+	// Most logos are public domain only as copyright goes (PD-textlogo) but still trademarks.
+	if (kind === 'logo' && /trademark/i.test(meta.Restrictions?.value ?? ''))
+		license += ' · trademark';
 	if (kind !== 'logo') {
 		if (!/^image\/(jpeg|png|webp)$/.test(info.mime)) return null;
 		if (info.width < THUMB || info.height < 400) return null;
@@ -576,7 +929,7 @@ async function photosFor(
 
 async function loadTargets(): Promise<{
 	targets: Target[];
-	brands: { slug: string; name: string }[];
+	brands: Brand[];
 }> {
 	if (modelsFile) {
 		const built = JSON.parse(await readFile(modelsFile, 'utf8')) as {
@@ -596,7 +949,7 @@ async function loadTargets(): Promise<{
 					photos: 0
 				}))
 			),
-			brands: built.map((b) => ({ slug: b.slug, name: b.name }))
+			brands: built.map((b) => ({ slug: b.slug, name: b.name, wikidata: null, hasLogo: false }))
 		};
 	}
 	const sql = db();
@@ -614,7 +967,10 @@ async function loadTargets(): Promise<{
 		select m.id, b.name as brand, b.slug as brand_slug, m.name as model, m.slug,
 			m.nl_count + m.variant_count * 50 as nl, m.photo_count as photos
 		from cars.models m join cars.brands b on b.id = m.brand_id`;
-	const brands = await sql<{ slug: string; name: string }[]>`select slug, name from cars.brands`;
+	const brands = (
+		await sql<{ slug: string; name: string; wikidata: string | null; logo: string | null }[]>`
+			select slug, name, wikidata, logo from cars.brands`
+	).map((b) => ({ slug: b.slug, name: b.name, wikidata: b.wikidata, hasLogo: !!b.logo }));
 	return {
 		targets: rows.map((r) => ({
 			id: r.id,
@@ -665,40 +1021,43 @@ async function main() {
 	const targets = loaded.targets.filter((t) => !missingOnly || t.photos === 0);
 	targets.sort((a, b) => b.nl - a.nl);
 	const wd = await loadWikidata();
-	const { match, makersOf } = matcher(wd);
+	const { match } = matcher(wd);
 	const matches = new Map<Target, Match>();
 	for (const t of targets) {
 		const m = match(t);
 		if (m) matches.set(t, m);
 	}
 	log(`matched ${matches.size}/${targets.length} models to Wikidata`);
-	const photos = await photosFor(targets, wd, matches, loaded.targets);
+	const photos = logosOnly
+		? new Map<Target, Photo[]>()
+		: await photosFor(targets, wd, matches, loaded.targets);
 
-	// Brand logos (P154 of the manufacturer/brand item whose name matches best).
-	const logos = new Map<string, { photo: Photo; wikidata: string; article: string | null }>();
-	const brandPick = new Map<
-		string,
-		{ id: string; logo: string | null; article: string | null; label: string }
-	>();
-	for (const b of brands) {
-		const cands = [...makersOf(b.name)].map((id) => wd.makers.get(id)!).filter(Boolean);
-		const exact = cands.filter((m) => cmp(m.label) === cmp(b.name));
-		const pick = [...exact, ...cands].find((m) => m.logo) ?? exact[0] ?? null;
-		if (pick) brandPick.set(b.slug, pick);
-	}
-	await imageInfo(
-		[...brandPick.values()].map((p) => p.logo).filter((l): l is string => !!l),
-		250
+	const { logos, brandPick } = await findLogos(
+		brands.filter((b) => !missingOnly || !b.hasLogo),
+		wd.makers
 	);
-	for (const [slug, p] of brandPick) {
-		const photo = p.logo ? toPhoto(p.logo, 'logo', null, 250) : null;
-		if (photo) logos.set(slug, { photo, wikidata: p.id, article: p.article });
-	}
 	await saveApiCache(true);
 
+	if (logosOnly && outFile)
+		await writeFile(
+			outFile,
+			JSON.stringify(
+				brands.map((b) => ({
+					brand: b.name,
+					item: brandPick.get(b.slug)?.id ?? null,
+					label: brandPick.get(b.slug)?.label ?? null,
+					logo: logos.get(b.slug)?.photo.file ?? null,
+					thumb: logos.get(b.slug)?.photo.thumb ?? null,
+					via: logos.get(b.slug)?.via ?? null,
+					license: logos.get(b.slug)?.photo.license ?? null
+				})),
+				null,
+				1
+			)
+		);
 	const total = [...photos.values()].reduce((a, p) => a + p.length, 0);
 	log(`photos: ${photos.size} models with photos, ${total} photos, ${logos.size} brand logos`);
-	if (outFile)
+	if (outFile && !logosOnly)
 		await writeFile(
 			outFile,
 			JSON.stringify(
@@ -718,11 +1077,11 @@ async function main() {
 
 	const sql = db();
 	await sql.begin(async (tx) => {
-		const ids = targets.map((t) => t.id!).filter(Boolean);
+		const ids = logosOnly ? [] : targets.map((t) => t.id!).filter(Boolean);
 		// Replace photos for every model we looked at (including ones that lost their match).
 		await tx`delete from cars.photos where model_id = any(${ids})`;
 		await tx`update cars.models set wikidata = null, wikipedia = null, photo = null, photo_count = 0, photos_checked_at = now() where id = any(${ids})`;
-		for (const t of targets) {
+		for (const t of logosOnly ? [] : targets) {
 			const m = matches.get(t);
 			if (!m || !t.id) continue;
 			const list = photos.get(t) ?? [];