// Finds freely licensed photos for the car pokedex (/trackers/cars) on Wikimedia Commons.
//
//   DB_URL=... bun scripts/cars-photos.ts               match every model, replace all photos
//   DB_URL=... bun scripts/cars-photos.ts --missing     only models that have no photos yet
//   DB_URL=... bun scripts/cars-photos.ts --enrich r.json   add photos to a cars-import
//                                                       --report-new file, print it to stdout
//   bun scripts/cars-photos.ts --dry-run --models built.json [--cache dir] [--out x.json]
//
// How a model gets matched (low-confidence matches are skipped, never guessed):
//  1. One SPARQL query lists every Wikidata item that is an instance of "car model"
//     (Q3231690) with its English label/aliases, enwiki article, manufacturer (P176),
//     brand (P1716), image (P18), Commons category (P373) and parent series (P179).
//  2. A model matches an item when "<brand> <model>" equals the item's label, alias or
//     article title, or when "<model>" does and the item's manufacturer/brand is that brand.
//     Among several hits the series root with an article and the most sitelinks wins.
//  3. Photos, in order: the item's own image, the images of its generations (items that
//     are P179 "part of the series" of it, newest first), images used in the enwiki article
//     and files in the Commons category. Article/category files must mention the model in
//     their file name; logos, maps, drawings, SVGs and small files are dropped.
// Nothing is downloaded: we store thumbnail URLs (standard widths) plus author and licence.
import postgres from 'postgres';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';

const args = process.argv.slice(2);
const flag = (name: string) => args.includes(name);
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
const dryRun = flag('--dry-run');
const missingOnly = flag('--missing');
const logosOnly = flag('--logos'); // brand logos only, models untouched
const enrichFile = opt('--enrich');
const modelsFile = opt('--models');
const cacheDir = opt('--cache');
const outFile = opt('--out');
const MAX_PHOTOS = 12;
const LOGO = 250; // logo render width (SVGs come back as PNG thumbnails)
const THUMB = 960; // Wikimedia standard thumbnail widths: 120 250 330 500 960 1280 1920
const UA =
	'StanRungeCarDex/1.0 (https://stanrunge.dev/trackers/cars; car pokedex photo matcher) bun';
const log = (...a: unknown[]) => console.error(...a);

// ---------------------------------------------------------------- http

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
let apiCache: Record<string, unknown> = {};
let apiCacheDirty = 0;
async function loadApiCache() {
	if (!cacheDir) return;
	await mkdir(cacheDir, { recursive: true });
	try {
		apiCache = JSON.parse(await readFile(join(cacheDir, 'api.json'), 'utf8'));
	} catch {
		apiCache = {};
	}
}
async function saveApiCache(force = false) {
	if (!cacheDir || (!force && apiCacheDirty < 200)) return;
	apiCacheDirty = 0;
	await writeFile(join(cacheDir, 'api.json'), JSON.stringify(apiCache));
}

async function getJson<T>(url: string, init?: RequestInit): Promise<T> {
	const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
	if (key in apiCache) return apiCache[key] as T;
	for (let attempt = 0; ; attempt++) {
		const res = await fetch(url, {
			...init,
			headers: {
				'user-agent': UA,
				'api-user-agent': UA,
				accept: 'application/json',
				...init?.headers
			}
		});
		const retryAfter = Number(res.headers.get('retry-after')) || 0;
		if (res.ok) {
			const body = (await res.json()) as T & { error?: { code?: string } };
			if (body?.error?.code === 'maxlag' && attempt < 8) {
				await sleep(Math.max(retryAfter, 5) * 1000);
				continue;
			}
			apiCache[key] = body;
			apiCacheDirty++;
			await saveApiCache();
			return body;
		}
		if ((res.status === 429 || res.status >= 500) && attempt < 6) {
			await sleep(Math.max(retryAfter * 1000, 2000 * 2 ** attempt));
			continue;
		}
		throw new Error(`${res.status} ${url.slice(0, 200)}`);
	}
}

let lastCall = 0;
/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
async function mw(host: string, params: Record<string, string>) {
	const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
	const key = `https://${host}/w/api.php?${body}`;
	if (!(key in apiCache)) {
		const wait = lastCall + 150 - Date.now();
		if (wait > 0) await sleep(wait);
		lastCall = Date.now();
	}
	return getJson<MwResponse>(key);
}
type MwPage = {
	title: string;
	missing?: boolean;
	images?: { title: string }[];
	imageinfo?: ImageInfo[];
};
type ImageInfo = {
	width: number;
	height: number;
	mime: string;
	thumburl?: string;
	descriptionurl?: string;
	extmetadata?: Record<string, { value: string }>;
};
type MwResponse = {
	query?: { pages?: MwPage[]; normalized?: { from: string; to: string }[] };
	continue?: Record<string, string>;
};

async function sparql<T>(query: string): Promise<T[]> {
	const res = await getJson<{ results: { bindings: Record<string, { value: string }>[] } }>(
		'https://query.wikidata.org/sparql',
		{
			method: 'POST',
			headers: {
				'content-type': 'application/x-www-form-urlencoded',
				accept: 'application/sparql-results+json'
			},
			body: new URLSearchParams({ query }).toString()
		}
	);
	return res.results.bindings.map(
		(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
	);
}

// ---------------------------------------------------------------- wikidata

const qid = (uri: string) => uri.split('/').at(-1)!;
const fileName = (uri: string) =>
	decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
const cmp = (s: string) =>
	s
		.normalize('NFD')
		.replace(/[\u0300-\u036f]/g, '')
		.toUpperCase()
		.replace(/&/g, ' AND ')
		.replace(/[^A-Z0-9]/g, '');
const tokens = (s: string) =>
	s
		.normalize('NFD')
		.replace(/[\u0300-\u036f]/g, '')
		.toUpperCase()
		.replace(/&/g, ' ')
		.split(/[^A-Z0-9]+/)
		.filter(Boolean);

type WdItem = {
	id: string;
	label: string;
	names: Set<string>; // cmp() of label, aliases, enwiki title
	primary: Set<string>; // cmp() of label and enwiki title only
	series: boolean; // "car model series" (Q59773381), i.e. spans generations
	article: string | null;
	makers: Set<string>;
	image: string | null;
	category: string | null;
	parents: Set<string>;
	links: number;
	year: number | null;
};

const split = (s: string | undefined) => (s ? s.split('|').filter(Boolean) : []);

async function loadWikidata() {
	type Row = Record<string, string | undefined>;
	log('wikidata: car models');
	const rows = await sparql<Row>(`
SELECT ?item (SAMPLE(?l) AS ?label) (SAMPLE(?a) AS ?article) (SAMPLE(?img) AS ?image) (SAMPLE(?c) AS ?cat)
  (SAMPLE(?links) AS ?sitelinks) (MIN(YEAR(?inc)) AS ?year)
  (GROUP_CONCAT(DISTINCT STRAFTER(STR(?m), "entity/"); separator="|") AS ?makers)
  (GROUP_CONCAT(DISTINCT STRAFTER(STR(?p), "entity/"); separator="|") AS ?parents)
  (GROUP_CONCAT(DISTINCT STRAFTER(STR(?cls), "entity/"); separator="|") AS ?classes)
WHERE {
  VALUES ?cls { wd:Q3231690 wd:Q59773381 }
  ?item wdt:P31 ?cls .
  OPTIONAL { ?item rdfs:label ?l FILTER(LANG(?l) = "en") }
  OPTIONAL { ?a schema:about ?item; schema:isPartOf <https://en.wikipedia.org/> }
  OPTIONAL { ?item wdt:P18 ?img }
  OPTIONAL { ?item wdt:P373 ?c }
  OPTIONAL { ?item wikibase:sitelinks ?links }
  OPTIONAL { ?item wdt:P571|wdt:P580|wdt:P5444 ?inc }
  OPTIONAL { ?item wdt:P176|wdt:P1716 ?m }
  OPTIONAL { ?item wdt:P179 ?p }
} GROUP BY ?item`);
	log(`wikidata: ${rows.length} items; aliases`);
	const aliases = await sparql<Row>(`
SELECT ?item ?alias WHERE { VALUES ?cls { wd:Q3231690 wd:Q59773381 } ?item wdt:P31 ?cls ; skos:altLabel ?alias FILTER(LANG(?alias) = "en") }`);
	const items = new Map<string, WdItem>();
	for (const r of rows) {
		const id = qid(r.item!);
		const article = r.article
			? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
			: null;
		const label = r.label ?? article ?? id;
		items.set(id, {
			id,
			label,
			names: new Set([cmp(label), ...(article ? [cmp(article.replace(/\s*\(.*\)$/, ''))] : [])]),
			primary: new Set([cmp(label), ...(article ? [cmp(article.replace(/\s*\(.*\)$/, ''))] : [])]),
			series: split(r.classes).includes('Q59773381'),
			article,
			makers: new Set(split(r.makers)),
			image: r.image ? fileName(r.image) : null,
			category: r.cat ?? null,
			parents: new Set(split(r.parents)),
			links: Number(r.sitelinks ?? 0),
			year: r.year ? Number(r.year) : null
		});
	}
	for (const a of aliases) items.get(qid(a.item!))?.names.add(cmp(a.alias!));
	const makerIds = [...new Set([...items.values()].flatMap((i) => [...i.makers]))];
	log(`wikidata: ${makerIds.length} manufacturers/brands`);
	const makers = new Map<
		string,
		{ id: string; label: string; logo: string | null; article: string | null }
	>();
	for (let i = 0; i < makerIds.length; i += 400) {
		const chunk = makerIds.slice(i, i + 400);
		const res = await sparql<Row>(`
SELECT ?m (SAMPLE(?l) AS ?label) (SAMPLE(?logo) AS ?logo) (SAMPLE(?a) AS ?article) WHERE {
  VALUES ?m { ${chunk.map((q) => `wd:${q}`).join(' ')} }
  OPTIONAL { ?m rdfs:label ?l FILTER(LANG(?l) = "en") }
  OPTIONAL { ?m wdt:P154 ?logo }
  OPTIONAL { ?a schema:about ?m; schema:isPartOf <https://en.wikipedia.org/> }
} GROUP BY ?m`);
		for (const r of res) {
			const id = qid(r.m!);
			makers.set(id, {
				id,
				label: r.label ?? id,
				logo: r.logo ? fileName(r.logo) : null,
				article: r.article
					? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
					: null
			});
		}
	}
	const children = new Map<string, WdItem[]>();
	for (const item of items.values())
		for (const p of item.parents) children.set(p, [...(children.get(p) ?? []), item]);
	return { items, makers, children };
}

// ---------------------------------------------------------------- brand logos

type Brand = { slug: string; name: string; wikidata: string | null; hasLogo: boolean };
type LogoItem = {
	id: string;
	label: string;
	article: string | null;
	category: string | null;
	links: number;
	classes: Set<string>;
	industries: Set<string>;
	logos: string[]; // best first
};

/** Registration names whose brand goes by another name on Wikidata/Wikipedia. */
const BRAND_NAMES: Record<string, string[]> = {
	'BMW Alpina': ['Alpina'],
	'Auto Bianchi': ['Autobianchi'],
	'VW-Porsche': ['Volkswagen-Porsche'],
	KGM: ['KG Mobility'],
	SsangYong: ['SsangYong Motor'],
	Dfsk: ['DFSK'],
	Faw: ['FAW', 'FAW Group'],
	Austin: ['Austin Motor Company'],
	Daimler: ['Daimler Company'],
	Nekaf: ['NEKAF'],
	INEOS: ['Ineos Automotive', 'Ineos Grenadier'],
	STI: ['Subaru Tecnica International'],
	'Lynk & Co': ['Lynk & Co'],
	Smart: ['Smart (marque)'],
	'Aston Martin': ['Aston Martin Lagonda'],
	Ram: ['Ram Trucks'],
	Think: ['Think Global'],
	Shelby: ['Shelby American', 'Carroll Shelby International']
};
const AUTO_INDUSTRY = new Set(['Q190117', 'Q1420', 'Q3266850']); // automotive industry, car, ...
const AUTO_CLASS = new Set([
	'Q786820', // automobile manufacturer
	'Q2001319', // automobile marque
	'Q16889133', // vehicle brand? (checked below against the data)
	'Q1137109', // car brand
	'Q19866001' // motor vehicle manufacturer
]);
const CORP_SUFFIX =
	/[\s,]+(motors?|motor cars?|motor car company|motor company|motor corporation|automobiles?|automobili|automobil|automotive|auto|cars?|company|corporation|corp\.?|inc\.?|ltd\.?|limited|llc|gmbh|kg|ag|co\.?|s\.?a\.?|s\.?p\.?a\.?|marque|brand)$/i;
/** "Bentley Motors Limited", "Mini (marque)" and "Škoda Auto" all reduce to the bare brand. */
const brandKey = (s: string) => {
	let t = s.replace(/\s*\([^)]*\)\s*$/, '').trim();
	for (let prev = ''; prev !== t;) {
		prev = t;
		t = t.replace(CORP_SUFFIX, '').trim();
	}
	return cmp(t);
};
const LOGO_WORD = /logo|wordmark|crest|emblem|marque|brand|logotip/i;
const FREE_PD = /public domain|^pd\b|cc0/i;
/** A file is only used when it is clearly a drawn logo, never a photo of a badge on a car. */
const BAD_LOGOS = new Set(['Morris motor logo.jpg']); // photos of badges despite the name
function logoFile(file: string, brand: string[]): Photo | null {
	if (BAD_LOGOS.has(file)) return null;
	const photo = toPhoto(file, 'logo', null, LOGO);
	if (!photo) return null;
	const named = brand.some((k) => cmp(file).includes(k));
	if (/\.svg$/i.test(file)) return photo;
	if (/\.(png|gif)$/i.test(file))
		return LOGO_WORD.test(file) || (FREE_PD.test(photo.license ?? '') && named) ? photo : null;
	if (/\.(jpe?g|webp)$/i.test(file))
		return LOGO_WORD.test(file) && FREE_PD.test(photo.license ?? '') ? photo : null;
	return null;
}

/** Category files must be "<Brand> logo 2021.svg"-like: no F1 team, motorcycle or parent names. */
const PLAIN_WORDS = new Set(
	'LOGO LOGOS LOGOTYPE WORDMARK EMBLEM BRAND MARQUE SYMBOL ICON BADGE CREST NEW OLD CURRENT SIMPLE FLAT COLOR COLOUR COLOURED COLORED BLACK WHITE GREY GRAY RED BLUE SILVER VERTICAL HORIZONTAL STACKED TRANSPARENT AUTOMOBILE AUTOMOBILES AUTOMOBILI CARS CAR MOTOR MOTORS COMPANY SVG PNG'.split(
		' '
	)
);
function plainLogoName(file: string, names: string[]) {
	const t = tokens(file.replace(/\.[a-z]+$/i, ''));
	for (const n of names) {
		const nt = tokens(n);
		const at = t.findIndex((_, i) => nt.every((x, j) => t[i + j] === x));
		if (at < 0) continue;
		const rest = [...t.slice(0, at), ...t.slice(at + nt.length)];
		if (
			rest.some((x) => x.startsWith('LOGO')) &&
			rest.every((x) => PLAIN_WORDS.has(x) || /^\d+$/.test(x))
		)
			return true;
	}
	return false;
}
/** Hand-checked Commons logos for brands whose Wikidata item has none (or only photos). */
const LOGO_FILES: Record<string, string> = {
	// Wikidata prefers the group/corporate logo for these; the car brand has its own.
	BMW: 'BMW logo (gray).svg',
	Renault: 'Renault 2021 Text.svg',
	Seat: 'SEAT Logo from 2017.svg',
	SsangYong: 'Ssangyong company logo.svg',
	Bentley: 'Bentley (Logo 1919).png',
	Caterham: 'Logo of Caterham Cars.png',
	NSU: 'NSU 1960 Logo.svg',
	Scion: 'Scion logo.png',
	Yugo: 'Yugo-logo.svg',
	Mahindra: 'Mahindra Automotive new logo.png'
};
/** Registration names that are not the company Wikidata knows under that name. */
const BRAND_SKIP = new Set(['Iav']);

async function logoItems(ids: string[]) {
	type Row = Record<string, string | undefined>;
	const out = new Map<string, LogoItem>();
	for (let i = 0; i < ids.length; i += 200) {
		const chunk = ids.slice(i, i + 200);
		const rows = await sparql<Row>(`
SELECT ?item (SAMPLE(?l) AS ?label) (SAMPLE(?a) AS ?article) (SAMPLE(?c) AS ?cat) (SAMPLE(?sl) AS ?links)
  (GROUP_CONCAT(DISTINCT STRAFTER(STR(?cls), "entity/"); separator="|") AS ?classes)
  (GROUP_CONCAT(DISTINCT STRAFTER(STR(?ind), "entity/"); separator="|") AS ?industries)
  (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(YEAR(?start)), ""), ";", IF(BOUND(?end), "end", "")); separator="|") AS ?logos)
WHERE {
  VALUES ?item { ${chunk.map((q) => `wd:${q}`).join(' ')} }
  OPTIONAL { ?item rdfs:label ?l FILTER(LANG(?l) = "en") }
  OPTIONAL { ?a schema:about ?item; schema:isPartOf <https://en.wikipedia.org/> }
  OPTIONAL { ?item wdt:P373 ?c }
  OPTIONAL { ?item wikibase:sitelinks ?sl }
  OPTIONAL { ?item wdt:P31 ?cls }
  OPTIONAL { ?item wdt:P452 ?ind }
  OPTIONAL {
    ?item p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank .
    FILTER(?rank != wikibase:DeprecatedRank)
    OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end }
  }
} GROUP BY ?item`);
		for (const r of rows) {
			const id = qid(r.item!);
			const logos = split(r.logos)
				.map((x) => {
					const [uri, rank, start, end] = x.split(';');
					return {
						file: fileName(uri),
						current: !end,
						score:
							(rank.endsWith('PreferredRank') ? 10000 : 0) + (end ? 0 : 5000) + (Number(start) || 0)
					};
				})
				.sort((a, b) => b.score - a.score);
			// Logos with an end date are history; only used when nothing current is known.
			const current = logos.filter((x) => x.current);
			out.set(id, {
				id,
				label: r.label ?? id,
				article: r.article
					? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
					: null,
				category: r.cat ?? null,
				links: Number(r.links ?? 0),
				classes: new Set(split(r.classes)),
				industries: new Set(split(r.industries)),
				logos: [...new Set((current.length ? current : logos).map((x) => x.file))]
			});
		}
	}
	return out;
}

/** The logo= file of an enwiki infobox, only if it lives on Commons (local files are fair use). */
async function infoboxLogo(article: string) {
	const res = (await mw('en.wikipedia.org', {
		action: 'query',
		prop: 'revisions',
		rvprop: 'content',
		rvslots: 'main',
		rvsection: '0',
		redirects: '1',
		titles: article
	})) as { query?: { pages?: { revisions?: { slots?: { main?: { content?: string } } }[] }[] } };
	const text = res.query?.pages?.[0]?.revisions?.[0]?.slots?.main?.content ?? '';
	const m = text.match(
		/\|\s*logo\s*=\s*(?:\[\[)?(?:File:|Image:)?\s*([^|\]\n{}<]+?\.(?:svg|png|gif))/i
	);
	return m ? m[1].trim().replace(/_/g, ' ') : null;
}

async function findLogos(
	brands: Brand[],
	makers: Map<string, { id: string; label: string }>
): Promise<{
	logos: Map<
		string,
		{ photo: Photo; wikidata: string | null; article: string | null; via: string }
	>;
	brandPick: Map<string, { id: string; label: string; article: string | null }>;
}> {
	brands = brands.filter((b) => !BRAND_SKIP.has(b.name));
	log(`logos: looking up ${brands.length} brands`);
	const names = new Map(brands.map((b) => [b.slug, [b.name, ...(BRAND_NAMES[b.name] ?? [])]]));
	const keys = new Map(brands.map((b) => [b.slug, [...new Set(names.get(b.slug)!.map(brandKey))]]));
	// Candidates: Wikidata search hits for every name, the stored item and car-model makers.
	const cands = new Map<string, Set<string>>();
	for (const b of brands) {
		const set = new Set<string>();
		if (b.wikidata) set.add(b.wikidata);
		for (const m of makers.values())
			if (keys.get(b.slug)!.includes(brandKey(m.label))) set.add(m.id);
		for (const n of names.get(b.slug)!) {
			const res = (await mw('www.wikidata.org', {
				action: 'wbsearchentities',
				search: n,
				language: 'en',
				type: 'item',
				limit: '20'
			})) as { search?: { id: string }[] };
			for (const hit of res.search ?? []) set.add(hit.id);
		}
		cands.set(b.slug, set);
	}
	const items = await logoItems([...new Set([...cands.values()].flatMap((s) => [...s]))]);
	await saveApiCache(true);

	const logos = new Map<
		string,
		{ photo: Photo; wikidata: string | null; article: string | null; via: string }
	>();
	const brandPick = new Map<string, { id: string; label: string; article: string | null }>();
	const ranked = new Map<string, LogoItem[]>();
	for (const b of brands) {
		const k = keys.get(b.slug)!;
		const ok = [...cands.get(b.slug)!]
			.map((id) => items.get(id))
			.filter((i): i is LogoItem => !!i)
			// Named like the brand (the article only counts for items without an English label).
			.filter((i) => k.includes(brandKey(i.label === i.id && i.article ? i.article : i.label)))
			.filter(
				(i) =>
					makers.has(i.id) ||
					[...i.industries].some((x) => AUTO_INDUSTRY.has(x)) ||
					[...i.classes].some((x) => AUTO_CLASS.has(x))
			);
		const score = (i: LogoItem) =>
			(makers.has(i.id) ? 4 : 0) +
			([...i.classes].some((x) => AUTO_CLASS.has(x)) ? 2 : 0) +
			([...i.industries].some((x) => AUTO_INDUSTRY.has(x)) ? 2 : 0) +
			(k.includes(cmp(i.label)) ? 1 : 0) +
			Math.min(i.links, 400) / 1000;
		ok.sort((a, b) => score(b) - score(a));
		ranked.set(b.slug, ok);
		if (ok[0]) brandPick.set(b.slug, ok[0]);
	}
	// 1. P154 of the brand's own item (never a parent company's).
	await imageInfo(
		[...ranked.values()].flatMap((l) => l.flatMap((i) => i.logos)),
		LOGO
	);
	// 0. Hand-checked files.
	await imageInfo(Object.values(LOGO_FILES), LOGO);
	for (const b of brands) {
		const file = LOGO_FILES[b.name];
		const photo = file ? toPhoto(file, 'logo', null, LOGO) : null;
		const i = brandPick.get(b.slug);
		if (photo)
			logos.set(b.slug, {
				photo,
				wikidata: i?.id ?? null,
				article: i?.article ?? null,
				via: 'curated'
			});
	}
	// A file named after the brand wins over e.g. a parent company's logo on a shared item.
	for (const b of brands)
		for (const named of [true, false]) {
			if (logos.has(b.slug)) break;
			for (const i of ranked.get(b.slug)!) {
				const photo = i.logos
					.filter((f) => !named || keys.get(b.slug)!.some((k) => cmp(f).includes(k)))
					.map((f) => logoFile(f, keys.get(b.slug)!))
					.find(Boolean);
				if (photo) {
					logos.set(b.slug, { photo, wikidata: i.id, article: i.article, via: `P154 ${i.id}` });
					break;
				}
			}
		}
	// 2. The logo in the brand article's infobox, when that file is on Commons.
	for (const b of brands) {
		if (logos.has(b.slug)) continue;
		for (const i of ranked
			.get(b.slug)!
			.filter((i) => i.article)
			.slice(0, 2)) {
			const file = await infoboxLogo(i.article!);
			if (!file) continue;
			await imageInfo([file], LOGO);
			const photo = logoFile(file, keys.get(b.slug)!);
			if (photo && (LOGO_WORD.test(file) || keys.get(b.slug)!.some((x) => cmp(file).includes(x)))) {
				logos.set(b.slug, {
					photo,
					wikidata: i.id,
					article: i.article,
					via: `infobox ${i.article}`
				});
				break;
			}
		}
	}
	// 3. "<Brand> logos" on Commons: a logo file named after the brand, newest year first.
	for (const b of brands) {
		if (logos.has(b.slug)) continue;
		const best = ranked.get(b.slug)![0];
		if (!best) continue; // without an automotive item the category could be anyone's
		// The item's own Commons category is trusted; a category merely named like the brand
		// (shared with e.g. a motorcycle maker) only if the brand's article uses the file.
		const used = best.article
			? new Set((await articleImages([best.article])).get(best.article) ?? [])
			: new Set<string>();
		const cats = [...new Set([best.category, ...names.get(b.slug)!].filter(Boolean))] as string[];
		for (const c of cats) {
			const files = (await categoryFiles(`${c} logos`))
				.filter((f) => /\.(svg|png)$/i.test(f) && /logo/i.test(f))
				.filter((f) => plainLogoName(f, names.get(b.slug)!))
				.filter((f) => c === best.category || used.has(f))
				.sort(
					(x, y) =>
						Math.max(0, ...(y.match(/(?:19|20)\d\d/g) ?? []).map(Number)) -
							Math.max(0, ...(x.match(/(?:19|20)\d\d/g) ?? []).map(Number)) ||
						Number(/\.svg$/i.test(y)) - Number(/\.svg$/i.test(x)) ||
						x.length - y.length
				);
			await imageInfo(files.slice(0, 5), LOGO);
			const file = files.slice(0, 5).find((f) => logoFile(f, keys.get(b.slug)!));
			if (file) {
				const photo = logoFile(file, keys.get(b.slug)!)!;
				logos.set(b.slug, {
					photo,
					wikidata: best.id,
					article: best.article,
					via: `Category:${c} logos`
				});
				break;
			}
		}
	}
	await saveApiCache(true);
	log(`logos: ${logos.size}/${brands.length} brands`);
	return { logos, brandPick };
}

// ---------------------------------------------------------------- matching

type Target = {
	id: number | null;
	brand: string;
	brandSlug: string;
	model: string;
	slug: string;
	nl: number;
	photos: number;
};
type Match = { item: WdItem; via: string };

/** Popular models whose Dutch/registration name differs from the Wikipedia one. */
const ALIASES: Record<string, string> = {
	'MINI|Cooper': 'Mini Hatch',
	'MINI|Countryman Cooper': 'Mini Countryman',
	'MINI|Clubman Cooper': 'Mini Clubman',
	'Renault|Megane Scenic': 'Renault Scénic',
	'Volvo|EX40': 'Volvo XC40',
	'Volvo|EC40': 'Volvo C40',
	'Hyundai|Atos-Prime': 'Hyundai Atos',
	'Suzuki|Wagon': 'Suzuki Wagon R',
	'Lynk & Co|01': 'Lynk & Co 01',
	"Kia|Cee'd": 'Kia Ceed'
};

function matcher(wd: Awaited<ReturnType<typeof loadWikidata>>) {
	const byName = new Map<string, WdItem[]>();
	for (const item of wd.items.values())
		for (const n of item.names) byName.set(n, [...(byName.get(n) ?? []), item]);
	const brandMakers = new Map<string, Set<string>>();
	const makersOf = (brand: string) => {
		let set = brandMakers.get(brand);
		if (set) return set;
		const bt = tokens(brand);
		set = new Set(
			[...wd.makers.values()]
				.filter((m) => {
					const mt = tokens(m.label);
					return bt.length && bt.every((t, i) => mt[i] === t);
				})
				.map((m) => m.id)
		);
		brandMakers.set(brand, set);
		return set;
	};
	// Prefer the item that is *called* this (not just aliased), spans generations, has an
	// article, is a series root and is well known.
	const score = (i: WdItem, names: string[]) =>
		(names.some((n) => i.primary.has(n)) ? 4000 : 0) +
		(i.series ? 800 : 0) +
		Math.min(10, wd.children.get(i.id)?.length ?? 0) * 60 +
		(i.article ? 1000 : 0) +
		(i.parents.size ? 0 : 500) +
		i.links;
	return {
		makersOf,
		match(t: Target): Match | null {
			const makers = makersOf(t.brand);
			const alias = ALIASES[`${t.brand}|${t.model}`];
			const bm = [
				...(byName.get(cmp(`${t.brand} ${t.model}`)) ?? []),
				...(alias ? (byName.get(cmp(alias)) ?? []) : [])
			];
			const m = byName.get(cmp(t.model)) ?? [];
			const ok = [
				...bm.filter((i) => !i.makers.size || [...i.makers].some((x) => makers.has(x))),
				...m.filter((i) => [...i.makers].some((x) => makers.has(x)))
			];
			// An item whose maker is known but belongs to someone else is never used.
			if (!ok.length) return null;
			const names = [cmp(`${t.brand} ${t.model}`), cmp(t.model), ...(alias ? [cmp(alias)] : [])];
			const best = [...new Set(ok)].sort((a, b) => score(b, names) - score(a, names))[0];
			return { item: best, via: bm.includes(best) ? 'brand+model name' : 'model name + maker' };
		}
	};
}

// ---------------------------------------------------------------- photos

export type Photo = {
	file: string;
	kind: 'lead' | 'generation' | 'article' | 'commons' | 'logo';
	caption: string | null;
	thumb: string;
	width: number;
	height: number;
	author: string | null;
	license: string | null;
	licenseUrl: string | null;
};

const BAD_FILE =
	/\.(svg|gif|tiff?|pdf|djvu|webm|ogv)$|logo|emblem|badge|wordmark|map\b|karte|locator|diagram|chart|graph|schema|schematic|drawing|sketch|blueprint|patent|icon|flag|coat of arms|wappen|sign\b|signature|symbol|plan\b|cutaway|silhouette|commons-|wiki|stub|poster|advert|brochure|stamp|timeline|people|crowd|queue|in line|factory|assembly|production line|plant\b|werk\b|museum entrance|dealership|showroom sign|\btoys?\b|model car|lego|scale model|diecast/i;
const stripHtml = (s: string) =>
	s
		.replace(/<[^>]+>/g, ' ')
		.replace(/&amp;/g, '&')
		.replace(/&quot;/g, '"')
		.replace(/&#0?39;/g, "'")
		.replace(/&nbsp;/g, ' ')
		.replace(/\s+/g, ' ')
		.trim();

const infoCache = new Map<string, ImageInfo | null>();
async function imageInfo(files: string[], width = THUMB) {
	const todo = [...new Set(files)].filter((f) => !infoCache.has(`${width}|${f}`));
	for (let i = 0; i < todo.length; i += 50) {
		const chunk = todo.slice(i, i + 50);
		const res = await mw('commons.wikimedia.org', {
			action: 'query',
			prop: 'imageinfo',
			iiprop: 'url|size|mime|extmetadata',
			iiurlwidth: String(width),
			iiextmetadatafilter: `Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree${
				width === LOGO ? '|Restrictions' : ''
			}`,
			titles: chunk.map((f) => `File:${f}`).join('|')
		});
		const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
		for (const p of res.query?.pages ?? []) {
			const original = (norm.get(p.title) ?? p.title).replace(/^File:/, '');
			infoCache.set(`${width}|${original}`, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]);
		}
		for (const f of chunk)
			if (!infoCache.has(`${width}|${f}`)) infoCache.set(`${width}|${f}`, null);
	}
}
function toPhoto(
	file: string,
	kind: Photo['kind'],
	caption: string | null,
	width = THUMB
): Photo | null {
	const info = infoCache.get(`${width}|${file}`);
	if (!info?.thumburl) return null;
	const meta = info.extmetadata ?? {};
	if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
	let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
	if (!license || /fair use|non-free/i.test(license)) return null;
	// Most logos are public domain only as copyright goes (PD-textlogo) but still trademarks.
	if (kind === 'logo' && /trademark/i.test(meta.Restrictions?.value ?? ''))
		license += ' · trademark';
	if (kind !== 'logo') {
		if (!/^image\/(jpeg|png|webp)$/.test(info.mime)) return null;
		if (info.width < THUMB || info.height < 400) return null;
		if (info.width / info.height > 3 || info.height / info.width > 1.6) return null;
	}
	const author = meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null;
	return {
		file,
		kind,
		caption,
		// Canonical host, no tracking query: https://upload.wikimedia.org/.../960px-<file>
		thumb: info.thumburl
			.replace(/\?.*$/, '')
			.replace('//thumb.wikimedia.org/', '//upload.wikimedia.org/'),
		width: info.width,
		height: info.height,
		author,
		license,
		licenseUrl: meta.LicenseUrl?.value ?? null
	};
}

async function articleImages(titles: string[]) {
	const out = new Map<string, string[]>();
	const todo = [...new Set(titles)];
	for (let i = 0; i < todo.length; i += 50) {
		const chunk = todo.slice(i, i + 50);
		let cont: Record<string, string> = {};
		for (let guard = 0; guard < 30; guard++) {
			const res = await mw('en.wikipedia.org', {
				action: 'query',
				prop: 'images',
				imlimit: 'max',
				redirects: '1',
				titles: chunk.join('|'),
				...cont
			});
			const norm = new Map<string, string>();
			for (const n of [
				...(res.query?.normalized ?? []),
				...((res.query as { redirects?: { from: string; to: string }[] })?.redirects ?? [])
			])
				norm.set(n.to, norm.get(n.from) ?? n.from);
			for (const p of res.query?.pages ?? []) {
				const key = norm.get(p.title) ?? p.title;
				out.set(key, [
					...(out.get(key) ?? []),
					...(p.images ?? []).map((im) => im.title.replace(/^File:/, ''))
				]);
			}
			if (!res.continue) break;
			cont = res.continue;
		}
	}
	return out;
}

async function categoryFiles(category: string): Promise<string[]> {
	const res = await mw('commons.wikimedia.org', {
		action: 'query',
		list: 'categorymembers',
		cmtitle: `Category:${category}`,
		cmtype: 'file',
		cmlimit: '60'
	});
	return ((res.query as { categorymembers?: { title: string }[] })?.categorymembers ?? []).map(
		(m) => m.title.replace(/^File:/, '')
	);
}

const GENERIC = new Set(
	'SEDAN SALOON COUPE HATCHBACK WAGON ESTATE TOURING VARIANT CONVERTIBLE CABRIOLET CABRIO ROADSTER SPIDER SPYDER VAN PLUS SPORT SPORTS SERIES CLASS MODEL GRAND NEW SUPER LONG SHORT DOOR DOORS FACELIFT GENERATION FIRST SECOND THIRD FOURTH FIFTH SIXTH SEVENTH EIGHTH NINTH CAR AUTO HYBRID ELECTRIC'.split(
		' '
	)
);
/** Words that identify a generation ("E46", "W205", "Mk7", "XP210"), not the brand or body style. */
function distinctive(label: string, t: Target): string[] {
	const skip = new Set([...tokens(t.brand), ...tokens(t.model)]);
	return tokens(label).filter(
		(w) =>
			!skip.has(w) &&
			!GENERIC.has(w) &&
			(/\d/.test(w) ? w.length >= 3 : w.length >= 4) &&
			!/^\d{4}$/.test(w)
	);
}

/**
 * Does the file name mention this model? ("Peugeot 208 GT.jpg" for 208, "BMW E46.jpg" for the
 * 3 Series via its generation codes). Files that name a longer sibling model of the same brand
 * ("Golf Sportsvan" on the Golf, "Mustang Mach-E" on the Mustang) are rejected.
 */
function mentions(
	file: string,
	t: Target,
	item: WdItem,
	genWords: string[],
	siblings: string[]
): boolean {
	const f = cmp(file);
	if (siblings.some((s) => f.includes(s))) return false;
	const model = cmp(t.model);
	const short = model.length < 3 || /^\d+$/.test(model);
	if (f.includes(model) && (!short || f.includes(cmp(t.brand)) || f.includes(cmp(item.label))))
		return true;
	if (f.includes(cmp(item.label))) return true;
	return genWords.some((w) => f.includes(w));
}

async function photosFor(
	targets: Target[],
	wd: Awaited<ReturnType<typeof loadWikidata>>,
	matches: Map<Target, Match>,
	all: Target[]
) {
	const articles = await articleImages(
		[...matches.values()].map((m) => m.item.article).filter((a): a is string => !!a)
	);
	const result = new Map<Target, Photo[]>();
	const byBrand = new Map<string, Target[]>();
	for (const t of all) byBrand.set(t.brandSlug, [...(byBrand.get(t.brandSlug) ?? []), t]);
	let done = 0;
	for (const t of targets) {
		const m = matches.get(t);
		if (!m) continue;
		const { item } = m;
		const children = wd.children.get(item.id) ?? [];
		const genWords = [...new Set(children.flatMap((c) => distinctive(c.label, t)))];
		const own = cmp(t.model);
		const siblings = (byBrand.get(t.brandSlug) ?? [])
			.map((o) => cmp(o.model))
			.filter((o) => o !== own && o.startsWith(own));
		const ok = (f: string) => !BAD_FILE.test(f) && mentions(f, t, item, genWords, siblings);
		const gens = children
			.filter((g) => g.image)
			.sort(
				(a, b) =>
					(b.year ?? 0) - (a.year ?? 0) || b.label.localeCompare(a.label, 'en', { numeric: true })
			);
		const picks: { file: string; kind: Photo['kind']; caption: string | null }[] = [];
		if (item.image) picks.push({ file: item.image, kind: 'lead', caption: null });
		for (const g of gens
			.filter((g) => ok(g.image!) || distinctive(g.label, t).some((w) => cmp(g.image!).includes(w)))
			.slice(0, 8))
			picks.push({ file: g.image!, kind: 'generation', caption: g.label });
		const fromArticle = (item.article ? (articles.get(item.article) ?? []) : []).filter(ok);
		for (const f of fromArticle) picks.push({ file: f, kind: 'article', caption: null });
		const unique = () => [...new Map(picks.map((p) => [p.file, p])).values()];
		await imageInfo(unique().map((p) => p.file));
		let photos = unique()
			.map((p) => toPhoto(p.file, p.kind, p.caption))
			.filter((p): p is Photo => !!p);
		if (photos.length < MAX_PHOTOS && item.category) {
			const files = (await categoryFiles(item.category)).filter(ok);
			// Spread picks across the (alphabetical) category instead of taking the first few.
			const want = MAX_PHOTOS * 2;
			const step = Math.max(1, files.length / want);
			const spread = Array.from(
				{ length: Math.min(want, files.length) },
				(_, i) => files[Math.floor(i * step)]
			);
			await imageInfo(spread);
			const seen = new Set(photos.map((p) => p.file));
			for (const f of spread) {
				if (seen.has(f)) continue;
				const p = toPhoto(f, 'commons', null);
				if (p) photos.push(p);
				seen.add(f);
			}
		}
		photos = photos.slice(0, MAX_PHOTOS);
		if (photos.length) result.set(t, photos);
		if (++done % 100 === 0) log(`photos: ${done}/${matches.size} models`);
	}
	return result;
}

// ---------------------------------------------------------------- main

async function loadTargets(): Promise<{
	targets: Target[];
	brands: Brand[];
}> {
	if (modelsFile) {
		const built = JSON.parse(await readFile(modelsFile, 'utf8')) as {
			slug: string;
			name: string;
			models: { slug: string; name: string; nlCount: number; variants: number }[];
		}[];
		return {
			targets: built.flatMap((b) =>
				b.models.map((m) => ({
					id: null,
					brand: b.name,
					brandSlug: b.slug,
					model: m.name,
					slug: m.slug,
					nl: m.nlCount + m.variants * 50,
					photos: 0
				}))
			),
			brands: built.map((b) => ({ slug: b.slug, name: b.name, wikidata: null, hasLogo: false }))
		};
	}
	const sql = db();
	const rows = await sql<
		{
			id: number;
			brand: string;
			brand_slug: string;
			model: string;
			slug: string;
			nl: number;
			photos: number;
		}[]
	>`
		select m.id, b.name as brand, b.slug as brand_slug, m.name as model, m.slug,
			m.nl_count + m.variant_count * 50 as nl, m.photo_count as photos
		from cars.models m join cars.brands b on b.id = m.brand_id`;
	const brands = (
		await sql<{ slug: string; name: string; wikidata: string | null; logo: string | null }[]>`
			select slug, name, wikidata, logo from cars.brands`
	).map((b) => ({ slug: b.slug, name: b.name, wikidata: b.wikidata, hasLogo: !!b.logo }));
	return {
		targets: rows.map((r) => ({
			id: r.id,
			brand: r.brand,
			brandSlug: r.brand_slug,
			model: r.model,
			slug: r.slug,
			nl: r.nl,
			photos: r.photos
		})),
		brands
	};
}

let sqlClient: ReturnType<typeof postgres> | null = null;
function db() {
	if (!process.env.DB_URL) throw new Error('Set DB_URL (or use --dry-run --models <built.json>)');
	sqlClient ??= postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
	return sqlClient;
}

async function enrich(file: string) {
	const report = JSON.parse(await readFile(file, 'utf8')) as {
		newModels?: { url: string; photo?: string | null }[];
	};
	const sql = db();
	for (const m of report.newModels ?? []) {
		const [, brand, model] =
			new URL(m.url).pathname.match(/\/trackers\/cars\/([^/]+)\/([^/]+)/) ?? [];
		const [row] = await sql<{ photo: string | null }[]>`
			select m.photo from cars.models m join cars.brands b on b.id = m.brand_id
			where b.slug = ${brand} and m.slug = ${model}`;
		m.photo = row?.photo ?? null;
	}
	console.log(JSON.stringify(report, null, 2));
}

async function main() {
	await loadApiCache();
	if (enrichFile && !flag('--fetch')) {
		// Enrich only: look up photos already written.
		await enrich(enrichFile);
		return;
	}
	const loaded = await loadTargets();
	const { brands } = loaded;
	// --missing: only models without photos are (re)matched; siblings still come from all models.
	const targets = loaded.targets.filter((t) => !missingOnly || t.photos === 0);
	targets.sort((a, b) => b.nl - a.nl);
	const wd = await loadWikidata();
	const { match } = matcher(wd);
	const matches = new Map<Target, Match>();
	for (const t of targets) {
		const m = match(t);
		if (m) matches.set(t, m);
	}
	log(`matched ${matches.size}/${targets.length} models to Wikidata`);
	const photos = logosOnly
		? new Map<Target, Photo[]>()
		: await photosFor(targets, wd, matches, loaded.targets);

	const { logos, brandPick } = await findLogos(
		brands.filter((b) => !missingOnly || !b.hasLogo),
		wd.makers
	);
	await saveApiCache(true);

	if (logosOnly && outFile)
		await writeFile(
			outFile,
			JSON.stringify(
				brands.map((b) => ({
					brand: b.name,
					item: brandPick.get(b.slug)?.id ?? null,
					label: brandPick.get(b.slug)?.label ?? null,
					logo: logos.get(b.slug)?.photo.file ?? null,
					thumb: logos.get(b.slug)?.photo.thumb ?? null,
					via: logos.get(b.slug)?.via ?? null,
					license: logos.get(b.slug)?.photo.license ?? null
				})),
				null,
				1
			)
		);
	const total = [...photos.values()].reduce((a, p) => a + p.length, 0);
	log(`photos: ${photos.size} models with photos, ${total} photos, ${logos.size} brand logos`);
	if (outFile && !logosOnly)
		await writeFile(
			outFile,
			JSON.stringify(
				targets
					.filter((t) => matches.has(t))
					.map((t) => ({
						brand: t.brand,
						model: t.model,
						item: matches.get(t)!.item.id,
						label: matches.get(t)!.item.label,
						via: matches.get(t)!.via,
						photos: (photos.get(t) ?? []).map((p) => `${p.kind}: ${p.file}`)
					}))
			)
		);
	if (dryRun) return;

	const sql = db();
	await sql.begin(async (tx) => {
		const ids = logosOnly ? [] : targets.map((t) => t.id!).filter(Boolean);
		// Replace photos for every model we looked at (including ones that lost their match).
		await tx`delete from cars.photos where model_id = any(${ids})`;
		await tx`update cars.models set wikidata = null, wikipedia = null, photo = null, photo_count = 0, photos_checked_at = now() where id = any(${ids})`;
		for (const t of logosOnly ? [] : targets) {
			const m = matches.get(t);
			if (!m || !t.id) continue;
			const list = photos.get(t) ?? [];
			await tx`update cars.models set wikidata = ${m.item.id}, wikipedia = ${m.item.article},
				photo = ${list[0]?.thumb ?? null}, photo_count = ${list.length} where id = ${t.id}`;
			if (list.length)
				await tx`insert into cars.photos ${tx(
					list.map((p, i) => ({
						model_id: t.id,
						position: i,
						kind: p.kind,
						file: p.file,
						caption: p.caption,
						thumb: p.thumb,
						width: p.width,
						height: p.height,
						author: p.author,
						license: p.license,
						license_url: p.licenseUrl
					}))
				)}`;
		}
		{
			// --missing only fills brands that have no logo yet (e.g. brand new ones).
			for (const b of brands) {
				const l = logos.get(b.slug);
				await tx`update cars.brands set wikidata = ${l?.wikidata ?? brandPick.get(b.slug)?.id ?? null},
					wikipedia = ${l?.article ?? brandPick.get(b.slug)?.article ?? null},
					logo = ${l?.photo.thumb ?? null}, logo_file = ${l?.photo.file ?? null},
					logo_license = ${l?.photo.license ?? null}, logo_author = ${l?.photo.author ?? null}
					where slug = ${b.slug} and (${!missingOnly} or logo is null)`;
			}
		}
		// Brand hero photo: the lead photo of its most common model that has one.
		await tx`update cars.brands b set photo = (
			select m.photo from cars.models m where m.brand_id = b.id and m.photo is not null
			order by m.nl_count desc, m.variant_count desc limit 1)`;
	});
	log('written');
	if (enrichFile) await enrich(enrichFile);
	await sql.end();
}

await main();
await sqlClient?.end();