cars: Wikimedia Commons photos (cars.photos, card heroes, model gallery + lightbox, brand hero/logo), cars:photos + cars:refresh with --report-new, fold duplicate names (Golf VI, Peugot, Mustang Mach 1, Model 3/Y, i10/i30, Ceed, Lynk & Co)

scripts/cars-import.ts

@@ -5,6 +5,7 @@
 //   bun scripts/cars-import.ts --cache /tmp/cars     reuse raw downloads from that folder
 //   bun scripts/cars-import.ts --dry-run --out x.json   also dump the built catalogue
 //   bun scripts/cars-import.ts --renumber           reassign Dex numbers from #1 (oldest first)
+//   bun scripts/cars-import.ts --report-new         JSON of brands/models new since the last run on stdout
 //
 // Sources (no API keys):
 //  - fueleconomy.gov vehicles.csv: every US EPA-tested car since 1984 (make, base model,
@@ -31,6 +32,11 @@ const dryRun = args.includes('--dry-run');
 const renumber = args.includes('--renumber');
 const cacheDir = args.includes('--cache') ? args[args.indexOf('--cache') + 1] : null;
 const outFile = args.includes('--out') ? args[args.indexOf('--out') + 1] : null;
+// --report-new: print a JSON summary of brands/models that did not exist before this run to
+// stdout (all progress logging moves to stderr so stdout stays machine-readable).
+const reportNew = args.includes('--report-new');
+const SITE = 'https://stanrunge.dev/trackers/cars';
+const log = (...a: unknown[]) => (reportNew ? console.error(...a) : console.log(...a));
 
 const FE_URL = 'https://www.fueleconomy.gov/feg/epadata/vehicles.csv';
 const RDW = 'https://opendata.rdw.nl/resource';
@@ -150,7 +156,7 @@ type RdwFuel = {
 
 async function loadSources() {
 	const fe = await cached('fe.json', async () => {
-		console.log('fetching fueleconomy.gov vehicles.csv');
+		log('fetching fueleconomy.gov vehicles.csv');
 		const res = await fetch(FE_URL, { signal: AbortSignal.timeout(300_000) });
 		if (!res.ok) throw new Error(`fueleconomy.gov ${res.status}`);
 		return parseCsv(await res.text());
@@ -162,7 +168,7 @@ async function loadSources() {
 		...(having ? { $having: having } : {}),
 		$limit: '2000000'
 	});
-	console.log('querying RDW aggregates');
+	log('querying RDW aggregates');
 	const [models, years, bodies] = await Promise.all([
 		cached('rdw-models.json', () =>
 			getJson<RdwModel[]>(
@@ -385,10 +391,35 @@ function cleanTradeName(raw: string, make: string): string {
 		else if (s.startsWith(`${make} `) && s.length > make.length + 1) s = s.slice(make.length + 1);
 		else break;
 	}
+	// Misspelt make as first word ("PEUGOT 205" under PEUGEOT).
+	const [w0, ...restW] = s.split(' ');
+	const mk0 = make.replace(/[^A-Z]/g, '');
+	if (
+		restW.length &&
+		mk0.length >= 5 &&
+		w0 !== mk0 &&
+		w0.slice(0, 4) === mk0.slice(0, 4) &&
+		Math.abs(w0.length - mk0.length) <= 1
+	)
+		s = restW.join(' ');
 	s = s.replace(/^\d{5,}\s+/, ''); // leading RDW type code, e.g. "155221 GOLF CABRIOLET"
 	return s.replace(/[-\s]+$/, '').trim();
 }
 
+/** Hand fixes for trade names that put the trim first ("MACH 1 MUSTANG" is a Mustang). */
+const TRADE_FIXES: Record<string, [RegExp, string][]> = {
+	FORD: [
+		[/^MACH ?(1|I)\b ?MUSTANG\b/, 'MUSTANG MACH 1'],
+		[/^MUSTANG( II)? MACH ?(1|I|ONE)\b/, 'MUSTANG$1']
+	],
+	LYNKCO: [[/^LYNK ?& ?CO ?/, '']],
+	KIA: [[/^CEE ?'? ?D\b/, "CEE'D"]]
+};
+function tradeFix(makeK: string, name: string): string {
+	for (const [re, to] of TRADE_FIXES[makeK] ?? []) name = name.replace(re, to);
+	return name.trim();
+}
+
 /** Brand-specific folding of engine-coded names into the model family. */
 function familyRule(makeK: string, name: string): string | null {
 	if (makeK === 'MERCEDESBENZ') {
@@ -650,7 +681,7 @@ function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
 		if (!r.handelsbenaming || /^(ONBEKEND|NIET|UNKNOWN|-|\?)/i.test(r.handelsbenaming)) continue;
 		if (!brands.has(mk) && (makeTotals.get(mk) ?? 0) < MIN_NL_MAKE) continue;
 		if (/^(ONBEKEND|DIVERSEN|ZELFBOUW|KIT ?CAR|EIGENBOUW)/.test(mk)) continue;
-		const clean = cleanTradeName(r.handelsbenaming, r.merk.toUpperCase());
+		const clean = tradeFix(mk, cleanTradeName(r.handelsbenaming, r.merk.toUpperCase()));
 		if (!clean || !/[A-Z0-9]/.test(clean) || clean.includes('*') || /^\d{5,}$/.test(clean))
 			continue;
 		rows.push({ ...r, clean, makeK: mk });
@@ -658,14 +689,29 @@ function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
 	// A trade name folds into a shorter family name when what follows is trim, body, engine
 	// or code noise ("205 GTI 1.9 U9" -> 205, "Astra Sports Tourer" -> Astra) but not when it
 	// names a different car ("Yaris Cross", "Corolla Verso" stay their own model).
-	const isNoise = (w: string) =>
-		/[\d./]/.test(w) || w.length === 1 || TRIM_WORDS.has(w.replace(/[^A-Z]/g, ''));
+	const ROMAN = /^(II|III|IV|V|VI|VII|VIII|IX|X)$/;
+	const isNoise = (w: string, prev = '') =>
+		/[\d./]/.test(w) ||
+		w.length === 1 ||
+		TRIM_WORDS.has(w.replace(/[^A-Z]/g, '')) ||
+		(ROMAN.test(w) && !/^(MK|MARK|SERIE|SERIES|SPORT)$/.test(prev));
 	// Words up to the first noise word: "IX1 EDRIVE20" -> "IX1", "ID.7 TOURER PRO" -> "ID.7".
+	// A short or generic first word needs the next one: "MODEL 3", "I 10", "CEE D", "P 1800".
+	const STEM = /^([A-Z]{1,3}|MODEL|CLASS|TYPE|SERIE|SERIES)$/;
+	const needsNext = (words: string[]) =>
+		words.length > 1 &&
+		STEM.test(words[0]) &&
+		!/[./]/.test(words[1]) &&
+		(words[0].length === 1 || words[1].length <= 2);
 	const core = (clean: string) => {
 		const words = clean.split(' ');
-		let i = 1;
-		while (i < words.length && !isNoise(words[i])) i++;
-		return words.slice(0, i).join(' ');
+		let i = needsNext(words) ? 2 : 1;
+		while (i < words.length && !isNoise(words[i], words[i - 1])) i++;
+		const out = words.slice(0, i);
+		// "IX 20" -> "IX20", "TR 3" -> "TR3", "P 1800" -> "P1800" (but "MODEL 3" keeps its space).
+		if (i >= 2 && words[0].length <= 2 && /^\d/.test(words[1]))
+			out.splice(0, 2, words[0] + words[1]);
+		return out.join(' ');
 	};
 	const candidates = new Map<string, Map<string, string>>(); // makeK -> cmpKey -> display upper name
 	const addCandidate = (mk: string, name: string) => {
@@ -693,7 +739,7 @@ function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
 		const words = clean.split(' ');
 		for (let i = 1; i < words.length; i++) {
 			const hit = cands.get(cmpKey(words.slice(0, i).join(' ')));
-			if (hit && isNoise(words[i])) return hit;
+			if (hit && isNoise(words[i], words[i - 1])) return hit;
 		}
 		return cands.get(cmpKey(clean)) ?? cands.get(cmpKey(core(clean))) ?? null;
 	}
@@ -906,12 +952,43 @@ function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
 
 // ---------------------------------------------------------------- writing
 
+type NewModel = {
+	brand: string;
+	model: string;
+	dex: number;
+	url: string;
+	firstYear: number | null;
+	lastYear: number | null;
+	fuels: string[];
+	bodies: string[];
+	powerKw: number | null;
+	hp: number | null;
+	nlCount: number;
+	nlPriceLatest: number | null;
+	usVersions: number;
+};
+const added = {
+	brands: [] as { brand: string; url: string; models: number; nlCount: number }[],
+	models: [] as NewModel[]
+};
+
 async function write(brands: ReturnType<typeof build>) {
 	const url = process.env.DB_URL;
 	if (!url) throw new Error('Set DB_URL (or use --dry-run)');
 	const sql = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} });
 	try {
 		await sql.begin(async (tx) => {
+			const oldBrands = new Set(
+				(await tx<{ slug: string }[]>`select slug from cars.brands`).map((r) => r.slug)
+			);
+			for (const b of brands)
+				if (oldBrands.size && !oldBrands.has(b.slug))
+					added.brands.push({
+						brand: b.name,
+						url: `${SITE}/${b.slug}`,
+						models: b.models.length,
+						nlCount: b.nlCount
+					});
 			const brandIds = new Map<string, number>();
 			for (const b of brands) {
 				const [row] = await tx`
@@ -930,6 +1007,7 @@ async function write(brands: ReturnType<typeof build>) {
 			const existing = await tx<{ id: number; dex: number; brand_id: number; slug: string }[]>`
 				select id, dex, brand_id, slug from cars.models`;
 			const byKey = new Map(existing.map((e) => [`${e.brand_id}|${e.slug}`, e]));
+			const firstRun = existing.length === 0;
 			let nextDex = renumber ? 1 : Math.max(0, ...existing.map((e) => e.dex)) + 1;
 			const all = brands
 				.flatMap((b) => b.models.map((m) => ({ b, m, brandId: brandIds.get(b.slug)! })))
@@ -996,6 +1074,24 @@ async function write(brands: ReturnType<typeof build>) {
 					on conflict (brand_id, slug) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'brand_id' && k !== 'slug') as (keyof typeof row)[])}
 					returning id`;
 				keep.push(saved.id);
+				if (!firstRun && !byKey.has(`${brandId}|${m.slug}`)) {
+					const b = brands.find((x) => brandIds.get(x.slug) === brandId)!;
+					added.models.push({
+						brand: b.name,
+						model: m.name,
+						dex,
+						url: `${SITE}/${b.slug}/${m.slug}`,
+						firstYear: m.firstYear,
+						lastYear: m.lastYear,
+						fuels: m.fuels,
+						bodies: m.bodies,
+						powerKw: m.powerKwMax,
+						hp: m.powerKwMax ? Math.round(m.powerKwMax * 1.341) : null,
+						nlCount: m.nlCount,
+						nlPriceLatest: m.nlPriceLatest,
+						usVersions: variants.length
+					});
+				}
 				for (let i = 0; i < variants.length; i += 500) {
 					const chunk = variants.slice(i, i + 500).map((v) => ({
 						model_id: saved.id,
@@ -1022,7 +1118,7 @@ async function write(brands: ReturnType<typeof build>) {
 				}
 			}
 			await tx`delete from cars.models where id <> all(${keep})`;
-			console.log(`wrote ${brands.length} brands, ${keep.length} models, ${variantCount} variants`);
+			log(`wrote ${brands.length} brands, ${keep.length} models, ${variantCount} variants`);
 		});
 	} finally {
 		await sql.end();
@@ -1032,17 +1128,17 @@ async function write(brands: ReturnType<typeof build>) {
 // ---------------------------------------------------------------- main
 
 const src = await loadSources();
-console.log(
+log(
 	`fueleconomy.gov rows: ${src.fe.length}; RDW names: ${src.models.length}, years: ${src.years.length}`
 );
 const plates = [...new Set(src.years.map((y) => y.k).filter(Boolean))];
 const fuel = await cached('rdw-fuel.json', () => {
-	console.log(`querying RDW fuel data for ${plates.length} sample cars`);
+	log(`querying RDW fuel data for ${plates.length} sample cars`);
 	return loadFuel(plates);
 });
 const brands = build(src, fuel);
 const models = brands.flatMap((b) => b.models);
-console.log(
+log(
 	`built ${brands.length} brands, ${models.length} models, ${models.reduce((a, m) => a + m.variants.length, 0)} variants`
 );
 if (outFile)
@@ -1058,5 +1154,20 @@ if (outFile)
 if (dryRun) {
 	const top = models.toSorted((a, c) => c.nlCount - a.nlCount).slice(0, 15);
 	for (const m of top)
-		console.log(`  ${m.name} nl=${m.nlCount} ${m.firstYear}-${m.lastYear} ${m.fuels.join('/')}`);
-} else await write(brands);
+		log(`  ${m.name} nl=${m.nlCount} ${m.firstYear}-${m.lastYear} ${m.fuels.join('/')}`);
+} else {
+	await write(brands);
+	if (reportNew)
+		console.log(
+			JSON.stringify(
+				{
+					generatedAt: new Date().toISOString(),
+					totals: { brands: brands.length, models: models.length },
+					newBrands: added.brands,
+					newModels: added.models.toSorted((a, b) => b.nlCount - a.nlCount)
+				},
+				null,
+				2
+			)
+		);
+}