Stan Runge
bf0a1e1d2a
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
1174 lines
40 KiB
TypeScript
1174 lines
40 KiB
TypeScript
// Builds the public car pokedex (/trackers/cars, schema `cars`) from open data.
|
|
//
|
|
// DB_URL=... bun scripts/cars-import.ts fetch sources and write to the database
|
|
// bun scripts/cars-import.ts --dry-run fetch + build only, print a summary
|
|
// bun scripts/cars-import.ts --cache /tmp/cars reuse raw downloads from that folder
|
|
// bun scripts/cars-import.ts --dry-run --out x.json also dump the built catalogue
|
|
// bun scripts/cars-import.ts --renumber reassign Dex numbers from #1 (oldest first)
|
|
// bun scripts/cars-import.ts --report-new JSON of brands/models new since the last run on stdout
|
|
//
|
|
// Sources (no API keys):
|
|
// - fueleconomy.gov vehicles.csv: every US EPA-tested car since 1984 (make, base model,
|
|
// year, engine, cylinders, transmission, drive, fuel, size class, MPG, CO2, EV range).
|
|
// - RDW open data (Socrata, aggregated server-side, nothing big is downloaded):
|
|
// m9d7-ebf2 Gekentekende voertuigen: passenger cars registered in NL grouped by
|
|
// make + trade name (count, years, catalogue price, mass, body, seats).
|
|
// 8ys7-d773 brandstof: fuel, power, consumption and CO2 for one sample car per
|
|
// model per first-registration year (the dataset has no make column, and
|
|
// Socrata here does not support joins).
|
|
// Re-runnable: brands/models are upserted by slug (Dex numbers stay stable, new models get
|
|
// the next number), models that vanished are deleted, variants are replaced.
|
|
import postgres from 'postgres';
|
|
|
|
const known = (xs: (number | null)[]) => xs.filter((x): x is number => x != null);
|
|
const minOrNull = (xs: (number | null)[]) => (known(xs).length ? Math.min(...known(xs)) : null);
|
|
const maxOrNull = (xs: (number | null)[]) => (known(xs).length ? Math.max(...known(xs)) : null);
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
|
|
const args = process.argv.slice(2);
|
|
const dryRun = args.includes('--dry-run');
|
|
// --renumber: hand out Dex numbers from #1 again instead of keeping existing ones.
|
|
const renumber = args.includes('--renumber');
|
|
const cacheDir = args.includes('--cache') ? args[args.indexOf('--cache') + 1] : null;
|
|
const outFile = args.includes('--out') ? args[args.indexOf('--out') + 1] : null;
|
|
// --report-new: print a JSON summary of brands/models that did not exist before this run to
|
|
// stdout (all progress logging moves to stderr so stdout stays machine-readable).
|
|
const reportNew = args.includes('--report-new');
|
|
const SITE = 'https://stanrunge.dev/trackers/cars';
|
|
const log = (...a: unknown[]) => (reportNew ? console.error(...a) : console.log(...a));
|
|
|
|
const FE_URL = 'https://www.fueleconomy.gov/feg/epadata/vehicles.csv';
|
|
const RDW = 'https://opendata.rdw.nl/resource';
|
|
const RDW_CARS = `${RDW}/m9d7-ebf2.json`;
|
|
const RDW_FUEL = `${RDW}/8ys7-d773.json`;
|
|
const SRC_FE = 'fueleconomy.gov';
|
|
const SRC_RDW = 'RDW open data';
|
|
// Passenger cars only; motorhomes, hearses, ambulances and wheelchair conversions are
|
|
// registered as "Personenauto" too but are not cars you would buy off a forecourt.
|
|
const RDW_WHERE =
|
|
"voertuigsoort='Personenauto' AND inrichting not in ('kampeerwagen','ambulance','lijkwagen','voor rolstoelen toegankelijk voertuig','huifopbouw')";
|
|
const MIN_NL_NAME = 5; // trade names with fewer NL registrations are mostly typos and one-offs
|
|
const MIN_NL_FAMILY = 20; // a trade name needs this many to stand as a model on its own
|
|
const MIN_NL_MAKE = 50; // NL-only makes need this many cars in total
|
|
|
|
// ---------------------------------------------------------------- fetching
|
|
|
|
async function cached<T>(name: string, load: () => Promise<T>): Promise<T> {
|
|
if (cacheDir) {
|
|
try {
|
|
return JSON.parse(await readFile(join(cacheDir, name), 'utf8')) as T;
|
|
} catch {
|
|
// not cached yet
|
|
}
|
|
}
|
|
const value = await load();
|
|
if (cacheDir) {
|
|
await mkdir(cacheDir, { recursive: true });
|
|
await writeFile(join(cacheDir, name), JSON.stringify(value));
|
|
}
|
|
return value;
|
|
}
|
|
|
|
async function getJson<T>(url: string, params: Record<string, string>, tries = 4): Promise<T> {
|
|
const u = new URL(url);
|
|
for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
|
|
for (let attempt = 1; ; attempt++) {
|
|
try {
|
|
const res = await fetch(u, { signal: AbortSignal.timeout(300_000) });
|
|
if (!res.ok) throw new Error(`${res.status} ${(await res.text()).slice(0, 200)}`);
|
|
return (await res.json()) as T;
|
|
} catch (err) {
|
|
if (attempt >= tries) throw err;
|
|
await Bun.sleep(2000 * attempt);
|
|
}
|
|
}
|
|
}
|
|
|
|
function parseCsv(text: string): Record<string, string>[] {
|
|
const rows: string[][] = [];
|
|
let row: string[] = [];
|
|
let field = '';
|
|
let quoted = false;
|
|
for (let i = 0; i < text.length; i++) {
|
|
const c = text[i];
|
|
if (quoted) {
|
|
if (c === '"') {
|
|
if (text[i + 1] === '"') {
|
|
field += '"';
|
|
i++;
|
|
} else quoted = false;
|
|
} else field += c;
|
|
} else if (c === '"') quoted = true;
|
|
else if (c === ',') {
|
|
row.push(field);
|
|
field = '';
|
|
} else if (c === '\n' || c === '\r') {
|
|
if (c === '\r' && text[i + 1] === '\n') i++;
|
|
row.push(field);
|
|
rows.push(row);
|
|
row = [];
|
|
field = '';
|
|
} else field += c;
|
|
}
|
|
if (field || row.length) rows.push([...row, field]);
|
|
const [header, ...body] = rows;
|
|
return body
|
|
.filter((r) => r.length === header.length)
|
|
.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i]])));
|
|
}
|
|
|
|
type RdwModel = {
|
|
merk: string;
|
|
handelsbenaming?: string;
|
|
n: string;
|
|
price?: string;
|
|
mass?: string;
|
|
cc?: string;
|
|
cyl?: string;
|
|
};
|
|
type RdwYear = {
|
|
merk: string;
|
|
handelsbenaming?: string;
|
|
y?: string;
|
|
n: string;
|
|
p?: string;
|
|
k: string;
|
|
};
|
|
type RdwBody = {
|
|
merk: string;
|
|
handelsbenaming?: string;
|
|
inrichting?: string;
|
|
n: string;
|
|
seats?: string;
|
|
doors?: string;
|
|
};
|
|
type RdwFuel = {
|
|
kenteken: string;
|
|
brandstof_omschrijving?: string;
|
|
klasse_hybride_elektrisch_voertuig?: string;
|
|
nettomaximumvermogen?: string;
|
|
brandstofverbruik_gecombineerd?: string;
|
|
brandstof_verbruik_gecombineerd_wltp?: string;
|
|
co2_uitstoot_gecombineerd?: string;
|
|
emissie_co2_gecombineerd_wltp?: string;
|
|
};
|
|
|
|
async function loadSources() {
|
|
const fe = await cached('fe.json', async () => {
|
|
log('fetching fueleconomy.gov vehicles.csv');
|
|
const res = await fetch(FE_URL, { signal: AbortSignal.timeout(300_000) });
|
|
if (!res.ok) throw new Error(`fueleconomy.gov ${res.status}`);
|
|
return parseCsv(await res.text());
|
|
});
|
|
const group = (select: string, groupBy: string, having?: string) => ({
|
|
$select: select,
|
|
$where: RDW_WHERE,
|
|
$group: groupBy,
|
|
...(having ? { $having: having } : {}),
|
|
$limit: '2000000'
|
|
});
|
|
log('querying RDW aggregates');
|
|
const [models, years, bodies] = await Promise.all([
|
|
cached('rdw-models.json', () =>
|
|
getJson<RdwModel[]>(
|
|
RDW_CARS,
|
|
group(
|
|
'merk,handelsbenaming,count(*) as n,avg(catalogusprijs) as price,avg(massa_rijklaar) as mass,avg(cilinderinhoud) as cc,avg(aantal_cilinders) as cyl',
|
|
'merk,handelsbenaming',
|
|
`n >= ${MIN_NL_NAME}`
|
|
)
|
|
)
|
|
),
|
|
cached('rdw-years.json', () =>
|
|
getJson<RdwYear[]>(
|
|
RDW_CARS,
|
|
group(
|
|
'merk,handelsbenaming,date_extract_y(datum_eerste_toelating_dt) as y,count(*) as n,avg(catalogusprijs) as p,max(kenteken) as k',
|
|
'merk,handelsbenaming,y',
|
|
'n >= 3'
|
|
)
|
|
)
|
|
),
|
|
cached('rdw-bodies.json', () =>
|
|
getJson<RdwBody[]>(
|
|
RDW_CARS,
|
|
group(
|
|
'merk,handelsbenaming,inrichting,count(*) as n,avg(aantal_zitplaatsen) as seats,avg(aantal_deuren) as doors',
|
|
'merk,handelsbenaming,inrichting',
|
|
'n >= 3'
|
|
)
|
|
)
|
|
)
|
|
]);
|
|
return { fe, models, years, bodies };
|
|
}
|
|
|
|
async function loadFuel(plates: string[]): Promise<RdwFuel[]> {
|
|
const out: RdwFuel[] = [];
|
|
const chunks: string[][] = [];
|
|
for (let i = 0; i < plates.length; i += 250) chunks.push(plates.slice(i, i + 250));
|
|
let next = 0;
|
|
async function worker() {
|
|
while (next < chunks.length) {
|
|
const chunk = chunks[next++];
|
|
const list = chunk.map((p) => `'${p.replace(/[^A-Z0-9]/g, '')}'`).join(',');
|
|
out.push(
|
|
...(await getJson<RdwFuel[]>(RDW_FUEL, {
|
|
$where: `kenteken in(${list})`,
|
|
$limit: '5000'
|
|
}))
|
|
);
|
|
}
|
|
}
|
|
await Promise.all(Array.from({ length: 4 }, worker));
|
|
return out;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- normalising
|
|
|
|
const MAKE_ALIASES: Record<string, string> = {
|
|
VW: 'VOLKSWAGEN',
|
|
BMWI: 'BMW',
|
|
BMWALPINA: 'ALPINA',
|
|
TESLAMOTORS: 'TESLA',
|
|
MERCEDES: 'MERCEDESBENZ',
|
|
MERCEDESAMG: 'MERCEDESBENZ',
|
|
DAIMLERBENZ: 'MERCEDESBENZ',
|
|
CITROEN: 'CITROEN',
|
|
LYNKCO: 'LYNKCO',
|
|
DSAUTOMOBILES: 'DS',
|
|
MCLARENAUTOMOTIVE: 'MCLAREN',
|
|
ROLLSROYCEMOTORCARS: 'ROLLSROYCE',
|
|
GENERALMOTORS: 'GM',
|
|
MGROVER: 'MG',
|
|
FORDCNGTECHNIK: 'FORD',
|
|
JAGUARCARS: 'JAGUAR',
|
|
MICROCOMPACTCARSMART: 'SMART',
|
|
MICROCOMPACTCAR: 'SMART',
|
|
QUATTRO: 'AUDI',
|
|
AUTOMOBILILAMBORGHINIS: 'LAMBORGHINI',
|
|
AUTOMOBILILAMBORGHINI: 'LAMBORGHINI',
|
|
LUCIDMOTORS: 'LUCID',
|
|
DAIMLERCHRYSLERAG: 'CHRYSLER',
|
|
DAIMLERCHRYSLERUSA: 'CHRYSLER',
|
|
LADAVAZ: 'LADA',
|
|
WILLYSOVERLAND: 'WILLYS',
|
|
DAXCOBRA: 'DAX',
|
|
ALPINERENAULT: 'ALPINE',
|
|
AMERICANMOTORS: 'AMC',
|
|
AMERICANMOTORSCORPORATION: 'AMC',
|
|
RUFAUTOMOBILEGMBH: 'RUF',
|
|
RUFAUTOMOBILE: 'RUF',
|
|
SALEENPERFORMANCE: 'SALEEN',
|
|
TVRENGINEERINGLTD: 'TVR',
|
|
BUGATTIRIMAC: 'BUGATTI',
|
|
INEOSAUTOMOTIVE: 'INEOS',
|
|
KGMOBILITY: 'KGM'
|
|
};
|
|
// Display names where neither source spells the brand the way it brands itself.
|
|
const MAKE_NAMES: Record<string, string> = {
|
|
LYNKCO: 'Lynk & Co',
|
|
MCLAREN: 'McLaren',
|
|
LUCID: 'Lucid',
|
|
NSU: 'NSU',
|
|
DKW: 'DKW',
|
|
DAF: 'DAF',
|
|
CITROEN: 'Citroën',
|
|
SKODA: 'Škoda',
|
|
NIO: 'NIO',
|
|
XPENG: 'XPeng',
|
|
BYD: 'BYD',
|
|
MG: 'MG',
|
|
DS: 'DS',
|
|
KGM: 'KGM',
|
|
INEOS: 'INEOS',
|
|
AMC: 'AMC',
|
|
RUF: 'RUF',
|
|
TVR: 'TVR',
|
|
DELOREAN: 'DeLorean',
|
|
SSANGYONG: 'SsangYong',
|
|
VWPORSCHE: 'VW-Porsche',
|
|
ALPINE: 'Alpine',
|
|
SMART: 'smart',
|
|
MINI: 'MINI'
|
|
};
|
|
const makeKey = (name: string) => {
|
|
const k = name
|
|
.normalize('NFD')
|
|
.replace(/[\u0300-\u036f]/g, '')
|
|
.toUpperCase()
|
|
.replace(/[^A-Z0-9]/g, '');
|
|
return MAKE_ALIASES[k] ?? k;
|
|
};
|
|
|
|
// Country of origin of the brand (where it was founded / is headquartered), for well-known
|
|
// brands only. Everything else stays blank rather than guessed.
|
|
const COUNTRIES: Record<string, string> = Object.fromEntries(
|
|
Object.entries({
|
|
Germany:
|
|
'VOLKSWAGEN AUDI BMW MERCEDESBENZ PORSCHE OPEL SMART MAYBACH ALPINA WIESMANN TRABANT WARTBURG BORGWARD AUTOUNION NSU DKW GUMPERT ARTEGA CUPRA_X',
|
|
Japan:
|
|
'TOYOTA HONDA NISSAN MAZDA SUBARU MITSUBISHI SUZUKI DAIHATSU LEXUS INFINITI ACURA ISUZU SCION DATSUN',
|
|
'United States':
|
|
'FORD CHEVROLET DODGE CHRYSLER JEEP CADILLAC BUICK GMC LINCOLN TESLA RAM PONTIAC OLDSMOBILE PLYMOUTH MERCURY SATURN HUMMER RIVIAN LUCID FISKER SALEEN SHELBY ROUSH EAGLE AMC GEO HENNESSEY KARMA VECTOR PANOZ STUDEBAKER',
|
|
Italy:
|
|
'FIAT FERRARI LAMBORGHINI MASERATI ALFAROMEO LANCIA ABARTH PAGANI DETOMASO AUTOBIANCHI INNOCENTI',
|
|
France:
|
|
'PEUGEOT RENAULT CITROEN DS ALPINE BUGATTI SIMCA TALBOT VENTURI MATRA AIXAM LIGIER MICROCAR CHATENET',
|
|
'United Kingdom':
|
|
'JAGUAR LANDROVER MINI ROLLSROYCE BENTLEY ASTONMARTIN LOTUS MCLAREN MG ROVER TRIUMPH MORGAN AUSTIN MORRIS CATERHAM TVR NOBLE ARIELA VAUXHALL DAIMLER JENSEN HILLMAN SUNBEAM LEYLAND AUSTINHEALEY RANGEROVER',
|
|
Sweden: 'VOLVO SAAB POLESTAR KOENIGSEGG',
|
|
'South Korea': 'HYUNDAI KIA GENESIS DAEWOO SSANGYONG KGM',
|
|
China:
|
|
'BYD NIO XPENG LEAPMOTOR GEELY LYNKCO ZEEKR GWM ORA WEY AIWAYS SERES VOYAH HONGQI MAXUS DONGFENG JAC CHERY OMODA JAECOO XIAOMI ZHIDOU BAIC SKYWELL DFSK FORTHING',
|
|
'Czech Republic': 'SKODA TATRA',
|
|
Spain: 'SEAT CUPRA',
|
|
Romania: 'DACIA',
|
|
Netherlands: 'DAF SPYKER DONKERVOORT LIGHTYEAR',
|
|
Russia: 'LADA',
|
|
India: 'TATA MAHINDRA',
|
|
Vietnam: 'VINFAST',
|
|
Malaysia: 'PROTON',
|
|
Croatia: 'RIMAC'
|
|
}).flatMap(([country, keys]) => keys.split(' ').map((k) => [k, country]))
|
|
);
|
|
|
|
const TRIM_WORDS = new Set(
|
|
`GTI GTD GTE GTX GT GTS GL GLS GLX GLI GX GXL XR XE XL XLI XLT XS XT DX LX EX SE SX SL SLX LS LT LTZ ST RS S
|
|
CABRIOLET CABRIO CABR CONVERTIBLE ROADSTER SPIDER SPYDER COUPE COUPÉ TARGA VARIANT AVANT SPORTBACK SPORTSBACK ESTATE
|
|
HATCHBACK HB SEDAN SALOON NOTCHBACK LIMOUSINE STATION STATIONWAGON SW WAGON TOURER SPORTS TOURING TOURISMO BREAK
|
|
KOMBI CARAVAN COMBI LIFTBACK FASTBACK GRANDCOUPE AUTOMAAT AUTOMATIC AUTOMATIK AUT AUTO DIESEL HYBRID HYBRIDE ELECTRIC
|
|
ELECTRIQUE ELEKTRISCH PHEV EHYBRID EV TURBO TDI TSI TFSI FSI HDI DCI CDI CRDI JTD JTDM TDCI ECOBOOST VTEC IDI BLUEHDI
|
|
BLUEMOTION KW PK CV DRS DEURS DEUR DOOR DOORS SPECIAL EDITION LIMITED EXECUTIVE COMFORT COMFORTLINE TRENDLINE HIGHLINE
|
|
BUSINESS PREMIUM LUXURY ELEGANCE AMBIENTE TITANIUM TREND ACTIVE STYLE SPORTLINE LINE PURE PLUS PRO MAX ACCENT
|
|
AMBITION ALLURE ACCESS EXPRESSION ESSENTIA ESSENTIAL INTENS ZEN LIFE ICON FUN BASE STANDARD DELUXE DL GLE CL CLX
|
|
INJECTION ETECH PLUGIN HYBRID4 XDRIVE SDRIVE EDRIVE QUATTRO MOTION MATIC AWD WD X`.split(/\s+/)
|
|
);
|
|
const KEEP_UPPER = new Set(
|
|
'GTI GTD GTE GTX GT AMG SUV MPV RS ST EV BEV PHEV TDI TSI SRT SS SVR SVT XL XLT CC SW TT GTS GTV RV CR-V HR-V EQA EQB EQC EQE EQS EQV EQT UX NX RX LX GX IS ES LS LC RC II III IV VI VII VIII IX XI XII'.split(
|
|
' '
|
|
)
|
|
);
|
|
function titleCase(s: string): string {
|
|
return s
|
|
.toLowerCase()
|
|
.split(/(\s+|-)/)
|
|
.map((w) => {
|
|
const up = w.toUpperCase();
|
|
if (/\d/.test(w) || (w.length <= 3 && /^[^aeiouy]+$/.test(w)) || KEEP_UPPER.has(up))
|
|
return up;
|
|
return w.charAt(0).toUpperCase() + w.slice(1);
|
|
})
|
|
.join('');
|
|
}
|
|
const slugify = (s: string) =>
|
|
s
|
|
.normalize('NFD')
|
|
.replace(/[\u0300-\u036f]/g, '')
|
|
.toLowerCase()
|
|
.replace(/&/g, ' and ')
|
|
.replace(/[^a-z0-9]+/g, '-')
|
|
.replace(/^-|-$/g, '') || 'x';
|
|
const cmpKey = (s: string) => s.toUpperCase().replace(/[^A-Z0-9]/g, '');
|
|
|
|
/** RDW trade name -> cleaned upper-case name ("TOYOTA AYGO; 1.0" -> "AYGO"). */
|
|
function cleanTradeName(raw: string, make: string): string {
|
|
let s = raw
|
|
.toUpperCase()
|
|
.replace(/[;,(].*$/, '')
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
const makeWords = new Set([
|
|
...make.split(/[\s-]+/),
|
|
make.replace(/[\s-]+/g, ''),
|
|
...(makeKey(make) === 'VOLKSWAGEN' ? ['VW'] : []),
|
|
...(makeKey(make) === 'MERCEDESBENZ' ? ['MERCEDES', 'BENZ', 'MERCEDES-BENZ', 'MB'] : [])
|
|
]);
|
|
for (let i = 0; i < 3; i++) {
|
|
const [first, ...rest] = s.split(' ');
|
|
if (rest.length && makeWords.has(first)) s = rest.join(' ');
|
|
else if (s.startsWith(`${make} `) && s.length > make.length + 1) s = s.slice(make.length + 1);
|
|
else break;
|
|
}
|
|
// Misspelt make as first word ("PEUGOT 205" under PEUGEOT).
|
|
const [w0, ...restW] = s.split(' ');
|
|
const mk0 = make.replace(/[^A-Z]/g, '');
|
|
if (
|
|
restW.length &&
|
|
mk0.length >= 5 &&
|
|
w0 !== mk0 &&
|
|
w0.slice(0, 4) === mk0.slice(0, 4) &&
|
|
Math.abs(w0.length - mk0.length) <= 1
|
|
)
|
|
s = restW.join(' ');
|
|
s = s.replace(/^\d{5,}\s+/, ''); // leading RDW type code, e.g. "155221 GOLF CABRIOLET"
|
|
return s.replace(/[-\s]+$/, '').trim();
|
|
}
|
|
|
|
/** Hand fixes for trade names that put the trim first ("MACH 1 MUSTANG" is a Mustang). */
|
|
const TRADE_FIXES: Record<string, [RegExp, string][]> = {
|
|
FORD: [
|
|
[/^MACH ?(1|I)\b ?MUSTANG\b/, 'MUSTANG MACH 1'],
|
|
[/^MUSTANG( II)? MACH ?(1|I|ONE)\b/, 'MUSTANG$1']
|
|
],
|
|
LYNKCO: [[/^LYNK ?& ?CO ?/, '']],
|
|
KIA: [[/^CEE ?'? ?D\b/, "CEE'D"]]
|
|
};
|
|
function tradeFix(makeK: string, name: string): string {
|
|
for (const [re, to] of TRADE_FIXES[makeK] ?? []) name = name.replace(re, to);
|
|
return name.trim();
|
|
}
|
|
|
|
/** Brand-specific folding of engine-coded names into the model family. */
|
|
function familyRule(makeK: string, name: string): string | null {
|
|
if (makeK === 'MERCEDESBENZ') {
|
|
if (/^(MERCEDES-)?AMG GT\b/.test(name)) return 'AMG GT';
|
|
name = name.replace(/^(MERCEDES-)?AMG /, '');
|
|
const m = name.match(
|
|
/^(A|B|C|E|S|G|V|R|X|CL|CLA|CLC|CLK|CLS|GL|GLA|GLB|GLC|GLE|GLK|GLS|ML|SL|SLC|SLK)[ -]?\d{2,3}/
|
|
);
|
|
if (m) return m[1] === 'ML' ? 'M-CLASS' : `${m[1]}-CLASS`;
|
|
const k = name.match(/^([A-Z]{1,3})[ -]KLASSE\b/);
|
|
if (k) return `${k[1]}-CLASS`;
|
|
const eq = name.match(/^(EQ[A-Z])\b/);
|
|
if (eq) return eq[1];
|
|
}
|
|
if (makeK === 'BMW') {
|
|
const reihe = name.match(/^([1-8])(ER| SERIES| REIHE|-SERIE)/);
|
|
if (reihe) return `${reihe[1]} SERIES`;
|
|
const m = name.match(/^([1-8])\d{2} ?[A-Z]{1,3}\b/);
|
|
if (m) return `${m[1]} SERIES`;
|
|
}
|
|
if (makeK === 'OPEL') {
|
|
// Opel generation letters: Corsa-B, Astra-G-CC, Meriva-A -> Corsa, Astra, Meriva
|
|
const g = name.match(/^([A-Z]{3,})-[A-K](\b|-)/);
|
|
if (g) return g[1];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// Consumer body type from the RDW body (inrichting) and the EPA size class.
|
|
const RDW_BODY: Record<string, string> = {
|
|
hatchback: 'hatchback',
|
|
stationwagen: 'wagon',
|
|
MPV: 'mpv',
|
|
sedan: 'sedan',
|
|
cabriolet: 'convertible',
|
|
coupe: 'coupe',
|
|
terreinvoertuig: 'suv',
|
|
limousine: 'sedan',
|
|
'pick-up truck': 'pickup',
|
|
truckstationwagen: 'suv'
|
|
};
|
|
function feBody(vclass: string): string {
|
|
const v = vclass.toLowerCase();
|
|
if (v.includes('pickup')) return 'pickup';
|
|
if (v.includes('sport utility') || v.includes('special purpose')) return 'suv';
|
|
if (v.includes('minivan')) return 'mpv';
|
|
if (v.includes('van')) return 'van';
|
|
if (v.includes('wagon')) return 'wagon';
|
|
if (v.includes('two seater')) return 'sports';
|
|
return 'car';
|
|
}
|
|
const feSizeClass = (vclass: string) =>
|
|
vclass
|
|
.replace(/\s*[-/]?\s*(2|4)wd$/i, '')
|
|
.replace(/ - $/, '')
|
|
.replace(/Sport Utility Vehicles?/, 'SUV')
|
|
.replace('Special Purpose Vehicles', 'Special Purpose Vehicle')
|
|
.replace('Vans Passenger', 'Vans, Passenger Type')
|
|
.trim();
|
|
function feFuel(r: Record<string, string>): string[] {
|
|
const atv = r.atvType;
|
|
const f = r.fuelType;
|
|
if (atv === 'EV' || f === 'Electricity') return ['electric'];
|
|
if (atv === 'Plug-in Hybrid' || /Electricity/.test(f)) return ['plug-in hybrid'];
|
|
if (atv === 'Hybrid') return ['hybrid'];
|
|
if (f === 'Diesel') return ['diesel'];
|
|
if (f === 'CNG') return ['cng'];
|
|
if (f === 'Hydrogen') return ['hydrogen'];
|
|
if (/propane/.test(f)) return ['petrol', 'lpg'];
|
|
if (/natural gas/.test(f)) return ['petrol', 'cng'];
|
|
return ['petrol'];
|
|
}
|
|
const FE_DRIVE: Record<string, string> = {
|
|
'Front-Wheel Drive': 'fwd',
|
|
'Rear-Wheel Drive': 'rwd',
|
|
'All-Wheel Drive': 'awd',
|
|
'4-Wheel or All-Wheel Drive': 'awd',
|
|
'4-Wheel Drive': '4wd',
|
|
'Part-time 4-Wheel Drive': '4wd'
|
|
};
|
|
const feTrans = (t: string) =>
|
|
/^Auto/.test(t) ? 'automatic' : /^Manual/.test(t) ? 'manual' : null;
|
|
const num = (v: string | undefined) => {
|
|
if (v == null || v === '') return null;
|
|
const n = Number(v);
|
|
return Number.isFinite(n) ? n : null;
|
|
};
|
|
|
|
// ---------------------------------------------------------------- building
|
|
|
|
type Model = {
|
|
makeK: string;
|
|
key: string; // cmpKey of the family name
|
|
name: string;
|
|
usNames: Set<string>;
|
|
years: Set<number>;
|
|
usYears: Set<number>;
|
|
bodies: Map<string, number>;
|
|
sizeClasses: Set<string>;
|
|
fuels: Map<string, number>;
|
|
drives: Set<string>;
|
|
trans: Set<string>;
|
|
engine: number[];
|
|
cyl: number[];
|
|
turbo: boolean;
|
|
kw: number[];
|
|
l100: number[];
|
|
co2: number[];
|
|
mpg: number[];
|
|
rangeMi: number[];
|
|
nlCount: number;
|
|
nlPriceSum: number;
|
|
nlPriceN: number;
|
|
nlMassSum: number;
|
|
nlMassN: number;
|
|
seatsSum: number;
|
|
doorsSum: number;
|
|
seatN: number;
|
|
nlYears: Map<number, { n: number; pSum: number; pN: number }>;
|
|
nlNames: Map<string, number>;
|
|
variants: Record<string, unknown>[];
|
|
sources: Set<string>;
|
|
};
|
|
type Brand = {
|
|
key: string;
|
|
name: string;
|
|
usName: string | null;
|
|
nlCount: number;
|
|
models: Map<string, Model>;
|
|
sources: Set<string>;
|
|
};
|
|
|
|
function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
|
|
const brands = new Map<string, Brand>();
|
|
const brand = (rawName: string, us: boolean) => {
|
|
const key = makeKey(rawName);
|
|
let b = brands.get(key);
|
|
if (!b) {
|
|
b = { key, name: rawName, usName: null, nlCount: 0, models: new Map(), sources: new Set() };
|
|
brands.set(key, b);
|
|
}
|
|
if (us && !b.usName) b.usName = rawName;
|
|
return b;
|
|
};
|
|
const model = (b: Brand, key: string, name: string): Model => {
|
|
let m = b.models.get(key);
|
|
if (!m) {
|
|
m = {
|
|
makeK: b.key,
|
|
key,
|
|
name,
|
|
usNames: new Set(),
|
|
years: new Set(),
|
|
usYears: new Set(),
|
|
bodies: new Map(),
|
|
sizeClasses: new Set(),
|
|
fuels: new Map(),
|
|
drives: new Set(),
|
|
trans: new Set(),
|
|
engine: [],
|
|
cyl: [],
|
|
turbo: false,
|
|
kw: [],
|
|
l100: [],
|
|
co2: [],
|
|
mpg: [],
|
|
rangeMi: [],
|
|
nlCount: 0,
|
|
nlPriceSum: 0,
|
|
nlPriceN: 0,
|
|
nlMassSum: 0,
|
|
nlMassN: 0,
|
|
seatsSum: 0,
|
|
doorsSum: 0,
|
|
seatN: 0,
|
|
nlYears: new Map(),
|
|
nlNames: new Map(),
|
|
variants: [],
|
|
sources: new Set()
|
|
};
|
|
b.models.set(key, m);
|
|
}
|
|
return m;
|
|
};
|
|
const bump = <K>(map: Map<K, number>, k: K, n = 1) => map.set(k, (map.get(k) ?? 0) + n);
|
|
|
|
// 1. US EPA: base model = family.
|
|
for (const r of src.fe) {
|
|
const year = Number(r.year);
|
|
if (!r.make || !r.baseModel || !year || r.baseModel === 'Postal Vehicle') continue;
|
|
const b = brand(r.make, true);
|
|
b.sources.add(SRC_FE);
|
|
const family = r.baseModel.split('/')[0].trim() || r.baseModel;
|
|
const m = model(b, cmpKey(family), family);
|
|
m.sources.add(SRC_FE);
|
|
m.usNames.add(r.model);
|
|
m.years.add(year);
|
|
m.usYears.add(year);
|
|
bump(m.bodies, feBody(r.VClass));
|
|
if (r.VClass) m.sizeClasses.add(feSizeClass(r.VClass));
|
|
for (const f of feFuel(r)) bump(m.fuels, f);
|
|
if (FE_DRIVE[r.drive]) m.drives.add(FE_DRIVE[r.drive]);
|
|
const t = feTrans(r.trany);
|
|
if (t) m.trans.add(t);
|
|
const displ = num(r.displ);
|
|
const cyl = num(r.cylinders);
|
|
if (displ) m.engine.push(displ);
|
|
if (cyl) m.cyl.push(cyl);
|
|
if (r.tCharger === 'T' || r.sCharger === 'S') m.turbo = true;
|
|
const comb = num(r.comb08);
|
|
if (comb) m.mpg.push(comb);
|
|
const range = num(r.range);
|
|
if (range) m.rangeMi.push(range);
|
|
m.variants.push({
|
|
sourceId: num(r.id),
|
|
year,
|
|
name: r.model,
|
|
engineL: displ,
|
|
cylinders: cyl,
|
|
transmission: r.trany || null,
|
|
drive: r.drive || null,
|
|
fuel: r.fuelType || null,
|
|
sizeClass: r.VClass || null,
|
|
mpgCity: num(r.city08),
|
|
mpgHwy: num(r.highway08),
|
|
mpgComb: comb,
|
|
co2Gpm: (() => {
|
|
const c = num(r.co2TailpipeGpm);
|
|
return c && c > 0 ? Math.round(c) : null;
|
|
})(),
|
|
rangeMi: range || null,
|
|
evMotor: r.evMotor || null,
|
|
turbo: r.tCharger === 'T',
|
|
supercharged: r.sCharger === 'S'
|
|
});
|
|
}
|
|
|
|
// 2. RDW: fold trade names into families.
|
|
const makeTotals = new Map<string, number>();
|
|
for (const r of src.models) bump(makeTotals, makeKey(r.merk), Number(r.n));
|
|
const fuelByPlate = new Map<string, RdwFuel[]>();
|
|
for (const f of fuel) fuelByPlate.set(f.kenteken, [...(fuelByPlate.get(f.kenteken) ?? []), f]);
|
|
const yearsByName = new Map<string, RdwYear[]>();
|
|
for (const y of src.years) {
|
|
const k = `${y.merk}|${y.handelsbenaming ?? ''}`;
|
|
yearsByName.set(k, [...(yearsByName.get(k) ?? []), y]);
|
|
}
|
|
const bodiesByName = new Map<string, RdwBody[]>();
|
|
for (const y of src.bodies) {
|
|
const k = `${y.merk}|${y.handelsbenaming ?? ''}`;
|
|
bodiesByName.set(k, [...(bodiesByName.get(k) ?? []), y]);
|
|
}
|
|
|
|
// Candidate families per make: US base models + RDW names that stand on their own.
|
|
type Row = RdwModel & { clean: string; makeK: string };
|
|
const rows: Row[] = [];
|
|
for (const r of src.models) {
|
|
const mk = makeKey(r.merk);
|
|
if (!r.handelsbenaming || /^(ONBEKEND|NIET|UNKNOWN|-|\?)/i.test(r.handelsbenaming)) continue;
|
|
if (!brands.has(mk) && (makeTotals.get(mk) ?? 0) < MIN_NL_MAKE) continue;
|
|
if (/^(ONBEKEND|DIVERSEN|ZELFBOUW|KIT ?CAR|EIGENBOUW)/.test(mk)) continue;
|
|
const clean = tradeFix(mk, cleanTradeName(r.handelsbenaming, r.merk.toUpperCase()));
|
|
if (!clean || !/[A-Z0-9]/.test(clean) || clean.includes('*') || /^\d{5,}$/.test(clean))
|
|
continue;
|
|
rows.push({ ...r, clean, makeK: mk });
|
|
}
|
|
// A trade name folds into a shorter family name when what follows is trim, body, engine
|
|
// or code noise ("205 GTI 1.9 U9" -> 205, "Astra Sports Tourer" -> Astra) but not when it
|
|
// names a different car ("Yaris Cross", "Corolla Verso" stay their own model).
|
|
const ROMAN = /^(II|III|IV|V|VI|VII|VIII|IX|X)$/;
|
|
const isNoise = (w: string, prev = '') =>
|
|
/[\d./]/.test(w) ||
|
|
w.length === 1 ||
|
|
TRIM_WORDS.has(w.replace(/[^A-Z]/g, '')) ||
|
|
(ROMAN.test(w) && !/^(MK|MARK|SERIE|SERIES|SPORT)$/.test(prev));
|
|
// Words up to the first noise word: "IX1 EDRIVE20" -> "IX1", "ID.7 TOURER PRO" -> "ID.7".
|
|
// A short or generic first word needs the next one: "MODEL 3", "I 10", "CEE D", "P 1800".
|
|
const STEM = /^([A-Z]{1,3}|MODEL|CLASS|TYPE|SERIE|SERIES)$/;
|
|
const needsNext = (words: string[]) =>
|
|
words.length > 1 &&
|
|
STEM.test(words[0]) &&
|
|
!/[./]/.test(words[1]) &&
|
|
(words[0].length === 1 || words[1].length <= 2);
|
|
const core = (clean: string) => {
|
|
const words = clean.split(' ');
|
|
let i = needsNext(words) ? 2 : 1;
|
|
while (i < words.length && !isNoise(words[i], words[i - 1])) i++;
|
|
const out = words.slice(0, i);
|
|
// "IX 20" -> "IX20", "TR 3" -> "TR3", "P 1800" -> "P1800" (but "MODEL 3" keeps its space).
|
|
if (i >= 2 && words[0].length <= 2 && /^\d/.test(words[1]))
|
|
out.splice(0, 2, words[0] + words[1]);
|
|
return out.join(' ');
|
|
};
|
|
const candidates = new Map<string, Map<string, string>>(); // makeK -> cmpKey -> display upper name
|
|
const addCandidate = (mk: string, name: string) => {
|
|
const map = candidates.get(mk) ?? new Map<string, string>();
|
|
candidates.set(mk, map);
|
|
if (!map.has(cmpKey(name))) map.set(cmpKey(name), name.toUpperCase());
|
|
};
|
|
for (const b of brands.values()) for (const m of b.models.values()) addCandidate(b.key, m.name);
|
|
const cleanTotals = new Map<string, number>();
|
|
for (const r of rows)
|
|
bump(cleanTotals, `${r.makeK}|${familyRule(r.makeK, r.clean) ?? core(r.clean)}`, Number(r.n));
|
|
for (const [k, n] of cleanTotals) {
|
|
if (n < MIN_NL_FAMILY) continue;
|
|
const [mk, name] = k.split('|');
|
|
addCandidate(mk, name);
|
|
}
|
|
function familyOf(mk: string, clean: string): string | null {
|
|
const ruled = familyRule(mk, clean);
|
|
if (ruled) {
|
|
addCandidate(mk, ruled);
|
|
return ruled;
|
|
}
|
|
const cands = candidates.get(mk);
|
|
if (!cands) return null;
|
|
const words = clean.split(' ');
|
|
for (let i = 1; i < words.length; i++) {
|
|
const hit = cands.get(cmpKey(words.slice(0, i).join(' ')));
|
|
if (hit && isNoise(words[i], words[i - 1])) return hit;
|
|
}
|
|
return cands.get(cmpKey(clean)) ?? cands.get(cmpKey(core(clean))) ?? null;
|
|
}
|
|
|
|
const plateInfo = (plate: string) => {
|
|
const rowsF = fuelByPlate.get(plate) ?? [];
|
|
const kinds = new Set(rowsF.map((f) => f.brandstof_omschrijving));
|
|
const cls = rowsF.find(
|
|
(f) => f.klasse_hybride_elektrisch_voertuig
|
|
)?.klasse_hybride_elektrisch_voertuig;
|
|
let fuel: string | null = null;
|
|
if (kinds.has('Waterstof')) fuel = 'hydrogen';
|
|
else if (cls?.startsWith('OVC')) fuel = 'plug-in hybrid';
|
|
else if (cls?.startsWith('NOVC')) fuel = 'hybrid';
|
|
else if (kinds.has('Elektriciteit') && kinds.size > 1) fuel = 'hybrid';
|
|
else if (kinds.has('Elektriciteit')) fuel = 'electric';
|
|
else if (kinds.has('Diesel')) fuel = 'diesel';
|
|
else if (kinds.has('LPG')) fuel = 'lpg';
|
|
else if (kinds.has('CNG')) fuel = 'cng';
|
|
else if (kinds.has('Benzine')) fuel = 'petrol';
|
|
const kw = Math.max(0, ...rowsF.map((f) => num(f.nettomaximumvermogen) ?? 0)) || null;
|
|
const l100 =
|
|
rowsF
|
|
.map(
|
|
(f) =>
|
|
num(f.brandstof_verbruik_gecombineerd_wltp) ?? num(f.brandstofverbruik_gecombineerd)
|
|
)
|
|
.find((v) => v && v > 0 && v < 40) ?? null;
|
|
const co2 =
|
|
rowsF
|
|
.map((f) => num(f.emissie_co2_gecombineerd_wltp) ?? num(f.co2_uitstoot_gecombineerd))
|
|
.find((v) => v != null && v >= 0 && v < 1000) ?? null;
|
|
return { fuel, kw, l100, co2 };
|
|
};
|
|
|
|
const thisYear = new Date().getFullYear();
|
|
for (const r of rows) {
|
|
const n = Number(r.n);
|
|
let fam = familyOf(r.makeK, r.clean);
|
|
if (!fam) {
|
|
if (n < MIN_NL_FAMILY) continue;
|
|
fam = core(r.clean);
|
|
}
|
|
const b = brand(r.merk, false);
|
|
const famKey = cmpKey(fam);
|
|
const existing = b.models.get(famKey);
|
|
const m = existing ?? model(b, famKey, titleCase(fam));
|
|
b.sources.add(SRC_RDW);
|
|
b.nlCount += n;
|
|
m.sources.add(SRC_RDW);
|
|
m.nlCount += n;
|
|
bump(m.nlNames, titleCase(r.clean), n);
|
|
const price = num(r.price);
|
|
if (price && price > 1000) {
|
|
// avg over registrations that have a price; weight by count as an approximation
|
|
m.nlPriceSum += price * n;
|
|
m.nlPriceN += n;
|
|
}
|
|
const mass = num(r.mass);
|
|
if (mass && mass > 300) {
|
|
m.nlMassSum += mass * n;
|
|
m.nlMassN += n;
|
|
}
|
|
const cc = num(r.cc);
|
|
if (cc && cc > 300) m.engine.push(Math.round(cc / 100) / 10);
|
|
const name = `${r.merk}|${r.handelsbenaming}`;
|
|
for (const y of yearsByName.get(name) ?? []) {
|
|
const year = Number(y.y);
|
|
if (!year || year < 1900 || year > thisYear) continue;
|
|
const yn = Number(y.n);
|
|
const e = m.nlYears.get(year) ?? { n: 0, pSum: 0, pN: 0 };
|
|
e.n += yn;
|
|
const p = num(y.p);
|
|
if (p && p > 1000) {
|
|
e.pSum += p * yn;
|
|
e.pN += yn;
|
|
}
|
|
m.nlYears.set(year, e);
|
|
if (y.k) {
|
|
const info = plateInfo(y.k);
|
|
if (info.fuel) bump(m.fuels, info.fuel, yn);
|
|
if (info.kw) m.kw.push(info.kw);
|
|
if (info.l100) m.l100.push(info.l100);
|
|
if (info.co2 != null) m.co2.push(info.co2);
|
|
}
|
|
}
|
|
for (const bd of bodiesByName.get(name) ?? []) {
|
|
const bn = Number(bd.n);
|
|
const body = bd.inrichting ? RDW_BODY[bd.inrichting] : undefined;
|
|
if (body) bump(m.bodies, body, bn);
|
|
const seats = num(bd.seats);
|
|
const doors = num(bd.doors);
|
|
if (seats && doors) {
|
|
m.seatsSum += seats * bn;
|
|
m.doorsSum += doors * bn;
|
|
m.seatN += bn;
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. Finalise.
|
|
const out = [];
|
|
for (const b of brands.values()) {
|
|
const name = MAKE_NAMES[b.key] ?? b.usName ?? titleCase(b.name);
|
|
const models = [];
|
|
for (const m of b.models.values()) {
|
|
const nlYearList = [...m.nlYears.entries()]
|
|
.sort((a, c) => a[0] - c[0])
|
|
.map(([y, e]) => ({ y, n: e.n, price: e.pN ? Math.round(e.pSum / e.pN) : null }));
|
|
const nlFirst = nlYearList[0]?.y ?? null;
|
|
const nlLast = nlYearList.at(-1)?.y ?? null;
|
|
const years = [...m.years, ...(nlFirst ? [nlFirst] : []), ...(nlLast ? [nlLast] : [])];
|
|
if (!years.length && !m.nlCount) continue;
|
|
const latestPrice = nlYearList.filter((y) => y.price).at(-1);
|
|
const minMax = (xs: number[]) =>
|
|
xs.length ? [Math.min(...xs), Math.max(...xs)] : [null, null];
|
|
const fuelTotal = [...m.fuels.values()].reduce((a, c) => a + c, 0);
|
|
const bodyTotal = [...m.bodies.values()].reduce((a, c) => a + c, 0);
|
|
// A fuel/body seen on under 1% of the samples is usually a conversion or data error.
|
|
const fuels = [...m.fuels.entries()]
|
|
.filter(([, n]) => n / fuelTotal >= 0.01)
|
|
.sort((a, c) => c[1] - a[1])
|
|
.map(([f]) => f);
|
|
const bodies = [...m.bodies.entries()]
|
|
.filter(([k, n]) => k !== 'car' && n / bodyTotal >= 0.03)
|
|
.sort((a, c) => c[1] - a[1])
|
|
.map(([k]) => k);
|
|
const [eMin, eMax] = minMax(m.engine);
|
|
const [cMin, cMax] = minMax(m.cyl);
|
|
const [kMin, kMax] = minMax(m.kw);
|
|
const [lMin, lMax] = minMax(m.l100);
|
|
const [coMin, coMax] = minMax(m.co2);
|
|
const [mMin, mMax] = minMax(m.mpg);
|
|
const displayName = m.usNames.size ? m.name : m.name;
|
|
models.push({
|
|
slug: slugify(displayName),
|
|
name: displayName,
|
|
search: [name, displayName, ...m.nlNames.keys(), ...[...m.usNames].slice(0, 40)]
|
|
.join(' ')
|
|
.toLowerCase()
|
|
.slice(0, 4000),
|
|
firstYear: years.length ? Math.min(...years) : null,
|
|
lastYear: years.length ? Math.max(...years) : null,
|
|
bodies,
|
|
sizeClasses: [...m.sizeClasses].sort(),
|
|
fuels,
|
|
drives: [...m.drives].sort(),
|
|
transmissions: [...m.trans].sort(),
|
|
engineLMin: eMin,
|
|
engineLMax: eMax,
|
|
cylindersMin: cMin,
|
|
cylindersMax: cMax,
|
|
turbo: m.usYears.size ? m.turbo : null,
|
|
powerKwMin: kMin,
|
|
powerKwMax: kMax,
|
|
l100Min: lMin,
|
|
l100Max: lMax,
|
|
co2Min: coMin == null ? null : Math.round(coMin),
|
|
co2Max: coMax == null ? null : Math.round(coMax),
|
|
mpgMin: mMin,
|
|
mpgMax: mMax,
|
|
rangeKmMax: m.rangeMi.length ? Math.round(Math.max(...m.rangeMi) * 1.609344) : null,
|
|
seats: m.seatN ? Math.round(m.seatsSum / m.seatN) : null,
|
|
doors: m.seatN ? Math.round(m.doorsSum / m.seatN) : null,
|
|
massKg: m.nlMassN ? Math.round(m.nlMassSum / m.nlMassN) : null,
|
|
nlCount: m.nlCount,
|
|
nlFirstYear: nlFirst,
|
|
nlLastYear: nlLast,
|
|
nlPrice: m.nlPriceN ? Math.round(m.nlPriceSum / m.nlPriceN) : null,
|
|
nlPriceLatest: latestPrice?.price ?? null,
|
|
nlPriceLatestYear: latestPrice?.y ?? null,
|
|
nlYears: nlYearList,
|
|
nlNames: [...m.nlNames.entries()]
|
|
.sort((a, c) => c[1] - a[1])
|
|
.slice(0, 30)
|
|
.map(([nm, n]) => ({ name: nm, n })),
|
|
usYears: [...m.usYears].sort((a, c) => a - c),
|
|
variants: m.variants,
|
|
sources: [...m.sources].sort()
|
|
});
|
|
}
|
|
if (!models.length) continue;
|
|
// Same slug twice in a brand (e.g. "C-MAX" vs "C MAX" already folded; rare leftovers).
|
|
const seen = new Map<string, number>();
|
|
for (const m of models) {
|
|
const c = seen.get(m.slug) ?? 0;
|
|
seen.set(m.slug, c + 1);
|
|
if (c) m.slug = `${m.slug}-${c + 1}`;
|
|
}
|
|
out.push({
|
|
slug: slugify(name),
|
|
name,
|
|
country: COUNTRIES[b.key] ?? null,
|
|
nlCount: b.nlCount,
|
|
firstYear: minOrNull(models.map((m) => m.firstYear)),
|
|
lastYear: maxOrNull(models.map((m) => m.lastYear)),
|
|
sources: [...b.sources].sort(),
|
|
models
|
|
});
|
|
}
|
|
// NL popularity rank across all models.
|
|
const ranked = out.flatMap((b) => b.models).filter((m) => m.nlCount > 0);
|
|
ranked.sort((a, c) => c.nlCount - a.nlCount);
|
|
const rank = new Map(ranked.map((m, i) => [m, i + 1]));
|
|
return out.map((b) => ({
|
|
...b,
|
|
models: b.models.map((m) => ({ ...m, nlRank: rank.get(m) ?? null }))
|
|
}));
|
|
}
|
|
|
|
// ---------------------------------------------------------------- writing
|
|
|
|
type NewModel = {
|
|
brand: string;
|
|
model: string;
|
|
dex: number;
|
|
url: string;
|
|
firstYear: number | null;
|
|
lastYear: number | null;
|
|
fuels: string[];
|
|
bodies: string[];
|
|
powerKw: number | null;
|
|
hp: number | null;
|
|
nlCount: number;
|
|
nlPriceLatest: number | null;
|
|
usVersions: number;
|
|
};
|
|
const added = {
|
|
brands: [] as { brand: string; url: string; models: number; nlCount: number }[],
|
|
models: [] as NewModel[]
|
|
};
|
|
|
|
async function write(brands: ReturnType<typeof build>) {
|
|
const url = process.env.DB_URL;
|
|
if (!url) throw new Error('Set DB_URL (or use --dry-run)');
|
|
const sql = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
|
try {
|
|
await sql.begin(async (tx) => {
|
|
const oldBrands = new Set(
|
|
(await tx<{ slug: string }[]>`select slug from cars.brands`).map((r) => r.slug)
|
|
);
|
|
for (const b of brands)
|
|
if (oldBrands.size && !oldBrands.has(b.slug))
|
|
added.brands.push({
|
|
brand: b.name,
|
|
url: `${SITE}/${b.slug}`,
|
|
models: b.models.length,
|
|
nlCount: b.nlCount
|
|
});
|
|
const brandIds = new Map<string, number>();
|
|
for (const b of brands) {
|
|
const [row] = await tx`
|
|
insert into cars.brands (slug, name, country, model_count, nl_count, first_year, last_year, sources)
|
|
values (${b.slug}, ${b.name}, ${b.country}, ${b.models.length}, ${b.nlCount}, ${b.firstYear}, ${b.lastYear}, ${b.sources})
|
|
on conflict (slug) do update set name = excluded.name, country = excluded.country,
|
|
model_count = excluded.model_count, nl_count = excluded.nl_count, first_year = excluded.first_year,
|
|
last_year = excluded.last_year, sources = excluded.sources
|
|
returning id`;
|
|
brandIds.set(b.slug, row.id);
|
|
}
|
|
await tx`delete from cars.brands where slug <> all(${[...brandIds.keys()]})`;
|
|
|
|
// Keep Dex numbers: existing (brand, slug) keep theirs; new models are numbered after
|
|
// the current maximum, oldest first.
|
|
const existing = await tx<{ id: number; dex: number; brand_id: number; slug: string }[]>`
|
|
select id, dex, brand_id, slug from cars.models`;
|
|
const byKey = new Map(existing.map((e) => [`${e.brand_id}|${e.slug}`, e]));
|
|
const firstRun = existing.length === 0;
|
|
let nextDex = renumber ? 1 : Math.max(0, ...existing.map((e) => e.dex)) + 1;
|
|
const all = brands
|
|
.flatMap((b) => b.models.map((m) => ({ b, m, brandId: brandIds.get(b.slug)! })))
|
|
.sort(
|
|
(x, y) =>
|
|
(x.m.firstYear ?? 9999) - (y.m.firstYear ?? 9999) ||
|
|
x.b.name.localeCompare(y.b.name) ||
|
|
x.m.name.localeCompare(y.m.name)
|
|
);
|
|
const keep: number[] = [];
|
|
// Park existing dex numbers out of the way while rows are rewritten (unique index).
|
|
await tx`update cars.models set dex = -dex`;
|
|
await tx`delete from cars.variants`;
|
|
let variantCount = 0;
|
|
for (const { m, brandId } of all) {
|
|
const prev = renumber ? undefined : byKey.get(`${brandId}|${m.slug}`);
|
|
const dex = prev?.dex ?? nextDex++;
|
|
const { variants, ...cols } = m;
|
|
const row = {
|
|
dex,
|
|
brand_id: brandId,
|
|
slug: cols.slug,
|
|
name: cols.name,
|
|
search: cols.search,
|
|
first_year: cols.firstYear,
|
|
last_year: cols.lastYear,
|
|
bodies: cols.bodies,
|
|
size_classes: cols.sizeClasses,
|
|
fuels: cols.fuels,
|
|
drives: cols.drives,
|
|
transmissions: cols.transmissions,
|
|
engine_l_min: cols.engineLMin,
|
|
engine_l_max: cols.engineLMax,
|
|
cylinders_min: cols.cylindersMin,
|
|
cylinders_max: cols.cylindersMax,
|
|
turbo: cols.turbo,
|
|
power_kw_min: cols.powerKwMin,
|
|
power_kw_max: cols.powerKwMax,
|
|
l100_min: cols.l100Min,
|
|
l100_max: cols.l100Max,
|
|
co2_min: cols.co2Min,
|
|
co2_max: cols.co2Max,
|
|
mpg_min: cols.mpgMin,
|
|
mpg_max: cols.mpgMax,
|
|
range_km_max: cols.rangeKmMax,
|
|
seats: cols.seats,
|
|
doors: cols.doors,
|
|
mass_kg: cols.massKg,
|
|
nl_count: cols.nlCount,
|
|
nl_rank: cols.nlRank,
|
|
nl_first_year: cols.nlFirstYear,
|
|
nl_last_year: cols.nlLastYear,
|
|
nl_price: cols.nlPrice,
|
|
nl_price_latest: cols.nlPriceLatest,
|
|
nl_price_latest_year: cols.nlPriceLatestYear,
|
|
nl_years: tx.json(cols.nlYears),
|
|
nl_names: tx.json(cols.nlNames),
|
|
us_years: cols.usYears,
|
|
variant_count: variants.length,
|
|
sources: cols.sources
|
|
};
|
|
const [saved] = await tx`
|
|
insert into cars.models ${tx(row)}
|
|
on conflict (brand_id, slug) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'brand_id' && k !== 'slug') as (keyof typeof row)[])}
|
|
returning id`;
|
|
keep.push(saved.id);
|
|
if (!firstRun && !byKey.has(`${brandId}|${m.slug}`)) {
|
|
const b = brands.find((x) => brandIds.get(x.slug) === brandId)!;
|
|
added.models.push({
|
|
brand: b.name,
|
|
model: m.name,
|
|
dex,
|
|
url: `${SITE}/${b.slug}/${m.slug}`,
|
|
firstYear: m.firstYear,
|
|
lastYear: m.lastYear,
|
|
fuels: m.fuels,
|
|
bodies: m.bodies,
|
|
powerKw: m.powerKwMax,
|
|
hp: m.powerKwMax ? Math.round(m.powerKwMax * 1.341) : null,
|
|
nlCount: m.nlCount,
|
|
nlPriceLatest: m.nlPriceLatest,
|
|
usVersions: variants.length
|
|
});
|
|
}
|
|
for (let i = 0; i < variants.length; i += 500) {
|
|
const chunk = variants.slice(i, i + 500).map((v) => ({
|
|
model_id: saved.id,
|
|
source_id: v.sourceId,
|
|
year: v.year,
|
|
name: v.name,
|
|
engine_l: v.engineL,
|
|
cylinders: v.cylinders,
|
|
transmission: v.transmission,
|
|
drive: v.drive,
|
|
fuel: v.fuel,
|
|
size_class: v.sizeClass,
|
|
mpg_city: v.mpgCity,
|
|
mpg_hwy: v.mpgHwy,
|
|
mpg_comb: v.mpgComb,
|
|
co2_gpm: v.co2Gpm,
|
|
range_mi: v.rangeMi,
|
|
ev_motor: v.evMotor,
|
|
turbo: v.turbo,
|
|
supercharged: v.supercharged
|
|
}));
|
|
await tx`insert into cars.variants ${tx(chunk)}`;
|
|
variantCount += chunk.length;
|
|
}
|
|
}
|
|
await tx`delete from cars.models where id <> all(${keep})`;
|
|
log(`wrote ${brands.length} brands, ${keep.length} models, ${variantCount} variants`);
|
|
});
|
|
} finally {
|
|
await sql.end();
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- main
|
|
|
|
const src = await loadSources();
|
|
log(
|
|
`fueleconomy.gov rows: ${src.fe.length}; RDW names: ${src.models.length}, years: ${src.years.length}`
|
|
);
|
|
const plates = [...new Set(src.years.map((y) => y.k).filter(Boolean))];
|
|
const fuel = await cached('rdw-fuel.json', () => {
|
|
log(`querying RDW fuel data for ${plates.length} sample cars`);
|
|
return loadFuel(plates);
|
|
});
|
|
const brands = build(src, fuel);
|
|
const models = brands.flatMap((b) => b.models);
|
|
log(
|
|
`built ${brands.length} brands, ${models.length} models, ${models.reduce((a, m) => a + m.variants.length, 0)} variants`
|
|
);
|
|
if (outFile)
|
|
await writeFile(
|
|
outFile,
|
|
JSON.stringify(
|
|
brands.map((b) => ({
|
|
...b,
|
|
models: b.models.map(({ variants, ...m }) => ({ ...m, variants: variants.length }))
|
|
}))
|
|
)
|
|
);
|
|
if (dryRun) {
|
|
const top = models.toSorted((a, c) => c.nlCount - a.nlCount).slice(0, 15);
|
|
for (const m of top)
|
|
log(` ${m.name} nl=${m.nlCount} ${m.firstYear}-${m.lastYear} ${m.fuels.join('/')}`);
|
|
} else {
|
|
await write(brands);
|
|
if (reportNew)
|
|
console.log(
|
|
JSON.stringify(
|
|
{
|
|
generatedAt: new Date().toISOString(),
|
|
totals: { brands: brands.length, models: models.length },
|
|
newBrands: added.brands,
|
|
newModels: added.models.toSorted((a, b) => b.nlCount - a.nlCount)
|
|
},
|
|
null,
|
|
2
|
|
)
|
|
);
|
|
}
|