cars: Wikimedia Commons photos (cars.photos, card heroes, model gallery + lightbox, brand hero/logo), cars:photos + cars:refresh with --report-new, fold duplicate names (Golf VI, Peugot, Mustang Mach 1, Model 3/Y, i10/i30, Ceed, Lynk & Co)
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
All checks were successful
Deploy production / deploy (push) Successful in 3m0s
This commit is contained in:
50
drizzle/20261024161853_cars_photos/migration.sql
Normal file
50
drizzle/20261024161853_cars_photos/migration.sql
Normal file
@@ -0,0 +1,50 @@
|
||||
CREATE TABLE "cars"."photos" (
|
||||
"id" serial PRIMARY KEY,
|
||||
"model_id" integer NOT NULL,
|
||||
"position" smallint NOT NULL,
|
||||
"kind" text NOT NULL,
|
||||
"file" text NOT NULL,
|
||||
"caption" text,
|
||||
"thumb" text NOT NULL,
|
||||
"width" integer NOT NULL,
|
||||
"height" integer NOT NULL,
|
||||
"author" text,
|
||||
"license" text,
|
||||
"license_url" text
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "wikidata" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "wikipedia" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "logo" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "logo_file" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "logo_author" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "logo_license" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."brands" ADD COLUMN "photo" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "wikidata" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "wikipedia" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "photo" text;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "photo_count" smallint DEFAULT 0 NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "cars"."models" ADD COLUMN "photos_checked_at" timestamp with time zone;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "photos_model_position_key" ON "cars"."photos" ("model_id","position");--> statement-breakpoint
|
||||
ALTER TABLE "cars"."photos" ADD CONSTRAINT "photos_model_id_models_id_fkey" FOREIGN KEY ("model_id") REFERENCES "cars"."models"("id") ON DELETE CASCADE;
|
||||
--> statement-breakpoint
|
||||
-- hand-written: grants + registry for the photos table
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_app') THEN
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON cars.photos TO personal_app;
|
||||
GRANT USAGE, SELECT ON SEQUENCE cars.photos_id_seq TO personal_app;
|
||||
END IF;
|
||||
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'personal_ro') THEN
|
||||
GRANT SELECT ON cars.photos TO personal_ro;
|
||||
END IF;
|
||||
END $$;
|
||||
--> statement-breakpoint
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('meta.tables') IS NOT NULL THEN
|
||||
INSERT INTO meta.tables (schema_name, table_name, slug, display_name, kind, editable, has_done, sort_order)
|
||||
VALUES ('cars','photos','cars-photos','Car photos','core',false,false,1000)
|
||||
ON CONFLICT DO NOTHING;
|
||||
END IF;
|
||||
END $$;
|
||||
20274
drizzle/20261024161853_cars_photos/snapshot.json
Normal file
20274
drizzle/20261024161853_cars_photos/snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,8 @@
|
||||
"db:studio": "drizzle-kit studio",
|
||||
"db:seed": "bun scripts/seed-catalog.ts",
|
||||
"cars:import": "bun scripts/cars-import.ts",
|
||||
"cars:photos": "bun scripts/cars-photos.ts",
|
||||
"cars:refresh": "scripts/cars-refresh.sh",
|
||||
"db:seed:cod": "bun scripts/seed-cod-catalog.ts",
|
||||
"cs-prices:ingest": "bun scripts/ingest-cs-prices.ts",
|
||||
"house:worker": "bun scripts/house-worker.ts",
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
// bun scripts/cars-import.ts --cache /tmp/cars reuse raw downloads from that folder
|
||||
// bun scripts/cars-import.ts --dry-run --out x.json also dump the built catalogue
|
||||
// bun scripts/cars-import.ts --renumber reassign Dex numbers from #1 (oldest first)
|
||||
// bun scripts/cars-import.ts --report-new JSON of brands/models new since the last run on stdout
|
||||
//
|
||||
// Sources (no API keys):
|
||||
// - fueleconomy.gov vehicles.csv: every US EPA-tested car since 1984 (make, base model,
|
||||
@@ -31,6 +32,11 @@ const dryRun = args.includes('--dry-run');
|
||||
const renumber = args.includes('--renumber');
|
||||
const cacheDir = args.includes('--cache') ? args[args.indexOf('--cache') + 1] : null;
|
||||
const outFile = args.includes('--out') ? args[args.indexOf('--out') + 1] : null;
|
||||
// --report-new: print a JSON summary of brands/models that did not exist before this run to
|
||||
// stdout (all progress logging moves to stderr so stdout stays machine-readable).
|
||||
const reportNew = args.includes('--report-new');
|
||||
const SITE = 'https://stanrunge.dev/trackers/cars';
|
||||
const log = (...a: unknown[]) => (reportNew ? console.error(...a) : console.log(...a));
|
||||
|
||||
const FE_URL = 'https://www.fueleconomy.gov/feg/epadata/vehicles.csv';
|
||||
const RDW = 'https://opendata.rdw.nl/resource';
|
||||
@@ -150,7 +156,7 @@ type RdwFuel = {
|
||||
|
||||
async function loadSources() {
|
||||
const fe = await cached('fe.json', async () => {
|
||||
console.log('fetching fueleconomy.gov vehicles.csv');
|
||||
log('fetching fueleconomy.gov vehicles.csv');
|
||||
const res = await fetch(FE_URL, { signal: AbortSignal.timeout(300_000) });
|
||||
if (!res.ok) throw new Error(`fueleconomy.gov ${res.status}`);
|
||||
return parseCsv(await res.text());
|
||||
@@ -162,7 +168,7 @@ async function loadSources() {
|
||||
...(having ? { $having: having } : {}),
|
||||
$limit: '2000000'
|
||||
});
|
||||
console.log('querying RDW aggregates');
|
||||
log('querying RDW aggregates');
|
||||
const [models, years, bodies] = await Promise.all([
|
||||
cached('rdw-models.json', () =>
|
||||
getJson<RdwModel[]>(
|
||||
@@ -385,10 +391,35 @@ function cleanTradeName(raw: string, make: string): string {
|
||||
else if (s.startsWith(`${make} `) && s.length > make.length + 1) s = s.slice(make.length + 1);
|
||||
else break;
|
||||
}
|
||||
// Misspelt make as first word ("PEUGOT 205" under PEUGEOT).
|
||||
const [w0, ...restW] = s.split(' ');
|
||||
const mk0 = make.replace(/[^A-Z]/g, '');
|
||||
if (
|
||||
restW.length &&
|
||||
mk0.length >= 5 &&
|
||||
w0 !== mk0 &&
|
||||
w0.slice(0, 4) === mk0.slice(0, 4) &&
|
||||
Math.abs(w0.length - mk0.length) <= 1
|
||||
)
|
||||
s = restW.join(' ');
|
||||
s = s.replace(/^\d{5,}\s+/, ''); // leading RDW type code, e.g. "155221 GOLF CABRIOLET"
|
||||
return s.replace(/[-\s]+$/, '').trim();
|
||||
}
|
||||
|
||||
/** Hand fixes for trade names that put the trim first ("MACH 1 MUSTANG" is a Mustang). */
|
||||
const TRADE_FIXES: Record<string, [RegExp, string][]> = {
|
||||
FORD: [
|
||||
[/^MACH ?(1|I)\b ?MUSTANG\b/, 'MUSTANG MACH 1'],
|
||||
[/^MUSTANG( II)? MACH ?(1|I|ONE)\b/, 'MUSTANG$1']
|
||||
],
|
||||
LYNKCO: [[/^LYNK ?& ?CO ?/, '']],
|
||||
KIA: [[/^CEE ?'? ?D\b/, "CEE'D"]]
|
||||
};
|
||||
function tradeFix(makeK: string, name: string): string {
|
||||
for (const [re, to] of TRADE_FIXES[makeK] ?? []) name = name.replace(re, to);
|
||||
return name.trim();
|
||||
}
|
||||
|
||||
/** Brand-specific folding of engine-coded names into the model family. */
|
||||
function familyRule(makeK: string, name: string): string | null {
|
||||
if (makeK === 'MERCEDESBENZ') {
|
||||
@@ -650,7 +681,7 @@ function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
|
||||
if (!r.handelsbenaming || /^(ONBEKEND|NIET|UNKNOWN|-|\?)/i.test(r.handelsbenaming)) continue;
|
||||
if (!brands.has(mk) && (makeTotals.get(mk) ?? 0) < MIN_NL_MAKE) continue;
|
||||
if (/^(ONBEKEND|DIVERSEN|ZELFBOUW|KIT ?CAR|EIGENBOUW)/.test(mk)) continue;
|
||||
const clean = cleanTradeName(r.handelsbenaming, r.merk.toUpperCase());
|
||||
const clean = tradeFix(mk, cleanTradeName(r.handelsbenaming, r.merk.toUpperCase()));
|
||||
if (!clean || !/[A-Z0-9]/.test(clean) || clean.includes('*') || /^\d{5,}$/.test(clean))
|
||||
continue;
|
||||
rows.push({ ...r, clean, makeK: mk });
|
||||
@@ -658,14 +689,29 @@ function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
|
||||
// A trade name folds into a shorter family name when what follows is trim, body, engine
|
||||
// or code noise ("205 GTI 1.9 U9" -> 205, "Astra Sports Tourer" -> Astra) but not when it
|
||||
// names a different car ("Yaris Cross", "Corolla Verso" stay their own model).
|
||||
const isNoise = (w: string) =>
|
||||
/[\d./]/.test(w) || w.length === 1 || TRIM_WORDS.has(w.replace(/[^A-Z]/g, ''));
|
||||
const ROMAN = /^(II|III|IV|V|VI|VII|VIII|IX|X)$/;
|
||||
const isNoise = (w: string, prev = '') =>
|
||||
/[\d./]/.test(w) ||
|
||||
w.length === 1 ||
|
||||
TRIM_WORDS.has(w.replace(/[^A-Z]/g, '')) ||
|
||||
(ROMAN.test(w) && !/^(MK|MARK|SERIE|SERIES|SPORT)$/.test(prev));
|
||||
// Words up to the first noise word: "IX1 EDRIVE20" -> "IX1", "ID.7 TOURER PRO" -> "ID.7".
|
||||
// A short or generic first word needs the next one: "MODEL 3", "I 10", "CEE D", "P 1800".
|
||||
const STEM = /^([A-Z]{1,3}|MODEL|CLASS|TYPE|SERIE|SERIES)$/;
|
||||
const needsNext = (words: string[]) =>
|
||||
words.length > 1 &&
|
||||
STEM.test(words[0]) &&
|
||||
!/[./]/.test(words[1]) &&
|
||||
(words[0].length === 1 || words[1].length <= 2);
|
||||
const core = (clean: string) => {
|
||||
const words = clean.split(' ');
|
||||
let i = 1;
|
||||
while (i < words.length && !isNoise(words[i])) i++;
|
||||
return words.slice(0, i).join(' ');
|
||||
let i = needsNext(words) ? 2 : 1;
|
||||
while (i < words.length && !isNoise(words[i], words[i - 1])) i++;
|
||||
const out = words.slice(0, i);
|
||||
// "IX 20" -> "IX20", "TR 3" -> "TR3", "P 1800" -> "P1800" (but "MODEL 3" keeps its space).
|
||||
if (i >= 2 && words[0].length <= 2 && /^\d/.test(words[1]))
|
||||
out.splice(0, 2, words[0] + words[1]);
|
||||
return out.join(' ');
|
||||
};
|
||||
const candidates = new Map<string, Map<string, string>>(); // makeK -> cmpKey -> display upper name
|
||||
const addCandidate = (mk: string, name: string) => {
|
||||
@@ -693,7 +739,7 @@ function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
|
||||
const words = clean.split(' ');
|
||||
for (let i = 1; i < words.length; i++) {
|
||||
const hit = cands.get(cmpKey(words.slice(0, i).join(' ')));
|
||||
if (hit && isNoise(words[i])) return hit;
|
||||
if (hit && isNoise(words[i], words[i - 1])) return hit;
|
||||
}
|
||||
return cands.get(cmpKey(clean)) ?? cands.get(cmpKey(core(clean))) ?? null;
|
||||
}
|
||||
@@ -906,12 +952,43 @@ function build(src: Awaited<ReturnType<typeof loadSources>>, fuel: RdwFuel[]) {
|
||||
|
||||
// ---------------------------------------------------------------- writing
|
||||
|
||||
type NewModel = {
|
||||
brand: string;
|
||||
model: string;
|
||||
dex: number;
|
||||
url: string;
|
||||
firstYear: number | null;
|
||||
lastYear: number | null;
|
||||
fuels: string[];
|
||||
bodies: string[];
|
||||
powerKw: number | null;
|
||||
hp: number | null;
|
||||
nlCount: number;
|
||||
nlPriceLatest: number | null;
|
||||
usVersions: number;
|
||||
};
|
||||
const added = {
|
||||
brands: [] as { brand: string; url: string; models: number; nlCount: number }[],
|
||||
models: [] as NewModel[]
|
||||
};
|
||||
|
||||
async function write(brands: ReturnType<typeof build>) {
|
||||
const url = process.env.DB_URL;
|
||||
if (!url) throw new Error('Set DB_URL (or use --dry-run)');
|
||||
const sql = postgres(url, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
||||
try {
|
||||
await sql.begin(async (tx) => {
|
||||
const oldBrands = new Set(
|
||||
(await tx<{ slug: string }[]>`select slug from cars.brands`).map((r) => r.slug)
|
||||
);
|
||||
for (const b of brands)
|
||||
if (oldBrands.size && !oldBrands.has(b.slug))
|
||||
added.brands.push({
|
||||
brand: b.name,
|
||||
url: `${SITE}/${b.slug}`,
|
||||
models: b.models.length,
|
||||
nlCount: b.nlCount
|
||||
});
|
||||
const brandIds = new Map<string, number>();
|
||||
for (const b of brands) {
|
||||
const [row] = await tx`
|
||||
@@ -930,6 +1007,7 @@ async function write(brands: ReturnType<typeof build>) {
|
||||
const existing = await tx<{ id: number; dex: number; brand_id: number; slug: string }[]>`
|
||||
select id, dex, brand_id, slug from cars.models`;
|
||||
const byKey = new Map(existing.map((e) => [`${e.brand_id}|${e.slug}`, e]));
|
||||
const firstRun = existing.length === 0;
|
||||
let nextDex = renumber ? 1 : Math.max(0, ...existing.map((e) => e.dex)) + 1;
|
||||
const all = brands
|
||||
.flatMap((b) => b.models.map((m) => ({ b, m, brandId: brandIds.get(b.slug)! })))
|
||||
@@ -996,6 +1074,24 @@ async function write(brands: ReturnType<typeof build>) {
|
||||
on conflict (brand_id, slug) do update set ${tx(row, Object.keys(row).filter((k) => k !== 'brand_id' && k !== 'slug') as (keyof typeof row)[])}
|
||||
returning id`;
|
||||
keep.push(saved.id);
|
||||
if (!firstRun && !byKey.has(`${brandId}|${m.slug}`)) {
|
||||
const b = brands.find((x) => brandIds.get(x.slug) === brandId)!;
|
||||
added.models.push({
|
||||
brand: b.name,
|
||||
model: m.name,
|
||||
dex,
|
||||
url: `${SITE}/${b.slug}/${m.slug}`,
|
||||
firstYear: m.firstYear,
|
||||
lastYear: m.lastYear,
|
||||
fuels: m.fuels,
|
||||
bodies: m.bodies,
|
||||
powerKw: m.powerKwMax,
|
||||
hp: m.powerKwMax ? Math.round(m.powerKwMax * 1.341) : null,
|
||||
nlCount: m.nlCount,
|
||||
nlPriceLatest: m.nlPriceLatest,
|
||||
usVersions: variants.length
|
||||
});
|
||||
}
|
||||
for (let i = 0; i < variants.length; i += 500) {
|
||||
const chunk = variants.slice(i, i + 500).map((v) => ({
|
||||
model_id: saved.id,
|
||||
@@ -1022,7 +1118,7 @@ async function write(brands: ReturnType<typeof build>) {
|
||||
}
|
||||
}
|
||||
await tx`delete from cars.models where id <> all(${keep})`;
|
||||
console.log(`wrote ${brands.length} brands, ${keep.length} models, ${variantCount} variants`);
|
||||
log(`wrote ${brands.length} brands, ${keep.length} models, ${variantCount} variants`);
|
||||
});
|
||||
} finally {
|
||||
await sql.end();
|
||||
@@ -1032,17 +1128,17 @@ async function write(brands: ReturnType<typeof build>) {
|
||||
// ---------------------------------------------------------------- main
|
||||
|
||||
const src = await loadSources();
|
||||
console.log(
|
||||
log(
|
||||
`fueleconomy.gov rows: ${src.fe.length}; RDW names: ${src.models.length}, years: ${src.years.length}`
|
||||
);
|
||||
const plates = [...new Set(src.years.map((y) => y.k).filter(Boolean))];
|
||||
const fuel = await cached('rdw-fuel.json', () => {
|
||||
console.log(`querying RDW fuel data for ${plates.length} sample cars`);
|
||||
log(`querying RDW fuel data for ${plates.length} sample cars`);
|
||||
return loadFuel(plates);
|
||||
});
|
||||
const brands = build(src, fuel);
|
||||
const models = brands.flatMap((b) => b.models);
|
||||
console.log(
|
||||
log(
|
||||
`built ${brands.length} brands, ${models.length} models, ${models.reduce((a, m) => a + m.variants.length, 0)} variants`
|
||||
);
|
||||
if (outFile)
|
||||
@@ -1058,5 +1154,20 @@ if (outFile)
|
||||
if (dryRun) {
|
||||
const top = models.toSorted((a, c) => c.nlCount - a.nlCount).slice(0, 15);
|
||||
for (const m of top)
|
||||
console.log(` ${m.name} nl=${m.nlCount} ${m.firstYear}-${m.lastYear} ${m.fuels.join('/')}`);
|
||||
} else await write(brands);
|
||||
log(` ${m.name} nl=${m.nlCount} ${m.firstYear}-${m.lastYear} ${m.fuels.join('/')}`);
|
||||
} else {
|
||||
await write(brands);
|
||||
if (reportNew)
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
generatedAt: new Date().toISOString(),
|
||||
totals: { brands: brands.length, models: models.length },
|
||||
newBrands: added.brands,
|
||||
newModels: added.models.toSorted((a, b) => b.nlCount - a.nlCount)
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
767
scripts/cars-photos.ts
Normal file
767
scripts/cars-photos.ts
Normal file
@@ -0,0 +1,767 @@
|
||||
// Finds freely licensed photos for the car pokedex (/trackers/cars) on Wikimedia Commons.
|
||||
//
|
||||
// DB_URL=... bun scripts/cars-photos.ts match every model, replace all photos
|
||||
// DB_URL=... bun scripts/cars-photos.ts --missing only models that have no photos yet
|
||||
// DB_URL=... bun scripts/cars-photos.ts --enrich r.json add photos to a cars-import
|
||||
// --report-new file, print it to stdout
|
||||
// bun scripts/cars-photos.ts --dry-run --models built.json [--cache dir] [--out x.json]
|
||||
//
|
||||
// How a model gets matched (low-confidence matches are skipped, never guessed):
|
||||
// 1. One SPARQL query lists every Wikidata item that is an instance of "car model"
|
||||
// (Q3231690) with its English label/aliases, enwiki article, manufacturer (P176),
|
||||
// brand (P1716), image (P18), Commons category (P373) and parent series (P179).
|
||||
// 2. A model matches an item when "<brand> <model>" equals the item's label, alias or
|
||||
// article title, or when "<model>" does and the item's manufacturer/brand is that brand.
|
||||
// Among several hits the series root with an article and the most sitelinks wins.
|
||||
// 3. Photos, in order: the item's own image, the images of its generations (items that
|
||||
// are P179 "part of the series" of it, newest first), images used in the enwiki article
|
||||
// and files in the Commons category. Article/category files must mention the model in
|
||||
// their file name; logos, maps, drawings, SVGs and small files are dropped.
|
||||
// Nothing is downloaded: we store thumbnail URLs (standard widths) plus author and licence.
|
||||
import postgres from 'postgres';
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const flag = (name: string) => args.includes(name);
|
||||
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
||||
const dryRun = flag('--dry-run');
|
||||
const missingOnly = flag('--missing');
|
||||
const enrichFile = opt('--enrich');
|
||||
const modelsFile = opt('--models');
|
||||
const cacheDir = opt('--cache');
|
||||
const outFile = opt('--out');
|
||||
const MAX_PHOTOS = 12;
|
||||
const THUMB = 960; // Wikimedia standard thumbnail widths: 120 250 330 500 960 1280 1920
|
||||
const UA =
|
||||
'StanRungeCarDex/1.0 (https://stanrunge.dev/trackers/cars; car pokedex photo matcher) bun';
|
||||
const log = (...a: unknown[]) => console.error(...a);
|
||||
|
||||
// ---------------------------------------------------------------- http
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
let apiCache: Record<string, unknown> = {};
|
||||
let apiCacheDirty = 0;
|
||||
async function loadApiCache() {
|
||||
if (!cacheDir) return;
|
||||
await mkdir(cacheDir, { recursive: true });
|
||||
try {
|
||||
apiCache = JSON.parse(await readFile(join(cacheDir, 'api.json'), 'utf8'));
|
||||
} catch {
|
||||
apiCache = {};
|
||||
}
|
||||
}
|
||||
async function saveApiCache(force = false) {
|
||||
if (!cacheDir || (!force && apiCacheDirty < 200)) return;
|
||||
apiCacheDirty = 0;
|
||||
await writeFile(join(cacheDir, 'api.json'), JSON.stringify(apiCache));
|
||||
}
|
||||
|
||||
async function getJson<T>(url: string, init?: RequestInit): Promise<T> {
|
||||
const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
|
||||
if (key in apiCache) return apiCache[key] as T;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const res = await fetch(url, {
|
||||
...init,
|
||||
headers: {
|
||||
'user-agent': UA,
|
||||
'api-user-agent': UA,
|
||||
accept: 'application/json',
|
||||
...init?.headers
|
||||
}
|
||||
});
|
||||
const retryAfter = Number(res.headers.get('retry-after')) || 0;
|
||||
if (res.ok) {
|
||||
const body = (await res.json()) as T & { error?: { code?: string } };
|
||||
if (body?.error?.code === 'maxlag' && attempt < 8) {
|
||||
await sleep(Math.max(retryAfter, 5) * 1000);
|
||||
continue;
|
||||
}
|
||||
apiCache[key] = body;
|
||||
apiCacheDirty++;
|
||||
await saveApiCache();
|
||||
return body;
|
||||
}
|
||||
if ((res.status === 429 || res.status >= 500) && attempt < 6) {
|
||||
await sleep(Math.max(retryAfter * 1000, 2000 * 2 ** attempt));
|
||||
continue;
|
||||
}
|
||||
throw new Error(`${res.status} ${url.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
|
||||
let lastCall = 0;
|
||||
/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
|
||||
async function mw(host: string, params: Record<string, string>) {
|
||||
const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
|
||||
const key = `https://${host}/w/api.php?${body}`;
|
||||
if (!(key in apiCache)) {
|
||||
const wait = lastCall + 150 - Date.now();
|
||||
if (wait > 0) await sleep(wait);
|
||||
lastCall = Date.now();
|
||||
}
|
||||
return getJson<MwResponse>(key);
|
||||
}
|
||||
type MwPage = {
|
||||
title: string;
|
||||
missing?: boolean;
|
||||
images?: { title: string }[];
|
||||
imageinfo?: ImageInfo[];
|
||||
};
|
||||
type ImageInfo = {
|
||||
width: number;
|
||||
height: number;
|
||||
mime: string;
|
||||
thumburl?: string;
|
||||
descriptionurl?: string;
|
||||
extmetadata?: Record<string, { value: string }>;
|
||||
};
|
||||
type MwResponse = {
|
||||
query?: { pages?: MwPage[]; normalized?: { from: string; to: string }[] };
|
||||
continue?: Record<string, string>;
|
||||
};
|
||||
|
||||
async function sparql<T>(query: string): Promise<T[]> {
|
||||
const res = await getJson<{ results: { bindings: Record<string, { value: string }>[] } }>(
|
||||
'https://query.wikidata.org/sparql',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/x-www-form-urlencoded',
|
||||
accept: 'application/sparql-results+json'
|
||||
},
|
||||
body: new URLSearchParams({ query }).toString()
|
||||
}
|
||||
);
|
||||
return res.results.bindings.map(
|
||||
(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- wikidata
|
||||
|
||||
const qid = (uri: string) => uri.split('/').at(-1)!;
|
||||
const fileName = (uri: string) =>
|
||||
decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
|
||||
const cmp = (s: string) =>
|
||||
s
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toUpperCase()
|
||||
.replace(/&/g, ' AND ')
|
||||
.replace(/[^A-Z0-9]/g, '');
|
||||
const tokens = (s: string) =>
|
||||
s
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toUpperCase()
|
||||
.replace(/&/g, ' ')
|
||||
.split(/[^A-Z0-9]+/)
|
||||
.filter(Boolean);
|
||||
|
||||
type WdItem = {
|
||||
id: string;
|
||||
label: string;
|
||||
names: Set<string>; // cmp() of label, aliases, enwiki title
|
||||
primary: Set<string>; // cmp() of label and enwiki title only
|
||||
series: boolean; // "car model series" (Q59773381), i.e. spans generations
|
||||
article: string | null;
|
||||
makers: Set<string>;
|
||||
image: string | null;
|
||||
category: string | null;
|
||||
parents: Set<string>;
|
||||
links: number;
|
||||
year: number | null;
|
||||
};
|
||||
|
||||
const split = (s: string | undefined) => (s ? s.split('|').filter(Boolean) : []);
|
||||
|
||||
async function loadWikidata() {
|
||||
type Row = Record<string, string | undefined>;
|
||||
log('wikidata: car models');
|
||||
const rows = await sparql<Row>(`
|
||||
SELECT ?item (SAMPLE(?l) AS ?label) (SAMPLE(?a) AS ?article) (SAMPLE(?img) AS ?image) (SAMPLE(?c) AS ?cat)
|
||||
(SAMPLE(?links) AS ?sitelinks) (MIN(YEAR(?inc)) AS ?year)
|
||||
(GROUP_CONCAT(DISTINCT STRAFTER(STR(?m), "entity/"); separator="|") AS ?makers)
|
||||
(GROUP_CONCAT(DISTINCT STRAFTER(STR(?p), "entity/"); separator="|") AS ?parents)
|
||||
(GROUP_CONCAT(DISTINCT STRAFTER(STR(?cls), "entity/"); separator="|") AS ?classes)
|
||||
WHERE {
|
||||
VALUES ?cls { wd:Q3231690 wd:Q59773381 }
|
||||
?item wdt:P31 ?cls .
|
||||
OPTIONAL { ?item rdfs:label ?l FILTER(LANG(?l) = "en") }
|
||||
OPTIONAL { ?a schema:about ?item; schema:isPartOf <https://en.wikipedia.org/> }
|
||||
OPTIONAL { ?item wdt:P18 ?img }
|
||||
OPTIONAL { ?item wdt:P373 ?c }
|
||||
OPTIONAL { ?item wikibase:sitelinks ?links }
|
||||
OPTIONAL { ?item wdt:P571|wdt:P580|wdt:P5444 ?inc }
|
||||
OPTIONAL { ?item wdt:P176|wdt:P1716 ?m }
|
||||
OPTIONAL { ?item wdt:P179 ?p }
|
||||
} GROUP BY ?item`);
|
||||
log(`wikidata: ${rows.length} items; aliases`);
|
||||
const aliases = await sparql<Row>(`
|
||||
SELECT ?item ?alias WHERE { VALUES ?cls { wd:Q3231690 wd:Q59773381 } ?item wdt:P31 ?cls ; skos:altLabel ?alias FILTER(LANG(?alias) = "en") }`);
|
||||
const items = new Map<string, WdItem>();
|
||||
for (const r of rows) {
|
||||
const id = qid(r.item!);
|
||||
const article = r.article
|
||||
? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
|
||||
: null;
|
||||
const label = r.label ?? article ?? id;
|
||||
items.set(id, {
|
||||
id,
|
||||
label,
|
||||
names: new Set([cmp(label), ...(article ? [cmp(article.replace(/\s*\(.*\)$/, ''))] : [])]),
|
||||
primary: new Set([cmp(label), ...(article ? [cmp(article.replace(/\s*\(.*\)$/, ''))] : [])]),
|
||||
series: split(r.classes).includes('Q59773381'),
|
||||
article,
|
||||
makers: new Set(split(r.makers)),
|
||||
image: r.image ? fileName(r.image) : null,
|
||||
category: r.cat ?? null,
|
||||
parents: new Set(split(r.parents)),
|
||||
links: Number(r.sitelinks ?? 0),
|
||||
year: r.year ? Number(r.year) : null
|
||||
});
|
||||
}
|
||||
for (const a of aliases) items.get(qid(a.item!))?.names.add(cmp(a.alias!));
|
||||
const makerIds = [...new Set([...items.values()].flatMap((i) => [...i.makers]))];
|
||||
log(`wikidata: ${makerIds.length} manufacturers/brands`);
|
||||
const makers = new Map<
|
||||
string,
|
||||
{ id: string; label: string; logo: string | null; article: string | null }
|
||||
>();
|
||||
for (let i = 0; i < makerIds.length; i += 400) {
|
||||
const chunk = makerIds.slice(i, i + 400);
|
||||
const res = await sparql<Row>(`
|
||||
SELECT ?m (SAMPLE(?l) AS ?label) (SAMPLE(?logo) AS ?logo) (SAMPLE(?a) AS ?article) WHERE {
|
||||
VALUES ?m { ${chunk.map((q) => `wd:${q}`).join(' ')} }
|
||||
OPTIONAL { ?m rdfs:label ?l FILTER(LANG(?l) = "en") }
|
||||
OPTIONAL { ?m wdt:P154 ?logo }
|
||||
OPTIONAL { ?a schema:about ?m; schema:isPartOf <https://en.wikipedia.org/> }
|
||||
} GROUP BY ?m`);
|
||||
for (const r of res) {
|
||||
const id = qid(r.m!);
|
||||
makers.set(id, {
|
||||
id,
|
||||
label: r.label ?? id,
|
||||
logo: r.logo ? fileName(r.logo) : null,
|
||||
article: r.article
|
||||
? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
|
||||
: null
|
||||
});
|
||||
}
|
||||
}
|
||||
const children = new Map<string, WdItem[]>();
|
||||
for (const item of items.values())
|
||||
for (const p of item.parents) children.set(p, [...(children.get(p) ?? []), item]);
|
||||
return { items, makers, children };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- matching
|
||||
|
||||
type Target = {
|
||||
id: number | null;
|
||||
brand: string;
|
||||
brandSlug: string;
|
||||
model: string;
|
||||
slug: string;
|
||||
nl: number;
|
||||
photos: number;
|
||||
};
|
||||
type Match = { item: WdItem; via: string };
|
||||
|
||||
/** Popular models whose Dutch/registration name differs from the Wikipedia one. */
|
||||
const ALIASES: Record<string, string> = {
|
||||
'MINI|Cooper': 'Mini Hatch',
|
||||
'MINI|Countryman Cooper': 'Mini Countryman',
|
||||
'MINI|Clubman Cooper': 'Mini Clubman',
|
||||
'Renault|Megane Scenic': 'Renault Scénic',
|
||||
'Volvo|EX40': 'Volvo XC40',
|
||||
'Volvo|EC40': 'Volvo C40',
|
||||
'Hyundai|Atos-Prime': 'Hyundai Atos',
|
||||
'Suzuki|Wagon': 'Suzuki Wagon R',
|
||||
'Lynk & Co|01': 'Lynk & Co 01',
|
||||
"Kia|Cee'd": 'Kia Ceed'
|
||||
};
|
||||
|
||||
function matcher(wd: Awaited<ReturnType<typeof loadWikidata>>) {
|
||||
const byName = new Map<string, WdItem[]>();
|
||||
for (const item of wd.items.values())
|
||||
for (const n of item.names) byName.set(n, [...(byName.get(n) ?? []), item]);
|
||||
const brandMakers = new Map<string, Set<string>>();
|
||||
const makersOf = (brand: string) => {
|
||||
let set = brandMakers.get(brand);
|
||||
if (set) return set;
|
||||
const bt = tokens(brand);
|
||||
set = new Set(
|
||||
[...wd.makers.values()]
|
||||
.filter((m) => {
|
||||
const mt = tokens(m.label);
|
||||
return bt.length && bt.every((t, i) => mt[i] === t);
|
||||
})
|
||||
.map((m) => m.id)
|
||||
);
|
||||
brandMakers.set(brand, set);
|
||||
return set;
|
||||
};
|
||||
// Prefer the item that is *called* this (not just aliased), spans generations, has an
|
||||
// article, is a series root and is well known.
|
||||
const score = (i: WdItem, names: string[]) =>
|
||||
(names.some((n) => i.primary.has(n)) ? 4000 : 0) +
|
||||
(i.series ? 800 : 0) +
|
||||
Math.min(10, wd.children.get(i.id)?.length ?? 0) * 60 +
|
||||
(i.article ? 1000 : 0) +
|
||||
(i.parents.size ? 0 : 500) +
|
||||
i.links;
|
||||
return {
|
||||
makersOf,
|
||||
match(t: Target): Match | null {
|
||||
const makers = makersOf(t.brand);
|
||||
const alias = ALIASES[`${t.brand}|${t.model}`];
|
||||
const bm = [
|
||||
...(byName.get(cmp(`${t.brand} ${t.model}`)) ?? []),
|
||||
...(alias ? (byName.get(cmp(alias)) ?? []) : [])
|
||||
];
|
||||
const m = byName.get(cmp(t.model)) ?? [];
|
||||
const ok = [
|
||||
...bm.filter((i) => !i.makers.size || [...i.makers].some((x) => makers.has(x))),
|
||||
...m.filter((i) => [...i.makers].some((x) => makers.has(x)))
|
||||
];
|
||||
// An item whose maker is known but belongs to someone else is never used.
|
||||
if (!ok.length) return null;
|
||||
const names = [cmp(`${t.brand} ${t.model}`), cmp(t.model), ...(alias ? [cmp(alias)] : [])];
|
||||
const best = [...new Set(ok)].sort((a, b) => score(b, names) - score(a, names))[0];
|
||||
return { item: best, via: bm.includes(best) ? 'brand+model name' : 'model name + maker' };
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- photos
|
||||
|
||||
export type Photo = {
|
||||
file: string;
|
||||
kind: 'lead' | 'generation' | 'article' | 'commons' | 'logo';
|
||||
caption: string | null;
|
||||
thumb: string;
|
||||
width: number;
|
||||
height: number;
|
||||
author: string | null;
|
||||
license: string | null;
|
||||
licenseUrl: string | null;
|
||||
};
|
||||
|
||||
const BAD_FILE =
|
||||
/\.(svg|gif|tiff?|pdf|djvu|webm|ogv)$|logo|emblem|badge|wordmark|map\b|karte|locator|diagram|chart|graph|schema|schematic|drawing|sketch|blueprint|patent|icon|flag|coat of arms|wappen|sign\b|signature|symbol|plan\b|cutaway|silhouette|commons-|wiki|stub|poster|advert|brochure|stamp|timeline|people|crowd|queue|in line|factory|assembly|production line|plant\b|werk\b|museum entrance|dealership|showroom sign|\btoys?\b|model car|lego|scale model|diecast/i;
|
||||
const stripHtml = (s: string) =>
|
||||
s
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/�?39;/g, "'")
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
const infoCache = new Map<string, ImageInfo | null>();
|
||||
async function imageInfo(files: string[], width = THUMB) {
|
||||
const todo = [...new Set(files)].filter((f) => !infoCache.has(`${width}|${f}`));
|
||||
for (let i = 0; i < todo.length; i += 50) {
|
||||
const chunk = todo.slice(i, i + 50);
|
||||
const res = await mw('commons.wikimedia.org', {
|
||||
action: 'query',
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|size|mime|extmetadata',
|
||||
iiurlwidth: String(width),
|
||||
iiextmetadatafilter: 'Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree',
|
||||
titles: chunk.map((f) => `File:${f}`).join('|')
|
||||
});
|
||||
const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
|
||||
for (const p of res.query?.pages ?? []) {
|
||||
const original = (norm.get(p.title) ?? p.title).replace(/^File:/, '');
|
||||
infoCache.set(`${width}|${original}`, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]);
|
||||
}
|
||||
for (const f of chunk)
|
||||
if (!infoCache.has(`${width}|${f}`)) infoCache.set(`${width}|${f}`, null);
|
||||
}
|
||||
}
|
||||
function toPhoto(
|
||||
file: string,
|
||||
kind: Photo['kind'],
|
||||
caption: string | null,
|
||||
width = THUMB
|
||||
): Photo | null {
|
||||
const info = infoCache.get(`${width}|${file}`);
|
||||
if (!info?.thumburl) return null;
|
||||
const meta = info.extmetadata ?? {};
|
||||
if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
|
||||
const license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
|
||||
if (!license || /fair use|non-free/i.test(license)) return null;
|
||||
if (kind !== 'logo') {
|
||||
if (!/^image\/(jpeg|png|webp)$/.test(info.mime)) return null;
|
||||
if (info.width < THUMB || info.height < 400) return null;
|
||||
if (info.width / info.height > 3 || info.height / info.width > 1.6) return null;
|
||||
}
|
||||
const author = meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null;
|
||||
return {
|
||||
file,
|
||||
kind,
|
||||
caption,
|
||||
thumb: info.thumburl,
|
||||
width: info.width,
|
||||
height: info.height,
|
||||
author,
|
||||
license,
|
||||
licenseUrl: meta.LicenseUrl?.value ?? null
|
||||
};
|
||||
}
|
||||
|
||||
async function articleImages(titles: string[]) {
|
||||
const out = new Map<string, string[]>();
|
||||
const todo = [...new Set(titles)];
|
||||
for (let i = 0; i < todo.length; i += 50) {
|
||||
const chunk = todo.slice(i, i + 50);
|
||||
let cont: Record<string, string> = {};
|
||||
for (let guard = 0; guard < 30; guard++) {
|
||||
const res = await mw('en.wikipedia.org', {
|
||||
action: 'query',
|
||||
prop: 'images',
|
||||
imlimit: 'max',
|
||||
redirects: '1',
|
||||
titles: chunk.join('|'),
|
||||
...cont
|
||||
});
|
||||
const norm = new Map<string, string>();
|
||||
for (const n of [
|
||||
...(res.query?.normalized ?? []),
|
||||
...((res.query as { redirects?: { from: string; to: string }[] })?.redirects ?? [])
|
||||
])
|
||||
norm.set(n.to, norm.get(n.from) ?? n.from);
|
||||
for (const p of res.query?.pages ?? []) {
|
||||
const key = norm.get(p.title) ?? p.title;
|
||||
out.set(key, [
|
||||
...(out.get(key) ?? []),
|
||||
...(p.images ?? []).map((im) => im.title.replace(/^File:/, ''))
|
||||
]);
|
||||
}
|
||||
if (!res.continue) break;
|
||||
cont = res.continue;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function categoryFiles(category: string): Promise<string[]> {
|
||||
const res = await mw('commons.wikimedia.org', {
|
||||
action: 'query',
|
||||
list: 'categorymembers',
|
||||
cmtitle: `Category:${category}`,
|
||||
cmtype: 'file',
|
||||
cmlimit: '60'
|
||||
});
|
||||
return ((res.query as { categorymembers?: { title: string }[] })?.categorymembers ?? []).map(
|
||||
(m) => m.title.replace(/^File:/, '')
|
||||
);
|
||||
}
|
||||
|
||||
const GENERIC = new Set(
|
||||
'SEDAN SALOON COUPE HATCHBACK WAGON ESTATE TOURING VARIANT CONVERTIBLE CABRIOLET CABRIO ROADSTER SPIDER SPYDER VAN PLUS SPORT SPORTS SERIES CLASS MODEL GRAND NEW SUPER LONG SHORT DOOR DOORS FACELIFT GENERATION FIRST SECOND THIRD FOURTH FIFTH SIXTH SEVENTH EIGHTH NINTH CAR AUTO HYBRID ELECTRIC'.split(
|
||||
' '
|
||||
)
|
||||
);
|
||||
/** Words that identify a generation ("E46", "W205", "Mk7", "XP210"), not the brand or body style. */
|
||||
function distinctive(label: string, t: Target): string[] {
|
||||
const skip = new Set([...tokens(t.brand), ...tokens(t.model)]);
|
||||
return tokens(label).filter(
|
||||
(w) =>
|
||||
!skip.has(w) &&
|
||||
!GENERIC.has(w) &&
|
||||
(/\d/.test(w) ? w.length >= 3 : w.length >= 4) &&
|
||||
!/^\d{4}$/.test(w)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Does the file name mention this model? ("Peugeot 208 GT.jpg" for 208, "BMW E46.jpg" for the
|
||||
* 3 Series via its generation codes). Files that name a longer sibling model of the same brand
|
||||
* ("Golf Sportsvan" on the Golf, "Mustang Mach-E" on the Mustang) are rejected.
|
||||
*/
|
||||
function mentions(
|
||||
file: string,
|
||||
t: Target,
|
||||
item: WdItem,
|
||||
genWords: string[],
|
||||
siblings: string[]
|
||||
): boolean {
|
||||
const f = cmp(file);
|
||||
if (siblings.some((s) => f.includes(s))) return false;
|
||||
const model = cmp(t.model);
|
||||
const short = model.length < 3 || /^\d+$/.test(model);
|
||||
if (f.includes(model) && (!short || f.includes(cmp(t.brand)) || f.includes(cmp(item.label))))
|
||||
return true;
|
||||
if (f.includes(cmp(item.label))) return true;
|
||||
return genWords.some((w) => f.includes(w));
|
||||
}
|
||||
|
||||
async function photosFor(
|
||||
targets: Target[],
|
||||
wd: Awaited<ReturnType<typeof loadWikidata>>,
|
||||
matches: Map<Target, Match>,
|
||||
all: Target[]
|
||||
) {
|
||||
const articles = await articleImages(
|
||||
[...matches.values()].map((m) => m.item.article).filter((a): a is string => !!a)
|
||||
);
|
||||
const result = new Map<Target, Photo[]>();
|
||||
const byBrand = new Map<string, Target[]>();
|
||||
for (const t of all) byBrand.set(t.brandSlug, [...(byBrand.get(t.brandSlug) ?? []), t]);
|
||||
let done = 0;
|
||||
for (const t of targets) {
|
||||
const m = matches.get(t);
|
||||
if (!m) continue;
|
||||
const { item } = m;
|
||||
const children = wd.children.get(item.id) ?? [];
|
||||
const genWords = [...new Set(children.flatMap((c) => distinctive(c.label, t)))];
|
||||
const own = cmp(t.model);
|
||||
const siblings = (byBrand.get(t.brandSlug) ?? [])
|
||||
.map((o) => cmp(o.model))
|
||||
.filter((o) => o !== own && o.startsWith(own));
|
||||
const ok = (f: string) => !BAD_FILE.test(f) && mentions(f, t, item, genWords, siblings);
|
||||
const gens = children
|
||||
.filter((g) => g.image)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
(b.year ?? 0) - (a.year ?? 0) || b.label.localeCompare(a.label, 'en', { numeric: true })
|
||||
);
|
||||
const picks: { file: string; kind: Photo['kind']; caption: string | null }[] = [];
|
||||
if (item.image) picks.push({ file: item.image, kind: 'lead', caption: null });
|
||||
for (const g of gens
|
||||
.filter((g) => ok(g.image!) || distinctive(g.label, t).some((w) => cmp(g.image!).includes(w)))
|
||||
.slice(0, 8))
|
||||
picks.push({ file: g.image!, kind: 'generation', caption: g.label });
|
||||
const fromArticle = (item.article ? (articles.get(item.article) ?? []) : []).filter(ok);
|
||||
for (const f of fromArticle) picks.push({ file: f, kind: 'article', caption: null });
|
||||
const unique = () => [...new Map(picks.map((p) => [p.file, p])).values()];
|
||||
await imageInfo(unique().map((p) => p.file));
|
||||
let photos = unique()
|
||||
.map((p) => toPhoto(p.file, p.kind, p.caption))
|
||||
.filter((p): p is Photo => !!p);
|
||||
if (photos.length < MAX_PHOTOS && item.category) {
|
||||
const files = (await categoryFiles(item.category)).filter(ok);
|
||||
// Spread picks across the (alphabetical) category instead of taking the first few.
|
||||
const want = MAX_PHOTOS * 2;
|
||||
const step = Math.max(1, files.length / want);
|
||||
const spread = Array.from(
|
||||
{ length: Math.min(want, files.length) },
|
||||
(_, i) => files[Math.floor(i * step)]
|
||||
);
|
||||
await imageInfo(spread);
|
||||
const seen = new Set(photos.map((p) => p.file));
|
||||
for (const f of spread) {
|
||||
if (seen.has(f)) continue;
|
||||
const p = toPhoto(f, 'commons', null);
|
||||
if (p) photos.push(p);
|
||||
seen.add(f);
|
||||
}
|
||||
}
|
||||
photos = photos.slice(0, MAX_PHOTOS);
|
||||
if (photos.length) result.set(t, photos);
|
||||
if (++done % 100 === 0) log(`photos: ${done}/${matches.size} models`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- main
|
||||
|
||||
async function loadTargets(): Promise<{
|
||||
targets: Target[];
|
||||
brands: { slug: string; name: string }[];
|
||||
}> {
|
||||
if (modelsFile) {
|
||||
const built = JSON.parse(await readFile(modelsFile, 'utf8')) as {
|
||||
slug: string;
|
||||
name: string;
|
||||
models: { slug: string; name: string; nlCount: number; variants: number }[];
|
||||
}[];
|
||||
return {
|
||||
targets: built.flatMap((b) =>
|
||||
b.models.map((m) => ({
|
||||
id: null,
|
||||
brand: b.name,
|
||||
brandSlug: b.slug,
|
||||
model: m.name,
|
||||
slug: m.slug,
|
||||
nl: m.nlCount + m.variants * 50,
|
||||
photos: 0
|
||||
}))
|
||||
),
|
||||
brands: built.map((b) => ({ slug: b.slug, name: b.name }))
|
||||
};
|
||||
}
|
||||
const sql = db();
|
||||
const rows = await sql<
|
||||
{
|
||||
id: number;
|
||||
brand: string;
|
||||
brand_slug: string;
|
||||
model: string;
|
||||
slug: string;
|
||||
nl: number;
|
||||
photos: number;
|
||||
}[]
|
||||
>`
|
||||
select m.id, b.name as brand, b.slug as brand_slug, m.name as model, m.slug,
|
||||
m.nl_count + m.variant_count * 50 as nl, m.photo_count as photos
|
||||
from cars.models m join cars.brands b on b.id = m.brand_id`;
|
||||
const brands = await sql<{ slug: string; name: string }[]>`select slug, name from cars.brands`;
|
||||
return {
|
||||
targets: rows.map((r) => ({
|
||||
id: r.id,
|
||||
brand: r.brand,
|
||||
brandSlug: r.brand_slug,
|
||||
model: r.model,
|
||||
slug: r.slug,
|
||||
nl: r.nl,
|
||||
photos: r.photos
|
||||
})),
|
||||
brands
|
||||
};
|
||||
}
|
||||
|
||||
let sqlClient: ReturnType<typeof postgres> | null = null;
|
||||
function db() {
|
||||
if (!process.env.DB_URL) throw new Error('Set DB_URL (or use --dry-run --models <built.json>)');
|
||||
sqlClient ??= postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
||||
return sqlClient;
|
||||
}
|
||||
|
||||
async function enrich(file: string) {
|
||||
const report = JSON.parse(await readFile(file, 'utf8')) as {
|
||||
newModels?: { url: string; photo?: string | null }[];
|
||||
};
|
||||
const sql = db();
|
||||
for (const m of report.newModels ?? []) {
|
||||
const [, brand, model] =
|
||||
new URL(m.url).pathname.match(/\/trackers\/cars\/([^/]+)\/([^/]+)/) ?? [];
|
||||
const [row] = await sql<{ photo: string | null }[]>`
|
||||
select m.photo from cars.models m join cars.brands b on b.id = m.brand_id
|
||||
where b.slug = ${brand} and m.slug = ${model}`;
|
||||
m.photo = row?.photo ?? null;
|
||||
}
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await loadApiCache();
|
||||
if (enrichFile && !flag('--fetch')) {
|
||||
// Enrich only: look up photos already written.
|
||||
await enrich(enrichFile);
|
||||
return;
|
||||
}
|
||||
const loaded = await loadTargets();
|
||||
const { brands } = loaded;
|
||||
// --missing: only models without photos are (re)matched; siblings still come from all models.
|
||||
const targets = loaded.targets.filter((t) => !missingOnly || t.photos === 0);
|
||||
targets.sort((a, b) => b.nl - a.nl);
|
||||
const wd = await loadWikidata();
|
||||
const { match, makersOf } = matcher(wd);
|
||||
const matches = new Map<Target, Match>();
|
||||
for (const t of targets) {
|
||||
const m = match(t);
|
||||
if (m) matches.set(t, m);
|
||||
}
|
||||
log(`matched ${matches.size}/${targets.length} models to Wikidata`);
|
||||
const photos = await photosFor(targets, wd, matches, loaded.targets);
|
||||
|
||||
// Brand logos (P154 of the manufacturer/brand item whose name matches best).
|
||||
const logos = new Map<string, { photo: Photo; wikidata: string; article: string | null }>();
|
||||
const brandPick = new Map<
|
||||
string,
|
||||
{ id: string; logo: string | null; article: string | null; label: string }
|
||||
>();
|
||||
for (const b of brands) {
|
||||
const cands = [...makersOf(b.name)].map((id) => wd.makers.get(id)!).filter(Boolean);
|
||||
const exact = cands.filter((m) => cmp(m.label) === cmp(b.name));
|
||||
const pick = [...exact, ...cands].find((m) => m.logo) ?? exact[0] ?? null;
|
||||
if (pick) brandPick.set(b.slug, pick);
|
||||
}
|
||||
await imageInfo(
|
||||
[...brandPick.values()].map((p) => p.logo).filter((l): l is string => !!l),
|
||||
250
|
||||
);
|
||||
for (const [slug, p] of brandPick) {
|
||||
const photo = p.logo ? toPhoto(p.logo, 'logo', null, 250) : null;
|
||||
if (photo) logos.set(slug, { photo, wikidata: p.id, article: p.article });
|
||||
}
|
||||
await saveApiCache(true);
|
||||
|
||||
const total = [...photos.values()].reduce((a, p) => a + p.length, 0);
|
||||
log(`photos: ${photos.size} models with photos, ${total} photos, ${logos.size} brand logos`);
|
||||
if (outFile)
|
||||
await writeFile(
|
||||
outFile,
|
||||
JSON.stringify(
|
||||
targets
|
||||
.filter((t) => matches.has(t))
|
||||
.map((t) => ({
|
||||
brand: t.brand,
|
||||
model: t.model,
|
||||
item: matches.get(t)!.item.id,
|
||||
label: matches.get(t)!.item.label,
|
||||
via: matches.get(t)!.via,
|
||||
photos: (photos.get(t) ?? []).map((p) => `${p.kind}: ${p.file}`)
|
||||
}))
|
||||
)
|
||||
);
|
||||
if (dryRun) return;
|
||||
|
||||
const sql = db();
|
||||
await sql.begin(async (tx) => {
|
||||
const ids = targets.map((t) => t.id!).filter(Boolean);
|
||||
// Replace photos for every model we looked at (including ones that lost their match).
|
||||
await tx`delete from cars.photos where model_id = any(${ids})`;
|
||||
await tx`update cars.models set wikidata = null, wikipedia = null, photo = null, photo_count = 0, photos_checked_at = now() where id = any(${ids})`;
|
||||
for (const t of targets) {
|
||||
const m = matches.get(t);
|
||||
if (!m || !t.id) continue;
|
||||
const list = photos.get(t) ?? [];
|
||||
await tx`update cars.models set wikidata = ${m.item.id}, wikipedia = ${m.item.article},
|
||||
photo = ${list[0]?.thumb ?? null}, photo_count = ${list.length} where id = ${t.id}`;
|
||||
if (list.length)
|
||||
await tx`insert into cars.photos ${tx(
|
||||
list.map((p, i) => ({
|
||||
model_id: t.id,
|
||||
position: i,
|
||||
kind: p.kind,
|
||||
file: p.file,
|
||||
caption: p.caption,
|
||||
thumb: p.thumb,
|
||||
width: p.width,
|
||||
height: p.height,
|
||||
author: p.author,
|
||||
license: p.license,
|
||||
license_url: p.licenseUrl
|
||||
}))
|
||||
)}`;
|
||||
}
|
||||
{
|
||||
// --missing only fills brands that have no logo yet (e.g. brand new ones).
|
||||
for (const b of brands) {
|
||||
const l = logos.get(b.slug);
|
||||
await tx`update cars.brands set wikidata = ${l?.wikidata ?? brandPick.get(b.slug)?.id ?? null},
|
||||
wikipedia = ${l?.article ?? brandPick.get(b.slug)?.article ?? null},
|
||||
logo = ${l?.photo.thumb ?? null}, logo_file = ${l?.photo.file ?? null},
|
||||
logo_license = ${l?.photo.license ?? null}, logo_author = ${l?.photo.author ?? null}
|
||||
where slug = ${b.slug} and (${!missingOnly} or logo is null)`;
|
||||
}
|
||||
}
|
||||
// Brand hero photo: the lead photo of its most common model that has one.
|
||||
await tx`update cars.brands b set photo = (
|
||||
select m.photo from cars.models m where m.brand_id = b.id and m.photo is not null
|
||||
order by m.nl_count desc, m.variant_count desc limit 1)`;
|
||||
});
|
||||
log('written');
|
||||
if (enrichFile) await enrich(enrichFile);
|
||||
await sql.end();
|
||||
}
|
||||
|
||||
await main();
|
||||
await sqlClient?.end();
|
||||
26
scripts/cars-refresh.sh
Executable file
26
scripts/cars-refresh.sh
Executable file
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# Weekly-ish refresh of the car pokedex (/trackers/cars) on stan-vm.
|
||||
#
|
||||
# ssh stan-vm 'cd ~/dev/website-cars && git fetch -q && git checkout -q --detach origin/main \
|
||||
# && bun install --frozen-lockfile >/dev/null && scripts/cars-refresh.sh'
|
||||
#
|
||||
# 1. bun scripts/cars-import.ts --report-new rebuilds brands/models/variants from RDW +
|
||||
# fueleconomy.gov and notes which brands/models did not exist before.
|
||||
# 2. bun scripts/cars-photos.ts --missing finds Wikimedia Commons photos for models
|
||||
# that have none yet (new ones included).
|
||||
# stdout: one JSON report {generatedAt, totals, newBrands, newModels[] (with photo)}.
|
||||
# Progress goes to stderr. DB_URL is read from the running app container and never printed.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ -z "${DB_URL:-}" ]; then
|
||||
DB_URL=$(sudo docker inspect personal-website-app-1 --format '{{range .Config.Env}}{{println .}}{{end}}' |
|
||||
sed -n 's/^DB_URL=//p' | sed 's/@db:5432/@127.0.0.1:5440/')
|
||||
export DB_URL
|
||||
fi
|
||||
|
||||
report=$(mktemp /tmp/cars-report.XXXXXX.json)
|
||||
trap 'rm -f "$report"' EXIT
|
||||
|
||||
bun scripts/cars-import.ts --report-new "$@" >"$report"
|
||||
bun scripts/cars-photos.ts --missing --enrich "$report" --fetch
|
||||
@@ -6,18 +6,40 @@
|
||||
dexLabel,
|
||||
fmtCompact,
|
||||
fmtEuro,
|
||||
brandHue,
|
||||
kwToHp,
|
||||
monogram,
|
||||
thumbAt,
|
||||
thumbSrcset,
|
||||
years,
|
||||
type CarCard
|
||||
} from './shared.js';
|
||||
|
||||
let { car, current }: { car: CarCard; current: number } = $props();
|
||||
let {
|
||||
car,
|
||||
current,
|
||||
eager = false
|
||||
}: { car: CarCard; current: number; eager?: boolean } = $props();
|
||||
const href = $derived(`/trackers/cars/${car.brandSlug}/${car.slug}`);
|
||||
</script>
|
||||
|
||||
<a {href} class="card">
|
||||
<div class="top">
|
||||
<BrandBadge name={car.brandName} />
|
||||
<div class="media" style:--hue={brandHue(car.brandName)}>
|
||||
{#if car.photo}
|
||||
<img
|
||||
src={thumbAt(car.photo, 330)}
|
||||
srcset={thumbSrcset(car.photo)}
|
||||
sizes="(min-width: 1024px) 270px, (min-width: 640px) 45vw, 100vw"
|
||||
alt="{car.brandName} {car.name}"
|
||||
loading={eager ? 'eager' : 'lazy'}
|
||||
decoding="async"
|
||||
width="330"
|
||||
height="206"
|
||||
/>
|
||||
{:else}
|
||||
<span class="placeholder" aria-hidden="true">{monogram(car.brandName)}</span>
|
||||
{/if}
|
||||
<span class="badge"><BrandBadge name={car.brandName} size="sm" /></span>
|
||||
<span class="dex">{dexLabel(car.dex)}</span>
|
||||
</div>
|
||||
<div class="min-w-0">
|
||||
@@ -68,15 +90,49 @@
|
||||
border-color: var(--site-muted);
|
||||
background: var(--site-hover);
|
||||
}
|
||||
.top {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
.media {
|
||||
position: relative;
|
||||
margin: -0.9rem -0.9rem 0.1rem;
|
||||
aspect-ratio: 16 / 10;
|
||||
overflow: hidden;
|
||||
border-radius: 0.85rem 0.85rem 0 0;
|
||||
background: radial-gradient(circle at 30% 20%, hsl(var(--hue) 35% 22%), hsl(var(--hue) 30% 9%));
|
||||
}
|
||||
.media img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
transition: transform 0.3s;
|
||||
}
|
||||
.card:hover .media img {
|
||||
transform: scale(1.04);
|
||||
}
|
||||
.placeholder {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
font-size: 3rem;
|
||||
font-weight: 800;
|
||||
letter-spacing: 0.05em;
|
||||
color: hsl(var(--hue) 50% 60% / 0.25);
|
||||
}
|
||||
.badge {
|
||||
position: absolute;
|
||||
left: 0.6rem;
|
||||
bottom: 0.6rem;
|
||||
}
|
||||
.dex {
|
||||
position: absolute;
|
||||
top: 0.5rem;
|
||||
right: 0.5rem;
|
||||
font-family: ui-monospace, monospace;
|
||||
font-size: 0.75rem;
|
||||
color: var(--site-muted);
|
||||
font-size: 0.72rem;
|
||||
color: #fff;
|
||||
background: rgb(0 0 0 / 0.55);
|
||||
border-radius: 999px;
|
||||
padding: 0.05rem 0.45rem;
|
||||
backdrop-filter: blur(4px);
|
||||
}
|
||||
.brand {
|
||||
font-size: 0.75rem;
|
||||
|
||||
268
src/lib/cars/Gallery.svelte
Normal file
268
src/lib/cars/Gallery.svelte
Normal file
@@ -0,0 +1,268 @@
|
||||
<script lang="ts">
|
||||
import ChevronLeft from 'lucide-svelte/icons/chevron-left';
|
||||
import ChevronRight from 'lucide-svelte/icons/chevron-right';
|
||||
import X from 'lucide-svelte/icons/x';
|
||||
import { commonsPage, thumbAt, thumbSrcset, type CarPhoto } from './shared.js';
|
||||
|
||||
let {
|
||||
photos,
|
||||
name,
|
||||
wikipedia = null
|
||||
}: { photos: CarPhoto[]; name: string; wikipedia?: string | null } = $props();
|
||||
|
||||
let dialog: HTMLDialogElement | undefined = $state();
|
||||
let index = $state(0);
|
||||
const current = $derived(photos[index]);
|
||||
const large = (p: CarPhoto) => thumbAt(p.thumb, p.width > 1280 ? 1280 : 960);
|
||||
const alt = (p: CarPhoto, i: number) =>
|
||||
`${p.caption ?? name}, photo ${i + 1} of ${photos.length}`;
|
||||
|
||||
// Links go to the Commons file page; with JS they open the lightbox instead.
|
||||
function open(event: MouseEvent, i: number) {
|
||||
if (event.metaKey || event.ctrlKey || event.shiftKey || event.button !== 0) return;
|
||||
event.preventDefault();
|
||||
index = i;
|
||||
dialog?.showModal();
|
||||
}
|
||||
const step = (d: number) => (index = (index + d + photos.length) % photos.length);
|
||||
function onkeydown(event: KeyboardEvent) {
|
||||
if (event.key === 'ArrowRight') step(1);
|
||||
else if (event.key === 'ArrowLeft') step(-1);
|
||||
else return;
|
||||
event.preventDefault();
|
||||
}
|
||||
// Click on the backdrop (outside the figure) closes.
|
||||
function onclick(event: MouseEvent) {
|
||||
if (event.target === dialog) dialog?.close();
|
||||
}
|
||||
</script>
|
||||
|
||||
{#snippet credit(p: CarPhoto)}
|
||||
<a href={commonsPage(p.file)} rel="noopener">{p.author ?? 'Wikimedia Commons'}</a>{#if p.license}
|
||||
· {#if p.licenseUrl}<a href={p.licenseUrl} rel="noopener license">{p.license}</a
|
||||
>{:else}{p.license}{/if}{/if}
|
||||
{/snippet}
|
||||
|
||||
<section class="gallery" aria-label="Photos of {name}">
|
||||
<ul class:single={photos.length === 1}>
|
||||
{#each photos as p, i (p.file)}
|
||||
<li class:lead={i === 0}>
|
||||
<a href={commonsPage(p.file)} onclick={(e) => open(e, i)} title={p.caption ?? undefined}>
|
||||
<img
|
||||
src={thumbAt(p.thumb, i === 0 ? 960 : 330)}
|
||||
srcset={i === 0
|
||||
? thumbSrcset(p.thumb, [500, 960, 1280])
|
||||
: thumbSrcset(p.thumb, [330, 500])}
|
||||
sizes={i === 0 ? '(min-width: 1152px) 680px, 100vw' : '(min-width: 1152px) 220px, 50vw'}
|
||||
alt={alt(p, i)}
|
||||
loading={i === 0 ? 'eager' : 'lazy'}
|
||||
fetchpriority={i === 0 ? 'high' : undefined}
|
||||
decoding="async"
|
||||
width={p.width}
|
||||
height={p.height}
|
||||
/>
|
||||
{#if p.caption}<span class="cap">{p.caption}</span>{/if}
|
||||
</a>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
<details class="credits">
|
||||
<summary>
|
||||
{photos.length} photo{photos.length === 1 ? '' : 's'} from Wikimedia Commons{#if wikipedia}
|
||||
· <a href="https://en.wikipedia.org/wiki/{wikipedia.replace(/ /g, '_')}" rel="noopener"
|
||||
>Wikipedia</a
|
||||
>{/if} · credits
|
||||
</summary>
|
||||
<ol>
|
||||
{#each photos as p, i (p.file)}
|
||||
<li>{i + 1}. {p.caption ?? p.file.replace(/\.\w+$/, '')}: {@render credit(p)}</li>
|
||||
{/each}
|
||||
</ol>
|
||||
</details>
|
||||
</section>
|
||||
|
||||
<dialog bind:this={dialog} {onkeydown} {onclick} aria-label="{name} photos">
|
||||
{#if current}
|
||||
<figure>
|
||||
{#key index}
|
||||
<img src={large(current)} alt={alt(current, index)} />
|
||||
{/key}
|
||||
<figcaption>
|
||||
<span
|
||||
>{index + 1}/{photos.length}{#if current.caption}
|
||||
· {current.caption}{/if}</span
|
||||
>
|
||||
<span class="credit">{@render credit(current)}</span>
|
||||
</figcaption>
|
||||
</figure>
|
||||
{#if photos.length > 1}
|
||||
<button class="nav prev" onclick={() => step(-1)} aria-label="Previous photo"
|
||||
><ChevronLeft size={28} /></button
|
||||
>
|
||||
<button class="nav next" onclick={() => step(1)} aria-label="Next photo"
|
||||
><ChevronRight size={28} /></button
|
||||
>
|
||||
{/if}
|
||||
<button class="close" onclick={() => dialog?.close()} aria-label="Close" autofocus
|
||||
><X size={22} /></button
|
||||
>
|
||||
{/if}
|
||||
</dialog>
|
||||
|
||||
<style>
|
||||
.gallery ul {
|
||||
display: grid;
|
||||
gap: 0.5rem;
|
||||
grid-template-columns: repeat(2, 1fr);
|
||||
}
|
||||
@media (min-width: 768px) {
|
||||
.gallery ul {
|
||||
grid-template-columns: repeat(4, 1fr);
|
||||
}
|
||||
.gallery li.lead {
|
||||
grid-column: span 2;
|
||||
grid-row: span 2;
|
||||
}
|
||||
.gallery ul.single {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
.gallery li.lead {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
@media (min-width: 768px) {
|
||||
.gallery li.lead {
|
||||
grid-column: span 2;
|
||||
}
|
||||
}
|
||||
.gallery a {
|
||||
position: relative;
|
||||
display: block;
|
||||
height: 100%;
|
||||
aspect-ratio: 3 / 2;
|
||||
overflow: hidden;
|
||||
border-radius: 0.7rem;
|
||||
border: 1px solid var(--site-border);
|
||||
background: var(--site-surface);
|
||||
}
|
||||
.gallery li.lead a {
|
||||
aspect-ratio: auto;
|
||||
min-height: 14rem;
|
||||
}
|
||||
.gallery img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
transition: transform 0.3s;
|
||||
}
|
||||
.gallery a:hover img,
|
||||
.gallery a:focus-visible img {
|
||||
transform: scale(1.04);
|
||||
}
|
||||
.cap {
|
||||
position: absolute;
|
||||
left: 0.4rem;
|
||||
bottom: 0.4rem;
|
||||
max-width: calc(100% - 0.8rem);
|
||||
padding: 0.05rem 0.4rem;
|
||||
border-radius: 0.35rem;
|
||||
font-size: 0.68rem;
|
||||
color: #fff;
|
||||
background: rgb(0 0 0 / 0.55);
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.credits {
|
||||
margin-top: 0.5rem;
|
||||
font-size: 0.75rem;
|
||||
color: var(--site-muted);
|
||||
}
|
||||
.credits summary {
|
||||
cursor: pointer;
|
||||
}
|
||||
.credits ol {
|
||||
margin-top: 0.4rem;
|
||||
display: grid;
|
||||
gap: 0.15rem;
|
||||
}
|
||||
.credits a,
|
||||
figcaption a {
|
||||
text-decoration: underline;
|
||||
}
|
||||
dialog {
|
||||
width: 100vw;
|
||||
height: 100vh;
|
||||
max-width: none;
|
||||
max-height: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
border: 0;
|
||||
background: rgb(0 0 0 / 0.92);
|
||||
color: #fff;
|
||||
}
|
||||
dialog::backdrop {
|
||||
background: rgb(0 0 0 / 0.6);
|
||||
}
|
||||
figure {
|
||||
position: absolute;
|
||||
inset: 3.5rem 1rem 1rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.6rem;
|
||||
pointer-events: none;
|
||||
}
|
||||
figure img {
|
||||
max-width: 100%;
|
||||
min-height: 0;
|
||||
flex: 0 1 auto;
|
||||
object-fit: contain;
|
||||
border-radius: 0.4rem;
|
||||
pointer-events: auto;
|
||||
}
|
||||
figcaption {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
gap: 0.3rem 1rem;
|
||||
font-size: 0.8rem;
|
||||
color: rgb(255 255 255 / 0.8);
|
||||
pointer-events: auto;
|
||||
text-align: center;
|
||||
}
|
||||
.nav,
|
||||
.close {
|
||||
position: absolute;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
border-radius: 999px;
|
||||
background: rgb(255 255 255 / 0.12);
|
||||
color: #fff;
|
||||
}
|
||||
.nav:hover,
|
||||
.close:hover,
|
||||
.nav:focus-visible,
|
||||
.close:focus-visible {
|
||||
background: rgb(255 255 255 / 0.25);
|
||||
}
|
||||
.nav {
|
||||
top: 50%;
|
||||
width: 3rem;
|
||||
height: 3rem;
|
||||
transform: translateY(-50%);
|
||||
}
|
||||
.prev {
|
||||
left: 0.75rem;
|
||||
}
|
||||
.next {
|
||||
right: 0.75rem;
|
||||
}
|
||||
.close {
|
||||
top: 0.75rem;
|
||||
right: 0.75rem;
|
||||
width: 2.5rem;
|
||||
height: 2.5rem;
|
||||
}
|
||||
</style>
|
||||
@@ -164,4 +164,25 @@ export type CarCard = {
|
||||
nlCount: number;
|
||||
nlPriceLatest: number | null;
|
||||
powerKwMax: number | null;
|
||||
photo: string | null;
|
||||
};
|
||||
|
||||
/** Same Wikimedia thumbnail at another standard width (120/250/330/500/960/1280). */
|
||||
export const thumbAt = (url: string, width: number) =>
|
||||
url.replace(/\/\d+px-([^/]+)$/, `/${width}px-$1`);
|
||||
export const thumbSrcset = (url: string, widths = [330, 500, 960]) =>
|
||||
widths.map((w) => `${thumbAt(url, w)} ${w}w`).join(', ');
|
||||
export const commonsPage = (file: string) =>
|
||||
`https://commons.wikimedia.org/wiki/File:${encodeURIComponent(file.replace(/ /g, '_'))}`;
|
||||
|
||||
export type CarPhoto = {
|
||||
file: string;
|
||||
kind: string;
|
||||
caption: string | null;
|
||||
thumb: string;
|
||||
width: number;
|
||||
height: number;
|
||||
author: string | null;
|
||||
license: string | null;
|
||||
licenseUrl: string | null;
|
||||
};
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Car pokedex queries (/trackers/cars). Public, read-only; data is written only by
|
||||
// scripts/cars-import.ts.
|
||||
import { db } from '#lib/server/db/index.js';
|
||||
import { carBrands, carModels, carVariants } from '#lib/server/db/schema/index.js';
|
||||
import { carBrands, carModels, carPhotos, carVariants } from '#lib/server/db/schema/index.js';
|
||||
import { and, arrayContains, asc, desc, eq, gt, gte, ilike, lte, sql, type SQL } from 'drizzle-orm';
|
||||
import { BODIES, DRIVES, FUELS, SORTS, type CarCard, type CarSort } from '#lib/cars/shared.js';
|
||||
|
||||
@@ -55,7 +55,8 @@ const cardColumns = {
|
||||
bodies: carModels.bodies,
|
||||
nlCount: carModels.nlCount,
|
||||
nlPriceLatest: carModels.nlPriceLatest,
|
||||
powerKwMax: carModels.powerKwMax
|
||||
powerKwMax: carModels.powerKwMax,
|
||||
photo: carModels.photo
|
||||
};
|
||||
|
||||
const escapeLike = (s: string) => s.replace(/[\\%_]/g, (c) => `\\${c}`);
|
||||
@@ -145,7 +146,21 @@ export async function getBrand(slug: string) {
|
||||
.innerJoin(carBrands, eq(carBrands.id, carModels.brandId))
|
||||
.where(eq(carModels.brandId, brand.id))
|
||||
.orderBy(desc(carModels.nlCount), desc(carModels.variantCount), asc(carModels.name));
|
||||
return { brand, models };
|
||||
const [hero] = brand.photo
|
||||
? await db
|
||||
.select({
|
||||
file: carPhotos.file,
|
||||
author: carPhotos.author,
|
||||
license: carPhotos.license,
|
||||
licenseUrl: carPhotos.licenseUrl,
|
||||
model: carModels.name
|
||||
})
|
||||
.from(carPhotos)
|
||||
.innerJoin(carModels, eq(carModels.id, carPhotos.modelId))
|
||||
.where(and(eq(carModels.brandId, brand.id), eq(carPhotos.thumb, brand.photo)))
|
||||
.limit(1)
|
||||
: [];
|
||||
return { brand, models, hero: hero ?? null };
|
||||
}
|
||||
|
||||
export async function getModel(brandSlug: string, modelSlug: string) {
|
||||
@@ -156,7 +171,7 @@ export async function getModel(brandSlug: string, modelSlug: string) {
|
||||
.where(and(eq(carBrands.slug, brandSlug), eq(carModels.slug, modelSlug)))
|
||||
.limit(1);
|
||||
if (!row) return null;
|
||||
const [variants, neighbours] = await Promise.all([
|
||||
const [variants, neighbours, photos] = await Promise.all([
|
||||
db
|
||||
.select()
|
||||
.from(carVariants)
|
||||
@@ -175,11 +190,27 @@ export async function getModel(brandSlug: string, modelSlug: string) {
|
||||
sql`${carModels.dex} in (
|
||||
(select max(dex) from cars.models where dex < ${row.model.dex}),
|
||||
(select min(dex) from cars.models where dex > ${row.model.dex}))`
|
||||
)
|
||||
),
|
||||
db
|
||||
.select({
|
||||
file: carPhotos.file,
|
||||
kind: carPhotos.kind,
|
||||
caption: carPhotos.caption,
|
||||
thumb: carPhotos.thumb,
|
||||
width: carPhotos.width,
|
||||
height: carPhotos.height,
|
||||
author: carPhotos.author,
|
||||
license: carPhotos.license,
|
||||
licenseUrl: carPhotos.licenseUrl
|
||||
})
|
||||
.from(carPhotos)
|
||||
.where(eq(carPhotos.modelId, row.model.id))
|
||||
.orderBy(asc(carPhotos.position))
|
||||
]);
|
||||
return {
|
||||
...row,
|
||||
variants,
|
||||
photos,
|
||||
prev: neighbours.find((n) => n.dex < row.model.dex) ?? null,
|
||||
next: neighbours.find((n) => n.dex > row.model.dex) ?? null
|
||||
};
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
real,
|
||||
smallint,
|
||||
text,
|
||||
timestamp,
|
||||
uniqueIndex
|
||||
} from 'drizzle-orm/pg-core';
|
||||
import { carsSchema } from './schemas';
|
||||
@@ -38,6 +39,16 @@ export const carBrands = carsSchema.table(
|
||||
firstYear: smallint('first_year'),
|
||||
lastYear: smallint('last_year'),
|
||||
sources: text('sources').array().notNull().default([]),
|
||||
/** Wikidata item of the brand/manufacturer and its enwiki article (scripts/cars-photos.ts). */
|
||||
wikidata: text('wikidata'),
|
||||
wikipedia: text('wikipedia'),
|
||||
/** Logo thumbnail (Wikimedia Commons, 250px) and its credit. */
|
||||
logo: text('logo'),
|
||||
logoFile: text('logo_file'),
|
||||
logoAuthor: text('logo_author'),
|
||||
logoLicense: text('logo_license'),
|
||||
/** Hero photo: lead photo of the brand's most common model. */
|
||||
photo: text('photo'),
|
||||
...timestamps
|
||||
},
|
||||
(t) => [uniqueIndex('brands_slug_key').on(t.slug)]
|
||||
@@ -96,6 +107,13 @@ export const carModels = carsSchema.table(
|
||||
usYears: smallint('us_years').array().notNull().default([]),
|
||||
variantCount: integer('variant_count').notNull().default(0),
|
||||
sources: text('sources').array().notNull().default([]),
|
||||
/** Matched Wikidata item / enwiki article (scripts/cars-photos.ts); null = no confident match. */
|
||||
wikidata: text('wikidata'),
|
||||
wikipedia: text('wikipedia'),
|
||||
/** Lead photo thumbnail URL (960px), denormalised from cars.photos for the card grid. */
|
||||
photo: text('photo'),
|
||||
photoCount: smallint('photo_count').notNull().default(0),
|
||||
photosCheckedAt: timestamp('photos_checked_at', { withTimezone: true }),
|
||||
...timestamps
|
||||
},
|
||||
(t) => [
|
||||
@@ -135,3 +153,28 @@ export const carVariants = carsSchema.table(
|
||||
},
|
||||
(t) => [index('variants_model_year_idx').on(t.modelId, t.year)]
|
||||
);
|
||||
|
||||
/** Freely licensed photos from Wikimedia Commons; only thumbnail URLs are stored, nothing is downloaded. */
|
||||
export const carPhotos = carsSchema.table(
|
||||
'photos',
|
||||
{
|
||||
id: id(),
|
||||
modelId: integer('model_id')
|
||||
.notNull()
|
||||
.references(() => carModels.id, { onDelete: 'cascade' }),
|
||||
position: smallint('position').notNull(),
|
||||
/** lead (Wikidata image), generation, article (enwiki), commons (Commons category). */
|
||||
kind: text('kind').notNull(),
|
||||
/** Commons file name without the "File:" prefix. */
|
||||
file: text('file').notNull(),
|
||||
caption: text('caption'),
|
||||
/** 960px thumbnail on upload.wikimedia.org; other standard widths are derived from it. */
|
||||
thumb: text('thumb').notNull(),
|
||||
width: integer('width').notNull(),
|
||||
height: integer('height').notNull(),
|
||||
author: text('author'),
|
||||
license: text('license'),
|
||||
licenseUrl: text('license_url')
|
||||
},
|
||||
(t) => [uniqueIndex('photos_model_position_key').on(t.modelId, t.position)]
|
||||
);
|
||||
|
||||
@@ -187,8 +187,8 @@
|
||||
|
||||
{#if data.cars.length}
|
||||
<ul class="grid">
|
||||
{#each data.cars as car (car.dex)}
|
||||
<li><CarCard {car} current={data.currentYear} /></li>
|
||||
{#each data.cars as car, i (car.dex)}
|
||||
<li><CarCard {car} current={data.currentYear} eager={i < 4} /></li>
|
||||
{/each}
|
||||
</ul>
|
||||
{:else}
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { PageServerLoad } from './$types';
|
||||
export const load: PageServerLoad = async ({ params }) => {
|
||||
const found = await getBrand(params.brand);
|
||||
if (!found) error(404, 'No such brand in the car pokedex');
|
||||
const { brand, models } = found;
|
||||
const { brand, models, hero } = found;
|
||||
return {
|
||||
brand: {
|
||||
slug: brand.slug,
|
||||
@@ -15,8 +15,15 @@ export const load: PageServerLoad = async ({ params }) => {
|
||||
nlCount: brand.nlCount,
|
||||
firstYear: brand.firstYear,
|
||||
lastYear: brand.lastYear,
|
||||
sources: brand.sources
|
||||
sources: brand.sources,
|
||||
wikipedia: brand.wikipedia,
|
||||
logo: brand.logo,
|
||||
logoFile: brand.logoFile,
|
||||
logoAuthor: brand.logoAuthor,
|
||||
logoLicense: brand.logoLicense,
|
||||
photo: brand.photo
|
||||
},
|
||||
hero,
|
||||
models,
|
||||
currentYear: new Date().getFullYear(),
|
||||
seo: {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<script lang="ts">
|
||||
import BrandBadge from '#lib/cars/BrandBadge.svelte';
|
||||
import CarCard from '#lib/cars/CarCard.svelte';
|
||||
import { flag, fmtNumber, years } from '#lib/cars/shared.js';
|
||||
import { commonsPage, flag, fmtNumber, thumbAt, thumbSrcset, years } from '#lib/cars/shared.js';
|
||||
import ArrowLeft from 'lucide-svelte/icons/arrow-left';
|
||||
import type { PageProps } from './$types';
|
||||
|
||||
@@ -11,8 +11,46 @@
|
||||
|
||||
<div class="mx-auto max-w-6xl px-4 py-12 sm:px-6 sm:py-16">
|
||||
<a href="/trackers/cars" class="back"><ArrowLeft size={15} /> all cars</a>
|
||||
{#if b.photo}
|
||||
<figure class="hero">
|
||||
<img
|
||||
src={thumbAt(b.photo, 960)}
|
||||
srcset={thumbSrcset(b.photo, [500, 960, 1280])}
|
||||
sizes="(min-width: 1152px) 1104px, 100vw"
|
||||
alt="{b.name} {data.hero?.model ?? ''}"
|
||||
fetchpriority="high"
|
||||
width="1280"
|
||||
height="480"
|
||||
/>
|
||||
{#if data.hero}
|
||||
<figcaption>
|
||||
{b.name}
|
||||
{data.hero.model} ·
|
||||
<a href={commonsPage(data.hero.file)} rel="noopener"
|
||||
>{data.hero.author ?? 'Wikimedia Commons'}</a
|
||||
>{#if data.hero.license}
|
||||
· {#if data.hero.licenseUrl}<a href={data.hero.licenseUrl} rel="noopener license"
|
||||
>{data.hero.license}</a
|
||||
>{:else}{data.hero.license}{/if}{/if}
|
||||
</figcaption>
|
||||
{/if}
|
||||
</figure>
|
||||
{/if}
|
||||
<header class="mt-6 mb-8 flex items-center gap-5">
|
||||
<BrandBadge name={b.name} size="lg" />
|
||||
{#if b.logo}
|
||||
<img
|
||||
class="logo"
|
||||
src={b.logo}
|
||||
alt="{b.name} logo"
|
||||
title="Logo: {b.logoAuthor ?? 'Wikimedia Commons'}{b.logoLicense
|
||||
? ` (${b.logoLicense})`
|
||||
: ''}"
|
||||
width="72"
|
||||
height="72"
|
||||
/>
|
||||
{:else}
|
||||
<BrandBadge name={b.name} size="lg" />
|
||||
{/if}
|
||||
<div>
|
||||
<h1 class="text-4xl font-semibold tracking-tight sm:text-5xl">{b.name}</h1>
|
||||
<p class="mt-2 text-[var(--site-muted)]">
|
||||
@@ -25,12 +63,18 @@
|
||||
</header>
|
||||
|
||||
<ul class="grid">
|
||||
{#each data.models as car (car.dex)}
|
||||
<li><CarCard {car} current={data.currentYear} /></li>
|
||||
{#each data.models as car, i (car.dex)}
|
||||
<li><CarCard {car} current={data.currentYear} eager={i < 4} /></li>
|
||||
{/each}
|
||||
</ul>
|
||||
<p class="mt-10 text-xs text-[var(--site-muted)]">
|
||||
Sorted by how many are registered in NL. Sources: {b.sources.join(', ')}.
|
||||
Sorted by how many are registered in NL. Sources: {b.sources.join(', ')}.{#if b.wikipedia}
|
||||
· <a href="https://en.wikipedia.org/wiki/{b.wikipedia.replace(/ /g, '_')}" rel="noopener"
|
||||
>{b.name} on Wikipedia</a
|
||||
>{/if}{#if b.logoFile}
|
||||
· Logo: <a href={commonsPage(b.logoFile)} rel="noopener"
|
||||
>{b.logoAuthor ?? 'Wikimedia Commons'}</a
|
||||
>{#if b.logoLicense}, {b.logoLicense}{/if}{/if}. Photos from Wikimedia Commons.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -45,6 +89,46 @@
|
||||
.back:hover {
|
||||
color: var(--site-text);
|
||||
}
|
||||
.hero {
|
||||
position: relative;
|
||||
margin-top: 1.25rem;
|
||||
aspect-ratio: 8 / 3;
|
||||
overflow: hidden;
|
||||
border-radius: 1rem;
|
||||
border: 1px solid var(--site-border);
|
||||
background: var(--site-surface);
|
||||
}
|
||||
.hero img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
.hero figcaption {
|
||||
position: absolute;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
max-width: 100%;
|
||||
padding: 0.25rem 0.6rem;
|
||||
font-size: 0.7rem;
|
||||
color: rgb(255 255 255 / 0.85);
|
||||
background: rgb(0 0 0 / 0.5);
|
||||
border-top-left-radius: 0.5rem;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.hero figcaption a {
|
||||
text-decoration: underline;
|
||||
}
|
||||
.logo {
|
||||
width: 4.5rem;
|
||||
height: 4.5rem;
|
||||
object-fit: contain;
|
||||
padding: 0.5rem;
|
||||
border-radius: 1rem;
|
||||
background: #fff;
|
||||
border: 1px solid var(--site-border);
|
||||
}
|
||||
.grid {
|
||||
display: grid;
|
||||
gap: 0.75rem;
|
||||
|
||||
@@ -14,6 +14,7 @@ function publicCar(m: Model) {
|
||||
delete out.search;
|
||||
delete out.createdAt;
|
||||
delete out.updatedAt;
|
||||
delete out.photosCheckedAt;
|
||||
return {
|
||||
...(out as Omit<Model, 'id' | 'brandId' | 'search' | 'createdAt' | 'updatedAt'>),
|
||||
updatedAt: m.updatedAt?.toISOString() ?? null
|
||||
@@ -30,11 +31,12 @@ function publicVariant(v: Variant) {
|
||||
export const load: PageServerLoad = async ({ params }) => {
|
||||
const found = await getModel(params.brand, params.model);
|
||||
if (!found) error(404, 'No such car in the pokedex');
|
||||
const { model, brand, variants, prev, next } = found;
|
||||
const { model, brand, variants, photos, prev, next } = found;
|
||||
return {
|
||||
car: publicCar(model),
|
||||
brand: { slug: brand.slug, name: brand.name, country: brand.country },
|
||||
variants: variants.map(publicVariant),
|
||||
photos,
|
||||
prev,
|
||||
next,
|
||||
currentYear: new Date().getFullYear(),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
<script lang="ts">
|
||||
import BrandBadge from '#lib/cars/BrandBadge.svelte';
|
||||
import FuelChip from '#lib/cars/FuelChip.svelte';
|
||||
import Gallery from '#lib/cars/Gallery.svelte';
|
||||
import {
|
||||
bodyLabel,
|
||||
dexLabel,
|
||||
@@ -174,6 +175,10 @@
|
||||
{/if}
|
||||
</header>
|
||||
|
||||
{#if data.photos.length}
|
||||
<Gallery photos={data.photos} name="{data.brand.name} {car.name}" wikipedia={car.wikipedia} />
|
||||
{/if}
|
||||
|
||||
{#if baseStats.length}
|
||||
<section class="card" aria-labelledby="stats">
|
||||
<h2 id="stats">base stats</h2>
|
||||
|
||||
Reference in New Issue
Block a user