Stan Runge
7636f0b2bc
All checks were successful
Deploy production / deploy (push) Successful in 2m34s
771 lines
27 KiB
TypeScript
771 lines
27 KiB
TypeScript
// Finds freely licensed photos for the car pokedex (/trackers/cars) on Wikimedia Commons.
|
|
//
|
|
// DB_URL=... bun scripts/cars-photos.ts match every model, replace all photos
|
|
// DB_URL=... bun scripts/cars-photos.ts --missing only models that have no photos yet
|
|
// DB_URL=... bun scripts/cars-photos.ts --enrich r.json add photos to a cars-import
|
|
// --report-new file, print it to stdout
|
|
// bun scripts/cars-photos.ts --dry-run --models built.json [--cache dir] [--out x.json]
|
|
//
|
|
// How a model gets matched (low-confidence matches are skipped, never guessed):
|
|
// 1. One SPARQL query lists every Wikidata item that is an instance of "car model"
|
|
// (Q3231690) with its English label/aliases, enwiki article, manufacturer (P176),
|
|
// brand (P1716), image (P18), Commons category (P373) and parent series (P179).
|
|
// 2. A model matches an item when "<brand> <model>" equals the item's label, alias or
|
|
// article title, or when "<model>" does and the item's manufacturer/brand is that brand.
|
|
// Among several hits the series root with an article and the most sitelinks wins.
|
|
// 3. Photos, in order: the item's own image, the images of its generations (items that
|
|
// are P179 "part of the series" of it, newest first), images used in the enwiki article
|
|
// and files in the Commons category. Article/category files must mention the model in
|
|
// their file name; logos, maps, drawings, SVGs and small files are dropped.
|
|
// Nothing is downloaded: we store thumbnail URLs (standard widths) plus author and licence.
|
|
import postgres from 'postgres';
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
|
|
const args = process.argv.slice(2);
|
|
const flag = (name: string) => args.includes(name);
|
|
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
|
const dryRun = flag('--dry-run');
|
|
const missingOnly = flag('--missing');
|
|
const enrichFile = opt('--enrich');
|
|
const modelsFile = opt('--models');
|
|
const cacheDir = opt('--cache');
|
|
const outFile = opt('--out');
|
|
const MAX_PHOTOS = 12;
|
|
const THUMB = 960; // Wikimedia standard thumbnail widths: 120 250 330 500 960 1280 1920
|
|
const UA =
|
|
'StanRungeCarDex/1.0 (https://stanrunge.dev/trackers/cars; car pokedex photo matcher) bun';
|
|
const log = (...a: unknown[]) => console.error(...a);
|
|
|
|
// ---------------------------------------------------------------- http
|
|
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
let apiCache: Record<string, unknown> = {};
|
|
let apiCacheDirty = 0;
|
|
async function loadApiCache() {
|
|
if (!cacheDir) return;
|
|
await mkdir(cacheDir, { recursive: true });
|
|
try {
|
|
apiCache = JSON.parse(await readFile(join(cacheDir, 'api.json'), 'utf8'));
|
|
} catch {
|
|
apiCache = {};
|
|
}
|
|
}
|
|
async function saveApiCache(force = false) {
|
|
if (!cacheDir || (!force && apiCacheDirty < 200)) return;
|
|
apiCacheDirty = 0;
|
|
await writeFile(join(cacheDir, 'api.json'), JSON.stringify(apiCache));
|
|
}
|
|
|
|
async function getJson<T>(url: string, init?: RequestInit): Promise<T> {
|
|
const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
|
|
if (key in apiCache) return apiCache[key] as T;
|
|
for (let attempt = 0; ; attempt++) {
|
|
const res = await fetch(url, {
|
|
...init,
|
|
headers: {
|
|
'user-agent': UA,
|
|
'api-user-agent': UA,
|
|
accept: 'application/json',
|
|
...init?.headers
|
|
}
|
|
});
|
|
const retryAfter = Number(res.headers.get('retry-after')) || 0;
|
|
if (res.ok) {
|
|
const body = (await res.json()) as T & { error?: { code?: string } };
|
|
if (body?.error?.code === 'maxlag' && attempt < 8) {
|
|
await sleep(Math.max(retryAfter, 5) * 1000);
|
|
continue;
|
|
}
|
|
apiCache[key] = body;
|
|
apiCacheDirty++;
|
|
await saveApiCache();
|
|
return body;
|
|
}
|
|
if ((res.status === 429 || res.status >= 500) && attempt < 6) {
|
|
await sleep(Math.max(retryAfter * 1000, 2000 * 2 ** attempt));
|
|
continue;
|
|
}
|
|
throw new Error(`${res.status} ${url.slice(0, 200)}`);
|
|
}
|
|
}
|
|
|
|
let lastCall = 0;
|
|
/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
|
|
async function mw(host: string, params: Record<string, string>) {
|
|
const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
|
|
const key = `https://${host}/w/api.php?${body}`;
|
|
if (!(key in apiCache)) {
|
|
const wait = lastCall + 150 - Date.now();
|
|
if (wait > 0) await sleep(wait);
|
|
lastCall = Date.now();
|
|
}
|
|
return getJson<MwResponse>(key);
|
|
}
|
|
type MwPage = {
|
|
title: string;
|
|
missing?: boolean;
|
|
images?: { title: string }[];
|
|
imageinfo?: ImageInfo[];
|
|
};
|
|
type ImageInfo = {
|
|
width: number;
|
|
height: number;
|
|
mime: string;
|
|
thumburl?: string;
|
|
descriptionurl?: string;
|
|
extmetadata?: Record<string, { value: string }>;
|
|
};
|
|
type MwResponse = {
|
|
query?: { pages?: MwPage[]; normalized?: { from: string; to: string }[] };
|
|
continue?: Record<string, string>;
|
|
};
|
|
|
|
async function sparql<T>(query: string): Promise<T[]> {
|
|
const res = await getJson<{ results: { bindings: Record<string, { value: string }>[] } }>(
|
|
'https://query.wikidata.org/sparql',
|
|
{
|
|
method: 'POST',
|
|
headers: {
|
|
'content-type': 'application/x-www-form-urlencoded',
|
|
accept: 'application/sparql-results+json'
|
|
},
|
|
body: new URLSearchParams({ query }).toString()
|
|
}
|
|
);
|
|
return res.results.bindings.map(
|
|
(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- wikidata
|
|
|
|
const qid = (uri: string) => uri.split('/').at(-1)!;
|
|
const fileName = (uri: string) =>
|
|
decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
|
|
const cmp = (s: string) =>
|
|
s
|
|
.normalize('NFD')
|
|
.replace(/[\u0300-\u036f]/g, '')
|
|
.toUpperCase()
|
|
.replace(/&/g, ' AND ')
|
|
.replace(/[^A-Z0-9]/g, '');
|
|
const tokens = (s: string) =>
|
|
s
|
|
.normalize('NFD')
|
|
.replace(/[\u0300-\u036f]/g, '')
|
|
.toUpperCase()
|
|
.replace(/&/g, ' ')
|
|
.split(/[^A-Z0-9]+/)
|
|
.filter(Boolean);
|
|
|
|
type WdItem = {
|
|
id: string;
|
|
label: string;
|
|
names: Set<string>; // cmp() of label, aliases, enwiki title
|
|
primary: Set<string>; // cmp() of label and enwiki title only
|
|
series: boolean; // "car model series" (Q59773381), i.e. spans generations
|
|
article: string | null;
|
|
makers: Set<string>;
|
|
image: string | null;
|
|
category: string | null;
|
|
parents: Set<string>;
|
|
links: number;
|
|
year: number | null;
|
|
};
|
|
|
|
const split = (s: string | undefined) => (s ? s.split('|').filter(Boolean) : []);
|
|
|
|
async function loadWikidata() {
|
|
type Row = Record<string, string | undefined>;
|
|
log('wikidata: car models');
|
|
const rows = await sparql<Row>(`
|
|
SELECT ?item (SAMPLE(?l) AS ?label) (SAMPLE(?a) AS ?article) (SAMPLE(?img) AS ?image) (SAMPLE(?c) AS ?cat)
|
|
(SAMPLE(?links) AS ?sitelinks) (MIN(YEAR(?inc)) AS ?year)
|
|
(GROUP_CONCAT(DISTINCT STRAFTER(STR(?m), "entity/"); separator="|") AS ?makers)
|
|
(GROUP_CONCAT(DISTINCT STRAFTER(STR(?p), "entity/"); separator="|") AS ?parents)
|
|
(GROUP_CONCAT(DISTINCT STRAFTER(STR(?cls), "entity/"); separator="|") AS ?classes)
|
|
WHERE {
|
|
VALUES ?cls { wd:Q3231690 wd:Q59773381 }
|
|
?item wdt:P31 ?cls .
|
|
OPTIONAL { ?item rdfs:label ?l FILTER(LANG(?l) = "en") }
|
|
OPTIONAL { ?a schema:about ?item; schema:isPartOf <https://en.wikipedia.org/> }
|
|
OPTIONAL { ?item wdt:P18 ?img }
|
|
OPTIONAL { ?item wdt:P373 ?c }
|
|
OPTIONAL { ?item wikibase:sitelinks ?links }
|
|
OPTIONAL { ?item wdt:P571|wdt:P580|wdt:P5444 ?inc }
|
|
OPTIONAL { ?item wdt:P176|wdt:P1716 ?m }
|
|
OPTIONAL { ?item wdt:P179 ?p }
|
|
} GROUP BY ?item`);
|
|
log(`wikidata: ${rows.length} items; aliases`);
|
|
const aliases = await sparql<Row>(`
|
|
SELECT ?item ?alias WHERE { VALUES ?cls { wd:Q3231690 wd:Q59773381 } ?item wdt:P31 ?cls ; skos:altLabel ?alias FILTER(LANG(?alias) = "en") }`);
|
|
const items = new Map<string, WdItem>();
|
|
for (const r of rows) {
|
|
const id = qid(r.item!);
|
|
const article = r.article
|
|
? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
|
|
: null;
|
|
const label = r.label ?? article ?? id;
|
|
items.set(id, {
|
|
id,
|
|
label,
|
|
names: new Set([cmp(label), ...(article ? [cmp(article.replace(/\s*\(.*\)$/, ''))] : [])]),
|
|
primary: new Set([cmp(label), ...(article ? [cmp(article.replace(/\s*\(.*\)$/, ''))] : [])]),
|
|
series: split(r.classes).includes('Q59773381'),
|
|
article,
|
|
makers: new Set(split(r.makers)),
|
|
image: r.image ? fileName(r.image) : null,
|
|
category: r.cat ?? null,
|
|
parents: new Set(split(r.parents)),
|
|
links: Number(r.sitelinks ?? 0),
|
|
year: r.year ? Number(r.year) : null
|
|
});
|
|
}
|
|
for (const a of aliases) items.get(qid(a.item!))?.names.add(cmp(a.alias!));
|
|
const makerIds = [...new Set([...items.values()].flatMap((i) => [...i.makers]))];
|
|
log(`wikidata: ${makerIds.length} manufacturers/brands`);
|
|
const makers = new Map<
|
|
string,
|
|
{ id: string; label: string; logo: string | null; article: string | null }
|
|
>();
|
|
for (let i = 0; i < makerIds.length; i += 400) {
|
|
const chunk = makerIds.slice(i, i + 400);
|
|
const res = await sparql<Row>(`
|
|
SELECT ?m (SAMPLE(?l) AS ?label) (SAMPLE(?logo) AS ?logo) (SAMPLE(?a) AS ?article) WHERE {
|
|
VALUES ?m { ${chunk.map((q) => `wd:${q}`).join(' ')} }
|
|
OPTIONAL { ?m rdfs:label ?l FILTER(LANG(?l) = "en") }
|
|
OPTIONAL { ?m wdt:P154 ?logo }
|
|
OPTIONAL { ?a schema:about ?m; schema:isPartOf <https://en.wikipedia.org/> }
|
|
} GROUP BY ?m`);
|
|
for (const r of res) {
|
|
const id = qid(r.m!);
|
|
makers.set(id, {
|
|
id,
|
|
label: r.label ?? id,
|
|
logo: r.logo ? fileName(r.logo) : null,
|
|
article: r.article
|
|
? decodeURIComponent(r.article.split('/wiki/').at(-1)!).replace(/_/g, ' ')
|
|
: null
|
|
});
|
|
}
|
|
}
|
|
const children = new Map<string, WdItem[]>();
|
|
for (const item of items.values())
|
|
for (const p of item.parents) children.set(p, [...(children.get(p) ?? []), item]);
|
|
return { items, makers, children };
|
|
}
|
|
|
|
// ---------------------------------------------------------------- matching
|
|
|
|
type Target = {
|
|
id: number | null;
|
|
brand: string;
|
|
brandSlug: string;
|
|
model: string;
|
|
slug: string;
|
|
nl: number;
|
|
photos: number;
|
|
};
|
|
type Match = { item: WdItem; via: string };
|
|
|
|
/** Popular models whose Dutch/registration name differs from the Wikipedia one. */
|
|
const ALIASES: Record<string, string> = {
|
|
'MINI|Cooper': 'Mini Hatch',
|
|
'MINI|Countryman Cooper': 'Mini Countryman',
|
|
'MINI|Clubman Cooper': 'Mini Clubman',
|
|
'Renault|Megane Scenic': 'Renault Scénic',
|
|
'Volvo|EX40': 'Volvo XC40',
|
|
'Volvo|EC40': 'Volvo C40',
|
|
'Hyundai|Atos-Prime': 'Hyundai Atos',
|
|
'Suzuki|Wagon': 'Suzuki Wagon R',
|
|
'Lynk & Co|01': 'Lynk & Co 01',
|
|
"Kia|Cee'd": 'Kia Ceed'
|
|
};
|
|
|
|
function matcher(wd: Awaited<ReturnType<typeof loadWikidata>>) {
|
|
const byName = new Map<string, WdItem[]>();
|
|
for (const item of wd.items.values())
|
|
for (const n of item.names) byName.set(n, [...(byName.get(n) ?? []), item]);
|
|
const brandMakers = new Map<string, Set<string>>();
|
|
const makersOf = (brand: string) => {
|
|
let set = brandMakers.get(brand);
|
|
if (set) return set;
|
|
const bt = tokens(brand);
|
|
set = new Set(
|
|
[...wd.makers.values()]
|
|
.filter((m) => {
|
|
const mt = tokens(m.label);
|
|
return bt.length && bt.every((t, i) => mt[i] === t);
|
|
})
|
|
.map((m) => m.id)
|
|
);
|
|
brandMakers.set(brand, set);
|
|
return set;
|
|
};
|
|
// Prefer the item that is *called* this (not just aliased), spans generations, has an
|
|
// article, is a series root and is well known.
|
|
const score = (i: WdItem, names: string[]) =>
|
|
(names.some((n) => i.primary.has(n)) ? 4000 : 0) +
|
|
(i.series ? 800 : 0) +
|
|
Math.min(10, wd.children.get(i.id)?.length ?? 0) * 60 +
|
|
(i.article ? 1000 : 0) +
|
|
(i.parents.size ? 0 : 500) +
|
|
i.links;
|
|
return {
|
|
makersOf,
|
|
match(t: Target): Match | null {
|
|
const makers = makersOf(t.brand);
|
|
const alias = ALIASES[`${t.brand}|${t.model}`];
|
|
const bm = [
|
|
...(byName.get(cmp(`${t.brand} ${t.model}`)) ?? []),
|
|
...(alias ? (byName.get(cmp(alias)) ?? []) : [])
|
|
];
|
|
const m = byName.get(cmp(t.model)) ?? [];
|
|
const ok = [
|
|
...bm.filter((i) => !i.makers.size || [...i.makers].some((x) => makers.has(x))),
|
|
...m.filter((i) => [...i.makers].some((x) => makers.has(x)))
|
|
];
|
|
// An item whose maker is known but belongs to someone else is never used.
|
|
if (!ok.length) return null;
|
|
const names = [cmp(`${t.brand} ${t.model}`), cmp(t.model), ...(alias ? [cmp(alias)] : [])];
|
|
const best = [...new Set(ok)].sort((a, b) => score(b, names) - score(a, names))[0];
|
|
return { item: best, via: bm.includes(best) ? 'brand+model name' : 'model name + maker' };
|
|
}
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------- photos
|
|
|
|
export type Photo = {
|
|
file: string;
|
|
kind: 'lead' | 'generation' | 'article' | 'commons' | 'logo';
|
|
caption: string | null;
|
|
thumb: string;
|
|
width: number;
|
|
height: number;
|
|
author: string | null;
|
|
license: string | null;
|
|
licenseUrl: string | null;
|
|
};
|
|
|
|
const BAD_FILE =
|
|
/\.(svg|gif|tiff?|pdf|djvu|webm|ogv)$|logo|emblem|badge|wordmark|map\b|karte|locator|diagram|chart|graph|schema|schematic|drawing|sketch|blueprint|patent|icon|flag|coat of arms|wappen|sign\b|signature|symbol|plan\b|cutaway|silhouette|commons-|wiki|stub|poster|advert|brochure|stamp|timeline|people|crowd|queue|in line|factory|assembly|production line|plant\b|werk\b|museum entrance|dealership|showroom sign|\btoys?\b|model car|lego|scale model|diecast/i;
|
|
const stripHtml = (s: string) =>
|
|
s
|
|
.replace(/<[^>]+>/g, ' ')
|
|
.replace(/&/g, '&')
|
|
.replace(/"/g, '"')
|
|
.replace(/�?39;/g, "'")
|
|
.replace(/ /g, ' ')
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
|
|
const infoCache = new Map<string, ImageInfo | null>();
|
|
async function imageInfo(files: string[], width = THUMB) {
|
|
const todo = [...new Set(files)].filter((f) => !infoCache.has(`${width}|${f}`));
|
|
for (let i = 0; i < todo.length; i += 50) {
|
|
const chunk = todo.slice(i, i + 50);
|
|
const res = await mw('commons.wikimedia.org', {
|
|
action: 'query',
|
|
prop: 'imageinfo',
|
|
iiprop: 'url|size|mime|extmetadata',
|
|
iiurlwidth: String(width),
|
|
iiextmetadatafilter: 'Artist|LicenseShortName|LicenseUrl|AttributionRequired|NonFree',
|
|
titles: chunk.map((f) => `File:${f}`).join('|')
|
|
});
|
|
const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
|
|
for (const p of res.query?.pages ?? []) {
|
|
const original = (norm.get(p.title) ?? p.title).replace(/^File:/, '');
|
|
infoCache.set(`${width}|${original}`, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]);
|
|
}
|
|
for (const f of chunk)
|
|
if (!infoCache.has(`${width}|${f}`)) infoCache.set(`${width}|${f}`, null);
|
|
}
|
|
}
|
|
function toPhoto(
|
|
file: string,
|
|
kind: Photo['kind'],
|
|
caption: string | null,
|
|
width = THUMB
|
|
): Photo | null {
|
|
const info = infoCache.get(`${width}|${file}`);
|
|
if (!info?.thumburl) return null;
|
|
const meta = info.extmetadata ?? {};
|
|
if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
|
|
const license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
|
|
if (!license || /fair use|non-free/i.test(license)) return null;
|
|
if (kind !== 'logo') {
|
|
if (!/^image\/(jpeg|png|webp)$/.test(info.mime)) return null;
|
|
if (info.width < THUMB || info.height < 400) return null;
|
|
if (info.width / info.height > 3 || info.height / info.width > 1.6) return null;
|
|
}
|
|
const author = meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null;
|
|
return {
|
|
file,
|
|
kind,
|
|
caption,
|
|
// Canonical host, no tracking query: https://upload.wikimedia.org/.../960px-<file>
|
|
thumb: info.thumburl
|
|
.replace(/\?.*$/, '')
|
|
.replace('//thumb.wikimedia.org/', '//upload.wikimedia.org/'),
|
|
width: info.width,
|
|
height: info.height,
|
|
author,
|
|
license,
|
|
licenseUrl: meta.LicenseUrl?.value ?? null
|
|
};
|
|
}
|
|
|
|
async function articleImages(titles: string[]) {
|
|
const out = new Map<string, string[]>();
|
|
const todo = [...new Set(titles)];
|
|
for (let i = 0; i < todo.length; i += 50) {
|
|
const chunk = todo.slice(i, i + 50);
|
|
let cont: Record<string, string> = {};
|
|
for (let guard = 0; guard < 30; guard++) {
|
|
const res = await mw('en.wikipedia.org', {
|
|
action: 'query',
|
|
prop: 'images',
|
|
imlimit: 'max',
|
|
redirects: '1',
|
|
titles: chunk.join('|'),
|
|
...cont
|
|
});
|
|
const norm = new Map<string, string>();
|
|
for (const n of [
|
|
...(res.query?.normalized ?? []),
|
|
...((res.query as { redirects?: { from: string; to: string }[] })?.redirects ?? [])
|
|
])
|
|
norm.set(n.to, norm.get(n.from) ?? n.from);
|
|
for (const p of res.query?.pages ?? []) {
|
|
const key = norm.get(p.title) ?? p.title;
|
|
out.set(key, [
|
|
...(out.get(key) ?? []),
|
|
...(p.images ?? []).map((im) => im.title.replace(/^File:/, ''))
|
|
]);
|
|
}
|
|
if (!res.continue) break;
|
|
cont = res.continue;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function categoryFiles(category: string): Promise<string[]> {
|
|
const res = await mw('commons.wikimedia.org', {
|
|
action: 'query',
|
|
list: 'categorymembers',
|
|
cmtitle: `Category:${category}`,
|
|
cmtype: 'file',
|
|
cmlimit: '60'
|
|
});
|
|
return ((res.query as { categorymembers?: { title: string }[] })?.categorymembers ?? []).map(
|
|
(m) => m.title.replace(/^File:/, '')
|
|
);
|
|
}
|
|
|
|
const GENERIC = new Set(
|
|
'SEDAN SALOON COUPE HATCHBACK WAGON ESTATE TOURING VARIANT CONVERTIBLE CABRIOLET CABRIO ROADSTER SPIDER SPYDER VAN PLUS SPORT SPORTS SERIES CLASS MODEL GRAND NEW SUPER LONG SHORT DOOR DOORS FACELIFT GENERATION FIRST SECOND THIRD FOURTH FIFTH SIXTH SEVENTH EIGHTH NINTH CAR AUTO HYBRID ELECTRIC'.split(
|
|
' '
|
|
)
|
|
);
|
|
/** Words that identify a generation ("E46", "W205", "Mk7", "XP210"), not the brand or body style. */
|
|
function distinctive(label: string, t: Target): string[] {
|
|
const skip = new Set([...tokens(t.brand), ...tokens(t.model)]);
|
|
return tokens(label).filter(
|
|
(w) =>
|
|
!skip.has(w) &&
|
|
!GENERIC.has(w) &&
|
|
(/\d/.test(w) ? w.length >= 3 : w.length >= 4) &&
|
|
!/^\d{4}$/.test(w)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Does the file name mention this model? ("Peugeot 208 GT.jpg" for 208, "BMW E46.jpg" for the
|
|
* 3 Series via its generation codes). Files that name a longer sibling model of the same brand
|
|
* ("Golf Sportsvan" on the Golf, "Mustang Mach-E" on the Mustang) are rejected.
|
|
*/
|
|
function mentions(
|
|
file: string,
|
|
t: Target,
|
|
item: WdItem,
|
|
genWords: string[],
|
|
siblings: string[]
|
|
): boolean {
|
|
const f = cmp(file);
|
|
if (siblings.some((s) => f.includes(s))) return false;
|
|
const model = cmp(t.model);
|
|
const short = model.length < 3 || /^\d+$/.test(model);
|
|
if (f.includes(model) && (!short || f.includes(cmp(t.brand)) || f.includes(cmp(item.label))))
|
|
return true;
|
|
if (f.includes(cmp(item.label))) return true;
|
|
return genWords.some((w) => f.includes(w));
|
|
}
|
|
|
|
async function photosFor(
|
|
targets: Target[],
|
|
wd: Awaited<ReturnType<typeof loadWikidata>>,
|
|
matches: Map<Target, Match>,
|
|
all: Target[]
|
|
) {
|
|
const articles = await articleImages(
|
|
[...matches.values()].map((m) => m.item.article).filter((a): a is string => !!a)
|
|
);
|
|
const result = new Map<Target, Photo[]>();
|
|
const byBrand = new Map<string, Target[]>();
|
|
for (const t of all) byBrand.set(t.brandSlug, [...(byBrand.get(t.brandSlug) ?? []), t]);
|
|
let done = 0;
|
|
for (const t of targets) {
|
|
const m = matches.get(t);
|
|
if (!m) continue;
|
|
const { item } = m;
|
|
const children = wd.children.get(item.id) ?? [];
|
|
const genWords = [...new Set(children.flatMap((c) => distinctive(c.label, t)))];
|
|
const own = cmp(t.model);
|
|
const siblings = (byBrand.get(t.brandSlug) ?? [])
|
|
.map((o) => cmp(o.model))
|
|
.filter((o) => o !== own && o.startsWith(own));
|
|
const ok = (f: string) => !BAD_FILE.test(f) && mentions(f, t, item, genWords, siblings);
|
|
const gens = children
|
|
.filter((g) => g.image)
|
|
.sort(
|
|
(a, b) =>
|
|
(b.year ?? 0) - (a.year ?? 0) || b.label.localeCompare(a.label, 'en', { numeric: true })
|
|
);
|
|
const picks: { file: string; kind: Photo['kind']; caption: string | null }[] = [];
|
|
if (item.image) picks.push({ file: item.image, kind: 'lead', caption: null });
|
|
for (const g of gens
|
|
.filter((g) => ok(g.image!) || distinctive(g.label, t).some((w) => cmp(g.image!).includes(w)))
|
|
.slice(0, 8))
|
|
picks.push({ file: g.image!, kind: 'generation', caption: g.label });
|
|
const fromArticle = (item.article ? (articles.get(item.article) ?? []) : []).filter(ok);
|
|
for (const f of fromArticle) picks.push({ file: f, kind: 'article', caption: null });
|
|
const unique = () => [...new Map(picks.map((p) => [p.file, p])).values()];
|
|
await imageInfo(unique().map((p) => p.file));
|
|
let photos = unique()
|
|
.map((p) => toPhoto(p.file, p.kind, p.caption))
|
|
.filter((p): p is Photo => !!p);
|
|
if (photos.length < MAX_PHOTOS && item.category) {
|
|
const files = (await categoryFiles(item.category)).filter(ok);
|
|
// Spread picks across the (alphabetical) category instead of taking the first few.
|
|
const want = MAX_PHOTOS * 2;
|
|
const step = Math.max(1, files.length / want);
|
|
const spread = Array.from(
|
|
{ length: Math.min(want, files.length) },
|
|
(_, i) => files[Math.floor(i * step)]
|
|
);
|
|
await imageInfo(spread);
|
|
const seen = new Set(photos.map((p) => p.file));
|
|
for (const f of spread) {
|
|
if (seen.has(f)) continue;
|
|
const p = toPhoto(f, 'commons', null);
|
|
if (p) photos.push(p);
|
|
seen.add(f);
|
|
}
|
|
}
|
|
photos = photos.slice(0, MAX_PHOTOS);
|
|
if (photos.length) result.set(t, photos);
|
|
if (++done % 100 === 0) log(`photos: ${done}/${matches.size} models`);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- main
|
|
|
|
async function loadTargets(): Promise<{
|
|
targets: Target[];
|
|
brands: { slug: string; name: string }[];
|
|
}> {
|
|
if (modelsFile) {
|
|
const built = JSON.parse(await readFile(modelsFile, 'utf8')) as {
|
|
slug: string;
|
|
name: string;
|
|
models: { slug: string; name: string; nlCount: number; variants: number }[];
|
|
}[];
|
|
return {
|
|
targets: built.flatMap((b) =>
|
|
b.models.map((m) => ({
|
|
id: null,
|
|
brand: b.name,
|
|
brandSlug: b.slug,
|
|
model: m.name,
|
|
slug: m.slug,
|
|
nl: m.nlCount + m.variants * 50,
|
|
photos: 0
|
|
}))
|
|
),
|
|
brands: built.map((b) => ({ slug: b.slug, name: b.name }))
|
|
};
|
|
}
|
|
const sql = db();
|
|
const rows = await sql<
|
|
{
|
|
id: number;
|
|
brand: string;
|
|
brand_slug: string;
|
|
model: string;
|
|
slug: string;
|
|
nl: number;
|
|
photos: number;
|
|
}[]
|
|
>`
|
|
select m.id, b.name as brand, b.slug as brand_slug, m.name as model, m.slug,
|
|
m.nl_count + m.variant_count * 50 as nl, m.photo_count as photos
|
|
from cars.models m join cars.brands b on b.id = m.brand_id`;
|
|
const brands = await sql<{ slug: string; name: string }[]>`select slug, name from cars.brands`;
|
|
return {
|
|
targets: rows.map((r) => ({
|
|
id: r.id,
|
|
brand: r.brand,
|
|
brandSlug: r.brand_slug,
|
|
model: r.model,
|
|
slug: r.slug,
|
|
nl: r.nl,
|
|
photos: r.photos
|
|
})),
|
|
brands
|
|
};
|
|
}
|
|
|
|
let sqlClient: ReturnType<typeof postgres> | null = null;
|
|
function db() {
|
|
if (!process.env.DB_URL) throw new Error('Set DB_URL (or use --dry-run --models <built.json>)');
|
|
sqlClient ??= postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
|
return sqlClient;
|
|
}
|
|
|
|
async function enrich(file: string) {
|
|
const report = JSON.parse(await readFile(file, 'utf8')) as {
|
|
newModels?: { url: string; photo?: string | null }[];
|
|
};
|
|
const sql = db();
|
|
for (const m of report.newModels ?? []) {
|
|
const [, brand, model] =
|
|
new URL(m.url).pathname.match(/\/trackers\/cars\/([^/]+)\/([^/]+)/) ?? [];
|
|
const [row] = await sql<{ photo: string | null }[]>`
|
|
select m.photo from cars.models m join cars.brands b on b.id = m.brand_id
|
|
where b.slug = ${brand} and m.slug = ${model}`;
|
|
m.photo = row?.photo ?? null;
|
|
}
|
|
console.log(JSON.stringify(report, null, 2));
|
|
}
|
|
|
|
async function main() {
|
|
await loadApiCache();
|
|
if (enrichFile && !flag('--fetch')) {
|
|
// Enrich only: look up photos already written.
|
|
await enrich(enrichFile);
|
|
return;
|
|
}
|
|
const loaded = await loadTargets();
|
|
const { brands } = loaded;
|
|
// --missing: only models without photos are (re)matched; siblings still come from all models.
|
|
const targets = loaded.targets.filter((t) => !missingOnly || t.photos === 0);
|
|
targets.sort((a, b) => b.nl - a.nl);
|
|
const wd = await loadWikidata();
|
|
const { match, makersOf } = matcher(wd);
|
|
const matches = new Map<Target, Match>();
|
|
for (const t of targets) {
|
|
const m = match(t);
|
|
if (m) matches.set(t, m);
|
|
}
|
|
log(`matched ${matches.size}/${targets.length} models to Wikidata`);
|
|
const photos = await photosFor(targets, wd, matches, loaded.targets);
|
|
|
|
// Brand logos (P154 of the manufacturer/brand item whose name matches best).
|
|
const logos = new Map<string, { photo: Photo; wikidata: string; article: string | null }>();
|
|
const brandPick = new Map<
|
|
string,
|
|
{ id: string; logo: string | null; article: string | null; label: string }
|
|
>();
|
|
for (const b of brands) {
|
|
const cands = [...makersOf(b.name)].map((id) => wd.makers.get(id)!).filter(Boolean);
|
|
const exact = cands.filter((m) => cmp(m.label) === cmp(b.name));
|
|
const pick = [...exact, ...cands].find((m) => m.logo) ?? exact[0] ?? null;
|
|
if (pick) brandPick.set(b.slug, pick);
|
|
}
|
|
await imageInfo(
|
|
[...brandPick.values()].map((p) => p.logo).filter((l): l is string => !!l),
|
|
250
|
|
);
|
|
for (const [slug, p] of brandPick) {
|
|
const photo = p.logo ? toPhoto(p.logo, 'logo', null, 250) : null;
|
|
if (photo) logos.set(slug, { photo, wikidata: p.id, article: p.article });
|
|
}
|
|
await saveApiCache(true);
|
|
|
|
const total = [...photos.values()].reduce((a, p) => a + p.length, 0);
|
|
log(`photos: ${photos.size} models with photos, ${total} photos, ${logos.size} brand logos`);
|
|
if (outFile)
|
|
await writeFile(
|
|
outFile,
|
|
JSON.stringify(
|
|
targets
|
|
.filter((t) => matches.has(t))
|
|
.map((t) => ({
|
|
brand: t.brand,
|
|
model: t.model,
|
|
item: matches.get(t)!.item.id,
|
|
label: matches.get(t)!.item.label,
|
|
via: matches.get(t)!.via,
|
|
photos: (photos.get(t) ?? []).map((p) => `${p.kind}: ${p.file}`)
|
|
}))
|
|
)
|
|
);
|
|
if (dryRun) return;
|
|
|
|
const sql = db();
|
|
await sql.begin(async (tx) => {
|
|
const ids = targets.map((t) => t.id!).filter(Boolean);
|
|
// Replace photos for every model we looked at (including ones that lost their match).
|
|
await tx`delete from cars.photos where model_id = any(${ids})`;
|
|
await tx`update cars.models set wikidata = null, wikipedia = null, photo = null, photo_count = 0, photos_checked_at = now() where id = any(${ids})`;
|
|
for (const t of targets) {
|
|
const m = matches.get(t);
|
|
if (!m || !t.id) continue;
|
|
const list = photos.get(t) ?? [];
|
|
await tx`update cars.models set wikidata = ${m.item.id}, wikipedia = ${m.item.article},
|
|
photo = ${list[0]?.thumb ?? null}, photo_count = ${list.length} where id = ${t.id}`;
|
|
if (list.length)
|
|
await tx`insert into cars.photos ${tx(
|
|
list.map((p, i) => ({
|
|
model_id: t.id,
|
|
position: i,
|
|
kind: p.kind,
|
|
file: p.file,
|
|
caption: p.caption,
|
|
thumb: p.thumb,
|
|
width: p.width,
|
|
height: p.height,
|
|
author: p.author,
|
|
license: p.license,
|
|
license_url: p.licenseUrl
|
|
}))
|
|
)}`;
|
|
}
|
|
{
|
|
// --missing only fills brands that have no logo yet (e.g. brand new ones).
|
|
for (const b of brands) {
|
|
const l = logos.get(b.slug);
|
|
await tx`update cars.brands set wikidata = ${l?.wikidata ?? brandPick.get(b.slug)?.id ?? null},
|
|
wikipedia = ${l?.article ?? brandPick.get(b.slug)?.article ?? null},
|
|
logo = ${l?.photo.thumb ?? null}, logo_file = ${l?.photo.file ?? null},
|
|
logo_license = ${l?.photo.license ?? null}, logo_author = ${l?.photo.author ?? null}
|
|
where slug = ${b.slug} and (${!missingOnly} or logo is null)`;
|
|
}
|
|
}
|
|
// Brand hero photo: the lead photo of its most common model that has one.
|
|
await tx`update cars.brands b set photo = (
|
|
select m.photo from cars.models m where m.brand_id = b.id and m.photo is not null
|
|
order by m.nl_count desc, m.variant_count desc limit 1)`;
|
|
});
|
|
log('written');
|
|
if (enrichFile) await enrich(enrichFile);
|
|
await sql.end();
|
|
}
|
|
|
|
await main();
|
|
await sqlClient?.end();
|