diff --git a/scripts/cars-photos.ts b/scripts/cars-photos.ts index d2a9c05..81bcfa4 100644 --- a/scripts/cars-photos.ts +++ b/scripts/cars-photos.ts @@ -1112,7 +1112,8 @@ async function main() { wikipedia = ${l?.article ?? brandPick.get(b.slug)?.article ?? null}, logo = ${l?.photo.thumb ?? null}, logo_file = ${l?.photo.file ?? null}, logo_license = ${l?.photo.license ?? null}, logo_author = ${l?.photo.author ?? null} - where slug = ${b.slug} and (${!missingOnly} or logo is null)`; + where slug = ${b.slug} and (${!missingOnly} or logo is null) + and not (${!l?.photo} and logo like '/spotters/logos/%')`; } } // Brand hero photo: the lead photo of its most common model that has one. diff --git a/scripts/cars-refresh.sh b/scripts/cars-refresh.sh index 5f63cd1..b989ece 100755 --- a/scripts/cars-refresh.sh +++ b/scripts/cars-refresh.sh @@ -10,6 +10,8 @@ # that have none yet (new ones included). # 3. bun scripts/spotters-activity.ts cars refreshes the per-country activity scores # ("most active in NL" sort; see that script). +# 4. bun scripts/spotters-logos.ts cars --new logos for new brands / makers / operators +# (Wikidata logo or icon, then Wikipedia infobox logos) # stdout: one JSON report {generatedAt, totals, newBrands, newModels[] (with photo)}, plus `nl` per new # entry and `newInNl` (new entries that are active in the Netherlands, see # scripts/spotters-report-nl.ts). --report-new is accepted (the report is always printed). @@ -31,4 +33,5 @@ trap 'rm -f "$report"' EXIT bun scripts/cars-import.ts --report-new "${args[@]}" >"$report" bun scripts/spotters-activity.ts cars >&2 +bun scripts/spotters-logos.ts cars --new >&2 bun scripts/cars-photos.ts --missing --enrich "$report" --fetch | bun scripts/spotters-report-nl.ts cars diff --git a/scripts/planes-photos.ts b/scripts/planes-photos.ts index b00ee37..7c714eb 100644 --- a/scripts/planes-photos.ts +++ b/scripts/planes-photos.ts @@ -666,7 +666,7 @@ async function main() { await tx`update planes.makers set wikipedia = ${l?.article ?? null}, logo = ${l?.photo?.thumb ?? null}, logo_file = ${l?.photo?.file ?? null}, logo_license = ${l?.photo?.license ?? null}, logo_author = ${l?.photo?.author ?? null} - where slug = ${m.slug}`; + where slug = ${m.slug} and not (${!l?.photo} and logo like '/spotters/logos/%')`; } await tx`update planes.makers k set photo = ( select t.photo from planes.types t where t.maker_id = k.id and t.photo is not null diff --git a/scripts/planes-refresh.sh b/scripts/planes-refresh.sh index 56dbb4d..0ff66be 100755 --- a/scripts/planes-refresh.sh +++ b/scripts/planes-refresh.sh @@ -11,6 +11,8 @@ # have none yet (new ones included) and maker logos that are missing. # 3. bun scripts/spotters-activity.ts planes refreshes the per-country activity scores # ("most active in NL" sort; OpenSky registrations, see that script). +# 4. bun scripts/spotters-logos.ts planes --new logos for new brands / makers / operators +# (Wikidata logo or icon, then Wikipedia infobox logos) # stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}, plus `nl` per new # entry and `newInNl` (new entries that are active in the Netherlands, see # scripts/spotters-report-nl.ts). --report-new is accepted (the report is always printed). @@ -32,4 +34,5 @@ trap 'rm -f "$report"' EXIT bun scripts/planes-import.ts --report-new "${args[@]}" >"$report" bun scripts/spotters-activity.ts planes >&2 +bun scripts/spotters-logos.ts planes --new >&2 bun scripts/planes-photos.ts --missing --enrich "$report" --fetch | bun scripts/spotters-report-nl.ts planes diff --git a/scripts/spotters-activity.ts b/scripts/spotters-activity.ts index 4df7032..b07ed05 100644 --- a/scripts/spotters-activity.ts +++ b/scripts/spotters-activity.ts @@ -306,28 +306,54 @@ async function planes() { for (let depth = 0; id && depth < 6; depth++, id = parent.get(id) ?? null) for (const [cc, n] of m) add(scores, id, cc, n); } - // Wikidata operators (P137) and their countries. + // Wikidata operators (P137) and their countries, as two cheap queries (the joined + // version times out on the public endpoint). const byQid = new Map(types.map((t) => [t.wikidata, t.id])); let ops = 0; let failed = 0; + const opsOf = new Map>(); + const fail = (e: unknown) => { + failed++; + log(`planes: operator chunk failed (${String(e).slice(0, 80)})`); + return []; + }; for (const c of chunks( [...byQid.keys()].filter((q): q is string => !!q), - 100 + 200 )) { // A failing chunk (query timeout) only loses its operator bonus; registrations stay. - const res = await sparql<{ item: string; iso: string; n: string }>(` - SELECT ?item ?iso (COUNT(DISTINCT ?op) AS ?n) WHERE { - VALUES ?item { ${c.map((q) => `wd:${q}`).join(' ')} } - ?item wdt:P137 ?op . ?op wdt:P17 ?country . ?country wdt:P297 ?iso . - } GROUP BY ?item ?iso`).catch((e: unknown) => { - failed++; - log(`planes: operator chunk failed (${String(e).slice(0, 80)})`); - return []; - }); + const res = await sparql<{ item: string; op: string }>(` + SELECT ?item ?op WHERE { VALUES ?item { ${c.map((q) => `wd:${q}`).join(' ')} } ?item wdt:P137 ?op . }`).catch( + fail + ); for (const r of res) { - const id = byQid.get(r.item.split('/').at(-1)!); - if (!id) continue; - add(scores, id, r.iso.toLowerCase(), 25 * Math.min(Number(r.n) || 1, 6)); + const item = r.item.split('/').at(-1)!; + const set = opsOf.get(item) ?? new Set(); + set.add(r.op.split('/').at(-1)!); + opsOf.set(item, set); + } + } + const isoOf = new Map>(); + for (const c of chunks([...new Set([...opsOf.values()].flatMap((s) => [...s]))], 300)) { + const res = await sparql<{ op: string; iso: string }>(` + SELECT ?op ?iso WHERE { VALUES ?op { ${c.map((q) => `wd:${q}`).join(' ')} } ?op wdt:P17 ?country . ?country wdt:P297 ?iso . }`).catch( + fail + ); + for (const r of res) { + const op = r.op.split('/').at(-1)!; + const set = isoOf.get(op) ?? new Set(); + set.add(r.iso.toLowerCase()); + isoOf.set(op, set); + } + } + for (const [item, opSet] of opsOf) { + const id = byQid.get(item); + if (!id) continue; + const perCountry = new Map(); + for (const op of opSet) + for (const cc of isoOf.get(op) ?? []) perCountry.set(cc, (perCountry.get(cc) ?? 0) + 1); + for (const [cc, n] of perCountry) { + add(scores, id, cc, 25 * Math.min(n, 6)); ops++; } } @@ -355,7 +381,7 @@ async function wildlife() { us: 'United States', gb: 'United Kingdom', cz: 'Czech Republic', - tr: 'Turkey', + tr: 'Türkiye', kr: 'South Korea' }[cc] ?? dn.of(cc.toUpperCase())!; const found = await inat<{ diff --git a/scripts/spotters-logos.ts b/scripts/spotters-logos.ts new file mode 100644 index 0000000..f7a4e55 --- /dev/null +++ b/scripts/spotters-logos.ts @@ -0,0 +1,520 @@ +// Fills missing brand / maker / operator logos for the spotter trackers and swaps very wide +// logos for a square icon when Wikidata has one, so the square logo tiles look right. +// +// DB_URL=... bun scripts/spotters-logos.ts [options] +// +// (default) orgs without a logo: Wikidata logo (P154), small logo / icon (P8972, +// P2910), the logo= field of the org's Wikipedia infobox in en, nl, de, +// fr, it, es, pl, cs, sv or ja (Commons files only) +// --square also orgs that already have a logo: if it is wide (>1.6:1) or tall and +// Wikidata has a square small logo / icon, use that instead +// --favicon last resort for orgs still without a logo: the largest square icon +// (apple-touch-icon, manifest icon, svg icon) of the org's official +// website (P856, current statement only, never for dissolved orgs), only +// when the site does not redirect to another domain. Saved as 160px webp +// in static/spotters/logos//.webp (commit + deploy them) +// --favicons-db point orgs without a logo at their static/spotters/logos file (run this +// after the files are deployed) +// --new only orgs created in the last 2 days (for the refresh scripts) +// --limit first n orgs by priority (most types, Dutch first) +// --dry-run print what would change, write nothing +// +// Logos stay strict: a file must be free on Commons (toPhoto-like check: no fair use, +// no non-free), come from the org's own Wikidata item or its own article's logo field, and +// a website icon must come from the org's own current website. +import postgres from 'postgres'; +import { createHash } from 'node:crypto'; +import { existsSync } from 'node:fs'; +import { mkdir, writeFile } from 'node:fs/promises'; +import sharp from 'sharp'; +import { + chunks, + fileName, + log, + mw, + qid, + sleep, + sparql, + stripHtml, + type ImageInfo +} from './planes-wiki'; + +const args = process.argv.slice(2); +const flag = (name: string) => args.includes(name); +const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null); +const which = args[0]; +const dryRun = flag('--dry-run'); +const square = flag('--square'); +const favicon = flag('--favicon'); +const faviconsDb = flag('--favicons-db'); +const onlyNew = flag('--new'); +const limit = Number(opt('--limit')) || Infinity; +if (!['cars', 'planes', 'trains', 'all'].includes(which ?? '')) + throw new Error('usage: spotters-logos.ts [--square] [--favicon]'); + +const LOGO = 250; +const STATIC = 'static/spotters/logos'; +const TABLES = [ + { + tracker: 'cars', + table: 'cars.brands', + dir: 'cars-brands', + select: `select b.slug, b.name, null::text as full_name, b.wikidata, b.logo, b.logo_file, b.model_count as types, + (b.country = 'Netherlands')::int + coalesce((select sum(a.score) from cars.activity a join cars.models m on m.id = a.model_id where m.brand_id = b.id and a.area = 'nl'), 0) as nl, + b.created_at from cars.brands b` + }, + { + tracker: 'planes', + table: 'planes.makers', + dir: 'planes-makers', + select: `select k.slug, k.name, k.full_name, k.wikidata, k.logo, k.logo_file, k.type_count as types, + coalesce((select sum(a.score) from planes.activity a join planes.types t on t.id = a.type_id where t.maker_id = k.id and a.area = 'nl'), 0) as nl, + k.created_at from planes.makers k` + }, + { + tracker: 'trains', + table: 'trains.makers', + dir: 'trains-makers', + select: `select k.slug, k.name, k.full_name, k.wikidata, k.logo, k.logo_file, k.type_count as types, + coalesce((select sum(a.score) from trains.activity a join trains.types t on t.id = a.type_id where t.maker_id = k.id and a.area = 'nl'), 0) as nl, + k.created_at from trains.makers k` + }, + { + tracker: 'trains', + table: 'trains.operators', + dir: 'trains-operators', + select: `select o.slug, o.name, o.full_name, o.wikidata, o.logo, o.logo_file, o.type_count as types, + (o.country = 'Netherlands')::int * 1000 as nl, o.created_at from trains.operators o` + } +] as const; +type Table = (typeof TABLES)[number]; +type Org = { + slug: string; + name: string; + full_name: string | null; + wikidata: string | null; + logo: string | null; + logo_file: string | null; + types: number | null; + nl: number | string | null; + created_at: Date; +}; +type Pick = { + logo: string; + file: string | null; + author: string | null; + license: string | null; + how: string; +}; + +if (!process.env.DB_URL) throw new Error('Set DB_URL'); +const sql = postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} }); + +// ---------------------------------------------------------------- commons +const info = new Map(); +async function imageInfo(files: string[]) { + const todo = [...new Set(files)].filter((f) => !info.has(f)); + for (const chunk of chunks(todo, 50)) { + const res = await mw('commons.wikimedia.org', { + action: 'query', + prop: 'imageinfo', + iiprop: 'url|size|mime|extmetadata', + iiurlwidth: String(LOGO), + iiextmetadatafilter: 'Artist|LicenseShortName|NonFree|Restrictions', + titles: chunk.map((f) => `File:${f}`).join('|') + }); + const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from])); + for (const p of res.query?.pages ?? []) { + const original = (norm.get(p.title) ?? p.title).replace(/^File:/, ''); + info.set(original, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]); + } + for (const f of chunk) if (!info.has(f)) info.set(f, null); + } +} +/** A free Commons image as a logo, or null (fair use / non-free / not an image). */ +function commonsLogo(file: string, how: string): (Pick & { ratio: number }) | null { + const i = info.get(file); + if (!i?.thumburl || !/^image\/(svg\+xml|png|gif|jpeg|webp)$/.test(i.mime)) return null; + const meta = i.extmetadata ?? {}; + if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null; + let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null; + if (!license || /fair use|non-free/i.test(license)) return null; + if (/trademark/i.test(meta.Restrictions?.value ?? '')) license += ' · trademark'; + if (i.width < 48 || i.height < 48) return null; + return { + logo: i.thumburl.replace(/\?.*$/, ''), + file, + author: meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null, + license, + how, + ratio: i.width / i.height + }; +} +const squarish = (r: number) => r >= 0.75 && r <= 1.34; + +// ---------------------------------------------------------------- wikidata +const WIKIS = ['en', 'nl', 'de', 'fr', 'it', 'es', 'pl', 'cs', 'sv', 'ja']; +type Wd = { + logos: string[]; + icons: string[]; + sites: string[]; + dissolved: boolean; + articles: Map; +}; +async function wikidata(ids: string[]) { + const out = new Map(); + for (const c of chunks(ids, 150)) { + let rows: Record[]; + try { + rows = await sparql(` +SELECT ?m (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(?end), "")); separator="|") AS ?logos) + (GROUP_CONCAT(DISTINCT STR(?icon); separator="|") AS ?icons) + (GROUP_CONCAT(DISTINCT STR(?site); separator="|") AS ?sites) + (GROUP_CONCAT(DISTINCT STR(?a); separator="|") AS ?articles) (SAMPLE(?gone) AS ?dissolved) +WHERE { + VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } + OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) + OPTIONAL { ?st pq:P582 ?end } } + OPTIONAL { ?m p:P8972|p:P2910 ?ist . ?ist ps:P8972|ps:P2910 ?icon ; wikibase:rank ?irank . + FILTER(?irank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?ist pq:P582 [] } } + OPTIONAL { ?m p:P856 ?wst . ?wst ps:P856 ?site ; wikibase:rank ?wrank . + FILTER(?wrank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?wst pq:P582 [] } } + OPTIONAL { ?a schema:about ?m ; schema:isPartOf ?wiki . + VALUES ?wiki { ${WIKIS.map((w) => ``).join(' ')} } } + OPTIONAL { ?m wdt:P576 ?gone } +} GROUP BY ?m`); + } catch (e) { + log(`wikidata chunk failed: ${(e as Error).message}`); + continue; + } + for (const r of rows) { + const logos = (r.logos ? r.logos.split('|').filter(Boolean) : []) + .map((x) => { + const [uri, rank, end] = x.split(';'); + return { + file: fileName(uri), + score: (rank.endsWith('PreferredRank') ? 2 : 0) + (end ? 0 : 4) + }; + }) + .sort((a, b) => b.score - a.score); + const articles = new Map(); + for (const a of r.articles ? r.articles.split('|') : []) { + const m = a.match(/^https:\/\/(\w+)\.wikipedia\.org\/wiki\/(.+)$/); + if (m) articles.set(m[1], decodeURIComponent(m[2]).replace(/_/g, ' ')); + } + out.set(qid(r.m), { + logos: [...new Set(logos.map((l) => l.file))], + icons: r.icons ? [...new Set(r.icons.split('|').map(fileName))] : [], + sites: r.sites ? r.sites.split('|').filter((s) => /^https?:\/\//.test(s)) : [], + dissolved: !!r.dissolved, + articles + }); + } + } + return out; +} + +// ---------------------------------------------------------------- wikipedia infoboxes +const INFOBOX_LOGO = + /\|\s*(?:logo|logo_image|image_logo|logotipo|logotyp|logo_bild|bild_logo|ロゴ)\s*=\s*(?:\[\[)?(?:[^\s:|\]=]{2,12}:)?\s*([^|\]\n{}<=]+?\.(?:svg|png|gif))/i; +async function infoboxLogos(wiki: string, titles: string[]) { + const out = new Map(); + for (const c of chunks(titles, 40)) { + const res = await mw(`${wiki}.wikipedia.org`, { + action: 'query', + prop: 'revisions', + rvprop: 'content', + rvslots: 'main', + rvsection: '0', + redirects: '1', + titles: c.join('|') + }).catch(() => null); + if (!res) continue; + const back = new Map(); + for (const n of res.query?.normalized ?? []) back.set(n.to, n.from); + for (const n of res.query?.redirects ?? []) back.set(n.to, back.get(n.from) ?? n.from); + for (const p of res.query?.pages ?? []) { + const text = p.revisions?.[0]?.slots?.main?.content ?? ''; + const m = text.match(INFOBOX_LOGO); + if (m) out.set(back.get(p.title) ?? p.title, m[1].trim().replace(/_/g, ' ')); + } + } + return out; +} + +// ---------------------------------------------------------------- website icons +const PARKED = + /domain (?:is |may be )?for sale|buy this domain|parked (?:free|domain)|this domain|sedo|hugedomains|afternic|dan\.com|bodis|godaddy|namecheap|domain expired|account suspended|default web site page|it works!|welcome to nginx/i; +const site2 = (host: string) => { + const parts = host.replace(/^www\./, '').split('.'); + const n = parts.length >= 3 && /^(co|com|org|net|ac|gov|ne|or)$/.test(parts.at(-2)!) ? 3 : 2; + return parts.slice(-n).join('.'); +}; +async function get(url: string, max = 1_500_000) { + const res = await fetch(url, { + redirect: 'follow', + signal: AbortSignal.timeout(15_000), + headers: { + 'user-agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36 (stanrunge.dev spotters logo check)', + accept: '*/*' + } + }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const buf = Buffer.from(await res.arrayBuffer()); + if (buf.length > max) throw new Error('too big'); + return { url: res.url, type: res.headers.get('content-type') ?? '', buf }; +} +const attr = (tag: string, name: string) => + tag + .match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i')) + ?.slice(1) + .find((x) => x != null) ?? null; +type IconCandidate = { url: string; size: number }; +async function websiteIcon( + site: string +): Promise<{ png: Buffer; host: string; hash: string } | { error: string }> { + let page; + try { + page = await get(site, 3_000_000); + } catch (e) { + return { error: `site: ${(e as Error).message}` }; + } + const from = new URL(site).hostname; + const to = new URL(page.url).hostname; + if (site2(from) !== site2(to)) return { error: `redirects to ${to}` }; + const html = page.buf.toString('utf8'); + const title = stripHtml(html.match(/]*>([\s\S]{0,300}?)<\/title>/i)?.[1] ?? ''); + if (PARKED.test(title) || html.length < 400) + return { error: `parked/empty (${title.slice(0, 60)})` }; + const cands: IconCandidate[] = []; + for (const tag of html.match(/]*>/gi) ?? []) { + const rel = (attr(tag, 'rel') ?? '').toLowerCase(); + const href = attr(tag, 'href'); + if (!href || !/\bicon\b|apple-touch-icon/.test(rel) || /mask-icon/.test(rel)) continue; + const sizes = attr(tag, 'sizes') ?? ''; + const n = Math.max( + 0, + ...[...sizes.matchAll(/(\d+)x(\d+)/gi)].map((m) => Math.min(+m[1], +m[2])) + ); + const svg = /\.svg(\?|$)/i.test(href) || /svg/i.test(attr(tag, 'type') ?? ''); + const size = svg ? 1000 : n || (/apple-touch/.test(rel) ? 180 : 32); + try { + cands.push({ url: new URL(href.replace(/&/g, '&'), page.url).href, size }); + } catch { + /* bad href */ + } + } + const manifest = html.match(/]*rel\s*=\s*["']?manifest[^>]*>/i)?.[0]; + const mhref = manifest ? attr(manifest, 'href') : null; + if (mhref) + try { + const murl = new URL(mhref.replace(/&/g, '&'), page.url).href; + const m = JSON.parse((await get(murl, 200_000)).buf.toString('utf8')) as { + icons?: { src: string; sizes?: string; purpose?: string }[]; + }; + for (const i of m.icons ?? []) { + if (/monochrome/.test(i.purpose ?? '')) continue; + const n = Math.max( + 0, + ...[...(i.sizes ?? '').matchAll(/(\d+)x(\d+)/gi)].map((x) => Math.min(+x[1], +x[2])) + ); + cands.push({ url: new URL(i.src, murl).href, size: Math.min(n || 192, 512) }); + } + } catch { + /* no manifest */ + } + cands.push({ url: new URL('/apple-touch-icon.png', page.url).href, size: 179 }); + cands.sort((a, b) => b.size - a.size); + const seen = new Set(); + for (const c of cands) { + if (c.size < 96 || seen.has(c.url) || /\.ico(\?|$)/i.test(c.url)) continue; + seen.add(c.url); + try { + const img = await get(c.url); + if ( + site2(new URL(img.url).hostname) !== site2(to) && + !/cdn|static|assets|img|media|cloudfront|akamai|wp\.com|squarespace|wixstatic|shopify/i.test( + new URL(img.url).hostname + ) + ) + continue; + const meta = await sharp(img.buf, { density: 300 }).metadata(); + const w = meta.width ?? 0; + const h = meta.height ?? 0; + if (meta.format !== 'svg' && Math.min(w, h) < 96) continue; + if (!squarish(w / h)) continue; + const stats = await sharp(img.buf, { density: 300 }) + .resize(64, 64, { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } }) + .ensureAlpha() + .stats(); + // A blank or single-colour square is a placeholder, not a logo. + const spread = stats.channels.reduce((s, ch) => s + ch.stdev, 0); + if (spread < 12) continue; + const png = await sharp(img.buf, { density: 300 }) + .resize(160, 160, { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } }) + .webp({ quality: 90 }) + .toBuffer(); + return { + png, + host: to.replace(/^www\./, ''), + hash: createHash('sha1').update(png).digest('hex') + }; + } catch { + /* next candidate */ + } + } + return { error: `no square icon >= 96px (${cands.length} candidates)` }; +} + +// ---------------------------------------------------------------- main +async function write(t: Table, slug: string, p: Pick) { + if (dryRun) return; + await sql.unsafe( + `update ${t.table} set logo = $1, logo_file = $2, logo_author = $3, logo_license = $4 where slug = $5`, + [p.logo, p.file, p.author, p.license, slug] + ); +} + +const tables = TABLES.filter((t) => which === 'all' || t.tracker === which); +const favHashes = new Map(); +for (const t of tables) { + let orgs = (await sql.unsafe(t.select)) as unknown as Org[]; + const before = orgs.filter((o) => o.logo).length; + if (onlyNew) orgs = orgs.filter((o) => Date.now() - +new Date(o.created_at) < 2 * 86400_000); + if (faviconsDb) { + let n = 0; + for (const o of orgs.filter((o) => !o.logo)) { + if (!existsSync(`${STATIC}/${t.dir}/${o.slug}.webp`)) continue; + n++; + await write(t, o.slug, { + logo: `/spotters/logos/${t.dir}/${o.slug}.webp`, + file: null, + author: null, + license: 'website icon · trademark', + how: 'favicon' + }); + } + log(`${t.table}: ${n} website icons linked`); + continue; + } + orgs = orgs + .filter((o) => (square || !o.logo) && o.wikidata) + .sort((a, b) => Number(b.nl ?? 0) - Number(a.nl ?? 0) || (b.types ?? 0) - (a.types ?? 0)) + .slice(0, limit); + log(`${t.table}: ${before} with a logo; checking ${orgs.length}`); + const wd = await wikidata([...new Set(orgs.map((o) => o.wikidata!))]); + await imageInfo([ + ...[...wd.values()].flatMap((w) => [...w.logos, ...w.icons]), + ...orgs.map((o) => o.logo_file).filter((f): f is string => !!f) + ]); + const picks = new Map(); + const stillMissing: Org[] = []; + for (const o of orgs) { + const w = wd.get(o.wikidata!); + const icons = (w?.icons ?? []) + .map((f) => commonsLogo(f, 'wikidata icon')) + .filter((x) => x && squarish(x.ratio)); + if (o.logo) { + // --square: swap a wide or tall logo for a square icon. + const cur = o.logo_file ? info.get(o.logo_file) : null; + const ratio = cur ? cur.width / cur.height : 1; + if (icons[0] && (ratio > 1.6 || ratio < 0.6) && icons[0].file !== o.logo_file) + picks.set(o.slug, icons[0]); + continue; + } + const logos = (w?.logos ?? []).map((f) => commonsLogo(f, 'wikidata logo')).filter((x) => !!x); + // Prefer a square logo, then a square icon, then any logo. + const pick = + logos.find((l) => squarish(l.ratio)) ?? + icons[0] ?? + logos[0] ?? + (w?.icons ?? []).map((f) => commonsLogo(f, 'wikidata icon')).find((x) => !!x); + if (pick) picks.set(o.slug, pick); + else stillMissing.push(o); + } + // Wikipedia infobox logo= fields, wiki by wiki. + let left = stillMissing; + for (const wiki of WIKIS) { + const want = left.filter((o) => wd.get(o.wikidata!)?.articles.has(wiki)); + if (!want.length) continue; + const found = await infoboxLogos( + wiki, + want.map((o) => wd.get(o.wikidata!)!.articles.get(wiki)!) + ); + const files = new Map(); + for (const o of want) { + const f = found.get(wd.get(o.wikidata!)!.articles.get(wiki)!); + if (f) files.set(o.slug, f); + } + await imageInfo([...files.values()]); + for (const [slug, f] of files) { + const p = commonsLogo(f, `${wiki}wiki infobox`); + if (p) picks.set(slug, p); + } + left = left.filter((o) => !picks.has(o.slug)); + } + for (const o of orgs) { + const p = picks.get(o.slug); + if (!p) continue; + log(` ${t.table} ${o.slug}: ${p.how} ${p.file}${o.logo ? ' (square swap)' : ''}`); + await write(t, o.slug, p); + } + const added = orgs.filter((o) => !o.logo && picks.has(o.slug)).length; + log( + `${t.table}: +${added} logos from Wikimedia, ${orgs.filter((o) => o.logo && picks.has(o.slug)).length} swapped for square icons, ${left.length} still missing` + ); + if (favicon) { + let tried = 0; + const fails: Record = {}; + for (const o of left) { + const w = wd.get(o.wikidata!); + if (!w?.sites.length) { + fails['no website'] = (fails['no website'] ?? 0) + 1; + continue; + } + if (w.dissolved) { + fails['dissolved'] = (fails['dissolved'] ?? 0) + 1; + continue; + } + tried++; + const r = await websiteIcon(w.sites[0]); + if ('error' in r) { + const k = r.error + .replace(/\(.*$/, '') + .replace(/HTTP \d+/, 'HTTP error') + .trim() + .slice(0, 40); + fails[k] = (fails[k] ?? 0) + 1; + log(` ${o.slug}: ${r.error}`); + } else { + const list = favHashes.get(r.hash) ?? []; + list.push({ t, slug: o.slug, host: r.host, png: r.png }); + favHashes.set(r.hash, list); + log(` ${o.slug}: icon from ${r.host}`); + } + await sleep(300); + } + log(`${t.table}: website icons tried ${tried}; failures ${JSON.stringify(fails)}`); + } +} +if (favicon) { + let saved = 0; + for (const list of favHashes.values()) { + // The same icon on different sites is a generic CMS / host icon: skip all of them. + if (new Set(list.map((x) => x.host)).size > 1) { + log(` generic icon shared by ${list.map((x) => x.slug).join(', ')}: skipped`); + continue; + } + for (const x of list) { + if (dryRun) continue; + await mkdir(`${STATIC}/${x.t.dir}`, { recursive: true }); + await writeFile(`${STATIC}/${x.t.dir}/${x.slug}.webp`, x.png); + saved++; + } + } + log( + `website icons saved: ${saved} (commit static/spotters/logos, deploy, then run --favicons-db)` + ); +} +await sql.end(); diff --git a/scripts/trains-photos.ts b/scripts/trains-photos.ts index 41a8e6c..739628d 100644 --- a/scripts/trains-photos.ts +++ b/scripts/trains-photos.ts @@ -738,10 +738,12 @@ async function main() { const p = l?.photo; if (m.table === 'makers') await tx`update trains.makers set wikipedia = ${l?.article ?? null}, logo = ${p?.thumb ?? null}, logo_file = ${p?.file ?? null}, - logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug}`; + logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug} + and not (${!p} and logo like '/spotters/logos/%')`; else await tx`update trains.operators set wikipedia = ${l?.article ?? null}, logo = ${p?.thumb ?? null}, logo_file = ${p?.file ?? null}, - logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug}`; + logo_license = ${p?.license ?? null}, logo_author = ${p?.author ?? null} where slug = ${m.slug} + and not (${!p} and logo like '/spotters/logos/%')`; } await tx`update trains.makers k set photo = ( select t.photo from trains.types t where t.maker_id = k.id and t.photo is not null diff --git a/scripts/trains-refresh.sh b/scripts/trains-refresh.sh index a0d9eaf..7048793 100755 --- a/scripts/trains-refresh.sh +++ b/scripts/trains-refresh.sh @@ -11,6 +11,8 @@ # have none yet (new ones included) and maker and operator logos that are missing. # 3. bun scripts/spotters-activity.ts trains refreshes the per-country activity scores # ("most active in NL" sort; see that script). +# 4. bun scripts/spotters-logos.ts trains --new logos for new brands / makers / operators +# (Wikidata logo or icon, then Wikipedia infobox logos) # stdout: one JSON report {generatedAt, totals, newMakers, newTypes[] (with photo)}, plus `nl` per new # entry and `newInNl` (new entries that are active in the Netherlands, see # scripts/spotters-report-nl.ts). --report-new is accepted (the report is always printed). @@ -32,4 +34,5 @@ trap 'rm -f "$report"' EXIT bun scripts/trains-import.ts --report-new "${args[@]}" >"$report" bun scripts/spotters-activity.ts trains >&2 +bun scripts/spotters-logos.ts trains --new >&2 bun scripts/trains-photos.ts --missing --enrich "$report" --fetch | bun scripts/spotters-report-nl.ts trains