Stan Runge
7aeb31c46a
All checks were successful
Deploy production / deploy (push) Successful in 1m9s
614 lines
23 KiB
TypeScript
614 lines
23 KiB
TypeScript
// Fills missing brand / maker / operator logos for the spotter trackers and swaps very wide
|
|
// logos for a square icon when Wikidata has one, so the square logo tiles look right.
|
|
//
|
|
// DB_URL=... bun scripts/spotters-logos.ts <cars|planes|trains|all> [options]
|
|
//
|
|
// (default) orgs without a logo: Wikidata logo (P154), small logo / icon (P8972,
|
|
// P2910), the logo= field of the org's Wikipedia infobox in en, nl, de,
|
|
// fr, it, es, pl, cs, sv or ja (Commons files only)
|
|
// --square also orgs that already have a logo: if it is wide (>1.6:1) or tall and
|
|
// Wikidata has a square small logo / icon, use that instead
|
|
// --favicon last resort for orgs still without a logo: the largest square icon
|
|
// (apple-touch-icon, manifest icon, svg icon) of the org's official
|
|
// website (P856, current statement only, never for dissolved orgs), only
|
|
// when the site does not redirect to another domain. Saved as 160px webp
|
|
// in static/spotters/logos/<table>/<slug>.webp (commit + deploy them)
|
|
// --favicons-db point orgs without a logo at their static/spotters/logos file (run this
|
|
// after the files are deployed)
|
|
// --new only orgs created in the last 2 days (for the refresh scripts)
|
|
// --limit <n> first n orgs by priority (most types, Dutch first)
|
|
// --recheck <log> re-test the Wikimedia picks listed in an earlier run's log with the
|
|
// current rules and clear the ones that fail
|
|
// --dry-run print what would change, write nothing
|
|
//
|
|
// Logos stay strict: a file must be free on Commons (toPhoto-like check: no fair use,
|
|
// no non-free), come from the org's own Wikidata item or its own article's logo field, and
|
|
// a website icon must come from the org's own current website.
|
|
import postgres from 'postgres';
|
|
import { createHash } from 'node:crypto';
|
|
import { existsSync } from 'node:fs';
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import sharp from 'sharp';
|
|
import {
|
|
chunks,
|
|
cmp,
|
|
fileName,
|
|
log,
|
|
mw,
|
|
qid,
|
|
sleep,
|
|
sparql,
|
|
stripHtml,
|
|
tokens,
|
|
type ImageInfo
|
|
} from './planes-wiki';
|
|
|
|
const args = process.argv.slice(2);
|
|
const flag = (name: string) => args.includes(name);
|
|
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
|
const which = args[0];
|
|
const dryRun = flag('--dry-run');
|
|
const square = flag('--square');
|
|
const favicon = flag('--favicon');
|
|
const faviconsDb = flag('--favicons-db');
|
|
const onlyNew = flag('--new');
|
|
const limit = Number(opt('--limit')) || Infinity;
|
|
const recheck = opt('--recheck');
|
|
if (!['cars', 'planes', 'trains', 'all'].includes(which ?? ''))
|
|
throw new Error('usage: spotters-logos.ts <cars|planes|trains|all> [--square] [--favicon]');
|
|
|
|
const LOGO = 250;
|
|
const STATIC = 'static/spotters/logos';
|
|
const TABLES = [
|
|
{
|
|
tracker: 'cars',
|
|
table: 'cars.brands',
|
|
dir: 'cars-brands',
|
|
select: `select b.slug, b.name, null::text as full_name, b.wikidata, b.logo, b.logo_file, b.model_count as types,
|
|
(b.country = 'Netherlands')::int + coalesce((select sum(a.score) from cars.activity a join cars.models m on m.id = a.model_id where m.brand_id = b.id and a.area = 'nl'), 0) as nl,
|
|
b.created_at from cars.brands b`
|
|
},
|
|
{
|
|
tracker: 'planes',
|
|
table: 'planes.makers',
|
|
dir: 'planes-makers',
|
|
select: `select k.slug, k.name, k.full_name, k.wikidata, k.logo, k.logo_file, k.type_count as types,
|
|
coalesce((select sum(a.score) from planes.activity a join planes.types t on t.id = a.type_id where t.maker_id = k.id and a.area = 'nl'), 0) as nl,
|
|
k.created_at from planes.makers k`
|
|
},
|
|
{
|
|
tracker: 'trains',
|
|
table: 'trains.makers',
|
|
dir: 'trains-makers',
|
|
select: `select k.slug, k.name, k.full_name, k.wikidata, k.logo, k.logo_file, k.type_count as types,
|
|
coalesce((select sum(a.score) from trains.activity a join trains.types t on t.id = a.type_id where t.maker_id = k.id and a.area = 'nl'), 0) as nl,
|
|
k.created_at from trains.makers k`
|
|
},
|
|
{
|
|
tracker: 'trains',
|
|
table: 'trains.operators',
|
|
dir: 'trains-operators',
|
|
select: `select o.slug, o.name, o.full_name, o.wikidata, o.logo, o.logo_file, o.type_count as types,
|
|
(o.country = 'Netherlands')::int * 1000 as nl, o.created_at from trains.operators o`
|
|
}
|
|
] as const;
|
|
type Table = (typeof TABLES)[number];
|
|
type Org = {
|
|
slug: string;
|
|
name: string;
|
|
full_name: string | null;
|
|
wikidata: string | null;
|
|
logo: string | null;
|
|
logo_file: string | null;
|
|
types: number | null;
|
|
nl: number | string | null;
|
|
created_at: Date;
|
|
};
|
|
type Pick = {
|
|
logo: string;
|
|
file: string | null;
|
|
author: string | null;
|
|
license: string | null;
|
|
how: string;
|
|
};
|
|
|
|
if (!process.env.DB_URL) throw new Error('Set DB_URL');
|
|
const sql = postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
|
|
|
// ---------------------------------------------------------------- commons
|
|
const info = new Map<string, ImageInfo | null>();
|
|
async function imageInfo(files: string[]) {
|
|
const todo = [...new Set(files)].filter((f) => !info.has(f));
|
|
for (const chunk of chunks(todo, 50)) {
|
|
const res = await mw('commons.wikimedia.org', {
|
|
action: 'query',
|
|
prop: 'imageinfo',
|
|
iiprop: 'url|size|mime|extmetadata',
|
|
iiurlwidth: String(LOGO),
|
|
iiextmetadatafilter: 'Artist|LicenseShortName|NonFree|Restrictions',
|
|
titles: chunk.map((f) => `File:${f}`).join('|')
|
|
});
|
|
const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
|
|
for (const p of res.query?.pages ?? []) {
|
|
const original = (norm.get(p.title) ?? p.title).replace(/^File:/, '');
|
|
info.set(original, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]);
|
|
}
|
|
for (const f of chunk) if (!info.has(f)) info.set(f, null);
|
|
}
|
|
}
|
|
/** A free Commons image as a logo, or null (fair use / non-free / not an image). */
|
|
function commonsLogo(
|
|
file: string,
|
|
how: string,
|
|
names: string[]
|
|
): (Pick & { ratio: number }) | null {
|
|
const i = info.get(file);
|
|
if (!i?.thumburl || !/^image\/(svg\+xml|png|gif|jpeg|webp)$/.test(i.mime)) return null;
|
|
const meta = i.extmetadata ?? {};
|
|
if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
|
|
let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
|
|
if (!license || /fair use|non-free/i.test(license)) return null;
|
|
if (/trademark/i.test(meta.Restrictions?.value ?? '')) license += ' · trademark';
|
|
if (i.width < 48 || i.height < 48) return null;
|
|
if (!looksLikeLogo(file, how, names, license)) return null;
|
|
return {
|
|
logo: i.thumburl.replace(/\?.*$/, ''),
|
|
file,
|
|
author: meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null,
|
|
license,
|
|
how,
|
|
ratio: i.width / i.height
|
|
};
|
|
}
|
|
const squarish = (r: number) => r >= 0.75 && r <= 1.34;
|
|
|
|
// Wikidata's logo property and infobox logo fields often hold photos (a badge on a car, a
|
|
// works plate, a building), old names or national insignia, so every file must also look
|
|
// like a logo of this org by its name.
|
|
const LOGO_WORD = /logo|logotype|logotip|wordmark|emblem|marque|insignia|signet|symbol|лого/i;
|
|
const NOT_LOGO =
|
|
/plate\b|plaque|schild|advertisement|werbung|relief|building|interior|museum|launch|drawing|left side|naca|\bimg|\bdsc|\(\d{6,}\)|cropped|flickr|coat of arms|reichsadler|\bflags?\b|\bseal\b|banner|roundel of|\bon a car|\bon st-|works\b/i;
|
|
const FREE_PD = /public domain|^pd\b|cc0/i;
|
|
const GENERIC = new Set(
|
|
'RAIL RAILWAY RAILWAYS AVIATION AIRCRAFT AIRCRAFTS AIRPLANE COMPANY WORKS WERKE GROUP CORPORATION CORP MOTOR MOTORS CARS LOCOMOTIVE LOCOMOTIVES ENGINEERING INDUSTRIES INDUSTRY AND THE GMBH LTD LIMITED INC FACTORY PLANT TRANSPORT TRANSPORTATION'.split(
|
|
' '
|
|
)
|
|
);
|
|
function looksLikeLogo(file: string, how: string, names: string[], license: string | null) {
|
|
if (NOT_LOGO.test(file)) return false;
|
|
const f = cmp(file);
|
|
const ft = new Set(tokens(file));
|
|
const full = names.some((n) => cmp(n).length >= 3 && f.includes(cmp(n)));
|
|
const token = names.some((n) =>
|
|
tokens(n).some((t) => t.length >= 4 && !GENERIC.has(t) && ft.has(t))
|
|
);
|
|
const word = LOGO_WORD.test(file);
|
|
const fromItem = how.startsWith('wikidata');
|
|
const many = names.some(
|
|
(n) => tokens(n).filter((t) => t.length >= 4 && !GENERIC.has(t) && ft.has(t)).length >= 2
|
|
);
|
|
if (/\.svg$/i.test(file)) return fromItem || many || (word && (full || token));
|
|
if (/\.(png|gif)$/i.test(file)) return fromItem ? word || full || token : full || (word && token);
|
|
return fromItem && word && (full || (token && FREE_PD.test(license ?? '')));
|
|
}
|
|
|
|
// ---------------------------------------------------------------- wikidata
|
|
const WIKIS = ['en', 'nl', 'de', 'fr', 'it', 'es', 'pl', 'cs', 'sv', 'ja'];
|
|
type Wd = {
|
|
logos: string[];
|
|
oldLogos: string[];
|
|
icons: string[];
|
|
sites: string[];
|
|
dissolved: boolean;
|
|
articles: Map<string, string>;
|
|
};
|
|
async function wikidata(ids: string[]) {
|
|
const out = new Map<string, Wd>();
|
|
const entry = (q: string) => {
|
|
let w = out.get(q);
|
|
if (!w)
|
|
out.set(
|
|
q,
|
|
(w = {
|
|
logos: [],
|
|
oldLogos: [],
|
|
icons: [],
|
|
sites: [],
|
|
dissolved: false,
|
|
articles: new Map()
|
|
})
|
|
);
|
|
return w;
|
|
};
|
|
// Four small queries per chunk: one joined query times out on the public endpoint.
|
|
const run = async (c: string[], body: string) => {
|
|
try {
|
|
return await sparql(
|
|
`SELECT * WHERE { VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } ${body} }`
|
|
);
|
|
} catch (e) {
|
|
log(`wikidata chunk failed: ${(e as Error).message}`);
|
|
return [];
|
|
}
|
|
};
|
|
for (const c of chunks(ids, 150)) {
|
|
const logos = await run(
|
|
c,
|
|
`?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) OPTIONAL { ?st pq:P582 ?end }`
|
|
);
|
|
const scored = new Map<string, { file: string; score: number }[]>();
|
|
for (const r of logos) {
|
|
const list = scored.get(qid(r.m)) ?? [];
|
|
list.push({
|
|
file: fileName(r.logo),
|
|
score: (r.rank.endsWith('PreferredRank') ? 2 : 0) + (r.end ? 0 : 4)
|
|
});
|
|
scored.set(qid(r.m), list);
|
|
}
|
|
for (const [q, list] of scored) {
|
|
const sorted = list.sort((x, y) => y.score - x.score);
|
|
// Ended logos only count for dissolved orgs (a renamed org must not keep its old logo).
|
|
entry(q).logos = [...new Set(sorted.filter((x) => x.score >= 4).map((x) => x.file))];
|
|
entry(q).oldLogos = [...new Set(sorted.map((x) => x.file))];
|
|
}
|
|
for (const r of await run(
|
|
c,
|
|
`?m p:P8972|p:P2910 ?st . ?st ps:P8972|ps:P2910 ?icon ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?st pq:P582 [] }`
|
|
))
|
|
entry(qid(r.m)).icons.push(fileName(r.icon));
|
|
for (const r of await run(
|
|
c,
|
|
`{ ?m p:P856 ?st . ?st ps:P856 ?site ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?st pq:P582 [] } } UNION { ?m wdt:P576 ?gone }`
|
|
)) {
|
|
if (r.gone) entry(qid(r.m)).dissolved = true;
|
|
else if (/^https?:\/\//.test(r.site ?? '')) entry(qid(r.m)).sites.push(r.site);
|
|
}
|
|
for (const r of await run(
|
|
c,
|
|
`?a schema:about ?m ; schema:isPartOf ?wiki . VALUES ?wiki { ${WIKIS.map((w) => `<https://${w}.wikipedia.org/>`).join(' ')} }`
|
|
)) {
|
|
const m = r.a.match(/^https:\/\/(\w+)\.wikipedia\.org\/wiki\/(.+)$/);
|
|
if (m) entry(qid(r.m)).articles.set(m[1], decodeURIComponent(m[2]).replace(/_/g, ' '));
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- wikipedia infoboxes
|
|
const INFOBOX_LOGO =
|
|
/\|\s*(?:logo|logo_image|image_logo|logotipo|logotyp|logo_bild|bild_logo|ロゴ)\s*=\s*(?:\[\[)?(?:[^\s:|\]=]{2,12}:)?\s*([^|\]\n{}<=]+?\.(?:svg|png|gif))/i;
|
|
async function infoboxLogos(wiki: string, titles: string[]) {
|
|
const out = new Map<string, string>();
|
|
for (const c of chunks(titles, 40)) {
|
|
const res = await mw(`${wiki}.wikipedia.org`, {
|
|
action: 'query',
|
|
prop: 'revisions',
|
|
rvprop: 'content',
|
|
rvslots: 'main',
|
|
rvsection: '0',
|
|
redirects: '1',
|
|
titles: c.join('|')
|
|
}).catch(() => null);
|
|
if (!res) continue;
|
|
const back = new Map<string, string>();
|
|
for (const n of res.query?.normalized ?? []) back.set(n.to, n.from);
|
|
for (const n of res.query?.redirects ?? []) back.set(n.to, back.get(n.from) ?? n.from);
|
|
for (const p of res.query?.pages ?? []) {
|
|
const text = p.revisions?.[0]?.slots?.main?.content ?? '';
|
|
const m = text.match(INFOBOX_LOGO);
|
|
if (m) out.set(back.get(p.title) ?? p.title, m[1].trim().replace(/_/g, ' '));
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- website icons
|
|
const PARKED =
|
|
/domain (?:is |may be )?for sale|buy this domain|parked (?:free|domain)|this domain|sedo|hugedomains|afternic|dan\.com|bodis|godaddy|namecheap|domain expired|account suspended|default web site page|it works!|welcome to nginx/i;
|
|
const site2 = (host: string) => {
|
|
const parts = host.replace(/^www\./, '').split('.');
|
|
const n = parts.length >= 3 && /^(co|com|org|net|ac|gov|ne|or)$/.test(parts.at(-2)!) ? 3 : 2;
|
|
return parts.slice(-n).join('.');
|
|
};
|
|
async function get(url: string, max = 1_500_000) {
|
|
const res = await fetch(url, {
|
|
redirect: 'follow',
|
|
signal: AbortSignal.timeout(15_000),
|
|
headers: {
|
|
'user-agent':
|
|
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36 (stanrunge.dev spotters logo check)',
|
|
accept: '*/*'
|
|
}
|
|
});
|
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
const buf = Buffer.from(await res.arrayBuffer());
|
|
if (buf.length > max) throw new Error('too big');
|
|
return { url: res.url, type: res.headers.get('content-type') ?? '', buf };
|
|
}
|
|
const attr = (tag: string, name: string) =>
|
|
tag
|
|
.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'))
|
|
?.slice(1)
|
|
.find((x) => x != null) ?? null;
|
|
type IconCandidate = { url: string; size: number };
|
|
async function websiteIcon(
|
|
site: string
|
|
): Promise<{ png: Buffer; host: string; hash: string } | { error: string }> {
|
|
let page;
|
|
try {
|
|
page = await get(site, 3_000_000);
|
|
} catch (e) {
|
|
return { error: `site: ${(e as Error).message}` };
|
|
}
|
|
const from = new URL(site).hostname;
|
|
const to = new URL(page.url).hostname;
|
|
if (site2(from) !== site2(to)) return { error: `redirects to ${to}` };
|
|
const html = page.buf.toString('utf8');
|
|
const title = stripHtml(html.match(/<title[^>]*>([\s\S]{0,300}?)<\/title>/i)?.[1] ?? '');
|
|
if (PARKED.test(title) || html.length < 400)
|
|
return { error: `parked/empty (${title.slice(0, 60)})` };
|
|
const cands: IconCandidate[] = [];
|
|
for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) {
|
|
const rel = (attr(tag, 'rel') ?? '').toLowerCase();
|
|
const href = attr(tag, 'href');
|
|
if (!href || !/\bicon\b|apple-touch-icon/.test(rel) || /mask-icon/.test(rel)) continue;
|
|
const sizes = attr(tag, 'sizes') ?? '';
|
|
const n = Math.max(
|
|
0,
|
|
...[...sizes.matchAll(/(\d+)x(\d+)/gi)].map((m) => Math.min(+m[1], +m[2]))
|
|
);
|
|
const svg = /\.svg(\?|$)/i.test(href) || /svg/i.test(attr(tag, 'type') ?? '');
|
|
const size = svg ? 1000 : n || (/apple-touch/.test(rel) ? 180 : 32);
|
|
try {
|
|
cands.push({ url: new URL(href.replace(/&/g, '&'), page.url).href, size });
|
|
} catch {
|
|
/* bad href */
|
|
}
|
|
}
|
|
const manifest = html.match(/<link\b[^>]*rel\s*=\s*["']?manifest[^>]*>/i)?.[0];
|
|
const mhref = manifest ? attr(manifest, 'href') : null;
|
|
if (mhref)
|
|
try {
|
|
const murl = new URL(mhref.replace(/&/g, '&'), page.url).href;
|
|
const m = JSON.parse((await get(murl, 200_000)).buf.toString('utf8')) as {
|
|
icons?: { src: string; sizes?: string; purpose?: string }[];
|
|
};
|
|
for (const i of m.icons ?? []) {
|
|
if (/monochrome/.test(i.purpose ?? '')) continue;
|
|
const n = Math.max(
|
|
0,
|
|
...[...(i.sizes ?? '').matchAll(/(\d+)x(\d+)/gi)].map((x) => Math.min(+x[1], +x[2]))
|
|
);
|
|
cands.push({ url: new URL(i.src, murl).href, size: Math.min(n || 192, 512) });
|
|
}
|
|
} catch {
|
|
/* no manifest */
|
|
}
|
|
cands.push({ url: new URL('/apple-touch-icon.png', page.url).href, size: 179 });
|
|
cands.sort((a, b) => b.size - a.size);
|
|
const seen = new Set<string>();
|
|
for (const c of cands) {
|
|
if (c.size < 96 || seen.has(c.url) || /\.ico(\?|$)/i.test(c.url)) continue;
|
|
seen.add(c.url);
|
|
try {
|
|
const img = await get(c.url);
|
|
if (
|
|
site2(new URL(img.url).hostname) !== site2(to) &&
|
|
!/cdn|static|assets|img|media|cloudfront|akamai|wp\.com|squarespace|wixstatic|shopify/i.test(
|
|
new URL(img.url).hostname
|
|
)
|
|
)
|
|
continue;
|
|
const meta = await sharp(img.buf, { density: 300 }).metadata();
|
|
const w = meta.width ?? 0;
|
|
const h = meta.height ?? 0;
|
|
if (meta.format !== 'svg' && Math.min(w, h) < 96) continue;
|
|
if (!squarish(w / h)) continue;
|
|
const stats = await sharp(img.buf, { density: 300 })
|
|
.resize(64, 64, { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
|
.ensureAlpha()
|
|
.stats();
|
|
// A blank or single-colour square is a placeholder, not a logo.
|
|
const spread = stats.channels.reduce((s, ch) => s + ch.stdev, 0);
|
|
if (spread < 12) continue;
|
|
const png = await sharp(img.buf, { density: 300 })
|
|
.resize(160, 160, { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
|
.webp({ quality: 90 })
|
|
.toBuffer();
|
|
return {
|
|
png,
|
|
host: to.replace(/^www\./, ''),
|
|
hash: createHash('sha1').update(png).digest('hex')
|
|
};
|
|
} catch {
|
|
/* next candidate */
|
|
}
|
|
}
|
|
return { error: `no square icon >= 96px (${cands.length} candidates)` };
|
|
}
|
|
|
|
// ---------------------------------------------------------------- main
|
|
async function write(t: Table, slug: string, p: Pick) {
|
|
if (dryRun) return;
|
|
await sql.unsafe(
|
|
`update ${t.table} set logo = $1, logo_file = $2, logo_author = $3, logo_license = $4 where slug = $5`,
|
|
[p.logo, p.file, p.author, p.license, slug]
|
|
);
|
|
}
|
|
|
|
if (recheck) {
|
|
const { readFile } = await import('node:fs/promises');
|
|
const picks = [
|
|
...(await readFile(recheck, 'utf8')).matchAll(
|
|
/^ {2}(\S+) (\S+): (wikidata logo|wikidata icon|\w+wiki infobox) (.+?)$/gm
|
|
)
|
|
].filter((m) => !m[4].endsWith(' (square swap)'));
|
|
await imageInfo(picks.map((m) => m[4]));
|
|
let cleared = 0;
|
|
for (const [, table, slug, how, file] of picks) {
|
|
const t = TABLES.find((x) => x.table === table)!;
|
|
const [o] = (await sql.unsafe(
|
|
`select name, ${t.table === 'cars.brands' ? 'null' : 'full_name'} as full_name from ${t.table} where slug = $1`,
|
|
[slug]
|
|
)) as unknown as { name: string; full_name: string | null }[];
|
|
if (!o || commonsLogo(file, how, [o.name, o.full_name ?? o.name])) continue;
|
|
cleared++;
|
|
log(` clear ${table} ${slug}: ${file}`);
|
|
if (!dryRun)
|
|
await sql.unsafe(
|
|
`update ${t.table} set logo = null, logo_file = null, logo_author = null, logo_license = null where slug = $1 and logo_file = $2`,
|
|
[slug, file]
|
|
);
|
|
}
|
|
log(`recheck: ${cleared}/${picks.length} picks cleared`);
|
|
await sql.end();
|
|
process.exit(0);
|
|
}
|
|
|
|
const tables = TABLES.filter((t) => which === 'all' || t.tracker === which);
|
|
const favHashes = new Map<string, { t: Table; slug: string; host: string; png: Buffer }[]>();
|
|
for (const t of tables) {
|
|
let orgs = (await sql.unsafe(t.select)) as unknown as Org[];
|
|
const before = orgs.filter((o) => o.logo).length;
|
|
if (onlyNew) orgs = orgs.filter((o) => Date.now() - +new Date(o.created_at) < 2 * 86400_000);
|
|
if (faviconsDb) {
|
|
let n = 0;
|
|
for (const o of orgs.filter((o) => !o.logo)) {
|
|
if (!existsSync(`${STATIC}/${t.dir}/${o.slug}.webp`)) continue;
|
|
n++;
|
|
await write(t, o.slug, {
|
|
logo: `/spotters/logos/${t.dir}/${o.slug}.webp`,
|
|
file: null,
|
|
author: null,
|
|
license: 'website icon · trademark',
|
|
how: 'favicon'
|
|
});
|
|
}
|
|
log(`${t.table}: ${n} website icons linked`);
|
|
continue;
|
|
}
|
|
orgs = orgs
|
|
.filter((o) => (square || !o.logo) && o.wikidata)
|
|
.sort((a, b) => Number(b.nl ?? 0) - Number(a.nl ?? 0) || (b.types ?? 0) - (a.types ?? 0))
|
|
.slice(0, limit);
|
|
log(`${t.table}: ${before} with a logo; checking ${orgs.length}`);
|
|
const wd = await wikidata([...new Set(orgs.map((o) => o.wikidata!))]);
|
|
await imageInfo([
|
|
...[...wd.values()].flatMap((w) => [...w.oldLogos, ...w.icons]),
|
|
...orgs.map((o) => o.logo_file).filter((f): f is string => !!f)
|
|
]);
|
|
const picks = new Map<string, Pick>();
|
|
const stillMissing: Org[] = [];
|
|
for (const o of orgs) {
|
|
const w = wd.get(o.wikidata!);
|
|
const names = [o.name, o.full_name ?? o.name];
|
|
const icons = (w?.icons ?? [])
|
|
.map((f) => commonsLogo(f, 'wikidata icon', names))
|
|
.filter((x) => x && squarish(x.ratio));
|
|
if (o.logo) {
|
|
// --square: swap a wide or tall logo for a square icon.
|
|
const cur = o.logo_file ? info.get(o.logo_file) : null;
|
|
const ratio = cur ? cur.width / cur.height : 1;
|
|
if (icons[0] && (ratio > 1.6 || ratio < 0.6) && icons[0].file !== o.logo_file)
|
|
picks.set(o.slug, icons[0]);
|
|
continue;
|
|
}
|
|
const logos = (w?.logos.length ? w.logos : w?.dissolved ? w.oldLogos : [])
|
|
.map((f) => commonsLogo(f, 'wikidata logo', names))
|
|
.filter((x) => !!x);
|
|
// Prefer a square logo, then a square icon, then any logo.
|
|
const pick =
|
|
logos.find((l) => squarish(l.ratio)) ??
|
|
icons[0] ??
|
|
logos[0] ??
|
|
(w?.icons ?? []).map((f) => commonsLogo(f, 'wikidata icon', names)).find((x) => !!x);
|
|
if (pick) picks.set(o.slug, pick);
|
|
else stillMissing.push(o);
|
|
}
|
|
// Wikipedia infobox logo= fields, wiki by wiki.
|
|
let left = stillMissing;
|
|
for (const wiki of WIKIS) {
|
|
const want = left.filter((o) => wd.get(o.wikidata!)?.articles.has(wiki));
|
|
if (!want.length) continue;
|
|
const found = await infoboxLogos(
|
|
wiki,
|
|
want.map((o) => wd.get(o.wikidata!)!.articles.get(wiki)!)
|
|
);
|
|
const files = new Map<string, string>();
|
|
for (const o of want) {
|
|
const f = found.get(wd.get(o.wikidata!)!.articles.get(wiki)!);
|
|
if (f) files.set(o.slug, f);
|
|
}
|
|
await imageInfo([...files.values()]);
|
|
for (const [slug, f] of files) {
|
|
const o = want.find((x) => x.slug === slug)!;
|
|
const p = commonsLogo(f, `${wiki}wiki infobox`, [o.name, o.full_name ?? o.name]);
|
|
if (p) picks.set(slug, p);
|
|
}
|
|
left = left.filter((o) => !picks.has(o.slug));
|
|
}
|
|
for (const o of orgs) {
|
|
const p = picks.get(o.slug);
|
|
if (!p) continue;
|
|
log(` ${t.table} ${o.slug}: ${p.how} ${p.file}${o.logo ? ' (square swap)' : ''}`);
|
|
await write(t, o.slug, p);
|
|
}
|
|
const added = orgs.filter((o) => !o.logo && picks.has(o.slug)).length;
|
|
log(
|
|
`${t.table}: +${added} logos from Wikimedia, ${orgs.filter((o) => o.logo && picks.has(o.slug)).length} swapped for square icons, ${left.length} still missing`
|
|
);
|
|
if (favicon) {
|
|
let tried = 0;
|
|
const fails: Record<string, number> = {};
|
|
for (const o of left) {
|
|
const w = wd.get(o.wikidata!);
|
|
if (!w?.sites.length) {
|
|
fails['no website'] = (fails['no website'] ?? 0) + 1;
|
|
continue;
|
|
}
|
|
if (w.dissolved) {
|
|
fails['dissolved'] = (fails['dissolved'] ?? 0) + 1;
|
|
continue;
|
|
}
|
|
tried++;
|
|
const r = await websiteIcon(w.sites[0]);
|
|
if ('error' in r) {
|
|
const k = r.error
|
|
.replace(/\(.*$/, '')
|
|
.replace(/HTTP \d+/, 'HTTP error')
|
|
.trim()
|
|
.slice(0, 40);
|
|
fails[k] = (fails[k] ?? 0) + 1;
|
|
log(` ${o.slug}: ${r.error}`);
|
|
} else {
|
|
const list = favHashes.get(r.hash) ?? [];
|
|
list.push({ t, slug: o.slug, host: r.host, png: r.png });
|
|
favHashes.set(r.hash, list);
|
|
log(` ${o.slug}: icon from ${r.host}`);
|
|
}
|
|
await sleep(300);
|
|
}
|
|
log(`${t.table}: website icons tried ${tried}; failures ${JSON.stringify(fails)}`);
|
|
}
|
|
}
|
|
if (favicon) {
|
|
let saved = 0;
|
|
for (const list of favHashes.values()) {
|
|
// The same icon on different sites is a generic CMS / host icon: skip all of them.
|
|
if (new Set(list.map((x) => x.host)).size > 1) {
|
|
log(` generic icon shared by ${list.map((x) => x.slug).join(', ')}: skipped`);
|
|
continue;
|
|
}
|
|
for (const x of list) {
|
|
if (dryRun) continue;
|
|
await mkdir(`${STATIC}/${x.t.dir}`, { recursive: true });
|
|
await writeFile(`${STATIC}/${x.t.dir}/${x.slug}.webp`, x.png);
|
|
saved++;
|
|
}
|
|
}
|
|
log(
|
|
`website icons saved: ${saved} (commit static/spotters/logos, deploy, then run --favicons-db)`
|
|
);
|
|
}
|
|
await sql.end();
|