|
|
|
|
@@ -0,0 +1,520 @@
|
|
|
|
|
// Fills missing brand / maker / operator logos for the spotter trackers and swaps very wide
|
|
|
|
|
// logos for a square icon when Wikidata has one, so the square logo tiles look right.
|
|
|
|
|
//
|
|
|
|
|
// DB_URL=... bun scripts/spotters-logos.ts <cars|planes|trains|all> [options]
|
|
|
|
|
//
|
|
|
|
|
// (default) orgs without a logo: Wikidata logo (P154), small logo / icon (P8972,
|
|
|
|
|
// P2910), the logo= field of the org's Wikipedia infobox in en, nl, de,
|
|
|
|
|
// fr, it, es, pl, cs, sv or ja (Commons files only)
|
|
|
|
|
// --square also orgs that already have a logo: if it is wide (>1.6:1) or tall and
|
|
|
|
|
// Wikidata has a square small logo / icon, use that instead
|
|
|
|
|
// --favicon last resort for orgs still without a logo: the largest square icon
|
|
|
|
|
// (apple-touch-icon, manifest icon, svg icon) of the org's official
|
|
|
|
|
// website (P856, current statement only, never for dissolved orgs), only
|
|
|
|
|
// when the site does not redirect to another domain. Saved as 160px webp
|
|
|
|
|
// in static/spotters/logos/<table>/<slug>.webp (commit + deploy them)
|
|
|
|
|
// --favicons-db point orgs without a logo at their static/spotters/logos file (run this
|
|
|
|
|
// after the files are deployed)
|
|
|
|
|
// --new only orgs created in the last 2 days (for the refresh scripts)
|
|
|
|
|
// --limit <n> first n orgs by priority (most types, Dutch first)
|
|
|
|
|
// --dry-run print what would change, write nothing
|
|
|
|
|
//
|
|
|
|
|
// Logos stay strict: a file must be free on Commons (toPhoto-like check: no fair use,
|
|
|
|
|
// no non-free), come from the org's own Wikidata item or its own article's logo field, and
|
|
|
|
|
// a website icon must come from the org's own current website.
|
|
|
|
|
import postgres from 'postgres';
|
|
|
|
|
import { createHash } from 'node:crypto';
|
|
|
|
|
import { existsSync } from 'node:fs';
|
|
|
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
|
|
|
import sharp from 'sharp';
|
|
|
|
|
import {
|
|
|
|
|
chunks,
|
|
|
|
|
fileName,
|
|
|
|
|
log,
|
|
|
|
|
mw,
|
|
|
|
|
qid,
|
|
|
|
|
sleep,
|
|
|
|
|
sparql,
|
|
|
|
|
stripHtml,
|
|
|
|
|
type ImageInfo
|
|
|
|
|
} from './planes-wiki';
|
|
|
|
|
|
|
|
|
|
const args = process.argv.slice(2);
|
|
|
|
|
const flag = (name: string) => args.includes(name);
|
|
|
|
|
const opt = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : null);
|
|
|
|
|
const which = args[0];
|
|
|
|
|
const dryRun = flag('--dry-run');
|
|
|
|
|
const square = flag('--square');
|
|
|
|
|
const favicon = flag('--favicon');
|
|
|
|
|
const faviconsDb = flag('--favicons-db');
|
|
|
|
|
const onlyNew = flag('--new');
|
|
|
|
|
const limit = Number(opt('--limit')) || Infinity;
|
|
|
|
|
if (!['cars', 'planes', 'trains', 'all'].includes(which ?? ''))
|
|
|
|
|
throw new Error('usage: spotters-logos.ts <cars|planes|trains|all> [--square] [--favicon]');
|
|
|
|
|
|
|
|
|
|
const LOGO = 250;
|
|
|
|
|
const STATIC = 'static/spotters/logos';
|
|
|
|
|
const TABLES = [
|
|
|
|
|
{
|
|
|
|
|
tracker: 'cars',
|
|
|
|
|
table: 'cars.brands',
|
|
|
|
|
dir: 'cars-brands',
|
|
|
|
|
select: `select b.slug, b.name, null::text as full_name, b.wikidata, b.logo, b.logo_file, b.model_count as types,
|
|
|
|
|
(b.country = 'Netherlands')::int + coalesce((select sum(a.score) from cars.activity a join cars.models m on m.id = a.model_id where m.brand_id = b.id and a.area = 'nl'), 0) as nl,
|
|
|
|
|
b.created_at from cars.brands b`
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
tracker: 'planes',
|
|
|
|
|
table: 'planes.makers',
|
|
|
|
|
dir: 'planes-makers',
|
|
|
|
|
select: `select k.slug, k.name, k.full_name, k.wikidata, k.logo, k.logo_file, k.type_count as types,
|
|
|
|
|
coalesce((select sum(a.score) from planes.activity a join planes.types t on t.id = a.type_id where t.maker_id = k.id and a.area = 'nl'), 0) as nl,
|
|
|
|
|
k.created_at from planes.makers k`
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
tracker: 'trains',
|
|
|
|
|
table: 'trains.makers',
|
|
|
|
|
dir: 'trains-makers',
|
|
|
|
|
select: `select k.slug, k.name, k.full_name, k.wikidata, k.logo, k.logo_file, k.type_count as types,
|
|
|
|
|
coalesce((select sum(a.score) from trains.activity a join trains.types t on t.id = a.type_id where t.maker_id = k.id and a.area = 'nl'), 0) as nl,
|
|
|
|
|
k.created_at from trains.makers k`
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
tracker: 'trains',
|
|
|
|
|
table: 'trains.operators',
|
|
|
|
|
dir: 'trains-operators',
|
|
|
|
|
select: `select o.slug, o.name, o.full_name, o.wikidata, o.logo, o.logo_file, o.type_count as types,
|
|
|
|
|
(o.country = 'Netherlands')::int * 1000 as nl, o.created_at from trains.operators o`
|
|
|
|
|
}
|
|
|
|
|
] as const;
|
|
|
|
|
type Table = (typeof TABLES)[number];
|
|
|
|
|
type Org = {
|
|
|
|
|
slug: string;
|
|
|
|
|
name: string;
|
|
|
|
|
full_name: string | null;
|
|
|
|
|
wikidata: string | null;
|
|
|
|
|
logo: string | null;
|
|
|
|
|
logo_file: string | null;
|
|
|
|
|
types: number | null;
|
|
|
|
|
nl: number | string | null;
|
|
|
|
|
created_at: Date;
|
|
|
|
|
};
|
|
|
|
|
type Pick = {
|
|
|
|
|
logo: string;
|
|
|
|
|
file: string | null;
|
|
|
|
|
author: string | null;
|
|
|
|
|
license: string | null;
|
|
|
|
|
how: string;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
if (!process.env.DB_URL) throw new Error('Set DB_URL');
|
|
|
|
|
const sql = postgres(process.env.DB_URL, { max: 1, connect_timeout: 10, onnotice: () => {} });
|
|
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------- commons
|
|
|
|
|
const info = new Map<string, ImageInfo | null>();
|
|
|
|
|
async function imageInfo(files: string[]) {
|
|
|
|
|
const todo = [...new Set(files)].filter((f) => !info.has(f));
|
|
|
|
|
for (const chunk of chunks(todo, 50)) {
|
|
|
|
|
const res = await mw('commons.wikimedia.org', {
|
|
|
|
|
action: 'query',
|
|
|
|
|
prop: 'imageinfo',
|
|
|
|
|
iiprop: 'url|size|mime|extmetadata',
|
|
|
|
|
iiurlwidth: String(LOGO),
|
|
|
|
|
iiextmetadatafilter: 'Artist|LicenseShortName|NonFree|Restrictions',
|
|
|
|
|
titles: chunk.map((f) => `File:${f}`).join('|')
|
|
|
|
|
});
|
|
|
|
|
const norm = new Map((res.query?.normalized ?? []).map((n) => [n.to, n.from]));
|
|
|
|
|
for (const p of res.query?.pages ?? []) {
|
|
|
|
|
const original = (norm.get(p.title) ?? p.title).replace(/^File:/, '');
|
|
|
|
|
info.set(original, p.missing || !p.imageinfo?.[0] ? null : p.imageinfo[0]);
|
|
|
|
|
}
|
|
|
|
|
for (const f of chunk) if (!info.has(f)) info.set(f, null);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
/** A free Commons image as a logo, or null (fair use / non-free / not an image). */
|
|
|
|
|
function commonsLogo(file: string, how: string): (Pick & { ratio: number }) | null {
|
|
|
|
|
const i = info.get(file);
|
|
|
|
|
if (!i?.thumburl || !/^image\/(svg\+xml|png|gif|jpeg|webp)$/.test(i.mime)) return null;
|
|
|
|
|
const meta = i.extmetadata ?? {};
|
|
|
|
|
if (meta.NonFree?.value && meta.NonFree.value !== 'false') return null;
|
|
|
|
|
let license = meta.LicenseShortName?.value ? stripHtml(meta.LicenseShortName.value) : null;
|
|
|
|
|
if (!license || /fair use|non-free/i.test(license)) return null;
|
|
|
|
|
if (/trademark/i.test(meta.Restrictions?.value ?? '')) license += ' · trademark';
|
|
|
|
|
if (i.width < 48 || i.height < 48) return null;
|
|
|
|
|
return {
|
|
|
|
|
logo: i.thumburl.replace(/\?.*$/, ''),
|
|
|
|
|
file,
|
|
|
|
|
author: meta.Artist?.value ? stripHtml(meta.Artist.value).slice(0, 200) || null : null,
|
|
|
|
|
license,
|
|
|
|
|
how,
|
|
|
|
|
ratio: i.width / i.height
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const squarish = (r: number) => r >= 0.75 && r <= 1.34;
|
|
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------- wikidata
|
|
|
|
|
const WIKIS = ['en', 'nl', 'de', 'fr', 'it', 'es', 'pl', 'cs', 'sv', 'ja'];
|
|
|
|
|
type Wd = {
|
|
|
|
|
logos: string[];
|
|
|
|
|
icons: string[];
|
|
|
|
|
sites: string[];
|
|
|
|
|
dissolved: boolean;
|
|
|
|
|
articles: Map<string, string>;
|
|
|
|
|
};
|
|
|
|
|
async function wikidata(ids: string[]) {
|
|
|
|
|
const out = new Map<string, Wd>();
|
|
|
|
|
for (const c of chunks(ids, 150)) {
|
|
|
|
|
let rows: Record<string, string>[];
|
|
|
|
|
try {
|
|
|
|
|
rows = await sparql(`
|
|
|
|
|
SELECT ?m (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(?end), "")); separator="|") AS ?logos)
|
|
|
|
|
(GROUP_CONCAT(DISTINCT STR(?icon); separator="|") AS ?icons)
|
|
|
|
|
(GROUP_CONCAT(DISTINCT STR(?site); separator="|") AS ?sites)
|
|
|
|
|
(GROUP_CONCAT(DISTINCT STR(?a); separator="|") AS ?articles) (SAMPLE(?gone) AS ?dissolved)
|
|
|
|
|
WHERE {
|
|
|
|
|
VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} }
|
|
|
|
|
OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank)
|
|
|
|
|
OPTIONAL { ?st pq:P582 ?end } }
|
|
|
|
|
OPTIONAL { ?m p:P8972|p:P2910 ?ist . ?ist ps:P8972|ps:P2910 ?icon ; wikibase:rank ?irank .
|
|
|
|
|
FILTER(?irank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?ist pq:P582 [] } }
|
|
|
|
|
OPTIONAL { ?m p:P856 ?wst . ?wst ps:P856 ?site ; wikibase:rank ?wrank .
|
|
|
|
|
FILTER(?wrank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?wst pq:P582 [] } }
|
|
|
|
|
OPTIONAL { ?a schema:about ?m ; schema:isPartOf ?wiki .
|
|
|
|
|
VALUES ?wiki { ${WIKIS.map((w) => `<https://${w}.wikipedia.org/>`).join(' ')} } }
|
|
|
|
|
OPTIONAL { ?m wdt:P576 ?gone }
|
|
|
|
|
} GROUP BY ?m`);
|
|
|
|
|
} catch (e) {
|
|
|
|
|
log(`wikidata chunk failed: ${(e as Error).message}`);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
for (const r of rows) {
|
|
|
|
|
const logos = (r.logos ? r.logos.split('|').filter(Boolean) : [])
|
|
|
|
|
.map((x) => {
|
|
|
|
|
const [uri, rank, end] = x.split(';');
|
|
|
|
|
return {
|
|
|
|
|
file: fileName(uri),
|
|
|
|
|
score: (rank.endsWith('PreferredRank') ? 2 : 0) + (end ? 0 : 4)
|
|
|
|
|
};
|
|
|
|
|
})
|
|
|
|
|
.sort((a, b) => b.score - a.score);
|
|
|
|
|
const articles = new Map<string, string>();
|
|
|
|
|
for (const a of r.articles ? r.articles.split('|') : []) {
|
|
|
|
|
const m = a.match(/^https:\/\/(\w+)\.wikipedia\.org\/wiki\/(.+)$/);
|
|
|
|
|
if (m) articles.set(m[1], decodeURIComponent(m[2]).replace(/_/g, ' '));
|
|
|
|
|
}
|
|
|
|
|
out.set(qid(r.m), {
|
|
|
|
|
logos: [...new Set(logos.map((l) => l.file))],
|
|
|
|
|
icons: r.icons ? [...new Set(r.icons.split('|').map(fileName))] : [],
|
|
|
|
|
sites: r.sites ? r.sites.split('|').filter((s) => /^https?:\/\//.test(s)) : [],
|
|
|
|
|
dissolved: !!r.dissolved,
|
|
|
|
|
articles
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------- wikipedia infoboxes
|
|
|
|
|
const INFOBOX_LOGO =
|
|
|
|
|
/\|\s*(?:logo|logo_image|image_logo|logotipo|logotyp|logo_bild|bild_logo|ロゴ)\s*=\s*(?:\[\[)?(?:[^\s:|\]=]{2,12}:)?\s*([^|\]\n{}<=]+?\.(?:svg|png|gif))/i;
|
|
|
|
|
async function infoboxLogos(wiki: string, titles: string[]) {
|
|
|
|
|
const out = new Map<string, string>();
|
|
|
|
|
for (const c of chunks(titles, 40)) {
|
|
|
|
|
const res = await mw(`${wiki}.wikipedia.org`, {
|
|
|
|
|
action: 'query',
|
|
|
|
|
prop: 'revisions',
|
|
|
|
|
rvprop: 'content',
|
|
|
|
|
rvslots: 'main',
|
|
|
|
|
rvsection: '0',
|
|
|
|
|
redirects: '1',
|
|
|
|
|
titles: c.join('|')
|
|
|
|
|
}).catch(() => null);
|
|
|
|
|
if (!res) continue;
|
|
|
|
|
const back = new Map<string, string>();
|
|
|
|
|
for (const n of res.query?.normalized ?? []) back.set(n.to, n.from);
|
|
|
|
|
for (const n of res.query?.redirects ?? []) back.set(n.to, back.get(n.from) ?? n.from);
|
|
|
|
|
for (const p of res.query?.pages ?? []) {
|
|
|
|
|
const text = p.revisions?.[0]?.slots?.main?.content ?? '';
|
|
|
|
|
const m = text.match(INFOBOX_LOGO);
|
|
|
|
|
if (m) out.set(back.get(p.title) ?? p.title, m[1].trim().replace(/_/g, ' '));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------- website icons
|
|
|
|
|
const PARKED =
|
|
|
|
|
/domain (?:is |may be )?for sale|buy this domain|parked (?:free|domain)|this domain|sedo|hugedomains|afternic|dan\.com|bodis|godaddy|namecheap|domain expired|account suspended|default web site page|it works!|welcome to nginx/i;
|
|
|
|
|
const site2 = (host: string) => {
|
|
|
|
|
const parts = host.replace(/^www\./, '').split('.');
|
|
|
|
|
const n = parts.length >= 3 && /^(co|com|org|net|ac|gov|ne|or)$/.test(parts.at(-2)!) ? 3 : 2;
|
|
|
|
|
return parts.slice(-n).join('.');
|
|
|
|
|
};
|
|
|
|
|
async function get(url: string, max = 1_500_000) {
|
|
|
|
|
const res = await fetch(url, {
|
|
|
|
|
redirect: 'follow',
|
|
|
|
|
signal: AbortSignal.timeout(15_000),
|
|
|
|
|
headers: {
|
|
|
|
|
'user-agent':
|
|
|
|
|
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36 (stanrunge.dev spotters logo check)',
|
|
|
|
|
accept: '*/*'
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
|
|
|
const buf = Buffer.from(await res.arrayBuffer());
|
|
|
|
|
if (buf.length > max) throw new Error('too big');
|
|
|
|
|
return { url: res.url, type: res.headers.get('content-type') ?? '', buf };
|
|
|
|
|
}
|
|
|
|
|
const attr = (tag: string, name: string) =>
|
|
|
|
|
tag
|
|
|
|
|
.match(new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'))
|
|
|
|
|
?.slice(1)
|
|
|
|
|
.find((x) => x != null) ?? null;
|
|
|
|
|
type IconCandidate = { url: string; size: number };
|
|
|
|
|
async function websiteIcon(
|
|
|
|
|
site: string
|
|
|
|
|
): Promise<{ png: Buffer; host: string; hash: string } | { error: string }> {
|
|
|
|
|
let page;
|
|
|
|
|
try {
|
|
|
|
|
page = await get(site, 3_000_000);
|
|
|
|
|
} catch (e) {
|
|
|
|
|
return { error: `site: ${(e as Error).message}` };
|
|
|
|
|
}
|
|
|
|
|
const from = new URL(site).hostname;
|
|
|
|
|
const to = new URL(page.url).hostname;
|
|
|
|
|
if (site2(from) !== site2(to)) return { error: `redirects to ${to}` };
|
|
|
|
|
const html = page.buf.toString('utf8');
|
|
|
|
|
const title = stripHtml(html.match(/<title[^>]*>([\s\S]{0,300}?)<\/title>/i)?.[1] ?? '');
|
|
|
|
|
if (PARKED.test(title) || html.length < 400)
|
|
|
|
|
return { error: `parked/empty (${title.slice(0, 60)})` };
|
|
|
|
|
const cands: IconCandidate[] = [];
|
|
|
|
|
for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) {
|
|
|
|
|
const rel = (attr(tag, 'rel') ?? '').toLowerCase();
|
|
|
|
|
const href = attr(tag, 'href');
|
|
|
|
|
if (!href || !/\bicon\b|apple-touch-icon/.test(rel) || /mask-icon/.test(rel)) continue;
|
|
|
|
|
const sizes = attr(tag, 'sizes') ?? '';
|
|
|
|
|
const n = Math.max(
|
|
|
|
|
0,
|
|
|
|
|
...[...sizes.matchAll(/(\d+)x(\d+)/gi)].map((m) => Math.min(+m[1], +m[2]))
|
|
|
|
|
);
|
|
|
|
|
const svg = /\.svg(\?|$)/i.test(href) || /svg/i.test(attr(tag, 'type') ?? '');
|
|
|
|
|
const size = svg ? 1000 : n || (/apple-touch/.test(rel) ? 180 : 32);
|
|
|
|
|
try {
|
|
|
|
|
cands.push({ url: new URL(href.replace(/&/g, '&'), page.url).href, size });
|
|
|
|
|
} catch {
|
|
|
|
|
/* bad href */
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const manifest = html.match(/<link\b[^>]*rel\s*=\s*["']?manifest[^>]*>/i)?.[0];
|
|
|
|
|
const mhref = manifest ? attr(manifest, 'href') : null;
|
|
|
|
|
if (mhref)
|
|
|
|
|
try {
|
|
|
|
|
const murl = new URL(mhref.replace(/&/g, '&'), page.url).href;
|
|
|
|
|
const m = JSON.parse((await get(murl, 200_000)).buf.toString('utf8')) as {
|
|
|
|
|
icons?: { src: string; sizes?: string; purpose?: string }[];
|
|
|
|
|
};
|
|
|
|
|
for (const i of m.icons ?? []) {
|
|
|
|
|
if (/monochrome/.test(i.purpose ?? '')) continue;
|
|
|
|
|
const n = Math.max(
|
|
|
|
|
0,
|
|
|
|
|
...[...(i.sizes ?? '').matchAll(/(\d+)x(\d+)/gi)].map((x) => Math.min(+x[1], +x[2]))
|
|
|
|
|
);
|
|
|
|
|
cands.push({ url: new URL(i.src, murl).href, size: Math.min(n || 192, 512) });
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
/* no manifest */
|
|
|
|
|
}
|
|
|
|
|
cands.push({ url: new URL('/apple-touch-icon.png', page.url).href, size: 179 });
|
|
|
|
|
cands.sort((a, b) => b.size - a.size);
|
|
|
|
|
const seen = new Set<string>();
|
|
|
|
|
for (const c of cands) {
|
|
|
|
|
if (c.size < 96 || seen.has(c.url) || /\.ico(\?|$)/i.test(c.url)) continue;
|
|
|
|
|
seen.add(c.url);
|
|
|
|
|
try {
|
|
|
|
|
const img = await get(c.url);
|
|
|
|
|
if (
|
|
|
|
|
site2(new URL(img.url).hostname) !== site2(to) &&
|
|
|
|
|
!/cdn|static|assets|img|media|cloudfront|akamai|wp\.com|squarespace|wixstatic|shopify/i.test(
|
|
|
|
|
new URL(img.url).hostname
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
continue;
|
|
|
|
|
const meta = await sharp(img.buf, { density: 300 }).metadata();
|
|
|
|
|
const w = meta.width ?? 0;
|
|
|
|
|
const h = meta.height ?? 0;
|
|
|
|
|
if (meta.format !== 'svg' && Math.min(w, h) < 96) continue;
|
|
|
|
|
if (!squarish(w / h)) continue;
|
|
|
|
|
const stats = await sharp(img.buf, { density: 300 })
|
|
|
|
|
.resize(64, 64, { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
|
|
|
|
.ensureAlpha()
|
|
|
|
|
.stats();
|
|
|
|
|
// A blank or single-colour square is a placeholder, not a logo.
|
|
|
|
|
const spread = stats.channels.reduce((s, ch) => s + ch.stdev, 0);
|
|
|
|
|
if (spread < 12) continue;
|
|
|
|
|
const png = await sharp(img.buf, { density: 300 })
|
|
|
|
|
.resize(160, 160, { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
|
|
|
|
.webp({ quality: 90 })
|
|
|
|
|
.toBuffer();
|
|
|
|
|
return {
|
|
|
|
|
png,
|
|
|
|
|
host: to.replace(/^www\./, ''),
|
|
|
|
|
hash: createHash('sha1').update(png).digest('hex')
|
|
|
|
|
};
|
|
|
|
|
} catch {
|
|
|
|
|
/* next candidate */
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return { error: `no square icon >= 96px (${cands.length} candidates)` };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---------------------------------------------------------------- main
|
|
|
|
|
async function write(t: Table, slug: string, p: Pick) {
|
|
|
|
|
if (dryRun) return;
|
|
|
|
|
await sql.unsafe(
|
|
|
|
|
`update ${t.table} set logo = $1, logo_file = $2, logo_author = $3, logo_license = $4 where slug = $5`,
|
|
|
|
|
[p.logo, p.file, p.author, p.license, slug]
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const tables = TABLES.filter((t) => which === 'all' || t.tracker === which);
|
|
|
|
|
const favHashes = new Map<string, { t: Table; slug: string; host: string; png: Buffer }[]>();
|
|
|
|
|
for (const t of tables) {
|
|
|
|
|
let orgs = (await sql.unsafe(t.select)) as unknown as Org[];
|
|
|
|
|
const before = orgs.filter((o) => o.logo).length;
|
|
|
|
|
if (onlyNew) orgs = orgs.filter((o) => Date.now() - +new Date(o.created_at) < 2 * 86400_000);
|
|
|
|
|
if (faviconsDb) {
|
|
|
|
|
let n = 0;
|
|
|
|
|
for (const o of orgs.filter((o) => !o.logo)) {
|
|
|
|
|
if (!existsSync(`${STATIC}/${t.dir}/${o.slug}.webp`)) continue;
|
|
|
|
|
n++;
|
|
|
|
|
await write(t, o.slug, {
|
|
|
|
|
logo: `/spotters/logos/${t.dir}/${o.slug}.webp`,
|
|
|
|
|
file: null,
|
|
|
|
|
author: null,
|
|
|
|
|
license: 'website icon · trademark',
|
|
|
|
|
how: 'favicon'
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
log(`${t.table}: ${n} website icons linked`);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
orgs = orgs
|
|
|
|
|
.filter((o) => (square || !o.logo) && o.wikidata)
|
|
|
|
|
.sort((a, b) => Number(b.nl ?? 0) - Number(a.nl ?? 0) || (b.types ?? 0) - (a.types ?? 0))
|
|
|
|
|
.slice(0, limit);
|
|
|
|
|
log(`${t.table}: ${before} with a logo; checking ${orgs.length}`);
|
|
|
|
|
const wd = await wikidata([...new Set(orgs.map((o) => o.wikidata!))]);
|
|
|
|
|
await imageInfo([
|
|
|
|
|
...[...wd.values()].flatMap((w) => [...w.logos, ...w.icons]),
|
|
|
|
|
...orgs.map((o) => o.logo_file).filter((f): f is string => !!f)
|
|
|
|
|
]);
|
|
|
|
|
const picks = new Map<string, Pick>();
|
|
|
|
|
const stillMissing: Org[] = [];
|
|
|
|
|
for (const o of orgs) {
|
|
|
|
|
const w = wd.get(o.wikidata!);
|
|
|
|
|
const icons = (w?.icons ?? [])
|
|
|
|
|
.map((f) => commonsLogo(f, 'wikidata icon'))
|
|
|
|
|
.filter((x) => x && squarish(x.ratio));
|
|
|
|
|
if (o.logo) {
|
|
|
|
|
// --square: swap a wide or tall logo for a square icon.
|
|
|
|
|
const cur = o.logo_file ? info.get(o.logo_file) : null;
|
|
|
|
|
const ratio = cur ? cur.width / cur.height : 1;
|
|
|
|
|
if (icons[0] && (ratio > 1.6 || ratio < 0.6) && icons[0].file !== o.logo_file)
|
|
|
|
|
picks.set(o.slug, icons[0]);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
const logos = (w?.logos ?? []).map((f) => commonsLogo(f, 'wikidata logo')).filter((x) => !!x);
|
|
|
|
|
// Prefer a square logo, then a square icon, then any logo.
|
|
|
|
|
const pick =
|
|
|
|
|
logos.find((l) => squarish(l.ratio)) ??
|
|
|
|
|
icons[0] ??
|
|
|
|
|
logos[0] ??
|
|
|
|
|
(w?.icons ?? []).map((f) => commonsLogo(f, 'wikidata icon')).find((x) => !!x);
|
|
|
|
|
if (pick) picks.set(o.slug, pick);
|
|
|
|
|
else stillMissing.push(o);
|
|
|
|
|
}
|
|
|
|
|
// Wikipedia infobox logo= fields, wiki by wiki.
|
|
|
|
|
let left = stillMissing;
|
|
|
|
|
for (const wiki of WIKIS) {
|
|
|
|
|
const want = left.filter((o) => wd.get(o.wikidata!)?.articles.has(wiki));
|
|
|
|
|
if (!want.length) continue;
|
|
|
|
|
const found = await infoboxLogos(
|
|
|
|
|
wiki,
|
|
|
|
|
want.map((o) => wd.get(o.wikidata!)!.articles.get(wiki)!)
|
|
|
|
|
);
|
|
|
|
|
const files = new Map<string, string>();
|
|
|
|
|
for (const o of want) {
|
|
|
|
|
const f = found.get(wd.get(o.wikidata!)!.articles.get(wiki)!);
|
|
|
|
|
if (f) files.set(o.slug, f);
|
|
|
|
|
}
|
|
|
|
|
await imageInfo([...files.values()]);
|
|
|
|
|
for (const [slug, f] of files) {
|
|
|
|
|
const p = commonsLogo(f, `${wiki}wiki infobox`);
|
|
|
|
|
if (p) picks.set(slug, p);
|
|
|
|
|
}
|
|
|
|
|
left = left.filter((o) => !picks.has(o.slug));
|
|
|
|
|
}
|
|
|
|
|
for (const o of orgs) {
|
|
|
|
|
const p = picks.get(o.slug);
|
|
|
|
|
if (!p) continue;
|
|
|
|
|
log(` ${t.table} ${o.slug}: ${p.how} ${p.file}${o.logo ? ' (square swap)' : ''}`);
|
|
|
|
|
await write(t, o.slug, p);
|
|
|
|
|
}
|
|
|
|
|
const added = orgs.filter((o) => !o.logo && picks.has(o.slug)).length;
|
|
|
|
|
log(
|
|
|
|
|
`${t.table}: +${added} logos from Wikimedia, ${orgs.filter((o) => o.logo && picks.has(o.slug)).length} swapped for square icons, ${left.length} still missing`
|
|
|
|
|
);
|
|
|
|
|
if (favicon) {
|
|
|
|
|
let tried = 0;
|
|
|
|
|
const fails: Record<string, number> = {};
|
|
|
|
|
for (const o of left) {
|
|
|
|
|
const w = wd.get(o.wikidata!);
|
|
|
|
|
if (!w?.sites.length) {
|
|
|
|
|
fails['no website'] = (fails['no website'] ?? 0) + 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (w.dissolved) {
|
|
|
|
|
fails['dissolved'] = (fails['dissolved'] ?? 0) + 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
tried++;
|
|
|
|
|
const r = await websiteIcon(w.sites[0]);
|
|
|
|
|
if ('error' in r) {
|
|
|
|
|
const k = r.error
|
|
|
|
|
.replace(/\(.*$/, '')
|
|
|
|
|
.replace(/HTTP \d+/, 'HTTP error')
|
|
|
|
|
.trim()
|
|
|
|
|
.slice(0, 40);
|
|
|
|
|
fails[k] = (fails[k] ?? 0) + 1;
|
|
|
|
|
log(` ${o.slug}: ${r.error}`);
|
|
|
|
|
} else {
|
|
|
|
|
const list = favHashes.get(r.hash) ?? [];
|
|
|
|
|
list.push({ t, slug: o.slug, host: r.host, png: r.png });
|
|
|
|
|
favHashes.set(r.hash, list);
|
|
|
|
|
log(` ${o.slug}: icon from ${r.host}`);
|
|
|
|
|
}
|
|
|
|
|
await sleep(300);
|
|
|
|
|
}
|
|
|
|
|
log(`${t.table}: website icons tried ${tried}; failures ${JSON.stringify(fails)}`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (favicon) {
|
|
|
|
|
let saved = 0;
|
|
|
|
|
for (const list of favHashes.values()) {
|
|
|
|
|
// The same icon on different sites is a generic CMS / host icon: skip all of them.
|
|
|
|
|
if (new Set(list.map((x) => x.host)).size > 1) {
|
|
|
|
|
log(` generic icon shared by ${list.map((x) => x.slug).join(', ')}: skipped`);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
for (const x of list) {
|
|
|
|
|
if (dryRun) continue;
|
|
|
|
|
await mkdir(`${STATIC}/${x.t.dir}`, { recursive: true });
|
|
|
|
|
await writeFile(`${STATIC}/${x.t.dir}/${x.slug}.webp`, x.png);
|
|
|
|
|
saved++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
log(
|
|
|
|
|
`website icons saved: ${saved} (commit static/spotters/logos, deploy, then run --favicons-db)`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
await sql.end();
|
