diff --git a/scripts/spotters-activity.ts b/scripts/spotters-activity.ts index b07ed05..1b9b544 100644 --- a/scripts/spotters-activity.ts +++ b/scripts/spotters-activity.ts @@ -193,8 +193,10 @@ async function planes() { makers: string[]; icao: string[]; wikidata: string; + current: boolean; }[] - >`select t.id, t.family_id as family, t.name, m.name as maker, t.makers, t.icao, t.wikidata + >`select t.id, t.family_id as family, t.name, m.name as maker, t.makers, t.icao, t.wikidata, + (t.status in ('in service', 'in production') or (t.retired_year is null and coalesce(t.introduced_year, t.first_flight_year, 0) >= 1960)) as current from planes.types t join planes.makers m on m.id = t.maker_id`; // Candidate keys per maker word: the type's name, without generic words, and its @@ -346,14 +348,18 @@ async function planes() { isoOf.set(op, set); } } + // Historic operators (a 1918 air force flying the DH.9) say little about what you can + // spot today: types that are retired or older than 1960 get a fifth of the bonus. + const current = new Map(types.map((t) => [t.id, t.current])); for (const [item, opSet] of opsOf) { const id = byQid.get(item); if (!id) continue; + const weight = current.get(id) ? 25 : 5; const perCountry = new Map(); for (const op of opSet) for (const cc of isoOf.get(op) ?? []) perCountry.set(cc, (perCountry.get(cc) ?? 0) + 1); for (const [cc, n] of perCountry) { - add(scores, id, cc, 25 * Math.min(n, 6)); + add(scores, id, cc, weight * Math.min(n, 6)); ops++; } } diff --git a/scripts/spotters-logos.ts b/scripts/spotters-logos.ts index 5609dbe..0d6401c 100644 --- a/scripts/spotters-logos.ts +++ b/scripts/spotters-logos.ts @@ -17,6 +17,8 @@ // after the files are deployed) // --new only orgs created in the last 2 days (for the refresh scripts) // --limit first n orgs by priority (most types, Dutch first) +// --recheck re-test the Wikimedia picks listed in an earlier run's log with the +// current rules and clear the ones that fail // --dry-run print what would change, write nothing // // Logos stay strict: a file must be free on Commons (toPhoto-like check: no fair use, @@ -29,6 +31,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; import sharp from 'sharp'; import { chunks, + cmp, fileName, log, mw, @@ -36,6 +39,7 @@ import { sleep, sparql, stripHtml, + tokens, type ImageInfo } from './planes-wiki'; @@ -49,6 +53,7 @@ const favicon = flag('--favicon'); const faviconsDb = flag('--favicons-db'); const onlyNew = flag('--new'); const limit = Number(opt('--limit')) || Infinity; +const recheck = opt('--recheck'); if (!['cars', 'planes', 'trains', 'all'].includes(which ?? '')) throw new Error('usage: spotters-logos.ts [--square] [--favicon]'); @@ -132,7 +137,11 @@ async function imageInfo(files: string[]) { } } /** A free Commons image as a logo, or null (fair use / non-free / not an image). */ -function commonsLogo(file: string, how: string): (Pick & { ratio: number }) | null { +function commonsLogo( + file: string, + how: string, + names: string[] +): (Pick & { ratio: number }) | null { const i = info.get(file); if (!i?.thumburl || !/^image\/(svg\+xml|png|gif|jpeg|webp)$/.test(i.mime)) return null; const meta = i.extmetadata ?? {}; @@ -141,6 +150,7 @@ function commonsLogo(file: string, how: string): (Pick & { ratio: number }) | nu if (!license || /fair use|non-free/i.test(license)) return null; if (/trademark/i.test(meta.Restrictions?.value ?? '')) license += ' · trademark'; if (i.width < 48 || i.height < 48) return null; + if (!looksLikeLogo(file, how, names, license)) return null; return { logo: i.thumburl.replace(/\?.*$/, ''), file, @@ -152,10 +162,41 @@ function commonsLogo(file: string, how: string): (Pick & { ratio: number }) | nu } const squarish = (r: number) => r >= 0.75 && r <= 1.34; +// Wikidata's logo property and infobox logo fields often hold photos (a badge on a car, a +// works plate, a building), old names or national insignia, so every file must also look +// like a logo of this org by its name. +const LOGO_WORD = /logo|logotype|logotip|wordmark|emblem|marque|insignia|signet|symbol|лого/i; +const NOT_LOGO = + /plate\b|plaque|schild|advertisement|werbung|relief|building|interior|museum|launch|drawing|left side|naca|\bimg|\bdsc|\(\d{6,}\)|cropped|flickr|coat of arms|reichsadler|\bflags?\b|\bseal\b|banner|roundel of|\bon a car|\bon st-|works\b/i; +const FREE_PD = /public domain|^pd\b|cc0/i; +const GENERIC = new Set( + 'RAIL RAILWAY RAILWAYS AVIATION AIRCRAFT AIRCRAFTS AIRPLANE COMPANY WORKS WERKE GROUP CORPORATION CORP MOTOR MOTORS CARS LOCOMOTIVE LOCOMOTIVES ENGINEERING INDUSTRIES INDUSTRY AND THE GMBH LTD LIMITED INC FACTORY PLANT TRANSPORT TRANSPORTATION'.split( + ' ' + ) +); +function looksLikeLogo(file: string, how: string, names: string[], license: string | null) { + if (NOT_LOGO.test(file)) return false; + const f = cmp(file); + const ft = new Set(tokens(file)); + const full = names.some((n) => cmp(n).length >= 3 && f.includes(cmp(n))); + const token = names.some((n) => + tokens(n).some((t) => t.length >= 4 && !GENERIC.has(t) && ft.has(t)) + ); + const word = LOGO_WORD.test(file); + const fromItem = how.startsWith('wikidata'); + const many = names.some( + (n) => tokens(n).filter((t) => t.length >= 4 && !GENERIC.has(t) && ft.has(t)).length >= 2 + ); + if (/\.svg$/i.test(file)) return fromItem || many || (word && (full || token)); + if (/\.(png|gif)$/i.test(file)) return fromItem ? word || full || token : full || (word && token); + return fromItem && word && (full || (token && FREE_PD.test(license ?? ''))); +} + // ---------------------------------------------------------------- wikidata const WIKIS = ['en', 'nl', 'de', 'fr', 'it', 'es', 'pl', 'cs', 'sv', 'ja']; type Wd = { logos: string[]; + oldLogos: string[]; icons: string[]; sites: string[]; dissolved: boolean; @@ -166,7 +207,17 @@ async function wikidata(ids: string[]) { const entry = (q: string) => { let w = out.get(q); if (!w) - out.set(q, (w = { logos: [], icons: [], sites: [], dissolved: false, articles: new Map() })); + out.set( + q, + (w = { + logos: [], + oldLogos: [], + icons: [], + sites: [], + dissolved: false, + articles: new Map() + }) + ); return w; }; // Four small queries per chunk: one joined query times out on the public endpoint. @@ -194,8 +245,12 @@ async function wikidata(ids: string[]) { }); scored.set(qid(r.m), list); } - for (const [q, list] of scored) - entry(q).logos = [...new Set(list.sort((x, y) => y.score - x.score).map((x) => x.file))]; + for (const [q, list] of scored) { + const sorted = list.sort((x, y) => y.score - x.score); + // Ended logos only count for dissolved orgs (a renamed org must not keep its old logo). + entry(q).logos = [...new Set(sorted.filter((x) => x.score >= 4).map((x) => x.file))]; + entry(q).oldLogos = [...new Set(sorted.map((x) => x.file))]; + } for (const r of await run( c, `?m p:P8972|p:P2910 ?st . ?st ps:P8972|ps:P2910 ?icon ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?st pq:P582 [] }` @@ -381,6 +436,35 @@ async function write(t: Table, slug: string, p: Pick) { ); } +if (recheck) { + const { readFile } = await import('node:fs/promises'); + const picks = [ + ...(await readFile(recheck, 'utf8')).matchAll( + /^ {2}(\S+) (\S+): (wikidata logo|wikidata icon|\w+wiki infobox) (.+?)$/gm + ) + ].filter((m) => !m[4].endsWith(' (square swap)')); + await imageInfo(picks.map((m) => m[4])); + let cleared = 0; + for (const [, table, slug, how, file] of picks) { + const t = TABLES.find((x) => x.table === table)!; + const [o] = (await sql.unsafe( + `select name, ${t.table === 'cars.brands' ? 'null' : 'full_name'} as full_name from ${t.table} where slug = $1`, + [slug] + )) as unknown as { name: string; full_name: string | null }[]; + if (!o || commonsLogo(file, how, [o.name, o.full_name ?? o.name])) continue; + cleared++; + log(` clear ${table} ${slug}: ${file}`); + if (!dryRun) + await sql.unsafe( + `update ${t.table} set logo = null, logo_file = null, logo_author = null, logo_license = null where slug = $1 and logo_file = $2`, + [slug, file] + ); + } + log(`recheck: ${cleared}/${picks.length} picks cleared`); + await sql.end(); + process.exit(0); +} + const tables = TABLES.filter((t) => which === 'all' || t.tracker === which); const favHashes = new Map(); for (const t of tables) { @@ -410,15 +494,16 @@ for (const t of tables) { log(`${t.table}: ${before} with a logo; checking ${orgs.length}`); const wd = await wikidata([...new Set(orgs.map((o) => o.wikidata!))]); await imageInfo([ - ...[...wd.values()].flatMap((w) => [...w.logos, ...w.icons]), + ...[...wd.values()].flatMap((w) => [...w.oldLogos, ...w.icons]), ...orgs.map((o) => o.logo_file).filter((f): f is string => !!f) ]); const picks = new Map(); const stillMissing: Org[] = []; for (const o of orgs) { const w = wd.get(o.wikidata!); + const names = [o.name, o.full_name ?? o.name]; const icons = (w?.icons ?? []) - .map((f) => commonsLogo(f, 'wikidata icon')) + .map((f) => commonsLogo(f, 'wikidata icon', names)) .filter((x) => x && squarish(x.ratio)); if (o.logo) { // --square: swap a wide or tall logo for a square icon. @@ -428,13 +513,15 @@ for (const t of tables) { picks.set(o.slug, icons[0]); continue; } - const logos = (w?.logos ?? []).map((f) => commonsLogo(f, 'wikidata logo')).filter((x) => !!x); + const logos = (w?.logos.length ? w.logos : w?.dissolved ? w.oldLogos : []) + .map((f) => commonsLogo(f, 'wikidata logo', names)) + .filter((x) => !!x); // Prefer a square logo, then a square icon, then any logo. const pick = logos.find((l) => squarish(l.ratio)) ?? icons[0] ?? logos[0] ?? - (w?.icons ?? []).map((f) => commonsLogo(f, 'wikidata icon')).find((x) => !!x); + (w?.icons ?? []).map((f) => commonsLogo(f, 'wikidata icon', names)).find((x) => !!x); if (pick) picks.set(o.slug, pick); else stillMissing.push(o); } @@ -454,7 +541,8 @@ for (const t of tables) { } await imageInfo([...files.values()]); for (const [slug, f] of files) { - const p = commonsLogo(f, `${wiki}wiki infobox`); + const o = want.find((x) => x.slug === slug)!; + const p = commonsLogo(f, `${wiki}wiki infobox`, [o.name, o.full_name ?? o.name]); if (p) picks.set(slug, p); } left = left.filter((o) => !picks.has(o.slug));