From 11fa8ec006b2b826947904f87b7543301a26a90f Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sun, 25 Oct 2026 14:10:52 +0000 Subject: [PATCH] spotters: fix 500 on detail pages of seen items (timestamp comes back as text); logos script uses small separate Wikidata queries --- scripts/spotters-logos.ts | 87 ++++++++++++++++++++------------------ src/lib/server/spotting.ts | 9 ++-- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/scripts/spotters-logos.ts b/scripts/spotters-logos.ts index f7a4e55..5609dbe 100644 --- a/scripts/spotters-logos.ts +++ b/scripts/spotters-logos.ts @@ -163,52 +163,57 @@ type Wd = { }; async function wikidata(ids: string[]) { const out = new Map(); - for (const c of chunks(ids, 150)) { - let rows: Record[]; + const entry = (q: string) => { + let w = out.get(q); + if (!w) + out.set(q, (w = { logos: [], icons: [], sites: [], dissolved: false, articles: new Map() })); + return w; + }; + // Four small queries per chunk: one joined query times out on the public endpoint. + const run = async (c: string[], body: string) => { try { - rows = await sparql(` -SELECT ?m (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(?end), "")); separator="|") AS ?logos) - (GROUP_CONCAT(DISTINCT STR(?icon); separator="|") AS ?icons) - (GROUP_CONCAT(DISTINCT STR(?site); separator="|") AS ?sites) - (GROUP_CONCAT(DISTINCT STR(?a); separator="|") AS ?articles) (SAMPLE(?gone) AS ?dissolved) -WHERE { - VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } - OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) - OPTIONAL { ?st pq:P582 ?end } } - OPTIONAL { ?m p:P8972|p:P2910 ?ist . ?ist ps:P8972|ps:P2910 ?icon ; wikibase:rank ?irank . - FILTER(?irank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?ist pq:P582 [] } } - OPTIONAL { ?m p:P856 ?wst . ?wst ps:P856 ?site ; wikibase:rank ?wrank . - FILTER(?wrank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?wst pq:P582 [] } } - OPTIONAL { ?a schema:about ?m ; schema:isPartOf ?wiki . - VALUES ?wiki { ${WIKIS.map((w) => ``).join(' ')} } } - OPTIONAL { ?m wdt:P576 ?gone } -} GROUP BY ?m`); + return await sparql( + `SELECT * WHERE { VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } ${body} }` + ); } catch (e) { log(`wikidata chunk failed: ${(e as Error).message}`); - continue; + return []; } - for (const r of rows) { - const logos = (r.logos ? r.logos.split('|').filter(Boolean) : []) - .map((x) => { - const [uri, rank, end] = x.split(';'); - return { - file: fileName(uri), - score: (rank.endsWith('PreferredRank') ? 2 : 0) + (end ? 0 : 4) - }; - }) - .sort((a, b) => b.score - a.score); - const articles = new Map(); - for (const a of r.articles ? r.articles.split('|') : []) { - const m = a.match(/^https:\/\/(\w+)\.wikipedia\.org\/wiki\/(.+)$/); - if (m) articles.set(m[1], decodeURIComponent(m[2]).replace(/_/g, ' ')); - } - out.set(qid(r.m), { - logos: [...new Set(logos.map((l) => l.file))], - icons: r.icons ? [...new Set(r.icons.split('|').map(fileName))] : [], - sites: r.sites ? r.sites.split('|').filter((s) => /^https?:\/\//.test(s)) : [], - dissolved: !!r.dissolved, - articles + }; + for (const c of chunks(ids, 150)) { + const logos = await run( + c, + `?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) OPTIONAL { ?st pq:P582 ?end }` + ); + const scored = new Map(); + for (const r of logos) { + const list = scored.get(qid(r.m)) ?? []; + list.push({ + file: fileName(r.logo), + score: (r.rank.endsWith('PreferredRank') ? 2 : 0) + (r.end ? 0 : 4) }); + scored.set(qid(r.m), list); + } + for (const [q, list] of scored) + entry(q).logos = [...new Set(list.sort((x, y) => y.score - x.score).map((x) => x.file))]; + for (const r of await run( + c, + `?m p:P8972|p:P2910 ?st . ?st ps:P8972|ps:P2910 ?icon ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?st pq:P582 [] }` + )) + entry(qid(r.m)).icons.push(fileName(r.icon)); + for (const r of await run( + c, + `{ ?m p:P856 ?st . ?st ps:P856 ?site ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?st pq:P582 [] } } UNION { ?m wdt:P576 ?gone }` + )) { + if (r.gone) entry(qid(r.m)).dissolved = true; + else if (/^https?:\/\//.test(r.site ?? '')) entry(qid(r.m)).sites.push(r.site); + } + for (const r of await run( + c, + `?a schema:about ?m ; schema:isPartOf ?wiki . VALUES ?wiki { ${WIKIS.map((w) => ``).join(' ')} }` + )) { + const m = r.a.match(/^https:\/\/(\w+)\.wikipedia\.org\/wiki\/(.+)$/); + if (m) entry(qid(r.m)).articles.set(m[1], decodeURIComponent(m[2]).replace(/_/g, ' ')); } } return out; diff --git a/src/lib/server/spotting.ts b/src/lib/server/spotting.ts index 7433b23..6b0dcec 100644 --- a/src/lib/server/spotting.ts +++ b/src/lib/server/spotting.ts @@ -62,11 +62,10 @@ export async function idsProgress(tracker: Tracker, ids: number[]) { export async function getSeen(tracker: Tracker, id: number): Promise { const { col } = T[tracker]; - const [r] = await client.unsafe<{ seen_on: string | null; note: string | null; at: Date }[]>( - `select seen_on::text, note, created_at as at from ${tracker}.seen where ${col} = $1`, - [id] - ); - return r ? { seenOn: r.seen_on, note: r.note, at: r.at.toISOString() } : null; + const [r] = await client.unsafe< + { seen_on: string | null; note: string | null; at: string | Date }[] + >(`select seen_on::text, note, created_at as at from ${tracker}.seen where ${col} = $1`, [id]); + return r ? { seenOn: r.seen_on, note: r.note, at: new Date(r.at).toISOString() } : null; } /** Marks (or unmarks) an item as seen. Returns false when the item does not exist. */