stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

spotters: fix 500 on detail pages of seen items (timestamp comes back as text); logos script uses small separate Wikidata queries
All checks were successful
Deploy production / deploy (push) Successful in 1m10s
Details

Browse Source
This commit is contained in:
Stan Runge
2026-10-25 14:10:52 +00:00
parent 0f8db8e432
commit 11fa8ec006
2 changed files with 50 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
scripts/spotters-logos.ts
View File

@@ -163,52 +163,57 @@ type Wd = {
}; };
async function wikidata(ids: string[]) { async function wikidata(ids: string[]) {
const out = new Map<string, Wd>(); const out = new Map<string, Wd>();
for (const c of chunks(ids, 150)) { const entry = (q: string) => {
let rows: Record<string, string>[]; let w = out.get(q);
if (!w)
out.set(q, (w = { logos: [], icons: [], sites: [], dissolved: false, articles: new Map() }));
return w;
};
// Four small queries per chunk: one joined query times out on the public endpoint.
const run = async (c: string[], body: string) => {
try { try {
rows = await sparql(` return await sparql(
SELECT ?m (GROUP_CONCAT(DISTINCT CONCAT(STR(?logo), ";", STR(?rank), ";", COALESCE(STR(?end), "")); separator="|") AS ?logos) `SELECT * WHERE { VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} } ${body} }`
(GROUP_CONCAT(DISTINCT STR(?icon); separator="|") AS ?icons) );
(GROUP_CONCAT(DISTINCT STR(?site); separator="|") AS ?sites)
(GROUP_CONCAT(DISTINCT STR(?a); separator="|") AS ?articles) (SAMPLE(?gone) AS ?dissolved)
WHERE {
VALUES ?m { ${c.map((x) => `wd:${x}`).join(' ')} }
OPTIONAL { ?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank)
OPTIONAL { ?st pq:P582 ?end } }
OPTIONAL { ?m p:P8972|p:P2910 ?ist . ?ist ps:P8972|ps:P2910 ?icon ; wikibase:rank ?irank .
FILTER(?irank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?ist pq:P582 [] } }
OPTIONAL { ?m p:P856 ?wst . ?wst ps:P856 ?site ; wikibase:rank ?wrank .
FILTER(?wrank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?wst pq:P582 [] } }
OPTIONAL { ?a schema:about ?m ; schema:isPartOf ?wiki .
VALUES ?wiki { ${WIKIS.map((w) => `<https://${w}.wikipedia.org/>`).join(' ')} } }
OPTIONAL { ?m wdt:P576 ?gone }
} GROUP BY ?m`);
} catch (e) { } catch (e) {
log(`wikidata chunk failed: ${(e as Error).message}`); log(`wikidata chunk failed: ${(e as Error).message}`);
continue; return [];
} }
for (const r of rows) { };
const logos = (r.logos ? r.logos.split('|').filter(Boolean) : []) for (const c of chunks(ids, 150)) {
.map((x) => { const logos = await run(
const [uri, rank, end] = x.split(';'); c,
return { `?m p:P154 ?st . ?st ps:P154 ?logo ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) OPTIONAL { ?st pq:P582 ?end }`
file: fileName(uri), );
score: (rank.endsWith('PreferredRank') ? 2 : 0) + (end ? 0 : 4) const scored = new Map<string, { file: string; score: number }[]>();
}; for (const r of logos) {
}) const list = scored.get(qid(r.m)) ?? [];
.sort((a, b) => b.score - a.score); list.push({
const articles = new Map<string, string>(); file: fileName(r.logo),
for (const a of r.articles ? r.articles.split('|') : []) { score: (r.rank.endsWith('PreferredRank') ? 2 : 0) + (r.end ? 0 : 4)
const m = a.match(/^https:\/\/(\w+)\.wikipedia\.org\/wiki\/(.+)$/);
if (m) articles.set(m[1], decodeURIComponent(m[2]).replace(/_/g, ' '));
}
out.set(qid(r.m), {
logos: [...new Set(logos.map((l) => l.file))],
icons: r.icons ? [...new Set(r.icons.split('|').map(fileName))] : [],
sites: r.sites ? r.sites.split('|').filter((s) => /^https?:\/\//.test(s)) : [],
dissolved: !!r.dissolved,
articles
}); });
scored.set(qid(r.m), list);
}
for (const [q, list] of scored)
entry(q).logos = [...new Set(list.sort((x, y) => y.score - x.score).map((x) => x.file))];
for (const r of await run(
c,
`?m p:P8972|p:P2910 ?st . ?st ps:P8972|ps:P2910 ?icon ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?st pq:P582 [] }`
))
entry(qid(r.m)).icons.push(fileName(r.icon));
for (const r of await run(
c,
`{ ?m p:P856 ?st . ?st ps:P856 ?site ; wikibase:rank ?rank . FILTER(?rank != wikibase:DeprecatedRank) FILTER NOT EXISTS { ?st pq:P582 [] } } UNION { ?m wdt:P576 ?gone }`
)) {
if (r.gone) entry(qid(r.m)).dissolved = true;
else if (/^https?:\/\//.test(r.site ?? '')) entry(qid(r.m)).sites.push(r.site);
}
for (const r of await run(
c,
`?a schema:about ?m ; schema:isPartOf ?wiki . VALUES ?wiki { ${WIKIS.map((w) => `<https://${w}.wikipedia.org/>`).join(' ')} }`
)) {
const m = r.a.match(/^https:\/\/(\w+)\.wikipedia\.org\/wiki\/(.+)$/);
if (m) entry(qid(r.m)).articles.set(m[1], decodeURIComponent(m[2]).replace(/_/g, ' '));
} }
} }
return out; return out;

9
src/lib/server/spotting.ts
View File

@@ -62,11 +62,10 @@ export async function idsProgress(tracker: Tracker, ids: number[]) {
export async function getSeen(tracker: Tracker, id: number): Promise<SeenEntry | null> { export async function getSeen(tracker: Tracker, id: number): Promise<SeenEntry | null> {
const { col } = T[tracker]; const { col } = T[tracker];
const [r] = await client.unsafe<{ seen_on: string | null; note: string | null; at: Date }[]>( const [r] = await client.unsafe<
`select seen_on::text, note, created_at as at from ${tracker}.seen where ${col} = $1`, { seen_on: string | null; note: string | null; at: string | Date }[]
[id] >(`select seen_on::text, note, created_at as at from ${tracker}.seen where ${col} = $1`, [id]);
); return r ? { seenOn: r.seen_on, note: r.note, at: new Date(r.at).toISOString() } : null;
return r ? { seenOn: r.seen_on, note: r.note, at: r.at.toISOString() } : null;
} }
/** Marks (or unmarks) an item as seen. Returns false when the item does not exist. */ /** Marks (or unmarks) an item as seen. Returns false when the item does not exist. */