trains: Wikidata/Wikipedia import, Commons photos and refresh scripts

scripts/trains-wiki.ts

@@ -0,0 +1,191 @@
+// Shared HTTP helpers for the train pokedex scripts (scripts/trains-import.ts,
+// scripts/trains-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries,
+// maxlag handling and an optional on-disk cache (--cache <dir>, one file per request).
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export const UA =
+	'StanRungeTrainDex/1.0 (https://stanrunge.dev/trackers/trains; train pokedex importer) bun';
+export const log = (...a: unknown[]) => console.error(...a);
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+let cacheDir: string | null = null;
+
+/**
+ * Use `<dir>` as a persistent response cache: one small file per request (keyed by a hash), so
+ * there is never one giant JSON in memory and an interrupted run resumes where it stopped.
+ */
+export async function useCache(dir: string | null) {
+	cacheDir = dir;
+	if (dir) await mkdir(dir, { recursive: true });
+}
+const cachePath = (key: string) =>
+	join(cacheDir!, `${createHash('sha1').update(key).digest('hex')}.json`);
+async function readCached<T>(key: string): Promise<T | undefined> {
+	if (!cacheDir) return undefined;
+	try {
+		return JSON.parse(await readFile(cachePath(key), 'utf8')) as T;
+	} catch {
+		return undefined;
+	}
+}
+async function writeCached(key: string, body: unknown) {
+	if (cacheDir) await writeFile(cachePath(key), JSON.stringify(body));
+}
+export async function hasCached(key: string) {
+	return (await readCached(key)) !== undefined;
+}
+
+export async function getJson<T>(url: string, init?: RequestInit, useCached = true): Promise<T> {
+	const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
+	if (useCached) {
+		const hit = await readCached<T>(key);
+		if (hit !== undefined) return hit;
+	}
+	for (let attempt = 0; ; attempt++) {
+		let res: Response;
+		try {
+			res = await fetch(url, {
+				...init,
+				headers: {
+					'user-agent': UA,
+					'api-user-agent': UA,
+					accept: 'application/json',
+					...init?.headers
+				},
+				signal: AbortSignal.timeout(120_000)
+			});
+		} catch (e) {
+			if (attempt < 6) {
+				await sleep(2000 * 2 ** attempt);
+				continue;
+			}
+			throw e;
+		}
+		const retryAfter = Number(res.headers.get('retry-after')) || 0;
+		if (res.ok) {
+			const body = (await res.json()) as T & { error?: { code?: string } };
+			if (body?.error?.code === 'maxlag' && attempt < 8) {
+				await sleep(Math.max(retryAfter, 5) * 1000);
+				continue;
+			}
+			if (useCached) await writeCached(key, body);
+			return body;
+		}
+		if ((res.status === 429 || res.status >= 500) && attempt < 6) {
+			await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt));
+			continue;
+		}
+		throw new Error(`${res.status} ${url.slice(0, 200)}`);
+	}
+}
+
+let lastCall = 0;
+/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
+export async function mw<T = MwResponse>(
+	host: string,
+	params: Record<string, string>,
+	useCached = true
+): Promise<T> {
+	const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
+	const key = `https://${host}/w/api.php?${body}`;
+	if (!useCached || !(await hasCached(key))) {
+		const wait = lastCall + 150 - Date.now();
+		if (wait > 0) await sleep(wait);
+		lastCall = Date.now();
+	}
+	return getJson<T>(key, undefined, useCached);
+}
+export type MwPage = {
+	pageid?: number;
+	title: string;
+	missing?: boolean;
+	lastrevid?: number;
+	extract?: string;
+	images?: { title: string }[];
+	imageinfo?: ImageInfo[];
+	revisions?: { revid?: number; slots?: { main?: { content?: string } } }[];
+	categorymembers?: { title: string }[];
+};
+export type ImageInfo = {
+	width: number;
+	height: number;
+	mime: string;
+	thumburl?: string;
+	descriptionurl?: string;
+	extmetadata?: Record<string, { value: string }>;
+};
+export type MwResponse = {
+	query?: {
+		pages?: MwPage[];
+		normalized?: { from: string; to: string }[];
+		redirects?: { from: string; to: string }[];
+		categorymembers?: { title: string }[];
+	};
+	continue?: Record<string, string>;
+};
+
+let lastSparql = 0;
+export async function sparql<T = Record<string, string>>(query: string): Promise<T[]> {
+	const body = new URLSearchParams({ query }).toString();
+	if (!(await hasCached(`https://query.wikidata.org/sparql#${body}`))) {
+		const wait = lastSparql + 400 - Date.now();
+		if (wait > 0) await sleep(wait);
+		lastSparql = Date.now();
+	}
+	const res = await getJson<{ results: { bindings: Record<string, { value: string }>[] } }>(
+		'https://query.wikidata.org/sparql',
+		{
+			method: 'POST',
+			headers: {
+				'content-type': 'application/x-www-form-urlencoded',
+				accept: 'application/sparql-results+json'
+			},
+			body
+		}
+	);
+	return res.results.bindings.map(
+		(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
+	);
+}
+
+export const qid = (uri: string) => uri.split('/').at(-1)!;
+export const fileName = (uri: string) =>
+	decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
+/** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */
+export const cmp = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/&/g, ' AND ')
+		.replace(/[^A-Z0-9]/g, '');
+export const tokens = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/&/g, ' ')
+		.split(/[^A-Z0-9]+/)
+		.filter(Boolean);
+export const slugify = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toLowerCase()
+		.replace(/&/g, ' and ')
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-+|-+$/g, '')
+		.slice(0, 80) || 'x';
+export const chunks = <T>(xs: T[], n: number) =>
+	Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n));
+export const stripHtml = (s: string) =>
+	s
+		.replace(/<[^>]+>/g, ' ')
+		.replace(/&amp;/g, '&')
+		.replace(/&quot;/g, '"')
+		.replace(/&#0?39;/g, "'")
+		.replace(/&nbsp;/g, ' ')
+		.replace(/\s+/g, ' ')
+		.trim();