planes: wikidata/wikipedia import, commons photos and refresh script

2026-10-24 20:10:36 +00:00
parent cc6c7c8ede
commit 2259965a28
5 changed files with 2346 additions and 0 deletions
--- a/scripts/planes-wiki.ts
+++ b/scripts/planes-wiki.ts
@@ -0,0 +1,184 @@
+// Shared HTTP helpers for the plane pokedex scripts (scripts/planes-import.ts,
+// scripts/planes-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries,
+// maxlag handling and an optional on-disk cache (--cache <dir>).
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export const UA =
+	'StanRungePlaneDex/1.0 (https://stanrunge.dev/trackers/planes; plane pokedex importer) bun';
+export const log = (...a: unknown[]) => console.error(...a);
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+let cacheDir: string | null = null;
+let cache: Record<string, unknown> = {};
+let dirty = 0;
+let cacheName = 'api.json';
+
+/** Use `<dir>/<name>` as a persistent response cache (only for dev runs and refreshes). */
+export async function useCache(dir: string | null, name = 'api.json') {
+	cacheDir = dir;
+	cacheName = name;
+	if (!dir) return;
+	await mkdir(dir, { recursive: true });
+	try {
+		cache = JSON.parse(await readFile(join(dir, name), 'utf8'));
+	} catch {
+		cache = {};
+	}
+}
+export async function saveCache(force = false) {
+	if (!cacheDir || (!force && dirty < 200)) return;
+	dirty = 0;
+	await writeFile(join(cacheDir, cacheName), JSON.stringify(cache));
+}
+
+export async function getJson<T>(url: string, init?: RequestInit, useCached = true): Promise<T> {
+	const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
+	if (useCached && key in cache) return cache[key] as T;
+	for (let attempt = 0; ; attempt++) {
+		let res: Response;
+		try {
+			res = await fetch(url, {
+				...init,
+				headers: {
+					'user-agent': UA,
+					'api-user-agent': UA,
+					accept: 'application/json',
+					...init?.headers
+				},
+				signal: AbortSignal.timeout(120_000)
+			});
+		} catch (e) {
+			if (attempt < 6) {
+				await sleep(2000 * 2 ** attempt);
+				continue;
+			}
+			throw e;
+		}
+		const retryAfter = Number(res.headers.get('retry-after')) || 0;
+		if (res.ok) {
+			const body = (await res.json()) as T & { error?: { code?: string } };
+			if (body?.error?.code === 'maxlag' && attempt < 8) {
+				await sleep(Math.max(retryAfter, 5) * 1000);
+				continue;
+			}
+			if (useCached && cacheDir) {
+				cache[key] = body;
+				dirty++;
+				await saveCache();
+			}
+			return body;
+		}
+		if ((res.status === 429 || res.status >= 500) && attempt < 6) {
+			await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt));
+			continue;
+		}
+		throw new Error(`${res.status} ${url.slice(0, 200)}`);
+	}
+}
+
+let lastCall = 0;
+/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
+export async function mw<T = MwResponse>(
+	host: string,
+	params: Record<string, string>,
+	useCached = true
+): Promise<T> {
+	const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
+	const key = `https://${host}/w/api.php?${body}`;
+	if (!useCached || !(key in cache)) {
+		const wait = lastCall + 150 - Date.now();
+		if (wait > 0) await sleep(wait);
+		lastCall = Date.now();
+	}
+	return getJson<T>(key, undefined, useCached);
+}
+export type MwPage = {
+	pageid?: number;
+	title: string;
+	missing?: boolean;
+	lastrevid?: number;
+	extract?: string;
+	images?: { title: string }[];
+	imageinfo?: ImageInfo[];
+	revisions?: { revid?: number; slots?: { main?: { content?: string } } }[];
+	categorymembers?: { title: string }[];
+};
+export type ImageInfo = {
+	width: number;
+	height: number;
+	mime: string;
+	thumburl?: string;
+	descriptionurl?: string;
+	extmetadata?: Record<string, { value: string }>;
+};
+export type MwResponse = {
+	query?: {
+		pages?: MwPage[];
+		normalized?: { from: string; to: string }[];
+		redirects?: { from: string; to: string }[];
+		categorymembers?: { title: string }[];
+	};
+	continue?: Record<string, string>;
+};
+
+let lastSparql = 0;
+export async function sparql<T = Record<string, string>>(query: string): Promise<T[]> {
+	const wait = lastSparql + 400 - Date.now();
+	if (wait > 0) await sleep(wait);
+	lastSparql = Date.now();
+	const res = await getJson<{ results: { bindings: Record<string, { value: string }>[] } }>(
+		'https://query.wikidata.org/sparql',
+		{
+			method: 'POST',
+			headers: {
+				'content-type': 'application/x-www-form-urlencoded',
+				accept: 'application/sparql-results+json'
+			},
+			body: new URLSearchParams({ query }).toString()
+		}
+	);
+	return res.results.bindings.map(
+		(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
+	);
+}
+
+export const qid = (uri: string) => uri.split('/').at(-1)!;
+export const fileName = (uri: string) =>
+	decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
+/** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */
+export const cmp = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/&/g, ' AND ')
+		.replace(/[^A-Z0-9]/g, '');
+export const tokens = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toUpperCase()
+		.replace(/&/g, ' ')
+		.split(/[^A-Z0-9]+/)
+		.filter(Boolean);
+export const slugify = (s: string) =>
+	s
+		.normalize('NFD')
+		.replace(/[\u0300-\u036f]/g, '')
+		.toLowerCase()
+		.replace(/&/g, ' and ')
+		.replace(/[^a-z0-9]+/g, '-')
+		.replace(/^-+|-+$/g, '')
+		.slice(0, 80) || 'x';
+export const chunks = <T>(xs: T[], n: number) =>
+	Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n));
+export const stripHtml = (s: string) =>
+	s
+		.replace(/<[^>]+>/g, ' ')
+		.replace(/&amp;/g, '&')
+		.replace(/&quot;/g, '"')
+		.replace(/&#0?39;/g, "'")
+		.replace(/&nbsp;/g, ' ')
+		.replace(/\s+/g, ' ')
+		.trim();