website/scripts/trains-wiki.ts

// Shared HTTP helpers for the train pokedex scripts (scripts/trains-import.ts,
// scripts/trains-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries,
// maxlag handling and an optional on-disk cache (--cache <dir>, one file per request).
import { createHash } from 'node:crypto';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';

export const UA =
	'StanRungeTrainDex/1.0 (https://stanrunge.dev/trackers/trains; train pokedex importer) bun';
export const log = (...a: unknown[]) => console.error(...a);
export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

let cacheDir: string | null = null;

/**
 * Use `<dir>` as a persistent response cache: one small file per request (keyed by a hash), so
 * there is never one giant JSON in memory and an interrupted run resumes where it stopped.
 */
export async function useCache(dir: string | null) {
	cacheDir = dir;
	if (dir) await mkdir(dir, { recursive: true });
}
const cachePath = (key: string) =>
	join(cacheDir!, `${createHash('sha1').update(key).digest('hex')}.json`);
async function readCached<T>(key: string): Promise<T | undefined> {
	if (!cacheDir) return undefined;
	try {
		return JSON.parse(await readFile(cachePath(key), 'utf8')) as T;
	} catch {
		return undefined;
	}
}
async function writeCached(key: string, body: unknown) {
	if (cacheDir) await writeFile(cachePath(key), JSON.stringify(body));
}
export async function hasCached(key: string) {
	return (await readCached(key)) !== undefined;
}

export async function getJson<T>(url: string, init?: RequestInit, useCached = true): Promise<T> {
	const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
	if (useCached) {
		const hit = await readCached<T>(key);
		if (hit !== undefined) return hit;
	}
	for (let attempt = 0; ; attempt++) {
		let res: Response;
		try {
			res = await fetch(url, {
				...init,
				headers: {
					'user-agent': UA,
					'api-user-agent': UA,
					accept: 'application/json',
					...init?.headers
				},
				signal: AbortSignal.timeout(120_000)
			});
		} catch (e) {
			if (attempt < 6) {
				await sleep(2000 * 2 ** attempt);
				continue;
			}
			throw e;
		}
		const retryAfter = Number(res.headers.get('retry-after')) || 0;
		if (res.ok) {
			const body = (await res.json()) as T & { error?: { code?: string } };
			if (body?.error?.code === 'maxlag' && attempt < 8) {
				await sleep(Math.max(retryAfter, 5) * 1000);
				continue;
			}
			if (useCached) await writeCached(key, body);
			return body;
		}
		if ((res.status === 429 || res.status >= 500) && attempt < 6) {
			await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt));
			continue;
		}
		throw new Error(`${res.status} ${url.slice(0, 200)}`);
	}
}

let lastCall = 0;
/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
export async function mw<T = MwResponse>(
	host: string,
	params: Record<string, string>,
	useCached = true
): Promise<T> {
	const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
	const key = `https://${host}/w/api.php?${body}`;
	if (!useCached || !(await hasCached(key))) {
		const wait = lastCall + 150 - Date.now();
		if (wait > 0) await sleep(wait);
		lastCall = Date.now();
	}
	return getJson<T>(key, undefined, useCached);
}
export type MwPage = {
	pageid?: number;
	title: string;
	missing?: boolean;
	lastrevid?: number;
	extract?: string;
	images?: { title: string }[];
	imageinfo?: ImageInfo[];
	revisions?: { revid?: number; slots?: { main?: { content?: string } } }[];
	categorymembers?: { title: string }[];
};
export type ImageInfo = {
	width: number;
	height: number;
	mime: string;
	thumburl?: string;
	descriptionurl?: string;
	extmetadata?: Record<string, { value: string }>;
};
export type MwResponse = {
	query?: {
		pages?: MwPage[];
		normalized?: { from: string; to: string }[];
		redirects?: { from: string; to: string }[];
		categorymembers?: { title: string }[];
	};
	continue?: Record<string, string>;
};

let lastSparql = 0;
export async function sparql<T = Record<string, string>>(query: string): Promise<T[]> {
	const body = new URLSearchParams({ query }).toString();
	if (!(await hasCached(`https://query.wikidata.org/sparql#${body}`))) {
		const wait = lastSparql + 400 - Date.now();
		if (wait > 0) await sleep(wait);
		lastSparql = Date.now();
	}
	const res = await getJson<{ results: { bindings: Record<string, { value: string }>[] } }>(
		'https://query.wikidata.org/sparql',
		{
			method: 'POST',
			headers: {
				'content-type': 'application/x-www-form-urlencoded',
				accept: 'application/sparql-results+json'
			},
			body
		}
	);
	return res.results.bindings.map(
		(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
	);
}

export const qid = (uri: string) => uri.split('/').at(-1)!;
export const fileName = (uri: string) =>
	decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
/** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */
export const cmp = (s: string) =>
	s
		.normalize('NFD')
		.replace(/[\u0300-\u036f]/g, '')
		.toUpperCase()
		.replace(/&/g, ' AND ')
		.replace(/[^A-Z0-9]/g, '');
export const tokens = (s: string) =>
	s
		.normalize('NFD')
		.replace(/[\u0300-\u036f]/g, '')
		.toUpperCase()
		.replace(/&/g, ' ')
		.split(/[^A-Z0-9]+/)
		.filter(Boolean);
export const slugify = (s: string) =>
	s
		.normalize('NFD')
		.replace(/[\u0300-\u036f]/g, '')
		.toLowerCase()
		.replace(/&/g, ' and ')
		.replace(/[^a-z0-9]+/g, '-')
		.replace(/^-+|-+$/g, '')
		.slice(0, 80) || 'x';
export const chunks = <T>(xs: T[], n: number) =>
	Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n));
export const stripHtml = (s: string) =>
	s
		.replace(/<[^>]+>/g, ' ')
		.replace(/&amp;/g, '&')
		.replace(/&quot;/g, '"')
		.replace(/&#0?39;/g, "'")
		.replace(/&nbsp;/g, ' ')
		.replace(/\s+/g, ' ')
		.trim();