// Shared HTTP helpers for the plane pokedex scripts (scripts/planes-import.ts, // scripts/planes-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries, // maxlag handling and an optional on-disk cache (--cache ). import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; export const UA = 'StanRungePlaneDex/1.0 (https://stanrunge.dev/trackers/planes; plane pokedex importer) bun'; export const log = (...a: unknown[]) => console.error(...a); export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); let cacheDir: string | null = null; let cache: Record = {}; let dirty = 0; let cacheName = 'api.json'; /** Use `/` as a persistent response cache (only for dev runs and refreshes). */ export async function useCache(dir: string | null, name = 'api.json') { cacheDir = dir; cacheName = name; if (!dir) return; await mkdir(dir, { recursive: true }); try { cache = JSON.parse(await readFile(join(dir, name), 'utf8')); } catch { cache = {}; } } export async function saveCache(force = false) { if (!cacheDir || (!force && dirty < 200)) return; dirty = 0; await writeFile(join(cacheDir, cacheName), JSON.stringify(cache)); } export async function getJson(url: string, init?: RequestInit, useCached = true): Promise { const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`; if (useCached && key in cache) return cache[key] as T; for (let attempt = 0; ; attempt++) { let res: Response; try { res = await fetch(url, { ...init, headers: { 'user-agent': UA, 'api-user-agent': UA, accept: 'application/json', ...init?.headers }, signal: AbortSignal.timeout(120_000) }); } catch (e) { if (attempt < 6) { await sleep(2000 * 2 ** attempt); continue; } throw e; } const retryAfter = Number(res.headers.get('retry-after')) || 0; if (res.ok) { const body = (await res.json()) as T & { error?: { code?: string } }; if (body?.error?.code === 'maxlag' && attempt < 8) { await sleep(Math.max(retryAfter, 5) * 1000); continue; } if (useCached && cacheDir) { cache[key] = body; dirty++; await saveCache(); } return body; } if ((res.status === 429 || res.status >= 500) && attempt < 6) { await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt)); continue; } throw new Error(`${res.status} ${url.slice(0, 200)}`); } } let lastCall = 0; /** MediaWiki action API, one request at a time with a small gap (API etiquette). */ export async function mw( host: string, params: Record, useCached = true ): Promise { const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params }); const key = `https://${host}/w/api.php?${body}`; if (!useCached || !(key in cache)) { const wait = lastCall + 150 - Date.now(); if (wait > 0) await sleep(wait); lastCall = Date.now(); } return getJson(key, undefined, useCached); } export type MwPage = { pageid?: number; title: string; missing?: boolean; lastrevid?: number; extract?: string; images?: { title: string }[]; imageinfo?: ImageInfo[]; revisions?: { revid?: number; slots?: { main?: { content?: string } } }[]; categorymembers?: { title: string }[]; }; export type ImageInfo = { width: number; height: number; mime: string; thumburl?: string; descriptionurl?: string; extmetadata?: Record; }; export type MwResponse = { query?: { pages?: MwPage[]; normalized?: { from: string; to: string }[]; redirects?: { from: string; to: string }[]; categorymembers?: { title: string }[]; }; continue?: Record; }; let lastSparql = 0; export async function sparql>(query: string): Promise { const wait = lastSparql + 400 - Date.now(); if (wait > 0) await sleep(wait); lastSparql = Date.now(); const res = await getJson<{ results: { bindings: Record[] } }>( 'https://query.wikidata.org/sparql', { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded', accept: 'application/sparql-results+json' }, body: new URLSearchParams({ query }).toString() } ); return res.results.bindings.map( (b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T ); } export const qid = (uri: string) => uri.split('/').at(-1)!; export const fileName = (uri: string) => decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' '); /** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */ export const cmp = (s: string) => s .normalize('NFD') .replace(/[\u0300-\u036f]/g, '') .toUpperCase() .replace(/&/g, ' AND ') .replace(/[^A-Z0-9]/g, ''); export const tokens = (s: string) => s .normalize('NFD') .replace(/[\u0300-\u036f]/g, '') .toUpperCase() .replace(/&/g, ' ') .split(/[^A-Z0-9]+/) .filter(Boolean); export const slugify = (s: string) => s .normalize('NFD') .replace(/[\u0300-\u036f]/g, '') .toLowerCase() .replace(/&/g, ' and ') .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') .slice(0, 80) || 'x'; export const chunks = (xs: T[], n: number) => Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n)); export const stripHtml = (s: string) => s .replace(/<[^>]+>/g, ' ') .replace(/&/g, '&') .replace(/"/g, '"') .replace(/�?39;/g, "'") .replace(/ /g, ' ') .replace(/\s+/g, ' ') .trim();