// Shared HTTP helpers for the plane pokedex scripts (scripts/planes-import.ts,
// scripts/planes-photos.ts): polite Wikidata SPARQL + MediaWiki API access with retries,
// maxlag handling and an optional on-disk cache (--cache
).
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
export const UA =
'StanRungePlaneDex/1.0 (https://stanrunge.dev/trackers/planes; plane pokedex importer) bun';
export const log = (...a: unknown[]) => console.error(...a);
export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
let cacheDir: string | null = null;
let cache: Record = {};
let dirty = 0;
let cacheName = 'api.json';
/** Use `/` as a persistent response cache (only for dev runs and refreshes). */
export async function useCache(dir: string | null, name = 'api.json') {
cacheDir = dir;
cacheName = name;
if (!dir) return;
await mkdir(dir, { recursive: true });
try {
cache = JSON.parse(await readFile(join(dir, name), 'utf8'));
} catch {
cache = {};
}
}
export async function saveCache(force = false) {
if (!cacheDir || (!force && dirty < 200)) return;
dirty = 0;
await writeFile(join(cacheDir, cacheName), JSON.stringify(cache));
}
export async function getJson(url: string, init?: RequestInit, useCached = true): Promise {
const key = `${url}${init?.body ? `#${String(init.body)}` : ''}`;
if (useCached && key in cache) return cache[key] as T;
for (let attempt = 0; ; attempt++) {
let res: Response;
try {
res = await fetch(url, {
...init,
headers: {
'user-agent': UA,
'api-user-agent': UA,
accept: 'application/json',
...init?.headers
},
signal: AbortSignal.timeout(120_000)
});
} catch (e) {
if (attempt < 6) {
await sleep(2000 * 2 ** attempt);
continue;
}
throw e;
}
const retryAfter = Number(res.headers.get('retry-after')) || 0;
if (res.ok) {
const body = (await res.json()) as T & { error?: { code?: string } };
if (body?.error?.code === 'maxlag' && attempt < 8) {
await sleep(Math.max(retryAfter, 5) * 1000);
continue;
}
if (useCached && cacheDir) {
cache[key] = body;
dirty++;
await saveCache();
}
return body;
}
if ((res.status === 429 || res.status >= 500) && attempt < 6) {
await sleep(Math.max(retryAfter * 1000, 3000 * 2 ** attempt));
continue;
}
throw new Error(`${res.status} ${url.slice(0, 200)}`);
}
}
let lastCall = 0;
/** MediaWiki action API, one request at a time with a small gap (API etiquette). */
export async function mw(
host: string,
params: Record,
useCached = true
): Promise {
const body = new URLSearchParams({ format: 'json', formatversion: '2', maxlag: '5', ...params });
const key = `https://${host}/w/api.php?${body}`;
if (!useCached || !(key in cache)) {
const wait = lastCall + 150 - Date.now();
if (wait > 0) await sleep(wait);
lastCall = Date.now();
}
return getJson(key, undefined, useCached);
}
export type MwPage = {
pageid?: number;
title: string;
missing?: boolean;
lastrevid?: number;
extract?: string;
images?: { title: string }[];
imageinfo?: ImageInfo[];
revisions?: { revid?: number; slots?: { main?: { content?: string } } }[];
categorymembers?: { title: string }[];
};
export type ImageInfo = {
width: number;
height: number;
mime: string;
thumburl?: string;
descriptionurl?: string;
extmetadata?: Record;
};
export type MwResponse = {
query?: {
pages?: MwPage[];
normalized?: { from: string; to: string }[];
redirects?: { from: string; to: string }[];
categorymembers?: { title: string }[];
};
continue?: Record;
};
let lastSparql = 0;
export async function sparql>(query: string): Promise {
const wait = lastSparql + 400 - Date.now();
if (wait > 0) await sleep(wait);
lastSparql = Date.now();
const res = await getJson<{ results: { bindings: Record[] } }>(
'https://query.wikidata.org/sparql',
{
method: 'POST',
headers: {
'content-type': 'application/x-www-form-urlencoded',
accept: 'application/sparql-results+json'
},
body: new URLSearchParams({ query }).toString()
}
);
return res.results.bindings.map(
(b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value])) as T
);
}
export const qid = (uri: string) => uri.split('/').at(-1)!;
export const fileName = (uri: string) =>
decodeURIComponent(uri.split('/Special:FilePath/').at(-1)!).replace(/_/g, ' ');
/** Uppercase letters and digits only: "Boeing 737-800" -> "BOEING737800". */
export const cmp = (s: string) =>
s
.normalize('NFD')
.replace(/[\u0300-\u036f]/g, '')
.toUpperCase()
.replace(/&/g, ' AND ')
.replace(/[^A-Z0-9]/g, '');
export const tokens = (s: string) =>
s
.normalize('NFD')
.replace(/[\u0300-\u036f]/g, '')
.toUpperCase()
.replace(/&/g, ' ')
.split(/[^A-Z0-9]+/)
.filter(Boolean);
export const slugify = (s: string) =>
s
.normalize('NFD')
.replace(/[\u0300-\u036f]/g, '')
.toLowerCase()
.replace(/&/g, ' and ')
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80) || 'x';
export const chunks = (xs: T[], n: number) =>
Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n));
export const stripHtml = (s: string) =>
s
.replace(/<[^>]+>/g, ' ')
.replace(/&/g, '&')
.replace(/"/g, '"')
.replace(/?39;/g, "'")
.replace(/ /g, ' ')
.replace(/\s+/g, ' ')
.trim();