From 2949d251eea2cda793215e44f078ee50ddd353d9 Mon Sep 17 00:00:00 2001 From: Stan Runge Date: Sun, 1 Mar 2026 17:21:02 +0000 Subject: [PATCH] feat: backfill mappool slot metadata and user/team avatars to R2, improving caching and reducing API calls --- scripts/backfill-r2.ts | 226 +++++++++++++----- src/lib/server/db/schema.ts | 13 +- .../(app)/mappools/[id]/+page.server.ts | 58 ++++- src/routes/(app)/matches/[id]/+page.server.ts | 41 +++- 4 files changed, 267 insertions(+), 71 deletions(-) diff --git a/scripts/backfill-r2.ts b/scripts/backfill-r2.ts index 0099c8c..5a3cce1 100644 --- a/scripts/backfill-r2.ts +++ b/scripts/backfill-r2.ts @@ -1,13 +1,20 @@ /** - * One-time script to upload existing user avatars and beatmap covers to R2. - * Run with: bun run scripts/backfill-r2.ts + * Backfill ALL images into R2: + * 1. Mappool slot covers (title/artist/version/coverUrl/listCoverUrl) + * 2. User avatars still pointing at osu! servers + * 3. Team avatars still pointing at osu! servers + * + * Run with: bun run scripts/backfill-r2-all.ts * Reads credentials from .env automatically. + * + * After running, do `bun run db:push` to add the new schema columns, + * then run this script to populate them. */ import postgres from 'postgres'; import { S3Client } from 'bun'; -// ── Config from env ──────────────────────────────────────────────────── +// ── Config ───────────────────────────────────────────────────────────── const R2_ACCOUNT_ID = process.env.R2_ACCOUNT_ID!; const R2_ACCESS_KEY_ID = process.env.R2_ACCESS_KEY_ID!; @@ -18,11 +25,14 @@ const DATABASE_URL = process.env.DATABASE_URL!; const OSU_CLIENT_ID = process.env.OSU_CLIENT_ID!; const OSU_CLIENT_SECRET = process.env.OSU_CLIENT_SECRET!; -for (const [k, v] of Object.entries({ R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, R2_BUCKET, R2_PUBLIC_URL, DATABASE_URL, OSU_CLIENT_ID, OSU_CLIENT_SECRET })) { +for (const [k, v] of Object.entries({ + R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, + R2_BUCKET, R2_PUBLIC_URL, DATABASE_URL, OSU_CLIENT_ID, OSU_CLIENT_SECRET +})) { if (!v) { console.error(`Missing env var: ${k}`); process.exit(1); } } -// ── R2 client ────────────────────────────────────────────────────────── +// ── R2 ───────────────────────────────────────────────────────────────── const r2 = new S3Client({ accessKeyId: R2_ACCESS_KEY_ID, @@ -38,20 +48,32 @@ function urlToKey(url: string): string { return `misc${u.pathname}`; } -async function uploadIfMissing(url: string): Promise<'uploaded' | 'exists' | 'failed'> { - const key = urlToKey(url); +async function proxyToR2(osuUrl: string): Promise { + const key = urlToKey(osuUrl); + const cdnUrl = `${R2_PUBLIC_URL}/${key}`; try { const file = r2.file(key); - if (await file.exists()) return 'exists'; + if (await file.exists()) return cdnUrl; - const res = await fetch(url); + const res = await fetch(osuUrl); if (!res.ok) throw new Error(`HTTP ${res.status}`); const contentType = res.headers.get('content-type') ?? 'image/jpeg'; await file.write(res, { type: contentType }); - return 'uploaded'; + console.log(` ↑ uploaded ${key}`); + return cdnUrl; } catch (err: unknown) { - console.error(` ✗ ${key} — ${err instanceof Error ? err.message : err}`); - return 'failed'; + console.error(` ✗ failed ${key}: ${err instanceof Error ? err.message : err}`); + return osuUrl; // Return original on failure + } +} + +function isOsuUrl(url: string | null): boolean { + if (!url) return false; + try { + const u = new URL(url); + return u.hostname.endsWith('.ppy.sh'); + } catch { + return false; } } @@ -81,18 +103,26 @@ async function getOsuToken(): Promise { return osuToken; } -async function getBeatmapCovers(beatmapId: string): Promise<{ card: string; list: string } | null> { +interface BeatmapData { + beatmapset: { + title: string; + artist: string; + covers: Record; + }; + version: string; + difficulty_rating: number; + bpm: number; + total_length: number; +} + +async function getBeatmap(beatmapId: string): Promise { try { const token = await getOsuToken(); const res = await fetch(`https://osu.ppy.sh/api/v2/beatmaps/${beatmapId}`, { headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' } }); if (!res.ok) return null; - const data = await res.json() as { beatmapset: { covers: Record } }; - return { - card: data.beatmapset.covers['card@2x'], - list: data.beatmapset.covers['list@2x'] - }; + return await res.json() as BeatmapData; } catch { return null; } @@ -102,57 +132,135 @@ async function getBeatmapCovers(beatmapId: string): Promise<{ card: string; list const sql = postgres(DATABASE_URL); -console.log('── Backfilling user avatars ─────────────────────────────'); +// ════════════════════════════════════════════════════════════════════════ +// 1. MAPPOOL SLOTS — backfill title/artist/version/coverUrl/listCoverUrl +// ════════════════════════════════════════════════════════════════════════ -const users = await sql<{ image: string }[]>` - SELECT image FROM "user" WHERE image IS NOT NULL AND image != '' +console.log('═══ 1/3: Backfilling mappool slot metadata + covers ═══\n'); + +const slots = await sql<{ id: string; beatmap_id: string; cover_url: string | null; title: string | null }[]>` + SELECT id, beatmap_id, cover_url, title FROM mappool_slot + WHERE cover_url IS NULL OR title IS NULL `; -let uploaded = 0, skipped = 0, failed = 0; -for (const { image } of users) { - if (!image.includes('ppy.sh') && !image.includes('a.ppy.sh')) { - // Non-osu avatar, skip for now - continue; - } - const result = await uploadIfMissing(image); - if (result === 'uploaded') { uploaded++; console.log(` ↑ ${urlToKey(image)}`); } - else if (result === 'exists') skipped++; - else failed++; -} -console.log(`Done: ${uploaded} uploaded, ${skipped} already existed, ${failed} failed\n`); +console.log(`Found ${slots.length} slots missing metadata\n`); -console.log('── Backfilling beatmap covers ────────────────────────────'); - -const slots = await sql<{ beatmap_id: string }[]>` - SELECT DISTINCT beatmap_id FROM mappool_slot -`; - -console.log(`Found ${slots.length} unique beatmaps\n`); - -uploaded = 0; skipped = 0; failed = 0; -for (const { beatmap_id } of slots) { - process.stdout.write(` beatmap ${beatmap_id} ... `); - const covers = await getBeatmapCovers(beatmap_id); - if (!covers) { +let slotOk = 0, slotFail = 0; +for (const slot of slots) { + process.stdout.write(` beatmap ${slot.beatmap_id} ... `); + const bm = await getBeatmap(slot.beatmap_id); + if (!bm) { console.log('✗ not found on osu!'); - failed++; + slotFail++; continue; } - let ok = true; - for (const url of [covers.card, covers.list]) { - if (!url) continue; - const result = await uploadIfMissing(url); - if (result === 'failed') ok = false; - if (result === 'uploaded') uploaded++; - if (result === 'exists') skipped++; - } - console.log(ok ? '✓' : '✗ partial'); + const cardUrl = bm.beatmapset.covers['card@2x']; + const listUrl = bm.beatmapset.covers['list@2x']; - // Be polite to osu! API — 10 req/s max - await sleep(120); + const coverUrl = cardUrl ? await proxyToR2(cardUrl) : null; + const listCoverUrl = listUrl ? await proxyToR2(listUrl) : null; + + await sql` + UPDATE mappool_slot SET + title = ${bm.beatmapset.title}, + artist = ${bm.beatmapset.artist}, + version = ${bm.version}, + star_rating = ${bm.difficulty_rating}, + bpm = ${bm.bpm}, + total_length = ${bm.total_length}, + cover_url = ${coverUrl}, + list_cover_url = ${listCoverUrl} + WHERE id = ${slot.id} + `; + + console.log('✓'); + slotOk++; + await sleep(120); // Rate limit osu! API } -console.log(`\nDone: ${uploaded} uploaded, ${skipped} already existed, ${failed} failed`); +console.log(`\nSlots: ${slotOk} updated, ${slotFail} failed\n`); + +// Also update any slots that have osu! URLs instead of R2 URLs (partially migrated) +const osuUrlSlots = await sql<{ id: string; cover_url: string | null; list_cover_url: string | null }[]>` + SELECT id, cover_url, list_cover_url FROM mappool_slot + WHERE (cover_url LIKE '%ppy.sh%' OR list_cover_url LIKE '%ppy.sh%') +`; + +if (osuUrlSlots.length > 0) { + console.log(`Found ${osuUrlSlots.length} slots with osu! URLs (need R2 proxy)\n`); + for (const slot of osuUrlSlots) { + const newCover = slot.cover_url && isOsuUrl(slot.cover_url) ? await proxyToR2(slot.cover_url) : slot.cover_url; + const newList = slot.list_cover_url && isOsuUrl(slot.list_cover_url) ? await proxyToR2(slot.list_cover_url) : slot.list_cover_url; + await sql`UPDATE mappool_slot SET cover_url = ${newCover}, list_cover_url = ${newList} WHERE id = ${slot.id}`; + } + console.log('Done re-proxying slot covers\n'); +} + +// ════════════════════════════════════════════════════════════════════════ +// 2. USER AVATARS — re-proxy any still pointing at ppy.sh +// ════════════════════════════════════════════════════════════════════════ + +console.log('═══ 2/3: Backfilling user avatars ═══\n'); + +const users = await sql<{ id: string; image: string | null }[]>` + SELECT id, image FROM "user" WHERE image IS NOT NULL +`; + +let avatarOk = 0, avatarSkip = 0; +for (const u of users) { + if (!isOsuUrl(u.image)) { + avatarSkip++; + continue; + } + process.stdout.write(` user ${u.id} (${u.image}) ... `); + const cdnUrl = await proxyToR2(u.image!); + if (cdnUrl !== u.image) { + await sql`UPDATE "user" SET image = ${cdnUrl} WHERE id = ${u.id}`; + console.log('✓ updated'); + avatarOk++; + } else { + console.log('— unchanged (proxy failed)'); + } +} + +console.log(`\nAvatars: ${avatarOk} updated, ${avatarSkip} already R2\n`); + +// ════════════════════════════════════════════════════════════════════════ +// 3. TEAM AVATARS — re-proxy any still pointing at ppy.sh +// ════════════════════════════════════════════════════════════════════════ + +console.log('═══ 3/3: Backfilling team avatars ═══\n'); + +const teams = await sql<{ id: string; avatar_url: string | null }[]>` + SELECT id, avatar_url FROM team WHERE avatar_url IS NOT NULL +`; + +let teamOk = 0, teamSkip = 0; +for (const t of teams) { + if (!isOsuUrl(t.avatar_url)) { + teamSkip++; + continue; + } + process.stdout.write(` team ${t.id} ... `); + const cdnUrl = await proxyToR2(t.avatar_url!); + if (cdnUrl !== t.avatar_url) { + await sql`UPDATE team SET avatar_url = ${cdnUrl} WHERE id = ${t.id}`; + console.log('✓ updated'); + teamOk++; + } else { + console.log('— unchanged'); + } +} + +console.log(`\nTeams: ${teamOk} updated, ${teamSkip} already R2\n`); + +// ════════════════════════════════════════════════════════════════════════ + +console.log('═══ All done! ═══'); +console.log('Summary:'); +console.log(` Mappool slots: ${slotOk} backfilled, ${slotFail} failed`); +console.log(` User avatars: ${avatarOk} re-proxied, ${avatarSkip} already R2`); +console.log(` Team avatars: ${teamOk} re-proxied, ${teamSkip} already R2`); await sql.end(); diff --git a/src/lib/server/db/schema.ts b/src/lib/server/db/schema.ts index ff6ba5a..b24b06b 100644 --- a/src/lib/server/db/schema.ts +++ b/src/lib/server/db/schema.ts @@ -1,8 +1,3 @@ -// src/lib/server/db/schema.ts -// -// CHANGE: Add `role` column to user table in auth.schema.ts (see that file). -// This file adds the permission helper utilities. - import { pgTable, integer, @@ -60,7 +55,13 @@ export const mappoolSlot = pgTable('mappool_slot', { starRating: real('star_rating'), bpm: real('bpm'), totalLength: integer('total_length'), - mods: text('mods').array().default([]).notNull() + mods: text('mods').array().default([]).notNull(), + // ── Cached beatmap metadata (stored at insert time → zero API calls on page load) ── + title: text('title'), + artist: text('artist'), + version: text('version'), + coverUrl: text('cover_url'), + listCoverUrl: text('list_cover_url') }); // ── Matches ───────────────────────────────────────────────────────────── diff --git a/src/routes/(app)/mappools/[id]/+page.server.ts b/src/routes/(app)/mappools/[id]/+page.server.ts index 95662b5..bd90916 100644 --- a/src/routes/(app)/mappools/[id]/+page.server.ts +++ b/src/routes/(app)/mappools/[id]/+page.server.ts @@ -21,11 +21,47 @@ export const load: PageServerLoad = async ({ params, locals }) => { if (!pool) error(404, 'Mappool not found'); - // Fetch beatmap metadata for all slots + // Use cached metadata from DB — only fall back to osu! API for legacy slots const slotsWithBeatmaps = await Promise.all( pool.slots.map(async (slot) => { + // Fast path: metadata already stored in DB (R2 URLs) + if (slot.title && slot.coverUrl) { + return { + ...slot, + beatmap: { + title: slot.title, + artist: slot.artist ?? '', + version: slot.version ?? '', + starRating: slot.starRating ?? 0, + bpm: slot.bpm ?? 0, + totalLength: slot.totalLength ?? 0, + coverUrl: slot.coverUrl, + url: `https://osu.ppy.sh/beatmaps/${slot.beatmapId}` + } + }; + } + + // Slow path: legacy slot without cached metadata — fetch + backfill try { const beatmap = await getBeatmap(slot.beatmapId); + const coverUrl = await proxyImage(beatmap.beatmapset.covers['card@2x']); + const listCoverUrl = await proxyImage(beatmap.beatmapset.covers['list@2x']); + + // Backfill the slot so future loads are instant + await db + .update(mappoolSlot) + .set({ + title: beatmap.beatmapset.title, + artist: beatmap.beatmapset.artist, + version: beatmap.version, + coverUrl, + listCoverUrl, + starRating: beatmap.difficulty_rating, + bpm: beatmap.bpm, + totalLength: beatmap.total_length + }) + .where(eq(mappoolSlot.id, slot.id)); + return { ...slot, beatmap: { @@ -35,7 +71,7 @@ export const load: PageServerLoad = async ({ params, locals }) => { starRating: beatmap.difficulty_rating, bpm: beatmap.bpm, totalLength: beatmap.total_length, - coverUrl: await proxyImage(beatmap.beatmapset.covers['card@2x']), + coverUrl, url: beatmap.url } }; @@ -105,6 +141,16 @@ export const actions: Actions = { where: and(eq(mappoolSlot.mappoolId, params.id), eq(mappoolSlot.category, category)) }); + // Proxy covers to R2 at insert time — pages never hit osu! again + let coverUrl: string | null = null; + let listCoverUrl: string | null = null; + try { + coverUrl = await proxyImage(beatmap.beatmapset.covers['card@2x']); + listCoverUrl = await proxyImage(beatmap.beatmapset.covers['list@2x']); + } catch (err: any) { + console.warn('[Mappool] Failed to proxy covers to R2:', err.message); + } + await db.insert(mappoolSlot).values({ mappoolId: params.id, beatmapId, @@ -113,7 +159,13 @@ export const actions: Actions = { starRating: beatmap.difficulty_rating, bpm: beatmap.bpm, totalLength: beatmap.total_length, - mods: category === 'NM' || category === 'TB' || category === 'FM' ? [] : [category] + mods: category === 'NM' || category === 'TB' || category === 'FM' ? [] : [category], + // Cached metadata — all R2 URLs + title: beatmap.beatmapset.title, + artist: beatmap.beatmapset.artist, + version: beatmap.version, + coverUrl, + listCoverUrl }); return { success: true }; diff --git a/src/routes/(app)/matches/[id]/+page.server.ts b/src/routes/(app)/matches/[id]/+page.server.ts index 5676147..ce0d6ee 100644 --- a/src/routes/(app)/matches/[id]/+page.server.ts +++ b/src/routes/(app)/matches/[id]/+page.server.ts @@ -1,5 +1,6 @@ import { user } from '$lib/server/db/schema'; import { db } from '$lib/server/db'; +import { mappoolSlot } from '$lib/server/db/schema'; import { getMatchFull, submitRoll, pickMap, cancelMatch } from '$lib/server/match/engine'; import { playPickedMap, closeLobby, forceStartGame } from '$lib/server/match/orchestrator'; import { getLobby, getLobbyStatus } from '$lib/server/bancho/client'; @@ -22,13 +23,32 @@ export const load: PageServerLoad = async ({ params, locals }) => { error(404, 'Match not found'); } - // Build beatmap metadata cache for all slots in the mappool + // Build beatmap metadata cache — prefer DB-stored R2 URLs, fall back to API const beatmapCache: Record = {}; if (m.mappool?.slots) { await Promise.allSettled( m.mappool.slots.map(async (slot) => { + // Fast path: metadata already stored in DB + if (slot.title && slot.coverUrl) { + beatmapCache[slot.beatmapId] = { + title: slot.title, + artist: slot.artist ?? '', + version: slot.version ?? '', + starRating: slot.starRating ?? 0, + bpm: slot.bpm ?? 0, + totalLength: slot.totalLength ?? 0, + coverUrl: slot.coverUrl, + listCoverUrl: slot.listCoverUrl + }; + return; + } + + // Slow path: legacy slot — fetch from API + backfill DB try { const bm = await getBeatmap(slot.beatmapId); + const coverUrl = await proxyImage(bm.beatmapset.covers['card@2x']); + const listCoverUrl = await proxyImage(bm.beatmapset.covers['list@2x']); + beatmapCache[slot.beatmapId] = { title: bm.beatmapset.title, artist: bm.beatmapset.artist, @@ -36,9 +56,24 @@ export const load: PageServerLoad = async ({ params, locals }) => { starRating: bm.difficulty_rating, bpm: bm.bpm, totalLength: bm.total_length, - coverUrl: await proxyImage(bm.beatmapset.covers['card@2x']), - listCoverUrl: await proxyImage(bm.beatmapset.covers['list@2x']) + coverUrl, + listCoverUrl }; + + // Backfill so future loads are instant + await db + .update(mappoolSlot) + .set({ + title: bm.beatmapset.title, + artist: bm.beatmapset.artist, + version: bm.version, + coverUrl, + listCoverUrl, + starRating: bm.difficulty_rating, + bpm: bm.bpm, + totalLength: bm.total_length + }) + .where(eq(mappoolSlot.id, slot.id)); } catch { /* skip */ } }) );