// Backfill the full Last.fm history of a user into music.plays (source 'lastfm'). // // LASTFM_API_KEY=... bun scripts/music-import-lastfm.ts [--user stanrunge] [--sleep 300] // [--max-pages N] [--reset] // // Walks user.getRecentTracks newest → oldest in 200-scrobble pages, always asking for page 1 // of a shrinking [from, to] window, so new scrobbles never shift the pages under it. // Progress is saved after every page in music.import_state ('lastfm:'): stop it any // time and re-run to resume; once complete, a re-run only fetches scrobbles newer than the // last completed run. Overlap with Spotify plays is hidden by the music.plays_deduped view. // Rate limit: one request per --sleep ms (default 300, Last.fm asks for at most 5/s), // exponential back-off on 429/5xx and Last.fm errors 8, 11, 16 and 29. import { parseRecentTracks } from '../src/lib/music/lastfm-parse'; import { getImportState, insertPlay, newCache, setImportState, upsertTrack } from '../src/lib/music/store.server'; import { arg, musicDb, sleep } from './music-db'; type State = { completeUntil: number; walk: { from: number; to: number; top: number } | null; imported: number; seen: number; }; const apiKey = process.env.LASTFM_API_KEY; if (!apiKey) throw new Error('Set LASTFM_API_KEY (https://www.last.fm/api/account/create)'); const user = arg('user') ?? 'stanrunge'; const pause = Number(arg('sleep') ?? 300); const maxPages = Number(arg('max-pages') ?? Infinity); const key = `lastfm:${user.toLowerCase()}`; const RETRYABLE = new Set([8, 11, 16, 29]); async function fetchPage(from: number, to: number) { const url = `https://ws.audioscrobbler.com/2.0/?${new URLSearchParams({ method: 'user.getrecenttracks', user, api_key: apiKey!, format: 'json', limit: '200', page: '1', from: String(from), to: String(to) })}`; for (let attempt = 0; ; attempt++) { let retry: boolean; let message: string; try { const response = await fetch(url, { headers: { 'User-Agent': 'stanrunge.dev music importer (stan@stanrunge.dev)' }, signal: AbortSignal.timeout(30_000) }); const body = (await response.json().catch(() => null)) as Record | null; if (response.ok && body && !('error' in body)) return body; const code = Number(body?.error); message = `HTTP ${response.status}${body?.message ? `: ${String(body.message)}` : ''}`; retry = response.status === 429 || response.status >= 500 || RETRYABLE.has(code); } catch (e) { message = e instanceof Error ? e.message : String(e); retry = true; } if (!retry || attempt >= 6) throw new Error(`Last.fm request failed: ${message}`); const wait = 2 ** attempt * 1000; console.warn(` ${message}; retrying in ${wait / 1000}s`); await sleep(wait); } } const sql = musicDb(); try { let state: State = (process.argv.includes('--reset') ? null : await getImportState(sql, key)) ?? { completeUntil: 0, walk: null, imported: 0, seen: 0 }; if (!state.walk) { const now = Math.floor(Date.now() / 1000); state = { ...state, walk: { from: state.completeUntil + 1, to: now, top: now } }; } const cache = newCache(); let pages = 0; console.log( `Last.fm ${user}: importing ${new Date(state.walk!.from * 1000).toISOString()} → ${new Date(state.walk!.to * 1000).toISOString()}` ); while (state.walk && pages < maxPages) { const walk = state.walk; const page = walk.from > walk.to ? null : parseRecentTracks(await fetchPage(walk.from, walk.to)); pages++; if (!page || page.plays.length === 0) { state = { ...state, completeUntil: walk.top, walk: null }; await setImportState(sql, key, state); console.log(`Done: ${state.imported} new plays stored (${state.seen} scrobbles seen).`); break; } let stored = 0; for (const play of page.plays) { const trackId = await upsertTrack(sql, play.track, false, cache); if ( await insertPlay(sql, { trackId, playedAt: play.playedAt, source: 'lastfm', origin: 'import' }) ) stored++; } const oldest = Math.min(...page.plays.map((p) => p.uts)); state = { ...state, imported: state.imported + stored, seen: state.seen + page.plays.length, walk: { ...walk, to: oldest >= walk.to ? walk.to - 1 : oldest } }; await setImportState(sql, key, state); console.log( ` page ${pages}: ${page.plays.length} scrobbles, ${stored} new, back to ${new Date(oldest * 1000).toISOString()} (~${page.total} left)` ); await sleep(pause); } if (state.walk) console.log(`Stopped after ${pages} pages; re-run to resume.`); } finally { await sql.end(); }