website/scripts/music-import-lastfm.ts

// Backfill the full Last.fm history of a user into music.plays (source 'lastfm').
//
//   LASTFM_API_KEY=... bun scripts/music-import-lastfm.ts [--user stanrunge] [--sleep 300]
//                                                        [--max-pages N] [--reset]
//
// Walks user.getRecentTracks newest → oldest in 200-scrobble pages, always asking for page 1
// of a shrinking [from, to] window, so new scrobbles never shift the pages under it.
// Progress is saved after every page in music.import_state ('lastfm:<user>'): stop it any
// time and re-run to resume; once complete, a re-run only fetches scrobbles newer than the
// last completed run. Overlap with Spotify plays is hidden by the music.plays_deduped view.
// Rate limit: one request per --sleep ms (default 300, Last.fm asks for at most 5/s),
// exponential back-off on 429/5xx and Last.fm errors 8, 11, 16 and 29.
import { parseRecentTracks } from '../src/lib/music/lastfm-parse';
import {
	getImportState,
	insertPlay,
	newCache,
	setImportState,
	upsertTrack
} from '../src/lib/music/store.server';
import { arg, musicDb, sleep } from './music-db';

type State = {
	completeUntil: number;
	walk: { from: number; to: number; top: number } | null;
	imported: number;
	seen: number;
};

const apiKey = process.env.LASTFM_API_KEY;
if (!apiKey) throw new Error('Set LASTFM_API_KEY (https://www.last.fm/api/account/create)');
const user = arg('user') ?? 'stanrunge';
const pause = Number(arg('sleep') ?? 300);
const maxPages = Number(arg('max-pages') ?? Infinity);
const key = `lastfm:${user.toLowerCase()}`;
const RETRYABLE = new Set([8, 11, 16, 29]);

async function fetchPage(from: number, to: number) {
	const url = `https://ws.audioscrobbler.com/2.0/?${new URLSearchParams({
		method: 'user.getrecenttracks',
		user,
		api_key: apiKey!,
		format: 'json',
		limit: '200',
		page: '1',
		from: String(from),
		to: String(to)
	})}`;
	for (let attempt = 0; ; attempt++) {
		let retry: boolean;
		let message: string;
		try {
			const response = await fetch(url, {
				headers: { 'User-Agent': 'stanrunge.dev music importer (stan@stanrunge.dev)' },
				signal: AbortSignal.timeout(30_000)
			});
			const body = (await response.json().catch(() => null)) as Record<string, unknown> | null;
			if (response.ok && body && !('error' in body)) return body;
			const code = Number(body?.error);
			message = `HTTP ${response.status}${body?.message ? `: ${String(body.message)}` : ''}`;
			retry = response.status === 429 || response.status >= 500 || RETRYABLE.has(code);
		} catch (e) {
			message = e instanceof Error ? e.message : String(e);
			retry = true;
		}
		if (!retry || attempt >= 6) throw new Error(`Last.fm request failed: ${message}`);
		const wait = 2 ** attempt * 1000;
		console.warn(`  ${message}; retrying in ${wait / 1000}s`);
		await sleep(wait);
	}
}

const sql = musicDb();
try {
	let state: State = (process.argv.includes('--reset')
		? null
		: await getImportState<State>(sql, key)) ?? {
		completeUntil: 0,
		walk: null,
		imported: 0,
		seen: 0
	};
	if (!state.walk) {
		const now = Math.floor(Date.now() / 1000);
		state = { ...state, walk: { from: state.completeUntil + 1, to: now, top: now } };
	}
	const cache = newCache();
	let pages = 0;
	console.log(
		`Last.fm ${user}: importing ${new Date(state.walk!.from * 1000).toISOString()} → ${new Date(state.walk!.to * 1000).toISOString()}`
	);
	while (state.walk && pages < maxPages) {
		const walk = state.walk;
		const page =
			walk.from > walk.to ? null : parseRecentTracks(await fetchPage(walk.from, walk.to));
		pages++;
		if (!page || page.plays.length === 0) {
			state = { ...state, completeUntil: walk.top, walk: null };
			await setImportState(sql, key, state);
			console.log(`Done: ${state.imported} new plays stored (${state.seen} scrobbles seen).`);
			break;
		}
		let stored = 0;
		for (const play of page.plays) {
			const trackId = await upsertTrack(sql, play.track, false, cache);
			if (
				await insertPlay(sql, {
					trackId,
					playedAt: play.playedAt,
					source: 'lastfm',
					origin: 'import'
				})
			)
				stored++;
		}
		const oldest = Math.min(...page.plays.map((p) => p.uts));
		state = {
			...state,
			imported: state.imported + stored,
			seen: state.seen + page.plays.length,
			walk: { ...walk, to: oldest >= walk.to ? walk.to - 1 : oldest }
		};
		await setImportState(sql, key, state);
		console.log(
			`  page ${pages}: ${page.plays.length} scrobbles, ${stored} new, back to ${new Date(oldest * 1000).toISOString()} (~${page.total} left)`
		);
		await sleep(pause);
	}
	if (state.walk) console.log(`Stopped after ${pages} pages; re-run to resume.`);
} finally {
	await sql.end();
}