music: self-hosted scrobbler (Spotify poller, Last.fm + streaming-history importers, /trackers/music, homepage listening card)

- music schema: artists/albums/tracks (+album art), plays with source + dedupe, now_playing,
  spotify_auth, import_state; plays_deduped view hides Last.fm scrobbles Spotify also has
- in-app Spotify poller (60s, leased) started from hooks init; no new role or env needed
- /api/spotify/connect + callback (owner only), /trackers/music public stats
- scripts: music-poll, music-import-lastfm (resumable), music-import-history
- deploy/music-collector: optional host timer with its own role

scripts/music-db.ts

@@ -0,0 +1,16 @@
+// Shared database connection for the music scripts: MUSIC_DB_URL (e.g. the optional
+// music_collector role) or DB_URL (the app role, as in the image's prod.env).
+import postgres from 'postgres';
+
+export function musicDb() {
+	const url = process.env.MUSIC_DB_URL || process.env.DB_URL;
+	if (!url) throw new Error('Set MUSIC_DB_URL or DB_URL');
+	return postgres(url, { max: 1, prepare: false, connect_timeout: 10, onnotice: () => {} });
+}
+
+export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export function arg(name: string): string | undefined {
+	const i = process.argv.indexOf(`--${name}`);
+	return i >= 0 ? process.argv[i + 1] : undefined;
+}

scripts/music-import-history.ts

@@ -0,0 +1,69 @@
+// Import a Spotify data export into music.plays (source 'spotify', origin 'export').
+//
+//   bun scripts/music-import-history.ts <file-or-dir> [...] [--min-ms 30000] [--dry-run]
+//
+// Accepts the Extended Streaming History (Streaming_History_Audio_*.json, older
+// endsong_*.json) and the short account-data export (StreamingHistory_music_*.json).
+// Podcasts/audiobooks and plays under --min-ms are skipped. Safe to re-run: a play already
+// stored (same artist+title within a minute, also from the live poller) is not added twice.
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { DEFAULT_MIN_MS, parseHistoryEntry, type HistoryPlay } from '../src/lib/music/history';
+import { insertPlay, newCache, upsertTrack } from '../src/lib/music/store.server';
+import { arg, musicDb } from './music-db';
+
+const minMs = Number(arg('min-ms') ?? DEFAULT_MIN_MS);
+const dryRun = process.argv.includes('--dry-run');
+const skip = new Set(['--min-ms', arg('min-ms')]);
+const inputs = process.argv.slice(2).filter((a) => !a.startsWith('--') && !skip.has(a));
+if (!inputs.length) throw new Error('Usage: bun scripts/music-import-history.ts <file-or-dir> ...');
+
+const NAME = /^(Streaming_History_Audio_.*|endsong_?\d*|StreamingHistory(_music)?_?\d*)\.json$/i;
+function files(path: string): string[] {
+	if (statSync(path).isDirectory())
+		return readdirSync(path).flatMap((name) => {
+			const child = join(path, name);
+			return statSync(child).isDirectory() ? files(child) : NAME.test(name) ? [child] : [];
+		});
+	return [path];
+}
+
+const plays: HistoryPlay[] = [];
+let entries = 0;
+for (const file of inputs.flatMap(files)) {
+	const data = JSON.parse(readFileSync(file, 'utf8')) as unknown[];
+	if (!Array.isArray(data)) continue;
+	entries += data.length;
+	for (const entry of data) {
+		const play = parseHistoryEntry(entry, minMs);
+		if (play) plays.push(play);
+	}
+	console.log(`${file}: ${data.length} entries`);
+}
+plays.sort((a, b) => a.playedAt.getTime() - b.playedAt.getTime());
+console.log(`${plays.length} music plays of ${minMs / 1000}s or more out of ${entries} entries.`);
+if (dryRun || !plays.length) process.exit(0);
+
+const sql = musicDb();
+try {
+	const cache = newCache();
+	let stored = 0;
+	for (const [i, play] of plays.entries()) {
+		const trackId = await upsertTrack(sql, play.track, false, cache);
+		if (
+			await insertPlay(sql, {
+				trackId,
+				playedAt: play.playedAt,
+				source: 'spotify',
+				origin: 'export',
+				msPlayed: play.msPlayed,
+				platform: play.platform
+			})
+		)
+			stored++;
+		if ((i + 1) % 5000 === 0) console.log(`  ${i + 1}/${plays.length} (${stored} new)`);
+	}
+	console.log(`Done: ${stored} new plays stored.`);
+} finally {
+	await sql.end();
+}

scripts/music-import-lastfm.ts

@@ -0,0 +1,132 @@
+// Backfill the full Last.fm history of a user into music.plays (source 'lastfm').
+//
+//   LASTFM_API_KEY=... bun scripts/music-import-lastfm.ts [--user stanrunge] [--sleep 300]
+//                                                        [--max-pages N] [--reset]
+//
+// Walks user.getRecentTracks newest → oldest in 200-scrobble pages, always asking for page 1
+// of a shrinking [from, to] window, so new scrobbles never shift the pages under it.
+// Progress is saved after every page in music.import_state ('lastfm:<user>'): stop it any
+// time and re-run to resume; once complete, a re-run only fetches scrobbles newer than the
+// last completed run. Overlap with Spotify plays is hidden by the music.plays_deduped view.
+// Rate limit: one request per --sleep ms (default 300, Last.fm asks for at most 5/s),
+// exponential back-off on 429/5xx and Last.fm errors 8, 11, 16 and 29.
+import { parseRecentTracks } from '../src/lib/music/lastfm-parse';
+import {
+	getImportState,
+	insertPlay,
+	newCache,
+	setImportState,
+	upsertTrack
+} from '../src/lib/music/store.server';
+import { arg, musicDb, sleep } from './music-db';
+
+type State = {
+	completeUntil: number;
+	walk: { from: number; to: number; top: number } | null;
+	imported: number;
+	seen: number;
+};
+
+const apiKey = process.env.LASTFM_API_KEY;
+if (!apiKey) throw new Error('Set LASTFM_API_KEY (https://www.last.fm/api/account/create)');
+const user = arg('user') ?? 'stanrunge';
+const pause = Number(arg('sleep') ?? 300);
+const maxPages = Number(arg('max-pages') ?? Infinity);
+const key = `lastfm:${user.toLowerCase()}`;
+const RETRYABLE = new Set([8, 11, 16, 29]);
+
+async function fetchPage(from: number, to: number) {
+	const url = `https://ws.audioscrobbler.com/2.0/?${new URLSearchParams({
+		method: 'user.getrecenttracks',
+		user,
+		api_key: apiKey!,
+		format: 'json',
+		limit: '200',
+		page: '1',
+		from: String(from),
+		to: String(to)
+	})}`;
+	for (let attempt = 0; ; attempt++) {
+		let retry: boolean;
+		let message: string;
+		try {
+			const response = await fetch(url, {
+				headers: { 'User-Agent': 'stanrunge.dev music importer (stan@stanrunge.dev)' },
+				signal: AbortSignal.timeout(30_000)
+			});
+			const body = (await response.json().catch(() => null)) as Record<string, unknown> | null;
+			if (response.ok && body && !('error' in body)) return body;
+			const code = Number(body?.error);
+			message = `HTTP ${response.status}${body?.message ? `: ${String(body.message)}` : ''}`;
+			retry = response.status === 429 || response.status >= 500 || RETRYABLE.has(code);
+		} catch (e) {
+			message = e instanceof Error ? e.message : String(e);
+			retry = true;
+		}
+		if (!retry || attempt >= 6) throw new Error(`Last.fm request failed: ${message}`);
+		const wait = 2 ** attempt * 1000;
+		console.warn(`  ${message}; retrying in ${wait / 1000}s`);
+		await sleep(wait);
+	}
+}
+
+const sql = musicDb();
+try {
+	let state: State = (process.argv.includes('--reset')
+		? null
+		: await getImportState<State>(sql, key)) ?? {
+		completeUntil: 0,
+		walk: null,
+		imported: 0,
+		seen: 0
+	};
+	if (!state.walk) {
+		const now = Math.floor(Date.now() / 1000);
+		state = { ...state, walk: { from: state.completeUntil + 1, to: now, top: now } };
+	}
+	const cache = newCache();
+	let pages = 0;
+	console.log(
+		`Last.fm ${user}: importing ${new Date(state.walk!.from * 1000).toISOString()} → ${new Date(state.walk!.to * 1000).toISOString()}`
+	);
+	while (state.walk && pages < maxPages) {
+		const walk = state.walk;
+		const page =
+			walk.from > walk.to ? null : parseRecentTracks(await fetchPage(walk.from, walk.to));
+		pages++;
+		if (!page || page.plays.length === 0) {
+			state = { ...state, completeUntil: walk.top, walk: null };
+			await setImportState(sql, key, state);
+			console.log(`Done: ${state.imported} new plays stored (${state.seen} scrobbles seen).`);
+			break;
+		}
+		let stored = 0;
+		for (const play of page.plays) {
+			const trackId = await upsertTrack(sql, play.track, false, cache);
+			if (
+				await insertPlay(sql, {
+					trackId,
+					playedAt: play.playedAt,
+					source: 'lastfm',
+					origin: 'import'
+				})
+			)
+				stored++;
+		}
+		const oldest = Math.min(...page.plays.map((p) => p.uts));
+		state = {
+			...state,
+			imported: state.imported + stored,
+			seen: state.seen + page.plays.length,
+			walk: { ...walk, to: oldest >= walk.to ? walk.to - 1 : oldest }
+		};
+		await setImportState(sql, key, state);
+		console.log(
+			`  page ${pages}: ${page.plays.length} scrobbles, ${stored} new, back to ${new Date(oldest * 1000).toISOString()} (~${page.total} left)`
+		);
+		await sleep(pause);
+	}
+	if (state.walk) console.log(`Stopped after ${pages} pages; re-run to resume.`);
+} finally {
+	await sql.end();
+}

scripts/music-poll.ts

@@ -0,0 +1,18 @@
+// One Spotify poll (now playing + recently played), for the optional host timer in
+// deploy/music-collector. The app normally does this itself every minute (see
+// src/lib/server/music/poller.ts); the lease in music.spotify_auth stops double polling.
+//   MUSIC_DB_URL=... SPOTIFY_CLIENT_ID=... SPOTIFY_CLIENT_SECRET=... bun scripts/music-poll.ts
+import { pollSpotify } from '../src/lib/music/collector.server';
+import { musicDb } from './music-db';
+
+const clientId = process.env.SPOTIFY_CLIENT_ID;
+const clientSecret = process.env.SPOTIFY_CLIENT_SECRET;
+if (!clientId || !clientSecret) throw new Error('Set SPOTIFY_CLIENT_ID and SPOTIFY_CLIENT_SECRET');
+const sql = musicDb();
+try {
+	const result = await pollSpotify(sql, { clientId, clientSecret });
+	console.log(JSON.stringify(result));
+	if (result.status === 'error') process.exitCode = 1;
+} finally {
+	await sql.end();
+}