music: self-hosted scrobbler (Spotify poller, Last.fm + streaming-history importers, /trackers/music, homepage listening card)

- music schema: artists/albums/tracks (+album art), plays with source + dedupe, now_playing,
  spotify_auth, import_state; plays_deduped view hides Last.fm scrobbles Spotify also has
- in-app Spotify poller (60s, leased) started from hooks init; no new role or env needed
- /api/spotify/connect + callback (owner only), /trackers/music public stats
- scripts: music-poll, music-import-lastfm (resumable), music-import-history
- deploy/music-collector: optional host timer with its own role

src/lib/server/db/index.ts

@@ -3,7 +3,8 @@ import { relations } from './relations';
 import { drizzle } from 'drizzle-orm/postgres-js';
 import postgres from 'postgres';
 
-const client = postgres(DB_URL, { prepare: false });
+/** Raw postgres.js client (same pool), for hand-written SQL such as the music stats. */
+export const client = postgres(DB_URL, { prepare: false });
 
 export const db = drizzle({
 	relations,

src/lib/server/db/schema/index.ts

@@ -18,3 +18,4 @@ export * from './tasks';
 export * from './groceries';
 export * from './home';
 export * from './cars';
+export * from './music';

src/lib/server/db/schema/music.ts

@@ -0,0 +1,180 @@
+import { sql } from 'drizzle-orm';
+import {
+	bigint,
+	boolean,
+	check,
+	index,
+	integer,
+	jsonb,
+	primaryKey,
+	smallint,
+	text,
+	uniqueIndex
+} from 'drizzle-orm/pg-core';
+import { musicSchema } from './schemas';
+import { id, timestamptz } from './helpers';
+
+// Listening history (a self-hosted Last.fm): every play from Spotify (polled every minute,
+// see src/lib/server/music/collector.ts) and later YouTube / local files, plus the
+// Extended Streaming History import (scripts/music-import-history.ts). Public read
+// (personal_ro gets SELECT) except music.spotify_auth, which holds the refresh token.
+// Rows without a spotify_id (imports, YouTube, local files) are matched by name.
+
+export const musicArtists = musicSchema.table(
+	'artists',
+	{
+		id: id(),
+		spotifyId: text('spotify_id'),
+		name: text('name').notNull(),
+		imageUrl: text('image_url'),
+		createdAt: timestamptz('created_at').notNull().defaultNow()
+	},
+	(t) => [
+		uniqueIndex('artists_spotify_id_key')
+			.on(t.spotifyId)
+			.where(sql`${t.spotifyId} is not null`),
+		index('artists_name_idx').on(sql`lower(${t.name})`)
+	]
+);
+
+export const musicAlbums = musicSchema.table(
+	'albums',
+	{
+		id: id(),
+		spotifyId: text('spotify_id'),
+		name: text('name').notNull(),
+		artistId: integer('artist_id').references(() => musicArtists.id, { onDelete: 'set null' }),
+		/** Largest cover (640 px on Spotify) and a small one (64–300 px) for lists. */
+		imageUrl: text('image_url'),
+		imageUrlSmall: text('image_url_small'),
+		releaseDate: text('release_date'),
+		createdAt: timestamptz('created_at').notNull().defaultNow()
+	},
+	(t) => [
+		uniqueIndex('albums_spotify_id_key')
+			.on(t.spotifyId)
+			.where(sql`${t.spotifyId} is not null`),
+		index('albums_name_idx').on(sql`lower(${t.name})`)
+	]
+);
+
+export const musicTracks = musicSchema.table(
+	'tracks',
+	{
+		id: id(),
+		spotifyId: text('spotify_id'),
+		name: text('name').notNull(),
+		albumId: integer('album_id').references(() => musicAlbums.id, { onDelete: 'set null' }),
+		/** Primary (first credited) artist, also in track_artists at position 0. */
+		artistId: integer('artist_id').references(() => musicArtists.id, { onDelete: 'set null' }),
+		durationMs: integer('duration_ms'),
+		explicit: boolean('explicit'),
+		/** normalised "primary artist|title" (src/lib/music/normalize.ts), for cross-source dedupe. */
+		matchKey: text('match_key').notNull(),
+		createdAt: timestamptz('created_at').notNull().defaultNow()
+	},
+	(t) => [
+		uniqueIndex('tracks_spotify_id_key')
+			.on(t.spotifyId)
+			.where(sql`${t.spotifyId} is not null`),
+		index('tracks_name_idx').on(sql`lower(${t.name})`),
+		index('tracks_match_key_idx').on(t.matchKey)
+	]
+);
+
+export const musicTrackArtists = musicSchema.table(
+	'track_artists',
+	{
+		trackId: integer('track_id')
+			.notNull()
+			.references(() => musicTracks.id, { onDelete: 'cascade' }),
+		artistId: integer('artist_id')
+			.notNull()
+			.references(() => musicArtists.id, { onDelete: 'cascade' }),
+		position: smallint('position').notNull().default(0)
+	},
+	(t) => [
+		primaryKey({ columns: [t.trackId, t.artistId] }),
+		index('track_artists_artist_idx').on(t.artistId)
+	]
+);
+
+export const musicPlays = musicSchema.table(
+	'plays',
+	{
+		id: id(),
+		trackId: integer('track_id')
+			.notNull()
+			.references(() => musicTracks.id, { onDelete: 'cascade' }),
+		/** Spotify: when the play ended (played_at / export ts). Last.fm: the scrobble time,
+		 * usually when the track started. See the music.plays_deduped view. */
+		playedAt: timestamptz('played_at').notNull(),
+		source: text('source').notNull().default('spotify'),
+		/** How the row got here: 'api' (poller), 'export' (Spotify streaming history),
+		 * 'import' (Last.fm and other backfills), 'manual'. */
+		origin: text('origin').notNull().default('api'),
+		/** Known for imports; null for API plays (Spotify only reports plays of 30 s or more). */
+		msPlayed: integer('ms_played'),
+		contextType: text('context_type'),
+		contextUri: text('context_uri'),
+		platform: text('platform'),
+		createdAt: timestamptz('created_at').notNull().defaultNow()
+	},
+	(t) => [
+		uniqueIndex('plays_dedupe_key').on(t.source, t.trackId, t.playedAt),
+		index('plays_played_at_idx').on(t.playedAt),
+		index('plays_source_played_idx').on(t.source, t.playedAt),
+		index('plays_track_played_idx').on(t.trackId, t.playedAt),
+		check('plays_source_check', sql`${t.source} in ('spotify', 'lastfm', 'youtube', 'local')`),
+		check('plays_origin_check', sql`${t.origin} in ('api', 'export', 'import', 'manual')`)
+	]
+);
+
+/** One row (id = 1): what is playing right now, refreshed by the poller every minute. */
+export const musicNowPlaying = musicSchema.table(
+	'now_playing',
+	{
+		id: smallint('id').primaryKey().default(1),
+		trackId: integer('track_id').references(() => musicTracks.id, { onDelete: 'set null' }),
+		isPlaying: boolean('is_playing').notNull().default(false),
+		progressMs: integer('progress_ms'),
+		deviceName: text('device_name'),
+		deviceType: text('device_type'),
+		contextUri: text('context_uri'),
+		updatedAt: timestamptz('updated_at').notNull().defaultNow()
+	},
+	(t) => [check('now_playing_single_row', sql`${t.id} = 1`)]
+);
+
+/**
+ * One row (id = 1): the owner's Spotify authorization (Authorization Code flow, stored by
+ * /api/spotify/callback) and the poller's state. Never granted to personal_ro.
+ */
+export const musicSpotifyAuth = musicSchema.table(
+	'spotify_auth',
+	{
+		id: smallint('id').primaryKey().default(1),
+		spotifyUserId: text('spotify_user_id').notNull(),
+		displayName: text('display_name'),
+		refreshToken: text('refresh_token').notNull(),
+		accessToken: text('access_token'),
+		accessExpiresAt: timestamptz('access_expires_at'),
+		scope: text('scope'),
+		/** recently-played `after` cursor: played_at (ms since epoch) of the newest stored play. */
+		recentCursor: bigint('recent_cursor', { mode: 'number' }),
+		needsReauth: boolean('needs_reauth').notNull().default(false),
+		leaseUntil: timestamptz('lease_until'),
+		lastPollAt: timestamptz('last_poll_at'),
+		lastError: text('last_error'),
+		connectedAt: timestamptz('connected_at').notNull().defaultNow(),
+		updatedAt: timestamptz('updated_at').notNull().defaultNow()
+	},
+	(t) => [check('spotify_auth_single_row', sql`${t.id} = 1`)]
+);
+
+/** Resumable backfill state, e.g. key 'lastfm:stanrunge' (see scripts/music-import-lastfm.ts). */
+export const musicImportState = musicSchema.table('import_state', {
+	key: text('key').primaryKey(),
+	data: jsonb('data').notNull(),
+	updatedAt: timestamptz('updated_at').notNull().defaultNow()
+});

src/lib/server/db/schema/schemas.ts

@@ -18,3 +18,4 @@ export const tasksSchema = pgSchema('tasks');
 export const groceriesSchema = pgSchema('groceries');
 export const homeSchema = pgSchema('home');
 export const carsSchema = pgSchema('cars');
+export const musicSchema = pgSchema('music');

src/lib/server/music/oauth.ts

@@ -0,0 +1,33 @@
+import { SPOTIFY_SCOPES } from '#lib/music/collector.server.js';
+
+export const STATE_COOKIE = 'spotify_oauth_state';
+export const CALLBACK_PATH = '/api/spotify/callback';
+
+/**
+ * The redirect URI registered in the Spotify app: https://stanrunge.dev/api/spotify/callback
+ * (prod) or https://dev.stanrunge.dev/api/spotify/callback (dev server). Caddy terminates TLS,
+ * so the request URL may say http; the public hosts are always https.
+ */
+export function publicOrigin(url: URL) {
+	if (url.hostname === 'stanrunge.dev' || url.hostname.endsWith('.stanrunge.dev'))
+		return `https://${url.hostname}`;
+	return url.origin;
+}
+
+export function authorizeUrl(clientId: string, redirectUri: string, state: string) {
+	return `https://accounts.spotify.com/authorize?${new URLSearchParams({
+		response_type: 'code',
+		client_id: clientId,
+		scope: SPOTIFY_SCOPES.join(' '),
+		redirect_uri: redirectUri,
+		state,
+		show_dialog: 'true'
+	})}`;
+}
+
+export function safeEqual(a: string, b: string) {
+	if (a.length !== b.length) return false;
+	let diff = 0;
+	for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+	return diff === 0;
+}

src/lib/server/music/poller.ts

@@ -0,0 +1,43 @@
+import { SPOTIFY_CLIENT_ID, SPOTIFY_CLIENT_SECRET } from '$app/env/private';
+import { client } from '#lib/server/db/index.js';
+import { pollSpotify } from '#lib/music/collector.server.js';
+
+// The Spotify poller runs inside the app (started from hooks.server.ts `init`): the app
+// already has the database role (personal_app) and the Spotify client credentials, so
+// no extra role, secret file or host timer is needed. It does nothing until the owner has
+// connected Spotify at /api/spotify/connect. MUSIC_POLLER=off disables it (e.g. when the
+// optional host timer, deploy/music-collector, is used instead).
+const INTERVAL_MS = 60_000;
+let timer: ReturnType<typeof setInterval> | undefined;
+let running = false;
+let lastLogged = '';
+
+export const spotifyCredentials = () => ({
+	clientId: SPOTIFY_CLIENT_ID,
+	clientSecret: SPOTIFY_CLIENT_SECRET
+});
+
+export async function pollNow() {
+	if (running || !SPOTIFY_CLIENT_ID || !SPOTIFY_CLIENT_SECRET) return;
+	running = true;
+	try {
+		const result = await pollSpotify(client, spotifyCredentials());
+		const message = result.status === 'error' ? result.error : '';
+		if (message && message !== lastLogged) console.error(`[music] poll failed: ${message}`);
+		lastLogged = message;
+	} catch (e) {
+		// e.g. the music schema is not migrated on this database yet; log once, keep trying.
+		const message = e instanceof Error ? e.message : String(e);
+		if (message !== lastLogged) console.error(`[music] poll failed: ${message}`);
+		lastLogged = message;
+	} finally {
+		running = false;
+	}
+}
+
+export function startMusicPoller() {
+	if (timer || process.env.MUSIC_POLLER === 'off') return;
+	if (!SPOTIFY_CLIENT_ID || !SPOTIFY_CLIENT_SECRET) return;
+	timer = setInterval(pollNow, INTERVAL_MS);
+	setTimeout(pollNow, 5_000);
+}

src/lib/server/music/stats.ts

@@ -0,0 +1,212 @@
+import { client as sql } from '#lib/server/db/index.js';
+
+// Read side of the listening tracker. Everything reads music.plays_deduped, which hides a
+// Last.fm scrobble when Spotify (live API or export) has the same play, so nothing is
+// counted twice. Listening time uses ms_played when known (imports) and otherwise the
+// track length (the API only reports plays of 30 s or more, not how long).
+const TZ = 'Europe/Amsterdam';
+// Fragments are built per query (postgres.js queries are single-use objects).
+const playMs = () => sql`coalesce(p.ms_played, t.duration_ms, 210000)`;
+
+export const RANGES = { '7d': 7, '30d': 30, all: null } as const;
+export type Range = keyof typeof RANGES;
+export const parseRange = (value: string | null): Range =>
+	value && value in RANGES ? (value as Range) : '30d';
+
+export type Listening = {
+	nowPlaying: boolean;
+	at: string;
+	track: string;
+	artists: string;
+	album: string | null;
+	image: string | null;
+	url: string | null;
+	device: string | null;
+};
+
+const artistsOf = () => sql`coalesce((select string_agg(ar.name, ', ' order by ta.position)
+	from music.track_artists ta join music.artists ar on ar.id = ta.artist_id
+	where ta.track_id = t.id), a.name, '')`;
+
+const toListening = (row: Record<string, unknown>, nowPlaying: boolean): Listening => ({
+	nowPlaying,
+	at: new Date(row.at as string).toISOString(),
+	track: row.track as string,
+	artists: row.artists as string,
+	album: (row.album as string) ?? null,
+	image: (row.image as string) ?? null,
+	url: row.spotify_id ? `https://open.spotify.com/track/${row.spotify_id}` : null,
+	device: (row.device as string) ?? null
+});
+
+/** Homepage card: what is playing now (fresh within 3 min), else the last play. Never throws. */
+export async function getListening(): Promise<Listening | null> {
+	try {
+		const [now] = await sql`
+			select np.updated_at as at, t.name as track, t.spotify_id, ${artistsOf()} as artists,
+			       al.name as album, coalesce(al.image_url_small, al.image_url) as image, np.device_name as device
+			from music.now_playing np
+			join music.tracks t on t.id = np.track_id
+			left join music.artists a on a.id = t.artist_id
+			left join music.albums al on al.id = t.album_id
+			where np.id = 1 and np.is_playing and np.updated_at > now() - interval '3 minutes'`;
+		if (now) return toListening(now, true);
+		const [last] = await sql`
+			select p.played_at as at, t.name as track, t.spotify_id, ${artistsOf()} as artists,
+			       al.name as album, coalesce(al.image_url_small, al.image_url) as image, null as device
+			from music.plays p
+			join music.tracks t on t.id = p.track_id
+			left join music.artists a on a.id = t.artist_id
+			left join music.albums al on al.id = t.album_id
+			order by p.played_at desc limit 1`;
+		return last ? toListening(last, false) : null;
+	} catch {
+		return null; // e.g. not migrated on this database: the homepage must still render
+	}
+}
+
+export async function getMusicStats(range: Range) {
+	const days = RANGES[range];
+	const since = (days ? new Date(Date.now() - days * 86_400_000) : new Date(0)).toISOString();
+	const chartDays = days ?? 90;
+	const [totals, artists, tracks, albums, daily, streak, recent, sources] = await Promise.all([
+		sql`
+			select count(*)::int as plays,
+			       coalesce(sum(${playMs()}), 0)::float8 / 3600000 as hours,
+			       count(distinct p.track_id)::int as tracks,
+			       count(distinct t.artist_id)::int as artists,
+			       min(p.played_at) as first
+			from music.plays_deduped p join music.tracks t on t.id = p.track_id
+			where p.played_at >= ${since}::timestamptz`,
+		sql`
+			select a.id, a.name, a.spotify_id, count(*)::int as plays,
+			       coalesce(sum(${playMs()}), 0)::float8 / 3600000 as hours,
+			       coalesce(a.image_url, (select coalesce(al.image_url_small, al.image_url) from music.albums al
+			         where al.artist_id = a.id and al.image_url is not null
+			         order by al.spotify_id is null, al.id desc limit 1)) as image
+			from music.plays_deduped p
+			join music.tracks t on t.id = p.track_id
+			join music.artists a on a.id = t.artist_id
+			where p.played_at >= ${since}::timestamptz
+			group by a.id order by plays desc, a.name limit 10`,
+		sql`
+			select t.id, t.name, t.spotify_id, ${artistsOf()} as artists, count(*)::int as plays,
+			       coalesce(al.image_url_small, al.image_url) as image
+			from music.plays_deduped p
+			join music.tracks t on t.id = p.track_id
+			left join music.artists a on a.id = t.artist_id
+			left join music.albums al on al.id = t.album_id
+			where p.played_at >= ${since}::timestamptz
+			group by t.id, a.name, al.id order by plays desc, t.name limit 10`,
+		sql`
+			select al.id, al.name, al.spotify_id, a.name as artist, count(*)::int as plays,
+			       coalesce(al.image_url_small, al.image_url) as image
+			from music.plays_deduped p
+			join music.tracks t on t.id = p.track_id
+			join music.albums al on al.id = t.album_id
+			left join music.artists a on a.id = al.artist_id
+			where p.played_at >= ${since}::timestamptz
+			group by al.id, a.name order by plays desc, al.name limit 10`,
+		sql`
+			with days as (
+				select generate_series((now() at time zone ${TZ})::date - ${chartDays - 1}::int,
+				                       (now() at time zone ${TZ})::date, interval '1 day')::date as day)
+			select d.day::text as day, count(p.id)::int as plays,
+			       coalesce(sum(${playMs()}), 0)::float8 / 3600000 as hours
+			from days d
+			left join music.plays_deduped p
+			  on p.played_at >= now() - make_interval(days => ${chartDays + 1})
+			 and (p.played_at at time zone ${TZ})::date = d.day
+			left join music.tracks t on t.id = p.track_id
+			group by d.day order by d.day`,
+		sql`
+			with d as (select distinct (played_at at time zone ${TZ})::date as day from music.plays_deduped),
+			g as (select day, day - (row_number() over (order by day))::int as grp from d),
+			s as (select min(day) as start_day, max(day) as end_day, count(*)::int as len from g group by grp)
+			select
+				coalesce((select len from s where end_day >= (now() at time zone ${TZ})::date - 1
+				          order by end_day desc limit 1), 0) as current,
+				coalesce((select len from s order by len desc, end_day desc limit 1), 0) as longest,
+				(select start_day::text from s order by len desc, end_day desc limit 1) as longest_start,
+				(select end_day::text from s order by len desc, end_day desc limit 1) as longest_end,
+				(select count(*)::int from d) as days`,
+		sql`
+			select p.id, p.played_at, p.source, t.name as track, t.spotify_id, ${artistsOf()} as artists,
+			       coalesce(al.image_url_small, al.image_url) as image
+			from music.plays_deduped p
+			join music.tracks t on t.id = p.track_id
+			left join music.artists a on a.id = t.artist_id
+			left join music.albums al on al.id = t.album_id
+			order by p.played_at desc limit 50`,
+		sql`select source, count(*)::int as plays from music.plays_deduped group by source order by plays desc`
+	]);
+	const t = totals[0];
+	return {
+		range,
+		totals: {
+			plays: t.plays as number,
+			hours: Number(t.hours),
+			tracks: t.tracks as number,
+			artists: t.artists as number,
+			first: t.first ? new Date(t.first).toISOString() : null
+		},
+		topArtists: artists.map((r) => ({
+			name: r.name as string,
+			plays: r.plays as number,
+			hours: Number(r.hours),
+			image: (r.image as string) ?? null,
+			url: r.spotify_id ? `https://open.spotify.com/artist/${r.spotify_id}` : null
+		})),
+		topTracks: tracks.map((r) => ({
+			name: r.name as string,
+			artists: r.artists as string,
+			plays: r.plays as number,
+			image: (r.image as string) ?? null,
+			url: r.spotify_id ? `https://open.spotify.com/track/${r.spotify_id}` : null
+		})),
+		topAlbums: albums.map((r) => ({
+			name: r.name as string,
+			artist: (r.artist as string) ?? '',
+			plays: r.plays as number,
+			image: (r.image as string) ?? null,
+			url: r.spotify_id ? `https://open.spotify.com/album/${r.spotify_id}` : null
+		})),
+		daily: daily.map((r) => ({
+			day: r.day as string,
+			plays: r.plays as number,
+			hours: Number(r.hours)
+		})),
+		streak: {
+			current: Number(streak[0].current),
+			longest: Number(streak[0].longest),
+			longestStart: (streak[0].longest_start as string) ?? null,
+			longestEnd: (streak[0].longest_end as string) ?? null,
+			days: Number(streak[0].days)
+		},
+		recent: recent.map((r) => ({
+			id: r.id as number,
+			at: new Date(r.played_at).toISOString(),
+			source: r.source as string,
+			track: r.track as string,
+			artists: r.artists as string,
+			image: (r.image as string) ?? null,
+			url: r.spotify_id ? `https://open.spotify.com/track/${r.spotify_id}` : null
+		})),
+		sources: sources.map((r) => ({ source: r.source as string, plays: r.plays as number }))
+	};
+}
+
+/** Owner-only connection status for /trackers/music. Never includes tokens. */
+export async function getSpotifyConnection() {
+	const [row] = await sql`
+		select spotify_user_id, display_name, needs_reauth, last_poll_at, last_error, connected_at
+		from music.spotify_auth where id = 1`;
+	if (!row) return null;
+	return {
+		user: (row.display_name as string) || (row.spotify_user_id as string),
+		needsReauth: row.needs_reauth as boolean,
+		lastPollAt: row.last_poll_at ? new Date(row.last_poll_at).toISOString() : null,
+		lastError: (row.last_error as string) ?? null,
+		connectedAt: new Date(row.connected_at).toISOString()
+	};
+}

src/lib/server/owner.ts

@@ -26,7 +26,9 @@ export const ownerOnlyPrefixes = [
 	'/trackers/groceries',
 	'/trackers/houses',
 	'/trackers/books/files',
-	'/lists/db'
+	'/lists/db',
+	// Spotify OAuth for the listening tracker (connect + callback 404 for everyone else).
+	'/api/spotify'
 ] as const;
 
 /**
@@ -46,7 +48,8 @@ export const ownerWritePrefixes = [
 	'/trackers/groceries',
 	'/trackers/houses',
 	'/trackers/books',
-	'/trackers/cars'
+	'/trackers/cars',
+	'/trackers/music'
 ] as const;
 
 const normalise = (pathname: string) => pathname.toLowerCase().replace(/\/+$/, '');