music: self-hosted scrobbler (Spotify poller, Last.fm + streaming-history importers, /trackers/music, homepage listening card)
All checks were successful
Deploy production / deploy (push) Successful in 2m37s
All checks were successful
Deploy production / deploy (push) Successful in 2m37s
- music schema: artists/albums/tracks (+album art), plays with source + dedupe, now_playing, spotify_auth, import_state; plays_deduped view hides Last.fm scrobbles Spotify also has - in-app Spotify poller (60s, leased) started from hooks init; no new role or env needed - /api/spotify/connect + callback (owner only), /trackers/music public stats - scripts: music-poll, music-import-lastfm (resumable), music-import-history - deploy/music-collector: optional host timer with its own role
This commit is contained in:
90
src/lib/music/ListeningCard.svelte
Normal file
90
src/lib/music/ListeningCard.svelte
Normal file
@@ -0,0 +1,90 @@
|
||||
<script lang="ts">
|
||||
import { relativeTime } from '#lib/relative-time.js';
|
||||
import Music from 'lucide-svelte/icons/music';
|
||||
|
||||
type Listening = {
|
||||
nowPlaying: boolean;
|
||||
at: string;
|
||||
track: string;
|
||||
artists: string;
|
||||
album: string | null;
|
||||
image: string | null;
|
||||
url: string | null;
|
||||
};
|
||||
let { listening, href = '/trackers/music' }: { listening: Listening; href?: string } = $props();
|
||||
|
||||
let now = $state(Date.now());
|
||||
$effect(() => {
|
||||
const timer = setInterval(() => (now = Date.now()), 30_000);
|
||||
return () => clearInterval(timer);
|
||||
});
|
||||
</script>
|
||||
|
||||
<a
|
||||
{href}
|
||||
class="flex items-center gap-4 rounded-md border border-[var(--site-border)] bg-[var(--site-surface)] p-3 hover:border-[var(--site-text)]"
|
||||
aria-label="{listening.nowPlaying
|
||||
? 'Listening to'
|
||||
: 'Last played'} {listening.track} by {listening.artists}"
|
||||
>
|
||||
{#if listening.image}
|
||||
<img
|
||||
src={listening.image}
|
||||
alt=""
|
||||
width="64"
|
||||
height="64"
|
||||
loading="lazy"
|
||||
class="size-16 shrink-0 rounded object-cover"
|
||||
/>
|
||||
{:else}
|
||||
<span class="flex size-16 shrink-0 items-center justify-center rounded bg-[var(--site-border)]"
|
||||
><Music size={24} aria-hidden="true" /></span
|
||||
>
|
||||
{/if}
|
||||
<span class="min-w-0">
|
||||
<span class="flex items-center gap-2 text-sm text-[var(--site-muted)]">
|
||||
{#if listening.nowPlaying}
|
||||
<span class="eq" aria-hidden="true"><i></i><i></i><i></i></span> listening now
|
||||
{:else}
|
||||
last played {relativeTime(listening.at, now)}
|
||||
{/if}
|
||||
</span>
|
||||
<span class="block truncate font-medium">{listening.track}</span>
|
||||
<span class="block truncate text-sm text-[var(--site-muted)]">{listening.artists}</span>
|
||||
</span>
|
||||
</a>
|
||||
|
||||
<style>
|
||||
.eq {
|
||||
display: inline-flex;
|
||||
align-items: flex-end;
|
||||
gap: 2px;
|
||||
height: 12px;
|
||||
}
|
||||
.eq i {
|
||||
width: 3px;
|
||||
background: #1db954;
|
||||
animation: eq 0.9s ease-in-out infinite;
|
||||
}
|
||||
.eq i:nth-child(2) {
|
||||
animation-delay: -0.3s;
|
||||
}
|
||||
.eq i:nth-child(3) {
|
||||
animation-delay: -0.6s;
|
||||
}
|
||||
@keyframes eq {
|
||||
0%,
|
||||
100% {
|
||||
height: 3px;
|
||||
}
|
||||
50% {
|
||||
height: 12px;
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.eq i {
|
||||
animation: none;
|
||||
height: 8px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
177
src/lib/music/collector.server.ts
Normal file
177
src/lib/music/collector.server.ts
Normal file
@@ -0,0 +1,177 @@
|
||||
// Spotify poller: one call does what a scrobbler does every minute.
|
||||
// 1. GET /me/player → music.now_playing (row 1)
|
||||
// 2. GET /me/player/recently-played?after=<cursor> → music.plays (Spotify lists plays of
|
||||
// 30 s or more, max 50 per call, so polling every minute never misses one)
|
||||
// The authorization (refresh token) lives in music.spotify_auth, written by
|
||||
// /api/spotify/callback. A lease column keeps two app containers (e.g. mid-deploy) or the
|
||||
// optional host timer (scripts/music-poll.ts) from polling at the same time.
|
||||
import { newCache, upsertTrack, insertPlay, type Db } from './store.server';
|
||||
import { parsePlayer, parseRecentlyPlayed } from './spotify-parse';
|
||||
|
||||
export type SpotifyCredentials = { clientId: string; clientSecret: string };
|
||||
type Fetch = typeof fetch;
|
||||
|
||||
export const SPOTIFY_SCOPES = [
|
||||
'user-read-currently-playing',
|
||||
'user-read-recently-played',
|
||||
'user-read-playback-state',
|
||||
'user-top-read'
|
||||
] as const;
|
||||
|
||||
export class SpotifyError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
message: string,
|
||||
readonly code?: string
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
type TokenResponse = {
|
||||
access_token: string;
|
||||
expires_in: number;
|
||||
refresh_token?: string;
|
||||
scope?: string;
|
||||
};
|
||||
|
||||
async function token(
|
||||
fetchFn: Fetch,
|
||||
creds: SpotifyCredentials,
|
||||
body: Record<string, string>
|
||||
): Promise<TokenResponse> {
|
||||
const response = await fetchFn('https://accounts.spotify.com/api/token', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Basic ${btoa(`${creds.clientId}:${creds.clientSecret}`)}`,
|
||||
'Content-Type': 'application/x-www-form-urlencoded'
|
||||
},
|
||||
body: new URLSearchParams(body),
|
||||
signal: AbortSignal.timeout(15_000)
|
||||
});
|
||||
const json = (await response.json().catch(() => ({}))) as Record<string, unknown>;
|
||||
if (!response.ok || typeof json.access_token !== 'string')
|
||||
throw new SpotifyError(
|
||||
response.status,
|
||||
`Spotify token request failed (${response.status} ${String(json.error ?? '')})`,
|
||||
typeof json.error === 'string' ? json.error : undefined
|
||||
);
|
||||
return json as unknown as TokenResponse;
|
||||
}
|
||||
|
||||
export const exchangeCode = (
|
||||
fetchFn: Fetch,
|
||||
creds: SpotifyCredentials,
|
||||
code: string,
|
||||
redirectUri: string
|
||||
) => token(fetchFn, creds, { grant_type: 'authorization_code', code, redirect_uri: redirectUri });
|
||||
|
||||
export const refreshAccessToken = (fetchFn: Fetch, creds: SpotifyCredentials, refresh: string) =>
|
||||
token(fetchFn, creds, { grant_type: 'refresh_token', refresh_token: refresh });
|
||||
|
||||
/** GET a Web API path; null for 204 No Content. */
|
||||
export async function spotifyGet(fetchFn: Fetch, access: string, path: string) {
|
||||
const response = await fetchFn(`https://api.spotify.com/v1/${path}`, {
|
||||
headers: { Authorization: `Bearer ${access}` },
|
||||
signal: AbortSignal.timeout(15_000)
|
||||
});
|
||||
if (response.status === 204) return null;
|
||||
if (!response.ok)
|
||||
throw new SpotifyError(
|
||||
response.status,
|
||||
`Spotify GET /${path.split('?')[0]} → ${response.status}`
|
||||
);
|
||||
const text = await response.text();
|
||||
return text ? (JSON.parse(text) as unknown) : null;
|
||||
}
|
||||
|
||||
export type PollResult =
|
||||
| { status: 'idle' }
|
||||
| { status: 'ok'; playing: boolean; newPlays: number }
|
||||
| { status: 'error'; error: string };
|
||||
|
||||
export async function pollSpotify(
|
||||
sql: Db,
|
||||
creds: SpotifyCredentials,
|
||||
fetchFn: Fetch = fetch
|
||||
): Promise<PollResult> {
|
||||
const [auth] = await sql`
|
||||
update music.spotify_auth set lease_until = now() + interval '55 seconds'
|
||||
where id = 1 and not needs_reauth and (lease_until is null or lease_until < now())
|
||||
returning refresh_token, access_token, access_expires_at, recent_cursor`;
|
||||
if (!auth) return { status: 'idle' };
|
||||
let error: string | null = null;
|
||||
let result: PollResult;
|
||||
try {
|
||||
let access: string | null = auth.access_token;
|
||||
const expires = auth.access_expires_at ? new Date(auth.access_expires_at).getTime() : 0;
|
||||
if (!access || expires < Date.now() + 60_000) {
|
||||
const fresh = await refreshAccessToken(fetchFn, creds, auth.refresh_token);
|
||||
access = fresh.access_token;
|
||||
await sql`
|
||||
update music.spotify_auth set access_token = ${access},
|
||||
access_expires_at = now() + make_interval(secs => ${fresh.expires_in}),
|
||||
refresh_token = coalesce(${fresh.refresh_token ?? null}, refresh_token),
|
||||
scope = coalesce(${fresh.scope ?? null}, scope), updated_at = now()
|
||||
where id = 1`;
|
||||
}
|
||||
const cache = newCache();
|
||||
|
||||
const player = parsePlayer(
|
||||
await spotifyGet(fetchFn, access!, 'me/player?additional_types=episode')
|
||||
);
|
||||
const trackId = player?.track ? await upsertTrack(sql, player.track, true, cache) : null;
|
||||
await sql`
|
||||
insert into music.now_playing (id, track_id, is_playing, progress_ms, device_name, device_type, context_uri, updated_at)
|
||||
values (1, ${trackId}, ${!!(player?.isPlaying && trackId)}, ${player?.progressMs ?? null},
|
||||
${player?.deviceName ?? null}, ${player?.deviceType ?? null}, ${player?.contextUri ?? null}, now())
|
||||
on conflict (id) do update set
|
||||
track_id = coalesce(excluded.track_id, music.now_playing.track_id),
|
||||
is_playing = excluded.is_playing, progress_ms = excluded.progress_ms,
|
||||
device_name = excluded.device_name, device_type = excluded.device_type,
|
||||
context_uri = excluded.context_uri, updated_at = now()`;
|
||||
|
||||
const cursor: number | null = auth.recent_cursor === null ? null : Number(auth.recent_cursor);
|
||||
const recent = parseRecentlyPlayed(
|
||||
await spotifyGet(
|
||||
fetchFn,
|
||||
access!,
|
||||
`me/player/recently-played?limit=50${cursor ? `&after=${cursor}` : ''}`
|
||||
)
|
||||
);
|
||||
let newPlays = 0;
|
||||
let newest = cursor ?? 0;
|
||||
for (const play of recent) {
|
||||
const at = play.playedAt.getTime();
|
||||
if (cursor && at <= cursor) continue;
|
||||
const id = await upsertTrack(sql, play.track, true, cache);
|
||||
if (
|
||||
await insertPlay(sql, {
|
||||
trackId: id,
|
||||
playedAt: play.playedAt,
|
||||
source: 'spotify',
|
||||
origin: 'api',
|
||||
contextType: play.contextType,
|
||||
contextUri: play.contextUri
|
||||
})
|
||||
)
|
||||
newPlays++;
|
||||
newest = Math.max(newest, at);
|
||||
}
|
||||
if (newest && newest !== cursor)
|
||||
await sql`update music.spotify_auth set recent_cursor = ${newest} where id = 1`;
|
||||
result = { status: 'ok', playing: !!player?.isPlaying, newPlays };
|
||||
} catch (e) {
|
||||
error = e instanceof Error ? e.message : String(e);
|
||||
result = { status: 'error', error };
|
||||
if (e instanceof SpotifyError && e.code === 'invalid_grant')
|
||||
// The refresh token was revoked: stop polling until /api/spotify/connect is used again.
|
||||
await sql`update music.spotify_auth set needs_reauth = true where id = 1`;
|
||||
else if (e instanceof SpotifyError && e.status === 401)
|
||||
await sql`update music.spotify_auth set access_token = null where id = 1`;
|
||||
} finally {
|
||||
await sql`update music.spotify_auth set lease_until = null, last_poll_at = now(),
|
||||
last_error = ${error} where id = 1`;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
55
src/lib/music/history.ts
Normal file
55
src/lib/music/history.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
// Pure parser for Spotify data exports, used by scripts/music-import-history.ts:
|
||||
// - Extended Streaming History (Streaming_History_Audio_*.json, older endsong_*.json):
|
||||
// { ts, ms_played, master_metadata_track_name, master_metadata_album_artist_name,
|
||||
// master_metadata_album_album_name, spotify_track_uri, platform, ... }
|
||||
// - Account data (StreamingHistory_music_*.json / StreamingHistory*.json):
|
||||
// { endTime: 'YYYY-MM-DD HH:mm' (UTC), artistName, trackName, msPlayed }
|
||||
// Podcast episodes, audiobooks and plays shorter than minMs are skipped (Spotify's own
|
||||
// recently-played list only counts plays of 30 s or more, so that is the default).
|
||||
import type { ParsedTrack } from './spotify-parse';
|
||||
|
||||
export type HistoryPlay = {
|
||||
playedAt: Date;
|
||||
msPlayed: number;
|
||||
platform: string | null;
|
||||
track: ParsedTrack;
|
||||
};
|
||||
|
||||
export const DEFAULT_MIN_MS = 30_000;
|
||||
|
||||
type Rec = Record<string, unknown>;
|
||||
const str = (value: unknown) => (typeof value === 'string' && value.trim() ? value.trim() : null);
|
||||
|
||||
export function parseHistoryEntry(value: unknown, minMs = DEFAULT_MIN_MS): HistoryPlay | null {
|
||||
if (!value || typeof value !== 'object') return null;
|
||||
const e = value as Rec;
|
||||
const extended = 'ts' in e;
|
||||
const name = str(extended ? e.master_metadata_track_name : e.trackName);
|
||||
const artist = str(extended ? e.master_metadata_album_artist_name : e.artistName);
|
||||
const msPlayed = Number(extended ? e.ms_played : e.msPlayed);
|
||||
if (!name || !artist || !Number.isFinite(msPlayed) || msPlayed < minMs) return null;
|
||||
const rawTime = str(extended ? e.ts : e.endTime);
|
||||
if (!rawTime) return null;
|
||||
const playedAt = new Date(
|
||||
extended ? rawTime : `${rawTime.replace(' ', 'T')}${rawTime.length <= 16 ? ':00' : ''}Z`
|
||||
);
|
||||
if (Number.isNaN(playedAt.getTime())) return null;
|
||||
const uri = str(e.spotify_track_uri);
|
||||
const id = uri?.match(/^spotify:track:([A-Za-z0-9]{22})$/)?.[1] ?? null;
|
||||
const album = extended ? str(e.master_metadata_album_album_name) : null;
|
||||
return {
|
||||
playedAt,
|
||||
msPlayed: Math.round(msPlayed),
|
||||
platform: extended ? str(e.platform) : null,
|
||||
track: {
|
||||
spotifyId: id,
|
||||
name,
|
||||
durationMs: null,
|
||||
explicit: null,
|
||||
artists: [{ spotifyId: null, name: artist }],
|
||||
album: album
|
||||
? { spotifyId: null, name: album, imageUrl: null, imageUrlSmall: null, releaseDate: null }
|
||||
: null
|
||||
}
|
||||
};
|
||||
}
|
||||
66
src/lib/music/lastfm-parse.ts
Normal file
66
src/lib/music/lastfm-parse.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
// Pure parser for Last.fm user.getRecentTracks (format=json, extended=0).
|
||||
import type { ParsedTrack } from './spotify-parse';
|
||||
|
||||
export type LastfmPlay = { playedAt: Date; uts: number; track: ParsedTrack };
|
||||
export type LastfmPage = { plays: LastfmPlay[]; page: number; totalPages: number; total: number };
|
||||
|
||||
type Rec = Record<string, unknown>;
|
||||
const rec = (value: unknown): Rec =>
|
||||
value && typeof value === 'object' && !Array.isArray(value) ? (value as Rec) : {};
|
||||
const text = (value: unknown) => {
|
||||
const raw = typeof value === 'string' ? value : (rec(value)['#text'] ?? rec(value).name);
|
||||
return typeof raw === 'string' && raw.trim() ? raw.trim() : null;
|
||||
};
|
||||
|
||||
export function parseRecentTracks(value: unknown): LastfmPage {
|
||||
const body = rec(rec(value).recenttracks);
|
||||
const attr = rec(body['@attr']);
|
||||
const raw = body.track;
|
||||
const list = Array.isArray(raw) ? raw : raw ? [raw] : [];
|
||||
const plays: LastfmPlay[] = [];
|
||||
for (const entry of list) {
|
||||
const t = rec(entry);
|
||||
if (rec(t['@attr']).nowplaying === 'true') continue; // not scrobbled yet
|
||||
const uts = Number(rec(t.date).uts);
|
||||
const name = text(t.name);
|
||||
const artist = text(t.artist);
|
||||
if (!name || !artist || !Number.isFinite(uts) || uts <= 0) continue;
|
||||
const album = text(t.album);
|
||||
const images = (Array.isArray(t.image) ? t.image : []).map(rec);
|
||||
const pick = (size: string) => {
|
||||
const url = images.find((i) => i.size === size)?.['#text'];
|
||||
// Last.fm's grey star placeholder is not a cover.
|
||||
return typeof url === 'string' &&
|
||||
url.startsWith('https://') &&
|
||||
!url.includes('2a96cbd8b46e442fc41c2b86b821562f')
|
||||
? url
|
||||
: null;
|
||||
};
|
||||
plays.push({
|
||||
uts,
|
||||
playedAt: new Date(uts * 1000),
|
||||
track: {
|
||||
spotifyId: null,
|
||||
name,
|
||||
durationMs: null,
|
||||
explicit: null,
|
||||
artists: [{ spotifyId: null, name: artist }],
|
||||
album: album
|
||||
? {
|
||||
spotifyId: null,
|
||||
name: album,
|
||||
imageUrl: pick('extralarge') ?? pick('large'),
|
||||
imageUrlSmall: pick('large') ?? pick('medium'),
|
||||
releaseDate: null
|
||||
}
|
||||
: null
|
||||
}
|
||||
});
|
||||
}
|
||||
return {
|
||||
plays,
|
||||
page: Number(attr.page) || 1,
|
||||
totalPages: Number(attr.totalPages) || 0,
|
||||
total: Number(attr.total) || 0
|
||||
};
|
||||
}
|
||||
167
src/lib/music/music.test.ts
Normal file
167
src/lib/music/music.test.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { parseHistoryEntry } from './history';
|
||||
import { parseRecentTracks } from './lastfm-parse';
|
||||
import { matchKey } from './normalize';
|
||||
import { parsePlayer, parseRecentlyPlayed, parseTrack, pickImages } from './spotify-parse';
|
||||
|
||||
const ID = '4uLU6hMCjMI75M1A2tKUQC';
|
||||
const spotifyTrack = {
|
||||
type: 'track',
|
||||
id: ID,
|
||||
name: 'Never Gonna Give You Up',
|
||||
duration_ms: 213573,
|
||||
explicit: false,
|
||||
artists: [{ id: '0gxyHStUsqpMadRV0Di1Qt', name: 'Rick Astley' }],
|
||||
album: {
|
||||
id: '6N9PS4QXF1D0OWPk0Sxtb4',
|
||||
name: 'Whenever You Need Somebody',
|
||||
release_date: '1987-11-12',
|
||||
images: [
|
||||
{ url: 'https://i.scdn.co/image/640', width: 640, height: 640 },
|
||||
{ url: 'https://i.scdn.co/image/300', width: 300, height: 300 },
|
||||
{ url: 'https://i.scdn.co/image/64', width: 64, height: 64 }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
describe('spotify parsing', () => {
|
||||
it('parses a track with album art', () => {
|
||||
const t = parseTrack(spotifyTrack)!;
|
||||
expect(t.spotifyId).toBe(ID);
|
||||
expect(t.artists).toEqual([{ spotifyId: '0gxyHStUsqpMadRV0Di1Qt', name: 'Rick Astley' }]);
|
||||
expect(t.album?.imageUrl).toBe('https://i.scdn.co/image/640');
|
||||
expect(t.album?.imageUrlSmall).toBe('https://i.scdn.co/image/300');
|
||||
});
|
||||
it('skips local files, episodes and nameless items', () => {
|
||||
expect(parseTrack({ ...spotifyTrack, is_local: true })).toBeNull();
|
||||
expect(parseTrack({ ...spotifyTrack, type: 'episode' })).toBeNull();
|
||||
expect(parseTrack({ ...spotifyTrack, name: ' ' })).toBeNull();
|
||||
});
|
||||
it('keeps only https images', () => {
|
||||
expect(pickImages([{ url: 'http://x/1', width: 640 }])).toEqual({ large: null, small: null });
|
||||
});
|
||||
it('sorts recently played oldest first', () => {
|
||||
const plays = parseRecentlyPlayed({
|
||||
items: [
|
||||
{
|
||||
track: spotifyTrack,
|
||||
played_at: '2026-10-24T15:00:00.123Z',
|
||||
context: { type: 'playlist', uri: 'spotify:playlist:x' }
|
||||
},
|
||||
{ track: spotifyTrack, played_at: '2026-10-24T14:00:00.000Z', context: null },
|
||||
{ track: null, played_at: '2026-10-24T13:00:00.000Z' }
|
||||
]
|
||||
});
|
||||
expect(plays.map((p) => p.playedAt.toISOString())).toEqual([
|
||||
'2026-10-24T14:00:00.000Z',
|
||||
'2026-10-24T15:00:00.123Z'
|
||||
]);
|
||||
expect(plays[1].contextType).toBe('playlist');
|
||||
});
|
||||
it('parses the player and ignores podcasts', () => {
|
||||
expect(parsePlayer(null)).toBeNull();
|
||||
const p = parsePlayer({
|
||||
is_playing: true,
|
||||
progress_ms: 1000,
|
||||
currently_playing_type: 'track',
|
||||
item: spotifyTrack,
|
||||
device: { name: 'Mac', type: 'Computer' }
|
||||
})!;
|
||||
expect(p).toMatchObject({ isPlaying: true, deviceName: 'Mac' });
|
||||
expect(p.track?.name).toBe('Never Gonna Give You Up');
|
||||
expect(
|
||||
parsePlayer({ is_playing: true, currently_playing_type: 'episode', item: {} })!.track
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('history export parsing', () => {
|
||||
it('parses extended streaming history and skips short plays and podcasts', () => {
|
||||
const play = parseHistoryEntry({
|
||||
ts: '2024-01-02T03:04:05Z',
|
||||
ms_played: 200000,
|
||||
platform: 'ios',
|
||||
master_metadata_track_name: 'Song',
|
||||
master_metadata_album_artist_name: 'Artist',
|
||||
master_metadata_album_album_name: 'Album',
|
||||
spotify_track_uri: `spotify:track:${ID}`
|
||||
})!;
|
||||
expect(play.track.spotifyId).toBe(ID);
|
||||
expect(play.playedAt.toISOString()).toBe('2024-01-02T03:04:05.000Z');
|
||||
expect(
|
||||
parseHistoryEntry({
|
||||
ts: '2024-01-02T03:04:05Z',
|
||||
ms_played: 5000,
|
||||
master_metadata_track_name: 'S',
|
||||
master_metadata_album_artist_name: 'A'
|
||||
})
|
||||
).toBeNull();
|
||||
expect(
|
||||
parseHistoryEntry({
|
||||
ts: '2024-01-02T03:04:05Z',
|
||||
ms_played: 900000,
|
||||
master_metadata_track_name: null,
|
||||
episode_name: 'Pod'
|
||||
})
|
||||
).toBeNull();
|
||||
});
|
||||
it('parses the account-data export (UTC endTime)', () => {
|
||||
const play = parseHistoryEntry({
|
||||
endTime: '2023-05-06 07:08',
|
||||
artistName: 'A',
|
||||
trackName: 'T',
|
||||
msPlayed: 60000
|
||||
})!;
|
||||
expect(play.playedAt.toISOString()).toBe('2023-05-06T07:08:00.000Z');
|
||||
expect(play.track.spotifyId).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('last.fm parsing', () => {
|
||||
it('skips the now-playing item and placeholder covers', () => {
|
||||
const page = parseRecentTracks({
|
||||
recenttracks: {
|
||||
'@attr': { page: '1', totalPages: '3', total: '500' },
|
||||
track: [
|
||||
{ '@attr': { nowplaying: 'true' }, name: 'Now', artist: { '#text': 'A' } },
|
||||
{
|
||||
name: 'Song',
|
||||
artist: { '#text': 'Artist' },
|
||||
album: { '#text': 'Album' },
|
||||
date: { uts: '1700000000' },
|
||||
image: [
|
||||
{ size: 'large', '#text': 'https://lastfm.freetls.fastly.net/i/u/174s/abc.png' },
|
||||
{
|
||||
size: 'extralarge',
|
||||
'#text': 'https://lastfm.freetls.fastly.net/i/u/300x300/abc.png'
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
name: 'Starless',
|
||||
artist: { '#text': 'B' },
|
||||
date: { uts: '1690000000' },
|
||||
image: [{ size: 'large', '#text': 'https://x/2a96cbd8b46e442fc41c2b86b821562f.png' }]
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
expect(page.totalPages).toBe(3);
|
||||
expect(page.plays).toHaveLength(2);
|
||||
expect(page.plays[0].playedAt.toISOString()).toBe('2023-11-14T22:13:20.000Z');
|
||||
expect(page.plays[0].track.album?.imageUrl).toContain('300x300');
|
||||
expect(page.plays[1].track.album).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('matchKey', () => {
|
||||
it('ignores remaster/feat/accents/punctuation differences', () => {
|
||||
const base = matchKey('Beyoncé', 'Crazy in Love');
|
||||
expect(matchKey('Beyonce', 'Crazy In Love (feat. Jay-Z)')).toBe(base);
|
||||
expect(matchKey('BEYONCÉ', 'Crazy in Love - 2003 Remaster')).toBe(base);
|
||||
expect(matchKey('Simon & Garfunkel', 'The Boxer')).toBe(
|
||||
matchKey('Simon and Garfunkel', 'The Boxer')
|
||||
);
|
||||
expect(matchKey('A', 'Song - Part 2')).not.toBe(matchKey('A', 'Song'));
|
||||
});
|
||||
});
|
||||
28
src/lib/music/normalize.ts
Normal file
28
src/lib/music/normalize.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
// Cross-source matching key: the same play reported by Spotify (live API or the streaming
|
||||
// history export) and Last.fm often differs in spelling ("Song - 2011 Remaster",
|
||||
// "Song (feat. X)", "&" vs "and", accents). Both sides reduce to "artist|title".
|
||||
|
||||
const VERSION_WORDS = /\b(remaster(ed)?|version|edit|mono|stereo|mix|live|deluxe|bonus|single)\b/;
|
||||
|
||||
function clean(value: string) {
|
||||
let s = value
|
||||
.normalize('NFKD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase();
|
||||
// "(feat. X)", "[with Y]", trailing " feat. Z"
|
||||
s = s.replace(/\s*[([](feat\.?|ft\.?|featuring|with)\s[^)\]]*[)\]]/g, '');
|
||||
s = s.replace(/\s+(feat\.?|ft\.?|featuring)\s.*$/, '');
|
||||
// "(2011 Remaster)", "[Live]" ... and " - 2011 Remaster", " - Radio Edit"
|
||||
s = s.replace(/\s*[([]([^)\]]*)[)\]]/g, (whole, inner: string) =>
|
||||
VERSION_WORDS.test(inner) ? '' : whole
|
||||
);
|
||||
s = s.replace(/\s+-\s+(.*)$/, (whole, tail: string) => (VERSION_WORDS.test(tail) ? '' : whole));
|
||||
return s
|
||||
.replace(/&/g, ' and ')
|
||||
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function matchKey(artist: string, title: string) {
|
||||
return `${clean(artist)}|${clean(title)}`;
|
||||
}
|
||||
122
src/lib/music/spotify-parse.ts
Normal file
122
src/lib/music/spotify-parse.ts
Normal file
@@ -0,0 +1,122 @@
|
||||
// Pure parsers for Spotify Web API responses (no I/O, no SvelteKit imports), shared by the
|
||||
// in-app poller, scripts/music-poll.ts and the tests. Spotify trims fields over time (see the
|
||||
// February 2026 dev-mode changes), so everything is optional and checked.
|
||||
|
||||
export type ParsedArtist = { spotifyId: string | null; name: string };
|
||||
export type ParsedAlbum = {
|
||||
spotifyId: string | null;
|
||||
name: string;
|
||||
imageUrl: string | null;
|
||||
imageUrlSmall: string | null;
|
||||
releaseDate: string | null;
|
||||
};
|
||||
export type ParsedTrack = {
|
||||
spotifyId: string | null;
|
||||
name: string;
|
||||
durationMs: number | null;
|
||||
explicit: boolean | null;
|
||||
artists: ParsedArtist[];
|
||||
album: ParsedAlbum | null;
|
||||
};
|
||||
export type ParsedRecentPlay = {
|
||||
track: ParsedTrack;
|
||||
playedAt: Date;
|
||||
contextType: string | null;
|
||||
contextUri: string | null;
|
||||
};
|
||||
export type ParsedPlayer = {
|
||||
isPlaying: boolean;
|
||||
progressMs: number | null;
|
||||
deviceName: string | null;
|
||||
deviceType: string | null;
|
||||
contextUri: string | null;
|
||||
/** Null for podcasts, ads and unknown items: only music tracks are stored. */
|
||||
track: ParsedTrack | null;
|
||||
};
|
||||
|
||||
type Rec = Record<string, unknown>;
|
||||
const rec = (value: unknown): Rec =>
|
||||
value && typeof value === 'object' && !Array.isArray(value) ? (value as Rec) : {};
|
||||
const str = (value: unknown) => (typeof value === 'string' && value.trim() ? value.trim() : null);
|
||||
const int = (value: unknown) =>
|
||||
typeof value === 'number' && Number.isFinite(value) ? Math.round(value) : null;
|
||||
const SPOTIFY_ID = /^[A-Za-z0-9]{22}$/;
|
||||
export const spotifyId = (value: unknown) => {
|
||||
const id = str(value);
|
||||
return id && SPOTIFY_ID.test(id) ? id : null;
|
||||
};
|
||||
|
||||
/** Spotify lists images largest first; keep the largest and the smallest of at least 64 px. */
|
||||
export function pickImages(images: unknown): { large: string | null; small: string | null } {
|
||||
const list = (Array.isArray(images) ? images : [])
|
||||
.map(rec)
|
||||
.filter((i) => typeof i.url === 'string' && (i.url as string).startsWith('https://'))
|
||||
.map((i) => ({ url: i.url as string, width: int(i.width) ?? 0 }))
|
||||
.sort((a, b) => b.width - a.width);
|
||||
if (!list.length) return { large: null, small: null };
|
||||
const small = [...list].reverse().find((i) => i.width >= 160) ?? list[list.length - 1];
|
||||
return { large: list[0].url, small: small.url };
|
||||
}
|
||||
|
||||
export function parseTrack(value: unknown): ParsedTrack | null {
|
||||
const track = rec(value);
|
||||
if (track.type !== undefined && track.type !== 'track') return null;
|
||||
if (track.is_local === true) return null;
|
||||
const name = str(track.name);
|
||||
if (!name) return null;
|
||||
const artists = (Array.isArray(track.artists) ? track.artists : [])
|
||||
.map(rec)
|
||||
.map((a) => ({ spotifyId: spotifyId(a.id), name: str(a.name) }))
|
||||
.filter((a): a is ParsedArtist => !!a.name);
|
||||
if (!artists.length) return null;
|
||||
const album = rec(track.album);
|
||||
const albumName = str(album.name);
|
||||
const images = pickImages(album.images);
|
||||
return {
|
||||
spotifyId: spotifyId(track.id),
|
||||
name,
|
||||
durationMs: int(track.duration_ms),
|
||||
explicit: typeof track.explicit === 'boolean' ? track.explicit : null,
|
||||
artists,
|
||||
album: albumName
|
||||
? {
|
||||
spotifyId: spotifyId(album.id),
|
||||
name: albumName,
|
||||
imageUrl: images.large,
|
||||
imageUrlSmall: images.small,
|
||||
releaseDate: str(album.release_date)
|
||||
}
|
||||
: null
|
||||
};
|
||||
}
|
||||
|
||||
/** GET /me/player/recently-played → plays, oldest first. */
|
||||
export function parseRecentlyPlayed(value: unknown): ParsedRecentPlay[] {
|
||||
const items = rec(value).items;
|
||||
const plays: ParsedRecentPlay[] = [];
|
||||
for (const entry of Array.isArray(items) ? items : []) {
|
||||
const item = rec(entry);
|
||||
const track = parseTrack(item.track);
|
||||
const playedAt = str(item.played_at) ? new Date(item.played_at as string) : null;
|
||||
if (!track || !playedAt || Number.isNaN(playedAt.getTime())) continue;
|
||||
const context = rec(item.context);
|
||||
plays.push({ track, playedAt, contextType: str(context.type), contextUri: str(context.uri) });
|
||||
}
|
||||
return plays.sort((a, b) => a.playedAt.getTime() - b.playedAt.getTime());
|
||||
}
|
||||
|
||||
/** GET /me/player (null body = 204, nothing active). */
|
||||
export function parsePlayer(value: unknown): ParsedPlayer | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
const player = rec(value);
|
||||
const device = rec(player.device);
|
||||
const type = str(player.currently_playing_type);
|
||||
return {
|
||||
isPlaying: player.is_playing === true,
|
||||
progressMs: int(player.progress_ms),
|
||||
deviceName: str(device.name),
|
||||
deviceType: str(device.type),
|
||||
contextUri: str(rec(player.context).uri),
|
||||
track: type === null || type === 'track' ? parseTrack(player.item) : null
|
||||
};
|
||||
}
|
||||
224
src/lib/music/store.server.ts
Normal file
224
src/lib/music/store.server.ts
Normal file
@@ -0,0 +1,224 @@
|
||||
// Writes to the music schema with postgres.js. No SvelteKit imports (relative imports only),
|
||||
// so the app, scripts/music-*.ts and the image's `bun scripts/...` runs all share it.
|
||||
import type { Sql } from 'postgres';
|
||||
import { matchKey } from './normalize';
|
||||
import type { ParsedAlbum, ParsedArtist, ParsedTrack } from './spotify-parse';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export type Db = Sql<any>;
|
||||
export type PlaySource = 'spotify' | 'lastfm' | 'youtube' | 'local';
|
||||
export type PlayOrigin = 'api' | 'export' | 'import' | 'manual';
|
||||
|
||||
/** Per-run cache so a poll or import does not look up the same artist/album/track twice. */
|
||||
export type StoreCache = Map<string, number>;
|
||||
export const newCache = (): StoreCache => new Map();
|
||||
|
||||
export async function upsertArtist(sql: Db, artist: ParsedArtist, cache = newCache()) {
|
||||
const cacheKey = `artist:${artist.spotifyId ?? `name:${artist.name.toLowerCase()}`}`;
|
||||
const cached = cache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
let id: number | undefined;
|
||||
if (artist.spotifyId) {
|
||||
const [found] =
|
||||
await sql`select id, name from music.artists where spotify_id = ${artist.spotifyId}`;
|
||||
if (found) {
|
||||
id = found.id;
|
||||
if (found.name !== artist.name)
|
||||
await sql`update music.artists set name = ${artist.name} where id = ${found.id}`;
|
||||
} else {
|
||||
// Adopt a row created by an import (name only) so its plays merge with live ones.
|
||||
const [adopted] = await sql`
|
||||
update music.artists set spotify_id = ${artist.spotifyId}, name = ${artist.name}
|
||||
where id = (select id from music.artists where spotify_id is null
|
||||
and lower(name) = lower(${artist.name}) order by id limit 1)
|
||||
returning id`;
|
||||
id =
|
||||
adopted?.id ??
|
||||
(
|
||||
await sql`
|
||||
insert into music.artists (spotify_id, name) values (${artist.spotifyId}, ${artist.name})
|
||||
on conflict (spotify_id) where spotify_id is not null
|
||||
do update set name = excluded.name returning id`
|
||||
)[0].id;
|
||||
}
|
||||
} else {
|
||||
const [found] = await sql`
|
||||
select id from music.artists where lower(name) = lower(${artist.name})
|
||||
order by (spotify_id is null), id limit 1`;
|
||||
id =
|
||||
found?.id ??
|
||||
(await sql`insert into music.artists (name) values (${artist.name}) returning id`)[0].id;
|
||||
}
|
||||
cache.set(cacheKey, id!);
|
||||
return id!;
|
||||
}
|
||||
|
||||
export async function upsertAlbum(
|
||||
sql: Db,
|
||||
album: ParsedAlbum,
|
||||
artistId: number | null,
|
||||
cache = newCache()
|
||||
) {
|
||||
const cacheKey = `album:${album.spotifyId ?? `name:${artistId}:${album.name.toLowerCase()}`}`;
|
||||
const cached = cache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
let row: { id: number; image_url: string | null; image_url_small: string | null } | undefined;
|
||||
if (album.spotifyId) {
|
||||
[row] =
|
||||
await sql`select id, image_url, image_url_small from music.albums where spotify_id = ${album.spotifyId}`;
|
||||
if (!row)
|
||||
[row] = await sql`
|
||||
update music.albums set spotify_id = ${album.spotifyId}, name = ${album.name}
|
||||
where id = (select id from music.albums where spotify_id is null
|
||||
and lower(name) = lower(${album.name})
|
||||
and (artist_id is not distinct from ${artistId} or artist_id is null)
|
||||
order by id limit 1)
|
||||
returning id, image_url, image_url_small`;
|
||||
if (!row)
|
||||
[row] = await sql`
|
||||
insert into music.albums (spotify_id, name, artist_id, image_url, image_url_small, release_date)
|
||||
values (${album.spotifyId}, ${album.name}, ${artistId}, ${album.imageUrl},
|
||||
${album.imageUrlSmall}, ${album.releaseDate})
|
||||
on conflict (spotify_id) where spotify_id is not null
|
||||
do update set name = excluded.name returning id, image_url, image_url_small`;
|
||||
} else {
|
||||
[row] = await sql`
|
||||
select id, image_url, image_url_small from music.albums
|
||||
where lower(name) = lower(${album.name}) and artist_id is not distinct from ${artistId}
|
||||
order by (spotify_id is null), id limit 1`;
|
||||
if (!row)
|
||||
[row] = await sql`
|
||||
insert into music.albums (name, artist_id, image_url, image_url_small)
|
||||
values (${album.name}, ${artistId}, ${album.imageUrl}, ${album.imageUrlSmall})
|
||||
returning id, image_url, image_url_small`;
|
||||
}
|
||||
// Spotify covers win; an import only fills a missing cover.
|
||||
const better = album.spotifyId
|
||||
? album.imageUrl && album.imageUrl !== row!.image_url
|
||||
: album.imageUrl && !row!.image_url;
|
||||
if (better)
|
||||
await sql`update music.albums set image_url = ${album.imageUrl},
|
||||
image_url_small = ${album.imageUrlSmall ?? album.imageUrl},
|
||||
release_date = coalesce(${album.releaseDate}, release_date) where id = ${row!.id}`;
|
||||
cache.set(cacheKey, row!.id);
|
||||
return row!.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find or create a track. `full` = complete Spotify API data (all artists, album, duration):
|
||||
* it may correct an existing row; partial import data never overwrites anything.
|
||||
*/
|
||||
export async function upsertTrack(sql: Db, track: ParsedTrack, full: boolean, cache = newCache()) {
|
||||
const artistIds: number[] = [];
|
||||
for (const artist of track.artists) artistIds.push(await upsertArtist(sql, artist, cache));
|
||||
const artistId = artistIds[0];
|
||||
const cacheKey = `track:${track.spotifyId ?? `name:${artistId}:${track.name.toLowerCase()}`}`;
|
||||
const cached = cache.get(cacheKey);
|
||||
if (cached) return cached;
|
||||
const albumId = track.album ? await upsertAlbum(sql, track.album, artistId, cache) : null;
|
||||
const key = matchKey(track.artists[0].name, track.name);
|
||||
let id: number | undefined;
|
||||
if (track.spotifyId) {
|
||||
const [found] = await sql`
|
||||
select id, album_id, artist_id, duration_ms from music.tracks where spotify_id = ${track.spotifyId}`;
|
||||
if (found) {
|
||||
id = found.id;
|
||||
if (
|
||||
full &&
|
||||
(found.album_id !== albumId ||
|
||||
found.artist_id !== artistId ||
|
||||
found.duration_ms !== track.durationMs)
|
||||
)
|
||||
await sql`update music.tracks set name = ${track.name}, album_id = ${albumId},
|
||||
artist_id = ${artistId}, duration_ms = ${track.durationMs},
|
||||
explicit = ${track.explicit}, match_key = ${key} where id = ${id}`;
|
||||
else if (!found.album_id && albumId)
|
||||
await sql`update music.tracks set album_id = ${albumId} where id = ${id}`;
|
||||
} else {
|
||||
const [adopted] = await sql`
|
||||
update music.tracks set spotify_id = ${track.spotifyId},
|
||||
album_id = coalesce(${albumId}, album_id),
|
||||
duration_ms = coalesce(${track.durationMs}, duration_ms),
|
||||
explicit = coalesce(${track.explicit}, explicit)
|
||||
where id = (select id from music.tracks where spotify_id is null
|
||||
and lower(name) = lower(${track.name}) and artist_id = ${artistId}
|
||||
order by id limit 1)
|
||||
returning id`;
|
||||
id =
|
||||
adopted?.id ??
|
||||
(
|
||||
await sql`
|
||||
insert into music.tracks (spotify_id, name, album_id, artist_id, duration_ms, explicit, match_key)
|
||||
values (${track.spotifyId}, ${track.name}, ${albumId}, ${artistId},
|
||||
${track.durationMs}, ${track.explicit}, ${key})
|
||||
on conflict (spotify_id) where spotify_id is not null
|
||||
do update set name = excluded.name returning id`
|
||||
)[0].id;
|
||||
}
|
||||
} else {
|
||||
const [found] = await sql`
|
||||
select id, album_id from music.tracks
|
||||
where lower(name) = lower(${track.name}) and artist_id = ${artistId}
|
||||
order by (spotify_id is null), id limit 1`;
|
||||
if (found) {
|
||||
id = found.id;
|
||||
if (!found.album_id && albumId)
|
||||
await sql`update music.tracks set album_id = ${albumId} where id = ${id}`;
|
||||
} else
|
||||
id = (
|
||||
await sql`
|
||||
insert into music.tracks (name, album_id, artist_id, duration_ms, match_key)
|
||||
values (${track.name}, ${albumId}, ${artistId}, ${track.durationMs}, ${key})
|
||||
returning id`
|
||||
)[0].id;
|
||||
}
|
||||
for (const [position, aid] of artistIds.entries())
|
||||
await sql`insert into music.track_artists (track_id, artist_id, position)
|
||||
values (${id!}, ${aid}, ${position}) on conflict do nothing`;
|
||||
cache.set(cacheKey, id!);
|
||||
return id!;
|
||||
}
|
||||
|
||||
export type PlayInput = {
|
||||
trackId: number;
|
||||
playedAt: Date;
|
||||
source: PlaySource;
|
||||
origin: PlayOrigin;
|
||||
msPlayed?: number | null;
|
||||
contextType?: string | null;
|
||||
contextUri?: string | null;
|
||||
platform?: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Insert a play unless the same source already has it: same normalised artist+title within
|
||||
* a minute (the live API and the Spotify export report the same end time to within seconds;
|
||||
* Last.fm re-imports have the exact same timestamp). Cross-source overlap (Last.fm vs
|
||||
* Spotify) is resolved at read time by the music.plays_deduped view. Returns true if stored.
|
||||
*/
|
||||
export async function insertPlay(sql: Db, play: PlayInput) {
|
||||
const at = play.playedAt.toISOString();
|
||||
const rows = await sql`
|
||||
insert into music.plays (track_id, played_at, source, origin, ms_played, context_type, context_uri, platform)
|
||||
select ${play.trackId}, ${at}::timestamptz, ${play.source}, ${play.origin}, ${play.msPlayed ?? null},
|
||||
${play.contextType ?? null}, ${play.contextUri ?? null}, ${play.platform ?? null}
|
||||
where not exists (
|
||||
select 1 from music.plays p join music.tracks t on t.id = p.track_id
|
||||
where p.source = ${play.source}
|
||||
and p.played_at between ${at}::timestamptz - interval '60 seconds'
|
||||
and ${at}::timestamptz + interval '60 seconds'
|
||||
and t.match_key = (select match_key from music.tracks where id = ${play.trackId}))
|
||||
on conflict do nothing
|
||||
returning id`;
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
export async function getImportState<T>(sql: Db, key: string): Promise<T | null> {
|
||||
const [row] = await sql`select data from music.import_state where key = ${key}`;
|
||||
return (row?.data as T) ?? null;
|
||||
}
|
||||
|
||||
export async function setImportState(sql: Db, key: string, data: unknown) {
|
||||
await sql`insert into music.import_state (key, data) values (${key}, ${sql.json(data as never)})
|
||||
on conflict (key) do update set data = excluded.data, updated_at = now()`;
|
||||
}
|
||||
Reference in New Issue
Block a user