stan/website
1
0
Fork 0
Code Issues 13 Pull Requests Actions Packages Projects Releases Wiki Activity

music: Last.fm-style stats with 7d/30d/1y/all ranges and genres
All checks were successful
Deploy production / deploy (push) Successful in 1m14s
Details

Browse Source 
- Range picker (?range=7d|30d|1y|all) drives every section; the hours chart
  covers the whole range (daily for 7d/30d, weekly for 1y, monthly for all).
- Genres from Last.fm artist top tags (music.artist_tags, normalised and
  filtered), split by artist listening time, with a per-bucket stacked chart.
- Listening clock, new artists, comparison with the previous period and top
  movers, streaks, milestones, top albums with art, rank changes.
- music.play_facts materialized view (refreshed by the app when plays arrive)
  plus a per-range cache keep the page fast.
- scripts/music-fetch-tags.ts backfills tags, most-played artists first.
This commit is contained in:
Stan Runge
2026-10-25 14:03:39 +00:00
parent 5e4cd83748
commit 516bc15545
9 changed files with 27811 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/lib/server/db/schema/music.ts
View File

@@ -190,3 +190,28 @@ export const musicImportState = musicSchema.table('import_state', {
data: jsonb('data').notNull(),
updatedAt: timestamptz('updated_at').notNull().defaultNow()
});
/**
* Genres: Last.fm artist.getTopTags per artist, normalised and filtered for junk tags
* (scripts/music-fetch-tags.ts). Weight is Last.fm's tag count (100 = the artist's top tag).
*/
export const musicArtistTags = musicSchema.table(
'artist_tags',
{
artistId: integer('artist_id')
.notNull()
.references(() => musicArtists.id, { onDelete: 'cascade' }),
tag: text('tag').notNull(),
weight: smallint('weight').notNull()
},
(t) => [primaryKey({ columns: [t.artistId, t.tag] }), index('artist_tags_tag_idx').on(t.tag)]
);
/** When tags were last fetched per artist: 'ok', 'no_tags' or 'not_found' (retry later). */
export const musicArtistTagFetch = musicSchema.table('artist_tag_fetch', {
artistId: integer('artist_id')
.primaryKey()
.references(() => musicArtists.id, { onDelete: 'cascade' }),
status: text('status').notNull(),
fetchedAt: timestamptz('fetched_at').notNull().defaultNow()
});

386
src/lib/server/music/stats.ts
View File

@@ -7,12 +7,14 @@ import type { LiveListening } from '#lib/music/now-playing.js';
// track length (the API only reports plays of 30 s or more, not how long).
const TZ = 'Europe/Amsterdam';
// Fragments are built per query (postgres.js queries are single-use objects).
const playMs = () => sql`coalesce(p.ms_played, t.duration_ms, 210000)`;
export const RANGES = { '7d': 7, '30d': 30, all: null } as const;
export const RANGES = { '7d': 7, '30d': 30, '1y': 365, all: null } as const;
export type Range = keyof typeof RANGES;
export const parseRange = (value: string | null): Range =>
value && value in RANGES ? (value as Range) : '30d';
/** Chart bucket per range: days for 7d/30d, weeks for a year, months for all time. */
export const BUCKETS = { '7d': 'day', '30d': 'day', '1y': 'week', all: 'month' } as const;
export type Bucket = (typeof BUCKETS)[Range];
export type Listening = {
nowPlaying: boolean;
@@ -108,98 +110,310 @@ export async function getLiveFromDb(): Promise<LiveListening | null> {
}
}
// The heavy aggregates read music.play_facts (a materialized copy of plays_deduped with
// artist, album, listening ms and local day/hour/weekday). The app refreshes it when new
// plays arrive (a definer function; REFRESH needs the owner) and caches results per range
// until the next refresh or for five minutes. Recent plays always read the live view.
const CACHE_MS = 5 * 60_000;
const MILESTONES = [
1, 1_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 250_000, 300_000, 400_000, 500_000
];
let factsVersion = 0;
let factsWatermark: number | null = null;
let refreshing: Promise<void> | null = null;
const statsCache = new Map<Range, { at: number; version: number; data: Promise<MusicStats> }>();
/** Re-materialize music.play_facts (single flight). */
export function refreshPlayFacts(): Promise<void> {
refreshing ??= (async () => {
const [row] = await sql`select coalesce(max(id), 0)::int as id from music.plays`;
await sql`select music.refresh_play_facts()`;
factsWatermark = row.id as number;
factsVersion++;
})()
.catch((e) =>
console.error(`[music] play_facts refresh failed: ${e instanceof Error ? e.message : e}`)
)
.finally(() => (refreshing = null));
return refreshing;
}
/** Kick off a refresh when plays were added since the last one (never blocks the page). */
async function checkFacts() {
const [row] = await sql`select coalesce(max(id), 0)::int as id from music.plays`;
if (row.id !== factsWatermark) void refreshPlayFacts();
}
const DAY_MS = 86_400_000;
const isoDay = (ms: number) => new Date(ms).toISOString().slice(0, 10);
const dayMs = (day: string) => Date.parse(`${day}T00:00:00Z`);
/** Today's date in Amsterdam, YYYY-MM-DD. */
const localToday = () =>
new Intl.DateTimeFormat('en-CA', {
timeZone: TZ,
year: 'numeric',
month: '2-digit',
day: '2-digit'
}).format(new Date());
const hoursOf = (v: unknown) => Number(v ?? 0) / 3_600_000;
export async function getMusicStats(range: Range) {
const [, stats, recent] = await Promise.all([
checkFacts().catch(() => {}),
cachedStats(range),
getRecentPlays()
]);
return { ...stats, recent };
}
function cachedStats(range: Range): Promise<MusicStats> {
const hit = statsCache.get(range);
if (hit && hit.version === factsVersion && Date.now() - hit.at < CACHE_MS) return hit.data;
const data = computeStats(range);
statsCache.set(range, { at: Date.now(), version: factsVersion, data });
data.catch(() => statsCache.delete(range));
return data;
}
/** The latest plays, live (not cached, not from play_facts). */
async function getRecentPlays() {
const rows = await sql`
select p.id, p.played_at, p.source, t.name as track, t.spotify_id, ${artistsOf()} as artists,
coalesce(al.image_url_small, al.image_url) as image
from music.plays_deduped p
join music.tracks t on t.id = p.track_id
left join music.artists a on a.id = t.artist_id
left join music.albums al on al.id = t.album_id
order by p.played_at desc limit 30`;
return rows.map((r) => ({
id: r.id as number,
at: new Date(r.played_at).toISOString(),
source: r.source as string,
track: r.track as string,
artists: r.artists as string,
image: (r.image as string) ?? null,
url: r.spotify_id ? `https://open.spotify.com/track/${r.spotify_id}` : null
}));
}
async function computeStats(range: Range) {
const days = RANGES[range];
const since = (days ? new Date(Date.now() - days * 86_400_000) : new Date(0)).toISOString();
const chartDays = days ?? 90;
const [totals, artists, tracks, albums, daily, streak, recent, sources] = await Promise.all([
const unit: Bucket = BUCKETS[range];
const today = localToday();
const from = days ? isoDay(dayMs(today) - (days - 1) * DAY_MS) : null;
const prevFrom = days && from ? isoDay(dayMs(from) - days * DAY_MS) : null;
// Fragments are single-use: build one per query.
const cur = () => (from ? sql`f.local_day >= ${from}::date` : sql`true`);
const prev = () => sql`f.local_day >= ${prevFrom}::date and f.local_day < ${from}::date`;
const step = `1 ${unit}`;
const tagShare =
() => sql`select artist_id, tag, weight::float8 / sum(weight) over (partition by artist_id) as share
from music.artist_tags`;
const totalsOf = (where: typeof cur) => sql`
select count(*)::int as plays, coalesce(sum(f.ms), 0)::float8 as ms,
count(distinct f.track_id)::int as tracks, count(distinct f.artist_id)::int as artists,
count(distinct f.album_id)::int as albums, count(distinct f.local_day)::int as days,
min(f.local_day)::text as first
from music.play_facts f where ${where()}`;
const ranksOf = (where: typeof cur, limit: number) => sql`
select f.artist_id as id, count(*)::int as plays,
(row_number() over (order by count(*) desc, f.artist_id))::int as rank
from music.play_facts f where ${where()} and f.artist_id is not null
group by f.artist_id order by plays desc, f.artist_id limit ${limit}`;
const [
totals,
prevTotals,
artists,
curRanks,
prevRanks,
tracks,
albums,
chart,
clock,
newArtists,
discovered,
streak,
milestones,
genres,
tagged,
genreTimeline,
sources
] = await Promise.all([
totalsOf(cur),
prevFrom ? totalsOf(prev) : Promise.resolve(null),
sql`
select count(*)::int as plays,
coalesce(sum(${playMs()}), 0)::float8 / 3600000 as hours,
count(distinct p.track_id)::int as tracks,
count(distinct t.artist_id)::int as artists,
min(p.played_at) as first
from music.plays_deduped p join music.tracks t on t.id = p.track_id
where p.played_at >= ${since}::timestamptz`,
sql`
select a.id, a.name, a.spotify_id, count(*)::int as plays,
coalesce(sum(${playMs()}), 0)::float8 / 3600000 as hours,
select a.id, a.name, a.spotify_id, x.plays, x.ms,
coalesce(a.image_url, (select coalesce(al.image_url_small, al.image_url) from music.albums al
where al.artist_id = a.id and al.image_url is not null
order by al.spotify_id is null, al.id desc limit 1)) as image
from music.plays_deduped p
join music.tracks t on t.id = p.track_id
join music.artists a on a.id = t.artist_id
where p.played_at >= ${since}::timestamptz
group by a.id order by plays desc, a.name limit 10`,
from (select f.artist_id, count(*)::int as plays, sum(f.ms)::float8 as ms
from music.play_facts f where ${cur()} and f.artist_id is not null
group by f.artist_id order by plays desc limit 10) x
join music.artists a on a.id = x.artist_id
order by x.plays desc, a.name`,
prevFrom ? ranksOf(cur, 100) : Promise.resolve([]),
prevFrom ? ranksOf(prev, 1000) : Promise.resolve([]),
sql`
select t.id, t.name, t.spotify_id, ${artistsOf()} as artists, count(*)::int as plays,
select t.id, t.name, t.spotify_id, ${artistsOf()} as artists, x.plays,
coalesce(al.image_url_small, al.image_url) as image
from music.plays_deduped p
join music.tracks t on t.id = p.track_id
from (select f.track_id, count(*)::int as plays from music.play_facts f where ${cur()}
group by f.track_id order by plays desc limit 10) x
join music.tracks t on t.id = x.track_id
left join music.artists a on a.id = t.artist_id
left join music.albums al on al.id = t.album_id
where p.played_at >= ${since}::timestamptz
group by t.id, a.name, al.id order by plays desc, t.name limit 10`,
order by x.plays desc, t.name`,
sql`
select al.id, al.name, al.spotify_id, a.name as artist, count(*)::int as plays,
coalesce(al.image_url_small, al.image_url) as image
from music.plays_deduped p
join music.tracks t on t.id = p.track_id
join music.albums al on al.id = t.album_id
select al.id, al.name, al.spotify_id, a.name as artist, x.plays, x.ms,
coalesce(al.image_url, al.image_url_small) as image
from (select f.album_id, count(*)::int as plays, sum(f.ms)::float8 as ms
from music.play_facts f where ${cur()} and f.album_id is not null
group by f.album_id order by plays desc limit 12) x
join music.albums al on al.id = x.album_id
left join music.artists a on a.id = al.artist_id
where p.played_at >= ${since}::timestamptz
group by al.id, a.name order by plays desc, al.name limit 10`,
order by x.plays desc, al.name`,
sql`
with days as (
select generate_series((now() at time zone ${TZ})::date - ${chartDays - 1}::int,
(now() at time zone ${TZ})::date, interval '1 day')::date as day)
select d.day::text as day, count(p.id)::int as plays,
coalesce(sum(${playMs()}), 0)::float8 / 3600000 as hours
from days d
left join music.plays_deduped p
on p.played_at >= now() - make_interval(days => ${chartDays + 1})
and (p.played_at at time zone ${TZ})::date = d.day
left join music.tracks t on t.id = p.track_id
group by d.day order by d.day`,
with b as (
select generate_series(
date_trunc(${unit}, coalesce(${from}::date, (select min(local_day) from music.play_facts), ${today}::date)),
date_trunc(${unit}, ${today}::date), ${step}::interval)::date as bucket)
select b.bucket::text as bucket, coalesce(x.plays, 0)::int as plays, coalesce(x.ms, 0)::float8 as ms
from b left join (
select date_trunc(${unit}, f.local_day)::date as bucket, count(*) as plays, sum(f.ms) as ms
from music.play_facts f where ${cur()} group by 1) x using (bucket)
order by b.bucket`,
sql`
with d as (select distinct (played_at at time zone ${TZ})::date as day from music.plays_deduped),
select f.dow, f.hour, count(*)::int as plays, sum(f.ms)::float8 as ms
from music.play_facts f where ${cur()} group by 1, 2`,
from
? sql`
with firsts as (
select artist_id, min(played_at) as first from music.play_facts
where artist_id is not null group by artist_id having min(local_day) >= ${from}::date)
select a.name, a.spotify_id, fi.first, count(*)::int as plays, (count(*) over ())::int as total
from firsts fi
join music.play_facts f on f.artist_id = fi.artist_id
join music.artists a on a.id = fi.artist_id
group by a.id, fi.first order by plays desc, a.name limit 10`
: Promise.resolve([]),
sql`
select extract(year from first)::int as year, count(*)::int as artists
from (select min(local_day) as first from music.play_facts where artist_id is not null
group by artist_id) x group by 1 order by 1`,
sql`
with d as (select distinct local_day as day from music.play_facts),
g as (select day, day - (row_number() over (order by day))::int as grp from d),
s as (select min(day) as start_day, max(day) as end_day, count(*)::int as len from g group by grp)
select
coalesce((select len from s where end_day >= (now() at time zone ${TZ})::date - 1
coalesce((select len from s where end_day >= ${today}::date - 1
order by end_day desc limit 1), 0) as current,
coalesce((select len from s order by len desc, end_day desc limit 1), 0) as longest,
(select start_day::text from s order by len desc, end_day desc limit 1) as longest_start,
(select end_day::text from s order by len desc, end_day desc limit 1) as longest_end,
(select count(*)::int from d) as days`,
sql`
select p.id, p.played_at, p.source, t.name as track, t.spotify_id, ${artistsOf()} as artists,
coalesce(al.image_url_small, al.image_url) as image
from music.plays_deduped p
join music.tracks t on t.id = p.track_id
select x.n::int as n, x.played_at, t.name as track, t.spotify_id, a.name as artist
from (select track_id, played_at, row_number() over (order by played_at, id) as n
from music.play_facts) x
join music.tracks t on t.id = x.track_id
left join music.artists a on a.id = t.artist_id
left join music.albums al on al.id = t.album_id
order by p.played_at desc limit 50`,
sql`select source, count(*)::int as plays from music.plays_deduped group by source order by plays desc`
where x.n = any(${MILESTONES}::int[]) order by x.n`,
sql`
with ah as (select f.artist_id, sum(f.ms)::float8 as ms from music.play_facts f
where ${cur()} and f.artist_id is not null group by 1),
tw as (${tagShare()})
select tw.tag, sum(ah.ms * tw.share) as ms, count(*)::int as artists
from ah join tw using (artist_id) group by tw.tag order by ms desc limit 20`,
sql`
select coalesce(sum(f.ms) filter (where exists (
select 1 from music.artist_tags at where at.artist_id = f.artist_id)), 0)::float8 as tagged,
coalesce(sum(f.ms), 0)::float8 as total
from music.play_facts f where ${cur()}`,
sql`
with ah as (select f.artist_id, sum(f.ms)::float8 as ms from music.play_facts f
where ${cur()} and f.artist_id is not null group by 1),
tw as (${tagShare()}),
top as (select tw.tag from ah join tw using (artist_id) group by tw.tag
order by sum(ah.ms * tw.share) desc limit 6),
bx as (select date_trunc(${unit}, f.local_day)::date as bucket, f.artist_id, sum(f.ms)::float8 as ms
from music.play_facts f where ${cur()} and f.artist_id is not null group by 1, 2)
select bx.bucket::text as bucket, tw.tag, sum(bx.ms * tw.share) as ms
from bx join (${tagShare()}) tw using (artist_id)
where tw.tag in (select tag from top) group by 1, 2`,
sql`select f.source, count(*)::int as plays from music.play_facts f where ${cur()}
group by f.source order by plays desc`
]);
const t = totals[0];
const pt = prevTotals?.[0];
const spanDays =
days ?? (t.first ? Math.round((dayMs(today) - dayMs(t.first as string)) / DAY_MS) + 1 : 1);
const prevRank = new Map(prevRanks.map((r) => [r.id as number, r]));
const curRank = new Map(curRanks.map((r) => [r.id as number, r]));
const names = new Map(artists.map((a) => [a.id as number, a.name as string]));
const topGenres = genres.slice(0, 6).map((g) => g.tag as string);
const timeline = new Map<string, Record<string, number>>();
for (const r of genreTimeline) {
const row = timeline.get(r.bucket as string) ?? {};
row[r.tag as string] = hoursOf(r.ms);
timeline.set(r.bucket as string, row);
}
// Movers: artists with the biggest gain in plays over the previous period.
const movers = curRanks
.map((r) => {
const p = prevRank.get(r.id as number);
return {
id: r.id as number,
rank: r.rank as number,
prevRank: (p?.rank as number) ?? null,
plays: r.plays as number,
prevPlays: (p?.plays as number) ?? 0
};
})
.filter((m) => m.plays > m.prevPlays)
.sort((a, b) => b.plays - b.prevPlays - (a.plays - a.prevPlays))
.slice(0, 5);
const moverIds = movers.map((m) => m.id).filter((id) => !names.has(id));
if (moverIds.length)
for (const r of await sql`select id, name from music.artists where id = any(${moverIds}::int[])`)
names.set(r.id as number, r.name as string);
return {
range,
unit,
from,
today,
totals: {
plays: t.plays as number,
hours: Number(t.hours),
hours: hoursOf(t.ms),
tracks: t.tracks as number,
artists: t.artists as number,
first: t.first ? new Date(t.first).toISOString() : null
albums: t.albums as number,
daysListened: t.days as number,
spanDays,
first: (t.first as string) ?? null
},
topArtists: artists.map((r) => ({
name: r.name as string,
plays: r.plays as number,
hours: Number(r.hours),
image: (r.image as string) ?? null,
url: r.spotify_id ? `https://open.spotify.com/artist/${r.spotify_id}` : null
})),
previous: pt
? {
plays: pt.plays as number,
hours: hoursOf(pt.ms),
artists: pt.artists as number,
tracks: pt.tracks as number
}
: null,
topArtists: artists.map((r) => {
const p = prevRank.get(r.id as number);
return {
name: r.name as string,
plays: r.plays as number,
hours: hoursOf(r.ms),
image: (r.image as string) ?? null,
url: r.spotify_id ? `https://open.spotify.com/artist/${r.spotify_id}` : null,
rank: (curRank.get(r.id as number)?.rank as number) ?? null,
prevRank: prevFrom ? ((p?.rank as number) ?? null) : undefined
};
}),
movers: movers.map((m) => ({ ...m, name: names.get(m.id) ?? '' })),
topTracks: tracks.map((r) => ({
name: r.name as string,
artists: r.artists as string,
@@ -211,14 +425,48 @@ export async function getMusicStats(range: Range) {
name: r.name as string,
artist: (r.artist as string) ?? '',
plays: r.plays as number,
hours: hoursOf(r.ms),
image: (r.image as string) ?? null,
url: r.spotify_id ? `https://open.spotify.com/album/${r.spotify_id}` : null
})),
daily: daily.map((r) => ({
day: r.day as string,
chart: chart.map((r) => ({
bucket: r.bucket as string,
plays: r.plays as number,
hours: Number(r.hours)
hours: hoursOf(r.ms)
})),
clock: clock.map((r) => ({
dow: r.dow as number,
hour: r.hour as number,
plays: r.plays as number,
hours: hoursOf(r.ms)
})),
newArtists: {
total: (newArtists[0]?.total as number) ?? 0,
top: newArtists.map((r) => ({
name: r.name as string,
plays: r.plays as number,
first: new Date(r.first).toISOString(),
url: r.spotify_id ? `https://open.spotify.com/artist/${r.spotify_id}` : null
}))
},
discoveredByYear: discovered.map((r) => ({
year: r.year as number,
artists: r.artists as number
})),
genres: {
top: genres.map((g) => ({
tag: g.tag as string,
hours: hoursOf(g.ms),
artists: g.artists as number
})),
coverage: Number(tagged[0].total) ? Number(tagged[0].tagged) / Number(tagged[0].total) : 0,
series: topGenres,
timeline: chart.map((r) => ({
bucket: r.bucket as string,
hours: hoursOf(r.ms),
tags: timeline.get(r.bucket as string) ?? {}
}))
},
streak: {
current: Number(streak[0].current),
longest: Number(streak[0].longest),
@@ -226,19 +474,19 @@ export async function getMusicStats(range: Range) {
longestEnd: (streak[0].longest_end as string) ?? null,
days: Number(streak[0].days)
},
recent: recent.map((r) => ({
id: r.id as number,
milestones: milestones.map((r) => ({
n: r.n as number,
at: new Date(r.played_at).toISOString(),
source: r.source as string,
track: r.track as string,
artists: r.artists as string,
image: (r.image as string) ?? null,
artist: (r.artist as string) ?? '',
url: r.spotify_id ? `https://open.spotify.com/track/${r.spotify_id}` : null
})),
sources: sources.map((r) => ({ source: r.source as string, plays: r.plays as number }))
};
}
export type MusicStats = Awaited<ReturnType<typeof computeStats>>;
/** Owner-only connection status for /trackers/music. Never includes tokens. */
export async function getSpotifyConnection() {
const [row] = await sql`