music: Last.fm-style stats with 7d/30d/1y/all ranges and genres
All checks were successful
Deploy production / deploy (push) Successful in 1m14s
All checks were successful
Deploy production / deploy (push) Successful in 1m14s
- Range picker (?range=7d|30d|1y|all) drives every section; the hours chart covers the whole range (daily for 7d/30d, weekly for 1y, monthly for all). - Genres from Last.fm artist top tags (music.artist_tags, normalised and filtered), split by artist listening time, with a per-bucket stacked chart. - Listening clock, new artists, comparison with the previous period and top movers, streaks, milestones, top albums with art, rank changes. - music.play_facts materialized view (refreshed by the app when plays arrive) plus a per-range cache keep the page fast. - scripts/music-fetch-tags.ts backfills tags, most-played artists first.
This commit is contained in:
85
scripts/music-fetch-tags.ts
Normal file
85
scripts/music-fetch-tags.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
// Fetch Last.fm artist top tags into music.artist_tags (genres for /trackers/music).
|
||||
//
|
||||
// LASTFM_API_KEY=... bun scripts/music-fetch-tags.ts [--limit N] [--sleep 250] [--refetch-days 90]
|
||||
//
|
||||
// Most-played artists first (all time). Artists fetched within --refetch-days are skipped
|
||||
// ('not_found' is retried after 30 days), so a re-run only picks up new artists. Tags are
|
||||
// normalised and filtered (src/lib/music/tags.ts); the top 5 genres per artist are kept.
|
||||
// Rate limit: one request per --sleep ms (Last.fm asks for at most 5/s), back-off on 429/5xx.
|
||||
import { pickTags } from '../src/lib/music/tags';
|
||||
import { arg, musicDb, sleep } from './music-db';
|
||||
|
||||
const apiKey = process.env.LASTFM_API_KEY;
|
||||
if (!apiKey) throw new Error('Set LASTFM_API_KEY');
|
||||
const limit = Number(arg('limit') ?? 100_000);
|
||||
const pause = Number(arg('sleep') ?? 250);
|
||||
const refetchDays = Number(arg('refetch-days') ?? 90);
|
||||
|
||||
type LastfmTags = { toptags?: { tag?: { name: string; count: number }[] }; error?: number };
|
||||
|
||||
async function topTags(name: string): Promise<LastfmTags> {
|
||||
const url = `https://ws.audioscrobbler.com/2.0/?${new URLSearchParams({
|
||||
method: 'artist.gettoptags',
|
||||
artist: name,
|
||||
autocorrect: '1',
|
||||
api_key: apiKey!,
|
||||
format: 'json'
|
||||
})}`;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(15_000) }).catch(() => null);
|
||||
const json = response ? ((await response.json().catch(() => ({}))) as LastfmTags) : {};
|
||||
const retry =
|
||||
!response ||
|
||||
response.status === 429 ||
|
||||
response.status >= 500 ||
|
||||
json.error === 29 ||
|
||||
json.error === 8 ||
|
||||
json.error === 11 ||
|
||||
json.error === 16;
|
||||
if (!retry || attempt >= 4) return json;
|
||||
await sleep(2_000 * 2 ** attempt);
|
||||
}
|
||||
}
|
||||
|
||||
const sql = musicDb();
|
||||
try {
|
||||
const artists = await sql`
|
||||
select a.id, a.name, count(*)::int as plays
|
||||
from music.plays p join music.tracks t on t.id = p.track_id
|
||||
join music.artists a on a.id = t.artist_id
|
||||
left join music.artist_tag_fetch tf on tf.artist_id = a.id
|
||||
where tf.artist_id is null
|
||||
or (tf.status = 'ok' and tf.fetched_at < now() - make_interval(days => ${refetchDays}))
|
||||
or tf.status like 'error%'
|
||||
or (tf.status <> 'ok' and tf.fetched_at < now() - interval '30 days')
|
||||
group by a.id order by plays desc, a.id limit ${limit}`;
|
||||
console.log(`${artists.length} artists to fetch.`);
|
||||
let tagged = 0;
|
||||
for (const [i, artist] of artists.entries()) {
|
||||
const json = await topTags(artist.name as string);
|
||||
const tags = pickTags(json.toptags?.tag ?? []);
|
||||
const status =
|
||||
json.error === 6
|
||||
? 'not_found'
|
||||
: json.error
|
||||
? `error ${json.error}`
|
||||
: tags.length
|
||||
? 'ok'
|
||||
: 'no_tags';
|
||||
await sql.begin(async (tx) => {
|
||||
if (!json.error || json.error === 6) {
|
||||
await tx`delete from music.artist_tags where artist_id = ${artist.id}`;
|
||||
for (const t of tags)
|
||||
await tx`insert into music.artist_tags (artist_id, tag, weight) values (${artist.id}, ${t.tag}, ${t.weight})`;
|
||||
}
|
||||
await tx`insert into music.artist_tag_fetch (artist_id, status) values (${artist.id}, ${status})
|
||||
on conflict (artist_id) do update set status = excluded.status, fetched_at = now()`;
|
||||
});
|
||||
if (tags.length) tagged++;
|
||||
if ((i + 1) % 250 === 0) console.log(` ${i + 1}/${artists.length} (${tagged} with genres)`);
|
||||
await sleep(pause);
|
||||
}
|
||||
console.log(`Done: ${tagged} of ${artists.length} artists got genres.`);
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
Reference in New Issue
Block a user