map finder: quick lists from my listening (top/recent/now/Spotify top/liked), Artist - Title without Spotify, owned playlists via the user token, rich beatmap cards (cover, mapper, status, BPM, length, star chips, preview, osu!direct), mode/status/star filters, DB result cache (music.osu_match_cache)
Some checks failed
Deploy production / deploy (push) Has been cancelled

This commit is contained in:
2026-10-24 16:15:56 +00:00
parent 0aa14cebf3
commit 46703f1a6c
21 changed files with 21050 additions and 178 deletions

View File

@@ -172,6 +172,13 @@ export const musicSpotifyAuth = musicSchema.table(
(t) => [check('spotify_auth_single_row', sql`${t.id} = 1`)]
);
/** Map finder (/projects/spotify-beatmap-finder): osu! search results per normalised track. */
export const musicOsuMatchCache = musicSchema.table('osu_match_cache', {
key: text('key').primaryKey(),
result: jsonb('result').notNull(),
searchedAt: timestamptz('searched_at').notNull().defaultNow()
});
/** Resumable backfill state, e.g. key 'lastfm:stanrunge' (see scripts/music-import-lastfm.ts). */
export const musicImportState = musicSchema.table('import_state', {
key: text('key').primaryKey(),

View File

@@ -0,0 +1,28 @@
import { client as sql } from '#lib/server/db/index.js';
import type { MatchStore } from '#lib/server/osu.js';
import type { ScoredBeatmapset } from '#lib/map-finder/types.js';
// osu! search results per normalised track, kept for a few days so repeat lookups (your
// top tracks, a playlist you check again) are instant and cost no osu! API calls.
const TTL_DAYS = 3;
export const matchStore: MatchStore = {
async get(key) {
const [row] = await sql`
select result from music.osu_match_cache
where key = ${key} and searched_at > now() - make_interval(days => ${TTL_DAYS})`;
if (!row) return null;
return (
typeof row.result === 'string' ? JSON.parse(row.result) : row.result
) as ScoredBeatmapset[];
},
async set(key, result) {
await sql`
insert into music.osu_match_cache (key, result, searched_at)
values (${key}, ${JSON.stringify(result)}::jsonb, now())
on conflict (key) do update set result = excluded.result, searched_at = now()`;
// Keep the table small: drop entries well past their TTL now and then.
if (Math.random() < 0.02)
await sql`delete from music.osu_match_cache where searched_at < now() - interval '30 days'`;
}
};

View File

@@ -0,0 +1,146 @@
import { client as sql } from '#lib/server/db/index.js';
import type { SpotifyCollection, SpotifyTrack } from '#lib/map-finder/types.js';
import { usableTrack } from '#lib/server/spotify.js';
import { ApiError } from '#lib/server/finder-http.js';
import { spotifyUserGet } from '#lib/server/music/user-token.js';
import { SpotifyError } from '#lib/music/collector.server.js';
// "Find maps for what I listen to": track lists from the listening tracker (music.* in the
// database, filled by the Spotify poller and the Last.fm import) and from Stan's Spotify
// account. History sources need no Spotify API call at all, only osu!.
export const SOURCES = {
top: { label: 'top tracks · 30 days', cost: 6, owner: false },
recent: { label: 'recently played', cost: 6, owner: false },
now: { label: 'now playing', cost: 1, owner: false },
'spotify-top': { label: 'all-time top (Spotify)', cost: 6, owner: false },
liked: { label: 'liked songs', cost: 6, owner: true }
} as const;
export type Source = keyof typeof SOURCES;
export const isSource = (value: unknown): value is Source =>
typeof value === 'string' && Object.hasOwn(SOURCES, value);
export const SOURCE_TRACKS = 30;
type Row = {
id: number;
spotify_id: string | null;
name: string;
artists: string[] | null;
album: string | null;
image: string | null;
note: string | null;
};
const artistsOf = () => sql`coalesce((select array_agg(ar.name order by ta.position)
from music.track_artists ta join music.artists ar on ar.id = ta.artist_id
where ta.track_id = t.id), array_remove(array[a.name], null))`;
function fromRow(row: Row): SpotifyTrack | null {
const artists = (row.artists ?? []).filter(Boolean).map((name) => ({ name }));
if (!artists.length || !row.name) return null;
return {
id: row.spotify_id ?? `db:${row.id}`,
name: row.name,
artists,
album: { name: row.album ?? '', images: row.image ? [{ url: row.image }] : [] },
url: row.spotify_id ? `https://open.spotify.com/track/${row.spotify_id}` : null,
note: row.note ?? undefined
};
}
async function history(source: 'top' | 'recent' | 'now'): Promise<SpotifyTrack[]> {
let rows: Row[];
if (source === 'top')
rows = await sql<Row[]>`
select t.id, t.spotify_id, t.name, ${artistsOf()} as artists, al.name as album,
coalesce(al.image_url_small, al.image_url) as image,
count(*)::text || case when count(*) = 1 then ' play' else ' plays' end as note
from music.plays_deduped p
join music.tracks t on t.id = p.track_id
left join music.artists a on a.id = t.artist_id
left join music.albums al on al.id = t.album_id
where p.played_at >= now() - interval '30 days'
group by t.id, a.name, al.id
order by count(*) desc, max(p.played_at) desc limit ${SOURCE_TRACKS}`;
else if (source === 'recent')
rows = await sql<Row[]>`
select t.id, t.spotify_id, t.name, ${artistsOf()} as artists, al.name as album,
coalesce(al.image_url_small, al.image_url) as image, null as note
from (select track_id, max(played_at) as at from music.plays_deduped
group by track_id order by at desc limit ${SOURCE_TRACKS}) p
join music.tracks t on t.id = p.track_id
left join music.artists a on a.id = t.artist_id
left join music.albums al on al.id = t.album_id
order by p.at desc`;
else
rows = await sql<Row[]>`
select * from (
select t.id, t.spotify_id, t.name, ${artistsOf()} as artists, al.name as album,
coalesce(al.image_url_small, al.image_url) as image, 'listening now' as note, 0 as rank
from music.now_playing n
join music.tracks t on t.id = n.track_id
left join music.artists a on a.id = t.artist_id
left join music.albums al on al.id = t.album_id
where n.is_playing and n.updated_at > now() - interval '3 minutes'
union all
(select t.id, t.spotify_id, t.name, ${artistsOf()}, al.name,
coalesce(al.image_url_small, al.image_url), 'last played', 1
from music.plays_deduped p
join music.tracks t on t.id = p.track_id
left join music.artists a on a.id = t.artist_id
left join music.albums al on al.id = t.album_id
order by p.played_at desc limit 1)
) x order by rank limit 1`;
return rows.map(fromRow).filter((t): t is SpotifyTrack => t !== null);
}
async function spotifyUser(source: 'spotify-top' | 'liked'): Promise<SpotifyTrack[]> {
let response: Record<string, unknown> | null;
try {
response = await spotifyUserGet(
source === 'liked'
? `me/tracks?limit=${SOURCE_TRACKS}`
: `me/top/tracks?time_range=long_term&limit=${SOURCE_TRACKS}`
);
} catch (e) {
if (e instanceof SpotifyError && (e.status === 401 || e.status === 403))
throw new ApiError(
'Spotify',
403,
source === 'liked'
? 'Liked songs need Spotify reconnected with library access (/api/spotify/connect).'
: 'Spotify refused the request. Reconnect Spotify at /api/spotify/connect.'
);
throw new ApiError('Spotify', 502, 'Spotify request failed. Please try again later.');
}
if (!response)
throw new ApiError(
'Spotify',
503,
'Spotify is not connected yet, so this list is unavailable.'
);
const items = Array.isArray(response.items) ? response.items : [];
const seen = new Set<string>();
const tracks: SpotifyTrack[] = [];
for (const item of items) {
const track = usableTrack(
source === 'liked' ? (item as Record<string, unknown> | null)?.track : item
);
if (track && !seen.has(track.id)) {
seen.add(track.id);
tracks.push({ ...track, url: `https://open.spotify.com/track/${track.id}` });
}
}
return tracks;
}
export async function getSourceTracks(
source: Source
): Promise<{ tracks: SpotifyTrack[]; collection: SpotifyCollection }> {
const tracks =
source === 'spotify-top' || source === 'liked'
? await spotifyUser(source)
: await history(source);
return {
tracks,
collection: { kind: 'library', id: source, name: `Stan's ${SOURCES[source].label}`, owner: '' }
};
}

View File

@@ -0,0 +1,45 @@
import { client as sql } from '#lib/server/db/index.js';
import { refreshAccessToken, spotifyGet } from '#lib/music/collector.server.js';
import { spotifyCredentials } from './poller';
// Spotify calls on Stan's behalf (his own playlists, liked songs, top tracks), using the
// authorization stored by /api/spotify/connect. Shares the access token with the poller.
let pending: Promise<string | null> | null = null;
async function fresh(): Promise<string | null> {
const creds = spotifyCredentials();
if (!creds.clientId || !creds.clientSecret) return null;
const [auth] = await sql`
select refresh_token, access_token, access_expires_at, needs_reauth
from music.spotify_auth where id = 1`.catch(() => []);
if (!auth || auth.needs_reauth) return null;
const expires = auth.access_expires_at ? new Date(auth.access_expires_at).getTime() : 0;
if (auth.access_token && expires > Date.now() + 60_000) return auth.access_token as string;
const token = await refreshAccessToken(fetch, creds, auth.refresh_token);
await sql`
update music.spotify_auth set access_token = ${token.access_token},
access_expires_at = now() + make_interval(secs => ${token.expires_in}),
refresh_token = coalesce(${token.refresh_token ?? null}, refresh_token),
scope = coalesce(${token.scope ?? null}, scope), updated_at = now()
where id = 1`;
return token.access_token;
}
/** A user access token, or null when Spotify is not connected (or needs reconnecting). */
export function spotifyUserToken(): Promise<string | null> {
pending ??= fresh().finally(() => (pending = null));
return pending;
}
/** GET an API path with the user token; null when not connected. */
export async function spotifyUserGet(path: string): Promise<Record<string, unknown> | null> {
const token = await spotifyUserToken();
if (!token) return null;
return ((await spotifyGet(fetch, token, path)) ?? {}) as Record<string, unknown>;
}
/** Scopes granted at connect time (to tell the owner to reconnect for new features). */
export async function grantedScopes(): Promise<string[]> {
const [row] = await sql`select scope from music.spotify_auth where id = 1`.catch(() => []);
return typeof row?.scope === 'string' ? row.scope.split(' ') : [];
}

View File

@@ -1,4 +1,4 @@
import { expect, it, vi } from 'vitest';
import { describe, expect, it, vi } from 'vitest';
vi.mock('$app/env/private', () => ({ OSU_CLIENT_ID: '1', OSU_CLIENT_SECRET: 'test' }));
import { getOsuMaps } from './osu';
it('falls back deliberately, merges IDs, reuses OAuth and caches matching results', async () => {
@@ -32,3 +32,31 @@ it('falls back deliberately, merges IDs, reuses OAuth and caches matching result
1
);
});
describe('parseBeatmapset', () => {
it('keeps the shown fields, fixes preview URLs and sorts difficulties', async () => {
const { parseBeatmapset } = await import('./osu');
const set = parseBeatmapset({
id: 7,
title: 'T',
artist: 'A',
creator: 'mapper',
status: 'ranked',
preview_url: '//b.ppy.sh/preview/7.mp3',
covers: { card: 'https://assets.ppy.sh/c.jpg', list: 'javascript:alert(1)' },
beatmaps: [
{ id: 2, mode: 'osu', difficulty_rating: 5.123, total_length: 90, version: 'Hard' },
{ id: 1, mode: 'osu', difficulty_rating: 2, total_length: 90, version: 'Easy' },
{ id: 3, mode: 'bogus' }
]
});
expect(set).toMatchObject({
id: 7,
creator: 'mapper',
preview_url: 'https://b.ppy.sh/preview/7.mp3'
});
expect(set?.covers?.list).toBeUndefined();
expect(set?.beatmaps?.map((d) => d.stars)).toEqual([2, 5.12]);
expect(parseBeatmapset({ id: 'x' })).toBeNull();
});
});

View File

@@ -1,5 +1,11 @@
import { OSU_CLIENT_ID, OSU_CLIENT_SECRET } from '$app/env/private';
import type { OsuBeatmapset, ScoredBeatmapset, SpotifyTrack } from '#lib/map-finder/types.js';
import type {
OsuBeatmapset,
OsuDifficulty,
OsuMode,
ScoredBeatmapset,
SpotifyTrack
} from '#lib/map-finder/types.js';
import {
cleanTitle,
normalize,
@@ -11,6 +17,12 @@ const token = createTokenCache();
const cache = new Map<string, { expires: number; result: ScoredBeatmapset[] }>();
const pending = new Map<string, Promise<ScoredBeatmapset[]>>();
export const MATCH_CONCURRENCY = 3;
/** Optional persistent cache (music.osu_match_cache) behind the in-memory one. */
export type MatchStore = {
get(key: string): Promise<ScoredBeatmapset[] | null>;
set(key: string, result: ScoredBeatmapset[]): Promise<void>;
};
export const osuConfigured = () => !!(OSU_CLIENT_ID && OSU_CLIENT_SECRET);
// Global permits also bound outbound work across simultaneous visitors.
let active = 0;
const queue: (() => void)[] = [];
@@ -25,10 +37,72 @@ async function limited<T>(work: () => Promise<T>): Promise<T> {
else active--;
}
}
const MODES = new Set<OsuMode>(['osu', 'taiko', 'fruits', 'mania']);
const str = (value: unknown) => (typeof value === 'string' ? value : undefined);
const num = (value: unknown) =>
typeof value === 'number' && Number.isFinite(value) ? value : undefined;
const https = (value: unknown) => {
const url = str(value);
// The API sends previews protocol-relative (//b.ppy.sh/preview/1.mp3).
if (url?.startsWith('//')) return `https:${url}`;
return url?.startsWith('https://') ? url : undefined;
};
function difficulty(value: unknown): OsuDifficulty | null {
if (!value || typeof value !== 'object') return null;
const d = value as Record<string, unknown>;
const mode = str(d.mode) as OsuMode | undefined;
if (!Number.isInteger(d.id) || !mode || !MODES.has(mode)) return null;
return {
id: d.id as number,
version: str(d.version) ?? '',
mode,
stars: Math.round((num(d.difficulty_rating) ?? 0) * 100) / 100,
length: num(d.total_length) ?? 0,
bpm: num(d.bpm),
status: str(d.status)
};
}
/** Keep only the fields the finder shows (the search response is large). */
export function parseBeatmapset(item: unknown): OsuBeatmapset | null {
if (!item || typeof item !== 'object') return null;
const b = item as Record<string, unknown>;
if (!Number.isInteger(b.id) || typeof b.title !== 'string' || typeof b.artist !== 'string')
return null;
const covers = (b.covers && typeof b.covers === 'object' ? b.covers : {}) as Record<
string,
unknown
>;
return {
id: b.id as number,
title: b.title,
artist: b.artist,
title_unicode: str(b.title_unicode),
artist_unicode: str(b.artist_unicode),
status: str(b.status),
creator: str(b.creator),
bpm: num(b.bpm),
play_count: num(b.play_count),
favourite_count: num(b.favourite_count),
preview_url: https(b.preview_url),
ranked_date: str(b.ranked_date) ?? null,
video: b.video === true,
nsfw: b.nsfw === true,
covers: {
card: https(covers.card),
'card@2x': https(covers['card@2x']),
list: https(covers.list)
},
beatmaps: Array.isArray(b.beatmaps)
? b.beatmaps
.map(difficulty)
.filter((d): d is OsuDifficulty => d !== null)
.sort((x, y) => x.mode.localeCompare(y.mode) || x.stars - y.stars)
: undefined
};
}
async function search(fetchFn: Fetch, query: string): Promise<OsuBeatmapset[]> {
const access = await token(() => {
if (!OSU_CLIENT_ID || !OSU_CLIENT_SECRET)
throw new ApiError('osu!', 503, 'osu! is not configured on this server.');
if (!osuConfigured()) throw new ApiError('osu!', 503, 'osu! is not configured on this server.');
return requestJson(fetchFn, 'osu!', 'https://osu.ppy.sh/oauth/token', {
method: 'POST',
headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
@@ -49,44 +123,45 @@ async function search(fetchFn: Fetch, query: string): Promise<OsuBeatmapset[]> {
if (!Array.isArray(response?.beatmapsets))
throw new ApiError('osu!', 502, 'osu! returned an invalid search response.');
return response.beatmapsets
.filter(
(item): item is OsuBeatmapset =>
!!item &&
typeof item === 'object' &&
Number.isInteger((item as OsuBeatmapset).id) &&
typeof (item as OsuBeatmapset).title === 'string' &&
typeof (item as OsuBeatmapset).artist === 'string'
)
.map((item) => ({
id: item.id,
title: item.title,
artist: item.artist,
title_unicode: typeof item.title_unicode === 'string' ? item.title_unicode : undefined,
artist_unicode: typeof item.artist_unicode === 'string' ? item.artist_unicode : undefined,
status: typeof item.status === 'string' ? item.status : undefined,
covers: { card: typeof item.covers?.card === 'string' ? item.covers.card : undefined },
beatmaps: Array.isArray(item.beatmaps) ? item.beatmaps.map(() => ({ id: 0 })) : undefined
}));
.map(parseBeatmapset)
.filter((item): item is OsuBeatmapset => item !== null);
}
export async function getOsuMaps(fetchFn: Fetch, track: SpotifyTrack): Promise<ScoredBeatmapset[]> {
const key = JSON.stringify([normalize(track.name), track.artists.map((a) => normalize(a.name))]);
const cacheError = (error: unknown) => {
console.warn('osu! match cache unavailable:', (error as Error)?.message);
return null;
};
export const matchCacheKey = (track: SpotifyTrack) =>
JSON.stringify([normalize(track.name), track.artists.map((a) => normalize(a.name))]);
export async function getOsuMaps(
fetchFn: Fetch,
track: SpotifyTrack,
store?: MatchStore
): Promise<ScoredBeatmapset[]> {
const key = matchCacheKey(track);
for (const [key, entry] of cache) if (entry.expires <= Date.now()) cache.delete(key);
const hit = cache.get(key);
if (hit) return hit.result;
const existing = pending.get(key);
if (existing) return existing;
const work = limited(async () => {
const maps = await search(fetchFn, `${track.artists[0].name} ${track.name}`);
let ranked = rankCandidates(track, maps);
if (!ranked.some((m) => m.score >= MEDIUM_CONFIDENCE))
ranked = rankCandidates(track, [
...maps,
...(await search(fetchFn, cleanTitle(track.name) || track.name))
]);
const remember = (result: ScoredBeatmapset[]) => {
if (cache.size >= 500) cache.delete(cache.keys().next().value!);
cache.set(key, { expires: Date.now() + 5 * 60_000, result: ranked });
return ranked;
}).finally(() => pending.delete(key));
cache.set(key, { expires: Date.now() + 5 * 60_000, result });
return result;
};
const work = (async () => {
const stored = await store?.get(key).catch(cacheError);
if (stored) return remember(stored);
const ranked = await limited(async () => {
const title = cleanTitle(track.name) || track.name;
const maps = await search(fetchFn, `${track.artists[0].name} ${title}`);
let ranked = rankCandidates(track, maps);
if (!ranked.some((m) => m.score >= MEDIUM_CONFIDENCE))
ranked = rankCandidates(track, [...maps, ...(await search(fetchFn, title))]);
return ranked;
});
await store?.set(key, ranked).catch(cacheError);
return remember(ranked);
})().finally(() => pending.delete(key));
pending.set(key, work);
return work;
}

View File

@@ -39,7 +39,8 @@ export function usableTrack(value: unknown): SpotifyTrack | null {
id: track.id,
name: track.name,
artists,
album: { name: typeof album.name === 'string' ? album.name : '', images }
album: { name: typeof album.name === 'string' ? album.name : '', images },
url: `https://open.spotify.com/track/${track.id}`
};
}
export function playlistTracks(items: unknown[], seen = new Set<string>()): SpotifyTrack[] {
@@ -55,19 +56,31 @@ export function playlistTracks(items: unknown[], seen = new Set<string>()): Spot
}
return result;
}
async function spotify(fetchFn: Fetch, path: string) {
const access = await token(() => {
if (!SPOTIFY_CLIENT_ID || !SPOTIFY_CLIENT_SECRET)
throw new ApiError('Spotify', 503, 'Spotify is not configured on this server.');
return requestJson(fetchFn, 'Spotify', 'https://accounts.spotify.com/api/token', {
method: 'POST',
headers: {
Authorization: `Basic ${btoa(`${SPOTIFY_CLIENT_ID}:${SPOTIFY_CLIENT_SECRET}`)}`,
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams({ grant_type: 'client_credentials' })
});
});
// Development-mode Spotify apps (2026) only return playlist contents to a user token, and
// only for playlists that user owns or collaborates on: use Stan's connected account.
async function userToken(): Promise<string | null> {
try {
const { spotifyUserToken } = await import('./music/user-token');
return await spotifyUserToken();
} catch {
return null;
}
}
async function spotify(fetchFn: Fetch, path: string, user: string | null = null) {
const access =
user ??
(await token(() => {
if (!SPOTIFY_CLIENT_ID || !SPOTIFY_CLIENT_SECRET)
throw new ApiError('Spotify', 503, 'Spotify is not configured on this server.');
return requestJson(fetchFn, 'Spotify', 'https://accounts.spotify.com/api/token', {
method: 'POST',
headers: {
Authorization: `Basic ${btoa(`${SPOTIFY_CLIENT_ID}:${SPOTIFY_CLIENT_SECRET}`)}`,
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams({ grant_type: 'client_credentials' })
});
}));
return record(
await requestJson(fetchFn, 'Spotify', `https://api.spotify.com/v1/${path}`, {
headers: { Authorization: `Bearer ${access}` }
@@ -78,7 +91,8 @@ export async function getSpotifyMetadata(
fetchFn: Fetch,
input:
| Extract<SpotifyInput, { kind: 'track' | 'playlist' | 'album' }>
| Extract<SpotifyInput, { kind: 'search' }>
| Extract<SpotifyInput, { kind: 'search' }>,
options: { owner?: boolean } = {}
): Promise<{ tracks: SpotifyTrack[]; collection?: SpotifyCollection; truncated: boolean }> {
if (input.kind === 'track' || input.kind === 'search') {
let response: RecordValue;
@@ -137,7 +151,14 @@ export async function getSpotifyMetadata(
: undefined
};
// Extended-quota responses use tracks/track; newer development-mode responses use items/item.
const modern = metadata.items !== undefined;
let user: string | null = null;
if (!isAlbum && metadata.items === undefined && metadata.tracks === undefined) {
if (metadata.public === false && !options.owner)
throw new ApiError('Spotify', 403, 'Private playlist.');
user = await userToken();
if (!user) throw new ApiError('Spotify', 403, 'Playlist contents need a user token.');
}
const modern = metadata.items !== undefined || user !== null;
let page = record(modern ? metadata.items : metadata.tracks);
let offset = 0;
const tracks: SpotifyTrack[] = [];
@@ -146,7 +167,8 @@ export async function getSpotifyMetadata(
if (!Array.isArray(page.items))
page = await spotify(
fetchFn,
`${resource}/${input.id}/${modern ? 'items' : 'tracks'}?limit=50&offset=${offset}`
`${resource}/${input.id}/${modern ? 'items' : 'tracks'}?limit=50&offset=${offset}`,
user
);
if (!Array.isArray(page.items))
throw new ApiError(
@@ -181,7 +203,7 @@ export async function getSpotifyMetadata(
error.status,
input.kind === 'album'
? 'Spotify album unavailable or not found. Try an individual track.'
: 'Spotify playlist unavailable or private, or this application does not have playlist access. Try a public playlist or an individual track.'
: "Spotify only lets this site read playlists that Stan owns (Spotify's 2026 API limits for small apps). Try a track or album link, or one of the quick lists below."
);
throw error;
}