feat: enhance IRC lobby management with reconnect handling and health check integration

src/lib/server/bancho/client.ts

@@ -8,6 +8,7 @@ function sleep(ms: number) {
 
 let client: any = null;
 let connecting: Promise<void> | null = null;
+let reconnecting = false;
 
 async function getClient() {
 	if (client?.isConnected?.()) return client;
@@ -25,13 +26,54 @@ async function getClient() {
 		banchoModule.BanchoClient ??
 		banchoModule.default?.BanchoClient ??
 		banchoModule.default;
+
 	client = new BanchoClient({
 		username: env.OSU_IRC_USERNAME,
-		password: env.OSU_IRC_PASSWORD
+		password: env.OSU_IRC_PASSWORD,
+		// bancho.js has built-in reconnect but we need to handle channel rejoin
+		apiKey: undefined
+	});
+
+	// ── Lifecycle logging ──
+	client.on('connected', () => {
+		console.log('[Bancho] Connected as', env.OSU_IRC_USERNAME);
+	});
+
+	client.on('disconnected', () => {
+		console.warn('[Bancho] Disconnected from IRC');
+	});
+
+	client.on('error', (err: any) => {
+		console.error('[Bancho] IRC error:', err?.message ?? err);
+	});
+
+	// ── Reconnect: re-join all active lobby channels ──
+	// bancho.js fires 'connected' again after an auto-reconnect
+	let initialConnect = true;
+	client.on('connected', async () => {
+		if (initialConnect) {
+			initialConnect = false;
+			return; // skip the first connect, lobbies haven't been created yet
+		}
+
+		reconnecting = true;
+		console.log('[Bancho] Reconnected — re-joining', lobbies.size, 'active lobby channels');
+
+		for (const [matchId, lobby] of lobbies.entries()) {
+			try {
+				await lobby._rejoinChannel(client);
+				console.log(`[Bancho] Re-joined channel for match ${matchId}`);
+			} catch (err: any) {
+				console.error(`[Bancho] Failed to re-join channel for match ${matchId}:`, err.message);
+				// Channel may have been closed by Bancho while we were offline
+				// Mark it as dead so orchestrator knows
+				lobby._dead = true;
+			}
+		}
+		reconnecting = false;
 	});
 
 	connecting = client.connect().then(() => {
-		console.log('[Bancho] Connected as', env.OSU_IRC_USERNAME);
 		connecting = null;
 	});
 
@@ -39,6 +81,20 @@ async function getClient() {
 	return client;
 }
 
+/**
+ * Check if the IRC client is currently connected.
+ */
+export function isBanchoConnected(): boolean {
+	return client?.isConnected?.() ?? false;
+}
+
+/**
+ * Check if we're in the middle of a reconnect (channels may be temporarily unavailable).
+ */
+export function isBanchoReconnecting(): boolean {
+	return reconnecting;
+}
+
 // ── Types ───────────────────────────────────────────────────────────────
 
 export interface PlayerScore {
@@ -54,6 +110,12 @@ export class TournamentLobby {
 	private channelName = '';
 	public osuMatchId = 0;
 
+	/** Set to true if the channel couldn't be re-joined after reconnect */
+	public _dead = false;
+
+	// The message handler function (stored so we can re-attach after reconnect)
+	private messageHandler: ((msg: any) => void) | null = null;
+
 	// Score collection
 	private collectedScores: PlayerScore[] = [];
 	private matchFinishedResolve: ((scores: PlayerScore[]) => void) | null = null;
@@ -103,8 +165,44 @@ export class TournamentLobby {
 	private async joinChannel(c: any) {
 		this.channel = c.getChannel(this.channelName);
 		await this.channel.join();
+		this.attachMessageHandler();
+	}
 
-		this.channel.on('message', (msg: any) => {
+	/**
+	 * Re-join the channel after a reconnect.
+	 * Called internally by the reconnect handler.
+	 */
+	async _rejoinChannel(c: any) {
+		if (!this.channelName) return;
+
+		// Remove old listener if any
+		if (this.channel && this.messageHandler) {
+			try {
+				this.channel.removeListener('message', this.messageHandler);
+			} catch {
+				/* old channel may be completely dead */
+			}
+		}
+
+		this.channel = c.getChannel(this.channelName);
+		await this.channel.join();
+		this.attachMessageHandler();
+		this._dead = false;
+	}
+
+	private attachMessageHandler() {
+		if (!this.channel) return;
+
+		// Remove previous handler if exists (prevent duplicates)
+		if (this.messageHandler) {
+			try {
+				this.channel.removeListener('message', this.messageHandler);
+			} catch {
+				/* ignore */
+			}
+		}
+
+		this.messageHandler = (msg: any) => {
 			const text: string = msg.message ?? String(msg);
 			const sender: string = msg.user?.ircUsername ?? '';
 
@@ -113,7 +211,9 @@ export class TournamentLobby {
 			} else if (sender) {
 				this.parsePlayerMessage(sender, text);
 			}
-		});
+		};
+
+		this.channel.on('message', this.messageHandler);
 	}
 
 	// ── Message Parsing ─────────────────────────────────────────────────
@@ -165,6 +265,13 @@ export class TournamentLobby {
 			}
 			return;
 		}
+
+		// Room closed (by Bancho or timeout)
+		if (text.includes('Closed the match')) {
+			console.log(`[Bancho] Room ${this.channelName} was closed`);
+			this._dead = true;
+			return;
+		}
 	}
 
 	private parsePlayerMessage(sender: string, text: string) {
@@ -184,8 +291,30 @@ export class TournamentLobby {
 
 	// ── IRC Commands ────────────────────────────────────────────────────
 
+	private ensureAlive() {
+		if (this._dead) {
+			throw new Error(`Lobby ${this.channelName} is dead (closed or lost after reconnect)`);
+		}
+		if (!this.channel) {
+			throw new Error('Lobby not created');
+		}
+	}
+
 	async send(cmd: string) {
-		if (!this.channel) throw new Error('Lobby not created');
+		this.ensureAlive();
+
+		// If we're in the middle of reconnecting, wait a bit
+		if (reconnecting) {
+			console.log(`[Bancho] Waiting for reconnect before sending to ${this.channelName}...`);
+			const start = Date.now();
+			while (reconnecting && Date.now() - start < 10_000) {
+				await sleep(500);
+			}
+			if (reconnecting) {
+				throw new Error('Timed out waiting for IRC reconnect');
+			}
+		}
+
 		console.log(`[Bancho] ${this.channelName} > ${cmd}`);
 		await this.channel.sendMessage(cmd);
 	}
@@ -219,9 +348,13 @@ export class TournamentLobby {
 
 	async close() {
 		try {
+			if (this._dead) {
+				console.log(`[Bancho] Lobby ${this.channelName} already dead, skipping close`);
+				return;
+			}
 			await this.send('!mp close');
-		} catch {
+		} catch (err: any) {
-			/* lobby may already be closed */
+			console.warn(`[Bancho] Failed to close ${this.channelName}:`, err.message);
 		}
 	}
 
@@ -229,6 +362,13 @@ export class TournamentLobby {
 		await this.send(message);
 	}
 
+	/**
+	 * Check if this lobby is still usable.
+	 */
+	get isAlive(): boolean {
+		return !this._dead && this.channel != null;
+	}
+
 	// ── Async Waiters ───────────────────────────────────────────────────
 
 	/**
@@ -271,11 +411,50 @@ export class TournamentLobby {
 const lobbies = new Map<string, TournamentLobby>();
 
 export function getLobby(matchId: string) {
-	return lobbies.get(matchId);
+	const lobby = lobbies.get(matchId);
+	if (lobby?._dead) {
+		console.warn(`[Bancho] Lobby for match ${matchId} is dead, removing from registry`);
+		lobbies.delete(matchId);
+		return undefined;
+	}
+	return lobby;
 }
+
 export function setLobby(matchId: string, lobby: TournamentLobby) {
 	lobbies.set(matchId, lobby);
 }
+
 export function removeLobby(matchId: string) {
 	lobbies.delete(matchId);
 }
+
+/**
+ * Get count of active (non-dead) lobbies.
+ * Useful for checking against the 4-lobby limit for non-bot accounts.
+ */
+export function getActiveLobbyCount(): number {
+	let count = 0;
+	for (const [id, lobby] of lobbies.entries()) {
+		if (lobby._dead) {
+			lobbies.delete(id);
+		} else {
+			count++;
+		}
+	}
+	return count;
+}
+
+/**
+ * Close all active lobbies. Useful for graceful shutdown.
+ */
+export async function closeAllLobbies() {
+	console.log(`[Bancho] Closing all ${lobbies.size} active lobbies...`);
+	for (const [matchId, lobby] of lobbies.entries()) {
+		try {
+			await lobby.close();
+		} catch (err: any) {
+			console.warn(`[Bancho] Failed to close lobby ${matchId}:`, err.message);
+		}
+		lobbies.delete(matchId);
+	}
+}

src/lib/server/match/orchestrator.ts

@@ -3,18 +3,42 @@ import { match, matchGame, mappoolSlot, user } from '$lib/server/db/schema';
 import { eq, and } from 'drizzle-orm';
 import { MATCH_STATES, type MatchConfig } from './types';
 import { getMatchFull, submitRoll, pickMap, submitGameScores } from './engine';
-import { TournamentLobby, setLobby, getLobby, removeLobby } from '../bancho/client';
+import {
+	TournamentLobby,
+	setLobby,
+	getLobby,
+	removeLobby,
+	getActiveLobbyCount,
+	isBanchoConnected,
+	isBanchoReconnecting
+} from '../bancho/client';
 
 function sleep(ms: number) {
 	return new Promise<void>((r) => setTimeout(r, ms));
 }
 
+/** Max concurrent lobbies for non-bot accounts */
+const MAX_LOBBIES = 4;
+
 /**
  * Create IRC lobby → configure → invite players → wire up chat handlers → move to ROLLING.
  */
 export async function initMatchLobby(matchId: string) {
 	const m = await getMatchFull(matchId);
 	const config = m.config as MatchConfig;
+
+	// Check lobby limit
+	const activeCount = getActiveLobbyCount();
+	if (activeCount >= MAX_LOBBIES) {
+		console.error(
+			`[Orchestrator] Cannot create lobby: ${activeCount}/${MAX_LOBBIES} lobbies active. ` +
+			`Close some matches first.`
+		);
+		throw new Error(
+			`Lobby limit reached (${activeCount}/${MAX_LOBBIES}). Close or finish existing matches first.`
+		);
+	}
+
 	const lobby = new TournamentLobby();
 
 	const lobbyName = `VASH: (${m.participants[0]?.team.name}) vs (${m.participants[1]?.team.name})`;
@@ -216,6 +240,12 @@ export async function playPickedMap(matchId: string, matchGameId: string) {
 		return;
 	}
 
+	if (!lobby.isAlive) {
+		console.warn('[Orchestrator] Lobby for', matchId, 'is dead — removing');
+		removeLobby(matchId);
+		return;
+	}
+
 	const game = await db.query.matchGame.findFirst({
 		where: eq(matchGame.id, matchGameId),
 		with: { slot: true }
@@ -244,7 +274,11 @@ export async function playPickedMap(matchId: string, matchGameId: string) {
 	} catch (err: any) {
 		// If ready times out, inform and don't start
 		console.warn('[Orchestrator] Ready timeout:', err.message);
+		try {
 			await lobby.chat('Ready timed out. Use the web UI to force start, or ready up!');
+		} catch {
+			// Lobby may be dead after timeout
+		}
 		return;
 	}
 
@@ -262,6 +296,11 @@ export async function forceStartGame(matchId: string) {
 	const lobby = getLobby(matchId);
 	if (!lobby) return;
 
+	if (!lobby.isAlive) {
+		console.warn('[Orchestrator] Cannot force start — lobby is dead');
+		return;
+	}
+
 	await lobby.chat('Force starting in 10s!');
 	await lobby.startGame(10);
 }
@@ -302,9 +341,13 @@ async function collectScores(matchId: string, matchGameId: string, lobby: Tourna
 	const updated = await getMatchFull(matchId);
 	if (updated.state === MATCH_STATES.FINISHED) {
 		const winner = updated.participants.find((p) => p.teamId === updated.winnerId);
+		try {
 			await lobby.chat(`GG! ${winner?.team.name ?? '?'} wins the match!`);
 			await sleep(5000);
 			await lobby.close();
+		} catch {
+			/* lobby may already be closed */
+		}
 		removeLobby(matchId);
 	} else {
 		// Announce next picker
@@ -313,9 +356,13 @@ async function collectScores(matchId: string, matchGameId: string, lobby: Tourna
 		);
 		const nextIdx = updated.games.length % sorted.length;
 		const nextPicker = sorted[nextIdx];
+		try {
 			await lobby.chat(
 				`${nextPicker?.team.name}'s turn to pick. Use !pick <slot> (e.g. !pick HD1) or pick in web UI.`
 			);
+		} catch {
+			/* lobby may be dead */
+		}
 	}
 }
 

src/routes/api/health/+server.ts

@@ -1,7 +1,7 @@
-// src/routes/api/health/+server.ts
 import { json } from '@sveltejs/kit';
 import { db } from '$lib/server/db';
 import { sql } from 'drizzle-orm';
+import { isBanchoConnected, isBanchoReconnecting, getActiveLobbyCount } from '$lib/server/bancho/client';
 import type { RequestHandler } from './$types';
 
 export const GET: RequestHandler = async () => {
@@ -12,7 +12,13 @@ export const GET: RequestHandler = async () => {
 		return json({
 			status: 'ok',
 			timestamp: new Date().toISOString(),
-			uptime: process.uptime()
+			uptime: process.uptime(),
+			irc: {
+				connected: isBanchoConnected(),
+				reconnecting: isBanchoReconnecting(),
+				activeLobbies: getActiveLobbyCount(),
+				maxLobbies: 4
+			}
 		});
 	} catch (e: any) {
 		return json(