- streamer/windows: vash-broadcast.exe (setup/login/start/stop/status/doctor) with per-instance OBS, tosu and FFmpeg, a supervised worker, OCR room selection and OBS scene reconciliation; built and tested on Linux only - streamer/relay.ts shared with Vash TV; Cloudflare Access proxy for dev/staging - worker status validation and live score helpers for the site - STREAMING_HANDOFF.md and streamer/ALPHA_TEST.md record what is unverified Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
901 lines
30 KiB
TypeScript
901 lines
30 KiB
TypeScript
/**
|
||
* The unattended worker for one instance, running in the signed-in desktop:
|
||
*
|
||
* Vash (/ws/agent) ──follow──▶ osu!tourney: right team size, right room
|
||
* tosu ──live scores──▶ Vash; tosu ──team names──▶ room confirmed
|
||
* Vash (/stream) ──Go live──▶ relay listening ──▶ OBS streams into it
|
||
* OBS: overlay over the captured tourney window, AMD hardware encoding
|
||
* relay ──▶ every destination + HLS recording on the Vash site
|
||
*
|
||
* It restarts whatever stops (OBS, tosu, osu!tourney, the relay), reports what
|
||
* it is doing to Vash and to state.json, and stops cleanly when asked.
|
||
*/
|
||
import { existsSync } from 'node:fs';
|
||
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
||
import { dirname, join } from 'node:path';
|
||
import { VashLink, type Follow } from '../link';
|
||
import { Obs } from '../obs';
|
||
import { Broadcaster, RtmpIngest } from '../relay';
|
||
import { TosuFeed } from '../tosu';
|
||
import { VashApi, type StreamConfig } from '../vash-api';
|
||
import {
|
||
accessHeaders,
|
||
layout,
|
||
loadConfig,
|
||
readJson,
|
||
secret,
|
||
VERSION,
|
||
writeJson,
|
||
type InstanceConfig,
|
||
type Layout
|
||
} from './config';
|
||
import { authorized, callWorker, type Runtime } from './control';
|
||
import { WindowsDesktop, type Desktop, type SessionInfo } from './desktop';
|
||
import { acquireLock, lockOwner, releaseLock } from './lock';
|
||
import { fileLogger } from './log';
|
||
import { ObsScene } from './obs-control';
|
||
import { encoderFromLog, latestObsLog, OBS_ARGS, writeObsSettings } from './obs-files';
|
||
import { startAccessProxy } from './proxy';
|
||
import { Supervised, type Child } from './supervise';
|
||
import { normalModeFor, pickMainWindow, readSavedLogin, TourneyController } from './tourney';
|
||
import { startViewer } from './viewer';
|
||
|
||
export interface WorkerState {
|
||
instance: string;
|
||
version: string;
|
||
pid: number;
|
||
state: string;
|
||
detail: string | null;
|
||
updatedAt: string;
|
||
startedAt: string;
|
||
onAir: boolean;
|
||
live: boolean;
|
||
obsStreaming: boolean;
|
||
relayReceiving: boolean;
|
||
encoder: string | null;
|
||
match: {
|
||
id: string | null;
|
||
title: string | null;
|
||
lobbyId: number | null;
|
||
roomName: string | null;
|
||
} | null;
|
||
tourney: { state: string; detail: string | null; verified: boolean };
|
||
components: Record<string, unknown>;
|
||
destinations: { label: string; state: string; error: string | null }[];
|
||
vash: { connected: boolean; rejected: boolean; channel: string | null; lastError: string | null };
|
||
desktop: SessionInfo | null;
|
||
warnings: string[];
|
||
}
|
||
|
||
/** The same program, as a command line: the compiled exe, or bun + this script in development. */
|
||
export function selfCommand(...args: string[]) {
|
||
const compiled =
|
||
Bun.main.startsWith('B:/~BUN') || Bun.main.startsWith('/$bunfs') || Bun.main.includes('~BUN');
|
||
return compiled ? [process.execPath, ...args] : [process.execPath, Bun.main, ...args];
|
||
}
|
||
|
||
const childOf = (process: import('bun').Subprocess): Child => ({
|
||
pid: process.pid,
|
||
exited: process.exited,
|
||
kill: () => process.kill()
|
||
});
|
||
|
||
/** Stops every process whose program lives in this instance's folder. */
|
||
export async function stopInstanceProcesses(
|
||
desktop: Desktop,
|
||
paths: Layout,
|
||
log: (...parts: unknown[]) => void
|
||
) {
|
||
const processes = await desktop.processes(paths.dir);
|
||
if (!processes.length) return 0;
|
||
log(`Stopping ${processes.length} leftover process(es) of this instance`);
|
||
return desktop.kill(processes.map((process) => process.pid));
|
||
}
|
||
|
||
/** Runs until asked to stop. Tests pass a fake desktop; Windows uses the PowerShell helper. */
|
||
export async function runWorker(name: string, desktop?: Desktop): Promise<number> {
|
||
const config = await loadConfig(name);
|
||
const paths = layout(name);
|
||
const log = fileLogger(paths.log);
|
||
log(
|
||
`vash-broadcast ${VERSION} worker starting for "${name}": ${config.server}, channel ${config.channel}`
|
||
);
|
||
const worker = new Worker(
|
||
config,
|
||
paths,
|
||
log,
|
||
desktop ?? new WindowsDesktop(paths.helperScript, log)
|
||
);
|
||
return worker.run();
|
||
}
|
||
|
||
class Worker {
|
||
private desktop: Desktop;
|
||
private api: VashApi;
|
||
private link: VashLink;
|
||
private tosu: TosuFeed;
|
||
private tourney: TourneyController;
|
||
private obs: Obs;
|
||
private scene: ObsScene;
|
||
private ingest: RtmpIngest;
|
||
private broadcaster: Broadcaster;
|
||
private obsProcess: Supervised;
|
||
private tosuProcess: Supervised;
|
||
private stopRequested = false;
|
||
private stopping = false;
|
||
private blockedBy: string | null = null;
|
||
private stream: StreamConfig | null = null;
|
||
private streamError: string | null = null;
|
||
private session: SessionInfo | null = null;
|
||
private follow: Follow | null = null;
|
||
private obsLaunchedWith = '';
|
||
private obsConnectedAt = 0;
|
||
private obsConnecting = false;
|
||
private obsStreaming = false;
|
||
private lastStartStream = 0;
|
||
private streamingSince = 0;
|
||
private encoder: string | null = null;
|
||
private warnings = new Set<string>();
|
||
private startedAt = new Date();
|
||
private controlToken = secret();
|
||
private headers: Record<string, string>;
|
||
private overlayUrl: string | null = null;
|
||
|
||
constructor(
|
||
private config: InstanceConfig,
|
||
private paths: Layout,
|
||
private log: (...parts: unknown[]) => void,
|
||
desktop: Desktop
|
||
) {
|
||
this.headers = accessHeaders(config);
|
||
this.desktop = desktop;
|
||
this.api = new VashApi(config.server, config.channel, config.token, this.headers);
|
||
this.tosu = new TosuFeed(
|
||
`ws://127.0.0.1:${config.ports.tosu}/websocket/v2`,
|
||
(frame) => this.link.sendScores(frame),
|
||
log
|
||
);
|
||
this.link = new VashLink({
|
||
server: config.server,
|
||
channel: config.channel,
|
||
token: config.token,
|
||
headers: this.headers,
|
||
log,
|
||
onFollow: (follow) => {
|
||
const changed =
|
||
follow.matchId !== this.follow?.matchId || follow.lobbyId !== this.follow?.lobbyId;
|
||
this.follow = follow;
|
||
if (changed)
|
||
log(
|
||
follow.title
|
||
? `Vash: now showing ${follow.title}${follow.lobbyId ? ` (mp/${follow.lobbyId})` : ''}`
|
||
: 'Vash: nothing to show'
|
||
);
|
||
this.tourney.follow(follow);
|
||
}
|
||
});
|
||
this.tourney = new TourneyController({
|
||
osuDir: paths.osu,
|
||
snapshots: paths.snapshots,
|
||
height: config.tourneyHeight,
|
||
windowTitle: `Vash osu!tourney (${config.name})`,
|
||
desktop: this.desktop,
|
||
tosu: this.tosu,
|
||
log
|
||
});
|
||
this.obs = new Obs(`ws://127.0.0.1:${config.ports.obs}`, config.obsPassword);
|
||
this.obs.onClose = (reason) => {
|
||
log(`OBS websocket closed: ${reason}`);
|
||
this.obsConnectedAt = 0;
|
||
this.obsStreaming = false;
|
||
this.scene.reset();
|
||
};
|
||
this.scene = new ObsScene(this.obs, log);
|
||
this.ingest = new RtmpIngest({
|
||
ffmpeg: paths.ffmpeg,
|
||
host: '127.0.0.1',
|
||
port: config.ports.ingest,
|
||
key: config.ingestKey,
|
||
log
|
||
});
|
||
this.broadcaster = new Broadcaster({
|
||
api: this.api,
|
||
ffmpeg: paths.ffmpeg,
|
||
hlsDir: paths.hls,
|
||
source: this.ingest,
|
||
log
|
||
});
|
||
this.obsProcess = new Supervised('OBS', () => this.launchObs(), { log });
|
||
this.tosuProcess = new Supervised(
|
||
'tosu',
|
||
() =>
|
||
childOf(
|
||
Bun.spawn([paths.tosuExe], {
|
||
cwd: paths.tosu,
|
||
stdout: 'ignore',
|
||
stderr: 'ignore',
|
||
windowsHide: true
|
||
})
|
||
),
|
||
{ log }
|
||
);
|
||
}
|
||
|
||
private async launchObs(): Promise<Child> {
|
||
const stream = this.stream;
|
||
const settings = {
|
||
websocketPort: this.config.ports.obs,
|
||
websocketPassword: this.config.obsPassword,
|
||
width: stream?.width ?? 1920,
|
||
height: stream?.height ?? 1080,
|
||
fps: stream?.fps ?? 60,
|
||
bitrateKbps: stream?.bitrateKbps ?? 6000,
|
||
encoder: this.config.encoder,
|
||
ingestUrl: this.ingest.url,
|
||
ingestKey: this.config.ingestKey
|
||
};
|
||
await writeObsSettings(this.paths.obsConfig, settings);
|
||
// Started before Vash answered: restart once the channel's real settings are known.
|
||
this.obsLaunchedWith = this.videoKey(stream) || 'defaults';
|
||
this.scene.reset();
|
||
const process = Bun.spawn([this.paths.obsExe, ...OBS_ARGS], {
|
||
cwd: dirname(this.paths.obsExe),
|
||
stdout: 'ignore',
|
||
stderr: 'ignore'
|
||
});
|
||
this.log(
|
||
`OBS started (${settings.width}×${settings.height} ${settings.fps} fps, ${settings.bitrateKbps} kbps, ${settings.encoder})`
|
||
);
|
||
return childOf(process);
|
||
}
|
||
|
||
/** OBS must restart (while off air) when these change. */
|
||
private videoKey(stream: StreamConfig | null) {
|
||
return stream ? `${stream.width}x${stream.height}@${stream.fps}/${stream.bitrateKbps}` : '';
|
||
}
|
||
|
||
async run(): Promise<number> {
|
||
const { paths, config, log } = this;
|
||
const server = Bun.serve({
|
||
hostname: '127.0.0.1',
|
||
port: config.ports.control,
|
||
fetch: async (request) => {
|
||
if (!authorized(request, this.controlToken))
|
||
return new Response('Forbidden', { status: 403 });
|
||
const url = new URL(request.url);
|
||
if (url.pathname === '/status') return Response.json(this.state());
|
||
if (url.pathname === '/stop' && request.method === 'POST') {
|
||
this.stopRequested = true;
|
||
return Response.json({ ok: true });
|
||
}
|
||
if (url.pathname === '/snapshot' && request.method === 'POST')
|
||
return Response.json(await this.snapshot());
|
||
return new Response('Not found', { status: 404 });
|
||
}
|
||
});
|
||
const runtime: Runtime = {
|
||
pid: process.pid,
|
||
watchdogPid: Number(process.env.VASH_WATCHDOG_PID) || null,
|
||
port: config.ports.control,
|
||
token: this.controlToken,
|
||
startedAt: this.startedAt.toISOString(),
|
||
version: VERSION
|
||
};
|
||
await writeJson(paths.runtime, runtime);
|
||
process.on('SIGTERM', () => (this.stopRequested = true));
|
||
process.on('SIGINT', () => (this.stopRequested = true));
|
||
|
||
// Leftovers of a crashed run would hold ports and windows; this instance's only.
|
||
await stopInstanceProcesses(this.desktop, paths, log).catch((err) => log('Cleanup:', err));
|
||
|
||
let proxy: ReturnType<typeof startAccessProxy> | null = null;
|
||
if (config.access) {
|
||
proxy = startAccessProxy({
|
||
port: config.ports.proxy,
|
||
upstream: config.server,
|
||
headers: this.headers,
|
||
log
|
||
});
|
||
log(`Overlay proxy for Cloudflare Access on 127.0.0.1:${config.ports.proxy}`);
|
||
}
|
||
|
||
this.link.start();
|
||
this.tosu.start();
|
||
// Sleep or a switched-off display would stop capture in the middle of a match.
|
||
await this.desktop.stayAwake(true).catch((err) => log('Could not keep the PC awake:', err));
|
||
|
||
let tick = 0;
|
||
while (!this.stopRequested) {
|
||
tick++;
|
||
try {
|
||
await this.step(tick);
|
||
} catch (err) {
|
||
log('Worker step failed:', err);
|
||
}
|
||
await this.writeState();
|
||
await Bun.sleep(1000);
|
||
}
|
||
await this.shutdown();
|
||
proxy?.stop(true);
|
||
server.stop(true);
|
||
return 0;
|
||
}
|
||
|
||
private async step(tick: number) {
|
||
const { paths, config, log } = this;
|
||
|
||
// One osu!tourney + tosu stack per PC.
|
||
if (this.blockedBy !== null || tick === 1) {
|
||
const lock = await acquireLock(paths.lock, config.name);
|
||
if (!lock.ok) {
|
||
if (this.blockedBy !== lock.owner.instance)
|
||
log(
|
||
`Waiting: instance "${lock.owner.instance}" runs osu!tourney on this PC (pid ${lock.owner.pid})`
|
||
);
|
||
this.blockedBy = lock.owner.instance;
|
||
// A holder whose worker has not answered for a minute is gone (its pid was reused).
|
||
const answering = await this.ownerAnswers(lock.owner.instance);
|
||
if (answering) this.silentSince = 0;
|
||
else if (!this.silentSince) this.silentSince = Date.now();
|
||
else if (Date.now() - this.silentSince > 60_000) {
|
||
log(
|
||
`Instance "${lock.owner.instance}" holds the tourney lock but does not answer; taking over`
|
||
);
|
||
await rm(paths.lock, { force: true });
|
||
this.silentSince = 0;
|
||
}
|
||
this.link.report(this.report());
|
||
await Bun.sleep(4000);
|
||
return;
|
||
}
|
||
if (this.blockedBy) log('The tourney stack is free; starting');
|
||
this.blockedBy = null;
|
||
}
|
||
|
||
if (tick % 2 === 1) this.session = await this.desktop.session().catch(() => null);
|
||
|
||
if (tick % 5 === 1) {
|
||
try {
|
||
const stream = await this.api.stream();
|
||
// OBS on this PC loads the overlay itself; behind Access, through the local proxy.
|
||
if (config.access)
|
||
stream.overlayUrl = stream.overlayUrl.replace(
|
||
/^https?:\/\/[^/]+/,
|
||
`http://127.0.0.1:${config.ports.proxy}`
|
||
);
|
||
this.stream = stream;
|
||
this.streamError = null;
|
||
} catch (err) {
|
||
if (this.streamError !== (err as Error).message) log('Vash stream settings:', err);
|
||
this.streamError = (err as Error).message;
|
||
}
|
||
}
|
||
|
||
// Programs: tosu and OBS always; osu!tourney once Vash said what to show (or after 15 s).
|
||
await this.tosuProcess.ensure();
|
||
if (
|
||
this.stream &&
|
||
this.obsLaunchedWith &&
|
||
this.obsLaunchedWith !== this.videoKey(this.stream) &&
|
||
!this.broadcaster.onAir
|
||
) {
|
||
log('Channel video settings changed; restarting OBS');
|
||
await this.stopObs();
|
||
}
|
||
await this.obsProcess.ensure();
|
||
await this.connectObs();
|
||
|
||
if (this.follow || Date.now() - this.startedAt.getTime() > 15_000)
|
||
if (tick % 2 === 0) await this.tourney.tick(this.session);
|
||
|
||
if (this.obs.connected && this.stream && tick % 3 === 0) {
|
||
const window = this.tourney.window;
|
||
await this.scene
|
||
.reconcile({
|
||
canvas: { width: this.stream.width, height: this.stream.height },
|
||
fps: this.stream.fps,
|
||
overlayUrl: this.stream.overlayUrl,
|
||
capture: window
|
||
? {
|
||
title: `Vash osu!tourney (${config.name})`,
|
||
className: window.className,
|
||
exe: 'osu!.exe'
|
||
}
|
||
: null,
|
||
showGameplay: this.tourney.state === 'spectating',
|
||
ingest: { server: this.ingest.url, key: config.ingestKey }
|
||
})
|
||
.catch((err) => log('OBS scene:', err));
|
||
}
|
||
|
||
// On and off air, following the channel's Go live switch.
|
||
if (this.stream && tick % 5 === 1)
|
||
await this.broadcaster.apply(this.stream).catch((err) => log('Broadcast:', err));
|
||
await this.broadcaster.tick();
|
||
await this.syncObsStream();
|
||
this.link.report(this.report());
|
||
}
|
||
|
||
private silentSince = 0;
|
||
|
||
private async ownerAnswers(instance: string) {
|
||
try {
|
||
return !!(await callWorker(layout(instance).runtime, '/status'));
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
private async connectObs() {
|
||
if (!this.obsProcess.running || this.obs.connected || this.obsConnecting) return;
|
||
this.obsConnecting = true;
|
||
try {
|
||
await this.obs.connect();
|
||
this.obsConnectedAt = Date.now();
|
||
this.log('OBS websocket connected');
|
||
const encoder = await this.scene.profileEncoder().catch(() => null);
|
||
if (this.config.encoder === 'amd' && encoder && encoder !== 'amd')
|
||
this.warn(
|
||
`OBS switched the encoder to ${encoder}: the AMD hardware encoder is not available (see OBS's log)`
|
||
);
|
||
} catch (err) {
|
||
const upFor = Date.now() - this.obsProcess.startedAt;
|
||
if (upFor > 90_000) {
|
||
this.log(
|
||
`OBS is running but its websocket does not answer (${(err as Error).message}); restarting OBS`
|
||
);
|
||
await this.stopObs();
|
||
}
|
||
} finally {
|
||
this.obsConnecting = false;
|
||
}
|
||
}
|
||
|
||
/** OBS streams into the relay while on air, and not otherwise. */
|
||
private async syncObsStream() {
|
||
if (!this.obs.connected) {
|
||
this.obsStreaming = false;
|
||
return;
|
||
}
|
||
const status = await this.scene.streaming().catch(() => null);
|
||
if (!status) return;
|
||
const was = this.obsStreaming;
|
||
this.obsStreaming = status.outputActive;
|
||
if (this.obsStreaming && !was) {
|
||
this.streamingSince = Date.now();
|
||
this.encoder = null;
|
||
}
|
||
const wanted =
|
||
this.broadcaster.onAir &&
|
||
this.ingest.running &&
|
||
Date.now() - this.ingest.listeningSince > 1500;
|
||
if (wanted && !status.outputActive && Date.now() - this.lastStartStream > 10_000) {
|
||
this.lastStartStream = Date.now();
|
||
this.log('Starting the OBS stream into the relay');
|
||
await this.scene.startStream().catch((err) => this.log('OBS StartStream:', err));
|
||
} else if (!this.broadcaster.onAir && status.outputActive) {
|
||
this.log('Stopping the OBS stream');
|
||
await this.scene.stopStream().catch((err) => this.log('OBS StopStream:', err));
|
||
}
|
||
// Which encoder actually runs, from OBS's own log, a few seconds into the stream.
|
||
if (this.obsStreaming && !this.encoder && Date.now() - this.streamingSince > 5000) {
|
||
const found = encoderFromLog(await latestObsLog(this.paths.obsConfig));
|
||
this.encoder =
|
||
found === 'amd'
|
||
? 'AMD AMF (hardware)'
|
||
: found === 'x264'
|
||
? 'x264 (software)'
|
||
: (found ?? 'unknown');
|
||
this.log(`OBS encoder in use: ${this.encoder}`);
|
||
if (this.config.encoder === 'amd' && found !== 'amd')
|
||
this.warn(`Streaming with ${this.encoder}, not the AMD hardware encoder`);
|
||
}
|
||
}
|
||
|
||
private warn(message: string) {
|
||
if (this.warnings.has(message)) return;
|
||
this.warnings.add(message);
|
||
this.log(`Warning: ${message}`);
|
||
}
|
||
|
||
private async stopObs() {
|
||
this.obs.close();
|
||
this.obsConnectedAt = 0;
|
||
await this.obsProcess.stop(async (child) => {
|
||
await this.desktop.close(child.pid);
|
||
});
|
||
}
|
||
|
||
/** The one state shown on the site and in `vash-broadcast status`. */
|
||
private overall(): { state: string; detail: string | null } {
|
||
if (this.stopping) return { state: 'stopping', detail: null };
|
||
if (this.blockedBy)
|
||
return {
|
||
state: 'blocked',
|
||
detail: `Instance "${this.blockedBy}" is running osu!tourney on this PC`
|
||
};
|
||
if (this.link.rejected)
|
||
return {
|
||
state: 'error',
|
||
detail: 'Vash refused the agent token; run setup again with a new token'
|
||
};
|
||
if (this.streamError && !this.stream)
|
||
return { state: 'error', detail: `Cannot reach Vash: ${this.streamError}` };
|
||
const tourney = this.tourney.state;
|
||
if (tourney === 'login-required')
|
||
return { state: 'login-required', detail: this.tourney.detail };
|
||
if (!this.obs.connected)
|
||
return {
|
||
state: this.obsProcess.restarts ? 'reconnecting' : 'starting',
|
||
detail: this.obsProcess.lastExit ? `OBS ${this.obsProcess.lastExit}` : 'starting OBS'
|
||
};
|
||
if (this.broadcaster.onAir && !this.obsStreaming && Date.now() - this.lastStartStream > 20_000)
|
||
return { state: 'reconnecting', detail: 'On air, but OBS is not streaming into the relay' };
|
||
if (tourney === 'stopped')
|
||
return { state: 'starting', detail: 'waiting for Vash to say what to show' };
|
||
const map: Record<string, string> = {
|
||
starting: 'starting',
|
||
'desktop-locked': 'desktop-locked',
|
||
idle: 'idle',
|
||
'waiting-for-lobby': 'waiting-for-lobby',
|
||
selecting: 'selecting',
|
||
spectating: 'spectating',
|
||
error: 'error'
|
||
};
|
||
return { state: map[tourney] ?? 'error', detail: this.tourney.detail };
|
||
}
|
||
|
||
private report() {
|
||
const { state, detail } = this.overall();
|
||
return {
|
||
state,
|
||
detail,
|
||
onAir: this.broadcaster.onAir && this.obsStreaming,
|
||
encoder: this.encoder,
|
||
instance: this.config.name,
|
||
version: VERSION
|
||
};
|
||
}
|
||
|
||
state(): WorkerState {
|
||
const { state, detail } = this.overall();
|
||
const follow = this.follow;
|
||
return {
|
||
instance: this.config.name,
|
||
version: VERSION,
|
||
pid: process.pid,
|
||
state,
|
||
detail,
|
||
updatedAt: new Date().toISOString(),
|
||
startedAt: this.startedAt.toISOString(),
|
||
onAir: this.broadcaster.onAir,
|
||
live: !!this.stream?.live,
|
||
obsStreaming: this.obsStreaming,
|
||
relayReceiving: this.ingest.receiving,
|
||
encoder: this.encoder,
|
||
match: follow
|
||
? {
|
||
id: follow.matchId,
|
||
title: follow.title,
|
||
lobbyId: follow.lobbyId,
|
||
roomName: follow.roomName
|
||
}
|
||
: null,
|
||
tourney: {
|
||
state: this.tourney.state,
|
||
detail: this.tourney.detail,
|
||
verified: this.tourney.verified
|
||
},
|
||
components: {
|
||
obs: { ...this.obsProcess.status(), websocket: this.obs.connected },
|
||
tosu: {
|
||
...this.tosuProcess.status(),
|
||
connected: this.tosu.connected,
|
||
players: this.tosu.fresh()?.clients.length ?? 0,
|
||
frameAt: this.tosu.latestAt ? new Date(this.tosu.latestAt).toISOString() : null
|
||
},
|
||
scores: {
|
||
players: this.link.scores.players,
|
||
sentAt: this.link.scores.sentAt ? new Date(this.link.scores.sentAt).toISOString() : null,
|
||
acceptedAt: this.link.scores.acceptedAt
|
||
? new Date(this.link.scores.acceptedAt).toISOString()
|
||
: null
|
||
},
|
||
relay: {
|
||
listening: this.ingest.running,
|
||
receiving: this.ingest.receiving,
|
||
recording: this.broadcaster.recording
|
||
}
|
||
},
|
||
destinations: this.broadcaster
|
||
.destinations()
|
||
.map(({ label, state, error }) => ({ label, state, error })),
|
||
vash: {
|
||
connected: this.link.connected,
|
||
rejected: this.link.rejected,
|
||
channel: this.link.channelName,
|
||
lastError: this.streamError
|
||
},
|
||
desktop: this.session,
|
||
warnings: [...this.warnings]
|
||
};
|
||
}
|
||
|
||
private lastStateWrite = 0;
|
||
private async writeState(force = false) {
|
||
if (!force && Date.now() - this.lastStateWrite < 2000) return;
|
||
this.lastStateWrite = Date.now();
|
||
await writeJson(this.paths.state, this.state()).catch(() => {});
|
||
}
|
||
|
||
/** Saves what OBS outputs and what the tourney window shows, for checking from SSH. */
|
||
private async snapshot() {
|
||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||
await mkdir(this.paths.snapshots, { recursive: true });
|
||
const files: string[] = [];
|
||
const errors: string[] = [];
|
||
if (this.obs.connected) {
|
||
try {
|
||
const file = join(this.paths.snapshots, `${stamp}-obs-output.png`);
|
||
await writeFile(file, await this.scene.screenshot());
|
||
files.push(file);
|
||
} catch (err) {
|
||
errors.push(`OBS: ${(err as Error).message}`);
|
||
}
|
||
} else errors.push('OBS is not connected');
|
||
const window = this.tourney.window;
|
||
if (window) {
|
||
try {
|
||
const file = join(this.paths.snapshots, `${stamp}-tourney-window.png`);
|
||
await this.desktop.capture({
|
||
x: window.clientX,
|
||
y: window.clientY,
|
||
width: window.clientWidth,
|
||
height: window.clientHeight,
|
||
path: file
|
||
});
|
||
files.push(file);
|
||
} catch (err) {
|
||
errors.push(`Tourney window: ${(err as Error).message}`);
|
||
}
|
||
} else errors.push('No osu!tourney window');
|
||
const panel = join(this.paths.snapshots, 'tourney-panel.png');
|
||
if (existsSync(panel)) files.push(panel, join(this.paths.snapshots, 'tourney-panel.json'));
|
||
return { files, errors };
|
||
}
|
||
|
||
private async shutdown() {
|
||
const { log, paths } = this;
|
||
this.stopping = true;
|
||
log('Stopping');
|
||
this.link.report(this.report());
|
||
await this.writeState(true);
|
||
if (this.obs.connected && this.obsStreaming) await this.scene.stopStream().catch(() => {});
|
||
await this.broadcaster.stop().catch((err) => log('Going off air:', err));
|
||
await this.tourney.stop().catch((err) => log('Stopping osu!tourney:', err));
|
||
await this.stopObs().catch(() => {});
|
||
await this.tosuProcess.stop();
|
||
this.tosu.stop();
|
||
this.link.stop();
|
||
await stopInstanceProcesses(this.desktop, paths, log).catch(() => {});
|
||
this.desktop.stop();
|
||
await releaseLock(paths.lock);
|
||
await writeJson(paths.state, { ...this.state(), state: 'stopped', detail: null }).catch(
|
||
() => {}
|
||
);
|
||
await rm(paths.runtime, { force: true });
|
||
log('Stopped');
|
||
}
|
||
}
|
||
|
||
// ── Watchdog ────────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* What the scheduled task runs. It restarts the worker if it crashes (with
|
||
* backoff) and exits when the worker stops on purpose. The request file says
|
||
* whether this start is the worker, the one-time sign-in, or a diagnosis.
|
||
*/
|
||
export async function runTask(name: string): Promise<number> {
|
||
const paths = layout(name);
|
||
const log = fileLogger(paths.log);
|
||
const request = await readJson<{ mode: string; at: number }>(paths.request);
|
||
await rm(paths.request, { force: true });
|
||
const mode = request && Date.now() - request.at < 5 * 60_000 ? request.mode : 'run';
|
||
if (mode === 'login') return runSignIn(name);
|
||
if (mode === 'doctor') return runDesktopDoctor(name);
|
||
|
||
const owner = await lockOwner(paths.lock);
|
||
const runtime = await readJson<Runtime>(paths.runtime);
|
||
if (runtime && owner?.pid === runtime.pid) {
|
||
log('The worker is already running; not starting another');
|
||
return 0;
|
||
}
|
||
let failures = 0;
|
||
for (;;) {
|
||
const started = Date.now();
|
||
const child = Bun.spawn(selfCommand('worker', '--instance', name), {
|
||
stdout: 'ignore',
|
||
stderr: 'pipe',
|
||
env: { ...process.env, VASH_WATCHDOG_PID: String(process.pid) },
|
||
windowsHide: true
|
||
});
|
||
let stderr = '';
|
||
void (async () => {
|
||
for await (const chunk of child.stderr)
|
||
stderr = (stderr + new TextDecoder().decode(chunk)).slice(-2000);
|
||
})();
|
||
const code = await child.exited;
|
||
if (code === 0) return 0;
|
||
if (Date.now() - started > 10 * 60_000) failures = 0;
|
||
const wait = Math.min(60_000, 2000 * 2 ** failures++);
|
||
log(
|
||
`Worker exited with ${code}${stderr ? `: ${stderr.trim().split('\n').slice(-3).join(' ')}` : ''}; restarting in ${Math.round(wait / 1000)}s`
|
||
);
|
||
if (existsSync(paths.request)) return 0;
|
||
await Bun.sleep(wait);
|
||
}
|
||
}
|
||
|
||
// ── One-time sign-in ────────────────────────────────────────────────────
|
||
|
||
export interface SignInState {
|
||
state: 'starting' | 'waiting' | 'closing' | 'done' | 'failed';
|
||
message: string;
|
||
port: number;
|
||
token: string;
|
||
username: string | null;
|
||
updatedAt: string;
|
||
}
|
||
|
||
async function runSignIn(name: string): Promise<number> {
|
||
const config = await loadConfig(name);
|
||
const paths = layout(name);
|
||
const log = fileLogger(paths.log);
|
||
const desktop = new WindowsDesktop(paths.helperScript, log);
|
||
const token = secret();
|
||
let finish = false;
|
||
let state: SignInState = {
|
||
state: 'starting',
|
||
message: 'Starting osu!…',
|
||
port: config.ports.viewer,
|
||
token,
|
||
username: null,
|
||
updatedAt: new Date().toISOString()
|
||
};
|
||
const set = async (next: Partial<SignInState>) => {
|
||
state = { ...state, ...next, updatedAt: new Date().toISOString() };
|
||
await writeJson(paths.login, state);
|
||
};
|
||
log('One-time sign-in: starting osu! normally');
|
||
await stopInstanceProcesses(desktop, paths, log).catch(() => {});
|
||
const restore = await normalModeFor(paths.osu);
|
||
const viewer = startViewer({
|
||
port: config.ports.viewer,
|
||
token,
|
||
desktop,
|
||
scratch: paths.snapshots,
|
||
title: `Vash broadcast sign-in (${name})`,
|
||
status: () => ({
|
||
message: state.message,
|
||
done: state.state === 'done' || state.state === 'failed'
|
||
}),
|
||
onFinish: () => (finish = true)
|
||
});
|
||
await mkdir(paths.snapshots, { recursive: true });
|
||
try {
|
||
Bun.spawn([paths.osuExe], { cwd: paths.osu, stdout: 'ignore', stderr: 'ignore' });
|
||
await set({
|
||
state: 'waiting',
|
||
message:
|
||
'osu! is starting (a fresh installation downloads its files first). Sign in with the spectator account, tick Remember username and Remember password, then press Finish.'
|
||
});
|
||
const deadline = Date.now() + 45 * 60_000;
|
||
while (!finish && Date.now() < deadline) {
|
||
await Bun.sleep(1000);
|
||
const window = pickMainWindow(await desktop.windows(paths.osu).catch(() => []));
|
||
if (window && state.message.startsWith('osu! is starting'))
|
||
await desktop.focus(window.hwnd).catch(() => false);
|
||
}
|
||
await set({ state: 'closing', message: 'Closing osu! so it saves the login…' });
|
||
for (const process of await desktop.processes(paths.osu))
|
||
await desktop.close(process.pid).catch(() => 0);
|
||
for (
|
||
let waited = 0;
|
||
waited < 20_000 && (await desktop.processes(paths.osu)).length;
|
||
waited += 1000
|
||
)
|
||
await Bun.sleep(1000);
|
||
await stopInstanceProcesses(desktop, paths, log).catch(() => {});
|
||
await restore();
|
||
const login = await readSavedLogin(paths.osu);
|
||
if (login.saved) {
|
||
await set({
|
||
state: 'done',
|
||
username: login.username,
|
||
message: `Signed in as ${login.username}; the login is saved.`
|
||
});
|
||
log(`One-time sign-in done: ${login.username}`);
|
||
} else {
|
||
await set({
|
||
state: 'failed',
|
||
message: finish
|
||
? 'osu! did not save a login. Run vash-broadcast login again and tick Remember username and Remember password.'
|
||
: 'The sign-in timed out after 45 minutes.'
|
||
});
|
||
}
|
||
} catch (err) {
|
||
await restore().catch(() => {});
|
||
await set({ state: 'failed', message: (err as Error).message });
|
||
} finally {
|
||
await Bun.sleep(3000);
|
||
viewer.stop(true);
|
||
desktop.stop();
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
// ── Diagnosis inside the desktop ─────────────────────────────────────────
|
||
|
||
async function runDesktopDoctor(name: string): Promise<number> {
|
||
const paths = layout(name);
|
||
const log = fileLogger(paths.log);
|
||
const desktop = new WindowsDesktop(paths.helperScript, log);
|
||
const checks: { name: string; ok: boolean; detail: string }[] = [];
|
||
const check = async (label: string, run: () => Promise<string>) => {
|
||
try {
|
||
checks.push({ name: label, ok: true, detail: await run() });
|
||
} catch (err) {
|
||
checks.push({ name: label, ok: false, detail: (err as Error).message });
|
||
}
|
||
};
|
||
await check('Desktop session', async () => {
|
||
const session = await desktop.session();
|
||
if (session.inputDesktop !== 'Default')
|
||
throw new Error(
|
||
`The desktop is locked or on a secure screen (${session.inputDesktop ?? 'no input desktop'}); rooms cannot be clicked`
|
||
);
|
||
if (session.sessionId !== session.consoleSessionId)
|
||
throw new Error(
|
||
`This session (${session.sessionId}) is not the one on the PC's screen (${session.consoleSessionId}); a disconnected Remote Desktop session has no display`
|
||
);
|
||
return `unlocked, ${session.screenWidth}×${session.screenHeight}, idle ${Math.round(session.idleMs / 1000)}s`;
|
||
});
|
||
await check('Screen capture', async () => {
|
||
await mkdir(paths.snapshots, { recursive: true });
|
||
const file = join(paths.snapshots, 'doctor-screen.jpg');
|
||
const size = await desktop.capture({
|
||
x: 0,
|
||
y: 0,
|
||
width: 640,
|
||
height: 360,
|
||
path: file,
|
||
format: 'jpeg'
|
||
});
|
||
return `captured ${size.width}×${size.height} to ${file}`;
|
||
});
|
||
await check('Windows OCR', async () => {
|
||
const ready = await desktop.ocrReady();
|
||
return `language ${ready.language}, images up to ${ready.maxDimension} px`;
|
||
});
|
||
await check('osu!tourney window', async () => {
|
||
const windows = await desktop.windows(paths.osu);
|
||
return windows.length
|
||
? windows
|
||
.map((window) => `"${window.title}" ${window.clientWidth}×${window.clientHeight}`)
|
||
.join('; ')
|
||
: 'osu! is not running (normal while the worker is stopped)';
|
||
});
|
||
await check('AMD encoder in OBS log', async () => {
|
||
const log = await latestObsLog(paths.obsConfig);
|
||
if (!log) return 'no OBS log yet; start the worker once';
|
||
const lines = log
|
||
.split(/\r?\n/)
|
||
.filter((line) => /amf|AMD|encoder/i.test(line))
|
||
.slice(0, 12);
|
||
return lines.join(' | ') || 'no encoder lines in the latest OBS log';
|
||
});
|
||
desktop.stop();
|
||
await writeJson(paths.doctor, { at: new Date().toISOString(), checks });
|
||
return 0;
|
||
}
|