feat: add native Windows broadcaster and live score modules
- streamer/windows: vash-broadcast.exe (setup/login/start/stop/status/doctor) with per-instance OBS, tosu and FFmpeg, a supervised worker, OCR room selection and OBS scene reconciliation; built and tested on Linux only - streamer/relay.ts shared with Vash TV; Cloudflare Access proxy for dev/staging - worker status validation and live score helpers for the site - STREAMING_HANDOFF.md and streamer/ALPHA_TEST.md record what is unverified Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
900
streamer/windows/worker.ts
Normal file
900
streamer/windows/worker.ts
Normal file
@@ -0,0 +1,900 @@
|
||||
/**
|
||||
* The unattended worker for one instance, running in the signed-in desktop:
|
||||
*
|
||||
* Vash (/ws/agent) ──follow──▶ osu!tourney: right team size, right room
|
||||
* tosu ──live scores──▶ Vash; tosu ──team names──▶ room confirmed
|
||||
* Vash (/stream) ──Go live──▶ relay listening ──▶ OBS streams into it
|
||||
* OBS: overlay over the captured tourney window, AMD hardware encoding
|
||||
* relay ──▶ every destination + HLS recording on the Vash site
|
||||
*
|
||||
* It restarts whatever stops (OBS, tosu, osu!tourney, the relay), reports what
|
||||
* it is doing to Vash and to state.json, and stops cleanly when asked.
|
||||
*/
|
||||
import { existsSync } from 'node:fs';
|
||||
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { VashLink, type Follow } from '../link';
|
||||
import { Obs } from '../obs';
|
||||
import { Broadcaster, RtmpIngest } from '../relay';
|
||||
import { TosuFeed } from '../tosu';
|
||||
import { VashApi, type StreamConfig } from '../vash-api';
|
||||
import {
|
||||
accessHeaders,
|
||||
layout,
|
||||
loadConfig,
|
||||
readJson,
|
||||
secret,
|
||||
VERSION,
|
||||
writeJson,
|
||||
type InstanceConfig,
|
||||
type Layout
|
||||
} from './config';
|
||||
import { authorized, callWorker, type Runtime } from './control';
|
||||
import { WindowsDesktop, type Desktop, type SessionInfo } from './desktop';
|
||||
import { acquireLock, lockOwner, releaseLock } from './lock';
|
||||
import { fileLogger } from './log';
|
||||
import { ObsScene } from './obs-control';
|
||||
import { encoderFromLog, latestObsLog, OBS_ARGS, writeObsSettings } from './obs-files';
|
||||
import { startAccessProxy } from './proxy';
|
||||
import { Supervised, type Child } from './supervise';
|
||||
import { normalModeFor, pickMainWindow, readSavedLogin, TourneyController } from './tourney';
|
||||
import { startViewer } from './viewer';
|
||||
|
||||
export interface WorkerState {
|
||||
instance: string;
|
||||
version: string;
|
||||
pid: number;
|
||||
state: string;
|
||||
detail: string | null;
|
||||
updatedAt: string;
|
||||
startedAt: string;
|
||||
onAir: boolean;
|
||||
live: boolean;
|
||||
obsStreaming: boolean;
|
||||
relayReceiving: boolean;
|
||||
encoder: string | null;
|
||||
match: {
|
||||
id: string | null;
|
||||
title: string | null;
|
||||
lobbyId: number | null;
|
||||
roomName: string | null;
|
||||
} | null;
|
||||
tourney: { state: string; detail: string | null; verified: boolean };
|
||||
components: Record<string, unknown>;
|
||||
destinations: { label: string; state: string; error: string | null }[];
|
||||
vash: { connected: boolean; rejected: boolean; channel: string | null; lastError: string | null };
|
||||
desktop: SessionInfo | null;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/** The same program, as a command line: the compiled exe, or bun + this script in development. */
|
||||
export function selfCommand(...args: string[]) {
|
||||
const compiled =
|
||||
Bun.main.startsWith('B:/~BUN') || Bun.main.startsWith('/$bunfs') || Bun.main.includes('~BUN');
|
||||
return compiled ? [process.execPath, ...args] : [process.execPath, Bun.main, ...args];
|
||||
}
|
||||
|
||||
const childOf = (process: import('bun').Subprocess): Child => ({
|
||||
pid: process.pid,
|
||||
exited: process.exited,
|
||||
kill: () => process.kill()
|
||||
});
|
||||
|
||||
/** Stops every process whose program lives in this instance's folder. */
|
||||
export async function stopInstanceProcesses(
|
||||
desktop: Desktop,
|
||||
paths: Layout,
|
||||
log: (...parts: unknown[]) => void
|
||||
) {
|
||||
const processes = await desktop.processes(paths.dir);
|
||||
if (!processes.length) return 0;
|
||||
log(`Stopping ${processes.length} leftover process(es) of this instance`);
|
||||
return desktop.kill(processes.map((process) => process.pid));
|
||||
}
|
||||
|
||||
/** Runs until asked to stop. Tests pass a fake desktop; Windows uses the PowerShell helper. */
|
||||
export async function runWorker(name: string, desktop?: Desktop): Promise<number> {
|
||||
const config = await loadConfig(name);
|
||||
const paths = layout(name);
|
||||
const log = fileLogger(paths.log);
|
||||
log(
|
||||
`vash-broadcast ${VERSION} worker starting for "${name}": ${config.server}, channel ${config.channel}`
|
||||
);
|
||||
const worker = new Worker(
|
||||
config,
|
||||
paths,
|
||||
log,
|
||||
desktop ?? new WindowsDesktop(paths.helperScript, log)
|
||||
);
|
||||
return worker.run();
|
||||
}
|
||||
|
||||
class Worker {
|
||||
private desktop: Desktop;
|
||||
private api: VashApi;
|
||||
private link: VashLink;
|
||||
private tosu: TosuFeed;
|
||||
private tourney: TourneyController;
|
||||
private obs: Obs;
|
||||
private scene: ObsScene;
|
||||
private ingest: RtmpIngest;
|
||||
private broadcaster: Broadcaster;
|
||||
private obsProcess: Supervised;
|
||||
private tosuProcess: Supervised;
|
||||
private stopRequested = false;
|
||||
private stopping = false;
|
||||
private blockedBy: string | null = null;
|
||||
private stream: StreamConfig | null = null;
|
||||
private streamError: string | null = null;
|
||||
private session: SessionInfo | null = null;
|
||||
private follow: Follow | null = null;
|
||||
private obsLaunchedWith = '';
|
||||
private obsConnectedAt = 0;
|
||||
private obsConnecting = false;
|
||||
private obsStreaming = false;
|
||||
private lastStartStream = 0;
|
||||
private streamingSince = 0;
|
||||
private encoder: string | null = null;
|
||||
private warnings = new Set<string>();
|
||||
private startedAt = new Date();
|
||||
private controlToken = secret();
|
||||
private headers: Record<string, string>;
|
||||
private overlayUrl: string | null = null;
|
||||
|
||||
constructor(
|
||||
private config: InstanceConfig,
|
||||
private paths: Layout,
|
||||
private log: (...parts: unknown[]) => void,
|
||||
desktop: Desktop
|
||||
) {
|
||||
this.headers = accessHeaders(config);
|
||||
this.desktop = desktop;
|
||||
this.api = new VashApi(config.server, config.channel, config.token, this.headers);
|
||||
this.tosu = new TosuFeed(
|
||||
`ws://127.0.0.1:${config.ports.tosu}/websocket/v2`,
|
||||
(frame) => this.link.sendScores(frame),
|
||||
log
|
||||
);
|
||||
this.link = new VashLink({
|
||||
server: config.server,
|
||||
channel: config.channel,
|
||||
token: config.token,
|
||||
headers: this.headers,
|
||||
log,
|
||||
onFollow: (follow) => {
|
||||
const changed =
|
||||
follow.matchId !== this.follow?.matchId || follow.lobbyId !== this.follow?.lobbyId;
|
||||
this.follow = follow;
|
||||
if (changed)
|
||||
log(
|
||||
follow.title
|
||||
? `Vash: now showing ${follow.title}${follow.lobbyId ? ` (mp/${follow.lobbyId})` : ''}`
|
||||
: 'Vash: nothing to show'
|
||||
);
|
||||
this.tourney.follow(follow);
|
||||
}
|
||||
});
|
||||
this.tourney = new TourneyController({
|
||||
osuDir: paths.osu,
|
||||
snapshots: paths.snapshots,
|
||||
height: config.tourneyHeight,
|
||||
windowTitle: `Vash osu!tourney (${config.name})`,
|
||||
desktop: this.desktop,
|
||||
tosu: this.tosu,
|
||||
log
|
||||
});
|
||||
this.obs = new Obs(`ws://127.0.0.1:${config.ports.obs}`, config.obsPassword);
|
||||
this.obs.onClose = (reason) => {
|
||||
log(`OBS websocket closed: ${reason}`);
|
||||
this.obsConnectedAt = 0;
|
||||
this.obsStreaming = false;
|
||||
this.scene.reset();
|
||||
};
|
||||
this.scene = new ObsScene(this.obs, log);
|
||||
this.ingest = new RtmpIngest({
|
||||
ffmpeg: paths.ffmpeg,
|
||||
host: '127.0.0.1',
|
||||
port: config.ports.ingest,
|
||||
key: config.ingestKey,
|
||||
log
|
||||
});
|
||||
this.broadcaster = new Broadcaster({
|
||||
api: this.api,
|
||||
ffmpeg: paths.ffmpeg,
|
||||
hlsDir: paths.hls,
|
||||
source: this.ingest,
|
||||
log
|
||||
});
|
||||
this.obsProcess = new Supervised('OBS', () => this.launchObs(), { log });
|
||||
this.tosuProcess = new Supervised(
|
||||
'tosu',
|
||||
() =>
|
||||
childOf(
|
||||
Bun.spawn([paths.tosuExe], {
|
||||
cwd: paths.tosu,
|
||||
stdout: 'ignore',
|
||||
stderr: 'ignore',
|
||||
windowsHide: true
|
||||
})
|
||||
),
|
||||
{ log }
|
||||
);
|
||||
}
|
||||
|
||||
private async launchObs(): Promise<Child> {
|
||||
const stream = this.stream;
|
||||
const settings = {
|
||||
websocketPort: this.config.ports.obs,
|
||||
websocketPassword: this.config.obsPassword,
|
||||
width: stream?.width ?? 1920,
|
||||
height: stream?.height ?? 1080,
|
||||
fps: stream?.fps ?? 60,
|
||||
bitrateKbps: stream?.bitrateKbps ?? 6000,
|
||||
encoder: this.config.encoder,
|
||||
ingestUrl: this.ingest.url,
|
||||
ingestKey: this.config.ingestKey
|
||||
};
|
||||
await writeObsSettings(this.paths.obsConfig, settings);
|
||||
// Started before Vash answered: restart once the channel's real settings are known.
|
||||
this.obsLaunchedWith = this.videoKey(stream) || 'defaults';
|
||||
this.scene.reset();
|
||||
const process = Bun.spawn([this.paths.obsExe, ...OBS_ARGS], {
|
||||
cwd: dirname(this.paths.obsExe),
|
||||
stdout: 'ignore',
|
||||
stderr: 'ignore'
|
||||
});
|
||||
this.log(
|
||||
`OBS started (${settings.width}×${settings.height} ${settings.fps} fps, ${settings.bitrateKbps} kbps, ${settings.encoder})`
|
||||
);
|
||||
return childOf(process);
|
||||
}
|
||||
|
||||
/** OBS must restart (while off air) when these change. */
|
||||
private videoKey(stream: StreamConfig | null) {
|
||||
return stream ? `${stream.width}x${stream.height}@${stream.fps}/${stream.bitrateKbps}` : '';
|
||||
}
|
||||
|
||||
async run(): Promise<number> {
|
||||
const { paths, config, log } = this;
|
||||
const server = Bun.serve({
|
||||
hostname: '127.0.0.1',
|
||||
port: config.ports.control,
|
||||
fetch: async (request) => {
|
||||
if (!authorized(request, this.controlToken))
|
||||
return new Response('Forbidden', { status: 403 });
|
||||
const url = new URL(request.url);
|
||||
if (url.pathname === '/status') return Response.json(this.state());
|
||||
if (url.pathname === '/stop' && request.method === 'POST') {
|
||||
this.stopRequested = true;
|
||||
return Response.json({ ok: true });
|
||||
}
|
||||
if (url.pathname === '/snapshot' && request.method === 'POST')
|
||||
return Response.json(await this.snapshot());
|
||||
return new Response('Not found', { status: 404 });
|
||||
}
|
||||
});
|
||||
const runtime: Runtime = {
|
||||
pid: process.pid,
|
||||
watchdogPid: Number(process.env.VASH_WATCHDOG_PID) || null,
|
||||
port: config.ports.control,
|
||||
token: this.controlToken,
|
||||
startedAt: this.startedAt.toISOString(),
|
||||
version: VERSION
|
||||
};
|
||||
await writeJson(paths.runtime, runtime);
|
||||
process.on('SIGTERM', () => (this.stopRequested = true));
|
||||
process.on('SIGINT', () => (this.stopRequested = true));
|
||||
|
||||
// Leftovers of a crashed run would hold ports and windows; this instance's only.
|
||||
await stopInstanceProcesses(this.desktop, paths, log).catch((err) => log('Cleanup:', err));
|
||||
|
||||
let proxy: ReturnType<typeof startAccessProxy> | null = null;
|
||||
if (config.access) {
|
||||
proxy = startAccessProxy({
|
||||
port: config.ports.proxy,
|
||||
upstream: config.server,
|
||||
headers: this.headers,
|
||||
log
|
||||
});
|
||||
log(`Overlay proxy for Cloudflare Access on 127.0.0.1:${config.ports.proxy}`);
|
||||
}
|
||||
|
||||
this.link.start();
|
||||
this.tosu.start();
|
||||
// Sleep or a switched-off display would stop capture in the middle of a match.
|
||||
await this.desktop.stayAwake(true).catch((err) => log('Could not keep the PC awake:', err));
|
||||
|
||||
let tick = 0;
|
||||
while (!this.stopRequested) {
|
||||
tick++;
|
||||
try {
|
||||
await this.step(tick);
|
||||
} catch (err) {
|
||||
log('Worker step failed:', err);
|
||||
}
|
||||
await this.writeState();
|
||||
await Bun.sleep(1000);
|
||||
}
|
||||
await this.shutdown();
|
||||
proxy?.stop(true);
|
||||
server.stop(true);
|
||||
return 0;
|
||||
}
|
||||
|
||||
private async step(tick: number) {
|
||||
const { paths, config, log } = this;
|
||||
|
||||
// One osu!tourney + tosu stack per PC.
|
||||
if (this.blockedBy !== null || tick === 1) {
|
||||
const lock = await acquireLock(paths.lock, config.name);
|
||||
if (!lock.ok) {
|
||||
if (this.blockedBy !== lock.owner.instance)
|
||||
log(
|
||||
`Waiting: instance "${lock.owner.instance}" runs osu!tourney on this PC (pid ${lock.owner.pid})`
|
||||
);
|
||||
this.blockedBy = lock.owner.instance;
|
||||
// A holder whose worker has not answered for a minute is gone (its pid was reused).
|
||||
const answering = await this.ownerAnswers(lock.owner.instance);
|
||||
if (answering) this.silentSince = 0;
|
||||
else if (!this.silentSince) this.silentSince = Date.now();
|
||||
else if (Date.now() - this.silentSince > 60_000) {
|
||||
log(
|
||||
`Instance "${lock.owner.instance}" holds the tourney lock but does not answer; taking over`
|
||||
);
|
||||
await rm(paths.lock, { force: true });
|
||||
this.silentSince = 0;
|
||||
}
|
||||
this.link.report(this.report());
|
||||
await Bun.sleep(4000);
|
||||
return;
|
||||
}
|
||||
if (this.blockedBy) log('The tourney stack is free; starting');
|
||||
this.blockedBy = null;
|
||||
}
|
||||
|
||||
if (tick % 2 === 1) this.session = await this.desktop.session().catch(() => null);
|
||||
|
||||
if (tick % 5 === 1) {
|
||||
try {
|
||||
const stream = await this.api.stream();
|
||||
// OBS on this PC loads the overlay itself; behind Access, through the local proxy.
|
||||
if (config.access)
|
||||
stream.overlayUrl = stream.overlayUrl.replace(
|
||||
/^https?:\/\/[^/]+/,
|
||||
`http://127.0.0.1:${config.ports.proxy}`
|
||||
);
|
||||
this.stream = stream;
|
||||
this.streamError = null;
|
||||
} catch (err) {
|
||||
if (this.streamError !== (err as Error).message) log('Vash stream settings:', err);
|
||||
this.streamError = (err as Error).message;
|
||||
}
|
||||
}
|
||||
|
||||
// Programs: tosu and OBS always; osu!tourney once Vash said what to show (or after 15 s).
|
||||
await this.tosuProcess.ensure();
|
||||
if (
|
||||
this.stream &&
|
||||
this.obsLaunchedWith &&
|
||||
this.obsLaunchedWith !== this.videoKey(this.stream) &&
|
||||
!this.broadcaster.onAir
|
||||
) {
|
||||
log('Channel video settings changed; restarting OBS');
|
||||
await this.stopObs();
|
||||
}
|
||||
await this.obsProcess.ensure();
|
||||
await this.connectObs();
|
||||
|
||||
if (this.follow || Date.now() - this.startedAt.getTime() > 15_000)
|
||||
if (tick % 2 === 0) await this.tourney.tick(this.session);
|
||||
|
||||
if (this.obs.connected && this.stream && tick % 3 === 0) {
|
||||
const window = this.tourney.window;
|
||||
await this.scene
|
||||
.reconcile({
|
||||
canvas: { width: this.stream.width, height: this.stream.height },
|
||||
fps: this.stream.fps,
|
||||
overlayUrl: this.stream.overlayUrl,
|
||||
capture: window
|
||||
? {
|
||||
title: `Vash osu!tourney (${config.name})`,
|
||||
className: window.className,
|
||||
exe: 'osu!.exe'
|
||||
}
|
||||
: null,
|
||||
showGameplay: this.tourney.state === 'spectating',
|
||||
ingest: { server: this.ingest.url, key: config.ingestKey }
|
||||
})
|
||||
.catch((err) => log('OBS scene:', err));
|
||||
}
|
||||
|
||||
// On and off air, following the channel's Go live switch.
|
||||
if (this.stream && tick % 5 === 1)
|
||||
await this.broadcaster.apply(this.stream).catch((err) => log('Broadcast:', err));
|
||||
await this.broadcaster.tick();
|
||||
await this.syncObsStream();
|
||||
this.link.report(this.report());
|
||||
}
|
||||
|
||||
private silentSince = 0;
|
||||
|
||||
private async ownerAnswers(instance: string) {
|
||||
try {
|
||||
return !!(await callWorker(layout(instance).runtime, '/status'));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async connectObs() {
|
||||
if (!this.obsProcess.running || this.obs.connected || this.obsConnecting) return;
|
||||
this.obsConnecting = true;
|
||||
try {
|
||||
await this.obs.connect();
|
||||
this.obsConnectedAt = Date.now();
|
||||
this.log('OBS websocket connected');
|
||||
const encoder = await this.scene.profileEncoder().catch(() => null);
|
||||
if (this.config.encoder === 'amd' && encoder && encoder !== 'amd')
|
||||
this.warn(
|
||||
`OBS switched the encoder to ${encoder}: the AMD hardware encoder is not available (see OBS's log)`
|
||||
);
|
||||
} catch (err) {
|
||||
const upFor = Date.now() - this.obsProcess.startedAt;
|
||||
if (upFor > 90_000) {
|
||||
this.log(
|
||||
`OBS is running but its websocket does not answer (${(err as Error).message}); restarting OBS`
|
||||
);
|
||||
await this.stopObs();
|
||||
}
|
||||
} finally {
|
||||
this.obsConnecting = false;
|
||||
}
|
||||
}
|
||||
|
||||
/** OBS streams into the relay while on air, and not otherwise. */
|
||||
private async syncObsStream() {
|
||||
if (!this.obs.connected) {
|
||||
this.obsStreaming = false;
|
||||
return;
|
||||
}
|
||||
const status = await this.scene.streaming().catch(() => null);
|
||||
if (!status) return;
|
||||
const was = this.obsStreaming;
|
||||
this.obsStreaming = status.outputActive;
|
||||
if (this.obsStreaming && !was) {
|
||||
this.streamingSince = Date.now();
|
||||
this.encoder = null;
|
||||
}
|
||||
const wanted =
|
||||
this.broadcaster.onAir &&
|
||||
this.ingest.running &&
|
||||
Date.now() - this.ingest.listeningSince > 1500;
|
||||
if (wanted && !status.outputActive && Date.now() - this.lastStartStream > 10_000) {
|
||||
this.lastStartStream = Date.now();
|
||||
this.log('Starting the OBS stream into the relay');
|
||||
await this.scene.startStream().catch((err) => this.log('OBS StartStream:', err));
|
||||
} else if (!this.broadcaster.onAir && status.outputActive) {
|
||||
this.log('Stopping the OBS stream');
|
||||
await this.scene.stopStream().catch((err) => this.log('OBS StopStream:', err));
|
||||
}
|
||||
// Which encoder actually runs, from OBS's own log, a few seconds into the stream.
|
||||
if (this.obsStreaming && !this.encoder && Date.now() - this.streamingSince > 5000) {
|
||||
const found = encoderFromLog(await latestObsLog(this.paths.obsConfig));
|
||||
this.encoder =
|
||||
found === 'amd'
|
||||
? 'AMD AMF (hardware)'
|
||||
: found === 'x264'
|
||||
? 'x264 (software)'
|
||||
: (found ?? 'unknown');
|
||||
this.log(`OBS encoder in use: ${this.encoder}`);
|
||||
if (this.config.encoder === 'amd' && found !== 'amd')
|
||||
this.warn(`Streaming with ${this.encoder}, not the AMD hardware encoder`);
|
||||
}
|
||||
}
|
||||
|
||||
private warn(message: string) {
|
||||
if (this.warnings.has(message)) return;
|
||||
this.warnings.add(message);
|
||||
this.log(`Warning: ${message}`);
|
||||
}
|
||||
|
||||
private async stopObs() {
|
||||
this.obs.close();
|
||||
this.obsConnectedAt = 0;
|
||||
await this.obsProcess.stop(async (child) => {
|
||||
await this.desktop.close(child.pid);
|
||||
});
|
||||
}
|
||||
|
||||
/** The one state shown on the site and in `vash-broadcast status`. */
|
||||
private overall(): { state: string; detail: string | null } {
|
||||
if (this.stopping) return { state: 'stopping', detail: null };
|
||||
if (this.blockedBy)
|
||||
return {
|
||||
state: 'blocked',
|
||||
detail: `Instance "${this.blockedBy}" is running osu!tourney on this PC`
|
||||
};
|
||||
if (this.link.rejected)
|
||||
return {
|
||||
state: 'error',
|
||||
detail: 'Vash refused the agent token; run setup again with a new token'
|
||||
};
|
||||
if (this.streamError && !this.stream)
|
||||
return { state: 'error', detail: `Cannot reach Vash: ${this.streamError}` };
|
||||
const tourney = this.tourney.state;
|
||||
if (tourney === 'login-required')
|
||||
return { state: 'login-required', detail: this.tourney.detail };
|
||||
if (!this.obs.connected)
|
||||
return {
|
||||
state: this.obsProcess.restarts ? 'reconnecting' : 'starting',
|
||||
detail: this.obsProcess.lastExit ? `OBS ${this.obsProcess.lastExit}` : 'starting OBS'
|
||||
};
|
||||
if (this.broadcaster.onAir && !this.obsStreaming && Date.now() - this.lastStartStream > 20_000)
|
||||
return { state: 'reconnecting', detail: 'On air, but OBS is not streaming into the relay' };
|
||||
if (tourney === 'stopped')
|
||||
return { state: 'starting', detail: 'waiting for Vash to say what to show' };
|
||||
const map: Record<string, string> = {
|
||||
starting: 'starting',
|
||||
'desktop-locked': 'desktop-locked',
|
||||
idle: 'idle',
|
||||
'waiting-for-lobby': 'waiting-for-lobby',
|
||||
selecting: 'selecting',
|
||||
spectating: 'spectating',
|
||||
error: 'error'
|
||||
};
|
||||
return { state: map[tourney] ?? 'error', detail: this.tourney.detail };
|
||||
}
|
||||
|
||||
private report() {
|
||||
const { state, detail } = this.overall();
|
||||
return {
|
||||
state,
|
||||
detail,
|
||||
onAir: this.broadcaster.onAir && this.obsStreaming,
|
||||
encoder: this.encoder,
|
||||
instance: this.config.name,
|
||||
version: VERSION
|
||||
};
|
||||
}
|
||||
|
||||
state(): WorkerState {
|
||||
const { state, detail } = this.overall();
|
||||
const follow = this.follow;
|
||||
return {
|
||||
instance: this.config.name,
|
||||
version: VERSION,
|
||||
pid: process.pid,
|
||||
state,
|
||||
detail,
|
||||
updatedAt: new Date().toISOString(),
|
||||
startedAt: this.startedAt.toISOString(),
|
||||
onAir: this.broadcaster.onAir,
|
||||
live: !!this.stream?.live,
|
||||
obsStreaming: this.obsStreaming,
|
||||
relayReceiving: this.ingest.receiving,
|
||||
encoder: this.encoder,
|
||||
match: follow
|
||||
? {
|
||||
id: follow.matchId,
|
||||
title: follow.title,
|
||||
lobbyId: follow.lobbyId,
|
||||
roomName: follow.roomName
|
||||
}
|
||||
: null,
|
||||
tourney: {
|
||||
state: this.tourney.state,
|
||||
detail: this.tourney.detail,
|
||||
verified: this.tourney.verified
|
||||
},
|
||||
components: {
|
||||
obs: { ...this.obsProcess.status(), websocket: this.obs.connected },
|
||||
tosu: {
|
||||
...this.tosuProcess.status(),
|
||||
connected: this.tosu.connected,
|
||||
players: this.tosu.fresh()?.clients.length ?? 0,
|
||||
frameAt: this.tosu.latestAt ? new Date(this.tosu.latestAt).toISOString() : null
|
||||
},
|
||||
scores: {
|
||||
players: this.link.scores.players,
|
||||
sentAt: this.link.scores.sentAt ? new Date(this.link.scores.sentAt).toISOString() : null,
|
||||
acceptedAt: this.link.scores.acceptedAt
|
||||
? new Date(this.link.scores.acceptedAt).toISOString()
|
||||
: null
|
||||
},
|
||||
relay: {
|
||||
listening: this.ingest.running,
|
||||
receiving: this.ingest.receiving,
|
||||
recording: this.broadcaster.recording
|
||||
}
|
||||
},
|
||||
destinations: this.broadcaster
|
||||
.destinations()
|
||||
.map(({ label, state, error }) => ({ label, state, error })),
|
||||
vash: {
|
||||
connected: this.link.connected,
|
||||
rejected: this.link.rejected,
|
||||
channel: this.link.channelName,
|
||||
lastError: this.streamError
|
||||
},
|
||||
desktop: this.session,
|
||||
warnings: [...this.warnings]
|
||||
};
|
||||
}
|
||||
|
||||
private lastStateWrite = 0;
|
||||
private async writeState(force = false) {
|
||||
if (!force && Date.now() - this.lastStateWrite < 2000) return;
|
||||
this.lastStateWrite = Date.now();
|
||||
await writeJson(this.paths.state, this.state()).catch(() => {});
|
||||
}
|
||||
|
||||
/** Saves what OBS outputs and what the tourney window shows, for checking from SSH. */
|
||||
private async snapshot() {
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
await mkdir(this.paths.snapshots, { recursive: true });
|
||||
const files: string[] = [];
|
||||
const errors: string[] = [];
|
||||
if (this.obs.connected) {
|
||||
try {
|
||||
const file = join(this.paths.snapshots, `${stamp}-obs-output.png`);
|
||||
await writeFile(file, await this.scene.screenshot());
|
||||
files.push(file);
|
||||
} catch (err) {
|
||||
errors.push(`OBS: ${(err as Error).message}`);
|
||||
}
|
||||
} else errors.push('OBS is not connected');
|
||||
const window = this.tourney.window;
|
||||
if (window) {
|
||||
try {
|
||||
const file = join(this.paths.snapshots, `${stamp}-tourney-window.png`);
|
||||
await this.desktop.capture({
|
||||
x: window.clientX,
|
||||
y: window.clientY,
|
||||
width: window.clientWidth,
|
||||
height: window.clientHeight,
|
||||
path: file
|
||||
});
|
||||
files.push(file);
|
||||
} catch (err) {
|
||||
errors.push(`Tourney window: ${(err as Error).message}`);
|
||||
}
|
||||
} else errors.push('No osu!tourney window');
|
||||
const panel = join(this.paths.snapshots, 'tourney-panel.png');
|
||||
if (existsSync(panel)) files.push(panel, join(this.paths.snapshots, 'tourney-panel.json'));
|
||||
return { files, errors };
|
||||
}
|
||||
|
||||
private async shutdown() {
|
||||
const { log, paths } = this;
|
||||
this.stopping = true;
|
||||
log('Stopping');
|
||||
this.link.report(this.report());
|
||||
await this.writeState(true);
|
||||
if (this.obs.connected && this.obsStreaming) await this.scene.stopStream().catch(() => {});
|
||||
await this.broadcaster.stop().catch((err) => log('Going off air:', err));
|
||||
await this.tourney.stop().catch((err) => log('Stopping osu!tourney:', err));
|
||||
await this.stopObs().catch(() => {});
|
||||
await this.tosuProcess.stop();
|
||||
this.tosu.stop();
|
||||
this.link.stop();
|
||||
await stopInstanceProcesses(this.desktop, paths, log).catch(() => {});
|
||||
this.desktop.stop();
|
||||
await releaseLock(paths.lock);
|
||||
await writeJson(paths.state, { ...this.state(), state: 'stopped', detail: null }).catch(
|
||||
() => {}
|
||||
);
|
||||
await rm(paths.runtime, { force: true });
|
||||
log('Stopped');
|
||||
}
|
||||
}
|
||||
|
||||
// ── Watchdog ────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* What the scheduled task runs. It restarts the worker if it crashes (with
|
||||
* backoff) and exits when the worker stops on purpose. The request file says
|
||||
* whether this start is the worker, the one-time sign-in, or a diagnosis.
|
||||
*/
|
||||
export async function runTask(name: string): Promise<number> {
|
||||
const paths = layout(name);
|
||||
const log = fileLogger(paths.log);
|
||||
const request = await readJson<{ mode: string; at: number }>(paths.request);
|
||||
await rm(paths.request, { force: true });
|
||||
const mode = request && Date.now() - request.at < 5 * 60_000 ? request.mode : 'run';
|
||||
if (mode === 'login') return runSignIn(name);
|
||||
if (mode === 'doctor') return runDesktopDoctor(name);
|
||||
|
||||
const owner = await lockOwner(paths.lock);
|
||||
const runtime = await readJson<Runtime>(paths.runtime);
|
||||
if (runtime && owner?.pid === runtime.pid) {
|
||||
log('The worker is already running; not starting another');
|
||||
return 0;
|
||||
}
|
||||
let failures = 0;
|
||||
for (;;) {
|
||||
const started = Date.now();
|
||||
const child = Bun.spawn(selfCommand('worker', '--instance', name), {
|
||||
stdout: 'ignore',
|
||||
stderr: 'pipe',
|
||||
env: { ...process.env, VASH_WATCHDOG_PID: String(process.pid) },
|
||||
windowsHide: true
|
||||
});
|
||||
let stderr = '';
|
||||
void (async () => {
|
||||
for await (const chunk of child.stderr)
|
||||
stderr = (stderr + new TextDecoder().decode(chunk)).slice(-2000);
|
||||
})();
|
||||
const code = await child.exited;
|
||||
if (code === 0) return 0;
|
||||
if (Date.now() - started > 10 * 60_000) failures = 0;
|
||||
const wait = Math.min(60_000, 2000 * 2 ** failures++);
|
||||
log(
|
||||
`Worker exited with ${code}${stderr ? `: ${stderr.trim().split('\n').slice(-3).join(' ')}` : ''}; restarting in ${Math.round(wait / 1000)}s`
|
||||
);
|
||||
if (existsSync(paths.request)) return 0;
|
||||
await Bun.sleep(wait);
|
||||
}
|
||||
}
|
||||
|
||||
// ── One-time sign-in ────────────────────────────────────────────────────
|
||||
|
||||
export interface SignInState {
|
||||
state: 'starting' | 'waiting' | 'closing' | 'done' | 'failed';
|
||||
message: string;
|
||||
port: number;
|
||||
token: string;
|
||||
username: string | null;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
async function runSignIn(name: string): Promise<number> {
|
||||
const config = await loadConfig(name);
|
||||
const paths = layout(name);
|
||||
const log = fileLogger(paths.log);
|
||||
const desktop = new WindowsDesktop(paths.helperScript, log);
|
||||
const token = secret();
|
||||
let finish = false;
|
||||
let state: SignInState = {
|
||||
state: 'starting',
|
||||
message: 'Starting osu!…',
|
||||
port: config.ports.viewer,
|
||||
token,
|
||||
username: null,
|
||||
updatedAt: new Date().toISOString()
|
||||
};
|
||||
const set = async (next: Partial<SignInState>) => {
|
||||
state = { ...state, ...next, updatedAt: new Date().toISOString() };
|
||||
await writeJson(paths.login, state);
|
||||
};
|
||||
log('One-time sign-in: starting osu! normally');
|
||||
await stopInstanceProcesses(desktop, paths, log).catch(() => {});
|
||||
const restore = await normalModeFor(paths.osu);
|
||||
const viewer = startViewer({
|
||||
port: config.ports.viewer,
|
||||
token,
|
||||
desktop,
|
||||
scratch: paths.snapshots,
|
||||
title: `Vash broadcast sign-in (${name})`,
|
||||
status: () => ({
|
||||
message: state.message,
|
||||
done: state.state === 'done' || state.state === 'failed'
|
||||
}),
|
||||
onFinish: () => (finish = true)
|
||||
});
|
||||
await mkdir(paths.snapshots, { recursive: true });
|
||||
try {
|
||||
Bun.spawn([paths.osuExe], { cwd: paths.osu, stdout: 'ignore', stderr: 'ignore' });
|
||||
await set({
|
||||
state: 'waiting',
|
||||
message:
|
||||
'osu! is starting (a fresh installation downloads its files first). Sign in with the spectator account, tick Remember username and Remember password, then press Finish.'
|
||||
});
|
||||
const deadline = Date.now() + 45 * 60_000;
|
||||
while (!finish && Date.now() < deadline) {
|
||||
await Bun.sleep(1000);
|
||||
const window = pickMainWindow(await desktop.windows(paths.osu).catch(() => []));
|
||||
if (window && state.message.startsWith('osu! is starting'))
|
||||
await desktop.focus(window.hwnd).catch(() => false);
|
||||
}
|
||||
await set({ state: 'closing', message: 'Closing osu! so it saves the login…' });
|
||||
for (const process of await desktop.processes(paths.osu))
|
||||
await desktop.close(process.pid).catch(() => 0);
|
||||
for (
|
||||
let waited = 0;
|
||||
waited < 20_000 && (await desktop.processes(paths.osu)).length;
|
||||
waited += 1000
|
||||
)
|
||||
await Bun.sleep(1000);
|
||||
await stopInstanceProcesses(desktop, paths, log).catch(() => {});
|
||||
await restore();
|
||||
const login = await readSavedLogin(paths.osu);
|
||||
if (login.saved) {
|
||||
await set({
|
||||
state: 'done',
|
||||
username: login.username,
|
||||
message: `Signed in as ${login.username}; the login is saved.`
|
||||
});
|
||||
log(`One-time sign-in done: ${login.username}`);
|
||||
} else {
|
||||
await set({
|
||||
state: 'failed',
|
||||
message: finish
|
||||
? 'osu! did not save a login. Run vash-broadcast login again and tick Remember username and Remember password.'
|
||||
: 'The sign-in timed out after 45 minutes.'
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
await restore().catch(() => {});
|
||||
await set({ state: 'failed', message: (err as Error).message });
|
||||
} finally {
|
||||
await Bun.sleep(3000);
|
||||
viewer.stop(true);
|
||||
desktop.stop();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ── Diagnosis inside the desktop ─────────────────────────────────────────
|
||||
|
||||
async function runDesktopDoctor(name: string): Promise<number> {
|
||||
const paths = layout(name);
|
||||
const log = fileLogger(paths.log);
|
||||
const desktop = new WindowsDesktop(paths.helperScript, log);
|
||||
const checks: { name: string; ok: boolean; detail: string }[] = [];
|
||||
const check = async (label: string, run: () => Promise<string>) => {
|
||||
try {
|
||||
checks.push({ name: label, ok: true, detail: await run() });
|
||||
} catch (err) {
|
||||
checks.push({ name: label, ok: false, detail: (err as Error).message });
|
||||
}
|
||||
};
|
||||
await check('Desktop session', async () => {
|
||||
const session = await desktop.session();
|
||||
if (session.inputDesktop !== 'Default')
|
||||
throw new Error(
|
||||
`The desktop is locked or on a secure screen (${session.inputDesktop ?? 'no input desktop'}); rooms cannot be clicked`
|
||||
);
|
||||
if (session.sessionId !== session.consoleSessionId)
|
||||
throw new Error(
|
||||
`This session (${session.sessionId}) is not the one on the PC's screen (${session.consoleSessionId}); a disconnected Remote Desktop session has no display`
|
||||
);
|
||||
return `unlocked, ${session.screenWidth}×${session.screenHeight}, idle ${Math.round(session.idleMs / 1000)}s`;
|
||||
});
|
||||
await check('Screen capture', async () => {
|
||||
await mkdir(paths.snapshots, { recursive: true });
|
||||
const file = join(paths.snapshots, 'doctor-screen.jpg');
|
||||
const size = await desktop.capture({
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 640,
|
||||
height: 360,
|
||||
path: file,
|
||||
format: 'jpeg'
|
||||
});
|
||||
return `captured ${size.width}×${size.height} to ${file}`;
|
||||
});
|
||||
await check('Windows OCR', async () => {
|
||||
const ready = await desktop.ocrReady();
|
||||
return `language ${ready.language}, images up to ${ready.maxDimension} px`;
|
||||
});
|
||||
await check('osu!tourney window', async () => {
|
||||
const windows = await desktop.windows(paths.osu);
|
||||
return windows.length
|
||||
? windows
|
||||
.map((window) => `"${window.title}" ${window.clientWidth}×${window.clientHeight}`)
|
||||
.join('; ')
|
||||
: 'osu! is not running (normal while the worker is stopped)';
|
||||
});
|
||||
await check('AMD encoder in OBS log', async () => {
|
||||
const log = await latestObsLog(paths.obsConfig);
|
||||
if (!log) return 'no OBS log yet; start the worker once';
|
||||
const lines = log
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => /amf|AMD|encoder/i.test(line))
|
||||
.slice(0, 12);
|
||||
return lines.join(' | ') || 'no encoder lines in the latest OBS log';
|
||||
});
|
||||
desktop.stop();
|
||||
await writeJson(paths.doctor, { at: new Date().toISOString(), checks });
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user