esports/streamer/windows/supervise.ts

/**
 * Keeps one child program running: restarts it when it exits, waiting longer
 * after each quick failure (2 s, 4 s, … up to a minute) and forgetting past
 * failures once it has stayed up for a while.
 */

export interface Child {
	readonly pid: number;
	readonly exited: Promise<number | null>;
	kill(): void;
}

export type SupervisedState = 'stopped' | 'starting' | 'running' | 'backoff';

export interface SupervisorOptions {
	log: (...parts: unknown[]) => void;
	now?: () => number;
	/** A run at least this long resets the backoff. */
	stableMs?: number;
	baseDelayMs?: number;
	maxDelayMs?: number;
}

export class Supervised {
	state: SupervisedState = 'stopped';
	child: Child | null = null;
	restarts = 0;
	failures = 0;
	lastExit: string | null = null;
	startedAt = 0;
	private retryAt = 0;
	private wanted = false;
	private now: () => number;

	constructor(
		readonly name: string,
		private launch: () => Child | Promise<Child>,
		private options: SupervisorOptions
	) {
		this.now = options.now ?? Date.now;
	}

	get running() {
		return this.state === 'running';
	}

	/** Starts the program if it should run and is not; call on every tick. */
	async ensure() {
		this.wanted = true;
		if (this.state === 'running' || this.state === 'starting') return;
		if (this.state === 'backoff' && this.now() < this.retryAt) return;
		this.state = 'starting';
		let child: Child;
		try {
			child = await this.launch();
		} catch (err) {
			this.onExit(`could not start: ${(err as Error).message}`);
			return;
		}
		if (!this.wanted) {
			child.kill();
			this.state = 'stopped';
			return;
		}
		this.child = child;
		this.startedAt = this.now();
		this.state = 'running';
		void child.exited.then((code) => {
			if (this.child !== child) return;
			this.child = null;
			this.onExit(`exited with ${code}`);
		});
	}

	private onExit(reason: string) {
		this.lastExit = reason;
		if (!this.wanted) {
			this.state = 'stopped';
			return;
		}
		const ranFor = this.startedAt ? this.now() - this.startedAt : 0;
		if (ranFor >= (this.options.stableMs ?? 5 * 60_000)) this.failures = 0;
		const delay = Math.min(
			this.options.maxDelayMs ?? 60_000,
			(this.options.baseDelayMs ?? 2000) * 2 ** this.failures
		);
		this.failures++;
		this.restarts++;
		this.startedAt = 0;
		this.retryAt = this.now() + delay;
		this.state = 'backoff';
		this.options.log(`${this.name} ${reason}; restarting in ${Math.round(delay / 1000)}s`);
	}

	/** Stops the program and keeps it stopped until the next ensure(). */
	async stop(graceful?: (child: Child) => Promise<void>) {
		this.wanted = false;
		const child = this.child;
		this.child = null;
		this.state = 'stopped';
		if (!child) return;
		if (graceful) {
			await graceful(child).catch(() => {});
			const done = await Promise.race([
				child.exited.then(() => true),
				Bun.sleep(10_000).then(() => false)
			]);
			if (done) return;
		}
		child.kill();
		await Promise.race([child.exited, Bun.sleep(5000)]);
	}

	status() {
		return {
			state: this.state,
			pid: this.child?.pid ?? null,
			restarts: this.restarts,
			lastExit: this.lastExit,
			upSec: this.state === 'running' ? Math.round((this.now() - this.startedAt) / 1000) : 0
		};
	}
}