/** * Keeps one child program running: restarts it when it exits, waiting longer * after each quick failure (2 s, 4 s, … up to a minute) and forgetting past * failures once it has stayed up for a while. */ export interface Child { readonly pid: number; readonly exited: Promise; kill(): void; } export type SupervisedState = 'stopped' | 'starting' | 'running' | 'backoff'; export interface SupervisorOptions { log: (...parts: unknown[]) => void; now?: () => number; /** A run at least this long resets the backoff. */ stableMs?: number; baseDelayMs?: number; maxDelayMs?: number; } export class Supervised { state: SupervisedState = 'stopped'; child: Child | null = null; restarts = 0; failures = 0; lastExit: string | null = null; startedAt = 0; private retryAt = 0; private wanted = false; private now: () => number; constructor( readonly name: string, private launch: () => Child | Promise, private options: SupervisorOptions ) { this.now = options.now ?? Date.now; } get running() { return this.state === 'running'; } /** Starts the program if it should run and is not; call on every tick. */ async ensure() { this.wanted = true; if (this.state === 'running' || this.state === 'starting') return; if (this.state === 'backoff' && this.now() < this.retryAt) return; this.state = 'starting'; let child: Child; try { child = await this.launch(); } catch (err) { this.onExit(`could not start: ${(err as Error).message}`); return; } if (!this.wanted) { child.kill(); this.state = 'stopped'; return; } this.child = child; this.startedAt = this.now(); this.state = 'running'; void child.exited.then((code) => { if (this.child !== child) return; this.child = null; this.onExit(`exited with ${code}`); }); } private onExit(reason: string) { this.lastExit = reason; if (!this.wanted) { this.state = 'stopped'; return; } const ranFor = this.startedAt ? this.now() - this.startedAt : 0; if (ranFor >= (this.options.stableMs ?? 5 * 60_000)) this.failures = 0; const delay = Math.min( this.options.maxDelayMs ?? 60_000, (this.options.baseDelayMs ?? 2000) * 2 ** this.failures ); this.failures++; this.restarts++; this.startedAt = 0; this.retryAt = this.now() + delay; this.state = 'backoff'; this.options.log(`${this.name} ${reason}; restarting in ${Math.round(delay / 1000)}s`); } /** Stops the program and keeps it stopped until the next ensure(). */ async stop(graceful?: (child: Child) => Promise) { this.wanted = false; const child = this.child; this.child = null; this.state = 'stopped'; if (!child) return; if (graceful) { await graceful(child).catch(() => {}); const done = await Promise.race([ child.exited.then(() => true), Bun.sleep(10_000).then(() => false) ]); if (done) return; } child.kill(); await Promise.race([child.exited, Bun.sleep(5000)]); } status() { return { state: this.state, pid: this.child?.pid ?? null, restarts: this.restarts, lastExit: this.lastExit, upSec: this.state === 'running' ? Math.round((this.now() - this.startedAt) / 1000) : 0 }; } }