/** * The desktop the worker drives: windows, screen capture, OCR and input. * On Windows this is a long-running PowerShell helper (desktop.ps1) in the * signed-in session; tests use a fake with the same interface. */ import { existsSync } from 'node:fs'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import type { Subprocess } from 'bun'; import helperScript from './desktop.ps1' with { type: 'text' }; export interface WindowInfo { hwnd: number; pid: number; title: string; className: string; exe: string | null; visible: boolean; minimized: boolean; x: number; y: number; width: number; height: number; /** The client area (no title bar or borders) in screen coordinates. */ clientX: number; clientY: number; clientWidth: number; clientHeight: number; } export interface SessionInfo { sessionId: number; consoleSessionId: number; /** "Default" when unlocked; "Winlogon" or null when locked or on a secure screen. */ inputDesktop: string | null; screenWidth: number; screenHeight: number; idleMs: number; } export interface OcrLine { text: string; x: number; y: number; width: number; height: number; } export interface CaptureOptions { x: number; y: number; width: number; height: number; path: string; format?: 'png' | 'jpeg'; maxWidth?: number; scale?: number; invert?: boolean; quality?: number; } export interface Desktop { session(): Promise; /** Top-level windows of processes whose executable is under a folder. */ windows(under: string): Promise; window(hwnd: number): Promise; processes(under: string): Promise<{ pid: number; path: string }[]>; kill(pids: number[]): Promise; /** Posts WM_CLOSE to a process's windows. */ close(pid: number): Promise; focus(hwnd: number): Promise; move(hwnd: number, x: number, y: number): Promise; setTitle(hwnd: number, title: string): Promise; capture(options: CaptureOptions): Promise<{ width: number; height: number }>; ocr(path: string): Promise; ocrReady(): Promise<{ language: string; maxDimension: number }>; click(x: number, y: number, right?: boolean): Promise; type(text: string): Promise; key(vk: number): Promise; /** Keeps the PC from sleeping and the display on while the helper runs. */ stayAwake(on: boolean): Promise; stop(): void; } interface Pending { resolve(value: unknown): void; reject(err: Error): void; timer: ReturnType; } export class WindowsDesktop implements Desktop { private process: Subprocess<'pipe', 'pipe', 'pipe'> | null = null; private pending = new Map(); private nextId = 1; private ready: Promise | null = null; private lastError = ''; constructor( private scriptPath: string, private log: (...parts: unknown[]) => void = () => {} ) {} /** * Writes the helper next to the program when it differs from this build's copy, * with a byte order mark: Windows PowerShell reads a script without one as ANSI. */ static async writeScript(path: string) { const content = `\uFEFF${helperScript.replace(/^\uFEFF/, '')}`; const current = existsSync(path) ? await readFile(path, 'utf8') : null; if (current === content) return; await mkdir(dirname(path), { recursive: true }); await writeFile(path, content); } private start(): Promise { if (this.ready) return this.ready; this.ready = (async () => { await WindowsDesktop.writeScript(this.scriptPath); const process = Bun.spawn( [ 'powershell.exe', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', this.scriptPath ], { stdin: 'pipe', stdout: 'pipe', stderr: 'pipe', windowsHide: true } ); this.process = process; let resolveReady!: () => void; let rejectReady!: (err: Error) => void; const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; }); this.pending.set(0, { resolve: () => resolveReady(), reject: rejectReady, timer: setTimeout(() => rejectReady(new Error('The desktop helper did not start')), 60_000) }); void this.readLines(process); void (async () => { for await (const chunk of process.stderr) this.lastError = new TextDecoder().decode(chunk).trim().slice(-500) || this.lastError; })(); void process.exited.then((code) => { if (this.process !== process) return; this.process = null; this.ready = null; const error = new Error( `The desktop helper exited (${code})${this.lastError ? `: ${this.lastError}` : ''}` ); for (const waiter of this.pending.values()) { clearTimeout(waiter.timer); waiter.reject(error); } this.pending.clear(); }); await ready; })(); this.ready.catch(() => { this.ready = null; this.process?.kill(); this.process = null; }); return this.ready; } private async readLines(process: Subprocess<'pipe', 'pipe', 'pipe'>) { let buffer = ''; const decoder = new TextDecoder(); for await (const chunk of process.stdout) { buffer += decoder.decode(chunk, { stream: true }); let newline: number; while ((newline = buffer.indexOf('\n')) >= 0) { const line = buffer.slice(0, newline).trim(); buffer = buffer.slice(newline + 1); if (!line) continue; let message: { id: number; ok: boolean; result?: unknown; error?: string }; try { message = JSON.parse(line.replace(/^\uFEFF/, '')); } catch { this.log('desktop helper:', line.slice(0, 200)); continue; } const waiter = this.pending.get(message.id); if (!waiter) continue; this.pending.delete(message.id); clearTimeout(waiter.timer); if (message.ok) waiter.resolve(message.result ?? null); else waiter.reject(new Error(message.error ?? 'desktop helper error')); } } } async call(cmd: string, args: Record = {}, timeoutMs = 20_000): Promise { await this.start(); const process = this.process; if (!process) throw new Error('The desktop helper is not running'); const id = this.nextId++; return new Promise((resolve, reject) => { const timer = setTimeout(() => { this.pending.delete(id); // A stuck helper is restarted on the next call. this.process = null; this.ready = null; process.kill(); reject(new Error(`The desktop helper did not answer ${cmd} in time`)); }, timeoutMs); this.pending.set(id, { resolve: resolve as (value: unknown) => void, reject, timer }); process.stdin.write(`${JSON.stringify({ id, cmd, ...args })}\n`); void process.stdin.flush(); }); } session() { return this.call('session'); } windows(under: string) { return this.call('windows', { under }); } window(hwnd: number) { return this.call('window', { hwnd }); } processes(under: string) { return this.call<{ pid: number; path: string }[]>('processes', { under }); } kill(pids: number[]) { return pids.length ? this.call('kill', { pids }) : Promise.resolve(0); } close(pid: number) { return this.call('close', { pid }); } focus(hwnd: number) { return this.call('focus', { hwnd }); } move(hwnd: number, x: number, y: number) { return this.call('move', { hwnd, x, y }); } setTitle(hwnd: number, title: string) { return this.call('title', { hwnd, title }); } capture(options: CaptureOptions) { return this.call<{ width: number; height: number }>('capture', { ...options }); } ocr(path: string) { return this.call('ocr', { path }, 40_000); } ocrReady() { return this.call<{ language: string; maxDimension: number }>('ocrReady', {}, 40_000); } click(x: number, y: number, right = false) { return this.call('click', { x, y, right }); } type(text: string) { return this.call('type', { text }); } async key(vk: number) { await this.call('key', { vk }); } stayAwake(on: boolean) { return this.call('awake', { on }); } stop() { const process = this.process; this.process = null; this.ready = null; try { process?.stdin.end(); } catch { /* already gone */ } setTimeout(() => process?.kill(), 1000); } }