diff --git a/deploy/home-collector/README.md b/deploy/home-collector/README.md index fbc46d0..918f748 100644 --- a/deploy/home-collector/README.md +++ b/deploy/home-collector/README.md @@ -13,6 +13,18 @@ every minute (systemd **user** timer `home-collector.timer`) and upserts two row The page reads both rows in the same parallel query batch as tasks and links, so a slow upstream never delays it; a snapshot older than 3 minutes is shown as stale. +## Live stats (1 s) + +On top of the minute snapshot, `/start` opens a Server-Sent Events stream (`/start/live`, +owner only) while the tab is visible. The app reads netdata on the host +(`NETDATA_URL`, default `http://100.116.93.104:19999`, the Tailscale-only bind) once a second +through one shared poller (`src/lib/server/live-stats.ts`): host CPU, RAM, network, disk IO +and per-container CPU/mem/state/health (netdata's docker and cgroups collectors), so the app +still never touches the Docker socket. The poller stops 5 s after the last viewer leaves. +The app container reaches netdata through one ufw rule: +`ufw allow in on br-c1585f0009ac from 172.26.0.0/16 to 100.116.93.104 port 19999 proto tcp` +(the `backend` compose network's bridge). + ## Setup 1. A login role that may only upsert snapshots (created once, cluster superuser): diff --git a/src/hooks.server.ts b/src/hooks.server.ts index b930f05..9f5e177 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -42,7 +42,9 @@ export const handle: Handle = async ({ event, resolve }) => { ); if (ownerOnly) { response.headers.set('x-robots-tag', 'noindex, nofollow, noarchive'); - response.headers.set('cache-control', 'private, no-store'); + // Keep a stricter value a route set itself (the /start/live stream adds no-transform). + if (!response.headers.get('cache-control')?.includes('no-store')) + response.headers.set('cache-control', 'private, no-store'); } return response; }; diff --git a/src/lib/components/Sparkline.svelte b/src/lib/components/Sparkline.svelte new file mode 100644 index 0000000..40eca67 --- /dev/null +++ b/src/lib/components/Sparkline.svelte @@ -0,0 +1,58 @@ + + + + {#each series as s, i (i)} + {@const p = points(s.values)} + {#if p.length > 1} + {#if s.fill} + + {/if} + + {/if} + {/each} + + + diff --git a/src/lib/home.ts b/src/lib/home.ts index b6cd798..3e6e4aa 100644 --- a/src/lib/home.ts +++ b/src/lib/home.ts @@ -67,6 +67,29 @@ export type HostSnapshot = { timers: TimerState[]; b2: Record | null; }; +/** One second of live stats from /start/live (see #lib/server/live-stats). */ +export type LiveContainer = { + /** CPU % of one core (like `docker stats`), memory in bytes. */ + cpu: number | null; + mem: number | null; + state: string | null; + health: string | null; +}; +export type LiveFrame = { + t: number; + /** Host CPU % (all cores = 100) and iowait share. */ + cpu: number; + iowait: number; + mem: { used: number; total: number }; + /** Bits per second. */ + net: { rx: number; tx: number }; + /** Bytes per second. */ + io: { read: number; write: number }; + load: number[]; + containers: Record; +}; +export type LiveHistory = Pick[]; + export type NtfyMessage = { id: string; topic: string; diff --git a/src/lib/server/live-stats.test.ts b/src/lib/server/live-stats.test.ts new file mode 100644 index 0000000..9b371ca --- /dev/null +++ b/src/lib/server/live-stats.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { parseAllmetrics } from './live-stats.js'; + +const d = (o: Record) => + Object.fromEntries(Object.entries(o).map(([k, v]) => [k, { name: k, value: v }])); + +describe('parseAllmetrics', () => { + it('turns netdata charts into a live frame', () => { + const f = parseAllmetrics( + { + 'system.cpu': { dimensions: d({ user: 30, system: 5, iowait: 4, idle: 61 }) }, + 'system.ram': { dimensions: d({ free: 1024, used: 2048, cached: 1024, buffers: 0 }) }, + 'system.net': { dimensions: d({ InOctets: 1000, OutOctets: -500 }) }, + 'system.io': { dimensions: d({ in: 10, out: -20 }) }, + 'system.load': { dimensions: d({ load1: 1.234, load5: 2, load15: 3 }) }, + 'cgroup_web-1.cpu': { dimensions: d({ user: 120, system: 30 }) }, + 'cgroup_web-1.mem_usage': { dimensions: d({ ram: 100, swap: 0 }) }, + 'docker_local.container_web-1_state': { + dimensions: d({ running: 1, exited: 0, paused: 0 }) + }, + 'docker_local.container_web-1_health_status': { + dimensions: d({ healthy: 0, not_running_unhealthy: 0, no_healthcheck: 1 }) + }, + 'cgroup_old-1.cpu': { dimensions: d({ user: 5, system: 0 }) }, + 'docker_local.container_old-1_state': { dimensions: d({ running: 0, exited: 1 }) }, + 'docker_local.container_old-1_health_status': { + dimensions: d({ healthy: 0, not_running_unhealthy: 1 }) + } + }, + 1 + ); + expect(f.cpu).toBe(39); + expect(f.iowait).toBe(4); + expect(f.mem).toEqual({ used: 2048 * 1024 * 1024, total: 4096 * 1024 * 1024 }); + expect(f.net).toEqual({ rx: 1_000_000, tx: 500_000 }); + expect(f.io).toEqual({ read: 10240, write: 20480 }); + expect(f.load).toEqual([1.23, 2, 3]); + expect(f.containers['web-1']).toEqual({ + cpu: 150, + mem: 100 * 1024 * 1024, + state: 'running', + health: null + }); + expect(f.containers['old-1']).toEqual({ + cpu: null, + mem: null, + state: 'exited', + health: 'unhealthy' + }); + }); +}); diff --git a/src/lib/server/live-stats.ts b/src/lib/server/live-stats.ts new file mode 100644 index 0000000..e31acb7 --- /dev/null +++ b/src/lib/server/live-stats.ts @@ -0,0 +1,203 @@ +// Live server stats for /start (CPU, RAM, network, disk IO and per-container CPU/mem/state, +// once a second). netdata on the host already samples all of this every second, including +// the Docker state/health of every container, so the app never needs the Docker socket: one +// shared poller reads netdata's allmetrics (one ~40 KB local request, ~5 ms) and fans the +// frame out to every open /start/live stream. It runs only while someone is watching and +// stops a few seconds after the last viewer leaves. +import type { LiveContainer, LiveFrame, LiveHistory } from '#lib/home.js'; + +const NETDATA = (process.env.NETDATA_URL || 'http://100.116.93.104:19999').replace(/\/$/, ''); +const FILTER = [ + 'system.cpu', + 'system.ram', + 'system.net', + 'system.io', + 'system.load', + 'cgroup_*.cpu', + 'cgroup_*.mem_usage', + 'docker_local.container_*_state', + 'docker_local.container_*_health_status' +].join(' '); +const HISTORY = 60; +const GRACE_MS = 5_000; +const MiB = 1024 * 1024; + +type Dims = Record; +type Chart = { dimensions: Dims }; + +const dim = (c: Chart | undefined, id: string) => Number(c?.dimensions[id]?.value ?? 0) || 0; +const sum = (c: Chart | undefined) => + Object.values(c?.dimensions ?? {}).reduce((n, d) => n + (Number(d.value) || 0), 0); +/** The dimension that is 1 in a netdata one-hot chart (container state / health). */ +const hot = (c: Chart | undefined) => + Object.entries(c?.dimensions ?? {}).find(([, d]) => d.value === 1)?.[0] ?? null; +const HEALTH: Record = { + healthy: 'healthy', + unhealthy: 'unhealthy', + not_running_unhealthy: 'unhealthy', + starting: 'starting', + no_healthcheck: null +}; + +async function get(path: string): Promise { + const r = await fetch(`${NETDATA}${path}`, { signal: AbortSignal.timeout(900) }); + if (!r.ok) throw new Error(`netdata ${r.status}`); + return r.json(); +} + +export function parseAllmetrics(all: Record, t = Date.now()): LiveFrame { + const cpu = all['system.cpu']; + const ram = all['system.ram']; + const containers: Record = {}; + for (const [id, chart] of Object.entries(all)) { + let m = /^cgroup_(.+)\.(cpu|mem_usage)$/.exec(id); + if (m) { + const c = (containers[m[1]] ??= { cpu: null, mem: null, state: null, health: null }); + if (m[2] === 'cpu') c.cpu = round(sum(chart), 2); + else c.mem = Math.round(dim(chart, 'ram') * MiB); + continue; + } + m = /^docker_local\.container_(.+)_(state|health_status)$/.exec(id); + if (m) { + const c = (containers[m[1]] ??= { cpu: null, mem: null, state: null, health: null }); + const v = hot(chart); + if (m[2] === 'state') c.state = v; + else c.health = v && v in HEALTH ? HEALTH[v] : v; + } + } + // A stopped container keeps a stale cgroup chart for a while; don't show its last CPU. + for (const c of Object.values(containers)) + if (c.state && c.state !== 'running') { + c.cpu = null; + c.mem = null; + } + return { + t, + cpu: cpu ? round(100 - dim(cpu, 'idle'), 1) : 0, + iowait: round(dim(cpu, 'iowait'), 1), + mem: { used: Math.round(dim(ram, 'used') * MiB), total: Math.round(sum(ram) * MiB) }, + net: { + rx: Math.round(Math.abs(dim(all['system.net'], 'InOctets')) * 1000), + tx: Math.round(Math.abs(dim(all['system.net'], 'OutOctets')) * 1000) + }, + io: { + read: Math.round(Math.abs(dim(all['system.io'], 'in')) * 1024), + write: Math.round(Math.abs(dim(all['system.io'], 'out')) * 1024) + }, + load: ['load1', 'load5', 'load15'].map((d) => round(dim(all['system.load'], d), 2)), + containers + }; +} + +const round = (n: number, d: number) => Math.round(n * 10 ** d) / 10 ** d; + +/** The last minute from netdata's own history, so sparklines are full on the first paint. */ +async function backfill(): Promise { + const q = (chart: string) => + get(`/api/v1/data?chart=${chart}&after=-${HISTORY}&points=${HISTORY}&format=json`) as Promise<{ + labels: string[]; + data: number[][]; + }>; + const [cpu, ram, net, io] = await Promise.all( + ['system.cpu', 'system.ram', 'system.net', 'system.io'].map(q) + ); + const col = (r: { labels: string[] }, name: string) => r.labels.indexOf(name); + const at = (r: { data: number[][] }, t: number) => r.data.find((row) => row[0] === t); + const iIdle = col(cpu, 'idle'); + const iUsed = col(ram, 'used'); + const out: LiveHistory = []; + for (const row of [...cpu.data].sort((a, b) => a[0] - b[0])) { + const t = row[0]; + const r = at(ram, t); + const n = at(net, t); + const d = at(io, t); + if (!r || !n || !d) continue; + out.push({ + t: t * 1000, + cpu: round(100 - row[iIdle], 1), + mem: { + used: Math.round(r[iUsed] * MiB), + total: Math.round(r.slice(1).reduce((a, b) => a + b, 0) * MiB) + }, + net: { rx: Math.round(Math.abs(n[1]) * 1000), tx: Math.round(Math.abs(n[2]) * 1000) }, + io: { read: Math.round(Math.abs(d[1]) * 1024), write: Math.round(Math.abs(d[2]) * 1024) } + }); + } + return out; +} + +type Listener = (event: 'frame' | 'fail', data: unknown) => void; + +const listeners = new Set(); +let history: LiveHistory = []; +let timer: ReturnType | null = null; +let stopTimer: ReturnType | null = null; +let inflight = false; +let last: LiveFrame | null = null; + +async function tick() { + if (inflight) return; // never pile up requests if netdata is slow + inflight = true; + try { + const all = (await get( + `/api/v1/allmetrics?format=json&filter=${encodeURIComponent(FILTER)}` + )) as Record; + const frame = parseAllmetrics(all); + last = frame; + history.push({ t: frame.t, cpu: frame.cpu, mem: frame.mem, net: frame.net, io: frame.io }); + if (history.length > HISTORY) history = history.slice(-HISTORY); + for (const l of listeners) l('frame', frame); + } catch (e) { + for (const l of listeners) l('fail', { message: e instanceof Error ? e.message : 'failed' }); + } finally { + inflight = false; + } +} + +function start() { + if (stopTimer) { + clearTimeout(stopTimer); + stopTimer = null; + } + if (timer) return; + timer = setInterval(tick, 1000); + void tick(); +} + +function stop() { + if (timer) clearInterval(timer); + timer = null; + history = []; + last = null; +} + +/** Subscribe to frames; returns the current minute of history and an unsubscribe. */ +export async function subscribe(fn: Listener) { + const fresh = !timer; + listeners.add(fn); + start(); + if (fresh || history.length < 5) { + try { + const h = await backfill(); + const lastT = h.at(-1)?.t ?? 0; + if (history.length < h.length) history = h.concat(history.filter((x) => x.t > lastT)); + } catch { + // sparklines just fill up live + } + } + return { + history: history.slice(-HISTORY), + last, + unsubscribe() { + listeners.delete(fn); + if (!listeners.size && !stopTimer) + stopTimer = setTimeout(() => { + stopTimer = null; + if (!listeners.size) stop(); + }, GRACE_MS); + } + }; +} + +/** For tests and diagnostics. */ +export const liveStatus = () => ({ viewers: listeners.size, running: !!timer }); diff --git a/src/routes/start/+page.svelte b/src/routes/start/+page.svelte index 5887e94..70a62b5 100644 --- a/src/routes/start/+page.svelte +++ b/src/routes/start/+page.svelte @@ -1,10 +1,20 @@