start: live server stats over SSE (1 s): CPU, RAM, net, disk IO with 60 s sparklines, per-container CPU/mem/state; one shared netdata poller that runs only while /start is open (owner-only /start/live)
All checks were successful
Deploy production / deploy (push) Successful in 1m29s
All checks were successful
Deploy production / deploy (push) Successful in 1m29s
This commit is contained in:
58
src/lib/components/Sparkline.svelte
Normal file
58
src/lib/components/Sparkline.svelte
Normal file
@@ -0,0 +1,58 @@
|
||||
<script lang="ts">
|
||||
// A tiny inline sparkline (no axes): the last `slots` samples, newest on the right. Pass
|
||||
// `max` for a fixed scale (e.g. 100 for %), otherwise it scales to the visible peak.
|
||||
type Series = { values: number[]; color: string; fill?: boolean };
|
||||
let {
|
||||
series,
|
||||
max,
|
||||
slots = 60,
|
||||
label = ''
|
||||
}: { series: Series[]; max?: number; slots?: number; label?: string } = $props();
|
||||
|
||||
const W = 100;
|
||||
const H = 20;
|
||||
const top = $derived(
|
||||
max ?? Math.max(1, ...series.flatMap((s) => s.values.slice(-slots).map((v) => v || 0)))
|
||||
);
|
||||
function points(values: number[]) {
|
||||
const v = values.slice(-slots);
|
||||
const off = slots - v.length;
|
||||
return v.map((n, i) => {
|
||||
const x = ((off + i) / (slots - 1)) * W;
|
||||
const y = H - 1 - (Math.min(top, Math.max(0, n || 0)) / top) * (H - 2);
|
||||
return `${x.toFixed(2)},${y.toFixed(2)}`;
|
||||
});
|
||||
}
|
||||
</script>
|
||||
|
||||
<svg class="spark" viewBox="0 0 {W} {H}" preserveAspectRatio="none" role="img" aria-label={label}>
|
||||
{#each series as s, i (i)}
|
||||
{@const p = points(s.values)}
|
||||
{#if p.length > 1}
|
||||
{#if s.fill}
|
||||
<polygon
|
||||
points="{p[0].split(',')[0]},{H} {p.join(' ')} {W},{H}"
|
||||
fill={s.color}
|
||||
fill-opacity="0.14"
|
||||
/>
|
||||
{/if}
|
||||
<polyline
|
||||
points={p.join(' ')}
|
||||
fill="none"
|
||||
stroke={s.color}
|
||||
stroke-width="1.25"
|
||||
stroke-linejoin="round"
|
||||
vector-effect="non-scaling-stroke"
|
||||
/>
|
||||
{/if}
|
||||
{/each}
|
||||
</svg>
|
||||
|
||||
<style>
|
||||
.spark {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 20px;
|
||||
overflow: visible;
|
||||
}
|
||||
</style>
|
||||
@@ -67,6 +67,29 @@ export type HostSnapshot = {
|
||||
timers: TimerState[];
|
||||
b2: Record<string, unknown> | null;
|
||||
};
|
||||
/** One second of live stats from /start/live (see #lib/server/live-stats). */
|
||||
export type LiveContainer = {
|
||||
/** CPU % of one core (like `docker stats`), memory in bytes. */
|
||||
cpu: number | null;
|
||||
mem: number | null;
|
||||
state: string | null;
|
||||
health: string | null;
|
||||
};
|
||||
export type LiveFrame = {
|
||||
t: number;
|
||||
/** Host CPU % (all cores = 100) and iowait share. */
|
||||
cpu: number;
|
||||
iowait: number;
|
||||
mem: { used: number; total: number };
|
||||
/** Bits per second. */
|
||||
net: { rx: number; tx: number };
|
||||
/** Bytes per second. */
|
||||
io: { read: number; write: number };
|
||||
load: number[];
|
||||
containers: Record<string, LiveContainer>;
|
||||
};
|
||||
export type LiveHistory = Pick<LiveFrame, 't' | 'cpu' | 'mem' | 'net' | 'io'>[];
|
||||
|
||||
export type NtfyMessage = {
|
||||
id: string;
|
||||
topic: string;
|
||||
|
||||
51
src/lib/server/live-stats.test.ts
Normal file
51
src/lib/server/live-stats.test.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { parseAllmetrics } from './live-stats.js';
|
||||
|
||||
const d = (o: Record<string, number>) =>
|
||||
Object.fromEntries(Object.entries(o).map(([k, v]) => [k, { name: k, value: v }]));
|
||||
|
||||
describe('parseAllmetrics', () => {
|
||||
it('turns netdata charts into a live frame', () => {
|
||||
const f = parseAllmetrics(
|
||||
{
|
||||
'system.cpu': { dimensions: d({ user: 30, system: 5, iowait: 4, idle: 61 }) },
|
||||
'system.ram': { dimensions: d({ free: 1024, used: 2048, cached: 1024, buffers: 0 }) },
|
||||
'system.net': { dimensions: d({ InOctets: 1000, OutOctets: -500 }) },
|
||||
'system.io': { dimensions: d({ in: 10, out: -20 }) },
|
||||
'system.load': { dimensions: d({ load1: 1.234, load5: 2, load15: 3 }) },
|
||||
'cgroup_web-1.cpu': { dimensions: d({ user: 120, system: 30 }) },
|
||||
'cgroup_web-1.mem_usage': { dimensions: d({ ram: 100, swap: 0 }) },
|
||||
'docker_local.container_web-1_state': {
|
||||
dimensions: d({ running: 1, exited: 0, paused: 0 })
|
||||
},
|
||||
'docker_local.container_web-1_health_status': {
|
||||
dimensions: d({ healthy: 0, not_running_unhealthy: 0, no_healthcheck: 1 })
|
||||
},
|
||||
'cgroup_old-1.cpu': { dimensions: d({ user: 5, system: 0 }) },
|
||||
'docker_local.container_old-1_state': { dimensions: d({ running: 0, exited: 1 }) },
|
||||
'docker_local.container_old-1_health_status': {
|
||||
dimensions: d({ healthy: 0, not_running_unhealthy: 1 })
|
||||
}
|
||||
},
|
||||
1
|
||||
);
|
||||
expect(f.cpu).toBe(39);
|
||||
expect(f.iowait).toBe(4);
|
||||
expect(f.mem).toEqual({ used: 2048 * 1024 * 1024, total: 4096 * 1024 * 1024 });
|
||||
expect(f.net).toEqual({ rx: 1_000_000, tx: 500_000 });
|
||||
expect(f.io).toEqual({ read: 10240, write: 20480 });
|
||||
expect(f.load).toEqual([1.23, 2, 3]);
|
||||
expect(f.containers['web-1']).toEqual({
|
||||
cpu: 150,
|
||||
mem: 100 * 1024 * 1024,
|
||||
state: 'running',
|
||||
health: null
|
||||
});
|
||||
expect(f.containers['old-1']).toEqual({
|
||||
cpu: null,
|
||||
mem: null,
|
||||
state: 'exited',
|
||||
health: 'unhealthy'
|
||||
});
|
||||
});
|
||||
});
|
||||
203
src/lib/server/live-stats.ts
Normal file
203
src/lib/server/live-stats.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
// Live server stats for /start (CPU, RAM, network, disk IO and per-container CPU/mem/state,
|
||||
// once a second). netdata on the host already samples all of this every second, including
|
||||
// the Docker state/health of every container, so the app never needs the Docker socket: one
|
||||
// shared poller reads netdata's allmetrics (one ~40 KB local request, ~5 ms) and fans the
|
||||
// frame out to every open /start/live stream. It runs only while someone is watching and
|
||||
// stops a few seconds after the last viewer leaves.
|
||||
import type { LiveContainer, LiveFrame, LiveHistory } from '#lib/home.js';
|
||||
|
||||
const NETDATA = (process.env.NETDATA_URL || 'http://100.116.93.104:19999').replace(/\/$/, '');
|
||||
const FILTER = [
|
||||
'system.cpu',
|
||||
'system.ram',
|
||||
'system.net',
|
||||
'system.io',
|
||||
'system.load',
|
||||
'cgroup_*.cpu',
|
||||
'cgroup_*.mem_usage',
|
||||
'docker_local.container_*_state',
|
||||
'docker_local.container_*_health_status'
|
||||
].join(' ');
|
||||
const HISTORY = 60;
|
||||
const GRACE_MS = 5_000;
|
||||
const MiB = 1024 * 1024;
|
||||
|
||||
type Dims = Record<string, { name: string; value: number | null }>;
|
||||
type Chart = { dimensions: Dims };
|
||||
|
||||
const dim = (c: Chart | undefined, id: string) => Number(c?.dimensions[id]?.value ?? 0) || 0;
|
||||
const sum = (c: Chart | undefined) =>
|
||||
Object.values(c?.dimensions ?? {}).reduce((n, d) => n + (Number(d.value) || 0), 0);
|
||||
/** The dimension that is 1 in a netdata one-hot chart (container state / health). */
|
||||
const hot = (c: Chart | undefined) =>
|
||||
Object.entries(c?.dimensions ?? {}).find(([, d]) => d.value === 1)?.[0] ?? null;
|
||||
const HEALTH: Record<string, string | null> = {
|
||||
healthy: 'healthy',
|
||||
unhealthy: 'unhealthy',
|
||||
not_running_unhealthy: 'unhealthy',
|
||||
starting: 'starting',
|
||||
no_healthcheck: null
|
||||
};
|
||||
|
||||
async function get(path: string): Promise<unknown> {
|
||||
const r = await fetch(`${NETDATA}${path}`, { signal: AbortSignal.timeout(900) });
|
||||
if (!r.ok) throw new Error(`netdata ${r.status}`);
|
||||
return r.json();
|
||||
}
|
||||
|
||||
export function parseAllmetrics(all: Record<string, Chart>, t = Date.now()): LiveFrame {
|
||||
const cpu = all['system.cpu'];
|
||||
const ram = all['system.ram'];
|
||||
const containers: Record<string, LiveContainer> = {};
|
||||
for (const [id, chart] of Object.entries(all)) {
|
||||
let m = /^cgroup_(.+)\.(cpu|mem_usage)$/.exec(id);
|
||||
if (m) {
|
||||
const c = (containers[m[1]] ??= { cpu: null, mem: null, state: null, health: null });
|
||||
if (m[2] === 'cpu') c.cpu = round(sum(chart), 2);
|
||||
else c.mem = Math.round(dim(chart, 'ram') * MiB);
|
||||
continue;
|
||||
}
|
||||
m = /^docker_local\.container_(.+)_(state|health_status)$/.exec(id);
|
||||
if (m) {
|
||||
const c = (containers[m[1]] ??= { cpu: null, mem: null, state: null, health: null });
|
||||
const v = hot(chart);
|
||||
if (m[2] === 'state') c.state = v;
|
||||
else c.health = v && v in HEALTH ? HEALTH[v] : v;
|
||||
}
|
||||
}
|
||||
// A stopped container keeps a stale cgroup chart for a while; don't show its last CPU.
|
||||
for (const c of Object.values(containers))
|
||||
if (c.state && c.state !== 'running') {
|
||||
c.cpu = null;
|
||||
c.mem = null;
|
||||
}
|
||||
return {
|
||||
t,
|
||||
cpu: cpu ? round(100 - dim(cpu, 'idle'), 1) : 0,
|
||||
iowait: round(dim(cpu, 'iowait'), 1),
|
||||
mem: { used: Math.round(dim(ram, 'used') * MiB), total: Math.round(sum(ram) * MiB) },
|
||||
net: {
|
||||
rx: Math.round(Math.abs(dim(all['system.net'], 'InOctets')) * 1000),
|
||||
tx: Math.round(Math.abs(dim(all['system.net'], 'OutOctets')) * 1000)
|
||||
},
|
||||
io: {
|
||||
read: Math.round(Math.abs(dim(all['system.io'], 'in')) * 1024),
|
||||
write: Math.round(Math.abs(dim(all['system.io'], 'out')) * 1024)
|
||||
},
|
||||
load: ['load1', 'load5', 'load15'].map((d) => round(dim(all['system.load'], d), 2)),
|
||||
containers
|
||||
};
|
||||
}
|
||||
|
||||
const round = (n: number, d: number) => Math.round(n * 10 ** d) / 10 ** d;
|
||||
|
||||
/** The last minute from netdata's own history, so sparklines are full on the first paint. */
|
||||
async function backfill(): Promise<LiveHistory> {
|
||||
const q = (chart: string) =>
|
||||
get(`/api/v1/data?chart=${chart}&after=-${HISTORY}&points=${HISTORY}&format=json`) as Promise<{
|
||||
labels: string[];
|
||||
data: number[][];
|
||||
}>;
|
||||
const [cpu, ram, net, io] = await Promise.all(
|
||||
['system.cpu', 'system.ram', 'system.net', 'system.io'].map(q)
|
||||
);
|
||||
const col = (r: { labels: string[] }, name: string) => r.labels.indexOf(name);
|
||||
const at = (r: { data: number[][] }, t: number) => r.data.find((row) => row[0] === t);
|
||||
const iIdle = col(cpu, 'idle');
|
||||
const iUsed = col(ram, 'used');
|
||||
const out: LiveHistory = [];
|
||||
for (const row of [...cpu.data].sort((a, b) => a[0] - b[0])) {
|
||||
const t = row[0];
|
||||
const r = at(ram, t);
|
||||
const n = at(net, t);
|
||||
const d = at(io, t);
|
||||
if (!r || !n || !d) continue;
|
||||
out.push({
|
||||
t: t * 1000,
|
||||
cpu: round(100 - row[iIdle], 1),
|
||||
mem: {
|
||||
used: Math.round(r[iUsed] * MiB),
|
||||
total: Math.round(r.slice(1).reduce((a, b) => a + b, 0) * MiB)
|
||||
},
|
||||
net: { rx: Math.round(Math.abs(n[1]) * 1000), tx: Math.round(Math.abs(n[2]) * 1000) },
|
||||
io: { read: Math.round(Math.abs(d[1]) * 1024), write: Math.round(Math.abs(d[2]) * 1024) }
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
type Listener = (event: 'frame' | 'fail', data: unknown) => void;
|
||||
|
||||
const listeners = new Set<Listener>();
|
||||
let history: LiveHistory = [];
|
||||
let timer: ReturnType<typeof setInterval> | null = null;
|
||||
let stopTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let inflight = false;
|
||||
let last: LiveFrame | null = null;
|
||||
|
||||
async function tick() {
|
||||
if (inflight) return; // never pile up requests if netdata is slow
|
||||
inflight = true;
|
||||
try {
|
||||
const all = (await get(
|
||||
`/api/v1/allmetrics?format=json&filter=${encodeURIComponent(FILTER)}`
|
||||
)) as Record<string, Chart>;
|
||||
const frame = parseAllmetrics(all);
|
||||
last = frame;
|
||||
history.push({ t: frame.t, cpu: frame.cpu, mem: frame.mem, net: frame.net, io: frame.io });
|
||||
if (history.length > HISTORY) history = history.slice(-HISTORY);
|
||||
for (const l of listeners) l('frame', frame);
|
||||
} catch (e) {
|
||||
for (const l of listeners) l('fail', { message: e instanceof Error ? e.message : 'failed' });
|
||||
} finally {
|
||||
inflight = false;
|
||||
}
|
||||
}
|
||||
|
||||
function start() {
|
||||
if (stopTimer) {
|
||||
clearTimeout(stopTimer);
|
||||
stopTimer = null;
|
||||
}
|
||||
if (timer) return;
|
||||
timer = setInterval(tick, 1000);
|
||||
void tick();
|
||||
}
|
||||
|
||||
function stop() {
|
||||
if (timer) clearInterval(timer);
|
||||
timer = null;
|
||||
history = [];
|
||||
last = null;
|
||||
}
|
||||
|
||||
/** Subscribe to frames; returns the current minute of history and an unsubscribe. */
|
||||
export async function subscribe(fn: Listener) {
|
||||
const fresh = !timer;
|
||||
listeners.add(fn);
|
||||
start();
|
||||
if (fresh || history.length < 5) {
|
||||
try {
|
||||
const h = await backfill();
|
||||
const lastT = h.at(-1)?.t ?? 0;
|
||||
if (history.length < h.length) history = h.concat(history.filter((x) => x.t > lastT));
|
||||
} catch {
|
||||
// sparklines just fill up live
|
||||
}
|
||||
}
|
||||
return {
|
||||
history: history.slice(-HISTORY),
|
||||
last,
|
||||
unsubscribe() {
|
||||
listeners.delete(fn);
|
||||
if (!listeners.size && !stopTimer)
|
||||
stopTimer = setTimeout(() => {
|
||||
stopTimer = null;
|
||||
if (!listeners.size) stop();
|
||||
}, GRACE_MS);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** For tests and diagnostics. */
|
||||
export const liveStatus = () => ({ viewers: listeners.size, running: !!timer });
|
||||
Reference in New Issue
Block a user