Stan Runge
2bd668bf5e
All checks were successful
Deploy production / deploy (push) Successful in 2m39s
- home schema: home.links (seeded, owner-editable) and home.snapshots (written by a host collector) - deploy/home-collector: stdlib Python + systemd user timer (60s) that snapshots /proc stats, docker containers, systemd timers, service HTTP checks and ntfy topics via a limited role - extras/newtab-extension: MV3 new-tab override, zipped to static/newtab-extension.zip - owner-only pages skip Google Analytics; /trackers/health#body anchor for weigh-ins
345 lines
12 KiB
Python
345 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Host-side collector for the /start dashboard on stanrunge.dev.
|
|
|
|
Runs every minute as a systemd user timer on the server (home-collector.timer) and upserts
|
|
two rows into home.snapshots with the limited `home_collector` role:
|
|
host CPU, RAM, swap, disk, load, uptime, Docker containers (state, health, restarts,
|
|
CPU/mem), systemd timer results (backups, disk-guard, scrapers) and service checks
|
|
ntfy the most recent messages from the owner's ntfy topics
|
|
|
|
The app container has no Docker socket and never gets one; this script uses the host's
|
|
passwordless `sudo docker`. ntfy topic names stay on the host: only a label is stored.
|
|
|
|
Config: ~/.config/home-collector.env (mode 600)
|
|
HOME_DB_URL=postgres://home_collector:...@127.0.0.1:5440/<db>
|
|
NTFY_EXTRA=label=https://ntfy.sh/topic,... (optional extra topics, e.g. backups)
|
|
Topics are also read from ~/.config/disk-guard.env (NTFY_URL) and
|
|
~/.config/house-scrape.env (HOUSE_NTFY_URL).
|
|
Stdlib only. `--print` prints the snapshot instead of writing it; HOME_DB_URL in the
|
|
environment overrides the env file (e.g. to fill the dev database).
|
|
"""
|
|
import concurrent.futures as cf
|
|
import json
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import socket
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
HOME = os.path.expanduser('~')
|
|
CFG = os.path.join(HOME, '.config')
|
|
TS = 'stan-vm.tail665403.ts.net'
|
|
|
|
# (name, link shown on the dashboard, url checked from the host or None = link only)
|
|
SERVICES = [
|
|
('Website', 'https://stanrunge.dev', 'http://127.0.0.1:3000/login'),
|
|
('Gitea', 'https://git.stanrunge.dev', 'http://127.0.0.1:3010/'),
|
|
('Immich', 'https://photos.stanrunge.dev', 'http://127.0.0.1:2283/'),
|
|
('Umami', 'https://analytics.stanrunge.dev', 'http://127.0.0.1:3110/'),
|
|
('pgAdmin', 'https://pgadmin.stanrunge.dev', 'http://127.0.0.1:5050/misc/ping'),
|
|
('Vash esports', 'https://esports.vash.software', 'http://127.0.0.1:3100/'),
|
|
('Vash staging', 'https://esports-staging.vash.software', 'http://127.0.0.1:3101/'),
|
|
('Dozzle logs', 'https://logs.vash.software', 'http://127.0.0.1:3111/'),
|
|
('Beeriokart', 'https://beeriokart.stanrunge.dev', 'http://127.0.0.1:3003/'),
|
|
('Stuhee', 'https://stuhee.stanrunge.dev', 'http://127.0.0.1:3004/'),
|
|
('Dev site', 'https://dev.stanrunge.dev', 'http://127.0.0.1:3002/'),
|
|
('Webmin', f'https://{TS}:10000', 'https://100.83.57.113:10000/'),
|
|
('Netdata', f'http://{TS}:19999', 'http://127.0.0.1:19999/api/v1/info'),
|
|
('Portainer', f'https://{TS}:9443', 'https://100.83.57.113:9443/'),
|
|
('ntfy', 'https://ntfy.sh/app', 'https://ntfy.sh/v1/health'),
|
|
]
|
|
|
|
# (label, unit, scope)
|
|
TIMERS = [
|
|
('Backup (restic → B2)', 'stan-backup', 'system'),
|
|
('Backup maintenance', 'stan-backup-maintain', 'system'),
|
|
('Postgres dump', 'postgres-backup', 'system'),
|
|
('Postgres dump prune', 'postgres-backup-prune', 'system'),
|
|
('Disk guard check', 'disk-guard-check', 'user'),
|
|
('Disk guard cleanup', 'disk-guard-cleanup', 'user'),
|
|
('House scrape', 'house-scrape', 'user'),
|
|
('B2 cost', 'b2-cost', 'user'),
|
|
('Groceries import', 'groceries-import', 'user'),
|
|
('Money sync', 'money-sync', 'user'),
|
|
('Energy (NED) hourly', 'ned-collector-hourly', 'user'),
|
|
('Energy (NED) daily', 'ned-collector-daily', 'user'),
|
|
]
|
|
|
|
|
|
def read_env(path):
|
|
out = {}
|
|
try:
|
|
with open(path) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith('#') or '=' not in line:
|
|
continue
|
|
k, v = line.split('=', 1)
|
|
out[k.strip()] = v.strip().strip('"').strip("'")
|
|
except OSError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def run(cmd, timeout=15):
|
|
try:
|
|
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
|
return r.stdout if r.returncode == 0 else ''
|
|
except Exception:
|
|
return ''
|
|
|
|
|
|
def cpu_times():
|
|
with open('/proc/stat') as f:
|
|
parts = [int(x) for x in f.readline().split()[1:]]
|
|
idle = parts[3] + (parts[4] if len(parts) > 4 else 0)
|
|
return idle, sum(parts[:8])
|
|
|
|
|
|
def meminfo():
|
|
m = {}
|
|
with open('/proc/meminfo') as f:
|
|
for line in f:
|
|
k, v = line.split(':', 1)
|
|
m[k] = int(v.split()[0]) * 1024
|
|
return m
|
|
|
|
|
|
def system_stats():
|
|
i1, t1 = cpu_times()
|
|
time.sleep(1)
|
|
i2, t2 = cpu_times()
|
|
cpu = 100.0 * (1 - (i2 - i1) / max(1, t2 - t1))
|
|
m = meminfo()
|
|
with open('/proc/uptime') as f:
|
|
uptime = float(f.read().split()[0])
|
|
load = list(os.getloadavg())
|
|
disks = []
|
|
for mount in ('/',):
|
|
u = shutil.disk_usage(mount)
|
|
disks.append({'mount': mount, 'total': u.total, 'used': u.used, 'free': u.free,
|
|
'pct': round(100 * u.used / u.total, 1)})
|
|
return {
|
|
'hostname': socket.gethostname(),
|
|
'cpus': os.cpu_count(),
|
|
'cpuPct': round(cpu, 1),
|
|
'load': [round(x, 2) for x in load],
|
|
'uptimeS': int(uptime),
|
|
'mem': {'total': m['MemTotal'], 'available': m['MemAvailable'],
|
|
'used': m['MemTotal'] - m['MemAvailable']},
|
|
'swap': {'total': m.get('SwapTotal', 0), 'used': m.get('SwapTotal', 0) - m.get('SwapFree', 0)},
|
|
'disks': disks,
|
|
}
|
|
|
|
|
|
def containers():
|
|
ids = run(['sudo', '-n', 'docker', 'ps', '-aq']).split()
|
|
if not ids:
|
|
return []
|
|
try:
|
|
info = json.loads(run(['sudo', '-n', 'docker', 'inspect', *ids]) or '[]')
|
|
except ValueError:
|
|
info = []
|
|
stats = {}
|
|
for line in run(['sudo', '-n', 'docker', 'stats', '--no-stream', '--format', '{{json .}}'], 20).splitlines():
|
|
try:
|
|
s = json.loads(line)
|
|
stats[s['Name']] = {'cpu': s.get('CPUPerc'), 'mem': s.get('MemUsage', '').split(' / ')[0]}
|
|
except (ValueError, KeyError):
|
|
pass
|
|
out = []
|
|
for c in info:
|
|
st = c.get('State', {})
|
|
name = c.get('Name', '').lstrip('/')
|
|
out.append({
|
|
'name': name,
|
|
'image': c.get('Config', {}).get('Image', '').split('@')[0][:80],
|
|
'state': st.get('Status'),
|
|
'health': (st.get('Health') or {}).get('Status'),
|
|
'restarts': c.get('RestartCount', 0),
|
|
'exitCode': st.get('ExitCode'),
|
|
'startedAt': st.get('StartedAt'),
|
|
'cpu': stats.get(name, {}).get('cpu'),
|
|
'mem': stats.get(name, {}).get('mem'),
|
|
})
|
|
out.sort(key=lambda c: c['name'])
|
|
return out
|
|
|
|
|
|
def unit_props(unit, scope, props):
|
|
cmd = ['systemctl'] + (['--user'] if scope == 'user' else []) + [
|
|
'show', unit, '-p', ','.join(props)]
|
|
d = {}
|
|
for line in run(cmd).splitlines():
|
|
if '=' in line:
|
|
k, v = line.split('=', 1)
|
|
d[k] = v
|
|
return d
|
|
|
|
|
|
def list_timers(scope):
|
|
cmd = ['systemctl'] + (['--user'] if scope == 'user' else []) + ['list-timers', '--all', '-o', 'json']
|
|
try:
|
|
return {t['unit']: t for t in json.loads(run(cmd) or '[]')}
|
|
except (ValueError, KeyError, TypeError):
|
|
return {}
|
|
|
|
|
|
def sec(usec):
|
|
try:
|
|
n = int(usec or 0)
|
|
return n // 1_000_000 if n > 0 else None
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def timers():
|
|
lists = {'user': list_timers('user'), 'system': list_timers('system')}
|
|
out = []
|
|
for label, unit, scope in TIMERS + [('Home collector', 'home-collector', 'user')]:
|
|
t = lists[scope].get(unit + '.timer')
|
|
if not t:
|
|
continue
|
|
s = unit_props(unit + '.service', scope, ['ActiveState', 'Result', 'ExecMainStatus'])
|
|
out.append({
|
|
'label': label, 'unit': unit, 'scope': scope,
|
|
'last': sec(t.get('last')),
|
|
'next': sec(t.get('next')),
|
|
'running': s.get('ActiveState') in ('activating', 'active', 'deactivating'),
|
|
'result': s.get('Result'),
|
|
'exitStatus': int(s.get('ExecMainStatus') or 0),
|
|
})
|
|
return out
|
|
|
|
|
|
_insecure = ssl.create_default_context()
|
|
_insecure.check_hostname = False
|
|
_insecure.verify_mode = ssl.CERT_NONE
|
|
|
|
|
|
def check(service):
|
|
name, link, url = service
|
|
res = {'name': name, 'url': link, 'ok': None, 'code': None, 'ms': None}
|
|
if not url:
|
|
return res
|
|
start = time.monotonic()
|
|
try:
|
|
req = urllib.request.Request(url, headers={'User-Agent': 'home-collector'})
|
|
with urllib.request.urlopen(req, timeout=5, context=_insecure) as r:
|
|
r.read(256)
|
|
res['code'] = r.status
|
|
except urllib.error.HTTPError as e:
|
|
res['code'] = e.code
|
|
except Exception:
|
|
res['code'] = 0
|
|
res['ms'] = round((time.monotonic() - start) * 1000)
|
|
res['ok'] = bool(res['code']) and res['code'] < 500
|
|
return res
|
|
|
|
|
|
def services():
|
|
with cf.ThreadPoolExecutor(8) as ex:
|
|
return list(ex.map(check, SERVICES))
|
|
|
|
|
|
def b2_cost():
|
|
d = os.path.join(HOME, 'backups-cost', 'daily')
|
|
try:
|
|
files = sorted(f for f in os.listdir(d) if f.endswith('.json'))
|
|
if files:
|
|
with open(os.path.join(d, files[-1])) as f:
|
|
return json.load(f)
|
|
except (OSError, ValueError):
|
|
pass
|
|
return None
|
|
|
|
|
|
def ntfy_topics(cfg):
|
|
topics = []
|
|
for label, path, key in (('disk-guard', 'disk-guard.env', 'NTFY_URL'),
|
|
('houses', 'house-scrape.env', 'HOUSE_NTFY_URL')):
|
|
url = read_env(os.path.join(CFG, path)).get(key)
|
|
if url:
|
|
topics.append((label, url))
|
|
for item in (cfg.get('NTFY_EXTRA') or '').split(','):
|
|
if '=' in item:
|
|
label, url = item.split('=', 1)
|
|
topics.append((label.strip(), url.strip()))
|
|
return topics
|
|
|
|
|
|
def ntfy(cfg):
|
|
msgs, seen, errors = [], set(), []
|
|
for label, url in ntfy_topics(cfg):
|
|
if url in seen:
|
|
continue
|
|
seen.add(url)
|
|
try:
|
|
req = urllib.request.Request(url.rstrip('/') + '/json?poll=1&since=72h',
|
|
headers={'User-Agent': 'home-collector'})
|
|
with urllib.request.urlopen(req, timeout=8) as r:
|
|
for line in r.read().decode('utf-8', 'replace').splitlines():
|
|
try:
|
|
m = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if m.get('event') != 'message':
|
|
continue
|
|
msgs.append({'id': m.get('id'), 'topic': label, 'time': m.get('time'),
|
|
'title': (m.get('title') or '')[:200],
|
|
'message': (m.get('message') or '')[:600],
|
|
'priority': m.get('priority', 3), 'tags': m.get('tags') or [],
|
|
'click': m.get('click')})
|
|
except Exception:
|
|
errors.append(label)
|
|
msgs.sort(key=lambda m: m['time'] or 0, reverse=True)
|
|
return {'messages': msgs[:40], 'topics': [t for t, _ in ntfy_topics(cfg)], 'errors': errors}
|
|
|
|
|
|
def write(db_url, rows):
|
|
tag = 'j' + secrets.token_hex(8)
|
|
sql = ['BEGIN;']
|
|
for key, data in rows.items():
|
|
payload = json.dumps(data, separators=(',', ':'))
|
|
sql.append(
|
|
f"INSERT INTO home.snapshots (key, data, collected_at) VALUES ('{key}', ${tag}${payload}${tag}$::jsonb, now()) "
|
|
"ON CONFLICT (key) DO UPDATE SET data = excluded.data, collected_at = excluded.collected_at;")
|
|
sql.append('COMMIT;')
|
|
r = subprocess.run(['psql', '-X', '-q', '-v', 'ON_ERROR_STOP=1', db_url], input='\n'.join(sql),
|
|
capture_output=True, text=True, timeout=20)
|
|
if r.returncode != 0:
|
|
# psql errors never contain the password; the URL is not echoed.
|
|
sys.stderr.write(r.stderr[-500:])
|
|
sys.exit(1)
|
|
|
|
|
|
def main():
|
|
cfg = read_env(os.path.join(CFG, 'home-collector.env'))
|
|
with cf.ThreadPoolExecutor(4) as ex:
|
|
f_sys = ex.submit(system_stats)
|
|
f_ct = ex.submit(containers)
|
|
f_svc = ex.submit(services)
|
|
f_ntfy = ex.submit(ntfy, cfg)
|
|
host = f_sys.result()
|
|
host['containers'] = f_ct.result()
|
|
host['services'] = f_svc.result()
|
|
host['timers'] = timers()
|
|
host['b2'] = b2_cost()
|
|
notes = f_ntfy.result()
|
|
rows = {'host': host, 'ntfy': notes}
|
|
if '--print' in sys.argv:
|
|
print(json.dumps(rows, indent=1))
|
|
return
|
|
url = os.environ.get('HOME_DB_URL') or cfg.get('HOME_DB_URL')
|
|
if not url:
|
|
sys.exit('HOME_DB_URL missing in ~/.config/home-collector.env')
|
|
write(url, rows)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|