server: add watchdog fail-safes

scripts/daemon/lesavka-hw-watchdog.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import os
+import signal
+import time
+
+path = os.environ.get("LESAVKA_WATCHDOG_DEV", "/dev/watchdog")
+interval = float(os.environ.get("LESAVKA_WATCHDOG_INTERVAL", "5"))
+
+fd = os.open(path, os.O_WRONLY)
+
+running = True
+
+
+def handle(_sig, _frame):
+    global running
+    running = False
+
+
+signal.signal(signal.SIGTERM, handle)
+signal.signal(signal.SIGINT, handle)
+
+try:
+    while running:
+        os.write(fd, b"\0")
+        time.sleep(interval)
+finally:
+    try:
+        os.write(fd, b"V")
+    except Exception:
+        pass
+    os.close(fd)

scripts/daemon/lesavka-hw-watchdog.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Lesavka hardware watchdog
+ConditionPathExists=/dev/watchdog
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/lesavka-hw-watchdog.py
+Restart=always
+RestartSec=2
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

scripts/daemon/lesavka-watchdog.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Lesavka reboot watchdog
+ConditionPathExists=!/etc/lesavka/no-reboot
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/lesavka-watchdog.sh
+
+[Install]
+WantedBy=multi-user.target

scripts/daemon/lesavka-watchdog.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+HEARTBEAT=/etc/lesavka/watchdog.touch
+MAX_AGE_SEC=${LESAVKA_WATCHDOG_MAX_AGE:-840}
+
+if [[ -f /etc/lesavka/no-reboot ]]; then
+    exit 0
+fi
+
+now=$(date +%s)
+if [[ ! -f "$HEARTBEAT" ]]; then
+    logger -t lesavka-watchdog "no heartbeat file; rebooting"
+    systemctl --no-wall reboot
+    exit 0
+fi
+
+mtime=$(stat -c %Y "$HEARTBEAT" 2>/dev/null || echo 0)
+age=$((now - mtime))
+if (( age > MAX_AGE_SEC )); then
+    logger -t lesavka-watchdog "heartbeat stale (${age}s); rebooting"
+    systemctl --no-wall reboot
+fi

scripts/daemon/lesavka-watchdog.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Lesavka reboot watchdog timer
+
+[Timer]
+OnBootSec=15min
+OnUnitActiveSec=15min
+AccuracySec=30s
+Persistent=true
+
+[Install]
+WantedBy=timers.target

scripts/install/server.sh

@@ -157,6 +157,8 @@ sudo install -Dm755 "$SRC_DIR/server/target/release/lesavka-server" /usr/local/b
 sudo install -Dm755 "$SRC_DIR/server/target/release/lesavka-uvc"    /usr/local/bin/lesavka-uvc
 sudo install -Dm755 "$SRC_DIR/scripts/daemon/lesavka-core.sh"       /usr/local/bin/lesavka-core.sh
 sudo install -Dm755 "$SRC_DIR/scripts/daemon/lesavka-uvc.sh"        /usr/local/bin/lesavka-uvc.sh
+sudo install -Dm755 "$SRC_DIR/scripts/daemon/lesavka-watchdog.sh"   /usr/local/bin/lesavka-watchdog.sh
+sudo install -Dm755 "$SRC_DIR/scripts/daemon/lesavka-hw-watchdog.py" /usr/local/bin/lesavka-hw-watchdog.py
 
 echo "==> 6a. Systemd units - lesavka-core"
 cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-core.service >/dev/null
@@ -243,6 +245,57 @@ User=root
 WantedBy=multi-user.target
 UNIT
 
+echo "==> 6d. Systemd units - watchdogs"
+cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-watchdog.service >/dev/null
+[Unit]
+Description=Lesavka reboot watchdog
+ConditionPathExists=!/etc/lesavka/no-reboot
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/lesavka-watchdog.sh
+
+[Install]
+WantedBy=multi-user.target
+UNIT
+
+cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-watchdog.timer >/dev/null
+[Unit]
+Description=Lesavka reboot watchdog timer
+
+[Timer]
+OnBootSec=15min
+OnUnitActiveSec=15min
+AccuracySec=30s
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+UNIT
+
+cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-hw-watchdog.service >/dev/null
+[Unit]
+Description=Lesavka hardware watchdog
+ConditionPathExists=/dev/watchdog
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/lesavka-hw-watchdog.py
+Restart=always
+RestartSec=2
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target
+UNIT
+
+sudo install -d /etc/lesavka
+sudo touch /etc/lesavka/watchdog.touch
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now lesavka-hw-watchdog
+sudo systemctl enable --now lesavka-watchdog.timer
+
 if [[ -n ${LESAVKA_ALLOW_GADGET_RESET:-} ]] || ! is_attached_state "$UDC_STATE"; then
   sudo systemctl restart lesavka-uvc
   echo "✅  lesavka-uvc installed and restarted..."