install: make recovery ladder server-only by default

Cargo.lock

@@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
 [[package]]
 name = "lesavka_client"
-version = "0.22.3"
+version = "0.22.4"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -1686,7 +1686,7 @@ dependencies = [
 
 [[package]]
 name = "lesavka_common"
-version = "0.22.3"
+version = "0.22.4"
 dependencies = [
  "anyhow",
  "base64",
@@ -1698,7 +1698,7 @@ dependencies = [
 
 [[package]]
 name = "lesavka_server"
-version = "0.22.3"
+version = "0.22.4"
 dependencies = [
  "anyhow",
  "base64",

client/Cargo.toml

@@ -4,7 +4,7 @@ path                    = "src/main.rs"
 
 [package]
 name                    = "lesavka_client"
-version                 = "0.22.3"
+version                 = "0.22.4"
 edition                 = "2024"
 
 [dependencies]

common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name                    = "lesavka_common"
-version                 = "0.22.3"
+version                 = "0.22.4"
 edition                 = "2024"
 build                   = "build.rs"
 

scripts/daemon/lesavka-recovery-ladder.sh

@@ -6,6 +6,7 @@ ACTION=${1:-recover}
 LOG_PATH=${LESAVKA_RECOVERY_LOG:-/var/log/lesavka/recovery-ladder.log}
 LAST_GOOD_DIR=${LESAVKA_RECOVERY_LAST_GOOD_DIR:-/var/lib/lesavka/recovery/last-good}
 CHECK_TIMEOUT_SECONDS=${LESAVKA_RECOVERY_TIMEOUT_SECONDS:-60}
+ALLOW_UVC_RESTART=${LESAVKA_RECOVERY_ALLOW_UVC_RESTART:-0}
 ALLOW_CORE_RESTART=${LESAVKA_RECOVERY_ALLOW_CORE_RESTART:-0}
 ALLOW_REBOOT=${LESAVKA_RECOVERY_ALLOW_REBOOT:-0}
 SERVER_BIND_ADDR=${LESAVKA_SERVER_BIND_ADDR:-0.0.0.0:50051}
@@ -116,6 +117,10 @@ restart_server_only() {
 }
 
 restart_uvc_and_server() {
+  if [[ $ALLOW_UVC_RESTART == 0 || $ALLOW_UVC_RESTART == false || $ALLOW_UVC_RESTART == no ]]; then
+    log "step 2: UVC helper restart disabled; preserving attached USB gadget"
+    return 1
+  fi
   log "step 2: restarting UVC helper and server"
   systemctl reset-failed lesavka-uvc.service lesavka-server.service >/dev/null 2>&1 || true
   systemctl restart lesavka-uvc.service
@@ -171,6 +176,11 @@ recover() {
 
   log "step 3: restoring last-known-good entrypoints"
   if restore_last_good; then
+    restart_server_only || true
+    if wait_for_health; then
+      snapshot_last_good
+      return 0
+    fi
     restart_uvc_and_server || true
     if wait_for_health; then
       snapshot_last_good

scripts/install/server.sh

@@ -1046,6 +1046,45 @@ install_verified_executable() {
   sudo chmod 0755 "$dest"
 }
 
+pause_recovery_ladder_timer() {
+  sudo systemctl stop lesavka-recovery-ladder.timer >/dev/null 2>&1 || true
+}
+
+install_recovery_ladder_units() {
+  cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.service >/dev/null
+[Unit]
+Description=lesavka soft recovery ladder
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/lesavka-recovery-ladder recover
+Environment=LESAVKA_RECOVERY_TIMEOUT_SECONDS=60
+Environment=LESAVKA_RECOVERY_ALLOW_UVC_RESTART=0
+Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0
+Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0
+EnvironmentFile=-/etc/lesavka/server.env
+EnvironmentFile=-/etc/lesavka/uvc.env
+UNIT
+
+  cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.timer >/dev/null
+[Unit]
+Description=periodically check and softly recover lesavka services
+
+[Timer]
+OnBootSec=90s
+OnUnitActiveSec=60s
+AccuracySec=10s
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+UNIT
+
+  sudo systemctl daemon-reload
+  sudo systemctl enable lesavka-recovery-ladder.timer
+}
+
 CAPTURE_DISCOVERY_RELAY_PRESENT=0
 CAPTURE_DISCOVERY_RELAY_WAS_ACTIVE=0
 CAPTURE_DISCOVERY_POWER_BORROWED=0
@@ -1094,6 +1133,7 @@ while [[ $# -gt 0 ]]; do
 done
 echo "==> Using git ref: $REF"
 mkdir -p "$TMPDIR"
+pause_recovery_ladder_timer
 
 if [[ -z $REPO_URL ]] && [[ -d $SCRIPT_REPO_ROOT/.git ]]; then
   REPO_URL=$(git -C "$SCRIPT_REPO_ROOT" config --get remote.origin.url || true)
@@ -1236,6 +1276,7 @@ install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-core.sh" /usr/local
 install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-uvc.sh" /usr/local/bin/lesavka-uvc.sh "lesavka-uvc.sh"
 install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-recovery-ladder.sh" /usr/local/bin/lesavka-recovery-ladder "lesavka-recovery-ladder"
 install_verified_executable "$SRC_DIR/scripts/manual/run_uac_output_sanity.sh" /usr/local/bin/lesavka-uac-sanity "lesavka-uac-sanity"
+install_recovery_ladder_units
 
 echo "==> 5b. Runtime environment defaults"
 sudo install -d -m 0755 /etc/lesavka
@@ -1358,8 +1399,10 @@ if [[ "$ATTACHED_UVC_CHANGE_DEFERRED" == "1" ]]; then
   rm -f "$SERVER_ENV_TMP" "$UVC_ENV_TMP"
   INSTALLED_VERSION=$(manifest_package_version "$SRC_DIR/server/Cargo.toml" 2>/dev/null || true)
   INSTALLED_SHA=$(git -C "$SCRIPT_REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true)
+  sudo systemctl start lesavka-recovery-ladder.timer >/dev/null 2>&1 || true
   echo "✅  lesavka-server binaries installed; live service restart deferred for attached-gadget safety."
   echo "➡️  Installed binaries: lesavka-server ${INSTALLED_VERSION:-unknown}${INSTALLED_SHA:+ ($INSTALLED_SHA)}"
+  echo "➡️  Recovery ladder: installed, enabled, and started in server-only mode"
   echo "➡️  Deferred UVC codec request: ${INSTALL_UVC_CODEC}"
   exit 0
 fi
@@ -1372,8 +1415,10 @@ if [[ "$ATTACHED_UVC_RESTART_DEFERRED" == "1" ]]; then
   echo "    Rerun the installer with the same UVC settings to restart services after the env is stable, or use LESAVKA_ALLOW_GADGET_RESET=1 LESAVKA_FORCE_GADGET_REBUILD=1 during a maintenance window." >&2
   INSTALLED_VERSION=$(manifest_package_version "$SRC_DIR/server/Cargo.toml" 2>/dev/null || true)
   INSTALLED_SHA=$(git -C "$SCRIPT_REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true)
+  sudo systemctl start lesavka-recovery-ladder.timer >/dev/null 2>&1 || true
   echo "✅  lesavka-server binaries and runtime env installed; live service restart deferred for attached-gadget safety."
   echo "➡️  Installed binaries: lesavka-server ${INSTALLED_VERSION:-unknown}${INSTALLED_SHA:+ ($INSTALLED_SHA)}"
+  echo "➡️  Recovery ladder: installed, enabled, and started in server-only mode"
   echo "➡️  Deferred restart UVC codec: ${INSTALL_UVC_CODEC}"
   exit 0
 fi
@@ -1532,37 +1577,7 @@ sudo rm -f /etc/systemd/system/lesavka-watchdog.timer \
           /etc/lesavka/watchdog.touch
 
 echo "==> 6e. Systemd units - recovery ladder"
-cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.service >/dev/null
+echo "✅  recovery ladder unit files installed and timer enabled; activation waits for successful service verification."
-[Unit]
-Description=lesavka soft recovery ladder
-After=network.target
-
-[Service]
-Type=oneshot
-ExecStart=/usr/local/bin/lesavka-recovery-ladder recover
-Environment=LESAVKA_RECOVERY_TIMEOUT_SECONDS=60
-Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0
-Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0
-EnvironmentFile=-/etc/lesavka/server.env
-EnvironmentFile=-/etc/lesavka/uvc.env
-UNIT
-
-cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.timer >/dev/null
-[Unit]
-Description=periodically check and softly recover lesavka services
-
-[Timer]
-OnBootSec=90s
-OnUnitActiveSec=60s
-AccuracySec=10s
-Persistent=true
-
-[Install]
-WantedBy=timers.target
-UNIT
-
-sudo systemctl daemon-reload
-sudo systemctl enable lesavka-recovery-ladder.timer
 
 if [[ "$UVC_ENV_CHANGED" == "1" ]] && systemctl is-active --quiet lesavka-uvc; then
   sudo systemctl restart lesavka-uvc

server/Cargo.toml

@@ -10,7 +10,7 @@ bench                   = false
 
 [package]
 name                    = "lesavka_server"
-version                 = "0.22.3"
+version                 = "0.22.4"
 edition                 = "2024"
 autobins                = false
 

tests/chaos/system/interrupted_install_safe_state_contract.rs

@@ -126,8 +126,10 @@ fn recovery_ladder_restores_before_rebooting_or_touching_the_core_gadget() {
         "restore_last_good",
         "restart_server_only",
         "restart_uvc_and_server",
+        "LESAVKA_RECOVERY_ALLOW_UVC_RESTART:-0",
         "LESAVKA_RECOVERY_ALLOW_CORE_RESTART:-0",
         "LESAVKA_RECOVERY_ALLOW_REBOOT:-0",
+        "UVC helper restart disabled; preserving attached USB gadget",
         "core restart disabled; preserving attached USB gadget",
         "reboot disabled; leaving host online for operator inspection",
     ] {

tests/installer/scripts/install/server_install_script_contract.rs

@@ -213,6 +213,10 @@ fn server_install_pins_hdmi_camera_and_display_defaults() {
         SERVER_INSTALL.contains("ATTACHED_UVC_CHANGE_DEFERRED=1"),
         "attached UVC descriptor/runtime changes should be deferred instead of half-applied"
     );
+    assert!(
+        SERVER_INSTALL.contains("pause_recovery_ladder_timer"),
+        "server installs should stop any stale recovery timer before a long build/install cycle"
+    );
     assert!(
         SERVER_INSTALL
             .contains("Leaving /etc/lesavka/server.env and /etc/lesavka/uvc.env unchanged"),
@@ -222,6 +226,11 @@ fn server_install_pins_hdmi_camera_and_display_defaults() {
         SERVER_INSTALL.contains("binaries were installed but running services were left untouched"),
         "deferred attached-gadget installs must not restart live services"
     );
+    assert!(
+        SERVER_INSTALL
+            .contains("Recovery ladder: installed, enabled, and started in server-only mode"),
+        "deferred attached-gadget installs should leave a safe recovery ladder in place"
+    );
     assert!(
         SERVER_INSTALL.contains("LIVE_UVC_DESCRIPTOR_MISMATCH"),
         "installer should catch partial previous runs where env already changed but live descriptors did not"
@@ -438,6 +447,7 @@ fn server_install_provisions_non_rebooting_recovery_ladder() {
         "lesavka-recovery-ladder.service",
         "lesavka-recovery-ladder.timer",
         "ExecStart=/usr/local/bin/lesavka-recovery-ladder recover",
+        "Environment=LESAVKA_RECOVERY_ALLOW_UVC_RESTART=0",
         "Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0",
         "Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0",
         "OnUnitActiveSec=60s",
@@ -456,6 +466,15 @@ fn server_install_provisions_non_rebooting_recovery_ladder() {
                 .unwrap(),
         "last-known-good snapshots should only be refreshed after the installed server is verified"
     );
+    assert!(
+        SERVER_INSTALL
+            .find("install_recovery_ladder_units")
+            .unwrap()
+            < SERVER_INSTALL
+                .find("ATTACHED_UVC_CHANGE_DEFERRED=0")
+                .unwrap(),
+        "recovery ladder units should be installed before any attached-gadget deferral can exit"
+    );
 }
 
 #[test]