From 0f1fe8913853e12ee35b3658b750aa3f40d12e71 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Mon, 11 May 2026 03:13:23 -0300 Subject: [PATCH] install: make recovery ladder server-only by default --- Cargo.lock | 6 +- client/Cargo.toml | 2 +- common/Cargo.toml | 2 +- scripts/daemon/lesavka-recovery-ladder.sh | 10 +++ scripts/install/server.sh | 77 +++++++++++-------- server/Cargo.toml | 2 +- ...interrupted_install_safe_state_contract.rs | 2 + .../install/server_install_script_contract.rs | 19 +++++ 8 files changed, 83 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d193161..1f8d318 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.22.3" +version = "0.22.4" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.22.3" +version = "0.22.4" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.22.3" +version = "0.22.4" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 7414b9a..ab7e23d 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.22.3" +version = "0.22.4" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 5ec5f48..bfcd507 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.22.3" +version = "0.22.4" edition = "2024" build = "build.rs" diff --git a/scripts/daemon/lesavka-recovery-ladder.sh b/scripts/daemon/lesavka-recovery-ladder.sh index adf7f7e..adb0019 100755 --- a/scripts/daemon/lesavka-recovery-ladder.sh +++ b/scripts/daemon/lesavka-recovery-ladder.sh @@ -6,6 +6,7 @@ ACTION=${1:-recover} LOG_PATH=${LESAVKA_RECOVERY_LOG:-/var/log/lesavka/recovery-ladder.log} LAST_GOOD_DIR=${LESAVKA_RECOVERY_LAST_GOOD_DIR:-/var/lib/lesavka/recovery/last-good} CHECK_TIMEOUT_SECONDS=${LESAVKA_RECOVERY_TIMEOUT_SECONDS:-60} +ALLOW_UVC_RESTART=${LESAVKA_RECOVERY_ALLOW_UVC_RESTART:-0} ALLOW_CORE_RESTART=${LESAVKA_RECOVERY_ALLOW_CORE_RESTART:-0} ALLOW_REBOOT=${LESAVKA_RECOVERY_ALLOW_REBOOT:-0} SERVER_BIND_ADDR=${LESAVKA_SERVER_BIND_ADDR:-0.0.0.0:50051} @@ -116,6 +117,10 @@ restart_server_only() { } restart_uvc_and_server() { + if [[ $ALLOW_UVC_RESTART == 0 || $ALLOW_UVC_RESTART == false || $ALLOW_UVC_RESTART == no ]]; then + log "step 2: UVC helper restart disabled; preserving attached USB gadget" + return 1 + fi log "step 2: restarting UVC helper and server" systemctl reset-failed lesavka-uvc.service lesavka-server.service >/dev/null 2>&1 || true systemctl restart lesavka-uvc.service @@ -171,6 +176,11 @@ recover() { log "step 3: restoring last-known-good entrypoints" if restore_last_good; then + restart_server_only || true + if wait_for_health; then + snapshot_last_good + return 0 + fi restart_uvc_and_server || true if wait_for_health; then snapshot_last_good diff --git a/scripts/install/server.sh b/scripts/install/server.sh index 120f78a..bf9f5dd 100755 --- a/scripts/install/server.sh +++ b/scripts/install/server.sh @@ -1046,6 +1046,45 @@ install_verified_executable() { sudo chmod 0755 "$dest" } +pause_recovery_ladder_timer() { + sudo systemctl stop lesavka-recovery-ladder.timer >/dev/null 2>&1 || true +} + +install_recovery_ladder_units() { + cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.service >/dev/null +[Unit] +Description=lesavka soft recovery ladder +After=network.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/lesavka-recovery-ladder recover +Environment=LESAVKA_RECOVERY_TIMEOUT_SECONDS=60 +Environment=LESAVKA_RECOVERY_ALLOW_UVC_RESTART=0 +Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0 +Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0 +EnvironmentFile=-/etc/lesavka/server.env +EnvironmentFile=-/etc/lesavka/uvc.env +UNIT + + cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.timer >/dev/null +[Unit] +Description=periodically check and softly recover lesavka services + +[Timer] +OnBootSec=90s +OnUnitActiveSec=60s +AccuracySec=10s +Persistent=true + +[Install] +WantedBy=timers.target +UNIT + + sudo systemctl daemon-reload + sudo systemctl enable lesavka-recovery-ladder.timer +} + CAPTURE_DISCOVERY_RELAY_PRESENT=0 CAPTURE_DISCOVERY_RELAY_WAS_ACTIVE=0 CAPTURE_DISCOVERY_POWER_BORROWED=0 @@ -1094,6 +1133,7 @@ while [[ $# -gt 0 ]]; do done echo "==> Using git ref: $REF" mkdir -p "$TMPDIR" +pause_recovery_ladder_timer if [[ -z $REPO_URL ]] && [[ -d $SCRIPT_REPO_ROOT/.git ]]; then REPO_URL=$(git -C "$SCRIPT_REPO_ROOT" config --get remote.origin.url || true) @@ -1236,6 +1276,7 @@ install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-core.sh" /usr/local install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-uvc.sh" /usr/local/bin/lesavka-uvc.sh "lesavka-uvc.sh" install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-recovery-ladder.sh" /usr/local/bin/lesavka-recovery-ladder "lesavka-recovery-ladder" install_verified_executable "$SRC_DIR/scripts/manual/run_uac_output_sanity.sh" /usr/local/bin/lesavka-uac-sanity "lesavka-uac-sanity" +install_recovery_ladder_units echo "==> 5b. Runtime environment defaults" sudo install -d -m 0755 /etc/lesavka @@ -1358,8 +1399,10 @@ if [[ "$ATTACHED_UVC_CHANGE_DEFERRED" == "1" ]]; then rm -f "$SERVER_ENV_TMP" "$UVC_ENV_TMP" INSTALLED_VERSION=$(manifest_package_version "$SRC_DIR/server/Cargo.toml" 2>/dev/null || true) INSTALLED_SHA=$(git -C "$SCRIPT_REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true) + sudo systemctl start lesavka-recovery-ladder.timer >/dev/null 2>&1 || true echo "✅ lesavka-server binaries installed; live service restart deferred for attached-gadget safety." echo "➡️ Installed binaries: lesavka-server ${INSTALLED_VERSION:-unknown}${INSTALLED_SHA:+ ($INSTALLED_SHA)}" + echo "➡️ Recovery ladder: installed, enabled, and started in server-only mode" echo "➡️ Deferred UVC codec request: ${INSTALL_UVC_CODEC}" exit 0 fi @@ -1372,8 +1415,10 @@ if [[ "$ATTACHED_UVC_RESTART_DEFERRED" == "1" ]]; then echo " Rerun the installer with the same UVC settings to restart services after the env is stable, or use LESAVKA_ALLOW_GADGET_RESET=1 LESAVKA_FORCE_GADGET_REBUILD=1 during a maintenance window." >&2 INSTALLED_VERSION=$(manifest_package_version "$SRC_DIR/server/Cargo.toml" 2>/dev/null || true) INSTALLED_SHA=$(git -C "$SCRIPT_REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true) + sudo systemctl start lesavka-recovery-ladder.timer >/dev/null 2>&1 || true echo "✅ lesavka-server binaries and runtime env installed; live service restart deferred for attached-gadget safety." echo "➡️ Installed binaries: lesavka-server ${INSTALLED_VERSION:-unknown}${INSTALLED_SHA:+ ($INSTALLED_SHA)}" + echo "➡️ Recovery ladder: installed, enabled, and started in server-only mode" echo "➡️ Deferred restart UVC codec: ${INSTALL_UVC_CODEC}" exit 0 fi @@ -1532,37 +1577,7 @@ sudo rm -f /etc/systemd/system/lesavka-watchdog.timer \ /etc/lesavka/watchdog.touch echo "==> 6e. Systemd units - recovery ladder" -cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.service >/dev/null -[Unit] -Description=lesavka soft recovery ladder -After=network.target - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/lesavka-recovery-ladder recover -Environment=LESAVKA_RECOVERY_TIMEOUT_SECONDS=60 -Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0 -Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0 -EnvironmentFile=-/etc/lesavka/server.env -EnvironmentFile=-/etc/lesavka/uvc.env -UNIT - -cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.timer >/dev/null -[Unit] -Description=periodically check and softly recover lesavka services - -[Timer] -OnBootSec=90s -OnUnitActiveSec=60s -AccuracySec=10s -Persistent=true - -[Install] -WantedBy=timers.target -UNIT - -sudo systemctl daemon-reload -sudo systemctl enable lesavka-recovery-ladder.timer +echo "✅ recovery ladder unit files installed and timer enabled; activation waits for successful service verification." if [[ "$UVC_ENV_CHANGED" == "1" ]] && systemctl is-active --quiet lesavka-uvc; then sudo systemctl restart lesavka-uvc diff --git a/server/Cargo.toml b/server/Cargo.toml index 3c6ba3f..bd85bd5 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.22.3" +version = "0.22.4" edition = "2024" autobins = false diff --git a/tests/chaos/system/interrupted_install_safe_state_contract.rs b/tests/chaos/system/interrupted_install_safe_state_contract.rs index d2214a3..546e6de 100644 --- a/tests/chaos/system/interrupted_install_safe_state_contract.rs +++ b/tests/chaos/system/interrupted_install_safe_state_contract.rs @@ -126,8 +126,10 @@ fn recovery_ladder_restores_before_rebooting_or_touching_the_core_gadget() { "restore_last_good", "restart_server_only", "restart_uvc_and_server", + "LESAVKA_RECOVERY_ALLOW_UVC_RESTART:-0", "LESAVKA_RECOVERY_ALLOW_CORE_RESTART:-0", "LESAVKA_RECOVERY_ALLOW_REBOOT:-0", + "UVC helper restart disabled; preserving attached USB gadget", "core restart disabled; preserving attached USB gadget", "reboot disabled; leaving host online for operator inspection", ] { diff --git a/tests/installer/scripts/install/server_install_script_contract.rs b/tests/installer/scripts/install/server_install_script_contract.rs index d8ddb98..408ef9e 100644 --- a/tests/installer/scripts/install/server_install_script_contract.rs +++ b/tests/installer/scripts/install/server_install_script_contract.rs @@ -213,6 +213,10 @@ fn server_install_pins_hdmi_camera_and_display_defaults() { SERVER_INSTALL.contains("ATTACHED_UVC_CHANGE_DEFERRED=1"), "attached UVC descriptor/runtime changes should be deferred instead of half-applied" ); + assert!( + SERVER_INSTALL.contains("pause_recovery_ladder_timer"), + "server installs should stop any stale recovery timer before a long build/install cycle" + ); assert!( SERVER_INSTALL .contains("Leaving /etc/lesavka/server.env and /etc/lesavka/uvc.env unchanged"), @@ -222,6 +226,11 @@ fn server_install_pins_hdmi_camera_and_display_defaults() { SERVER_INSTALL.contains("binaries were installed but running services were left untouched"), "deferred attached-gadget installs must not restart live services" ); + assert!( + SERVER_INSTALL + .contains("Recovery ladder: installed, enabled, and started in server-only mode"), + "deferred attached-gadget installs should leave a safe recovery ladder in place" + ); assert!( SERVER_INSTALL.contains("LIVE_UVC_DESCRIPTOR_MISMATCH"), "installer should catch partial previous runs where env already changed but live descriptors did not" @@ -438,6 +447,7 @@ fn server_install_provisions_non_rebooting_recovery_ladder() { "lesavka-recovery-ladder.service", "lesavka-recovery-ladder.timer", "ExecStart=/usr/local/bin/lesavka-recovery-ladder recover", + "Environment=LESAVKA_RECOVERY_ALLOW_UVC_RESTART=0", "Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0", "Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0", "OnUnitActiveSec=60s", @@ -456,6 +466,15 @@ fn server_install_provisions_non_rebooting_recovery_ladder() { .unwrap(), "last-known-good snapshots should only be refreshed after the installed server is verified" ); + assert!( + SERVER_INSTALL + .find("install_recovery_ladder_units") + .unwrap() + < SERVER_INSTALL + .find("ATTACHED_UVC_CHANGE_DEFERRED=0") + .unwrap(), + "recovery ladder units should be installed before any attached-gadget deferral can exit" + ); } #[test]