bstein/lesavka
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

install: make recovery ladder server-only by default

Browse Source
This commit is contained in:
Brad Stein 2026-05-11 03:13:23 -03:00
parent 4bc5264513
commit 0f1fe89138
8 changed files with 83 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "lesavka_client"
version = "0.22.3"
version = "0.22.4"
dependencies = [
"anyhow",
"async-stream",
@ -1686,7 +1686,7 @@ dependencies = [
[[package]]
name = "lesavka_common"
version = "0.22.3"
version = "0.22.4"
dependencies = [
"anyhow",
"base64",
@ -1698,7 +1698,7 @@ dependencies = [
[[package]]
name = "lesavka_server"
version = "0.22.3"
version = "0.22.4"
dependencies = [
"anyhow",
"base64",

2
client/Cargo.toml
View File

@ -4,7 +4,7 @@ path = "src/main.rs"
[package]
name = "lesavka_client"
version = "0.22.3"
version = "0.22.4"
edition = "2024"
[dependencies]

2
common/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "lesavka_common"
version = "0.22.3"
version = "0.22.4"
edition = "2024"
build = "build.rs"

10
scripts/daemon/lesavka-recovery-ladder.sh
View File

@ -6,6 +6,7 @@ ACTION=${1:-recover}
LOG_PATH=${LESAVKA_RECOVERY_LOG:-/var/log/lesavka/recovery-ladder.log}
LAST_GOOD_DIR=${LESAVKA_RECOVERY_LAST_GOOD_DIR:-/var/lib/lesavka/recovery/last-good}
CHECK_TIMEOUT_SECONDS=${LESAVKA_RECOVERY_TIMEOUT_SECONDS:-60}
ALLOW_UVC_RESTART=${LESAVKA_RECOVERY_ALLOW_UVC_RESTART:-0}
ALLOW_CORE_RESTART=${LESAVKA_RECOVERY_ALLOW_CORE_RESTART:-0}
ALLOW_REBOOT=${LESAVKA_RECOVERY_ALLOW_REBOOT:-0}
SERVER_BIND_ADDR=${LESAVKA_SERVER_BIND_ADDR:-0.0.0.0:50051}
@ -116,6 +117,10 @@ restart_server_only() {
}
restart_uvc_and_server() {
if [[ $ALLOW_UVC_RESTART == 0 || $ALLOW_UVC_RESTART == false || $ALLOW_UVC_RESTART == no ]]; then
log "step 2: UVC helper restart disabled; preserving attached USB gadget"
return 1
fi
log "step 2: restarting UVC helper and server"
systemctl reset-failed lesavka-uvc.service lesavka-server.service >/dev/null 2>&1 || true
systemctl restart lesavka-uvc.service
@ -171,6 +176,11 @@ recover() {
log "step 3: restoring last-known-good entrypoints"
if restore_last_good; then
restart_server_only || true
if wait_for_health; then
snapshot_last_good
return 0
fi
restart_uvc_and_server || true
if wait_for_health; then
snapshot_last_good

77
scripts/install/server.sh
View File

@ -1046,6 +1046,45 @@ install_verified_executable() {
sudo chmod 0755 "$dest"
}
pause_recovery_ladder_timer() {
sudo systemctl stop lesavka-recovery-ladder.timer >/dev/null 2>&1 || true
}
install_recovery_ladder_units() {
cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.service >/dev/null
[Unit]
Description=lesavka soft recovery ladder
After=network.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/lesavka-recovery-ladder recover
Environment=LESAVKA_RECOVERY_TIMEOUT_SECONDS=60
Environment=LESAVKA_RECOVERY_ALLOW_UVC_RESTART=0
Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0
Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0
EnvironmentFile=-/etc/lesavka/server.env
EnvironmentFile=-/etc/lesavka/uvc.env
UNIT
cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.timer >/dev/null
[Unit]
Description=periodically check and softly recover lesavka services
[Timer]
OnBootSec=90s
OnUnitActiveSec=60s
AccuracySec=10s
Persistent=true
[Install]
WantedBy=timers.target
UNIT
sudo systemctl daemon-reload
sudo systemctl enable lesavka-recovery-ladder.timer
}
CAPTURE_DISCOVERY_RELAY_PRESENT=0
CAPTURE_DISCOVERY_RELAY_WAS_ACTIVE=0
CAPTURE_DISCOVERY_POWER_BORROWED=0
@ -1094,6 +1133,7 @@ while [[ $# -gt 0 ]]; do
done
echo "==> Using git ref: $REF"
mkdir -p "$TMPDIR"
pause_recovery_ladder_timer
if [[ -z $REPO_URL ]] && [[ -d $SCRIPT_REPO_ROOT/.git ]]; then
REPO_URL=$(git -C "$SCRIPT_REPO_ROOT" config --get remote.origin.url || true)
@ -1236,6 +1276,7 @@ install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-core.sh" /usr/local
install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-uvc.sh" /usr/local/bin/lesavka-uvc.sh "lesavka-uvc.sh"
install_verified_executable "$SRC_DIR/scripts/daemon/lesavka-recovery-ladder.sh" /usr/local/bin/lesavka-recovery-ladder "lesavka-recovery-ladder"
install_verified_executable "$SRC_DIR/scripts/manual/run_uac_output_sanity.sh" /usr/local/bin/lesavka-uac-sanity "lesavka-uac-sanity"
install_recovery_ladder_units
echo "==> 5b. Runtime environment defaults"
sudo install -d -m 0755 /etc/lesavka
@ -1358,8 +1399,10 @@ if [[ "$ATTACHED_UVC_CHANGE_DEFERRED" == "1" ]]; then
rm -f "$SERVER_ENV_TMP" "$UVC_ENV_TMP"
INSTALLED_VERSION=$(manifest_package_version "$SRC_DIR/server/Cargo.toml" 2>/dev/null || true)
INSTALLED_SHA=$(git -C "$SCRIPT_REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true)
sudo systemctl start lesavka-recovery-ladder.timer >/dev/null 2>&1 || true
echo "✅ lesavka-server binaries installed; live service restart deferred for attached-gadget safety."
echo "➡️ Installed binaries: lesavka-server ${INSTALLED_VERSION:-unknown}${INSTALLED_SHA:+ ($INSTALLED_SHA)}"
echo "➡️ Recovery ladder: installed, enabled, and started in server-only mode"
echo "➡️ Deferred UVC codec request: ${INSTALL_UVC_CODEC}"
exit 0
fi
@ -1372,8 +1415,10 @@ if [[ "$ATTACHED_UVC_RESTART_DEFERRED" == "1" ]]; then
echo " Rerun the installer with the same UVC settings to restart services after the env is stable, or use LESAVKA_ALLOW_GADGET_RESET=1 LESAVKA_FORCE_GADGET_REBUILD=1 during a maintenance window." >&2
INSTALLED_VERSION=$(manifest_package_version "$SRC_DIR/server/Cargo.toml" 2>/dev/null || true)
INSTALLED_SHA=$(git -C "$SCRIPT_REPO_ROOT" rev-parse --short HEAD 2>/dev/null || true)
sudo systemctl start lesavka-recovery-ladder.timer >/dev/null 2>&1 || true
echo "✅ lesavka-server binaries and runtime env installed; live service restart deferred for attached-gadget safety."
echo "➡️ Installed binaries: lesavka-server ${INSTALLED_VERSION:-unknown}${INSTALLED_SHA:+ ($INSTALLED_SHA)}"
echo "➡️ Recovery ladder: installed, enabled, and started in server-only mode"
echo "➡️ Deferred restart UVC codec: ${INSTALL_UVC_CODEC}"
exit 0
fi
@ -1532,37 +1577,7 @@ sudo rm -f /etc/systemd/system/lesavka-watchdog.timer \
/etc/lesavka/watchdog.touch
echo "==> 6e. Systemd units - recovery ladder"
cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.service >/dev/null
[Unit]
Description=lesavka soft recovery ladder
After=network.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/lesavka-recovery-ladder recover
Environment=LESAVKA_RECOVERY_TIMEOUT_SECONDS=60
Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0
Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0
EnvironmentFile=-/etc/lesavka/server.env
EnvironmentFile=-/etc/lesavka/uvc.env
UNIT
cat <<'UNIT' | sudo tee /etc/systemd/system/lesavka-recovery-ladder.timer >/dev/null
[Unit]
Description=periodically check and softly recover lesavka services
[Timer]
OnBootSec=90s
OnUnitActiveSec=60s
AccuracySec=10s
Persistent=true
[Install]
WantedBy=timers.target
UNIT
sudo systemctl daemon-reload
sudo systemctl enable lesavka-recovery-ladder.timer
echo "✅ recovery ladder unit files installed and timer enabled; activation waits for successful service verification."
if [[ "$UVC_ENV_CHANGED" == "1" ]] && systemctl is-active --quiet lesavka-uvc; then
sudo systemctl restart lesavka-uvc

2
server/Cargo.toml
View File

@ -10,7 +10,7 @@ bench = false
[package]
name = "lesavka_server"
version = "0.22.3"
version = "0.22.4"
edition = "2024"
autobins = false

2
tests/chaos/system/interrupted_install_safe_state_contract.rs
View File

@ -126,8 +126,10 @@ fn recovery_ladder_restores_before_rebooting_or_touching_the_core_gadget() {
"restore_last_good",
"restart_server_only",
"restart_uvc_and_server",
"LESAVKA_RECOVERY_ALLOW_UVC_RESTART:-0",
"LESAVKA_RECOVERY_ALLOW_CORE_RESTART:-0",
"LESAVKA_RECOVERY_ALLOW_REBOOT:-0",
"UVC helper restart disabled; preserving attached USB gadget",
"core restart disabled; preserving attached USB gadget",
"reboot disabled; leaving host online for operator inspection",
] {

19
tests/installer/scripts/install/server_install_script_contract.rs
View File

@ -213,6 +213,10 @@ fn server_install_pins_hdmi_camera_and_display_defaults() {
SERVER_INSTALL.contains("ATTACHED_UVC_CHANGE_DEFERRED=1"),
"attached UVC descriptor/runtime changes should be deferred instead of half-applied"
);
assert!(
SERVER_INSTALL.contains("pause_recovery_ladder_timer"),
"server installs should stop any stale recovery timer before a long build/install cycle"
);
assert!(
SERVER_INSTALL
.contains("Leaving /etc/lesavka/server.env and /etc/lesavka/uvc.env unchanged"),
@ -222,6 +226,11 @@ fn server_install_pins_hdmi_camera_and_display_defaults() {
SERVER_INSTALL.contains("binaries were installed but running services were left untouched"),
"deferred attached-gadget installs must not restart live services"
);
assert!(
SERVER_INSTALL
.contains("Recovery ladder: installed, enabled, and started in server-only mode"),
"deferred attached-gadget installs should leave a safe recovery ladder in place"
);
assert!(
SERVER_INSTALL.contains("LIVE_UVC_DESCRIPTOR_MISMATCH"),
"installer should catch partial previous runs where env already changed but live descriptors did not"
@ -438,6 +447,7 @@ fn server_install_provisions_non_rebooting_recovery_ladder() {
"lesavka-recovery-ladder.service",
"lesavka-recovery-ladder.timer",
"ExecStart=/usr/local/bin/lesavka-recovery-ladder recover",
"Environment=LESAVKA_RECOVERY_ALLOW_UVC_RESTART=0",
"Environment=LESAVKA_RECOVERY_ALLOW_CORE_RESTART=0",
"Environment=LESAVKA_RECOVERY_ALLOW_REBOOT=0",
"OnUnitActiveSec=60s",
@ -456,6 +466,15 @@ fn server_install_provisions_non_rebooting_recovery_ladder() {
.unwrap(),
"last-known-good snapshots should only be refreshed after the installed server is verified"
);
assert!(
SERVER_INSTALL
.find("install_recovery_ladder_units")
.unwrap()
< SERVER_INSTALL
.find("ATTACHED_UVC_CHANGE_DEFERRED=0")
.unwrap(),
"recovery ladder units should be installed before any attached-gadget deferral can exit"
);
}
#[test]