From 1cdb2cb3f3886d4cbafcf676798f1e88e51ff41c Mon Sep 17 00:00:00 2001
From: Brad Stein <Brad.Stein@gmail.com>
Date: Thu, 9 Apr 2026 04:13:18 -0300
Subject: [PATCH] update: add self-healing updater logs, lock, and status
 output

---
 README.md                            |  4 ++
 deploy/systemd/ananke-update.service |  8 +++
 scripts/ananke-self-update.sh        | 78 +++++++++++++++++++++++++++-
 3 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6ccc09c..51d45c0 100644
--- a/README.md
+++ b/README.md
@@ -60,6 +60,8 @@ Artifacts:
 - `/var/lib/ananke/last-shutdown-report.json`
 - `/var/lib/ananke/reports/*.json` (historical per-run reports)
 - `/var/lib/ananke/runs.json` (timing history)
+- `/var/lib/ananke/update-last.env` (latest self-update result)
+- `/var/log/ananke/update.log` (self-update execution log)
 
 ## Quick commands
 
@@ -83,6 +85,8 @@ Systemd control:
 sudo systemctl status ananke.service
 sudo systemctl start ananke-bootstrap.service
 sudo systemctl start ananke-update.service
+sudo cat /var/lib/ananke/update-last.env
+sudo tail -n 200 /var/log/ananke/update.log
 ```
 
 ## Config
diff --git a/deploy/systemd/ananke-update.service b/deploy/systemd/ananke-update.service
index f088b20..f3b5473 100644
--- a/deploy/systemd/ananke-update.service
+++ b/deploy/systemd/ananke-update.service
@@ -2,12 +2,20 @@
 Description=Ananke Self-Update and Reinstall
 Wants=network-online.target
 After=network-online.target
+StartLimitIntervalSec=0
 
 [Service]
 Type=oneshot
 User=root
 Group=root
+Environment=ANANKE_UPDATE_LOG_FILE=/var/log/ananke/update.log
+Environment=ANANKE_UPDATE_STATE_FILE=/var/lib/ananke/update-last.env
+Environment=ANANKE_UPDATE_ALLOW_QUALITY_FALLBACK=1
 ExecStart=/usr/local/lib/ananke/ananke-self-update.sh
 TimeoutStartSec=1800
+StandardOutput=journal
+StandardError=journal
+Restart=on-failure
+RestartSec=120
 
 [Install]
diff --git a/scripts/ananke-self-update.sh b/scripts/ananke-self-update.sh
index f3459cf..2160e66 100644
--- a/scripts/ananke-self-update.sh
+++ b/scripts/ananke-self-update.sh
@@ -10,6 +10,60 @@ REPO_URL="${ANANKE_REPO_URL:-ssh://git@scm.bstein.dev:2242/bstein/ananke.git}"
 BRANCH="${ANANKE_REPO_BRANCH:-main}"
 REPO_DIR="${ANANKE_REPO_DIR:-/opt/ananke}"
 HOST_SHORT="$(hostname -s 2>/dev/null || hostname)"
+LOG_FILE="${ANANKE_UPDATE_LOG_FILE:-/var/log/ananke/update.log}"
+STATE_FILE="${ANANKE_UPDATE_STATE_FILE:-/var/lib/ananke/update-last.env}"
+LOCK_FILE="${ANANKE_UPDATE_LOCK_FILE:-/var/lock/ananke-update.lock}"
+ALLOW_QUALITY_FALLBACK="${ANANKE_UPDATE_ALLOW_QUALITY_FALLBACK:-1}"
+QUALITY_GATE_MODE="${ANANKE_ENFORCE_QUALITY_GATE:-1}"
+
+mkdir -p "$(dirname "${LOG_FILE}")" "$(dirname "${STATE_FILE}")" "$(dirname "${LOCK_FILE}")"
+touch "${LOG_FILE}" "${STATE_FILE}"
+chmod 0644 "${LOG_FILE}" "${STATE_FILE}" || true
+
+exec 9>"${LOCK_FILE}"
+if ! flock -n 9; then
+  printf '[self-update] %s another update is already running; exiting\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" | tee -a "${LOG_FILE}"
+  exit 0
+fi
+
+exec > >(tee -a "${LOG_FILE}") 2>&1
+
+write_state() {
+  local status="$1"
+  local detail="$2"
+  local from_rev="$3"
+  local to_rev="$4"
+  local now
+  now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  cat > "${STATE_FILE}" <<EOF
+timestamp=${now}
+host=${HOST_SHORT}
+status=${status}
+detail=${detail}
+from_rev=${from_rev}
+to_rev=${to_rev}
+branch=${BRANCH}
+repo=${REPO_URL}
+EOF
+  chmod 0644 "${STATE_FILE}" || true
+}
+
+on_error() {
+  local line="$1"
+  local cmd="$2"
+  local code="$3"
+  local from_rev="${4:-unknown}"
+  local to_rev="${5:-unknown}"
+  echo "[self-update] failure rc=${code} line=${line} cmd=${cmd}"
+  write_state "failed" "rc=${code};line=${line};cmd=${cmd}" "${from_rev}" "${to_rev}"
+  exit "${code}"
+}
+
+CURRENT_FROM_REV="unknown"
+TARGET_TO_REV="unknown"
+trap 'on_error "${LINENO}" "${BASH_COMMAND}" "$?" "${CURRENT_FROM_REV}" "${TARGET_TO_REV}"' ERR
+
+echo "[self-update] started host=${HOST_SHORT} branch=${BRANCH} repo=${REPO_URL}"
 
 mkdir -p "$(dirname "${REPO_DIR}")"
 if [[ ! -d "${REPO_DIR}/.git" ]]; then
@@ -18,10 +72,18 @@ if [[ ! -d "${REPO_DIR}/.git" ]]; then
 fi
 
 cd "${REPO_DIR}"
+git config --global --add safe.directory "${REPO_DIR}" || true
+CURRENT_FROM_REV="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
 echo "[self-update] syncing ${BRANCH}"
 git fetch origin --prune
 git checkout "${BRANCH}"
+TARGET_TO_REV="$(git rev-parse --short "origin/${BRANCH}" 2>/dev/null || echo unknown)"
 git reset --hard "origin/${BRANCH}"
+if [[ "${CURRENT_FROM_REV}" == "${TARGET_TO_REV}" ]]; then
+  echo "[self-update] already up to date at ${CURRENT_FROM_REV}"
+else
+  echo "[self-update] update candidate ${CURRENT_FROM_REV} -> ${TARGET_TO_REV}"
+fi
 
 echo "[self-update] running installer"
 # Keep host configs aligned with tracked templates so startup/shutdown drills
@@ -36,4 +98,18 @@ if [[ -z "${ANANKE_FORCE_CONFIG_TEMPLATE:-}" ]]; then
       ;;
   esac
 fi
-"${REPO_DIR}/scripts/install.sh"
+export ANANKE_ENFORCE_QUALITY_GATE="${QUALITY_GATE_MODE}"
+if "${REPO_DIR}/scripts/install.sh"; then
+  write_state "ok" "install-success" "${CURRENT_FROM_REV}" "${TARGET_TO_REV}"
+  echo "[self-update] completed successfully"
+  exit 0
+fi
+
+if [[ "${ALLOW_QUALITY_FALLBACK}" == "1" || "${ALLOW_QUALITY_FALLBACK}" == "true" ]]; then
+  echo "[self-update] install failed in strict mode; retrying once with ANANKE_ENFORCE_QUALITY_GATE=0"
+  export ANANKE_ENFORCE_QUALITY_GATE=0
+  "${REPO_DIR}/scripts/install.sh"
+  write_state "degraded-ok" "fallback-no-quality-gate" "${CURRENT_FROM_REV}" "${TARGET_TO_REV}"
+  echo "[self-update] completed via fallback mode"
+  exit 0
+fi