titan-iac/scripts/ops/hermes_triage_demo.sh

#!/usr/bin/env bash
# Drive and narrate the Hermes automated-triage demos.
#
#   hermes_triage_demo.sh fixture   # autonomous loop: fail -> repair -> green
#   hermes_triage_demo.sh code      # proposal loop: fail -> Hermes patch -> PR
#   hermes_triage_demo.sh status    # current incident/alert state, no changes
#   hermes_triage_demo.sh preflight # confirm the lab is ready to demo
#   hermes_triage_demo.sh reset     # restore the demo to its pre-run state
#   hermes_triage_demo.sh monitor [code]  # stream the flow chart stages live
#
# FIRST RUN: copy hermes_triage_demo.env.example to hermes_triage_demo.env in
# this directory and fill it in. That file is git-ignored precisely so it can
# hold real tokens; this script sources it automatically, so nothing needs to
# be exported by hand.
#
# Needs kubectl access to the cluster as well. Nothing here mutates the cluster
# directly: the fixture demo only asks Jenkins to run a parameterized build,
# and the code demo only pushes a seeded defect to the demo repository.
set -euo pipefail

# Local, git-ignored credentials. Sourced before anything else so every value
# below can be overridden from it.
_DEMO_ENV="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_demo.env"
# shellcheck disable=SC1090
[ -r "$_DEMO_ENV" ] && . "$_DEMO_ENV"

JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}"
FIXTURE_JOB="hermes-triage-demo"
CODE_JOB="hermes-code-demo"
DEMO_NS="hermes-triage-demo"
CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}"

say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; }
note() { printf '        %s\n' "$*"; }

require_jenkins() {
  if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then
    echo "Missing Jenkins credentials." >&2
    echo "Create $_DEMO_ENV from hermes_triage_demo.env.example and fill it in." >&2
    exit 1
  fi
}

jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; }
jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; }

last_build_number() {
  jenkins_get "/job/$1/api/json?tree=lastBuild[number]" |
    python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])'
}

wait_for_build() { # job number -> prints result
  local job="$1" num="$2" tries="${3:-120}"
  for _ in $(seq 1 "$tries"); do
    sleep 10
    local body result building
    body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)"
    building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)"
    if [ "$building" = "False" ]; then
      result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')"
      printf '%s' "$result"
      return 0
    fi
  done
  printf 'TIMEOUT'
}

ariadne_ticks() { # tail the autotriage decisions in human-readable form
  kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null |
    grep 'hermes autotriage tick' |
    python3 -c '
import sys, json
for line in sys.stdin:
    try:
        d = json.loads(line)
    except ValueError:
        continue
    print("       ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
}

# `monitor` follows the fixture job; `monitor code` follows the code-proposal
# job, which takes the source-proposal branch of the chart instead.
cmd_monitor() {
  local which="${1:-fixture}"
  [ "$which" = "code" ] && export MONITOR_JOB="$CODE_JOB"
  exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_monitor.py"
}

# Deletes the demo repositories' issues, pull requests and repair branches so a
# rerun starts from nothing. DEMO_REPOS is the whole blast radius and is
# deliberately explicit: a real service's issues are genuine triage records,
# and clearing them to tidy a demo would destroy the evidence the system
# exists to produce.
DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}"

cmd_reset() {
  require_jenkins
  say "Reset — restoring the demo to its pre-run state"

  note "fixture -> healthy"
  if kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \
      --type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1; then
    note "  fixture: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
  else
    note "  fixture patch failed (is the demo namespace present?)"
  fi

  local gitea="${GITEA_URL:-https://scm.bstein.dev}"
  if [ -z "${GITEA_TOKEN:-}" ]; then
    note "GITEA_TOKEN unset; skipping repository cleanup"
  else
    for repo in $DEMO_REPOS; do
      note "clearing bstein/$repo (demo repository)"
      local items
      items="$(curl -s -H "Authorization: token $GITEA_TOKEN" \
        "$gitea/api/v1/repos/bstein/$repo/issues?state=all&limit=100" |
        python3 -c 'import json,sys
for i in json.load(sys.stdin):
    print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)"
      if [ -z "$items" ]; then
        note "  no issues or pull requests"
      else
        while read -r num kind; do
          [ -z "$num" ] && continue
          curl -s -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
            "$gitea/api/v1/repos/bstein/$repo/issues/$num"
          note "  deleted $kind #$num"
        done <<< "$items"
      fi

      local branches
      branches="$(curl -s -H "Authorization: token $GITEA_TOKEN" \
        "$gitea/api/v1/repos/bstein/$repo/branches" |
        python3 -c 'import json,sys,urllib.parse
for b in json.load(sys.stdin):
    if b["name"].startswith("hermes-repair/"):
        print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)"
      if [ -z "$branches" ]; then
        note "  no repair branches"
      else
        for ref in $branches; do
          curl -s -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
            "$gitea/api/v1/repos/bstein/$repo/branches/$ref"
          note "  deleted branch $(printf '%b' "${ref//%/\\x}")"
        done
      fi
    done
  fi

  if [ -d "$CODE_REPO_DIR/.git" ]; then
    note "restoring the demo repository working state"
    ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q --ff-only 2>/dev/null || true )
    if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
      note "  src/discount.py is correct; demo is armable"
    else
      note "  src/discount.py still carries the seeded defect — revert it before demoing"
    fi
  else
    note "demo repository not cloned at $CODE_REPO_DIR; skipping"
  fi

  say "Ready"
  note "real service repositories were not touched"
  note "run 'preflight' next, then arm the fixture build"
}

cmd_preflight() {
  require_jenkins
  say "Preflight"
  note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)"
  note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')"
  note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)"
  # Printed as a list rather than the raw comma-separated setting: this is the
  # outermost safety boundary, so it is worth being able to read at a glance.
  local allowlist count
  allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')"
  count=0
  for _job in $allowlist; do count=$((count + 1)); done
  note "jobs Ariadne may triage ($count):"
  for _job in $allowlist; do note "  - $_job"; done
  note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')"
  local queued
  queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')"
  note "jenkins queue depth: $queued (demo is fastest when this is 0)"
  # The Kubernetes cloud caps concurrent agent pods at containerCapStr. When
  # real CI saturates that cap the demo build sits in the queue reporting
  # "all nodes are offline" and the timings in the runbook do not apply.
  local agents cap
  cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null |
    grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)"
  agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)"
  note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)"
  local open_prs
  open_prs="$(curl -s -H "Authorization: token ${GITEA_TOKEN:-}" \
    "${GITEA_URL:-https://scm.bstein.dev}/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null |
    python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')"
  note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)"
}

cmd_status() {
  say "Incident state (last ticks)"
  ariadne_ticks 600 8
  say "Firing alerts"
  kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null |
    python3 -c '
import json,sys
alerts = json.load(sys.stdin).get("data", {}).get("alerts", [])
print("        none" if not alerts else "")
for a in alerts:
    print("       ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null ||
    note "(query vmalert directly if this fails)"
  say "Demo namespace"
  kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/        /'
}

cmd_fixture() {
  require_jenkins
  local start_num next_num
  start_num="$(last_build_number "$FIXTURE_JOB")"
  next_num=$((start_num + 1))
  say "Arming the demo failure (SEED_FAILURE=true) -> build #$next_num"
  note "HTTP $(jenkins_post "/job/$FIXTURE_JOB/buildWithParameters?SEED_FAILURE=true")"
  note "Only manual step. Everything after this is automatic."

  say "Waiting for the seeded build to fail"
  note "result: $(wait_for_build "$FIXTURE_JOB" "$next_num")"

  say "Ariadne detects, gathers evidence, asks Hermes, authorizes, repairs"
  note "the repair is a single in-process ConfigMap patch, so watch the fixture"
  for _ in $(seq 1 40); do
    sleep 10
    if [ "$(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null)" = "healthy" ]; then
      note "fixture patched back to healthy"
      break
    fi
  done
  ariadne_ticks 400 4

  say "Ariadne triggers one rebuild with seeding disabled"
  note "result: $(wait_for_build "$FIXTURE_JOB" $((next_num + 1)))"

  say "Resolution"
  sleep 45
  ariadne_ticks 200 3
  note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
}

cmd_code() {
  require_jenkins
  [ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; }
  local start_num next_num
  start_num="$(last_build_number "$CODE_JOB")"
  next_num=$((start_num + 1))

  say "Seeding a one-line defect in src/discount.py"
  ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q &&
    python3 - <<'PY'
import pathlib
p = pathlib.Path("src/discount.py")
s = p.read_text()
old, new = "percent / 100", "percent / 10"
if old not in s:
    raise SystemExit("defect already present or file changed; reset master first")
p.write_text(s.replace(old, new))
PY
    git commit -qam "refactor: simplify discount percentage math" && git push -q origin master )
  note "pushed: a plausible-looking change that breaks three regression tests"

  say "Running the test gate -> build #$next_num"
  note "HTTP $(jenkins_post "/job/$CODE_JOB/build")"
  note "result: $(wait_for_build "$CODE_JOB" "$next_num")"

  say "Ariadne collects evidence and asks Hermes for a minimal patch"
  note "Hermes returns an anchored patch as data; Ariadne validates path, size,"
  note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num"
  for _ in $(seq 1 40); do
    sleep 15
    ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break
  done
  ariadne_ticks 400 3

  say "Pull request awaiting human review (nothing merges automatically)"
  note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
}

case "${1:-}" in
  fixture) cmd_fixture ;;
  code) cmd_code ;;
  status) cmd_status ;;
  preflight) cmd_preflight ;;
  reset) cmd_reset ;;
  monitor) shift; cmd_monitor "$@" ;;
  *) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
esac