#!/usr/bin/env bash # Drive and narrate the Hermes automated-triage demos. # # hermes_triage_demo.sh fixture # autonomous loop: fail -> repair -> green # hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR # hermes_triage_demo.sh status # current incident/alert state, no changes # hermes_triage_demo.sh preflight # confirm the lab is ready to demo # # Requires JENKINS_USER and JENKINS_TOKEN in the environment (a Jenkins API # token) plus kubectl access to the cluster. Nothing here mutates the cluster # directly: the fixture demo only asks Jenkins to run a parameterized build, # and the code demo only pushes a seeded defect to the demo repository. set -euo pipefail JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}" FIXTURE_JOB="hermes-triage-demo" CODE_JOB="hermes-code-demo" DEMO_NS="hermes-triage-demo" CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}" say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; } note() { printf ' %s\n' "$*"; } require_jenkins() { : "${JENKINS_USER:?set JENKINS_USER}" "${JENKINS_TOKEN:?set JENKINS_TOKEN}" } jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; } jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; } last_build_number() { jenkins_get "/job/$1/api/json?tree=lastBuild[number]" | python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])' } wait_for_build() { # job number -> prints result local job="$1" num="$2" tries="${3:-120}" for _ in $(seq 1 "$tries"); do sleep 10 local body result building body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)" building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)" if [ "$building" = "False" ]; then result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')" printf '%s' "$result" return 0 fi done printf 'TIMEOUT' } ariadne_ticks() { # tail the autotriage decisions in human-readable form kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null | grep 'hermes autotriage tick' | python3 -c ' import sys, json for line in sys.stdin: try: d = json.loads(line) except ValueError: continue print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}" } cmd_preflight() { require_jenkins say "Preflight" note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)" note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')" note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)" note "allowlisted jobs: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null)" note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')" local queued queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')" note "jenkins queue depth: $queued (demo is fastest when this is 0)" } cmd_status() { say "Incident state (last ticks)" ariadne_ticks 600 8 say "Firing alerts" kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null | python3 -c ' import json,sys alerts = json.load(sys.stdin).get("data", {}).get("alerts", []) print(" none" if not alerts else "") for a in alerts: print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null || note "(query vmalert directly if this fails)" say "Demo namespace" kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/ /' } cmd_fixture() { require_jenkins local start_num next_num start_num="$(last_build_number "$FIXTURE_JOB")" next_num=$((start_num + 1)) say "Arming the demo failure (SEED_FAILURE=true) -> build #$next_num" note "HTTP $(jenkins_post "/job/$FIXTURE_JOB/buildWithParameters?SEED_FAILURE=true")" note "Only manual step. Everything after this is automatic." say "Waiting for the seeded build to fail" note "result: $(wait_for_build "$FIXTURE_JOB" "$next_num")" say "Ariadne detects, gathers evidence, asks Hermes, authorizes, repairs" local repair="hermes-demo-repair-$next_num" for _ in $(seq 1 40); do sleep 10 if [ "$(kubectl -n "$DEMO_NS" get job "$repair" -o jsonpath='{.status.succeeded}' 2>/dev/null)" = "1" ]; then note "repair job $repair succeeded" break fi done ariadne_ticks 400 4 say "Ariadne triggers one rebuild with seeding disabled" note "result: $(wait_for_build "$FIXTURE_JOB" $((next_num + 1)))" say "Resolution" sleep 45 ariadne_ticks 200 3 note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')" } cmd_code() { require_jenkins [ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; } local start_num next_num start_num="$(last_build_number "$CODE_JOB")" next_num=$((start_num + 1)) say "Seeding a one-line defect in src/discount.py" ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q && python3 - <<'PY' import pathlib p = pathlib.Path("src/discount.py") s = p.read_text() old, new = "percent / 100", "percent / 10" if old not in s: raise SystemExit("defect already present or file changed; reset master first") p.write_text(s.replace(old, new)) PY git commit -qam "refactor: simplify discount percentage math" && git push -q origin master ) note "pushed: a plausible-looking change that breaks three regression tests" say "Running the test gate -> build #$next_num" note "HTTP $(jenkins_post "/job/$CODE_JOB/build")" note "result: $(wait_for_build "$CODE_JOB" "$next_num")" say "Ariadne collects evidence and asks Hermes for a minimal patch" note "Hermes returns an anchored patch as data; Ariadne validates path, size," note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num" for _ in $(seq 1 40); do sleep 15 ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break done ariadne_ticks 400 3 say "Pull request awaiting human review (nothing merges automatically)" note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls" } case "${1:-}" in fixture) cmd_fixture ;; code) cmd_code ;; status) cmd_status ;; preflight) cmd_preflight ;; *) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;; esac