151cd574a9
The code demo seeds its defect by pushing to master, and the fix only lands if someone merges the pull request - which, by design, nobody does during a demo. So master stays broken, and reset only printed "revert it before demoing" while leaving it that way. The second run of the day then aborted on "defect already present" before anything started. Reset now reverts it on master. The substitution is anchored so it cannot match an already-correct divisor, which keeps reset idempotent: running it on a healthy repository changes nothing rather than corrupting the file. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
317 lines
14 KiB
Bash
Executable File
317 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Drive and narrate the Hermes automated-triage demos.
|
|
#
|
|
# hermes_triage_demo.sh fixture # autonomous loop: fail -> repair -> green
|
|
# hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR
|
|
# hermes_triage_demo.sh status # current incident/alert state, no changes
|
|
# hermes_triage_demo.sh preflight # confirm the lab is ready to demo
|
|
# hermes_triage_demo.sh reset # restore the demo to its pre-run state
|
|
# hermes_triage_demo.sh monitor [code] # stream the flow chart stages live
|
|
#
|
|
# FIRST RUN: copy hermes_triage_demo.env.example to hermes_triage_demo.env in
|
|
# this directory and fill it in. That file is git-ignored precisely so it can
|
|
# hold real tokens; this script sources it automatically, so nothing needs to
|
|
# be exported by hand.
|
|
#
|
|
# Needs kubectl access to the cluster as well. Nothing here mutates the cluster
|
|
# directly: the fixture demo only asks Jenkins to run a parameterized build,
|
|
# and the code demo only pushes a seeded defect to the demo repository.
|
|
set -euo pipefail
|
|
|
|
# Local, git-ignored credentials. Sourced before anything else so every value
|
|
# below can be overridden from it.
|
|
_DEMO_ENV="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_demo.env"
|
|
# shellcheck disable=SC1090
|
|
[ -r "$_DEMO_ENV" ] && . "$_DEMO_ENV"
|
|
|
|
JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}"
|
|
FIXTURE_JOB="hermes-triage-demo"
|
|
CODE_JOB="hermes-code-demo"
|
|
DEMO_NS="hermes-triage-demo"
|
|
CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}"
|
|
|
|
say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; }
|
|
note() { printf ' %s\n' "$*"; }
|
|
|
|
require_jenkins() {
|
|
if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then
|
|
echo "Missing Jenkins credentials." >&2
|
|
echo "Create $_DEMO_ENV from hermes_triage_demo.env.example and fill it in." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; }
|
|
jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; }
|
|
|
|
last_build_number() {
|
|
jenkins_get "/job/$1/api/json?tree=lastBuild[number]" |
|
|
python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])'
|
|
}
|
|
|
|
wait_for_build() { # job number -> prints result
|
|
local job="$1" num="$2" tries="${3:-120}"
|
|
for _ in $(seq 1 "$tries"); do
|
|
sleep 10
|
|
local body result building
|
|
body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)"
|
|
building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)"
|
|
if [ "$building" = "False" ]; then
|
|
result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')"
|
|
printf '%s' "$result"
|
|
return 0
|
|
fi
|
|
done
|
|
printf 'TIMEOUT'
|
|
}
|
|
|
|
ariadne_ticks() { # tail the autotriage decisions in human-readable form
|
|
kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null |
|
|
grep 'hermes autotriage tick' |
|
|
python3 -c '
|
|
import sys, json
|
|
for line in sys.stdin:
|
|
try:
|
|
d = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
|
|
}
|
|
|
|
# `monitor` follows the fixture job; `monitor code` follows the code-proposal
|
|
# job, which takes the source-proposal branch of the chart instead.
|
|
cmd_monitor() {
|
|
local which="${1:-fixture}"
|
|
[ "$which" = "code" ] && export MONITOR_JOB="$CODE_JOB"
|
|
exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_monitor.py"
|
|
}
|
|
|
|
# Deletes the demo repositories' issues, pull requests and repair branches so a
|
|
# rerun starts from nothing. DEMO_REPOS is the whole blast radius and is
|
|
# deliberately explicit: a real service's issues are genuine triage records,
|
|
# and clearing them to tidy a demo would destroy the evidence the system
|
|
# exists to produce.
|
|
DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}"
|
|
|
|
cmd_reset() {
|
|
require_jenkins
|
|
say "Reset — restoring the demo to its pre-run state"
|
|
|
|
note "fixture -> healthy"
|
|
if kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \
|
|
--type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1; then
|
|
note " fixture: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
|
|
else
|
|
note " fixture patch failed (is the demo namespace present?)"
|
|
fi
|
|
|
|
local gitea="${GITEA_URL:-https://scm.bstein.dev}"
|
|
if [ -z "${GITEA_TOKEN:-}" ]; then
|
|
note "GITEA_TOKEN unset; skipping repository cleanup"
|
|
else
|
|
for repo in $DEMO_REPOS; do
|
|
note "clearing bstein/$repo (demo repository)"
|
|
local items
|
|
items="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \
|
|
"$gitea/api/v1/repos/bstein/$repo/issues?state=all&limit=100" |
|
|
python3 -c 'import json,sys
|
|
for i in json.load(sys.stdin):
|
|
print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)"
|
|
if [ -z "$items" ]; then
|
|
note " no issues or pull requests"
|
|
else
|
|
while read -r num kind; do
|
|
[ -z "$num" ] && continue
|
|
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
|
"$gitea/api/v1/repos/bstein/$repo/issues/$num"
|
|
note " deleted $kind #$num"
|
|
done <<< "$items"
|
|
fi
|
|
|
|
local branches
|
|
branches="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \
|
|
"$gitea/api/v1/repos/bstein/$repo/branches" |
|
|
python3 -c 'import json,sys,urllib.parse
|
|
for b in json.load(sys.stdin):
|
|
if b["name"].startswith("hermes-repair/"):
|
|
print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)"
|
|
if [ -z "$branches" ]; then
|
|
note " no repair branches"
|
|
else
|
|
for ref in $branches; do
|
|
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
|
"$gitea/api/v1/repos/bstein/$repo/branches/$ref"
|
|
note " deleted branch $(printf '%b' "${ref//%/\\x}")"
|
|
done
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# The code demo seeds its defect by pushing to master, and the fix only lands
|
|
# if someone merges the pull request. A demo that ended without a merge - the
|
|
# usual case, since the point is that nothing merges itself - leaves master
|
|
# broken, and the next run aborts on "defect already present". Reset has to
|
|
# undo the seed rather than just report it, or the second demo of the day
|
|
# fails before it starts.
|
|
if [ -d "$CODE_REPO_DIR/.git" ]; then
|
|
note "restoring the demo repository working state"
|
|
( cd "$CODE_REPO_DIR" && git checkout -q master && git fetch -q origin &&
|
|
git reset -q --hard origin/master ) || note " could not sync master"
|
|
if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
|
|
note " src/discount.py is correct; demo is armable"
|
|
else
|
|
note " src/discount.py carries the seeded defect; reverting it on master"
|
|
( cd "$CODE_REPO_DIR" &&
|
|
python3 - <<'PY'
|
|
import pathlib, re, sys
|
|
|
|
path = pathlib.Path("src/discount.py")
|
|
source = path.read_text()
|
|
# Matches the seeded `percent / 10` without also matching a correct
|
|
# `percent / 100`, so re-running reset on a healthy file changes nothing.
|
|
fixed = re.sub(r"percent / 10(?!\d)", "percent / 100", source)
|
|
if fixed == source:
|
|
sys.exit("unrecognised defect; fix src/discount.py by hand")
|
|
path.write_text(fixed)
|
|
PY
|
|
git commit -qam "revert: restore the discount divisor" && git push -q origin master &&
|
|
note " reverted and pushed; demo is armable" ) || note " revert failed — fix src/discount.py by hand"
|
|
fi
|
|
else
|
|
note "demo repository not cloned at $CODE_REPO_DIR; skipping"
|
|
fi
|
|
|
|
say "Ready"
|
|
note "real service repositories were not touched"
|
|
note "run 'preflight' next, then arm the fixture build"
|
|
}
|
|
|
|
cmd_preflight() {
|
|
require_jenkins
|
|
say "Preflight"
|
|
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)"
|
|
note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
|
note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)"
|
|
# Printed as a list rather than the raw comma-separated setting: this is the
|
|
# outermost safety boundary, so it is worth being able to read at a glance.
|
|
local allowlist count
|
|
allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')"
|
|
count=0
|
|
for _job in $allowlist; do count=$((count + 1)); done
|
|
note "jobs Ariadne may triage ($count):"
|
|
for _job in $allowlist; do note " - $_job"; done
|
|
note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')"
|
|
local queued
|
|
queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')"
|
|
note "jenkins queue depth: $queued (demo is fastest when this is 0)"
|
|
# The Kubernetes cloud caps concurrent agent pods at containerCapStr. When
|
|
# real CI saturates that cap the demo build sits in the queue reporting
|
|
# "all nodes are offline" and the timings in the runbook do not apply.
|
|
local agents cap
|
|
cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null |
|
|
grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)"
|
|
agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)"
|
|
note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)"
|
|
local open_prs
|
|
open_prs="$(curl -s --max-time 25 -H "Authorization: token ${GITEA_TOKEN:-}" \
|
|
"${GITEA_URL:-https://scm.bstein.dev}/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null |
|
|
python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')"
|
|
note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)"
|
|
}
|
|
|
|
cmd_status() {
|
|
say "Incident state (last ticks)"
|
|
ariadne_ticks 600 8
|
|
say "Firing alerts"
|
|
kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null |
|
|
python3 -c '
|
|
import json,sys
|
|
alerts = json.load(sys.stdin).get("data", {}).get("alerts", [])
|
|
print(" none" if not alerts else "")
|
|
for a in alerts:
|
|
print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null ||
|
|
note "(query vmalert directly if this fails)"
|
|
say "Demo namespace"
|
|
kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/ /'
|
|
}
|
|
|
|
cmd_fixture() {
|
|
require_jenkins
|
|
local start_num next_num
|
|
start_num="$(last_build_number "$FIXTURE_JOB")"
|
|
next_num=$((start_num + 1))
|
|
say "Arming the demo failure (SEED_FAILURE=true) -> build #$next_num"
|
|
note "HTTP $(jenkins_post "/job/$FIXTURE_JOB/buildWithParameters?SEED_FAILURE=true")"
|
|
note "Only manual step. Everything after this is automatic."
|
|
|
|
say "Waiting for the seeded build to fail"
|
|
note "result: $(wait_for_build "$FIXTURE_JOB" "$next_num")"
|
|
|
|
say "Ariadne detects, gathers evidence, asks Hermes, authorizes, repairs"
|
|
note "the repair is a single in-process ConfigMap patch, so watch the fixture"
|
|
for _ in $(seq 1 40); do
|
|
sleep 10
|
|
if [ "$(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null)" = "healthy" ]; then
|
|
note "fixture patched back to healthy"
|
|
break
|
|
fi
|
|
done
|
|
ariadne_ticks 400 4
|
|
|
|
say "Ariadne triggers one rebuild with seeding disabled"
|
|
note "result: $(wait_for_build "$FIXTURE_JOB" $((next_num + 1)))"
|
|
|
|
say "Resolution"
|
|
sleep 45
|
|
ariadne_ticks 200 3
|
|
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
|
|
}
|
|
|
|
cmd_code() {
|
|
require_jenkins
|
|
[ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; }
|
|
local start_num next_num
|
|
start_num="$(last_build_number "$CODE_JOB")"
|
|
next_num=$((start_num + 1))
|
|
|
|
say "Seeding a one-line defect in src/discount.py"
|
|
( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q &&
|
|
python3 - <<'PY'
|
|
import pathlib
|
|
p = pathlib.Path("src/discount.py")
|
|
s = p.read_text()
|
|
old, new = "percent / 100", "percent / 10"
|
|
if old not in s:
|
|
raise SystemExit("defect already present or file changed; reset master first")
|
|
p.write_text(s.replace(old, new))
|
|
PY
|
|
git commit -qam "refactor: simplify discount percentage math" && git push -q origin master )
|
|
note "pushed: a plausible-looking change that breaks three regression tests"
|
|
|
|
say "Running the test gate -> build #$next_num"
|
|
note "HTTP $(jenkins_post "/job/$CODE_JOB/build")"
|
|
note "result: $(wait_for_build "$CODE_JOB" "$next_num")"
|
|
|
|
say "Ariadne collects evidence and asks Hermes for a minimal patch"
|
|
note "Hermes returns an anchored patch as data; Ariadne validates path, size,"
|
|
note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num"
|
|
for _ in $(seq 1 40); do
|
|
sleep 15
|
|
ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break
|
|
done
|
|
ariadne_ticks 400 3
|
|
|
|
say "Pull request awaiting human review (nothing merges automatically)"
|
|
note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
fixture) cmd_fixture ;;
|
|
code) cmd_code ;;
|
|
status) cmd_status ;;
|
|
preflight) cmd_preflight ;;
|
|
reset) cmd_reset ;;
|
|
monitor) shift; cmd_monitor "$@" ;;
|
|
*) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
|
|
esac
|