4ffcae8b3c
Gitea wedged at 2073Mi against a 2Gi limit: its API stopped answering even on its own loopback, and the repeated SSH LoginGraceTime drops in its log were starvation symptoms rather than a separate fault. Raised to 3Gi. The reset command's Gitea calls had no --max-time, so a slow service became an indefinite hang with no output - the script appeared frozen after 'clearing bstein/hermes-code-demo'. They now fail after 25 seconds and say so. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
295 lines
12 KiB
Bash
Executable File
295 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Drive and narrate the Hermes automated-triage demos.
|
|
#
|
|
# hermes_triage_demo.sh fixture # autonomous loop: fail -> repair -> green
|
|
# hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR
|
|
# hermes_triage_demo.sh status # current incident/alert state, no changes
|
|
# hermes_triage_demo.sh preflight # confirm the lab is ready to demo
|
|
# hermes_triage_demo.sh reset # restore the demo to its pre-run state
|
|
# hermes_triage_demo.sh monitor [code] # stream the flow chart stages live
|
|
#
|
|
# FIRST RUN: copy hermes_triage_demo.env.example to hermes_triage_demo.env in
|
|
# this directory and fill it in. That file is git-ignored precisely so it can
|
|
# hold real tokens; this script sources it automatically, so nothing needs to
|
|
# be exported by hand.
|
|
#
|
|
# Needs kubectl access to the cluster as well. Nothing here mutates the cluster
|
|
# directly: the fixture demo only asks Jenkins to run a parameterized build,
|
|
# and the code demo only pushes a seeded defect to the demo repository.
|
|
set -euo pipefail
|
|
|
|
# Local, git-ignored credentials. Sourced before anything else so every value
|
|
# below can be overridden from it.
|
|
_DEMO_ENV="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_demo.env"
|
|
# shellcheck disable=SC1090
|
|
[ -r "$_DEMO_ENV" ] && . "$_DEMO_ENV"
|
|
|
|
JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}"
|
|
FIXTURE_JOB="hermes-triage-demo"
|
|
CODE_JOB="hermes-code-demo"
|
|
DEMO_NS="hermes-triage-demo"
|
|
CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}"
|
|
|
|
say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; }
|
|
note() { printf ' %s\n' "$*"; }
|
|
|
|
require_jenkins() {
|
|
if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then
|
|
echo "Missing Jenkins credentials." >&2
|
|
echo "Create $_DEMO_ENV from hermes_triage_demo.env.example and fill it in." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; }
|
|
jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; }
|
|
|
|
last_build_number() {
|
|
jenkins_get "/job/$1/api/json?tree=lastBuild[number]" |
|
|
python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])'
|
|
}
|
|
|
|
wait_for_build() { # job number -> prints result
|
|
local job="$1" num="$2" tries="${3:-120}"
|
|
for _ in $(seq 1 "$tries"); do
|
|
sleep 10
|
|
local body result building
|
|
body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)"
|
|
building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)"
|
|
if [ "$building" = "False" ]; then
|
|
result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')"
|
|
printf '%s' "$result"
|
|
return 0
|
|
fi
|
|
done
|
|
printf 'TIMEOUT'
|
|
}
|
|
|
|
ariadne_ticks() { # tail the autotriage decisions in human-readable form
|
|
kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null |
|
|
grep 'hermes autotriage tick' |
|
|
python3 -c '
|
|
import sys, json
|
|
for line in sys.stdin:
|
|
try:
|
|
d = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
|
|
}
|
|
|
|
# `monitor` follows the fixture job; `monitor code` follows the code-proposal
|
|
# job, which takes the source-proposal branch of the chart instead.
|
|
cmd_monitor() {
|
|
local which="${1:-fixture}"
|
|
[ "$which" = "code" ] && export MONITOR_JOB="$CODE_JOB"
|
|
exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_monitor.py"
|
|
}
|
|
|
|
# Deletes the demo repositories' issues, pull requests and repair branches so a
|
|
# rerun starts from nothing. DEMO_REPOS is the whole blast radius and is
|
|
# deliberately explicit: a real service's issues are genuine triage records,
|
|
# and clearing them to tidy a demo would destroy the evidence the system
|
|
# exists to produce.
|
|
DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}"
|
|
|
|
cmd_reset() {
|
|
require_jenkins
|
|
say "Reset — restoring the demo to its pre-run state"
|
|
|
|
note "fixture -> healthy"
|
|
if kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \
|
|
--type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1; then
|
|
note " fixture: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
|
|
else
|
|
note " fixture patch failed (is the demo namespace present?)"
|
|
fi
|
|
|
|
local gitea="${GITEA_URL:-https://scm.bstein.dev}"
|
|
if [ -z "${GITEA_TOKEN:-}" ]; then
|
|
note "GITEA_TOKEN unset; skipping repository cleanup"
|
|
else
|
|
for repo in $DEMO_REPOS; do
|
|
note "clearing bstein/$repo (demo repository)"
|
|
local items
|
|
items="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \
|
|
"$gitea/api/v1/repos/bstein/$repo/issues?state=all&limit=100" |
|
|
python3 -c 'import json,sys
|
|
for i in json.load(sys.stdin):
|
|
print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)"
|
|
if [ -z "$items" ]; then
|
|
note " no issues or pull requests"
|
|
else
|
|
while read -r num kind; do
|
|
[ -z "$num" ] && continue
|
|
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
|
"$gitea/api/v1/repos/bstein/$repo/issues/$num"
|
|
note " deleted $kind #$num"
|
|
done <<< "$items"
|
|
fi
|
|
|
|
local branches
|
|
branches="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \
|
|
"$gitea/api/v1/repos/bstein/$repo/branches" |
|
|
python3 -c 'import json,sys,urllib.parse
|
|
for b in json.load(sys.stdin):
|
|
if b["name"].startswith("hermes-repair/"):
|
|
print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)"
|
|
if [ -z "$branches" ]; then
|
|
note " no repair branches"
|
|
else
|
|
for ref in $branches; do
|
|
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
|
"$gitea/api/v1/repos/bstein/$repo/branches/$ref"
|
|
note " deleted branch $(printf '%b' "${ref//%/\\x}")"
|
|
done
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ -d "$CODE_REPO_DIR/.git" ]; then
|
|
note "restoring the demo repository working state"
|
|
( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q --ff-only 2>/dev/null || true )
|
|
if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
|
|
note " src/discount.py is correct; demo is armable"
|
|
else
|
|
note " src/discount.py still carries the seeded defect — revert it before demoing"
|
|
fi
|
|
else
|
|
note "demo repository not cloned at $CODE_REPO_DIR; skipping"
|
|
fi
|
|
|
|
say "Ready"
|
|
note "real service repositories were not touched"
|
|
note "run 'preflight' next, then arm the fixture build"
|
|
}
|
|
|
|
cmd_preflight() {
|
|
require_jenkins
|
|
say "Preflight"
|
|
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)"
|
|
note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
|
note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)"
|
|
# Printed as a list rather than the raw comma-separated setting: this is the
|
|
# outermost safety boundary, so it is worth being able to read at a glance.
|
|
local allowlist count
|
|
allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')"
|
|
count=0
|
|
for _job in $allowlist; do count=$((count + 1)); done
|
|
note "jobs Ariadne may triage ($count):"
|
|
for _job in $allowlist; do note " - $_job"; done
|
|
note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')"
|
|
local queued
|
|
queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')"
|
|
note "jenkins queue depth: $queued (demo is fastest when this is 0)"
|
|
# The Kubernetes cloud caps concurrent agent pods at containerCapStr. When
|
|
# real CI saturates that cap the demo build sits in the queue reporting
|
|
# "all nodes are offline" and the timings in the runbook do not apply.
|
|
local agents cap
|
|
cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null |
|
|
grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)"
|
|
agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)"
|
|
note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)"
|
|
local open_prs
|
|
open_prs="$(curl -s --max-time 25 -H "Authorization: token ${GITEA_TOKEN:-}" \
|
|
"${GITEA_URL:-https://scm.bstein.dev}/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null |
|
|
python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')"
|
|
note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)"
|
|
}
|
|
|
|
cmd_status() {
|
|
say "Incident state (last ticks)"
|
|
ariadne_ticks 600 8
|
|
say "Firing alerts"
|
|
kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null |
|
|
python3 -c '
|
|
import json,sys
|
|
alerts = json.load(sys.stdin).get("data", {}).get("alerts", [])
|
|
print(" none" if not alerts else "")
|
|
for a in alerts:
|
|
print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null ||
|
|
note "(query vmalert directly if this fails)"
|
|
say "Demo namespace"
|
|
kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/ /'
|
|
}
|
|
|
|
cmd_fixture() {
|
|
require_jenkins
|
|
local start_num next_num
|
|
start_num="$(last_build_number "$FIXTURE_JOB")"
|
|
next_num=$((start_num + 1))
|
|
say "Arming the demo failure (SEED_FAILURE=true) -> build #$next_num"
|
|
note "HTTP $(jenkins_post "/job/$FIXTURE_JOB/buildWithParameters?SEED_FAILURE=true")"
|
|
note "Only manual step. Everything after this is automatic."
|
|
|
|
say "Waiting for the seeded build to fail"
|
|
note "result: $(wait_for_build "$FIXTURE_JOB" "$next_num")"
|
|
|
|
say "Ariadne detects, gathers evidence, asks Hermes, authorizes, repairs"
|
|
note "the repair is a single in-process ConfigMap patch, so watch the fixture"
|
|
for _ in $(seq 1 40); do
|
|
sleep 10
|
|
if [ "$(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null)" = "healthy" ]; then
|
|
note "fixture patched back to healthy"
|
|
break
|
|
fi
|
|
done
|
|
ariadne_ticks 400 4
|
|
|
|
say "Ariadne triggers one rebuild with seeding disabled"
|
|
note "result: $(wait_for_build "$FIXTURE_JOB" $((next_num + 1)))"
|
|
|
|
say "Resolution"
|
|
sleep 45
|
|
ariadne_ticks 200 3
|
|
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
|
|
}
|
|
|
|
cmd_code() {
|
|
require_jenkins
|
|
[ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; }
|
|
local start_num next_num
|
|
start_num="$(last_build_number "$CODE_JOB")"
|
|
next_num=$((start_num + 1))
|
|
|
|
say "Seeding a one-line defect in src/discount.py"
|
|
( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q &&
|
|
python3 - <<'PY'
|
|
import pathlib
|
|
p = pathlib.Path("src/discount.py")
|
|
s = p.read_text()
|
|
old, new = "percent / 100", "percent / 10"
|
|
if old not in s:
|
|
raise SystemExit("defect already present or file changed; reset master first")
|
|
p.write_text(s.replace(old, new))
|
|
PY
|
|
git commit -qam "refactor: simplify discount percentage math" && git push -q origin master )
|
|
note "pushed: a plausible-looking change that breaks three regression tests"
|
|
|
|
say "Running the test gate -> build #$next_num"
|
|
note "HTTP $(jenkins_post "/job/$CODE_JOB/build")"
|
|
note "result: $(wait_for_build "$CODE_JOB" "$next_num")"
|
|
|
|
say "Ariadne collects evidence and asks Hermes for a minimal patch"
|
|
note "Hermes returns an anchored patch as data; Ariadne validates path, size,"
|
|
note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num"
|
|
for _ in $(seq 1 40); do
|
|
sleep 15
|
|
ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break
|
|
done
|
|
ariadne_ticks 400 3
|
|
|
|
say "Pull request awaiting human review (nothing merges automatically)"
|
|
note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
fixture) cmd_fixture ;;
|
|
code) cmd_code ;;
|
|
status) cmd_status ;;
|
|
preflight) cmd_preflight ;;
|
|
reset) cmd_reset ;;
|
|
monitor) shift; cmd_monitor "$@" ;;
|
|
*) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
|
|
esac
|