refactor(demo): split the two demos into two scripts
Some checks failed
Tests / Declarative: Post Actions failed: 2, passed: 142
Some checks failed
Tests / Declarative: Post Actions failed: 2, passed: 142
One script with a 'fixture' and a 'code' subcommand meant the wrong subcommand was always one word away, in front of an audience, with different blast radii behind each: the triage demo touches only a ConfigMap, the code demo pushes to a repository and deletes issues. Those should not share a command line. Each demo is now its own driver with the same five verbs - monitor, reset, preflight, run, status - so knowing one teaches the other. What they genuinely share (credentials, Jenkins access, the tick reader, the lab-wide preflight checks) moved to hermes_demo_lib.sh rather than being duplicated, because the reason to split was clarity at the command line, not two copies of the same helper drifting apart. Each reset now covers only its own demo. The triage reset no longer reaches into a Gitea repository it never writes to, and the code reset owns the repository cleanup entirely. The credentials file is now hermes_demo.env since both read it; the old hermes_triage_demo.env is still sourced as a fallback so a filled-in file keeps working, and both names stay git-ignored. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
jenkins
parent
5fd5e6d729
commit
35ae4d4bab
1
.gitignore
vendored
1
.gitignore
vendored
@ -23,4 +23,5 @@ crash.log
|
||||
terraform/atlas/generated/
|
||||
|
||||
# Local demo credentials (never commit)
|
||||
scripts/ops/hermes_demo.env
|
||||
scripts/ops/hermes_triage_demo.env
|
||||
|
||||
186
scripts/ops/hermes_code_demo.sh
Executable file
186
scripts/ops/hermes_code_demo.sh
Executable file
@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env bash
|
||||
# Drive and narrate the Hermes code-proposal demo.
|
||||
#
|
||||
# This is the proposal loop: a build fails on a real defect, Ariadne asks
|
||||
# Hermes for a minimal patch, validates it as data, pushes a branch and opens a
|
||||
# pull request. Nothing merges. The point of this half is the stop, not the fix.
|
||||
#
|
||||
# hermes_code_demo.sh monitor # follow the Test Automation Diagram live
|
||||
# hermes_code_demo.sh reset # restore the demo repository to pre-run state
|
||||
# hermes_code_demo.sh preflight # confirm the lab is ready to demo
|
||||
# hermes_code_demo.sh run # seed the defect and narrate the loop
|
||||
# hermes_code_demo.sh status # current incident/alert state, no changes
|
||||
#
|
||||
# The autonomous-repair demo is a separate script: hermes_triage_demo.sh.
|
||||
#
|
||||
# FIRST RUN: copy hermes_demo.env.example to hermes_demo.env in this directory
|
||||
# and fill it in. That file is git-ignored precisely so it can hold real
|
||||
# tokens; this script sources it automatically. You also need bstein/
|
||||
# hermes-code-demo cloned locally (default ~/Development/hermes-code-demo,
|
||||
# override with CODE_REPO_DIR).
|
||||
#
|
||||
# The only thing this mutates is the demo repository: it pushes a seeded defect
|
||||
# to master and reverts it on reset.
|
||||
set -euo pipefail
|
||||
|
||||
# shellcheck source=scripts/ops/hermes_demo_lib.sh
|
||||
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_demo_lib.sh"
|
||||
|
||||
# DEMO_REPOS is the whole blast radius and is deliberately explicit: a real
|
||||
# service's issues are genuine triage records, and clearing them to tidy a demo
|
||||
# would destroy the evidence the system exists to produce.
|
||||
DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}"
|
||||
|
||||
cmd_reset() {
|
||||
say "Reset — restoring the code demo to its pre-run state"
|
||||
|
||||
if [ -z "${GITEA_TOKEN:-}" ]; then
|
||||
note "GITEA_TOKEN unset; skipping repository cleanup"
|
||||
else
|
||||
for repo in $DEMO_REPOS; do
|
||||
note "clearing bstein/$repo (demo repository)"
|
||||
local items
|
||||
items="$(gitea_get "/api/v1/repos/bstein/$repo/issues?state=all&limit=100" |
|
||||
python3 -c 'import json,sys
|
||||
for i in json.load(sys.stdin):
|
||||
print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)"
|
||||
if [ -z "$items" ]; then
|
||||
note " no issues or pull requests"
|
||||
else
|
||||
while read -r num kind; do
|
||||
[ -z "$num" ] && continue
|
||||
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
||||
"$GITEA_URL/api/v1/repos/bstein/$repo/issues/$num"
|
||||
note " deleted $kind #$num"
|
||||
done <<< "$items"
|
||||
fi
|
||||
|
||||
local branches
|
||||
branches="$(gitea_get "/api/v1/repos/bstein/$repo/branches" |
|
||||
python3 -c 'import json,sys,urllib.parse
|
||||
for b in json.load(sys.stdin):
|
||||
if b["name"].startswith("hermes-repair/"):
|
||||
print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)"
|
||||
if [ -z "$branches" ]; then
|
||||
note " no repair branches"
|
||||
else
|
||||
for ref in $branches; do
|
||||
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
||||
"$GITEA_URL/api/v1/repos/bstein/$repo/branches/$ref"
|
||||
note " deleted branch $(printf '%b' "${ref//%/\\x}")"
|
||||
done
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# The demo seeds its defect by pushing to master, and the fix only lands if
|
||||
# someone merges the pull request - which, by design, nobody does during a
|
||||
# demo. So master stays broken, and the next run aborts on "defect already
|
||||
# present". Reset has to undo the seed rather than just report it, or the
|
||||
# second demo of the day fails before it starts.
|
||||
if [ -d "$CODE_REPO_DIR/.git" ]; then
|
||||
note "restoring the demo repository working state"
|
||||
( cd "$CODE_REPO_DIR" && git checkout -q master && git fetch -q origin &&
|
||||
git reset -q --hard origin/master ) || note " could not sync master"
|
||||
if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
|
||||
note " src/discount.py is correct; demo is armable"
|
||||
else
|
||||
note " src/discount.py carries the seeded defect; reverting it on master"
|
||||
( cd "$CODE_REPO_DIR" &&
|
||||
python3 - <<'PY'
|
||||
import pathlib, re, sys
|
||||
|
||||
path = pathlib.Path("src/discount.py")
|
||||
source = path.read_text()
|
||||
# Matches the seeded `percent / 10` without also matching a correct
|
||||
# `percent / 100`, so re-running reset on a healthy file changes nothing.
|
||||
fixed = re.sub(r"percent / 10(?!\d)", "percent / 100", source)
|
||||
if fixed == source:
|
||||
sys.exit("unrecognised defect; fix src/discount.py by hand")
|
||||
path.write_text(fixed)
|
||||
PY
|
||||
git commit -qam "revert: restore the discount divisor" && git push -q origin master &&
|
||||
note " reverted and pushed; demo is armable" ) || note " revert failed — fix src/discount.py by hand"
|
||||
fi
|
||||
else
|
||||
note "demo repository not cloned at $CODE_REPO_DIR; skipping"
|
||||
fi
|
||||
|
||||
say "Ready"
|
||||
note "real service repositories were not touched"
|
||||
note "run 'preflight' next, then 'run'"
|
||||
}
|
||||
|
||||
cmd_preflight() {
|
||||
require_jenkins
|
||||
say "Preflight — code demo"
|
||||
if [ -d "$CODE_REPO_DIR/.git" ]; then
|
||||
if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
|
||||
note "demo repository: src/discount.py is correct; armable"
|
||||
else
|
||||
note "demo repository: src/discount.py carries a defect — run 'reset' first"
|
||||
fi
|
||||
else
|
||||
note "demo repository: NOT CLONED at $CODE_REPO_DIR"
|
||||
fi
|
||||
local open_prs
|
||||
open_prs="$(gitea_get "/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null |
|
||||
python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')"
|
||||
note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)"
|
||||
note "code proposals enabled: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_CODE_ENABLED 2>/dev/null)"
|
||||
note "fix categories: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_FIX_CATEGORIES 2>/dev/null)"
|
||||
shared_preflight
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
shared_status
|
||||
say "Open proposals"
|
||||
note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
|
||||
}
|
||||
|
||||
cmd_run() {
|
||||
require_jenkins
|
||||
[ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; }
|
||||
local start_num next_num
|
||||
start_num="$(last_build_number "$CODE_JOB")"
|
||||
next_num=$((start_num + 1))
|
||||
|
||||
say "Seeding a one-line defect in src/discount.py"
|
||||
( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q &&
|
||||
python3 - <<'PY'
|
||||
import pathlib
|
||||
p = pathlib.Path("src/discount.py")
|
||||
s = p.read_text()
|
||||
old, new = "percent / 100", "percent / 10"
|
||||
if old not in s:
|
||||
raise SystemExit("defect already present or file changed; run reset first")
|
||||
p.write_text(s.replace(old, new))
|
||||
PY
|
||||
git commit -qam "refactor: simplify discount percentage math" && git push -q origin master )
|
||||
note "pushed: a plausible-looking change that breaks three regression tests"
|
||||
|
||||
say "Running the test gate -> build #$next_num"
|
||||
note "HTTP $(jenkins_post "/job/$CODE_JOB/build")"
|
||||
note "result: $(wait_for_build "$CODE_JOB" "$next_num")"
|
||||
|
||||
say "Ariadne collects evidence and asks Hermes for a minimal patch"
|
||||
note "Hermes returns an anchored patch as data; Ariadne validates path, size,"
|
||||
note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num"
|
||||
for _ in $(seq 1 40); do
|
||||
sleep 15
|
||||
ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break
|
||||
done
|
||||
ariadne_ticks 400 3
|
||||
|
||||
say "Pull request awaiting human review (nothing merges automatically)"
|
||||
note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
run|code) cmd_run ;;
|
||||
status) cmd_status ;;
|
||||
preflight) cmd_preflight ;;
|
||||
reset) cmd_reset ;;
|
||||
monitor) run_monitor "$CODE_JOB" ;;
|
||||
*) sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
|
||||
esac
|
||||
@ -1,12 +1,14 @@
|
||||
# Copy to hermes_triage_demo.env (same directory) and fill in.
|
||||
# That filename is git-ignored so it can hold real tokens.
|
||||
# Copy to hermes_demo.env (same directory) and fill in. That filename is
|
||||
# git-ignored so it can hold real tokens. Both demo drivers read it:
|
||||
# hermes_triage_demo.sh and hermes_code_demo.sh.
|
||||
|
||||
# Jenkins API token: https://ci.bstein.dev -> your user -> Configure -> API Token
|
||||
export JENKINS_USER="your-jenkins-user"
|
||||
export JENKINS_TOKEN="your-jenkins-api-token"
|
||||
|
||||
# Gitea token, used only to report open pull requests during preflight.
|
||||
# Preflight still works without it; that one line will read "?".
|
||||
# Gitea token. The code demo needs it to clear its own demo repository on
|
||||
# reset and to report open pull requests during preflight; the triage demo
|
||||
# touches no repository and works without it.
|
||||
export GITEA_TOKEN="your-gitea-token"
|
||||
|
||||
# Override only if you are not pointing at the usual lab.
|
||||
136
scripts/ops/hermes_demo_lib.sh
Normal file
136
scripts/ops/hermes_demo_lib.sh
Normal file
@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared plumbing for the two Hermes demo drivers.
|
||||
#
|
||||
# The triage demo and the code demo are separate scripts on purpose: they prove
|
||||
# different halves of the Test Automation Diagram, they reset different things,
|
||||
# and mixing them behind one command invited exactly the confusion of running
|
||||
# the wrong subcommand in front of an audience. What they genuinely share -
|
||||
# credentials, Jenkins access, the Ariadne tick reader - lives here, so a fix
|
||||
# to any of it applies to both instead of being made twice and drifting.
|
||||
#
|
||||
# Not executable on its own; both drivers source it.
|
||||
|
||||
# Local, git-ignored credentials. `hermes_demo.env` is the current name;
|
||||
# `hermes_triage_demo.env` is still read so an existing filled-in file keeps
|
||||
# working after the split.
|
||||
_DEMO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEMO_ENV=""
|
||||
for _candidate in "$_DEMO_DIR/hermes_demo.env" "$_DEMO_DIR/hermes_triage_demo.env"; do
|
||||
if [ -r "$_candidate" ]; then
|
||||
DEMO_ENV="$_candidate"
|
||||
# shellcheck disable=SC1090
|
||||
. "$_candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "$DEMO_ENV" ] || DEMO_ENV="$_DEMO_DIR/hermes_demo.env"
|
||||
|
||||
JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}"
|
||||
GITEA_URL="${GITEA_URL:-https://scm.bstein.dev}"
|
||||
FIXTURE_JOB="hermes-triage-demo"
|
||||
CODE_JOB="hermes-code-demo"
|
||||
DEMO_NS="hermes-triage-demo"
|
||||
CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}"
|
||||
|
||||
say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
note() { printf ' %s\n' "$*"; }
|
||||
|
||||
require_jenkins() {
|
||||
if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then
|
||||
echo "Missing Jenkins credentials." >&2
|
||||
echo "Create $DEMO_ENV from hermes_demo.env.example and fill it in." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Every remote call is time-bounded. A hung curl during a demo is worse than a
|
||||
# failed one: a failure says what to do next, a hang says nothing at all.
|
||||
jenkins_get() { curl -sk --max-time 25 -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; }
|
||||
jenkins_post() {
|
||||
curl -sk --max-time 25 -o /dev/null -w '%{http_code}' \
|
||||
-u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"
|
||||
}
|
||||
gitea_get() { curl -s --max-time 25 -H "Authorization: token ${GITEA_TOKEN:-}" "$GITEA_URL$1"; }
|
||||
|
||||
last_build_number() {
|
||||
jenkins_get "/job/$1/api/json?tree=lastBuild[number]" |
|
||||
python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])'
|
||||
}
|
||||
|
||||
wait_for_build() { # job number [tries] -> prints result
|
||||
local job="$1" num="$2" tries="${3:-120}"
|
||||
for _ in $(seq 1 "$tries"); do
|
||||
sleep 10
|
||||
local body result building
|
||||
body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)"
|
||||
building="$(printf '%s' "$body" |
|
||||
python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)"
|
||||
if [ "$building" = "False" ]; then
|
||||
result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')"
|
||||
printf '%s' "$result"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
printf 'TIMEOUT'
|
||||
}
|
||||
|
||||
ariadne_ticks() { # tail the autotriage decisions in human-readable form
|
||||
kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null |
|
||||
grep 'hermes autotriage tick' |
|
||||
python3 -c '
|
||||
import sys, json
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
|
||||
}
|
||||
|
||||
# Both drivers narrate the same Test Automation Diagram; the monitor selects
|
||||
# which branch of it to follow from MONITOR_JOB.
|
||||
run_monitor() { # job
|
||||
export MONITOR_JOB="$1"
|
||||
exec python3 "$_DEMO_DIR/hermes_triage_monitor.py"
|
||||
}
|
||||
|
||||
# The checks that are true of the lab regardless of which demo is running.
|
||||
shared_preflight() {
|
||||
note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)"
|
||||
# Printed as a list rather than the raw comma-separated setting: this is the
|
||||
# outermost safety boundary, so it is worth being able to read at a glance.
|
||||
local allowlist count
|
||||
allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')"
|
||||
count=0
|
||||
for _job in $allowlist; do count=$((count + 1)); done
|
||||
note "jobs Ariadne may triage ($count):"
|
||||
for _job in $allowlist; do note " - $_job"; done
|
||||
note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')"
|
||||
local queued
|
||||
queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')"
|
||||
note "jenkins queue depth: $queued (demo is fastest when this is 0)"
|
||||
# The Kubernetes cloud caps concurrent agent pods at containerCapStr. When
|
||||
# real CI saturates that cap the demo build sits in the queue reporting
|
||||
# "all nodes are offline" and the timings in the runbook do not apply.
|
||||
local agents cap
|
||||
cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null |
|
||||
grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)"
|
||||
agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)"
|
||||
note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)"
|
||||
}
|
||||
|
||||
# The alert and incident state both demos are judged by.
|
||||
shared_status() {
|
||||
say "Incident state (last ticks)"
|
||||
ariadne_ticks 600 8
|
||||
say "Firing alerts"
|
||||
kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null |
|
||||
python3 -c '
|
||||
import json,sys
|
||||
alerts = json.load(sys.stdin).get("data", {}).get("alerts", [])
|
||||
print(" none" if not alerts else "")
|
||||
for a in alerts:
|
||||
print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null ||
|
||||
note "(query vmalert directly if this fails)"
|
||||
}
|
||||
@ -1,103 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# Drive and narrate the Hermes automated-triage demos.
|
||||
# Drive and narrate the Hermes automated-triage demo.
|
||||
#
|
||||
# hermes_triage_demo.sh fixture # autonomous loop: fail -> repair -> green
|
||||
# hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR
|
||||
# hermes_triage_demo.sh status # current incident/alert state, no changes
|
||||
# hermes_triage_demo.sh preflight # confirm the lab is ready to demo
|
||||
# hermes_triage_demo.sh reset # restore the demo to its pre-run state
|
||||
# hermes_triage_demo.sh monitor [code] # follow the Test Automation Diagram live
|
||||
# This is the autonomous loop: a build fails, Ariadne diagnoses it through
|
||||
# Hermes, authorizes a predefined repair, performs it, and rebuilds green
|
||||
# without a human touching anything.
|
||||
#
|
||||
# FIRST RUN: copy hermes_triage_demo.env.example to hermes_triage_demo.env in
|
||||
# this directory and fill it in. That file is git-ignored precisely so it can
|
||||
# hold real tokens; this script sources it automatically, so nothing needs to
|
||||
# be exported by hand.
|
||||
# hermes_triage_demo.sh monitor # follow the Test Automation Diagram live
|
||||
# hermes_triage_demo.sh reset # restore the demo to its pre-run state
|
||||
# hermes_triage_demo.sh preflight # confirm the lab is ready to demo
|
||||
# hermes_triage_demo.sh run # arm the failure and narrate the loop
|
||||
# hermes_triage_demo.sh status # current incident/alert state, no changes
|
||||
#
|
||||
# The code-proposal demo is a separate script: hermes_code_demo.sh. They prove
|
||||
# different halves of the diagram and reset different things, so they are kept
|
||||
# apart rather than behind one command.
|
||||
#
|
||||
# FIRST RUN: copy hermes_demo.env.example to hermes_demo.env in this directory
|
||||
# and fill it in. That file is git-ignored precisely so it can hold real
|
||||
# tokens; this script sources it automatically.
|
||||
#
|
||||
# Needs kubectl access to the cluster as well. Nothing here mutates the cluster
|
||||
# directly: the fixture demo only asks Jenkins to run a parameterized build,
|
||||
# and the code demo only pushes a seeded defect to the demo repository.
|
||||
# directly: the demo only asks Jenkins to run a parameterized build.
|
||||
set -euo pipefail
|
||||
|
||||
# Local, git-ignored credentials. Sourced before anything else so every value
|
||||
# below can be overridden from it.
|
||||
_DEMO_ENV="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_demo.env"
|
||||
# shellcheck disable=SC1090
|
||||
[ -r "$_DEMO_ENV" ] && . "$_DEMO_ENV"
|
||||
|
||||
JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}"
|
||||
FIXTURE_JOB="hermes-triage-demo"
|
||||
CODE_JOB="hermes-code-demo"
|
||||
DEMO_NS="hermes-triage-demo"
|
||||
CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}"
|
||||
|
||||
say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
note() { printf ' %s\n' "$*"; }
|
||||
|
||||
require_jenkins() {
|
||||
if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then
|
||||
echo "Missing Jenkins credentials." >&2
|
||||
echo "Create $_DEMO_ENV from hermes_triage_demo.env.example and fill it in." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; }
|
||||
jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; }
|
||||
|
||||
last_build_number() {
|
||||
jenkins_get "/job/$1/api/json?tree=lastBuild[number]" |
|
||||
python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])'
|
||||
}
|
||||
|
||||
wait_for_build() { # job number -> prints result
|
||||
local job="$1" num="$2" tries="${3:-120}"
|
||||
for _ in $(seq 1 "$tries"); do
|
||||
sleep 10
|
||||
local body result building
|
||||
body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)"
|
||||
building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)"
|
||||
if [ "$building" = "False" ]; then
|
||||
result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')"
|
||||
printf '%s' "$result"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
printf 'TIMEOUT'
|
||||
}
|
||||
|
||||
ariadne_ticks() { # tail the autotriage decisions in human-readable form
|
||||
kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null |
|
||||
grep 'hermes autotriage tick' |
|
||||
python3 -c '
|
||||
import sys, json
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
|
||||
}
|
||||
|
||||
# `monitor` follows the fixture job; `monitor code` follows the code-proposal
|
||||
# job, which takes the source-proposal branch of the Test Automation Diagram
|
||||
# instead. Both narrate the same diagram, stage for stage.
|
||||
cmd_monitor() {
|
||||
local which="${1:-fixture}"
|
||||
[ "$which" = "code" ] && export MONITOR_JOB="$CODE_JOB"
|
||||
exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_monitor.py"
|
||||
}
|
||||
|
||||
# Deletes the demo repositories' issues, pull requests and repair branches so a
|
||||
# rerun starts from nothing. DEMO_REPOS is the whole blast radius and is
|
||||
# deliberately explicit: a real service's issues are genuine triage records,
|
||||
# and clearing them to tidy a demo would destroy the evidence the system
|
||||
# exists to produce.
|
||||
DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}"
|
||||
# shellcheck source=scripts/ops/hermes_demo_lib.sh
|
||||
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_demo_lib.sh"
|
||||
|
||||
# The fixture is a ConfigMap the demo job reads. Resetting it is the whole
|
||||
# blast radius of this script: no repository is touched, because the triage
|
||||
# loop repairs infrastructure rather than source.
|
||||
cmd_reset() {
|
||||
require_jenkins
|
||||
say "Reset — restoring the demo to its pre-run state"
|
||||
|
||||
say "Reset — restoring the triage demo to its pre-run state"
|
||||
note "fixture -> healthy"
|
||||
if kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \
|
||||
--type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1; then
|
||||
@ -105,138 +38,25 @@ cmd_reset() {
|
||||
else
|
||||
note " fixture patch failed (is the demo namespace present?)"
|
||||
fi
|
||||
|
||||
local gitea="${GITEA_URL:-https://scm.bstein.dev}"
|
||||
if [ -z "${GITEA_TOKEN:-}" ]; then
|
||||
note "GITEA_TOKEN unset; skipping repository cleanup"
|
||||
else
|
||||
for repo in $DEMO_REPOS; do
|
||||
note "clearing bstein/$repo (demo repository)"
|
||||
local items
|
||||
items="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \
|
||||
"$gitea/api/v1/repos/bstein/$repo/issues?state=all&limit=100" |
|
||||
python3 -c 'import json,sys
|
||||
for i in json.load(sys.stdin):
|
||||
print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)"
|
||||
if [ -z "$items" ]; then
|
||||
note " no issues or pull requests"
|
||||
else
|
||||
while read -r num kind; do
|
||||
[ -z "$num" ] && continue
|
||||
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
||||
"$gitea/api/v1/repos/bstein/$repo/issues/$num"
|
||||
note " deleted $kind #$num"
|
||||
done <<< "$items"
|
||||
fi
|
||||
|
||||
local branches
|
||||
branches="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \
|
||||
"$gitea/api/v1/repos/bstein/$repo/branches" |
|
||||
python3 -c 'import json,sys,urllib.parse
|
||||
for b in json.load(sys.stdin):
|
||||
if b["name"].startswith("hermes-repair/"):
|
||||
print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)"
|
||||
if [ -z "$branches" ]; then
|
||||
note " no repair branches"
|
||||
else
|
||||
for ref in $branches; do
|
||||
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
|
||||
"$gitea/api/v1/repos/bstein/$repo/branches/$ref"
|
||||
note " deleted branch $(printf '%b' "${ref//%/\\x}")"
|
||||
done
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# The code demo seeds its defect by pushing to master, and the fix only lands
|
||||
# if someone merges the pull request. A demo that ended without a merge - the
|
||||
# usual case, since the point is that nothing merges itself - leaves master
|
||||
# broken, and the next run aborts on "defect already present". Reset has to
|
||||
# undo the seed rather than just report it, or the second demo of the day
|
||||
# fails before it starts.
|
||||
if [ -d "$CODE_REPO_DIR/.git" ]; then
|
||||
note "restoring the demo repository working state"
|
||||
( cd "$CODE_REPO_DIR" && git checkout -q master && git fetch -q origin &&
|
||||
git reset -q --hard origin/master ) || note " could not sync master"
|
||||
if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
|
||||
note " src/discount.py is correct; demo is armable"
|
||||
else
|
||||
note " src/discount.py carries the seeded defect; reverting it on master"
|
||||
( cd "$CODE_REPO_DIR" &&
|
||||
python3 - <<'PY'
|
||||
import pathlib, re, sys
|
||||
|
||||
path = pathlib.Path("src/discount.py")
|
||||
source = path.read_text()
|
||||
# Matches the seeded `percent / 10` without also matching a correct
|
||||
# `percent / 100`, so re-running reset on a healthy file changes nothing.
|
||||
fixed = re.sub(r"percent / 10(?!\d)", "percent / 100", source)
|
||||
if fixed == source:
|
||||
sys.exit("unrecognised defect; fix src/discount.py by hand")
|
||||
path.write_text(fixed)
|
||||
PY
|
||||
git commit -qam "revert: restore the discount divisor" && git push -q origin master &&
|
||||
note " reverted and pushed; demo is armable" ) || note " revert failed — fix src/discount.py by hand"
|
||||
fi
|
||||
else
|
||||
note "demo repository not cloned at $CODE_REPO_DIR; skipping"
|
||||
fi
|
||||
|
||||
say "Ready"
|
||||
note "real service repositories were not touched"
|
||||
note "run 'preflight' next, then arm the fixture build"
|
||||
note "no repository was touched; this demo repairs infrastructure, not source"
|
||||
note "run 'preflight' next, then 'run'"
|
||||
}
|
||||
|
||||
cmd_preflight() {
|
||||
require_jenkins
|
||||
say "Preflight"
|
||||
say "Preflight — triage demo"
|
||||
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)"
|
||||
note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)"
|
||||
# Printed as a list rather than the raw comma-separated setting: this is the
|
||||
# outermost safety boundary, so it is worth being able to read at a glance.
|
||||
local allowlist count
|
||||
allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')"
|
||||
count=0
|
||||
for _job in $allowlist; do count=$((count + 1)); done
|
||||
note "jobs Ariadne may triage ($count):"
|
||||
for _job in $allowlist; do note " - $_job"; done
|
||||
note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')"
|
||||
local queued
|
||||
queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')"
|
||||
note "jenkins queue depth: $queued (demo is fastest when this is 0)"
|
||||
# The Kubernetes cloud caps concurrent agent pods at containerCapStr. When
|
||||
# real CI saturates that cap the demo build sits in the queue reporting
|
||||
# "all nodes are offline" and the timings in the runbook do not apply.
|
||||
local agents cap
|
||||
cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null |
|
||||
grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)"
|
||||
agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)"
|
||||
note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)"
|
||||
local open_prs
|
||||
open_prs="$(curl -s --max-time 25 -H "Authorization: token ${GITEA_TOKEN:-}" \
|
||||
"${GITEA_URL:-https://scm.bstein.dev}/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null |
|
||||
python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')"
|
||||
note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)"
|
||||
shared_preflight
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
say "Incident state (last ticks)"
|
||||
ariadne_ticks 600 8
|
||||
say "Firing alerts"
|
||||
kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null |
|
||||
python3 -c '
|
||||
import json,sys
|
||||
alerts = json.load(sys.stdin).get("data", {}).get("alerts", [])
|
||||
print(" none" if not alerts else "")
|
||||
for a in alerts:
|
||||
print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null ||
|
||||
note "(query vmalert directly if this fails)"
|
||||
shared_status
|
||||
say "Demo namespace"
|
||||
kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/ /'
|
||||
}
|
||||
|
||||
cmd_fixture() {
|
||||
cmd_run() {
|
||||
require_jenkins
|
||||
local start_num next_num
|
||||
start_num="$(last_build_number "$FIXTURE_JOB")"
|
||||
@ -268,50 +88,11 @@ cmd_fixture() {
|
||||
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
|
||||
}
|
||||
|
||||
cmd_code() {
|
||||
require_jenkins
|
||||
[ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; }
|
||||
local start_num next_num
|
||||
start_num="$(last_build_number "$CODE_JOB")"
|
||||
next_num=$((start_num + 1))
|
||||
|
||||
say "Seeding a one-line defect in src/discount.py"
|
||||
( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q &&
|
||||
python3 - <<'PY'
|
||||
import pathlib
|
||||
p = pathlib.Path("src/discount.py")
|
||||
s = p.read_text()
|
||||
old, new = "percent / 100", "percent / 10"
|
||||
if old not in s:
|
||||
raise SystemExit("defect already present or file changed; reset master first")
|
||||
p.write_text(s.replace(old, new))
|
||||
PY
|
||||
git commit -qam "refactor: simplify discount percentage math" && git push -q origin master )
|
||||
note "pushed: a plausible-looking change that breaks three regression tests"
|
||||
|
||||
say "Running the test gate -> build #$next_num"
|
||||
note "HTTP $(jenkins_post "/job/$CODE_JOB/build")"
|
||||
note "result: $(wait_for_build "$CODE_JOB" "$next_num")"
|
||||
|
||||
say "Ariadne collects evidence and asks Hermes for a minimal patch"
|
||||
note "Hermes returns an anchored patch as data; Ariadne validates path, size,"
|
||||
note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num"
|
||||
for _ in $(seq 1 40); do
|
||||
sleep 15
|
||||
ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break
|
||||
done
|
||||
ariadne_ticks 400 3
|
||||
|
||||
say "Pull request awaiting human review (nothing merges automatically)"
|
||||
note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
fixture) cmd_fixture ;;
|
||||
code) cmd_code ;;
|
||||
run|fixture) cmd_run ;;
|
||||
status) cmd_status ;;
|
||||
preflight) cmd_preflight ;;
|
||||
reset) cmd_reset ;;
|
||||
monitor) shift; cmd_monitor "$@" ;;
|
||||
*) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
|
||||
monitor) run_monitor "$FIXTURE_JOB" ;;
|
||||
*) sed -n '2,17p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
|
||||
esac
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user