diff --git a/.gitignore b/.gitignore index b041e596b..b6ad4c480 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ crash.log terraform/atlas/generated/ # Local demo credentials (never commit) +scripts/ops/hermes_demo.env scripts/ops/hermes_triage_demo.env diff --git a/scripts/ops/hermes_code_demo.sh b/scripts/ops/hermes_code_demo.sh new file mode 100755 index 000000000..7fdb94663 --- /dev/null +++ b/scripts/ops/hermes_code_demo.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Drive and narrate the Hermes code-proposal demo. +# +# This is the proposal loop: a build fails on a real defect, Ariadne asks +# Hermes for a minimal patch, validates it as data, pushes a branch and opens a +# pull request. Nothing merges. The point of this half is the stop, not the fix. +# +# hermes_code_demo.sh monitor # follow the Test Automation Diagram live +# hermes_code_demo.sh reset # restore the demo repository to pre-run state +# hermes_code_demo.sh preflight # confirm the lab is ready to demo +# hermes_code_demo.sh run # seed the defect and narrate the loop +# hermes_code_demo.sh status # current incident/alert state, no changes +# +# The autonomous-repair demo is a separate script: hermes_triage_demo.sh. +# +# FIRST RUN: copy hermes_demo.env.example to hermes_demo.env in this directory +# and fill it in. That file is git-ignored precisely so it can hold real +# tokens; this script sources it automatically. You also need bstein/ +# hermes-code-demo cloned locally (default ~/Development/hermes-code-demo, +# override with CODE_REPO_DIR). +# +# The only thing this mutates is the demo repository: it pushes a seeded defect +# to master and reverts it on reset. +set -euo pipefail + +# shellcheck source=scripts/ops/hermes_demo_lib.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_demo_lib.sh" + +# DEMO_REPOS is the whole blast radius and is deliberately explicit: a real +# service's issues are genuine triage records, and clearing them to tidy a demo +# would destroy the evidence the system exists to produce. +DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}" + +cmd_reset() { + say "Reset — restoring the code demo to its pre-run state" + + if [ -z "${GITEA_TOKEN:-}" ]; then + note "GITEA_TOKEN unset; skipping repository cleanup" + else + for repo in $DEMO_REPOS; do + note "clearing bstein/$repo (demo repository)" + local items + items="$(gitea_get "/api/v1/repos/bstein/$repo/issues?state=all&limit=100" | + python3 -c 'import json,sys +for i in json.load(sys.stdin): + print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)" + if [ -z "$items" ]; then + note " no issues or pull requests" + else + while read -r num kind; do + [ -z "$num" ] && continue + curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \ + "$GITEA_URL/api/v1/repos/bstein/$repo/issues/$num" + note " deleted $kind #$num" + done <<< "$items" + fi + + local branches + branches="$(gitea_get "/api/v1/repos/bstein/$repo/branches" | + python3 -c 'import json,sys,urllib.parse +for b in json.load(sys.stdin): + if b["name"].startswith("hermes-repair/"): + print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)" + if [ -z "$branches" ]; then + note " no repair branches" + else + for ref in $branches; do + curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \ + "$GITEA_URL/api/v1/repos/bstein/$repo/branches/$ref" + note " deleted branch $(printf '%b' "${ref//%/\\x}")" + done + fi + done + fi + + # The demo seeds its defect by pushing to master, and the fix only lands if + # someone merges the pull request - which, by design, nobody does during a + # demo. So master stays broken, and the next run aborts on "defect already + # present". Reset has to undo the seed rather than just report it, or the + # second demo of the day fails before it starts. + if [ -d "$CODE_REPO_DIR/.git" ]; then + note "restoring the demo repository working state" + ( cd "$CODE_REPO_DIR" && git checkout -q master && git fetch -q origin && + git reset -q --hard origin/master ) || note " could not sync master" + if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then + note " src/discount.py is correct; demo is armable" + else + note " src/discount.py carries the seeded defect; reverting it on master" + ( cd "$CODE_REPO_DIR" && + python3 - <<'PY' +import pathlib, re, sys + +path = pathlib.Path("src/discount.py") +source = path.read_text() +# Matches the seeded `percent / 10` without also matching a correct +# `percent / 100`, so re-running reset on a healthy file changes nothing. +fixed = re.sub(r"percent / 10(?!\d)", "percent / 100", source) +if fixed == source: + sys.exit("unrecognised defect; fix src/discount.py by hand") +path.write_text(fixed) +PY + git commit -qam "revert: restore the discount divisor" && git push -q origin master && + note " reverted and pushed; demo is armable" ) || note " revert failed — fix src/discount.py by hand" + fi + else + note "demo repository not cloned at $CODE_REPO_DIR; skipping" + fi + + say "Ready" + note "real service repositories were not touched" + note "run 'preflight' next, then 'run'" +} + +cmd_preflight() { + require_jenkins + say "Preflight — code demo" + if [ -d "$CODE_REPO_DIR/.git" ]; then + if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then + note "demo repository: src/discount.py is correct; armable" + else + note "demo repository: src/discount.py carries a defect — run 'reset' first" + fi + else + note "demo repository: NOT CLONED at $CODE_REPO_DIR" + fi + local open_prs + open_prs="$(gitea_get "/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null | + python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')" + note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)" + note "code proposals enabled: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_CODE_ENABLED 2>/dev/null)" + note "fix categories: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_FIX_CATEGORIES 2>/dev/null)" + shared_preflight +} + +cmd_status() { + shared_status + say "Open proposals" + note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls" +} + +cmd_run() { + require_jenkins + [ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; } + local start_num next_num + start_num="$(last_build_number "$CODE_JOB")" + next_num=$((start_num + 1)) + + say "Seeding a one-line defect in src/discount.py" + ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q && + python3 - <<'PY' +import pathlib +p = pathlib.Path("src/discount.py") +s = p.read_text() +old, new = "percent / 100", "percent / 10" +if old not in s: + raise SystemExit("defect already present or file changed; run reset first") +p.write_text(s.replace(old, new)) +PY + git commit -qam "refactor: simplify discount percentage math" && git push -q origin master ) + note "pushed: a plausible-looking change that breaks three regression tests" + + say "Running the test gate -> build #$next_num" + note "HTTP $(jenkins_post "/job/$CODE_JOB/build")" + note "result: $(wait_for_build "$CODE_JOB" "$next_num")" + + say "Ariadne collects evidence and asks Hermes for a minimal patch" + note "Hermes returns an anchored patch as data; Ariadne validates path, size," + note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num" + for _ in $(seq 1 40); do + sleep 15 + ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break + done + ariadne_ticks 400 3 + + say "Pull request awaiting human review (nothing merges automatically)" + note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls" +} + +case "${1:-}" in + run|code) cmd_run ;; + status) cmd_status ;; + preflight) cmd_preflight ;; + reset) cmd_reset ;; + monitor) run_monitor "$CODE_JOB" ;; + *) sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;; +esac diff --git a/scripts/ops/hermes_triage_demo.env.example b/scripts/ops/hermes_demo.env.example similarity index 51% rename from scripts/ops/hermes_triage_demo.env.example rename to scripts/ops/hermes_demo.env.example index e03be508c..e72c130ae 100644 --- a/scripts/ops/hermes_triage_demo.env.example +++ b/scripts/ops/hermes_demo.env.example @@ -1,12 +1,14 @@ -# Copy to hermes_triage_demo.env (same directory) and fill in. -# That filename is git-ignored so it can hold real tokens. +# Copy to hermes_demo.env (same directory) and fill in. That filename is +# git-ignored so it can hold real tokens. Both demo drivers read it: +# hermes_triage_demo.sh and hermes_code_demo.sh. # Jenkins API token: https://ci.bstein.dev -> your user -> Configure -> API Token export JENKINS_USER="your-jenkins-user" export JENKINS_TOKEN="your-jenkins-api-token" -# Gitea token, used only to report open pull requests during preflight. -# Preflight still works without it; that one line will read "?". +# Gitea token. The code demo needs it to clear its own demo repository on +# reset and to report open pull requests during preflight; the triage demo +# touches no repository and works without it. export GITEA_TOKEN="your-gitea-token" # Override only if you are not pointing at the usual lab. diff --git a/scripts/ops/hermes_demo_lib.sh b/scripts/ops/hermes_demo_lib.sh new file mode 100644 index 000000000..53fbd29e9 --- /dev/null +++ b/scripts/ops/hermes_demo_lib.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Shared plumbing for the two Hermes demo drivers. +# +# The triage demo and the code demo are separate scripts on purpose: they prove +# different halves of the Test Automation Diagram, they reset different things, +# and mixing them behind one command invited exactly the confusion of running +# the wrong subcommand in front of an audience. What they genuinely share - +# credentials, Jenkins access, the Ariadne tick reader - lives here, so a fix +# to any of it applies to both instead of being made twice and drifting. +# +# Not executable on its own; both drivers source it. + +# Local, git-ignored credentials. `hermes_demo.env` is the current name; +# `hermes_triage_demo.env` is still read so an existing filled-in file keeps +# working after the split. +_DEMO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEMO_ENV="" +for _candidate in "$_DEMO_DIR/hermes_demo.env" "$_DEMO_DIR/hermes_triage_demo.env"; do + if [ -r "$_candidate" ]; then + DEMO_ENV="$_candidate" + # shellcheck disable=SC1090 + . "$_candidate" + break + fi +done +[ -n "$DEMO_ENV" ] || DEMO_ENV="$_DEMO_DIR/hermes_demo.env" + +JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}" +GITEA_URL="${GITEA_URL:-https://scm.bstein.dev}" +FIXTURE_JOB="hermes-triage-demo" +CODE_JOB="hermes-code-demo" +DEMO_NS="hermes-triage-demo" +CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}" + +say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; } +note() { printf ' %s\n' "$*"; } + +require_jenkins() { + if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then + echo "Missing Jenkins credentials." >&2 + echo "Create $DEMO_ENV from hermes_demo.env.example and fill it in." >&2 + exit 1 + fi +} + +# Every remote call is time-bounded. A hung curl during a demo is worse than a +# failed one: a failure says what to do next, a hang says nothing at all. +jenkins_get() { curl -sk --max-time 25 -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; } +jenkins_post() { + curl -sk --max-time 25 -o /dev/null -w '%{http_code}' \ + -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1" +} +gitea_get() { curl -s --max-time 25 -H "Authorization: token ${GITEA_TOKEN:-}" "$GITEA_URL$1"; } + +last_build_number() { + jenkins_get "/job/$1/api/json?tree=lastBuild[number]" | + python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])' +} + +wait_for_build() { # job number [tries] -> prints result + local job="$1" num="$2" tries="${3:-120}" + for _ in $(seq 1 "$tries"); do + sleep 10 + local body result building + body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)" + building="$(printf '%s' "$body" | + python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)" + if [ "$building" = "False" ]; then + result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')" + printf '%s' "$result" + return 0 + fi + done + printf 'TIMEOUT' +} + +ariadne_ticks() { # tail the autotriage decisions in human-readable form + kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null | + grep 'hermes autotriage tick' | + python3 -c ' +import sys, json +for line in sys.stdin: + try: + d = json.loads(line) + except ValueError: + continue + print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}" +} + +# Both drivers narrate the same Test Automation Diagram; the monitor selects +# which branch of it to follow from MONITOR_JOB. +run_monitor() { # job + export MONITOR_JOB="$1" + exec python3 "$_DEMO_DIR/hermes_triage_monitor.py" +} + +# The checks that are true of the lab regardless of which demo is running. +shared_preflight() { + note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')" + note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)" + # Printed as a list rather than the raw comma-separated setting: this is the + # outermost safety boundary, so it is worth being able to read at a glance. + local allowlist count + allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')" + count=0 + for _job in $allowlist; do count=$((count + 1)); done + note "jobs Ariadne may triage ($count):" + for _job in $allowlist; do note " - $_job"; done + note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')" + local queued + queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')" + note "jenkins queue depth: $queued (demo is fastest when this is 0)" + # The Kubernetes cloud caps concurrent agent pods at containerCapStr. When + # real CI saturates that cap the demo build sits in the queue reporting + # "all nodes are offline" and the timings in the runbook do not apply. + local agents cap + cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null | + grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)" + agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)" + note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)" +} + +# The alert and incident state both demos are judged by. +shared_status() { + say "Incident state (last ticks)" + ariadne_ticks 600 8 + say "Firing alerts" + kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null | + python3 -c ' +import json,sys +alerts = json.load(sys.stdin).get("data", {}).get("alerts", []) +print(" none" if not alerts else "") +for a in alerts: + print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null || + note "(query vmalert directly if this fails)" +} diff --git a/scripts/ops/hermes_triage_demo.sh b/scripts/ops/hermes_triage_demo.sh index 450c63fb1..a9e9341d8 100755 --- a/scripts/ops/hermes_triage_demo.sh +++ b/scripts/ops/hermes_triage_demo.sh @@ -1,103 +1,36 @@ #!/usr/bin/env bash -# Drive and narrate the Hermes automated-triage demos. +# Drive and narrate the Hermes automated-triage demo. # -# hermes_triage_demo.sh fixture # autonomous loop: fail -> repair -> green -# hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR -# hermes_triage_demo.sh status # current incident/alert state, no changes -# hermes_triage_demo.sh preflight # confirm the lab is ready to demo -# hermes_triage_demo.sh reset # restore the demo to its pre-run state -# hermes_triage_demo.sh monitor [code] # follow the Test Automation Diagram live +# This is the autonomous loop: a build fails, Ariadne diagnoses it through +# Hermes, authorizes a predefined repair, performs it, and rebuilds green +# without a human touching anything. # -# FIRST RUN: copy hermes_triage_demo.env.example to hermes_triage_demo.env in -# this directory and fill it in. That file is git-ignored precisely so it can -# hold real tokens; this script sources it automatically, so nothing needs to -# be exported by hand. +# hermes_triage_demo.sh monitor # follow the Test Automation Diagram live +# hermes_triage_demo.sh reset # restore the demo to its pre-run state +# hermes_triage_demo.sh preflight # confirm the lab is ready to demo +# hermes_triage_demo.sh run # arm the failure and narrate the loop +# hermes_triage_demo.sh status # current incident/alert state, no changes +# +# The code-proposal demo is a separate script: hermes_code_demo.sh. They prove +# different halves of the diagram and reset different things, so they are kept +# apart rather than behind one command. +# +# FIRST RUN: copy hermes_demo.env.example to hermes_demo.env in this directory +# and fill it in. That file is git-ignored precisely so it can hold real +# tokens; this script sources it automatically. # # Needs kubectl access to the cluster as well. Nothing here mutates the cluster -# directly: the fixture demo only asks Jenkins to run a parameterized build, -# and the code demo only pushes a seeded defect to the demo repository. +# directly: the demo only asks Jenkins to run a parameterized build. set -euo pipefail -# Local, git-ignored credentials. Sourced before anything else so every value -# below can be overridden from it. -_DEMO_ENV="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_demo.env" -# shellcheck disable=SC1090 -[ -r "$_DEMO_ENV" ] && . "$_DEMO_ENV" - -JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}" -FIXTURE_JOB="hermes-triage-demo" -CODE_JOB="hermes-code-demo" -DEMO_NS="hermes-triage-demo" -CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}" - -say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; } -note() { printf ' %s\n' "$*"; } - -require_jenkins() { - if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then - echo "Missing Jenkins credentials." >&2 - echo "Create $_DEMO_ENV from hermes_triage_demo.env.example and fill it in." >&2 - exit 1 - fi -} - -jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; } -jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; } - -last_build_number() { - jenkins_get "/job/$1/api/json?tree=lastBuild[number]" | - python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])' -} - -wait_for_build() { # job number -> prints result - local job="$1" num="$2" tries="${3:-120}" - for _ in $(seq 1 "$tries"); do - sleep 10 - local body result building - body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)" - building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)" - if [ "$building" = "False" ]; then - result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')" - printf '%s' "$result" - return 0 - fi - done - printf 'TIMEOUT' -} - -ariadne_ticks() { # tail the autotriage decisions in human-readable form - kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null | - grep 'hermes autotriage tick' | - python3 -c ' -import sys, json -for line in sys.stdin: - try: - d = json.loads(line) - except ValueError: - continue - print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}" -} - -# `monitor` follows the fixture job; `monitor code` follows the code-proposal -# job, which takes the source-proposal branch of the Test Automation Diagram -# instead. Both narrate the same diagram, stage for stage. -cmd_monitor() { - local which="${1:-fixture}" - [ "$which" = "code" ] && export MONITOR_JOB="$CODE_JOB" - exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_monitor.py" -} - -# Deletes the demo repositories' issues, pull requests and repair branches so a -# rerun starts from nothing. DEMO_REPOS is the whole blast radius and is -# deliberately explicit: a real service's issues are genuine triage records, -# and clearing them to tidy a demo would destroy the evidence the system -# exists to produce. -DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}" +# shellcheck source=scripts/ops/hermes_demo_lib.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_demo_lib.sh" +# The fixture is a ConfigMap the demo job reads. Resetting it is the whole +# blast radius of this script: no repository is touched, because the triage +# loop repairs infrastructure rather than source. cmd_reset() { - require_jenkins - say "Reset — restoring the demo to its pre-run state" - + say "Reset — restoring the triage demo to its pre-run state" note "fixture -> healthy" if kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \ --type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1; then @@ -105,138 +38,25 @@ cmd_reset() { else note " fixture patch failed (is the demo namespace present?)" fi - - local gitea="${GITEA_URL:-https://scm.bstein.dev}" - if [ -z "${GITEA_TOKEN:-}" ]; then - note "GITEA_TOKEN unset; skipping repository cleanup" - else - for repo in $DEMO_REPOS; do - note "clearing bstein/$repo (demo repository)" - local items - items="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \ - "$gitea/api/v1/repos/bstein/$repo/issues?state=all&limit=100" | - python3 -c 'import json,sys -for i in json.load(sys.stdin): - print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)" - if [ -z "$items" ]; then - note " no issues or pull requests" - else - while read -r num kind; do - [ -z "$num" ] && continue - curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \ - "$gitea/api/v1/repos/bstein/$repo/issues/$num" - note " deleted $kind #$num" - done <<< "$items" - fi - - local branches - branches="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \ - "$gitea/api/v1/repos/bstein/$repo/branches" | - python3 -c 'import json,sys,urllib.parse -for b in json.load(sys.stdin): - if b["name"].startswith("hermes-repair/"): - print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)" - if [ -z "$branches" ]; then - note " no repair branches" - else - for ref in $branches; do - curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \ - "$gitea/api/v1/repos/bstein/$repo/branches/$ref" - note " deleted branch $(printf '%b' "${ref//%/\\x}")" - done - fi - done - fi - - # The code demo seeds its defect by pushing to master, and the fix only lands - # if someone merges the pull request. A demo that ended without a merge - the - # usual case, since the point is that nothing merges itself - leaves master - # broken, and the next run aborts on "defect already present". Reset has to - # undo the seed rather than just report it, or the second demo of the day - # fails before it starts. - if [ -d "$CODE_REPO_DIR/.git" ]; then - note "restoring the demo repository working state" - ( cd "$CODE_REPO_DIR" && git checkout -q master && git fetch -q origin && - git reset -q --hard origin/master ) || note " could not sync master" - if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then - note " src/discount.py is correct; demo is armable" - else - note " src/discount.py carries the seeded defect; reverting it on master" - ( cd "$CODE_REPO_DIR" && - python3 - <<'PY' -import pathlib, re, sys - -path = pathlib.Path("src/discount.py") -source = path.read_text() -# Matches the seeded `percent / 10` without also matching a correct -# `percent / 100`, so re-running reset on a healthy file changes nothing. -fixed = re.sub(r"percent / 10(?!\d)", "percent / 100", source) -if fixed == source: - sys.exit("unrecognised defect; fix src/discount.py by hand") -path.write_text(fixed) -PY - git commit -qam "revert: restore the discount divisor" && git push -q origin master && - note " reverted and pushed; demo is armable" ) || note " revert failed — fix src/discount.py by hand" - fi - else - note "demo repository not cloned at $CODE_REPO_DIR; skipping" - fi - say "Ready" - note "real service repositories were not touched" - note "run 'preflight' next, then arm the fixture build" + note "no repository was touched; this demo repairs infrastructure, not source" + note "run 'preflight' next, then 'run'" } cmd_preflight() { require_jenkins - say "Preflight" + say "Preflight — triage demo" note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)" - note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')" - note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)" - # Printed as a list rather than the raw comma-separated setting: this is the - # outermost safety boundary, so it is worth being able to read at a glance. - local allowlist count - allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')" - count=0 - for _job in $allowlist; do count=$((count + 1)); done - note "jobs Ariadne may triage ($count):" - for _job in $allowlist; do note " - $_job"; done - note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')" - local queued - queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')" - note "jenkins queue depth: $queued (demo is fastest when this is 0)" - # The Kubernetes cloud caps concurrent agent pods at containerCapStr. When - # real CI saturates that cap the demo build sits in the queue reporting - # "all nodes are offline" and the timings in the runbook do not apply. - local agents cap - cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null | - grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)" - agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)" - note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)" - local open_prs - open_prs="$(curl -s --max-time 25 -H "Authorization: token ${GITEA_TOKEN:-}" \ - "${GITEA_URL:-https://scm.bstein.dev}/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null | - python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')" - note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)" + shared_preflight } cmd_status() { - say "Incident state (last ticks)" - ariadne_ticks 600 8 - say "Firing alerts" - kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null | - python3 -c ' -import json,sys -alerts = json.load(sys.stdin).get("data", {}).get("alerts", []) -print(" none" if not alerts else "") -for a in alerts: - print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null || - note "(query vmalert directly if this fails)" + shared_status say "Demo namespace" kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/ /' } -cmd_fixture() { +cmd_run() { require_jenkins local start_num next_num start_num="$(last_build_number "$FIXTURE_JOB")" @@ -268,50 +88,11 @@ cmd_fixture() { note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')" } -cmd_code() { - require_jenkins - [ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; } - local start_num next_num - start_num="$(last_build_number "$CODE_JOB")" - next_num=$((start_num + 1)) - - say "Seeding a one-line defect in src/discount.py" - ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q && - python3 - <<'PY' -import pathlib -p = pathlib.Path("src/discount.py") -s = p.read_text() -old, new = "percent / 100", "percent / 10" -if old not in s: - raise SystemExit("defect already present or file changed; reset master first") -p.write_text(s.replace(old, new)) -PY - git commit -qam "refactor: simplify discount percentage math" && git push -q origin master ) - note "pushed: a plausible-looking change that breaks three regression tests" - - say "Running the test gate -> build #$next_num" - note "HTTP $(jenkins_post "/job/$CODE_JOB/build")" - note "result: $(wait_for_build "$CODE_JOB" "$next_num")" - - say "Ariadne collects evidence and asks Hermes for a minimal patch" - note "Hermes returns an anchored patch as data; Ariadne validates path, size," - note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num" - for _ in $(seq 1 40); do - sleep 15 - ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break - done - ariadne_ticks 400 3 - - say "Pull request awaiting human review (nothing merges automatically)" - note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls" -} - case "${1:-}" in - fixture) cmd_fixture ;; - code) cmd_code ;; + run|fixture) cmd_run ;; status) cmd_status ;; preflight) cmd_preflight ;; reset) cmd_reset ;; - monitor) shift; cmd_monitor "$@" ;; - *) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;; + monitor) run_monitor "$FIXTURE_JOB" ;; + *) sed -n '2,17p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;; esac