feat(hermes-code): multibranch validation for hermes-repair/* proposal branches
Adds demo driver script and branch-level test gate so a Hermes-proposed pull request carries a green build before human merge. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jenkins
parent
029779724c
commit
a555fc0c96
170
scripts/ops/hermes_triage_demo.sh
Executable file
170
scripts/ops/hermes_triage_demo.sh
Executable file
@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
# Drive and narrate the Hermes automated-triage demos.
|
||||
#
|
||||
# hermes_triage_demo.sh fixture # autonomous loop: fail -> repair -> green
|
||||
# hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR
|
||||
# hermes_triage_demo.sh status # current incident/alert state, no changes
|
||||
# hermes_triage_demo.sh preflight # confirm the lab is ready to demo
|
||||
#
|
||||
# Requires JENKINS_USER and JENKINS_TOKEN in the environment (a Jenkins API
|
||||
# token) plus kubectl access to the cluster. Nothing here mutates the cluster
|
||||
# directly: the fixture demo only asks Jenkins to run a parameterized build,
|
||||
# and the code demo only pushes a seeded defect to the demo repository.
|
||||
set -euo pipefail
|
||||
|
||||
JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}"
|
||||
FIXTURE_JOB="hermes-triage-demo"
|
||||
CODE_JOB="hermes-code-demo"
|
||||
DEMO_NS="hermes-triage-demo"
|
||||
CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}"
|
||||
|
||||
say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
note() { printf ' %s\n' "$*"; }
|
||||
|
||||
require_jenkins() {
|
||||
: "${JENKINS_USER:?set JENKINS_USER}" "${JENKINS_TOKEN:?set JENKINS_TOKEN}"
|
||||
}
|
||||
|
||||
jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; }
|
||||
jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; }
|
||||
|
||||
last_build_number() {
|
||||
jenkins_get "/job/$1/api/json?tree=lastBuild[number]" |
|
||||
python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])'
|
||||
}
|
||||
|
||||
wait_for_build() { # job number -> prints result
|
||||
local job="$1" num="$2" tries="${3:-120}"
|
||||
for _ in $(seq 1 "$tries"); do
|
||||
sleep 10
|
||||
local body result building
|
||||
body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)"
|
||||
building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)"
|
||||
if [ "$building" = "False" ]; then
|
||||
result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')"
|
||||
printf '%s' "$result"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
printf 'TIMEOUT'
|
||||
}
|
||||
|
||||
ariadne_ticks() { # tail the autotriage decisions in human-readable form
|
||||
kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null |
|
||||
grep 'hermes autotriage tick' |
|
||||
python3 -c '
|
||||
import sys, json
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
|
||||
}
|
||||
|
||||
cmd_preflight() {
|
||||
require_jenkins
|
||||
say "Preflight"
|
||||
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)"
|
||||
note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)"
|
||||
note "allowlisted jobs: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null)"
|
||||
note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')"
|
||||
local queued
|
||||
queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')"
|
||||
note "jenkins queue depth: $queued (demo is fastest when this is 0)"
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
say "Incident state (last ticks)"
|
||||
ariadne_ticks 600 8
|
||||
say "Firing alerts"
|
||||
kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null |
|
||||
python3 -c '
|
||||
import json,sys
|
||||
alerts = json.load(sys.stdin).get("data", {}).get("alerts", [])
|
||||
print(" none" if not alerts else "")
|
||||
for a in alerts:
|
||||
print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null ||
|
||||
note "(query vmalert directly if this fails)"
|
||||
say "Demo namespace"
|
||||
kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/ /'
|
||||
}
|
||||
|
||||
cmd_fixture() {
|
||||
require_jenkins
|
||||
local start_num next_num
|
||||
start_num="$(last_build_number "$FIXTURE_JOB")"
|
||||
next_num=$((start_num + 1))
|
||||
say "Arming the demo failure (SEED_FAILURE=true) -> build #$next_num"
|
||||
note "HTTP $(jenkins_post "/job/$FIXTURE_JOB/buildWithParameters?SEED_FAILURE=true")"
|
||||
note "Only manual step. Everything after this is automatic."
|
||||
|
||||
say "Waiting for the seeded build to fail"
|
||||
note "result: $(wait_for_build "$FIXTURE_JOB" "$next_num")"
|
||||
|
||||
say "Ariadne detects, gathers evidence, asks Hermes, authorizes, repairs"
|
||||
local repair="hermes-demo-repair-$next_num"
|
||||
for _ in $(seq 1 40); do
|
||||
sleep 10
|
||||
if [ "$(kubectl -n "$DEMO_NS" get job "$repair" -o jsonpath='{.status.succeeded}' 2>/dev/null)" = "1" ]; then
|
||||
note "repair job $repair succeeded"
|
||||
break
|
||||
fi
|
||||
done
|
||||
ariadne_ticks 400 4
|
||||
|
||||
say "Ariadne triggers one rebuild with seeding disabled"
|
||||
note "result: $(wait_for_build "$FIXTURE_JOB" $((next_num + 1)))"
|
||||
|
||||
say "Resolution"
|
||||
sleep 45
|
||||
ariadne_ticks 200 3
|
||||
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
|
||||
}
|
||||
|
||||
cmd_code() {
|
||||
require_jenkins
|
||||
[ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; }
|
||||
local start_num next_num
|
||||
start_num="$(last_build_number "$CODE_JOB")"
|
||||
next_num=$((start_num + 1))
|
||||
|
||||
say "Seeding a one-line defect in src/discount.py"
|
||||
( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q &&
|
||||
python3 - <<'PY'
|
||||
import pathlib
|
||||
p = pathlib.Path("src/discount.py")
|
||||
s = p.read_text()
|
||||
old, new = "percent / 100", "percent / 10"
|
||||
if old not in s:
|
||||
raise SystemExit("defect already present or file changed; reset master first")
|
||||
p.write_text(s.replace(old, new))
|
||||
PY
|
||||
git commit -qam "refactor: simplify discount percentage math" && git push -q origin master )
|
||||
note "pushed: a plausible-looking change that breaks three regression tests"
|
||||
|
||||
say "Running the test gate -> build #$next_num"
|
||||
note "HTTP $(jenkins_post "/job/$CODE_JOB/build")"
|
||||
note "result: $(wait_for_build "$CODE_JOB" "$next_num")"
|
||||
|
||||
say "Ariadne collects evidence and asks Hermes for a minimal patch"
|
||||
note "Hermes returns an anchored patch as data; Ariadne validates path, size,"
|
||||
note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num"
|
||||
for _ in $(seq 1 40); do
|
||||
sleep 15
|
||||
ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break
|
||||
done
|
||||
ariadne_ticks 400 3
|
||||
|
||||
say "Pull request awaiting human review (nothing merges automatically)"
|
||||
note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
fixture) cmd_fixture ;;
|
||||
code) cmd_code ;;
|
||||
status) cmd_status ;;
|
||||
preflight) cmd_preflight ;;
|
||||
*) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
|
||||
esac
|
||||
@ -447,6 +447,35 @@ data:
|
||||
}
|
||||
}
|
||||
}
|
||||
multibranchPipelineJob('hermes-code-demo-branches') {
|
||||
description('Branch validation for the Hermes code-repair demo. Builds hermes-repair/* proposal branches so a pull request carries a green test gate before a human merges it.')
|
||||
branchSources {
|
||||
branchSource {
|
||||
source {
|
||||
git {
|
||||
id('hermes-code-demo-branches')
|
||||
remote('https://scm.bstein.dev/bstein/hermes-code-demo.git')
|
||||
credentialsId('gitea-pat')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
factory {
|
||||
workflowBranchProjectFactory {
|
||||
scriptPath('Jenkinsfile')
|
||||
}
|
||||
}
|
||||
orphanedItemStrategy {
|
||||
discardOldItems {
|
||||
numToKeep(20)
|
||||
}
|
||||
}
|
||||
triggers {
|
||||
periodicFolderTrigger {
|
||||
interval('1m')
|
||||
}
|
||||
}
|
||||
}
|
||||
pipelineJob('hermes-code-demo') {
|
||||
disabled(false)
|
||||
description('Hermes code-repair demo: pytest gate over a small pricing helper. A seeded one-line defect fails the gate; Hermes proposes a minimal patch and Ariadne opens a pull request for human review.')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user