diff --git a/scripts/ops/hermes_triage_demo.sh b/scripts/ops/hermes_triage_demo.sh new file mode 100755 index 000000000..280cd9fbf --- /dev/null +++ b/scripts/ops/hermes_triage_demo.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# Drive and narrate the Hermes automated-triage demos. +# +# hermes_triage_demo.sh fixture # autonomous loop: fail -> repair -> green +# hermes_triage_demo.sh code # proposal loop: fail -> Hermes patch -> PR +# hermes_triage_demo.sh status # current incident/alert state, no changes +# hermes_triage_demo.sh preflight # confirm the lab is ready to demo +# +# Requires JENKINS_USER and JENKINS_TOKEN in the environment (a Jenkins API +# token) plus kubectl access to the cluster. Nothing here mutates the cluster +# directly: the fixture demo only asks Jenkins to run a parameterized build, +# and the code demo only pushes a seeded defect to the demo repository. +set -euo pipefail + +JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}" +FIXTURE_JOB="hermes-triage-demo" +CODE_JOB="hermes-code-demo" +DEMO_NS="hermes-triage-demo" +CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}" + +say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; } +note() { printf ' %s\n' "$*"; } + +require_jenkins() { + : "${JENKINS_USER:?set JENKINS_USER}" "${JENKINS_TOKEN:?set JENKINS_TOKEN}" +} + +jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; } +jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; } + +last_build_number() { + jenkins_get "/job/$1/api/json?tree=lastBuild[number]" | + python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])' +} + +wait_for_build() { # job number -> prints result + local job="$1" num="$2" tries="${3:-120}" + for _ in $(seq 1 "$tries"); do + sleep 10 + local body result building + body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)" + building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)" + if [ "$building" = "False" ]; then + result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')" + printf '%s' "$result" + return 0 + fi + done + printf 'TIMEOUT' +} + +ariadne_ticks() { # tail the autotriage decisions in human-readable form + kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null | + grep 'hermes autotriage tick' | + python3 -c ' +import sys, json +for line in sys.stdin: + try: + d = json.loads(line) + except ValueError: + continue + print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}" +} + +cmd_preflight() { + require_jenkins + say "Preflight" + note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)" + note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')" + note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)" + note "allowlisted jobs: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null)" + note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')" + local queued + queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')" + note "jenkins queue depth: $queued (demo is fastest when this is 0)" +} + +cmd_status() { + say "Incident state (last ticks)" + ariadne_ticks 600 8 + say "Firing alerts" + kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null | + python3 -c ' +import json,sys +alerts = json.load(sys.stdin).get("data", {}).get("alerts", []) +print(" none" if not alerts else "") +for a in alerts: + print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null || + note "(query vmalert directly if this fails)" + say "Demo namespace" + kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/ /' +} + +cmd_fixture() { + require_jenkins + local start_num next_num + start_num="$(last_build_number "$FIXTURE_JOB")" + next_num=$((start_num + 1)) + say "Arming the demo failure (SEED_FAILURE=true) -> build #$next_num" + note "HTTP $(jenkins_post "/job/$FIXTURE_JOB/buildWithParameters?SEED_FAILURE=true")" + note "Only manual step. Everything after this is automatic." + + say "Waiting for the seeded build to fail" + note "result: $(wait_for_build "$FIXTURE_JOB" "$next_num")" + + say "Ariadne detects, gathers evidence, asks Hermes, authorizes, repairs" + local repair="hermes-demo-repair-$next_num" + for _ in $(seq 1 40); do + sleep 10 + if [ "$(kubectl -n "$DEMO_NS" get job "$repair" -o jsonpath='{.status.succeeded}' 2>/dev/null)" = "1" ]; then + note "repair job $repair succeeded" + break + fi + done + ariadne_ticks 400 4 + + say "Ariadne triggers one rebuild with seeding disabled" + note "result: $(wait_for_build "$FIXTURE_JOB" $((next_num + 1)))" + + say "Resolution" + sleep 45 + ariadne_ticks 200 3 + note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')" +} + +cmd_code() { + require_jenkins + [ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; } + local start_num next_num + start_num="$(last_build_number "$CODE_JOB")" + next_num=$((start_num + 1)) + + say "Seeding a one-line defect in src/discount.py" + ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q && + python3 - <<'PY' +import pathlib +p = pathlib.Path("src/discount.py") +s = p.read_text() +old, new = "percent / 100", "percent / 10" +if old not in s: + raise SystemExit("defect already present or file changed; reset master first") +p.write_text(s.replace(old, new)) +PY + git commit -qam "refactor: simplify discount percentage math" && git push -q origin master ) + note "pushed: a plausible-looking change that breaks three regression tests" + + say "Running the test gate -> build #$next_num" + note "HTTP $(jenkins_post "/job/$CODE_JOB/build")" + note "result: $(wait_for_build "$CODE_JOB" "$next_num")" + + say "Ariadne collects evidence and asks Hermes for a minimal patch" + note "Hermes returns an anchored patch as data; Ariadne validates path, size," + note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num" + for _ in $(seq 1 40); do + sleep 15 + ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break + done + ariadne_ticks 400 3 + + say "Pull request awaiting human review (nothing merges automatically)" + note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls" +} + +case "${1:-}" in + fixture) cmd_fixture ;; + code) cmd_code ;; + status) cmd_status ;; + preflight) cmd_preflight ;; + *) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;; +esac diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index 48a0503da..a3d173d82 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -447,6 +447,35 @@ data: } } } + multibranchPipelineJob('hermes-code-demo-branches') { + description('Branch validation for the Hermes code-repair demo. Builds hermes-repair/* proposal branches so a pull request carries a green test gate before a human merges it.') + branchSources { + branchSource { + source { + git { + id('hermes-code-demo-branches') + remote('https://scm.bstein.dev/bstein/hermes-code-demo.git') + credentialsId('gitea-pat') + } + } + } + } + factory { + workflowBranchProjectFactory { + scriptPath('Jenkinsfile') + } + } + orphanedItemStrategy { + discardOldItems { + numToKeep(20) + } + } + triggers { + periodicFolderTrigger { + interval('1m') + } + } + } pipelineJob('hermes-code-demo') { disabled(false) description('Hermes code-repair demo: pytest gate over a small pricing helper. A seeded one-line defect fails the gate; Hermes proposes a minimal patch and Ariadne opens a pull request for human review.')