diff --git a/services/hermes/configmap.yaml b/services/hermes/configmap.yaml index 4f7d9196e..c89c7c159 100644 --- a/services/hermes/configmap.yaml +++ b/services/hermes/configmap.yaml @@ -110,6 +110,10 @@ data: read-only state. Turn repeated successful triage paths into reusable skills or memory when the pattern is stable. + When Brad says "triage", "triage now", "check the latest failures", or + "what is broken", begin the triage-titan-test-failures workflow + immediately. Do not make him choose a tool, page, or evidence source first. + Stay Flux-first. Do not mutate the cluster directly. Explain evidence, recommend the smallest repo-side change, and name the exact verification commands a human should run after Flux reconciles. @@ -119,6 +123,11 @@ data: You are Hermes running inside the Titan Kubernetes cluster as a read-only testing and operations triage assistant. + Treat short requests such as `triage`, `triage now`, `check the latest + failures`, and `what is broken` as instructions to load and run the + `triage-titan-test-failures` skill immediately. Ask only when approval is + required for a state-changing evidence refresh. + Ariadne owns deterministic evidence collection and local diagnosis. Start every testing triage by reading: @@ -152,3 +161,28 @@ data: Do not run mutating commands such as `kubectl apply`, `delete`, `scale`, `patch`, `cordon`, `uncordon`, `drain`, or `rollout restart`. Do not read Kubernetes Secret values. Draft repo changes or operator steps instead. + START-HERE.md: | + # Hermes on Atlas: start here + + Use Chat for a new investigation or Sessions to resume a previous one. + The shortest useful prompts are: + + - `Triage the latest test failure.` + - `What is broken in the cluster right now?` + - `Triage build and save the report.` + + Hermes will use Ariadne evidence, Jenkins log excerpts, quality metrics, + recent Git changes, Flux state, Grafana context, and Kubernetes read-only + state. It will separate facts, inference, and unknowns and will ask before + triggering a fresh evidence collection. + + Two real examples are already available under Sessions: + + 1. `Proof 1 - Soteria 269 SonarQube triage` + 2. `Proof 2 - Veles registry outage triage` + + Their saved reports are in Files under `triage-proof/`. + + Hermes may inspect workload and delivery metadata, logs, metrics, and + events. Its Kubernetes identity cannot read Secret values or mutate cluster + resources. Apply fixes through the titan-iac GitOps workflow after review. diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 74d141253..b4d5154f2 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback ai.bstein.dev/role: testing-triage ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred) - ai.bstein.dev/config-rev: "20260802-session-stability" + ai.bstein.dev/config-rev: "20260802-triage-ready" spec: serviceAccountName: hermes-triage automountServiceAccountToken: true @@ -82,10 +82,11 @@ spec: - -c - | set -eu - mkdir -p /opt/data/workspace /opt/data/home/.local/bin /opt/data/logs + mkdir -p /opt/data/workspace/triage-proof /opt/data/home/.local/bin /opt/data/logs cp /config/config.yaml /opt/data/config.yaml cp /config/SOUL.md /opt/data/SOUL.md cp /config/AGENTS.md /opt/data/workspace/AGENTS.md + cp /config/START-HERE.md /opt/data/workspace/START-HERE.md touch /opt/data/.env if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" diff --git a/services/hermes/rbac.yaml b/services/hermes/rbac.yaml index ae44ff15c..bad9a4d29 100644 --- a/services/hermes/rbac.yaml +++ b/services/hermes/rbac.yaml @@ -22,6 +22,7 @@ rules: - pods - pods/log - replicationcontrollers + - serviceaccounts - services verbs: ["get", "list", "watch"] - apiGroups: ["apps"] @@ -54,6 +55,12 @@ rules: - gitrepositories - helmrepositories verbs: ["get", "list", "watch"] + - apiGroups: ["image.toolkit.fluxcd.io"] + resources: + - imagepolicies + - imagerepositories + - imageupdateautomations + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -67,4 +74,3 @@ subjects: - kind: ServiceAccount name: hermes-triage namespace: hermes - diff --git a/services/hermes/skills/triage-titan-test-failures/SKILL.md b/services/hermes/skills/triage-titan-test-failures/SKILL.md index b1600a9e6..c4a4d551c 100644 --- a/services/hermes/skills/triage-titan-test-failures/SKILL.md +++ b/services/hermes/skills/triage-titan-test-failures/SKILL.md @@ -52,6 +52,47 @@ Work in this order: 5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry. 6. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause. +For recent Git context, map the canonical suite to its Gitea repository and +read the latest commits. The current mappings are: + +- `ananke` -> `bstein/ananke` +- `ariadne` -> `bstein/Ariadne` +- `atlasbot` -> `bstein/atlasbot` +- `bstein_home` -> `bstein/bstein-dev-home` +- `data_prepper` -> `bstein/titan-iac` +- `metis` -> `bstein/metis` +- `pegasus` -> `bstein/pegasus` +- `soteria` -> `bstein/soteria` +- `titan_iac` -> `bstein/titan-iac` + +Use the read-only Gitea API and compare commit timestamps to the failing build: + +```sh +curl -fsS "$GITEA_BASE_URL/api/v1/repos///commits?limit=10" +``` + +For quality telemetry, query VictoriaMetrics directly when the bundle is stale +or a specific label needs confirmation. URL-encode PromQL rather than manually +escaping it: + +```sh +curl -G -fsS --data-urlencode \ + 'query=platform_quality_gate_runs_total{suite=""}' \ + "$VICTORIA_METRICS_URL/api/v1/query" +curl -G -fsS --data-urlencode \ + 'query=_quality_gate_checks_total' \ + "$VICTORIA_METRICS_URL/api/v1/query" +``` + +For image-pull failures, inspect the pod event, its ServiceAccount metadata, +and Flux image objects without reading referenced Secret values: + +```sh +kubectl -n describe pod +kubectl -n get serviceaccount -o yaml +kubectl get imagerepositories,imagepolicies,imageupdateautomations -A +``` + If the latest diagnosis is unavailable, the minimum acceptable result still analyzes the deterministic bundle's failed suites, build/check evidence, freshness, and environment observations. Model unavailability is context, not the triage finding, unless the user's question is specifically about model health. Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands. diff --git a/services/maintenance/ariadne-deployment.yaml b/services/maintenance/ariadne-deployment.yaml index 6b57cf858..d40b4a4ac 100644 --- a/services/maintenance/ariadne-deployment.yaml +++ b/services/maintenance/ariadne-deployment.yaml @@ -431,7 +431,7 @@ spec: - name: ARIADNE_VM_URL value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 - name: ARIADNE_CLUSTER_STATE_VM_TIMEOUT_SEC - value: "5" + value: "30" - name: ARIADNE_ALERTMANAGER_URL value: http://alertmanager.monitoring.svc.cluster.local - name: OPENSEARCH_URL