feat(hermes-triage): human-required alert rule + Ariadne autotriage config

- vmalert: hermes-triage.yaml alert group (HermesTriageHumanRequired on
  ariadne_hermes_triage_incident{status="human_required"}, 1m interval),
  wire Alertmanager notifier into vmalert-atlas-availability
- Ariadne deployment: hermes autotriage env (observe mode: autotriage on,
  autoremediation OFF), 1m schedule, allowlist hermes-triage-demo only,
  HERMES_API_KEY from optional hermes-api-server-key Secret
This commit is contained in:
jenkins 2026-08-05 16:51:48 -03:00
parent b166f22830
commit 363adedd75
2 changed files with 40 additions and 1 deletions

View File

@ -456,6 +456,30 @@ spec:
value: "45 */6 * * *"
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
value: "*/15 * * * *"
- name: ARIADNE_SCHEDULE_HERMES_AUTOTRIAGE
value: "* * * * *"
- name: ARIADNE_HERMES_AUTOTRIAGE_ENABLED
value: "true"
- name: ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST
value: hermes-triage-demo
- name: ARIADNE_HERMES_AUTOREMEDIATION_ENABLED
value: "false"
- name: ARIADNE_HERMES_ALLOWED_ACTIONS
value: repair_demo_fixture
- name: ARIADNE_HERMES_MIN_CONFIDENCE
value: "0.85"
- name: ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT
value: "1"
- name: ARIADNE_HERMES_API_URL
value: http://hermes.hermes.svc.cluster.local:8642
- name: ARIADNE_HERMES_RUN_TIMEOUT_SECONDS
value: "420"
- name: ARIADNE_HERMES_API_KEY
valueFrom:
secretKeyRef:
name: hermes-api-server-key
key: api-key
optional: true
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
value: http://hermes-model-gate.hermes.svc.cluster.local:11434
- name: ARIADNE_TESTING_TRIAGE_MODEL

View File

@ -5,6 +5,20 @@ metadata:
name: vmalert-atlas-availability-rules
namespace: monitoring
data:
hermes-triage.yaml: |
groups:
- name: hermes.triage
interval: 1m
rules:
- alert: HermesTriageHumanRequired
expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1
for: 2m
labels:
severity: warning
service: hermes-triage
annotations:
summary: "Hermes triage needs a human: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
description: "Ariadne marked incident {{ $labels.jenkins_job }}/{{ $labels.build }} as human_required (unsupported, ambiguous, or unsafe failure). Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/ - Ariadne diagnosis: GET /api/admin/testing/triage/latest or /api/admin/audit/events on ariadne.maintenance."
atlas-availability.yaml: |
groups:
- name: atlas.availability.gateway
@ -326,7 +340,7 @@ spec:
labels:
app: vmalert-atlas-availability
annotations:
bstein.dev/rules-revision: "2026-05-20-platform-quality-category-rollups"
bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
spec:
serviceAccountName: vmalert-atlas-availability
affinity:
@ -349,6 +363,7 @@ spec:
- -rule=/etc/vmalert/rules/*.yaml
- -configCheckInterval=30s
- -evaluationInterval=15m
- -notifier.url=http://alertmanager.monitoring.svc.cluster.local:9093
- -httpListenAddr=:8880
ports:
- name: http