feat(hermes-triage): human-required alert rule + Ariadne autotriage config
- vmalert: hermes-triage.yaml alert group (HermesTriageHumanRequired on
ariadne_hermes_triage_incident{status="human_required"}, 1m interval),
wire Alertmanager notifier into vmalert-atlas-availability
- Ariadne deployment: hermes autotriage env (observe mode: autotriage on,
autoremediation OFF), 1m schedule, allowlist hermes-triage-demo only,
HERMES_API_KEY from optional hermes-api-server-key Secret
This commit is contained in:
parent
b166f22830
commit
363adedd75
@ -456,6 +456,30 @@ spec:
|
||||
value: "45 */6 * * *"
|
||||
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
|
||||
value: "*/15 * * * *"
|
||||
- name: ARIADNE_SCHEDULE_HERMES_AUTOTRIAGE
|
||||
value: "* * * * *"
|
||||
- name: ARIADNE_HERMES_AUTOTRIAGE_ENABLED
|
||||
value: "true"
|
||||
- name: ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST
|
||||
value: hermes-triage-demo
|
||||
- name: ARIADNE_HERMES_AUTOREMEDIATION_ENABLED
|
||||
value: "false"
|
||||
- name: ARIADNE_HERMES_ALLOWED_ACTIONS
|
||||
value: repair_demo_fixture
|
||||
- name: ARIADNE_HERMES_MIN_CONFIDENCE
|
||||
value: "0.85"
|
||||
- name: ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT
|
||||
value: "1"
|
||||
- name: ARIADNE_HERMES_API_URL
|
||||
value: http://hermes.hermes.svc.cluster.local:8642
|
||||
- name: ARIADNE_HERMES_RUN_TIMEOUT_SECONDS
|
||||
value: "420"
|
||||
- name: ARIADNE_HERMES_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hermes-api-server-key
|
||||
key: api-key
|
||||
optional: true
|
||||
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
|
||||
value: http://hermes-model-gate.hermes.svc.cluster.local:11434
|
||||
- name: ARIADNE_TESTING_TRIAGE_MODEL
|
||||
|
||||
@ -5,6 +5,20 @@ metadata:
|
||||
name: vmalert-atlas-availability-rules
|
||||
namespace: monitoring
|
||||
data:
|
||||
hermes-triage.yaml: |
|
||||
groups:
|
||||
- name: hermes.triage
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: HermesTriageHumanRequired
|
||||
expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
service: hermes-triage
|
||||
annotations:
|
||||
summary: "Hermes triage needs a human: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
|
||||
description: "Ariadne marked incident {{ $labels.jenkins_job }}/{{ $labels.build }} as human_required (unsupported, ambiguous, or unsafe failure). Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/ - Ariadne diagnosis: GET /api/admin/testing/triage/latest or /api/admin/audit/events on ariadne.maintenance."
|
||||
atlas-availability.yaml: |
|
||||
groups:
|
||||
- name: atlas.availability.gateway
|
||||
@ -326,7 +340,7 @@ spec:
|
||||
labels:
|
||||
app: vmalert-atlas-availability
|
||||
annotations:
|
||||
bstein.dev/rules-revision: "2026-05-20-platform-quality-category-rollups"
|
||||
bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
|
||||
spec:
|
||||
serviceAccountName: vmalert-atlas-availability
|
||||
affinity:
|
||||
@ -349,6 +363,7 @@ spec:
|
||||
- -rule=/etc/vmalert/rules/*.yaml
|
||||
- -configCheckInterval=30s
|
||||
- -evaluationInterval=15m
|
||||
- -notifier.url=http://alertmanager.monitoring.svc.cluster.local:9093
|
||||
- -httpListenAddr=:8880
|
||||
ports:
|
||||
- name: http
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user