feat(hermes-triage): human-required alert rule + Ariadne autotriage config
- vmalert: hermes-triage.yaml alert group (HermesTriageHumanRequired on
ariadne_hermes_triage_incident{status="human_required"}, 1m interval),
wire Alertmanager notifier into vmalert-atlas-availability
- Ariadne deployment: hermes autotriage env (observe mode: autotriage on,
autoremediation OFF), 1m schedule, allowlist hermes-triage-demo only,
HERMES_API_KEY from optional hermes-api-server-key Secret
This commit is contained in:
parent
b166f22830
commit
363adedd75
@ -456,6 +456,30 @@ spec:
|
|||||||
value: "45 */6 * * *"
|
value: "45 */6 * * *"
|
||||||
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
|
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
|
||||||
value: "*/15 * * * *"
|
value: "*/15 * * * *"
|
||||||
|
- name: ARIADNE_SCHEDULE_HERMES_AUTOTRIAGE
|
||||||
|
value: "* * * * *"
|
||||||
|
- name: ARIADNE_HERMES_AUTOTRIAGE_ENABLED
|
||||||
|
value: "true"
|
||||||
|
- name: ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST
|
||||||
|
value: hermes-triage-demo
|
||||||
|
- name: ARIADNE_HERMES_AUTOREMEDIATION_ENABLED
|
||||||
|
value: "false"
|
||||||
|
- name: ARIADNE_HERMES_ALLOWED_ACTIONS
|
||||||
|
value: repair_demo_fixture
|
||||||
|
- name: ARIADNE_HERMES_MIN_CONFIDENCE
|
||||||
|
value: "0.85"
|
||||||
|
- name: ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT
|
||||||
|
value: "1"
|
||||||
|
- name: ARIADNE_HERMES_API_URL
|
||||||
|
value: http://hermes.hermes.svc.cluster.local:8642
|
||||||
|
- name: ARIADNE_HERMES_RUN_TIMEOUT_SECONDS
|
||||||
|
value: "420"
|
||||||
|
- name: ARIADNE_HERMES_API_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: hermes-api-server-key
|
||||||
|
key: api-key
|
||||||
|
optional: true
|
||||||
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
|
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
|
||||||
value: http://hermes-model-gate.hermes.svc.cluster.local:11434
|
value: http://hermes-model-gate.hermes.svc.cluster.local:11434
|
||||||
- name: ARIADNE_TESTING_TRIAGE_MODEL
|
- name: ARIADNE_TESTING_TRIAGE_MODEL
|
||||||
|
|||||||
@ -5,6 +5,20 @@ metadata:
|
|||||||
name: vmalert-atlas-availability-rules
|
name: vmalert-atlas-availability-rules
|
||||||
namespace: monitoring
|
namespace: monitoring
|
||||||
data:
|
data:
|
||||||
|
hermes-triage.yaml: |
|
||||||
|
groups:
|
||||||
|
- name: hermes.triage
|
||||||
|
interval: 1m
|
||||||
|
rules:
|
||||||
|
- alert: HermesTriageHumanRequired
|
||||||
|
expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: hermes-triage
|
||||||
|
annotations:
|
||||||
|
summary: "Hermes triage needs a human: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
|
||||||
|
description: "Ariadne marked incident {{ $labels.jenkins_job }}/{{ $labels.build }} as human_required (unsupported, ambiguous, or unsafe failure). Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/ - Ariadne diagnosis: GET /api/admin/testing/triage/latest or /api/admin/audit/events on ariadne.maintenance."
|
||||||
atlas-availability.yaml: |
|
atlas-availability.yaml: |
|
||||||
groups:
|
groups:
|
||||||
- name: atlas.availability.gateway
|
- name: atlas.availability.gateway
|
||||||
@ -326,7 +340,7 @@ spec:
|
|||||||
labels:
|
labels:
|
||||||
app: vmalert-atlas-availability
|
app: vmalert-atlas-availability
|
||||||
annotations:
|
annotations:
|
||||||
bstein.dev/rules-revision: "2026-05-20-platform-quality-category-rollups"
|
bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
|
||||||
spec:
|
spec:
|
||||||
serviceAccountName: vmalert-atlas-availability
|
serviceAccountName: vmalert-atlas-availability
|
||||||
affinity:
|
affinity:
|
||||||
@ -349,6 +363,7 @@ spec:
|
|||||||
- -rule=/etc/vmalert/rules/*.yaml
|
- -rule=/etc/vmalert/rules/*.yaml
|
||||||
- -configCheckInterval=30s
|
- -configCheckInterval=30s
|
||||||
- -evaluationInterval=15m
|
- -evaluationInterval=15m
|
||||||
|
- -notifier.url=http://alertmanager.monitoring.svc.cluster.local:9093
|
||||||
- -httpListenAddr=:8880
|
- -httpListenAddr=:8880
|
||||||
ports:
|
ports:
|
||||||
- name: http
|
- name: http
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user