From e3e40f4618046caead393d839c25d8d4779b7d92 Mon Sep 17 00:00:00 2001
From: jenkins <jenkins@bstein.dev>
Date: Thu, 6 Aug 2026 00:26:30 -0300
Subject: [PATCH] feat(monitoring): make Hermes triage email mean something

Every human_required escalation already files an issue in the failing
service's own repository, and mailing on each one made the inbox the loudest
and least informative output of the system. Replace the blanket alert with
two narrow ones: a repair that ran and failed, which is the only case where
the automation acted and left things no better, and an escalation still
untouched after six hours.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 .../vmalert-atlas-availability.yaml           | 23 +++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml
index 2c2bdaed8..8d8a3c3b5 100644
--- a/services/monitoring/vmalert-atlas-availability.yaml
+++ b/services/monitoring/vmalert-atlas-availability.yaml
@@ -10,15 +10,30 @@ data:
       - name: hermes.triage
         interval: 1m
         rules:
-          - alert: HermesTriageHumanRequired
+          # Escalation on its own is NOT worth an email. Every human_required
+          # incident already files an issue in the failing service's own repo,
+          # and that issue is the durable artifact. Mailing on every escalation
+          # made the inbox the loudest and least useful output of the system.
+          # Two things do deserve mail: a repair that ran and failed, and an
+          # escalation nobody has touched in hours.
+          - alert: HermesTriageActionFailed
+            expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="failed"}) == 1
+            for: 1m
+            labels:
+              severity: critical
+              service: hermes-triage
+            annotations:
+              summary: "Hermes repair FAILED: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
+              description: "Ariadne authorized a remediation for {{ $labels.jenkins_job }}/{{ $labels.build }} and it did not succeed, or its rebuild failed. This is the one case where the automation acted and did not leave things better. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
+          - alert: HermesTriageUnattended
             expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1
-            for: 2m
+            for: 6h
             labels:
               severity: warning
               service: hermes-triage
             annotations:
-              summary: "Hermes triage needs a human: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
-              description: "Ariadne marked incident {{ $labels.jenkins_job }}/{{ $labels.build }} as human_required (unsupported, ambiguous, or unsafe failure). Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/ - Ariadne diagnosis: GET /api/admin/testing/triage/latest or /api/admin/audit/events on ariadne.maintenance."
+              summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
+              description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
   atlas-availability.yaml: |
     groups:
       - name: atlas.availability.gateway