diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index 2c2bdaed8..8d8a3c3b5 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -10,15 +10,30 @@ data: - name: hermes.triage interval: 1m rules: - - alert: HermesTriageHumanRequired + # Escalation on its own is NOT worth an email. Every human_required + # incident already files an issue in the failing service's own repo, + # and that issue is the durable artifact. Mailing on every escalation + # made the inbox the loudest and least useful output of the system. + # Two things do deserve mail: a repair that ran and failed, and an + # escalation nobody has touched in hours. + - alert: HermesTriageActionFailed + expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="failed"}) == 1 + for: 1m + labels: + severity: critical + service: hermes-triage + annotations: + summary: "Hermes repair FAILED: incident {{ $labels.jenkins_job }}/{{ $labels.build }}" + description: "Ariadne authorized a remediation for {{ $labels.jenkins_job }}/{{ $labels.build }} and it did not succeed, or its rebuild failed. This is the one case where the automation acted and did not leave things better. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/" + - alert: HermesTriageUnattended expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1 - for: 2m + for: 6h labels: severity: warning service: hermes-triage annotations: - summary: "Hermes triage needs a human: incident {{ $labels.jenkins_job }}/{{ $labels.build }}" - description: "Ariadne marked incident {{ $labels.jenkins_job }}/{{ $labels.build }} as human_required (unsupported, ambiguous, or unsafe failure). Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/ - Ariadne diagnosis: GET /api/admin/testing/triage/latest or /api/admin/audit/events on ariadne.maintenance." + summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}" + description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/" atlas-availability.yaml: | groups: - name: atlas.availability.gateway