monitoring(alerts): watch soteria authz denial spikes

services/monitoring/grafana-alerting-config.yaml

@@ -543,6 +543,54 @@ data:
               summary: "One or more PVCs are stale, missing, or failed per Soteria backup health"
             labels:
               severity: warning
+          - uid: maint-soteria-authz-denials
+            title: "Soteria authorization denials elevated"
+            condition: C
+            for: "10m"
+            data:
+              - refId: A
+                relativeTimeRange:
+                  from: 900
+                  to: 0
+                datasourceUid: atlas-vm
+                model:
+                  expr: sum(increase(soteria_authz_denials_total[15m])) or on() vector(0)
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  legendFormat: soteria-authz-denials-15m
+                  datasource:
+                    type: prometheus
+                    uid: atlas-vm
+              - refId: B
+                datasourceUid: __expr__
+                model:
+                  expression: A
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  reducer: last
+                  type: reduce
+              - refId: C
+                datasourceUid: __expr__
+                model:
+                  expression: B
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  type: threshold
+                  conditions:
+                    - evaluator:
+                        params: [10]
+                        type: gt
+                      operator:
+                        type: and
+                      reducer:
+                        type: last
+                      type: query
+            noDataState: OK
+            execErrState: Alerting
+            annotations:
+              summary: "Soteria saw >10 authorization denials in 15m"
+            labels:
+              severity: warning
       - orgId: 1
         name: ariadne
         folder: Alerts