e3e40f4618
Every human_required escalation already files an issue in the failing service's own repository, and mailing on each one made the inbox the loudest and least informative output of the system. Replace the blanket alert with two narrow ones: a repair that ran and failed, which is the only case where the automation acted and left things no better, and an escalation still untouched after six hours. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
429 lines
19 KiB
YAML
429 lines
19 KiB
YAML
# services/monitoring/vmalert-atlas-availability.yaml
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: vmalert-atlas-availability-rules
|
|
namespace: monitoring
|
|
data:
|
|
hermes-triage.yaml: |
|
|
groups:
|
|
- name: hermes.triage
|
|
interval: 1m
|
|
rules:
|
|
# Escalation on its own is NOT worth an email. Every human_required
|
|
# incident already files an issue in the failing service's own repo,
|
|
# and that issue is the durable artifact. Mailing on every escalation
|
|
# made the inbox the loudest and least useful output of the system.
|
|
# Two things do deserve mail: a repair that ran and failed, and an
|
|
# escalation nobody has touched in hours.
|
|
- alert: HermesTriageActionFailed
|
|
expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="failed"}) == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
service: hermes-triage
|
|
annotations:
|
|
summary: "Hermes repair FAILED: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
|
|
description: "Ariadne authorized a remediation for {{ $labels.jenkins_job }}/{{ $labels.build }} and it did not succeed, or its rebuild failed. This is the one case where the automation acted and did not leave things better. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
|
|
- alert: HermesTriageUnattended
|
|
expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1
|
|
for: 6h
|
|
labels:
|
|
severity: warning
|
|
service: hermes-triage
|
|
annotations:
|
|
summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
|
|
description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
|
|
atlas-availability.yaml: |
|
|
groups:
|
|
- name: atlas.availability.gateway
|
|
interval: 1h
|
|
eval_offset: 59m
|
|
rules:
|
|
- record: atlas:availability:requests_1h
|
|
expr: |
|
|
sum(increase(
|
|
traefik_entrypoint_requests_total{
|
|
entrypoint="websecure",
|
|
protocol="http",
|
|
code=~"[1-5].."
|
|
}[1h]
|
|
))
|
|
labels:
|
|
definition: request-v4
|
|
scope: atlas
|
|
rollup: hourly
|
|
- record: atlas:availability:failures_1h
|
|
expr: |
|
|
sum(increase(
|
|
traefik_entrypoint_requests_total{
|
|
entrypoint="websecure",
|
|
protocol="http",
|
|
code=~"5.."
|
|
}[1h]
|
|
))
|
|
labels:
|
|
definition: request-v4
|
|
scope: atlas
|
|
rollup: hourly
|
|
- name: atlas.availability.rollup
|
|
interval: 1d
|
|
eval_offset: 23h59m
|
|
rules:
|
|
- record: atlas:availability:requests_1d
|
|
expr: |
|
|
sum(increase(
|
|
traefik_entrypoint_requests_total{
|
|
entrypoint="websecure",
|
|
protocol="http",
|
|
code=~"[1-5].."
|
|
}[1d]
|
|
))
|
|
labels:
|
|
definition: request-v4
|
|
scope: atlas
|
|
rollup: daily
|
|
- record: atlas:availability:failures_1d
|
|
expr: |
|
|
sum(increase(
|
|
traefik_entrypoint_requests_total{
|
|
entrypoint="websecure",
|
|
protocol="http",
|
|
code=~"5.."
|
|
}[1d]
|
|
))
|
|
labels:
|
|
definition: request-v4
|
|
scope: atlas
|
|
rollup: daily
|
|
platform-quality.yaml: |
|
|
groups:
|
|
- name: platform.quality
|
|
interval: 5m
|
|
rules:
|
|
- record: platform_quality:test_case_status:count_1h
|
|
expr: |
|
|
sum by (suite, branch, test, category, status) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=""},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:test_case_pass_rate:percent_1h
|
|
expr: |
|
|
100 * (
|
|
(
|
|
sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status="passed"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status="passed"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
or on(suite, branch, test, category)
|
|
(
|
|
0 * sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
/
|
|
clamp_min(
|
|
sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
),
|
|
1
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:test_case_health_rate:percent_1h
|
|
expr: |
|
|
100 * (
|
|
(
|
|
sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|skipped|not_applicable|na|n/a"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|skipped|not_applicable|na|n/a"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
or on(suite, branch, test, category)
|
|
(
|
|
0 * sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
/
|
|
clamp_min(
|
|
sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
),
|
|
1
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:test_category_health_rate:percent_1h
|
|
expr: |
|
|
avg by (suite, branch, category) (
|
|
platform_quality:test_case_health_rate:percent_1h{suite!="",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"}
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:suite_runs:increase_24h
|
|
expr: |
|
|
sum by (suite, branch, status) (
|
|
(
|
|
increase((
|
|
max without(instance, job) (
|
|
platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch!="",status!=""}
|
|
)
|
|
)[24h:1m])
|
|
)
|
|
or (
|
|
increase((
|
|
max without(instance, job) (
|
|
platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch="",status!=""}
|
|
)
|
|
)[24h:1m])
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:suite_coverage_percent:latest_1h
|
|
expr: |
|
|
max by (suite, branch) (
|
|
platform_quality_gate_workspace_line_coverage_percent{exported_job="platform-quality-ci",suite!=""}
|
|
)
|
|
or on (suite, branch)
|
|
max by (suite, branch) (
|
|
{__name__=~".*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!=""}
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:suite_source_files_total:latest_1h
|
|
expr: |
|
|
max by (suite, branch) (
|
|
platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!=""}
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:suite_source_lines_over_500_total:latest_1h
|
|
expr: |
|
|
max by (suite, branch) (
|
|
platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!=""}
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:check_status:present_1h
|
|
expr: |
|
|
sum by (suite, branch, check, status) (
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",status!=""}
|
|
or label_replace(
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",result!="",status=""},
|
|
"status", "$1", "result", "(.*)"
|
|
)
|
|
or (
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""}
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or (
|
|
label_replace(
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""},
|
|
"status", "$1", "result", "(.*)"
|
|
)
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or (
|
|
label_replace(
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""},
|
|
"branch", "unknown", "__name__", ".*"
|
|
)
|
|
unless on (suite)
|
|
max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or (
|
|
label_replace(
|
|
label_replace(
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""},
|
|
"status", "$1", "result", "(.*)"
|
|
),
|
|
"branch", "unknown", "__name__", ".*"
|
|
)
|
|
unless on (suite)
|
|
max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:check_seen_flag:present_1h
|
|
expr: |
|
|
clamp_max(
|
|
max by (suite, branch, check) (
|
|
platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!=""} > 0
|
|
),
|
|
1
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:check_failed_flag:present_1h
|
|
expr: |
|
|
clamp_max(
|
|
max by (suite, branch, check) (
|
|
platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0
|
|
),
|
|
1
|
|
)
|
|
or on (suite, branch, check)
|
|
(0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""})
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:check_healthy_flag:present_1h
|
|
expr: |
|
|
(
|
|
clamp_max(
|
|
max by (suite, branch, check) (
|
|
platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status=~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0
|
|
),
|
|
1
|
|
)
|
|
unless on (suite, branch, check)
|
|
(platform_quality:check_failed_flag:present_1h{suite!="",branch!="",check!=""} > 0)
|
|
)
|
|
or on (suite, branch, check)
|
|
(0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""})
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:sonar_gate_health_percent:latest_1h
|
|
expr: |
|
|
100 * max by (project_key) (
|
|
sonarqube_project_quality_gate_pass{project_key!=""}
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
|
|
---
|
|
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: vmalert-atlas-availability
|
|
namespace: monitoring
|
|
|
|
---
|
|
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vmalert-atlas-availability
|
|
namespace: monitoring
|
|
labels:
|
|
app: vmalert-atlas-availability
|
|
spec:
|
|
replicas: 1
|
|
revisionHistoryLimit: 3
|
|
selector:
|
|
matchLabels:
|
|
app: vmalert-atlas-availability
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: vmalert-atlas-availability
|
|
annotations:
|
|
bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
|
|
spec:
|
|
serviceAccountName: vmalert-atlas-availability
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: kubernetes.io/hostname
|
|
operator: NotIn
|
|
values:
|
|
- titan-22
|
|
- titan-24
|
|
containers:
|
|
- name: vmalert
|
|
image: victoriametrics/vmalert:v1.113.0
|
|
args:
|
|
- -datasource.url=http://victoria-metrics-single-server:8428
|
|
- -datasource.queryStep=1h
|
|
- -remoteWrite.url=http://victoria-metrics-single-server:8428
|
|
- -rule=/etc/vmalert/rules/*.yaml
|
|
- -configCheckInterval=30s
|
|
- -evaluationInterval=15m
|
|
- -notifier.url=http://alertmanager.monitoring.svc.cluster.local:9093
|
|
- -httpListenAddr=:8880
|
|
ports:
|
|
- name: http
|
|
containerPort: 8880
|
|
readinessProbe:
|
|
tcpSocket:
|
|
port: http
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
tcpSocket:
|
|
port: http
|
|
initialDelaySeconds: 20
|
|
periodSeconds: 30
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
volumeMounts:
|
|
- name: rules
|
|
mountPath: /etc/vmalert/rules
|
|
readOnly: true
|
|
volumes:
|
|
- name: rules
|
|
configMap:
|
|
name: vmalert-atlas-availability-rules
|
|
|
|
---
|
|
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: vmalert-atlas-availability
|
|
namespace: monitoring
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "8880"
|
|
spec:
|
|
selector:
|
|
app: vmalert-atlas-availability
|
|
ports:
|
|
- name: http
|
|
port: 8880
|
|
targetPort: http
|