# services/monitoring/vmalert-atlas-availability.yaml apiVersion: v1 kind: ConfigMap metadata: name: vmalert-atlas-availability-rules namespace: monitoring data: hermes-triage.yaml: | groups: - name: hermes.triage interval: 1m rules: # Escalation on its own is NOT worth an email. Every human_required # incident already files an issue in the failing service's own repo, # and that issue is the durable artifact. Mailing on every escalation # made the inbox the loudest and least useful output of the system. # Two things do deserve mail: a repair that ran and failed, and an # escalation nobody has touched in hours. - alert: HermesTriageActionFailed expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="failed"}) == 1 for: 1m labels: severity: critical service: hermes-triage annotations: summary: "Hermes repair FAILED: incident {{ $labels.jenkins_job }}/{{ $labels.build }}" description: "Ariadne authorized a remediation for {{ $labels.jenkins_job }}/{{ $labels.build }} and it did not succeed, or its rebuild failed. This is the one case where the automation acted and did not leave things better. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/" - alert: HermesTriageUnattended expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1 for: 6h labels: severity: warning service: hermes-triage annotations: summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}" description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/" atlas-availability.yaml: | groups: - name: atlas.availability.gateway interval: 1h eval_offset: 59m rules: - record: atlas:availability:requests_1h expr: | sum(increase( traefik_entrypoint_requests_total{ entrypoint="websecure", protocol="http", code=~"[1-5].." }[1h] )) labels: definition: request-v4 scope: atlas rollup: hourly - record: atlas:availability:failures_1h expr: | sum(increase( traefik_entrypoint_requests_total{ entrypoint="websecure", protocol="http", code=~"5.." }[1h] )) labels: definition: request-v4 scope: atlas rollup: hourly - name: atlas.availability.rollup interval: 1d eval_offset: 23h59m rules: - record: atlas:availability:requests_1d expr: | sum(increase( traefik_entrypoint_requests_total{ entrypoint="websecure", protocol="http", code=~"[1-5].." }[1d] )) labels: definition: request-v4 scope: atlas rollup: daily - record: atlas:availability:failures_1d expr: | sum(increase( traefik_entrypoint_requests_total{ entrypoint="websecure", protocol="http", code=~"5.." }[1d] )) labels: definition: request-v4 scope: atlas rollup: daily platform-quality.yaml: | groups: - name: platform.quality interval: 5m rules: - record: platform_quality:test_case_status:count_1h expr: | sum by (suite, branch, test, category, status) ( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"} or label_replace( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=""}, "category", "uncategorized", "__name__", ".*" ) ) labels: rollup: hourly - record: platform_quality:test_case_pass_rate:percent_1h expr: | 100 * ( ( sum by (suite, branch, test, category) ( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status="passed"} or label_replace( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status="passed"}, "category", "uncategorized", "__name__", ".*" ) ) or on(suite, branch, test, category) ( 0 * sum by (suite, branch, test, category) ( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"} or label_replace( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"}, "category", "uncategorized", "__name__", ".*" ) ) ) ) ) / clamp_min( sum by (suite, branch, test, category) ( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"} or label_replace( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"}, "category", "uncategorized", "__name__", ".*" ) ), 1 ) labels: rollup: hourly - record: platform_quality:test_case_health_rate:percent_1h expr: | 100 * ( ( sum by (suite, branch, test, category) ( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|skipped|not_applicable|na|n/a"} or label_replace( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|skipped|not_applicable|na|n/a"}, "category", "uncategorized", "__name__", ".*" ) ) or on(suite, branch, test, category) ( 0 * sum by (suite, branch, test, category) ( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"} or label_replace( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}, "category", "uncategorized", "__name__", ".*" ) ) ) ) ) / clamp_min( sum by (suite, branch, test, category) ( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"} or label_replace( platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}, "category", "uncategorized", "__name__", ".*" ) ), 1 ) labels: rollup: hourly - record: platform_quality:test_category_health_rate:percent_1h expr: | avg by (suite, branch, category) ( platform_quality:test_case_health_rate:percent_1h{suite!="",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"} ) labels: rollup: hourly - record: platform_quality:suite_runs:increase_24h expr: | sum by (suite, branch, status) ( ( increase(( max without(instance, job) ( platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch!="",status!=""} ) )[24h:1m]) ) or ( increase(( max without(instance, job) ( platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch="",status!=""} ) )[24h:1m]) * on (suite) group_left(branch) max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""}) ) ) labels: rollup: hourly - record: platform_quality:suite_coverage_percent:latest_1h expr: | max by (suite, branch) ( platform_quality_gate_workspace_line_coverage_percent{exported_job="platform-quality-ci",suite!=""} ) or on (suite, branch) max by (suite, branch) ( {__name__=~".*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!=""} ) labels: rollup: hourly - record: platform_quality:suite_source_files_total:latest_1h expr: | max by (suite, branch) ( platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!=""} ) labels: rollup: hourly - record: platform_quality:suite_source_lines_over_500_total:latest_1h expr: | max by (suite, branch) ( platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!=""} ) labels: rollup: hourly - record: platform_quality:check_status:present_1h expr: | sum by (suite, branch, check, status) ( {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",status!=""} or label_replace( {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",result!="",status=""}, "status", "$1", "result", "(.*)" ) or ( {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""} * on (suite) group_left(branch) max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""}) ) or ( label_replace( {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""}, "status", "$1", "result", "(.*)" ) * on (suite) group_left(branch) max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""}) ) or ( label_replace( {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""}, "branch", "unknown", "__name__", ".*" ) unless on (suite) max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""}) ) or ( label_replace( label_replace( {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""}, "status", "$1", "result", "(.*)" ), "branch", "unknown", "__name__", ".*" ) unless on (suite) max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""}) ) ) labels: rollup: hourly - record: platform_quality:check_seen_flag:present_1h expr: | clamp_max( max by (suite, branch, check) ( platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!=""} > 0 ), 1 ) labels: rollup: hourly - record: platform_quality:check_failed_flag:present_1h expr: | clamp_max( max by (suite, branch, check) ( platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0 ), 1 ) or on (suite, branch, check) (0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""}) labels: rollup: hourly - record: platform_quality:check_healthy_flag:present_1h expr: | ( clamp_max( max by (suite, branch, check) ( platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status=~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0 ), 1 ) unless on (suite, branch, check) (platform_quality:check_failed_flag:present_1h{suite!="",branch!="",check!=""} > 0) ) or on (suite, branch, check) (0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""}) labels: rollup: hourly - record: platform_quality:sonar_gate_health_percent:latest_1h expr: | 100 * max by (project_key) ( sonarqube_project_quality_gate_pass{project_key!=""} ) labels: rollup: hourly --- apiVersion: v1 kind: ServiceAccount metadata: name: vmalert-atlas-availability namespace: monitoring --- apiVersion: apps/v1 kind: Deployment metadata: name: vmalert-atlas-availability namespace: monitoring labels: app: vmalert-atlas-availability spec: replicas: 1 revisionHistoryLimit: 3 selector: matchLabels: app: vmalert-atlas-availability template: metadata: labels: app: vmalert-atlas-availability annotations: bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert" spec: serviceAccountName: vmalert-atlas-availability affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: NotIn values: - titan-22 - titan-24 containers: - name: vmalert image: victoriametrics/vmalert:v1.113.0 args: - -datasource.url=http://victoria-metrics-single-server:8428 - -datasource.queryStep=1h - -remoteWrite.url=http://victoria-metrics-single-server:8428 - -rule=/etc/vmalert/rules/*.yaml - -configCheckInterval=30s - -evaluationInterval=15m - -notifier.url=http://alertmanager.monitoring.svc.cluster.local:9093 - -httpListenAddr=:8880 ports: - name: http containerPort: 8880 readinessProbe: tcpSocket: port: http initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: tcpSocket: port: http initialDelaySeconds: 20 periodSeconds: 30 resources: requests: cpu: 25m memory: 64Mi limits: cpu: 500m memory: 256Mi volumeMounts: - name: rules mountPath: /etc/vmalert/rules readOnly: true volumes: - name: rules configMap: name: vmalert-atlas-availability-rules --- apiVersion: v1 kind: Service metadata: name: vmalert-atlas-availability namespace: monitoring annotations: prometheus.io/scrape: "true" prometheus.io/port: "8880" spec: selector: app: vmalert-atlas-availability ports: - name: http port: 8880 targetPort: http