titan-iac/services/monitoring/vmalert-atlas-availability.yaml

# services/monitoring/vmalert-atlas-availability.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: vmalert-atlas-availability-rules
  namespace: monitoring
data:
  hermes-triage.yaml: |
    groups:
      - name: hermes.triage
        interval: 1m
        rules:
          # Escalation on its own is NOT worth an email. Every human_required
          # incident already files an issue in the failing service's own repo,
          # and that issue is the durable artifact. Mailing on every escalation
          # made the inbox the loudest and least useful output of the system.
          # Two things do deserve mail: a repair that ran and failed, and an
          # escalation nobody has touched in hours.
          - alert: HermesTriageActionFailed
            expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="failed"}) == 1
            for: 1m
            labels:
              severity: critical
              service: hermes-triage
            annotations:
              summary: "Hermes repair FAILED: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
              description: "Ariadne authorized a remediation for {{ $labels.jenkins_job }}/{{ $labels.build }} and it did not succeed, or its rebuild failed. This is the one case where the automation acted and did not leave things better. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
          - alert: HermesTriageUnattended
            expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1
            for: 6h
            labels:
              severity: warning
              service: hermes-triage
            annotations:
              summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
              description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
  atlas-availability.yaml: |
    groups:
      - name: atlas.availability.gateway
        interval: 1h
        eval_offset: 59m
        rules:
          - record: atlas:availability:requests_1h
            expr: |
              sum(increase(
                traefik_entrypoint_requests_total{
                  entrypoint="websecure",
                  protocol="http",
                  code=~"[1-5].."
                }[1h]
              ))
            labels:
              definition: request-v4
              scope: atlas
              rollup: hourly
          - record: atlas:availability:failures_1h
            expr: |
              sum(increase(
                traefik_entrypoint_requests_total{
                  entrypoint="websecure",
                  protocol="http",
                  code=~"5.."
                }[1h]
              ))
            labels:
              definition: request-v4
              scope: atlas
              rollup: hourly
      - name: atlas.availability.rollup
        interval: 1d
        eval_offset: 23h59m
        rules:
          - record: atlas:availability:requests_1d
            expr: |
              sum(increase(
                traefik_entrypoint_requests_total{
                  entrypoint="websecure",
                  protocol="http",
                  code=~"[1-5].."
                }[1d]
              ))
            labels:
              definition: request-v4
              scope: atlas
              rollup: daily
          - record: atlas:availability:failures_1d
            expr: |
              sum(increase(
                traefik_entrypoint_requests_total{
                  entrypoint="websecure",
                  protocol="http",
                  code=~"5.."
                }[1d]
              ))
            labels:
              definition: request-v4
              scope: atlas
              rollup: daily
  platform-quality.yaml: |
    groups:
      - name: platform.quality
        interval: 5m
        rules:
          - record: platform_quality:test_case_status:count_1h
            expr: |
              sum by (suite, branch, test, category, status) (
                platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"}
                or label_replace(
                  platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=""},
                  "category", "uncategorized", "__name__", ".*"
                )
              )
            labels:
              rollup: hourly
          - record: platform_quality:test_case_pass_rate:percent_1h
            expr: |
              100 * (
                (
                  sum by (suite, branch, test, category) (
                    platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status="passed"}
                    or label_replace(
                      platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status="passed"},
                      "category", "uncategorized", "__name__", ".*"
                    )
                  )
                  or on(suite, branch, test, category)
                  (
                    0 * sum by (suite, branch, test, category) (
                      platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"}
                      or label_replace(
                        platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"},
                        "category", "uncategorized", "__name__", ".*"
                      )
                    )
                  )
                )
              )
              /
              clamp_min(
                sum by (suite, branch, test, category) (
                  platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"}
                  or label_replace(
                    platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"},
                    "category", "uncategorized", "__name__", ".*"
                  )
                ),
                1
              )
            labels:
              rollup: hourly
          - record: platform_quality:test_case_health_rate:percent_1h
            expr: |
              100 * (
                (
                  sum by (suite, branch, test, category) (
                    platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|skipped|not_applicable|na|n/a"}
                    or label_replace(
                      platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|skipped|not_applicable|na|n/a"},
                      "category", "uncategorized", "__name__", ".*"
                    )
                  )
                  or on(suite, branch, test, category)
                  (
                    0 * sum by (suite, branch, test, category) (
                      platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}
                      or label_replace(
                        platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"},
                        "category", "uncategorized", "__name__", ".*"
                      )
                    )
                  )
                )
              )
              /
              clamp_min(
                sum by (suite, branch, test, category) (
                  platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}
                  or label_replace(
                    platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"},
                    "category", "uncategorized", "__name__", ".*"
                  )
                ),
                1
              )
            labels:
              rollup: hourly
          - record: platform_quality:test_category_health_rate:percent_1h
            expr: |
              avg by (suite, branch, category) (
                platform_quality:test_case_health_rate:percent_1h{suite!="",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"}
              )
            labels:
              rollup: hourly
          - record: platform_quality:suite_runs:increase_24h
            expr: |
              sum by (suite, branch, status) (
                (
                  increase((
                    max without(instance, job) (
                      platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch!="",status!=""}
                    )
                  )[24h:1m])
                )
                or (
                  increase((
                    max without(instance, job) (
                      platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch="",status!=""}
                    )
                  )[24h:1m])
                  * on (suite) group_left(branch)
                    max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
              )
            labels:
              rollup: hourly
          - record: platform_quality:suite_coverage_percent:latest_1h
            expr: |
              max by (suite, branch) (
                platform_quality_gate_workspace_line_coverage_percent{exported_job="platform-quality-ci",suite!=""}
              )
              or on (suite, branch)
              max by (suite, branch) (
                {__name__=~".*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!=""}
              )
            labels:
              rollup: hourly
          - record: platform_quality:suite_source_files_total:latest_1h
            expr: |
              max by (suite, branch) (
                platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!=""}
              )
            labels:
              rollup: hourly
          - record: platform_quality:suite_source_lines_over_500_total:latest_1h
            expr: |
              max by (suite, branch) (
                platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!=""}
              )
            labels:
              rollup: hourly
          - record: platform_quality:check_status:present_1h
            expr: |
              sum by (suite, branch, check, status) (
                {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",status!=""}
                or label_replace(
                  {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",result!="",status=""},
                  "status", "$1", "result", "(.*)"
                )
                or (
                  {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""}
                  * on (suite) group_left(branch)
                    max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
                or (
                  label_replace(
                    {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""},
                    "status", "$1", "result", "(.*)"
                  )
                  * on (suite) group_left(branch)
                    max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
                or (
                  label_replace(
                    {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""},
                    "branch", "unknown", "__name__", ".*"
                  )
                  unless on (suite)
                    max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
                or (
                  label_replace(
                    label_replace(
                      {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""},
                      "status", "$1", "result", "(.*)"
                    ),
                    "branch", "unknown", "__name__", ".*"
                  )
                  unless on (suite)
                    max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
              )
            labels:
              rollup: hourly
          - record: platform_quality:check_seen_flag:present_1h
            expr: |
              clamp_max(
                max by (suite, branch, check) (
                  platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!=""} > 0
                ),
                1
              )
            labels:
              rollup: hourly
          - record: platform_quality:check_failed_flag:present_1h
            expr: |
              clamp_max(
                max by (suite, branch, check) (
                  platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0
                ),
                1
              )
              or on (suite, branch, check)
              (0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""})
            labels:
              rollup: hourly
          - record: platform_quality:check_healthy_flag:present_1h
            expr: |
              (
                clamp_max(
                  max by (suite, branch, check) (
                    platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status=~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0
                  ),
                  1
                )
                unless on (suite, branch, check)
                (platform_quality:check_failed_flag:present_1h{suite!="",branch!="",check!=""} > 0)
              )
              or on (suite, branch, check)
              (0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""})
            labels:
              rollup: hourly
          - record: platform_quality:sonar_gate_health_percent:latest_1h
            expr: |
              100 * max by (project_key) (
                sonarqube_project_quality_gate_pass{project_key!=""}
              )
            labels:
              rollup: hourly

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: vmalert-atlas-availability
  namespace: monitoring

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vmalert-atlas-availability
  namespace: monitoring
  labels:
    app: vmalert-atlas-availability
spec:
  replicas: 1
  revisionHistoryLimit: 3
  selector:
    matchLabels:
      app: vmalert-atlas-availability
  template:
    metadata:
      labels:
        app: vmalert-atlas-availability
      annotations:
        bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
    spec:
      serviceAccountName: vmalert-atlas-availability
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/hostname
                    operator: NotIn
                    values:
                      - titan-22
                      - titan-24
      containers:
        - name: vmalert
          image: victoriametrics/vmalert:v1.113.0
          args:
            - -datasource.url=http://victoria-metrics-single-server:8428
            - -datasource.queryStep=1h
            - -remoteWrite.url=http://victoria-metrics-single-server:8428
            - -rule=/etc/vmalert/rules/*.yaml
            - -configCheckInterval=30s
            - -evaluationInterval=15m
            - -notifier.url=http://alertmanager.monitoring.svc.cluster.local:9093
            - -httpListenAddr=:8880
          ports:
            - name: http
              containerPort: 8880
          readinessProbe:
            tcpSocket:
              port: http
            initialDelaySeconds: 5
            periodSeconds: 10
          livenessProbe:
            tcpSocket:
              port: http
            initialDelaySeconds: 20
            periodSeconds: 30
          resources:
            requests:
              cpu: 25m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 256Mi
          volumeMounts:
            - name: rules
              mountPath: /etc/vmalert/rules
              readOnly: true
      volumes:
        - name: rules
          configMap:
            name: vmalert-atlas-availability-rules

---

apiVersion: v1
kind: Service
metadata:
  name: vmalert-atlas-availability
  namespace: monitoring
  annotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "8880"
spec:
  selector:
    app: vmalert-atlas-availability
  ports:
    - name: http
      port: 8880
      targetPort: http