atlas-iac/services/monitoring/vmalert-atlas-availability.yaml

# services/monitoring/vmalert-atlas-availability.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: vmalert-atlas-availability-rules
  namespace: monitoring
data:
  hermes-triage.yaml: |
    groups:
      - name: hermes.triage
        interval: 1m
        rules:
          # Escalation on its own is NOT worth an email. Every human_required
          # incident already files an issue in the failing service's own repo,
          # and that issue is the durable artifact. Mailing on every escalation
          # made the inbox the loudest and least useful output of the system.
          # Two things do deserve mail: a repair that ran and failed, and an
          # escalation nobody has touched in hours.
          - alert: HermesTriageActionFailed
            expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="failed"}) == 1
            for: 1m
            labels:
              severity: critical
              service: hermes-triage
            annotations:
              summary: "Hermes repair FAILED: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
              description: "Ariadne authorized a remediation for {{ $labels.jenkins_job }}/{{ $labels.build }} and it did not succeed, or its rebuild failed. This is the one case where the automation acted and did not leave things better. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
          - alert: HermesTriageUnattended
            expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1
            for: 6h
            labels:
              severity: warning
              service: hermes-triage
            annotations:
              summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
              description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
  atlas-availability.yaml: |
    groups:
      - name: atlas.availability.gateway
        interval: 1h
        eval_offset: 59m
        rules:
          - record: atlas:availability:requests_1h
            expr: |
              sum(increase(
                traefik_entrypoint_requests_total{
                  entrypoint="websecure",
                  protocol="http",
                  code=~"[1-5].."
                }[1h]
              ))
            labels:
              definition: request-v4
              scope: atlas
              rollup: hourly
          - record: atlas:availability:failures_1h
            expr: |
              sum(increase(
                traefik_entrypoint_requests_total{
                  entrypoint="websecure",
                  protocol="http",
                  code=~"5.."
                }[1h]
              ))
            labels:
              definition: request-v4
              scope: atlas
              rollup: hourly
      - name: atlas.availability.rollup
        interval: 1d
        eval_offset: 23h59m
        rules:
          - record: atlas:availability:requests_1d
            expr: |
              sum(increase(
                traefik_entrypoint_requests_total{
                  entrypoint="websecure",
                  protocol="http",
                  code=~"[1-5].."
                }[1d]
              ))
            labels:
              definition: request-v4
              scope: atlas
              rollup: daily
          - record: atlas:availability:failures_1d
            expr: |
              sum(increase(
                traefik_entrypoint_requests_total{
                  entrypoint="websecure",
                  protocol="http",
                  code=~"5.."
                }[1d]
              ))
            labels:
              definition: request-v4
              scope: atlas
              rollup: daily
  platform-quality.yaml: |
    groups:
      - name: platform.quality
        # Stay inside the five-minute instant-query lookback, including the
        # group's evaluation and remote-write time.
        interval: 4m
        eval_offset: 1m
        rules:
          - record: platform_quality:test_case_status:count_1h
            expr: |
              sum by (suite, branch, test, category, status) (
                platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"}
                or label_replace(
                  platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=""},
                  "category", "uncategorized", "__name__", ".*"
                )
              )
            labels:
              rollup: hourly
          - record: platform_quality:test_case_pass_rate:percent_1h
            expr: |
              100 * (
                (
                  sum by (suite, branch, test, category) (
                    platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status="passed"}
                    or label_replace(
                      platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status="passed"},
                      "category", "uncategorized", "__name__", ".*"
                    )
                  )
                  or on(suite, branch, test, category)
                  (
                    0 * sum by (suite, branch, test, category) (
                      platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"}
                      or label_replace(
                        platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"},
                        "category", "uncategorized", "__name__", ".*"
                      )
                    )
                  )
                )
              )
              /
              clamp_min(
                sum by (suite, branch, test, category) (
                  platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"}
                  or label_replace(
                    platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"},
                    "category", "uncategorized", "__name__", ".*"
                  )
                ),
                1
              )
            labels:
              rollup: hourly
          - record: platform_quality:test_case_health_rate:percent_1h
            expr: |
              100 * (
                (
                  sum by (suite, branch, test, category) (
                    platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|skipped|not_applicable|na|n/a"}
                    or label_replace(
                      platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|skipped|not_applicable|na|n/a"},
                      "category", "uncategorized", "__name__", ".*"
                    )
                  )
                  or on(suite, branch, test, category)
                  (
                    0 * sum by (suite, branch, test, category) (
                      platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}
                      or label_replace(
                        platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"},
                        "category", "uncategorized", "__name__", ".*"
                      )
                    )
                  )
                )
              )
              /
              clamp_min(
                sum by (suite, branch, test, category) (
                  platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}
                  or label_replace(
                    platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"},
                    "category", "uncategorized", "__name__", ".*"
                  )
                ),
                1
              )
            labels:
              rollup: hourly
          - record: platform_quality:test_category_health_rate:percent_1h
            expr: |
              avg by (suite, branch, category) (
                platform_quality:test_case_health_rate:percent_1h{suite!="",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"}
              )
            labels:
              rollup: hourly
          - record: platform_quality:suite_runs:increase_24h
            expr: |
              sum by (suite, branch, status) (
                (
                  increase((
                    max without(instance, job) (
                      platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch!="",status!=""}
                    )
                  )[24h:1m])
                )
                or (
                  increase((
                    max without(instance, job) (
                      platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch="",status!=""}
                    )
                  )[24h:1m])
                  * on (suite) group_left(branch)
                    max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
              )
            labels:
              rollup: hourly
          - record: platform_quality:suite_coverage_percent:latest_1h
            expr: |
              max by (suite, branch) (
                {__name__=~"platform_quality_gate_workspace_line_coverage_percent|.*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!="",branch!=""}
              )
              or on (suite, branch)
              (
                max by (suite) (
                  {__name__=~"platform_quality_gate_workspace_line_coverage_percent|.*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!="",branch=""}
                )
                * on (suite) group_left(branch)
                  max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
              )
              or on (suite, branch)
              (
                label_replace(
                  max by (suite) (
                    {__name__=~"platform_quality_gate_workspace_line_coverage_percent|.*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!="",branch=""}
                  ),
                  "branch", "unknown", "__name__", ".*"
                )
                unless on (suite)
                  max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
              )
            labels:
              rollup: hourly
          - record: platform_quality:suite_source_files_total:latest_1h
            expr: |
              max by (suite, branch) (
                platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!="",branch!=""}
              )
              or on (suite, branch)
              (
                max by (suite) (
                  platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!="",branch=""}
                )
                * on (suite) group_left(branch)
                  max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
              )
              or on (suite, branch)
              (
                label_replace(
                  max by (suite) (
                    platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!="",branch=""}
                  ),
                  "branch", "unknown", "__name__", ".*"
                )
                unless on (suite)
                  max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
              )
            labels:
              rollup: hourly
          - record: platform_quality:suite_source_lines_over_500_total:latest_1h
            expr: |
              max by (suite, branch) (
                platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!="",branch!=""}
              )
              or on (suite, branch)
              (
                max by (suite) (
                  platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!="",branch=""}
                )
                * on (suite) group_left(branch)
                  max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
              )
              or on (suite, branch)
              (
                label_replace(
                  max by (suite) (
                    platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!="",branch=""}
                  ),
                  "branch", "unknown", "__name__", ".*"
                )
                unless on (suite)
                  max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
              )
            labels:
              rollup: hourly
          - record: platform_quality:check_status:present_1h
            expr: |
              sum by (suite, branch, check, status) (
                {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",status!=""}
                or label_replace(
                  {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",result!="",status=""},
                  "status", "$1", "result", "(.*)"
                )
                or (
                  {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""}
                  * on (suite) group_left(branch)
                    max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
                or (
                  label_replace(
                    {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""},
                    "status", "$1", "result", "(.*)"
                  )
                  * on (suite) group_left(branch)
                    max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
                or (
                  label_replace(
                    {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""},
                    "branch", "unknown", "__name__", ".*"
                  )
                  unless on (suite)
                    max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
                or (
                  label_replace(
                    label_replace(
                      {__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""},
                      "status", "$1", "result", "(.*)"
                    ),
                    "branch", "unknown", "__name__", ".*"
                  )
                  unless on (suite)
                    max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
                )
              )
            labels:
              rollup: hourly
          - record: platform_quality:check_seen_flag:present_1h
            expr: |
              clamp_max(
                max by (suite, branch, check) (
                  platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!=""} > 0
                ),
                1
              )
            labels:
              rollup: hourly
          - record: platform_quality:check_failed_flag:present_1h
            expr: |
              clamp_max(
                max by (suite, branch, check) (
                  platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0
                ),
                1
              )
              or on (suite, branch, check)
              (0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""})
            labels:
              rollup: hourly
          - record: platform_quality:check_healthy_flag:present_1h
            expr: |
              (
                clamp_max(
                  max by (suite, branch, check) (
                    platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status=~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0
                  ),
                  1
                )
                unless on (suite, branch, check)
                (platform_quality:check_failed_flag:present_1h{suite!="",branch!="",check!=""} > 0)
              )
              or on (suite, branch, check)
              (0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""})
            labels:
              rollup: hourly
          - record: platform_quality:sonar_gate_health_percent:latest_1h
            expr: |
              100 * max by (project_key) (
                sonarqube_project_quality_gate_pass{project_key!=""}
              )
            labels:
              rollup: hourly
  titan-media-capacity.yaml: |
    groups:
      - name: atlas.titan-media-capacity
        # Covers the titan-20/21/22 evidence audit from t_26da4c88: titan-20
        # runs at 227 MiB free at its worst point in a day with no coverage
        # at all, and titan-22 is Jellyfin's protected media-primary host —
        # any future opportunistic workload there needs contention alerts
        # to exist before it ships, not after.
        interval: 1m
        rules:
          - alert: Titan20NodeMemoryCritical
            expr: |
              (node_memory_MemAvailable_bytes{node="titan-20"} / node_memory_MemTotal_bytes{node="titan-20"}) < 0.05
            for: 10m
            labels:
              severity: critical
              service: hermes-ai-llm
            annotations:
              summary: "titan-20 (Hermes local LLM fallback/classifier) is within 5% of an OOM"
              description: "titan-20 hosts the hard-pinned Ollama fallback/classifier for Hermes Chat. Available memory has been under 5% of total for 10m. This node runs with no memory slack under normal load (98%+ used is typical); this alert fires only when it is at real risk of an OOM kill. Do not add workloads to titan-20 to relieve this — see docs/titan_20_21_22_capacity_optimization.md."
          - alert: TitanMediaHostCPUPressure
            expr: |
              (1 - avg(rate(node_cpu_seconds_total{mode="idle",node="titan-22"}[5m]))) > 0.85
            for: 10m
            labels:
              severity: warning
              service: jellyfin
            annotations:
              summary: "titan-22 (Jellyfin media-primary host) CPU busy > 85% for 10m"
              description: "titan-22 is normally under 30% CPU (p95, 24h baseline in docs/titan_20_21_22_capacity_optimization.md). Sustained pressure this high risks Jellyfin transcode quality and is the signal to check for a misbehaving opportunistic/scavenger workload."
          - alert: TitanMediaHostMemoryPressure
            expr: |
              (node_memory_MemAvailable_bytes{node="titan-22"} / node_memory_MemTotal_bytes{node="titan-22"}) < 0.15
            for: 10m
            labels:
              severity: warning
              service: jellyfin
            annotations:
              summary: "titan-22 (Jellyfin media-primary host) available memory under 15%"
              description: "titan-22 is normally ~90% free (24h baseline in docs/titan_20_21_22_capacity_optimization.md). This protects the reservation budget any future opportunistic/scavenger workload on this node must respect."
          - alert: JellyfinCPUThrottled
            expr: |
              (
                rate(container_cpu_cfs_throttled_periods_total{namespace="jellyfin",container="jellyfin"}[5m])
                /
                rate(container_cpu_cfs_periods_total{namespace="jellyfin",container="jellyfin"}[5m])
              ) > 0.05
            for: 5m
            labels:
              severity: critical
              service: jellyfin
            annotations:
              summary: "Jellyfin container is CPU-throttled against its own limit"
              description: "container_cpu_cfs_throttled_periods_total ratio for jellyfin/jellyfin has been over 5% for 5m. Sustained throttling degrades transcode encode speed and can cause playback stutter. Baseline in docs/titan_20_21_22_capacity_optimization.md measured 0% throttling over 24h; any sustained value here is a regression, most likely node-level contention from a co-scheduled workload."
          - alert: TitanMediaGPUMemorySaturation
            expr: |
              (
                (
                  label_replace(label_replace((DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE)), "pod", "$1", "Hostname", "(.*)"), "namespace", "monitoring", "", "")
                  * on(namespace,pod) group_left(node) kube_pod_info{namespace="monitoring"}
                )
                * on(node) group_left() node_uname_info{node="titan-22"}
              ) > 0.85
            for: 5m
            labels:
              severity: critical
              service: jellyfin
            annotations:
              summary: "titan-22 GPU VRAM over 85% used"
              description: "titan-22's shared RTX 3050 Ti (~8 GiB) is normally under 2% VRAM used (24h peak was 1.27 GiB in the t_26da4c88 audit). This is the concrete failure mode a second shared-GPU tenant (nvidia.com/gpu.shared) would cause for an active Jellyfin hardware transcode; GPU sharing on this node is time-sliced with no VRAM isolation, so this alert is the closest available proxy for contention."
          - alert: TitanVoiceHostCPUPressure
            expr: |
              (1 - avg(rate(node_cpu_seconds_total{mode="idle",node="titan-21"}[5m]))) > 0.90
            for: 15m
            labels:
              severity: warning
              service: hermes-voice
            annotations:
              summary: "titan-21 (Hermes STT/TTS host) CPU busy > 90% for 15m"
              description: "titan-21 already runs with 93% of its CPU requested (docs/titan_20_21_22_capacity_optimization.md); this alert catches sustained real contention that would add latency to Hermes Chat's hands-free voice path (feeds t_60d2deb6)."

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: vmalert-atlas-availability
  namespace: monitoring

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vmalert-atlas-availability
  namespace: monitoring
  labels:
    app: vmalert-atlas-availability
spec:
  replicas: 1
  revisionHistoryLimit: 3
  selector:
    matchLabels:
      app: vmalert-atlas-availability
  template:
    metadata:
      labels:
        app: vmalert-atlas-availability
      annotations:
        bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
    spec:
      serviceAccountName: vmalert-atlas-availability
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/hostname
                    operator: NotIn
                    values:
                      - titan-22
                      - titan-24
      containers:
        - name: vmalert
          image: victoriametrics/vmalert:v1.113.0
          args:
            - -datasource.url=http://victoria-metrics-single-server:8428
            - -datasource.queryStep=1h
            - -remoteWrite.url=http://victoria-metrics-single-server:8428
            - -rule=/etc/vmalert/rules/*.yaml
            - -configCheckInterval=30s
            - -evaluationInterval=15m
            - -notifier.url=http://alertmanager.monitoring.svc.cluster.local:9093
            - -httpListenAddr=:8880
          ports:
            - name: http
              containerPort: 8880
          readinessProbe:
            tcpSocket:
              port: http
            initialDelaySeconds: 5
            periodSeconds: 10
          livenessProbe:
            tcpSocket:
              port: http
            initialDelaySeconds: 20
            periodSeconds: 30
          resources:
            requests:
              cpu: 25m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 256Mi
          volumeMounts:
            - name: rules
              mountPath: /etc/vmalert/rules
              readOnly: true
      volumes:
        - name: rules
          configMap:
            name: vmalert-atlas-availability-rules

---

apiVersion: v1
kind: Service
metadata:
  name: vmalert-atlas-availability
  namespace: monitoring
  annotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "8880"
spec:
  selector:
    app: vmalert-atlas-availability
  ports:
    - name: http
      port: 8880
      targetPort: http