monitoring: prevent query pool starvation

scripts/tests/test_monitoring_query_capacity.py

@@ -0,0 +1,59 @@
+"""Protect the monitoring backend from known query-starvation regressions."""
+
+from pathlib import Path
+
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def _documents(path: Path) -> list[dict]:
+    """Load every non-empty YAML document from a repository manifest."""
+    return [document for document in yaml.safe_load_all(path.read_text()) if document]
+
+
+def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
+    """Keep search capacity and pod resources above the proven failure floor."""
+    manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml")
+    release = next(
+        manifest
+        for manifest in manifests
+        if manifest.get("kind") == "HelmRelease"
+        and manifest.get("metadata", {}).get("name") == "victoria-metrics-single"
+    )
+    server = release["spec"]["values"]["server"]
+
+    assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4
+    assert server["extraArgs"]["search.maxQueryDuration"] == "1m"
+    assert server["extraArgs"]["search.maxQueueDuration"] == "30s"
+    assert server["resources"]["requests"]["memory"] == "2Gi"
+    assert server["resources"]["limits"]["cpu"] == "2"
+    assert server["resources"]["limits"]["memory"] == "4Gi"
+
+
+def test_yearly_availability_reuses_the_hourly_rollup() -> None:
+    """Prevent the yearly rule from rescanning raw cluster metrics for 365 days."""
+    manifest = _documents(
+        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
+    )[0]
+    rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
+    yearly = next(
+        rule
+        for rule in rules["rules"]
+        if rule["record"] == "atlas:availability:ratio_365d"
+    )
+
+    assert "atlas:availability:ratio_1h" in yearly["expr"]
+    assert "kube_node_status_condition" not in yearly["expr"]
+    assert "[365d:1h]" not in yearly["expr"]
+
+
+def test_quality_rollups_do_not_run_every_minute() -> None:
+    """Keep high-cardinality quality rollups below the backend saturation cadence."""
+    manifest = _documents(
+        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
+    )[0]
+    quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0]
+
+    assert quality["interval"] == "5m"

services/monitoring/helmrelease.yaml

@@ -85,17 +85,20 @@ spec:
       extraArgs:
         retentionPeriod: "1y"   # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11}
         promscrape.configCheckInterval: "30s"
+        search.maxConcurrentRequests: "4"
+        search.maxQueryDuration: "1m"
+        search.maxQueueDuration: "30s"
 
       persistentVolume:
         enabled: true
         size: 100Gi
       resources:
         requests:
-          cpu: 250m
+          cpu: 500m
-          memory: 1Gi
-        limits:
-          cpu: "1"
           memory: 2Gi
+        limits:
+          cpu: "2"
+          memory: 4Gi
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -112,6 +115,21 @@ spec:
                       - titan-21
                       - titan-22
                       - titan-24
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi4
 
       # Enable built-in Kubernetes scraping
       scrape:

services/monitoring/vmalert-atlas-availability.yaml

@@ -30,119 +30,16 @@ data:
               rollup: hourly
           - record: atlas:availability:ratio_365d
             expr: |
-              clamp_max((
+              avg_over_time(
-                (
+                atlas:availability:ratio_1h{scope="atlas"}[365d]
-                  sum(sum_over_time((
-                    min(
-                      (
-                        sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
-                        / 3
-                      ),
-                      (
-                        sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
-                        / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
               )
-                    )
-                  )[365d:1h]))
-                  or on() vector(0)
-                )
-                +
-                clamp_min(
-                  8761
-                  -
-                  (
-                    clamp_min(
-                      floor(
-                        (
-                          time()
-                          -
-                          (
-                            min(min_over_time(timestamp(
-                              min(
-                                (
-                                  sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
-                                  / 3
-                                ),
-                                (
-                                  sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
-                                  / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
-                                )
-                              )
-                            )[365d:1h]))
-                            or on() vector(time() + 3600)
-                          )
-                        )
-                        / 3600
-                      )
-                      + 1,
-                      0
-                    )
-                  ),
-                  0
-                )
-              )
-              /
-              clamp_min(
-                (
-                  (
-                    sum(count_over_time((
-                      min(
-                        (
-                          sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
-                          / 3
-                        ),
-                        (
-                          sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
-                          / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
-                        )
-                      )
-                    )[365d:1h]))
-                    or on() vector(0)
-                  )
-                  +
-                  clamp_min(
-                    8761
-                    -
-                    (
-                      clamp_min(
-                        floor(
-                          (
-                            time()
-                            -
-                            (
-                              min(min_over_time(timestamp(
-                                min(
-                                  (
-                                    sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
-                                    / 3
-                                  ),
-                                  (
-                                    sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
-                                    / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
-                                  )
-                                )
-                              )[365d:1h]))
-                              or on() vector(time() + 3600)
-                            )
-                          )
-                          / 3600
-                        )
-                        + 1,
-                        0
-                      )
-                    ),
-                    0
-                  )
-                ),
-                1
-              ), 1)
             labels:
               scope: atlas
               rollup: yearly
   platform-quality.yaml: |
     groups:
       - name: platform.quality
-        interval: 1m
+        interval: 5m
         rules:
           - record: platform_quality:test_case_status:count_1h
             expr: |