diff --git a/scripts/tests/test_monitoring_query_capacity.py b/scripts/tests/test_monitoring_query_capacity.py new file mode 100644 index 000000000..afd44a720 --- /dev/null +++ b/scripts/tests/test_monitoring_query_capacity.py @@ -0,0 +1,59 @@ +"""Protect the monitoring backend from known query-starvation regressions.""" + +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _documents(path: Path) -> list[dict]: + """Load every non-empty YAML document from a repository manifest.""" + return [document for document in yaml.safe_load_all(path.read_text()) if document] + + +def test_victoria_metrics_has_dashboard_burst_headroom() -> None: + """Keep search capacity and pod resources above the proven failure floor.""" + manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml") + release = next( + manifest + for manifest in manifests + if manifest.get("kind") == "HelmRelease" + and manifest.get("metadata", {}).get("name") == "victoria-metrics-single" + ) + server = release["spec"]["values"]["server"] + + assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4 + assert server["extraArgs"]["search.maxQueryDuration"] == "1m" + assert server["extraArgs"]["search.maxQueueDuration"] == "30s" + assert server["resources"]["requests"]["memory"] == "2Gi" + assert server["resources"]["limits"]["cpu"] == "2" + assert server["resources"]["limits"]["memory"] == "4Gi" + + +def test_yearly_availability_reuses_the_hourly_rollup() -> None: + """Prevent the yearly rule from rescanning raw cluster metrics for 365 days.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + )[0] + rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0] + yearly = next( + rule + for rule in rules["rules"] + if rule["record"] == "atlas:availability:ratio_365d" + ) + + assert "atlas:availability:ratio_1h" in yearly["expr"] + assert "kube_node_status_condition" not in yearly["expr"] + assert "[365d:1h]" not in yearly["expr"] + + +def test_quality_rollups_do_not_run_every_minute() -> None: + """Keep high-cardinality quality rollups below the backend saturation cadence.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + )[0] + quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0] + + assert quality["interval"] == "5m" diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index 3774d165d..f83c7df10 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -85,17 +85,20 @@ spec: extraArgs: retentionPeriod: "1y" # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11} promscrape.configCheckInterval: "30s" + search.maxConcurrentRequests: "4" + search.maxQueryDuration: "1m" + search.maxQueueDuration: "30s" persistentVolume: enabled: true size: 100Gi resources: requests: - cpu: 250m - memory: 1Gi - limits: - cpu: "1" + cpu: 500m memory: 2Gi + limits: + cpu: "2" + memory: 4Gi affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -112,6 +115,21 @@ spec: - titan-21 - titan-22 - titan-24 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi4 # Enable built-in Kubernetes scraping scrape: diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index a1138d753..d1738fc1a 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -30,119 +30,16 @@ data: rollup: hourly - record: atlas:availability:ratio_365d expr: | - clamp_max(( - ( - sum(sum_over_time(( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) - ) - ) - )[365d:1h])) - or on() vector(0) - ) - + - clamp_min( - 8761 - - - ( - clamp_min( - floor( - ( - time() - - - ( - min(min_over_time(timestamp( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) - ) - ) - )[365d:1h])) - or on() vector(time() + 3600) - ) - ) - / 3600 - ) - + 1, - 0 - ) - ), - 0 - ) + avg_over_time( + atlas:availability:ratio_1h{scope="atlas"}[365d] ) - / - clamp_min( - ( - ( - sum(count_over_time(( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) - ) - ) - )[365d:1h])) - or on() vector(0) - ) - + - clamp_min( - 8761 - - - ( - clamp_min( - floor( - ( - time() - - - ( - min(min_over_time(timestamp( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) - ) - ) - )[365d:1h])) - or on() vector(time() + 3600) - ) - ) - / 3600 - ) - + 1, - 0 - ) - ), - 0 - ) - ), - 1 - ), 1) labels: scope: atlas rollup: yearly platform-quality.yaml: | groups: - name: platform.quality - interval: 1m + interval: 5m rules: - record: platform_quality:test_case_status:count_1h expr: |