"""Protect the monitoring backend from known query-starvation regressions.""" from pathlib import Path import yaml REPO_ROOT = Path(__file__).resolve().parents[2] def _documents(path: Path) -> list[dict]: """Load every non-empty YAML document from a repository manifest.""" return [document for document in yaml.safe_load_all(path.read_text()) if document] def test_victoria_metrics_has_dashboard_burst_headroom() -> None: """Keep search capacity and pod resources above the proven failure floor.""" manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml") release = next( manifest for manifest in manifests if manifest.get("kind") == "HelmRelease" and manifest.get("metadata", {}).get("name") == "victoria-metrics-single" ) server = release["spec"]["values"]["server"] assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4 assert server["extraArgs"]["search.maxQueryDuration"] == "1m" assert server["extraArgs"]["search.maxQueueDuration"] == "30s" assert server["resources"]["requests"]["memory"] == "2Gi" assert server["resources"]["limits"]["cpu"] == "2" assert server["resources"]["limits"]["memory"] == "4Gi" required_terms = server["affinity"]["nodeAffinity"][ "requiredDuringSchedulingIgnoredDuringExecution" ]["nodeSelectorTerms"] hostname_rule = next( expression for term in required_terms for expression in term["matchExpressions"] if expression["key"] == "kubernetes.io/hostname" ) assert hostname_rule["operator"] == "NotIn" assert {"titan-14", "titan-18"} <= set(hostname_rule["values"]) def test_yearly_availability_reuses_the_hourly_rollup() -> None: """Prevent the yearly rule from rescanning raw cluster metrics for 365 days.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] rules = [rule for group in groups for rule in group["rules"]] yearly = next( rule for rule in rules if rule["record"] == "atlas:availability:ratio_365d" ) assert "atlas:availability:gateway_requests_1h" in yearly["expr"] assert "atlas:availability:gateway_failures_1h" in yearly["expr"] assert 'definition="gateway-v3"' in yearly["expr"] assert "sum_over_time" in yearly["expr"] assert "[365d]" in yearly["expr"] assert "traefik_entrypoint_requests_total" not in yearly["expr"] assert yearly["labels"]["definition"] == "gateway-v3" def test_availability_uses_gateway_failures_instead_of_replica_capacity() -> None: """Measure request outcomes without treating redundant replica loss as downtime.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] rules = [rule for group in groups for rule in group["rules"]] requests = next( rule for rule in rules if rule["record"] == "atlas:availability:gateway_requests_1h" ) failures = next( rule for rule in rules if rule["record"] == "atlas:availability:gateway_failures_1h" ) assert 'code=~"[1-5].."' in requests["expr"] assert 'code=~"502|503|504"' in failures["expr"] assert "traefik_entrypoint_requests_total" in requests["expr"] assert "traefik_entrypoint_requests_total" in failures["expr"] assert "kube_node_status_condition" not in repr(rules) assert "kube_deployment_status_replicas_available" not in repr(rules) assert requests["labels"]["definition"] == "gateway-v3" assert failures["labels"]["definition"] == "gateway-v3" def test_availability_backfill_replays_the_same_gateway_sli() -> None: """Backfill retained history with the exact rules used for future samples.""" manifests = _documents( REPO_ROOT / "services/monitoring/availability-backfill-v3-job.yaml" ) config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap") job = next(manifest for manifest in manifests if manifest["kind"] == "Job") backfill = yaml.safe_load(config["data"]["atlas-gateway-history.yaml"]) expressions = { rule["record"]: rule["expr"] for group in backfill["groups"] for rule in group["rules"] } assert 'code=~"[1-5].."' in expressions["atlas:availability:gateway_requests_1h"] assert 'code=~"502|503|504"' in expressions["atlas:availability:gateway_failures_1h"] args = job["spec"]["template"]["spec"]["containers"][0]["args"] assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args assert "-replay.timeTo=2026-08-04T23:00:00Z" in args assert "-replay.maxDatapointsPerQuery=48" in args def test_quality_rollups_do_not_run_every_minute() -> None: """Keep high-cardinality quality rollups below the backend saturation cadence.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0] assert quality["interval"] == "5m" def test_vmalert_reloads_updated_rule_files() -> None: """Make Flux ConfigMap updates take effect without manual pod revision bumps.""" manifests = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" ) deployment = next( manifest for manifest in manifests if manifest.get("kind") == "Deployment" ) args = deployment["spec"]["template"]["spec"]["containers"][0]["args"] assert "-configCheckInterval=30s" in args