"""Protect the monitoring backend from known query-starvation regressions.""" from pathlib import Path import yaml REPO_ROOT = Path(__file__).resolve().parents[2] def _documents(path: Path) -> list[dict]: """Load every non-empty YAML document from a repository manifest.""" return [document for document in yaml.safe_load_all(path.read_text()) if document] def test_victoria_metrics_has_dashboard_burst_headroom() -> None: """Keep search capacity and pod resources above the proven failure floor.""" manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml") release = next( manifest for manifest in manifests if manifest.get("kind") == "HelmRelease" and manifest.get("metadata", {}).get("name") == "victoria-metrics-single" ) server = release["spec"]["values"]["server"] assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4 assert server["extraArgs"]["search.maxQueryDuration"] == "1m" assert server["extraArgs"]["search.maxQueueDuration"] == "30s" assert server["resources"]["requests"]["memory"] == "2Gi" assert server["resources"]["limits"]["cpu"] == "2" assert server["resources"]["limits"]["memory"] == "4Gi" required_terms = server["affinity"]["nodeAffinity"][ "requiredDuringSchedulingIgnoredDuringExecution" ]["nodeSelectorTerms"] hostname_rule = next( expression for term in required_terms for expression in term["matchExpressions"] if expression["key"] == "kubernetes.io/hostname" ) assert hostname_rule["operator"] == "NotIn" assert {"titan-14", "titan-18"} <= set(hostname_rule["values"]) def test_yearly_availability_reuses_the_hourly_rollup() -> None: """Prevent the yearly rule from rescanning raw cluster metrics for 365 days.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] rules = [rule for group in groups for rule in group["rules"]] yearly = next( rule for rule in rules if rule["record"] == "atlas:availability:ratio_365d" ) assert "atlas:availability:requests_1d" in yearly["expr"] assert "atlas:availability:failures_1d" in yearly["expr"] assert 'definition="request-v4"' in yearly["expr"] assert "sum_over_time" in yearly["expr"] assert "[365d]" in yearly["expr"] assert "traefik_entrypoint_requests_total" not in yearly["expr"] assert yearly["labels"]["definition"] == "request-v4" def test_daily_availability_rollups_use_the_same_request_sli() -> None: """Keep annual availability cheap without changing its request definition.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] daily_group = next( group for group in groups if group["name"] == "atlas.availability.rollup" ) daily = {rule["record"]: rule for rule in daily_group["rules"]} assert daily_group["interval"] == "1d" assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"] assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"] assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values()) def test_availability_uses_request_failures_instead_of_replica_capacity() -> None: """Measure HTTP outcomes without treating redundant replica loss as downtime.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] rules = [rule for group in groups for rule in group["rules"]] requests = next( rule for rule in rules if rule["record"] == "atlas:availability:requests_1h" ) failures = next( rule for rule in rules if rule["record"] == "atlas:availability:failures_1h" ) assert 'code=~"[1-5].."' in requests["expr"] assert 'code=~"5.."' in failures["expr"] assert "traefik_entrypoint_requests_total" in requests["expr"] assert "traefik_entrypoint_requests_total" in failures["expr"] assert "kube_node_status_condition" not in repr(rules) assert "kube_deployment_status_replicas_available" not in repr(rules) assert requests["labels"]["definition"] == "request-v4" assert failures["labels"]["definition"] == "request-v4" def test_availability_backfill_replays_the_same_request_sli() -> None: """Backfill retained history with the exact rules used for future samples.""" manifests = _documents( REPO_ROOT / "services/monitoring/availability-backfill-v4-job.yaml" ) config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap") job = next(manifest for manifest in manifests if manifest["kind"] == "Job") backfill = yaml.safe_load(config["data"]["atlas-request-history.yaml"]) expressions = { rule["record"]: rule["expr"] for group in backfill["groups"] for rule in group["rules"] } assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1h"] assert 'code=~"5.."' in expressions["atlas:availability:failures_1h"] args = job["spec"]["template"]["spec"]["containers"][0]["args"] assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args assert "-replay.timeTo=2026-08-04T23:00:00Z" in args assert "-replay.maxDatapointsPerQuery=48" in args def test_daily_availability_backfill_stays_below_query_timeout() -> None: """Replay daily buckets in small chunks so raw history never starves Grafana.""" manifests = _documents( REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml" ) config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap") job = next(manifest for manifest in manifests if manifest["kind"] == "Job") backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"]) group = backfill["groups"][0] expressions = {rule["record"]: rule["expr"] for rule in group["rules"]} assert group["interval"] == "1d" assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"] assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"] args = job["spec"]["template"]["spec"]["containers"][0]["args"] assert "-replay.maxDatapointsPerQuery=4" in args def test_quality_rollups_do_not_run_every_minute() -> None: """Keep high-cardinality quality rollups below the backend saturation cadence.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0] assert quality["interval"] == "5m" def test_vmalert_reloads_updated_rule_files() -> None: """Make Flux ConfigMap updates take effect without manual pod revision bumps.""" manifests = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" ) deployment = next( manifest for manifest in manifests if manifest.get("kind") == "Deployment" ) args = deployment["spec"]["template"]["spec"]["containers"][0]["args"] assert "-configCheckInterval=30s" in args