177 lines
7.5 KiB
Python
177 lines
7.5 KiB
Python
"""Protect the monitoring backend from known query-starvation regressions."""
|
|
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _documents(path: Path) -> list[dict]:
|
|
"""Load every non-empty YAML document from a repository manifest."""
|
|
return [document for document in yaml.safe_load_all(path.read_text()) if document]
|
|
|
|
|
|
def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
|
|
"""Keep search capacity and pod resources above the proven failure floor."""
|
|
manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml")
|
|
release = next(
|
|
manifest
|
|
for manifest in manifests
|
|
if manifest.get("kind") == "HelmRelease"
|
|
and manifest.get("metadata", {}).get("name") == "victoria-metrics-single"
|
|
)
|
|
server = release["spec"]["values"]["server"]
|
|
|
|
assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4
|
|
assert server["extraArgs"]["search.maxQueryDuration"] == "1m"
|
|
assert server["extraArgs"]["search.maxQueueDuration"] == "30s"
|
|
assert server["resources"]["requests"]["memory"] == "2Gi"
|
|
assert server["resources"]["limits"]["cpu"] == "2"
|
|
assert server["resources"]["limits"]["memory"] == "4Gi"
|
|
|
|
required_terms = server["affinity"]["nodeAffinity"][
|
|
"requiredDuringSchedulingIgnoredDuringExecution"
|
|
]["nodeSelectorTerms"]
|
|
hostname_rule = next(
|
|
expression
|
|
for term in required_terms
|
|
for expression in term["matchExpressions"]
|
|
if expression["key"] == "kubernetes.io/hostname"
|
|
)
|
|
assert hostname_rule["operator"] == "NotIn"
|
|
assert {"titan-14", "titan-18"} <= set(hostname_rule["values"])
|
|
|
|
|
|
def test_yearly_availability_is_published_outside_the_query_pool() -> None:
|
|
"""Keep all long-range availability work out of Grafana and MetricsQL."""
|
|
manifest = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)[0]
|
|
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
|
|
rules = [rule for group in groups for rule in group["rules"]]
|
|
manifests = _documents(
|
|
REPO_ROOT / "services/monitoring/availability-rollup-cronjob.yaml"
|
|
)
|
|
cronjob = next(manifest for manifest in manifests if manifest["kind"] == "CronJob")
|
|
source = (
|
|
REPO_ROOT / "services/monitoring/scripts/availability_rollup.py"
|
|
).read_text()
|
|
|
|
assert all(rule["record"] != "atlas:availability:ratio_365d" for rule in rules)
|
|
assert "atlas:availability:requests_1d" in source
|
|
assert "atlas:availability:failures_1d" in source
|
|
assert "/api/v1/export" in source
|
|
assert "/api/v1/import/prometheus" in source
|
|
assert cronjob["spec"]["schedule"] == "10 0 * * *"
|
|
assert cronjob["spec"]["concurrencyPolicy"] == "Forbid"
|
|
|
|
|
|
def test_daily_availability_rollups_use_the_same_request_sli() -> None:
|
|
"""Keep annual availability cheap without changing its request definition."""
|
|
manifest = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)[0]
|
|
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
|
|
daily_group = next(
|
|
group for group in groups if group["name"] == "atlas.availability.rollup"
|
|
)
|
|
daily = {rule["record"]: rule for rule in daily_group["rules"]}
|
|
|
|
assert daily_group["interval"] == "1d"
|
|
assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"]
|
|
assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"]
|
|
assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values())
|
|
|
|
|
|
def test_availability_uses_request_failures_instead_of_replica_capacity() -> None:
|
|
"""Measure HTTP outcomes without treating redundant replica loss as downtime."""
|
|
manifest = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)[0]
|
|
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
|
|
rules = [rule for group in groups for rule in group["rules"]]
|
|
requests = next(
|
|
rule
|
|
for rule in rules
|
|
if rule["record"] == "atlas:availability:requests_1h"
|
|
)
|
|
failures = next(
|
|
rule
|
|
for rule in rules
|
|
if rule["record"] == "atlas:availability:failures_1h"
|
|
)
|
|
|
|
assert 'code=~"[1-5].."' in requests["expr"]
|
|
assert 'code=~"5.."' in failures["expr"]
|
|
assert "traefik_entrypoint_requests_total" in requests["expr"]
|
|
assert "traefik_entrypoint_requests_total" in failures["expr"]
|
|
assert "kube_node_status_condition" not in repr(rules)
|
|
assert "kube_deployment_status_replicas_available" not in repr(rules)
|
|
assert requests["labels"]["definition"] == "request-v4"
|
|
assert failures["labels"]["definition"] == "request-v4"
|
|
|
|
|
|
def test_availability_backfill_replays_the_same_request_sli() -> None:
|
|
"""Backfill retained history with the exact rules used for future samples."""
|
|
manifests = _documents(
|
|
REPO_ROOT / "services/monitoring/availability-backfill-v4-job.yaml"
|
|
)
|
|
config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
|
|
job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
|
|
backfill = yaml.safe_load(config["data"]["atlas-request-history.yaml"])
|
|
expressions = {
|
|
rule["record"]: rule["expr"]
|
|
for group in backfill["groups"]
|
|
for rule in group["rules"]
|
|
}
|
|
|
|
assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1h"]
|
|
assert 'code=~"5.."' in expressions["atlas:availability:failures_1h"]
|
|
args = job["spec"]["template"]["spec"]["containers"][0]["args"]
|
|
assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args
|
|
assert "-replay.timeTo=2026-08-04T23:00:00Z" in args
|
|
assert "-replay.maxDatapointsPerQuery=48" in args
|
|
|
|
|
|
def test_daily_availability_backfill_stays_below_query_timeout() -> None:
|
|
"""Replay daily buckets in small chunks so raw history never starves Grafana."""
|
|
manifests = _documents(
|
|
REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml"
|
|
)
|
|
config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
|
|
job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
|
|
backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"])
|
|
group = backfill["groups"][0]
|
|
expressions = {rule["record"]: rule["expr"] for rule in group["rules"]}
|
|
|
|
assert group["interval"] == "1d"
|
|
assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"]
|
|
assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"]
|
|
args = job["spec"]["template"]["spec"]["containers"][0]["args"]
|
|
assert "-replay.maxDatapointsPerQuery=4" in args
|
|
|
|
|
|
def test_quality_rollups_do_not_run_every_minute() -> None:
|
|
"""Keep high-cardinality quality rollups below the backend saturation cadence."""
|
|
manifest = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)[0]
|
|
quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0]
|
|
|
|
assert quality["interval"] == "5m"
|
|
|
|
|
|
def test_vmalert_reloads_updated_rule_files() -> None:
|
|
"""Make Flux ConfigMap updates take effect without manual pod revision bumps."""
|
|
manifests = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)
|
|
deployment = next(
|
|
manifest for manifest in manifests if manifest.get("kind") == "Deployment"
|
|
)
|
|
args = deployment["spec"]["template"]["spec"]["containers"][0]["args"]
|
|
|
|
assert "-configCheckInterval=30s" in args
|