0da9e4c82d
- Move flat service manifests into structured subdirs (apps/, bootstrap-jobs/, repair-jobs/, migration-jobs/, validation-jobs/, node-ops/, networking/) - Retire oneoffs/ directories across services - Remove oceanus cluster and its host roles; add aether cluster + terraform scaffolding - Reorganize scripts/ into ops/, render/, sync/, manual-tests/ - Add Makefile with render/validate/test/flux targets and repo-structure tests - Update flux-system application CRs to the new paths - Add hermes-automated-triage-24h-plan knowledge doc (+ comms mirror) - Refresh knowledge catalogs, dashboards, vmalert rules, quality contract Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
107 lines
4.0 KiB
Python
107 lines
4.0 KiB
Python
"""Protect the monitoring backend from known query-starvation regressions."""
|
|
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _documents(path: Path) -> list[dict]:
|
|
"""Load every non-empty YAML document from a repository manifest."""
|
|
return [document for document in yaml.safe_load_all(path.read_text()) if document]
|
|
|
|
|
|
def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
|
|
"""Keep search capacity and pod resources above the proven failure floor."""
|
|
manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml")
|
|
release = next(
|
|
manifest
|
|
for manifest in manifests
|
|
if manifest.get("kind") == "HelmRelease"
|
|
and manifest.get("metadata", {}).get("name") == "victoria-metrics-single"
|
|
)
|
|
server = release["spec"]["values"]["server"]
|
|
|
|
assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4
|
|
assert server["extraArgs"]["search.maxQueryDuration"] == "1m"
|
|
assert server["extraArgs"]["search.maxQueueDuration"] == "30s"
|
|
assert server["resources"]["requests"]["memory"] == "2Gi"
|
|
assert server["resources"]["limits"]["cpu"] == "2"
|
|
assert server["resources"]["limits"]["memory"] == "4Gi"
|
|
|
|
required_terms = server["affinity"]["nodeAffinity"][
|
|
"requiredDuringSchedulingIgnoredDuringExecution"
|
|
]["nodeSelectorTerms"]
|
|
hostname_rule = next(
|
|
expression
|
|
for term in required_terms
|
|
for expression in term["matchExpressions"]
|
|
if expression["key"] == "kubernetes.io/hostname"
|
|
)
|
|
assert hostname_rule["operator"] == "NotIn"
|
|
assert {"titan-14", "titan-18"} <= set(hostname_rule["values"])
|
|
|
|
|
|
def test_yearly_availability_reuses_the_hourly_rollup() -> None:
|
|
"""Prevent the yearly rule from rescanning raw cluster metrics for 365 days."""
|
|
manifest = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)[0]
|
|
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
|
|
yearly = next(
|
|
rule
|
|
for rule in rules["rules"]
|
|
if rule["record"] == "atlas:availability:ratio_365d"
|
|
)
|
|
|
|
assert "atlas:availability:ratio_1h" in yearly["expr"]
|
|
assert 'definition="serving-v2"' in yearly["expr"]
|
|
assert 'definition=""' in yearly["expr"]
|
|
assert "share_gt_over_time" in yearly["expr"]
|
|
assert "[365d:15m]" in yearly["expr"]
|
|
assert "kube_node_status_condition" not in yearly["expr"]
|
|
assert "[365d:1h]" not in yearly["expr"]
|
|
|
|
|
|
def test_availability_counts_serving_state_instead_of_replica_capacity() -> None:
|
|
"""Treat quorum and any serving ingress replica as available binary states."""
|
|
manifest = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)[0]
|
|
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
|
|
hourly = next(
|
|
rule
|
|
for rule in rules["rules"]
|
|
if rule["record"] == "atlas:availability:ratio_1h"
|
|
)
|
|
|
|
assert ">= bool 2" in hourly["expr"]
|
|
assert "> bool 0" in hourly["expr"]
|
|
assert "/ 3" not in hourly["expr"]
|
|
assert hourly["labels"]["definition"] == "serving-v2"
|
|
|
|
|
|
def test_quality_rollups_do_not_run_every_minute() -> None:
|
|
"""Keep high-cardinality quality rollups below the backend saturation cadence."""
|
|
manifest = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)[0]
|
|
quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0]
|
|
|
|
assert quality["interval"] == "5m"
|
|
|
|
|
|
def test_vmalert_reloads_updated_rule_files() -> None:
|
|
"""Make Flux ConfigMap updates take effect without manual pod revision bumps."""
|
|
manifests = _documents(
|
|
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
|
|
)
|
|
deployment = next(
|
|
manifest for manifest in manifests if manifest.get("kind") == "Deployment"
|
|
)
|
|
args = deployment["spec"]["template"]["spec"]["containers"][0]["args"]
|
|
|
|
assert "-configCheckInterval=30s" in args
|