75 lines
2.9 KiB
Python
75 lines
2.9 KiB
Python
|
"""Keep the metrics database writable so the public dashboards can render."""
|
||
|
|
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import yaml
|
||
|
|
|
||
|
|
|
||
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||
|
|
|
||
|
|
# VictoriaMetrics stops accepting samples once free space under -storageDataPath
|
||
|
|
# falls below its 10MB floor. On 2026-08-18T15:25Z the volume filled, the
|
||
|
|
# database flipped read-only, and every panel on the public dashboards rendered
|
||
|
|
# "No data" while Grafana and all 119 scrape targets stayed healthy.
|
||
|
|
MINIMUM_STORAGE_GIB = 400
|
||
|
|
|
||
|
|
|
||
|
|
def _documents(path: Path) -> list[dict]:
|
||
|
|
"""Load every non-empty YAML document from a repository manifest."""
|
||
|
|
return [document for document in yaml.safe_load_all(path.read_text()) if document]
|
||
|
|
|
||
|
|
|
||
|
|
def _victoria_metrics_server() -> dict:
|
||
|
|
"""Return the VictoriaMetrics server values from its HelmRelease."""
|
||
|
|
manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml")
|
||
|
|
release = next(
|
||
|
|
manifest
|
||
|
|
for manifest in manifests
|
||
|
|
if manifest.get("kind") == "HelmRelease"
|
||
|
|
and manifest.get("metadata", {}).get("name") == "victoria-metrics-single"
|
||
|
|
)
|
||
|
|
return release["spec"]["values"]["server"]
|
||
|
|
|
||
|
|
|
||
|
|
def _alert_rules() -> dict[str, dict]:
|
||
|
|
"""Return every provisioned Grafana alert rule keyed by uid."""
|
||
|
|
manifest = _documents(
|
||
|
|
REPO_ROOT / "services/monitoring/grafana-alerting-config.yaml"
|
||
|
|
)[0]
|
||
|
|
groups = yaml.safe_load(manifest["data"]["rules.yaml"])["groups"]
|
||
|
|
return {rule["uid"]: rule for group in groups for rule in group["rules"]}
|
||
|
|
|
||
|
|
|
||
|
|
def test_metrics_storage_holds_a_full_retention_period() -> None:
|
||
|
|
"""Size the volume for the retention actually configured, plus headroom."""
|
||
|
|
server = _victoria_metrics_server()
|
||
|
|
size = server["persistentVolume"]["size"]
|
||
|
|
|
||
|
|
assert size.endswith("Gi")
|
||
|
|
assert int(size.removesuffix("Gi")) >= MINIMUM_STORAGE_GIB
|
||
|
|
assert server["extraArgs"]["retentionPeriod"] == "1y"
|
||
|
|
|
||
|
|
|
||
|
|
def test_full_metrics_storage_pages_before_it_blocks_ingestion() -> None:
|
||
|
|
"""Warn on shrinking headroom and page once the database refuses writes."""
|
||
|
|
rules = _alert_rules()
|
||
|
|
low = rules["victoria-metrics-storage-low"]
|
||
|
|
read_only = rules["victoria-metrics-storage-readonly"]
|
||
|
|
|
||
|
|
assert "vm_free_disk_space_bytes" in low["data"][0]["model"]["expr"]
|
||
|
|
assert low["labels"]["severity"] == "warning"
|
||
|
|
assert "vm_storage_is_read_only" in read_only["data"][0]["model"]["expr"]
|
||
|
|
assert read_only["labels"]["severity"] == "critical"
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_silent_metrics_database_still_pages() -> None:
|
||
|
|
"""A stalled database returns no samples, so absent data must alert."""
|
||
|
|
rules = _alert_rules()
|
||
|
|
stalled = rules["victoria-metrics-ingestion-stalled"]
|
||
|
|
|
||
|
|
assert "vm_rows_added_to_storage_total" in stalled["data"][0]["model"]["expr"]
|
||
|
|
assert stalled["noDataState"] == "Alerting"
|
||
|
|
assert stalled["execErrState"] == "Alerting"
|
||
|
|
assert stalled["labels"]["severity"] == "critical"
|
||
|
|
assert rules["victoria-metrics-storage-readonly"]["noDataState"] == "Alerting"
|