"""Keep the metrics database writable so the public dashboards can render.""" from pathlib import Path import yaml REPO_ROOT = Path(__file__).resolve().parents[2] # VictoriaMetrics stops accepting samples once free space under -storageDataPath # falls below its 10MB floor. On 2026-08-18T15:25Z the volume filled, the # database flipped read-only, and every panel on the public dashboards rendered # "No data" while Grafana and all 119 scrape targets stayed healthy. MINIMUM_STORAGE_GIB = 400 def _documents(path: Path) -> list[dict]: """Load every non-empty YAML document from a repository manifest.""" return [document for document in yaml.safe_load_all(path.read_text()) if document] def _victoria_metrics_server() -> dict: """Return the VictoriaMetrics server values from its HelmRelease.""" manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml") release = next( manifest for manifest in manifests if manifest.get("kind") == "HelmRelease" and manifest.get("metadata", {}).get("name") == "victoria-metrics-single" ) return release["spec"]["values"]["server"] def _alert_rules() -> dict[str, dict]: """Return every provisioned Grafana alert rule keyed by uid.""" manifest = _documents( REPO_ROOT / "services/monitoring/grafana-alerting-config.yaml" )[0] groups = yaml.safe_load(manifest["data"]["rules.yaml"])["groups"] return {rule["uid"]: rule for group in groups for rule in group["rules"]} def test_metrics_storage_holds_a_full_retention_period() -> None: """Size the volume for the retention actually configured, plus headroom.""" server = _victoria_metrics_server() size = server["persistentVolume"]["size"] assert size.endswith("Gi") assert int(size.removesuffix("Gi")) >= MINIMUM_STORAGE_GIB assert server["extraArgs"]["retentionPeriod"] == "1y" def test_full_metrics_storage_pages_before_it_blocks_ingestion() -> None: """Warn on shrinking headroom and page once the database refuses writes.""" rules = _alert_rules() low = rules["victoria-metrics-storage-low"] read_only = rules["victoria-metrics-storage-readonly"] assert "vm_free_disk_space_bytes" in low["data"][0]["model"]["expr"] assert low["labels"]["severity"] == "warning" assert "vm_storage_is_read_only" in read_only["data"][0]["model"]["expr"] assert read_only["labels"]["severity"] == "critical" def test_a_silent_metrics_database_still_pages() -> None: """A stalled database returns no samples, so absent data must alert.""" rules = _alert_rules() stalled = rules["victoria-metrics-ingestion-stalled"] assert "vm_rows_added_to_storage_total" in stalled["data"][0]["model"]["expr"] assert stalled["noDataState"] == "Alerting" assert stalled["execErrState"] == "Alerting" assert stalled["labels"]["severity"] == "critical" assert rules["victoria-metrics-storage-readonly"]["noDataState"] == "Alerting"