From c4eac8ceee1af2cb09248a64372e017f41126f4c Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Wed, 19 Aug 2026 10:37:54 +0000 Subject: [PATCH] fix(monitoring): keep VictoriaMetrics writable --- .../tests/test_monitoring_storage_capacity.py | 74 +++++++++ .../monitoring/grafana-alerting-config.yaml | 148 ++++++++++++++++++ services/monitoring/helmrelease.yaml | 7 +- 3 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 scripts/tests/test_monitoring_storage_capacity.py diff --git a/scripts/tests/test_monitoring_storage_capacity.py b/scripts/tests/test_monitoring_storage_capacity.py new file mode 100644 index 00000000..71c39a5f --- /dev/null +++ b/scripts/tests/test_monitoring_storage_capacity.py @@ -0,0 +1,74 @@ +"""Keep the metrics database writable so the public dashboards can render.""" + +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# VictoriaMetrics stops accepting samples once free space under -storageDataPath +# falls below its 10MB floor. On 2026-08-18T15:25Z the volume filled, the +# database flipped read-only, and every panel on the public dashboards rendered +# "No data" while Grafana and all 119 scrape targets stayed healthy. +MINIMUM_STORAGE_GIB = 400 + + +def _documents(path: Path) -> list[dict]: + """Load every non-empty YAML document from a repository manifest.""" + return [document for document in yaml.safe_load_all(path.read_text()) if document] + + +def _victoria_metrics_server() -> dict: + """Return the VictoriaMetrics server values from its HelmRelease.""" + manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml") + release = next( + manifest + for manifest in manifests + if manifest.get("kind") == "HelmRelease" + and manifest.get("metadata", {}).get("name") == "victoria-metrics-single" + ) + return release["spec"]["values"]["server"] + + +def _alert_rules() -> dict[str, dict]: + """Return every provisioned Grafana alert rule keyed by uid.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/grafana-alerting-config.yaml" + )[0] + groups = yaml.safe_load(manifest["data"]["rules.yaml"])["groups"] + return {rule["uid"]: rule for group in groups for rule in group["rules"]} + + +def test_metrics_storage_holds_a_full_retention_period() -> None: + """Size the volume for the retention actually configured, plus headroom.""" + server = _victoria_metrics_server() + size = server["persistentVolume"]["size"] + + assert size.endswith("Gi") + assert int(size.removesuffix("Gi")) >= MINIMUM_STORAGE_GIB + assert server["extraArgs"]["retentionPeriod"] == "1y" + + +def test_full_metrics_storage_pages_before_it_blocks_ingestion() -> None: + """Warn on shrinking headroom and page once the database refuses writes.""" + rules = _alert_rules() + low = rules["victoria-metrics-storage-low"] + read_only = rules["victoria-metrics-storage-readonly"] + + assert "vm_free_disk_space_bytes" in low["data"][0]["model"]["expr"] + assert low["labels"]["severity"] == "warning" + assert "vm_storage_is_read_only" in read_only["data"][0]["model"]["expr"] + assert read_only["labels"]["severity"] == "critical" + + +def test_a_silent_metrics_database_still_pages() -> None: + """A stalled database returns no samples, so absent data must alert.""" + rules = _alert_rules() + stalled = rules["victoria-metrics-ingestion-stalled"] + + assert "vm_rows_added_to_storage_total" in stalled["data"][0]["model"]["expr"] + assert stalled["noDataState"] == "Alerting" + assert stalled["execErrState"] == "Alerting" + assert stalled["labels"]["severity"] == "critical" + assert rules["victoria-metrics-storage-readonly"]["noDataState"] == "Alerting" diff --git a/services/monitoring/grafana-alerting-config.yaml b/services/monitoring/grafana-alerting-config.yaml index 6e408752..e0909f42 100644 --- a/services/monitoring/grafana-alerting-config.yaml +++ b/services/monitoring/grafana-alerting-config.yaml @@ -250,6 +250,154 @@ data: summary: "VictoriaMetrics is unavailable for >30m" labels: severity: critical + # The database can be up, scraped, and answering queries while + # refusing every new sample. These three rules watch the write path + # rather than liveness, and they alert on absent data because a + # stalled database stops producing the very series they read. + - uid: victoria-metrics-storage-low + title: "VictoriaMetrics storage headroom low (<15%)" + condition: C + for: "30m" + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: atlas-vm + model: + intervalMs: 60000 + maxDataPoints: 43200 + expr: 100 * sum(vm_free_disk_space_bytes{job="victoriametrics"}) / clamp_min(sum(vm_free_disk_space_bytes{job="victoriametrics"}) + sum(vm_data_size_bytes{job="victoriametrics"}), 1) + legendFormat: free % + datasource: + type: prometheus + uid: atlas-vm + - refId: B + datasourceUid: __expr__ + model: + expression: A + intervalMs: 60000 + maxDataPoints: 43200 + reducer: last + type: reduce + - refId: C + datasourceUid: __expr__ + model: + expression: B + intervalMs: 60000 + maxDataPoints: 43200 + type: threshold + conditions: + - evaluator: + params: [15] + type: lt + operator: + type: and + reducer: + type: last + type: query + noDataState: Alerting + execErrState: Alerting + annotations: + summary: "VictoriaMetrics has <15% free space on /storage; expand the volume before it stops accepting samples" + labels: + severity: warning + - uid: victoria-metrics-storage-readonly + title: "VictoriaMetrics storage is read-only" + condition: C + for: "5m" + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: atlas-vm + model: + intervalMs: 60000 + maxDataPoints: 43200 + expr: max(vm_storage_is_read_only{job="victoriametrics"}) or on() vector(0) + legendFormat: read-only + datasource: + type: prometheus + uid: atlas-vm + - refId: B + datasourceUid: __expr__ + model: + expression: A + intervalMs: 60000 + maxDataPoints: 43200 + reducer: last + type: reduce + - refId: C + datasourceUid: __expr__ + model: + expression: B + intervalMs: 60000 + maxDataPoints: 43200 + type: threshold + conditions: + - evaluator: + params: [0] + type: gt + operator: + type: and + reducer: + type: last + type: query + noDataState: Alerting + execErrState: Alerting + annotations: + summary: "VictoriaMetrics flipped /storage to read-only and is discarding every new sample" + labels: + severity: critical + - uid: victoria-metrics-ingestion-stalled + title: "VictoriaMetrics is not storing samples" + condition: C + for: "15m" + data: + - refId: A + relativeTimeRange: + from: 900 + to: 0 + datasourceUid: atlas-vm + model: + intervalMs: 60000 + maxDataPoints: 43200 + expr: sum(rate(vm_rows_added_to_storage_total{job="victoriametrics"}[10m])) or on() vector(0) + legendFormat: rows/s + datasource: + type: prometheus + uid: atlas-vm + - refId: B + datasourceUid: __expr__ + model: + expression: A + intervalMs: 60000 + maxDataPoints: 43200 + reducer: last + type: reduce + - refId: C + datasourceUid: __expr__ + model: + expression: B + intervalMs: 60000 + maxDataPoints: 43200 + type: threshold + conditions: + - evaluator: + params: [1] + type: lt + operator: + type: and + reducer: + type: last + type: query + noDataState: Alerting + execErrState: Alerting + annotations: + summary: "VictoriaMetrics stored no samples for 15m; every dashboard is about to read empty" + labels: + severity: critical - orgId: 1 name: maintenance folder: Alerts diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index ad30aa34..a938d8e2 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -91,7 +91,12 @@ spec: persistentVolume: enabled: true - size: 100Gi + # A year of Atlas cardinality is ~210Gi today, and this 100Gi never + # matched the 200Gi volume actually bound in the cluster. That volume + # filled on 2026-08-18: VictoriaMetrics went read-only, ingestion + # stopped, and every public dashboard panel rendered "No data" while + # all 119 scrape targets stayed healthy. + size: 400Gi resources: requests: cpu: 500m