fix(monitoring): keep VictoriaMetrics writable

scripts/tests/test_monitoring_storage_capacity.py

@@ -0,0 +1,74 @@
+"""Keep the metrics database writable so the public dashboards can render."""
+
+from pathlib import Path
+
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+# VictoriaMetrics stops accepting samples once free space under -storageDataPath
+# falls below its 10MB floor. On 2026-08-18T15:25Z the volume filled, the
+# database flipped read-only, and every panel on the public dashboards rendered
+# "No data" while Grafana and all 119 scrape targets stayed healthy.
+MINIMUM_STORAGE_GIB = 400
+
+
+def _documents(path: Path) -> list[dict]:
+    """Load every non-empty YAML document from a repository manifest."""
+    return [document for document in yaml.safe_load_all(path.read_text()) if document]
+
+
+def _victoria_metrics_server() -> dict:
+    """Return the VictoriaMetrics server values from its HelmRelease."""
+    manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml")
+    release = next(
+        manifest
+        for manifest in manifests
+        if manifest.get("kind") == "HelmRelease"
+        and manifest.get("metadata", {}).get("name") == "victoria-metrics-single"
+    )
+    return release["spec"]["values"]["server"]
+
+
+def _alert_rules() -> dict[str, dict]:
+    """Return every provisioned Grafana alert rule keyed by uid."""
+    manifest = _documents(
+        REPO_ROOT / "services/monitoring/grafana-alerting-config.yaml"
+    )[0]
+    groups = yaml.safe_load(manifest["data"]["rules.yaml"])["groups"]
+    return {rule["uid"]: rule for group in groups for rule in group["rules"]}
+
+
+def test_metrics_storage_holds_a_full_retention_period() -> None:
+    """Size the volume for the retention actually configured, plus headroom."""
+    server = _victoria_metrics_server()
+    size = server["persistentVolume"]["size"]
+
+    assert size.endswith("Gi")
+    assert int(size.removesuffix("Gi")) >= MINIMUM_STORAGE_GIB
+    assert server["extraArgs"]["retentionPeriod"] == "1y"
+
+
+def test_full_metrics_storage_pages_before_it_blocks_ingestion() -> None:
+    """Warn on shrinking headroom and page once the database refuses writes."""
+    rules = _alert_rules()
+    low = rules["victoria-metrics-storage-low"]
+    read_only = rules["victoria-metrics-storage-readonly"]
+
+    assert "vm_free_disk_space_bytes" in low["data"][0]["model"]["expr"]
+    assert low["labels"]["severity"] == "warning"
+    assert "vm_storage_is_read_only" in read_only["data"][0]["model"]["expr"]
+    assert read_only["labels"]["severity"] == "critical"
+
+
+def test_a_silent_metrics_database_still_pages() -> None:
+    """A stalled database returns no samples, so absent data must alert."""
+    rules = _alert_rules()
+    stalled = rules["victoria-metrics-ingestion-stalled"]
+
+    assert "vm_rows_added_to_storage_total" in stalled["data"][0]["model"]["expr"]
+    assert stalled["noDataState"] == "Alerting"
+    assert stalled["execErrState"] == "Alerting"
+    assert stalled["labels"]["severity"] == "critical"
+    assert rules["victoria-metrics-storage-readonly"]["noDataState"] == "Alerting"

services/monitoring/grafana-alerting-config.yaml

@@ -250,6 +250,154 @@ data:
               summary: "VictoriaMetrics is unavailable for >30m"
             labels:
               severity: critical
+          # The database can be up, scraped, and answering queries while
+          # refusing every new sample. These three rules watch the write path
+          # rather than liveness, and they alert on absent data because a
+          # stalled database stops producing the very series they read.
+          - uid: victoria-metrics-storage-low
+            title: "VictoriaMetrics storage headroom low (<15%)"
+            condition: C
+            for: "30m"
+            data:
+              - refId: A
+                relativeTimeRange:
+                  from: 600
+                  to: 0
+                datasourceUid: atlas-vm
+                model:
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  expr: 100 * sum(vm_free_disk_space_bytes{job="victoriametrics"}) / clamp_min(sum(vm_free_disk_space_bytes{job="victoriametrics"}) + sum(vm_data_size_bytes{job="victoriametrics"}), 1)
+                  legendFormat: free %
+                  datasource:
+                    type: prometheus
+                    uid: atlas-vm
+              - refId: B
+                datasourceUid: __expr__
+                model:
+                  expression: A
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  reducer: last
+                  type: reduce
+              - refId: C
+                datasourceUid: __expr__
+                model:
+                  expression: B
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  type: threshold
+                  conditions:
+                    - evaluator:
+                        params: [15]
+                        type: lt
+                      operator:
+                        type: and
+                      reducer:
+                        type: last
+                      type: query
+            noDataState: Alerting
+            execErrState: Alerting
+            annotations:
+              summary: "VictoriaMetrics has <15% free space on /storage; expand the volume before it stops accepting samples"
+            labels:
+              severity: warning
+          - uid: victoria-metrics-storage-readonly
+            title: "VictoriaMetrics storage is read-only"
+            condition: C
+            for: "5m"
+            data:
+              - refId: A
+                relativeTimeRange:
+                  from: 600
+                  to: 0
+                datasourceUid: atlas-vm
+                model:
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  expr: max(vm_storage_is_read_only{job="victoriametrics"}) or on() vector(0)
+                  legendFormat: read-only
+                  datasource:
+                    type: prometheus
+                    uid: atlas-vm
+              - refId: B
+                datasourceUid: __expr__
+                model:
+                  expression: A
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  reducer: last
+                  type: reduce
+              - refId: C
+                datasourceUid: __expr__
+                model:
+                  expression: B
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  type: threshold
+                  conditions:
+                    - evaluator:
+                        params: [0]
+                        type: gt
+                      operator:
+                        type: and
+                      reducer:
+                        type: last
+                      type: query
+            noDataState: Alerting
+            execErrState: Alerting
+            annotations:
+              summary: "VictoriaMetrics flipped /storage to read-only and is discarding every new sample"
+            labels:
+              severity: critical
+          - uid: victoria-metrics-ingestion-stalled
+            title: "VictoriaMetrics is not storing samples"
+            condition: C
+            for: "15m"
+            data:
+              - refId: A
+                relativeTimeRange:
+                  from: 900
+                  to: 0
+                datasourceUid: atlas-vm
+                model:
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  expr: sum(rate(vm_rows_added_to_storage_total{job="victoriametrics"}[10m])) or on() vector(0)
+                  legendFormat: rows/s
+                  datasource:
+                    type: prometheus
+                    uid: atlas-vm
+              - refId: B
+                datasourceUid: __expr__
+                model:
+                  expression: A
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  reducer: last
+                  type: reduce
+              - refId: C
+                datasourceUid: __expr__
+                model:
+                  expression: B
+                  intervalMs: 60000
+                  maxDataPoints: 43200
+                  type: threshold
+                  conditions:
+                    - evaluator:
+                        params: [1]
+                        type: lt
+                      operator:
+                        type: and
+                      reducer:
+                        type: last
+                      type: query
+            noDataState: Alerting
+            execErrState: Alerting
+            annotations:
+              summary: "VictoriaMetrics stored no samples for 15m; every dashboard is about to read empty"
+            labels:
+              severity: critical
       - orgId: 1
         name: maintenance
         folder: Alerts

services/monitoring/helmrelease.yaml

@@ -91,7 +91,12 @@ spec:
 
       persistentVolume:
         enabled: true
-        size: 100Gi
+        # A year of Atlas cardinality is ~210Gi today, and this 100Gi never
+        # matched the 200Gi volume actually bound in the cluster. That volume
+        # filled on 2026-08-18: VictoriaMetrics went read-only, ingestion
+        # stopped, and every public dashboard panel rendered "No data" while
+        # all 119 scrape targets stayed healthy.
+        size: 400Gi
       resources:
         requests:
           cpu: 500m