From bbab7330ce3fefbb05aca66c17e583804f08092f Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 4 Aug 2026 18:06:43 -0300 Subject: [PATCH] monitoring: avoid unstable volume hosts --- scripts/dashboards_render_atlas.py | 10 +++--- scripts/tests/test_dashboards_render_atlas.py | 11 ++++--- .../tests/test_monitoring_query_capacity.py | 31 +++++++++++++++++++ .../monitoring/dashboards/atlas-overview.json | 6 ++-- .../grafana-dashboard-overview.yaml | 6 ++-- services/monitoring/helmrelease.yaml | 9 ++++++ .../vmalert-atlas-availability.yaml | 8 +++-- 7 files changed, 64 insertions(+), 17 deletions(-) diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index ef6ccca0a..eb46a4df8 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -456,7 +456,9 @@ STUCK_TERMINATING_EXPR = ( ) UPTIME_WINDOW = "365d" # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series. -UPTIME_RECORDING_METRIC = f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas"}}' +UPTIME_RECORDING_METRIC = ( + f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}' +) UPTIME_RECORDING_EXPR = f"last_over_time({UPTIME_RECORDING_METRIC}[24h])" TRAEFIK_READY_EXPR = ( "(" @@ -1898,7 +1900,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = { "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.", "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.", "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.", - "Atlas Availability (365d)": "Rolling one-year Atlas availability; higher is better, below target means users saw downtime.", + "Atlas Serving Availability": "Observed availability with control-plane quorum and at least one Traefik replica serving; partial replica capacity remains available.", "Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.", "CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.", "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.", @@ -2131,7 +2133,7 @@ def build_overview(): }, { "id": 27, - "title": "Atlas Availability (365d)", + "title": "Atlas Serving Availability", "expr": UPTIME_PERCENT_EXPR, "kind": "stat", "thresholds": UPTIME_PERCENT_THRESHOLDS, @@ -2139,7 +2141,7 @@ def build_overview(): "decimals": 4, "text_mode": "value", "instant": True, - "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data.", + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h.", }, { "id": 4, diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 8b31c4448..a38c063c0 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -57,11 +57,14 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): dashboard = mod.build_overview() panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27) - assert panel["title"] == "Atlas Availability (365d)" - assert panel["targets"][0]["expr"] == 'last_over_time(atlas:availability:ratio_365d{scope="atlas"}[24h])' + assert panel["title"] == "Atlas Serving Availability" + assert panel["targets"][0]["expr"] == ( + 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])' + ) assert panel["targets"][0]["instant"] is True - assert "precomputed" in panel["description"] - assert "last successful rollup for up to 24h" in panel["description"] + assert "at least one Traefik replica serving" in panel["description"] + assert "Partial replica capacity remains available" in panel["description"] + assert "monitoring gaps are excluded" in panel["description"] def test_overview_uses_readable_quality_power_and_gitops_panels(): diff --git a/scripts/tests/test_monitoring_query_capacity.py b/scripts/tests/test_monitoring_query_capacity.py index 8b33e68c0..f5d5da36a 100644 --- a/scripts/tests/test_monitoring_query_capacity.py +++ b/scripts/tests/test_monitoring_query_capacity.py @@ -31,6 +31,18 @@ def test_victoria_metrics_has_dashboard_burst_headroom() -> None: assert server["resources"]["limits"]["cpu"] == "2" assert server["resources"]["limits"]["memory"] == "4Gi" + required_terms = server["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"] + hostname_rule = next( + expression + for term in required_terms + for expression in term["matchExpressions"] + if expression["key"] == "kubernetes.io/hostname" + ) + assert hostname_rule["operator"] == "NotIn" + assert {"titan-14", "titan-18"} <= set(hostname_rule["values"]) + def test_yearly_availability_reuses_the_hourly_rollup() -> None: """Prevent the yearly rule from rescanning raw cluster metrics for 365 days.""" @@ -45,10 +57,29 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None: ) assert "atlas:availability:ratio_1h" in yearly["expr"] + assert 'definition="serving-v2"' in yearly["expr"] assert "kube_node_status_condition" not in yearly["expr"] assert "[365d:1h]" not in yearly["expr"] +def test_availability_counts_serving_state_instead_of_replica_capacity() -> None: + """Treat quorum and any serving ingress replica as available binary states.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + )[0] + rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0] + hourly = next( + rule + for rule in rules["rules"] + if rule["record"] == "atlas:availability:ratio_1h" + ) + + assert ">= bool 2" in hourly["expr"] + assert "> bool 0" in hourly["expr"] + assert "/ 3" not in hourly["expr"] + assert hourly["labels"]["definition"] == "serving-v2" + + def test_quality_rollups_do_not_run_every_minute() -> None: """Keep high-cardinality quality rollups below the backend saturation cadence.""" manifest = _documents( diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index a97ec2db4..03e580be0 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -216,7 +216,7 @@ { "id": 27, "type": "stat", - "title": "Atlas Availability (365d)", + "title": "Atlas Serving Availability", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\"}[24h])", + "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h])", "refId": "A", "instant": true } @@ -286,7 +286,7 @@ }, "textMode": "value" }, - "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data." + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h." }, { "id": 4, diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index 377ff7c3e..b9f869a38 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -225,7 +225,7 @@ data: { "id": 27, "type": "stat", - "title": "Atlas Availability (365d)", + "title": "Atlas Serving Availability", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\"}[24h])", + "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h])", "refId": "A", "instant": true } @@ -295,7 +295,7 @@ data: }, "textMode": "value" }, - "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data." + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Partial replica capacity remains available; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h." }, { "id": 4, diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index f83c7df10..7242b6c15 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -111,6 +111,10 @@ spec: - key: kubernetes.io/hostname operator: NotIn values: + # Longhorn engine-image probes repeatedly fail on these + # attachment hosts and can surface volume I/O errors. + - titan-14 + - titan-18 - titan-20 - titan-21 - titan-22 @@ -130,6 +134,11 @@ spec: operator: In values: - rpi4 + - weight: 25 + preference: + matchExpressions: + - key: node-role.kubernetes.io/storage-backbone + operator: Exists # Enable built-in Kubernetes scraping scrape: diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index d387fa0b4..901ad1f6c 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -17,23 +17,25 @@ data: min( ( sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 + >= bool 2 ), ( sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) + > bool 0 ) ) )[1h:5m]) labels: + definition: serving-v2 scope: atlas rollup: hourly - record: atlas:availability:ratio_365d expr: | avg_over_time( - atlas:availability:ratio_1h{scope="atlas"}[365d] + atlas:availability:ratio_1h{scope="atlas",definition="serving-v2"}[365d] ) labels: + definition: serving-v2 scope: atlas rollup: yearly platform-quality.yaml: |