From 7c47a92610aa59e646a470c3b839ddf7f6e2a0ab Mon Sep 17 00:00:00 2001 From: jenkins Date: Tue, 4 Aug 2026 21:14:06 -0300 Subject: [PATCH] monitoring: add compact availability fallback --- scripts/dashboards_render_atlas.py | 9 ++++++++- scripts/tests/test_dashboards_render_atlas.py | 5 ++++- services/monitoring/dashboards/atlas-overview.json | 4 ++-- services/monitoring/grafana-dashboard-overview.yaml | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index f36f41714..13310f7ea 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -471,8 +471,15 @@ UPTIME_LIVE_FALLBACK_EXPR = ( f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / " f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))" ) +UPTIME_COMPACT_FALLBACK_EXPR = ( + "(1 - (sum_over_time(atlas:availability:failures_1h{" + 'scope="atlas",definition="request-v4"}[365d]) / ' + "clamp_min(sum_over_time(atlas:availability:requests_1h{" + 'scope="atlas",definition="request-v4"}[365d]), 1)))' +) UPTIME_RECORDING_EXPR = ( f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) " + f"or on() {UPTIME_COMPACT_FALLBACK_EXPR} " f"or on() {UPTIME_LIVE_FALLBACK_EXPR})" ) @@ -2142,7 +2149,7 @@ def build_overview(): "decimals": 4, "text_mode": "value", "instant": True, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour.", + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists.", }, { "id": 4, diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 08938be4c..a51e98d69 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -65,12 +65,15 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): ) assert 'code=~"5.."' in availability_expr assert 'code=~"[1-5].."' in availability_expr + assert "atlas:availability:failures_1h" in availability_expr + assert "atlas:availability:requests_1h" in availability_expr + assert "sum_over_time" in availability_expr assert "kube_node_status_condition" not in availability_expr assert "kube_deployment_status_replicas_available" not in availability_expr assert panel["targets"][0]["instant"] is True assert "Every server-side 5xx" in panel["description"] assert "Replica counts, Grafana health" in panel["description"] - assert "falls back to the same request SLI" in panel["description"] + assert "can rebuild it from compact hourly rollups" in panel["description"] def test_overview_uses_readable_quality_power_and_gitops_panels(): diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index f05295731..a205253ee 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -286,7 +286,7 @@ }, "textMode": "value" }, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists." }, { "id": 4, diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index b742e2e9b..fe18513a6 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -295,7 +295,7 @@ data: }, "textMode": "value" }, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists." }, { "id": 4,