bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring: add compact availability fallback

Browse Source
This commit is contained in:
jenkins 2026-08-04 21:14:06 -03:00
parent f724e72a96
commit 7c47a92610
4 changed files with 16 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/dashboards_render_atlas.py
View File

@ -471,8 +471,15 @@ UPTIME_LIVE_FALLBACK_EXPR = (
f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / "
f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
)
UPTIME_COMPACT_FALLBACK_EXPR = (
"(1 - (sum_over_time(atlas:availability:failures_1h{"
'scope="atlas",definition="request-v4"}[365d]) / '
"clamp_min(sum_over_time(atlas:availability:requests_1h{"
'scope="atlas",definition="request-v4"}[365d]), 1)))'
)
UPTIME_RECORDING_EXPR = (
f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
f"or on() {UPTIME_COMPACT_FALLBACK_EXPR} "
f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
)
@ -2142,7 +2149,7 @@ def build_overview():
"decimals": 4,
"text_mode": "value",
"instant": True,
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour.",
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists.",
},
{
"id": 4,

5
scripts/tests/test_dashboards_render_atlas.py
View File

@ -65,12 +65,15 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
)
assert 'code=~"5.."' in availability_expr
assert 'code=~"[1-5].."' in availability_expr
assert "atlas:availability:failures_1h" in availability_expr
assert "atlas:availability:requests_1h" in availability_expr
assert "sum_over_time" in availability_expr
assert "kube_node_status_condition" not in availability_expr
assert "kube_deployment_status_replicas_available" not in availability_expr
assert panel["targets"][0]["instant"] is True
assert "Every server-side 5xx" in panel["description"]
assert "Replica counts, Grafana health" in panel["description"]
assert "falls back to the same request SLI" in panel["description"]
assert "can rebuild it from compact hourly rollups" in panel["description"]
def test_overview_uses_readable_quality_power_and_gitops_panels():

4
services/monitoring/dashboards/atlas-overview.json
View File

@ -229,7 +229,7 @@
},
"targets": [
{
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"refId": "A",
"instant": true
}
@ -286,7 +286,7 @@
},
"textMode": "value"
},
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour."
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists."
},
{
"id": 4,

4
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -238,7 +238,7 @@ data:
},
"targets": [
{
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"refId": "A",
"instant": true
}
@ -295,7 +295,7 @@ data:
},
"textMode": "value"
},
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour."
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful annual sample for up to 24 hours, can rebuild it from compact hourly rollups, and only falls back to the same one-hour request SLI before history exists."
},
{
"id": 4,