bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring: use request success for availability

Browse Source
This commit is contained in:
jenkins 2026-08-04 21:12:31 -03:00
parent 4906fb73d8
commit f724e72a96
8 changed files with 55 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scripts/dashboards_render_atlas.py
View File

@ -457,19 +457,19 @@ STUCK_TERMINATING_EXPR = (
UPTIME_WINDOW = "365d"
# vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series.
UPTIME_RECORDING_METRIC = (
f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="gateway-v3"}}'
f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="request-v4"}}'
)
GATEWAY_REQUESTS_1H_EXPR = (
AVAILABILITY_REQUESTS_1H_EXPR = (
'sum(increase(traefik_entrypoint_requests_total{'
'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))'
)
GATEWAY_FAILURES_1H_EXPR = (
AVAILABILITY_FAILURES_1H_EXPR = (
'sum(increase(traefik_entrypoint_requests_total{'
'entrypoint="websecure",protocol="http",code=~"502|503|504"}[1h]))'
'entrypoint="websecure",protocol="http",code=~"5.."}[1h]))'
)
UPTIME_LIVE_FALLBACK_EXPR = (
f"(1 - (({GATEWAY_FAILURES_1H_EXPR} or on() vector(0)) / "
f"clamp_min({GATEWAY_REQUESTS_1H_EXPR}, 1)))"
f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / "
f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
)
UPTIME_RECORDING_EXPR = (
f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
@ -1901,7 +1901,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
"Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.",
"Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.",
"Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.",
"Atlas Availability (365d)": "Request-weighted Atlas ingress availability; only Traefik 502, 503, and 504 gateway failures count as downtime.",
"Atlas Availability (365d)": "Request-weighted Atlas ingress availability; every server-side 5xx response counts as a failed request.",
"Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.",
"CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.",
"Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.",
@ -2142,7 +2142,7 @@ def build_overview():
"decimals": 4,
"text_mode": "value",
"instant": True,
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour.",
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour.",
},
{
"id": 4,

8
scripts/tests/test_dashboards_render_atlas.py
View File

@ -60,17 +60,17 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
assert panel["title"] == "Atlas Availability (365d)"
availability_expr = panel["targets"][0]["expr"]
assert (
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="gateway-v3"}[24h])'
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[24h])'
in availability_expr
)
assert 'code=~"502|503|504"' in availability_expr
assert 'code=~"5.."' in availability_expr
assert 'code=~"[1-5].."' in availability_expr
assert "kube_node_status_condition" not in availability_expr
assert "kube_deployment_status_replicas_available" not in availability_expr
assert panel["targets"][0]["instant"] is True
assert "502, 503, and 504 gateway failures" in panel["description"]
assert "Every server-side 5xx" in panel["description"]
assert "Replica counts, Grafana health" in panel["description"]
assert "falls back to the same gateway SLI" in panel["description"]
assert "falls back to the same request SLI" in panel["description"]
def test_overview_uses_readable_quality_power_and_gitops_panels():

32
scripts/tests/test_monitoring_query_capacity.py
View File

@ -57,17 +57,17 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None:
if rule["record"] == "atlas:availability:ratio_365d"
)
assert "atlas:availability:gateway_requests_1h" in yearly["expr"]
assert "atlas:availability:gateway_failures_1h" in yearly["expr"]
assert 'definition="gateway-v3"' in yearly["expr"]
assert "atlas:availability:requests_1h" in yearly["expr"]
assert "atlas:availability:failures_1h" in yearly["expr"]
assert 'definition="request-v4"' in yearly["expr"]
assert "sum_over_time" in yearly["expr"]
assert "[365d]" in yearly["expr"]
assert "traefik_entrypoint_requests_total" not in yearly["expr"]
assert yearly["labels"]["definition"] == "gateway-v3"
assert yearly["labels"]["definition"] == "request-v4"
def test_availability_uses_gateway_failures_instead_of_replica_capacity() -> None:
"""Measure request outcomes without treating redundant replica loss as downtime."""
def test_availability_uses_request_failures_instead_of_replica_capacity() -> None:
"""Measure HTTP outcomes without treating redundant replica loss as downtime."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
@ -76,40 +76,40 @@ def test_availability_uses_gateway_failures_instead_of_replica_capacity() -> Non
requests = next(
rule
for rule in rules
if rule["record"] == "atlas:availability:gateway_requests_1h"
if rule["record"] == "atlas:availability:requests_1h"
)
failures = next(
rule
for rule in rules
if rule["record"] == "atlas:availability:gateway_failures_1h"
if rule["record"] == "atlas:availability:failures_1h"
)
assert 'code=~"[1-5].."' in requests["expr"]
assert 'code=~"502|503|504"' in failures["expr"]
assert 'code=~"5.."' in failures["expr"]
assert "traefik_entrypoint_requests_total" in requests["expr"]
assert "traefik_entrypoint_requests_total" in failures["expr"]
assert "kube_node_status_condition" not in repr(rules)
assert "kube_deployment_status_replicas_available" not in repr(rules)
assert requests["labels"]["definition"] == "gateway-v3"
assert failures["labels"]["definition"] == "gateway-v3"
assert requests["labels"]["definition"] == "request-v4"
assert failures["labels"]["definition"] == "request-v4"
def test_availability_backfill_replays_the_same_gateway_sli() -> None:
def test_availability_backfill_replays_the_same_request_sli() -> None:
"""Backfill retained history with the exact rules used for future samples."""
manifests = _documents(
REPO_ROOT / "services/monitoring/availability-backfill-v3-job.yaml"
REPO_ROOT / "services/monitoring/availability-backfill-v4-job.yaml"
)
config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
backfill = yaml.safe_load(config["data"]["atlas-gateway-history.yaml"])
backfill = yaml.safe_load(config["data"]["atlas-request-history.yaml"])
expressions = {
rule["record"]: rule["expr"]
for group in backfill["groups"]
for rule in group["rules"]
}
assert 'code=~"[1-5].."' in expressions["atlas:availability:gateway_requests_1h"]
assert 'code=~"502|503|504"' in expressions["atlas:availability:gateway_failures_1h"]
assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1h"]
assert 'code=~"5.."' in expressions["atlas:availability:failures_1h"]
args = job["spec"]["template"]["spec"]["containers"][0]["args"]
assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args
assert "-replay.timeTo=2026-08-04T23:00:00Z" in args

24
services/monitoring/availability-backfill-v3-job.yaml → services/monitoring/availability-backfill-v4-job.yaml
View File

@ -1,16 +1,16 @@
# services/monitoring/availability-backfill-v3-job.yaml
# services/monitoring/availability-backfill-v4-job.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: atlas-availability-gateway-v3-backfill-rules
name: atlas-availability-request-v4-backfill-rules
namespace: monitoring
data:
atlas-gateway-history.yaml: |
atlas-request-history.yaml: |
groups:
- name: atlas.availability.gateway.backfill
- name: atlas.availability.request.backfill
interval: 1h
rules:
- record: atlas:availability:gateway_requests_1h
- record: atlas:availability:requests_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
@ -20,20 +20,20 @@ data:
}[1h]
))
labels:
definition: gateway-v3
definition: request-v4
scope: atlas
rollup: hourly
- record: atlas:availability:gateway_failures_1h
- record: atlas:availability:failures_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"502|503|504"
code=~"5.."
}[1h]
))
labels:
definition: gateway-v3
definition: request-v4
scope: atlas
rollup: hourly
@ -42,14 +42,14 @@ data:
apiVersion: batch/v1
kind: Job
metadata:
name: atlas-availability-gateway-v3-backfill
name: atlas-availability-request-v4-backfill
namespace: monitoring
spec:
backoffLimit: 2
template:
metadata:
labels:
app: atlas-availability-gateway-v3-backfill
app: atlas-availability-request-v4-backfill
spec:
restartPolicy: Never
affinity:
@ -89,4 +89,4 @@ spec:
volumes:
- name: rules
configMap:
name: atlas-availability-gateway-v3-backfill-rules
name: atlas-availability-request-v4-backfill-rules

4
services/monitoring/dashboards/atlas-overview.json
View File

@ -229,7 +229,7 @@
},
"targets": [
{
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"gateway-v3\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"502|503|504\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"refId": "A",
"instant": true
}
@ -286,7 +286,7 @@
},
"textMode": "value"
},
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour."
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour."
},
{
"id": 4,

4
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -238,7 +238,7 @@ data:
},
"targets": [
{
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"gateway-v3\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"502|503|504\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"refId": "A",
"instant": true
}
@ -295,7 +295,7 @@ data:
},
"textMode": "value"
},
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour."
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour."
},
{
"id": 4,

2
services/monitoring/kustomization.yaml
View File

@ -19,7 +19,7 @@ resources:
- grafana-dashboard-mail.yaml
- grafana-dashboard-testing.yaml
- vmalert-atlas-availability.yaml
- availability-backfill-v3-job.yaml
- availability-backfill-v4-job.yaml
- dcgm-exporter.yaml
- nvidia-process-exporter.yaml
- jetson-tegrastats-exporter.yaml

20
services/monitoring/vmalert-atlas-availability.yaml
View File

@ -11,7 +11,7 @@ data:
interval: 1h
eval_offset: 59m
rules:
- record: atlas:availability:gateway_requests_1h
- record: atlas:availability:requests_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
@ -21,20 +21,20 @@ data:
}[1h]
))
labels:
definition: gateway-v3
definition: request-v4
scope: atlas
rollup: hourly
- record: atlas:availability:gateway_failures_1h
- record: atlas:availability:failures_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"502|503|504"
code=~"5.."
}[1h]
))
labels:
definition: gateway-v3
definition: request-v4
scope: atlas
rollup: hourly
- name: atlas.availability.rollup
@ -45,24 +45,24 @@ data:
expr: |
1 - (
sum_over_time(
atlas:availability:gateway_failures_1h{
atlas:availability:failures_1h{
scope="atlas",
definition="gateway-v3"
definition="request-v4"
}[365d]
)
/
clamp_min(
sum_over_time(
atlas:availability:gateway_requests_1h{
atlas:availability:requests_1h{
scope="atlas",
definition="gateway-v3"
definition="request-v4"
}[365d]
),
1
)
)
labels:
definition: gateway-v3
definition: request-v4
scope: atlas
rollup: yearly
platform-quality.yaml: |