diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index 67d7e78f2..f36f41714 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -457,19 +457,19 @@ STUCK_TERMINATING_EXPR = ( UPTIME_WINDOW = "365d" # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series. UPTIME_RECORDING_METRIC = ( - f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="gateway-v3"}}' + f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="request-v4"}}' ) -GATEWAY_REQUESTS_1H_EXPR = ( +AVAILABILITY_REQUESTS_1H_EXPR = ( 'sum(increase(traefik_entrypoint_requests_total{' 'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))' ) -GATEWAY_FAILURES_1H_EXPR = ( +AVAILABILITY_FAILURES_1H_EXPR = ( 'sum(increase(traefik_entrypoint_requests_total{' - 'entrypoint="websecure",protocol="http",code=~"502|503|504"}[1h]))' + 'entrypoint="websecure",protocol="http",code=~"5.."}[1h]))' ) UPTIME_LIVE_FALLBACK_EXPR = ( - f"(1 - (({GATEWAY_FAILURES_1H_EXPR} or on() vector(0)) / " - f"clamp_min({GATEWAY_REQUESTS_1H_EXPR}, 1)))" + f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / " + f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))" ) UPTIME_RECORDING_EXPR = ( f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) " @@ -1901,7 +1901,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = { "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.", "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.", "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.", - "Atlas Availability (365d)": "Request-weighted Atlas ingress availability; only Traefik 502, 503, and 504 gateway failures count as downtime.", + "Atlas Availability (365d)": "Request-weighted Atlas ingress availability; every server-side 5xx response counts as a failed request.", "Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.", "CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.", "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.", @@ -2142,7 +2142,7 @@ def build_overview(): "decimals": 4, "text_mode": "value", "instant": True, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour.", + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour.", }, { "id": 4, diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index f9bb4d024..08938be4c 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -60,17 +60,17 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): assert panel["title"] == "Atlas Availability (365d)" availability_expr = panel["targets"][0]["expr"] assert ( - 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="gateway-v3"}[24h])' + 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[24h])' in availability_expr ) - assert 'code=~"502|503|504"' in availability_expr + assert 'code=~"5.."' in availability_expr assert 'code=~"[1-5].."' in availability_expr assert "kube_node_status_condition" not in availability_expr assert "kube_deployment_status_replicas_available" not in availability_expr assert panel["targets"][0]["instant"] is True - assert "502, 503, and 504 gateway failures" in panel["description"] + assert "Every server-side 5xx" in panel["description"] assert "Replica counts, Grafana health" in panel["description"] - assert "falls back to the same gateway SLI" in panel["description"] + assert "falls back to the same request SLI" in panel["description"] def test_overview_uses_readable_quality_power_and_gitops_panels(): diff --git a/scripts/tests/test_monitoring_query_capacity.py b/scripts/tests/test_monitoring_query_capacity.py index 0b53a89f7..a40a7b071 100644 --- a/scripts/tests/test_monitoring_query_capacity.py +++ b/scripts/tests/test_monitoring_query_capacity.py @@ -57,17 +57,17 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None: if rule["record"] == "atlas:availability:ratio_365d" ) - assert "atlas:availability:gateway_requests_1h" in yearly["expr"] - assert "atlas:availability:gateway_failures_1h" in yearly["expr"] - assert 'definition="gateway-v3"' in yearly["expr"] + assert "atlas:availability:requests_1h" in yearly["expr"] + assert "atlas:availability:failures_1h" in yearly["expr"] + assert 'definition="request-v4"' in yearly["expr"] assert "sum_over_time" in yearly["expr"] assert "[365d]" in yearly["expr"] assert "traefik_entrypoint_requests_total" not in yearly["expr"] - assert yearly["labels"]["definition"] == "gateway-v3" + assert yearly["labels"]["definition"] == "request-v4" -def test_availability_uses_gateway_failures_instead_of_replica_capacity() -> None: - """Measure request outcomes without treating redundant replica loss as downtime.""" +def test_availability_uses_request_failures_instead_of_replica_capacity() -> None: + """Measure HTTP outcomes without treating redundant replica loss as downtime.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] @@ -76,40 +76,40 @@ def test_availability_uses_gateway_failures_instead_of_replica_capacity() -> Non requests = next( rule for rule in rules - if rule["record"] == "atlas:availability:gateway_requests_1h" + if rule["record"] == "atlas:availability:requests_1h" ) failures = next( rule for rule in rules - if rule["record"] == "atlas:availability:gateway_failures_1h" + if rule["record"] == "atlas:availability:failures_1h" ) assert 'code=~"[1-5].."' in requests["expr"] - assert 'code=~"502|503|504"' in failures["expr"] + assert 'code=~"5.."' in failures["expr"] assert "traefik_entrypoint_requests_total" in requests["expr"] assert "traefik_entrypoint_requests_total" in failures["expr"] assert "kube_node_status_condition" not in repr(rules) assert "kube_deployment_status_replicas_available" not in repr(rules) - assert requests["labels"]["definition"] == "gateway-v3" - assert failures["labels"]["definition"] == "gateway-v3" + assert requests["labels"]["definition"] == "request-v4" + assert failures["labels"]["definition"] == "request-v4" -def test_availability_backfill_replays_the_same_gateway_sli() -> None: +def test_availability_backfill_replays_the_same_request_sli() -> None: """Backfill retained history with the exact rules used for future samples.""" manifests = _documents( - REPO_ROOT / "services/monitoring/availability-backfill-v3-job.yaml" + REPO_ROOT / "services/monitoring/availability-backfill-v4-job.yaml" ) config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap") job = next(manifest for manifest in manifests if manifest["kind"] == "Job") - backfill = yaml.safe_load(config["data"]["atlas-gateway-history.yaml"]) + backfill = yaml.safe_load(config["data"]["atlas-request-history.yaml"]) expressions = { rule["record"]: rule["expr"] for group in backfill["groups"] for rule in group["rules"] } - assert 'code=~"[1-5].."' in expressions["atlas:availability:gateway_requests_1h"] - assert 'code=~"502|503|504"' in expressions["atlas:availability:gateway_failures_1h"] + assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1h"] + assert 'code=~"5.."' in expressions["atlas:availability:failures_1h"] args = job["spec"]["template"]["spec"]["containers"][0]["args"] assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args assert "-replay.timeTo=2026-08-04T23:00:00Z" in args diff --git a/services/monitoring/availability-backfill-v3-job.yaml b/services/monitoring/availability-backfill-v4-job.yaml similarity index 78% rename from services/monitoring/availability-backfill-v3-job.yaml rename to services/monitoring/availability-backfill-v4-job.yaml index 5f420ce74..3aedd0c1f 100644 --- a/services/monitoring/availability-backfill-v3-job.yaml +++ b/services/monitoring/availability-backfill-v4-job.yaml @@ -1,16 +1,16 @@ -# services/monitoring/availability-backfill-v3-job.yaml +# services/monitoring/availability-backfill-v4-job.yaml apiVersion: v1 kind: ConfigMap metadata: - name: atlas-availability-gateway-v3-backfill-rules + name: atlas-availability-request-v4-backfill-rules namespace: monitoring data: - atlas-gateway-history.yaml: | + atlas-request-history.yaml: | groups: - - name: atlas.availability.gateway.backfill + - name: atlas.availability.request.backfill interval: 1h rules: - - record: atlas:availability:gateway_requests_1h + - record: atlas:availability:requests_1h expr: | sum(increase( traefik_entrypoint_requests_total{ @@ -20,20 +20,20 @@ data: }[1h] )) labels: - definition: gateway-v3 + definition: request-v4 scope: atlas rollup: hourly - - record: atlas:availability:gateway_failures_1h + - record: atlas:availability:failures_1h expr: | sum(increase( traefik_entrypoint_requests_total{ entrypoint="websecure", protocol="http", - code=~"502|503|504" + code=~"5.." }[1h] )) labels: - definition: gateway-v3 + definition: request-v4 scope: atlas rollup: hourly @@ -42,14 +42,14 @@ data: apiVersion: batch/v1 kind: Job metadata: - name: atlas-availability-gateway-v3-backfill + name: atlas-availability-request-v4-backfill namespace: monitoring spec: backoffLimit: 2 template: metadata: labels: - app: atlas-availability-gateway-v3-backfill + app: atlas-availability-request-v4-backfill spec: restartPolicy: Never affinity: @@ -89,4 +89,4 @@ spec: volumes: - name: rules configMap: - name: atlas-availability-gateway-v3-backfill-rules + name: atlas-availability-request-v4-backfill-rules diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index ba1814315..f05295731 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"gateway-v3\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"502|503|504\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -286,7 +286,7 @@ }, "textMode": "value" }, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour." }, { "id": 4, diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index a03a3fc9e..b742e2e9b 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"gateway-v3\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"502|503|504\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -295,7 +295,7 @@ data: }, "textMode": "value" }, - "description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same request SLI over the last hour." }, { "id": 4, diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index de41b951e..d54b86f5c 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -19,7 +19,7 @@ resources: - grafana-dashboard-mail.yaml - grafana-dashboard-testing.yaml - vmalert-atlas-availability.yaml - - availability-backfill-v3-job.yaml + - availability-backfill-v4-job.yaml - dcgm-exporter.yaml - nvidia-process-exporter.yaml - jetson-tegrastats-exporter.yaml diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index 333493fe5..68ec402ae 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -11,7 +11,7 @@ data: interval: 1h eval_offset: 59m rules: - - record: atlas:availability:gateway_requests_1h + - record: atlas:availability:requests_1h expr: | sum(increase( traefik_entrypoint_requests_total{ @@ -21,20 +21,20 @@ data: }[1h] )) labels: - definition: gateway-v3 + definition: request-v4 scope: atlas rollup: hourly - - record: atlas:availability:gateway_failures_1h + - record: atlas:availability:failures_1h expr: | sum(increase( traefik_entrypoint_requests_total{ entrypoint="websecure", protocol="http", - code=~"502|503|504" + code=~"5.." }[1h] )) labels: - definition: gateway-v3 + definition: request-v4 scope: atlas rollup: hourly - name: atlas.availability.rollup @@ -45,24 +45,24 @@ data: expr: | 1 - ( sum_over_time( - atlas:availability:gateway_failures_1h{ + atlas:availability:failures_1h{ scope="atlas", - definition="gateway-v3" + definition="request-v4" }[365d] ) / clamp_min( sum_over_time( - atlas:availability:gateway_requests_1h{ + atlas:availability:requests_1h{ scope="atlas", - definition="gateway-v3" + definition="request-v4" }[365d] ), 1 ) ) labels: - definition: gateway-v3 + definition: request-v4 scope: atlas rollup: yearly platform-quality.yaml: |