diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index 9133a4d70..67d7e78f2 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -457,21 +457,20 @@ STUCK_TERMINATING_EXPR = ( UPTIME_WINDOW = "365d" # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series. UPTIME_RECORDING_METRIC = ( - f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}' + f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="gateway-v3"}}' ) -TRAEFIK_READY_EXPR = ( - "(" - 'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})' - " > bool 0)" +GATEWAY_REQUESTS_1H_EXPR = ( + 'sum(increase(traefik_entrypoint_requests_total{' + 'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))' ) -CONTROL_READY_FRACTION_EXPR = ( - f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})" - " >= bool 2)" +GATEWAY_FAILURES_1H_EXPR = ( + 'sum(increase(traefik_entrypoint_requests_total{' + 'entrypoint="websecure",protocol="http",code=~"502|503|504"}[1h]))' ) -UPTIME_AVAIL_EXPR = ( - f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))" +UPTIME_LIVE_FALLBACK_EXPR = ( + f"(1 - (({GATEWAY_FAILURES_1H_EXPR} or on() vector(0)) / " + f"clamp_min({GATEWAY_REQUESTS_1H_EXPR}, 1)))" ) -UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])" UPTIME_RECORDING_EXPR = ( f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) " f"or on() {UPTIME_LIVE_FALLBACK_EXPR})" @@ -1902,7 +1901,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = { "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.", "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.", "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.", - "Atlas Serving Availability": "Observed availability with control-plane quorum and at least one Traefik replica serving; partial replica capacity remains available.", + "Atlas Availability (365d)": "Request-weighted Atlas ingress availability; only Traefik 502, 503, and 504 gateway failures count as downtime.", "Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.", "CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.", "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.", @@ -2135,7 +2134,7 @@ def build_overview(): }, { "id": 27, - "title": "Atlas Serving Availability", + "title": "Atlas Availability (365d)", "expr": UPTIME_PERCENT_EXPR, "kind": "stat", "thresholds": UPTIME_PERCENT_THRESHOLDS, @@ -2143,7 +2142,7 @@ def build_overview(): "decimals": 4, "text_mode": "value", "instant": True, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.", + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour.", }, { "id": 4, diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 61791040b..f9bb4d024 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -57,20 +57,20 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): dashboard = mod.build_overview() panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27) - assert panel["title"] == "Atlas Serving Availability" + assert panel["title"] == "Atlas Availability (365d)" availability_expr = panel["targets"][0]["expr"] assert ( - 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])' + 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="gateway-v3"}[24h])' in availability_expr ) - assert ">= bool 2" in availability_expr - assert "> bool 0" in availability_expr - assert "[1h:5m]" in availability_expr + assert 'code=~"502|503|504"' in availability_expr + assert 'code=~"[1-5].."' in availability_expr + assert "kube_node_status_condition" not in availability_expr + assert "kube_deployment_status_replicas_available" not in availability_expr assert panel["targets"][0]["instant"] is True - assert "at least one Traefik replica serving" in panel["description"] - assert "partial-capacity samples remain available" in panel["description"] - assert "monitoring gaps are excluded" in panel["description"] - assert "falls back to the live binary serving state" in panel["description"] + assert "502, 503, and 504 gateway failures" in panel["description"] + assert "Replica counts, Grafana health" in panel["description"] + assert "falls back to the same gateway SLI" in panel["description"] def test_overview_uses_readable_quality_power_and_gitops_panels(): diff --git a/scripts/tests/test_monitoring_query_capacity.py b/scripts/tests/test_monitoring_query_capacity.py index 714679910..0b53a89f7 100644 --- a/scripts/tests/test_monitoring_query_capacity.py +++ b/scripts/tests/test_monitoring_query_capacity.py @@ -49,38 +49,71 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None: manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] - rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0] + groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] + rules = [rule for group in groups for rule in group["rules"]] yearly = next( rule - for rule in rules["rules"] + for rule in rules if rule["record"] == "atlas:availability:ratio_365d" ) - assert "atlas:availability:ratio_1h" in yearly["expr"] - assert 'definition="serving-v2"' in yearly["expr"] - assert 'definition=""' in yearly["expr"] - assert "share_gt_over_time" in yearly["expr"] - assert "[365d:15m]" in yearly["expr"] - assert "kube_node_status_condition" not in yearly["expr"] - assert "[365d:1h]" not in yearly["expr"] + assert "atlas:availability:gateway_requests_1h" in yearly["expr"] + assert "atlas:availability:gateway_failures_1h" in yearly["expr"] + assert 'definition="gateway-v3"' in yearly["expr"] + assert "sum_over_time" in yearly["expr"] + assert "[365d]" in yearly["expr"] + assert "traefik_entrypoint_requests_total" not in yearly["expr"] + assert yearly["labels"]["definition"] == "gateway-v3" -def test_availability_counts_serving_state_instead_of_replica_capacity() -> None: - """Treat quorum and any serving ingress replica as available binary states.""" +def test_availability_uses_gateway_failures_instead_of_replica_capacity() -> None: + """Measure request outcomes without treating redundant replica loss as downtime.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] - rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0] - hourly = next( + groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] + rules = [rule for group in groups for rule in group["rules"]] + requests = next( rule - for rule in rules["rules"] - if rule["record"] == "atlas:availability:ratio_1h" + for rule in rules + if rule["record"] == "atlas:availability:gateway_requests_1h" + ) + failures = next( + rule + for rule in rules + if rule["record"] == "atlas:availability:gateway_failures_1h" ) - assert ">= bool 2" in hourly["expr"] - assert "> bool 0" in hourly["expr"] - assert "/ 3" not in hourly["expr"] - assert hourly["labels"]["definition"] == "serving-v2" + assert 'code=~"[1-5].."' in requests["expr"] + assert 'code=~"502|503|504"' in failures["expr"] + assert "traefik_entrypoint_requests_total" in requests["expr"] + assert "traefik_entrypoint_requests_total" in failures["expr"] + assert "kube_node_status_condition" not in repr(rules) + assert "kube_deployment_status_replicas_available" not in repr(rules) + assert requests["labels"]["definition"] == "gateway-v3" + assert failures["labels"]["definition"] == "gateway-v3" + + +def test_availability_backfill_replays_the_same_gateway_sli() -> None: + """Backfill retained history with the exact rules used for future samples.""" + manifests = _documents( + REPO_ROOT / "services/monitoring/availability-backfill-v3-job.yaml" + ) + config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap") + job = next(manifest for manifest in manifests if manifest["kind"] == "Job") + backfill = yaml.safe_load(config["data"]["atlas-gateway-history.yaml"]) + expressions = { + rule["record"]: rule["expr"] + for group in backfill["groups"] + for rule in group["rules"] + } + + assert 'code=~"[1-5].."' in expressions["atlas:availability:gateway_requests_1h"] + assert 'code=~"502|503|504"' in expressions["atlas:availability:gateway_failures_1h"] + args = job["spec"]["template"]["spec"]["containers"][0]["args"] + assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args + assert "-replay.timeTo=2026-08-04T23:00:00Z" in args + assert "-replay.maxDatapointsPerQuery=48" in args def test_quality_rollups_do_not_run_every_minute() -> None: diff --git a/services/monitoring/availability-backfill-v3-job.yaml b/services/monitoring/availability-backfill-v3-job.yaml new file mode 100644 index 000000000..5f420ce74 --- /dev/null +++ b/services/monitoring/availability-backfill-v3-job.yaml @@ -0,0 +1,92 @@ +# services/monitoring/availability-backfill-v3-job.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlas-availability-gateway-v3-backfill-rules + namespace: monitoring +data: + atlas-gateway-history.yaml: | + groups: + - name: atlas.availability.gateway.backfill + interval: 1h + rules: + - record: atlas:availability:gateway_requests_1h + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"[1-5].." + }[1h] + )) + labels: + definition: gateway-v3 + scope: atlas + rollup: hourly + - record: atlas:availability:gateway_failures_1h + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"502|503|504" + }[1h] + )) + labels: + definition: gateway-v3 + scope: atlas + rollup: hourly + +--- + +apiVersion: batch/v1 +kind: Job +metadata: + name: atlas-availability-gateway-v3-backfill + namespace: monitoring +spec: + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-gateway-v3-backfill + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: vmalert-replay + image: victoriametrics/vmalert:v1.113.0 + args: + - -datasource.url=http://victoria-metrics-single-server:8428 + - -remoteWrite.url=http://victoria-metrics-single-server:8428 + - -remoteWrite.flushInterval=1s + - -rule=/etc/vmalert/backfill/*.yaml + - -replay.timeFrom=2026-05-01T00:00:00Z + - -replay.timeTo=2026-08-04T23:00:00Z + - -replay.maxDatapointsPerQuery=48 + - -replay.rulesDelay=2s + - -replay.disableProgressBar + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi + volumeMounts: + - name: rules + mountPath: /etc/vmalert/backfill + readOnly: true + volumes: + - name: rules + configMap: + name: atlas-availability-gateway-v3-backfill-rules diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index b7d0509b9..ba1814315 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -216,7 +216,7 @@ { "id": 27, "type": "stat", - "title": "Atlas Serving Availability", + "title": "Atlas Availability (365d)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"gateway-v3\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"502|503|504\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -286,7 +286,7 @@ }, "textMode": "value" }, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour." }, { "id": 4, diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index cc7862205..a03a3fc9e 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -225,7 +225,7 @@ data: { "id": 27, "type": "stat", - "title": "Atlas Serving Availability", + "title": "Atlas Availability (365d)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"gateway-v3\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"502|503|504\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -295,7 +295,7 @@ data: }, "textMode": "value" }, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour." }, { "id": 4, diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index 6bc0549a8..de41b951e 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -19,6 +19,7 @@ resources: - grafana-dashboard-mail.yaml - grafana-dashboard-testing.yaml - vmalert-atlas-availability.yaml + - availability-backfill-v3-job.yaml - dcgm-exporter.yaml - nvidia-process-exporter.yaml - jetson-tegrastats-exporter.yaml diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index 1bfcc1e49..333493fe5 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -7,38 +7,62 @@ metadata: data: atlas-availability.yaml: | groups: - - name: atlas.availability + - name: atlas.availability.gateway + interval: 1h + eval_offset: 59m + rules: + - record: atlas:availability:gateway_requests_1h + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"[1-5].." + }[1h] + )) + labels: + definition: gateway-v3 + scope: atlas + rollup: hourly + - record: atlas:availability:gateway_failures_1h + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"502|503|504" + }[1h] + )) + labels: + definition: gateway-v3 + scope: atlas + rollup: hourly + - name: atlas.availability.rollup interval: 15m eval_offset: 14m rules: - - record: atlas:availability:ratio_1h - expr: | - avg_over_time(( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - >= bool 2 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - > bool 0 - ) - ) - )[1h:5m]) - labels: - definition: serving-v2 - scope: atlas - rollup: hourly - record: atlas:availability:ratio_365d expr: | - share_gt_over_time(( - max by (scope, rollup) ( - atlas:availability:ratio_1h{scope="atlas",definition="serving-v2"} - or atlas:availability:ratio_1h{scope="atlas",definition=""} + 1 - ( + sum_over_time( + atlas:availability:gateway_failures_1h{ + scope="atlas", + definition="gateway-v3" + }[365d] ) - )[365d:15m], 0) + / + clamp_min( + sum_over_time( + atlas:availability:gateway_requests_1h{ + scope="atlas", + definition="gateway-v3" + }[365d] + ), + 1 + ) + ) labels: - definition: serving-v2 + definition: gateway-v3 scope: atlas rollup: yearly platform-quality.yaml: |