bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring: measure gateway availability

Browse Source
This commit is contained in:
jenkins 2026-08-04 21:02:19 -03:00
parent 27e80eec96
commit 4906fb73d8
8 changed files with 222 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
scripts/dashboards_render_atlas.py
View File

@ -457,21 +457,20 @@ STUCK_TERMINATING_EXPR = (
UPTIME_WINDOW = "365d"
# vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series.
UPTIME_RECORDING_METRIC = (
f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}'
f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="gateway-v3"}}'
)
TRAEFIK_READY_EXPR = (
"("
'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
" > bool 0)"
GATEWAY_REQUESTS_1H_EXPR = (
'sum(increase(traefik_entrypoint_requests_total{'
'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))'
)
CONTROL_READY_FRACTION_EXPR = (
f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
" >= bool 2)"
GATEWAY_FAILURES_1H_EXPR = (
'sum(increase(traefik_entrypoint_requests_total{'
'entrypoint="websecure",protocol="http",code=~"502|503|504"}[1h]))'
)
UPTIME_AVAIL_EXPR = (
f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
UPTIME_LIVE_FALLBACK_EXPR = (
f"(1 - (({GATEWAY_FAILURES_1H_EXPR} or on() vector(0)) / "
f"clamp_min({GATEWAY_REQUESTS_1H_EXPR}, 1)))"
)
UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])"
UPTIME_RECORDING_EXPR = (
f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
@ -1902,7 +1901,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
"Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.",
"Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.",
"Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.",
"Atlas Serving Availability": "Observed availability with control-plane quorum and at least one Traefik replica serving; partial replica capacity remains available.",
"Atlas Availability (365d)": "Request-weighted Atlas ingress availability; only Traefik 502, 503, and 504 gateway failures count as downtime.",
"Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.",
"CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.",
"Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.",
@ -2135,7 +2134,7 @@ def build_overview():
},
{
"id": 27,
"title": "Atlas Serving Availability",
"title": "Atlas Availability (365d)",
"expr": UPTIME_PERCENT_EXPR,
"kind": "stat",
"thresholds": UPTIME_PERCENT_THRESHOLDS,
@ -2143,7 +2142,7 @@ def build_overview():
"decimals": 4,
"text_mode": "value",
"instant": True,
"description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.",
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour.",
},
{
"id": 4,

18
scripts/tests/test_dashboards_render_atlas.py
View File

@ -57,20 +57,20 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
dashboard = mod.build_overview()
panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
assert panel["title"] == "Atlas Serving Availability"
assert panel["title"] == "Atlas Availability (365d)"
availability_expr = panel["targets"][0]["expr"]
assert (
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])'
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="gateway-v3"}[24h])'
in availability_expr
)
assert ">= bool 2" in availability_expr
assert "> bool 0" in availability_expr
assert "[1h:5m]" in availability_expr
assert 'code=~"502|503|504"' in availability_expr
assert 'code=~"[1-5].."' in availability_expr
assert "kube_node_status_condition" not in availability_expr
assert "kube_deployment_status_replicas_available" not in availability_expr
assert panel["targets"][0]["instant"] is True
assert "at least one Traefik replica serving" in panel["description"]
assert "partial-capacity samples remain available" in panel["description"]
assert "monitoring gaps are excluded" in panel["description"]
assert "falls back to the live binary serving state" in panel["description"]
assert "502, 503, and 504 gateway failures" in panel["description"]
assert "Replica counts, Grafana health" in panel["description"]
assert "falls back to the same gateway SLI" in panel["description"]
def test_overview_uses_readable_quality_power_and_gitops_panels():

71
scripts/tests/test_monitoring_query_capacity.py
View File

@ -49,38 +49,71 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None:
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
rules = [rule for group in groups for rule in group["rules"]]
yearly = next(
rule
for rule in rules["rules"]
for rule in rules
if rule["record"] == "atlas:availability:ratio_365d"
)
assert "atlas:availability:ratio_1h" in yearly["expr"]
assert 'definition="serving-v2"' in yearly["expr"]
assert 'definition=""' in yearly["expr"]
assert "share_gt_over_time" in yearly["expr"]
assert "[365d:15m]" in yearly["expr"]
assert "kube_node_status_condition" not in yearly["expr"]
assert "[365d:1h]" not in yearly["expr"]
assert "atlas:availability:gateway_requests_1h" in yearly["expr"]
assert "atlas:availability:gateway_failures_1h" in yearly["expr"]
assert 'definition="gateway-v3"' in yearly["expr"]
assert "sum_over_time" in yearly["expr"]
assert "[365d]" in yearly["expr"]
assert "traefik_entrypoint_requests_total" not in yearly["expr"]
assert yearly["labels"]["definition"] == "gateway-v3"
def test_availability_counts_serving_state_instead_of_replica_capacity() -> None:
"""Treat quorum and any serving ingress replica as available binary states."""
def test_availability_uses_gateway_failures_instead_of_replica_capacity() -> None:
"""Measure request outcomes without treating redundant replica loss as downtime."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
hourly = next(
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
rules = [rule for group in groups for rule in group["rules"]]
requests = next(
rule
for rule in rules["rules"]
if rule["record"] == "atlas:availability:ratio_1h"
for rule in rules
if rule["record"] == "atlas:availability:gateway_requests_1h"
)
failures = next(
rule
for rule in rules
if rule["record"] == "atlas:availability:gateway_failures_1h"
)
assert ">= bool 2" in hourly["expr"]
assert "> bool 0" in hourly["expr"]
assert "/ 3" not in hourly["expr"]
assert hourly["labels"]["definition"] == "serving-v2"
assert 'code=~"[1-5].."' in requests["expr"]
assert 'code=~"502|503|504"' in failures["expr"]
assert "traefik_entrypoint_requests_total" in requests["expr"]
assert "traefik_entrypoint_requests_total" in failures["expr"]
assert "kube_node_status_condition" not in repr(rules)
assert "kube_deployment_status_replicas_available" not in repr(rules)
assert requests["labels"]["definition"] == "gateway-v3"
assert failures["labels"]["definition"] == "gateway-v3"
def test_availability_backfill_replays_the_same_gateway_sli() -> None:
"""Backfill retained history with the exact rules used for future samples."""
manifests = _documents(
REPO_ROOT / "services/monitoring/availability-backfill-v3-job.yaml"
)
config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
backfill = yaml.safe_load(config["data"]["atlas-gateway-history.yaml"])
expressions = {
rule["record"]: rule["expr"]
for group in backfill["groups"]
for rule in group["rules"]
}
assert 'code=~"[1-5].."' in expressions["atlas:availability:gateway_requests_1h"]
assert 'code=~"502|503|504"' in expressions["atlas:availability:gateway_failures_1h"]
args = job["spec"]["template"]["spec"]["containers"][0]["args"]
assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args
assert "-replay.timeTo=2026-08-04T23:00:00Z" in args
assert "-replay.maxDatapointsPerQuery=48" in args
def test_quality_rollups_do_not_run_every_minute() -> None:

92
services/monitoring/availability-backfill-v3-job.yaml Normal file
View File

@ -0,0 +1,92 @@
# services/monitoring/availability-backfill-v3-job.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: atlas-availability-gateway-v3-backfill-rules
namespace: monitoring
data:
atlas-gateway-history.yaml: |
groups:
- name: atlas.availability.gateway.backfill
interval: 1h
rules:
- record: atlas:availability:gateway_requests_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"[1-5].."
}[1h]
))
labels:
definition: gateway-v3
scope: atlas
rollup: hourly
- record: atlas:availability:gateway_failures_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"502|503|504"
}[1h]
))
labels:
definition: gateway-v3
scope: atlas
rollup: hourly
---
apiVersion: batch/v1
kind: Job
metadata:
name: atlas-availability-gateway-v3-backfill
namespace: monitoring
spec:
backoffLimit: 2
template:
metadata:
labels:
app: atlas-availability-gateway-v3-backfill
spec:
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-22
- titan-24
containers:
- name: vmalert-replay
image: victoriametrics/vmalert:v1.113.0
args:
- -datasource.url=http://victoria-metrics-single-server:8428
- -remoteWrite.url=http://victoria-metrics-single-server:8428
- -remoteWrite.flushInterval=1s
- -rule=/etc/vmalert/backfill/*.yaml
- -replay.timeFrom=2026-05-01T00:00:00Z
- -replay.timeTo=2026-08-04T23:00:00Z
- -replay.maxDatapointsPerQuery=48
- -replay.rulesDelay=2s
- -replay.disableProgressBar
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: "1"
memory: 512Mi
volumeMounts:
- name: rules
mountPath: /etc/vmalert/backfill
readOnly: true
volumes:
- name: rules
configMap:
name: atlas-availability-gateway-v3-backfill-rules

6
services/monitoring/dashboards/atlas-overview.json
View File

@ -216,7 +216,7 @@
{
"id": 27,
"type": "stat",
"title": "Atlas Serving Availability",
"title": "Atlas Availability (365d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
@ -229,7 +229,7 @@
},
"targets": [
{
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))",
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"gateway-v3\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"502|503|504\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"refId": "A",
"instant": true
}
@ -286,7 +286,7 @@
},
"textMode": "value"
},
"description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available."
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour."
},
{
"id": 4,

6
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -225,7 +225,7 @@ data:
{
"id": 27,
"type": "stat",
"title": "Atlas Serving Availability",
"title": "Atlas Availability (365d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
@ -238,7 +238,7 @@ data:
},
"targets": [
{
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))",
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"gateway-v3\"}[24h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"502|503|504\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"refId": "A",
"instant": true
}
@ -295,7 +295,7 @@ data:
},
"textMode": "value"
},
"description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available."
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress. Every HTTP response counts as served except Traefik 502, 503, and 504 gateway failures. Client 4xx and application 500/501 responses do not mark the cluster unavailable because Atlas remained reachable. Replica counts, Grafana health, and monitoring gaps are not treated as downtime. Grafana keeps the last successful rollup for up to 24 hours and falls back to the same gateway SLI over the last hour."
},
{
"id": 4,

1
services/monitoring/kustomization.yaml
View File

@ -19,6 +19,7 @@ resources:
- grafana-dashboard-mail.yaml
- grafana-dashboard-testing.yaml
- vmalert-atlas-availability.yaml
- availability-backfill-v3-job.yaml
- dcgm-exporter.yaml
- nvidia-process-exporter.yaml
- jetson-tegrastats-exporter.yaml

74
services/monitoring/vmalert-atlas-availability.yaml
View File

@ -7,38 +7,62 @@ metadata:
data:
atlas-availability.yaml: |
groups:
- name: atlas.availability
- name: atlas.availability.gateway
interval: 1h
eval_offset: 59m
rules:
- record: atlas:availability:gateway_requests_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"[1-5].."
}[1h]
))
labels:
definition: gateway-v3
scope: atlas
rollup: hourly
- record: atlas:availability:gateway_failures_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"502|503|504"
}[1h]
))
labels:
definition: gateway-v3
scope: atlas
rollup: hourly
- name: atlas.availability.rollup
interval: 15m
eval_offset: 14m
rules:
- record: atlas:availability:ratio_1h
expr: |
avg_over_time((
min(
(
sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
>= bool 2
),
(
sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
> bool 0
)
)
)[1h:5m])
labels:
definition: serving-v2
scope: atlas
rollup: hourly
- record: atlas:availability:ratio_365d
expr: |
share_gt_over_time((
max by (scope, rollup) (
atlas:availability:ratio_1h{scope="atlas",definition="serving-v2"}
or atlas:availability:ratio_1h{scope="atlas",definition=""}
1 - (
sum_over_time(
atlas:availability:gateway_failures_1h{
scope="atlas",
definition="gateway-v3"
}[365d]
)
)[365d:15m], 0)
/
clamp_min(
sum_over_time(
atlas:availability:gateway_requests_1h{
scope="atlas",
definition="gateway-v3"
}[365d]
),
1
)
)
labels:
definition: serving-v2
definition: gateway-v3
scope: atlas
rollup: yearly
platform-quality.yaml: |