bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring: add daily availability rollups

Browse Source
This commit is contained in:
jenkins 2026-08-04 21:24:47 -03:00
parent 7c47a92610
commit a8dda9815d
8 changed files with 168 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/dashboards_render_atlas.py
View File

@ -472,9 +472,9 @@ UPTIME_LIVE_FALLBACK_EXPR = (
f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
)
UPTIME_COMPACT_FALLBACK_EXPR = (
"(1 - (sum_over_time(atlas:availability:failures_1h{"
"(1 - (sum_over_time(atlas:availability:failures_1d{"
'scope="atlas",definition="request-v4"}[365d]) / '
"clamp_min(sum_over_time(atlas:availability:requests_1h{"
"clamp_min(sum_over_time(atlas:availability:requests_1d{"
'scope="atlas",definition="request-v4"}[365d]), 1)))'
)
UPTIME_RECORDING_EXPR = (

4
scripts/tests/test_dashboards_render_atlas.py
View File

@ -65,8 +65,8 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
)
assert 'code=~"5.."' in availability_expr
assert 'code=~"[1-5].."' in availability_expr
assert "atlas:availability:failures_1h" in availability_expr
assert "atlas:availability:requests_1h" in availability_expr
assert "atlas:availability:failures_1d" in availability_expr
assert "atlas:availability:requests_1d" in availability_expr
assert "sum_over_time" in availability_expr
assert "kube_node_status_condition" not in availability_expr
assert "kube_deployment_status_replicas_available" not in availability_expr

39
scripts/tests/test_monitoring_query_capacity.py
View File

@ -57,8 +57,8 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None:
if rule["record"] == "atlas:availability:ratio_365d"
)
assert "atlas:availability:requests_1h" in yearly["expr"]
assert "atlas:availability:failures_1h" in yearly["expr"]
assert "atlas:availability:requests_1d" in yearly["expr"]
assert "atlas:availability:failures_1d" in yearly["expr"]
assert 'definition="request-v4"' in yearly["expr"]
assert "sum_over_time" in yearly["expr"]
assert "[365d]" in yearly["expr"]
@ -66,6 +66,23 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None:
assert yearly["labels"]["definition"] == "request-v4"
def test_daily_availability_rollups_use_the_same_request_sli() -> None:
"""Keep annual availability cheap without changing its request definition."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
daily_group = next(
group for group in groups if group["name"] == "atlas.availability.rollup"
)
daily = {rule["record"]: rule for rule in daily_group["rules"]}
assert daily_group["interval"] == "1d"
assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"]
assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"]
assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values())
def test_availability_uses_request_failures_instead_of_replica_capacity() -> None:
"""Measure HTTP outcomes without treating redundant replica loss as downtime."""
manifest = _documents(
@ -116,6 +133,24 @@ def test_availability_backfill_replays_the_same_request_sli() -> None:
assert "-replay.maxDatapointsPerQuery=48" in args
def test_daily_availability_backfill_stays_below_query_timeout() -> None:
"""Replay daily buckets in small chunks so raw history never starves Grafana."""
manifests = _documents(
REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml"
)
config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"])
group = backfill["groups"][0]
expressions = {rule["record"]: rule["expr"] for rule in group["rules"]}
assert group["interval"] == "1d"
assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"]
assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"]
args = job["spec"]["template"]["spec"]["containers"][0]["args"]
assert "-replay.maxDatapointsPerQuery=4" in args
def test_quality_rollups_do_not_run_every_minute() -> None:
"""Keep high-cardinality quality rollups below the backend saturation cadence."""
manifest = _documents(

92
services/monitoring/availability-daily-backfill-v4-job.yaml Normal file
View File

@ -0,0 +1,92 @@
# services/monitoring/availability-daily-backfill-v4-job.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: atlas-availability-request-v4-daily-backfill-rules
namespace: monitoring
data:
atlas-request-daily-history.yaml: |
groups:
- name: atlas.availability.request.daily.backfill
interval: 1d
rules:
- record: atlas:availability:requests_1d
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"[1-5].."
}[1d]
))
labels:
definition: request-v4
scope: atlas
rollup: daily
- record: atlas:availability:failures_1d
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"5.."
}[1d]
))
labels:
definition: request-v4
scope: atlas
rollup: daily
---
apiVersion: batch/v1
kind: Job
metadata:
name: atlas-availability-request-v4-daily-backfill
namespace: monitoring
spec:
backoffLimit: 2
template:
metadata:
labels:
app: atlas-availability-request-v4-daily-backfill
spec:
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-22
- titan-24
containers:
- name: vmalert-replay
image: victoriametrics/vmalert:v1.113.0
args:
- -datasource.url=http://victoria-metrics-single-server:8428
- -remoteWrite.url=http://victoria-metrics-single-server:8428
- -remoteWrite.flushInterval=1s
- -rule=/etc/vmalert/backfill/*.yaml
- -replay.timeFrom=2026-05-01T00:00:00Z
- -replay.timeTo=2026-08-04T00:00:00Z
- -replay.maxDatapointsPerQuery=4
- -replay.rulesDelay=2s
- -replay.disableProgressBar
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: "1"
memory: 512Mi
volumeMounts:
- name: rules
mountPath: /etc/vmalert/backfill
readOnly: true
volumes:
- name: rules
configMap:
name: atlas-availability-request-v4-daily-backfill-rules

2
services/monitoring/dashboards/atlas-overview.json
View File

@ -229,7 +229,7 @@
},
"targets": [
{
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"refId": "A",
"instant": true
}

2
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -238,7 +238,7 @@ data:
},
"targets": [
{
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))",
"refId": "A",
"instant": true
}

1
services/monitoring/kustomization.yaml
View File

@ -20,6 +20,7 @@ resources:
- grafana-dashboard-testing.yaml
- vmalert-atlas-availability.yaml
- availability-backfill-v4-job.yaml
- availability-daily-backfill-v4-job.yaml
- dcgm-exporter.yaml
- nvidia-process-exporter.yaml
- jetson-tegrastats-exporter.yaml

34
services/monitoring/vmalert-atlas-availability.yaml
View File

@ -38,6 +38,36 @@ data:
scope: atlas
rollup: hourly
- name: atlas.availability.rollup
interval: 1d
eval_offset: 23h59m
rules:
- record: atlas:availability:requests_1d
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"[1-5].."
}[1d]
))
labels:
definition: request-v4
scope: atlas
rollup: daily
- record: atlas:availability:failures_1d
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"5.."
}[1d]
))
labels:
definition: request-v4
scope: atlas
rollup: daily
- name: atlas.availability.annual
interval: 15m
eval_offset: 14m
rules:
@ -45,7 +75,7 @@ data:
expr: |
1 - (
sum_over_time(
atlas:availability:failures_1h{
atlas:availability:failures_1d{
scope="atlas",
definition="request-v4"
}[365d]
@ -53,7 +83,7 @@ data:
/
clamp_min(
sum_over_time(
atlas:availability:requests_1h{
atlas:availability:requests_1d{
scope="atlas",
definition="request-v4"
}[365d]