diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index 13310f7ea..5df8eb1bd 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -472,9 +472,9 @@ UPTIME_LIVE_FALLBACK_EXPR = ( f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))" ) UPTIME_COMPACT_FALLBACK_EXPR = ( - "(1 - (sum_over_time(atlas:availability:failures_1h{" + "(1 - (sum_over_time(atlas:availability:failures_1d{" 'scope="atlas",definition="request-v4"}[365d]) / ' - "clamp_min(sum_over_time(atlas:availability:requests_1h{" + "clamp_min(sum_over_time(atlas:availability:requests_1d{" 'scope="atlas",definition="request-v4"}[365d]), 1)))' ) UPTIME_RECORDING_EXPR = ( diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index a51e98d69..6ef5ab7f9 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -65,8 +65,8 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): ) assert 'code=~"5.."' in availability_expr assert 'code=~"[1-5].."' in availability_expr - assert "atlas:availability:failures_1h" in availability_expr - assert "atlas:availability:requests_1h" in availability_expr + assert "atlas:availability:failures_1d" in availability_expr + assert "atlas:availability:requests_1d" in availability_expr assert "sum_over_time" in availability_expr assert "kube_node_status_condition" not in availability_expr assert "kube_deployment_status_replicas_available" not in availability_expr diff --git a/scripts/tests/test_monitoring_query_capacity.py b/scripts/tests/test_monitoring_query_capacity.py index a40a7b071..0f7c7c656 100644 --- a/scripts/tests/test_monitoring_query_capacity.py +++ b/scripts/tests/test_monitoring_query_capacity.py @@ -57,8 +57,8 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None: if rule["record"] == "atlas:availability:ratio_365d" ) - assert "atlas:availability:requests_1h" in yearly["expr"] - assert "atlas:availability:failures_1h" in yearly["expr"] + assert "atlas:availability:requests_1d" in yearly["expr"] + assert "atlas:availability:failures_1d" in yearly["expr"] assert 'definition="request-v4"' in yearly["expr"] assert "sum_over_time" in yearly["expr"] assert "[365d]" in yearly["expr"] @@ -66,6 +66,23 @@ def test_yearly_availability_reuses_the_hourly_rollup() -> None: assert yearly["labels"]["definition"] == "request-v4" +def test_daily_availability_rollups_use_the_same_request_sli() -> None: + """Keep annual availability cheap without changing its request definition.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + )[0] + groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] + daily_group = next( + group for group in groups if group["name"] == "atlas.availability.rollup" + ) + daily = {rule["record"]: rule for rule in daily_group["rules"]} + + assert daily_group["interval"] == "1d" + assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"] + assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"] + assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values()) + + def test_availability_uses_request_failures_instead_of_replica_capacity() -> None: """Measure HTTP outcomes without treating redundant replica loss as downtime.""" manifest = _documents( @@ -116,6 +133,24 @@ def test_availability_backfill_replays_the_same_request_sli() -> None: assert "-replay.maxDatapointsPerQuery=48" in args +def test_daily_availability_backfill_stays_below_query_timeout() -> None: + """Replay daily buckets in small chunks so raw history never starves Grafana.""" + manifests = _documents( + REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml" + ) + config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap") + job = next(manifest for manifest in manifests if manifest["kind"] == "Job") + backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"]) + group = backfill["groups"][0] + expressions = {rule["record"]: rule["expr"] for rule in group["rules"]} + + assert group["interval"] == "1d" + assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"] + assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"] + args = job["spec"]["template"]["spec"]["containers"][0]["args"] + assert "-replay.maxDatapointsPerQuery=4" in args + + def test_quality_rollups_do_not_run_every_minute() -> None: """Keep high-cardinality quality rollups below the backend saturation cadence.""" manifest = _documents( diff --git a/services/monitoring/availability-daily-backfill-v4-job.yaml b/services/monitoring/availability-daily-backfill-v4-job.yaml new file mode 100644 index 000000000..dbd04bd78 --- /dev/null +++ b/services/monitoring/availability-daily-backfill-v4-job.yaml @@ -0,0 +1,92 @@ +# services/monitoring/availability-daily-backfill-v4-job.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlas-availability-request-v4-daily-backfill-rules + namespace: monitoring +data: + atlas-request-daily-history.yaml: | + groups: + - name: atlas.availability.request.daily.backfill + interval: 1d + rules: + - record: atlas:availability:requests_1d + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"[1-5].." + }[1d] + )) + labels: + definition: request-v4 + scope: atlas + rollup: daily + - record: atlas:availability:failures_1d + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"5.." + }[1d] + )) + labels: + definition: request-v4 + scope: atlas + rollup: daily + +--- + +apiVersion: batch/v1 +kind: Job +metadata: + name: atlas-availability-request-v4-daily-backfill + namespace: monitoring +spec: + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-request-v4-daily-backfill + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: vmalert-replay + image: victoriametrics/vmalert:v1.113.0 + args: + - -datasource.url=http://victoria-metrics-single-server:8428 + - -remoteWrite.url=http://victoria-metrics-single-server:8428 + - -remoteWrite.flushInterval=1s + - -rule=/etc/vmalert/backfill/*.yaml + - -replay.timeFrom=2026-05-01T00:00:00Z + - -replay.timeTo=2026-08-04T00:00:00Z + - -replay.maxDatapointsPerQuery=4 + - -replay.rulesDelay=2s + - -replay.disableProgressBar + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi + volumeMounts: + - name: rules + mountPath: /etc/vmalert/backfill + readOnly: true + volumes: + - name: rules + configMap: + name: atlas-availability-request-v4-daily-backfill-rules diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index a205253ee..991d24414 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index fe18513a6..be1238c1b 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1h{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[24h]) or on() (1 - (sum_over_time(atlas:availability:failures_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]) / clamp_min(sum_over_time(atlas:availability:requests_1d{scope=\"atlas\",definition=\"request-v4\"}[365d]), 1))) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index d54b86f5c..b96b64baa 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -20,6 +20,7 @@ resources: - grafana-dashboard-testing.yaml - vmalert-atlas-availability.yaml - availability-backfill-v4-job.yaml + - availability-daily-backfill-v4-job.yaml - dcgm-exporter.yaml - nvidia-process-exporter.yaml - jetson-tegrastats-exporter.yaml diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index 68ec402ae..f30946db2 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -38,6 +38,36 @@ data: scope: atlas rollup: hourly - name: atlas.availability.rollup + interval: 1d + eval_offset: 23h59m + rules: + - record: atlas:availability:requests_1d + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"[1-5].." + }[1d] + )) + labels: + definition: request-v4 + scope: atlas + rollup: daily + - record: atlas:availability:failures_1d + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"5.." + }[1d] + )) + labels: + definition: request-v4 + scope: atlas + rollup: daily + - name: atlas.availability.annual interval: 15m eval_offset: 14m rules: @@ -45,7 +75,7 @@ data: expr: | 1 - ( sum_over_time( - atlas:availability:failures_1h{ + atlas:availability:failures_1d{ scope="atlas", definition="request-v4" }[365d] @@ -53,7 +83,7 @@ data: / clamp_min( sum_over_time( - atlas:availability:requests_1h{ + atlas:availability:requests_1d{ scope="atlas", definition="request-v4" }[365d]