bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring: prevent query pool starvation

Browse Source
This commit is contained in:
jenkins 2026-08-04 12:22:05 -03:00
parent 59747de76d
commit b0599f4318
3 changed files with 84 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
scripts/tests/test_monitoring_query_capacity.py Normal file
View File

@ -0,0 +1,59 @@
"""Protect the monitoring backend from known query-starvation regressions."""
from pathlib import Path
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
def _documents(path: Path) -> list[dict]:
"""Load every non-empty YAML document from a repository manifest."""
return [document for document in yaml.safe_load_all(path.read_text()) if document]
def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
"""Keep search capacity and pod resources above the proven failure floor."""
manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml")
release = next(
manifest
for manifest in manifests
if manifest.get("kind") == "HelmRelease"
and manifest.get("metadata", {}).get("name") == "victoria-metrics-single"
)
server = release["spec"]["values"]["server"]
assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4
assert server["extraArgs"]["search.maxQueryDuration"] == "1m"
assert server["extraArgs"]["search.maxQueueDuration"] == "30s"
assert server["resources"]["requests"]["memory"] == "2Gi"
assert server["resources"]["limits"]["cpu"] == "2"
assert server["resources"]["limits"]["memory"] == "4Gi"
def test_yearly_availability_reuses_the_hourly_rollup() -> None:
"""Prevent the yearly rule from rescanning raw cluster metrics for 365 days."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
yearly = next(
rule
for rule in rules["rules"]
if rule["record"] == "atlas:availability:ratio_365d"
)
assert "atlas:availability:ratio_1h" in yearly["expr"]
assert "kube_node_status_condition" not in yearly["expr"]
assert "[365d:1h]" not in yearly["expr"]
def test_quality_rollups_do_not_run_every_minute() -> None:
"""Keep high-cardinality quality rollups below the backend saturation cadence."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0]
assert quality["interval"] == "5m"

26
services/monitoring/helmrelease.yaml
View File

@ -85,17 +85,20 @@ spec:
extraArgs:
retentionPeriod: "1y" # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11}
promscrape.configCheckInterval: "30s"
search.maxConcurrentRequests: "4"
search.maxQueryDuration: "1m"
search.maxQueueDuration: "30s"
persistentVolume:
enabled: true
size: 100Gi
resources:
requests:
cpu: 250m
memory: 1Gi
limits:
cpu: "1"
cpu: 500m
memory: 2Gi
limits:
cpu: "2"
memory: 4Gi
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
@ -112,6 +115,21 @@ spec:
- titan-21
- titan-22
- titan-24
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
- weight: 50
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi4
# Enable built-in Kubernetes scraping
scrape:

109
services/monitoring/vmalert-atlas-availability.yaml
View File

@ -30,119 +30,16 @@ data:
rollup: hourly
- record: atlas:availability:ratio_365d
expr: |
clamp_max((
(
sum(sum_over_time((
min(
(
sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
/ 3
),
(
sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
/ clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
)
)
)[365d:1h]))
or on() vector(0)
)
+
clamp_min(
8761
-
(
clamp_min(
floor(
(
time()
-
(
min(min_over_time(timestamp(
min(
(
sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
/ 3
),
(
sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
/ clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
)
)
)[365d:1h]))
or on() vector(time() + 3600)
)
)
/ 3600
)
+ 1,
0
)
),
0
)
avg_over_time(
atlas:availability:ratio_1h{scope="atlas"}[365d]
)
/
clamp_min(
(
(
sum(count_over_time((
min(
(
sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
/ 3
),
(
sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
/ clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
)
)
)[365d:1h]))
or on() vector(0)
)
+
clamp_min(
8761
-
(
clamp_min(
floor(
(
time()
-
(
min(min_over_time(timestamp(
min(
(
sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
/ 3
),
(
sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
/ clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
)
)
)[365d:1h]))
or on() vector(time() + 3600)
)
)
/ 3600
)
+ 1,
0
)
),
0
)
),
1
), 1)
labels:
scope: atlas
rollup: yearly
platform-quality.yaml: |
groups:
- name: platform.quality
interval: 1m
interval: 5m
rules:
- record: platform_quality:test_case_status:count_1h
expr: |