fbb401e4d5
Evidence-led capacity/QoS audit of titan-20 (Hermes LLM fallback/ classifier), titan-21 (STT/TTS), and titan-22 (Jellyfin media-primary, restored) for t_26da4c88. titan-20/21 are CPU-committed with no safe headroom (titan-20 at 227 MiB free memory at its 24h worst point); titan-22 has real idle CPU/RAM but its shared-GPU time-slicing has no VRAM/engine isolation, so no workload is relocated. Adds alerting for the sharpest gaps found (titan-20 memory exhaustion, titan-22 CPU/RAM/ GPU-VRAM pressure, Jellyfin CPU throttling, titan-21 CPU pressure) and two Atlas GPU dashboard panels (VRAM, NVENC/NVDEC utilization) so a future opportunistic-workload PR or a live transcode incident is visible without a promql session. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
565 lines
27 KiB
YAML
565 lines
27 KiB
YAML
# services/monitoring/vmalert-atlas-availability.yaml
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: vmalert-atlas-availability-rules
|
|
namespace: monitoring
|
|
data:
|
|
hermes-triage.yaml: |
|
|
groups:
|
|
- name: hermes.triage
|
|
interval: 1m
|
|
rules:
|
|
# Escalation on its own is NOT worth an email. Every human_required
|
|
# incident already files an issue in the failing service's own repo,
|
|
# and that issue is the durable artifact. Mailing on every escalation
|
|
# made the inbox the loudest and least useful output of the system.
|
|
# Two things do deserve mail: a repair that ran and failed, and an
|
|
# escalation nobody has touched in hours.
|
|
- alert: HermesTriageActionFailed
|
|
expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="failed"}) == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
service: hermes-triage
|
|
annotations:
|
|
summary: "Hermes repair FAILED: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
|
|
description: "Ariadne authorized a remediation for {{ $labels.jenkins_job }}/{{ $labels.build }} and it did not succeed, or its rebuild failed. This is the one case where the automation acted and did not leave things better. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
|
|
- alert: HermesTriageUnattended
|
|
expr: max by (jenkins_job, build) (ariadne_hermes_triage_incident{status="human_required"}) == 1
|
|
for: 6h
|
|
labels:
|
|
severity: warning
|
|
service: hermes-triage
|
|
annotations:
|
|
summary: "Hermes triage unattended 6h: incident {{ $labels.jenkins_job }}/{{ $labels.build }}"
|
|
description: "Incident {{ $labels.jenkins_job }}/{{ $labels.build }} has been human_required for six hours. Its issue was filed in the failing service's repository at the time of escalation; this alert only means nobody has resolved it since. Jenkins build: https://ci.bstein.dev/job/{{ $labels.jenkins_job }}/{{ $labels.build }}/"
|
|
atlas-availability.yaml: |
|
|
groups:
|
|
- name: atlas.availability.gateway
|
|
interval: 1h
|
|
eval_offset: 59m
|
|
rules:
|
|
- record: atlas:availability:requests_1h
|
|
expr: |
|
|
sum(increase(
|
|
traefik_entrypoint_requests_total{
|
|
entrypoint="websecure",
|
|
protocol="http",
|
|
code=~"[1-5].."
|
|
}[1h]
|
|
))
|
|
labels:
|
|
definition: request-v4
|
|
scope: atlas
|
|
rollup: hourly
|
|
- record: atlas:availability:failures_1h
|
|
expr: |
|
|
sum(increase(
|
|
traefik_entrypoint_requests_total{
|
|
entrypoint="websecure",
|
|
protocol="http",
|
|
code=~"5.."
|
|
}[1h]
|
|
))
|
|
labels:
|
|
definition: request-v4
|
|
scope: atlas
|
|
rollup: hourly
|
|
- name: atlas.availability.rollup
|
|
interval: 1d
|
|
eval_offset: 23h59m
|
|
rules:
|
|
- record: atlas:availability:requests_1d
|
|
expr: |
|
|
sum(increase(
|
|
traefik_entrypoint_requests_total{
|
|
entrypoint="websecure",
|
|
protocol="http",
|
|
code=~"[1-5].."
|
|
}[1d]
|
|
))
|
|
labels:
|
|
definition: request-v4
|
|
scope: atlas
|
|
rollup: daily
|
|
- record: atlas:availability:failures_1d
|
|
expr: |
|
|
sum(increase(
|
|
traefik_entrypoint_requests_total{
|
|
entrypoint="websecure",
|
|
protocol="http",
|
|
code=~"5.."
|
|
}[1d]
|
|
))
|
|
labels:
|
|
definition: request-v4
|
|
scope: atlas
|
|
rollup: daily
|
|
platform-quality.yaml: |
|
|
groups:
|
|
- name: platform.quality
|
|
# Stay inside the five-minute instant-query lookback, including the
|
|
# group's evaluation and remote-write time.
|
|
interval: 4m
|
|
eval_offset: 1m
|
|
rules:
|
|
- record: platform_quality:test_case_status:count_1h
|
|
expr: |
|
|
sum by (suite, branch, test, category, status) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=""},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:test_case_pass_rate:percent_1h
|
|
expr: |
|
|
100 * (
|
|
(
|
|
sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status="passed"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status="passed"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
or on(suite, branch, test, category)
|
|
(
|
|
0 * sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
/
|
|
clamp_min(
|
|
sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
),
|
|
1
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:test_case_health_rate:percent_1h
|
|
expr: |
|
|
100 * (
|
|
(
|
|
sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|skipped|not_applicable|na|n/a"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|skipped|not_applicable|na|n/a"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
or on(suite, branch, test, category)
|
|
(
|
|
0 * sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
/
|
|
clamp_min(
|
|
sum by (suite, branch, test, category) (
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit",status=~"passed|failed|error|skipped|not_applicable|na|n/a"}
|
|
or label_replace(
|
|
platform_quality_gate_test_case_result{exported_job="platform-quality-ci",branch!="",test!="",test!="__no_test_cases__",category="",status=~"passed|failed|error|skipped|not_applicable|na|n/a"},
|
|
"category", "uncategorized", "__name__", ".*"
|
|
)
|
|
),
|
|
1
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:test_category_health_rate:percent_1h
|
|
expr: |
|
|
avg by (suite, branch, category) (
|
|
platform_quality:test_case_health_rate:percent_1h{suite!="",branch!="",test!="",test!="__no_test_cases__",category=~"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit"}
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:suite_runs:increase_24h
|
|
expr: |
|
|
sum by (suite, branch, status) (
|
|
(
|
|
increase((
|
|
max without(instance, job) (
|
|
platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch!="",status!=""}
|
|
)
|
|
)[24h:1m])
|
|
)
|
|
or (
|
|
increase((
|
|
max without(instance, job) (
|
|
platform_quality_gate_runs_total{exported_job="platform-quality-ci",suite!="",branch="",status!=""}
|
|
)
|
|
)[24h:1m])
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:suite_coverage_percent:latest_1h
|
|
expr: |
|
|
max by (suite, branch) (
|
|
{__name__=~"platform_quality_gate_workspace_line_coverage_percent|.*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!="",branch!=""}
|
|
)
|
|
or on (suite, branch)
|
|
(
|
|
max by (suite) (
|
|
{__name__=~"platform_quality_gate_workspace_line_coverage_percent|.*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!="",branch=""}
|
|
)
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or on (suite, branch)
|
|
(
|
|
label_replace(
|
|
max by (suite) (
|
|
{__name__=~"platform_quality_gate_workspace_line_coverage_percent|.*_quality_gate_coverage_percent",exported_job="platform-quality-ci",suite!="",branch=""}
|
|
),
|
|
"branch", "unknown", "__name__", ".*"
|
|
)
|
|
unless on (suite)
|
|
max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:suite_source_files_total:latest_1h
|
|
expr: |
|
|
max by (suite, branch) (
|
|
platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!="",branch!=""}
|
|
)
|
|
or on (suite, branch)
|
|
(
|
|
max by (suite) (
|
|
platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!="",branch=""}
|
|
)
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or on (suite, branch)
|
|
(
|
|
label_replace(
|
|
max by (suite) (
|
|
platform_quality_gate_source_files_total{exported_job="platform-quality-ci",suite!="",branch=""}
|
|
),
|
|
"branch", "unknown", "__name__", ".*"
|
|
)
|
|
unless on (suite)
|
|
max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:suite_source_lines_over_500_total:latest_1h
|
|
expr: |
|
|
max by (suite, branch) (
|
|
platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!="",branch!=""}
|
|
)
|
|
or on (suite, branch)
|
|
(
|
|
max by (suite) (
|
|
platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!="",branch=""}
|
|
)
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or on (suite, branch)
|
|
(
|
|
label_replace(
|
|
max by (suite) (
|
|
platform_quality_gate_source_lines_over_500_total{exported_job="platform-quality-ci",suite!="",branch=""}
|
|
),
|
|
"branch", "unknown", "__name__", ".*"
|
|
)
|
|
unless on (suite)
|
|
max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:check_status:present_1h
|
|
expr: |
|
|
sum by (suite, branch, check, status) (
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",status!=""}
|
|
or label_replace(
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch!="",check!="",result!="",status=""},
|
|
"status", "$1", "result", "(.*)"
|
|
)
|
|
or (
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""}
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or (
|
|
label_replace(
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""},
|
|
"status", "$1", "result", "(.*)"
|
|
)
|
|
* on (suite) group_left(branch)
|
|
max by (suite, branch) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or (
|
|
label_replace(
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",status!=""},
|
|
"branch", "unknown", "__name__", ".*"
|
|
)
|
|
unless on (suite)
|
|
max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
or (
|
|
label_replace(
|
|
label_replace(
|
|
{__name__=~".*_quality_gate_checks_total",exported_job="platform-quality-ci",suite!="",branch="",check!="",result!="",status=""},
|
|
"status", "$1", "result", "(.*)"
|
|
),
|
|
"branch", "unknown", "__name__", ".*"
|
|
)
|
|
unless on (suite)
|
|
max by (suite) (platform_quality_gate_build_info{exported_job="platform-quality-ci",suite!="",branch!=""})
|
|
)
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:check_seen_flag:present_1h
|
|
expr: |
|
|
clamp_max(
|
|
max by (suite, branch, check) (
|
|
platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!=""} > 0
|
|
),
|
|
1
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:check_failed_flag:present_1h
|
|
expr: |
|
|
clamp_max(
|
|
max by (suite, branch, check) (
|
|
platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status!~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0
|
|
),
|
|
1
|
|
)
|
|
or on (suite, branch, check)
|
|
(0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""})
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:check_healthy_flag:present_1h
|
|
expr: |
|
|
(
|
|
clamp_max(
|
|
max by (suite, branch, check) (
|
|
platform_quality:check_status:present_1h{suite!="",branch!="",check!="",status=~"ok|passed|success|not_applicable|skipped|na|n/a"} > 0
|
|
),
|
|
1
|
|
)
|
|
unless on (suite, branch, check)
|
|
(platform_quality:check_failed_flag:present_1h{suite!="",branch!="",check!=""} > 0)
|
|
)
|
|
or on (suite, branch, check)
|
|
(0 * platform_quality:check_seen_flag:present_1h{suite!="",branch!="",check!=""})
|
|
labels:
|
|
rollup: hourly
|
|
- record: platform_quality:sonar_gate_health_percent:latest_1h
|
|
expr: |
|
|
100 * max by (project_key) (
|
|
sonarqube_project_quality_gate_pass{project_key!=""}
|
|
)
|
|
labels:
|
|
rollup: hourly
|
|
titan-media-capacity.yaml: |
|
|
groups:
|
|
- name: atlas.titan-media-capacity
|
|
# Covers the titan-20/21/22 evidence audit from t_26da4c88: titan-20
|
|
# runs at 227 MiB free at its worst point in a day with no coverage
|
|
# at all, and titan-22 is Jellyfin's protected media-primary host —
|
|
# any future opportunistic workload there needs contention alerts
|
|
# to exist before it ships, not after.
|
|
interval: 1m
|
|
rules:
|
|
- alert: Titan20NodeMemoryCritical
|
|
expr: |
|
|
(node_memory_MemAvailable_bytes{node="titan-20"} / node_memory_MemTotal_bytes{node="titan-20"}) < 0.05
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
service: hermes-ai-llm
|
|
annotations:
|
|
summary: "titan-20 (Hermes local LLM fallback/classifier) is within 5% of an OOM"
|
|
description: "titan-20 hosts the hard-pinned Ollama fallback/classifier for Hermes Chat. Available memory has been under 5% of total for 10m. This node runs with no memory slack under normal load (98%+ used is typical); this alert fires only when it is at real risk of an OOM kill. Do not add workloads to titan-20 to relieve this — see docs/titan_20_21_22_capacity_optimization.md."
|
|
- alert: TitanMediaHostCPUPressure
|
|
expr: |
|
|
(1 - avg(rate(node_cpu_seconds_total{mode="idle",node="titan-22"}[5m]))) > 0.85
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
service: jellyfin
|
|
annotations:
|
|
summary: "titan-22 (Jellyfin media-primary host) CPU busy > 85% for 10m"
|
|
description: "titan-22 is normally under 30% CPU (p95, 24h baseline in docs/titan_20_21_22_capacity_optimization.md). Sustained pressure this high risks Jellyfin transcode quality and is the signal to check for a misbehaving opportunistic/scavenger workload."
|
|
- alert: TitanMediaHostMemoryPressure
|
|
expr: |
|
|
(node_memory_MemAvailable_bytes{node="titan-22"} / node_memory_MemTotal_bytes{node="titan-22"}) < 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
service: jellyfin
|
|
annotations:
|
|
summary: "titan-22 (Jellyfin media-primary host) available memory under 15%"
|
|
description: "titan-22 is normally ~90% free (24h baseline in docs/titan_20_21_22_capacity_optimization.md). This protects the reservation budget any future opportunistic/scavenger workload on this node must respect."
|
|
- alert: JellyfinCPUThrottled
|
|
expr: |
|
|
(
|
|
rate(container_cpu_cfs_throttled_periods_total{namespace="jellyfin",container="jellyfin"}[5m])
|
|
/
|
|
rate(container_cpu_cfs_periods_total{namespace="jellyfin",container="jellyfin"}[5m])
|
|
) > 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: jellyfin
|
|
annotations:
|
|
summary: "Jellyfin container is CPU-throttled against its own limit"
|
|
description: "container_cpu_cfs_throttled_periods_total ratio for jellyfin/jellyfin has been over 5% for 5m. Sustained throttling degrades transcode encode speed and can cause playback stutter. Baseline in docs/titan_20_21_22_capacity_optimization.md measured 0% throttling over 24h; any sustained value here is a regression, most likely node-level contention from a co-scheduled workload."
|
|
- alert: TitanMediaGPUMemorySaturation
|
|
expr: |
|
|
(
|
|
(
|
|
label_replace(label_replace((DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE)), "pod", "$1", "Hostname", "(.*)"), "namespace", "monitoring", "", "")
|
|
* on(namespace,pod) group_left(node) kube_pod_info{namespace="monitoring"}
|
|
)
|
|
* on(node) group_left() node_uname_info{node="titan-22"}
|
|
) > 0.85
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: jellyfin
|
|
annotations:
|
|
summary: "titan-22 GPU VRAM over 85% used"
|
|
description: "titan-22's shared RTX 3050 Ti (~8 GiB) is normally under 2% VRAM used (24h peak was 1.27 GiB in the t_26da4c88 audit). This is the concrete failure mode a second shared-GPU tenant (nvidia.com/gpu.shared) would cause for an active Jellyfin hardware transcode; GPU sharing on this node is time-sliced with no VRAM isolation, so this alert is the closest available proxy for contention."
|
|
- alert: TitanVoiceHostCPUPressure
|
|
expr: |
|
|
(1 - avg(rate(node_cpu_seconds_total{mode="idle",node="titan-21"}[5m]))) > 0.90
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
service: hermes-voice
|
|
annotations:
|
|
summary: "titan-21 (Hermes STT/TTS host) CPU busy > 90% for 15m"
|
|
description: "titan-21 already runs with 93% of its CPU requested (docs/titan_20_21_22_capacity_optimization.md); this alert catches sustained real contention that would add latency to Hermes Chat's hands-free voice path (feeds t_60d2deb6)."
|
|
|
|
---
|
|
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: vmalert-atlas-availability
|
|
namespace: monitoring
|
|
|
|
---
|
|
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vmalert-atlas-availability
|
|
namespace: monitoring
|
|
labels:
|
|
app: vmalert-atlas-availability
|
|
spec:
|
|
replicas: 1
|
|
revisionHistoryLimit: 3
|
|
selector:
|
|
matchLabels:
|
|
app: vmalert-atlas-availability
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: vmalert-atlas-availability
|
|
annotations:
|
|
bstein.dev/rules-revision: "2026-08-05-hermes-triage-alert"
|
|
spec:
|
|
serviceAccountName: vmalert-atlas-availability
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: kubernetes.io/hostname
|
|
operator: NotIn
|
|
values:
|
|
- titan-22
|
|
- titan-24
|
|
containers:
|
|
- name: vmalert
|
|
image: victoriametrics/vmalert:v1.113.0
|
|
args:
|
|
- -datasource.url=http://victoria-metrics-single-server:8428
|
|
- -datasource.queryStep=1h
|
|
- -remoteWrite.url=http://victoria-metrics-single-server:8428
|
|
- -rule=/etc/vmalert/rules/*.yaml
|
|
- -configCheckInterval=30s
|
|
- -evaluationInterval=15m
|
|
- -notifier.url=http://alertmanager.monitoring.svc.cluster.local:9093
|
|
- -httpListenAddr=:8880
|
|
ports:
|
|
- name: http
|
|
containerPort: 8880
|
|
readinessProbe:
|
|
tcpSocket:
|
|
port: http
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
tcpSocket:
|
|
port: http
|
|
initialDelaySeconds: 20
|
|
periodSeconds: 30
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
volumeMounts:
|
|
- name: rules
|
|
mountPath: /etc/vmalert/rules
|
|
readOnly: true
|
|
volumes:
|
|
- name: rules
|
|
configMap:
|
|
name: vmalert-atlas-availability-rules
|
|
|
|
---
|
|
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: vmalert-atlas-availability
|
|
namespace: monitoring
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "8880"
|
|
spec:
|
|
selector:
|
|
app: vmalert-atlas-availability
|
|
ports:
|
|
- name: http
|
|
port: 8880
|
|
targetPort: http
|