6df688adcc
Deterministic coverage for the quota-aware lane: threshold boundaries (14.9/15/15.1), both-below preference, fetch-failure fail-open, cooldown elapsed-vs-not hysteresis, quota-reset recovery (never for auth), explicit fail-closed in both directions, bounded double-failure block, failure-reason classification, metrics emission, and worker env key stripping. Based on PR #15 (fix/hermes-result-decomposition-reliability). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
117 lines
4.3 KiB
Python
117 lines
4.3 KiB
Python
"""Lane metrics registry semantics and the pod-local scrape endpoint."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from testing.tests.test_hermes_cli_support import lanes
|
|
|
|
|
|
def test_registry_renders_counters_and_gauges_with_stable_labels():
|
|
registry = lanes.LaneMetricsRegistry()
|
|
registry.increment(
|
|
"hermes_cli_route_decisions_total",
|
|
{"provider": "codex", "effort": "high", "classifier": "c", "outcome": "completed"},
|
|
)
|
|
registry.increment(
|
|
"hermes_cli_route_decisions_total",
|
|
{"provider": "codex", "effort": "high", "classifier": "c", "outcome": "completed"},
|
|
)
|
|
registry.increment("hermes_cli_quota_fetch_failures_total")
|
|
registry.set_value(
|
|
"hermes_cli_quota_remaining_percent", {"provider": 'we"ird\n\\'}, 12.5
|
|
)
|
|
rendered = registry.render().decode("utf-8")
|
|
|
|
assert "# TYPE hermes_cli_route_decisions_total counter" in rendered
|
|
assert "# TYPE hermes_cli_quota_remaining_percent gauge" in rendered
|
|
assert (
|
|
'hermes_cli_route_decisions_total{classifier="c",effort="high",'
|
|
'outcome="completed",provider="codex"} 2' in rendered
|
|
)
|
|
assert "hermes_cli_quota_fetch_failures_total 1" in rendered
|
|
assert (
|
|
'hermes_cli_quota_remaining_percent{provider="we\\"ird\\n\\\\"} 12.5'
|
|
in rendered
|
|
)
|
|
|
|
|
|
def test_recording_helpers_feed_the_shared_registry():
|
|
lanes.record_route_decision("codex", "xhigh", "metrics-test", "completed")
|
|
lanes.record_provider_fallback("codex", "claude", "rate-limit")
|
|
lanes.record_router_selection_failure()
|
|
lanes.record_quota_fetch_failure()
|
|
lanes.record_provider_quota("metrics-test-provider", 42.0, 777.0)
|
|
lanes.record_provider_quota("metrics-test-no-reset", 41.0, None)
|
|
lanes.record_soft_exclusion("metrics-test-provider", True)
|
|
rendered = lanes.METRICS.render().decode("utf-8")
|
|
|
|
assert (
|
|
'hermes_cli_route_decisions_total{classifier="metrics-test",'
|
|
'effort="xhigh",outcome="completed",provider="codex"} 1' in rendered
|
|
)
|
|
assert (
|
|
'hermes_cli_provider_fallbacks_total{from_provider="codex",'
|
|
'reason="rate-limit",to_provider="claude"}' in rendered
|
|
)
|
|
assert "hermes_cli_router_selection_failures_total" in rendered
|
|
assert "hermes_cli_quota_fetch_failures_total" in rendered
|
|
assert (
|
|
'hermes_cli_quota_remaining_percent{provider="metrics-test-provider"} 42'
|
|
in rendered
|
|
)
|
|
assert (
|
|
'hermes_cli_quota_reset_timestamp_seconds{provider="metrics-test-provider"} 777'
|
|
in rendered
|
|
)
|
|
assert (
|
|
'hermes_cli_quota_reset_timestamp_seconds{provider="metrics-test-no-reset"}'
|
|
not in rendered
|
|
)
|
|
assert (
|
|
'hermes_cli_provider_soft_excluded{provider="metrics-test-provider"} 1'
|
|
in rendered
|
|
)
|
|
|
|
|
|
def test_metrics_server_serves_scrapes_health_and_rejects_unknown_paths():
|
|
server = lanes.start_metrics_server(0)
|
|
assert server is not None
|
|
try:
|
|
port = server.server_address[1]
|
|
lanes.record_route_decision("codex", "low", "scrape-test", "completed")
|
|
with urllib.request.urlopen(f"http://127.0.0.1:{port}/metrics") as response:
|
|
assert response.status == 200
|
|
assert b'classifier="scrape-test"' in response.read()
|
|
with urllib.request.urlopen(f"http://127.0.0.1:{port}/healthz") as response:
|
|
assert response.status == 200
|
|
try:
|
|
urllib.request.urlopen(f"http://127.0.0.1:{port}/nope")
|
|
raise AssertionError("unknown paths must 404")
|
|
except urllib.error.HTTPError as error:
|
|
assert error.code == 404
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
|
|
|
|
def test_metrics_server_reads_port_from_environment(monkeypatch):
|
|
monkeypatch.setenv("HERMES_CLI_LANE_METRICS_PORT", "0")
|
|
server = lanes.start_metrics_server()
|
|
assert server is not None
|
|
server.shutdown()
|
|
server.server_close()
|
|
|
|
|
|
def test_metrics_server_declines_conflicting_port(capsys):
|
|
first = lanes.start_metrics_server(0)
|
|
assert first is not None
|
|
try:
|
|
second = lanes.start_metrics_server(first.server_address[1])
|
|
assert second is None
|
|
assert "metrics server unavailable" in capsys.readouterr().err
|
|
finally:
|
|
first.shutdown()
|
|
first.server_close()
|