6df688adcc
Deterministic coverage for the quota-aware lane: threshold boundaries (14.9/15/15.1), both-below preference, fetch-failure fail-open, cooldown elapsed-vs-not hysteresis, quota-reset recovery (never for auth), explicit fail-closed in both directions, bounded double-failure block, failure-reason classification, metrics emission, and worker env key stripping. Based on PR #15 (fix/hermes-result-decomposition-reliability). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
300 lines
11 KiB
Python
300 lines
11 KiB
Python
"""Quota-aware soft routing thresholds, parsing, and manifest wiring."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from testing.tests.test_hermes_cli_support import (
|
|
HERMES,
|
|
Path,
|
|
lanes,
|
|
yaml,
|
|
)
|
|
|
|
|
|
EXPORTER_TEXT = """\
|
|
# HELP atlas_ai_quota_remaining_percent Remaining percentage.
|
|
# TYPE atlas_ai_quota_remaining_percent gauge
|
|
atlas_ai_quota_remaining_percent{limit="codex",provider="openai",window="five_hour"} 37.5
|
|
atlas_ai_quota_remaining_percent{limit="codex",provider="openai",window="seven_day"} 80
|
|
atlas_ai_quota_remaining_percent{limit="bonus-credits",provider="openai",window="five_hour"} 1
|
|
atlas_ai_quota_remaining_percent{limit="claude",provider="anthropic",window="five_hour"} 62
|
|
atlas_ai_quota_remaining_percent{limit="claude",provider="anthropic",window="seven_day"} 40
|
|
atlas_ai_quota_remaining_percent{limit="claude",provider="anthropic",window="seven_day_opus"} 1
|
|
atlas_ai_quota_remaining_percent{limit="other",provider="somebody",window="five_hour"} 2
|
|
atlas_ai_quota_reset_timestamp_seconds{limit="codex",provider="openai",window="five_hour"} 111
|
|
atlas_ai_quota_reset_timestamp_seconds{limit="codex",provider="openai",window="seven_day"} 222
|
|
atlas_ai_quota_reset_timestamp_seconds{limit="claude",provider="anthropic",window="seven_day"} 444
|
|
atlas_ai_quota_fetch_success{provider="openai"} 1
|
|
atlas_ai_quota_fetch_success{provider="anthropic"} 1
|
|
not a metric line at all
|
|
"""
|
|
|
|
|
|
class _MetricsResponse:
|
|
def __init__(self, text: str):
|
|
self._payload = text.encode("utf-8")
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self, _limit: int) -> bytes:
|
|
return self._payload
|
|
|
|
|
|
def test_parse_reduces_each_provider_to_its_binding_account_window():
|
|
snapshot = lanes.parse_quota_metrics(EXPORTER_TEXT)
|
|
|
|
assert snapshot["codex"].remaining_percent == 37.5
|
|
assert snapshot["codex"].reset_timestamp == 111
|
|
# The model-specific seven_day_opus window must not veto claude; the
|
|
# binding account window is seven_day at 40% with its own reset.
|
|
assert snapshot["claude"].remaining_percent == 40
|
|
assert snapshot["claude"].reset_timestamp == 444
|
|
|
|
|
|
def test_parse_drops_provider_after_failed_exporter_fetch():
|
|
text = EXPORTER_TEXT.replace(
|
|
'atlas_ai_quota_fetch_success{provider="openai"} 1',
|
|
'atlas_ai_quota_fetch_success{provider="openai"} 0',
|
|
)
|
|
snapshot = lanes.parse_quota_metrics(text)
|
|
|
|
assert "codex" not in snapshot
|
|
assert snapshot["claude"].remaining_percent == 40
|
|
|
|
|
|
def test_parse_reports_no_reset_when_binding_window_lacks_one():
|
|
text = (
|
|
'atlas_ai_quota_remaining_percent{limit="claude",provider="anthropic",window="five_hour"} 9\n'
|
|
)
|
|
snapshot = lanes.parse_quota_metrics(text)
|
|
|
|
assert snapshot == {"claude": lanes.ProviderQuota(9.0, None)}
|
|
|
|
|
|
def test_fetch_snapshot_fails_open_on_transport_error(monkeypatch, capsys):
|
|
failures = []
|
|
monkeypatch.setattr(lanes, "record_quota_fetch_failure", lambda: failures.append(1))
|
|
|
|
def refused(_url, timeout):
|
|
raise OSError("connection refused")
|
|
|
|
assert lanes.fetch_quota_snapshot(open_url=refused) == {}
|
|
assert failures == [1]
|
|
assert "fails open" in capsys.readouterr().err
|
|
|
|
|
|
def test_fetch_snapshot_counts_empty_signal_and_parses_live_payload(
|
|
monkeypatch, capsys
|
|
):
|
|
failures = []
|
|
monkeypatch.setattr(lanes, "record_quota_fetch_failure", lambda: failures.append(1))
|
|
|
|
empty = lanes.fetch_quota_snapshot(
|
|
open_url=lambda _url, timeout: _MetricsResponse("")
|
|
)
|
|
assert empty == {}
|
|
assert failures == [1]
|
|
assert "no gated windows" in capsys.readouterr().err
|
|
|
|
live = lanes.fetch_quota_snapshot(
|
|
open_url=lambda _url, timeout: _MetricsResponse(EXPORTER_TEXT)
|
|
)
|
|
assert live["codex"].remaining_percent == 37.5
|
|
assert failures == [1]
|
|
|
|
|
|
def test_threshold_boundary_is_strictly_below():
|
|
for remaining, expected in ((14.9, "codex"), (15.0, None), (15.1, None)):
|
|
snapshot = {
|
|
"codex": lanes.ProviderQuota(remaining, None),
|
|
"claude": lanes.ProviderQuota(80.0, None),
|
|
}
|
|
excluded, note = lanes.quota_soft_exclusion(snapshot, 15.0)
|
|
assert excluded == expected
|
|
if expected is None:
|
|
assert note is None
|
|
else:
|
|
assert "14.9% remaining" in note
|
|
assert "threshold 15%" in note
|
|
|
|
|
|
def test_both_providers_below_threshold_prefers_more_remaining():
|
|
snapshot = {
|
|
"codex": lanes.ProviderQuota(3.0, None),
|
|
"claude": lanes.ProviderQuota(9.0, None),
|
|
}
|
|
excluded, note = lanes.quota_soft_exclusion(snapshot, 15.0)
|
|
|
|
assert excluded == "codex"
|
|
assert "every provider is below" in note
|
|
assert "preferring claude (9% left) over codex (3% left)" in note
|
|
|
|
|
|
def test_both_below_tie_excludes_deterministically():
|
|
snapshot = {
|
|
"codex": lanes.ProviderQuota(5.0, None),
|
|
"claude": lanes.ProviderQuota(5.0, None),
|
|
}
|
|
excluded, _note = lanes.quota_soft_exclusion(snapshot, 15.0)
|
|
|
|
assert excluded == "claude"
|
|
|
|
|
|
def test_single_known_provider_below_threshold_is_excluded():
|
|
snapshot = {"claude": lanes.ProviderQuota(2.0, None)}
|
|
excluded, note = lanes.quota_soft_exclusion(snapshot, 15.0)
|
|
|
|
assert excluded == "claude"
|
|
assert "routing new auto work" in note
|
|
|
|
|
|
def test_kanban_setting_reads_deployed_config_and_fails_to_default(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path)
|
|
assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0
|
|
|
|
config = tmp_path / "config.yaml"
|
|
config.write_text("kanban:\n provider_quota_min_remaining_percent: 22\n")
|
|
assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 22.0
|
|
|
|
config.write_text("kanban: 5\n")
|
|
assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0
|
|
|
|
config.write_text("just a string\n")
|
|
assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0
|
|
|
|
config.write_text("kanban:\n provider_quota_min_remaining_percent: true\n")
|
|
assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0
|
|
|
|
config.write_text("kanban:\n provider_quota_min_remaining_percent: '20'\n")
|
|
assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0
|
|
|
|
config.write_text("kanban: [\n")
|
|
assert lanes.kanban_setting("provider_quota_min_remaining_percent", 15.0) == 15.0
|
|
|
|
|
|
def test_selection_constraint_never_gates_explicit_lanes():
|
|
def forbidden():
|
|
raise AssertionError("explicit lanes must not read quota state")
|
|
|
|
constraint = lanes.selection_constraint("cli-codex-high", snapshot_from=forbidden)
|
|
|
|
assert constraint == lanes.SelectionConstraint(None, None, None, ())
|
|
|
|
|
|
def test_selection_constraint_health_exclusion_wins_and_carries_resets(monkeypatch):
|
|
observed = {}
|
|
|
|
def health(now=None, quota_resets=None):
|
|
observed["resets"] = quota_resets
|
|
return "codex"
|
|
|
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", health)
|
|
snapshot = {"claude": lanes.ProviderQuota(50.0, 1234.0)}
|
|
constraint = lanes.selection_constraint(
|
|
"cli-auto", snapshot_from=lambda: snapshot
|
|
)
|
|
|
|
assert constraint.exclude_provider == "codex"
|
|
assert constraint.source == "health"
|
|
assert constraint.notes == ()
|
|
assert observed["resets"] == {"claude": 1234.0}
|
|
|
|
|
|
def test_selection_constraint_quota_gate_publishes_gauges(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path)
|
|
monkeypatch.setattr(
|
|
lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None
|
|
)
|
|
snapshot = {
|
|
"codex": lanes.ProviderQuota(10.0, 555.0),
|
|
"claude": lanes.ProviderQuota(60.0, None),
|
|
}
|
|
constraint = lanes.selection_constraint("cli-auto", snapshot_from=lambda: snapshot)
|
|
|
|
assert constraint.exclude_provider == "codex"
|
|
assert constraint.source == "quota"
|
|
assert constraint.exclude_reason == "is below its remaining-quota routing threshold"
|
|
assert len(constraint.notes) == 1 and "Quota guard: codex" in constraint.notes[0]
|
|
rendered = lanes.METRICS.render().decode("utf-8")
|
|
assert 'hermes_cli_quota_remaining_percent{provider="codex"} 10' in rendered
|
|
assert 'hermes_cli_quota_reset_timestamp_seconds{provider="codex"} 555' in rendered
|
|
assert 'hermes_cli_provider_soft_excluded{provider="codex"} 1' in rendered
|
|
assert 'hermes_cli_provider_soft_excluded{provider="claude"} 0' in rendered
|
|
|
|
|
|
def test_selection_constraint_fails_open_without_quota_signal(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path)
|
|
monkeypatch.setattr(
|
|
lanes, "fresh_unavailable_provider", lambda *_args, **_kwargs: None
|
|
)
|
|
constraint = lanes.selection_constraint("cli-auto", snapshot_from=dict)
|
|
|
|
assert constraint == lanes.SelectionConstraint(None, None, None, ())
|
|
|
|
|
|
def _agent_config_kanban() -> dict:
|
|
document = yaml.safe_load((HERMES / "agent-configmap.yaml").read_text())
|
|
return yaml.safe_load(document["data"]["config.yaml"])["kanban"]
|
|
|
|
|
|
def test_configmap_declares_quota_routing_defaults():
|
|
kanban = _agent_config_kanban()
|
|
|
|
assert kanban["provider_quota_min_remaining_percent"] == 15
|
|
assert kanban["provider_capacity_cooldown_seconds"] == 300
|
|
assert kanban["provider_auth_cooldown_seconds"] == 3600
|
|
|
|
|
|
def test_lane_metrics_are_wired_for_scraping():
|
|
deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text())
|
|
containers = deployment["spec"]["template"]["spec"]["containers"]
|
|
lane = next(item for item in containers if item["name"] == "cli-lane-runner")
|
|
env = {entry["name"]: entry.get("value") for entry in lane["env"]}
|
|
|
|
assert {"name": "lane-metrics", "containerPort": 9011, "protocol": "TCP"} in lane[
|
|
"ports"
|
|
]
|
|
assert env["HERMES_CLI_LANE_METRICS_PORT"] == "9011"
|
|
assert env["HERMES_CLI_QUOTA_METRICS_URL"] == "http://127.0.0.1:9010/metrics"
|
|
|
|
services = {
|
|
item["metadata"]["name"]: item
|
|
for item in yaml.safe_load_all((HERMES / "service.yaml").read_text())
|
|
if item
|
|
}
|
|
metrics_service = services["hermes-cli-lane-metrics"]
|
|
annotations = metrics_service["metadata"]["annotations"]
|
|
assert annotations["prometheus.io/scrape"] == "true"
|
|
assert annotations["prometheus.io/port"] == "9011"
|
|
assert metrics_service["spec"]["selector"] == {"app": "hermes-agent"}
|
|
assert metrics_service["spec"]["ports"][0]["targetPort"] == "lane-metrics"
|
|
|
|
policies = {
|
|
item["metadata"]["name"]: item
|
|
for item in yaml.safe_load_all((HERMES / "networkpolicy.yaml").read_text())
|
|
if item
|
|
}
|
|
agent_ingress = policies["hermes-agent-isolation"]["spec"]["ingress"]
|
|
monitoring_ports = [
|
|
port["port"]
|
|
for rule in agent_ingress
|
|
for port in rule.get("ports", [])
|
|
if any(
|
|
source.get("namespaceSelector", {})
|
|
.get("matchLabels", {})
|
|
.get("kubernetes.io/metadata.name")
|
|
== "monitoring"
|
|
for source in rule.get("from", [])
|
|
)
|
|
]
|
|
assert 9010 in monitoring_ports and 9011 in monitoring_ports
|