atlas-iac/testing/tests/test_hermes_cli_provider_health.py

"""Lane-side provider health writes, cooldown hysteresis, and recovery."""

from __future__ import annotations

from testing.tests.test_hermes_cli_support import (
    Path,
    json,
    lanes,
    pytest,
)


@pytest.fixture
def health_paths(tmp_path: Path, monkeypatch) -> dict[str, Path]:
    paths = {
        "codex": tmp_path / "health/codex.json",
        "claude": tmp_path / "health/claude.json",
    }
    monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths)
    monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path)
    return paths


@pytest.mark.parametrize(
    ("output", "expected"),
    [
        ("You have hit your usage limit for this billing cycle.", "quota"),
        ("credit balance exhausted", "quota"),
        ("HTTP 429: too many requests", "rate-limit"),
        ("overloaded_error (529)", "rate-limit"),
        ("OAuth token expired; run codex login", "auth"),
        ("401 Unauthorized", "auth"),
        ("stream disconnected before completion", "transport"),
    ],
)
def test_capacity_failures_are_classified_actionably(output: str, expected: str):
    assert lanes.classify_capacity_failure(output) == expected


def test_capacity_failure_records_cooldown_with_defaults(health_paths):
    until = lanes.record_provider_failure("codex", "quota", now=1000.0)

    value = json.loads(health_paths["codex"].read_text())
    assert until == 1300.0
    assert value["state"] == "capacity-limited"
    assert value["authenticated"] is True
    assert value["failure_reason"] == "quota"
    assert value["failed_at"] == 1000.0
    assert value["cooldown_until"] == 1300.0
    assert value["source"] == "cli-lane-runner"


def test_auth_failure_uses_longer_cooldown_and_marks_unauthenticated(health_paths):
    until = lanes.record_provider_failure("claude", "auth", now=1000.0)

    value = json.loads(health_paths["claude"].read_text())
    assert until == 4600.0
    assert value["state"] == "unavailable"
    assert value["authenticated"] is False


def test_cooldowns_are_configurable_through_the_kanban_block(
    health_paths, tmp_path: Path
):
    (tmp_path / "config.yaml").write_text(
        "kanban:\n"
        "  provider_capacity_cooldown_seconds: 60\n"
        "  provider_auth_cooldown_seconds: 120\n"
    )

    assert lanes.record_provider_failure("codex", "rate-limit", now=1000.0) == 1060.0
    assert lanes.record_provider_failure("codex", "auth", now=1000.0) == 1120.0


def test_lane_writes_merge_with_broker_health_snapshots(health_paths):
    health_paths["codex"].parent.mkdir(parents=True)
    health_paths["codex"].write_text(
        json.dumps({"transport": "codex-chatgpt-subscription", "state": "available"})
    )

    lanes.record_provider_failure("codex", "quota", now=1000.0)

    value = json.loads(health_paths["codex"].read_text())
    assert value["transport"] == "codex-chatgpt-subscription"
    assert value["state"] == "capacity-limited"


def test_success_clears_failure_evidence_and_readmits(health_paths):
    lanes.record_provider_failure("codex", "quota", now=1000.0)
    lanes.record_provider_success("codex")

    value = json.loads(health_paths["codex"].read_text())
    assert value["state"] == "available"
    assert value["authenticated"] is True
    for stale in ("failure_reason", "failed_at", "cooldown_until"):
        assert stale not in value
    assert lanes.fresh_unavailable_provider(now=1001.0) is None


def test_health_write_failures_never_break_the_lane(health_paths, tmp_path: Path):
    blocker = tmp_path / "health"
    blocker.write_text("a file where the directory must go")

    lanes.record_provider_failure("codex", "quota", now=1000.0)


def test_broker_reported_capacity_limited_state_is_excluded(health_paths):
    health_paths["codex"].parent.mkdir(parents=True)
    health_paths["codex"].write_text('{"state":"capacity-limited"}\n')

    now = health_paths["codex"].stat().st_mtime
    assert lanes.fresh_unavailable_provider(now=now) == "codex"


def test_cooldown_must_fully_elapse_before_readmission(health_paths):
    lanes.record_provider_failure("codex", "quota", now=1000.0)

    assert lanes.fresh_unavailable_provider(now=1299.9) == "codex"
    assert lanes.fresh_unavailable_provider(now=1300.0) is None


def test_quota_reset_readmits_capacity_limited_provider_early(health_paths):
    lanes.record_provider_failure("codex", "quota", now=1000.0)

    # Reset before the failure proves nothing; a future reset has not passed.
    assert (
        lanes.fresh_unavailable_provider(now=1100.0, quota_resets={"codex": 900.0})
        == "codex"
    )
    assert (
        lanes.fresh_unavailable_provider(now=1100.0, quota_resets={"codex": 1150.0})
        == "codex"
    )
    # A reset between the failure and now means the window rolled over.
    assert (
        lanes.fresh_unavailable_provider(now=1200.0, quota_resets={"codex": 1150.0})
        is None
    )


def test_quota_reset_never_readmits_auth_failures(health_paths):
    lanes.record_provider_failure("codex", "auth", now=1000.0)

    assert (
        lanes.fresh_unavailable_provider(now=1200.0, quota_resets={"codex": 1150.0})
        == "codex"
    )


def test_both_providers_in_cooldown_defer_to_live_attempts(health_paths):
    lanes.record_provider_failure("codex", "quota", now=1000.0)
    lanes.record_provider_failure("claude", "quota", now=1000.0)

    assert lanes.fresh_unavailable_provider(now=1100.0) is None


def test_vanished_health_file_is_not_excluded(health_paths, monkeypatch):
    monkeypatch.setattr(lanes, "load_json", lambda _path: {"state": "unavailable"})

    assert lanes.fresh_unavailable_provider(now=1000.0) is None


def test_malformed_cooldown_falls_back_to_freshness_window(health_paths):
    health_paths["claude"].parent.mkdir(parents=True)
    health_paths["claude"].write_text(
        '{"state":"capacity-limited","cooldown_until":true}\n'
    )

    now = health_paths["claude"].stat().st_mtime
    assert lanes.fresh_unavailable_provider(now=now) == "claude"
    assert lanes.fresh_unavailable_provider(now=now + 10 * 60) is None


def test_health_number_rejects_booleans_and_strings():
    assert lanes._health_number(True) is None
    assert lanes._health_number("5") is None
    assert lanes._health_number(5) == 5.0


def test_worker_environment_strips_metered_api_keys(monkeypatch):
    for name in (
        "ANTHROPIC_API_KEY",
        "CLAUDE_API_KEY",
        "OPENAI_API_KEY",
        "API_SERVER_KEY",
    ):
        monkeypatch.setenv(name, "must-not-leak")
    monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex")

    env = lanes._base_env()

    for name in (
        "ANTHROPIC_API_KEY",
        "CLAUDE_API_KEY",
        "OPENAI_API_KEY",
        "API_SERVER_KEY",
    ):
        assert name not in env
    assert env["CODEX_HOME"] == "/runtime-access/codex"
    assert env["GIT_TERMINAL_PROMPT"] == "0"