"""Lane-side provider health writes, cooldown hysteresis, and recovery.""" from __future__ import annotations from testing.tests.test_hermes_cli_support import ( Path, json, lanes, pytest, ) @pytest.fixture def health_paths(tmp_path: Path, monkeypatch) -> dict[str, Path]: paths = { "codex": tmp_path / "health/codex.json", "claude": tmp_path / "health/claude.json", } monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths) monkeypatch.setattr(lanes, "DATA_ROOT", tmp_path) return paths @pytest.mark.parametrize( ("output", "expected"), [ ("You have hit your usage limit for this billing cycle.", "quota"), ("credit balance exhausted", "quota"), ("HTTP 429: too many requests", "rate-limit"), ("overloaded_error (529)", "rate-limit"), ("OAuth token expired; run codex login", "auth"), ("401 Unauthorized", "auth"), ("stream disconnected before completion", "transport"), ], ) def test_capacity_failures_are_classified_actionably(output: str, expected: str): assert lanes.classify_capacity_failure(output) == expected def test_capacity_failure_records_cooldown_with_defaults(health_paths): until = lanes.record_provider_failure("codex", "quota", now=1000.0) value = json.loads(health_paths["codex"].read_text()) assert until == 1300.0 assert value["state"] == "capacity-limited" assert value["authenticated"] is True assert value["failure_reason"] == "quota" assert value["failed_at"] == 1000.0 assert value["cooldown_until"] == 1300.0 assert value["source"] == "cli-lane-runner" def test_auth_failure_uses_longer_cooldown_and_marks_unauthenticated(health_paths): until = lanes.record_provider_failure("claude", "auth", now=1000.0) value = json.loads(health_paths["claude"].read_text()) assert until == 4600.0 assert value["state"] == "unavailable" assert value["authenticated"] is False def test_cooldowns_are_configurable_through_the_kanban_block( health_paths, tmp_path: Path ): (tmp_path / "config.yaml").write_text( "kanban:\n" " provider_capacity_cooldown_seconds: 60\n" " provider_auth_cooldown_seconds: 120\n" ) assert lanes.record_provider_failure("codex", "rate-limit", now=1000.0) == 1060.0 assert lanes.record_provider_failure("codex", "auth", now=1000.0) == 1120.0 def test_lane_writes_merge_with_broker_health_snapshots(health_paths): health_paths["codex"].parent.mkdir(parents=True) health_paths["codex"].write_text( json.dumps({"transport": "codex-chatgpt-subscription", "state": "available"}) ) lanes.record_provider_failure("codex", "quota", now=1000.0) value = json.loads(health_paths["codex"].read_text()) assert value["transport"] == "codex-chatgpt-subscription" assert value["state"] == "capacity-limited" def test_success_clears_failure_evidence_and_readmits(health_paths): lanes.record_provider_failure("codex", "quota", now=1000.0) lanes.record_provider_success("codex") value = json.loads(health_paths["codex"].read_text()) assert value["state"] == "available" assert value["authenticated"] is True for stale in ("failure_reason", "failed_at", "cooldown_until"): assert stale not in value assert lanes.fresh_unavailable_provider(now=1001.0) is None def test_health_write_failures_never_break_the_lane(health_paths, tmp_path: Path): blocker = tmp_path / "health" blocker.write_text("a file where the directory must go") lanes.record_provider_failure("codex", "quota", now=1000.0) def test_broker_reported_capacity_limited_state_is_excluded(health_paths): health_paths["codex"].parent.mkdir(parents=True) health_paths["codex"].write_text('{"state":"capacity-limited"}\n') now = health_paths["codex"].stat().st_mtime assert lanes.fresh_unavailable_provider(now=now) == "codex" def test_cooldown_must_fully_elapse_before_readmission(health_paths): lanes.record_provider_failure("codex", "quota", now=1000.0) assert lanes.fresh_unavailable_provider(now=1299.9) == "codex" assert lanes.fresh_unavailable_provider(now=1300.0) is None def test_quota_reset_readmits_capacity_limited_provider_early(health_paths): lanes.record_provider_failure("codex", "quota", now=1000.0) # Reset before the failure proves nothing; a future reset has not passed. assert ( lanes.fresh_unavailable_provider(now=1100.0, quota_resets={"codex": 900.0}) == "codex" ) assert ( lanes.fresh_unavailable_provider(now=1100.0, quota_resets={"codex": 1150.0}) == "codex" ) # A reset between the failure and now means the window rolled over. assert ( lanes.fresh_unavailable_provider(now=1200.0, quota_resets={"codex": 1150.0}) is None ) def test_quota_reset_never_readmits_auth_failures(health_paths): lanes.record_provider_failure("codex", "auth", now=1000.0) assert ( lanes.fresh_unavailable_provider(now=1200.0, quota_resets={"codex": 1150.0}) == "codex" ) def test_both_providers_in_cooldown_defer_to_live_attempts(health_paths): lanes.record_provider_failure("codex", "quota", now=1000.0) lanes.record_provider_failure("claude", "quota", now=1000.0) assert lanes.fresh_unavailable_provider(now=1100.0) is None def test_vanished_health_file_is_not_excluded(health_paths, monkeypatch): monkeypatch.setattr(lanes, "load_json", lambda _path: {"state": "unavailable"}) assert lanes.fresh_unavailable_provider(now=1000.0) is None def test_malformed_cooldown_falls_back_to_freshness_window(health_paths): health_paths["claude"].parent.mkdir(parents=True) health_paths["claude"].write_text( '{"state":"capacity-limited","cooldown_until":true}\n' ) now = health_paths["claude"].stat().st_mtime assert lanes.fresh_unavailable_provider(now=now) == "claude" assert lanes.fresh_unavailable_provider(now=now + 10 * 60) is None def test_health_number_rejects_booleans_and_strings(): assert lanes._health_number(True) is None assert lanes._health_number("5") is None assert lanes._health_number(5) == 5.0 def test_worker_environment_strips_metered_api_keys(monkeypatch): for name in ( "ANTHROPIC_API_KEY", "CLAUDE_API_KEY", "OPENAI_API_KEY", "API_SERVER_KEY", ): monkeypatch.setenv(name, "must-not-leak") monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex") env = lanes._base_env() for name in ( "ANTHROPIC_API_KEY", "CLAUDE_API_KEY", "OPENAI_API_KEY", "API_SERVER_KEY", ): assert name not in env assert env["CODEX_HOME"] == "/runtime-access/codex" assert env["GIT_TERMINAL_PROMPT"] == "0"