From ba590c25187e3b4ae2f5bce617fa6d873870b37f Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 13:47:19 -0300 Subject: [PATCH] hermes: retain native auth failure health --- services/hermes/agent-deployment.yaml | 2 +- services/hermes/scripts/cli_lane_runner.py | 8 +++++++- testing/tests/test_hermes_cli_lanes.py | 23 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 3d8cd0da..845eb6f9 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260816-goal-judge-timeout-v5" + ai.bstein.dev/config-rev: "20260816-auth-health-quarantine-v6" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 5191ef2d..3eba3453 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -41,6 +41,7 @@ DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60 DEFAULT_MAX_RUNTIME = 12 * 60 * 60 HEARTBEAT_SECONDS = 20 PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60 +PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60 KANBAN_STORAGE_ATTEMPTS = 5 PROVIDER_HEALTH_PATHS = { "codex": DATA_ROOT / "provider-health/codex.json", @@ -164,8 +165,13 @@ def fresh_unavailable_provider(now: float | None = None) -> str | None: age = current - path.stat().st_mtime except OSError: continue + max_age = ( + PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS + if health.get("authenticated") is False + else PROVIDER_HEALTH_MAX_AGE_SECONDS + ) if ( - 0 <= age <= PROVIDER_HEALTH_MAX_AGE_SECONDS + 0 <= age <= max_age and health.get("state") == "unavailable" ): unavailable.append(provider) diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index cd4ad422..98093629 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -197,6 +197,29 @@ def test_fresh_native_health_excludes_only_one_proven_down_provider( assert lanes.fresh_unavailable_provider(now=now) is None +def test_explicit_auth_failure_survives_restart_health_gap(tmp_path: Path, monkeypatch): + paths = { + "codex": tmp_path / "codex.json", + "claude": tmp_path / "claude.json", + } + paths["codex"].write_text( + '{"state":"available","authenticated":true}\n', encoding="utf-8" + ) + paths["claude"].write_text( + '{"state":"unavailable","authenticated":false}\n', encoding="utf-8" + ) + monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths) + + stale_during_rollout = paths["claude"].stat().st_mtime + 10 * 60 + assert lanes.fresh_unavailable_provider(now=stale_during_rollout) == "claude" + + paths["claude"].write_text( + '{"state":"unavailable","authenticated":true}\n', encoding="utf-8" + ) + transient_stale = paths["claude"].stat().st_mtime + 10 * 60 + assert lanes.fresh_unavailable_provider(now=transient_stale) is None + + def test_worker_environment_preserves_vault_backed_cli_homes(monkeypatch): """Kanban workers must not fall back to credential-free persistent homes.""" monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex")