titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

hermes: retain native auth failure health

Browse Source
This commit is contained in:
jenkins 2026-08-16 13:47:19 -03:00
parent e8b4f0c57c
commit ba590c2518
3 changed files with 31 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
services/hermes/agent-deployment.yaml
View File

@ -25,7 +25,7 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
ai.bstein.dev/config-rev: "20260816-goal-judge-timeout-v5" ai.bstein.dev/config-rev: "20260816-auth-health-quarantine-v6"
prometheus.io/scrape: "true" prometheus.io/scrape: "true"
prometheus.io/path: /metrics prometheus.io/path: /metrics
prometheus.io/port: "9010" prometheus.io/port: "9010"

8
services/hermes/scripts/cli_lane_runner.py
View File

@ -41,6 +41,7 @@ DEFAULT_CLAIM_TTL = 7 * 24 * 60 * 60
DEFAULT_MAX_RUNTIME = 12 * 60 * 60 DEFAULT_MAX_RUNTIME = 12 * 60 * 60
HEARTBEAT_SECONDS = 20 HEARTBEAT_SECONDS = 20
PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60 PROVIDER_HEALTH_MAX_AGE_SECONDS = 5 * 60
PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS = 12 * 60 * 60
KANBAN_STORAGE_ATTEMPTS = 5 KANBAN_STORAGE_ATTEMPTS = 5
PROVIDER_HEALTH_PATHS = { PROVIDER_HEALTH_PATHS = {
"codex": DATA_ROOT / "provider-health/codex.json", "codex": DATA_ROOT / "provider-health/codex.json",
@ -164,8 +165,13 @@ def fresh_unavailable_provider(now: float | None = None) -> str | None:
age = current - path.stat().st_mtime age = current - path.stat().st_mtime
except OSError: except OSError:
continue continue
max_age = (
PROVIDER_AUTH_FAILURE_MAX_AGE_SECONDS
if health.get("authenticated") is False
else PROVIDER_HEALTH_MAX_AGE_SECONDS
)
if ( if (
0 <= age <= PROVIDER_HEALTH_MAX_AGE_SECONDS 0 <= age <= max_age
and health.get("state") == "unavailable" and health.get("state") == "unavailable"
): ):
unavailable.append(provider) unavailable.append(provider)

23
testing/tests/test_hermes_cli_lanes.py
View File

@ -197,6 +197,29 @@ def test_fresh_native_health_excludes_only_one_proven_down_provider(
assert lanes.fresh_unavailable_provider(now=now) is None assert lanes.fresh_unavailable_provider(now=now) is None
def test_explicit_auth_failure_survives_restart_health_gap(tmp_path: Path, monkeypatch):
paths = {
"codex": tmp_path / "codex.json",
"claude": tmp_path / "claude.json",
}
paths["codex"].write_text(
'{"state":"available","authenticated":true}\n', encoding="utf-8"
)
paths["claude"].write_text(
'{"state":"unavailable","authenticated":false}\n', encoding="utf-8"
)
monkeypatch.setattr(lanes, "PROVIDER_HEALTH_PATHS", paths)
stale_during_rollout = paths["claude"].stat().st_mtime + 10 * 60
assert lanes.fresh_unavailable_provider(now=stale_during_rollout) == "claude"
paths["claude"].write_text(
'{"state":"unavailable","authenticated":true}\n', encoding="utf-8"
)
transient_stale = paths["claude"].stat().st_mtime + 10 * 60
assert lanes.fresh_unavailable_provider(now=transient_stale) is None
def test_worker_environment_preserves_vault_backed_cli_homes(monkeypatch): def test_worker_environment_preserves_vault_backed_cli_homes(monkeypatch):
"""Kanban workers must not fall back to credential-free persistent homes.""" """Kanban workers must not fall back to credential-free persistent homes."""
monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex") monkeypatch.setenv("CODEX_HOME", "/runtime-access/codex")