From 42ac76fe6765fdd592d29a04f6302d91a4bcfcb7 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 16 Aug 2026 11:27:24 -0300 Subject: [PATCH] hermes: carry goal-loop controller evidence --- services/hermes/agent-deployment.yaml | 2 +- services/hermes/scripts/cli_lane_runner.py | 32 +++++++++++++++++--- testing/tests/test_hermes_cli_lanes.py | 35 +++++++++++++++++++--- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 1589eb44..d0c5495f 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260816-goal-completion-judge-v1" + ai.bstein.dev/config-rev: "20260816-goal-completion-judge-v2" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" diff --git a/services/hermes/scripts/cli_lane_runner.py b/services/hermes/scripts/cli_lane_runner.py index 2b63b42c..2ac3c99b 100644 --- a/services/hermes/scripts/cli_lane_runner.py +++ b/services/hermes/scripts/cli_lane_runner.py @@ -862,6 +862,7 @@ def execute_claim(board: str, task_id: str) -> None: excluded_provider = ( fresh_unavailable_provider() if assignee == "cli-auto" else None ) + unavailable_provider = excluded_provider route = select_route( context, assignee, @@ -914,6 +915,7 @@ def execute_claim(board: str, task_id: str) -> None: remaining, ) if result.capacity_failure: + unavailable_provider = route.provider retry_context = ( context + "\n\nRouting boundary: the first provider failed from capacity/authentication. " @@ -975,8 +977,18 @@ def execute_claim(board: str, task_id: str) -> None: and goal_mode ): heartbeat("local goal-completion judge active") + rejection_history = state.get("goal_rejections", []) + if not isinstance(rejection_history, list): + rejection_history = [] + judge_context = context + if goal_turn > 1 or rejection_history: + judge_context += ( + "\n\nAuthoritative Hermes goal-controller evidence: " + f"current turn {goal_turn}/{goal_max_turns}; prior rejected " + f"reports: {json.dumps(rejection_history[-5:])}." + ) accepted, judge_reason = cli_lane_goal.judge_goal_completion( - context, + judge_context, structured, ) metadata["goal_judge_reason"] = judge_reason @@ -1010,6 +1022,13 @@ def execute_claim(board: str, task_id: str) -> None: and deadline - time.monotonic() > 30 ) if can_continue: + rejection_history = state.get("goal_rejections", []) + if not isinstance(rejection_history, list): + rejection_history = [] + state["goal_rejections"] = [ + *rejection_history[-4:], + completion_problem, + ] goal_turn += 1 comment( f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}", @@ -1020,9 +1039,9 @@ def execute_claim(board: str, task_id: str) -> None: + completion_problem + "\nSelect a route that can finish and verify the remaining work." ) - excluded = ( - fresh_unavailable_provider() if assignee == "cli-auto" else None - ) + excluded = unavailable_provider + if excluded is None and assignee == "cli-auto": + excluded = fresh_unavailable_provider() next_route = select_route( escalation_context, assignee, @@ -1031,6 +1050,11 @@ def execute_claim(board: str, task_id: str) -> None: if excluded else None, ) + comment( + f"Goal route {goal_turn}/{goal_max_turns}: " + f"{next_route.provider}/{next_route.model} at {next_route.effort}; " + f"classifier={next_route.classifier}; {next_route.reason}", + ) handoff = ( git_handoff(workspace, result.output) if next_route.provider != route.provider diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index 31327578..899cb07a 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -727,12 +727,29 @@ def test_goal_card_continues_after_local_judge_rejects_progress( "state_path", lambda _board, _task_id: tmp_path / "state.json", ) - route = lanes.Route( - "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, () + claude_low = lanes.Route( + "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, () ) - monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route) + codex_low = lanes.Route( + "codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, () + ) + codex_xhigh = lanes.Route( + "codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, () + ) + route_calls = [] + + def select_route(_prompt, assignee, **kwargs): + route_calls.append((assignee, kwargs)) + if assignee == "cli-codex-low": + return codex_low + if len(route_calls) == 1: + return claude_low + return codex_xhigh + + monkeypatch.setattr(lanes, "select_route", select_route) monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) reports = [ + lanes.ProcessResult(1, "authentication expired", None, True), lanes.ProcessResult( 0, "first turn", @@ -767,10 +784,16 @@ def test_goal_card_continues_after_local_judge_rejects_progress( (True, "all explicit acceptance criteria have evidence"), ] ) + judge_contexts = [] + + def judge_goal_completion(objective, *_args, **_kwargs): + judge_contexts.append(objective) + return next(verdicts) + monkeypatch.setattr( lanes.cli_lane_goal, "judge_goal_completion", - lambda *_args, **_kwargs: next(verdicts), + judge_goal_completion, ) lanes.execute_claim("cassandra", "t_goal") @@ -778,6 +801,10 @@ def test_goal_card_continues_after_local_judge_rejects_progress( assert calls[0][0] == "complete" assert calls[0][1]["metadata"]["goal_turn"] == 2 assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments) + assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments) + assert route_calls[2][1]["exclude_provider"] == "claude" + assert "prior rejected reports" in judge_contexts[1] + assert "commit, push, and remote verification are missing" in judge_contexts[1] assert reports == []