hermes: carry goal-loop controller evidence

This commit is contained in:
jenkins 2026-08-16 11:27:24 -03:00
parent 64dd9c9be4
commit 42ac76fe67
3 changed files with 60 additions and 9 deletions

View File

@ -25,7 +25,7 @@ spec:
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
ai.bstein.dev/config-rev: "20260816-goal-completion-judge-v1" ai.bstein.dev/config-rev: "20260816-goal-completion-judge-v2"
prometheus.io/scrape: "true" prometheus.io/scrape: "true"
prometheus.io/path: /metrics prometheus.io/path: /metrics
prometheus.io/port: "9010" prometheus.io/port: "9010"

View File

@ -862,6 +862,7 @@ def execute_claim(board: str, task_id: str) -> None:
excluded_provider = ( excluded_provider = (
fresh_unavailable_provider() if assignee == "cli-auto" else None fresh_unavailable_provider() if assignee == "cli-auto" else None
) )
unavailable_provider = excluded_provider
route = select_route( route = select_route(
context, context,
assignee, assignee,
@ -914,6 +915,7 @@ def execute_claim(board: str, task_id: str) -> None:
remaining, remaining,
) )
if result.capacity_failure: if result.capacity_failure:
unavailable_provider = route.provider
retry_context = ( retry_context = (
context context
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. " + "\n\nRouting boundary: the first provider failed from capacity/authentication. "
@ -975,8 +977,18 @@ def execute_claim(board: str, task_id: str) -> None:
and goal_mode and goal_mode
): ):
heartbeat("local goal-completion judge active") heartbeat("local goal-completion judge active")
rejection_history = state.get("goal_rejections", [])
if not isinstance(rejection_history, list):
rejection_history = []
judge_context = context
if goal_turn > 1 or rejection_history:
judge_context += (
"\n\nAuthoritative Hermes goal-controller evidence: "
f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
f"reports: {json.dumps(rejection_history[-5:])}."
)
accepted, judge_reason = cli_lane_goal.judge_goal_completion( accepted, judge_reason = cli_lane_goal.judge_goal_completion(
context, judge_context,
structured, structured,
) )
metadata["goal_judge_reason"] = judge_reason metadata["goal_judge_reason"] = judge_reason
@ -1010,6 +1022,13 @@ def execute_claim(board: str, task_id: str) -> None:
and deadline - time.monotonic() > 30 and deadline - time.monotonic() > 30
) )
if can_continue: if can_continue:
rejection_history = state.get("goal_rejections", [])
if not isinstance(rejection_history, list):
rejection_history = []
state["goal_rejections"] = [
*rejection_history[-4:],
completion_problem,
]
goal_turn += 1 goal_turn += 1
comment( comment(
f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}", f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
@ -1020,9 +1039,9 @@ def execute_claim(board: str, task_id: str) -> None:
+ completion_problem + completion_problem
+ "\nSelect a route that can finish and verify the remaining work." + "\nSelect a route that can finish and verify the remaining work."
) )
excluded = ( excluded = unavailable_provider
fresh_unavailable_provider() if assignee == "cli-auto" else None if excluded is None and assignee == "cli-auto":
) excluded = fresh_unavailable_provider()
next_route = select_route( next_route = select_route(
escalation_context, escalation_context,
assignee, assignee,
@ -1031,6 +1050,11 @@ def execute_claim(board: str, task_id: str) -> None:
if excluded if excluded
else None, else None,
) )
comment(
f"Goal route {goal_turn}/{goal_max_turns}: "
f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
f"classifier={next_route.classifier}; {next_route.reason}",
)
handoff = ( handoff = (
git_handoff(workspace, result.output) git_handoff(workspace, result.output)
if next_route.provider != route.provider if next_route.provider != route.provider

View File

@ -727,12 +727,29 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
"state_path", "state_path",
lambda _board, _task_id: tmp_path / "state.json", lambda _board, _task_id: tmp_path / "state.json",
) )
route = lanes.Route( claude_low = lanes.Route(
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, () "claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, ()
) )
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route) codex_low = lanes.Route(
"codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, ()
)
codex_xhigh = lanes.Route(
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, ()
)
route_calls = []
def select_route(_prompt, assignee, **kwargs):
route_calls.append((assignee, kwargs))
if assignee == "cli-codex-low":
return codex_low
if len(route_calls) == 1:
return claude_low
return codex_xhigh
monkeypatch.setattr(lanes, "select_route", select_route)
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None) monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
reports = [ reports = [
lanes.ProcessResult(1, "authentication expired", None, True),
lanes.ProcessResult( lanes.ProcessResult(
0, 0,
"first turn", "first turn",
@ -767,10 +784,16 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
(True, "all explicit acceptance criteria have evidence"), (True, "all explicit acceptance criteria have evidence"),
] ]
) )
judge_contexts = []
def judge_goal_completion(objective, *_args, **_kwargs):
judge_contexts.append(objective)
return next(verdicts)
monkeypatch.setattr( monkeypatch.setattr(
lanes.cli_lane_goal, lanes.cli_lane_goal,
"judge_goal_completion", "judge_goal_completion",
lambda *_args, **_kwargs: next(verdicts), judge_goal_completion,
) )
lanes.execute_claim("cassandra", "t_goal") lanes.execute_claim("cassandra", "t_goal")
@ -778,6 +801,10 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
assert calls[0][0] == "complete" assert calls[0][0] == "complete"
assert calls[0][1]["metadata"]["goal_turn"] == 2 assert calls[0][1]["metadata"]["goal_turn"] == 2
assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments) assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments)
assert route_calls[2][1]["exclude_provider"] == "claude"
assert "prior rejected reports" in judge_contexts[1]
assert "commit, push, and remote verification are missing" in judge_contexts[1]
assert reports == [] assert reports == []