hermes: carry goal-loop controller evidence
This commit is contained in:
parent
64dd9c9be4
commit
42ac76fe67
@ -25,7 +25,7 @@ spec:
|
|||||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||||
ai.bstein.dev/config-rev: "20260816-goal-completion-judge-v1"
|
ai.bstein.dev/config-rev: "20260816-goal-completion-judge-v2"
|
||||||
prometheus.io/scrape: "true"
|
prometheus.io/scrape: "true"
|
||||||
prometheus.io/path: /metrics
|
prometheus.io/path: /metrics
|
||||||
prometheus.io/port: "9010"
|
prometheus.io/port: "9010"
|
||||||
|
|||||||
@ -862,6 +862,7 @@ def execute_claim(board: str, task_id: str) -> None:
|
|||||||
excluded_provider = (
|
excluded_provider = (
|
||||||
fresh_unavailable_provider() if assignee == "cli-auto" else None
|
fresh_unavailable_provider() if assignee == "cli-auto" else None
|
||||||
)
|
)
|
||||||
|
unavailable_provider = excluded_provider
|
||||||
route = select_route(
|
route = select_route(
|
||||||
context,
|
context,
|
||||||
assignee,
|
assignee,
|
||||||
@ -914,6 +915,7 @@ def execute_claim(board: str, task_id: str) -> None:
|
|||||||
remaining,
|
remaining,
|
||||||
)
|
)
|
||||||
if result.capacity_failure:
|
if result.capacity_failure:
|
||||||
|
unavailable_provider = route.provider
|
||||||
retry_context = (
|
retry_context = (
|
||||||
context
|
context
|
||||||
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
|
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
|
||||||
@ -975,8 +977,18 @@ def execute_claim(board: str, task_id: str) -> None:
|
|||||||
and goal_mode
|
and goal_mode
|
||||||
):
|
):
|
||||||
heartbeat("local goal-completion judge active")
|
heartbeat("local goal-completion judge active")
|
||||||
|
rejection_history = state.get("goal_rejections", [])
|
||||||
|
if not isinstance(rejection_history, list):
|
||||||
|
rejection_history = []
|
||||||
|
judge_context = context
|
||||||
|
if goal_turn > 1 or rejection_history:
|
||||||
|
judge_context += (
|
||||||
|
"\n\nAuthoritative Hermes goal-controller evidence: "
|
||||||
|
f"current turn {goal_turn}/{goal_max_turns}; prior rejected "
|
||||||
|
f"reports: {json.dumps(rejection_history[-5:])}."
|
||||||
|
)
|
||||||
accepted, judge_reason = cli_lane_goal.judge_goal_completion(
|
accepted, judge_reason = cli_lane_goal.judge_goal_completion(
|
||||||
context,
|
judge_context,
|
||||||
structured,
|
structured,
|
||||||
)
|
)
|
||||||
metadata["goal_judge_reason"] = judge_reason
|
metadata["goal_judge_reason"] = judge_reason
|
||||||
@ -1010,6 +1022,13 @@ def execute_claim(board: str, task_id: str) -> None:
|
|||||||
and deadline - time.monotonic() > 30
|
and deadline - time.monotonic() > 30
|
||||||
)
|
)
|
||||||
if can_continue:
|
if can_continue:
|
||||||
|
rejection_history = state.get("goal_rejections", [])
|
||||||
|
if not isinstance(rejection_history, list):
|
||||||
|
rejection_history = []
|
||||||
|
state["goal_rejections"] = [
|
||||||
|
*rejection_history[-4:],
|
||||||
|
completion_problem,
|
||||||
|
]
|
||||||
goal_turn += 1
|
goal_turn += 1
|
||||||
comment(
|
comment(
|
||||||
f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
|
f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
|
||||||
@ -1020,9 +1039,9 @@ def execute_claim(board: str, task_id: str) -> None:
|
|||||||
+ completion_problem
|
+ completion_problem
|
||||||
+ "\nSelect a route that can finish and verify the remaining work."
|
+ "\nSelect a route that can finish and verify the remaining work."
|
||||||
)
|
)
|
||||||
excluded = (
|
excluded = unavailable_provider
|
||||||
fresh_unavailable_provider() if assignee == "cli-auto" else None
|
if excluded is None and assignee == "cli-auto":
|
||||||
)
|
excluded = fresh_unavailable_provider()
|
||||||
next_route = select_route(
|
next_route = select_route(
|
||||||
escalation_context,
|
escalation_context,
|
||||||
assignee,
|
assignee,
|
||||||
@ -1031,6 +1050,11 @@ def execute_claim(board: str, task_id: str) -> None:
|
|||||||
if excluded
|
if excluded
|
||||||
else None,
|
else None,
|
||||||
)
|
)
|
||||||
|
comment(
|
||||||
|
f"Goal route {goal_turn}/{goal_max_turns}: "
|
||||||
|
f"{next_route.provider}/{next_route.model} at {next_route.effort}; "
|
||||||
|
f"classifier={next_route.classifier}; {next_route.reason}",
|
||||||
|
)
|
||||||
handoff = (
|
handoff = (
|
||||||
git_handoff(workspace, result.output)
|
git_handoff(workspace, result.output)
|
||||||
if next_route.provider != route.provider
|
if next_route.provider != route.provider
|
||||||
|
|||||||
@ -727,12 +727,29 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
|
|||||||
"state_path",
|
"state_path",
|
||||||
lambda _board, _task_id: tmp_path / "state.json",
|
lambda _board, _task_id: tmp_path / "state.json",
|
||||||
)
|
)
|
||||||
route = lanes.Route(
|
claude_low = lanes.Route(
|
||||||
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()
|
"claude", "claude-fable-5", "low", "claude-low", "jetson", "vote", 1, ()
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
codex_low = lanes.Route(
|
||||||
|
"codex", "gpt-5.6-luna", "low", "codex-low", "manual", "fallback", 1, ()
|
||||||
|
)
|
||||||
|
codex_xhigh = lanes.Route(
|
||||||
|
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "escalated", 1, ()
|
||||||
|
)
|
||||||
|
route_calls = []
|
||||||
|
|
||||||
|
def select_route(_prompt, assignee, **kwargs):
|
||||||
|
route_calls.append((assignee, kwargs))
|
||||||
|
if assignee == "cli-codex-low":
|
||||||
|
return codex_low
|
||||||
|
if len(route_calls) == 1:
|
||||||
|
return claude_low
|
||||||
|
return codex_xhigh
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes, "select_route", select_route)
|
||||||
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
|
||||||
reports = [
|
reports = [
|
||||||
|
lanes.ProcessResult(1, "authentication expired", None, True),
|
||||||
lanes.ProcessResult(
|
lanes.ProcessResult(
|
||||||
0,
|
0,
|
||||||
"first turn",
|
"first turn",
|
||||||
@ -767,10 +784,16 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
|
|||||||
(True, "all explicit acceptance criteria have evidence"),
|
(True, "all explicit acceptance criteria have evidence"),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
judge_contexts = []
|
||||||
|
|
||||||
|
def judge_goal_completion(objective, *_args, **_kwargs):
|
||||||
|
judge_contexts.append(objective)
|
||||||
|
return next(verdicts)
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
lanes.cli_lane_goal,
|
lanes.cli_lane_goal,
|
||||||
"judge_goal_completion",
|
"judge_goal_completion",
|
||||||
lambda *_args, **_kwargs: next(verdicts),
|
judge_goal_completion,
|
||||||
)
|
)
|
||||||
|
|
||||||
lanes.execute_claim("cassandra", "t_goal")
|
lanes.execute_claim("cassandra", "t_goal")
|
||||||
@ -778,6 +801,10 @@ def test_goal_card_continues_after_local_judge_rejects_progress(
|
|||||||
assert calls[0][0] == "complete"
|
assert calls[0][0] == "complete"
|
||||||
assert calls[0][1]["metadata"]["goal_turn"] == 2
|
assert calls[0][1]["metadata"]["goal_turn"] == 2
|
||||||
assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
|
assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
|
||||||
|
assert any("Goal route 2/3: codex/gpt-5.6-sol at xhigh" in item for item in comments)
|
||||||
|
assert route_calls[2][1]["exclude_provider"] == "claude"
|
||||||
|
assert "prior rejected reports" in judge_contexts[1]
|
||||||
|
assert "commit, push, and remote verification are missing" in judge_contexts[1]
|
||||||
assert reports == []
|
assert reports == []
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user