hermes: enforce goal completion evidence

This commit is contained in:
jenkins 2026-08-16 10:51:34 -03:00
parent 468ab278ea
commit 1527fd1af0
6 changed files with 539 additions and 67 deletions

View File

@ -61,6 +61,7 @@ configMapGenerator:
- ai_usage_exporter.py=scripts/ai_usage_exporter.py
- claude=scripts/claude
- claude_command_policy.py=scripts/claude_command_policy.py
- cli_lane_goal.py=scripts/cli_lane_goal.py
- cli_lane_runner.py=scripts/cli_lane_runner.py
- codex=scripts/codex
- configure_agent_clients.py=scripts/configure_agent_clients.py

View File

@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""Fail-closed completion checks for durable external Kanban workers."""
from __future__ import annotations
import json
import os
import re
import urllib.error
import urllib.request
from typing import Any, Callable
GOAL_JUDGE_URL = os.environ.get(
"HERMES_GOAL_JUDGE_URL",
"http://hermes-model-gate.hermes.svc.cluster.local:11434/v1/chat/completions",
)
GOAL_JUDGE_MODEL = os.environ.get(
"HERMES_GOAL_JUDGE_MODEL",
"qwen2.5:14b-instruct-q4_0",
)
RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"})
GOAL_JUDGE_SCHEMA: dict[str, Any] = {
"type": "object",
"additionalProperties": False,
"required": ["verdict", "reason"],
"properties": {
"verdict": {"type": "string", "enum": ["complete", "continue"]},
"reason": {"type": "string"},
},
}
UNFINISHED_EVIDENCE = re.compile(
r"(?:\bin[ -]?progress\b|\bstill\s+(?:active|pending|running)\b|"
r"\bremains?\s+(?:active|pending|running|unfinished)\b|"
r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b"
r"\s+(?:is|are|remains?|still|currently|has|have)\s+"
r"(?:pending|running|unfinished|not\s+(?:yet\s+)?(?:done|finished|run|complete))\b|"
r"\b(?:pending|running|unfinished)\s+"
r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)",
re.IGNORECASE | re.DOTALL,
)
def _bounded(value: str, limit: int) -> str:
"""Retain both ends of evidence while bounding local-judge context."""
if len(value) <= limit:
return value
marker = "\n...[goal evidence compacted]...\n"
remaining = max(0, limit - len(marker))
head = remaining // 2
return f"{value[:head]}{marker}{value[-(remaining - head):]}"
def unfinished_result_reason(result: dict[str, Any]) -> str | None:
"""Reject self-contradictory completion reports without model inference."""
status = str(result.get("status") or "")
summary = str(result.get("summary") or "").strip()
blockers = result.get("blockers")
if status == "incomplete":
return summary or "worker explicitly reported incomplete work"
if status != "completed":
return None
if isinstance(blockers, list) and any(str(item).strip() for item in blockers):
return "worker reported blockers while claiming completion"
tests = result.get("tests_run")
evidence = [summary]
if isinstance(tests, list):
evidence.extend(str(item) for item in tests)
match = UNFINISHED_EVIDENCE.search("\n".join(evidence))
if match:
return f"completion evidence says work is unfinished: {match.group(0).strip()}"
return None
def judge_goal_completion(
objective: str,
result: dict[str, Any],
*,
url: str = GOAL_JUDGE_URL,
open_request: Callable[..., Any] = urllib.request.urlopen,
) -> tuple[bool, str]:
"""Use the local model as a fail-closed judge for explicit goal cards."""
deterministic_reason = unfinished_result_reason(result)
if deterministic_reason:
return False, deterministic_reason
payload = {
"model": GOAL_JUDGE_MODEL,
"messages": [
{
"role": "system",
"content": (
"You are a fail-closed completion judge for an engineering task. "
"Compare the objective and acceptance criteria with the worker report. "
"Return complete only when the report gives concrete evidence that every "
"explicit requirement finished. Return continue when any work, test, command, "
"commit, push, review, or verification is pending, in progress, omitted, or "
"inconclusive. Do not trust the report status field."
),
},
{
"role": "user",
"content": json.dumps(
{
"objective": _bounded(objective, 6000),
"worker_report": _bounded(
json.dumps(result, default=str), 4000
),
},
separators=(",", ":"),
),
},
],
"stream": False,
"temperature": 0,
"max_tokens": 160,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "goal_completion_verdict",
"strict": True,
"schema": GOAL_JUDGE_SCHEMA,
},
},
}
request = urllib.request.Request(
url,
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with open_request(request, timeout=60) as response:
document = json.load(response)
content = document["choices"][0]["message"]["content"]
verdict = json.loads(content)
if verdict.get("verdict") not in {"complete", "continue"}:
raise ValueError("local judge returned an invalid verdict")
reason = str(verdict.get("reason") or "local judge supplied no reason")
return verdict.get("verdict") == "complete", reason
except (
AttributeError,
IndexError,
KeyError,
OSError,
TypeError,
ValueError,
json.JSONDecodeError,
urllib.error.URLError,
) as error:
return False, f"local completion judge unavailable: {type(error).__name__}: {error}"

View File

@ -22,6 +22,8 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable
import cli_lane_goal
DATA_ROOT = Path(os.environ.get("HERMES_HOME", "/opt/data"))
SWITCHYARD_URL = os.environ.get(
@ -60,7 +62,10 @@ RESULT_SCHEMA: dict[str, Any] = {
"additionalProperties": False,
"required": ["status", "summary", "changed_files", "tests_run", "artifacts", "blockers"],
"properties": {
"status": {"type": "string", "enum": ["completed", "blocked"]},
"status": {
"type": "string",
"enum": sorted(cli_lane_goal.RESULT_STATUSES),
},
"summary": {"type": "string"},
"changed_files": {"type": "array", "items": {"type": "string"}},
"tests_run": {"type": "array", "items": {"type": "string"}},
@ -272,7 +277,7 @@ Workspace: {workspace}
Operate autonomously inside the workspace. Inspect before editing, preserve unrelated user changes, run proportionate tests, and do not claim completion without evidence. You have owner-level Kubernetes access in every namespace. Prefer Flux-tracked manifests for durable changes, but use kubectl, Flux, exec, port-forwarding, rollout operations, and existing Vault workflows when the objective or incident requires them. Persist any desired-state mutation back to Git. Do not force-push, hard-reset, clean untracked files, or expose credentials.
Return a final JSON object matching the supplied schema. Use status=blocked only for a concrete unresolved blocker. List changed files, tests run, durable artifact paths, and blockers explicitly.
Return a final JSON object matching the supplied schema. Use status=incomplete when required work, tests, commands, commits, pushes, or verification are still running or remain to be done. Use status=blocked only for a concrete unresolved blocker. Never use status=completed for a progress report. List changed files, tests run, durable artifact paths, and blockers explicitly.
"""
@ -320,7 +325,7 @@ def workspace_artifacts(workspace: Path, values: Any) -> list[str]:
def _extract_json(value: Any) -> dict[str, Any] | None:
if isinstance(value, dict) and value.get("status") in {"completed", "blocked"}:
if isinstance(value, dict) and value.get("status") in cli_lane_goal.RESULT_STATUSES:
return value
if not isinstance(value, str):
return None
@ -333,7 +338,7 @@ def _extract_json(value: Any) -> dict[str, Any] | None:
parsed = json.loads(candidate)
except json.JSONDecodeError:
continue
if isinstance(parsed, dict) and parsed.get("status") in {"completed", "blocked"}:
if isinstance(parsed, dict) and parsed.get("status") in cli_lane_goal.RESULT_STATUSES:
return parsed
return None
@ -670,7 +675,7 @@ def run_provider(
max_runtime=max_runtime,
)
file_result = load_json(result_file)
if file_result.get("status") in {"completed", "blocked"}:
if file_result.get("status") in cli_lane_goal.RESULT_STATUSES:
result.structured = file_result
return result
@ -890,83 +895,178 @@ def execute_claim(board: str, task_id: str) -> None:
max_runtime = int(
_task_value(task, "max_runtime_seconds", 0) or DEFAULT_MAX_RUNTIME
)
result = run_provider(route, prompt, workspace, state, state_file, log_path, heartbeat, max_runtime)
if result.capacity_failure:
retry_context = (
context
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+ "Select the alternate hosted provider at an appropriate effort."
)
alternate = "claude" if route.provider == "codex" else "codex"
fallback = select_route(
retry_context,
f"cli-{alternate}-{route.effort}",
)
comment(
f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.",
)
goal_mode = bool(_task_value(task, "goal_mode", False))
goal_max_turns = max(1, int(_task_value(task, "goal_max_turns", 1) or 1))
goal_turn = max(1, int(state.get("goal_turn", 0) or 0) + 1)
deadline = time.monotonic() + max_runtime
while True:
state["goal_turn"] = goal_turn
atomic_json(state_file, state)
remaining = max(1, int(deadline - time.monotonic()))
result = run_provider(
fallback,
build_prompt(context, workspace, git_handoff(workspace, result.output)),
route,
prompt,
workspace,
state,
state_file,
log_path,
heartbeat,
max_runtime,
remaining,
)
route = fallback
structured = result.structured
if structured:
structured = dict(structured)
structured["artifacts"] = workspace_artifacts(
workspace,
structured.get("artifacts"),
)
metadata = {
"executor": "direct-cli-lane",
"provider": route.provider,
"model": route.model,
"effort": route.effort,
"classifier": route.classifier,
"state_file": str(state_file),
"codex_thread_id": state.get("codex_thread_id"),
"claude_session_id": state.get("claude_session_id"),
}
if structured:
for key in ("changed_files", "tests_run", "artifacts", "blockers"):
value = structured.get(key)
metadata[key] = value if isinstance(value, list) else []
if structured and structured.get("status") == "completed" and result.returncode == 0:
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.complete_task(
fresh,
task_id,
result=json.dumps(structured, sort_keys=True),
summary=str(structured.get("summary") or "Completed"),
metadata=metadata,
expected_run_id=run_id,
),
)
else:
reason = (
"; ".join(str(item) for item in (structured or {}).get("blockers", []))
if structured
else result.output[-4000:]
if result.capacity_failure:
retry_context = (
context
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
+ "Select the alternate hosted provider at an appropriate effort."
)
alternate = "claude" if route.provider == "codex" else "codex"
fallback = select_route(
retry_context,
f"cli-{alternate}-{route.effort}",
)
comment(
f"Provider fallback: {route.provider} -> {fallback.provider}; Jetson reclassified the retry boundary.",
)
result = run_provider(
fallback,
build_prompt(
context,
workspace,
git_handoff(workspace, result.output),
),
workspace,
state,
state_file,
log_path,
heartbeat,
max(1, int(deadline - time.monotonic())),
)
route = fallback
structured = result.structured
if structured:
structured = dict(structured)
structured["artifacts"] = workspace_artifacts(
workspace,
structured.get("artifacts"),
)
metadata = {
"executor": "direct-cli-lane",
"provider": route.provider,
"model": route.model,
"effort": route.effort,
"classifier": route.classifier,
"state_file": str(state_file),
"codex_thread_id": state.get("codex_thread_id"),
"claude_session_id": state.get("claude_session_id"),
"goal_mode": goal_mode,
"goal_turn": goal_turn,
}
if structured:
for key in ("changed_files", "tests_run", "artifacts", "blockers"):
value = structured.get(key)
metadata[key] = value if isinstance(value, list) else []
completion_problem = None
if structured and result.returncode == 0:
completion_problem = cli_lane_goal.unfinished_result_reason(structured)
if (
structured.get("status") == "completed"
and completion_problem is None
and goal_mode
):
heartbeat("local goal-completion judge active")
accepted, judge_reason = cli_lane_goal.judge_goal_completion(
context,
structured,
)
metadata["goal_judge_reason"] = judge_reason
if not accepted:
completion_problem = f"local goal judge requested continuation: {judge_reason}"
if (
structured
and structured.get("status") == "completed"
and result.returncode == 0
and completion_problem is None
):
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.complete_task(
fresh,
task_id,
result=json.dumps(structured, sort_keys=True),
summary=str(structured.get("summary") or "Completed"),
metadata=metadata,
expected_run_id=run_id,
),
)
break
can_continue = (
goal_mode
and completion_problem is not None
and goal_turn < goal_max_turns
and deadline - time.monotonic() > 30
)
if can_continue:
goal_turn += 1
comment(
f"Goal completion rejected; continuing turn {goal_turn}/{goal_max_turns}: {completion_problem}",
)
escalation_context = (
context
+ "\n\nThe previous worker turn missed its completion quality mark: "
+ completion_problem
+ "\nSelect a route that can finish and verify the remaining work."
)
excluded = (
fresh_unavailable_provider() if assignee == "cli-auto" else None
)
next_route = select_route(
escalation_context,
assignee,
exclude_provider=excluded,
exclude_reason="is unavailable according to fresh native health"
if excluded
else None,
)
handoff = (
git_handoff(workspace, result.output)
if next_route.provider != route.provider
else ""
)
route = next_route
prompt = build_prompt(
context,
workspace,
handoff
+ "\nGoal-loop continuation: the previous final report was rejected because "
+ completion_problem
+ ". Reinspect live state, finish the outstanding work, and return new final evidence.",
)
continue
reason = completion_problem
if reason is None and structured:
blockers = structured.get("blockers", [])
reason = "; ".join(str(item) for item in blockers)
reason = reason or str(structured.get("summary") or "")
if reason is None:
reason = result.output[-4000:]
_board_call(
kanban_db,
board,
lambda fresh: kanban_db.block_task(
fresh,
task_id,
reason=reason or f"{route.provider} worker failed with exit {result.returncode}",
reason=reason
or f"{route.provider} worker failed with exit {result.returncode}",
kind="transient" if result.capacity_failure else "capability",
expected_run_id=run_id,
),
)
break
except Exception as error:
failure_reason = f"Direct CLI lane failed: {type(error).__name__}: {error}"
_board_call(

View File

@ -0,0 +1,111 @@
"""Completion-gate tests for Hermes external Kanban workers."""
from __future__ import annotations
import importlib.util
import json
import sys
from pathlib import Path
import pytest
SCRIPT = (
Path(__file__).parents[2]
/ "services/hermes/scripts/cli_lane_goal.py"
)
SPEC = importlib.util.spec_from_file_location("cli_lane_goal_test", SCRIPT)
assert SPEC and SPEC.loader
goal = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = goal
SPEC.loader.exec_module(goal)
def _result(**overrides):
value = {
"status": "completed",
"summary": "All acceptance criteria passed and the branch was pushed.",
"changed_files": ["src/example.py"],
"tests_run": ["pytest -q: 12 passed"],
"artifacts": [],
"blockers": [],
}
value.update(overrides)
return value
@pytest.mark.parametrize(
"result",
[
_result(status="incomplete", summary="The full suite is still running."),
_result(summary="The broad rerun remains active before the final push."),
_result(tests_run=["Full suite — in progress (11m)"]),
_result(blockers=["remote head was not verified"]),
],
)
def test_unfinished_completion_evidence_is_rejected(result):
assert goal.unfinished_result_reason(result)
def test_completed_pending_state_test_name_is_not_a_false_positive():
result = _result(tests_run=["pytest tests/test_pending_build_state.py: 8 passed"])
assert goal.unfinished_result_reason(result) is None
class JudgeResponse:
def __init__(self, verdict: str, reason: str):
self.body = json.dumps(
{
"choices": [
{
"message": {
"content": json.dumps(
{"verdict": verdict, "reason": reason}
)
}
}
]
}
).encode()
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self):
return self.body
def test_goal_judge_uses_local_structured_verdict():
observed = {}
def request(req, timeout):
observed["payload"] = json.loads(req.data)
observed["timeout"] = timeout
return JudgeResponse("continue", "remote head evidence is missing")
accepted, reason = goal.judge_goal_completion(
"Run tests, push, and verify the remote head.",
_result(summary="Tests passed."),
open_request=request,
)
assert accepted is False
assert reason == "remote head evidence is missing"
assert observed["payload"]["model"] == "qwen2.5:14b-instruct-q4_0"
assert observed["payload"]["response_format"]["json_schema"]["strict"] is True
assert observed["timeout"] == 60
def test_goal_judge_fails_closed_on_invalid_response():
accepted, reason = goal.judge_goal_completion(
"Finish the task.",
_result(),
open_request=lambda *_args, **_kwargs: JudgeResponse("unknown", "bad"),
)
assert accepted is False
assert "local completion judge unavailable" in reason

View File

@ -8,9 +8,9 @@ import sys
from pathlib import Path
SCRIPT = (
Path(__file__).parents[2] / "services/hermes/scripts/cli_lane_runner.py"
)
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
sys.path.insert(0, str(SCRIPTS))
SCRIPT = SCRIPTS / "cli_lane_runner.py"
SPEC = importlib.util.spec_from_file_location("cli_lane_routing", SCRIPT)
assert SPEC and SPEC.loader
lanes = importlib.util.module_from_spec(SPEC)

View File

@ -16,6 +16,7 @@ import yaml
SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts"
sys.path.insert(0, str(SCRIPTS))
HERMES = Path(__file__).parents[2] / "services/hermes"
KEYCLOAK = Path(__file__).parents[2] / "services/keycloak"
FLUX_HERMES = (
@ -590,6 +591,22 @@ def test_board_call_retries_storage_faults_on_fresh_connections():
),
"complete",
),
(
lanes.ProcessResult(
0,
"",
{
"status": "completed",
"summary": "The full test suite is still running.",
"changed_files": ["src/a.py"],
"tests_run": ["pytest -q — in progress"],
"artifacts": [],
"blockers": [],
},
False,
),
"block",
),
],
)
def test_claim_requires_structured_evidence_and_surfaces_artifacts(
@ -671,6 +688,99 @@ def test_claim_requires_structured_evidence_and_surfaces_artifacts(
assert all(connection.closed for connection in connections)
def test_goal_card_continues_after_local_judge_rejects_progress(
tmp_path: Path,
monkeypatch,
):
task = SimpleNamespace(
id="t_goal",
current_run_id=12,
assignee="cli-auto",
max_runtime_seconds=300,
goal_mode=True,
goal_max_turns=3,
)
calls = []
comments = []
class Connection:
def close(self):
return None
fake_db = SimpleNamespace(
scoped_current_board=lambda _board: nullcontext(),
connect=lambda board: Connection(),
get_task=lambda _conn, _task_id: task,
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_goal"),
set_branch_name=lambda *_args: None,
set_workspace_path=lambda *_args: None,
build_worker_context=lambda *_args: "Run tests, commit, push, and verify remote HEAD.",
heartbeat_worker=lambda *_args, **_kwargs: True,
add_comment=lambda _conn, _task_id, _author, body: comments.append(body),
complete_task=lambda *_args, **kwargs: calls.append(("complete", kwargs)),
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
)
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
monkeypatch.setattr(
lanes,
"state_path",
lambda _board, _task_id: tmp_path / "state.json",
)
route = lanes.Route(
"codex", "gpt-5.6-sol", "xhigh", "codex-xhigh", "jetson", "vote", 1, ()
)
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
reports = [
lanes.ProcessResult(
0,
"first turn",
{
"status": "completed",
"summary": "Focused tests passed.",
"changed_files": ["src/a.py"],
"tests_run": ["pytest focused: passed"],
"artifacts": [],
"blockers": [],
},
False,
),
lanes.ProcessResult(
0,
"second turn",
{
"status": "completed",
"summary": "Full tests passed; commit pushed and remote HEAD verified.",
"changed_files": ["src/a.py"],
"tests_run": ["pytest full: passed"],
"artifacts": [],
"blockers": [],
},
False,
),
]
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
verdicts = iter(
[
(False, "commit, push, and remote verification are missing"),
(True, "all explicit acceptance criteria have evidence"),
]
)
monkeypatch.setattr(
lanes.cli_lane_goal,
"judge_goal_completion",
lambda *_args, **_kwargs: next(verdicts),
)
lanes.execute_claim("cassandra", "t_goal")
assert calls[0][0] == "complete"
assert calls[0][1]["metadata"]["goal_turn"] == 2
assert any("Goal completion rejected; continuing turn 2/3" in item for item in comments)
assert reports == []
def test_workspace_preparation_failure_durably_blocks_the_claim(tmp_path: Path, monkeypatch):
task = SimpleNamespace(id="t_bad_worktree", current_run_id=7, assignee="cli-auto")
calls = []