#!/usr/bin/env python3 """Fail-closed completion checks for durable external Kanban workers.""" from __future__ import annotations import json import os import re import urllib.error import urllib.request from typing import Any, Callable GOAL_JUDGE_URL = os.environ.get( "HERMES_GOAL_JUDGE_URL", "http://hermes-model-gate.hermes.svc.cluster.local:11434/v1/chat/completions", ) GOAL_JUDGE_MODEL = os.environ.get( "HERMES_GOAL_JUDGE_MODEL", "qwen2.5:14b-instruct-q4_0", ) GOAL_JUDGE_TIMEOUT_SECONDS = max( 30, min(180, int(os.environ.get("HERMES_GOAL_JUDGE_TIMEOUT_SECONDS", "120"))), ) RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"}) GOAL_JUDGE_SCHEMA: dict[str, Any] = { "type": "object", "additionalProperties": False, "required": ["verdict", "reason"], "properties": { "verdict": {"type": "string", "enum": ["complete", "continue"]}, "reason": {"type": "string"}, }, } UNFINISHED_EVIDENCE = re.compile( r"(?:\bin[ -]?progress\b|\bstill\s+(?:active|pending|running)\b|" r"\bremains?\s+(?:active|pending|running|unfinished)\b|" r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b" r"\s+(?:is|are|remains?|still|currently|has|have)\s+" r"(?:pending|running|unfinished|not\s+(?:yet\s+)?(?:done|finished|run|complete))\b|" r"\b(?:pending|running|unfinished)\s+" r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)", re.IGNORECASE | re.DOTALL, ) def _bounded(value: str, limit: int) -> str: """Retain both ends of evidence while bounding local-judge context.""" if len(value) <= limit: return value marker = "\n...[goal evidence compacted]...\n" remaining = max(0, limit - len(marker)) head = remaining // 2 return f"{value[:head]}{marker}{value[-(remaining - head):]}" def unfinished_result_reason(result: dict[str, Any]) -> str | None: """Reject self-contradictory completion reports without model inference.""" status = str(result.get("status") or "") summary = str(result.get("summary") or "").strip() blockers = result.get("blockers") if status == "incomplete": return summary or "worker explicitly reported incomplete work" if status != "completed": return None if isinstance(blockers, list) and any(str(item).strip() for item in blockers): return "worker reported blockers while claiming completion" tests = result.get("tests_run") evidence = [summary] if isinstance(tests, list): evidence.extend(str(item) for item in tests) match = UNFINISHED_EVIDENCE.search("\n".join(evidence)) if match: return f"completion evidence says work is unfinished: {match.group(0).strip()}" return None def judge_goal_completion( objective: str, result: dict[str, Any], *, url: str = GOAL_JUDGE_URL, open_request: Callable[..., Any] = urllib.request.urlopen, ) -> tuple[bool, str]: """Use the local model as a fail-closed judge for explicit goal cards.""" deterministic_reason = unfinished_result_reason(result) if deterministic_reason: return False, deterministic_reason payload = { "model": GOAL_JUDGE_MODEL, "messages": [ { "role": "system", "content": ( "You are a fail-closed completion judge for an engineering task. " "Compare the objective and acceptance criteria with the worker report. " "Return complete only when the report gives concrete evidence that every " "explicit requirement finished. Return continue when any work, test, command, " "commit, push, review, or verification is pending, in progress, omitted, or " "inconclusive. Do not trust the report status field." ), }, { "role": "user", "content": json.dumps( { "objective": _bounded(objective, 6000), "worker_report": _bounded( json.dumps(result, default=str), 4000 ), }, separators=(",", ":"), ), }, ], "stream": False, "temperature": 0, "max_tokens": 160, "response_format": { "type": "json_schema", "json_schema": { "name": "goal_completion_verdict", "strict": True, "schema": GOAL_JUDGE_SCHEMA, }, }, } request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) try: with open_request(request, timeout=GOAL_JUDGE_TIMEOUT_SECONDS) as response: document = json.load(response) content = document["choices"][0]["message"]["content"] verdict = json.loads(content) if verdict.get("verdict") not in {"complete", "continue"}: raise ValueError("local judge returned an invalid verdict") reason = str(verdict.get("reason") or "local judge supplied no reason") return verdict.get("verdict") == "complete", reason except ( AttributeError, IndexError, KeyError, OSError, TypeError, ValueError, json.JSONDecodeError, urllib.error.URLError, ) as error: return False, f"local completion judge unavailable: {type(error).__name__}: {error}"