#!/usr/bin/env python3 """Role-aware completion checks for durable external Kanban workers. Completion is judged against the action the card assigned, not against the health of whatever the card asked the worker to look at. * Implementation cards keep the fail-closed local goal judge: a report is accepted only when the model agrees every explicit requirement finished. * Review, audit, and diagnostic cards deliver a *verdict*. A completed report carrying a truthful SHIP or BLOCK verdict with evidence finalizes the card even when the verdict says the reviewed artifact must not ship. Findings are the deliverable, not a task blocker, so the card is never resumed with an instruction to repair an implementation the reviewer may not touch. The task role comes from explicit card metadata (``Hermes-Task-Role: review``) whenever the card author supplies it. Cards written before that contract fall back to a deliberately narrow inference that requires a read-only scope, a requested verdict, no requested mutation deliverable, and a report that changed no files. Review completion is fail-closed on a missing, unrecognized, or self-contradictory verdict and on missing evidence. """ from __future__ import annotations import json import os import re import urllib.error import urllib.request from typing import Any, Callable GOAL_JUDGE_URL = os.environ.get( "HERMES_GOAL_JUDGE_URL", "http://hermes-model-gate.hermes.svc.cluster.local:11434/v1/chat/completions", ) GOAL_JUDGE_MODEL = os.environ.get( "HERMES_GOAL_JUDGE_MODEL", "qwen2.5:14b-instruct-q4_0", ) GOAL_JUDGE_TIMEOUT_SECONDS = max( 30, min(180, int(os.environ.get("HERMES_GOAL_JUDGE_TIMEOUT_SECONDS", "120"))), ) RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"}) REVIEW_ROLE = "review" IMPLEMENTATION_ROLE = "implementation" TASK_ROLES = frozenset({REVIEW_ROLE, IMPLEMENTATION_ROLE}) ROLE_ALIASES = { "audit": REVIEW_ROLE, "assessment": REVIEW_ROLE, "diagnosis": REVIEW_ROLE, "diagnostic": REVIEW_ROLE, "inspection": REVIEW_ROLE, "review": REVIEW_ROLE, "reviewer": REVIEW_ROLE, "verdict": REVIEW_ROLE, "build": IMPLEMENTATION_ROLE, "change": IMPLEMENTATION_ROLE, "implement": IMPLEMENTATION_ROLE, "implementation": IMPLEMENTATION_ROLE, "repair": IMPLEMENTATION_ROLE, } JUDGE_REASON_LIMIT = 600 MIN_REVIEW_SUMMARY_CHARS = 80 READ_ONLY_GUARD = ( "the assigned action is a read-only review: report the verdict and evidence, " "and do not modify the reviewed implementation" ) GOAL_JUDGE_SCHEMA: dict[str, Any] = { "type": "object", "additionalProperties": False, "required": ["verdict", "reason"], "properties": { "verdict": {"type": "string", "enum": ["complete", "continue"]}, "reason": {"type": "string"}, }, } UNFINISHED_EVIDENCE = re.compile( r"(?:\bin[ -]?progress\b|\bstill\s+(?:active|pending|running)\b|" r"\bremains?\s+(?:active|pending|running|unfinished)\b|" r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b" r"\s+(?:is|are|remains?|still|currently|has|have)\s+" r"(?:pending|running|unfinished|not\s+(?:yet\s+)?(?:done|finished|run|complete))\b|" r"\b(?:pending|running|unfinished)\s+" r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)", re.IGNORECASE | re.DOTALL, ) # The worker context appends prior attempts, parent results, cross-task history # and comments after the card the worker was actually given. Role resolution # reads only the card itself so an unrelated sentence quoted from an earlier run # cannot reassign the task role. HISTORY_SECTION = re.compile( r"^##\s+(?:Prior attempts\b|Parent task results\b|Recent work by\b|Comment thread\b)", re.IGNORECASE | re.MULTILINE, ) # Explicit, machine-readable card metadata. This is the supported contract; # every pattern below it only keeps pre-contract cards working. ROLE_DIRECTIVE = re.compile( r"^[\s>#*\-]*(?:hermes[\s_-]*)?task[\s_-]*role\s*[:=]\s*([A-Za-z_-]+)\s*$", re.IGNORECASE | re.MULTILINE, ) OUTPUT_DIRECTIVE = re.compile( r"^[\s>#*\-]*(?:hermes[\s_-]*)?expected[\s_-]*output\s*[:=]\s*([A-Za-z_-]+)\s*$", re.IGNORECASE | re.MULTILINE, ) READ_ONLY_SCOPE = re.compile( r"(?:\b(?:do|does|must|should|shall|may|will)\s+not\s+(?:\w+\s+){0,3}?" r"(?:edit|modify|change|alter|patch|rewrite|implement|repair)\b" r"|\bdon'?t\s+(?:\w+\s+){0,3}?(?:edit|modify|change|alter|patch)\b" r"|\bwithout\s+(?:editing|modifying|changing|altering|patching)\b" r"|\bno\s+(?:code|file|source|implementation)\s+(?:edits?|changes?|modifications?)\b" r"|\bmake\s+no\s+(?:code|file|source)?\s*changes\b" r"|\bread[\s-]?only\s+(?:scope|review|audit|analysis|assessment|inspection|diagnos\w+)\b)", re.IGNORECASE, ) VERDICT_DELIVERABLE = re.compile( r"(?:\bship\s+or\s+block\b|\bblock\s+or\s+ship\b|\bship\s*/\s*block\b" r"|\breturn\s+(?:a\s+|one\s+|the\s+)?(?:strict\s+)?verdict\b" r"|\b(?:review|audit|release|diagnostic)\s+verdict\b|\bverdict\s*[:=])", re.IGNORECASE, ) MUTATION_DELIVERABLE = re.compile( r"(?:\bpushed\s+(?:sha|commit|head|branch|revision)\b" r"|\bpush\s+(?:the\s+|your\s+|one\s+)?(?:branch|commit|change\w*|fix\w*)\b" r"|\bcommit\s+and\s+push\b" r"|\bopen\s+(?:a|an|one|the)\s+(?:new\s+)?(?:draft\s+)?(?:pr\b|pull\s+request)" r"|\bdraft\s+pr\s+only\b)", re.IGNORECASE, ) VERDICT_DIRECTIVE = re.compile( r"\b(?:final\s+|strict\s+|overall\s+)?(?:verdict|recommendation|decision)\s*" r"(?:is\s*)?[:=—-]\s*[*_`\s]*(ship|block)\b", re.IGNORECASE, ) VERDICT_TOKEN = re.compile(r"(? str: """Retain both ends of evidence while bounding local-judge context.""" if len(value) <= limit: return value marker = "\n...[goal evidence compacted]...\n" remaining = max(0, limit - len(marker)) head = remaining // 2 return f"{value[:head]}{marker}{value[-(remaining - head):]}" def sanitize_reason(value: Any, limit: int = JUDGE_REASON_LIMIT) -> str: """Return one bounded, single-line, secret-free judge reason.""" text = SECRET_LIKE.sub("[redacted]", str(value or "")) text = " ".join(CONTROL_CHARACTERS.sub(" ", text).split()) if not text: return "local judge supplied no reason" return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..." def _strings(value: Any) -> list[str]: """Return the non-empty string entries of a reported evidence list.""" if not isinstance(value, list): return [] return [str(item).strip() for item in value if str(item).strip()] def card_scope(objective: str) -> str: """Return the card the worker was assigned, without appended run history.""" text = str(objective or "") match = HISTORY_SECTION.search(text) return text[: match.start()] if match else text def _declared_role(objective: str) -> str | None: """Return the role a card declared through explicit worker metadata.""" for pattern in (ROLE_DIRECTIVE, OUTPUT_DIRECTIVE): roles = { ROLE_ALIASES[value] for value in ( match.group(1).strip().lower() for match in pattern.finditer(objective) ) if value in ROLE_ALIASES } if len(roles) == 1: return roles.pop() if roles: # A card that declares two different roles has no usable contract. # Hold it to the stricter implementation regime rather than letting # the more permissive review path win an ambiguous declaration. return IMPLEMENTATION_ROLE return None def task_role(objective: str, result: dict[str, Any] | None = None) -> tuple[str, str]: """Resolve the assigned task role and how that resolution was reached.""" text = card_scope(objective) declared = _declared_role(text) if declared is not None: return declared, "directive" changed = _strings((result or {}).get("changed_files")) if ( not changed and READ_ONLY_SCOPE.search(text) and VERDICT_DELIVERABLE.search(text) and not MUTATION_DELIVERABLE.search(text) ): return REVIEW_ROLE, "inferred" return IMPLEMENTATION_ROLE, "default" def review_verdict(result: dict[str, Any]) -> tuple[str | None, str | None]: """Return the declared SHIP/BLOCK verdict, or why it cannot be trusted.""" declared: set[str] = set() field = result.get("verdict") if field is not None: value = str(field).strip().upper() if value not in {"SHIP", "BLOCK"}: return None, f"review verdict field is not SHIP or BLOCK: {value[:40]!r}" declared.add(value) summary = str(result.get("summary") or "") declared.update(match.upper() for match in VERDICT_DIRECTIVE.findall(summary)) if len(declared) > 1: return None, "review report declares both SHIP and BLOCK verdicts" if declared: return declared.pop(), None tokens = {match.upper() for match in VERDICT_TOKEN.findall(summary)} if len(tokens) > 1: return None, "review summary names both SHIP and BLOCK without a verdict line" if tokens: return tokens.pop(), None return None, "review report declares no explicit SHIP or BLOCK verdict" def review_completion_problem(result: dict[str, Any]) -> str | None: """Fail closed on a review report that does not carry a usable verdict.""" verdict, problem = review_verdict(result) if problem or verdict is None: return problem or "review report declares no explicit SHIP or BLOCK verdict" summary = str(result.get("summary") or "").strip() if len(summary) < MIN_REVIEW_SUMMARY_CHARS: return f"{verdict} verdict carries no reviewable rationale in its summary" findings = _strings(result.get("findings")) if verdict == "BLOCK" and not findings: return "BLOCK verdict reports no findings to justify it" if not findings and not _strings(result.get("tests_run")) and not _strings( result.get("artifacts") ): return f"{verdict} verdict reports no findings, tests, or artifacts as evidence" return None def _reported_role(result: dict[str, Any]) -> str: """Classify a report shape when no objective is available to the caller.""" if _strings(result.get("changed_files")): return IMPLEMENTATION_ROLE if result.get("verdict") is not None or VERDICT_DIRECTIVE.search( str(result.get("summary") or "") ): return REVIEW_ROLE return IMPLEMENTATION_ROLE def unfinished_result_reason( result: dict[str, Any], *, role: str | None = None, ) -> str | None: """Reject self-contradictory completion reports without model inference.""" status = str(result.get("status") or "") summary = str(result.get("summary") or "").strip() blockers = result.get("blockers") if status == "incomplete": return sanitize_reason(summary or "worker explicitly reported incomplete work") if status != "completed": return None if isinstance(blockers, list) and any(str(item).strip() for item in blockers): return "worker reported blockers while claiming completion" if (role or _reported_role(result)) == REVIEW_ROLE: # A review's prose describes the reviewed artifact, so the unfinished # work heuristic below would read the artifact's state as the review's # own. The verdict contract is the deterministic gate instead. return review_completion_problem(result) tests = result.get("tests_run") evidence = [summary] if isinstance(tests, list): evidence.extend(str(item) for item in tests) match = UNFINISHED_EVIDENCE.search("\n".join(evidence)) if match: return f"completion evidence says work is unfinished: {match.group(0).strip()}" return None def _completion_claimed(result: dict[str, Any]) -> bool: """Report whether the worker actually claimed a finished task.""" return str(result.get("status") or "") == "completed" def _judge_payload(objective: str, result: dict[str, Any], role: str) -> dict[str, Any]: """Build the fail-closed local judge request for an implementation card.""" return { "model": GOAL_JUDGE_MODEL, "messages": [ { "role": "system", "content": ( "You are a fail-closed completion judge for an engineering task. " "Compare the objective and acceptance criteria with the worker report. " "Return complete only when the report gives concrete evidence that every " "explicit requirement finished. Return continue when any work, test, command, " "commit, push, review, or verification is pending, in progress, omitted, or " "inconclusive. Defects the worker reports about some other artifact are " "findings, not unfinished work on this task. Do not trust the report status " "field." ), }, { "role": "user", "content": json.dumps( { "task_role": role, "objective": _bounded(objective, 6000), "worker_report": _bounded( json.dumps(result, default=str), 4000 ), }, separators=(",", ":"), ), }, ], "stream": False, "temperature": 0, "max_tokens": 160, "response_format": { "type": "json_schema", "json_schema": { "name": "goal_completion_verdict", "strict": True, "schema": GOAL_JUDGE_SCHEMA, }, }, } def judge_goal_completion( objective: str, result: dict[str, Any], *, url: str = GOAL_JUDGE_URL, open_request: Callable[..., Any] = urllib.request.urlopen, ) -> tuple[bool, str]: """Judge one worker report against the action its card actually assigned.""" role, source = task_role(objective, result) deterministic_reason = unfinished_result_reason(result, role=role) if deterministic_reason is None and not _completion_claimed(result): deterministic_reason = ( f"worker reported status {str(result.get('status') or 'unknown')!r} " "rather than a finished task" ) if deterministic_reason: if role == REVIEW_ROLE: return False, sanitize_reason(f"{READ_ONLY_GUARD}; {deterministic_reason}") return False, sanitize_reason(deterministic_reason) if role == REVIEW_ROLE: # The requested deliverable is present and self-consistent. A verdict # that the reviewed artifact must not ship is a finished review, so the # card finalizes instead of resuming a worker that may not repair it. verdict, _ = review_verdict(result) findings = len(_strings(result.get("findings"))) return True, sanitize_reason( f"review deliverable complete: {verdict} verdict with {findings} finding(s); " f"task role {role} resolved by {source}" ) request = urllib.request.Request( url, data=json.dumps(_judge_payload(objective, result, role)).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) try: with open_request(request, timeout=GOAL_JUDGE_TIMEOUT_SECONDS) as response: document = json.load(response) content = document["choices"][0]["message"]["content"] verdict = json.loads(content) if verdict.get("verdict") not in {"complete", "continue"}: raise ValueError("local judge returned an invalid verdict") reason = sanitize_reason(verdict.get("reason")) return verdict.get("verdict") == "complete", reason except ( AttributeError, IndexError, KeyError, OSError, TypeError, ValueError, json.JSONDecodeError, urllib.error.URLError, ) as error: return False, sanitize_reason( f"local completion judge unavailable: {type(error).__name__}: {error}" )