#!/usr/bin/env python3 """Role-aware completion checks for durable external Kanban workers. Completion is judged against the action the card assigned, not against the health of whatever the card asked the worker to look at. * Implementation cards keep the fail-closed local goal judge: a report is accepted only when the model agrees every explicit requirement finished. * Review, audit, and diagnostic cards deliver a *verdict*. A completed report carrying a truthful SHIP or BLOCK verdict with evidence finalizes the card even when the verdict says the reviewed artifact must not ship. Findings are the deliverable, not a task blocker, so the card is never resumed with an instruction to repair an implementation the reviewer may not touch. Three deliberate boundaries keep that regime honest. First, ``role`` belongs to the caller that holds the card: ``unfinished_result_reason`` cannot classify a card it was never handed, so every caller holding the objective MUST pass ``role``; the role-blind default is the replay contract below, never a weaker classifier that could short-circuit the role-aware gate. Second, the verdict contract binds in single-shot mode exactly as in the goal loop: a review card's deliverable *is* its verdict, so the lane fails closed rather than record a verdictless review as done. Two single-shot outcomes change against main d8f2d818, deliberately - a completed review whose prose calls the *reviewed* artifact pending now completes instead of being discarded, and one carrying no usable verdict now blocks instead of completing. Third, a journalled terminal record stays valid across an upgrade. PR15 re-runs this gate on replay without a card, so ``role=None`` keeps the unfinished-work heuristic - that integrity check still works - and skips it only for a report that satisfies the whole review contract and changed no files. That is precisely the set the role-aware lane accepts as a review, so a record accepted under one version re-validates under every other instead of being quarantined into a re-dispatch of an already-accepted task. The task role comes from explicit card metadata (``Hermes-Task-Role: review``) whenever the card author supplies it, read from real newlines and from the literal ``\\n`` escapes the board stores in single-line bodies. Pre-contract cards fall back to a narrow inference needing a read-only scope, a requested SHIP/BLOCK verdict, no requested mutation deliverable, and a report that changed no files. A card whose deliverable is a findings list rather than a verdict deliberately stays on the implementation regime: the verdict *is* the review contract's gate, so inferring a review role for a card that never asked for one could only fail closed. A report that changed files never resolves to the review role, even when the card declares it. """ from __future__ import annotations import json import os import re import urllib.error import urllib.request from typing import Any, Callable try: # The lane runs inside the agent image, which owns the canonical redactor. from agent.redact import redact_sensitive_text as _canonical_redact except Exception: # pragma: no cover - only when the agent runtime is absent _canonical_redact = None GOAL_JUDGE_URL = os.environ.get( "HERMES_GOAL_JUDGE_URL", "http://hermes-model-gate.hermes.svc.cluster.local:11434/v1/chat/completions", ) GOAL_JUDGE_MODEL = os.environ.get( "HERMES_GOAL_JUDGE_MODEL", "qwen2.5:14b-instruct-q4_0", ) GOAL_JUDGE_TIMEOUT_SECONDS = max( 30, min(180, int(os.environ.get("HERMES_GOAL_JUDGE_TIMEOUT_SECONDS", "120"))), ) RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"}) REVIEW_ROLE = "review" IMPLEMENTATION_ROLE = "implementation" TASK_ROLES = frozenset({REVIEW_ROLE, IMPLEMENTATION_ROLE}) ROLE_ALIASES = { "audit": REVIEW_ROLE, "assessment": REVIEW_ROLE, "diagnosis": REVIEW_ROLE, "diagnostic": REVIEW_ROLE, "inspection": REVIEW_ROLE, "review": REVIEW_ROLE, "reviewer": REVIEW_ROLE, "verdict": REVIEW_ROLE, "build": IMPLEMENTATION_ROLE, "change": IMPLEMENTATION_ROLE, "implement": IMPLEMENTATION_ROLE, "implementation": IMPLEMENTATION_ROLE, "repair": IMPLEMENTATION_ROLE, } JUDGE_REASON_LIMIT = 600 MIN_REVIEW_SUMMARY_CHARS = 80 READ_ONLY_GUARD = ( "the assigned action is a read-only review: report the verdict and evidence, " "and do not modify the reviewed implementation" ) # The runner appends its own turn/rejection evidence to the judge objective. # It is emitted under this heading so ``card_scope`` drops it with every other # non-card section instead of letting a quoted worker sentence reassign a role. CONTROLLER_EVIDENCE_HEADING = "## Hermes goal-controller evidence" # ``hermes_cli.kanban_db.build_worker_context`` (Hermes runtime 0.18.2) renders # the worker context as H2 sections and emits exactly CARD_SECTIONS followed by # HISTORY_SECTIONS; ``test_hermes_cli_review_context.py`` pins that contract # against the installed runtime. The cut below allow-lists the two card # sections, so an upstream rename fails closed - the card just ends early. CARD_SECTIONS = ("Body", "Attachments") HISTORY_SECTIONS = ( "Prior attempts on this task", "Parent task results", "Recent work by", "Comment thread", ) GOAL_JUDGE_SCHEMA: dict[str, Any] = { "type": "object", "additionalProperties": False, "required": ["verdict", "reason"], "properties": { "verdict": {"type": "string", "enum": ["complete", "continue"]}, "reason": {"type": "string"}, }, } UNFINISHED_EVIDENCE = re.compile( r"(?:\bin[ -]?progress\b|\bstill\s+(?:active|pending|running)\b|" r"\bremains?\s+(?:active|pending|running|unfinished)\b|" r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b" r"\s+(?:is|are|remains?|still|currently|has|have)\s+" r"(?:pending|running|unfinished|not\s+(?:yet\s+)?(?:done|finished|run|complete))\b|" r"\b(?:pending|running|unfinished)\s+" r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)", re.IGNORECASE | re.DOTALL, ) HISTORY_SECTION = re.compile( r"^##\s+(?!(?:Body|Attachments)\b)" r"|^Authoritative Hermes goal-controller evidence\b", re.IGNORECASE | re.MULTILINE, ) # Explicit, machine-readable card metadata. This is the supported contract; # every pattern below it only keeps pre-contract cards working. ROLE_DIRECTIVE = re.compile( r"^[\s>#*\-]*(?:hermes[\s_-]*)?task[\s_-]*role\s*[:=]\s*([A-Za-z_-]+)\s*$", re.IGNORECASE | re.MULTILINE, ) OUTPUT_DIRECTIVE = re.compile( r"^[\s>#*\-]*(?:hermes[\s_-]*)?expected[\s_-]*output\s*[:=]\s*([A-Za-z_-]+)\s*$", re.IGNORECASE | re.MULTILINE, ) READ_ONLY_SCOPE = re.compile( r"(?:\b(?:do|does|must|should|shall|may|will)\s+not\s+(?:\w+\s+){0,3}?" r"(?:edit|modify|change|alter|patch|rewrite|implement|repair|author)\b" r"|\bdon'?t\s+(?:\w+\s+){0,3}?(?:edit|modify|change|alter|patch)\b" r"|\bwithout\s+(?:editing|modifying|changing|altering|patching)\b" r"|\bno\s+(?:\w+\s+){0,2}?(?:edits?|changes?|modifications?)\b" r"|\bmake\s+no\s+(?:code|file|source)?\s*changes\b" r"|\bpristine\b|\bread[\s-]?only\b)", re.IGNORECASE, ) # A SHIP/BLOCK-shaped deliverable, not merely a mention of the word "verdict": # cards that audit some other component's verdict machinery must not be read as # owing one themselves. VERDICT_DELIVERABLE = re.compile( r"(?:\bship\b[^.\n]{0,120}?\bblock\b|\bblock\b[^.\n]{0,120}?\bship\b" r"|\bship\s*/\s*not-?ship\b|\bverdict\s+of\b" r"|\b(?:return|record|report|give|provide|deliver|state|produce|emit)\b" r"(?:\s+[-\w']+){0,6}?\s+verdict\b)", re.IGNORECASE, ) MUTATION_DELIVERABLE = re.compile( r"(?:\bpushed\s+(?:sha|commit|head|branch|revision)\b" r"|\bpush\s+(?:\w+\s+){0,3}?(?:branch|commit|change\w*|fix\w*)\b" r"|\bcommit\s+(?:\w+\s+){0,4}?and\s+push\b|\bamend\b" r"|\bopen\s+(?:a|an|one|the)\s+(?:new\s+)?(?:draft\s+)?(?:pr\b|pull\s+request)" r"|\bdraft\s+pr\s+only\b)", re.IGNORECASE, ) VERDICT_DIRECTIVE = re.compile( r"\b(?:final\s+|strict\s+|overall\s+)?(?:verdict|recommendation|decision)\s*" r"(?:is\s*)?[:=—-]\s*[*_`\s]*(ship|block)\b", re.IGNORECASE, ) VERDICT_TOKEN = re.compile(r"(? str: """Retain both ends of evidence while bounding local-judge context.""" if len(value) <= limit: return value marker = "\n...[goal evidence compacted]...\n" remaining = max(0, limit - len(marker)) head = remaining // 2 return f"{value[:head]}{marker}{value[-(remaining - head):]}" def redact(value: str) -> str: """Redact with the runtime's canonical helper plus this lane's own gaps.""" text = SECRET_LIKE.sub("[redacted]", value) text = URL_USERINFO.sub(r"\1:[redacted]@", text) text = CREDENTIAL_CONTEXT.sub(r"\1[redacted]", text) return _canonical_redact(text, force=True) if _canonical_redact else text def sanitize_reason(value: Any, limit: int = JUDGE_REASON_LIMIT) -> str: """Return one bounded, single-line, secret-free judge reason.""" text = redact(str(value or "")) text = " ".join(CONTROL_CHARACTERS.sub(" ", text).split()) if not text: return "local judge supplied no reason" return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..." def _strings(value: Any) -> list[str]: """Return the non-empty string entries of a reported evidence list.""" if not isinstance(value, list): return [] return [str(item).strip() for item in value if str(item).strip()] def card_scope(objective: str) -> str: """Return the card the worker was assigned, without appended run history. Literal ``\\n`` escapes are expanded first: the board stores most bodies on a single physical line, and every pattern here is line-anchored or depends on word boundaries the escape sequence would otherwise destroy. Expanding before the cut is monotone - it can only move the first history heading earlier - so it cannot smuggle appended history into the card. The normalization is for analysis only; the judge objective is never rewritten. """ text = str(objective or "").replace("\\r\\n", "\n").replace("\\n", "\n") match = HISTORY_SECTION.search(text) return text[: match.start()] if match else text def _declared_role(objective: str) -> str | None: """Return the role a card declared through explicit worker metadata.""" for pattern in (ROLE_DIRECTIVE, OUTPUT_DIRECTIVE): roles = { ROLE_ALIASES[value] for value in ( match.group(1).strip().lower() for match in pattern.finditer(objective) ) if value in ROLE_ALIASES } if len(roles) == 1: return roles.pop() if roles: # A card that declares two different roles has no usable contract. # Hold it to the stricter implementation regime rather than letting # the more permissive review path win an ambiguous declaration. return IMPLEMENTATION_ROLE return None def task_role(objective: str, result: dict[str, Any] | None = None) -> tuple[str, str]: """Resolve the assigned task role and how that resolution was reached.""" text = card_scope(objective) changed = _strings((result or {}).get("changed_files")) declared = _declared_role(text) if declared == REVIEW_ROLE and changed: # Mutation evidence outranks a declaration: a card cannot label itself # read-only and then self-certify a report that edited the tree. return IMPLEMENTATION_ROLE, "conflict" if declared is not None: return declared, "directive" if ( not changed and READ_ONLY_SCOPE.search(text) and VERDICT_DELIVERABLE.search(text) and not MUTATION_DELIVERABLE.search(text) ): return REVIEW_ROLE, "inferred" return IMPLEMENTATION_ROLE, "default" def review_verdict(result: dict[str, Any]) -> tuple[str | None, str | None]: """Return the declared SHIP/BLOCK verdict, or why it cannot be trusted.""" declared: set[str] = set() field = result.get("verdict") if field is not None: value = str(field).strip().upper() if value not in {"SHIP", "BLOCK"}: return None, f"review verdict field is not SHIP or BLOCK: {value[:40]!r}" declared.add(value) summary = str(result.get("summary") or "") declared.update(match.upper() for match in VERDICT_DIRECTIVE.findall(summary)) if len(declared) > 1: return None, "review report declares both SHIP and BLOCK verdicts" if declared: return declared.pop(), None tokens = {match.upper() for match in VERDICT_TOKEN.findall(summary)} if len(tokens) > 1: return None, "review summary names both SHIP and BLOCK without a verdict line" if tokens: return tokens.pop(), None return None, "review report declares no explicit SHIP or BLOCK verdict" def review_completion_problem(result: dict[str, Any]) -> str | None: """Fail closed on a review report that does not carry a usable verdict.""" verdict, problem = review_verdict(result) if problem or verdict is None: return problem or "review report declares no explicit SHIP or BLOCK verdict" summary = str(result.get("summary") or "").strip() if len(summary) < MIN_REVIEW_SUMMARY_CHARS: return f"{verdict} verdict carries no reviewable rationale in its summary" findings = _strings(result.get("findings")) if verdict == "BLOCK" and not findings: return "BLOCK verdict reports no findings to justify it" if not findings and not _strings(result.get("tests_run")) and not _strings( result.get("artifacts") ): return f"{verdict} verdict reports no findings, tests, or artifacts as evidence" return None def _completion_problem(result: dict[str, Any], role: str | None) -> str | None: """Return the unguarded reason this report is not a finished deliverable.""" status = str(result.get("status") or "") if status == "incomplete": summary = str(result.get("summary") or "").strip() return summary or "worker explicitly reported incomplete work" if status != "completed": return None blockers = result.get("blockers") if isinstance(blockers, list) and any(str(item).strip() for item in blockers): return "worker reported blockers while claiming completion" if role == REVIEW_ROLE: # A review's prose describes the reviewed artifact, so the unfinished # work heuristic below would read the artifact's state as the review's # own. The verdict contract is the deterministic gate instead. return review_completion_problem(result) if ( role is None and not _strings(result.get("changed_files")) and review_completion_problem(result) is None ): # A replay caller holds no card. This is exactly the shape the # role-aware lane accepts as a review, so honouring it here keeps a # journalled record valid without weakening the heuristic for anything # that is not already a complete review deliverable. return None tests = result.get("tests_run") evidence = [str(result.get("summary") or "").strip()] if isinstance(tests, list): evidence.extend(str(item) for item in tests) match = UNFINISHED_EVIDENCE.search("\n".join(evidence)) if match: return f"completion evidence says work is unfinished: {match.group(0).strip()}" return None def unfinished_result_reason( result: dict[str, Any], *, role: str | None = None, ) -> str | None: """Reject self-contradictory completion reports without model inference. ``role`` must be supplied by any caller holding the card; see the module docstring for the role-blind replay contract a caller without one gets. """ problem = _completion_problem(result, role) if problem is None: return None if role == REVIEW_ROLE: # Every reason a reviewer may ever read carries the guard, so a resumed # review is never told to repair the implementation it must not touch. problem = f"{READ_ONLY_GUARD}; {problem}" return sanitize_reason(problem) def _completion_claimed(result: dict[str, Any]) -> bool: """Report whether the worker actually claimed a finished task.""" return str(result.get("status") or "") == "completed" def _judge_payload(objective: str, result: dict[str, Any], role: str) -> dict[str, Any]: """Build the fail-closed local judge request for an implementation card.""" return { "model": GOAL_JUDGE_MODEL, "messages": [ { "role": "system", "content": ( "You are a fail-closed completion judge for an engineering task. " "Compare the objective and acceptance criteria with the worker report. " "Return complete only when the report gives concrete evidence that every " "explicit requirement finished. Return continue when any work, test, command, " "commit, push, review, or verification is pending, in progress, omitted, or " "inconclusive. Defects the worker reports about some other artifact are " "findings, not unfinished work on this task. Do not trust the report status " "field." ), }, { "role": "user", "content": json.dumps( { "task_role": role, "objective": _bounded(objective, 6000), "worker_report": _bounded( json.dumps(result, default=str), 4000 ), }, separators=(",", ":"), ), }, ], "stream": False, "temperature": 0, "max_tokens": 160, "response_format": { "type": "json_schema", "json_schema": { "name": "goal_completion_verdict", "strict": True, "schema": GOAL_JUDGE_SCHEMA, }, }, } def judge_goal_completion( objective: str, result: dict[str, Any], *, url: str = GOAL_JUDGE_URL, open_request: Callable[..., Any] = urllib.request.urlopen, ) -> tuple[bool, str]: """Judge one worker report against the action its card actually assigned.""" role, source = task_role(objective, result) reason = unfinished_result_reason(result, role=role) if reason is None and not _completion_claimed(result): reason = ( f"worker reported status {str(result.get('status') or 'unknown')!r} " "rather than a finished task" ) if role == REVIEW_ROLE: reason = f"{READ_ONLY_GUARD}; {reason}" if reason: return False, sanitize_reason(reason) if role == REVIEW_ROLE: # The requested deliverable is present and self-consistent. A verdict # that the reviewed artifact must not ship is a finished review, so the # card finalizes instead of resuming a worker that may not repair it. verdict, _ = review_verdict(result) findings = len(_strings(result.get("findings"))) return True, sanitize_reason( f"review deliverable complete: {verdict} verdict with {findings} finding(s); " f"task role {role} resolved by {source}" ) request = urllib.request.Request( url, data=json.dumps(_judge_payload(objective, result, role)).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) try: with open_request(request, timeout=GOAL_JUDGE_TIMEOUT_SECONDS) as response: document = json.load(response) content = document["choices"][0]["message"]["content"] verdict = json.loads(content) if verdict.get("verdict") not in {"complete", "continue"}: raise ValueError("local judge returned an invalid verdict") reason = sanitize_reason(verdict.get("reason")) return verdict.get("verdict") == "complete", reason except ( AttributeError, IndexError, KeyError, OSError, TypeError, ValueError, json.JSONDecodeError, urllib.error.URLError, ) as error: return False, sanitize_reason( f"local completion judge unavailable: {type(error).__name__}: {error}" )