atlas-iac/services/hermes/scripts/cli_lane_goal.py

#!/usr/bin/env python3
"""Fail-closed completion checks for durable external Kanban workers."""

from __future__ import annotations

import json
import os
import re
import urllib.error
import urllib.request
from typing import Any, Callable


GOAL_JUDGE_URL = os.environ.get(
    "HERMES_GOAL_JUDGE_URL",
    "http://hermes-model-gate.hermes.svc.cluster.local:11434/v1/chat/completions",
)
GOAL_JUDGE_MODEL = os.environ.get(
    "HERMES_GOAL_JUDGE_MODEL",
    "qwen2.5:14b-instruct-q4_0",
)
GOAL_JUDGE_TIMEOUT_SECONDS = max(
    30,
    min(180, int(os.environ.get("HERMES_GOAL_JUDGE_TIMEOUT_SECONDS", "120"))),
)
RESULT_STATUSES = frozenset({"completed", "blocked", "incomplete"})
GOAL_JUDGE_SCHEMA: dict[str, Any] = {
    "type": "object",
    "additionalProperties": False,
    "required": ["verdict", "reason"],
    "properties": {
        "verdict": {"type": "string", "enum": ["complete", "continue"]},
        "reason": {"type": "string"},
    },
}
UNFINISHED_EVIDENCE = re.compile(
    r"(?:\bin[ -]?progress\b|\bstill\s+(?:active|pending|running)\b|"
    r"\bremains?\s+(?:active|pending|running|unfinished)\b|"
    r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b"
    r"\s+(?:is|are|remains?|still|currently|has|have)\s+"
    r"(?:pending|running|unfinished|not\s+(?:yet\s+)?(?:done|finished|run|complete))\b|"
    r"\b(?:pending|running|unfinished)\s+"
    r"\b(?:tests?|checks?|build|validation|verification|commit|push)\b)",
    re.IGNORECASE | re.DOTALL,
)


def _bounded(value: str, limit: int) -> str:
    """Retain both ends of evidence while bounding local-judge context."""
    if len(value) <= limit:
        return value
    marker = "\n...[goal evidence compacted]...\n"
    remaining = max(0, limit - len(marker))
    head = remaining // 2
    return f"{value[:head]}{marker}{value[-(remaining - head):]}"


def unfinished_result_reason(result: dict[str, Any]) -> str | None:
    """Reject self-contradictory completion reports without model inference."""
    status = str(result.get("status") or "")
    summary = str(result.get("summary") or "").strip()
    blockers = result.get("blockers")
    if status == "incomplete":
        return summary or "worker explicitly reported incomplete work"
    if status != "completed":
        return None
    if isinstance(blockers, list) and any(str(item).strip() for item in blockers):
        return "worker reported blockers while claiming completion"
    tests = result.get("tests_run")
    evidence = [summary]
    if isinstance(tests, list):
        evidence.extend(str(item) for item in tests)
    match = UNFINISHED_EVIDENCE.search("\n".join(evidence))
    if match:
        return f"completion evidence says work is unfinished: {match.group(0).strip()}"
    return None


def judge_goal_completion(
    objective: str,
    result: dict[str, Any],
    *,
    url: str = GOAL_JUDGE_URL,
    open_request: Callable[..., Any] = urllib.request.urlopen,
) -> tuple[bool, str]:
    """Use the local model as a fail-closed judge for explicit goal cards."""
    deterministic_reason = unfinished_result_reason(result)
    if deterministic_reason:
        return False, deterministic_reason
    payload = {
        "model": GOAL_JUDGE_MODEL,
        "messages": [
            {
                "role": "system",
                "content": (
                    "You are a fail-closed completion judge for an engineering task. "
                    "Compare the objective and acceptance criteria with the worker report. "
                    "Return complete only when the report gives concrete evidence that every "
                    "explicit requirement finished. Return continue when any work, test, command, "
                    "commit, push, review, or verification is pending, in progress, omitted, or "
                    "inconclusive. Do not trust the report status field."
                ),
            },
            {
                "role": "user",
                "content": json.dumps(
                    {
                        "objective": _bounded(objective, 6000),
                        "worker_report": _bounded(
                            json.dumps(result, default=str), 4000
                        ),
                    },
                    separators=(",", ":"),
                ),
            },
        ],
        "stream": False,
        "temperature": 0,
        "max_tokens": 160,
        "response_format": {
            "type": "json_schema",
            "json_schema": {
                "name": "goal_completion_verdict",
                "strict": True,
                "schema": GOAL_JUDGE_SCHEMA,
            },
        },
    }
    request = urllib.request.Request(
        url,
        data=json.dumps(payload).encode("utf-8"),
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    try:
        with open_request(request, timeout=GOAL_JUDGE_TIMEOUT_SECONDS) as response:
            document = json.load(response)
        content = document["choices"][0]["message"]["content"]
        verdict = json.loads(content)
        if verdict.get("verdict") not in {"complete", "continue"}:
            raise ValueError("local judge returned an invalid verdict")
        reason = str(verdict.get("reason") or "local judge supplied no reason")
        return verdict.get("verdict") == "complete", reason
    except (
        AttributeError,
        IndexError,
        KeyError,
        OSError,
        TypeError,
        ValueError,
        json.JSONDecodeError,
        urllib.error.URLError,
    ) as error:
        return False, f"local completion judge unavailable: {type(error).__name__}: {error}"