atlas-iac/services/hermes/scripts/execution_pool_scm.py

#!/usr/bin/env python3
"""Assignment-bound Git gate routed exclusively through the PR14 SCM broker."""
from __future__ import annotations
import json
import hashlib
import os
import re
import stat
import subprocess
import threading
import time
import urllib.parse
from pathlib import Path
from collections.abc import Callable
from typing import Any

import scm_broker_client
from scm_task_grants import ZERO_SHA, sign_grant
from execution_pool_project import ATLAS_REPO, validate_branch
from execution_pool_protocol import ProtocolError, atomic_json, canonical_json, verify_envelope

WORKSPACE_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
SCM_ROOT = Path(os.environ.get("HERMES_SCM_STATE_ROOT", "/scm-state"))
ORDINAL = int(os.environ.get("HERMES_WORKER_ORDINAL", "-1"))
BROKER_ORIGIN = scm_broker_client.BROKER_ORIGIN.rstrip("/")
MAX_STATUS_BYTES = 4 * 1024 * 1024
MAX_PULL_PAGES = 20

def _broker_push_args(grant: str, target: str) -> tuple[str, ...]:
    """Build a self-contained receive-pack push for the broker scanner."""
    return ("-c", f"http.extraHeader=X-Hermes-Task-Grant: {grant}", "-c", "pack.window=0", "-c", "pack.depth=0", "push", "--no-thin", "hermes-broker", f"HEAD:refs/heads/{target}")


def _push_failure(error: RuntimeError) -> str:
    """Return bounded remediation without reflecting Git headers or grant text."""
    detail = str(error).lower()
    if any(marker in detail for marker in (
        "task branch head changed", "non-fast-forward", "fetch first",
    )):
        return "task branch changed; fetch and merge before retrying"
    if any(marker in detail for marker in (
        "signature", "not registered", "owned by another task",
        "authentication", "unauthorized", "forbidden", "permission denied",
        "http 401", "http 403",
    )):
        return (
            "SCM broker authorization rejected the branch update; "
            "inspect task ownership before retrying"
        )
    if any(marker in detail for marker in (
        "timed out", "connection", "could not resolve", "http 502",
        "http 503", "http 504",
    )):
        return "SCM broker transport failed; retry the preserved local commit"
    return "SCM broker rejected the branch update; inspect broker or upstream state before retrying"


def _git_environment() -> dict[str, str]:
    """Run Git without credentials, prompts, ambient config, or hook execution."""
    return {
        "HOME": "/nonexistent",
        "PATH": "/usr/bin:/bin",
        "GIT_CONFIG_NOSYSTEM": "1",
        "GIT_TERMINAL_PROMPT": "0",
    }


def _run(*arguments: str, cwd: Path | None = None, timeout: int = 300) -> str:
    command = [
        "/usr/bin/git",
        "-c",
        "core.hooksPath=/dev/null",
        "-c",
        "core.fsmonitor=false",
        *arguments,
    ]
    try:
        completed = subprocess.run(
            command,
            cwd=cwd,
            env=_git_environment(),
            stdin=subprocess.DEVNULL,
            text=True,
            capture_output=True,
            timeout=timeout,
            check=False,
        )
    except subprocess.TimeoutExpired as error:
        # Do not serialize ``error``: it includes the full command and its grant header.
        raise RuntimeError("SCM command timed out") from error
    if completed.returncode:
        message = (completed.stderr or completed.stdout or "SCM operation failed")[-2000:]
        raise RuntimeError(message.strip())
    if len(completed.stdout.encode()) > MAX_STATUS_BYTES:
        raise ProtocolError("SCM command output exceeds the safe limit")
    return completed.stdout.strip()


def _binding(envelope: dict[str, Any]) -> tuple[dict[str, Any], str, str, str]:
    if envelope["kind"] != "assignment" or int(envelope["worker_ordinal"]) != ORDINAL:
        raise ProtocolError("assignment does not belong to this worker ordinal")
    payload = envelope.get("payload")
    if not isinstance(payload, dict):
        raise ProtocolError("assignment payload must be an object")
    repo = str(payload.get("repo_url") or "")
    match = ATLAS_REPO.fullmatch(repo)
    if not match:
        raise ProtocolError("assignment repository is outside Atlas")
    try:
        branch = validate_branch(payload.get("branch"), feature=True)
        base = validate_branch(payload.get("base_branch"), feature=False)
    except ValueError as error:
        raise ProtocolError(str(error)) from error
    return payload, match.group("repo"), branch, base


def _broker_repo(repo: str) -> str:
    return f"{BROKER_ORIGIN}/git/atlas/{repo}.git"


def workspace_path(envelope: dict[str, Any]) -> Path:
    """Derive a private ordinal path; no caller-provided path is accepted."""
    parts = tuple(str(envelope[name]) for name in ("board", "task_id", "run_id"))
    identifier = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
    if any(not identifier.fullmatch(part) for part in parts):
        raise ProtocolError("assignment path binding is invalid")
    if WORKSPACE_ROOT.is_symlink():
        raise ProtocolError("workspace root must not be a symlink")
    workspace_root = WORKSPACE_ROOT.resolve()
    root = WORKSPACE_ROOT / "runs"
    if root.is_symlink():
        raise ProtocolError("workspace run root must not be a symlink")
    root.mkdir(mode=0o700, parents=True, exist_ok=True)
    root = root.resolve()
    root.relative_to(workspace_root)
    candidate = root.joinpath(*parts)
    current = root
    for part in parts[:-1]:
        current /= part
        if current.is_symlink():
            raise ProtocolError("workspace parent must not be a symlink")
        current.mkdir(mode=0o700, exist_ok=True)
    if candidate.is_symlink():
        raise ProtocolError("workspace must not be a symlink")
    candidate.resolve(strict=False).relative_to(root)
    return candidate


def _regular_text(path: Path, maximum: int) -> str:
    descriptor = os.open(
        path, os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
    )
    try:
        info = os.fstat(descriptor)
        if not stat.S_ISREG(info.st_mode) or info.st_size > maximum:
            raise ProtocolError("private SCM state is invalid")
        raw = os.read(descriptor, maximum + 1)
    finally:
        os.close(descriptor)
    try:
        return raw.decode().strip()
    except UnicodeError as error:
        raise ProtocolError("private SCM state is malformed") from error


def _state_path(envelope: dict[str, Any]) -> Path:
    if SCM_ROOT.is_symlink():
        raise ProtocolError("private SCM root must not be a symlink")
    root = SCM_ROOT.resolve()
    root.mkdir(mode=0o700, parents=True, exist_ok=True)
    name = "-".join(
        str(envelope[name]) for name in ("board", "task_id", "run_id")
    )
    path = root / f"{name}.json"
    if path.is_symlink():
        raise ProtocolError("private SCM state must not be a symlink")
    path.resolve(strict=False).relative_to(root)
    return path


def submission_refs(branch: str, _attempt: int, _head: str) -> tuple[str, ...]:
    """A root task has one branch and one continuing pull request."""
    try:
        return (validate_branch(branch, feature=True),)
    except ValueError as error:
        raise ProtocolError("no reviewed branch name is available for this task") from error


def _remote_heads(destination: Path, refs: tuple[str, ...]) -> dict[str, str]:
    """Read the broker's current head for each candidate ref, read-only."""
    output = _run(
        "ls-remote", "--heads", "hermes-broker",
        *(f"refs/heads/{ref}" for ref in refs),
        cwd=destination, timeout=300,
    )
    heads: dict[str, str] = {}
    for line in output.splitlines():
        fields = line.split()
        if len(fields) == 2 and fields[1].startswith("refs/heads/"):
            heads[fields[1].removeprefix("refs/heads/")] = fields[0]
    return heads


def _workspace_identity(destination: Path, repo: str, branch: str) -> str:
    """Validate the checkout using bounded Git plumbing with all hooks disabled."""
    if destination.is_symlink() or not (destination / ".git").is_dir():
        raise ProtocolError("workspace Git metadata is unavailable")
    ref_path = destination / ".git/refs/heads"
    for part in branch.split("/"):
        ref_path /= part
        if ref_path.is_symlink():
            raise ProtocolError("workspace branch ref must not be a symlink")
    origin = _run("remote", "get-url", "origin", cwd=destination)
    if origin != f"https://scm.bstein.dev/titan/{repo}.git":
        raise ProtocolError("workspace origin does not match assignment")
    broker = _run("remote", "get-url", "hermes-broker", cwd=destination)
    if broker != _broker_repo(repo):
        raise ProtocolError("workspace broker remote does not match assignment")
    current = _run("symbolic-ref", "--short", "HEAD", cwd=destination)
    if current != branch:
        raise ProtocolError("workspace branch does not match assignment")
    head = _run("rev-parse", "--verify", "HEAD", cwd=destination)
    if not re.fullmatch(r"[0-9a-f]{40,64}", head):
        raise ProtocolError("workspace HEAD is invalid")
    return head


def bootstrap_resume_artifact(key: bytes, assignment: Any, terminal: Any, *, title: str, body: str) -> dict[str, Any]:
    """Issue one receipt inside the mediator from two retained signed envelopes."""
    boundary = Boundary(key)
    source = boundary.verify(assignment)
    result = verify_envelope(key, terminal, expected_kind="result")
    names = ("board", "task_id", "run_id", "worker_ordinal", "attempt")
    if any(source[name] != result[name] for name in names):
        raise ProtocolError("retained terminal result does not match assignment")
    payload = result.get("payload")
    if not isinstance(payload, dict) or not isinstance(payload.get("structured"), dict):
        raise ProtocolError("retained terminal result is malformed")
    return boundary.resume_artifact(source, {"title": title, "body": body}, payload["structured"])


class Boundary:
    """The only process allowed to turn model output into an SCM/result handoff."""

    def __init__(self, key: bytes):
        self.key = key
        self.lock = threading.RLock()

    def verify(self, raw: Any) -> dict[str, Any]:
        return verify_envelope(self.key, raw, expected_kind="assignment")

    @staticmethod
    def _grant(envelope: dict[str, Any], repo: str, branch: str, base: str, old: str, head: str) -> str:
        """Bind a five-minute update authorization to the verified assignment."""
        payload = envelope.get("payload")
        root = payload.get("root_task_id") if isinstance(payload, dict) else None
        root = root if isinstance(root, str) and root else str(envelope["task_id"])
        return sign_grant({
            "repo": repo, "ref": branch, "base": base, "board": str(envelope["board"]),
            "root_task_id": root, "assignment_task_id": str(envelope["task_id"]),
            "run": str(envelope["run_id"]), "ordinal": int(envelope["worker_ordinal"]),
            "expires": int(time.time()) + 300, "expected_old": old, "new_head": head,
            "continuation_kind": str(payload.get("continuation_kind") or "") if isinstance(payload, dict) else "",
        })

    def checkout(self, envelope: dict[str, Any]) -> dict[str, Any]:
        _payload, repo, branch, base = _binding(envelope)
        destination = workspace_path(envelope)
        state_path = _state_path(envelope)
        with self.lock:
            if (destination / ".git").exists():
                _workspace_identity(destination, repo, branch)
                state = json.loads(_regular_text(state_path, 16 * 1024))
                baseline = state.get("baseline_sha") if isinstance(state, dict) else None
                if not isinstance(baseline, str) or not re.fullmatch(
                    r"[0-9a-f]{40,64}", baseline
                ):
                    raise ProtocolError("private SCM baseline is unavailable")
                return {"workspace": str(destination), "baseline_sha": baseline}
            if destination.exists() and any(destination.iterdir()):
                raise ProtocolError("workspace is non-empty and unmanaged")
            destination.parent.mkdir(parents=True, exist_ok=True)
            broker = _broker_repo(repo)
            try:
                _run(
                    "clone", "--single-branch", "--branch", branch, "--no-tags",
                    broker, str(destination), timeout=900,
                )
            except RuntimeError as error:
                if destination.exists() and any(destination.iterdir()):
                    raise ProtocolError(
                        "failed branch checkout left unmanaged workspace state"
                    ) from error
                if destination.exists():
                    destination.rmdir()
                _run(
                    "clone", "--single-branch", "--branch", base, "--no-tags",
                    broker, str(destination), timeout=900,
                )
                _run("checkout", "-b", branch, cwd=destination)
            _run(
                "remote", "set-url", "origin",
                f"https://scm.bstein.dev/titan/{repo}.git", cwd=destination,
            )
            _run("remote", "add", "hermes-broker", broker, cwd=destination)
            _run("config", "user.name", "Hermes Execution Worker", cwd=destination)
            _run("config", "user.email", "hermes@bstein.dev", cwd=destination)
            baseline = _workspace_identity(destination, repo, branch)
            atomic_json(
                state_path,
                {"baseline_sha": baseline, "repo": repo, "branch": branch},
            )
        return {"workspace": str(destination), "baseline_sha": baseline}

    @staticmethod
    def _draft(repo: str, branch: str, base: str, head: str, title: str, body: str, grant: str = "", *, refresh: bool = True, existing_only: bool = False) -> str:
        for page in range(1, MAX_PULL_PAGES + 1):
            query = urllib.parse.urlencode({"state": "open", "limit": 50, "page": page})
            existing = json.loads(
                scm_broker_client.read(f"/api/v1/repos/titan/{repo}/pulls?{query}")
            )
            if not isinstance(existing, list):
                raise ProtocolError("open pull-request discovery is malformed")
            for item in existing:
                if not isinstance(item, dict):
                    continue
                source, target = item.get("head"), item.get("base")
                if not isinstance(source, dict) or not isinstance(target, dict):
                    continue
                if source.get("ref") == branch and target.get("ref") == base:
                    number = item.get("number")
                    if not refresh:
                        return str(item.get("html_url") or "")
                    if not isinstance(number, int) or not grant:
                        raise ProtocolError("existing task draft cannot be refreshed")
                    updated = json.loads(scm_broker_client.update_draft(grant, number, title, body))
                    return str(updated.get("html_url") or "")
            if len(existing) < 50:
                break
        else:
            raise ProtocolError("open pull-request discovery exceeds the safe page limit")
        if existing_only:
            raise ProtocolError("review continuation has no existing pull request")
        created = json.loads(
            scm_broker_client.create_draft(
                repo,
                base=base,
                head=branch,
                head_sha=head,
                title=title,
                body=body,
            )
        )
        return str(created.get("html_url") or "")

    def submit(self, envelope: dict[str, Any], request: dict[str, Any], *, checkpoint: Callable[[], None] | None = None) -> dict[str, Any]:
        """Enforce clean/committed state, broker push, and reviewed draft creation."""
        _payload, repo, branch, base = _binding(envelope)
        destination = workspace_path(envelope)
        title = str(request.get("title") or f"Hermes task {envelope['task_id']}").strip()
        body = str(request.get("body") or "Automated Hermes draft.")
        if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
            raise ProtocolError("pull-request metadata exceeds the safe limit")
        with self.lock:
            if checkpoint:
                checkpoint()
            head = _workspace_identity(destination, repo, branch)
            state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
            baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
            if not isinstance(baseline, str) or not re.fullmatch(
                r"[0-9a-f]{40,64}", baseline
            ):
                raise ProtocolError("private SCM baseline is unavailable")
            status = _run(
                "status", "--porcelain=v1", "--untracked-files=all", cwd=destination
            )
            if status:
                raise ProtocolError("workspace has uncommitted or untracked files")
            target = submission_refs(branch, int(envelope["attempt"]), head)[0]
            kind = _payload.get("continuation_kind")
            repair, review = kind == "repair", kind == "review"
            if repair:
                # Do not advance an owned branch after its human PR was closed.
                self._draft(repo, target, base, head, title, body, refresh=False, existing_only=True)
            heads = _remote_heads(destination, (target,))
            if checkpoint:
                checkpoint()
            remote = heads.get(target)
            ahead = int(_run("rev-list", "--count", f"{baseline}..{head}", cwd=destination))
            if ahead <= 0 and remote is None:
                return {"workspace": str(destination), "branch": branch, "pull_request": ""}
            if remote == head:
                if checkpoint:
                    checkpoint()
                grant = self._grant(envelope, repo, target, base, remote, head)
                pull = self._draft(repo, target, base, head, title, body, grant) if not (repair or review) else self._draft(
                    repo, target, base, head, title, body, grant, refresh=not review, existing_only=True
                )
                return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}
            expected = remote or ZERO_SHA
            grant = self._grant(envelope, repo, target, base, expected, head)
            if remote is None:
                scm_broker_client.register_task(grant)
            try:
                if checkpoint:
                    checkpoint()
                _run(
                    *_broker_push_args(grant, target),
                    cwd=destination, timeout=900,
                )
            except RuntimeError as error:
                raise ProtocolError(_push_failure(error)) from error
            if checkpoint:
                checkpoint()
            grant = self._grant(envelope, repo, target, base, head, head)
            pull = self._draft(repo, target, base, head, title, body, grant) if not (repair or review) else self._draft(
                repo, target, base, head, title, body, grant, refresh=not review, existing_only=True
            )
        return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}

    def resume_artifact(self, envelope: dict[str, Any], request: dict[str, Any], structured: dict[str, Any]) -> dict[str, Any]:
        """Bind one clean, locally durable commit for a later fresh-grant retry."""
        payload, repo, branch, base = _binding(envelope)
        destination = workspace_path(envelope)
        head = _workspace_identity(destination, repo, branch)
        state = json.loads(_regular_text(_state_path(envelope), 16 * 1024))
        baseline = state.get("baseline_sha") if isinstance(state, dict) else ""
        title, body = str(request.get("title") or "").strip(), str(request.get("body") or "")
        if not isinstance(baseline, str) or not re.fullmatch(r"[0-9a-f]{40,64}", baseline):
            raise ProtocolError("private SCM baseline is unavailable")
        if _run("status", "--porcelain=v1", "--untracked-files=all", cwd=destination):
            raise ProtocolError("workspace has uncommitted or untracked files")
        if not title or len(title.encode()) > 512 or len(body.encode()) > 32 * 1024:
            raise ProtocolError("pull-request metadata exceeds the safe limit")
        root = payload.get("root_task_id")
        if not isinstance(root, str) or not root:
            raise ProtocolError("assignment continuation root is unavailable")
        source = {name: envelope[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
        source.update({"repo_url": payload.get("repo_url"), "branch": branch, "base_branch": base, "root_task_id": root})
        evidence = {
            "structured": json.loads(canonical_json(structured)),
            "title": title,
            "body": body,
        }
        return {"source": source, "baseline_sha": baseline, "head": head, **evidence,
                "result_digest": hashlib.sha256(canonical_json(evidence)).hexdigest()}

    def resume(self, envelope: dict[str, Any], artifact: Any, *, checkpoint: Callable[[], None] | None = None) -> dict[str, Any]:
        """Publish a verified preserved head with only the new assignment grant."""
        payload, repo, branch, base = _binding(envelope)
        required = {"source", "baseline_sha", "head", "title", "body", "structured", "result_digest"}
        if not isinstance(artifact, dict) or set(artifact) != required:
            raise ProtocolError("SCM resume artifact is malformed")
        source = artifact["source"]
        source_keys = {"board", "task_id", "run_id", "worker_ordinal", "attempt", "repo_url", "branch", "base_branch", "root_task_id"}
        if not isinstance(source, dict) or set(source) != source_keys:
            raise ProtocolError("SCM resume source binding is malformed")
        if (source["board"] != envelope["board"] or source["task_id"] != envelope["task_id"]
                or source["run_id"] == envelope["run_id"] or not isinstance(source["attempt"], int) or source["attempt"] < 1
                or not isinstance(source["worker_ordinal"], int) or source["worker_ordinal"] != envelope["worker_ordinal"]
                or source["repo_url"] != payload.get("repo_url")
                or source["branch"] != branch or source["base_branch"] != base or source["root_task_id"] != payload.get("root_task_id")):
            raise ProtocolError("SCM resume source does not match assignment")
        evidence = {name: artifact[name] for name in ("structured", "title", "body")}
        if not isinstance(artifact["result_digest"], str) or artifact["result_digest"] != hashlib.sha256(canonical_json(evidence)).hexdigest():
            raise ProtocolError("SCM resume evidence digest is invalid")
        head, baseline = artifact["head"], artifact["baseline_sha"]
        if not all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40,64}", value) for value in (head, baseline)):
            raise ProtocolError("SCM resume revisions are invalid")
        if not isinstance(artifact["title"], str) or not artifact["title"] or len(artifact["title"].encode()) > 512 or not isinstance(artifact["body"], str) or len(artifact["body"].encode()) > 32 * 1024:
            raise ProtocolError("SCM resume metadata is invalid")
        source_envelope = {name: source[name] for name in ("board", "task_id", "run_id", "worker_ordinal", "attempt")}
        destination = workspace_path(source_envelope)
        if checkpoint:
            checkpoint()
        if _workspace_identity(destination, repo, branch) != head:
            raise ProtocolError("preserved SCM workspace head changed")
        state = json.loads(_regular_text(_state_path(source_envelope), 16 * 1024))
        if not isinstance(state, dict) or state.get("baseline_sha") != baseline:
            raise ProtocolError("preserved SCM baseline changed")
        if _run("status", "--porcelain=v1", "--untracked-files=all", cwd=destination):
            raise ProtocolError("preserved SCM workspace is not clean")
        if checkpoint:
            checkpoint()
        target = submission_refs(branch, int(envelope["attempt"]), head)[0]
        self._draft(repo, target, base, head, artifact["title"], artifact["body"], refresh=False, existing_only=True)
        remote = _remote_heads(destination, (target,)).get(target)
        if remote not in {baseline, head}:
            raise ProtocolError("task branch changed; preserved commit needs reconciliation")
        grant = self._grant(envelope, repo, target, base, head if remote == head else baseline, head)
        if remote != head:
            if checkpoint:
                checkpoint()
            _run(*_broker_push_args(grant, target), cwd=destination, timeout=900)
            if checkpoint:
                checkpoint()
        if checkpoint:
            checkpoint()
        pull = self._draft(repo, target, base, head, artifact["title"], artifact["body"], self._grant(envelope, repo, target, base, head, head), existing_only=True)
        return {"workspace": str(destination), "branch": target, "pull_request": pull, "head": head}