feat(hermes-triage): failure-region console evidence + duplicate-proposal guard

Real-service triage returned "undetermined" because the console tail held
only post-build noise; this Jenkins has no junit or stage-view plugin, so
console text is the only structured evidence available.

- hermes_console_evidence: scan the full console for failure markers
  (pytest, build, tool gates, k8s/agent), capture context windows, merge
  overlaps, collapse repeats, and prefer the earliest regions under a byte
  budget; bundle gains jenkins.console_failures and console_truncated
- evidence: fetch the full console (head + tail bounded at 2MB) instead of
  the last 8KB; signature detection now also scans regions
- prompt: one line explaining that the earliest region usually holds the
  first enforced failure
- code flow: check for an already-open hermes-repair/* pull request before
  spending model tokens; fail open so a Gitea error cannot suppress work

Verified on a synthetic 4018-line pipeline: the real failure at line 7 is
now captured where the previous tail-only slice missed it entirely.

95 new tests; 281 pass in the hermes suite.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

ariadne/services/hermes_autotriage.py

@@ -41,6 +41,7 @@ _PROMPT_TEMPLATE = """Use $triage-titan-test-failures.
 Analyze incident __INCIDENT_ID__.
 Treat the attached Ariadne bundle as the source of truth.
 Identify the first enforced failure.
+The jenkins.console_failures array holds excerpts around detected failure markers in chronological order; the earliest region usually contains the first enforced failure, and jenkins.console_tail is the end of the build which often only shows downstream noise.
 Distinguish facts from inference.
 Return ONLY a single JSON object with exactly these keys and no others:
 {"incident_id": "<must equal __INCIDENT_ID__>", "classification": "<string; use known_demo_fixture_failure only when the evidence shows the hermes-triage-demo fixture unhealthy signature>", "confidence": <0..1>, "facts": [{"statement": "...", "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", "reference": "..."}], "inferences": ["..."], "first_failed_gate": "<string>", "requested_action": {"type": "run_ariadne_job", "id": "repair_demo_fixture"} or null, "human_required": <bool>, "reason": "<string>"}

ariadne/services/hermes_autotriage_evidence.py

@@ -7,13 +7,15 @@ import httpx
 
 from ..settings import settings
 from .hermes_autotriage_logs import collect_log_evidence
+from .hermes_console_evidence import extract_console_evidence
 
 
 SIGNATURE_MARKER = "hermes_demo_test_failure"
 SIGNATURE_TEST_NAME = "fixture-state-check"
 
-_CONSOLE_TAIL_LINES = 100
-_CONSOLE_TAIL_MAX_BYTES = 8192
+_CONSOLE_MAX_CHARS = 2_000_000
+_CONSOLE_TAIL_KEEP_CHARS = 200_000
+_CONSOLE_TRUNCATION_NOTE = "... [ariadne] console truncated in the middle ..."
 _MAX_FAILED_TESTS = 10
 _MAX_ERROR_DETAILS_CHARS = 2000
 _FAILED_TEST_STATUSES = {"FAILED", "REGRESSION"}
@@ -41,6 +43,8 @@ def collect_evidence(incident_id: str, job: str, last_build: dict) -> dict[str,
         "duration_seconds": _millis_to_seconds(last_build.get("duration")),
         "first_failed_stage": None,
         "console_tail": None,
+        "console_failures": [],
+        "console_truncated": False,
         "failed_tests": [],
     }
     build_number = jenkins["build_number"]
@@ -48,7 +52,10 @@ def collect_evidence(incident_id: str, job: str, last_build: dict) -> dict[str,
         with httpx.Client(**_client_kwargs()) as client:
             jenkins["first_failed_stage"] = _first_failed_stage(client, job, build_number)
             jenkins["failed_tests"] = _failed_tests(client, job, build_number)
-            jenkins["console_tail"] = _console_tail(client, job, build_number)
+            console = _console_evidence(client, job, build_number)
+        jenkins["console_tail"] = console.get("tail") or None
+        jenkins["console_failures"] = console.get("regions") or []
+        jenkins["console_truncated"] = bool(console.get("truncated"))
     except Exception:
         pass
     return {
@@ -64,8 +71,8 @@ def evidence_has_signature(bundle: dict, incident_id: str) -> bool:
 
     Inputs: an evidence bundle from `collect_evidence` and the incident id.
     Outputs: True when the fixture-state-check test failed, the console tail
-    contains the demo failure marker, or the log records contain both the
-    marker and the incident id.
+    or any console failure region contains the demo failure marker, or the
+    log records contain both the marker and the incident id.
     """
 
     jenkins = bundle.get("jenkins") if isinstance(bundle.get("jenkins"), dict) else {}
@@ -73,7 +80,7 @@ def evidence_has_signature(bundle: dict, incident_id: str) -> bool:
     for test in failed_tests:
         if isinstance(test, dict) and test.get("name") == SIGNATURE_TEST_NAME:
             return True
-    if SIGNATURE_MARKER in str(jenkins.get("console_tail") or ""):
+    if SIGNATURE_MARKER in _console_signature_text(jenkins):
         return True
     log_evidence = bundle.get("log_evidence") if isinstance(bundle.get("log_evidence"), dict) else {}
     records = log_evidence.get("records") if isinstance(log_evidence.get("records"), list) else []
@@ -81,6 +88,17 @@ def evidence_has_signature(bundle: dict, incident_id: str) -> bool:
     return SIGNATURE_MARKER in messages and incident_id in messages
 
 
+def _console_signature_text(jenkins: dict[str, Any]) -> str:
+    """Join the console tail and every failure region into one search string."""
+
+    regions = jenkins.get("console_failures")
+    parts = [str(jenkins.get("console_tail") or "")]
+    for region in regions if isinstance(regions, list) else []:
+        if isinstance(region, dict):
+            parts.append(str(region.get("text") or ""))
+    return "\n".join(parts)
+
+
 def _build_window(last_build: dict) -> tuple[str, str]:
     """Return the build start/end window as ISO-8601 UTC strings."""
 
@@ -154,17 +172,30 @@ def _failed_test(case: dict[str, Any]) -> dict[str, Any]:
     }
 
 
-def _console_tail(client: httpx.Client, job: str, build_number: int) -> str | None:
-    """Return the byte-capped tail of the build console log, or None."""
+def _console_evidence(client: httpx.Client, job: str, build_number: int) -> dict[str, Any]:
+    """Return failure regions plus the tail of the build console log.
+
+    The whole console is fetched (bounded by `_CONSOLE_MAX_CHARS`) because on
+    long pipelines the first enforced failure is thousands of lines above the
+    tail. Returns an empty dict when the console cannot be read.
+    """
 
     try:
         response = client.get(f"{_base_url()}/job/{job}/{build_number}/consoleText")
         response.raise_for_status()
         text = response.text
     except Exception:
-        return None
-    tail = "\n".join(text.splitlines()[-_CONSOLE_TAIL_LINES:])
-    return tail[-_CONSOLE_TAIL_MAX_BYTES:]
+        return {}
+    return extract_console_evidence(_bounded_console(text))
+
+
+def _bounded_console(text: str) -> str:
+    """Cap console text, keeping both the start and the end of huge logs."""
+
+    if len(text) <= _CONSOLE_MAX_CHARS:
+        return text
+    head = text[: _CONSOLE_MAX_CHARS - _CONSOLE_TAIL_KEEP_CHARS]
+    return f"{head}\n{_CONSOLE_TRUNCATION_NOTE}\n{text[-_CONSOLE_TAIL_KEEP_CHARS:]}"
 
 
 def _base_url() -> str:

ariadne/services/hermes_code_flow.py

@@ -14,6 +14,7 @@ CODE_PROPOSAL_EVENT_TYPE = "hermes_autotriage_code_proposal"
 
 _RUN_COMPLETED = "completed"
 _GITEA_TIMEOUT_SECONDS = 15.0
+_EXISTING_PROPOSAL_REASON = "existing_proposal_open"
 
 _PATCH_PROMPT_TEMPLATE = """Use $triage-titan-test-failures.
 You are proposing a MINIMAL source fix for incident __INCIDENT_ID__.
@@ -99,6 +100,9 @@ def _propose(
 ) -> tuple[dict[str, Any], dict[str, Any]]:
     """Run fetch, diagnosis, validation, and publication for one incident."""
 
+    duplicate = _duplicate_proposal(code_cfg, incident_id)
+    if duplicate is not None:
+        return duplicate
     path = str(code_cfg.get("candidate_path") or "")
     contents, fetch_error = hermes_code_repair.fetch_file(code_cfg, path)
     if contents is None:
@@ -120,6 +124,47 @@ def _propose(
     return _publish(code_cfg, incident_id, build_number, proposal)
 
 
+def _duplicate_proposal(
+    code_cfg: dict, incident_id: str
+) -> tuple[dict[str, Any], dict[str, Any]] | None:
+    """Suppress this proposal when an earlier repair pull request is still open.
+
+    Runs before the candidate fetch and before Hermes, so a duplicate costs no
+    model tokens. Returns the human_required result/event pair when an open
+    proposal exists, otherwise None so the flow continues — including when the
+    lookup itself failed, which fails open rather than dropping real work.
+    """
+
+    existing = hermes_code_repair.find_open_proposal(code_cfg)
+    error = existing.get("error")
+    if error:
+        logger.info(
+            "hermes open proposal lookup failed",
+            extra={
+                "event": "hermes_code_flow",
+                "status": "error",
+                "incident_id": incident_id,
+                "detail": error,
+            },
+        )
+        return None
+    if not existing.get("found"):
+        return None
+    identity = {
+        "pr_number": existing.get("pr_number"),
+        "url": existing.get("url"),
+        "branch": existing.get("branch"),
+    }
+    result = {"status": "human_required", "reason": _EXISTING_PROPOSAL_REASON, **identity}
+    event = {
+        "run_id": None,
+        "validated": False,
+        "reject_reason": _EXISTING_PROPOSAL_REASON,
+        **identity,
+    }
+    return result, event
+
+
 def _validated_patch(
     raw_output: str, incident_id: str, path: str, code_cfg: dict, contents: str
 ) -> tuple[hermes_code_patch.ProposedPatch | None, str]:

ariadne/services/hermes_code_repair.py

@@ -18,6 +18,8 @@ HTTP_UNPROCESSABLE = 422
 
 _DEFAULT_TIMEOUT_SECONDS = 15.0
 _PROTECTED_BRANCHES = {"master", "main"}
+_REPAIR_BRANCH_PREFIX = "hermes-repair/"
+_OPEN_PULLS_LIMIT = 50
 _COMMIT_IDENTITY = {"name": "Hermes Agent", "email": "hermes@bstein.dev"}
 _COMMIT_OK_STATUSES = {HTTP_OK, HTTP_CREATED}
 _BRANCH_RETRY_STATUSES = {HTTP_NOT_FOUND, HTTP_UNPROCESSABLE}
@@ -46,6 +48,39 @@ def fetch_file(cfg: dict, path: str) -> tuple[str | None, str | None]:
     return response.text, None
 
 
+def find_open_proposal(cfg: dict) -> dict[str, Any]:
+    """Find an already-open Hermes repair pull request for this repository.
+
+    Inputs: `cfg` as for `fetch_file`. Outputs:
+    {"found", "pr_number", "url", "branch", "error"}. An open pull request
+    counts as an existing proposal when its head branch starts with
+    `hermes-repair/` and its base branch equals cfg["base_branch"]; the
+    lowest-numbered match (the original proposal) is returned so repeated
+    checks stay stable while the branch sits unmerged.
+
+    Fails open by design: every HTTP, parse, or transport failure returns
+    found=False with `error` set, because a false "duplicate" would silently
+    suppress legitimate repair work — a missed duplicate is only noise, a
+    wrong duplicate is lost work. Never raises and never logs the token.
+    """
+
+    base_url = _base_url(cfg)
+    if not base_url:
+        return _no_proposal("gitea base url is empty")
+    try:
+        with httpx.Client(timeout=_timeout(cfg)) as client:
+            response = client.get(
+                f"{base_url}/api/v1/repos/{_owner(cfg)}/{_repo(cfg)}/pulls",
+                headers=_headers(cfg),
+                params={"state": "open", "limit": _OPEN_PULLS_LIMIT},
+            )
+    except Exception as exc:
+        return _no_proposal(f"open proposal lookup failed: {exc}")
+    if response.status_code != HTTP_OK:
+        return _no_proposal(f"open proposal lookup http {response.status_code}")
+    return _oldest_repair_pull(response, _base_branch(cfg))
+
+
 def push_branch(
     cfg: dict, incident_id: str, build_number: int, patch: Any, patched_contents: str
 ) -> dict[str, Any]:
@@ -162,6 +197,49 @@ def _file_sha(client: httpx.Client, cfg: dict, path: str) -> tuple[str | None, s
     return str(sha), None
 
 
+def _oldest_repair_pull(response: Any, base_branch: str) -> dict[str, Any]:
+    """Pick the lowest-numbered repair proposal out of an open-pulls payload."""
+
+    try:
+        payload = response.json()
+    except Exception as exc:
+        return _no_proposal(f"open proposal parse failed: {exc}")
+    if not isinstance(payload, list):
+        return _no_proposal("open proposal payload is not a list")
+    matches = [pull for pull in payload if _is_repair_pull(pull, base_branch)]
+    if not matches:
+        return _no_proposal(None)
+    oldest = min(matches, key=lambda pull: int(pull["number"]))
+    return {
+        "found": True,
+        "pr_number": int(oldest["number"]),
+        "url": str(oldest.get("html_url") or "") or None,
+        "branch": str(oldest["head"].get("ref") or "") or None,
+        "error": None,
+    }
+
+
+def _is_repair_pull(pull: Any, base_branch: str) -> bool:
+    """Report whether one open pull request is a Hermes repair proposal."""
+
+    if not isinstance(pull, dict):
+        return False
+    number = pull.get("number")
+    if isinstance(number, bool) or not isinstance(number, int):
+        return False
+    head, base = pull.get("head"), pull.get("base")
+    if not isinstance(head, dict) or not isinstance(base, dict):
+        return False
+    head_ref = str(head.get("ref") or "")
+    return head_ref.startswith(_REPAIR_BRANCH_PREFIX) and str(base.get("ref") or "") == base_branch
+
+
+def _no_proposal(error: str | None) -> dict[str, Any]:
+    """Build the fail-open result meaning "no existing proposal found"."""
+
+    return {"found": False, "pr_number": None, "url": None, "branch": None, "error": error}
+
+
 def _pr_result(response: Any) -> dict[str, Any]:
     """Map a pull-request response payload to the result shape."""
 

ariadne/services/hermes_console_evidence.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+FAILURE_MARKERS: tuple[str, ...] = (
+    # kubernetes / build-agent signals: long and unambiguous, so they are
+    # matched first and never mislabelled as a generic test failure
+    "ImagePullBackOff",
+    "OOMKilled",
+    "Failed to establish a new connection",
+    "Temporary failure in name resolution",
+    # pytest / unittest signals: they name the real failure
+    "=== FAILURES ===",
+    "short test summary",
+    "ERROR at setup",
+    "Traceback (most recent call last)",
+    "AssertionError",
+    "FAILED ",
+    "E   ",
+    # generic build signals
+    "BUILD FAILED",
+    "FAILURE:",
+    "non-zero exit",
+    "exit code",
+    "command not found",
+    "No such file",
+    "Exception",
+    "ERROR:",
+    # tool gates
+    "Quality gate",
+    "fail-under",
+    "coverage",
+    "docstring",
+    "Semgrep",
+    "SonarQube",
+    "Trivy",
+    "ruff",
+    "[loc]",
+)
+
+# Markers that are only meaningful at the start of a (stripped) line; matching
+# them anywhere would swallow every line that merely says "... failed ...".
+# All matching is case-insensitive, so these are stored lowercased.
+PREFIX_ONLY_MARKERS: frozenset[str] = frozenset({"e   ", "failed "})
+
+_DEFAULTS: dict[str, int] = {
+    "context_before": 6,
+    "context_after": 12,
+    "tail_lines": 40,
+    "max_total_bytes": 12000,
+    "max_tail_bytes": 4000,
+    "max_regions": 6,
+}
+
+_EMPTY: dict[str, Any] = {"regions": [], "tail": "", "total_lines": 0, "truncated": False}
+_DIGITS = re.compile(r"\d+")
+
+
+def extract_console_evidence(console_text: str, cfg: dict | None = None) -> dict:
+    """Extract failure regions and the tail from a Jenkins console log.
+
+    Inputs: the full `consoleText` of a build and an optional config dict
+    (`context_before`, `context_after`, `tail_lines`, `max_total_bytes`,
+    `max_tail_bytes`, `max_regions`; each falls back to its default).
+    Outputs: {"regions": [{"marker", "line_number", "text"}, ...] in
+    chronological order, "tail": str, "total_lines": int, "truncated": bool}.
+    The tail alone is unreliable on long pipelines, so regions carry the
+    earliest enforced failure. Pure function, no I/O, never raises.
+    """
+
+    try:
+        options = _options(cfg)
+        lines = _lines(console_text)
+        regions, truncated = _budgeted(_deduped(_regions(lines, options)), options)
+        return {
+            "regions": regions,
+            "tail": _tail(lines, options),
+            "total_lines": len(lines),
+            "truncated": truncated,
+        }
+    except Exception:
+        return dict(_EMPTY)
+
+
+def marker_for_line(line: str) -> str | None:
+    """Return the highest-priority failure marker matching one console line.
+
+    Inputs: a single console line. Outputs: the marker as declared in
+    `FAILURE_MARKERS`, or None when the line carries no failure signal.
+    """
+
+    lowered = str(line).lower()
+    stripped = lowered.lstrip()
+    for marker in FAILURE_MARKERS:
+        needle = marker.lower()
+        if needle in PREFIX_ONLY_MARKERS:
+            if stripped.startswith(needle):
+                return marker
+        elif needle in lowered:
+            return marker
+    return None
+
+
+def _lines(console_text: Any) -> list[str]:
+    """Split console text into lines, tolerating None and non-strings."""
+
+    return console_text.splitlines() if isinstance(console_text, str) else []
+
+
+def _options(cfg: dict | None) -> dict[str, int]:
+    """Merge caller config over the module defaults, ignoring bad values."""
+
+    source = cfg if isinstance(cfg, dict) else {}
+    return {key: _bounded_int(source.get(key), default) for key, default in _DEFAULTS.items()}
+
+
+def _bounded_int(value: Any, default: int) -> int:
+    """Coerce a config value to a non-negative int, else return the default."""
+
+    try:
+        number = int(value)
+    except (TypeError, ValueError):
+        return default
+    return number if number >= 0 else default
+
+
+def _regions(lines: list[str], options: dict[str, int]) -> list[dict[str, Any]]:
+    """Build merged context windows around every failure marker hit."""
+
+    spans: list[dict[str, Any]] = []
+    for index, line in enumerate(lines):
+        marker = marker_for_line(line)
+        if marker is None:
+            continue
+        start = max(0, index - options["context_before"])
+        end = min(len(lines), index + options["context_after"] + 1)
+        if spans and start <= spans[-1]["end"]:
+            spans[-1]["end"] = max(spans[-1]["end"], end)
+            continue
+        spans.append({"marker": marker, "line_number": index + 1, "start": start, "end": end})
+    return [
+        {
+            "marker": span["marker"],
+            "line_number": span["line_number"],
+            "text": "\n".join(lines[span["start"] : span["end"]]),
+        }
+        for span in spans
+    ]
+
+
+def _deduped(regions: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Collapse near-identical repeated regions into one annotated region."""
+
+    first: dict[str, dict[str, Any]] = {}
+    counts: dict[str, int] = {}
+    order: list[str] = []
+    for region in regions:
+        key = _fingerprint(region["text"])
+        if key in first:
+            counts[key] += 1
+            continue
+        first[key] = region
+        counts[key] = 1
+        order.append(key)
+    return [_with_repeat_note(first[key], counts[key]) for key in order]
+
+
+def _fingerprint(text: str) -> str:
+    """Normalize region text so retries differing only in numbers collapse."""
+
+    return _DIGITS.sub("#", " ".join(text.split()))
+
+
+def _with_repeat_note(region: dict[str, Any], count: int) -> dict[str, Any]:
+    """Append a repeat note to a region seen more than once."""
+
+    if count <= 1:
+        return region
+    return {**region, "text": f"{region['text']}\n(repeated {count} times)"}
+
+
+def _budgeted(
+    regions: list[dict[str, Any]], options: dict[str, int]
+) -> tuple[list[dict[str, Any]], bool]:
+    """Keep the earliest regions that fit the byte and count budgets."""
+
+    limit = options["max_total_bytes"]
+    kept: list[dict[str, Any]] = []
+    used = 0
+    truncated = False
+    for region in regions:
+        if len(kept) >= options["max_regions"]:
+            return kept, True
+        size = _byte_len(region["text"])
+        if used + size > limit:
+            if kept:
+                return kept, True
+            region = {**region, "text": _clip_head(region["text"], limit)}
+            size = _byte_len(region["text"])
+            truncated = True
+        kept.append(region)
+        used += size
+    return kept, truncated
+
+
+def _tail(lines: list[str], options: dict[str, int]) -> str:
+    """Return the byte-capped last lines of the console log."""
+
+    tail_lines = options["tail_lines"]
+    if tail_lines <= 0:
+        return ""
+    return _clip_tail("\n".join(lines[-tail_lines:]), options["max_tail_bytes"])
+
+
+def _byte_len(text: str) -> int:
+    """Return the UTF-8 byte length of a string."""
+
+    return len(text.encode("utf-8", "ignore"))
+
+
+def _clip_head(text: str, limit: int) -> str:
+    """Return at most `limit` bytes from the start of the text."""
+
+    if limit <= 0:
+        return ""
+    return text.encode("utf-8", "ignore")[:limit].decode("utf-8", "ignore")
+
+
+def _clip_tail(text: str, limit: int) -> str:
+    """Return at most `limit` bytes from the end of the text."""
+
+    if limit <= 0:
+        return ""
+    return text.encode("utf-8", "ignore")[-limit:].decode("utf-8", "ignore")

tests/test_hermes_autotriage_evidence.py

@@ -138,6 +138,8 @@ def test_collect_evidence_full_bundle(monkeypatch) -> None:
         {"name": "flaky", "className": "demo.Flaky", "errorDetails": None},
     ]
     assert jenkins["console_tail"] == "line1\nline2\nhermes_demo_test_failure seen"
+    assert jenkins["console_failures"] == []
+    assert jenkins["console_truncated"] is False
     assert bundle["log_evidence"]["records"] == []
     config, incident_id, window_start, window_end = calls["log"][0]
     assert incident_id == INCIDENT_ID
@@ -179,14 +181,60 @@ def test_console_tail_caps_lines_and_bytes(monkeypatch) -> None:
     _install(monkeypatch, routes)
     tail = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"]["console_tail"]
     lines = tail.splitlines()
-    assert len(lines) == 100
-    assert lines[0] == "line-200"
+    assert len(lines) == 40
+    assert lines[0] == "line-260"
     assert lines[-1] == "line-299"
 
     routes["/consoleText"] = FakeResponse(text="x" * 20000)
     _install(monkeypatch, routes)
     tail = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"]["console_tail"]
-    assert len(tail) == 8192
+    assert len(tail) == 4000
+
+
+def _long_console() -> str:
+    head = [
+        "[Pipeline] stage: Unit Tests",
+        "+ python -m pytest tests -q",
+        "=================================== FAILURES ===================================",
+        "E       assert 0 == 100",
+        "FAILED tests/test_ledger.py::test_balance",
+    ]
+    noise = [f"[Pipeline] teardown {i} completed ok" for i in range(2000)]
+    tail = [
+        "java.io.IOException: Failed to archive artifacts",
+        "ERROR: script returned exit code 1",
+        "Finished: FAILURE",
+    ]
+    return "\n".join(head + noise + tail)
+
+
+def test_console_failure_regions_capture_the_early_failure(monkeypatch) -> None:
+    routes = _all_failing_routes()
+    routes["/consoleText"] = FakeResponse(text=_long_console())
+    _install(monkeypatch, routes)
+    jenkins = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"]
+
+    regions = jenkins["console_failures"]
+    assert regions
+    assert "FAILED tests/test_ledger.py::test_balance" in regions[0]["text"]
+    assert regions[0]["line_number"] <= 5
+    assert set(regions[0]) == {"marker", "line_number", "text"}
+    assert "FAILED tests/test_ledger.py::test_balance" not in jenkins["console_tail"]
+    assert "ERROR: script returned exit code 1" in jenkins["console_tail"]
+    assert jenkins["console_truncated"] is False
+
+
+def test_console_read_is_bounded_but_keeps_head_and_tail(monkeypatch) -> None:
+    text = "\n".join(
+        ["ERROR: early boom"] + ["filler line that is long enough to add up" * 3] * 60000 + ["ERROR: late boom"]
+    )
+    assert len(text) > module._CONSOLE_MAX_CHARS
+    routes = _all_failing_routes()
+    routes["/consoleText"] = FakeResponse(text=text)
+    _install(monkeypatch, routes)
+    jenkins = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"]
+    assert "ERROR: early boom" in jenkins["console_failures"][0]["text"]
+    assert "ERROR: late boom" in jenkins["console_tail"]
 
 
 def test_failed_tests_are_capped_and_error_details_truncated(monkeypatch) -> None:
@@ -234,13 +282,28 @@ def test_evidence_tolerates_odd_jenkins_payloads(monkeypatch) -> None:
     assert jenkins["failed_tests"] == [{"name": "t", "className": "c", "errorDetails": None}]
 
 
-def _bundle(failed_tests=(), console_tail="", records=()):  # type: ignore[no-untyped-def]
+def _bundle(failed_tests=(), console_tail="", records=(), console_failures=()):  # type: ignore[no-untyped-def]
     return {
-        "jenkins": {"failed_tests": list(failed_tests), "console_tail": console_tail},
+        "jenkins": {
+            "failed_tests": list(failed_tests),
+            "console_tail": console_tail,
+            "console_failures": list(console_failures),
+        },
         "log_evidence": {"records": list(records)},
     }
 
 
+def test_signature_from_console_failure_region() -> None:
+    bundle = _bundle(
+        console_tail="ERROR: script returned exit code 1",
+        console_failures=[
+            {"marker": "FAILED ", "line_number": 12, "text": "FAILED hermes_demo_test_failure check"},
+            "not-a-region",
+        ],
+    )
+    assert module.evidence_has_signature(bundle, INCIDENT_ID) is True
+
+
 def test_signature_from_failed_test_name() -> None:
     bundle = _bundle(failed_tests=[{"name": "fixture-state-check", "className": "c", "errorDetails": None}])
     assert module.evidence_has_signature(bundle, INCIDENT_ID) is True

tests/test_hermes_code_flow.py

@@ -81,8 +81,20 @@ def _run(status: str = "completed", output=None) -> HermesRunResult:  # type: ig
     )
 
 
-def _install(monkeypatch, *, fetch=None, run=None, push=None, pull=None) -> dict:  # type: ignore[no-untyped-def]
-    calls: dict = {"fetches": [], "runs": [], "pushes": [], "pulls": []}
+def _no_existing(**overrides) -> dict:  # type: ignore[no-untyped-def]
+    base = {"found": False, "pr_number": None, "url": None, "branch": None, "error": None}
+    base.update(overrides)
+    return base
+
+
+def _install(monkeypatch, *, fetch=None, run=None, push=None, pull=None, existing=None) -> dict:  # type: ignore[no-untyped-def]
+    calls: dict = {"fetches": [], "runs": [], "pushes": [], "pulls": [], "lookups": []}
+
+    def fake_find(cfg):  # type: ignore[no-untyped-def]
+        calls["lookups"].append(cfg)
+        return existing if existing is not None else _no_existing()
+
+    monkeypatch.setattr(module.hermes_code_repair, "find_open_proposal", fake_find)
 
     def fake_fetch(cfg, path):  # type: ignore[no-untyped-def]
         calls["fetches"].append((cfg, path))
@@ -155,6 +167,50 @@ def test_happy_path_opens_pull_request(monkeypatch) -> None:
     assert "secret-token" not in serialized
 
 
+def test_existing_open_proposal_suppresses_duplicate(monkeypatch) -> None:
+    existing = {
+        "found": True,
+        "pr_number": 1,
+        "url": "https://scm.example/pulls/1",
+        "branch": "hermes-repair/4",
+        "error": None,
+    }
+    storage, calls, result = _propose(monkeypatch, existing=existing)
+    assert result == {
+        "status": "human_required",
+        "reason": "existing_proposal_open",
+        "pr_number": 1,
+        "url": "https://scm.example/pulls/1",
+        "branch": "hermes-repair/4",
+    }
+    assert calls["lookups"] == [_code_cfg()]
+    assert calls["fetches"] == []
+    assert calls["runs"] == []
+    assert calls["pushes"] == []
+    assert calls["pulls"] == []
+    assert _event(storage) == {
+        "incident_id": INCIDENT_ID,
+        "job": JOB,
+        "build_number": 7,
+        "run_id": None,
+        "validated": False,
+        "reject_reason": "existing_proposal_open",
+        "branch": "hermes-repair/4",
+        "pr_number": 1,
+        "url": "https://scm.example/pulls/1",
+    }
+
+
+def test_proposal_lookup_error_fails_open(monkeypatch) -> None:
+    storage, calls, result = _propose(
+        monkeypatch, existing=_no_existing(error="open proposal lookup http 503")
+    )
+    assert result["status"] == "pr_opened"
+    assert len(calls["fetches"]) == 1
+    assert len(calls["runs"]) == 1
+    assert _event(storage)["validated"] is True
+
+
 def test_prompt_is_frozen_shape(monkeypatch) -> None:
     _, calls, _ = _propose(monkeypatch)
     cfg, prompt = calls["runs"][0]

tests/test_hermes_code_repair.py

@@ -123,6 +123,96 @@ def test_fetch_file_uses_default_timeout(monkeypatch) -> None:
     assert calls["kwargs"] == {"timeout": 15.0}
 
 
+def _pull(number: int, head: str = BRANCH, base: str = "master") -> dict:
+    return {
+        "number": number,
+        "html_url": f"https://scm.example/pulls/{number}",
+        "head": {"ref": head},
+        "base": {"ref": base},
+    }
+
+
+def _none_found(error=None) -> dict:  # type: ignore[no-untyped-def]
+    return {"found": False, "pr_number": None, "url": None, "branch": None, "error": error}
+
+
+def test_find_open_proposal_matches_repair_branch(monkeypatch) -> None:
+    calls = _install_http(monkeypatch, [FakeResponse(200, [_pull(2)])])
+    result = module.find_open_proposal(_cfg())
+    assert result == {
+        "found": True,
+        "pr_number": 2,
+        "url": "https://scm.example/pulls/2",
+        "branch": BRANCH,
+        "error": None,
+    }
+    method, url, kwargs = calls["requests"][0]
+    assert (method, url) == ("GET", "https://scm.example/api/v1/repos/bstein/hermes-code-demo/pulls")
+    assert kwargs["params"] == {"state": "open", "limit": 50}
+    assert kwargs["headers"] == {"Authorization": "token secret-token"}
+
+
+def test_find_open_proposal_ignores_other_branch_prefixes(monkeypatch) -> None:
+    _install_http(monkeypatch, [FakeResponse(200, [_pull(3, head="feature/thing"), _pull(4, head="renovate/x")])])
+    assert module.find_open_proposal(_cfg()) == _none_found()
+
+
+def test_find_open_proposal_ignores_wrong_base_branch(monkeypatch) -> None:
+    _install_http(monkeypatch, [FakeResponse(200, [_pull(3, base="develop")])])
+    assert module.find_open_proposal(_cfg()) == _none_found()
+
+
+def test_find_open_proposal_returns_lowest_numbered_match(monkeypatch) -> None:
+    payload = [
+        _pull(9, head="hermes-repair/9"),
+        _pull(2, head="hermes-repair/4"),
+        _pull(1, head="other/1"),
+        _pull(5, head="hermes-repair/5"),
+    ]
+    _install_http(monkeypatch, [FakeResponse(200, payload)])
+    result = module.find_open_proposal(_cfg())
+    assert (result["found"], result["pr_number"], result["branch"]) == (True, 2, "hermes-repair/4")
+
+
+def test_find_open_proposal_skips_malformed_entries(monkeypatch) -> None:
+    payload = ["not a dict", {"number": 1}, {"number": True, "head": {"ref": BRANCH}, "base": {"ref": "master"}}]
+    _install_http(monkeypatch, [FakeResponse(200, payload)])
+    assert module.find_open_proposal(_cfg()) == _none_found()
+
+
+def test_find_open_proposal_empty_list(monkeypatch) -> None:
+    _install_http(monkeypatch, [FakeResponse(200, [])])
+    assert module.find_open_proposal(_cfg()) == _none_found()
+
+
+def test_find_open_proposal_http_error_fails_open(monkeypatch) -> None:
+    _install_http(monkeypatch, [FakeResponse(503)])
+    assert module.find_open_proposal(_cfg()) == _none_found("open proposal lookup http 503")
+
+
+def test_find_open_proposal_malformed_json_fails_open(monkeypatch) -> None:
+    _install_http(monkeypatch, [FakeResponse(200)])
+    result = module.find_open_proposal(_cfg())
+    assert result["found"] is False
+    assert result["error"] == "open proposal parse failed: no json body"
+
+
+def test_find_open_proposal_non_list_payload_fails_open(monkeypatch) -> None:
+    _install_http(monkeypatch, [FakeResponse(200, {"message": "nope"})])
+    assert module.find_open_proposal(_cfg()) == _none_found("open proposal payload is not a list")
+
+
+def test_find_open_proposal_request_exception_fails_open(monkeypatch) -> None:
+    _install_http(monkeypatch, [RuntimeError("connection reset")])
+    assert module.find_open_proposal(_cfg()) == _none_found("open proposal lookup failed: connection reset")
+
+
+def test_find_open_proposal_without_base_url(monkeypatch) -> None:
+    calls = _install_http(monkeypatch, [])
+    assert module.find_open_proposal(_cfg(gitea_base_url="")) == _none_found("gitea base url is empty")
+    assert calls["requests"] == []
+
+
 def test_push_branch_success_with_new_branch_payload(monkeypatch) -> None:
     calls, result = _push(monkeypatch, [FakeResponse(200, {"sha": "abc123"}), FakeResponse(201, {})])
     assert result == {"branch": BRANCH, "committed": True, "error": None}

tests/test_hermes_console_evidence.py

@@ -0,0 +1,300 @@
+from __future__ import annotations
+
+import pytest
+
+from ariadne.services import hermes_console_evidence as module
+
+
+def _console(lines) -> str:  # type: ignore[no-untyped-def]
+    return "\n".join(lines)
+
+
+def _noise(count: int, prefix: str = "step") -> list[str]:
+    return [f"[Pipeline] {prefix} {index} completed ok" for index in range(count)]
+
+
+@pytest.mark.parametrize(
+    ("line", "marker"),
+    [
+        ("=== FAILURES ===", "=== FAILURES ==="),
+        ("=========== short test summary info ==========", "short test summary"),
+        ("ERROR at setup of test_thing", "ERROR at setup"),
+        ("Traceback (most recent call last):", "Traceback (most recent call last)"),
+        ("    raise AssertionError(x)", "AssertionError"),
+        ("FAILED tests/test_a.py::test_b - boom", "FAILED "),
+        ("E       assert 1 == 2", "E   "),
+        ("BUILD FAILED in 3s", "BUILD FAILED"),
+        ("FAILURE: gradle task :check reported problems", "FAILURE:"),
+        ("process exited with non-zero exit status", "non-zero exit"),
+        ("script returned exit code 1", "exit code"),
+        ("bash: pytest: command not found", "command not found"),
+        ("cp: No such file or directory", "No such file"),
+        ("java.io.IOException: Exception while archiving", "Exception"),
+        ("ERROR: something broke", "ERROR:"),
+        ("error: lowercase also matches", "ERROR:"),
+        ("Quality gate failed for project", "Quality gate"),
+        ("Required test coverage of 80% not reached", "coverage"),
+        ("FAIL Required test --fail-under=80", "fail-under"),
+        ("missing docstring on public function", "docstring"),
+        ("Semgrep found 3 findings", "Semgrep"),
+        ("SonarQube analysis failed", "SonarQube"),
+        ("Trivy detected HIGH severity", "Trivy"),
+        ("ruff check found 2 problems", "ruff"),
+        ("[loc] file exceeds 500 lines", "[loc]"),
+        ("pod status ImagePullBackOff", "ImagePullBackOff"),
+        ("container was OOMKilled", "OOMKilled"),
+        (
+            "Failed to establish a new connection: [Errno 111]",
+            "Failed to establish a new connection",
+        ),
+        (
+            "Temporary failure in name resolution for jenkins",
+            "Temporary failure in name resolution",
+        ),
+    ],
+)
+def test_marker_families_are_detected(line: str, marker: str) -> None:
+    assert module.marker_for_line(line) == marker
+    result = module.extract_console_evidence(line)
+    assert result["regions"][0]["marker"] == marker
+    assert result["regions"][0]["line_number"] == 1
+
+
+def test_clean_lines_produce_no_regions() -> None:
+    result = module.extract_console_evidence(_console(_noise(20)))
+    assert result["regions"] == []
+    assert result["truncated"] is False
+    assert result["total_lines"] == 20
+
+
+def test_prefix_only_marker_does_not_match_mid_line() -> None:
+    assert module.marker_for_line("phase   done   nothing wrong here") is None
+    assert module.marker_for_line("   E   assert False") == "E   "
+
+
+def test_prefix_only_failed_marker_ignores_prose() -> None:
+    assert module.marker_for_line("Quality gate FAILED for project") == "Quality gate"
+    assert module.marker_for_line("  FAILED tests/test_a.py::test_b") == "FAILED "
+
+
+def test_priority_order_prefers_pytest_signal() -> None:
+    assert module.marker_for_line("ERROR: raised AssertionError while loading") == "AssertionError"
+
+
+def test_context_window_before_and_after() -> None:
+    lines = [f"line-{index}" for index in range(60)]
+    lines[30] = "ERROR: boom"
+    result = module.extract_console_evidence(_console(lines))
+    region = result["regions"][0]
+    assert region["line_number"] == 31
+    text = region["text"].splitlines()
+    assert text[0] == "line-24"
+    assert text[-1] == "line-42"
+    assert len(text) == 19
+
+
+def test_context_window_is_configurable_and_clamped_at_edges() -> None:
+    lines = ["ERROR: boom"] + [f"line-{index}" for index in range(3)]
+    result = module.extract_console_evidence(
+        _console(lines), {"context_before": 2, "context_after": 1}
+    )
+    assert result["regions"][0]["text"] == "ERROR: boom\nline-0"
+
+
+def test_overlapping_regions_are_merged() -> None:
+    lines = [f"line-{index}" for index in range(40)]
+    lines[10] = "ERROR: first"
+    lines[15] = "ERROR: second"
+    result = module.extract_console_evidence(_console(lines))
+    assert len(result["regions"]) == 1
+    region = result["regions"][0]
+    assert region["line_number"] == 11
+    assert "ERROR: first" in region["text"]
+    assert "ERROR: second" in region["text"]
+    assert region["text"].splitlines()[-1] == "line-27"
+
+
+def test_distant_regions_are_kept_separate() -> None:
+    lines = [f"line-{index}" for index in range(200)]
+    lines[10] = "ERROR: first"
+    lines[150] = "ERROR: second"
+    result = module.extract_console_evidence(_console(lines))
+    assert [region["line_number"] for region in result["regions"]] == [11, 151]
+
+
+def test_repeated_regions_are_deduplicated_with_a_note() -> None:
+    block = ["connecting to registry", "ERROR: connection reset by peer (attempt 1)", "retrying"]
+    lines: list[str] = []
+    for attempt in range(5):
+        lines.extend(_noise(30, "pad"))
+        lines.extend([block[0], block[1].replace("1", str(attempt + 1)), block[2]])
+    lines.extend(_noise(30, "pad"))
+    result = module.extract_console_evidence(_console(lines))
+    assert len(result["regions"]) == 1
+    assert result["regions"][0]["text"].endswith("(repeated 5 times)")
+    assert result["regions"][0]["line_number"] == 32
+
+
+def test_distinct_regions_are_not_deduplicated() -> None:
+    lines = _noise(30) + ["ERROR: alpha broke"] + _noise(30, "mid") + ["ERROR: beta broke"]
+    result = module.extract_console_evidence(_console(lines))
+    assert len(result["regions"]) == 2
+    assert "repeated" not in result["regions"][1]["text"]
+
+
+def test_byte_budget_prefers_earliest_regions() -> None:
+    lines: list[str] = []
+    for index in range(6):
+        lines.extend(_noise(40, "pad"))
+        lines.append(f"ERROR: distinct failure {chr(ord('a') + index)} " + "x" * 400)
+    result = module.extract_console_evidence(_console(lines), {"max_total_bytes": 2000})
+    assert result["truncated"] is True
+    markers = [region["line_number"] for region in result["regions"]]
+    assert markers == sorted(markers)
+    assert result["regions"][0]["line_number"] == 41
+    assert "distinct failure a" in result["regions"][0]["text"]
+    assert len(result["regions"]) < 6
+    assert sum(len(region["text"]) for region in result["regions"]) <= 2000
+
+
+def test_first_region_is_kept_and_clipped_when_it_alone_exceeds_budget() -> None:
+    lines = ["ERROR: huge " + "y" * 5000, "ERROR: later " + "z" * 5000]
+    result = module.extract_console_evidence(_console(lines), {"max_total_bytes": 100})
+    assert len(result["regions"]) == 1
+    assert len(result["regions"][0]["text"]) == 100
+    assert result["truncated"] is True
+
+
+def test_zero_byte_budget_still_reports_one_empty_region() -> None:
+    result = module.extract_console_evidence("ERROR: boom", {"max_total_bytes": 0})
+    assert result["regions"] == [{"marker": "ERROR:", "line_number": 1, "text": ""}]
+    assert result["truncated"] is True
+
+
+def test_max_regions_caps_the_region_count() -> None:
+    lines: list[str] = []
+    for index in range(8):
+        lines.extend(_noise(40, "pad"))
+        lines.append(f"ERROR: unique failure {chr(ord('a') + index)}")
+    result = module.extract_console_evidence(_console(lines), {"max_regions": 3})
+    assert len(result["regions"]) == 3
+    assert result["truncated"] is True
+    assert "ERROR: unique failure a" in result["regions"][0]["text"]
+
+
+def test_all_regions_within_budget_are_not_truncated() -> None:
+    lines = _noise(30) + ["ERROR: only one"] + _noise(30, "post")
+    result = module.extract_console_evidence(_console(lines))
+    assert len(result["regions"]) == 1
+    assert result["truncated"] is False
+
+
+def test_tail_is_capped_by_lines_and_bytes() -> None:
+    lines = [f"line-{index}" for index in range(200)]
+    result = module.extract_console_evidence(_console(lines))
+    tail = result["tail"].splitlines()
+    assert len(tail) == 40
+    assert tail[0] == "line-160"
+    assert tail[-1] == "line-199"
+
+    wide = module.extract_console_evidence(_console(["q" * 500] * 50), {"tail_lines": 20})
+    assert len(wide["tail"]) == 4000
+    assert wide["tail"].endswith("q")
+
+    assert module.extract_console_evidence("a\nb\nc", {"tail_lines": 0})["tail"] == ""
+    assert module.extract_console_evidence("a\nb\nc", {"max_tail_bytes": 0})["tail"] == ""
+
+
+def test_tail_shorter_than_the_window_is_returned_whole() -> None:
+    result = module.extract_console_evidence("a\nb\nc")
+    assert result["tail"] == "a\nb\nc"
+    assert result["total_lines"] == 3
+
+
+def test_empty_and_none_input() -> None:
+    for value in ("", None, 12345, b"bytes"):
+        result = module.extract_console_evidence(value)  # type: ignore[arg-type]
+        assert result == {"regions": [], "tail": "", "total_lines": 0, "truncated": False}
+
+
+def test_invalid_cfg_values_fall_back_to_defaults() -> None:
+    lines = [f"line-{index}" for index in range(200)]
+    cfg = {"tail_lines": "not-an-int", "context_before": -5, "max_regions": None}
+    result = module.extract_console_evidence(_console(lines), cfg)
+    assert len(result["tail"].splitlines()) == 40
+
+    numeric = module.extract_console_evidence(_console(lines), {"tail_lines": "7"})
+    assert len(numeric["tail"].splitlines()) == 7
+
+    assert module.extract_console_evidence(_console(lines), "not-a-dict")["total_lines"] == 200  # type: ignore[arg-type]
+
+
+def test_extractor_never_raises(monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    def boom(lines, options):  # type: ignore[no-untyped-def]
+        raise RuntimeError("kaboom")
+
+    monkeypatch.setattr(module, "_regions", boom)
+    assert module.extract_console_evidence("ERROR: boom") == {
+        "regions": [],
+        "tail": "",
+        "total_lines": 0,
+        "truncated": False,
+    }
+
+
+def _long_pipeline_console() -> str:
+    lines: list[str] = ["Started by user jenkins", "[Pipeline] node", "Running on agent-1"]
+    lines.extend(_noise(120, "checkout"))
+    lines.extend(
+        [
+            "[Pipeline] stage: Unit Tests",
+            "+ python -m pytest tests -q",
+            "tests/test_wallet.py .........",
+            "tests/test_ledger.py ..F",
+            "=================================== FAILURES ===================================",
+            "____________________ test_ledger_balance_after_migration _______________________",
+            "    def test_ledger_balance_after_migration():",
+            "        balance = ledger.balance('acct-7')",
+            ">       assert balance == 100",
+            "E       assert 0 == 100",
+            "tests/test_ledger.py:42: AssertionError",
+            "FAILED tests/test_ledger.py::test_ledger_balance_after_migration",
+        ]
+    )
+    lines.extend(_noise(3000, "teardown"))
+    lines.extend(
+        [
+            "[Pipeline] stage: Archive",
+            "java.io.IOException: Failed to archive artifacts",
+            "    at hudson.FilePath.copyRecursiveTo(FilePath.java:2810)",
+            "[Pipeline] End of Pipeline",
+            "ERROR: script returned exit code 1",
+            "Finished: FAILURE",
+        ]
+    )
+    return _console(lines)
+
+
+def test_long_pipeline_keeps_the_early_failure_that_the_tail_loses() -> None:
+    console = _long_pipeline_console()
+    result = module.extract_console_evidence(console)
+
+    assert result["total_lines"] > 3000
+    assert "FAILED tests/test_ledger.py::test_ledger_balance_after_migration" not in result["tail"]
+    assert "script returned exit code 1" in result["tail"]
+
+    first = result["regions"][0]
+    assert first["marker"] == "=== FAILURES ==="
+    assert "test_ledger_balance_after_migration" in first["text"]
+    assert "assert 0 == 100" in first["text"]
+    assert "FAILED tests/test_ledger.py::test_ledger_balance_after_migration" in first["text"]
+    assert first["line_number"] < 200
+    assert sum(len(region["text"]) for region in result["regions"]) <= 12000
+
+
+def test_long_pipeline_regions_stay_chronological_and_reach_the_end() -> None:
+    result = module.extract_console_evidence(_long_pipeline_console())
+    numbers = [region["line_number"] for region in result["regions"]]
+    assert numbers == sorted(numbers)
+    assert len(numbers) >= 2
+    assert "script returned exit code 1" in result["regions"][-1]["text"]