feat(hermes): show the failure in the issue instead of citing where it is

Issue #2 on bstein/ariadne cites its evidence as "console_failures marker
'=== FAILURES ===' at line 1949". That is precise and unusable: the
maintainer has to open Jenkins, find build 408, scroll a 2000-line console and
reconstruct what the diagnosis had already read. The whole point of filing an
issue is that someone can act on it without doing that.

The traceback was not merely unrendered - it was never collected. The junit
query asked for errorDetails and not errorStackTrace, so the bundle carried
the assertion that failed but not the line it failed on, and no amount of
rendering would have found it. Both halves are fixed: the trace is fetched and
bounded, and the issue now opens an Evidence section with it.

Console regions are the fallback for a build that published no test results,
clipped to their last lines because the tail of a failure region holds the
failure while the head only approaches it. A fence inside an excerpt is
escaped so raw log text cannot break out of the code block and spill into the
rendered page.

Bounded deliberately. The body has a hard character budget, and an issue that
spends it on console noise buries the one sentence saying why a person is
needed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

ariadne/services/hermes_autotriage_evidence.py

@@ -18,8 +18,11 @@ _CONSOLE_TAIL_KEEP_CHARS = 200_000
 _CONSOLE_TRUNCATION_NOTE = "... [ariadne] console truncated in the middle ..."
 _MAX_FAILED_TESTS = 10
 _MAX_ERROR_DETAILS_CHARS = 2000
+# The traceback is what a maintainer reads first. errorDetails is only the
+# assertion line, which says what was wrong but not where.
+_MAX_STACK_TRACE_CHARS = 4000
 _FAILED_TEST_STATUSES = {"FAILED", "REGRESSION"}
-_TEST_REPORT_TREE = "suites[cases[className,name,status,errorDetails]]"
+_TEST_REPORT_TREE = "suites[cases[className,name,status,errorDetails,errorStackTrace]]"
 # Jenkins agent pods are where CI failures actually happen, so this is the
 # namespace that matters for every job and is always included.
 _LOG_EXTRA_NAMESPACES = ("jenkins",)
@@ -167,10 +170,12 @@ def _failed_test(case: dict[str, Any]) -> dict[str, Any]:
     """Map one test-report case to the bounded failed-test shape."""
 
     details = case.get("errorDetails")
+    stack = case.get("errorStackTrace")
     return {
         "name": str(case.get("name") or ""),
         "className": str(case.get("className") or ""),
         "errorDetails": details[:_MAX_ERROR_DETAILS_CHARS] if isinstance(details, str) else None,
+        "errorStackTrace": stack[:_MAX_STACK_TRACE_CHARS] if isinstance(stack, str) else None,
     }
 
 

ariadne/services/hermes_incident_body.py

@@ -15,6 +15,7 @@ from __future__ import annotations
 import re
 from typing import Any
 
+from ariadne.services import hermes_incident_evidence_section as evidence_section
 from ariadne.services import hermes_suggested_remediation as suggestion_field
 
 
@@ -119,6 +120,7 @@ def issue_body(context: dict, max_chars: int = DEFAULT_MAX_BODY_CHARS) -> str:
         _summary_line(context),
         _human_section(context),
         _facts_section(context),
+        evidence_section.evidence_section(context.get("bundle")),
         _inferences_section(context),
         suggestion_field.issue_section(context.get("suggested_remediation")),
         _links_section(context),

ariadne/services/hermes_incident_evidence_section.py

@@ -0,0 +1,113 @@
+"""Show the failure itself in the issue, not just a citation of it.
+
+A diagnosis cites its evidence by location - "console_failures marker
+'=== FAILURES ===' at line 1949". That is precise and completely unusable: the
+maintainer reading the issue has to open Jenkins, find build 408, scroll a
+2000-line console, and reconstruct what the diagnosis already read. The
+evidence was in the bundle the whole time and simply never reached the page.
+
+So the raw excerpt goes in the issue. The traceback first, because it is the
+first thing anyone reads and it names the line the assertion actually failed
+on; the earliest console failure region second, for a build that published no
+test results at all. Everything is fenced and bounded - the body has a hard
+character budget, and an issue that spends all of it on console noise buries
+the one sentence saying why a person is needed.
+
+This is a presentation module. It never decides anything and never fetches
+anything; it renders what the bundle already carries.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+MAX_TRACE_CHARS = 2400
+MAX_REGION_CHARS = 1200
+MAX_REGION_LINES = 24
+
+_HEADING = "## Evidence"
+_TRACE_INTRO = "The failing test's traceback, as the build reported it:"
+_REGION_INTRO = "The earliest failure region of the build console:"
+_TRUNCATED = "\n... (truncated; the full log is in the linked build)"
+# A fenced block containing a stray fence would end early and spill raw log
+# text into the rendered issue.
+_FENCE = "```"
+_FENCE_ESCAPE = "`​``"
+
+
+def evidence_section(bundle: Any) -> str:
+    """Render the raw failure excerpt for the incident issue body.
+
+    Inputs: the evidence bundle collected for the incident. Outputs: the
+    markdown section, or "" when the bundle carries nothing worth showing -
+    an escalation with no console and no test results has nothing to quote,
+    and an empty heading is worse than no heading.
+    """
+
+    try:
+        jenkins = _jenkins(bundle)
+        trace = _first_trace(jenkins)
+        if trace:
+            return "\n".join([_HEADING, _TRACE_INTRO, _fenced(trace, MAX_TRACE_CHARS)])
+        region = _first_region(jenkins)
+        if region:
+            return "\n".join([_HEADING, _REGION_INTRO, _fenced(region, MAX_REGION_CHARS)])
+        return ""
+    except Exception:
+        return ""
+
+
+def _jenkins(bundle: Any) -> dict[str, Any]:
+    """Return the bundle's jenkins section, tolerating any shape."""
+
+    if not isinstance(bundle, dict):
+        return {}
+    jenkins = bundle.get("jenkins")
+    return jenkins if isinstance(jenkins, dict) else {}
+
+
+def _first_trace(jenkins: dict[str, Any]) -> str:
+    """Return the first failing test's traceback, or its assertion.
+
+    Prefers the stack trace: the assertion says what was wrong, the trace says
+    where. Falls back to the assertion when the runner published no trace.
+    """
+
+    tests = jenkins.get("failed_tests")
+    for test in tests if isinstance(tests, list) else []:
+        if not isinstance(test, dict):
+            continue
+        for key in ("errorStackTrace", "errorDetails"):
+            value = test.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+    return ""
+
+
+def _first_region(jenkins: dict[str, Any]) -> str:
+    """Return the earliest console failure region, clipped to its last lines.
+
+    The tail of a failure region holds the failure; the head is usually the
+    approach to it. When the region has to be cut, cutting the front keeps the
+    part worth reading.
+    """
+
+    regions = jenkins.get("console_failures")
+    for region in regions if isinstance(regions, list) else []:
+        if not isinstance(region, dict):
+            continue
+        text = str(region.get("text") or "").strip()
+        if text:
+            lines = text.split("\n")
+            return "\n".join(lines[-MAX_REGION_LINES:])
+    return ""
+
+
+def _fenced(text: str, limit: int) -> str:
+    """Wrap an excerpt in a code fence, clipped to its budget."""
+
+    body = text.replace(_FENCE, _FENCE_ESCAPE)
+    if len(body) > limit:
+        body = body[:limit].rstrip() + _TRUNCATED
+    return f"{_FENCE}\n{body}\n{_FENCE}"

ariadne/services/hermes_incident_issue.py

@@ -203,6 +203,9 @@ def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str,
         "facts": [body.fact_fields(fact) for fact in getattr(decision, "facts", None) or []],
         "inferences": list(getattr(decision, "inferences", None) or []),
         "suggested_remediation": getattr(decision, "suggested_remediation", None),
+        # The raw excerpt, so the issue shows the failure instead of citing
+        # where in Jenkins the failure can be found.
+        "bundle": bundle,
         "authorize_reason": authorize_reason,
         # An escalation that never reached a model has no cited facts, so its
         # console text is the whole explanation and must not be dropped.

tests/test_hermes_autotriage_evidence.py

@@ -134,8 +134,13 @@ def test_collect_evidence_full_bundle(monkeypatch) -> None:
     }
     assert jenkins["first_failed_stage"] == "Test"
     assert jenkins["failed_tests"] == [
-        {"name": "fixture-state-check", "className": "demo.Fixture", "errorDetails": "boom"},
-        {"name": "flaky", "className": "demo.Flaky", "errorDetails": None},
+        {
+            "name": "fixture-state-check",
+            "className": "demo.Fixture",
+            "errorDetails": "boom",
+            "errorStackTrace": None,
+        },
+        {"name": "flaky", "className": "demo.Flaky", "errorDetails": None, "errorStackTrace": None},
     ]
     assert jenkins["console_tail"] == "line1\nline2\nhermes_demo_test_failure seen"
     assert jenkins["console_failures"] == []
@@ -279,7 +284,9 @@ def test_evidence_tolerates_odd_jenkins_payloads(monkeypatch) -> None:
     jenkins = bundle["jenkins"]
     assert jenkins["build_number"] == 0
     assert jenkins["first_failed_stage"] is None
-    assert jenkins["failed_tests"] == [{"name": "t", "className": "c", "errorDetails": None}]
+    assert jenkins["failed_tests"] == [
+        {"name": "t", "className": "c", "errorDetails": None, "errorStackTrace": None}
+    ]
 
 
 def _bundle(failed_tests=(), console_tail="", records=(), console_failures=()):  # type: ignore[no-untyped-def]
@@ -372,3 +379,25 @@ def test_an_unmapped_job_adds_nothing(monkeypatch) -> None:
 def test_a_namespace_is_never_duplicated(monkeypatch) -> None:
     monkeypatch.setattr(module, "settings", _settings_with({"j": "jenkins"}))
     assert module._log_config("j")["extra_namespaces"].count("jenkins") == 1
+
+
+
+def test_the_stack_trace_is_collected_and_bounded(monkeypatch) -> None:
+    """The traceback names the line; the assertion alone does not."""
+
+    case = {
+        "name": "t",
+        "className": "c",
+        "status": "FAILED",
+        "errorDetails": "AssertionError",
+        "errorStackTrace": "T" * 9000,
+    }
+    routes = _all_failing_routes()
+    routes["/testReport/api/json"] = FakeResponse({"suites": [{"cases": [case]}]})
+    calls = _install(monkeypatch, routes)
+
+    failed = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"]["failed_tests"]
+
+    assert len(failed[0]["errorStackTrace"]) == module._MAX_STACK_TRACE_CHARS
+    trees = [str(params.get("tree", "")) for _url, params in calls["gets"] if params]
+    assert any("errorStackTrace" in tree for tree in trees)

tests/test_hermes_incident_evidence_section.py

@@ -0,0 +1,155 @@
+"""Tests for showing the failure itself in a triage issue."""
+
+from __future__ import annotations
+
+import pytest
+
+from ariadne.services import hermes_incident_evidence_section as module
+from ariadne.services import hermes_incident_body as body
+
+
+TRACE = (
+    "tests/test_utils.py:212: in test_safe_error_detail\n"
+    "    assert 'bad things' in safe_error_detail(exc)\n"
+    "E   AssertionError: assert 'bad things' in 'http 400'"
+)
+
+
+def _bundle(**jenkins):
+    return {"jenkins": jenkins}
+
+
+def test_the_traceback_is_shown_not_merely_cited() -> None:
+    """Citing a console line number makes a maintainer reconstruct the failure."""
+
+    section = module.evidence_section(
+        _bundle(failed_tests=[{"name": "t", "errorStackTrace": TRACE}])
+    )
+
+    assert section.startswith("## Evidence")
+    assert "The failing test's traceback" in section
+    assert "AssertionError: assert 'bad things' in 'http 400'" in section
+    assert section.count("```") == 2
+
+
+def test_the_stack_trace_is_preferred_over_the_assertion_line() -> None:
+    """The assertion says what was wrong; the trace says where."""
+
+    section = module.evidence_section(
+        _bundle(failed_tests=[{"errorDetails": "AssertionError: nope", "errorStackTrace": TRACE}])
+    )
+
+    assert "tests/test_utils.py:212" in section
+    assert "AssertionError: nope" not in section
+
+
+def test_the_assertion_is_used_when_no_trace_was_published() -> None:
+    section = module.evidence_section(
+        _bundle(failed_tests=[{"errorDetails": "AssertionError: nope"}])
+    )
+
+    assert "AssertionError: nope" in section
+
+
+def test_a_test_without_either_field_is_skipped() -> None:
+    section = module.evidence_section(
+        _bundle(
+            failed_tests=[
+                {"name": "a"},
+                {"name": "b", "errorStackTrace": "   "},
+                "not-a-dict",
+                {"name": "c", "errorStackTrace": TRACE},
+            ]
+        )
+    )
+
+    assert "tests/test_utils.py:212" in section
+
+
+def test_the_console_region_is_used_when_no_tests_were_published() -> None:
+    """A build whose runner published nothing still has a console to quote."""
+
+    section = module.evidence_section(
+        _bundle(console_failures=[{"text": "Traceback\n  File x\nValueError: boom"}])
+    )
+
+    assert "The earliest failure region" in section
+    assert "ValueError: boom" in section
+
+
+def test_the_region_keeps_its_last_lines_when_it_is_long() -> None:
+    """The tail of a failure region holds the failure; the head approaches it."""
+
+    text = "\n".join([f"line {i}" for i in range(100)] + ["ValueError: boom"])
+    section = module.evidence_section(_bundle(console_failures=[{"text": text}]))
+
+    assert "ValueError: boom" in section
+    assert "line 0" not in section
+
+
+def test_a_long_trace_is_truncated_with_a_pointer_to_the_build() -> None:
+    section = module.evidence_section(
+        _bundle(failed_tests=[{"errorStackTrace": "x" * 9000}])
+    )
+
+    assert len(section) < 3000
+    assert "truncated" in section
+    assert section.rstrip().endswith("```")
+
+
+def test_a_fence_inside_the_excerpt_cannot_end_the_block_early() -> None:
+    """Raw log text must not be able to break out into the rendered issue."""
+
+    section = module.evidence_section(
+        _bundle(failed_tests=[{"errorStackTrace": "before\n```\nafter"}])
+    )
+
+    assert section.count("```") == 2
+    assert "after" in section
+
+
+@pytest.mark.parametrize(
+    "bundle",
+    [
+        None,
+        {},
+        "not-a-dict",
+        {"jenkins": "not-a-dict"},
+        {"jenkins": {}},
+        {"jenkins": {"failed_tests": [], "console_failures": []}},
+        {"jenkins": {"failed_tests": None, "console_failures": None}},
+        {"jenkins": {"console_failures": [{"text": "   "}, "not-a-dict"]}},
+    ],
+)
+def test_a_bundle_with_nothing_to_quote_renders_no_heading(bundle) -> None:
+    """An empty Evidence heading is worse than no heading."""
+
+    assert module.evidence_section(bundle) == ""
+
+
+def test_the_issue_body_carries_the_excerpt_and_keeps_its_marker() -> None:
+    rendered = body.issue_body(
+        {
+            "incident_id": "ariadne/408",
+            "job": "ariadne",
+            "build_number": 408,
+            "classification": "pytest_test_failure",
+            "reason": "a repository test failure",
+            "run_id": "run-1",
+            "bundle": _bundle(failed_tests=[{"errorStackTrace": TRACE}]),
+        }
+    )
+
+    assert "## Evidence" in rendered
+    assert "tests/test_utils.py:212" in rendered
+    assert rendered.index("## Evidence") < rendered.index("## Links")
+    assert rendered.rstrip().endswith("-->")
+
+
+def test_an_issue_without_a_bundle_still_renders() -> None:
+    rendered = body.issue_body(
+        {"incident_id": "a/1", "job": "a", "build_number": 1, "reason": "r", "run_id": "x"}
+    )
+
+    assert "## Evidence" not in rendered
+    assert rendered.rstrip().endswith("-->")