diff --git a/ariadne/services/hermes_autotriage_evidence.py b/ariadne/services/hermes_autotriage_evidence.py index 3ac8859..88e00a6 100644 --- a/ariadne/services/hermes_autotriage_evidence.py +++ b/ariadne/services/hermes_autotriage_evidence.py @@ -18,8 +18,11 @@ _CONSOLE_TAIL_KEEP_CHARS = 200_000 _CONSOLE_TRUNCATION_NOTE = "... [ariadne] console truncated in the middle ..." _MAX_FAILED_TESTS = 10 _MAX_ERROR_DETAILS_CHARS = 2000 +# The traceback is what a maintainer reads first. errorDetails is only the +# assertion line, which says what was wrong but not where. +_MAX_STACK_TRACE_CHARS = 4000 _FAILED_TEST_STATUSES = {"FAILED", "REGRESSION"} -_TEST_REPORT_TREE = "suites[cases[className,name,status,errorDetails]]" +_TEST_REPORT_TREE = "suites[cases[className,name,status,errorDetails,errorStackTrace]]" # Jenkins agent pods are where CI failures actually happen, so this is the # namespace that matters for every job and is always included. _LOG_EXTRA_NAMESPACES = ("jenkins",) @@ -167,10 +170,12 @@ def _failed_test(case: dict[str, Any]) -> dict[str, Any]: """Map one test-report case to the bounded failed-test shape.""" details = case.get("errorDetails") + stack = case.get("errorStackTrace") return { "name": str(case.get("name") or ""), "className": str(case.get("className") or ""), "errorDetails": details[:_MAX_ERROR_DETAILS_CHARS] if isinstance(details, str) else None, + "errorStackTrace": stack[:_MAX_STACK_TRACE_CHARS] if isinstance(stack, str) else None, } diff --git a/ariadne/services/hermes_incident_body.py b/ariadne/services/hermes_incident_body.py index 661fa6e..1abbe59 100644 --- a/ariadne/services/hermes_incident_body.py +++ b/ariadne/services/hermes_incident_body.py @@ -15,6 +15,7 @@ from __future__ import annotations import re from typing import Any +from ariadne.services import hermes_incident_evidence_section as evidence_section from ariadne.services import hermes_suggested_remediation as suggestion_field @@ -119,6 +120,7 @@ def issue_body(context: dict, max_chars: int = DEFAULT_MAX_BODY_CHARS) -> str: _summary_line(context), _human_section(context), _facts_section(context), + evidence_section.evidence_section(context.get("bundle")), _inferences_section(context), suggestion_field.issue_section(context.get("suggested_remediation")), _links_section(context), diff --git a/ariadne/services/hermes_incident_evidence_section.py b/ariadne/services/hermes_incident_evidence_section.py new file mode 100644 index 0000000..2081fa5 --- /dev/null +++ b/ariadne/services/hermes_incident_evidence_section.py @@ -0,0 +1,113 @@ +"""Show the failure itself in the issue, not just a citation of it. + +A diagnosis cites its evidence by location - "console_failures marker +'=== FAILURES ===' at line 1949". That is precise and completely unusable: the +maintainer reading the issue has to open Jenkins, find build 408, scroll a +2000-line console, and reconstruct what the diagnosis already read. The +evidence was in the bundle the whole time and simply never reached the page. + +So the raw excerpt goes in the issue. The traceback first, because it is the +first thing anyone reads and it names the line the assertion actually failed +on; the earliest console failure region second, for a build that published no +test results at all. Everything is fenced and bounded - the body has a hard +character budget, and an issue that spends all of it on console noise buries +the one sentence saying why a person is needed. + +This is a presentation module. It never decides anything and never fetches +anything; it renders what the bundle already carries. +""" + +from __future__ import annotations + +from typing import Any + + +MAX_TRACE_CHARS = 2400 +MAX_REGION_CHARS = 1200 +MAX_REGION_LINES = 24 + +_HEADING = "## Evidence" +_TRACE_INTRO = "The failing test's traceback, as the build reported it:" +_REGION_INTRO = "The earliest failure region of the build console:" +_TRUNCATED = "\n... (truncated; the full log is in the linked build)" +# A fenced block containing a stray fence would end early and spill raw log +# text into the rendered issue. +_FENCE = "```" +_FENCE_ESCAPE = "`​``" + + +def evidence_section(bundle: Any) -> str: + """Render the raw failure excerpt for the incident issue body. + + Inputs: the evidence bundle collected for the incident. Outputs: the + markdown section, or "" when the bundle carries nothing worth showing - + an escalation with no console and no test results has nothing to quote, + and an empty heading is worse than no heading. + """ + + try: + jenkins = _jenkins(bundle) + trace = _first_trace(jenkins) + if trace: + return "\n".join([_HEADING, _TRACE_INTRO, _fenced(trace, MAX_TRACE_CHARS)]) + region = _first_region(jenkins) + if region: + return "\n".join([_HEADING, _REGION_INTRO, _fenced(region, MAX_REGION_CHARS)]) + return "" + except Exception: + return "" + + +def _jenkins(bundle: Any) -> dict[str, Any]: + """Return the bundle's jenkins section, tolerating any shape.""" + + if not isinstance(bundle, dict): + return {} + jenkins = bundle.get("jenkins") + return jenkins if isinstance(jenkins, dict) else {} + + +def _first_trace(jenkins: dict[str, Any]) -> str: + """Return the first failing test's traceback, or its assertion. + + Prefers the stack trace: the assertion says what was wrong, the trace says + where. Falls back to the assertion when the runner published no trace. + """ + + tests = jenkins.get("failed_tests") + for test in tests if isinstance(tests, list) else []: + if not isinstance(test, dict): + continue + for key in ("errorStackTrace", "errorDetails"): + value = test.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def _first_region(jenkins: dict[str, Any]) -> str: + """Return the earliest console failure region, clipped to its last lines. + + The tail of a failure region holds the failure; the head is usually the + approach to it. When the region has to be cut, cutting the front keeps the + part worth reading. + """ + + regions = jenkins.get("console_failures") + for region in regions if isinstance(regions, list) else []: + if not isinstance(region, dict): + continue + text = str(region.get("text") or "").strip() + if text: + lines = text.split("\n") + return "\n".join(lines[-MAX_REGION_LINES:]) + return "" + + +def _fenced(text: str, limit: int) -> str: + """Wrap an excerpt in a code fence, clipped to its budget.""" + + body = text.replace(_FENCE, _FENCE_ESCAPE) + if len(body) > limit: + body = body[:limit].rstrip() + _TRUNCATED + return f"{_FENCE}\n{body}\n{_FENCE}" diff --git a/ariadne/services/hermes_incident_issue.py b/ariadne/services/hermes_incident_issue.py index 1597984..a31879b 100644 --- a/ariadne/services/hermes_incident_issue.py +++ b/ariadne/services/hermes_incident_issue.py @@ -203,6 +203,9 @@ def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str, "facts": [body.fact_fields(fact) for fact in getattr(decision, "facts", None) or []], "inferences": list(getattr(decision, "inferences", None) or []), "suggested_remediation": getattr(decision, "suggested_remediation", None), + # The raw excerpt, so the issue shows the failure instead of citing + # where in Jenkins the failure can be found. + "bundle": bundle, "authorize_reason": authorize_reason, # An escalation that never reached a model has no cited facts, so its # console text is the whole explanation and must not be dropped. diff --git a/tests/test_hermes_autotriage_evidence.py b/tests/test_hermes_autotriage_evidence.py index 1547565..89cb8d6 100644 --- a/tests/test_hermes_autotriage_evidence.py +++ b/tests/test_hermes_autotriage_evidence.py @@ -134,8 +134,13 @@ def test_collect_evidence_full_bundle(monkeypatch) -> None: } assert jenkins["first_failed_stage"] == "Test" assert jenkins["failed_tests"] == [ - {"name": "fixture-state-check", "className": "demo.Fixture", "errorDetails": "boom"}, - {"name": "flaky", "className": "demo.Flaky", "errorDetails": None}, + { + "name": "fixture-state-check", + "className": "demo.Fixture", + "errorDetails": "boom", + "errorStackTrace": None, + }, + {"name": "flaky", "className": "demo.Flaky", "errorDetails": None, "errorStackTrace": None}, ] assert jenkins["console_tail"] == "line1\nline2\nhermes_demo_test_failure seen" assert jenkins["console_failures"] == [] @@ -279,7 +284,9 @@ def test_evidence_tolerates_odd_jenkins_payloads(monkeypatch) -> None: jenkins = bundle["jenkins"] assert jenkins["build_number"] == 0 assert jenkins["first_failed_stage"] is None - assert jenkins["failed_tests"] == [{"name": "t", "className": "c", "errorDetails": None}] + assert jenkins["failed_tests"] == [ + {"name": "t", "className": "c", "errorDetails": None, "errorStackTrace": None} + ] def _bundle(failed_tests=(), console_tail="", records=(), console_failures=()): # type: ignore[no-untyped-def] @@ -372,3 +379,25 @@ def test_an_unmapped_job_adds_nothing(monkeypatch) -> None: def test_a_namespace_is_never_duplicated(monkeypatch) -> None: monkeypatch.setattr(module, "settings", _settings_with({"j": "jenkins"})) assert module._log_config("j")["extra_namespaces"].count("jenkins") == 1 + + + +def test_the_stack_trace_is_collected_and_bounded(monkeypatch) -> None: + """The traceback names the line; the assertion alone does not.""" + + case = { + "name": "t", + "className": "c", + "status": "FAILED", + "errorDetails": "AssertionError", + "errorStackTrace": "T" * 9000, + } + routes = _all_failing_routes() + routes["/testReport/api/json"] = FakeResponse({"suites": [{"cases": [case]}]}) + calls = _install(monkeypatch, routes) + + failed = module.collect_evidence(INCIDENT_ID, JOB, _last_build())["jenkins"]["failed_tests"] + + assert len(failed[0]["errorStackTrace"]) == module._MAX_STACK_TRACE_CHARS + trees = [str(params.get("tree", "")) for _url, params in calls["gets"] if params] + assert any("errorStackTrace" in tree for tree in trees) diff --git a/tests/test_hermes_incident_evidence_section.py b/tests/test_hermes_incident_evidence_section.py new file mode 100644 index 0000000..1012af9 --- /dev/null +++ b/tests/test_hermes_incident_evidence_section.py @@ -0,0 +1,155 @@ +"""Tests for showing the failure itself in a triage issue.""" + +from __future__ import annotations + +import pytest + +from ariadne.services import hermes_incident_evidence_section as module +from ariadne.services import hermes_incident_body as body + + +TRACE = ( + "tests/test_utils.py:212: in test_safe_error_detail\n" + " assert 'bad things' in safe_error_detail(exc)\n" + "E AssertionError: assert 'bad things' in 'http 400'" +) + + +def _bundle(**jenkins): + return {"jenkins": jenkins} + + +def test_the_traceback_is_shown_not_merely_cited() -> None: + """Citing a console line number makes a maintainer reconstruct the failure.""" + + section = module.evidence_section( + _bundle(failed_tests=[{"name": "t", "errorStackTrace": TRACE}]) + ) + + assert section.startswith("## Evidence") + assert "The failing test's traceback" in section + assert "AssertionError: assert 'bad things' in 'http 400'" in section + assert section.count("```") == 2 + + +def test_the_stack_trace_is_preferred_over_the_assertion_line() -> None: + """The assertion says what was wrong; the trace says where.""" + + section = module.evidence_section( + _bundle(failed_tests=[{"errorDetails": "AssertionError: nope", "errorStackTrace": TRACE}]) + ) + + assert "tests/test_utils.py:212" in section + assert "AssertionError: nope" not in section + + +def test_the_assertion_is_used_when_no_trace_was_published() -> None: + section = module.evidence_section( + _bundle(failed_tests=[{"errorDetails": "AssertionError: nope"}]) + ) + + assert "AssertionError: nope" in section + + +def test_a_test_without_either_field_is_skipped() -> None: + section = module.evidence_section( + _bundle( + failed_tests=[ + {"name": "a"}, + {"name": "b", "errorStackTrace": " "}, + "not-a-dict", + {"name": "c", "errorStackTrace": TRACE}, + ] + ) + ) + + assert "tests/test_utils.py:212" in section + + +def test_the_console_region_is_used_when_no_tests_were_published() -> None: + """A build whose runner published nothing still has a console to quote.""" + + section = module.evidence_section( + _bundle(console_failures=[{"text": "Traceback\n File x\nValueError: boom"}]) + ) + + assert "The earliest failure region" in section + assert "ValueError: boom" in section + + +def test_the_region_keeps_its_last_lines_when_it_is_long() -> None: + """The tail of a failure region holds the failure; the head approaches it.""" + + text = "\n".join([f"line {i}" for i in range(100)] + ["ValueError: boom"]) + section = module.evidence_section(_bundle(console_failures=[{"text": text}])) + + assert "ValueError: boom" in section + assert "line 0" not in section + + +def test_a_long_trace_is_truncated_with_a_pointer_to_the_build() -> None: + section = module.evidence_section( + _bundle(failed_tests=[{"errorStackTrace": "x" * 9000}]) + ) + + assert len(section) < 3000 + assert "truncated" in section + assert section.rstrip().endswith("```") + + +def test_a_fence_inside_the_excerpt_cannot_end_the_block_early() -> None: + """Raw log text must not be able to break out into the rendered issue.""" + + section = module.evidence_section( + _bundle(failed_tests=[{"errorStackTrace": "before\n```\nafter"}]) + ) + + assert section.count("```") == 2 + assert "after" in section + + +@pytest.mark.parametrize( + "bundle", + [ + None, + {}, + "not-a-dict", + {"jenkins": "not-a-dict"}, + {"jenkins": {}}, + {"jenkins": {"failed_tests": [], "console_failures": []}}, + {"jenkins": {"failed_tests": None, "console_failures": None}}, + {"jenkins": {"console_failures": [{"text": " "}, "not-a-dict"]}}, + ], +) +def test_a_bundle_with_nothing_to_quote_renders_no_heading(bundle) -> None: + """An empty Evidence heading is worse than no heading.""" + + assert module.evidence_section(bundle) == "" + + +def test_the_issue_body_carries_the_excerpt_and_keeps_its_marker() -> None: + rendered = body.issue_body( + { + "incident_id": "ariadne/408", + "job": "ariadne", + "build_number": 408, + "classification": "pytest_test_failure", + "reason": "a repository test failure", + "run_id": "run-1", + "bundle": _bundle(failed_tests=[{"errorStackTrace": TRACE}]), + } + ) + + assert "## Evidence" in rendered + assert "tests/test_utils.py:212" in rendered + assert rendered.index("## Evidence") < rendered.index("## Links") + assert rendered.rstrip().endswith("-->") + + +def test_an_issue_without_a_bundle_still_renders() -> None: + rendered = body.issue_body( + {"incident_id": "a/1", "job": "a", "build_number": 1, "reason": "r", "run_id": "x"} + ) + + assert "## Evidence" not in rendered + assert rendered.rstrip().endswith("-->")