From dca283a17b98d6ad2cc0a005d5431c73a813f04f Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 6 Aug 2026 06:01:27 -0300 Subject: [PATCH] fix(hermes): stop crediting a diagnosis that never happened The hung-build path escalates without calling a model, but the issue footer said unconditionally that the finding came "from a Hermes Agent diagnosis (run `unknown`)". Live example: lesavka issue #3, filed for a build that overran, attributed a direct observation to a model conclusion that was never requested. That misrepresents where the finding came from, which matters most in exactly the artifact a human reads first. Attribute to Ariadne when there was no run, and carry the recorded console text into the body so the issue says what was actually observed instead of leaving "Why a human is needed: hung_build" as the whole explanation. Co-Authored-By: Claude Opus 5 --- ariadne/services/hermes_incident_body.py | 24 +++++++++- ariadne/services/hermes_incident_issue.py | 13 ++++++ tests/test_hermes_incident_attribution.py | 54 +++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 tests/test_hermes_incident_attribution.py diff --git a/ariadne/services/hermes_incident_body.py b/ariadne/services/hermes_incident_body.py index 7f9df3a..e6540eb 100644 --- a/ariadne/services/hermes_incident_body.py +++ b/ariadne/services/hermes_incident_body.py @@ -37,6 +37,14 @@ _FOOTER_TEMPLATE = ( "Filed automatically by Ariadne from a Hermes Agent diagnosis (run `{run_id}`). " "Hermes has no write access to this repository; no files or infrastructure were changed." ) +# Some escalations never reach a model at all - a build still running has no +# finished console to diagnose. Claiming a diagnosis that did not happen would +# misrepresent where the conclusion came from. +_NO_RUN_FOOTER = ( + "Filed automatically by Ariadne. No Hermes diagnosis was requested for this incident: " + "the finding is a direct observation, not a model conclusion. " + "Hermes has no write access to this repository; no files or infrastructure were changed." +) _MARKER_PATTERN = re.compile( r"" @@ -111,7 +119,7 @@ def issue_body(context: dict, max_chars: int = DEFAULT_MAX_BODY_CHARS) -> str: _facts_section(context), _inferences_section(context), _links_section(context), - _FOOTER_TEMPLATE.format(run_id=context.get("run_id") or "unknown"), + _footer(context), ] body = "\n\n".join(section for section in sections if section) return f"{_bounded_body(body, max_chars - len(marker) - 2)}\n\n{marker}" @@ -140,10 +148,24 @@ def _summary_line(context: dict) -> str: ) +def _footer(context: dict) -> str: + """Attribute the finding to a Hermes run, or to Ariadne when there was none.""" + + run_id = str(context.get("run_id") or "") + if not run_id or run_id == "unknown": + return _NO_RUN_FOOTER + return _FOOTER_TEMPLATE.format(run_id=run_id) + + def _human_section(context: dict) -> str: """Render the section explaining why the incident needs a person.""" lines = ["## Why a human is needed", str(context.get("reason") or "no reason recorded")] + observation = str(context.get("observation") or "").strip() + if observation: + # Escalations that never reached a model carry their explanation here + # rather than in facts, which are only populated from a diagnosis. + lines.append(observation) authorize_reason = str(context.get("authorize_reason") or "") if authorize_reason: lines.append(f"Ariadne did not authorize automated remediation: `{authorize_reason}`.") diff --git a/ariadne/services/hermes_incident_issue.py b/ariadne/services/hermes_incident_issue.py index 3410f37..dc7ee18 100644 --- a/ariadne/services/hermes_incident_issue.py +++ b/ariadne/services/hermes_incident_issue.py @@ -199,11 +199,24 @@ def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str, "facts": [body.fact_fields(fact) for fact in getattr(decision, "facts", None) or []], "inferences": list(getattr(decision, "inferences", None) or []), "authorize_reason": authorize_reason, + # An escalation that never reached a model has no cited facts, so its + # console text is the whole explanation and must not be dropped. + "observation": _observation(jenkins) if decision is None else "", "run_id": diagnosis.get("run_id"), "code_proposal_url": str(proposal.get("url") or "") if isinstance(proposal, dict) else "", } +def _observation(jenkins: Any) -> str: + """Return the console text Ariadne recorded without consulting a model.""" + + if not isinstance(jenkins, dict): + return "" + if jenkins.get("console_failures"): + return "" + return str(jenkins.get("console_tail") or "").strip() + + def _file_issue( storage: Any, config: Any, diff --git a/tests/test_hermes_incident_attribution.py b/tests/test_hermes_incident_attribution.py new file mode 100644 index 0000000..2c8e1af --- /dev/null +++ b/tests/test_hermes_incident_attribution.py @@ -0,0 +1,54 @@ +"""Tests that an issue attributes its finding to whoever actually made it.""" + +from __future__ import annotations + +from ariadne.services import hermes_incident_body as body +from ariadne.services import hermes_incident_issue as module + +def test_body_does_not_claim_a_diagnosis_that_never_happened() -> None: + """A hung build is escalated without any model call. + + Saying it came "from a Hermes Agent diagnosis" would misattribute a direct + observation to a model conclusion. + """ + + context = module.issue_context( + {"incident_id": "metis/272", "job": "metis", "build_number": 272}, + { + "bundle": { + "jenkins": { + "url": "https://ci.example/job/metis/272/", + "console_failures": [], + "console_tail": "Build metis #272 has been running for 75.0 minutes.", + } + }, + "outcome": None, + "authorize_reason": "hung_build", + "run_id": None, + }, + ) + rendered = body.issue_body(context) + + assert "No Hermes diagnosis was requested" in rendered + assert "from a Hermes Agent diagnosis" not in rendered + assert "has been running for 75.0 minutes" in rendered + + +def test_body_keeps_the_hermes_attribution_when_a_run_happened() -> None: + context = module.issue_context( + {"incident_id": "metis/272", "job": "metis", "build_number": 272}, + {"bundle": {}, "outcome": None, "authorize_reason": "x", "run_id": "run_abc"}, + ) + rendered = body.issue_body(context) + + assert "run `run_abc`" in rendered + assert "No Hermes diagnosis was requested" not in rendered + + +def test_observation_is_omitted_when_a_diagnosis_cited_facts() -> None: + """Console text is a fallback, not a duplicate of a real diagnosis.""" + + jenkins = {"console_failures": [{"text": "boom"}], "console_tail": "tail text"} + assert module._observation(jenkins) == "" + assert module._observation(None) == "" + assert module._observation({"console_tail": " only tail "}) == "only tail"