From 011bc2c993a374c67b6a1f22851276e02c4ad9f8 Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 6 Aug 2026 06:20:13 -0300 Subject: [PATCH] fix(hermes): give a non-model finding an honest headline and label Two remaining misattributions in the artifact a human reads first. The summary line opened with "Hermes auto-triage classified" even when no run happened, and the hung-build path fell through to the generic "undiagnosed" label because HUNG_CLASSIFICATION was defined and never wired, so lesavka issue #3 was titled 'undiagnosed' for a build that plainly overran. The headline now reads "Ariadne recorded incident X as Y" when there was no run, and a caller may supply an explicit classification that is used only when no diagnosis produced one - a real diagnosis is never overridden. Co-Authored-By: Claude Opus 5 --- ariadne/services/hermes_autotriage.py | 3 +- ariadne/services/hermes_incident_body.py | 11 +++++- ariadne/services/hermes_incident_issue.py | 6 ++- tests/test_hermes_incident_attribution.py | 47 +++++++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/ariadne/services/hermes_autotriage.py b/ariadne/services/hermes_autotriage.py index 7270af8..8579b76 100644 --- a/ariadne/services/hermes_autotriage.py +++ b/ariadne/services/hermes_autotriage.py @@ -104,7 +104,8 @@ def _handle_running_build( base = {"incident_id": incident_id, "job": job, "build_number": _int_value(last_build.get("number"))} hermes_events.record_incident(storage, base, "human_required", {"reason": reason}) bundle = hermes_hung_builds.hung_bundle(job, last_build, cap) - _file_incident_issue(storage, base, _diagnosis(bundle, None, reason, None), tick_state) + hung = {"classification": hermes_hung_builds.HUNG_CLASSIFICATION} + _file_incident_issue(storage, base, {**_diagnosis(bundle, None, reason, None), **hung}, tick_state) return {"status": "human_required", "incident_id": incident_id, "reason": reason} diff --git a/ariadne/services/hermes_incident_body.py b/ariadne/services/hermes_incident_body.py index e6540eb..100efac 100644 --- a/ariadne/services/hermes_incident_body.py +++ b/ariadne/services/hermes_incident_body.py @@ -140,10 +140,17 @@ def fact_fields(fact: Any) -> dict[str, str]: def _summary_line(context: dict) -> str: """Render the one-line summary that opens the issue body.""" + incident_id = context.get("incident_id") + classification = context.get("classification") + run_id = str(context.get("run_id") or "") + if not run_id or run_id == "unknown": + # The headline is the first thing read, so it must not credit a model + # for a finding Ariadne made on its own. + return f"Ariadne recorded incident `{incident_id}` as **{classification}**." confidence = context.get("confidence") return ( - f"Hermes auto-triage classified incident `{context.get('incident_id')}` as " - f"**{context.get('classification')}** (confidence {'n/a' if confidence is None else confidence}); " + f"Hermes auto-triage classified incident `{incident_id}` as " + f"**{classification}** (confidence {'n/a' if confidence is None else confidence}); " f"first failed gate: `{context.get('first_failed_gate') or 'unknown'}`." ) diff --git a/ariadne/services/hermes_incident_issue.py b/ariadne/services/hermes_incident_issue.py index dc7ee18..92a260d 100644 --- a/ariadne/services/hermes_incident_issue.py +++ b/ariadne/services/hermes_incident_issue.py @@ -192,7 +192,11 @@ def issue_context(base: dict[str, Any], diagnosis: dict[str, Any]) -> dict[str, "job": str(base.get("job") or ""), "build_number": base.get("build_number"), "build_url": str(jenkins.get("url") or "") if isinstance(jenkins, dict) else "", - "classification": str(getattr(decision, "classification", "") or body.UNDIAGNOSED), + "classification": str( + getattr(decision, "classification", "") + or diagnosis.get("classification") + or body.UNDIAGNOSED + ), "confidence": getattr(decision, "confidence", None), "first_failed_gate": str(getattr(decision, "first_failed_gate", "") or ""), "reason": str(getattr(decision, "reason", "") or authorize_reason), diff --git a/tests/test_hermes_incident_attribution.py b/tests/test_hermes_incident_attribution.py index 2c8e1af..bffdd3b 100644 --- a/tests/test_hermes_incident_attribution.py +++ b/tests/test_hermes_incident_attribution.py @@ -2,6 +2,8 @@ from __future__ import annotations +from types import SimpleNamespace + from ariadne.services import hermes_incident_body as body from ariadne.services import hermes_incident_issue as module @@ -52,3 +54,48 @@ def test_observation_is_omitted_when_a_diagnosis_cited_facts() -> None: assert module._observation(jenkins) == "" assert module._observation(None) == "" assert module._observation({"console_tail": " only tail "}) == "only tail" + + +def test_headline_credits_ariadne_when_no_model_ran() -> None: + """The summary line is read first; it must not credit a model either.""" + + context = module.issue_context( + {"incident_id": "lesavka/584", "job": "lesavka", "build_number": 584}, + {"bundle": {}, "outcome": None, "authorize_reason": "hung_build", + "run_id": None, "classification": "build_exceeded_time_cap"}, + ) + rendered = body.issue_body(context) + + assert rendered.startswith("Ariadne recorded incident `lesavka/584` as **build_exceeded_time_cap**.") + assert "Hermes auto-triage classified" not in rendered + + +def test_headline_keeps_hermes_wording_and_confidence_for_a_real_run() -> None: + decision = SimpleNamespace( + classification="pytest_test_failure", confidence=0.99, + first_failed_gate="tests", reason="r", facts=[], inferences=[], + ) + context = module.issue_context( + {"incident_id": "ariadne/409", "job": "ariadne", "build_number": 409}, + {"bundle": {}, "outcome": SimpleNamespace(decision=decision), + "authorize_reason": "human_required", "run_id": "run_abc"}, + ) + rendered = body.issue_body(context) + + assert "Hermes auto-triage classified incident `ariadne/409`" in rendered + assert "confidence 0.99" in rendered + + +def test_explicit_classification_is_used_only_without_a_decision() -> None: + """A real diagnosis must never be overridden by a fallback label.""" + + decision = SimpleNamespace( + classification="pytest_test_failure", confidence=0.9, + first_failed_gate="t", reason="r", facts=[], inferences=[], + ) + ctx = module.issue_context( + {"incident_id": "a/1", "job": "a", "build_number": 1}, + {"bundle": {}, "outcome": SimpleNamespace(decision=decision), + "classification": "build_exceeded_time_cap", "run_id": "run_x"}, + ) + assert ctx["classification"] == "pytest_test_failure"