011bc2c993
All checks were successful
Tests / Declarative: Post Actions passed: 1183
Two remaining misattributions in the artifact a human reads first. The summary line opened with "Hermes auto-triage classified" even when no run happened, and the hung-build path fell through to the generic "undiagnosed" label because HUNG_CLASSIFICATION was defined and never wired, so lesavka issue #3 was titled 'undiagnosed' for a build that plainly overran. The headline now reads "Ariadne recorded incident X as Y" when there was no run, and a caller may supply an explicit classification that is used only when no diagnosis produced one - a real diagnosis is never overridden. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
102 lines
3.9 KiB
Python
102 lines
3.9 KiB
Python
"""Tests that an issue attributes its finding to whoever actually made it."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
from ariadne.services import hermes_incident_body as body
|
|
from ariadne.services import hermes_incident_issue as module
|
|
|
|
def test_body_does_not_claim_a_diagnosis_that_never_happened() -> None:
|
|
"""A hung build is escalated without any model call.
|
|
|
|
Saying it came "from a Hermes Agent diagnosis" would misattribute a direct
|
|
observation to a model conclusion.
|
|
"""
|
|
|
|
context = module.issue_context(
|
|
{"incident_id": "metis/272", "job": "metis", "build_number": 272},
|
|
{
|
|
"bundle": {
|
|
"jenkins": {
|
|
"url": "https://ci.example/job/metis/272/",
|
|
"console_failures": [],
|
|
"console_tail": "Build metis #272 has been running for 75.0 minutes.",
|
|
}
|
|
},
|
|
"outcome": None,
|
|
"authorize_reason": "hung_build",
|
|
"run_id": None,
|
|
},
|
|
)
|
|
rendered = body.issue_body(context)
|
|
|
|
assert "No Hermes diagnosis was requested" in rendered
|
|
assert "from a Hermes Agent diagnosis" not in rendered
|
|
assert "has been running for 75.0 minutes" in rendered
|
|
|
|
|
|
def test_body_keeps_the_hermes_attribution_when_a_run_happened() -> None:
|
|
context = module.issue_context(
|
|
{"incident_id": "metis/272", "job": "metis", "build_number": 272},
|
|
{"bundle": {}, "outcome": None, "authorize_reason": "x", "run_id": "run_abc"},
|
|
)
|
|
rendered = body.issue_body(context)
|
|
|
|
assert "run `run_abc`" in rendered
|
|
assert "No Hermes diagnosis was requested" not in rendered
|
|
|
|
|
|
def test_observation_is_omitted_when_a_diagnosis_cited_facts() -> None:
|
|
"""Console text is a fallback, not a duplicate of a real diagnosis."""
|
|
|
|
jenkins = {"console_failures": [{"text": "boom"}], "console_tail": "tail text"}
|
|
assert module._observation(jenkins) == ""
|
|
assert module._observation(None) == ""
|
|
assert module._observation({"console_tail": " only tail "}) == "only tail"
|
|
|
|
|
|
def test_headline_credits_ariadne_when_no_model_ran() -> None:
|
|
"""The summary line is read first; it must not credit a model either."""
|
|
|
|
context = module.issue_context(
|
|
{"incident_id": "lesavka/584", "job": "lesavka", "build_number": 584},
|
|
{"bundle": {}, "outcome": None, "authorize_reason": "hung_build",
|
|
"run_id": None, "classification": "build_exceeded_time_cap"},
|
|
)
|
|
rendered = body.issue_body(context)
|
|
|
|
assert rendered.startswith("Ariadne recorded incident `lesavka/584` as **build_exceeded_time_cap**.")
|
|
assert "Hermes auto-triage classified" not in rendered
|
|
|
|
|
|
def test_headline_keeps_hermes_wording_and_confidence_for_a_real_run() -> None:
|
|
decision = SimpleNamespace(
|
|
classification="pytest_test_failure", confidence=0.99,
|
|
first_failed_gate="tests", reason="r", facts=[], inferences=[],
|
|
)
|
|
context = module.issue_context(
|
|
{"incident_id": "ariadne/409", "job": "ariadne", "build_number": 409},
|
|
{"bundle": {}, "outcome": SimpleNamespace(decision=decision),
|
|
"authorize_reason": "human_required", "run_id": "run_abc"},
|
|
)
|
|
rendered = body.issue_body(context)
|
|
|
|
assert "Hermes auto-triage classified incident `ariadne/409`" in rendered
|
|
assert "confidence 0.99" in rendered
|
|
|
|
|
|
def test_explicit_classification_is_used_only_without_a_decision() -> None:
|
|
"""A real diagnosis must never be overridden by a fallback label."""
|
|
|
|
decision = SimpleNamespace(
|
|
classification="pytest_test_failure", confidence=0.9,
|
|
first_failed_gate="t", reason="r", facts=[], inferences=[],
|
|
)
|
|
ctx = module.issue_context(
|
|
{"incident_id": "a/1", "job": "a", "build_number": 1},
|
|
{"bundle": {}, "outcome": SimpleNamespace(decision=decision),
|
|
"classification": "build_exceeded_time_cap", "run_id": "run_x"},
|
|
)
|
|
assert ctx["classification"] == "pytest_test_failure"
|