ariadne/tests/test_hermes_suggested_remediation.py

"""Tests for remediations Hermes proposes but Ariadne cannot perform."""

from __future__ import annotations

import json

import pytest

from ariadne.services import hermes_autotriage_decision as decision_module
from ariadne.services import hermes_incident_body as body
from ariadne.services import hermes_suggested_remediation as module


SUGGESTION = {
    "action_id": "restart_stuck_flux_reconciler",
    "summary": "Suspend and resume the Flux Kustomization so the stalled reconcile is retried.",
    "evidence_required": "A Kustomization reporting the same revision as not-ready for over an hour.",
}


def _payload(**overrides):
    payload = {
        "incident_id": "lesavka/9",
        "classification": "flux_reconcile_stalled",
        "confidence": 0.9,
        "facts": [],
        "inferences": [],
        "first_failed_gate": "deploy",
        "requested_action": None,
        "human_required": True,
        "reason": "the reconcile never converged",
    }
    payload.update(overrides)
    return payload


def _parse(payload):
    return decision_module.parse_triage_response(json.dumps(payload), "lesavka/9")


def test_a_response_without_the_field_still_validates() -> None:
    """The field is optional, so responses predating it must keep parsing."""

    outcome = _parse(_payload())

    assert outcome.valid
    assert outcome.decision.suggested_remediation is None


def test_a_null_suggestion_is_accepted() -> None:
    outcome = _parse(_payload(suggested_remediation=None))

    assert outcome.valid
    assert outcome.decision.suggested_remediation is None


def test_a_well_formed_suggestion_is_parsed() -> None:
    outcome = _parse(_payload(suggested_remediation=dict(SUGGESTION)))

    suggestion = outcome.decision.suggested_remediation
    assert suggestion.action_id == "restart_stuck_flux_reconciler"
    assert suggestion.summary.startswith("Suspend and resume")
    assert "not-ready" in suggestion.evidence_required


def test_a_suggestion_alongside_a_requested_action_is_rejected() -> None:
    """The field reports that nothing fit, so something fitting contradicts it."""

    outcome = _parse(
        _payload(
            requested_action={"type": "run_ariadne_job", "id": "retry_transient_infra"},
            suggested_remediation=dict(SUGGESTION),
        )
    )

    assert not outcome.valid
    assert "must be null when an action is requested" in outcome.reject_reason


@pytest.mark.parametrize(
    ("suggestion", "expected"),
    [
        ("not-an-object", "must be an object or null"),
        ({"action_id": "a_b_c"}, "must have exactly action_id"),
        ({**SUGGESTION, "summary": "   "}, "summary must be a non-empty string"),
        ({**SUGGESTION, "evidence_required": 7}, "evidence_required must be a non-empty string"),
        ({**SUGGESTION, "action_id": "Restart Flux"}, "is not snake_case"),
        ({**SUGGESTION, "action_id": "ab"}, "is not snake_case"),
    ],
)
def test_a_malformed_suggestion_is_rejected(suggestion, expected) -> None:
    outcome = _parse(_payload(suggested_remediation=suggestion))

    assert not outcome.valid
    assert expected in outcome.reject_reason


def test_an_unknown_key_is_still_rejected() -> None:
    """Making one key optional must not make the schema open-ended."""

    outcome = _parse(_payload(something_else=1))

    assert not outcome.valid
    assert "unexpected_keys: something_else" in outcome.reject_reason


def test_a_long_proposal_is_clipped() -> None:
    outcome = _parse(
        _payload(suggested_remediation={**SUGGESTION, "summary": "word " * 400})
    )

    assert len(outcome.decision.suggested_remediation.summary) <= 400


def test_a_suggestion_never_widens_what_ariadne_may_run() -> None:
    """The proposed id is not an allowlist entry, and the gates must say so."""

    outcome = _parse(_payload(suggested_remediation=dict(SUGGESTION), human_required=False))
    allowed, reason = decision_module.authorize_action(
        outcome,
        {
            "allowed_actions": ["retry_transient_infra"],
            "action_classifications": {"flux_reconcile_stalled": "restart_stuck_flux_reconciler"},
            "autoremediation_enabled": True,
            "min_confidence": 0.5,
        },
        prior_action_count=0,
        build_is_terminal_failure=True,
        job_allowlisted=True,
        evidence_has_signature=True,
    )

    assert not allowed
    assert reason == "requested_action_missing"


def test_the_detail_recorded_for_the_audit_trail() -> None:
    suggestion = module.from_payload({"suggested_remediation": dict(SUGGESTION)})

    assert module.as_detail(suggestion) == {
        "action_id": "restart_stuck_flux_reconciler",
        "summary": SUGGESTION["summary"],
        "evidence_required": SUGGESTION["evidence_required"],
    }
    assert module.as_detail(None) is None


def test_the_issue_section_says_plainly_that_it_did_not_happen() -> None:
    """A reader must not mistake a proposal for something that was attempted."""

    section = module.issue_section(module.from_payload({"suggested_remediation": dict(SUGGESTION)}))

    assert "## Suggested remediation (not available)" in section
    assert "it was not performed" in section
    assert "`restart_stuck_flux_reconciler`" in section
    assert module.issue_section(None) == ""


def test_the_issue_body_carries_the_proposal() -> None:
    rendered = body.issue_body(
        {
            "incident_id": "lesavka/9",
            "job": "lesavka",
            "build_number": 9,
            "classification": "flux_reconcile_stalled",
            "reason": "the reconcile never converged",
            "run_id": "run-1",
            "suggested_remediation": module.from_payload(
                {"suggested_remediation": dict(SUGGESTION)}
            ),
        }
    )

    assert "Suggested remediation (not available)" in rendered
    assert "restart_stuck_flux_reconciler" in rendered
    assert rendered.rstrip().endswith("-->")


def test_a_proposal_is_counted_when_the_diagnosis_is_recorded() -> None:
    """The counter measures how often the allowlist fell short, once per diagnosis."""

    from ariadne.services import hermes_autotriage_events as events
    from ariadne.services.hermes_autotriage_metrics import HERMES_TRIAGE_SUGGESTION_TOTAL

    class _Storage:
        def __init__(self):
            self.events = []

        def record_event(self, event_type, detail):
            self.events.append((event_type, detail))

    run = type(
        "Run",
        (),
        {
            "status": "completed", "run_id": "r1", "session_id": "s1", "error": None,
            "duration_seconds": 1.0, "denied_approvals": 0,
        },
    )()
    authorization = events.Authorization(allowed=False, reason="requested_action_missing", evidence_marker=None)
    before = HERMES_TRIAGE_SUGGESTION_TOTAL._value.get()
    storage = _Storage()
    events.record_diagnosis(storage, {"incident_id": "lesavka/9"}, run, _parse(_payload()), authorization)
    assert HERMES_TRIAGE_SUGGESTION_TOTAL._value.get() == before

    outcome = _parse(_payload(suggested_remediation=dict(SUGGESTION)))
    events.record_diagnosis(storage, {"incident_id": "lesavka/9"}, run, outcome, authorization)

    assert HERMES_TRIAGE_SUGGESTION_TOTAL._value.get() == before + 1
    detail = storage.events[-1][1]
    assert detail["outcome"]["suggested_remediation"]["action_id"] == "restart_stuck_flux_reconciler"