"""Tests for remediations Hermes proposes but Ariadne cannot perform.""" from __future__ import annotations import json import pytest from ariadne.services import hermes_autotriage_decision as decision_module from ariadne.services import hermes_incident_body as body from ariadne.services import hermes_suggested_remediation as module SUGGESTION = { "action_id": "restart_stuck_flux_reconciler", "summary": "Suspend and resume the Flux Kustomization so the stalled reconcile is retried.", "evidence_required": "A Kustomization reporting the same revision as not-ready for over an hour.", } def _payload(**overrides): payload = { "incident_id": "lesavka/9", "classification": "flux_reconcile_stalled", "confidence": 0.9, "facts": [], "inferences": [], "first_failed_gate": "deploy", "requested_action": None, "human_required": True, "reason": "the reconcile never converged", } payload.update(overrides) return payload def _parse(payload): return decision_module.parse_triage_response(json.dumps(payload), "lesavka/9") def test_a_response_without_the_field_still_validates() -> None: """The field is optional, so responses predating it must keep parsing.""" outcome = _parse(_payload()) assert outcome.valid assert outcome.decision.suggested_remediation is None def test_a_null_suggestion_is_accepted() -> None: outcome = _parse(_payload(suggested_remediation=None)) assert outcome.valid assert outcome.decision.suggested_remediation is None def test_a_well_formed_suggestion_is_parsed() -> None: outcome = _parse(_payload(suggested_remediation=dict(SUGGESTION))) suggestion = outcome.decision.suggested_remediation assert suggestion.action_id == "restart_stuck_flux_reconciler" assert suggestion.summary.startswith("Suspend and resume") assert "not-ready" in suggestion.evidence_required def test_a_suggestion_alongside_a_requested_action_is_rejected() -> None: """The field reports that nothing fit, so something fitting contradicts it.""" outcome = _parse( _payload( requested_action={"type": "run_ariadne_job", "id": "retry_transient_infra"}, suggested_remediation=dict(SUGGESTION), ) ) assert not outcome.valid assert "must be null when an action is requested" in outcome.reject_reason @pytest.mark.parametrize( ("suggestion", "expected"), [ ("not-an-object", "must be an object or null"), ({"action_id": "a_b_c"}, "must have exactly action_id"), ({**SUGGESTION, "summary": " "}, "summary must be a non-empty string"), ({**SUGGESTION, "evidence_required": 7}, "evidence_required must be a non-empty string"), ({**SUGGESTION, "action_id": "Restart Flux"}, "is not snake_case"), ({**SUGGESTION, "action_id": "ab"}, "is not snake_case"), ], ) def test_a_malformed_suggestion_is_rejected(suggestion, expected) -> None: outcome = _parse(_payload(suggested_remediation=suggestion)) assert not outcome.valid assert expected in outcome.reject_reason def test_an_unknown_key_is_still_rejected() -> None: """Making one key optional must not make the schema open-ended.""" outcome = _parse(_payload(something_else=1)) assert not outcome.valid assert "unexpected_keys: something_else" in outcome.reject_reason def test_a_long_proposal_is_clipped() -> None: outcome = _parse( _payload(suggested_remediation={**SUGGESTION, "summary": "word " * 400}) ) assert len(outcome.decision.suggested_remediation.summary) <= 400 def test_a_suggestion_never_widens_what_ariadne_may_run() -> None: """The proposed id is not an allowlist entry, and the gates must say so.""" outcome = _parse(_payload(suggested_remediation=dict(SUGGESTION), human_required=False)) allowed, reason = decision_module.authorize_action( outcome, { "allowed_actions": ["retry_transient_infra"], "action_classifications": {"flux_reconcile_stalled": "restart_stuck_flux_reconciler"}, "autoremediation_enabled": True, "min_confidence": 0.5, }, prior_action_count=0, build_is_terminal_failure=True, job_allowlisted=True, evidence_has_signature=True, ) assert not allowed assert reason == "requested_action_missing" def test_the_detail_recorded_for_the_audit_trail() -> None: suggestion = module.from_payload({"suggested_remediation": dict(SUGGESTION)}) assert module.as_detail(suggestion) == { "action_id": "restart_stuck_flux_reconciler", "summary": SUGGESTION["summary"], "evidence_required": SUGGESTION["evidence_required"], } assert module.as_detail(None) is None def test_the_issue_section_says_plainly_that_it_did_not_happen() -> None: """A reader must not mistake a proposal for something that was attempted.""" section = module.issue_section(module.from_payload({"suggested_remediation": dict(SUGGESTION)})) assert "## Suggested remediation (not available)" in section assert "it was not performed" in section assert "`restart_stuck_flux_reconciler`" in section assert module.issue_section(None) == "" def test_the_issue_body_carries_the_proposal() -> None: rendered = body.issue_body( { "incident_id": "lesavka/9", "job": "lesavka", "build_number": 9, "classification": "flux_reconcile_stalled", "reason": "the reconcile never converged", "run_id": "run-1", "suggested_remediation": module.from_payload( {"suggested_remediation": dict(SUGGESTION)} ), } ) assert "Suggested remediation (not available)" in rendered assert "restart_stuck_flux_reconciler" in rendered assert rendered.rstrip().endswith("-->") def test_a_proposal_is_counted_when_the_diagnosis_is_recorded() -> None: """The counter measures how often the allowlist fell short, once per diagnosis.""" from ariadne.services import hermes_autotriage_events as events from ariadne.services.hermes_autotriage_metrics import HERMES_TRIAGE_SUGGESTION_TOTAL class _Storage: def __init__(self): self.events = [] def record_event(self, event_type, detail): self.events.append((event_type, detail)) run = type( "Run", (), { "status": "completed", "run_id": "r1", "session_id": "s1", "error": None, "duration_seconds": 1.0, "denied_approvals": 0, }, )() authorization = events.Authorization(allowed=False, reason="requested_action_missing", evidence_marker=None) before = HERMES_TRIAGE_SUGGESTION_TOTAL._value.get() storage = _Storage() events.record_diagnosis(storage, {"incident_id": "lesavka/9"}, run, _parse(_payload()), authorization) assert HERMES_TRIAGE_SUGGESTION_TOTAL._value.get() == before outcome = _parse(_payload(suggested_remediation=dict(SUGGESTION))) events.record_diagnosis(storage, {"incident_id": "lesavka/9"}, run, outcome, authorization) assert HERMES_TRIAGE_SUGGESTION_TOTAL._value.get() == before + 1 detail = storage.events[-1][1] assert detail["outcome"]["suggested_remediation"]["action_id"] == "restart_stuck_flux_reconciler"