from __future__ import annotations import json import pytest from ariadne.services import hermes_autotriage_decision as module INCIDENT_ID = "inc-42" def _payload(**overrides) -> dict: # type: ignore[no-untyped-def] base = { "incident_id": INCIDENT_ID, "classification": "known_demo_fixture_failure", "confidence": 0.93, "facts": [ { "statement": "Build demo #12 failed in the fixture stage.", "source": "jenkins", "reference": "https://ci.bstein.dev/job/demo/12/", } ], "inferences": ["The fixture dataset drifted."], "first_failed_gate": "unit-tests", "requested_action": {"type": "run_ariadne_job", "id": "demo-fixture-reset"}, "human_required": False, "reason": "Known fixture failure signature matched.", } base.update(overrides) return base def _parse(payload: dict | None = None, raw: str | None = None, incident: str = INCIDENT_ID): # type: ignore[no-untyped-def] text = raw if raw is not None else json.dumps(payload if payload is not None else _payload()) return module.parse_triage_response(text, incident) def _cfg(**overrides) -> dict: # type: ignore[no-untyped-def] base = { "autoremediation_enabled": True, "allowed_actions": ["demo-fixture-reset"], "min_confidence": 0.8, "expected_classification": "known_demo_fixture_failure", "max_actions_per_incident": 1, } base.update(overrides) return base def _authorize(outcome=None, cfg=None, **overrides): # type: ignore[no-untyped-def] kwargs = { "prior_action_count": 0, "build_is_terminal_failure": True, "job_allowlisted": True, "evidence_has_signature": True, } kwargs.update(overrides) return module.authorize_action(outcome if outcome is not None else _parse(), cfg if cfg is not None else _cfg(), **kwargs) def test_parse_valid_response() -> None: outcome = _parse() assert outcome.valid is True assert outcome.human_required is False assert outcome.reject_reason is None decision = outcome.decision assert decision is not None assert decision.incident_id == INCIDENT_ID assert decision.classification == "known_demo_fixture_failure" assert decision.confidence == 0.93 assert decision.facts == [ module.TriageFact( statement="Build demo #12 failed in the fixture stage.", source="jenkins", reference="https://ci.bstein.dev/job/demo/12/", ) ] assert decision.inferences == ["The fixture dataset drifted."] assert decision.first_failed_gate == "unit-tests" assert decision.requested_action == module.RequestedAction(type="run_ariadne_job", id="demo-fixture-reset") assert decision.reason == "Known fixture failure signature matched." def test_parse_accepts_markdown_fenced_json() -> None: raw = "Here is my triage.\n```json\n" + json.dumps(_payload()) + "\n```\nLet me know." outcome = _parse(raw=raw) assert outcome.valid is True assert outcome.decision is not None assert outcome.decision.requested_action.id == "demo-fixture-reset" def test_parse_accepts_prose_and_braces_inside_strings() -> None: payload = _payload(reason='Matched the {fixture} signature with a "quoted" note and a \\ escape.') raw = "Prose before } stray brace... " + json.dumps(payload) + ' Trailing prose {"not": "parsed"}' outcome = _parse(raw=raw) assert outcome.valid is True assert outcome.decision is not None assert "{fixture}" in outcome.decision.reason def test_parse_accepts_null_requested_action_and_int_confidence() -> None: outcome = _parse(_payload(requested_action=None, confidence=1)) assert outcome.valid is True assert outcome.decision is not None assert outcome.decision.requested_action is None assert outcome.decision.confidence == 1.0 def test_parse_valid_response_with_human_required_true() -> None: outcome = _parse(_payload(human_required=True)) assert outcome.valid is True assert outcome.human_required is True assert outcome.reject_reason is None @pytest.mark.parametrize( "raw", ["", "no braces at all", '{"incident_id": "inc-42"', "closing only }"], ) def test_parse_rejects_missing_json_object(raw: str) -> None: outcome = _parse(raw=raw) assert outcome.valid is False assert outcome.decision is None assert outcome.human_required is True assert outcome.reject_reason == "no_json_object_found" def test_parse_rejects_invalid_json() -> None: outcome = _parse(raw="{bad json}") assert outcome.valid is False assert outcome.human_required is True assert outcome.reject_reason is not None assert outcome.reject_reason.startswith("invalid_json:") def test_parse_rejects_missing_and_extra_keys() -> None: missing = _payload() missing.pop("reason") outcome = _parse(missing) assert outcome.valid is False assert outcome.reject_reason == "missing_keys: reason" outcome = _parse(_payload(notes="extra")) assert outcome.valid is False assert outcome.reject_reason == "unexpected_keys: notes" @pytest.mark.parametrize("field", ["incident_id", "classification", "first_failed_gate", "reason"]) def test_parse_rejects_non_string_fields(field: str) -> None: outcome = _parse(_payload(**{field: 7})) assert outcome.valid is False assert outcome.reject_reason == f"field_type_invalid: {field} must be a string" def test_parse_rejects_non_boolean_human_required_and_non_list_inferences() -> None: outcome = _parse(_payload(human_required="yes")) assert outcome.reject_reason == "field_type_invalid: human_required must be a boolean" outcome = _parse(_payload(inferences={"not": "a list"})) assert outcome.reject_reason == "field_type_invalid: inferences must be a list" @pytest.mark.parametrize("confidence", ["high", None, True]) def test_parse_rejects_non_numeric_confidence(confidence) -> None: # type: ignore[no-untyped-def] outcome = _parse(_payload(confidence=confidence)) assert outcome.valid is False assert outcome.reject_reason == "field_type_invalid: confidence must be a number" @pytest.mark.parametrize("confidence", [1.5, -0.2]) def test_parse_rejects_out_of_range_confidence(confidence: float) -> None: outcome = _parse(_payload(confidence=confidence)) assert outcome.valid is False assert outcome.reject_reason == f"confidence_out_of_range: {confidence}" def test_parse_rejects_bad_facts() -> None: outcome = _parse(_payload(facts={"not": "a list"})) assert outcome.reject_reason == "field_type_invalid: facts must be a list" outcome = _parse(_payload(facts=["not a dict"])) assert outcome.reject_reason == "fact_invalid: facts[0] must be an object" outcome = _parse(_payload(facts=[{"statement": "s", "source": "jenkins"}])) assert outcome.reject_reason == "fact_invalid: facts[0] must have exactly statement, source, reference" fact = {"statement": "s", "source": "jenkins", "reference": "r", "extra": "x"} outcome = _parse(_payload(facts=[fact])) assert outcome.reject_reason == "fact_invalid: facts[0] must have exactly statement, source, reference" outcome = _parse(_payload(facts=[{"statement": 5, "source": "jenkins", "reference": "r"}])) assert outcome.reject_reason == "fact_invalid: facts[0] fields must be strings" def test_parse_rejects_unlisted_fact_source() -> None: good = _payload()["facts"][0] bad = dict(good, source="bing") outcome = _parse(_payload(facts=[good, bad])) assert outcome.valid is False assert outcome.reject_reason == "fact_source_invalid: facts[1] source 'bing'" def test_parse_rejects_malformed_requested_action() -> None: outcome = _parse(_payload(requested_action=["run_ariadne_job"])) assert outcome.reject_reason == "requested_action_invalid: must be an object or null" outcome = _parse(_payload(requested_action={"type": "run_ariadne_job"})) assert outcome.reject_reason == "requested_action_invalid: must have exactly type and id" outcome = _parse(_payload(requested_action={"type": "run_ariadne_job", "id": "x", "why": "extra"})) assert outcome.reject_reason == "requested_action_invalid: must have exactly type and id" outcome = _parse(_payload(requested_action={"type": "delete_pod", "id": "x"})) assert outcome.reject_reason == "requested_action_invalid: unsupported type 'delete_pod'" outcome = _parse(_payload(requested_action={"type": "run_ariadne_job", "id": " "})) assert outcome.reject_reason == "requested_action_invalid: id must be a non-empty string" outcome = _parse(_payload(requested_action={"type": "run_ariadne_job", "id": 7})) assert outcome.reject_reason == "requested_action_invalid: id must be a non-empty string" def test_parse_rejects_incident_id_mismatch() -> None: outcome = _parse(incident="inc-99") assert outcome.valid is False assert outcome.human_required is True assert outcome.reject_reason == "incident_id_mismatch: got 'inc-42' expected 'inc-99'" def test_authorize_all_gates_pass() -> None: assert _authorize() == (True, "authorized") def test_authorize_rejects_invalid_outcome() -> None: allowed, reason = _authorize(outcome=_parse(raw="not json")) assert allowed is False assert reason == "response_invalid: no_json_object_found" def test_authorize_rejects_human_required_decision() -> None: allowed, reason = _authorize(outcome=_parse(_payload(human_required=True))) assert allowed is False assert reason == "human_required" def test_authorize_requires_terminal_failed_build() -> None: assert _authorize(build_is_terminal_failure=False) == (False, "build_not_terminal_failure") def test_authorize_requires_allowlisted_job() -> None: assert _authorize(job_allowlisted=False) == (False, "job_not_allowlisted") def test_authorize_requires_expected_classification() -> None: allowed, reason = _authorize(outcome=_parse(_payload(classification="novel_failure"))) assert allowed is False assert reason == "classification_mismatch: got 'novel_failure' expected 'known_demo_fixture_failure'" def test_authorize_requires_requested_action() -> None: allowed, reason = _authorize(outcome=_parse(_payload(requested_action=None))) assert allowed is False assert reason == "requested_action_missing" def test_authorize_rejects_wrong_action_type_on_handcrafted_decision() -> None: base = _parse().decision assert base is not None decision = module.TriageDecision( incident_id=base.incident_id, classification=base.classification, confidence=base.confidence, facts=base.facts, inferences=base.inferences, first_failed_gate=base.first_failed_gate, requested_action=module.RequestedAction(type="delete_everything", id="demo-fixture-reset"), human_required=False, reason=base.reason, ) outcome = module.DecisionOutcome(valid=True, decision=decision, human_required=False, reject_reason=None) assert _authorize(outcome=outcome) == (False, "requested_action_type_invalid") def test_authorize_requires_action_in_allowlist() -> None: allowed, reason = _authorize(cfg=_cfg(allowed_actions=["other-job"])) assert allowed is False assert reason == "action_not_allowlisted: 'demo-fixture-reset'" def test_authorize_requires_minimum_confidence() -> None: allowed, reason = _authorize(outcome=_parse(_payload(confidence=0.5))) assert allowed is False assert reason == "confidence_below_minimum: 0.5 < 0.8" def test_authorize_requires_evidence_signature() -> None: assert _authorize(evidence_has_signature=False) == (False, "evidence_signature_missing") def test_authorize_enforces_action_budget() -> None: assert _authorize(prior_action_count=1) == (False, "max_actions_reached") assert _authorize(prior_action_count=0, cfg=_cfg(max_actions_per_incident=2)) == (True, "authorized") def test_authorize_requires_autoremediation_enabled() -> None: allowed, reason = _authorize(cfg=_cfg(autoremediation_enabled=False)) assert allowed is False assert reason == "autoremediation_disabled" def test_authorize_names_first_failing_gate_in_order() -> None: allowed, reason = _authorize( outcome=_parse(_payload(human_required=True)), cfg=_cfg(autoremediation_enabled=False), build_is_terminal_failure=False, evidence_has_signature=False, ) assert allowed is False assert reason == "human_required" def test_authorize_with_empty_cfg_defaults_closed() -> None: allowed, reason = _authorize(cfg={}) assert allowed is False assert reason == "action_not_allowlisted: 'demo-fixture-reset'" def test_authorize_falls_back_to_conservative_cfg_values() -> None: cfg = _cfg(min_confidence="high", max_actions_per_incident="many") allowed, reason = _authorize(cfg=cfg) assert allowed is False assert reason == "confidence_below_minimum: 0.93 < 1.0" def _mapping_cfg(**overrides) -> dict: # type: ignore[no-untyped-def] base = { "allowed_actions": ["repair_demo_fixture", "retry_transient_infra"], "action_classifications": { "known_demo_fixture_failure": "repair_demo_fixture", "transient_infra_failure": "retry_transient_infra", }, } base.update(overrides) return _cfg(**base) def _mapped_payload(classification: str, action_id: str) -> dict: return _payload( classification=classification, requested_action={"type": "run_ariadne_job", "id": action_id}, ) def test_authorize_accepts_each_registered_classification() -> None: for classification, action_id in _mapping_cfg()["action_classifications"].items(): outcome = _parse(_mapped_payload(classification, action_id)) assert _authorize(outcome=outcome, cfg=_mapping_cfg()) == (True, "authorized") def test_authorize_rejects_unregistered_classification() -> None: outcome = _parse(_mapped_payload("novel_failure", "retry_transient_infra")) allowed, reason = _authorize(outcome=outcome, cfg=_mapping_cfg()) assert allowed is False assert reason == "classification_not_supported: 'novel_failure'" def test_authorize_rejects_action_mapped_to_another_classification() -> None: outcome = _parse(_mapped_payload("transient_infra_failure", "repair_demo_fixture")) allowed, reason = _authorize(outcome=outcome, cfg=_mapping_cfg()) assert allowed is False assert reason == ( "action_does_not_match_classification: 'repair_demo_fixture' expected 'retry_transient_infra'" ) def test_authorize_still_requires_the_mapped_action_to_be_allowlisted() -> None: outcome = _parse(_mapped_payload("transient_infra_failure", "retry_transient_infra")) cfg = _mapping_cfg(allowed_actions=["repair_demo_fixture"]) allowed, reason = _authorize(outcome=outcome, cfg=cfg) assert allowed is False assert reason == "action_not_allowlisted: 'retry_transient_infra'" def test_authorize_reports_missing_action_before_the_mapping_gate() -> None: outcome = _parse(_payload(classification="transient_infra_failure", requested_action=None)) assert _authorize(outcome=outcome, cfg=_mapping_cfg()) == (False, "requested_action_missing") def test_authorize_mapping_still_enforces_the_remaining_gates() -> None: outcome = _parse(_mapped_payload("transient_infra_failure", "retry_transient_infra")) assert _authorize(outcome=outcome, cfg=_mapping_cfg(), evidence_has_signature=False) == ( False, "evidence_signature_missing", ) assert _authorize(outcome=outcome, cfg=_mapping_cfg(), prior_action_count=1) == ( False, "max_actions_reached", ) assert _authorize(outcome=outcome, cfg=_mapping_cfg(autoremediation_enabled=False)) == ( False, "autoremediation_disabled", ) def test_authorize_ignores_an_empty_or_malformed_mapping() -> None: for mapping in ({}, None, "known_demo_fixture_failure=repair_demo_fixture", []): cfg = _cfg(action_classifications=mapping) assert _authorize(cfg=cfg) == (True, "authorized") allowed, reason = _authorize(outcome=_parse(_payload(classification="other")), cfg=cfg) assert allowed is False assert reason == "classification_mismatch: got 'other' expected 'known_demo_fixture_failure'"