8c65b7bd60
Second entry in the action registry, proving it is a real extension point. No cluster mutation: the action is one Jenkins rebuild. - hermes_infra_signals: reviewable marker set across DNS/connectivity, image pull, upstream 5xx and agent-channel loss; Ariadne independently confirms a marker in the evidence before any retry, and records which marker justified it. "no space left on device" is deliberately excluded because a retry lands on the same full volume. - decision: classification -> action registry (action_classifications), falling back to the previous single-classification behavior - repair: retry_build posts to /build for unparameterized real jobs and buildWithParameters for the fixture demo job - orchestrator: retry path records requested/accepted/executed and moves the incident to awaiting_rebuild so the existing success path resolves it; one action per incident still enforced, so a retry cannot loop - events layer split out of the orchestrator to stay under the LOC cap Motivated by real failures tonight: pip DNS resolution and a Gitea 443 connect timeout, plus live incident metis/271 (SCM checkout timeout). 30 new tests; 368 pass in the hermes suite. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
447 lines
16 KiB
Python
447 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from ariadne.services import hermes_autotriage_decision as module
|
|
|
|
|
|
INCIDENT_ID = "inc-42"
|
|
|
|
|
|
def _payload(**overrides) -> dict: # type: ignore[no-untyped-def]
|
|
base = {
|
|
"incident_id": INCIDENT_ID,
|
|
"classification": "known_demo_fixture_failure",
|
|
"confidence": 0.93,
|
|
"facts": [
|
|
{
|
|
"statement": "Build demo #12 failed in the fixture stage.",
|
|
"source": "jenkins",
|
|
"reference": "https://ci.bstein.dev/job/demo/12/",
|
|
}
|
|
],
|
|
"inferences": ["The fixture dataset drifted."],
|
|
"first_failed_gate": "unit-tests",
|
|
"requested_action": {"type": "run_ariadne_job", "id": "demo-fixture-reset"},
|
|
"human_required": False,
|
|
"reason": "Known fixture failure signature matched.",
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _parse(payload: dict | None = None, raw: str | None = None, incident: str = INCIDENT_ID): # type: ignore[no-untyped-def]
|
|
text = raw if raw is not None else json.dumps(payload if payload is not None else _payload())
|
|
return module.parse_triage_response(text, incident)
|
|
|
|
|
|
def _cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
|
|
base = {
|
|
"autoremediation_enabled": True,
|
|
"allowed_actions": ["demo-fixture-reset"],
|
|
"min_confidence": 0.8,
|
|
"expected_classification": "known_demo_fixture_failure",
|
|
"max_actions_per_incident": 1,
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _authorize(outcome=None, cfg=None, **overrides): # type: ignore[no-untyped-def]
|
|
kwargs = {
|
|
"prior_action_count": 0,
|
|
"build_is_terminal_failure": True,
|
|
"job_allowlisted": True,
|
|
"evidence_has_signature": True,
|
|
}
|
|
kwargs.update(overrides)
|
|
return module.authorize_action(outcome if outcome is not None else _parse(), cfg if cfg is not None else _cfg(), **kwargs)
|
|
|
|
|
|
def test_parse_valid_response() -> None:
|
|
outcome = _parse()
|
|
|
|
assert outcome.valid is True
|
|
assert outcome.human_required is False
|
|
assert outcome.reject_reason is None
|
|
decision = outcome.decision
|
|
assert decision is not None
|
|
assert decision.incident_id == INCIDENT_ID
|
|
assert decision.classification == "known_demo_fixture_failure"
|
|
assert decision.confidence == 0.93
|
|
assert decision.facts == [
|
|
module.TriageFact(
|
|
statement="Build demo #12 failed in the fixture stage.",
|
|
source="jenkins",
|
|
reference="https://ci.bstein.dev/job/demo/12/",
|
|
)
|
|
]
|
|
assert decision.inferences == ["The fixture dataset drifted."]
|
|
assert decision.first_failed_gate == "unit-tests"
|
|
assert decision.requested_action == module.RequestedAction(type="run_ariadne_job", id="demo-fixture-reset")
|
|
assert decision.reason == "Known fixture failure signature matched."
|
|
|
|
|
|
def test_parse_accepts_markdown_fenced_json() -> None:
|
|
raw = "Here is my triage.\n```json\n" + json.dumps(_payload()) + "\n```\nLet me know."
|
|
|
|
outcome = _parse(raw=raw)
|
|
|
|
assert outcome.valid is True
|
|
assert outcome.decision is not None
|
|
assert outcome.decision.requested_action.id == "demo-fixture-reset"
|
|
|
|
|
|
def test_parse_accepts_prose_and_braces_inside_strings() -> None:
|
|
payload = _payload(reason='Matched the {fixture} signature with a "quoted" note and a \\ escape.')
|
|
raw = "Prose before } stray brace... " + json.dumps(payload) + ' Trailing prose {"not": "parsed"}'
|
|
|
|
outcome = _parse(raw=raw)
|
|
|
|
assert outcome.valid is True
|
|
assert outcome.decision is not None
|
|
assert "{fixture}" in outcome.decision.reason
|
|
|
|
|
|
def test_parse_accepts_null_requested_action_and_int_confidence() -> None:
|
|
outcome = _parse(_payload(requested_action=None, confidence=1))
|
|
|
|
assert outcome.valid is True
|
|
assert outcome.decision is not None
|
|
assert outcome.decision.requested_action is None
|
|
assert outcome.decision.confidence == 1.0
|
|
|
|
|
|
def test_parse_valid_response_with_human_required_true() -> None:
|
|
outcome = _parse(_payload(human_required=True))
|
|
|
|
assert outcome.valid is True
|
|
assert outcome.human_required is True
|
|
assert outcome.reject_reason is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raw",
|
|
["", "no braces at all", '{"incident_id": "inc-42"', "closing only }"],
|
|
)
|
|
def test_parse_rejects_missing_json_object(raw: str) -> None:
|
|
outcome = _parse(raw=raw)
|
|
|
|
assert outcome.valid is False
|
|
assert outcome.decision is None
|
|
assert outcome.human_required is True
|
|
assert outcome.reject_reason == "no_json_object_found"
|
|
|
|
|
|
def test_parse_rejects_invalid_json() -> None:
|
|
outcome = _parse(raw="{bad json}")
|
|
|
|
assert outcome.valid is False
|
|
assert outcome.human_required is True
|
|
assert outcome.reject_reason is not None
|
|
assert outcome.reject_reason.startswith("invalid_json:")
|
|
|
|
|
|
def test_parse_rejects_missing_and_extra_keys() -> None:
|
|
missing = _payload()
|
|
missing.pop("reason")
|
|
outcome = _parse(missing)
|
|
assert outcome.valid is False
|
|
assert outcome.reject_reason == "missing_keys: reason"
|
|
|
|
outcome = _parse(_payload(notes="extra"))
|
|
assert outcome.valid is False
|
|
assert outcome.reject_reason == "unexpected_keys: notes"
|
|
|
|
|
|
@pytest.mark.parametrize("field", ["incident_id", "classification", "first_failed_gate", "reason"])
|
|
def test_parse_rejects_non_string_fields(field: str) -> None:
|
|
outcome = _parse(_payload(**{field: 7}))
|
|
|
|
assert outcome.valid is False
|
|
assert outcome.reject_reason == f"field_type_invalid: {field} must be a string"
|
|
|
|
|
|
def test_parse_rejects_non_boolean_human_required_and_non_list_inferences() -> None:
|
|
outcome = _parse(_payload(human_required="yes"))
|
|
assert outcome.reject_reason == "field_type_invalid: human_required must be a boolean"
|
|
|
|
outcome = _parse(_payload(inferences={"not": "a list"}))
|
|
assert outcome.reject_reason == "field_type_invalid: inferences must be a list"
|
|
|
|
|
|
@pytest.mark.parametrize("confidence", ["high", None, True])
|
|
def test_parse_rejects_non_numeric_confidence(confidence) -> None: # type: ignore[no-untyped-def]
|
|
outcome = _parse(_payload(confidence=confidence))
|
|
|
|
assert outcome.valid is False
|
|
assert outcome.reject_reason == "field_type_invalid: confidence must be a number"
|
|
|
|
|
|
@pytest.mark.parametrize("confidence", [1.5, -0.2])
|
|
def test_parse_rejects_out_of_range_confidence(confidence: float) -> None:
|
|
outcome = _parse(_payload(confidence=confidence))
|
|
|
|
assert outcome.valid is False
|
|
assert outcome.reject_reason == f"confidence_out_of_range: {confidence}"
|
|
|
|
|
|
def test_parse_rejects_bad_facts() -> None:
|
|
outcome = _parse(_payload(facts={"not": "a list"}))
|
|
assert outcome.reject_reason == "field_type_invalid: facts must be a list"
|
|
|
|
outcome = _parse(_payload(facts=["not a dict"]))
|
|
assert outcome.reject_reason == "fact_invalid: facts[0] must be an object"
|
|
|
|
outcome = _parse(_payload(facts=[{"statement": "s", "source": "jenkins"}]))
|
|
assert outcome.reject_reason == "fact_invalid: facts[0] must have exactly statement, source, reference"
|
|
|
|
fact = {"statement": "s", "source": "jenkins", "reference": "r", "extra": "x"}
|
|
outcome = _parse(_payload(facts=[fact]))
|
|
assert outcome.reject_reason == "fact_invalid: facts[0] must have exactly statement, source, reference"
|
|
|
|
outcome = _parse(_payload(facts=[{"statement": 5, "source": "jenkins", "reference": "r"}]))
|
|
assert outcome.reject_reason == "fact_invalid: facts[0] fields must be strings"
|
|
|
|
|
|
def test_parse_rejects_unlisted_fact_source() -> None:
|
|
good = _payload()["facts"][0]
|
|
bad = dict(good, source="bing")
|
|
outcome = _parse(_payload(facts=[good, bad]))
|
|
|
|
assert outcome.valid is False
|
|
assert outcome.reject_reason == "fact_source_invalid: facts[1] source 'bing'"
|
|
|
|
|
|
def test_parse_rejects_malformed_requested_action() -> None:
|
|
outcome = _parse(_payload(requested_action=["run_ariadne_job"]))
|
|
assert outcome.reject_reason == "requested_action_invalid: must be an object or null"
|
|
|
|
outcome = _parse(_payload(requested_action={"type": "run_ariadne_job"}))
|
|
assert outcome.reject_reason == "requested_action_invalid: must have exactly type and id"
|
|
|
|
outcome = _parse(_payload(requested_action={"type": "run_ariadne_job", "id": "x", "why": "extra"}))
|
|
assert outcome.reject_reason == "requested_action_invalid: must have exactly type and id"
|
|
|
|
outcome = _parse(_payload(requested_action={"type": "delete_pod", "id": "x"}))
|
|
assert outcome.reject_reason == "requested_action_invalid: unsupported type 'delete_pod'"
|
|
|
|
outcome = _parse(_payload(requested_action={"type": "run_ariadne_job", "id": " "}))
|
|
assert outcome.reject_reason == "requested_action_invalid: id must be a non-empty string"
|
|
|
|
outcome = _parse(_payload(requested_action={"type": "run_ariadne_job", "id": 7}))
|
|
assert outcome.reject_reason == "requested_action_invalid: id must be a non-empty string"
|
|
|
|
|
|
def test_parse_rejects_incident_id_mismatch() -> None:
|
|
outcome = _parse(incident="inc-99")
|
|
|
|
assert outcome.valid is False
|
|
assert outcome.human_required is True
|
|
assert outcome.reject_reason == "incident_id_mismatch: got 'inc-42' expected 'inc-99'"
|
|
|
|
|
|
def test_authorize_all_gates_pass() -> None:
|
|
assert _authorize() == (True, "authorized")
|
|
|
|
|
|
def test_authorize_rejects_invalid_outcome() -> None:
|
|
allowed, reason = _authorize(outcome=_parse(raw="not json"))
|
|
|
|
assert allowed is False
|
|
assert reason == "response_invalid: no_json_object_found"
|
|
|
|
|
|
def test_authorize_rejects_human_required_decision() -> None:
|
|
allowed, reason = _authorize(outcome=_parse(_payload(human_required=True)))
|
|
|
|
assert allowed is False
|
|
assert reason == "human_required"
|
|
|
|
|
|
def test_authorize_requires_terminal_failed_build() -> None:
|
|
assert _authorize(build_is_terminal_failure=False) == (False, "build_not_terminal_failure")
|
|
|
|
|
|
def test_authorize_requires_allowlisted_job() -> None:
|
|
assert _authorize(job_allowlisted=False) == (False, "job_not_allowlisted")
|
|
|
|
|
|
def test_authorize_requires_expected_classification() -> None:
|
|
allowed, reason = _authorize(outcome=_parse(_payload(classification="novel_failure")))
|
|
|
|
assert allowed is False
|
|
assert reason == "classification_mismatch: got 'novel_failure' expected 'known_demo_fixture_failure'"
|
|
|
|
|
|
def test_authorize_requires_requested_action() -> None:
|
|
allowed, reason = _authorize(outcome=_parse(_payload(requested_action=None)))
|
|
|
|
assert allowed is False
|
|
assert reason == "requested_action_missing"
|
|
|
|
|
|
def test_authorize_rejects_wrong_action_type_on_handcrafted_decision() -> None:
|
|
base = _parse().decision
|
|
assert base is not None
|
|
decision = module.TriageDecision(
|
|
incident_id=base.incident_id,
|
|
classification=base.classification,
|
|
confidence=base.confidence,
|
|
facts=base.facts,
|
|
inferences=base.inferences,
|
|
first_failed_gate=base.first_failed_gate,
|
|
requested_action=module.RequestedAction(type="delete_everything", id="demo-fixture-reset"),
|
|
human_required=False,
|
|
reason=base.reason,
|
|
)
|
|
outcome = module.DecisionOutcome(valid=True, decision=decision, human_required=False, reject_reason=None)
|
|
|
|
assert _authorize(outcome=outcome) == (False, "requested_action_type_invalid")
|
|
|
|
|
|
def test_authorize_requires_action_in_allowlist() -> None:
|
|
allowed, reason = _authorize(cfg=_cfg(allowed_actions=["other-job"]))
|
|
|
|
assert allowed is False
|
|
assert reason == "action_not_allowlisted: 'demo-fixture-reset'"
|
|
|
|
|
|
def test_authorize_requires_minimum_confidence() -> None:
|
|
allowed, reason = _authorize(outcome=_parse(_payload(confidence=0.5)))
|
|
|
|
assert allowed is False
|
|
assert reason == "confidence_below_minimum: 0.5 < 0.8"
|
|
|
|
|
|
def test_authorize_requires_evidence_signature() -> None:
|
|
assert _authorize(evidence_has_signature=False) == (False, "evidence_signature_missing")
|
|
|
|
|
|
def test_authorize_enforces_action_budget() -> None:
|
|
assert _authorize(prior_action_count=1) == (False, "max_actions_reached")
|
|
assert _authorize(prior_action_count=0, cfg=_cfg(max_actions_per_incident=2)) == (True, "authorized")
|
|
|
|
|
|
def test_authorize_requires_autoremediation_enabled() -> None:
|
|
allowed, reason = _authorize(cfg=_cfg(autoremediation_enabled=False))
|
|
|
|
assert allowed is False
|
|
assert reason == "autoremediation_disabled"
|
|
|
|
|
|
def test_authorize_names_first_failing_gate_in_order() -> None:
|
|
allowed, reason = _authorize(
|
|
outcome=_parse(_payload(human_required=True)),
|
|
cfg=_cfg(autoremediation_enabled=False),
|
|
build_is_terminal_failure=False,
|
|
evidence_has_signature=False,
|
|
)
|
|
|
|
assert allowed is False
|
|
assert reason == "human_required"
|
|
|
|
|
|
def test_authorize_with_empty_cfg_defaults_closed() -> None:
|
|
allowed, reason = _authorize(cfg={})
|
|
|
|
assert allowed is False
|
|
assert reason == "action_not_allowlisted: 'demo-fixture-reset'"
|
|
|
|
|
|
def test_authorize_falls_back_to_conservative_cfg_values() -> None:
|
|
cfg = _cfg(min_confidence="high", max_actions_per_incident="many")
|
|
allowed, reason = _authorize(cfg=cfg)
|
|
|
|
assert allowed is False
|
|
assert reason == "confidence_below_minimum: 0.93 < 1.0"
|
|
|
|
|
|
def _mapping_cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
|
|
base = {
|
|
"allowed_actions": ["repair_demo_fixture", "retry_transient_infra"],
|
|
"action_classifications": {
|
|
"known_demo_fixture_failure": "repair_demo_fixture",
|
|
"transient_infra_failure": "retry_transient_infra",
|
|
},
|
|
}
|
|
base.update(overrides)
|
|
return _cfg(**base)
|
|
|
|
|
|
def _mapped_payload(classification: str, action_id: str) -> dict:
|
|
return _payload(
|
|
classification=classification,
|
|
requested_action={"type": "run_ariadne_job", "id": action_id},
|
|
)
|
|
|
|
|
|
def test_authorize_accepts_each_registered_classification() -> None:
|
|
for classification, action_id in _mapping_cfg()["action_classifications"].items():
|
|
outcome = _parse(_mapped_payload(classification, action_id))
|
|
assert _authorize(outcome=outcome, cfg=_mapping_cfg()) == (True, "authorized")
|
|
|
|
|
|
def test_authorize_rejects_unregistered_classification() -> None:
|
|
outcome = _parse(_mapped_payload("novel_failure", "retry_transient_infra"))
|
|
|
|
allowed, reason = _authorize(outcome=outcome, cfg=_mapping_cfg())
|
|
|
|
assert allowed is False
|
|
assert reason == "classification_not_supported: 'novel_failure'"
|
|
|
|
|
|
def test_authorize_rejects_action_mapped_to_another_classification() -> None:
|
|
outcome = _parse(_mapped_payload("transient_infra_failure", "repair_demo_fixture"))
|
|
|
|
allowed, reason = _authorize(outcome=outcome, cfg=_mapping_cfg())
|
|
|
|
assert allowed is False
|
|
assert reason == (
|
|
"action_does_not_match_classification: 'repair_demo_fixture' expected 'retry_transient_infra'"
|
|
)
|
|
|
|
|
|
def test_authorize_still_requires_the_mapped_action_to_be_allowlisted() -> None:
|
|
outcome = _parse(_mapped_payload("transient_infra_failure", "retry_transient_infra"))
|
|
cfg = _mapping_cfg(allowed_actions=["repair_demo_fixture"])
|
|
|
|
allowed, reason = _authorize(outcome=outcome, cfg=cfg)
|
|
|
|
assert allowed is False
|
|
assert reason == "action_not_allowlisted: 'retry_transient_infra'"
|
|
|
|
|
|
def test_authorize_reports_missing_action_before_the_mapping_gate() -> None:
|
|
outcome = _parse(_payload(classification="transient_infra_failure", requested_action=None))
|
|
|
|
assert _authorize(outcome=outcome, cfg=_mapping_cfg()) == (False, "requested_action_missing")
|
|
|
|
|
|
def test_authorize_mapping_still_enforces_the_remaining_gates() -> None:
|
|
outcome = _parse(_mapped_payload("transient_infra_failure", "retry_transient_infra"))
|
|
|
|
assert _authorize(outcome=outcome, cfg=_mapping_cfg(), evidence_has_signature=False) == (
|
|
False,
|
|
"evidence_signature_missing",
|
|
)
|
|
assert _authorize(outcome=outcome, cfg=_mapping_cfg(), prior_action_count=1) == (
|
|
False,
|
|
"max_actions_reached",
|
|
)
|
|
assert _authorize(outcome=outcome, cfg=_mapping_cfg(autoremediation_enabled=False)) == (
|
|
False,
|
|
"autoremediation_disabled",
|
|
)
|
|
|
|
|
|
def test_authorize_ignores_an_empty_or_malformed_mapping() -> None:
|
|
for mapping in ({}, None, "known_demo_fixture_failure=repair_demo_fixture", []):
|
|
cfg = _cfg(action_classifications=mapping)
|
|
assert _authorize(cfg=cfg) == (True, "authorized")
|
|
allowed, reason = _authorize(outcome=_parse(_payload(classification="other")), cfg=cfg)
|
|
assert allowed is False
|
|
assert reason == "classification_mismatch: got 'other' expected 'known_demo_fixture_failure'"
|