ariadne/tests/test_hermes_autotriage_retry.py

from __future__ import annotations

from ariadne.services import hermes_autotriage as module
from tests.hermes_autotriage_harness import (
    INCIDENT_ID,
    INFRA_MARKER,
    JOB,
    _build,
    _counter,
    _events,
    _gauge,
    _prepare,
    _retry_output,
    _run,
    _seed_incident,
    _settings,
    _statuses,
)


OTHER_JOB = "titan-iac"
BOTH_ACTIONS = ["repair_demo_fixture", "retry_transient_infra"]


def _retry_settings(**overrides):  # type: ignore[no-untyped-def]
    values = {"hermes_allowed_actions": BOTH_ACTIONS}
    values.update(overrides)
    return _settings(**values)


def _retry_env(monkeypatch, **overrides):  # type: ignore[no-untyped-def]
    kwargs = {"cfg": _retry_settings(), "run": _run(output=_retry_output())}
    kwargs.update(overrides)
    return _prepare(monkeypatch, **kwargs)


def test_transient_infra_failure_is_retried(monkeypatch) -> None:
    success_before = _counter("retry_transient_infra", "success")
    env = _retry_env(monkeypatch)

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB] == {
        "status": "awaiting_rebuild",
        "incident_id": INCIDENT_ID,
        "action": "retry_transient_infra",
        "evidence_marker": INFRA_MARKER,
    }
    assert _statuses(env.storage) == ["detected", "diagnosed", "awaiting_rebuild"]
    assert env.calls["retries"] == [(JOB, True)]
    assert env.calls["repairs"] == []
    assert env.calls["rebuilds"] == []
    assert _counter("retry_transient_infra", "success") == success_before + 1.0
    assert _gauge("12", "awaiting_rebuild") == 1.0


def test_retry_records_the_matched_marker_on_every_event(monkeypatch) -> None:
    env = _retry_env(monkeypatch)

    module.run_hermes_autotriage(env.storage)

    actions = _events(env.storage, module.ACTION_EVENT_TYPE)
    assert [action["result"] for action in actions] == ["requested", "accepted", "executed"]
    assert all(action["action"] == "retry_transient_infra" for action in actions)
    assert actions[0]["detail"] == {"evidence_marker": INFRA_MARKER}
    assert actions[2]["detail"] == {"evidence_marker": INFRA_MARKER}
    diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0]
    assert diagnosis["authorized"] is True
    assert diagnosis["authorize_reason"] == "authorized"
    assert diagnosis["evidence_marker"] == INFRA_MARKER
    assert diagnosis["outcome"]["classification"] == "transient_infra_failure"
    incident = _events(env.storage, module.INCIDENT_EVENT_TYPE)[-1]
    assert incident["phase"] == {"action": "retry_transient_infra", "evidence_marker": INFRA_MARKER}


def test_retry_resolves_on_the_next_green_build(monkeypatch) -> None:
    env = _retry_env(monkeypatch)
    module.run_hermes_autotriage(env.storage)

    green = _prepare(
        monkeypatch,
        cfg=_retry_settings(),
        last_build=_build(13, "SUCCESS"),
        storage=env.storage,
    )
    summary = module.run_hermes_autotriage(green.storage)

    assert summary["jobs"][JOB] == {"status": "healthy", "resolved": [INCIDENT_ID]}
    assert _statuses(env.storage)[-1] == "resolved"
    assert _gauge("12", "resolved") == 1.0


def test_unparameterized_job_is_retried_without_parameters(monkeypatch) -> None:
    cfg = _retry_settings(
        hermes_autotriage_job_allowlist=[OTHER_JOB],
        hermes_parameterized_jobs=[JOB],
    )
    env = _prepare(
        monkeypatch,
        cfg=cfg,
        last_build=_build(12, "FAILURE", job=OTHER_JOB),
        run=_run(output=_retry_output(job=OTHER_JOB)),
    )

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][OTHER_JOB]["status"] == "awaiting_rebuild"
    assert env.calls["retries"] == [(OTHER_JOB, False)]


def test_retry_without_infra_signature_requires_a_human(monkeypatch) -> None:
    env = _retry_env(monkeypatch, infra=(False, None))

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing"
    assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"]
    assert env.calls["retries"] == []
    assert _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0]["evidence_marker"] is None


def test_retry_ignores_the_fixture_signature(monkeypatch) -> None:
    env = _retry_env(monkeypatch, signature=False)

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB]["status"] == "awaiting_rebuild"
    assert env.calls["retries"] == [(JOB, True)]


def test_fixture_action_ignores_the_infra_signature(monkeypatch) -> None:
    env = _prepare(monkeypatch, cfg=_retry_settings(), signature=False, infra=(True, INFRA_MARKER))

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing"
    assert env.calls["retries"] == []


def test_retry_is_capped_at_one_action_per_incident(monkeypatch) -> None:
    env = _retry_env(monkeypatch)
    _seed_incident(env.storage, "detected")
    env.storage.record_event(
        module.ACTION_EVENT_TYPE,
        {"incident_id": INCIDENT_ID, "action": "retry_transient_infra", "result": "requested"},
    )

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB]["reason"] == "max_actions_reached"
    assert env.calls["retries"] == []


def test_retry_trigger_failure_marks_failed_and_human(monkeypatch) -> None:
    failed_before = _counter("retry_transient_infra", "failed")
    env = _retry_env(monkeypatch, retry={"requested": False, "error": "retry http 500"})

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB] == {
        "status": "failed",
        "incident_id": INCIDENT_ID,
        "reason": "retry http 500",
    }
    assert _statuses(env.storage) == ["detected", "diagnosed", "failed"]
    actions = _events(env.storage, module.ACTION_EVENT_TYPE)
    assert [action["result"] for action in actions] == ["requested", "accepted", "failed"]
    assert _counter("retry_transient_infra", "failed") == failed_before + 1.0
    assert _gauge("12", "human_required") == 1.0


def test_retry_action_is_rejected_while_not_allowlisted(monkeypatch) -> None:
    rejected_before = _counter("retry_transient_infra", "rejected")
    env = _prepare(monkeypatch, run=_run(output=_retry_output()))

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB]["reason"] == "action_not_allowlisted: 'retry_transient_infra'"
    assert env.calls["retries"] == []
    assert _counter("retry_transient_infra", "rejected") == rejected_before + 1.0


def test_action_must_match_its_classification(monkeypatch) -> None:
    output = _retry_output(requested_action={"type": "run_ariadne_job", "id": "repair_demo_fixture"})
    env = _retry_env(monkeypatch, run=_run(output=output))

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB]["reason"] == (
        "action_does_not_match_classification: 'repair_demo_fixture' expected 'retry_transient_infra'"
    )
    assert env.calls["retries"] == []
    assert env.calls["repairs"] == []


def test_unregistered_classification_requires_a_human(monkeypatch) -> None:
    env = _retry_env(monkeypatch, run=_run(output=_retry_output(classification="flaky_network_guess")))

    summary = module.run_hermes_autotriage(env.storage)

    assert summary["jobs"][JOB]["reason"] == "classification_not_supported: 'flaky_network_guess'"
    assert env.calls["retries"] == []


def test_prompt_documents_the_retry_action(monkeypatch) -> None:
    env = _retry_env(monkeypatch)
    module.run_hermes_autotriage(env.storage)

    _, prompt = env.calls["triage"][0]
    assert "Use classification transient_infra_failure with requested_action" in prompt
    assert '{"type": "run_ariadne_job", "id": "retry_transient_infra"}' in prompt
    assert "unrelated to the repository's code or tests" in prompt
    assert "Otherwise leave requested_action null" in prompt