ariadne/tests/test_hermes_triage_prompt.py

"""Tests for the frozen Hermes triage prompt and its job scoping."""

from __future__ import annotations

from ariadne.services import hermes_triage_prompt


BUNDLE = {"incident_id": "metis/272", "jenkins": {"job": "metis"}}


def test_demo_job_prompt_permits_the_demo_action() -> None:
    prompt = hermes_triage_prompt.build_prompt(
        "hermes-triage-demo/12", "hermes-triage-demo", {"incident_id": "hermes-triage-demo/12"}
    )
    assert "for the Jenkins job hermes-triage-demo" in prompt
    assert "known_demo_fixture_failure" in prompt
    assert "repair_demo_fixture" in prompt


def test_real_job_prompt_forbids_the_demo_classification() -> None:
    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
    assert "for the Jenkins job metis" in prompt
    # The fixture rules are not stated at all, so they cannot be repeated into
    # a real service's issue tracker.
    assert "known_demo_fixture_failure" not in prompt
    assert "repair_demo_fixture" not in prompt
    assert "hermes-triage-demo" not in prompt


def test_bundle_is_appended_compactly() -> None:
    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
    assert prompt.endswith('Bundle:\n{"incident_id":"metis/272","jenkins":{"job":"metis"}}')


def test_no_placeholders_survive_rendering() -> None:
    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
    for placeholder in ("__INCIDENT_ID__", "__JOB__", "__DEMO_JOB__", "__BUNDLE__"):
        assert placeholder not in prompt


def test_a_real_service_prompt_never_mentions_the_triage_system() -> None:
    """Whatever Hermes reasons about is published into the service's tracker.

    Policy talk belongs to Ariadne; in a service repository it reads as though
    the system exists to serve a demonstration.
    """

    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
    for leak in ("demo", "fixture", "allowlist", "hermes-triage-demo"):
        assert leak not in prompt.lower(), leak


def test_the_prompt_asks_for_maintainer_facing_language() -> None:
    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
    assert "who has no knowledge of how this triage system is configured" in prompt
    assert "Do not discuss classifications, actions, policies" in prompt


def test_structured_test_evidence_is_pointed_at_first() -> None:
    """failed_tests is better evidence than console text and must be preferred."""

    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
    assert prompt.index("jenkins.failed_tests") < prompt.index("jenkins.console_failures")