9c17ed5196
Hermes classified a real bstein-dev-home build as known_demo_fixture_failure at 0.99 confidence because a missing node binary superficially resembled the fixture signature. Ariadne's evidence gate correctly refused the action, but the filed issue carried a meaningless classification. Extract the frozen prompt into its own module and interpolate the job under analysis, so the demo classification and repair_demo_fixture action are forbidden outright on any job other than hermes-triage-demo. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
38 lines
1.5 KiB
Python
38 lines
1.5 KiB
Python
"""Tests for the frozen Hermes triage prompt and its job scoping."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from ariadne.services import hermes_triage_prompt
|
|
|
|
|
|
BUNDLE = {"incident_id": "metis/272", "jenkins": {"job": "metis"}}
|
|
|
|
|
|
def test_demo_job_prompt_permits_the_demo_action() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt(
|
|
"hermes-triage-demo/12", "hermes-triage-demo", {"incident_id": "hermes-triage-demo/12"}
|
|
)
|
|
assert "The Jenkins job under analysis is hermes-triage-demo." in prompt
|
|
assert "Set human_required to false when hermes-triage-demo is hermes-triage-demo" in prompt
|
|
|
|
|
|
def test_real_job_prompt_forbids_the_demo_classification() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert "The Jenkins job under analysis is metis." in prompt
|
|
assert (
|
|
"reserved for the job hermes-triage-demo. When metis is any other job they are forbidden"
|
|
in prompt
|
|
)
|
|
assert "Set human_required to false when metis is hermes-triage-demo" in prompt
|
|
|
|
|
|
def test_bundle_is_appended_compactly() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert prompt.endswith('Bundle:\n{"incident_id":"metis/272","jenkins":{"job":"metis"}}')
|
|
|
|
|
|
def test_no_placeholders_survive_rendering() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
for placeholder in ("__INCIDENT_ID__", "__JOB__", "__DEMO_JOB__", "__BUNDLE__"):
|
|
assert placeholder not in prompt
|