d3a2c94f80
All checks were successful
Tests / Declarative: Post Actions passed: 1205
Two changes to how this reads and behaves on real service repositories. The fixture rules were stated to Hermes on every job, so it reasoned about them out loud and that reasoning was published verbatim into service issue trackers - ariadne/404 opened with 'The job is ariadne, not hermes-triage-demo, so the reserved demo fixture classification and repair action are forbidden'. That reads as though the system exists to serve a demonstration. Those rules are now appended only for the fixture job, so a real service is never told about them and cannot repeat them; the demo classification is unreachable elsewhere by construction rather than by instruction. The prompt also asks for language aimed at a maintainer who knows nothing about how triage is configured, and points at the structured test evidence first now that junit publishes it. The duplicate guard refused a proposal whenever any repair pull request was open, which meant one unreviewed fix blocked every later one across the repository. It now enforces a ceiling instead, ARIADNE_HERMES_CODE_MAX_OPEN_PROPOSALS, default 64. That is a review-capacity limit, not a correctness one: proposals are cheap to make and expensive to read. Auto-triage settings move to their own module; they had grown a section's worth and pushed settings_sections.py past its size budget. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
64 lines
2.6 KiB
Python
64 lines
2.6 KiB
Python
"""Tests for the frozen Hermes triage prompt and its job scoping."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from ariadne.services import hermes_triage_prompt
|
|
|
|
|
|
BUNDLE = {"incident_id": "metis/272", "jenkins": {"job": "metis"}}
|
|
|
|
|
|
def test_demo_job_prompt_permits_the_demo_action() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt(
|
|
"hermes-triage-demo/12", "hermes-triage-demo", {"incident_id": "hermes-triage-demo/12"}
|
|
)
|
|
assert "for the Jenkins job hermes-triage-demo" in prompt
|
|
assert "known_demo_fixture_failure" in prompt
|
|
assert "repair_demo_fixture" in prompt
|
|
|
|
|
|
def test_real_job_prompt_forbids_the_demo_classification() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert "for the Jenkins job metis" in prompt
|
|
# The fixture rules are not stated at all, so they cannot be repeated into
|
|
# a real service's issue tracker.
|
|
assert "known_demo_fixture_failure" not in prompt
|
|
assert "repair_demo_fixture" not in prompt
|
|
assert "hermes-triage-demo" not in prompt
|
|
|
|
|
|
def test_bundle_is_appended_compactly() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert prompt.endswith('Bundle:\n{"incident_id":"metis/272","jenkins":{"job":"metis"}}')
|
|
|
|
|
|
def test_no_placeholders_survive_rendering() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
for placeholder in ("__INCIDENT_ID__", "__JOB__", "__DEMO_JOB__", "__BUNDLE__"):
|
|
assert placeholder not in prompt
|
|
|
|
|
|
def test_a_real_service_prompt_never_mentions_the_triage_system() -> None:
|
|
"""Whatever Hermes reasons about is published into the service's tracker.
|
|
|
|
Policy talk belongs to Ariadne; in a service repository it reads as though
|
|
the system exists to serve a demonstration.
|
|
"""
|
|
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
for leak in ("demo", "fixture", "allowlist", "hermes-triage-demo"):
|
|
assert leak not in prompt.lower(), leak
|
|
|
|
|
|
def test_the_prompt_asks_for_maintainer_facing_language() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert "who has no knowledge of how this triage system is configured" in prompt
|
|
assert "Do not discuss classifications, actions, policies" in prompt
|
|
|
|
|
|
def test_structured_test_evidence_is_pointed_at_first() -> None:
|
|
"""failed_tests is better evidence than console text and must be preferred."""
|
|
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert prompt.index("jenkins.failed_tests") < prompt.index("jenkins.console_failures")
|