9f24f8d999
All checks were successful
Tests / Declarative: Post Actions passed: 1275
Every action in the allowlist got there because someone hit the failure by hand, recognised the pattern, and wired a job for it. That loop only closes if a person happens to read enough issues to notice the same failure recurring, so a failure nobody reviews twice never earns an action. Until now a diagnosis that fit nothing could only say a human was needed; it could not say what the human should build. An optional suggested_remediation field closes the other half of the loop. When nothing in the allowlist fits, Hermes may name the remediation it believes would work and the evidence that should be required before running it is safe. It lands in the incident issue under a heading that states plainly the remediation does not exist and was not performed, and in the audit event, where the same proposal recurring across unrelated incidents is the evidence that building it is worth the effort. The field is inert by construction. No gate reads it, and an id that is not already allowlisted still fails action_not_allowlisted exactly as before - naming a remediation and being granted one stay different things, and only the second needs a human to change a deployment. It is rejected outright alongside a requested action: the field reports that nothing fit, so something fitting contradicts it, and allowing both would invite a rationale to be attached to a request the gates must judge on evidence alone. The key is optional rather than required so a response written before today still validates unchanged, and unknown keys are still refused. Also names reclaim_workspace_storage, clear_stuck_agent_pods and abort_hung_build in KNOWN_ACTION_LABELS. They were reporting as "unknown" in metrics on any deployment that had not enabled them, which is the one case where you most want to see the real name. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
75 lines
3.0 KiB
Python
75 lines
3.0 KiB
Python
"""Tests for the frozen Hermes triage prompt and its job scoping."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from ariadne.services import hermes_triage_prompt
|
|
|
|
|
|
BUNDLE = {"incident_id": "metis/272", "jenkins": {"job": "metis"}}
|
|
|
|
|
|
def test_demo_job_prompt_permits_the_demo_action() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt(
|
|
"hermes-triage-demo/12", "hermes-triage-demo", {"incident_id": "hermes-triage-demo/12"}
|
|
)
|
|
assert "for the Jenkins job hermes-triage-demo" in prompt
|
|
assert "known_demo_fixture_failure" in prompt
|
|
assert "repair_demo_fixture" in prompt
|
|
|
|
|
|
def test_real_job_prompt_forbids_the_demo_classification() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert "for the Jenkins job metis" in prompt
|
|
# The fixture rules are not stated at all, so they cannot be repeated into
|
|
# a real service's issue tracker.
|
|
assert "known_demo_fixture_failure" not in prompt
|
|
assert "repair_demo_fixture" not in prompt
|
|
assert "hermes-triage-demo" not in prompt
|
|
|
|
|
|
def test_bundle_is_appended_compactly() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert prompt.endswith('Bundle:\n{"incident_id":"metis/272","jenkins":{"job":"metis"}}')
|
|
|
|
|
|
def test_no_placeholders_survive_rendering() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
for placeholder in ("__INCIDENT_ID__", "__JOB__", "__DEMO_JOB__", "__BUNDLE__"):
|
|
assert placeholder not in prompt
|
|
|
|
|
|
def test_a_real_service_prompt_never_mentions_the_triage_system() -> None:
|
|
"""Whatever Hermes reasons about is published into the service's tracker.
|
|
|
|
Policy talk belongs to Ariadne; in a service repository it reads as though
|
|
the system exists to serve a demonstration.
|
|
"""
|
|
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
for leak in ("demo", "fixture", "allowlist", "hermes-triage-demo"):
|
|
assert leak not in prompt.lower(), leak
|
|
|
|
|
|
def test_the_prompt_asks_for_maintainer_facing_language() -> None:
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert "who has no knowledge of how this triage system is configured" in prompt
|
|
assert "Do not discuss classifications, actions, policies" in prompt
|
|
|
|
|
|
def test_structured_test_evidence_is_pointed_at_first() -> None:
|
|
"""failed_tests is better evidence than console text and must be preferred."""
|
|
|
|
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
|
|
assert prompt.index("jenkins.failed_tests") < prompt.index("jenkins.console_failures")
|
|
|
|
|
|
def test_prompt_offers_a_way_to_propose_a_remediation_that_does_not_exist() -> None:
|
|
"""Hermes cannot report a gap in the allowlist unless it is told it may."""
|
|
|
|
prompt = hermes_triage_prompt.build_prompt("lesavka/9", "lesavka", {})
|
|
|
|
assert '"suggested_remediation": <object or null>' in prompt
|
|
assert "evidence_required" in prompt
|
|
assert "must be null whenever requested_action is set" in prompt
|
|
assert "does not request anything" in prompt
|