bstein/ariadne
1
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(hermes): stop the triage system talking about itself, and raise the PR ceiling
All checks were successful
Tests / Declarative: Post Actions passed: 1205
Details

Browse Source 
Two changes to how this reads and behaves on real service repositories.

The fixture rules were stated to Hermes on every job, so it reasoned about
them out loud and that reasoning was published verbatim into service issue
trackers - ariadne/404 opened with 'The job is ariadne, not
hermes-triage-demo, so the reserved demo fixture classification and repair
action are forbidden'. That reads as though the system exists to serve a
demonstration. Those rules are now appended only for the fixture job, so a
real service is never told about them and cannot repeat them; the demo
classification is unreachable elsewhere by construction rather than by
instruction. The prompt also asks for language aimed at a maintainer who
knows nothing about how triage is configured, and points at the structured
test evidence first now that junit publishes it.

The duplicate guard refused a proposal whenever any repair pull request was
open, which meant one unreviewed fix blocked every later one across the
repository. It now enforces a ceiling instead, ARIADNE_HERMES_CODE_MAX_OPEN_PROPOSALS,
default 64. That is a review-capacity limit, not a correctness one: proposals
are cheap to make and expensive to read.

Auto-triage settings move to their own module; they had grown a section's
worth and pushed settings_sections.py past its size budget.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
codex 2026-08-06 13:08:18 -03:00
parent 136caa8477
commit d3a2c94f80
12 changed files with 192 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
ariadne/services/hermes_code_flow.py
View File

@ -37,7 +37,10 @@ logger = get_logger(__name__)
CODE_PROPOSAL_EVENT_TYPE = "hermes_autotriage_code_proposal"
_RUN_COMPLETED = "completed"
_EXISTING_PROPOSAL_REASON = "existing_proposal_open"
_PROPOSAL_LIMIT_REASON = "open_proposal_limit_reached"
# Review capacity, not correctness: proposals are cheap to make and
# expensive to read, so the ceiling protects the reviewer.
DEFAULT_MAX_OPEN_PROPOSALS = 64
_NO_REPO_MAPPING_REASON = "no_repo_mapping"
_NO_CANDIDATE_FILES_REASON = "no_candidate_files"
@ -368,16 +371,22 @@ def _duplicate_proposal(
return None
if not existing.get("found"):
return None
# Several proposals may sit open at once; only the ceiling suppresses new
# work. Below it, an open proposal on the same repository is no reason to
# withhold a fix for a different failure.
limit = _int_value(code_cfg.get("max_open_proposals")) or DEFAULT_MAX_OPEN_PROPOSALS
if _int_value(existing.get("open_count")) < limit:
return None
identity = {
"pr_number": existing.get("pr_number"),
"url": existing.get("url"),
"branch": existing.get("branch"),
}
result = {"status": "human_required", "reason": _EXISTING_PROPOSAL_REASON, **identity}
result = {"status": "human_required", "reason": _PROPOSAL_LIMIT_REASON, **identity}
event = {
"run_id": None,
"validated": False,
"reject_reason": _EXISTING_PROPOSAL_REASON,
"reject_reason": _PROPOSAL_LIMIT_REASON,
**identity,
}
return result, event

5
ariadne/services/hermes_code_repair.py
View File

@ -208,10 +208,11 @@ def _oldest_repair_pull(response: Any, base_branch: str) -> dict[str, Any]:
return _no_proposal("open proposal payload is not a list")
matches = [pull for pull in payload if _is_repair_pull(pull, base_branch)]
if not matches:
return _no_proposal(None)
return {**_no_proposal(None), "open_count": 0}
oldest = min(matches, key=lambda pull: int(pull["number"]))
return {
"found": True,
"open_count": len(matches),
"pr_number": int(oldest["number"]),
"url": str(oldest.get("html_url") or "") or None,
"branch": str(oldest["head"].get("ref") or "") or None,
@ -237,7 +238,7 @@ def _is_repair_pull(pull: Any, base_branch: str) -> bool:
def _no_proposal(error: str | None) -> dict[str, Any]:
"""Build the fail-open result meaning "no existing proposal found"."""
return {"found": False, "pr_number": None, "url": None, "branch": None, "error": error}
return {"found": False, "open_count": 0, "pr_number": None, "url": None, "branch": None, "error": error}
def _pr_result(response: Any) -> dict[str, Any]:

1
ariadne/services/hermes_code_repos.py
View File

@ -47,6 +47,7 @@ def build_config(config: Any) -> dict[str, Any]:
"gitea_token": config.hermes_gitea_token,
"owner": config.hermes_code_owner,
"repo": config.hermes_code_repo,
"max_open_proposals": getattr(config, "hermes_code_max_open_proposals", 0),
"base_branch": config.hermes_code_base_branch,
"timeout_seconds": _GITEA_TIMEOUT_SECONDS,
"legacy_job": str(getattr(config, "hermes_code_job", "") or ""),

43
ariadne/services/hermes_triage_prompt.py
View File

@ -2,6 +2,11 @@
The prompt lives in its own module so its wording is reviewed as a unit and
so changing it never competes for room with the orchestrator's logic.
The fixture-specific rules are appended only for the fixture job. Stating them
to every job made Hermes reason about them out loud, and that reasoning was
published verbatim into issues and pull requests on real service repositories,
which read as though the whole system existed to serve a demonstration.
"""
from __future__ import annotations
@ -12,22 +17,27 @@ from typing import Any
DEMO_JOB = "hermes-triage-demo"
_TEMPLATE = """Use $triage-titan-test-failures.
Analyze incident __INCIDENT_ID__.
_BASE = """Use $triage-titan-test-failures.
Analyze incident __INCIDENT_ID__ for the Jenkins job __JOB__.
Treat the attached Ariadne bundle as the source of truth.
Identify the first enforced failure.
jenkins.failed_tests carries the structured test results when the build published any; when it is populated it names the failing test, its class, and its assertion, and it is better evidence than console text.
jenkins.first_failed_stage names the pipeline stage that failed when the build reported stages.
The jenkins.console_failures array holds excerpts around detected failure markers in chronological order; the earliest region usually contains the first enforced failure, and jenkins.console_tail is the end of the build which often only shows downstream noise.
Distinguish facts from inference.
Return ONLY a single JSON object with exactly these keys and no others:
{"incident_id": "<must equal __INCIDENT_ID__>", "classification": "<string>", "confidence": <0..1>, "facts": [{"statement": "...", "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", "reference": "..."}], "inferences": ["..."], "first_failed_gate": "<string>", "requested_action": {"type": "run_ariadne_job", "id": "repair_demo_fixture"} or null, "human_required": <bool>, "reason": "<string>"}
You are diagnosing only; you do not execute anything. Ariadne separately validates and executes the requested action under its own authorization policy.
The Jenkins job under analysis is __JOB__.
The classification known_demo_fixture_failure and the action repair_demo_fixture are reserved for the job __DEMO_JOB__. When __JOB__ is any other job they are forbidden, no matter how closely the failure resembles that fixture; classify what the evidence actually shows instead, naming the failing tool or gate.
Set human_required to false when __JOB__ is __DEMO_JOB__ and the evidence matches the demo fixture signature, because the appropriate response is then the predefined repair_demo_fixture action.
Set human_required to true only when the failure does not match a known signature, decisive evidence is missing, or no allowlisted action fits.
{"incident_id": "<must equal __INCIDENT_ID__>", "classification": "<short snake_case name for what actually failed>", "confidence": <0..1>, "facts": [{"statement": "...", "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", "reference": "..."}], "inferences": ["..."], "first_failed_gate": "<string>", "requested_action": <object or null>, "human_required": <bool>, "reason": "<string>"}
You are diagnosing only; you do not execute anything. Ariadne separately validates and executes any requested action under its own authorization policy, and will refuse anything its own reading of the evidence does not support.
Write the reason and the inferences for an engineer who maintains this service and who has no knowledge of how this triage system is configured. Describe the failure and what a fix would involve. Do not discuss classifications, actions, policies, or which of them are permitted; those are Ariadne's concern and are meaningless in the service's issue tracker.
Use classification transient_infra_failure with requested_action {"type": "run_ariadne_job", "id": "retry_transient_infra"} only when the evidence shows an infrastructure, connectivity, or registry error unrelated to the repository's code or tests (DNS resolution failure, connection refused, reset, or timed out, TLS handshake failure, image pull failure, or a 5xx from a registry or SCM host), because re-running the same commit is then the whole remediation.
Otherwise leave requested_action null and set human_required per the rules above.
Do not perform mutations.
Otherwise leave requested_action null.
Set human_required to true when decisive evidence is missing or the failure needs a judgement only a maintainer can make; otherwise set it false and say plainly what you believe is wrong.
Do not perform mutations."""
_FIXTURE_RULES = """
This job is the fixture job. Use classification known_demo_fixture_failure with requested_action {"type": "run_ariadne_job", "id": "repair_demo_fixture"} when the evidence matches the fixture unhealthy signature, and set human_required false, because the predefined repair is then the whole remediation."""
_BUNDLE_SUFFIX = """
Bundle:
__BUNDLE__"""
@ -37,15 +47,18 @@ def build_prompt(incident_id: str, job: str, bundle: dict[str, Any]) -> str:
"""Render the frozen triage prompt for one incident.
Inputs: the incident id, the Jenkins job the incident belongs to, and the
evidence bundle. Outputs: the prompt string sent to Hermes. The job name is
interpolated so the demo-only classification and action stay scoped to the
demo job rather than leaking onto real services.
evidence bundle. Outputs: the prompt string sent to Hermes.
The fixture rules are appended only for the fixture job, so a real service
is never told about them and cannot repeat them into its own issues. The
demo classification therefore cannot be produced elsewhere by construction
rather than by instruction.
"""
template = _BASE + (_FIXTURE_RULES if job == DEMO_JOB else "") + _BUNDLE_SUFFIX
compact = json.dumps(bundle, separators=(",", ":"), ensure_ascii=True)
return (
_TEMPLATE.replace("__INCIDENT_ID__", incident_id)
.replace("__DEMO_JOB__", DEMO_JOB)
template.replace("__INCIDENT_ID__", incident_id)
.replace("__JOB__", job)
.replace("__BUNDLE__", compact)
)

1
ariadne/settings.py
View File

@ -192,6 +192,7 @@ class Settings:
hermes_hung_build_minutes: float
hermes_max_branches: int
hermes_job_namespaces: dict
hermes_code_max_open_proposals: int
hermes_max_actions_per_incident: int
hermes_api_url: str
hermes_api_key: str

9
ariadne/settings_env.py
View File

@ -26,3 +26,12 @@ def _env_float(name: str, default: float) -> float:
return float(raw)
except ValueError:
return default
def _pair_map(raw: str) -> dict[str, str]:
mapping: dict[str, str] = {}
for item in raw.split(","):
key, _, value = item.partition("=")
if key.strip() and value.strip():
mapping[key.strip()] = value.strip()
return mapping

53
ariadne/settings_hermes.py Normal file
View File

@ -0,0 +1,53 @@
"""Auto-triage settings, kept apart because they have grown a section's worth.
Splitting them out keeps settings_sections.py within its size budget and
puts every triage knob in one place to review.
"""
from __future__ import annotations
from typing import Any
from .settings_env import _env, _env_bool, _env_float, _env_int, _pair_map
def _hermes_autotriage_config() -> dict[str, Any]:
return {
"hermes_autotriage_enabled": _env_bool("ARIADNE_HERMES_AUTOTRIAGE_ENABLED", "false"),
"hermes_autotriage_job_allowlist": [
item.strip()
for item in _env("ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST", "hermes-triage-demo").split(",")
if item.strip()
],
"hermes_autoremediation_enabled": _env_bool("ARIADNE_HERMES_AUTOREMEDIATION_ENABLED", "false"),
"hermes_allowed_actions": [
item.strip()
for item in _env("ARIADNE_HERMES_ALLOWED_ACTIONS", "repair_demo_fixture").split(",")
if item.strip()
],
"hermes_action_classifications": _pair_map(
_env(
"ARIADNE_HERMES_ACTION_CLASSIFICATIONS",
"known_demo_fixture_failure=repair_demo_fixture,transient_infra_failure=retry_transient_infra",
)
),
"hermes_parameterized_jobs": [
item.strip()
for item in _env("ARIADNE_HERMES_PARAMETERIZED_JOBS", "hermes-triage-demo").split(",")
if item.strip()
],
"hermes_min_confidence": _env_float("ARIADNE_HERMES_MIN_CONFIDENCE", 0.85),
"hermes_hung_build_minutes": _env_float("ARIADNE_HERMES_HUNG_BUILD_MINUTES", 45.0),
"hermes_max_branches": _env_int("ARIADNE_HERMES_MAX_BRANCHES", 5),
"hermes_job_namespaces": _pair_map(_env("ARIADNE_HERMES_JOB_NAMESPACES", "")),
"hermes_code_max_open_proposals": _env_int("ARIADNE_HERMES_CODE_MAX_OPEN_PROPOSALS", 64),
"hermes_max_actions_per_incident": _env_int("ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT", 1),
"hermes_api_url": _env(
"ARIADNE_HERMES_API_URL",
"http://hermes.hermes.svc.cluster.local:8642",
).rstrip("/"),
"hermes_api_key": _env("ARIADNE_HERMES_API_KEY", ""),
"hermes_run_timeout_seconds": _env_float("ARIADNE_HERMES_RUN_TIMEOUT_SECONDS", 420.0),
"hermes_demo_namespace": _env("ARIADNE_HERMES_DEMO_NAMESPACE", "hermes-triage-demo"),
"hermes_demo_fixture_configmap": _env("ARIADNE_HERMES_DEMO_FIXTURE_CONFIGMAP", "hermes-triage-demo-fixture"),
}

53
ariadne/settings_sections.py
View File

@ -3,7 +3,8 @@ from __future__ import annotations
import json
from typing import Any
from .settings_env import _env, _env_bool, _env_float, _env_int
from .settings_hermes import _hermes_autotriage_config
from .settings_env import _env, _env_bool, _env_float, _env_int, _pair_map
def _keycloak_config() -> dict[str, Any]:
@ -254,56 +255,6 @@ def _testing_triage_config() -> dict[str, Any]:
}
def _pair_map(raw: str) -> dict[str, str]:
mapping: dict[str, str] = {}
for item in raw.split(","):
key, _, value = item.partition("=")
if key.strip() and value.strip():
mapping[key.strip()] = value.strip()
return mapping
def _hermes_autotriage_config() -> dict[str, Any]:
return {
"hermes_autotriage_enabled": _env_bool("ARIADNE_HERMES_AUTOTRIAGE_ENABLED", "false"),
"hermes_autotriage_job_allowlist": [
item.strip()
for item in _env("ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST", "hermes-triage-demo").split(",")
if item.strip()
],
"hermes_autoremediation_enabled": _env_bool("ARIADNE_HERMES_AUTOREMEDIATION_ENABLED", "false"),
"hermes_allowed_actions": [
item.strip()
for item in _env("ARIADNE_HERMES_ALLOWED_ACTIONS", "repair_demo_fixture").split(",")
if item.strip()
],
"hermes_action_classifications": _pair_map(
_env(
"ARIADNE_HERMES_ACTION_CLASSIFICATIONS",
"known_demo_fixture_failure=repair_demo_fixture,transient_infra_failure=retry_transient_infra",
)
),
"hermes_parameterized_jobs": [
item.strip()
for item in _env("ARIADNE_HERMES_PARAMETERIZED_JOBS", "hermes-triage-demo").split(",")
if item.strip()
],
"hermes_min_confidence": _env_float("ARIADNE_HERMES_MIN_CONFIDENCE", 0.85),
"hermes_hung_build_minutes": _env_float("ARIADNE_HERMES_HUNG_BUILD_MINUTES", 45.0),
"hermes_max_branches": _env_int("ARIADNE_HERMES_MAX_BRANCHES", 5),
"hermes_job_namespaces": _pair_map(_env("ARIADNE_HERMES_JOB_NAMESPACES", "")),
"hermes_max_actions_per_incident": _env_int("ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT", 1),
"hermes_api_url": _env(
"ARIADNE_HERMES_API_URL",
"http://hermes.hermes.svc.cluster.local:8642",
).rstrip("/"),
"hermes_api_key": _env("ARIADNE_HERMES_API_KEY", ""),
"hermes_run_timeout_seconds": _env_float("ARIADNE_HERMES_RUN_TIMEOUT_SECONDS", 420.0),
"hermes_demo_namespace": _env("ARIADNE_HERMES_DEMO_NAMESPACE", "hermes-triage-demo"),
"hermes_demo_fixture_configmap": _env("ARIADNE_HERMES_DEMO_FIXTURE_CONFIGMAP", "hermes-triage-demo-fixture"),
}
def _hermes_code_config() -> dict[str, Any]:
return {
"hermes_code_enabled": _env_bool("ARIADNE_HERMES_CODE_ENABLED", "false"),

9
tests/test_hermes_autotriage.py
View File

@ -114,12 +114,13 @@ def test_prompt_is_frozen_shape(monkeypatch) -> None:
"total_timeout_seconds": 420.0,
}
assert prompt.startswith("Use $triage-titan-test-failures.\n")
assert f"Analyze incident {INCIDENT_ID}." in prompt
assert f"Analyze incident {INCIDENT_ID} for the Jenkins job" in prompt
assert f'"<must equal {INCIDENT_ID}>"' in prompt
assert "You are diagnosing only; you do not execute anything." in prompt
assert "The Jenkins job under analysis is hermes-triage-demo." in prompt
assert "reserved for the job hermes-triage-demo" in prompt
assert "Do not perform mutations.\n\nBundle:\n" in prompt
assert "for the Jenkins job hermes-triage-demo" in prompt
assert "repair_demo_fixture" in prompt
assert "Do not perform mutations." in prompt
assert "\n\nBundle:\n" in prompt
assert prompt.rstrip().endswith('"log_evidence":{"records":[]}}')

46
tests/test_hermes_code_flow.py
View File

@ -45,6 +45,7 @@ def _code_cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
"gitea_token": "secret-token",
"owner": "bstein",
"repo": "hermes-code-demo",
"max_open_proposals": 0,
"base_branch": "master",
"timeout_seconds": 15.0,
"legacy_job": JOB,
@ -128,10 +129,11 @@ def _install(monkeypatch, *, fetch=None, run=None, push=None, pull=None, existin
return calls
def _propose(monkeypatch, **kwargs): # type: ignore[no-untyped-def]
def _propose(monkeypatch, code_cfg=None, **kwargs): # type: ignore[no-untyped-def]
storage = FakeStorage()
calls = _install(monkeypatch, **kwargs)
result = module.propose_code_fix(storage, INCIDENT_ID, JOB, 7, BUNDLE, _hermes_cfg(), _code_cfg())
cfg = code_cfg if code_cfg is not None else _code_cfg()
result = module.propose_code_fix(storage, INCIDENT_ID, JOB, 7, BUNDLE, _hermes_cfg(), cfg)
return storage, calls, result
@ -180,8 +182,11 @@ def test_happy_path_opens_pull_request(monkeypatch) -> None:
def test_existing_open_proposal_suppresses_duplicate(monkeypatch) -> None:
"""Only the ceiling suppresses new work, not a single open proposal."""
existing = {
"found": True,
"open_count": module.DEFAULT_MAX_OPEN_PROPOSALS,
"pr_number": 1,
"url": "https://scm.example/pulls/1",
"branch": "hermes-repair/4",
@ -190,7 +195,7 @@ def test_existing_open_proposal_suppresses_duplicate(monkeypatch) -> None:
storage, calls, result = _propose(monkeypatch, existing=existing)
assert result == {
"status": "human_required",
"reason": "existing_proposal_open",
"reason": "open_proposal_limit_reached",
"pr_number": 1,
"url": "https://scm.example/pulls/1",
"branch": "hermes-repair/4",
@ -206,7 +211,7 @@ def test_existing_open_proposal_suppresses_duplicate(monkeypatch) -> None:
"build_number": 7,
"run_id": None,
"validated": False,
"reject_reason": "existing_proposal_open",
"reject_reason": "open_proposal_limit_reached",
"branch": "hermes-repair/4",
"pr_number": 1,
"url": "https://scm.example/pulls/1",
@ -451,3 +456,36 @@ def test_orchestrator_skips_code_path_for_other_jobs(monkeypatch) -> None:
assert summary["jobs"][JOB]["reason"] == "hermes_run_error"
assert calls == []
assert len(triage_calls) == 1
def test_open_proposals_below_the_ceiling_do_not_suppress_new_work(monkeypatch) -> None:
"""A fix for one failure must not be withheld because another awaits review."""
existing = {
"found": True,
"open_count": 3,
"pr_number": 1,
"url": "https://scm.example/pulls/1",
"branch": "hermes-repair/4",
"error": None,
}
_storage, _calls, result = _propose(monkeypatch, existing=existing)
assert result["status"] == "pr_opened"
def test_the_ceiling_is_configurable(monkeypatch) -> None:
"""Review capacity is a property of the reviewer, not of the code."""
existing = {
"found": True,
"open_count": 2,
"pr_number": 1,
"url": "https://scm.example/pulls/1",
"branch": "hermes-repair/4",
"error": None,
}
cfg = {**_code_cfg(), "max_open_proposals": 2}
_storage, _calls, result = _propose(monkeypatch, existing=existing, code_cfg=cfg)
assert result["reason"] == "open_proposal_limit_reached"

3
tests/test_hermes_code_repair.py
View File

@ -133,7 +133,7 @@ def _pull(number: int, head: str = BRANCH, base: str = "master") -> dict:
def _none_found(error=None) -> dict: # type: ignore[no-untyped-def]
return {"found": False, "pr_number": None, "url": None, "branch": None, "error": error}
return {"found": False, "open_count": 0, "pr_number": None, "url": None, "branch": None, "error": error}
def test_find_open_proposal_matches_repair_branch(monkeypatch) -> None:
@ -141,6 +141,7 @@ def test_find_open_proposal_matches_repair_branch(monkeypatch) -> None:
result = module.find_open_proposal(_cfg())
assert result == {
"found": True,
"open_count": 1,
"pr_number": 2,
"url": "https://scm.example/pulls/2",
"branch": BRANCH,

42
tests/test_hermes_triage_prompt.py
View File

@ -12,18 +12,19 @@ def test_demo_job_prompt_permits_the_demo_action() -> None:
prompt = hermes_triage_prompt.build_prompt(
"hermes-triage-demo/12", "hermes-triage-demo", {"incident_id": "hermes-triage-demo/12"}
)
assert "The Jenkins job under analysis is hermes-triage-demo." in prompt
assert "Set human_required to false when hermes-triage-demo is hermes-triage-demo" in prompt
assert "for the Jenkins job hermes-triage-demo" in prompt
assert "known_demo_fixture_failure" in prompt
assert "repair_demo_fixture" in prompt
def test_real_job_prompt_forbids_the_demo_classification() -> None:
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
assert "The Jenkins job under analysis is metis." in prompt
assert (
"reserved for the job hermes-triage-demo. When metis is any other job they are forbidden"
in prompt
)
assert "Set human_required to false when metis is hermes-triage-demo" in prompt
assert "for the Jenkins job metis" in prompt
# The fixture rules are not stated at all, so they cannot be repeated into
# a real service's issue tracker.
assert "known_demo_fixture_failure" not in prompt
assert "repair_demo_fixture" not in prompt
assert "hermes-triage-demo" not in prompt
def test_bundle_is_appended_compactly() -> None:
@ -35,3 +36,28 @@ def test_no_placeholders_survive_rendering() -> None:
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
for placeholder in ("__INCIDENT_ID__", "__JOB__", "__DEMO_JOB__", "__BUNDLE__"):
assert placeholder not in prompt
def test_a_real_service_prompt_never_mentions_the_triage_system() -> None:
"""Whatever Hermes reasons about is published into the service's tracker.
Policy talk belongs to Ariadne; in a service repository it reads as though
the system exists to serve a demonstration.
"""
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
for leak in ("demo", "fixture", "allowlist", "hermes-triage-demo"):
assert leak not in prompt.lower(), leak
def test_the_prompt_asks_for_maintainer_facing_language() -> None:
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
assert "who has no knowledge of how this triage system is configured" in prompt
assert "Do not discuss classifications, actions, policies" in prompt
def test_structured_test_evidence_is_pointed_at_first() -> None:
"""failed_tests is better evidence than console text and must be preferred."""
prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
assert prompt.index("jenkins.failed_tests") < prompt.index("jenkins.console_failures")