fix(hermes): scope the demo classification to the demo job

Hermes classified a real bstein-dev-home build as known_demo_fixture_failure
at 0.99 confidence because a missing node binary superficially resembled the
fixture signature. Ariadne's evidence gate correctly refused the action, but
the filed issue carried a meaningless classification.

Extract the frozen prompt into its own module and interpolate the job under
analysis, so the demo classification and repair_demo_fixture action are
forbidden outright on any job other than hermes-triage-demo.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

ariadne/services/hermes_autotriage.py

@@ -9,6 +9,7 @@ import httpx
 from ..settings import settings
 from ..utils.logging import get_logger
 from . import hermes_agent_client, hermes_autotriage_repair, hermes_code_flow, hermes_incident_issue
+from . import hermes_triage_prompt
 from . import hermes_infra_signals
 from . import hermes_autotriage_decision as hermes_decision
 from . import hermes_autotriage_events as hermes_events
@@ -38,23 +39,6 @@ _LAST_BUILD_TREE = "lastBuild[number,result,building,timestamp,duration,url]"
 _RUN_COMPLETED = "completed"
 _UNKNOWN_ACTION_LABEL = "unknown"
 
-_PROMPT_TEMPLATE = """Use $triage-titan-test-failures.
-Analyze incident __INCIDENT_ID__.
-Treat the attached Ariadne bundle as the source of truth.
-Identify the first enforced failure.
-The jenkins.console_failures array holds excerpts around detected failure markers in chronological order; the earliest region usually contains the first enforced failure, and jenkins.console_tail is the end of the build which often only shows downstream noise.
-Distinguish facts from inference.
-Return ONLY a single JSON object with exactly these keys and no others:
-{"incident_id": "<must equal __INCIDENT_ID__>", "classification": "<string; use known_demo_fixture_failure only when the evidence shows the hermes-triage-demo fixture unhealthy signature>", "confidence": <0..1>, "facts": [{"statement": "...", "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", "reference": "..."}], "inferences": ["..."], "first_failed_gate": "<string>", "requested_action": {"type": "run_ariadne_job", "id": "repair_demo_fixture"} or null, "human_required": <bool>, "reason": "<string>"}
-You are diagnosing only; you do not execute anything. Ariadne separately validates and executes the requested action under its own authorization policy.
-Set human_required to false when the evidence matches the known demo fixture signature and the appropriate response is the predefined repair_demo_fixture action.
-Set human_required to true only when the failure does not match a known signature, decisive evidence is missing, or no allowlisted action fits.
-Use classification transient_infra_failure with requested_action {"type": "run_ariadne_job", "id": "retry_transient_infra"} only when the evidence shows an infrastructure, connectivity, or registry error unrelated to the repository's code or tests (DNS resolution failure, connection refused, reset, or timed out, TLS handshake failure, image pull failure, or a 5xx from a registry or SCM host), because re-running the same commit is then the whole remediation.
-Otherwise leave requested_action null and set human_required per the rules above.
-Do not perform mutations.
-
-Bundle:
-__BUNDLE__"""
 
 def run_hermes_autotriage(storage: Any) -> dict[str, Any]:
     """Run one Hermes auto-triage tick over the allowlisted Jenkins jobs.
@@ -222,7 +206,7 @@ def _run_pipeline(  # noqa: PLR0913 - the tick's issue budget travels with the i
     if settings.hermes_code_enabled and job == settings.hermes_code_job:
         return _propose_code_fix(storage, base, bundle)
     phase_started = time.time()
-    run = hermes_agent_client.run_triage(_hermes_run_config(), _build_prompt(incident_id, bundle))
+    run = hermes_agent_client.run_triage(_hermes_run_config(), hermes_triage_prompt.build_prompt(incident_id, job, bundle))
     HERMES_TRIAGE_DURATION_SECONDS.labels(phase="diagnosis").set(time.time() - phase_started)
     if run.status != _RUN_COMPLETED or not run.output:
         reason = f"hermes_run_{run.status}"
@@ -420,13 +404,6 @@ def _fail_action(storage: Any, base: dict[str, Any], action_id: str, error: str)
     return {"status": "failed", "incident_id": str(base["incident_id"]), "reason": error}
 
 
-def _build_prompt(incident_id: str, bundle: dict[str, Any]) -> str:
-    """Render the frozen triage prompt with the incident id and bundle."""
-
-    compact = json.dumps(bundle, separators=(",", ":"), ensure_ascii=True)
-    return _PROMPT_TEMPLATE.replace("__INCIDENT_ID__", incident_id).replace("__BUNDLE__", compact)
-
-
 def _requested_action_id(outcome: Any) -> str:
     """Return the raw action id a triage response asked for, if any."""
 

ariadne/services/hermes_triage_prompt.py

@@ -0,0 +1,51 @@
+"""Frozen triage prompt sent to Hermes for every Jenkins incident.
+
+The prompt lives in its own module so its wording is reviewed as a unit and
+so changing it never competes for room with the orchestrator's logic.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+DEMO_JOB = "hermes-triage-demo"
+
+_TEMPLATE = """Use $triage-titan-test-failures.
+Analyze incident __INCIDENT_ID__.
+Treat the attached Ariadne bundle as the source of truth.
+Identify the first enforced failure.
+The jenkins.console_failures array holds excerpts around detected failure markers in chronological order; the earliest region usually contains the first enforced failure, and jenkins.console_tail is the end of the build which often only shows downstream noise.
+Distinguish facts from inference.
+Return ONLY a single JSON object with exactly these keys and no others:
+{"incident_id": "<must equal __INCIDENT_ID__>", "classification": "<string>", "confidence": <0..1>, "facts": [{"statement": "...", "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", "reference": "..."}], "inferences": ["..."], "first_failed_gate": "<string>", "requested_action": {"type": "run_ariadne_job", "id": "repair_demo_fixture"} or null, "human_required": <bool>, "reason": "<string>"}
+You are diagnosing only; you do not execute anything. Ariadne separately validates and executes the requested action under its own authorization policy.
+The Jenkins job under analysis is __JOB__.
+The classification known_demo_fixture_failure and the action repair_demo_fixture are reserved for the job __DEMO_JOB__. When __JOB__ is any other job they are forbidden, no matter how closely the failure resembles that fixture; classify what the evidence actually shows instead, naming the failing tool or gate.
+Set human_required to false when __JOB__ is __DEMO_JOB__ and the evidence matches the demo fixture signature, because the appropriate response is then the predefined repair_demo_fixture action.
+Set human_required to true only when the failure does not match a known signature, decisive evidence is missing, or no allowlisted action fits.
+Use classification transient_infra_failure with requested_action {"type": "run_ariadne_job", "id": "retry_transient_infra"} only when the evidence shows an infrastructure, connectivity, or registry error unrelated to the repository's code or tests (DNS resolution failure, connection refused, reset, or timed out, TLS handshake failure, image pull failure, or a 5xx from a registry or SCM host), because re-running the same commit is then the whole remediation.
+Otherwise leave requested_action null and set human_required per the rules above.
+Do not perform mutations.
+
+Bundle:
+__BUNDLE__"""
+
+
+def build_prompt(incident_id: str, job: str, bundle: dict[str, Any]) -> str:
+    """Render the frozen triage prompt for one incident.
+
+    Inputs: the incident id, the Jenkins job the incident belongs to, and the
+    evidence bundle. Outputs: the prompt string sent to Hermes. The job name is
+    interpolated so the demo-only classification and action stay scoped to the
+    demo job rather than leaking onto real services.
+    """
+
+    compact = json.dumps(bundle, separators=(",", ":"), ensure_ascii=True)
+    return (
+        _TEMPLATE.replace("__INCIDENT_ID__", incident_id)
+        .replace("__DEMO_JOB__", DEMO_JOB)
+        .replace("__JOB__", job)
+        .replace("__BUNDLE__", compact)
+    )

tests/test_hermes_autotriage.py

@@ -115,7 +115,8 @@ def test_prompt_is_frozen_shape(monkeypatch) -> None:
     assert f"Analyze incident {INCIDENT_ID}." in prompt
     assert f'"<must equal {INCIDENT_ID}>"' in prompt
     assert "You are diagnosing only; you do not execute anything." in prompt
-    assert "Set human_required to false when the evidence matches" in prompt
+    assert "The Jenkins job under analysis is hermes-triage-demo." in prompt
+    assert "reserved for the job hermes-triage-demo" in prompt
     assert "Do not perform mutations.\n\nBundle:\n" in prompt
     assert prompt.rstrip().endswith('"log_evidence":{"records":[]}}')
 

tests/test_hermes_triage_prompt.py

@@ -0,0 +1,37 @@
+"""Tests for the frozen Hermes triage prompt and its job scoping."""
+
+from __future__ import annotations
+
+from ariadne.services import hermes_triage_prompt
+
+
+BUNDLE = {"incident_id": "metis/272", "jenkins": {"job": "metis"}}
+
+
+def test_demo_job_prompt_permits_the_demo_action() -> None:
+    prompt = hermes_triage_prompt.build_prompt(
+        "hermes-triage-demo/12", "hermes-triage-demo", {"incident_id": "hermes-triage-demo/12"}
+    )
+    assert "The Jenkins job under analysis is hermes-triage-demo." in prompt
+    assert "Set human_required to false when hermes-triage-demo is hermes-triage-demo" in prompt
+
+
+def test_real_job_prompt_forbids_the_demo_classification() -> None:
+    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
+    assert "The Jenkins job under analysis is metis." in prompt
+    assert (
+        "reserved for the job hermes-triage-demo. When metis is any other job they are forbidden"
+        in prompt
+    )
+    assert "Set human_required to false when metis is hermes-triage-demo" in prompt
+
+
+def test_bundle_is_appended_compactly() -> None:
+    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
+    assert prompt.endswith('Bundle:\n{"incident_id":"metis/272","jenkins":{"job":"metis"}}')
+
+
+def test_no_placeholders_survive_rendering() -> None:
+    prompt = hermes_triage_prompt.build_prompt("metis/272", "metis", BUNDLE)
+    for placeholder in ("__INCIDENT_ID__", "__JOB__", "__DEMO_JOB__", "__BUNDLE__"):
+        assert placeholder not in prompt