feat(hermes): clear stuck agent pods, and teach Hermes the new remediations

A build whose agent never started is a distinct failure from one that lost a
connection mid-run: retrying can work, but when the pool is already full of
stuck pods the retry queues behind them and fails identically. Clearing first
is what makes the retry worth making. The clear is Ariadne's existing
scheduled pod cleanup, which only removes pods that have already succeeded or
failed, so nothing running is touched. This is the failure behind lesavka's
open issue and behind two stalled demo runs tonight.

Critically, all three remediations are now described in the triage prompt.
They were wired in Ariadne but absent from what Hermes is told, so Hermes
could never have requested them - the allowlist would have advertised
capability that could not fire. A test now asserts every allowlisted action id
appears in the prompt, so the two cannot drift apart again.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

ariadne/services/hermes_autotriage.py

@@ -39,6 +39,7 @@ REPAIR_ACTION = "repair_demo_fixture"
 RETRY_ACTION = "retry_transient_infra"
 RECLAIM_ACTION = "reclaim_workspace_storage"
 ABORT_ACTION = "abort_hung_build"
+AGENT_ACTION = "clear_stuck_agent_pods"
 REBUILD_FAILED_REASON = "repair rebuild failed"
 CODE_FIX_PROPOSED_REASON = "code_fix_proposed"
 
@@ -334,6 +335,8 @@ def _evidence_signature(outcome: Any, bundle: dict[str, Any], incident_id: str)
         return hermes_infra_signals.has_transient_infra_signature(bundle)
     if requested == RECLAIM_ACTION:
         return hermes_storage_signals.has_storage_exhaustion_signature(bundle)
+    if requested == AGENT_ACTION:
+        return hermes_infra_signals.has_agent_provisioning_signature(bundle)
     return hermes_evidence.evidence_has_signature(bundle, incident_id), None
 
 

ariadne/services/hermes_autotriage_actions.py

@@ -14,7 +14,7 @@ from typing import Any
 from ..settings import settings
 from . import hermes_autotriage as hermes_autotriage
 from . import hermes_autotriage_events as hermes_events
-from . import hermes_autotriage_repair, jenkins_workspace_cleanup
+from . import hermes_autotriage_repair, jenkins_workspace_cleanup, pod_cleaner
 from .hermes_autotriage_metrics import (
     HERMES_TRIAGE_ACTION_TOTAL,
     HERMES_TRIAGE_DURATION_SECONDS,
@@ -31,9 +31,36 @@ def execute_action(
         return _retry_transient_infra(storage, base, marker)
     if action_id == hermes_autotriage.RECLAIM_ACTION:
         return _reclaim_workspace_storage(storage, base, marker)
+    if action_id == hermes_autotriage.AGENT_ACTION:
+        return _clear_stuck_agent_pods(storage, base, marker)
     return _repair_demo_fixture(storage, base, action_id)
 
 
+def _clear_stuck_agent_pods(storage: Any, base: dict[str, Any], marker: str | None) -> dict[str, Any]:
+    """Clear finished agent pods, then request one rebuild.
+
+    A build whose agent never started often succeeds on a retry, but when the
+    pool is already full of stuck pods the retry queues behind them and fails
+    identically. Clearing first is what makes the retry worth making. The
+    clear is Ariadne's existing scheduled pod cleanup, which only removes pods
+    that have already succeeded or failed, so nothing running is touched.
+    """
+
+    action = hermes_autotriage.AGENT_ACTION
+    hermes_events.record_action(storage, base, action, "requested", {"marker": marker})
+    try:
+        summary = pod_cleaner.clean_finished_pods()
+        cleared = getattr(summary, "deleted", 0)
+    except Exception as exc:
+        return fail_action(storage, base, action, f"pod cleanup failed: {exc}")
+    hermes_events.record_action(storage, base, action, "executed", {"cleared": cleared})
+    rebuild = hermes_autotriage_repair.trigger_rebuild(settings, str(base["job"]))
+    if not rebuild.get("requested"):
+        return fail_action(storage, base, action, str(rebuild.get("error") or "rebuild failed"))
+    hermes_events.record_incident(storage, base, "awaiting_rebuild", {"action": action, "cleared": cleared})
+    return {"status": "awaiting_rebuild", "incident_id": str(base["incident_id"]), "cleared": cleared}
+
+
 def _reclaim_workspace_storage(storage: Any, base: dict[str, Any], marker: str | None) -> dict[str, Any]:
     """Reclaim stale Jenkins workspace storage, then request one rebuild.
 

ariadne/services/hermes_infra_signals.py

@@ -102,3 +102,42 @@ def _first_marker(text: str) -> str | None:
             best = marker
             best_index = index
     return best
+
+
+# A build whose agent never came up is not the same as one that lost a
+# connection mid-run. Retrying alone can work, but when the agent pool is
+# already full of stuck pods the retry queues behind them and fails the same
+# way, so the pods are cleared first.
+AGENT_MARKERS: tuple[str, ...] = (
+    "are offline",
+    "error in provisioning",
+    "waiting to start: podinitializing",
+    "still waiting to schedule task",
+    "containercreating",
+)
+
+
+def has_agent_provisioning_signature(bundle: dict) -> tuple[bool, str | None]:
+    """Report whether the build failed because its agent never started.
+
+    Inputs: an evidence bundle from `collect_evidence`. Outputs: (matched,
+    marker), scanning the failure regions in order before the tail so the
+    earliest enforced failure wins. Never raises.
+    """
+
+    try:
+        jenkins = bundle.get("jenkins") if isinstance(bundle.get("jenkins"), dict) else {}
+        texts = []
+        regions = jenkins.get("console_failures")
+        for region in regions if isinstance(regions, list) else []:
+            if isinstance(region, dict):
+                texts.append(str(region.get("text") or ""))
+        texts.append(str(jenkins.get("console_tail") or ""))
+        for text in texts:
+            lowered = text.lower()
+            for marker in AGENT_MARKERS:
+                if marker in lowered:
+                    return True, marker
+        return False, None
+    except Exception:
+        return False, None

ariadne/services/hermes_triage_prompt.py

@@ -29,7 +29,10 @@ Return ONLY a single JSON object with exactly these keys and no others:
 {"incident_id": "<must equal __INCIDENT_ID__>", "classification": "<short snake_case name for what actually failed>", "confidence": <0..1>, "facts": [{"statement": "...", "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", "reference": "..."}], "inferences": ["..."], "first_failed_gate": "<string>", "requested_action": <object or null>, "human_required": <bool>, "reason": "<string>"}
 You are diagnosing only; you do not execute anything. Ariadne separately validates and executes any requested action under its own authorization policy, and will refuse anything its own reading of the evidence does not support.
 Write the reason and the inferences for an engineer who maintains this service and who has no knowledge of how this triage system is configured. Describe the failure and what a fix would involve. Do not discuss classifications, actions, policies, or which of them are permitted; those are Ariadne's concern and are meaningless in the service's issue tracker.
-Use classification transient_infra_failure with requested_action {"type": "run_ariadne_job", "id": "retry_transient_infra"} only when the evidence shows an infrastructure, connectivity, or registry error unrelated to the repository's code or tests (DNS resolution failure, connection refused, reset, or timed out, TLS handshake failure, image pull failure, or a 5xx from a registry or SCM host), because re-running the same commit is then the whole remediation.
+Three classifications have a predefined remediation. Use one only when the evidence plainly shows that failure; otherwise leave requested_action null.
+Use transient_infra_failure with requested_action {"type": "run_ariadne_job", "id": "retry_transient_infra"} when the evidence shows an infrastructure, connectivity, or registry error unrelated to the repository's code or tests (DNS resolution failure, connection refused, reset, or timed out, TLS handshake failure, image pull failure, or a 5xx from a registry or SCM host), because re-running the same commit is then the whole remediation.
+Use workspace_storage_exhausted with requested_action {"type": "run_ariadne_job", "id": "reclaim_workspace_storage"} when the evidence shows the build ran out of disk on its workspace volume (no space left on device, disk quota exceeded) while writing under the agent workspace. Do not classify this as transient_infra_failure: a plain rebuild lands on the same full volume, so the remediation must reclaim the stale workspace storage first.
+Use jenkins_agent_provisioning_failure with requested_action {"type": "run_ariadne_job", "id": "clear_stuck_agent_pods"} when the evidence shows the build never got an agent (all nodes of a label offline, an agent pod stuck ContainerCreating or Pending, or an error in provisioning) rather than failing once it was running.
 Otherwise leave requested_action null.
 Set human_required to true when decisive evidence is missing or the failure needs a judgement only a maintainer can make; otherwise set it false and say plainly what you believe is wrong.
 Do not perform mutations."""

tests/test_hermes_autotriage_actions.py

@@ -119,3 +119,45 @@ def test_abort_survives_a_transport_failure(monkeypatch) -> None:
     cfg = SimpleNamespace(jenkins_base_url="https://ci.example", jenkins_api_user="u",
                           jenkins_api_token="t", jenkins_api_timeout_sec=5)
     assert repair.abort_build(cfg, "j", 1)["requested"] is False
+
+
+def test_clearing_stuck_agent_pods_then_rebuilds(monkeypatch) -> None:
+    """A retry queued behind stuck pods fails the same way, so clear first."""
+
+    calls = []
+    monkeypatch.setattr(
+        module.pod_cleaner, "clean_finished_pods",
+        lambda: calls.append("clean") or SimpleNamespace(deleted=4),
+    )
+    monkeypatch.setattr(
+        module.hermes_autotriage_repair, "trigger_rebuild",
+        lambda cfg, job: calls.append(f"rebuild:{job}") or {"requested": True, "error": None},
+    )
+    result = module._clear_stuck_agent_pods(_Storage(), dict(BASE), "are offline")
+
+    assert calls == ["clean", "rebuild:lesavka"]
+    assert result["status"] == "awaiting_rebuild"
+    assert result["cleared"] == 4
+
+
+def test_a_failed_pod_clear_never_rebuilds(monkeypatch) -> None:
+    rebuilt = []
+    monkeypatch.setattr(
+        module.pod_cleaner, "clean_finished_pods",
+        lambda: (_ for _ in ()).throw(RuntimeError("boom")),
+    )
+    monkeypatch.setattr(
+        module.hermes_autotriage_repair, "trigger_rebuild",
+        lambda cfg, job: rebuilt.append(job) or {"requested": True},
+    )
+    assert module._clear_stuck_agent_pods(_Storage(), dict(BASE), "m")["status"] == "failed"
+    assert rebuilt == []
+
+
+def test_a_failed_rebuild_after_clearing_is_escalated(monkeypatch) -> None:
+    monkeypatch.setattr(module.pod_cleaner, "clean_finished_pods", lambda: SimpleNamespace(deleted=1))
+    monkeypatch.setattr(
+        module.hermes_autotriage_repair, "trigger_rebuild",
+        lambda cfg, job: {"requested": False, "error": "http 500"},
+    )
+    assert module._clear_stuck_agent_pods(_Storage(), dict(BASE), "m")["status"] == "failed"

tests/test_hermes_autotriage_retry.py

@@ -207,7 +207,12 @@ def test_prompt_documents_the_retry_action(monkeypatch) -> None:
     module.run_hermes_autotriage(env.storage)
 
     _, prompt = env.calls["triage"][0]
-    assert "Use classification transient_infra_failure with requested_action" in prompt
+    assert "Use transient_infra_failure with requested_action" in prompt
     assert '{"type": "run_ariadne_job", "id": "retry_transient_infra"}' in prompt
     assert "unrelated to the repository's code or tests" in prompt
     assert "Otherwise leave requested_action null" in prompt
+    # Every allowlisted action must be described, or Hermes can never request
+    # it and the allowlist would claim a capability that cannot fire.
+    assert '"id": "reclaim_workspace_storage"' in prompt
+    assert '"id": "clear_stuck_agent_pods"' in prompt
+    assert "lands on the same full volume" in prompt

tests/test_hermes_storage_signals.py

@@ -41,3 +41,34 @@ def test_a_workspace_mention_without_a_storage_marker_does_not_match() -> None:
 def test_never_raises_on_hostile_input() -> None:
     for bad in (None, 7, {}, {"jenkins": None}, {"jenkins": {"console_failures": "no"}}):
         assert module.has_storage_exhaustion_signature(bad) == (False, None)
+
+
+def test_agent_provisioning_failures_are_recognised() -> None:
+    """The build never got an agent, rather than failing once running."""
+
+    from ariadne.services import hermes_infra_signals
+
+    for text in (
+        "All nodes of label 'lesavka_583-abc' are offline",
+        "Error in provisioning; agent=KubernetesSlave[x]",
+        "Still waiting to schedule task",
+    ):
+        matched, marker = hermes_infra_signals.has_agent_provisioning_signature(_bundle(text))
+        assert matched is True, text
+        assert marker
+
+
+def test_an_ordinary_failure_is_not_an_agent_problem() -> None:
+    from ariadne.services import hermes_infra_signals
+
+    matched, _ = hermes_infra_signals.has_agent_provisioning_signature(
+        _bundle("AssertionError: expected 1 got 2")
+    )
+    assert matched is False
+
+
+def test_agent_signature_never_raises() -> None:
+    from ariadne.services import hermes_infra_signals
+
+    for bad in (None, 7, {}, {"jenkins": {"console_failures": "no"}}):
+        assert hermes_infra_signals.has_agent_provisioning_signature(bad) == (False, None)