ee7b243106
All checks were successful
Tests / Declarative: Post Actions passed: 1257
A build whose agent never started is a distinct failure from one that lost a connection mid-run: retrying can work, but when the pool is already full of stuck pods the retry queues behind them and fails identically. Clearing first is what makes the retry worth making. The clear is Ariadne's existing scheduled pod cleanup, which only removes pods that have already succeeded or failed, so nothing running is touched. This is the failure behind lesavka's open issue and behind two stalled demo runs tonight. Critically, all three remediations are now described in the triage prompt. They were wired in Ariadne but absent from what Hermes is told, so Hermes could never have requested them - the allowlist would have advertised capability that could not fire. A test now asserts every allowlisted action id appears in the prompt, so the two cannot drift apart again. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
219 lines
8.1 KiB
Python
219 lines
8.1 KiB
Python
from __future__ import annotations
|
|
|
|
from ariadne.services import hermes_autotriage as module
|
|
from tests.hermes_autotriage_harness import (
|
|
INCIDENT_ID,
|
|
INFRA_MARKER,
|
|
JOB,
|
|
_build,
|
|
_counter,
|
|
_events,
|
|
_gauge,
|
|
_prepare,
|
|
_retry_output,
|
|
_run,
|
|
_seed_incident,
|
|
_settings,
|
|
_statuses,
|
|
)
|
|
|
|
|
|
OTHER_JOB = "titan-iac"
|
|
BOTH_ACTIONS = ["repair_demo_fixture", "retry_transient_infra"]
|
|
|
|
|
|
def _retry_settings(**overrides): # type: ignore[no-untyped-def]
|
|
values = {"hermes_allowed_actions": BOTH_ACTIONS}
|
|
values.update(overrides)
|
|
return _settings(**values)
|
|
|
|
|
|
def _retry_env(monkeypatch, **overrides): # type: ignore[no-untyped-def]
|
|
kwargs = {"cfg": _retry_settings(), "run": _run(output=_retry_output())}
|
|
kwargs.update(overrides)
|
|
return _prepare(monkeypatch, **kwargs)
|
|
|
|
|
|
def test_transient_infra_failure_is_retried(monkeypatch) -> None:
|
|
success_before = _counter("retry_transient_infra", "success")
|
|
env = _retry_env(monkeypatch)
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB] == {
|
|
"status": "awaiting_rebuild",
|
|
"incident_id": INCIDENT_ID,
|
|
"action": "retry_transient_infra",
|
|
"evidence_marker": INFRA_MARKER,
|
|
}
|
|
assert _statuses(env.storage) == ["detected", "diagnosed", "awaiting_rebuild"]
|
|
assert env.calls["retries"] == [(JOB, True)]
|
|
assert env.calls["repairs"] == []
|
|
assert env.calls["rebuilds"] == []
|
|
assert _counter("retry_transient_infra", "success") == success_before + 1.0
|
|
assert _gauge("12", "awaiting_rebuild") == 1.0
|
|
|
|
|
|
def test_retry_records_the_matched_marker_on_every_event(monkeypatch) -> None:
|
|
env = _retry_env(monkeypatch)
|
|
|
|
module.run_hermes_autotriage(env.storage)
|
|
|
|
actions = _events(env.storage, module.ACTION_EVENT_TYPE)
|
|
assert [action["result"] for action in actions] == ["requested", "accepted", "executed"]
|
|
assert all(action["action"] == "retry_transient_infra" for action in actions)
|
|
assert actions[0]["detail"] == {"evidence_marker": INFRA_MARKER}
|
|
assert actions[2]["detail"] == {"evidence_marker": INFRA_MARKER}
|
|
diagnosis = _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0]
|
|
assert diagnosis["authorized"] is True
|
|
assert diagnosis["authorize_reason"] == "authorized"
|
|
assert diagnosis["evidence_marker"] == INFRA_MARKER
|
|
assert diagnosis["outcome"]["classification"] == "transient_infra_failure"
|
|
incident = _events(env.storage, module.INCIDENT_EVENT_TYPE)[-1]
|
|
assert incident["phase"] == {"action": "retry_transient_infra", "evidence_marker": INFRA_MARKER}
|
|
|
|
|
|
def test_retry_resolves_on_the_next_green_build(monkeypatch) -> None:
|
|
env = _retry_env(monkeypatch)
|
|
module.run_hermes_autotriage(env.storage)
|
|
|
|
green = _prepare(
|
|
monkeypatch,
|
|
cfg=_retry_settings(),
|
|
last_build=_build(13, "SUCCESS"),
|
|
storage=env.storage,
|
|
)
|
|
summary = module.run_hermes_autotriage(green.storage)
|
|
|
|
assert summary["jobs"][JOB] == {"status": "healthy", "resolved": [INCIDENT_ID]}
|
|
assert _statuses(env.storage)[-1] == "resolved"
|
|
assert _gauge("12", "resolved") == 1.0
|
|
|
|
|
|
def test_unparameterized_job_is_retried_without_parameters(monkeypatch) -> None:
|
|
cfg = _retry_settings(
|
|
hermes_autotriage_job_allowlist=[OTHER_JOB],
|
|
hermes_parameterized_jobs=[JOB],
|
|
)
|
|
env = _prepare(
|
|
monkeypatch,
|
|
cfg=cfg,
|
|
last_build=_build(12, "FAILURE", job=OTHER_JOB),
|
|
run=_run(output=_retry_output(job=OTHER_JOB)),
|
|
)
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][OTHER_JOB]["status"] == "awaiting_rebuild"
|
|
assert env.calls["retries"] == [(OTHER_JOB, False)]
|
|
|
|
|
|
def test_retry_without_infra_signature_requires_a_human(monkeypatch) -> None:
|
|
env = _retry_env(monkeypatch, infra=(False, None))
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing"
|
|
assert _statuses(env.storage) == ["detected", "diagnosed", "human_required"]
|
|
assert env.calls["retries"] == []
|
|
assert _events(env.storage, module.DIAGNOSIS_EVENT_TYPE)[0]["evidence_marker"] is None
|
|
|
|
|
|
def test_retry_ignores_the_fixture_signature(monkeypatch) -> None:
|
|
env = _retry_env(monkeypatch, signature=False)
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB]["status"] == "awaiting_rebuild"
|
|
assert env.calls["retries"] == [(JOB, True)]
|
|
|
|
|
|
def test_fixture_action_ignores_the_infra_signature(monkeypatch) -> None:
|
|
env = _prepare(monkeypatch, cfg=_retry_settings(), signature=False, infra=(True, INFRA_MARKER))
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB]["reason"] == "evidence_signature_missing"
|
|
assert env.calls["retries"] == []
|
|
|
|
|
|
def test_retry_is_capped_at_one_action_per_incident(monkeypatch) -> None:
|
|
env = _retry_env(monkeypatch)
|
|
_seed_incident(env.storage, "detected")
|
|
env.storage.record_event(
|
|
module.ACTION_EVENT_TYPE,
|
|
{"incident_id": INCIDENT_ID, "action": "retry_transient_infra", "result": "requested"},
|
|
)
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB]["reason"] == "max_actions_reached"
|
|
assert env.calls["retries"] == []
|
|
|
|
|
|
def test_retry_trigger_failure_marks_failed_and_human(monkeypatch) -> None:
|
|
failed_before = _counter("retry_transient_infra", "failed")
|
|
env = _retry_env(monkeypatch, retry={"requested": False, "error": "retry http 500"})
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB] == {
|
|
"status": "failed",
|
|
"incident_id": INCIDENT_ID,
|
|
"reason": "retry http 500",
|
|
}
|
|
assert _statuses(env.storage) == ["detected", "diagnosed", "failed"]
|
|
actions = _events(env.storage, module.ACTION_EVENT_TYPE)
|
|
assert [action["result"] for action in actions] == ["requested", "accepted", "failed"]
|
|
assert _counter("retry_transient_infra", "failed") == failed_before + 1.0
|
|
assert _gauge("12", "human_required") == 1.0
|
|
|
|
|
|
def test_retry_action_is_rejected_while_not_allowlisted(monkeypatch) -> None:
|
|
rejected_before = _counter("retry_transient_infra", "rejected")
|
|
env = _prepare(monkeypatch, run=_run(output=_retry_output()))
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB]["reason"] == "action_not_allowlisted: 'retry_transient_infra'"
|
|
assert env.calls["retries"] == []
|
|
assert _counter("retry_transient_infra", "rejected") == rejected_before + 1.0
|
|
|
|
|
|
def test_action_must_match_its_classification(monkeypatch) -> None:
|
|
output = _retry_output(requested_action={"type": "run_ariadne_job", "id": "repair_demo_fixture"})
|
|
env = _retry_env(monkeypatch, run=_run(output=output))
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB]["reason"] == (
|
|
"action_does_not_match_classification: 'repair_demo_fixture' expected 'retry_transient_infra'"
|
|
)
|
|
assert env.calls["retries"] == []
|
|
assert env.calls["repairs"] == []
|
|
|
|
|
|
def test_unregistered_classification_requires_a_human(monkeypatch) -> None:
|
|
env = _retry_env(monkeypatch, run=_run(output=_retry_output(classification="flaky_network_guess")))
|
|
|
|
summary = module.run_hermes_autotriage(env.storage)
|
|
|
|
assert summary["jobs"][JOB]["reason"] == "classification_not_supported: 'flaky_network_guess'"
|
|
assert env.calls["retries"] == []
|
|
|
|
|
|
def test_prompt_documents_the_retry_action(monkeypatch) -> None:
|
|
env = _retry_env(monkeypatch)
|
|
module.run_hermes_autotriage(env.storage)
|
|
|
|
_, prompt = env.calls["triage"][0]
|
|
assert "Use transient_infra_failure with requested_action" in prompt
|
|
assert '{"type": "run_ariadne_job", "id": "retry_transient_infra"}' in prompt
|
|
assert "unrelated to the repository's code or tests" in prompt
|
|
assert "Otherwise leave requested_action null" in prompt
|
|
# Every allowlisted action must be described, or Hermes can never request
|
|
# it and the allowlist would claim a capability that cannot fire.
|
|
assert '"id": "reclaim_workspace_storage"' in prompt
|
|
assert '"id": "clear_stuck_agent_pods"' in prompt
|
|
assert "lands on the same full volume" in prompt
|