ariadne/tests/test_hermes_autotriage_repair.py
codex b2fcad4116 refactor(hermes-triage): repair the fixture in-process, not via a spawned Job
The repair action no longer creates a Kubernetes Job and polls it. Ariadne
patches the fixture ConfigMap directly through its own k8s client, which
removes roughly 40 seconds of pod scheduling from the loop, drops the two
failure modes that Job introduced (volume attach and node selection), and
turns an opaque pod log into an Ariadne event.

- execute_repair returns {action, target, succeeded, error} and issues one
  merge patch; no Job, no polling, no injectable clock, never raises
- the patch writes the same terminal value every time, so idempotency needs
  no duplicate guard; one action per incident is still enforced upstream
- orchestrator records {repair, target}; the state machine, rebuild trigger
  and failure path are unchanged
- retires the unused repair-image setting

Ariadne's service account now needs get+patch on that one ConfigMap by name
instead of Job create; the batch/jobs grant and the hermes-demo-repair
service account can be retired.

480 pass in the hermes suite.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-05 22:29:57 -03:00

251 lines
9.4 KiB
Python

from __future__ import annotations
from types import SimpleNamespace
from ariadne.k8s import client as k8s_client
from ariadne.services import hermes_autotriage_repair as module
INCIDENT_ID = "hermes-triage-demo/12"
CONFIGMAP_PATH = "/api/v1/namespaces/hermes-triage-demo/configmaps/hermes-triage-demo-fixture"
TARGET = "hermes-triage-demo/hermes-triage-demo-fixture"
class FakeResponse:
def __init__(self, status_code: int) -> None:
self.status_code = status_code
def _cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
base = {
"namespace": "hermes-triage-demo",
"fixture_configmap": "hermes-triage-demo-fixture",
}
base.update(overrides)
return base
def _jenkins_settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def]
values = {
"jenkins_base_url": "https://ci.example",
"jenkins_api_user": "user",
"jenkins_api_token": "token",
"jenkins_api_timeout_sec": 5.0,
}
values.update(overrides)
return SimpleNamespace(**values)
def _install_k8s(monkeypatch, patch_exc=None) -> dict: # type: ignore[no-untyped-def]
"""Record every Kubernetes call the repair path makes."""
calls: dict = {"patches": [], "posts": [], "gets": []}
def fake_patch(path, payload): # type: ignore[no-untyped-def]
calls["patches"].append((path, payload))
if patch_exc is not None:
raise patch_exc
return {"metadata": {"name": "hermes-triage-demo-fixture"}}
def fake_post(path, payload): # type: ignore[no-untyped-def]
calls["posts"].append((path, payload))
return {}
def fake_get(path): # type: ignore[no-untyped-def]
calls["gets"].append(path)
return {}
monkeypatch.setattr(module, "patch_json", fake_patch)
monkeypatch.setattr(k8s_client, "post_json", fake_post)
monkeypatch.setattr(k8s_client, "get_json", fake_get)
return calls
def _http_error(status_code: int) -> Exception:
error = RuntimeError(f"http {status_code}")
error.response = SimpleNamespace(status_code=status_code) # type: ignore[attr-defined]
return error
def test_execute_repair_patches_the_fixture_configmap(monkeypatch) -> None:
calls = _install_k8s(monkeypatch)
result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert result == {
"action": "configmap_patch",
"target": TARGET,
"succeeded": True,
"error": None,
}
assert calls["patches"] == [(CONFIGMAP_PATH, {"data": {"state": "healthy"}})]
def test_execute_repair_creates_no_kubernetes_job(monkeypatch) -> None:
calls = _install_k8s(monkeypatch)
module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert calls["posts"] == []
assert calls["gets"] == []
assert not any("/jobs" in path for path, _ in calls["patches"])
assert not hasattr(module, "_job_payload")
assert not hasattr(module, "_wait_for_completion")
def test_execute_repair_reports_non_2xx_status(monkeypatch) -> None:
calls = _install_k8s(monkeypatch, patch_exc=_http_error(403))
result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert result["action"] == "configmap_patch"
assert result["target"] == TARGET
assert result["succeeded"] is False
assert result["error"] == "configmap patch http 403"
assert len(calls["patches"]) == 1
def test_execute_repair_survives_a_transport_error(monkeypatch) -> None:
_install_k8s(monkeypatch, patch_exc=RuntimeError("connection reset by peer"))
result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert result["succeeded"] is False
assert result["error"] == "configmap patch failed: connection reset by peer"
def test_execute_repair_without_namespace_issues_no_request(monkeypatch) -> None:
calls = _install_k8s(monkeypatch)
result = module.execute_repair(_cfg(namespace=""), INCIDENT_ID, 12)
assert result["succeeded"] is False
assert result["error"] == "fixture target incomplete"
assert calls["patches"] == []
def test_execute_repair_without_configmap_name_issues_no_request(monkeypatch) -> None:
calls = _install_k8s(monkeypatch)
result = module.execute_repair(_cfg(fixture_configmap=None), INCIDENT_ID, 12)
assert result["succeeded"] is False
assert result["error"] == "fixture target incomplete"
assert calls["patches"] == []
def test_execute_repair_is_idempotent_across_repeats(monkeypatch) -> None:
calls = _install_k8s(monkeypatch)
first = module.execute_repair(_cfg(), INCIDENT_ID, 12)
second = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert first == second
assert calls["patches"] == [
(CONFIGMAP_PATH, {"data": {"state": "healthy"}}),
(CONFIGMAP_PATH, {"data": {"state": "healthy"}}),
]
def _install_http(monkeypatch, response=None, exc=None) -> dict: # type: ignore[no-untyped-def]
calls: dict = {"posts": [], "kwargs": None}
class FakeClient:
def __init__(self, **kwargs) -> None: # type: ignore[no-untyped-def]
calls["kwargs"] = kwargs
def __enter__(self): # type: ignore[no-untyped-def]
return self
def __exit__(self, *args) -> None: # type: ignore[no-untyped-def]
return None
def post(self, url, data=None): # type: ignore[no-untyped-def]
calls["posts"].append((url, data))
if exc is not None:
raise exc
return response
monkeypatch.setattr(module.httpx, "Client", FakeClient)
return calls
def test_trigger_rebuild_success(monkeypatch) -> None:
calls = _install_http(monkeypatch, response=FakeResponse(201))
result = module.trigger_rebuild(_jenkins_settings(), "hermes-triage-demo")
assert result == {"requested": True, "error": None}
url, data = calls["posts"][0]
assert url == "https://ci.example/job/hermes-triage-demo/buildWithParameters"
assert data == {"SEED_FAILURE": "false"}
assert calls["kwargs"]["auth"] == ("user", "token")
assert calls["kwargs"]["timeout"] == 5.0
def test_trigger_rebuild_non_created_status(monkeypatch) -> None:
_install_http(monkeypatch, response=FakeResponse(500))
result = module.trigger_rebuild(_jenkins_settings(), "hermes-triage-demo")
assert result == {"requested": False, "error": "rebuild http 500"}
def test_trigger_rebuild_request_failure(monkeypatch) -> None:
_install_http(monkeypatch, exc=RuntimeError("connect refused"))
result = module.trigger_rebuild(_jenkins_settings(), "hermes-triage-demo")
assert result["requested"] is False
assert result["error"] == "rebuild request failed: connect refused"
def test_trigger_rebuild_without_base_url(monkeypatch) -> None:
calls = _install_http(monkeypatch, response=FakeResponse(201))
result = module.trigger_rebuild(_jenkins_settings(jenkins_base_url=""), "hermes-triage-demo")
assert result == {"requested": False, "error": "jenkins base url is empty"}
assert calls["posts"] == []
def test_trigger_rebuild_without_credentials(monkeypatch) -> None:
calls = _install_http(monkeypatch, response=FakeResponse(201))
config = _jenkins_settings(jenkins_api_user="", jenkins_api_token="")
result = module.trigger_rebuild(config, "hermes-triage-demo")
assert result["requested"] is True
assert "auth" not in calls["kwargs"]
def test_retry_build_parameterized_reuses_the_rebuild_form(monkeypatch) -> None:
calls = _install_http(monkeypatch, response=FakeResponse(201))
result = module.retry_build(_jenkins_settings(), "hermes-triage-demo", parameterized=True)
assert result == {"requested": True, "error": None}
url, data = calls["posts"][0]
assert url == "https://ci.example/job/hermes-triage-demo/buildWithParameters"
assert data == {"SEED_FAILURE": "false"}
assert calls["kwargs"]["auth"] == ("user", "token")
def test_retry_build_unparameterized_posts_no_parameters(monkeypatch) -> None:
calls = _install_http(monkeypatch, response=FakeResponse(201))
result = module.retry_build(_jenkins_settings(), "titan-iac", parameterized=False)
assert result == {"requested": True, "error": None}
url, data = calls["posts"][0]
assert url == "https://ci.example/job/titan-iac/build"
assert data is None
def test_retry_build_non_created_status(monkeypatch) -> None:
_install_http(monkeypatch, response=FakeResponse(500))
result = module.retry_build(_jenkins_settings(), "titan-iac", parameterized=False)
assert result == {"requested": False, "error": "retry http 500"}
def test_retry_build_request_failure(monkeypatch) -> None:
_install_http(monkeypatch, exc=RuntimeError("connection reset by peer"))
result = module.retry_build(_jenkins_settings(), "titan-iac", parameterized=False)
assert result == {"requested": False, "error": "retry request failed: connection reset by peer"}
def test_retry_build_without_base_url(monkeypatch) -> None:
calls = _install_http(monkeypatch, response=FakeResponse(201))
result = module.retry_build(_jenkins_settings(jenkins_base_url=""), "titan-iac", parameterized=True)
assert result == {"requested": False, "error": "jenkins base url is empty"}
assert calls["posts"] == []
def test_retry_build_without_credentials(monkeypatch) -> None:
calls = _install_http(monkeypatch, response=FakeResponse(201))
config = _jenkins_settings(jenkins_api_user="", jenkins_api_token="")
result = module.retry_build(config, "titan-iac", parameterized=False)
assert result["requested"] is True
assert "auth" not in calls["kwargs"]