The repair action no longer creates a Kubernetes Job and polls it. Ariadne
patches the fixture ConfigMap directly through its own k8s client, which
removes roughly 40 seconds of pod scheduling from the loop, drops the two
failure modes that Job introduced (volume attach and node selection), and
turns an opaque pod log into an Ariadne event.
- execute_repair returns {action, target, succeeded, error} and issues one
merge patch; no Job, no polling, no injectable clock, never raises
- the patch writes the same terminal value every time, so idempotency needs
no duplicate guard; one action per incident is still enforced upstream
- orchestrator records {repair, target}; the state machine, rebuild trigger
and failure path are unchanged
- retires the unused repair-image setting
Ariadne's service account now needs get+patch on that one ConfigMap by name
instead of Job create; the batch/jobs grant and the hermes-demo-repair
service account can be retired.
480 pass in the hermes suite.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
251 lines
9.4 KiB
Python
251 lines
9.4 KiB
Python
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
from ariadne.k8s import client as k8s_client
|
|
from ariadne.services import hermes_autotriage_repair as module
|
|
|
|
|
|
INCIDENT_ID = "hermes-triage-demo/12"
|
|
CONFIGMAP_PATH = "/api/v1/namespaces/hermes-triage-demo/configmaps/hermes-triage-demo-fixture"
|
|
TARGET = "hermes-triage-demo/hermes-triage-demo-fixture"
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(self, status_code: int) -> None:
|
|
self.status_code = status_code
|
|
|
|
|
|
def _cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
|
|
base = {
|
|
"namespace": "hermes-triage-demo",
|
|
"fixture_configmap": "hermes-triage-demo-fixture",
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def _jenkins_settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def]
|
|
values = {
|
|
"jenkins_base_url": "https://ci.example",
|
|
"jenkins_api_user": "user",
|
|
"jenkins_api_token": "token",
|
|
"jenkins_api_timeout_sec": 5.0,
|
|
}
|
|
values.update(overrides)
|
|
return SimpleNamespace(**values)
|
|
|
|
|
|
def _install_k8s(monkeypatch, patch_exc=None) -> dict: # type: ignore[no-untyped-def]
|
|
"""Record every Kubernetes call the repair path makes."""
|
|
|
|
calls: dict = {"patches": [], "posts": [], "gets": []}
|
|
|
|
def fake_patch(path, payload): # type: ignore[no-untyped-def]
|
|
calls["patches"].append((path, payload))
|
|
if patch_exc is not None:
|
|
raise patch_exc
|
|
return {"metadata": {"name": "hermes-triage-demo-fixture"}}
|
|
|
|
def fake_post(path, payload): # type: ignore[no-untyped-def]
|
|
calls["posts"].append((path, payload))
|
|
return {}
|
|
|
|
def fake_get(path): # type: ignore[no-untyped-def]
|
|
calls["gets"].append(path)
|
|
return {}
|
|
|
|
monkeypatch.setattr(module, "patch_json", fake_patch)
|
|
monkeypatch.setattr(k8s_client, "post_json", fake_post)
|
|
monkeypatch.setattr(k8s_client, "get_json", fake_get)
|
|
return calls
|
|
|
|
|
|
def _http_error(status_code: int) -> Exception:
|
|
error = RuntimeError(f"http {status_code}")
|
|
error.response = SimpleNamespace(status_code=status_code) # type: ignore[attr-defined]
|
|
return error
|
|
|
|
|
|
def test_execute_repair_patches_the_fixture_configmap(monkeypatch) -> None:
|
|
calls = _install_k8s(monkeypatch)
|
|
result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
|
|
|
|
assert result == {
|
|
"action": "configmap_patch",
|
|
"target": TARGET,
|
|
"succeeded": True,
|
|
"error": None,
|
|
}
|
|
assert calls["patches"] == [(CONFIGMAP_PATH, {"data": {"state": "healthy"}})]
|
|
|
|
|
|
def test_execute_repair_creates_no_kubernetes_job(monkeypatch) -> None:
|
|
calls = _install_k8s(monkeypatch)
|
|
module.execute_repair(_cfg(), INCIDENT_ID, 12)
|
|
|
|
assert calls["posts"] == []
|
|
assert calls["gets"] == []
|
|
assert not any("/jobs" in path for path, _ in calls["patches"])
|
|
assert not hasattr(module, "_job_payload")
|
|
assert not hasattr(module, "_wait_for_completion")
|
|
|
|
|
|
def test_execute_repair_reports_non_2xx_status(monkeypatch) -> None:
|
|
calls = _install_k8s(monkeypatch, patch_exc=_http_error(403))
|
|
result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
|
|
|
|
assert result["action"] == "configmap_patch"
|
|
assert result["target"] == TARGET
|
|
assert result["succeeded"] is False
|
|
assert result["error"] == "configmap patch http 403"
|
|
assert len(calls["patches"]) == 1
|
|
|
|
|
|
def test_execute_repair_survives_a_transport_error(monkeypatch) -> None:
|
|
_install_k8s(monkeypatch, patch_exc=RuntimeError("connection reset by peer"))
|
|
result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
|
|
|
|
assert result["succeeded"] is False
|
|
assert result["error"] == "configmap patch failed: connection reset by peer"
|
|
|
|
|
|
def test_execute_repair_without_namespace_issues_no_request(monkeypatch) -> None:
|
|
calls = _install_k8s(monkeypatch)
|
|
result = module.execute_repair(_cfg(namespace=""), INCIDENT_ID, 12)
|
|
|
|
assert result["succeeded"] is False
|
|
assert result["error"] == "fixture target incomplete"
|
|
assert calls["patches"] == []
|
|
|
|
|
|
def test_execute_repair_without_configmap_name_issues_no_request(monkeypatch) -> None:
|
|
calls = _install_k8s(monkeypatch)
|
|
result = module.execute_repair(_cfg(fixture_configmap=None), INCIDENT_ID, 12)
|
|
|
|
assert result["succeeded"] is False
|
|
assert result["error"] == "fixture target incomplete"
|
|
assert calls["patches"] == []
|
|
|
|
|
|
def test_execute_repair_is_idempotent_across_repeats(monkeypatch) -> None:
|
|
calls = _install_k8s(monkeypatch)
|
|
first = module.execute_repair(_cfg(), INCIDENT_ID, 12)
|
|
second = module.execute_repair(_cfg(), INCIDENT_ID, 12)
|
|
|
|
assert first == second
|
|
assert calls["patches"] == [
|
|
(CONFIGMAP_PATH, {"data": {"state": "healthy"}}),
|
|
(CONFIGMAP_PATH, {"data": {"state": "healthy"}}),
|
|
]
|
|
|
|
|
|
def _install_http(monkeypatch, response=None, exc=None) -> dict: # type: ignore[no-untyped-def]
|
|
calls: dict = {"posts": [], "kwargs": None}
|
|
|
|
class FakeClient:
|
|
def __init__(self, **kwargs) -> None: # type: ignore[no-untyped-def]
|
|
calls["kwargs"] = kwargs
|
|
|
|
def __enter__(self): # type: ignore[no-untyped-def]
|
|
return self
|
|
|
|
def __exit__(self, *args) -> None: # type: ignore[no-untyped-def]
|
|
return None
|
|
|
|
def post(self, url, data=None): # type: ignore[no-untyped-def]
|
|
calls["posts"].append((url, data))
|
|
if exc is not None:
|
|
raise exc
|
|
return response
|
|
|
|
monkeypatch.setattr(module.httpx, "Client", FakeClient)
|
|
return calls
|
|
|
|
|
|
def test_trigger_rebuild_success(monkeypatch) -> None:
|
|
calls = _install_http(monkeypatch, response=FakeResponse(201))
|
|
result = module.trigger_rebuild(_jenkins_settings(), "hermes-triage-demo")
|
|
assert result == {"requested": True, "error": None}
|
|
url, data = calls["posts"][0]
|
|
assert url == "https://ci.example/job/hermes-triage-demo/buildWithParameters"
|
|
assert data == {"SEED_FAILURE": "false"}
|
|
assert calls["kwargs"]["auth"] == ("user", "token")
|
|
assert calls["kwargs"]["timeout"] == 5.0
|
|
|
|
|
|
def test_trigger_rebuild_non_created_status(monkeypatch) -> None:
|
|
_install_http(monkeypatch, response=FakeResponse(500))
|
|
result = module.trigger_rebuild(_jenkins_settings(), "hermes-triage-demo")
|
|
assert result == {"requested": False, "error": "rebuild http 500"}
|
|
|
|
|
|
def test_trigger_rebuild_request_failure(monkeypatch) -> None:
|
|
_install_http(monkeypatch, exc=RuntimeError("connect refused"))
|
|
result = module.trigger_rebuild(_jenkins_settings(), "hermes-triage-demo")
|
|
assert result["requested"] is False
|
|
assert result["error"] == "rebuild request failed: connect refused"
|
|
|
|
|
|
def test_trigger_rebuild_without_base_url(monkeypatch) -> None:
|
|
calls = _install_http(monkeypatch, response=FakeResponse(201))
|
|
result = module.trigger_rebuild(_jenkins_settings(jenkins_base_url=""), "hermes-triage-demo")
|
|
assert result == {"requested": False, "error": "jenkins base url is empty"}
|
|
assert calls["posts"] == []
|
|
|
|
|
|
def test_trigger_rebuild_without_credentials(monkeypatch) -> None:
|
|
calls = _install_http(monkeypatch, response=FakeResponse(201))
|
|
config = _jenkins_settings(jenkins_api_user="", jenkins_api_token="")
|
|
result = module.trigger_rebuild(config, "hermes-triage-demo")
|
|
assert result["requested"] is True
|
|
assert "auth" not in calls["kwargs"]
|
|
|
|
|
|
def test_retry_build_parameterized_reuses_the_rebuild_form(monkeypatch) -> None:
|
|
calls = _install_http(monkeypatch, response=FakeResponse(201))
|
|
result = module.retry_build(_jenkins_settings(), "hermes-triage-demo", parameterized=True)
|
|
|
|
assert result == {"requested": True, "error": None}
|
|
url, data = calls["posts"][0]
|
|
assert url == "https://ci.example/job/hermes-triage-demo/buildWithParameters"
|
|
assert data == {"SEED_FAILURE": "false"}
|
|
assert calls["kwargs"]["auth"] == ("user", "token")
|
|
|
|
|
|
def test_retry_build_unparameterized_posts_no_parameters(monkeypatch) -> None:
|
|
calls = _install_http(monkeypatch, response=FakeResponse(201))
|
|
result = module.retry_build(_jenkins_settings(), "titan-iac", parameterized=False)
|
|
|
|
assert result == {"requested": True, "error": None}
|
|
url, data = calls["posts"][0]
|
|
assert url == "https://ci.example/job/titan-iac/build"
|
|
assert data is None
|
|
|
|
|
|
def test_retry_build_non_created_status(monkeypatch) -> None:
|
|
_install_http(monkeypatch, response=FakeResponse(500))
|
|
result = module.retry_build(_jenkins_settings(), "titan-iac", parameterized=False)
|
|
assert result == {"requested": False, "error": "retry http 500"}
|
|
|
|
|
|
def test_retry_build_request_failure(monkeypatch) -> None:
|
|
_install_http(monkeypatch, exc=RuntimeError("connection reset by peer"))
|
|
result = module.retry_build(_jenkins_settings(), "titan-iac", parameterized=False)
|
|
assert result == {"requested": False, "error": "retry request failed: connection reset by peer"}
|
|
|
|
|
|
def test_retry_build_without_base_url(monkeypatch) -> None:
|
|
calls = _install_http(monkeypatch, response=FakeResponse(201))
|
|
result = module.retry_build(_jenkins_settings(jenkins_base_url=""), "titan-iac", parameterized=True)
|
|
assert result == {"requested": False, "error": "jenkins base url is empty"}
|
|
assert calls["posts"] == []
|
|
|
|
|
|
def test_retry_build_without_credentials(monkeypatch) -> None:
|
|
calls = _install_http(monkeypatch, response=FakeResponse(201))
|
|
config = _jenkins_settings(jenkins_api_user="", jenkins_api_token="")
|
|
result = module.retry_build(config, "titan-iac", parameterized=False)
|
|
assert result["requested"] is True
|
|
assert "auth" not in calls["kwargs"]
|