refactor(hermes-triage): repair the fixture in-process, not via a spawned Job

The repair action no longer creates a Kubernetes Job and polls it. Ariadne
patches the fixture ConfigMap directly through its own k8s client, which
removes roughly 40 seconds of pod scheduling from the loop, drops the two
failure modes that Job introduced (volume attach and node selection), and
turns an opaque pod log into an Ariadne event.

- execute_repair returns {action, target, succeeded, error} and issues one
  merge patch; no Job, no polling, no injectable clock, never raises
- the patch writes the same terminal value every time, so idempotency needs
  no duplicate guard; one action per incident is still enforced upstream
- orchestrator records {repair, target}; the state machine, rebuild trigger
  and failure path are unchanged
- retires the unused repair-image setting

Ariadne's service account now needs get+patch on that one ConfigMap by name
instead of Job create; the batch/jobs grant and the hermes-demo-repair
service account can be retired.

480 pass in the hermes suite.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
codex 2026-08-05 22:29:57 -03:00
parent 950d014707
commit b2fcad4116
7 changed files with 161 additions and 213 deletions

View File

@ -355,7 +355,13 @@ def _execute_action(
def _repair_demo_fixture(storage: Any, base: dict[str, Any], action_id: str) -> dict[str, Any]: def _repair_demo_fixture(storage: Any, base: dict[str, Any], action_id: str) -> dict[str, Any]:
"""Run the demo-fixture repair Job and request the verification rebuild.""" """Patch the demo fixture healthy and request the verification rebuild.
The repair runs in-process against the cluster API rather than as a
Kubernetes Job, so the executed action carries the patched target instead
of a Job name; the one-action-per-incident budget upstream is still what
stops the repair from repeating.
"""
hermes_events.record_action(storage, base, action_id, "requested", None) hermes_events.record_action(storage, base, action_id, "requested", None)
hermes_events.record_action(storage, base, action_id, "accepted", None) hermes_events.record_action(storage, base, action_id, "accepted", None)
@ -370,18 +376,10 @@ def _repair_demo_fixture(storage: Any, base: dict[str, Any], action_id: str) ->
rebuild = hermes_autotriage_repair.trigger_rebuild(settings, str(base["job"])) rebuild = hermes_autotriage_repair.trigger_rebuild(settings, str(base["job"]))
if not rebuild.get("requested"): if not rebuild.get("requested"):
return _fail_action(storage, base, action_id, str(rebuild.get("error") or "rebuild trigger failed")) return _fail_action(storage, base, action_id, str(rebuild.get("error") or "rebuild trigger failed"))
hermes_events.record_action(storage, base, action_id, "executed", {"repair_job": repair.get("job_name")}) detail = {"repair": repair.get("action"), "target": repair.get("target")}
hermes_events.record_incident( hermes_events.record_action(storage, base, action_id, "executed", detail)
storage, hermes_events.record_incident(storage, base, "awaiting_rebuild", {"action": action_id, **detail})
base, return {"status": "awaiting_rebuild", "incident_id": str(base["incident_id"]), **detail}
"awaiting_rebuild",
{"action": action_id, "repair_job": repair.get("job_name")},
)
return {
"status": "awaiting_rebuild",
"incident_id": str(base["incident_id"]),
"repair_job": repair.get("job_name"),
}
def _retry_transient_infra(storage: Any, base: dict[str, Any], marker: str | None) -> dict[str, Any]: def _retry_transient_infra(storage: Any, base: dict[str, Any], marker: str | None) -> dict[str, Any]:
@ -480,12 +478,11 @@ def _decision_config() -> dict[str, Any]:
def _repair_config() -> dict[str, Any]: def _repair_config() -> dict[str, Any]:
"""Build the config passed to the repair Job executor.""" """Build the config naming the ConfigMap the repair action patches."""
return { return {
"namespace": settings.hermes_demo_namespace, "namespace": settings.hermes_demo_namespace,
"fixture_configmap": settings.hermes_demo_fixture_configmap, "fixture_configmap": settings.hermes_demo_fixture_configmap,
"image": settings.hermes_repair_image,
} }

View File

@ -1,54 +1,55 @@
"""Remediation executors for the Hermes auto-triage demo incident flow.
The fixture repair is an in-process Ariadne action, not a Kubernetes Job: it
merge-patches the demo fixture ConfigMap back to healthy over the cluster API
using Ariadne's own service account, which needs get/patch on that one
ConfigMap. Because the patch always writes the same terminal value, the
repair is naturally idempotent and needs no duplicate guard of its own; the
one-action-per-incident budget upstream (the prior-action-count check in the
authorization chain) remains the only limit on how often it runs.
"""
from __future__ import annotations from __future__ import annotations
import json
import time
from typing import Any from typing import Any
import httpx import httpx
from ..k8s.client import get_json, post_json from ..k8s.client import patch_json
from ..utils.logging import get_logger from ..utils.logging import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
HTTP_CREATED = 201 HTTP_CREATED = 201
HTTP_CONFLICT = 409
_DEFAULT_WAIT_TIMEOUT_SECONDS = 120.0 _REPAIR_ACTION = "configmap_patch"
_POLL_INTERVAL_SECONDS = 2.0
_JOB_TTL_SECONDS = 3600
_REPAIR_MESSAGE = "fixture state reset to healthy"
_PARAMETERIZED_ENDPOINT = "buildWithParameters" _PARAMETERIZED_ENDPOINT = "buildWithParameters"
_PLAIN_ENDPOINT = "build" _PLAIN_ENDPOINT = "build"
_BUILD_PARAMETERS = {"SEED_FAILURE": "false"} _BUILD_PARAMETERS = {"SEED_FAILURE": "false"}
def execute_repair(cfg: dict, incident_id: str, build_number: int) -> dict[str, Any]: def execute_repair(cfg: dict, incident_id: str, build_number: int) -> dict[str, Any]:
"""Run the hardcoded demo-fixture repair Job and wait for its outcome. """Patch the demo fixture ConfigMap back to healthy in one API call.
Inputs: `cfg` with namespace, fixture_configmap, and image (plus optional Inputs: `cfg` with namespace and fixture_configmap, the incident id used
wait_timeout_seconds); the incident id and the failed build number that for the log line, and the failed build number (kept for the executor
names the Job. Outputs: {"job_name", "succeeded", "error"}; a 409 on signature; the patch is the same whatever built). Outputs: {"action",
creation is reported as a "duplicate job" failure. Never raises. "target", "succeeded", "error"}. No Job, no polling: a non-2xx response,
a transport error, or an incomplete target returns succeeded False with a
short reason. Never raises.
""" """
namespace = str(cfg.get("namespace") or "") namespace = str(cfg.get("namespace") or "").strip()
job_name = f"hermes-demo-repair-{build_number}" name = str(cfg.get("fixture_configmap") or "").strip()
target = f"{namespace}/{name}"
if not namespace or not name:
return _repair_result(target, False, "fixture target incomplete", incident_id)
try: try:
post_json( patch_json(f"/api/v1/namespaces/{namespace}/configmaps/{name}", {"data": {"state": "healthy"}})
f"/apis/batch/v1/namespaces/{namespace}/jobs",
_job_payload(cfg, job_name, incident_id),
)
except Exception as exc: except Exception as exc:
error = "duplicate job" if _status_code(exc) == HTTP_CONFLICT else f"job create failed: {exc}" return _repair_result(target, False, _patch_error(exc), incident_id)
logger.info( return _repair_result(target, True, None, incident_id)
"hermes demo repair job creation failed",
extra={"event": "hermes_autotriage_repair", "status": "error", "job": job_name, "detail": error},
)
return {"job_name": job_name, "succeeded": False, "error": error}
timeout_seconds = _number(cfg.get("wait_timeout_seconds"), _DEFAULT_WAIT_TIMEOUT_SECONDS)
return _wait_for_completion(namespace, job_name, timeout_seconds)
def trigger_rebuild(config: Any, job: str) -> dict[str, Any]: def trigger_rebuild(config: Any, job: str) -> dict[str, Any]:
@ -99,73 +100,28 @@ def _post_build(
return {"requested": False, "error": f"{label} http {response.status_code}"} return {"requested": False, "error": f"{label} http {response.status_code}"}
def _job_payload(cfg: dict, job_name: str, incident_id: str) -> dict[str, Any]: def _repair_result(target: str, succeeded: bool, error: str | None, incident_id: str) -> dict[str, Any]:
"""Build the hardcoded batch Job manifest that resets the demo fixture.""" """Log one repair outcome line and return the executor's result dict."""
marker = json.dumps( logger.info(
{"event": "hermes_demo_repair", "incident_id": incident_id, "message": _REPAIR_MESSAGE}, "hermes demo fixture repair",
separators=(",", ":"), extra={
) "event": "hermes_autotriage_repair",
namespace = str(cfg.get("namespace") or "") "status": "ok" if succeeded else "error",
fixture = str(cfg.get("fixture_configmap") or "hermes-triage-demo-fixture") "target": target,
patch = '{"data":{"state":"healthy"}}' "incident_id": incident_id,
script = (
f"kubectl -n {namespace} patch configmap {fixture} "
f"--type merge -p '{patch}' && echo '{marker}'"
)
return {
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": job_name,
"namespace": str(cfg.get("namespace") or ""),
"labels": {
"atlas.bstein.dev/trigger": "ariadne",
"app.kubernetes.io/part-of": "hermes-triage-demo",
},
}, },
"spec": { )
"backoffLimit": 0, return {"action": _REPAIR_ACTION, "target": target, "succeeded": succeeded, "error": error}
"ttlSecondsAfterFinished": _JOB_TTL_SECONDS,
"template": {
"metadata": {"labels": {"app.kubernetes.io/part-of": "hermes-triage-demo"}},
"spec": {
"restartPolicy": "Never",
"serviceAccountName": "hermes-demo-repair",
"nodeSelector": {"node-role.kubernetes.io/worker": "true"},
"containers": [
{
"name": "repair",
"image": str(cfg.get("image") or ""),
"command": ["sh", "-c", script],
}
],
},
},
},
}
def _wait_for_completion(namespace: str, job_name: str, timeout_seconds: float) -> dict[str, Any]: def _patch_error(exc: Exception) -> str:
"""Poll the repair Job until it succeeds, fails, or times out.""" """Render a short reason for a failed ConfigMap patch."""
deadline = time.time() + timeout_seconds code = _status_code(exc)
while time.time() < deadline: if code is not None:
try: return f"configmap patch http {code}"
job = get_json(f"/apis/batch/v1/namespaces/{namespace}/jobs/{job_name}") return f"configmap patch failed: {exc}"
except Exception as exc:
return {"job_name": job_name, "succeeded": False, "error": f"job status read failed: {exc}"}
status = job.get("status") if isinstance(job.get("status"), dict) else {}
if _int_value(status.get("succeeded")) > 0:
return {"job_name": job_name, "succeeded": True, "error": None}
if _int_value(status.get("failed")) > 0:
return {"job_name": job_name, "succeeded": False, "error": "repair job failed"}
time.sleep(_POLL_INTERVAL_SECONDS)
return {
"job_name": job_name,
"succeeded": False,
"error": f"repair job timeout after {timeout_seconds}s",
}
def _jenkins_client_kwargs(config: Any) -> dict[str, Any]: def _jenkins_client_kwargs(config: Any) -> dict[str, Any]:
@ -196,12 +152,3 @@ def _number(value: Any, default: float) -> float:
return float(value) return float(value)
except (TypeError, ValueError): except (TypeError, ValueError):
return default return default
def _int_value(value: Any) -> int:
"""Coerce a value to int, defaulting to zero."""
try:
return int(value)
except (TypeError, ValueError):
return 0

View File

@ -195,7 +195,6 @@ class Settings:
hermes_run_timeout_seconds: float hermes_run_timeout_seconds: float
hermes_demo_namespace: str hermes_demo_namespace: str
hermes_demo_fixture_configmap: str hermes_demo_fixture_configmap: str
hermes_repair_image: str
hermes_code_enabled: bool hermes_code_enabled: bool
hermes_code_job: str hermes_code_job: str
hermes_code_owner: str hermes_code_owner: str

View File

@ -298,7 +298,6 @@ def _hermes_autotriage_config() -> dict[str, Any]:
"hermes_run_timeout_seconds": _env_float("ARIADNE_HERMES_RUN_TIMEOUT_SECONDS", 420.0), "hermes_run_timeout_seconds": _env_float("ARIADNE_HERMES_RUN_TIMEOUT_SECONDS", 420.0),
"hermes_demo_namespace": _env("ARIADNE_HERMES_DEMO_NAMESPACE", "hermes-triage-demo"), "hermes_demo_namespace": _env("ARIADNE_HERMES_DEMO_NAMESPACE", "hermes-triage-demo"),
"hermes_demo_fixture_configmap": _env("ARIADNE_HERMES_DEMO_FIXTURE_CONFIGMAP", "hermes-triage-demo-fixture"), "hermes_demo_fixture_configmap": _env("ARIADNE_HERMES_DEMO_FIXTURE_CONFIGMAP", "hermes-triage-demo-fixture"),
"hermes_repair_image": _env("ARIADNE_HERMES_REPAIR_IMAGE", "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131"),
} }

View File

@ -64,7 +64,6 @@ def _settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def]
"hermes_run_timeout_seconds": 420.0, "hermes_run_timeout_seconds": 420.0,
"hermes_demo_namespace": "hermes-triage-demo", "hermes_demo_namespace": "hermes-triage-demo",
"hermes_demo_fixture_configmap": "hermes-triage-demo-fixture", "hermes_demo_fixture_configmap": "hermes-triage-demo-fixture",
"hermes_repair_image": "busybox:1.37",
"hermes_code_enabled": False, "hermes_code_enabled": False,
"hermes_code_job": "hermes-code-demo", "hermes_code_job": "hermes-code-demo",
"hermes_code_owner": "bstein", "hermes_code_owner": "bstein",
@ -217,7 +216,12 @@ def _prepare( # type: ignore[no-untyped-def] # noqa: PLR0913
calls["repairs"].append((repair_cfg, incident_id, build_number)) calls["repairs"].append((repair_cfg, incident_id, build_number))
if repair is not None: if repair is not None:
return repair return repair
return {"job_name": "hermes-demo-repair-12", "succeeded": True, "error": None} return {
"action": "configmap_patch",
"target": "hermes-triage-demo/hermes-triage-demo-fixture",
"succeeded": True,
"error": None,
}
def fake_trigger_rebuild(rebuild_cfg, job): # type: ignore[no-untyped-def] def fake_trigger_rebuild(rebuild_cfg, job): # type: ignore[no-untyped-def]
calls["rebuilds"].append(job) calls["rebuilds"].append(job)

View File

@ -56,7 +56,8 @@ def test_new_failure_full_happy_path(monkeypatch) -> None:
summary = module.run_hermes_autotriage(env.storage) summary = module.run_hermes_autotriage(env.storage)
job_summary = summary["jobs"][JOB] job_summary = summary["jobs"][JOB]
assert job_summary["status"] == "awaiting_rebuild" assert job_summary["status"] == "awaiting_rebuild"
assert job_summary["repair_job"] == "hermes-demo-repair-12" assert job_summary["repair"] == "configmap_patch"
assert job_summary["target"] == "hermes-triage-demo/hermes-triage-demo-fixture"
assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "awaiting_rebuild"] assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "awaiting_rebuild"]
actions = _events(env.storage, module.ACTION_EVENT_TYPE) actions = _events(env.storage, module.ACTION_EVENT_TYPE)
assert [action["result"] for action in actions] == ["requested", "accepted", "executed"] assert [action["result"] for action in actions] == ["requested", "accepted", "executed"]
@ -78,12 +79,21 @@ def test_new_failure_full_happy_path(monkeypatch) -> None:
{ {
"namespace": "hermes-triage-demo", "namespace": "hermes-triage-demo",
"fixture_configmap": "hermes-triage-demo-fixture", "fixture_configmap": "hermes-triage-demo-fixture",
"image": "busybox:1.37",
}, },
INCIDENT_ID, INCIDENT_ID,
12, 12,
) )
] ]
assert actions[-1]["detail"] == {
"repair": "configmap_patch",
"target": "hermes-triage-demo/hermes-triage-demo-fixture",
}
awaiting = _events(env.storage, module.INCIDENT_EVENT_TYPE)[-1]
assert awaiting["phase"] == {
"action": "repair_demo_fixture",
"repair": "configmap_patch",
"target": "hermes-triage-demo/hermes-triage-demo-fixture",
}
assert env.calls["rebuilds"] == [JOB] assert env.calls["rebuilds"] == [JOB]
assert _counter("repair_demo_fixture", "success") == success_before + 1.0 assert _counter("repair_demo_fixture", "success") == success_before + 1.0
assert _gauge("12", "awaiting_rebuild") == 1.0 assert _gauge("12", "awaiting_rebuild") == 1.0
@ -224,13 +234,18 @@ def test_repair_failure_marks_failed_and_human(monkeypatch) -> None:
failed_before = _counter("repair_demo_fixture", "failed") failed_before = _counter("repair_demo_fixture", "failed")
env = _prepare( env = _prepare(
monkeypatch, monkeypatch,
repair={"job_name": "hermes-demo-repair-12", "succeeded": False, "error": "repair job failed"}, repair={
"action": "configmap_patch",
"target": "hermes-triage-demo/hermes-triage-demo-fixture",
"succeeded": False,
"error": "configmap patch http 403",
},
) )
summary = module.run_hermes_autotriage(env.storage) summary = module.run_hermes_autotriage(env.storage)
assert summary["jobs"][JOB] == { assert summary["jobs"][JOB] == {
"status": "failed", "status": "failed",
"incident_id": INCIDENT_ID, "incident_id": INCIDENT_ID,
"reason": "repair job failed", "reason": "configmap patch http 403",
} }
assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "failed"] assert _statuses(env.storage) == ["detected", "diagnosed", "repairing", "failed"]
actions = _events(env.storage, module.ACTION_EVENT_TYPE) actions = _events(env.storage, module.ACTION_EVENT_TYPE)

View File

@ -2,23 +2,13 @@ from __future__ import annotations
from types import SimpleNamespace from types import SimpleNamespace
from ariadne.k8s import client as k8s_client
from ariadne.services import hermes_autotriage_repair as module from ariadne.services import hermes_autotriage_repair as module
INCIDENT_ID = "hermes-triage-demo/12" INCIDENT_ID = "hermes-triage-demo/12"
CONFIGMAP_PATH = "/api/v1/namespaces/hermes-triage-demo/configmaps/hermes-triage-demo-fixture"
TARGET = "hermes-triage-demo/hermes-triage-demo-fixture"
class FakeClock:
def __init__(self) -> None:
self.now = 0.0
self.sleeps: list[float] = []
def time(self) -> float:
return self.now
def sleep(self, seconds: float) -> None:
self.sleeps.append(seconds)
self.now += seconds
class FakeResponse: class FakeResponse:
@ -30,7 +20,6 @@ def _cfg(**overrides) -> dict: # type: ignore[no-untyped-def]
base = { base = {
"namespace": "hermes-triage-demo", "namespace": "hermes-triage-demo",
"fixture_configmap": "hermes-triage-demo-fixture", "fixture_configmap": "hermes-triage-demo-fixture",
"image": "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131",
} }
base.update(overrides) base.update(overrides)
return base return base
@ -47,110 +36,108 @@ def _jenkins_settings(**overrides) -> SimpleNamespace: # type: ignore[no-untype
return SimpleNamespace(**values) return SimpleNamespace(**values)
def _install_k8s(monkeypatch, statuses, post_exc=None) -> dict: # type: ignore[no-untyped-def] def _install_k8s(monkeypatch, patch_exc=None) -> dict: # type: ignore[no-untyped-def]
calls: dict = {"posts": [], "gets": []} """Record every Kubernetes call the repair path makes."""
queue = list(statuses)
calls: dict = {"patches": [], "posts": [], "gets": []}
def fake_patch(path, payload): # type: ignore[no-untyped-def]
calls["patches"].append((path, payload))
if patch_exc is not None:
raise patch_exc
return {"metadata": {"name": "hermes-triage-demo-fixture"}}
def fake_post(path, payload): # type: ignore[no-untyped-def] def fake_post(path, payload): # type: ignore[no-untyped-def]
calls["posts"].append((path, payload)) calls["posts"].append((path, payload))
if post_exc is not None: return {}
raise post_exc
return {"metadata": {"name": payload["metadata"]["name"]}}
def fake_get(path): # type: ignore[no-untyped-def] def fake_get(path): # type: ignore[no-untyped-def]
calls["gets"].append(path) calls["gets"].append(path)
item = queue.pop(0) if queue else {"status": {"active": 1}} return {}
if isinstance(item, Exception):
raise item
return item
monkeypatch.setattr(module, "post_json", fake_post) monkeypatch.setattr(module, "patch_json", fake_patch)
monkeypatch.setattr(module, "get_json", fake_get) monkeypatch.setattr(k8s_client, "post_json", fake_post)
clock = FakeClock() monkeypatch.setattr(k8s_client, "get_json", fake_get)
monkeypatch.setattr(module, "time", clock)
calls["clock"] = clock
return calls return calls
def test_repair_job_payload_contract(monkeypatch) -> None: def _http_error(status_code: int) -> Exception:
calls = _install_k8s(monkeypatch, [{"status": {"succeeded": 1}}]) error = RuntimeError(f"http {status_code}")
error.response = SimpleNamespace(status_code=status_code) # type: ignore[attr-defined]
return error
def test_execute_repair_patches_the_fixture_configmap(monkeypatch) -> None:
calls = _install_k8s(monkeypatch)
result = module.execute_repair(_cfg(), INCIDENT_ID, 12) result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert result == {"job_name": "hermes-demo-repair-12", "succeeded": True, "error": None} assert result == {
path, payload = calls["posts"][0] "action": "configmap_patch",
assert path == "/apis/batch/v1/namespaces/hermes-triage-demo/jobs" "target": TARGET,
assert payload["apiVersion"] == "batch/v1" "succeeded": True,
assert payload["kind"] == "Job" "error": None,
assert payload["metadata"]["name"] == "hermes-demo-repair-12"
assert payload["metadata"]["namespace"] == "hermes-triage-demo"
assert payload["metadata"]["labels"] == {
"atlas.bstein.dev/trigger": "ariadne",
"app.kubernetes.io/part-of": "hermes-triage-demo",
} }
spec = payload["spec"] assert calls["patches"] == [(CONFIGMAP_PATH, {"data": {"state": "healthy"}})]
assert spec["backoffLimit"] == 0
assert spec["ttlSecondsAfterFinished"] == 3600
pod = spec["template"]["spec"]
assert pod["restartPolicy"] == "Never"
assert pod["nodeSelector"] == {"node-role.kubernetes.io/worker": "true"}
assert pod["serviceAccountName"] == "hermes-demo-repair"
container = pod["containers"][0]
assert container["image"].startswith("bitnami/kubectl@sha256:")
assert container["command"][:2] == ["sh", "-c"]
script = container["command"][2]
assert "kubectl -n hermes-triage-demo patch configmap hermes-triage-demo-fixture" in script
assert '{"data":{"state":"healthy"}}' in script
assert '"event":"hermes_demo_repair"' in script
assert f'"incident_id":"{INCIDENT_ID}"' in script
assert '"message":"fixture state reset to healthy"' in script
assert "volumeMounts" not in container
assert "volumes" not in pod
def test_execute_repair_waits_through_active_polls(monkeypatch) -> None: def test_execute_repair_creates_no_kubernetes_job(monkeypatch) -> None:
calls = _install_k8s( calls = _install_k8s(monkeypatch)
monkeypatch, module.execute_repair(_cfg(), INCIDENT_ID, 12)
[{"status": {"active": 1}}, {"status": {"succeeded": 1}}],
) assert calls["posts"] == []
assert calls["gets"] == []
assert not any("/jobs" in path for path, _ in calls["patches"])
assert not hasattr(module, "_job_payload")
assert not hasattr(module, "_wait_for_completion")
def test_execute_repair_reports_non_2xx_status(monkeypatch) -> None:
calls = _install_k8s(monkeypatch, patch_exc=_http_error(403))
result = module.execute_repair(_cfg(), INCIDENT_ID, 12) result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert result["succeeded"] is True
assert len(calls["gets"]) == 2
assert calls["clock"].sleeps == [2.0]
assert result["action"] == "configmap_patch"
def test_execute_repair_job_failure(monkeypatch) -> None: assert result["target"] == TARGET
_install_k8s(monkeypatch, [{"status": {"failed": 1}}])
result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert result == {"job_name": "hermes-demo-repair-12", "succeeded": False, "error": "repair job failed"}
def test_execute_repair_timeout(monkeypatch) -> None:
_install_k8s(monkeypatch, [])
result = module.execute_repair(_cfg(wait_timeout_seconds=6), INCIDENT_ID, 12)
assert result["succeeded"] is False assert result["succeeded"] is False
assert result["error"] == "repair job timeout after 6.0s" assert result["error"] == "configmap patch http 403"
assert len(calls["patches"]) == 1
def test_execute_repair_duplicate_job(monkeypatch) -> None: def test_execute_repair_survives_a_transport_error(monkeypatch) -> None:
conflict = RuntimeError("conflict") _install_k8s(monkeypatch, patch_exc=RuntimeError("connection reset by peer"))
conflict.response = SimpleNamespace(status_code=409) # type: ignore[attr-defined]
_install_k8s(monkeypatch, [], post_exc=conflict)
result = module.execute_repair(_cfg(), INCIDENT_ID, 12) result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert result == {"job_name": "hermes-demo-repair-12", "succeeded": False, "error": "duplicate job"}
def test_execute_repair_create_error(monkeypatch) -> None:
_install_k8s(monkeypatch, [], post_exc=ValueError("nope"))
result = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert result["succeeded"] is False assert result["succeeded"] is False
assert result["error"] == "job create failed: nope" assert result["error"] == "configmap patch failed: connection reset by peer"
def test_execute_repair_status_read_error(monkeypatch) -> None: def test_execute_repair_without_namespace_issues_no_request(monkeypatch) -> None:
_install_k8s(monkeypatch, [RuntimeError("api down")]) calls = _install_k8s(monkeypatch)
result = module.execute_repair(_cfg(), INCIDENT_ID, 12) result = module.execute_repair(_cfg(namespace=""), INCIDENT_ID, 12)
assert result["succeeded"] is False assert result["succeeded"] is False
assert result["error"] == "job status read failed: api down" assert result["error"] == "fixture target incomplete"
assert calls["patches"] == []
def test_execute_repair_without_configmap_name_issues_no_request(monkeypatch) -> None:
calls = _install_k8s(monkeypatch)
result = module.execute_repair(_cfg(fixture_configmap=None), INCIDENT_ID, 12)
assert result["succeeded"] is False
assert result["error"] == "fixture target incomplete"
assert calls["patches"] == []
def test_execute_repair_is_idempotent_across_repeats(monkeypatch) -> None:
calls = _install_k8s(monkeypatch)
first = module.execute_repair(_cfg(), INCIDENT_ID, 12)
second = module.execute_repair(_cfg(), INCIDENT_ID, 12)
assert first == second
assert calls["patches"] == [
(CONFIGMAP_PATH, {"data": {"state": "healthy"}}),
(CONFIGMAP_PATH, {"data": {"state": "healthy"}}),
]
def _install_http(monkeypatch, response=None, exc=None) -> dict: # type: ignore[no-untyped-def] def _install_http(monkeypatch, response=None, exc=None) -> dict: # type: ignore[no-untyped-def]