From 1cbed3f6a569c89266935bb12d9b1c60b8bd568c Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 5 Aug 2026 17:42:00 -0300 Subject: [PATCH] refactor(hermes-autotriage): repair via ConfigMap patch, not PVC write Longhorn RWO attach latency and per-node engine availability made the PVC fixture unreliable for a fast repeatable demo. The fixture is now a ConfigMap; the repair Job runs bitnami/kubectl under the least-privilege hermes-demo-repair SA and patches state=healthy. Co-Authored-By: Claude Fable 5 --- ariadne/services/hermes_autotriage.py | 2 +- ariadne/services/hermes_autotriage_repair.py | 18 +++++++++--------- ariadne/settings.py | 2 +- ariadne/settings_sections.py | 4 ++-- tests/test_hermes_autotriage.py | 4 ++-- tests/test_hermes_autotriage_repair.py | 16 ++++++++-------- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/ariadne/services/hermes_autotriage.py b/ariadne/services/hermes_autotriage.py index dfd732b..c2b95c7 100644 --- a/ariadne/services/hermes_autotriage.py +++ b/ariadne/services/hermes_autotriage.py @@ -477,7 +477,7 @@ def _repair_config() -> dict[str, Any]: return { "namespace": settings.hermes_demo_namespace, - "fixture_pvc": settings.hermes_demo_fixture_pvc, + "fixture_configmap": settings.hermes_demo_fixture_configmap, "image": settings.hermes_repair_image, } diff --git a/ariadne/services/hermes_autotriage_repair.py b/ariadne/services/hermes_autotriage_repair.py index 6207c11..d4f0aa8 100644 --- a/ariadne/services/hermes_autotriage_repair.py +++ b/ariadne/services/hermes_autotriage_repair.py @@ -24,7 +24,7 @@ _REPAIR_MESSAGE = "fixture state reset to healthy" def execute_repair(cfg: dict, incident_id: str, build_number: int) -> dict[str, Any]: """Run the hardcoded demo-fixture repair Job and wait for its outcome. - Inputs: `cfg` with namespace, fixture_pvc, and image (plus optional + Inputs: `cfg` with namespace, fixture_configmap, and image (plus optional wait_timeout_seconds); the incident id and the failed build number that names the Job. Outputs: {"job_name", "succeeded", "error"}; a 409 on creation is reported as a "duplicate job" failure. Never raises. @@ -80,7 +80,13 @@ def _job_payload(cfg: dict, job_name: str, incident_id: str) -> dict[str, Any]: {"event": "hermes_demo_repair", "incident_id": incident_id, "message": _REPAIR_MESSAGE}, separators=(",", ":"), ) - script = f"printf 'healthy' > /fixture/state && echo '{marker}'" + namespace = str(cfg.get("namespace") or "") + fixture = str(cfg.get("fixture_configmap") or "hermes-triage-demo-fixture") + patch = '{"data":{"state":"healthy"}}' + script = ( + f"kubectl -n {namespace} patch configmap {fixture} " + f"--type merge -p '{patch}' && echo '{marker}'" + ) return { "apiVersion": "batch/v1", "kind": "Job", @@ -99,19 +105,13 @@ def _job_payload(cfg: dict, job_name: str, incident_id: str) -> dict[str, Any]: "metadata": {"labels": {"app.kubernetes.io/part-of": "hermes-triage-demo"}}, "spec": { "restartPolicy": "Never", + "serviceAccountName": "hermes-demo-repair", "nodeSelector": {"node-role.kubernetes.io/worker": "true"}, "containers": [ { "name": "repair", "image": str(cfg.get("image") or ""), "command": ["sh", "-c", script], - "volumeMounts": [{"name": "fixture", "mountPath": "/fixture"}], - } - ], - "volumes": [ - { - "name": "fixture", - "persistentVolumeClaim": {"claimName": str(cfg.get("fixture_pvc") or "")}, } ], }, diff --git a/ariadne/settings.py b/ariadne/settings.py index 4c33f11..76fee9f 100644 --- a/ariadne/settings.py +++ b/ariadne/settings.py @@ -190,7 +190,7 @@ class Settings: hermes_api_key: str hermes_run_timeout_seconds: float hermes_demo_namespace: str - hermes_demo_fixture_pvc: str + hermes_demo_fixture_configmap: str hermes_repair_image: str vaultwarden_namespace: str diff --git a/ariadne/settings_sections.py b/ariadne/settings_sections.py index e047d96..f7ac66c 100644 --- a/ariadne/settings_sections.py +++ b/ariadne/settings_sections.py @@ -277,8 +277,8 @@ def _hermes_autotriage_config() -> dict[str, Any]: "hermes_api_key": _env("ARIADNE_HERMES_API_KEY", ""), "hermes_run_timeout_seconds": _env_float("ARIADNE_HERMES_RUN_TIMEOUT_SECONDS", 420.0), "hermes_demo_namespace": _env("ARIADNE_HERMES_DEMO_NAMESPACE", "hermes-triage-demo"), - "hermes_demo_fixture_pvc": _env("ARIADNE_HERMES_DEMO_FIXTURE_PVC", "hermes-triage-demo-fixture"), - "hermes_repair_image": _env("ARIADNE_HERMES_REPAIR_IMAGE", "busybox:1.37"), + "hermes_demo_fixture_configmap": _env("ARIADNE_HERMES_DEMO_FIXTURE_CONFIGMAP", "hermes-triage-demo-fixture"), + "hermes_repair_image": _env("ARIADNE_HERMES_REPAIR_IMAGE", "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131"), } diff --git a/tests/test_hermes_autotriage.py b/tests/test_hermes_autotriage.py index 81cfc2c..7a1998b 100644 --- a/tests/test_hermes_autotriage.py +++ b/tests/test_hermes_autotriage.py @@ -52,7 +52,7 @@ def _settings(**overrides) -> SimpleNamespace: # type: ignore[no-untyped-def] "hermes_api_key": "key", "hermes_run_timeout_seconds": 420.0, "hermes_demo_namespace": "hermes-triage-demo", - "hermes_demo_fixture_pvc": "hermes-triage-demo-fixture", + "hermes_demo_fixture_configmap": "hermes-triage-demo-fixture", "hermes_repair_image": "busybox:1.37", "jenkins_base_url": "https://ci.example", "jenkins_api_user": "user", @@ -261,7 +261,7 @@ def test_new_failure_full_happy_path(monkeypatch) -> None: ( { "namespace": "hermes-triage-demo", - "fixture_pvc": "hermes-triage-demo-fixture", + "fixture_configmap": "hermes-triage-demo-fixture", "image": "busybox:1.37", }, INCIDENT_ID, diff --git a/tests/test_hermes_autotriage_repair.py b/tests/test_hermes_autotriage_repair.py index 2e210a0..8c334a4 100644 --- a/tests/test_hermes_autotriage_repair.py +++ b/tests/test_hermes_autotriage_repair.py @@ -29,8 +29,8 @@ class FakeResponse: def _cfg(**overrides) -> dict: # type: ignore[no-untyped-def] base = { "namespace": "hermes-triage-demo", - "fixture_pvc": "hermes-triage-demo-fixture", - "image": "busybox:1.37", + "fixture_configmap": "hermes-triage-demo-fixture", + "image": "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131", } base.update(overrides) return base @@ -93,18 +93,18 @@ def test_repair_job_payload_contract(monkeypatch) -> None: pod = spec["template"]["spec"] assert pod["restartPolicy"] == "Never" assert pod["nodeSelector"] == {"node-role.kubernetes.io/worker": "true"} + assert pod["serviceAccountName"] == "hermes-demo-repair" container = pod["containers"][0] - assert container["image"] == "busybox:1.37" + assert container["image"].startswith("bitnami/kubectl@sha256:") assert container["command"][:2] == ["sh", "-c"] script = container["command"][2] - assert "printf 'healthy' > /fixture/state" in script + assert "kubectl -n hermes-triage-demo patch configmap hermes-triage-demo-fixture" in script + assert '{"data":{"state":"healthy"}}' in script assert '"event":"hermes_demo_repair"' in script assert f'"incident_id":"{INCIDENT_ID}"' in script assert '"message":"fixture state reset to healthy"' in script - assert container["volumeMounts"] == [{"name": "fixture", "mountPath": "/fixture"}] - assert pod["volumes"] == [ - {"name": "fixture", "persistentVolumeClaim": {"claimName": "hermes-triage-demo-fixture"}} - ] + assert "volumeMounts" not in container + assert "volumes" not in pod def test_execute_repair_waits_through_active_polls(monkeypatch) -> None: