248 lines
9.6 KiB
Python
248 lines
9.6 KiB
Python
"""Placement and rollout-cost contracts for the distributed worker pool.
|
|
|
|
The independent review found that mediator-N pinned itself hard to worker-N while
|
|
sharing a ReadWriteOnce claim with it, so a drain or preemption that moved only
|
|
the worker deadlocked the ordinal on Multi-Attach until an operator deleted a
|
|
Pod. It also found that the provider-CLI install was redone on every Pod start
|
|
inside the 10m Flux health window for the whole ``hermes`` app.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
from testing.tests.test_hermes_execution_pool_manifests import ( # noqa: E402
|
|
FLUX,
|
|
HERMES,
|
|
documents,
|
|
mediators,
|
|
node,
|
|
placeable,
|
|
worker,
|
|
)
|
|
|
|
|
|
SCRIPTS = HERMES / "scripts"
|
|
|
|
|
|
def test_the_shared_workspace_claim_cannot_deadlock_on_multi_attach():
|
|
claims = {
|
|
item["metadata"]["name"]: item["spec"]["accessModes"]
|
|
for item in worker()["spec"]["volumeClaimTemplates"]
|
|
}
|
|
assert claims["workspace"] == ["ReadWriteMany"]
|
|
assert claims["tools"] == ["ReadWriteOnce"]
|
|
assert claims["provider-access"] == ["ReadWriteOnce"]
|
|
|
|
# Every claim the mediator co-mounts with the worker must be multi-attachable.
|
|
for ordinal, deployment in enumerate(mediators()):
|
|
shared = {
|
|
volume["persistentVolumeClaim"]["claimName"]
|
|
for volume in deployment["spec"]["template"]["spec"]["volumes"]
|
|
if "persistentVolumeClaim" in volume
|
|
}
|
|
assert f"workspace-hermes-execution-worker-{ordinal}" in shared
|
|
for name in shared:
|
|
if name.startswith("workspace-hermes-execution-worker-"):
|
|
assert claims["workspace"] == ["ReadWriteMany"]
|
|
else:
|
|
assert name == f"hermes-execution-mediator-state-{ordinal}"
|
|
|
|
|
|
def test_mediator_placement_is_a_preference_and_shares_the_worker_priority():
|
|
stateful = worker()["spec"]["template"]["spec"]
|
|
worker_terms = stateful["affinity"]["nodeAffinity"][
|
|
"requiredDuringSchedulingIgnoredDuringExecution"
|
|
]["nodeSelectorTerms"]
|
|
for ordinal, deployment in enumerate(mediators()):
|
|
pod = deployment["spec"]["template"]["spec"]
|
|
assert pod["priorityClassName"] == stateful["priorityClassName"] == "scavenger"
|
|
assert deployment["spec"]["strategy"]["type"] == "Recreate"
|
|
affinity = pod["affinity"]
|
|
# No hard podAffinity: the mediator must never be unschedulable purely
|
|
# because its worker is Pending or placed on a different node.
|
|
assert "requiredDuringSchedulingIgnoredDuringExecution" not in affinity["podAffinity"]
|
|
preferred = affinity["podAffinity"][
|
|
"preferredDuringSchedulingIgnoredDuringExecution"
|
|
]
|
|
assert preferred[0]["podAffinityTerm"]["labelSelector"]["matchLabels"] == {
|
|
"statefulset.kubernetes.io/pod-name": f"hermes-execution-worker-{ordinal}"
|
|
}
|
|
assert affinity["nodeAffinity"][
|
|
"requiredDuringSchedulingIgnoredDuringExecution"
|
|
]["nodeSelectorTerms"] == worker_terms
|
|
|
|
|
|
def test_a_preemption_that_moves_the_worker_leaves_both_pods_schedulable():
|
|
"""The exact drain/pressure sequence that used to need a manual Pod delete."""
|
|
nodes = [node(f"titan-{index:02d}") for index in (5, 6, 7, 8)]
|
|
stateful = worker()["spec"]["template"]["spec"]
|
|
mediator = mediators()[1]["spec"]["template"]["spec"]
|
|
|
|
# Steady state: worker-1 and mediator-1 are colocated on titan-05.
|
|
assert "titan-05" in placeable(stateful, nodes)
|
|
assert "titan-05" in placeable(mediator, nodes)
|
|
|
|
# titan-05 comes under pressure and evicts the scavenger-priority worker,
|
|
# which reschedules elsewhere while the mediator has not moved yet.
|
|
survivors = placeable(stateful, nodes, occupied=("titan-05",))
|
|
assert survivors and "titan-05" not in survivors
|
|
|
|
# The mediator still serves from titan-05 -- the shared claim is RWX, so the
|
|
# rescheduled worker does not block on Multi-Attach.
|
|
assert "titan-05" in placeable(mediator, nodes)
|
|
|
|
# And when titan-05 itself is drained the mediator follows independently,
|
|
# because nothing pins it to a Pod that may still be Pending.
|
|
drained = [
|
|
node(item["name"], cordoned=item["name"] == "titan-05") for item in nodes
|
|
]
|
|
rescheduled = placeable(mediator, drained)
|
|
assert rescheduled and "titan-05" not in rescheduled
|
|
|
|
|
|
def test_three_workers_still_spread_across_distinct_eligible_nodes():
|
|
stateful = worker()["spec"]["template"]["spec"]
|
|
nodes = [node(f"titan-{index:02d}") for index in (5, 6, 7)] + [
|
|
node("titan-13"), node("titan-22"),
|
|
]
|
|
occupied: list[str] = []
|
|
for _replica in range(3):
|
|
allowed = placeable(stateful, nodes, occupied=tuple(occupied))
|
|
assert allowed, f"replica {len(occupied)} is unschedulable"
|
|
occupied.append(allowed[0])
|
|
assert sorted(occupied) == ["titan-05", "titan-06", "titan-07"]
|
|
|
|
|
|
def test_provider_clis_install_once_onto_a_durable_verified_volume():
|
|
stateful = worker()
|
|
pod = stateful["spec"]["template"]["spec"]
|
|
claims = {item["metadata"]["name"] for item in stateful["spec"]["volumeClaimTemplates"]}
|
|
assert "tools" in claims
|
|
ephemeral = {
|
|
volume["name"] for volume in pod["volumes"] if "emptyDir" in volume
|
|
}
|
|
assert "tools" not in ephemeral, "the CLI cache must survive a Pod restart"
|
|
|
|
install = next(
|
|
item for item in pod["initContainers"] if item["name"] == "install-provider-clis"
|
|
)
|
|
script = install["args"][0]
|
|
# The marker alone is not trusted: a pruned or partial cache reinstalls.
|
|
assert 'if [ ! -f "${marker}" ] || [ ! -x "${tools}/bin/codex" ]' in script
|
|
assert 'rm -f "${marker}"' in script
|
|
# And the install itself is bounded, so a stalled registry fails fast.
|
|
assert "timeout 900 npm install" in script
|
|
assert "--fetch-timeout=120000" in script
|
|
assert script.count("npm install") == 1
|
|
|
|
worker_mount = next(
|
|
item for item in pod["containers"][0]["volumeMounts"]
|
|
if item["name"] == "tools"
|
|
)
|
|
assert worker_mount["readOnly"] is True
|
|
|
|
|
|
def test_flux_health_for_hermes_does_not_wait_on_the_best_effort_pool():
|
|
kustomization = documents(FLUX / "hermes/kustomization.yaml")[0]
|
|
gated = {
|
|
(item["kind"], item["name"])
|
|
for item in kustomization["spec"]["healthChecks"]
|
|
}
|
|
assert ("StatefulSet", "hermes-execution-worker") not in gated
|
|
assert not any(name.startswith("hermes-execution-") for _kind, name in gated)
|
|
# The rest of the app's health contract is untouched, so its dependents keep
|
|
# the same guarantees they had before the pool existed.
|
|
assert ("Deployment", "hermes-agent") in gated
|
|
assert ("Deployment", "hermes-switchyard") in gated
|
|
assert kustomization["spec"]["timeout"] == "10m"
|
|
|
|
for name in ("hermes-chat", "hermes-observer-bindings"):
|
|
dependent = documents(FLUX / name / "kustomization.yaml")[0]
|
|
assert {item["name"] for item in dependent["spec"]["dependsOn"]} >= {"hermes"}
|
|
|
|
|
|
def test_the_pool_still_reports_its_own_readiness():
|
|
pod = worker()["spec"]["template"]["spec"]
|
|
assert pod["containers"][0]["readinessProbe"]["exec"]["command"][-1].startswith(
|
|
"test -w /workspace"
|
|
)
|
|
for deployment in mediators():
|
|
container = deployment["spec"]["template"]["spec"]["containers"][0]
|
|
assert container["readinessProbe"]["httpGet"] == {
|
|
"path": "/ready", "port": "mediator"
|
|
}
|
|
assert container["startupProbe"]["failureThreshold"] == 60
|
|
|
|
|
|
def test_the_rendered_pool_configmap_carries_every_mounted_module():
|
|
kustomization = yaml.safe_load((HERMES / "kustomization.yaml").read_text())
|
|
generator = next(
|
|
item for item in kustomization["configMapGenerator"]
|
|
if item["name"] == "hermes-execution-pool"
|
|
)
|
|
keys = {entry.split("=", 1)[0] for entry in generator["files"]}
|
|
assert {
|
|
"execution_pool_store.py",
|
|
"execution_pool_maintenance.py",
|
|
"execution_pool_coordinator.py",
|
|
"execution_pool_server.py",
|
|
} <= keys
|
|
modules = sorted(
|
|
path.name for path in SCRIPTS.glob("execution_pool_*.py")
|
|
)
|
|
assert set(modules) <= keys, "a pool module is missing from its own mount"
|
|
assert json.dumps(sorted(keys))
|
|
|
|
|
|
def test_rendered_pool_configmap_imports_its_own_route_dependencies(tmp_path):
|
|
"""Workers import routing code only from their mounted ConfigMap payload."""
|
|
rendered = subprocess.run(
|
|
["kustomize", "build", str(HERMES)],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
configmap = next(
|
|
item
|
|
for item in yaml.safe_load_all(rendered.stdout)
|
|
if item
|
|
and item.get("kind") == "ConfigMap"
|
|
and item.get("metadata", {}).get("name", "").startswith(
|
|
"hermes-execution-pool-"
|
|
)
|
|
)
|
|
for name, content in configmap["data"].items():
|
|
if name.endswith(".py"):
|
|
(tmp_path / name).write_text(content, encoding="utf-8")
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-I",
|
|
"-c",
|
|
f"import sys; sys.path.insert(0, {str(tmp_path)!r}); import cli_lane_routing",
|
|
],
|
|
cwd=tmp_path,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
pod = worker()["spec"]["template"]["spec"]
|
|
container = pod["containers"][0]
|
|
assert {
|
|
"name": "routing-catalog",
|
|
"mountPath": "/routing-catalog",
|
|
"readOnly": True,
|
|
} in container["volumeMounts"]
|
|
volume = next(item for item in pod["volumes"] if item["name"] == "routing-catalog")
|
|
assert volume["persistentVolumeClaim"] == {
|
|
"claimName": "hermes-routing-catalog",
|
|
"readOnly": True,
|
|
}
|