From 1e66a0bedf8e34b4b399c82958122b75e994a2ac Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 23 Aug 2026 11:01:20 -0300 Subject: [PATCH] fix(hermes): tolerate worker lease handoff --- .../hermes/scripts/execution_pool_worker.py | 9 +++++- .../test_hermes_execution_pool_worker_v2.py | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/services/hermes/scripts/execution_pool_worker.py b/services/hermes/scripts/execution_pool_worker.py index 29d09483..6a5d7779 100644 --- a/services/hermes/scripts/execution_pool_worker.py +++ b/services/hermes/scripts/execution_pool_worker.py @@ -430,7 +430,14 @@ def readiness() -> None: atomic_json( cli_lane_runner.RESULT_SCHEMA_PATH, cli_lane_runner.RESULT_SCHEMA, 0o644 ) - _poll() + try: + _poll() + except urllib.error.HTTPError as error: + # A replacement pod can overlap the prior ordinal's coordinator lease. + # The main loop already defers this transient conflict until ownership + # transfers, so keep the container alive instead of CrashLooping. + if error.code != 409: + raise def main() -> int: diff --git a/testing/tests/test_hermes_execution_pool_worker_v2.py b/testing/tests/test_hermes_execution_pool_worker_v2.py index 7bef1f84..d06fc239 100644 --- a/testing/tests/test_hermes_execution_pool_worker_v2.py +++ b/testing/tests/test_hermes_execution_pool_worker_v2.py @@ -6,6 +6,7 @@ import json import os import subprocess import sys +import urllib.error from pathlib import Path import pytest @@ -288,3 +289,34 @@ def test_readiness_checks_ordinal_paths_credentials_and_mediator(tmp_path, monke claude_token.unlink() with pytest.raises(protocol.ProtocolError, match="credential"): worker.readiness() + + +def test_readiness_defers_transient_ordinal_lease_conflict(tmp_path, monkeypatch): + worker_root = tmp_path / "worker" + data_root = tmp_path / "data" + claude_token = tmp_path / "claude-oauth/token" + for path in (worker_root, worker_root / "provider-state", data_root): + path.mkdir(parents=True, exist_ok=True) + claude_token.parent.mkdir() + claude_token.write_text("setup-token") + monkeypatch.setattr(worker, "ORDINAL", 1) + monkeypatch.setattr(worker, "ROOT", worker_root) + monkeypatch.setattr(worker.cli_lane_runner, "DATA_ROOT", data_root) + monkeypatch.setattr( + worker.cli_lane_runner, "RESULT_SCHEMA_PATH", tmp_path / "schema/result.json" + ) + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN_FILE", str(claude_token)) + + def conflict(): + raise urllib.error.HTTPError("http://mediator", 409, "Conflict", {}, None) + + monkeypatch.setattr(worker, "_poll", conflict) + worker.readiness() + + def unauthorized(): + raise urllib.error.HTTPError("http://mediator", 401, "Unauthorized", {}, None) + + monkeypatch.setattr(worker, "_poll", unauthorized) + with pytest.raises(urllib.error.HTTPError) as error: + worker.readiness() + assert error.value.code == 401