diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 31e1df4b6..77228b6e6 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260812-tool-call-failover" + ai.bstein.dev/config-rev: "20260812-stream-tool-failover" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens diff --git a/services/hermes/scripts/codex_broker.py b/services/hermes/scripts/codex_broker.py index a4a8a447a..6f34c4520 100644 --- a/services/hermes/scripts/codex_broker.py +++ b/services/hermes/scripts/codex_broker.py @@ -160,7 +160,31 @@ def _completed_response(lines: Iterable[str]) -> dict[str, Any]: """Collapse a Codex SSE stream for a non-streaming Responses caller.""" terminal_response: dict[str, Any] | None = None output_items: dict[int, dict[str, Any]] = {} + function_argument_deltas: dict[str, list[str]] = {} + completed_function_arguments: set[str] = set() upstream_error = "" + + def function_key(event: dict[str, Any]) -> str: + """Identify one streamed function call without retaining its content.""" + item_id = str(event.get("item_id") or "").strip() + if item_id: + return item_id + output_index = event.get("output_index") + return f"output:{output_index}" if isinstance(output_index, int) else "" + + def validate_arguments(arguments: Any) -> None: + """Reject an incomplete JSON object so Switchyard can fail over.""" + if not isinstance(arguments, str): + raise RuntimeError("Codex returned malformed function arguments") + try: + parsed_arguments = json.loads(arguments) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise RuntimeError( + "Codex returned retryable malformed function arguments" + ) from exc + if not isinstance(parsed_arguments, dict): + raise RuntimeError("Codex returned malformed function arguments") + for line in lines: if not line.startswith("data:"): continue @@ -175,6 +199,19 @@ def _completed_response(lines: Iterable[str]) -> dict[str, Any]: continue event_type = str(event.get("type") or "") response = event.get("response") + if event_type == "response.function_call_arguments.delta": + key = function_key(event) + delta = event.get("delta") + if key and isinstance(delta, str): + function_argument_deltas.setdefault(key, []).append(delta) + elif event_type == "response.function_call_arguments.done": + key = function_key(event) + arguments = event.get("arguments") + if not isinstance(arguments, str) and key: + arguments = "".join(function_argument_deltas.get(key, [])) + validate_arguments(arguments) + if key: + completed_function_arguments.add(key) if event_type == "response.output_item.done": item = event.get("item") output_index = event.get("output_index") @@ -192,6 +229,9 @@ def _completed_response(lines: Iterable[str]) -> dict[str, Any]: upstream_error = str(error.get("message") or error.get("type") or "") else: upstream_error = str(error or "") + for key, deltas in function_argument_deltas.items(): + if key not in completed_function_arguments: + validate_arguments("".join(deltas)) if terminal_response is not None: status = str(terminal_response.get("status") or "").lower() if status != "completed": @@ -224,16 +264,7 @@ def _completed_response(lines: Iterable[str]) -> dict[str, Any]: if str(item.get("status") or "completed").lower() == "incomplete": raise RuntimeError("Codex returned a retryable incomplete tool call") arguments = item.get("arguments") - if not isinstance(arguments, str): - raise RuntimeError("Codex returned malformed function arguments") - try: - parsed_arguments = json.loads(arguments) - except (TypeError, ValueError, json.JSONDecodeError) as exc: - raise RuntimeError( - "Codex returned retryable malformed function arguments" - ) from exc - if not isinstance(parsed_arguments, dict): - raise RuntimeError("Codex returned malformed function arguments") + validate_arguments(arguments) return terminal_response raise RuntimeError(upstream_error or "Codex stream ended without a terminal response") diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index 2d1388cc0..864c04533 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -311,7 +311,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "codex_terra_high", "claude_sonnet_medium", "codex_sol_medium", "codex_terra_medium", "claude_haiku_low", "codex_luna_low", "codex_terra_low"] + targets = ["claude_opus_xhigh", "codex_sol_xhigh", "claude_sonnet_high", "codex_sol_high", "claude_opus_high", "codex_terra_high", "claude_sonnet_medium", "codex_sol_medium", "codex_terra_medium"] default_target = "claude_sonnet_high" session_affinity = false recent_turn_window = 12 @@ -353,14 +353,15 @@ data: unavailable, failed, exhausted, rate-limited, or out of capacity; use the other provider at the same floor. - 4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, - high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, + 4. Map exactly: Codex medium=codex_terra_medium, + high=codex_sol_high, xhigh=codex_sol_xhigh; Claude medium=claude_sonnet_medium, high=claude_sonnet_high, - xhigh=claude_opus_xhigh. Re-evaluate every boundary and resolve "continue" - or "do it" from recent context. + xhigh=claude_opus_xhigh. Low-tier targets are intentionally unavailable on + this route. Re-evaluate every boundary and resolve "continue" or "do it" + from recent context. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_opus_xhigh","codex_luna_low","claude_haiku_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_deep.policy] @@ -374,7 +375,7 @@ data: classifier_target = "classifier" # Switchyard falls through this list after a request-local target failure. # Keep both xhigh providers first so recovery can escalate, never downgrade. - targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium", "codex_luna_low", "claude_haiku_low", "codex_terra_low"] + targets = ["codex_sol_xhigh", "claude_opus_xhigh", "codex_sol_high", "claude_opus_high", "claude_sonnet_high", "codex_terra_high", "codex_sol_medium", "claude_sonnet_medium", "codex_terra_medium"] default_target = "codex_sol_high" session_affinity = false recent_turn_window = 16 @@ -416,14 +417,15 @@ data: unavailable, failed, exhausted, rate-limited, or out of capacity; use the other provider at the same floor. - 4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium, - high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low, + 4. Map exactly: Codex medium=codex_terra_medium, + high=codex_sol_high, xhigh=codex_sol_xhigh; Claude medium=claude_sonnet_medium, high=claude_sonnet_high, - xhigh=claude_opus_xhigh. Re-evaluate every boundary and resolve "continue" - or "do it" from recent context. + xhigh=claude_opus_xhigh. Low-tier targets are intentionally unavailable on + this route. Re-evaluate every boundary and resolve "continue" or "do it" + from recent context. """ response_schema = ''' - {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","codex_terra_medium","claude_sonnet_medium","codex_sol_medium","codex_luna_low","claude_haiku_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} + {"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","codex_terra_medium","claude_sonnet_medium","codex_sol_medium"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false} ''' [routes.auto_maximum.policy] diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 57ba23e75..80b4cb210 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260812-tool-call-effort-floor" + ai.bstein.dev/config-rev: "20260812-enforced-tool-floor" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index ff0265d4a..2532d6333 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -743,6 +743,34 @@ def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): + json.dumps({"type": "response.completed", "response": completed}), ] )["output"] == [valid_tool_item] + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_1","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + streamed_tool = module._completed_response( + [ + "event: response.function_call_arguments.delta", + 'data: {"type":"response.function_call_arguments.delta",' + '"item_id":"call_2","output_index":0,' + '"delta":"{\\"path\\":\\"/tmp\\",\\"offset\\":"}', + "event: response.function_call_arguments.done", + 'data: {"type":"response.function_call_arguments.done",' + '"item_id":"call_2","output_index":0,' + '"arguments":"{\\"path\\":\\"/tmp\\",\\"offset\\":0}"}', + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + assert streamed_tool["status"] == "completed" auth_dir = tmp_path / ".codex" auth_dir.mkdir() @@ -981,8 +1009,8 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): assert "Claude xhigh=worker_claude_opus_xhigh" in switchyard assert "Anthropic and Claude name the same provider" in switchyard assert "OpenAI and Codex name the same provider" in switchyard - assert switchyard.count("Codex low=codex_luna_low") == 4 - assert switchyard.count("Claude low=claude_haiku_low") == 4 + assert switchyard.count("Codex low=codex_luna_low") == 2 + assert switchyard.count("Claude low=claude_haiku_low") == 2 assert switchyard.count('Treat "think hard"') == 4 assert switchyard.count("Never choose below the") >= 5 routes = tomllib.loads(switchyard)["routes"] @@ -994,7 +1022,9 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed(): targets = routes[route_name]["targets"] selector_targets = routes[route_name]["response_schema"] assert not any(target.startswith("local_") for target in targets) + assert not any(target.endswith("_low") for target in targets) assert "local_qwen" not in selector_targets + assert "_low" not in selector_targets assert "not eligible for foreground" in routes[route_name]["prompt"] for route_name in ("auto_fast", "auto_balanced", "manual_local_qwen"): assert any(