From 8ed32c2b06b4d1b0174384cb1e064ef9ceb0bf0e Mon Sep 17 00:00:00 2001 From: jenkins Date: Wed, 12 Aug 2026 07:13:32 -0300 Subject: [PATCH] hermes: fail over malformed tool calls --- services/hermes/agent-deployment.yaml | 2 +- services/hermes/scripts/codex_broker.py | 22 +++++++++++ services/hermes/switchyard-configmap.yaml | 16 ++++++++ services/hermes/switchyard-deployment.yaml | 2 +- testing/tests/test_hermes_chat_quality.py | 43 ++++++++++++++++++++++ 5 files changed, 83 insertions(+), 2 deletions(-) diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index e770bb3b1..31e1df4b6 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,7 +25,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260812-codex-stream-terminal" + ai.bstein.dev/config-rev: "20260812-tool-call-failover" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens diff --git a/services/hermes/scripts/codex_broker.py b/services/hermes/scripts/codex_broker.py index 15e2834b8..a4a8a447a 100644 --- a/services/hermes/scripts/codex_broker.py +++ b/services/hermes/scripts/codex_broker.py @@ -212,6 +212,28 @@ def _completed_response(lines: Iterable[str]) -> dict[str, Any]: terminal_response["output"] = [ output_items[index] for index in sorted(output_items) ] + # A Responses stream can be marked ``completed`` even when a function + # call was cut off at the provider's output boundary. Passing that + # downstream as HTTP 200 makes Hermes retry the same broken turn until + # it emits the unhelpful "Response truncated" message. Reject malformed + # terminal tool calls here so Switchyard can fail over to another + # eligible provider/model for this boundary. + for item in terminal_response.get("output") or []: + if not isinstance(item, dict) or item.get("type") != "function_call": + continue + if str(item.get("status") or "completed").lower() == "incomplete": + raise RuntimeError("Codex returned a retryable incomplete tool call") + arguments = item.get("arguments") + if not isinstance(arguments, str): + raise RuntimeError("Codex returned malformed function arguments") + try: + parsed_arguments = json.loads(arguments) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise RuntimeError( + "Codex returned retryable malformed function arguments" + ) from exc + if not isinstance(parsed_arguments, dict): + raise RuntimeError("Codex returned malformed function arguments") return terminal_response raise RuntimeError(upstream_error or "Codex stream ended without a terminal response") diff --git a/services/hermes/switchyard-configmap.yaml b/services/hermes/switchyard-configmap.yaml index a80d61e0b..2d1388cc0 100644 --- a/services/hermes/switchyard-configmap.yaml +++ b/services/hermes/switchyard-configmap.yaml @@ -208,6 +208,10 @@ data: analysis, or bounded architecture; low only for simple conversation, formatting, lookup, or mechanical reversible work. Never choose below the floor and never exceed xhigh. + If tools are present or the boundary may emit a tool call, the floor is + medium even when the user's wording is short. Filesystem, shell, + repository, cluster, browser, and image-generation/edit operations are + tool work; never route those boundaries to Luna, Haiku, or local-low. 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a request to raise capability by at least one tier when the safety floor is @@ -266,6 +270,10 @@ data: analysis, or bounded architecture; low only for simple conversation, formatting, lookup, or mechanical reversible work. Never choose below the floor and never exceed xhigh. + If tools are present or the boundary may emit a tool call, the floor is + medium even when the user's wording is short. Filesystem, shell, + repository, cluster, browser, and image-generation/edit operations are + tool work; never route those boundaries to Luna, Haiku, or local-low. 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a request to raise capability by at least one tier when the safety floor is @@ -324,6 +332,10 @@ data: ordinary diagnosis, implementation, tool use, analysis, or bounded architecture; low only for simple notification summaries, lookup, or mechanical reversible work. Never choose below the floor or above xhigh. + If tools are present or the boundary may emit a tool call, the floor is + medium even when the user's wording is short. Filesystem, shell, + repository, cluster, browser, and image-generation/edit operations are + tool work; never route those boundaries to Luna or Haiku. 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a request to raise capability by at least one tier when the safety floor is @@ -383,6 +395,10 @@ data: implementation, tests, tool use, analysis, or bounded architecture; low only for lookup or truly mechanical reversible work. Never choose below the floor or above xhigh. + If tools are present or the boundary may emit a tool call, the floor is + medium even when the user's wording is short. Filesystem, shell, + repository, cluster, browser, and image-generation/edit operations are + tool work; never route those boundaries to Luna or Haiku. 2. Treat "think hard", "deeply", "carefully", and equivalent intent as a request to raise capability by at least one tier when the safety floor is diff --git a/services/hermes/switchyard-deployment.yaml b/services/hermes/switchyard-deployment.yaml index 7b5ecd365..57ba23e75 100644 --- a/services/hermes/switchyard-deployment.yaml +++ b/services/hermes/switchyard-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: hermes-switchyard annotations: - ai.bstein.dev/config-rev: "20260812-quality-route-local-exclusion" + ai.bstein.dev/config-rev: "20260812-tool-call-effort-floor" prometheus.io/scrape: "true" prometheus.io/port: "9005" prometheus.io/path: /metrics diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 7e965695f..ff0265d4a 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -700,6 +700,49 @@ def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch): 'data: {"type":"error","error":{"message":"provider unavailable"}}', ] ) + malformed_tool_item = { + "type": "function_call", + "name": "search_files", + "status": "completed", + "arguments": '{"path":"","offset":', + } + with pytest.raises(RuntimeError, match="malformed function arguments"): + module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": malformed_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + ) + valid_tool_item = { + **malformed_tool_item, + "arguments": '{"path":"","offset":0}', + } + assert module._completed_response( + [ + "event: response.output_item.done", + "data: " + + json.dumps( + { + "type": "response.output_item.done", + "output_index": 0, + "item": valid_tool_item, + } + ), + "event: response.completed", + "data: " + + json.dumps({"type": "response.completed", "response": completed}), + ] + )["output"] == [valid_tool_item] auth_dir = tmp_path / ".codex" auth_dir.mkdir()