hermes: fail over malformed tool calls
All checks were successful
Tests / Declarative: Post Actions passed: 251
All checks were successful
Tests / Declarative: Post Actions passed: 251
This commit is contained in:
parent
a1ef36dbea
commit
8ed32c2b06
@ -25,7 +25,7 @@ spec:
|
|||||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||||
ai.bstein.dev/config-rev: "20260812-codex-stream-terminal"
|
ai.bstein.dev/config-rev: "20260812-tool-call-failover"
|
||||||
vault.hashicorp.com/agent-inject: "true"
|
vault.hashicorp.com/agent-inject: "true"
|
||||||
vault.hashicorp.com/role: hermes-agent
|
vault.hashicorp.com/role: hermes-agent
|
||||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||||
|
|||||||
@ -212,6 +212,28 @@ def _completed_response(lines: Iterable[str]) -> dict[str, Any]:
|
|||||||
terminal_response["output"] = [
|
terminal_response["output"] = [
|
||||||
output_items[index] for index in sorted(output_items)
|
output_items[index] for index in sorted(output_items)
|
||||||
]
|
]
|
||||||
|
# A Responses stream can be marked ``completed`` even when a function
|
||||||
|
# call was cut off at the provider's output boundary. Passing that
|
||||||
|
# downstream as HTTP 200 makes Hermes retry the same broken turn until
|
||||||
|
# it emits the unhelpful "Response truncated" message. Reject malformed
|
||||||
|
# terminal tool calls here so Switchyard can fail over to another
|
||||||
|
# eligible provider/model for this boundary.
|
||||||
|
for item in terminal_response.get("output") or []:
|
||||||
|
if not isinstance(item, dict) or item.get("type") != "function_call":
|
||||||
|
continue
|
||||||
|
if str(item.get("status") or "completed").lower() == "incomplete":
|
||||||
|
raise RuntimeError("Codex returned a retryable incomplete tool call")
|
||||||
|
arguments = item.get("arguments")
|
||||||
|
if not isinstance(arguments, str):
|
||||||
|
raise RuntimeError("Codex returned malformed function arguments")
|
||||||
|
try:
|
||||||
|
parsed_arguments = json.loads(arguments)
|
||||||
|
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Codex returned retryable malformed function arguments"
|
||||||
|
) from exc
|
||||||
|
if not isinstance(parsed_arguments, dict):
|
||||||
|
raise RuntimeError("Codex returned malformed function arguments")
|
||||||
return terminal_response
|
return terminal_response
|
||||||
raise RuntimeError(upstream_error or "Codex stream ended without a terminal response")
|
raise RuntimeError(upstream_error or "Codex stream ended without a terminal response")
|
||||||
|
|
||||||
|
|||||||
@ -208,6 +208,10 @@ data:
|
|||||||
analysis, or bounded architecture; low only for simple conversation,
|
analysis, or bounded architecture; low only for simple conversation,
|
||||||
formatting, lookup, or mechanical reversible work. Never choose below the
|
formatting, lookup, or mechanical reversible work. Never choose below the
|
||||||
floor and never exceed xhigh.
|
floor and never exceed xhigh.
|
||||||
|
If tools are present or the boundary may emit a tool call, the floor is
|
||||||
|
medium even when the user's wording is short. Filesystem, shell,
|
||||||
|
repository, cluster, browser, and image-generation/edit operations are
|
||||||
|
tool work; never route those boundaries to Luna, Haiku, or local-low.
|
||||||
|
|
||||||
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
request to raise capability by at least one tier when the safety floor is
|
request to raise capability by at least one tier when the safety floor is
|
||||||
@ -266,6 +270,10 @@ data:
|
|||||||
analysis, or bounded architecture; low only for simple conversation,
|
analysis, or bounded architecture; low only for simple conversation,
|
||||||
formatting, lookup, or mechanical reversible work. Never choose below the
|
formatting, lookup, or mechanical reversible work. Never choose below the
|
||||||
floor and never exceed xhigh.
|
floor and never exceed xhigh.
|
||||||
|
If tools are present or the boundary may emit a tool call, the floor is
|
||||||
|
medium even when the user's wording is short. Filesystem, shell,
|
||||||
|
repository, cluster, browser, and image-generation/edit operations are
|
||||||
|
tool work; never route those boundaries to Luna, Haiku, or local-low.
|
||||||
|
|
||||||
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
request to raise capability by at least one tier when the safety floor is
|
request to raise capability by at least one tier when the safety floor is
|
||||||
@ -324,6 +332,10 @@ data:
|
|||||||
ordinary diagnosis, implementation, tool use, analysis, or bounded
|
ordinary diagnosis, implementation, tool use, analysis, or bounded
|
||||||
architecture; low only for simple notification summaries, lookup, or
|
architecture; low only for simple notification summaries, lookup, or
|
||||||
mechanical reversible work. Never choose below the floor or above xhigh.
|
mechanical reversible work. Never choose below the floor or above xhigh.
|
||||||
|
If tools are present or the boundary may emit a tool call, the floor is
|
||||||
|
medium even when the user's wording is short. Filesystem, shell,
|
||||||
|
repository, cluster, browser, and image-generation/edit operations are
|
||||||
|
tool work; never route those boundaries to Luna or Haiku.
|
||||||
|
|
||||||
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
request to raise capability by at least one tier when the safety floor is
|
request to raise capability by at least one tier when the safety floor is
|
||||||
@ -383,6 +395,10 @@ data:
|
|||||||
implementation, tests, tool use, analysis, or bounded architecture; low
|
implementation, tests, tool use, analysis, or bounded architecture; low
|
||||||
only for lookup or truly mechanical reversible work. Never choose below the
|
only for lookup or truly mechanical reversible work. Never choose below the
|
||||||
floor or above xhigh.
|
floor or above xhigh.
|
||||||
|
If tools are present or the boundary may emit a tool call, the floor is
|
||||||
|
medium even when the user's wording is short. Filesystem, shell,
|
||||||
|
repository, cluster, browser, and image-generation/edit operations are
|
||||||
|
tool work; never route those boundaries to Luna or Haiku.
|
||||||
|
|
||||||
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
request to raise capability by at least one tier when the safety floor is
|
request to raise capability by at least one tier when the safety floor is
|
||||||
|
|||||||
@ -22,7 +22,7 @@ spec:
|
|||||||
labels:
|
labels:
|
||||||
app: hermes-switchyard
|
app: hermes-switchyard
|
||||||
annotations:
|
annotations:
|
||||||
ai.bstein.dev/config-rev: "20260812-quality-route-local-exclusion"
|
ai.bstein.dev/config-rev: "20260812-tool-call-effort-floor"
|
||||||
prometheus.io/scrape: "true"
|
prometheus.io/scrape: "true"
|
||||||
prometheus.io/port: "9005"
|
prometheus.io/port: "9005"
|
||||||
prometheus.io/path: /metrics
|
prometheus.io/path: /metrics
|
||||||
|
|||||||
@ -700,6 +700,49 @@ def test_codex_broker_auth_and_request_contract(tmp_path: Path, monkeypatch):
|
|||||||
'data: {"type":"error","error":{"message":"provider unavailable"}}',
|
'data: {"type":"error","error":{"message":"provider unavailable"}}',
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
malformed_tool_item = {
|
||||||
|
"type": "function_call",
|
||||||
|
"name": "search_files",
|
||||||
|
"status": "completed",
|
||||||
|
"arguments": '{"path":"","offset":',
|
||||||
|
}
|
||||||
|
with pytest.raises(RuntimeError, match="malformed function arguments"):
|
||||||
|
module._completed_response(
|
||||||
|
[
|
||||||
|
"event: response.output_item.done",
|
||||||
|
"data: "
|
||||||
|
+ json.dumps(
|
||||||
|
{
|
||||||
|
"type": "response.output_item.done",
|
||||||
|
"output_index": 0,
|
||||||
|
"item": malformed_tool_item,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"event: response.completed",
|
||||||
|
"data: "
|
||||||
|
+ json.dumps({"type": "response.completed", "response": completed}),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
valid_tool_item = {
|
||||||
|
**malformed_tool_item,
|
||||||
|
"arguments": '{"path":"","offset":0}',
|
||||||
|
}
|
||||||
|
assert module._completed_response(
|
||||||
|
[
|
||||||
|
"event: response.output_item.done",
|
||||||
|
"data: "
|
||||||
|
+ json.dumps(
|
||||||
|
{
|
||||||
|
"type": "response.output_item.done",
|
||||||
|
"output_index": 0,
|
||||||
|
"item": valid_tool_item,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"event: response.completed",
|
||||||
|
"data: "
|
||||||
|
+ json.dumps({"type": "response.completed", "response": completed}),
|
||||||
|
]
|
||||||
|
)["output"] == [valid_tool_item]
|
||||||
|
|
||||||
auth_dir = tmp_path / ".codex"
|
auth_dir = tmp_path / ".codex"
|
||||||
auth_dir.mkdir()
|
auth_dir.mkdir()
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user