diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 9244feef8..9a40babc9 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260811-codex-app-server-permissions" + ai.bstein.dev/config-rev: "20260811-codex-cross-provider-fallback" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens @@ -253,6 +253,10 @@ spec: - /patched/codex_app_server_session.py - /opt/hermes/agent/codex_runtime.py - /patched/codex_runtime.py + - /opt/hermes/agent/chat_completion_helpers.py + - /patched/chat_completion_helpers.py + - /opt/hermes/agent/conversation_loop.py + - /patched/conversation_loop.py securityContext: allowPrivilegeEscalation: false runAsUser: 10000 @@ -385,6 +389,8 @@ spec: - {name: codex-runtime-patch, mountPath: /opt/hermes/hermes_cli/runtime_provider.py, subPath: runtime_provider.py} - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/transports/codex_app_server_session.py, subPath: codex_app_server_session.py} - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/codex_runtime.py, subPath: codex_runtime.py} + - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/chat_completion_helpers.py, subPath: chat_completion_helpers.py} + - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py} - {name: tui-gateway-patch, mountPath: /opt/hermes/tui_gateway/server.py, subPath: server.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} @@ -533,6 +539,8 @@ spec: - {name: codex-runtime-patch, mountPath: /opt/hermes/hermes_cli/runtime_provider.py, subPath: runtime_provider.py} - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/transports/codex_app_server_session.py, subPath: codex_app_server_session.py} - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/codex_runtime.py, subPath: codex_runtime.py} + - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/chat_completion_helpers.py, subPath: chat_completion_helpers.py} + - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py} - {name: tui-gateway-patch, mountPath: /opt/hermes/tui_gateway/server.py, subPath: server.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true} diff --git a/services/hermes/scripts/patch_codex_runtime.py b/services/hermes/scripts/patch_codex_runtime.py index 4aa591a8f..a02c51e09 100644 --- a/services/hermes/scripts/patch_codex_runtime.py +++ b/services/hermes/scripts/patch_codex_runtime.py @@ -86,6 +86,143 @@ TURN_AFTER = ''' reasoning = getattr(agent, "reasoning_config", None) ) ''' +FALLBACK_RESOLUTION_BEFORE = ''' # Use centralized router for client construction. + # raw_codex=True because the main agent needs direct responses.stream() + # access for Codex providers. + try: + from agent.auxiliary_client import resolve_provider_client +''' +FALLBACK_RESOLUTION_AFTER = ''' # The Codex subscription is exposed through the authenticated local CLI, + # not as a bearer token that the OpenAI-compatible fallback resolver can + # consume. Activate that runtime directly and let conversation_loop hand + # the already-built turn context to codex app-server. + try: + if fb_provider == "openai-codex": + from hermes_cli.runtime_provider import resolve_runtime_provider + + runtime = resolve_runtime_provider( + requested=fb_provider, + target_model=fb_model, + ) + if runtime.get("api_mode") == "codex_app_server": + try: + from hermes_cli.model_normalize import normalize_model_for_provider + + fb_model = normalize_model_for_provider(fb_model, fb_provider) + except Exception as _norm_err: + logger.warning( + "Could not normalize fallback model %r for provider %r: %s", + fb_model, fb_provider, _norm_err, + ) + + old_model = agent.model + agent._config_context_length = None + agent.model = fb_model + agent.provider = fb_provider + agent.base_url = str(runtime.get("base_url") or "") + agent.api_key = str(runtime.get("api_key") or "codex-cli-runtime") + agent.api_mode = "codex_app_server" + agent.client = None + agent._anthropic_client = None + agent._credential_pool = None + agent._fallback_activated = True + agent._codex_cross_provider_fallback = True + if hasattr(agent, "_transport_cache"): + agent._transport_cache.clear() + rewrite_prompt_model_identity(agent, fb_model, fb_provider) + agent._buffer_status( + f"🔄 Primary model failed — switching to fallback: " + f"{fb_model} via {fb_provider}" + ) + logger.info( + "Fallback activated through Codex app-server: %s → %s (%s)", + old_model, fb_model, fb_provider, + ) + _reset_stale_streak(agent) + return True + + from agent.auxiliary_client import resolve_provider_client +''' + +FALLBACK_DISPATCH_BEFORE = ''' while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call: + # Reset per-turn checkpoint dedup so each iteration can take one snapshot +''' +FALLBACK_DISPATCH_AFTER = ''' while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call: + # A provider failure can activate the CLI-backed Codex runtime after + # this function's initial runtime dispatch. Re-enter the native Codex + # adapter here instead of attempting an OpenAI-compatible request with + # credentials that deliberately do not exist in Hermes' auth store. + if agent.api_mode == "codex_app_server": + fallback_user_message = user_message + if getattr(agent, "_codex_cross_provider_fallback", False): + from agent.codex_runtime import build_cross_provider_codex_prompt + + fallback_user_message = build_cross_provider_codex_prompt( + messages, + user_message, + ) + agent._codex_cross_provider_fallback = False + return agent._run_codex_app_server_turn( + user_message=fallback_user_message, + original_user_message=original_user_message, + messages=messages, + effective_task_id=effective_task_id, + should_review_memory=_should_review_memory, + ) + + # Reset per-turn checkpoint dedup so each iteration can take one snapshot +''' + +FALLBACK_CONTEXT_BEFORE = '''def run_codex_app_server_turn( +''' +FALLBACK_CONTEXT_AFTER = '''def build_cross_provider_codex_prompt( + messages: List[Dict[str, Any]], + user_message: str, + *, + max_chars: int = 120_000, +) -> str: + """Bridge recent user/assistant context into a new Codex fallback thread.""" + + def _text(value: Any) -> str: + if isinstance(value, str): + return value + if isinstance(value, list): + parts = [] + for item in value: + if isinstance(item, dict): + candidate = item.get("text") or item.get("content") + if isinstance(candidate, str): + parts.append(candidate) + return "\\n".join(parts) + return "" + + exchanges = [] + for message in messages: + if not isinstance(message, dict): + continue + role = str(message.get("role") or "").strip().lower() + if role not in {"user", "assistant"}: + continue + content = _text(message.get("content")).strip() + if content: + exchanges.append(f"{role.upper()}: {content}") + + transcript = "\\n\\n".join(exchanges) + if len(transcript) > max_chars: + transcript = "[Earlier conversation omitted for context size.]\\n\\n" + transcript[-max_chars:] + if not transcript: + transcript = f"USER: {user_message}" + return ( + "Continue this Hermes conversation after the previously selected " + "provider exhausted its capacity. Preserve the objective and answer " + "the latest user request; do not restart or discard completed work.\\n\\n" + f"\\n{transcript}\\n" + ) + + +def run_codex_app_server_turn( +''' + def _replace_once(content: str, before: str, after: str, label: str) -> str: """Apply one exact replacement and fail closed when upstream drifts.""" @@ -129,8 +266,14 @@ def patch_session(source: Path, destination: Path) -> None: def patch_turn(source: Path, destination: Path) -> None: - """Forward live Hermes route metadata into the Codex session adapter.""" + """Forward route metadata and preserve context on Codex failover.""" content = source.read_text(encoding="utf-8") + content = _replace_once( + content, + FALLBACK_CONTEXT_BEFORE, + FALLBACK_CONTEXT_AFTER, + "Codex fallback context", + ) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text( _replace_once(content, TURN_BEFORE, TURN_AFTER, "Codex turn"), @@ -138,6 +281,36 @@ def patch_turn(source: Path, destination: Path) -> None: ) +def patch_fallback(source: Path, destination: Path) -> None: + """Activate the CLI-backed Codex runtime in Hermes' fallback chain.""" + content = source.read_text(encoding="utf-8") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + _replace_once( + content, + FALLBACK_RESOLUTION_BEFORE, + FALLBACK_RESOLUTION_AFTER, + "Codex fallback activation", + ), + encoding="utf-8", + ) + + +def patch_loop(source: Path, destination: Path) -> None: + """Dispatch a mid-turn Codex fallback through app-server.""" + content = source.read_text(encoding="utf-8") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + _replace_once( + content, + FALLBACK_DISPATCH_BEFORE, + FALLBACK_DISPATCH_AFTER, + "Codex fallback dispatch", + ), + encoding="utf-8", + ) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("provider_source", type=Path) @@ -146,10 +319,16 @@ def main() -> int: parser.add_argument("session_destination", type=Path) parser.add_argument("turn_source", type=Path) parser.add_argument("turn_destination", type=Path) + parser.add_argument("fallback_source", type=Path) + parser.add_argument("fallback_destination", type=Path) + parser.add_argument("loop_source", type=Path) + parser.add_argument("loop_destination", type=Path) args = parser.parse_args() patch_provider(args.provider_source, args.provider_destination) patch_session(args.session_source, args.session_destination) patch_turn(args.turn_source, args.turn_destination) + patch_fallback(args.fallback_source, args.fallback_destination) + patch_loop(args.loop_source, args.loop_destination) return 0 diff --git a/testing/tests/test_hermes_cli_lanes.py b/testing/tests/test_hermes_cli_lanes.py index 2663346fa..b506ae0a4 100644 --- a/testing/tests/test_hermes_cli_lanes.py +++ b/testing/tests/test_hermes_cli_lanes.py @@ -881,10 +881,35 @@ def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path): assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content turn = tmp_path / "codex_runtime.py" - turn.write_text(codex_runtime_patch.TURN_BEFORE, encoding="utf-8") + turn.write_text( + codex_runtime_patch.FALLBACK_CONTEXT_BEFORE + + codex_runtime_patch.TURN_BEFORE, + encoding="utf-8", + ) turn_out = tmp_path / "patched/codex_runtime.py" codex_runtime_patch.patch_turn(turn, turn_out) - assert "model=str(getattr(agent" in turn_out.read_text() + turn_content = turn_out.read_text() + assert "model=str(getattr(agent" in turn_content + assert "build_cross_provider_codex_prompt" in turn_content + + fallback = tmp_path / "chat_completion_helpers.py" + fallback.write_text( + codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE, + encoding="utf-8", + ) + fallback_out = tmp_path / "patched/chat_completion_helpers.py" + codex_runtime_patch.patch_fallback(fallback, fallback_out) + fallback_content = fallback_out.read_text() + assert 'agent.api_mode = "codex_app_server"' in fallback_content + assert "agent._codex_cross_provider_fallback = True" in fallback_content + + loop = tmp_path / "conversation_loop.py" + loop.write_text(codex_runtime_patch.FALLBACK_DISPATCH_BEFORE, encoding="utf-8") + loop_out = tmp_path / "patched/conversation_loop.py" + codex_runtime_patch.patch_loop(loop, loop_out) + loop_content = loop_out.read_text() + assert 'if agent.api_mode == "codex_app_server"' in loop_content + assert "build_cross_provider_codex_prompt" in loop_content def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path):