hermes(agent): preserve Codex fallback context

services/hermes/agent-deployment.yaml

@@ -24,7 +24,7 @@ spec:
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
         ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
-        ai.bstein.dev/config-rev: "20260811-codex-app-server-permissions"
+        ai.bstein.dev/config-rev: "20260811-codex-cross-provider-fallback"
         vault.hashicorp.com/agent-inject: "true"
         vault.hashicorp.com/role: hermes-agent
         vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
@@ -253,6 +253,10 @@ spec:
             - /patched/codex_app_server_session.py
             - /opt/hermes/agent/codex_runtime.py
             - /patched/codex_runtime.py
+            - /opt/hermes/agent/chat_completion_helpers.py
+            - /patched/chat_completion_helpers.py
+            - /opt/hermes/agent/conversation_loop.py
+            - /patched/conversation_loop.py
           securityContext:
             allowPrivilegeEscalation: false
             runAsUser: 10000
@@ -385,6 +389,8 @@ spec:
             - {name: codex-runtime-patch, mountPath: /opt/hermes/hermes_cli/runtime_provider.py, subPath: runtime_provider.py}
             - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/transports/codex_app_server_session.py, subPath: codex_app_server_session.py}
             - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/codex_runtime.py, subPath: codex_runtime.py}
+            - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/chat_completion_helpers.py, subPath: chat_completion_helpers.py}
+            - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py}
             - {name: tui-gateway-patch, mountPath: /opt/hermes/tui_gateway/server.py, subPath: server.py}
             - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
             - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
@@ -533,6 +539,8 @@ spec:
             - {name: codex-runtime-patch, mountPath: /opt/hermes/hermes_cli/runtime_provider.py, subPath: runtime_provider.py}
             - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/transports/codex_app_server_session.py, subPath: codex_app_server_session.py}
             - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/codex_runtime.py, subPath: codex_runtime.py}
+            - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/chat_completion_helpers.py, subPath: chat_completion_helpers.py}
+            - {name: codex-runtime-patch, mountPath: /opt/hermes/agent/conversation_loop.py, subPath: conversation_loop.py}
             - {name: tui-gateway-patch, mountPath: /opt/hermes/tui_gateway/server.py, subPath: server.py}
             - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
             - {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}

services/hermes/scripts/patch_codex_runtime.py

@@ -86,6 +86,143 @@ TURN_AFTER = '''    reasoning = getattr(agent, "reasoning_config", None)
         )
 '''
 
+FALLBACK_RESOLUTION_BEFORE = '''    # Use centralized router for client construction.
+    # raw_codex=True because the main agent needs direct responses.stream()
+    # access for Codex providers.
+    try:
+        from agent.auxiliary_client import resolve_provider_client
+'''
+FALLBACK_RESOLUTION_AFTER = '''    # The Codex subscription is exposed through the authenticated local CLI,
+    # not as a bearer token that the OpenAI-compatible fallback resolver can
+    # consume. Activate that runtime directly and let conversation_loop hand
+    # the already-built turn context to codex app-server.
+    try:
+        if fb_provider == "openai-codex":
+            from hermes_cli.runtime_provider import resolve_runtime_provider
+
+            runtime = resolve_runtime_provider(
+                requested=fb_provider,
+                target_model=fb_model,
+            )
+            if runtime.get("api_mode") == "codex_app_server":
+                try:
+                    from hermes_cli.model_normalize import normalize_model_for_provider
+
+                    fb_model = normalize_model_for_provider(fb_model, fb_provider)
+                except Exception as _norm_err:
+                    logger.warning(
+                        "Could not normalize fallback model %r for provider %r: %s",
+                        fb_model, fb_provider, _norm_err,
+                    )
+
+                old_model = agent.model
+                agent._config_context_length = None
+                agent.model = fb_model
+                agent.provider = fb_provider
+                agent.base_url = str(runtime.get("base_url") or "")
+                agent.api_key = str(runtime.get("api_key") or "codex-cli-runtime")
+                agent.api_mode = "codex_app_server"
+                agent.client = None
+                agent._anthropic_client = None
+                agent._credential_pool = None
+                agent._fallback_activated = True
+                agent._codex_cross_provider_fallback = True
+                if hasattr(agent, "_transport_cache"):
+                    agent._transport_cache.clear()
+                rewrite_prompt_model_identity(agent, fb_model, fb_provider)
+                agent._buffer_status(
+                    f"🔄 Primary model failed — switching to fallback: "
+                    f"{fb_model} via {fb_provider}"
+                )
+                logger.info(
+                    "Fallback activated through Codex app-server: %s → %s (%s)",
+                    old_model, fb_model, fb_provider,
+                )
+                _reset_stale_streak(agent)
+                return True
+
+        from agent.auxiliary_client import resolve_provider_client
+'''
+
+FALLBACK_DISPATCH_BEFORE = '''    while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call:
+        # Reset per-turn checkpoint dedup so each iteration can take one snapshot
+'''
+FALLBACK_DISPATCH_AFTER = '''    while (api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call:
+        # A provider failure can activate the CLI-backed Codex runtime after
+        # this function's initial runtime dispatch. Re-enter the native Codex
+        # adapter here instead of attempting an OpenAI-compatible request with
+        # credentials that deliberately do not exist in Hermes' auth store.
+        if agent.api_mode == "codex_app_server":
+            fallback_user_message = user_message
+            if getattr(agent, "_codex_cross_provider_fallback", False):
+                from agent.codex_runtime import build_cross_provider_codex_prompt
+
+                fallback_user_message = build_cross_provider_codex_prompt(
+                    messages,
+                    user_message,
+                )
+                agent._codex_cross_provider_fallback = False
+            return agent._run_codex_app_server_turn(
+                user_message=fallback_user_message,
+                original_user_message=original_user_message,
+                messages=messages,
+                effective_task_id=effective_task_id,
+                should_review_memory=_should_review_memory,
+            )
+
+        # Reset per-turn checkpoint dedup so each iteration can take one snapshot
+'''
+
+FALLBACK_CONTEXT_BEFORE = '''def run_codex_app_server_turn(
+'''
+FALLBACK_CONTEXT_AFTER = '''def build_cross_provider_codex_prompt(
+    messages: List[Dict[str, Any]],
+    user_message: str,
+    *,
+    max_chars: int = 120_000,
+) -> str:
+    """Bridge recent user/assistant context into a new Codex fallback thread."""
+
+    def _text(value: Any) -> str:
+        if isinstance(value, str):
+            return value
+        if isinstance(value, list):
+            parts = []
+            for item in value:
+                if isinstance(item, dict):
+                    candidate = item.get("text") or item.get("content")
+                    if isinstance(candidate, str):
+                        parts.append(candidate)
+            return "\\n".join(parts)
+        return ""
+
+    exchanges = []
+    for message in messages:
+        if not isinstance(message, dict):
+            continue
+        role = str(message.get("role") or "").strip().lower()
+        if role not in {"user", "assistant"}:
+            continue
+        content = _text(message.get("content")).strip()
+        if content:
+            exchanges.append(f"{role.upper()}: {content}")
+
+    transcript = "\\n\\n".join(exchanges)
+    if len(transcript) > max_chars:
+        transcript = "[Earlier conversation omitted for context size.]\\n\\n" + transcript[-max_chars:]
+    if not transcript:
+        transcript = f"USER: {user_message}"
+    return (
+        "Continue this Hermes conversation after the previously selected "
+        "provider exhausted its capacity. Preserve the objective and answer "
+        "the latest user request; do not restart or discard completed work.\\n\\n"
+        f"<hermes_conversation>\\n{transcript}\\n</hermes_conversation>"
+    )
+
+
+def run_codex_app_server_turn(
+'''
+
 
 def _replace_once(content: str, before: str, after: str, label: str) -> str:
     """Apply one exact replacement and fail closed when upstream drifts."""
@@ -129,8 +266,14 @@ def patch_session(source: Path, destination: Path) -> None:
 
 
 def patch_turn(source: Path, destination: Path) -> None:
-    """Forward live Hermes route metadata into the Codex session adapter."""
+    """Forward route metadata and preserve context on Codex failover."""
     content = source.read_text(encoding="utf-8")
+    content = _replace_once(
+        content,
+        FALLBACK_CONTEXT_BEFORE,
+        FALLBACK_CONTEXT_AFTER,
+        "Codex fallback context",
+    )
     destination.parent.mkdir(parents=True, exist_ok=True)
     destination.write_text(
         _replace_once(content, TURN_BEFORE, TURN_AFTER, "Codex turn"),
@@ -138,6 +281,36 @@ def patch_turn(source: Path, destination: Path) -> None:
     )
 
 
+def patch_fallback(source: Path, destination: Path) -> None:
+    """Activate the CLI-backed Codex runtime in Hermes' fallback chain."""
+    content = source.read_text(encoding="utf-8")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    destination.write_text(
+        _replace_once(
+            content,
+            FALLBACK_RESOLUTION_BEFORE,
+            FALLBACK_RESOLUTION_AFTER,
+            "Codex fallback activation",
+        ),
+        encoding="utf-8",
+    )
+
+
+def patch_loop(source: Path, destination: Path) -> None:
+    """Dispatch a mid-turn Codex fallback through app-server."""
+    content = source.read_text(encoding="utf-8")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    destination.write_text(
+        _replace_once(
+            content,
+            FALLBACK_DISPATCH_BEFORE,
+            FALLBACK_DISPATCH_AFTER,
+            "Codex fallback dispatch",
+        ),
+        encoding="utf-8",
+    )
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("provider_source", type=Path)
@@ -146,10 +319,16 @@ def main() -> int:
     parser.add_argument("session_destination", type=Path)
     parser.add_argument("turn_source", type=Path)
     parser.add_argument("turn_destination", type=Path)
+    parser.add_argument("fallback_source", type=Path)
+    parser.add_argument("fallback_destination", type=Path)
+    parser.add_argument("loop_source", type=Path)
+    parser.add_argument("loop_destination", type=Path)
     args = parser.parse_args()
     patch_provider(args.provider_source, args.provider_destination)
     patch_session(args.session_source, args.session_destination)
     patch_turn(args.turn_source, args.turn_destination)
+    patch_fallback(args.fallback_source, args.fallback_destination)
+    patch_loop(args.loop_source, args.loop_destination)
     return 0
 
 

testing/tests/test_hermes_cli_lanes.py

@@ -881,10 +881,35 @@ def test_codex_runtime_patch_uses_cli_and_forwards_route(tmp_path: Path):
     assert '"sandboxPolicy": {"type": "dangerFullAccess"}' in session_content
 
     turn = tmp_path / "codex_runtime.py"
-    turn.write_text(codex_runtime_patch.TURN_BEFORE, encoding="utf-8")
+    turn.write_text(
+        codex_runtime_patch.FALLBACK_CONTEXT_BEFORE
+        + codex_runtime_patch.TURN_BEFORE,
+        encoding="utf-8",
+    )
     turn_out = tmp_path / "patched/codex_runtime.py"
     codex_runtime_patch.patch_turn(turn, turn_out)
-    assert "model=str(getattr(agent" in turn_out.read_text()
+    turn_content = turn_out.read_text()
+    assert "model=str(getattr(agent" in turn_content
+    assert "build_cross_provider_codex_prompt" in turn_content
+
+    fallback = tmp_path / "chat_completion_helpers.py"
+    fallback.write_text(
+        codex_runtime_patch.FALLBACK_RESOLUTION_BEFORE,
+        encoding="utf-8",
+    )
+    fallback_out = tmp_path / "patched/chat_completion_helpers.py"
+    codex_runtime_patch.patch_fallback(fallback, fallback_out)
+    fallback_content = fallback_out.read_text()
+    assert 'agent.api_mode = "codex_app_server"' in fallback_content
+    assert "agent._codex_cross_provider_fallback = True" in fallback_content
+
+    loop = tmp_path / "conversation_loop.py"
+    loop.write_text(codex_runtime_patch.FALLBACK_DISPATCH_BEFORE, encoding="utf-8")
+    loop_out = tmp_path / "patched/conversation_loop.py"
+    codex_runtime_patch.patch_loop(loop, loop_out)
+    loop_content = loop_out.read_text()
+    assert 'if agent.api_mode == "codex_app_server"' in loop_content
+    assert "build_cross_provider_codex_prompt" in loop_content
 
 
 def test_codex_runtime_patch_fails_closed_on_upstream_drift(tmp_path: Path):