diff --git a/services/hermes/scripts/suite_backends.py b/services/hermes/scripts/suite_backends.py index 5d91dac5..e2f7a79c 100644 --- a/services/hermes/scripts/suite_backends.py +++ b/services/hermes/scripts/suite_backends.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import os from pathlib import Path +import re import signal import subprocess import tempfile @@ -129,6 +130,7 @@ def stop(process): def parse_claude(raw, expected_model): """Normalize only a completed, uncompacted CLI result with a pinned model.""" final, initialized = None, False + aliases = {expected_model, expected_model + "[1m]"} for line in raw.splitlines(): try: event = json.loads(line) @@ -140,7 +142,7 @@ def parse_claude(raw, expected_model): raise Problem("compaction_detected", 502) if event.get("type") == "system" and event.get("subtype") == "init": initialized = True - if (event.get("model") != expected_model or event.get("mcp_servers") or + if (event.get("model") not in aliases or event.get("mcp_servers") or event.get("plugins") or set(event.get("tools", [])) - {"StructuredOutput"}): raise Problem("worker_isolation_failed", 502) if event.get("type") == "result": @@ -156,11 +158,15 @@ def parse_claude(raw, expected_model): if final.get("stop_reason") in {"max_tokens", "model_context_window_exceeded"}: raise Problem("incomplete_generation", 502) models = final.get("modelUsage", {}) - if set(models) != {expected_model}: - raise Problem("model_changed", 502) - limits = models[expected_model] - if limits.get("contextWindow") != 1000000 or limits.get("maxOutputTokens") != 64000: - raise Problem("backend_capabilities_changed", 502) + if not models or set(models) - aliases: + observed = [name if re.fullmatch(r"claude-[a-z0-9.-]+(?:\[1m\])?", name) + else "unrecognized" for name in models] + raise Problem("model_changed", 502, observed_models=observed) + for limits in models.values(): + if (limits.get("contextWindow") != 1000000 or limits.get("maxOutputTokens") != 64000 + or limits.get("canonicalModel") != expected_model + or limits.get("provider") != "firstParty"): + raise Problem("backend_capabilities_changed", 502) usage = final.get("usage", {}) if any(usage.get("server_tool_use", {}).values()): raise Problem("worker_isolation_failed", 502) diff --git a/services/hermes/suite-planner-deployment.yaml b/services/hermes/suite-planner-deployment.yaml index 70513be2..d4cc7a6d 100644 --- a/services/hermes/suite-planner-deployment.yaml +++ b/services/hermes/suite-planner-deployment.yaml @@ -36,7 +36,7 @@ spec: app: hermes-suite-planner annotations: fluentbit.io/exclude: "true" - ai.bstein.dev/config-rev: suite-v2-20260929 + ai.bstein.dev/config-rev: suite-v3-20260929 vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-pre-populate-only: "true" vault.hashicorp.com/agent-init-first: "true"