diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 1a0361592..bbb782165 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,7 +24,7 @@ spec: ai.bstein.dev/execution: Herdr-supervised Codex and Claude Code ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260810-unattended-workers-v2" + ai.bstein.dev/config-rev: "20260810-context-aware-routing" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens diff --git a/services/hermes/plugins/auto-router/__init__.py b/services/hermes/plugins/auto-router/__init__.py index b0361c221..bc671ea7b 100644 --- a/services/hermes/plugins/auto-router/__init__.py +++ b/services/hermes/plugins/auto-router/__init__.py @@ -73,6 +73,15 @@ COMPLEX_TERMS = { "performance", "root cause", } +CONTEXTUAL_FOLLOWUP_PATTERNS = ( + r"\bcontinue\b", + r"\bresume\b", + r"\bkeep (?:going|working)\b", + r"\bloop through\b", + r"\b(?:all|remaining|outstanding)\b.{0,80}\b(?:work|tasks?|items?)\b", + r"\bdo (?:it|that|this)\b", + r"\bfinish (?:it|that|this|everything|all)\b", +) @dataclass(frozen=True) @@ -96,6 +105,50 @@ def _tokens(text: str) -> set[str]: return words +def _is_contextual_followup(text: str) -> bool: + """Return whether an instruction depends on work described earlier.""" + lowered = text.lower() + return any( + re.search(pattern, lowered, re.DOTALL) + for pattern in CONTEXTUAL_FOLLOWUP_PATTERNS + ) + + +def _message_text(message: dict[str, Any]) -> str: + """Flatten the text-bearing parts of one conversation-history message.""" + content = message.get("content", "") + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + parts: list[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + elif isinstance(block, dict): + value = block.get("text") or block.get("content") + if isinstance(value, str): + parts.append(value) + return "\n".join(parts) + + +def _task_with_recent_context( + text: str, conversation_history: list[dict[str, Any]] | None +) -> tuple[str, bool]: + """Resolve a referential follow-up against the latest assistant response.""" + if not _is_contextual_followup(text): + return text, False + for message in reversed(conversation_history or []): + if not isinstance(message, dict): + continue + if str(message.get("role") or "").lower() != "assistant": + continue + prior = _message_text(message).strip() + if prior: + return f"{text}\n\nRecent assistant context:\n{prior[-6000:]}", True + return text, False + + def heuristic_decision(text: str) -> Decision: """Return a safe, deterministic route when local classification is unavailable.""" tokens = _tokens(text) @@ -134,6 +187,14 @@ def heuristic_decision(text: str) -> Decision: "heuristic", "analysis or independent review task", ) + if _is_contextual_followup(text): + return Decision( + "implementation", + "high", + "codex", + "heuristic", + "continuation of material outstanding work", + ) if word_count <= 24: return Decision( "question", @@ -207,13 +268,32 @@ def jetson_decision(text: str, timeout: float = 1.8) -> Decision | None: return _validated_local_effort(value, latency_ms) -def classify_task(text: str) -> Decision: +def classify_task( + text: str, conversation_history: list[dict[str, Any]] | None = None +) -> Decision: """Combine local classification with deterministic safety and quality floors.""" - baseline = heuristic_decision(text) + effective_text, used_context = _task_with_recent_context(text, conversation_history) + baseline = heuristic_decision(effective_text) if baseline.effort in {"low", "xhigh"}: + if used_context: + return Decision( + baseline.shape, + baseline.effort, + baseline.provider, + "heuristic-context", + f"{baseline.reason}; resolved against recent assistant context", + ) return baseline - local = jetson_decision(text) + local = jetson_decision(effective_text) if local is None: + if used_context: + return Decision( + baseline.shape, + baseline.effort, + baseline.provider, + "heuristic-context", + f"{baseline.reason}; resolved against recent assistant context", + ) return baseline # Deterministic policy owns task shape, provider preference, xhigh, and the @@ -224,8 +304,9 @@ def classify_task(text: str) -> Decision: baseline.shape, effort, baseline.provider, - "jetson", - "Jetson effort classification with deterministic routing guardrails", + "jetson-context" if used_context else "jetson", + "Jetson effort classification with deterministic routing guardrails" + + (" and recent assistant context" if used_context else ""), local.latency_ms, ) @@ -456,7 +537,7 @@ def _pre_turn_route(ctx: Any, **kwargs: Any) -> None: if provider not in PROVIDERS or effort not in EFFORTS: policy = {"mode": "auto"} _write_policy(policy) - decision = classify_task(text) + decision = classify_task(text, kwargs.get("conversation_history")) plan = select_route(_load_json(ROUTING_PATH), decision) else: decision = Decision( @@ -464,7 +545,7 @@ def _pre_turn_route(ctx: Any, **kwargs: Any) -> None: ) plan = select_route(_load_json(ROUTING_PATH), decision, model) else: - decision = classify_task(text) + decision = classify_task(text, kwargs.get("conversation_history")) plan = select_route(_load_json(ROUTING_PATH), decision) _apply_route(ctx, agent, plan) _record_plan(policy, plan) @@ -476,7 +557,12 @@ def _pre_turn_route(ctx: Any, **kwargs: Any) -> None: f"{plan['effort']} · automatic capacity fallback remains enabled" ) else: - source = "Jetson" if plan["classifier"] == "jetson" else "fast fallback" + source = { + "jetson": "Jetson", + "jetson-context": "Jetson + recent context", + "heuristic-context": "recent-context policy", + "heuristic": "deterministic policy", + }.get(str(plan["classifier"]), "deterministic policy") emit( f"AUTO target → {plan['provider']}/{plan['model']} · " f"{plan['effort']} ({source}) · automatic capacity fallback enabled" diff --git a/testing/tests/test_hermes_auto_router.py b/testing/tests/test_hermes_auto_router.py index df9ff8ac2..b3562bf6d 100644 --- a/testing/tests/test_hermes_auto_router.py +++ b/testing/tests/test_hermes_auto_router.py @@ -113,6 +113,62 @@ def test_deterministic_low_and_xhigh_routes_skip_local_latency(monkeypatch): assert router.classify_task("Migrate production Vault credentials").effort == "xhigh" +def test_referential_outstanding_work_never_uses_low_route(monkeypatch): + monkeypatch.setattr( + router, + "jetson_decision", + lambda text: router.Decision( + "question", "low", "codex", "jetson", "test", 10 + ), + ) + + decision = router.classify_task( + "Look, loop through all of the still outstanding work that you identified. " + "Do it to the best of your abilities." + ) + + assert (decision.shape, decision.effort, decision.provider) == ( + "implementation", + "high", + "codex", + ) + + +def test_referential_followup_uses_recent_context_for_risk_floor(monkeypatch): + monkeypatch.setattr( + router, + "jetson_decision", + lambda text: (_ for _ in ()).throw(AssertionError("xhigh skips Jetson")), + ) + history = [ + {"role": "user", "content": "Is the work complete?"}, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": ( + "Still outstanding: production deployment retry, provider " + "switching, runtime experiments, and a full repository test pass." + ), + } + ], + }, + ] + + decision = router.classify_task( + "Loop through all outstanding work and finish it.", history + ) + + assert (decision.shape, decision.effort, decision.provider) == ( + "review", + "xhigh", + "claude", + ) + assert decision.classifier == "heuristic-context" + assert "recent assistant context" in decision.reason + + def test_manual_policy_is_reapplied_on_every_non_command_turn(monkeypatch): monkeypatch.setattr( router,