#!/usr/bin/env python3 """Bounded local route advice for provisional Hermes voice transcripts.""" from __future__ import annotations import json import os import re import threading import time from typing import Any, Final import httpx UPSTREAM: Final = os.environ.get( "HERMES_CLASSIFIER_BROKER_UPSTREAM", "http://ollama.ai.svc.cluster.local:11434", ).rstrip("/") MODEL: Final = os.environ.get( "HERMES_VOICE_PREFLIGHT_MODEL", "qwen2.5:14b-instruct-q4_0" ) MAX_BODY_BYTES: Final = 2 * 1024 MAX_TEXT_CHARS: Final = 512 TIMEOUT_SECONDS: Final = max( 0.1, min(0.75, float(os.environ.get("HERMES_VOICE_PREFLIGHT_TIMEOUT", "0.7"))) ) TIERS: Final = {"fast", "balanced", "deep", "maximum"} TURN_PATTERN: Final = re.compile(r"[A-Za-z0-9_.:-]{1,128}\Z") SEEN_TTL_SECONDS: Final = 120.0 MAX_SEEN_TURNS: Final = 256 def validate_request(payload: dict[str, Any]) -> tuple[str, int, str]: """Validate one bounded provisional transcript without identity coercion.""" turn_id = payload.get("turn_id") revision = payload.get("revision") transcript = payload.get("transcript") if not isinstance(turn_id, str) or not TURN_PATTERN.fullmatch(turn_id): raise ValueError("invalid turn_id") if isinstance(revision, bool) or not isinstance(revision, int): raise ValueError("invalid revision") if revision < 1 or revision > 1_000_000: raise ValueError("invalid revision") if not isinstance(transcript, str): raise ValueError("invalid transcript") transcript = " ".join(transcript.split()) if len(transcript) < 12 or len(transcript) > MAX_TEXT_CHARS: raise ValueError("invalid transcript") return turn_id, revision, transcript def inference_payload(transcript: str) -> dict[str, Any]: """Build the tiny local-only classification request used during speech.""" return { "model": MODEL, "messages": [ { "role": "system", "content": ( "Classify this incomplete spoken request into one advisory effort " "tier. Return JSON only as {\"tier\":\"fast|balanced|deep|maximum\"}. " "Use fast for simple conversation or facts, balanced for ordinary " "assistance, deep for multi-step work, and maximum only for explicit " "high-stakes or unusually complex work. This is not authoritative." ), }, {"role": "user", "content": transcript}, ], "temperature": 0, "max_tokens": 16, "stream": True, "response_format": {"type": "json_object"}, } class Coordinator: """Serialize disposable inference and give final classification priority.""" def __init__(self) -> None: self.condition = threading.Condition() self.authoritative = 0 self.active = False self.cancel: threading.Event | None = None self.client: Any = None self.response: Any = None self.seen: dict[str, float] = {} self.stats = { "admitted": 0, "rejected": 0, "success": 0, "timeout": 0, "cancelled": 0, "failure": 0, "preempted": 0, } self.duration_count = 0 self.duration_sum = 0.0 def _prune(self, now: float) -> None: self.seen = { turn: created for turn, created in self.seen.items() if now - created <= SEEN_TTL_SECONDS } if len(self.seen) > MAX_SEEN_TURNS: self.seen = dict(sorted(self.seen.items(), key=lambda item: item[1])[-MAX_SEEN_TURNS:]) def begin_authoritative(self) -> None: """Cancel advisory I/O as soon as a real classifier request arrives.""" with self.condition: self.authoritative += 1 if self.cancel is not None: self.cancel.set() self.stats["preempted"] += 1 resources = (self.response, self.client) for resource in resources: if resource is not None: try: resource.close() except Exception: pass def end_authoritative(self) -> None: """Release the real-classifier priority marker.""" with self.condition: self.authoritative = max(0, self.authoritative - 1) self.condition.notify_all() def begin(self, turn_id: str) -> threading.Event | None: """Admit at most one advisory globally and once per browser turn.""" now = time.monotonic() with self.condition: self._prune(now) if self.active or self.authoritative or turn_id in self.seen: self.stats["rejected"] += 1 return None self.seen[turn_id] = now self.active = True self.cancel = threading.Event() self.stats["admitted"] += 1 return self.cancel def register( self, cancel: threading.Event, *, client: Any = None, response: Any = None ) -> bool: """Publish cancellable resources unless authority already preempted them.""" with self.condition: if ( not self.active or self.cancel is not cancel or cancel.is_set() or self.authoritative ): cancel.set() return False if client is not None: self.client = client if response is not None: self.response = response return True def end(self) -> None: """Release the disposable local lane after its bounded request.""" with self.condition: self.active = False self.cancel = None self.client = None self.response = None self.condition.notify_all() def _record(self, outcome: str, started: float) -> None: """Record bounded low-cardinality timing without transcript or turn labels.""" elapsed = max(0.0, time.monotonic() - started) with self.condition: self.stats[outcome] += 1 self.duration_count += 1 self.duration_sum += elapsed print( f"voice-route-preflight outcome={outcome} duration_ms={elapsed * 1000:.1f}", flush=True, ) def metrics(self) -> str: """Render low-cardinality Prometheus telemetry with no user content.""" with self.condition: stats = dict(self.stats) duration_count = self.duration_count duration_sum = self.duration_sum authority = self.authoritative active = int(self.active) lines = [ "# HELP hermes_voice_route_preflight_total Local voice route preflight outcomes.", "# TYPE hermes_voice_route_preflight_total counter", ] lines.extend( f'hermes_voice_route_preflight_total{{outcome="{name}"}} {value}' for name, value in sorted(stats.items()) ) lines.extend( [ "# TYPE hermes_voice_route_preflight_duration_seconds summary", f"hermes_voice_route_preflight_duration_seconds_count {duration_count}", f"hermes_voice_route_preflight_duration_seconds_sum {duration_sum:.6f}", "# TYPE hermes_voice_route_preflight_active gauge", f"hermes_voice_route_preflight_active {active}", "# TYPE hermes_classifier_authoritative_active gauge", f"hermes_classifier_authoritative_active {authority}", ] ) return "\n".join(lines) + "\n" def stream(self, transcript: str, cancel: threading.Event) -> str: """Run one hard-bounded cancellable local Qwen advisory decode.""" started = time.monotonic() outcome = "failure" deadline = time.monotonic() + TIMEOUT_SECONDS timeout = httpx.Timeout( min(0.35, TIMEOUT_SECONDS), connect=min(0.25, TIMEOUT_SECONDS), read=min(0.35, TIMEOUT_SECONDS), write=min(0.25, TIMEOUT_SECONDS), pool=min(0.1, TIMEOUT_SECONDS), ) client = httpx.Client(timeout=timeout) if not self.register(cancel, client=client): client.close() self._record("cancelled", started) return "" response = None def expire() -> None: cancel.set() for resource in (response, client): if resource is not None: try: resource.close() except Exception: pass timer = threading.Timer(max(0.01, deadline - time.monotonic()), expire) timer.daemon = True timer.start() content = "" try: request = client.build_request( "POST", f"{UPSTREAM}/v1/chat/completions", headers={"Content-Type": "application/json"}, json=inference_payload(transcript), ) response = client.send(request, stream=True) if not self.register(cancel, response=response): outcome = "cancelled" response.close() return "" response.raise_for_status() for line in response.iter_lines(): if cancel.is_set() or time.monotonic() >= deadline: break if not line.startswith("data:"): continue event = line[5:].strip() if event == "[DONE]": break try: delta = (json.loads(event)["choices"][0].get("delta") or {}).get( "content", "" ) except (KeyError, IndexError, TypeError, json.JSONDecodeError): continue if isinstance(delta, str): content += delta if len(content) > 128: cancel.set() break if cancel.is_set() or time.monotonic() >= deadline: outcome = "timeout" if time.monotonic() >= deadline else "cancelled" return "" decoded = json.loads(content) tier = decoded.get("tier") if isinstance(decoded, dict) else None if tier in TIERS: outcome = "success" return tier return "" except Exception: outcome = "cancelled" if cancel.is_set() else "failure" return "" finally: cancel.set() timer.cancel() if response is not None: try: response.close() except Exception: pass try: client.close() except Exception: pass self._record(outcome, started) COORDINATOR = Coordinator()