306 lines
11 KiB
Python
306 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded local route advice for provisional Hermes voice transcripts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import threading
|
|
import time
|
|
from typing import Any, Final
|
|
|
|
import httpx
|
|
|
|
|
|
UPSTREAM: Final = os.environ.get(
|
|
"HERMES_CLASSIFIER_BROKER_UPSTREAM",
|
|
"http://ollama.ai.svc.cluster.local:11434",
|
|
).rstrip("/")
|
|
MODEL: Final = os.environ.get(
|
|
"HERMES_VOICE_PREFLIGHT_MODEL", "qwen2.5:14b-instruct-q4_0"
|
|
)
|
|
MAX_BODY_BYTES: Final = 2 * 1024
|
|
MAX_TEXT_CHARS: Final = 512
|
|
TIMEOUT_SECONDS: Final = max(
|
|
0.1, min(0.75, float(os.environ.get("HERMES_VOICE_PREFLIGHT_TIMEOUT", "0.7")))
|
|
)
|
|
TIERS: Final = {"fast", "balanced", "deep", "maximum"}
|
|
TURN_PATTERN: Final = re.compile(r"[A-Za-z0-9_.:-]{1,128}\Z")
|
|
SEEN_TTL_SECONDS: Final = 120.0
|
|
MAX_SEEN_TURNS: Final = 256
|
|
|
|
|
|
def validate_request(payload: dict[str, Any]) -> tuple[str, int, str]:
|
|
"""Validate one bounded provisional transcript without identity coercion."""
|
|
turn_id = payload.get("turn_id")
|
|
revision = payload.get("revision")
|
|
transcript = payload.get("transcript")
|
|
if not isinstance(turn_id, str) or not TURN_PATTERN.fullmatch(turn_id):
|
|
raise ValueError("invalid turn_id")
|
|
if isinstance(revision, bool) or not isinstance(revision, int):
|
|
raise ValueError("invalid revision")
|
|
if revision < 1 or revision > 1_000_000:
|
|
raise ValueError("invalid revision")
|
|
if not isinstance(transcript, str):
|
|
raise ValueError("invalid transcript")
|
|
transcript = " ".join(transcript.split())
|
|
if len(transcript) < 12 or len(transcript) > MAX_TEXT_CHARS:
|
|
raise ValueError("invalid transcript")
|
|
return turn_id, revision, transcript
|
|
|
|
|
|
def inference_payload(transcript: str) -> dict[str, Any]:
|
|
"""Build the tiny local-only classification request used during speech."""
|
|
return {
|
|
"model": MODEL,
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": (
|
|
"Classify this incomplete spoken request into one advisory effort "
|
|
"tier. Return JSON only as {\"tier\":\"fast|balanced|deep|maximum\"}. "
|
|
"Use fast for simple conversation or facts, balanced for ordinary "
|
|
"assistance, deep for multi-step work, and maximum only for explicit "
|
|
"high-stakes or unusually complex work. This is not authoritative."
|
|
),
|
|
},
|
|
{"role": "user", "content": transcript},
|
|
],
|
|
"temperature": 0,
|
|
"max_tokens": 16,
|
|
"stream": True,
|
|
"response_format": {"type": "json_object"},
|
|
}
|
|
|
|
|
|
class Coordinator:
|
|
"""Serialize disposable inference and give final classification priority."""
|
|
|
|
def __init__(self) -> None:
|
|
self.condition = threading.Condition()
|
|
self.authoritative = 0
|
|
self.active = False
|
|
self.cancel: threading.Event | None = None
|
|
self.client: Any = None
|
|
self.response: Any = None
|
|
self.seen: dict[str, float] = {}
|
|
self.stats = {
|
|
"admitted": 0,
|
|
"rejected": 0,
|
|
"success": 0,
|
|
"timeout": 0,
|
|
"cancelled": 0,
|
|
"failure": 0,
|
|
"preempted": 0,
|
|
}
|
|
self.duration_count = 0
|
|
self.duration_sum = 0.0
|
|
|
|
def _prune(self, now: float) -> None:
|
|
self.seen = {
|
|
turn: created
|
|
for turn, created in self.seen.items()
|
|
if now - created <= SEEN_TTL_SECONDS
|
|
}
|
|
if len(self.seen) > MAX_SEEN_TURNS:
|
|
self.seen = dict(sorted(self.seen.items(), key=lambda item: item[1])[-MAX_SEEN_TURNS:])
|
|
|
|
def begin_authoritative(self) -> None:
|
|
"""Cancel advisory I/O as soon as a real classifier request arrives."""
|
|
with self.condition:
|
|
self.authoritative += 1
|
|
if self.cancel is not None:
|
|
self.cancel.set()
|
|
self.stats["preempted"] += 1
|
|
resources = (self.response, self.client)
|
|
for resource in resources:
|
|
if resource is not None:
|
|
try:
|
|
resource.close()
|
|
except Exception:
|
|
pass
|
|
|
|
def end_authoritative(self) -> None:
|
|
"""Release the real-classifier priority marker."""
|
|
with self.condition:
|
|
self.authoritative = max(0, self.authoritative - 1)
|
|
self.condition.notify_all()
|
|
|
|
def begin(self, turn_id: str) -> threading.Event | None:
|
|
"""Admit at most one advisory globally and once per browser turn."""
|
|
now = time.monotonic()
|
|
with self.condition:
|
|
self._prune(now)
|
|
if self.active or self.authoritative or turn_id in self.seen:
|
|
self.stats["rejected"] += 1
|
|
return None
|
|
self.seen[turn_id] = now
|
|
self.active = True
|
|
self.cancel = threading.Event()
|
|
self.stats["admitted"] += 1
|
|
return self.cancel
|
|
|
|
def register(
|
|
self, cancel: threading.Event, *, client: Any = None, response: Any = None
|
|
) -> bool:
|
|
"""Publish cancellable resources unless authority already preempted them."""
|
|
with self.condition:
|
|
if (
|
|
not self.active
|
|
or self.cancel is not cancel
|
|
or cancel.is_set()
|
|
or self.authoritative
|
|
):
|
|
cancel.set()
|
|
return False
|
|
if client is not None:
|
|
self.client = client
|
|
if response is not None:
|
|
self.response = response
|
|
return True
|
|
|
|
def end(self) -> None:
|
|
"""Release the disposable local lane after its bounded request."""
|
|
with self.condition:
|
|
self.active = False
|
|
self.cancel = None
|
|
self.client = None
|
|
self.response = None
|
|
self.condition.notify_all()
|
|
|
|
def _record(self, outcome: str, started: float) -> None:
|
|
"""Record bounded low-cardinality timing without transcript or turn labels."""
|
|
elapsed = max(0.0, time.monotonic() - started)
|
|
with self.condition:
|
|
self.stats[outcome] += 1
|
|
self.duration_count += 1
|
|
self.duration_sum += elapsed
|
|
print(
|
|
f"voice-route-preflight outcome={outcome} duration_ms={elapsed * 1000:.1f}",
|
|
flush=True,
|
|
)
|
|
|
|
def metrics(self) -> str:
|
|
"""Render low-cardinality Prometheus telemetry with no user content."""
|
|
with self.condition:
|
|
stats = dict(self.stats)
|
|
duration_count = self.duration_count
|
|
duration_sum = self.duration_sum
|
|
authority = self.authoritative
|
|
active = int(self.active)
|
|
lines = [
|
|
"# HELP hermes_voice_route_preflight_total Local voice route preflight outcomes.",
|
|
"# TYPE hermes_voice_route_preflight_total counter",
|
|
]
|
|
lines.extend(
|
|
f'hermes_voice_route_preflight_total{{outcome="{name}"}} {value}'
|
|
for name, value in sorted(stats.items())
|
|
)
|
|
lines.extend(
|
|
[
|
|
"# TYPE hermes_voice_route_preflight_duration_seconds summary",
|
|
f"hermes_voice_route_preflight_duration_seconds_count {duration_count}",
|
|
f"hermes_voice_route_preflight_duration_seconds_sum {duration_sum:.6f}",
|
|
"# TYPE hermes_voice_route_preflight_active gauge",
|
|
f"hermes_voice_route_preflight_active {active}",
|
|
"# TYPE hermes_classifier_authoritative_active gauge",
|
|
f"hermes_classifier_authoritative_active {authority}",
|
|
]
|
|
)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
def stream(self, transcript: str, cancel: threading.Event) -> str:
|
|
"""Run one hard-bounded cancellable local Qwen advisory decode."""
|
|
started = time.monotonic()
|
|
outcome = "failure"
|
|
deadline = time.monotonic() + TIMEOUT_SECONDS
|
|
timeout = httpx.Timeout(
|
|
min(0.35, TIMEOUT_SECONDS),
|
|
connect=min(0.25, TIMEOUT_SECONDS),
|
|
read=min(0.35, TIMEOUT_SECONDS),
|
|
write=min(0.25, TIMEOUT_SECONDS),
|
|
pool=min(0.1, TIMEOUT_SECONDS),
|
|
)
|
|
client = httpx.Client(timeout=timeout)
|
|
if not self.register(cancel, client=client):
|
|
client.close()
|
|
self._record("cancelled", started)
|
|
return ""
|
|
|
|
response = None
|
|
|
|
def expire() -> None:
|
|
cancel.set()
|
|
for resource in (response, client):
|
|
if resource is not None:
|
|
try:
|
|
resource.close()
|
|
except Exception:
|
|
pass
|
|
|
|
timer = threading.Timer(max(0.01, deadline - time.monotonic()), expire)
|
|
timer.daemon = True
|
|
timer.start()
|
|
content = ""
|
|
try:
|
|
request = client.build_request(
|
|
"POST",
|
|
f"{UPSTREAM}/v1/chat/completions",
|
|
headers={"Content-Type": "application/json"},
|
|
json=inference_payload(transcript),
|
|
)
|
|
response = client.send(request, stream=True)
|
|
if not self.register(cancel, response=response):
|
|
outcome = "cancelled"
|
|
response.close()
|
|
return ""
|
|
response.raise_for_status()
|
|
for line in response.iter_lines():
|
|
if cancel.is_set() or time.monotonic() >= deadline:
|
|
break
|
|
if not line.startswith("data:"):
|
|
continue
|
|
event = line[5:].strip()
|
|
if event == "[DONE]":
|
|
break
|
|
try:
|
|
delta = (json.loads(event)["choices"][0].get("delta") or {}).get(
|
|
"content", ""
|
|
)
|
|
except (KeyError, IndexError, TypeError, json.JSONDecodeError):
|
|
continue
|
|
if isinstance(delta, str):
|
|
content += delta
|
|
if len(content) > 128:
|
|
cancel.set()
|
|
break
|
|
if cancel.is_set() or time.monotonic() >= deadline:
|
|
outcome = "timeout" if time.monotonic() >= deadline else "cancelled"
|
|
return ""
|
|
decoded = json.loads(content)
|
|
tier = decoded.get("tier") if isinstance(decoded, dict) else None
|
|
if tier in TIERS:
|
|
outcome = "success"
|
|
return tier
|
|
return ""
|
|
except Exception:
|
|
outcome = "cancelled" if cancel.is_set() else "failure"
|
|
return ""
|
|
finally:
|
|
cancel.set()
|
|
timer.cancel()
|
|
if response is not None:
|
|
try:
|
|
response.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
client.close()
|
|
except Exception:
|
|
pass
|
|
self._record(outcome, started)
|
|
|
|
|
|
COORDINATOR = Coordinator()
|