fix(hermes): keep Jetson router model loaded
This commit is contained in:
parent
3b7c48fcf2
commit
cd4b00893c
@ -438,7 +438,9 @@ def _jetson_route(text: str, timeout: float) -> tuple[tuple[str, str, str] | Non
|
|||||||
"required": ["provider", "effort", "priority"],
|
"required": ["provider", "effort", "priority"],
|
||||||
"additionalProperties": False,
|
"additionalProperties": False,
|
||||||
},
|
},
|
||||||
"keep_alive": "-1",
|
# Ollama accepts a numeric negative duration as "keep loaded". A
|
||||||
|
# string without a unit is rejected by current releases with HTTP 400.
|
||||||
|
"keep_alive": -1,
|
||||||
"options": {"temperature": 0, "num_ctx": 2048, "num_predict": 48},
|
"options": {"temperature": 0, "num_ctx": 2048, "num_predict": 48},
|
||||||
"messages": [
|
"messages": [
|
||||||
{"role": "system", "content": system_prompt},
|
{"role": "system", "content": system_prompt},
|
||||||
@ -927,7 +929,7 @@ def _rewarm_classifier() -> None:
|
|||||||
"model": JETSON_MODEL,
|
"model": JETSON_MODEL,
|
||||||
"prompt": "Reply with P",
|
"prompt": "Reply with P",
|
||||||
"stream": False,
|
"stream": False,
|
||||||
"keep_alive": "-1",
|
"keep_alive": -1,
|
||||||
"options": {"temperature": 0, "num_ctx": 128, "num_predict": 1},
|
"options": {"temperature": 0, "num_ctx": 128, "num_predict": 1},
|
||||||
}
|
}
|
||||||
request = urllib.request.Request(
|
request = urllib.request.Request(
|
||||||
|
|||||||
@ -3,6 +3,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import io
|
||||||
|
import json
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@ -172,6 +174,32 @@ def test_jetson_requests_one_structured_provider_effort_priority_vote(monkeypatc
|
|||||||
assert calls == [("Review the architecture", 2.5)]
|
assert calls == [("Review the architecture", 2.5)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_ollama_requests_use_numeric_keep_alive(monkeypatch):
|
||||||
|
payloads = []
|
||||||
|
|
||||||
|
def urlopen(request, timeout):
|
||||||
|
payloads.append(json.loads(request.data))
|
||||||
|
if request.full_url.endswith("/chat"):
|
||||||
|
body = {
|
||||||
|
"message": {
|
||||||
|
"content": '{"provider":"C","effort":"L","priority":"F"}'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
body = {"response": "P"}
|
||||||
|
return io.BytesIO(json.dumps(body).encode())
|
||||||
|
|
||||||
|
monkeypatch.setattr(router.urllib.request, "urlopen", urlopen)
|
||||||
|
|
||||||
|
vote, _ = router._jetson_route("Answer quickly", 2.5)
|
||||||
|
assert vote == ("C", "L", "F")
|
||||||
|
assert payloads[0]["keep_alive"] == -1
|
||||||
|
|
||||||
|
router._classifier_warm_lock.acquire()
|
||||||
|
router._rewarm_classifier()
|
||||||
|
assert payloads[1]["keep_alive"] == -1
|
||||||
|
|
||||||
|
|
||||||
def test_every_auto_classification_consults_jetson_and_keeps_safety_floors(monkeypatch):
|
def test_every_auto_classification_consults_jetson_and_keeps_safety_floors(monkeypatch):
|
||||||
calls = []
|
calls = []
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user