hermes: preserve exact routes and accept Telegram images

This commit is contained in:
jenkins 2026-08-15 21:48:30 -03:00
parent 89b9f1a4c7
commit 8c73906e6e
8 changed files with 468 additions and 11 deletions

View File

@ -449,6 +449,49 @@ if delegate.count(delegate_before) != 1:
f"found {delegate.count(delegate_before)}"
)
delegate_path.write_text(delegate.replace(delegate_before, delegate_after, 1))
delegate = delegate_path.read_text()
background_before = ''' is_subagent = getattr(parent_agent, "_delegate_depth", 0) > 0
return not is_subagent
'''
background_after = ''' is_subagent = getattr(parent_agent, "_delegate_depth", 0) > 0
# A one-shot process has no durable event loop to receive a detached
# child's result after its parent exits. Keep delegation synchronous so
# the child verdict is returned to—and persisted by—the parent turn.
if bool(getattr(parent_agent, "_hermes_oneshot", False)):
return False
return not is_subagent
'''
if delegate.count(background_before) != 1:
raise SystemExit(
"Hermes oneshot delegation context changed: expected 1, "
f"found {delegate.count(background_before)}"
)
delegate_path.write_text(delegate.replace(background_before, background_after, 1))
oneshot_path = Path("/opt/hermes/hermes_cli/oneshot.py")
oneshot = oneshot_path.read_text()
oneshot_before = ''' # Belt-and-braces: make sure AIAgent doesn't invoke any streaming
# display callbacks that would bypass our stdout capture.
agent.suppress_status_output = True
'''
oneshot_after = ''' # Preserve a caller's explicit --model route across every plugin routing
# boundary. Config/env defaults remain automatic policy inputs.
agent._hermes_explicit_model_pick = bool((model or "").strip())
# Detached delegation cannot deliver back into a process that exits after
# this turn; delegate_tool uses this marker to keep children synchronous.
agent._hermes_oneshot = True
# Belt-and-braces: make sure AIAgent doesn't invoke any streaming
# display callbacks that would bypass our stdout capture.
agent.suppress_status_output = True
'''
if oneshot.count(oneshot_before) != 1:
raise SystemExit(
"Hermes oneshot explicit-model context changed: expected 1, "
f"found {oneshot.count(oneshot_before)}"
)
oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1))
PY
# Hermes WebUI sends its model/provider/reasoning selection on /v1/runs.
@ -1236,6 +1279,7 @@ RUN cd /opt/hermes/web \
/opt/hermes/gateway/platforms/api_server.py \
/opt/hermes/agent/turn_context.py \
/opt/hermes/agent/conversation_loop.py \
/opt/hermes/hermes_cli/oneshot.py \
/opt/hermes/tools/delegate_tool.py \
/opt/hermes/tools/web_tools.py \
/opt/hermes/tools/python_sandbox_tool.py \

View File

@ -51,6 +51,14 @@ MANUAL_ROUTES = frozenset(
)
ALL_ROUTES = AUTO_ROUTES | MANUAL_ROUTES
EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
ROUTABLE_EFFORTS = frozenset({"low", "medium", "high", "xhigh"})
EXACT_MANUAL_ROUTES = frozenset(
f"{route}/{effort}"
for route in MANUAL_ROUTES
if not route.endswith("/local/qwen-14b")
for effort in ROUTABLE_EFFORTS
)
ALL_REQUEST_ROUTES = ALL_ROUTES | EXACT_MANUAL_ROUTES
PROVIDER_DEFAULT = {
"codex": "atlas/manual/codex/terra",
"claude": "atlas/manual/claude/sonnet",
@ -136,11 +144,11 @@ def _normalise_manual_route(provider: str, model: str = "") -> str:
def _explicit_ui_route(agent: Any) -> str:
"""Return a route selected in the WebUI model picker for this request."""
"""Return an exact route selected by the WebUI or command-line caller."""
if not bool(getattr(agent, "_hermes_explicit_model_pick", False)):
return ""
route = str(getattr(agent, "model", "") or "").strip()
return route if route in ALL_ROUTES else ""
return route if route in ALL_REQUEST_ROUTES else ""
def _explicit_ui_effort(agent: Any) -> str:
@ -158,6 +166,9 @@ def _boundary_selection(agent: Any) -> tuple[str, str, str]:
ui_effort = _explicit_ui_effort(agent)
if ui_route:
mode = "auto" if ui_route in AUTO_ROUTES else "manual"
if ui_route in EXACT_MANUAL_ROUTES:
effort = ui_route.rsplit("/", 1)[-1]
return ui_route, effort, f"ui-{mode}"
return ui_route, ui_effort, f"ui-{mode}"
if policy["mode"] == "manual":
@ -179,9 +190,11 @@ def _boundary_selection(agent: Any) -> tuple[str, str, str]:
def _resolved_route(route: str, effort: str) -> str:
"""Bind a manual family and UI effort to an exact Switchyard route."""
if route in EXACT_MANUAL_ROUTES:
return route
if route not in MANUAL_ROUTES or route.endswith("/local/qwen-14b"):
return route
provider_effort = effort if effort in {"low", "medium", "high", "xhigh"} else "low"
provider_effort = effort if effort in ROUTABLE_EFFORTS else "low"
return f"{route}/{provider_effort}"

View File

@ -27,6 +27,7 @@ type telegramBot struct {
config telegramConfig
router *tenantRouter
apiBase string
fileBase string
client *http.Client
mediaClient *http.Client
agentClient *http.Client
@ -45,10 +46,19 @@ type telegramUpdate struct {
}
type telegramMessage struct {
MessageID int64 `json:"message_id"`
From *telegramUser `json:"from"`
Chat telegramChat `json:"chat"`
Text string `json:"text"`
MessageID int64 `json:"message_id"`
From *telegramUser `json:"from"`
Chat telegramChat `json:"chat"`
Text string `json:"text"`
Caption string `json:"caption"`
Photo []telegramPhotoSize `json:"photo"`
}
type telegramPhotoSize struct {
FileID string `json:"file_id"`
FileSize int64 `json:"file_size"`
Width int `json:"width"`
Height int `json:"height"`
}
type telegramUser struct {
@ -89,6 +99,7 @@ func newTelegramBot(config telegramConfig, router *tenantRouter) *telegramBot {
config: config,
router: router,
apiBase: "https://api.telegram.org/bot" + config.BotToken,
fileBase: "https://api.telegram.org/file/bot" + config.BotToken,
client: &http.Client{Timeout: 70 * time.Second},
mediaClient: newTenantMediaClient(),
agentClient: &http.Client{Timeout: 15 * time.Minute},
@ -265,7 +276,7 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) {
return
}
if command == "help" {
_ = bot.sendText(message.Chat.ID, "Send any text to chat with Hermes. Use /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.")
_ = bot.sendText(message.Chat.ID, "Send text or a photo to chat with Hermes. Use /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.")
return
}
slot, linked := bot.router.telegramSlot(userID)
@ -274,8 +285,15 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) {
return
}
text := strings.TrimSpace(message.Text)
hasPhoto := len(message.Photo) > 0
if hasPhoto {
text = strings.TrimSpace(message.Caption)
if text == "" {
text = "Describe this image and answer any question it implies."
}
}
if text == "" {
_ = bot.sendText(message.Chat.ID, "Text messages are supported now; attachment support will be added separately.")
_ = bot.sendText(message.Chat.ID, "Hermes currently accepts text and image attachments. Please send other file types through Hermes WebUI.")
return
}
if utf8.RuneCountInString(text) > 12000 {
@ -286,7 +304,19 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) {
lock.Lock()
defer lock.Unlock()
_ = bot.sendAction(message.Chat.ID, "typing")
reply, err := bot.askTenant(slot, text, update.UpdateID)
var reply string
var err error
if hasPhoto {
media, fetchErr := bot.fetchTelegramPhoto(message.Photo)
if fetchErr != nil {
_ = bot.sendText(message.Chat.ID, "Hermes could not safely read that image. Please try a smaller JPEG, PNG, or WebP image.")
return
}
defer os.Remove(media.Path)
reply, err = bot.askTenantImage(slot, text, media, update.UpdateID)
} else {
reply, err = bot.askTenant(slot, text, update.UpdateID)
}
if err != nil {
_ = bot.sendText(message.Chat.ID, "Hermes could not answer right now. Please try again shortly.")
return
@ -303,13 +333,20 @@ func (bot *telegramBot) askTenant(slot int, text string, updateID int64) (string
"conversation": "telegram",
"store": true,
})
return bot.askTenantRequest(slot, bytes.NewReader(payload), updateID)
}
func (bot *telegramBot) askTenantRequest(slot int, bodyReader io.Reader, updateID int64) (string, error) {
if bot.router.backendAPIURL == nil {
return "", errors.New("tenant API unavailable")
}
ctx, cancel := context.WithTimeout(context.Background(), 14*time.Minute)
defer cancel()
request, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
bot.router.backendAPIURL(slot)+"/v1/responses",
bytes.NewReader(payload),
bodyReader,
)
if err != nil {
return "", err

View File

@ -0,0 +1,195 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"strings"
"time"
)
const telegramInputImageLimit = 20 << 20
type telegramInputMedia struct {
MIME string
Path string
}
func largestTelegramPhoto(photos []telegramPhotoSize) (telegramPhotoSize, error) {
var selected telegramPhotoSize
for _, photo := range photos {
if strings.TrimSpace(photo.FileID) == "" {
continue
}
if selected.FileID == "" || int64(photo.Width)*int64(photo.Height) > int64(selected.Width)*int64(selected.Height) {
selected = photo
}
}
if selected.FileID == "" {
return telegramPhotoSize{}, errors.New("Telegram photo has no file identifier")
}
if selected.FileSize > telegramInputImageLimit {
return telegramPhotoSize{}, errors.New("Telegram photo exceeds input limit")
}
return selected, nil
}
func safeTelegramFilePath(raw string) (string, error) {
value := strings.TrimSpace(raw)
if value == "" || strings.Contains(value, `\`) || strings.ContainsRune(value, '\x00') {
return "", errors.New("invalid Telegram file path")
}
cleaned := path.Clean(value)
if cleaned == "." || path.IsAbs(cleaned) || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
return "", errors.New("invalid Telegram file path")
}
return cleaned, nil
}
func escapedTelegramFilePath(value string) string {
components := strings.Split(value, "/")
for index := range components {
components[index] = url.PathEscape(components[index])
}
return strings.Join(components, "/")
}
func supportedTelegramImageType(header []byte) (string, error) {
detected := http.DetectContentType(header)
switch detected {
case "image/jpeg", "image/png", "image/webp", "image/gif":
return detected, nil
default:
return "", errors.New("unsupported Telegram image type")
}
}
func (bot *telegramBot) fetchTelegramPhoto(photos []telegramPhotoSize) (telegramInputMedia, error) {
selected, err := largestTelegramPhoto(photos)
if err != nil {
return telegramInputMedia{}, err
}
ctx, cancel := context.WithTimeout(context.Background(), 65*time.Second)
defer cancel()
var remote struct {
FilePath string `json:"file_path"`
FileSize int64 `json:"file_size"`
}
if err := bot.call(ctx, "getFile", url.Values{"file_id": {selected.FileID}}, &remote); err != nil {
return telegramInputMedia{}, errors.New("Telegram file lookup failed")
}
if remote.FileSize > telegramInputImageLimit {
return telegramInputMedia{}, errors.New("Telegram photo exceeds input limit")
}
remotePath, err := safeTelegramFilePath(remote.FilePath)
if err != nil {
return telegramInputMedia{}, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(bot.fileBase, "/")+"/"+escapedTelegramFilePath(remotePath), nil)
if err != nil {
return telegramInputMedia{}, errors.New("create Telegram file request")
}
response, err := bot.client.Do(request)
if err != nil {
return telegramInputMedia{}, errors.New("Telegram file download failed")
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return telegramInputMedia{}, errors.New("Telegram file download rejected")
}
if response.ContentLength > telegramInputImageLimit {
return telegramInputMedia{}, errors.New("Telegram photo exceeds input limit")
}
temporary, err := os.CreateTemp("", "hermes-telegram-input-*")
if err != nil {
return telegramInputMedia{}, errors.New("stage Telegram image")
}
temporaryPath := temporary.Name()
keep := false
defer func() {
_ = temporary.Close()
if !keep {
_ = os.Remove(temporaryPath)
}
}()
written, err := io.Copy(temporary, io.LimitReader(response.Body, telegramInputImageLimit+1))
if err != nil {
return telegramInputMedia{}, errors.New("read Telegram image")
}
if written > telegramInputImageLimit {
return telegramInputMedia{}, errors.New("Telegram photo exceeds input limit")
}
if _, err := temporary.Seek(0, io.SeekStart); err != nil {
return telegramInputMedia{}, errors.New("inspect Telegram image")
}
header := make([]byte, 512)
read, readErr := temporary.Read(header)
if readErr != nil && readErr != io.EOF {
return telegramInputMedia{}, errors.New("inspect Telegram image")
}
mimeType, err := supportedTelegramImageType(header[:read])
if err != nil {
return telegramInputMedia{}, err
}
if err := temporary.Close(); err != nil {
return telegramInputMedia{}, errors.New("stage Telegram image")
}
keep = true
return telegramInputMedia{MIME: mimeType, Path: temporaryPath}, nil
}
func streamTelegramImageRequest(writer io.Writer, prompt string, media telegramInputMedia) error {
promptJSON, err := json.Marshal(prompt)
if err != nil {
return err
}
if _, err := fmt.Fprintf(writer, `{"input":[{"role":"user","content":[{"type":"input_text","text":%s},{"type":"input_image","image_url":"data:%s;base64,`, promptJSON, media.MIME); err != nil {
return err
}
file, err := os.Open(media.Path)
if err != nil {
return err
}
encoder := base64.NewEncoder(base64.StdEncoding, writer)
_, copyErr := io.Copy(encoder, file)
closeEncoderErr := encoder.Close()
closeFileErr := file.Close()
if copyErr != nil {
return copyErr
}
if closeEncoderErr != nil {
return closeEncoderErr
}
if closeFileErr != nil {
return closeFileErr
}
_, err = io.WriteString(writer, `"}]}],"conversation":"telegram","store":true}`)
return err
}
func (bot *telegramBot) askTenantImage(slot int, prompt string, media telegramInputMedia, updateID int64) (string, error) {
reader, writer := io.Pipe()
writeDone := make(chan error, 1)
go func() {
err := streamTelegramImageRequest(writer, prompt, media)
_ = writer.CloseWithError(err)
writeDone <- err
}()
answer, requestErr := bot.askTenantRequest(slot, reader, updateID)
_ = reader.Close()
writeErr := <-writeDone
if requestErr != nil {
return "", requestErr
}
if writeErr != nil {
return "", errors.New("encode Telegram image")
}
return answer, nil
}

View File

@ -1,7 +1,9 @@
package main
import (
"encoding/base64"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
@ -91,3 +93,146 @@ func TestTelegramCallIncludesAPIDescription(t *testing.T) {
t.Fatalf("expected Telegram API description, got %v", err)
}
}
func jpegInputFixture() []byte {
return append(
[]byte{0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 'J', 'F', 'I', 'F', 0x00},
[]byte("telegram-input-image")...,
)
}
func TestFetchTelegramPhotoStagesLargestImageWithoutTrustingPath(t *testing.T) {
image := jpegInputFixture()
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
switch request.URL.Path {
case "/getFile":
if err := request.ParseForm(); err != nil {
t.Fatal(err)
}
if request.Form.Get("file_id") != "large" {
t.Fatalf("unexpected file id %q", request.Form.Get("file_id"))
}
writer.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(writer).Encode(map[string]any{
"ok": true,
"result": map[string]any{
"file_path": "photos/input.jpg",
"file_size": len(image),
},
})
case "/file/photos/input.jpg":
writer.Header().Set("Content-Type", "application/octet-stream")
_, _ = writer.Write(image)
default:
t.Fatalf("unexpected Telegram path %s", request.URL.Path)
}
}))
defer server.Close()
bot := &telegramBot{
apiBase: server.URL,
fileBase: server.URL + "/file",
client: &http.Client{Timeout: time.Second},
}
media, err := bot.fetchTelegramPhoto([]telegramPhotoSize{
{FileID: "small", Width: 100, Height: 100, FileSize: 50},
{FileID: "large", Width: 800, Height: 600, FileSize: int64(len(image))},
})
if err != nil {
t.Fatal(err)
}
defer os.Remove(media.Path)
actual, err := os.ReadFile(media.Path)
if err != nil {
t.Fatal(err)
}
if media.MIME != "image/jpeg" || string(actual) != string(image) {
t.Fatalf("unexpected staged image: mime=%q bytes=%x", media.MIME, actual)
}
info, err := os.Stat(media.Path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0600 {
t.Fatalf("staged image mode is %o", info.Mode().Perm())
}
}
func TestTelegramImageRequestStreamsMultimodalInput(t *testing.T) {
image := jpegInputFixture()
imagePath := filepath.Join(t.TempDir(), "input.jpg")
if err := os.WriteFile(imagePath, image, 0600); err != nil {
t.Fatal(err)
}
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.Header.Get("Authorization") != "Bearer relay-secret" {
t.Fatal("relay authentication was not set")
}
body, err := io.ReadAll(request.Body)
if err != nil {
t.Fatal(err)
}
var payload struct {
Input []struct {
Role string `json:"role"`
Content []struct {
Type string `json:"type"`
Text string `json:"text"`
ImageURL string `json:"image_url"`
} `json:"content"`
} `json:"input"`
Conversation string `json:"conversation"`
Store bool `json:"store"`
}
if err := json.Unmarshal(body, &payload); err != nil {
t.Fatalf("decode streamed request: %v\n%s", err, body)
}
if payload.Conversation != "telegram" || !payload.Store || len(payload.Input) != 1 || len(payload.Input[0].Content) != 2 {
t.Fatalf("unexpected image payload: %#v", payload)
}
if payload.Input[0].Role != "user" || payload.Input[0].Content[0].Text != "What is this?" || payload.Input[0].Content[1].Type != "input_image" {
t.Fatalf("unexpected image content: %#v", payload.Input[0])
}
encoded := strings.TrimPrefix(payload.Input[0].Content[1].ImageURL, "data:image/jpeg;base64,")
decoded, err := base64.StdEncoding.DecodeString(encoded)
if err != nil || string(decoded) != string(image) {
t.Fatalf("unexpected encoded image: err=%v bytes=%x", err, decoded)
}
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"It is a test image."}]}]}`))
}))
defer server.Close()
router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(int) string { return "" })
if err != nil {
t.Fatal(err)
}
router.backendAPIURL = func(int) string { return server.URL }
bot := newTelegramBot(telegramConfig{RelayKey: "relay-secret"}, router)
answer, err := bot.askTenantImage(0, "What is this?", telegramInputMedia{
MIME: "image/jpeg",
Path: imagePath,
}, 99)
if err != nil {
t.Fatal(err)
}
if answer != "It is a test image." {
t.Fatalf("unexpected answer %q", answer)
}
}
func TestTelegramInputRejectsUnsafePathsAndNonImages(t *testing.T) {
for _, value := range []string{"", "../secret", "/absolute/image.jpg", `photos\image.jpg`, "photos/../../secret"} {
if _, err := safeTelegramFilePath(value); err == nil {
t.Errorf("unsafe path accepted: %q", value)
}
}
if _, err := supportedTelegramImageType([]byte("not an image")); err == nil {
t.Fatal("non-image input was accepted")
}
if _, err := largestTelegramPhoto([]telegramPhotoSize{{
FileID: "too-large", Width: 1, Height: 1, FileSize: telegramInputImageLimit + 1,
}}); err == nil {
t.Fatal("oversized Telegram image was accepted")
}
}

View File

@ -24,6 +24,7 @@ const telegramPage = `<!doctype html>
<a class="hermes-back" href="/"> Back to Hermes</a>
<h1>Hermes on Telegram</h1>
<p>The operator configures one shared Hermes bot. Link your own Telegram account once so direct messages use this Keycloak account's isolated Hermes tenant.</p>
<p>After linking, you can send text or photos for analysis. Images Hermes creates or revises are returned directly in the same private chat.</p>
<p><strong>Account-link commands go only to the private chat with the Hermes botnot to Hermes WebUI and not to BotFather.</strong></p>
<p id="telegram-status">Checking Telegram</p>
<div class="hermes-link-actions">

View File

@ -107,6 +107,25 @@ def test_ui_manual_model_and_effort_are_forwarded_as_constraints(
)
def test_cli_exact_manual_route_remains_exact_across_boundaries(
tmp_path, monkeypatch
):
monkeypatch.setattr(router, "POLICY_PATH", tmp_path / "route-policy.json")
agent = _agent(
model="atlas/manual/claude/opus/xhigh",
_hermes_explicit_model_pick=True,
)
route, effort, source = router._boundary_selection(agent)
assert (route, effort, source) == (
"atlas/manual/claude/opus/xhigh",
"xhigh",
"ui-manual",
)
assert router._resolved_route(route, effort) == route
def test_manual_family_and_effort_resolve_to_one_exact_switchyard_route():
assert router._resolved_route("atlas/manual/codex/terra", "xhigh") == (
"atlas/manual/codex/terra/xhigh"

View File

@ -158,6 +158,9 @@ def test_gateway_image_honors_ui_model_and_caps_reasoning():
assert "pre_internal_route hook failed" in dockerfile
assert '"pre_subagent_route"' in dockerfile
assert "pre_subagent_route hook failed" in dockerfile
assert "agent._hermes_explicit_model_pick = bool" in dockerfile
assert "agent._hermes_oneshot = True" in dockerfile
assert 'getattr(parent_agent, "_hermes_oneshot", False)' in dockerfile
assert 'var != "CLAUDE_CODE_OAUTH_TOKEN"' in dockerfile
assert '"source": "managed_claude_code_subscription"' in dockerfile
assert '"name": "Claude Code subscription (Switchyard)"' in dockerfile