From 8c73906e6eadb0adb91b58fa293bba8fb937fd07 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sat, 15 Aug 2026 21:48:30 -0300 Subject: [PATCH] hermes: preserve exact routes and accept Telegram images --- dockerfiles/Dockerfile.hermes-agent | 44 ++++ .../hermes/plugins/auto-router/__init__.py | 19 +- services/hermes/router/telegram.go | 53 ++++- services/hermes/router/telegram_input.go | 195 ++++++++++++++++++ services/hermes/router/telegram_test.go | 145 +++++++++++++ services/hermes/router/web.go | 1 + testing/tests/test_hermes_auto_router.py | 19 ++ testing/tests/test_hermes_chat_quality.py | 3 + 8 files changed, 468 insertions(+), 11 deletions(-) create mode 100644 services/hermes/router/telegram_input.go diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index 6ece284a..a5f25fd6 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -449,6 +449,49 @@ if delegate.count(delegate_before) != 1: f"found {delegate.count(delegate_before)}" ) delegate_path.write_text(delegate.replace(delegate_before, delegate_after, 1)) + +delegate = delegate_path.read_text() +background_before = ''' is_subagent = getattr(parent_agent, "_delegate_depth", 0) > 0 + return not is_subagent +''' +background_after = ''' is_subagent = getattr(parent_agent, "_delegate_depth", 0) > 0 + # A one-shot process has no durable event loop to receive a detached + # child's result after its parent exits. Keep delegation synchronous so + # the child verdict is returned to—and persisted by—the parent turn. + if bool(getattr(parent_agent, "_hermes_oneshot", False)): + return False + return not is_subagent +''' +if delegate.count(background_before) != 1: + raise SystemExit( + "Hermes oneshot delegation context changed: expected 1, " + f"found {delegate.count(background_before)}" + ) +delegate_path.write_text(delegate.replace(background_before, background_after, 1)) + +oneshot_path = Path("/opt/hermes/hermes_cli/oneshot.py") +oneshot = oneshot_path.read_text() +oneshot_before = ''' # Belt-and-braces: make sure AIAgent doesn't invoke any streaming + # display callbacks that would bypass our stdout capture. + agent.suppress_status_output = True +''' +oneshot_after = ''' # Preserve a caller's explicit --model route across every plugin routing + # boundary. Config/env defaults remain automatic policy inputs. + agent._hermes_explicit_model_pick = bool((model or "").strip()) + # Detached delegation cannot deliver back into a process that exits after + # this turn; delegate_tool uses this marker to keep children synchronous. + agent._hermes_oneshot = True + + # Belt-and-braces: make sure AIAgent doesn't invoke any streaming + # display callbacks that would bypass our stdout capture. + agent.suppress_status_output = True +''' +if oneshot.count(oneshot_before) != 1: + raise SystemExit( + "Hermes oneshot explicit-model context changed: expected 1, " + f"found {oneshot.count(oneshot_before)}" + ) +oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1)) PY # Hermes WebUI sends its model/provider/reasoning selection on /v1/runs. @@ -1236,6 +1279,7 @@ RUN cd /opt/hermes/web \ /opt/hermes/gateway/platforms/api_server.py \ /opt/hermes/agent/turn_context.py \ /opt/hermes/agent/conversation_loop.py \ + /opt/hermes/hermes_cli/oneshot.py \ /opt/hermes/tools/delegate_tool.py \ /opt/hermes/tools/web_tools.py \ /opt/hermes/tools/python_sandbox_tool.py \ diff --git a/services/hermes/plugins/auto-router/__init__.py b/services/hermes/plugins/auto-router/__init__.py index f962e876..3a879534 100644 --- a/services/hermes/plugins/auto-router/__init__.py +++ b/services/hermes/plugins/auto-router/__init__.py @@ -51,6 +51,14 @@ MANUAL_ROUTES = frozenset( ) ALL_ROUTES = AUTO_ROUTES | MANUAL_ROUTES EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) +ROUTABLE_EFFORTS = frozenset({"low", "medium", "high", "xhigh"}) +EXACT_MANUAL_ROUTES = frozenset( + f"{route}/{effort}" + for route in MANUAL_ROUTES + if not route.endswith("/local/qwen-14b") + for effort in ROUTABLE_EFFORTS +) +ALL_REQUEST_ROUTES = ALL_ROUTES | EXACT_MANUAL_ROUTES PROVIDER_DEFAULT = { "codex": "atlas/manual/codex/terra", "claude": "atlas/manual/claude/sonnet", @@ -136,11 +144,11 @@ def _normalise_manual_route(provider: str, model: str = "") -> str: def _explicit_ui_route(agent: Any) -> str: - """Return a route selected in the WebUI model picker for this request.""" + """Return an exact route selected by the WebUI or command-line caller.""" if not bool(getattr(agent, "_hermes_explicit_model_pick", False)): return "" route = str(getattr(agent, "model", "") or "").strip() - return route if route in ALL_ROUTES else "" + return route if route in ALL_REQUEST_ROUTES else "" def _explicit_ui_effort(agent: Any) -> str: @@ -158,6 +166,9 @@ def _boundary_selection(agent: Any) -> tuple[str, str, str]: ui_effort = _explicit_ui_effort(agent) if ui_route: mode = "auto" if ui_route in AUTO_ROUTES else "manual" + if ui_route in EXACT_MANUAL_ROUTES: + effort = ui_route.rsplit("/", 1)[-1] + return ui_route, effort, f"ui-{mode}" return ui_route, ui_effort, f"ui-{mode}" if policy["mode"] == "manual": @@ -179,9 +190,11 @@ def _boundary_selection(agent: Any) -> tuple[str, str, str]: def _resolved_route(route: str, effort: str) -> str: """Bind a manual family and UI effort to an exact Switchyard route.""" + if route in EXACT_MANUAL_ROUTES: + return route if route not in MANUAL_ROUTES or route.endswith("/local/qwen-14b"): return route - provider_effort = effort if effort in {"low", "medium", "high", "xhigh"} else "low" + provider_effort = effort if effort in ROUTABLE_EFFORTS else "low" return f"{route}/{provider_effort}" diff --git a/services/hermes/router/telegram.go b/services/hermes/router/telegram.go index e84aa0d6..a9dca006 100644 --- a/services/hermes/router/telegram.go +++ b/services/hermes/router/telegram.go @@ -27,6 +27,7 @@ type telegramBot struct { config telegramConfig router *tenantRouter apiBase string + fileBase string client *http.Client mediaClient *http.Client agentClient *http.Client @@ -45,10 +46,19 @@ type telegramUpdate struct { } type telegramMessage struct { - MessageID int64 `json:"message_id"` - From *telegramUser `json:"from"` - Chat telegramChat `json:"chat"` - Text string `json:"text"` + MessageID int64 `json:"message_id"` + From *telegramUser `json:"from"` + Chat telegramChat `json:"chat"` + Text string `json:"text"` + Caption string `json:"caption"` + Photo []telegramPhotoSize `json:"photo"` +} + +type telegramPhotoSize struct { + FileID string `json:"file_id"` + FileSize int64 `json:"file_size"` + Width int `json:"width"` + Height int `json:"height"` } type telegramUser struct { @@ -89,6 +99,7 @@ func newTelegramBot(config telegramConfig, router *tenantRouter) *telegramBot { config: config, router: router, apiBase: "https://api.telegram.org/bot" + config.BotToken, + fileBase: "https://api.telegram.org/file/bot" + config.BotToken, client: &http.Client{Timeout: 70 * time.Second}, mediaClient: newTenantMediaClient(), agentClient: &http.Client{Timeout: 15 * time.Minute}, @@ -265,7 +276,7 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) { return } if command == "help" { - _ = bot.sendText(message.Chat.ID, "Send any text to chat with Hermes. Use /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.") + _ = bot.sendText(message.Chat.ID, "Send text or a photo to chat with Hermes. Use /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.") return } slot, linked := bot.router.telegramSlot(userID) @@ -274,8 +285,15 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) { return } text := strings.TrimSpace(message.Text) + hasPhoto := len(message.Photo) > 0 + if hasPhoto { + text = strings.TrimSpace(message.Caption) + if text == "" { + text = "Describe this image and answer any question it implies." + } + } if text == "" { - _ = bot.sendText(message.Chat.ID, "Text messages are supported now; attachment support will be added separately.") + _ = bot.sendText(message.Chat.ID, "Hermes currently accepts text and image attachments. Please send other file types through Hermes WebUI.") return } if utf8.RuneCountInString(text) > 12000 { @@ -286,7 +304,19 @@ func (bot *telegramBot) handleUpdate(update telegramUpdate) { lock.Lock() defer lock.Unlock() _ = bot.sendAction(message.Chat.ID, "typing") - reply, err := bot.askTenant(slot, text, update.UpdateID) + var reply string + var err error + if hasPhoto { + media, fetchErr := bot.fetchTelegramPhoto(message.Photo) + if fetchErr != nil { + _ = bot.sendText(message.Chat.ID, "Hermes could not safely read that image. Please try a smaller JPEG, PNG, or WebP image.") + return + } + defer os.Remove(media.Path) + reply, err = bot.askTenantImage(slot, text, media, update.UpdateID) + } else { + reply, err = bot.askTenant(slot, text, update.UpdateID) + } if err != nil { _ = bot.sendText(message.Chat.ID, "Hermes could not answer right now. Please try again shortly.") return @@ -303,13 +333,20 @@ func (bot *telegramBot) askTenant(slot int, text string, updateID int64) (string "conversation": "telegram", "store": true, }) + return bot.askTenantRequest(slot, bytes.NewReader(payload), updateID) +} + +func (bot *telegramBot) askTenantRequest(slot int, bodyReader io.Reader, updateID int64) (string, error) { + if bot.router.backendAPIURL == nil { + return "", errors.New("tenant API unavailable") + } ctx, cancel := context.WithTimeout(context.Background(), 14*time.Minute) defer cancel() request, err := http.NewRequestWithContext( ctx, http.MethodPost, bot.router.backendAPIURL(slot)+"/v1/responses", - bytes.NewReader(payload), + bodyReader, ) if err != nil { return "", err diff --git a/services/hermes/router/telegram_input.go b/services/hermes/router/telegram_input.go new file mode 100644 index 00000000..ff9c1f79 --- /dev/null +++ b/services/hermes/router/telegram_input.go @@ -0,0 +1,195 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" +) + +const telegramInputImageLimit = 20 << 20 + +type telegramInputMedia struct { + MIME string + Path string +} + +func largestTelegramPhoto(photos []telegramPhotoSize) (telegramPhotoSize, error) { + var selected telegramPhotoSize + for _, photo := range photos { + if strings.TrimSpace(photo.FileID) == "" { + continue + } + if selected.FileID == "" || int64(photo.Width)*int64(photo.Height) > int64(selected.Width)*int64(selected.Height) { + selected = photo + } + } + if selected.FileID == "" { + return telegramPhotoSize{}, errors.New("Telegram photo has no file identifier") + } + if selected.FileSize > telegramInputImageLimit { + return telegramPhotoSize{}, errors.New("Telegram photo exceeds input limit") + } + return selected, nil +} + +func safeTelegramFilePath(raw string) (string, error) { + value := strings.TrimSpace(raw) + if value == "" || strings.Contains(value, `\`) || strings.ContainsRune(value, '\x00') { + return "", errors.New("invalid Telegram file path") + } + cleaned := path.Clean(value) + if cleaned == "." || path.IsAbs(cleaned) || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", errors.New("invalid Telegram file path") + } + return cleaned, nil +} + +func escapedTelegramFilePath(value string) string { + components := strings.Split(value, "/") + for index := range components { + components[index] = url.PathEscape(components[index]) + } + return strings.Join(components, "/") +} + +func supportedTelegramImageType(header []byte) (string, error) { + detected := http.DetectContentType(header) + switch detected { + case "image/jpeg", "image/png", "image/webp", "image/gif": + return detected, nil + default: + return "", errors.New("unsupported Telegram image type") + } +} + +func (bot *telegramBot) fetchTelegramPhoto(photos []telegramPhotoSize) (telegramInputMedia, error) { + selected, err := largestTelegramPhoto(photos) + if err != nil { + return telegramInputMedia{}, err + } + ctx, cancel := context.WithTimeout(context.Background(), 65*time.Second) + defer cancel() + var remote struct { + FilePath string `json:"file_path"` + FileSize int64 `json:"file_size"` + } + if err := bot.call(ctx, "getFile", url.Values{"file_id": {selected.FileID}}, &remote); err != nil { + return telegramInputMedia{}, errors.New("Telegram file lookup failed") + } + if remote.FileSize > telegramInputImageLimit { + return telegramInputMedia{}, errors.New("Telegram photo exceeds input limit") + } + remotePath, err := safeTelegramFilePath(remote.FilePath) + if err != nil { + return telegramInputMedia{}, err + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(bot.fileBase, "/")+"/"+escapedTelegramFilePath(remotePath), nil) + if err != nil { + return telegramInputMedia{}, errors.New("create Telegram file request") + } + response, err := bot.client.Do(request) + if err != nil { + return telegramInputMedia{}, errors.New("Telegram file download failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return telegramInputMedia{}, errors.New("Telegram file download rejected") + } + if response.ContentLength > telegramInputImageLimit { + return telegramInputMedia{}, errors.New("Telegram photo exceeds input limit") + } + temporary, err := os.CreateTemp("", "hermes-telegram-input-*") + if err != nil { + return telegramInputMedia{}, errors.New("stage Telegram image") + } + temporaryPath := temporary.Name() + keep := false + defer func() { + _ = temporary.Close() + if !keep { + _ = os.Remove(temporaryPath) + } + }() + written, err := io.Copy(temporary, io.LimitReader(response.Body, telegramInputImageLimit+1)) + if err != nil { + return telegramInputMedia{}, errors.New("read Telegram image") + } + if written > telegramInputImageLimit { + return telegramInputMedia{}, errors.New("Telegram photo exceeds input limit") + } + if _, err := temporary.Seek(0, io.SeekStart); err != nil { + return telegramInputMedia{}, errors.New("inspect Telegram image") + } + header := make([]byte, 512) + read, readErr := temporary.Read(header) + if readErr != nil && readErr != io.EOF { + return telegramInputMedia{}, errors.New("inspect Telegram image") + } + mimeType, err := supportedTelegramImageType(header[:read]) + if err != nil { + return telegramInputMedia{}, err + } + if err := temporary.Close(); err != nil { + return telegramInputMedia{}, errors.New("stage Telegram image") + } + keep = true + return telegramInputMedia{MIME: mimeType, Path: temporaryPath}, nil +} + +func streamTelegramImageRequest(writer io.Writer, prompt string, media telegramInputMedia) error { + promptJSON, err := json.Marshal(prompt) + if err != nil { + return err + } + if _, err := fmt.Fprintf(writer, `{"input":[{"role":"user","content":[{"type":"input_text","text":%s},{"type":"input_image","image_url":"data:%s;base64,`, promptJSON, media.MIME); err != nil { + return err + } + file, err := os.Open(media.Path) + if err != nil { + return err + } + encoder := base64.NewEncoder(base64.StdEncoding, writer) + _, copyErr := io.Copy(encoder, file) + closeEncoderErr := encoder.Close() + closeFileErr := file.Close() + if copyErr != nil { + return copyErr + } + if closeEncoderErr != nil { + return closeEncoderErr + } + if closeFileErr != nil { + return closeFileErr + } + _, err = io.WriteString(writer, `"}]}],"conversation":"telegram","store":true}`) + return err +} + +func (bot *telegramBot) askTenantImage(slot int, prompt string, media telegramInputMedia, updateID int64) (string, error) { + reader, writer := io.Pipe() + writeDone := make(chan error, 1) + go func() { + err := streamTelegramImageRequest(writer, prompt, media) + _ = writer.CloseWithError(err) + writeDone <- err + }() + answer, requestErr := bot.askTenantRequest(slot, reader, updateID) + _ = reader.Close() + writeErr := <-writeDone + if requestErr != nil { + return "", requestErr + } + if writeErr != nil { + return "", errors.New("encode Telegram image") + } + return answer, nil +} diff --git a/services/hermes/router/telegram_test.go b/services/hermes/router/telegram_test.go index 81cbaf4c..73b852d6 100644 --- a/services/hermes/router/telegram_test.go +++ b/services/hermes/router/telegram_test.go @@ -1,7 +1,9 @@ package main import ( + "encoding/base64" "encoding/json" + "io" "net/http" "net/http/httptest" "os" @@ -91,3 +93,146 @@ func TestTelegramCallIncludesAPIDescription(t *testing.T) { t.Fatalf("expected Telegram API description, got %v", err) } } + +func jpegInputFixture() []byte { + return append( + []byte{0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 'J', 'F', 'I', 'F', 0x00}, + []byte("telegram-input-image")..., + ) +} + +func TestFetchTelegramPhotoStagesLargestImageWithoutTrustingPath(t *testing.T) { + image := jpegInputFixture() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/getFile": + if err := request.ParseForm(); err != nil { + t.Fatal(err) + } + if request.Form.Get("file_id") != "large" { + t.Fatalf("unexpected file id %q", request.Form.Get("file_id")) + } + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{ + "ok": true, + "result": map[string]any{ + "file_path": "photos/input.jpg", + "file_size": len(image), + }, + }) + case "/file/photos/input.jpg": + writer.Header().Set("Content-Type", "application/octet-stream") + _, _ = writer.Write(image) + default: + t.Fatalf("unexpected Telegram path %s", request.URL.Path) + } + })) + defer server.Close() + + bot := &telegramBot{ + apiBase: server.URL, + fileBase: server.URL + "/file", + client: &http.Client{Timeout: time.Second}, + } + media, err := bot.fetchTelegramPhoto([]telegramPhotoSize{ + {FileID: "small", Width: 100, Height: 100, FileSize: 50}, + {FileID: "large", Width: 800, Height: 600, FileSize: int64(len(image))}, + }) + if err != nil { + t.Fatal(err) + } + defer os.Remove(media.Path) + actual, err := os.ReadFile(media.Path) + if err != nil { + t.Fatal(err) + } + if media.MIME != "image/jpeg" || string(actual) != string(image) { + t.Fatalf("unexpected staged image: mime=%q bytes=%x", media.MIME, actual) + } + info, err := os.Stat(media.Path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Fatalf("staged image mode is %o", info.Mode().Perm()) + } +} + +func TestTelegramImageRequestStreamsMultimodalInput(t *testing.T) { + image := jpegInputFixture() + imagePath := filepath.Join(t.TempDir(), "input.jpg") + if err := os.WriteFile(imagePath, image, 0600); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Header.Get("Authorization") != "Bearer relay-secret" { + t.Fatal("relay authentication was not set") + } + body, err := io.ReadAll(request.Body) + if err != nil { + t.Fatal(err) + } + var payload struct { + Input []struct { + Role string `json:"role"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + ImageURL string `json:"image_url"` + } `json:"content"` + } `json:"input"` + Conversation string `json:"conversation"` + Store bool `json:"store"` + } + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode streamed request: %v\n%s", err, body) + } + if payload.Conversation != "telegram" || !payload.Store || len(payload.Input) != 1 || len(payload.Input[0].Content) != 2 { + t.Fatalf("unexpected image payload: %#v", payload) + } + if payload.Input[0].Role != "user" || payload.Input[0].Content[0].Text != "What is this?" || payload.Input[0].Content[1].Type != "input_image" { + t.Fatalf("unexpected image content: %#v", payload.Input[0]) + } + encoded := strings.TrimPrefix(payload.Input[0].Content[1].ImageURL, "data:image/jpeg;base64,") + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || string(decoded) != string(image) { + t.Fatalf("unexpected encoded image: err=%v bytes=%x", err, decoded) + } + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"It is a test image."}]}]}`)) + })) + defer server.Close() + + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(int) string { return "" }) + if err != nil { + t.Fatal(err) + } + router.backendAPIURL = func(int) string { return server.URL } + bot := newTelegramBot(telegramConfig{RelayKey: "relay-secret"}, router) + answer, err := bot.askTenantImage(0, "What is this?", telegramInputMedia{ + MIME: "image/jpeg", + Path: imagePath, + }, 99) + if err != nil { + t.Fatal(err) + } + if answer != "It is a test image." { + t.Fatalf("unexpected answer %q", answer) + } +} + +func TestTelegramInputRejectsUnsafePathsAndNonImages(t *testing.T) { + for _, value := range []string{"", "../secret", "/absolute/image.jpg", `photos\image.jpg`, "photos/../../secret"} { + if _, err := safeTelegramFilePath(value); err == nil { + t.Errorf("unsafe path accepted: %q", value) + } + } + if _, err := supportedTelegramImageType([]byte("not an image")); err == nil { + t.Fatal("non-image input was accepted") + } + if _, err := largestTelegramPhoto([]telegramPhotoSize{{ + FileID: "too-large", Width: 1, Height: 1, FileSize: telegramInputImageLimit + 1, + }}); err == nil { + t.Fatal("oversized Telegram image was accepted") + } +} diff --git a/services/hermes/router/web.go b/services/hermes/router/web.go index b93beae6..0ea02a20 100644 --- a/services/hermes/router/web.go +++ b/services/hermes/router/web.go @@ -24,6 +24,7 @@ const telegramPage = ` ← Back to Hermes

Hermes on Telegram

The operator configures one shared Hermes bot. Link your own Telegram account once so direct messages use this Keycloak account's isolated Hermes tenant.

+

After linking, you can send text or photos for analysis. Images Hermes creates or revises are returned directly in the same private chat.

Account-link commands go only to the private chat with the Hermes bot—not to Hermes WebUI and not to BotFather.

Checking Telegram…