hermes(chat): separate local image routing
All checks were successful
Tests / Declarative: Post Actions passed: 247
All checks were successful
Tests / Declarative: Post Actions passed: 247
This commit is contained in:
parent
85e4102ba6
commit
021a6b4593
@ -116,12 +116,14 @@ data:
|
||||
private workspace as typed conversations. Whisper and speech synthesis are
|
||||
transport services only; they do not select or replace the answering model.
|
||||
|
||||
When a user asks to create or edit an image, use the image generation tool.
|
||||
Set its per-call `route` to `local` when the request says local, private,
|
||||
on my hardware, or FLUX; set it to `hosted` when the request says OpenAI,
|
||||
hosted, GPT Image, or highest hosted quality; otherwise set it to `auto`.
|
||||
AUTO tries GPT Image 2 High first and falls back to local FLUX when the
|
||||
hosted route fails or refuses. Honor requested aspect ratio
|
||||
When a user asks to create or edit an image, use an image generation tool.
|
||||
Use `image_generate_local` when the request says local, private, on my
|
||||
hardware, or FLUX. Use `image_generate_hosted` when the request says
|
||||
OpenAI, hosted, GPT Image, or highest hosted quality. Otherwise use the
|
||||
standard `image_generate` tool in AUTO mode; AUTO tries GPT Image 2 High
|
||||
first and falls back to local FLUX when the hosted route fails or refuses.
|
||||
Never use standard `image_generate` or the hosted tool for an explicitly
|
||||
local request. Honor requested aspect ratio
|
||||
and use uploaded images as references when provided. Return the generated
|
||||
image inline so the WebUI offers its normal preview and download controls.
|
||||
The local FLUX route is already provisioned: never tell a user to install
|
||||
|
||||
@ -28,7 +28,7 @@ spec:
|
||||
ai.bstein.dev/role: isolated-user-chat
|
||||
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
||||
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
||||
ai.bstein.dev/config-rev: "20260811-route-aware-image-tool"
|
||||
ai.bstein.dev/config-rev: "20260811-explicit-image-route-tools"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-chat
|
||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||
|
||||
@ -48,55 +48,54 @@ MODELS = {
|
||||
}
|
||||
MAX_INPUT_BYTES = 25 << 20
|
||||
|
||||
IMAGE_GENERATE_SCHEMA = {
|
||||
"name": "image_generate",
|
||||
"description": (
|
||||
"Generate or edit an image with the private Atlas image studio. The "
|
||||
"backend is selectable on every call: route='local' uses the local "
|
||||
"FLUX model on Atlas hardware, route='hosted' uses GPT Image at its "
|
||||
"highest quality, and route='auto' tries hosted first with local "
|
||||
"fallback. You MUST honor the current user's requested route. Words "
|
||||
"such as local, private, FLUX, or on my hardware mean route='local'; "
|
||||
"OpenAI, GPT Image, or hosted mean route='hosted'. Use route='auto' "
|
||||
"when the user does not express a preference. The backend is already "
|
||||
"provisioned; never ask for an endpoint or tell the user to install "
|
||||
"Diffusers or ComfyUI. Return the generated image path to the user "
|
||||
"with the platform's normal file-delivery convention."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "Detailed description of the image or requested edit.",
|
||||
},
|
||||
"route": {
|
||||
"type": "string",
|
||||
"enum": ["auto", "local", "hosted"],
|
||||
"default": "auto",
|
||||
"description": (
|
||||
"Image compute route for this call. Honor an explicit user "
|
||||
"preference; otherwise use auto."
|
||||
),
|
||||
},
|
||||
"aspect_ratio": {
|
||||
"type": "string",
|
||||
"enum": ["landscape", "square", "portrait"],
|
||||
"default": DEFAULT_ASPECT_RATIO,
|
||||
"description": "Requested output aspect ratio.",
|
||||
},
|
||||
"image_url": {
|
||||
"type": "string",
|
||||
"description": "Optional uploaded image URL or private local path to edit.",
|
||||
},
|
||||
"reference_image_urls": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional reference images for the generation or edit.",
|
||||
},
|
||||
IMAGE_GENERATE_PARAMETERS = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "Detailed description of the image or requested edit.",
|
||||
},
|
||||
"aspect_ratio": {
|
||||
"type": "string",
|
||||
"enum": ["landscape", "square", "portrait"],
|
||||
"default": DEFAULT_ASPECT_RATIO,
|
||||
"description": "Requested output aspect ratio.",
|
||||
},
|
||||
"image_url": {
|
||||
"type": "string",
|
||||
"description": "Optional uploaded image URL or private local path to edit.",
|
||||
},
|
||||
"reference_image_urls": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional reference images for the generation or edit.",
|
||||
},
|
||||
"required": ["prompt"],
|
||||
},
|
||||
"required": ["prompt"],
|
||||
}
|
||||
|
||||
LOCAL_IMAGE_SCHEMA = {
|
||||
"name": "image_generate_local",
|
||||
"description": (
|
||||
"Generate or edit an image only with the private local FLUX model on "
|
||||
"Atlas hardware. Use this tool whenever the user says local, private, "
|
||||
"FLUX, on my hardware, or otherwise explicitly rejects a hosted image "
|
||||
"provider. Never substitute the generic image_generate tool for an "
|
||||
"explicit local request. The backend is already provisioned; never ask "
|
||||
"for an endpoint or tell the user to install Diffusers or ComfyUI."
|
||||
),
|
||||
"parameters": IMAGE_GENERATE_PARAMETERS,
|
||||
}
|
||||
|
||||
HOSTED_IMAGE_SCHEMA = {
|
||||
"name": "image_generate_hosted",
|
||||
"description": (
|
||||
"Generate or edit an image only with hosted OpenAI GPT Image at the "
|
||||
"highest configured quality. Use this tool when the user explicitly "
|
||||
"asks for OpenAI, GPT Image, or hosted image generation. Do not use it "
|
||||
"when the user explicitly requests local or private generation."
|
||||
),
|
||||
"parameters": IMAGE_GENERATE_PARAMETERS,
|
||||
}
|
||||
|
||||
|
||||
@ -295,8 +294,8 @@ class AtlasBrokerImageProvider(ImageGenProvider):
|
||||
)
|
||||
|
||||
|
||||
def _handle_image_generate(args: dict[str, Any], **_kwargs: Any) -> str:
|
||||
"""Dispatch one image request while honoring the per-call route."""
|
||||
def _handle_image_generate(args: dict[str, Any], route: str) -> str:
|
||||
"""Dispatch one image request through a fixed route."""
|
||||
prompt = str(args.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
return json.dumps(
|
||||
@ -307,17 +306,9 @@ def _handle_image_generate(args: dict[str, Any], **_kwargs: Any) -> str:
|
||||
"error_type": "invalid_argument",
|
||||
}
|
||||
)
|
||||
route = str(args.get("route") or "auto").strip().lower()
|
||||
model = ROUTE_MODELS.get(route)
|
||||
if model is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"image": None,
|
||||
"error": "route must be auto, local, or hosted",
|
||||
"error_type": "invalid_argument",
|
||||
}
|
||||
)
|
||||
if model is None: # pragma: no cover - routes are fixed by the wrappers
|
||||
raise ValueError(f"unsupported image route: {route}")
|
||||
provider = AtlasBrokerImageProvider()
|
||||
result = provider.generate(
|
||||
prompt,
|
||||
@ -331,23 +322,43 @@ def _handle_image_generate(args: dict[str, Any], **_kwargs: Any) -> str:
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
|
||||
def _handle_local_image(args: dict[str, Any], **_kwargs: Any) -> str:
|
||||
"""Generate an image without allowing hosted provider substitution."""
|
||||
return _handle_image_generate(args, "local")
|
||||
|
||||
|
||||
def _handle_hosted_image(args: dict[str, Any], **_kwargs: Any) -> str:
|
||||
"""Generate an image through the highest-quality hosted route."""
|
||||
return _handle_image_generate(args, "hosted")
|
||||
|
||||
|
||||
def _image_tool_available() -> bool:
|
||||
"""Expose the tool when at least one broker route is healthy."""
|
||||
return AtlasBrokerImageProvider().is_available()
|
||||
|
||||
|
||||
def register(ctx: Any) -> None:
|
||||
"""Register the broker and its route-aware image tool contract."""
|
||||
"""Register the broker and explicit local/hosted image tools."""
|
||||
ctx.register_image_gen_provider(AtlasBrokerImageProvider())
|
||||
ctx.register_tool(
|
||||
name="image_generate",
|
||||
name="image_generate_local",
|
||||
toolset="image_gen",
|
||||
schema=IMAGE_GENERATE_SCHEMA,
|
||||
handler=_handle_image_generate,
|
||||
schema=LOCAL_IMAGE_SCHEMA,
|
||||
handler=_handle_local_image,
|
||||
check_fn=_image_tool_available,
|
||||
requires_env=[],
|
||||
is_async=False,
|
||||
description=IMAGE_GENERATE_SCHEMA["description"],
|
||||
description=LOCAL_IMAGE_SCHEMA["description"],
|
||||
emoji="🎨",
|
||||
)
|
||||
ctx.register_tool(
|
||||
name="image_generate_hosted",
|
||||
toolset="image_gen",
|
||||
schema=HOSTED_IMAGE_SCHEMA,
|
||||
handler=_handle_hosted_image,
|
||||
check_fn=_image_tool_available,
|
||||
requires_env=[],
|
||||
is_async=False,
|
||||
description=HOSTED_IMAGE_SCHEMA["description"],
|
||||
emoji="🎨",
|
||||
override=True,
|
||||
)
|
||||
|
||||
@ -319,7 +319,8 @@ def test_chat_image_generation_uses_private_owner_broker():
|
||||
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
||||
assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"]
|
||||
assert "local FLUX waits" in configmap["data"]["SOUL.md"]
|
||||
assert "Set its per-call `route` to `local`" in configmap["data"]["SOUL.md"]
|
||||
assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"]
|
||||
assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"]
|
||||
assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"]
|
||||
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
||||
assert config["image_gen"] == {
|
||||
@ -341,11 +342,11 @@ def test_chat_image_generation_uses_private_owner_broker():
|
||||
assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.")
|
||||
|
||||
plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text()
|
||||
assert '"enum": ["auto", "local", "hosted"]' in plugin
|
||||
assert '"local": "flux-2-klein-4b-local"' in plugin
|
||||
assert '"hosted": "gpt-image-2-high"' in plugin
|
||||
assert 'name="image_generate"' in plugin
|
||||
assert "override=True" in plugin
|
||||
assert 'name="image_generate_local"' in plugin
|
||||
assert 'name="image_generate_hosted"' in plugin
|
||||
assert "override=True" not in plugin
|
||||
|
||||
agent = _documents(HERMES / "agent-deployment.yaml")[0]
|
||||
containers = agent["spec"]["template"]["spec"]["containers"]
|
||||
@ -439,7 +440,7 @@ def test_chat_reasoning_uses_private_owner_codex_broker():
|
||||
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
||||
assert statefulset["spec"]["template"]["metadata"]["annotations"][
|
||||
"ai.bstein.dev/config-rev"
|
||||
] == "20260811-route-aware-image-tool"
|
||||
] == "20260811-explicit-image-route-tools"
|
||||
hermes = next(
|
||||
item
|
||||
for item in statefulset["spec"]["template"]["spec"]["containers"]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user