hermes(chat): separate local image routing
All checks were successful
Tests / Declarative: Post Actions passed: 247
All checks were successful
Tests / Declarative: Post Actions passed: 247
This commit is contained in:
parent
85e4102ba6
commit
021a6b4593
@ -116,12 +116,14 @@ data:
|
|||||||
private workspace as typed conversations. Whisper and speech synthesis are
|
private workspace as typed conversations. Whisper and speech synthesis are
|
||||||
transport services only; they do not select or replace the answering model.
|
transport services only; they do not select or replace the answering model.
|
||||||
|
|
||||||
When a user asks to create or edit an image, use the image generation tool.
|
When a user asks to create or edit an image, use an image generation tool.
|
||||||
Set its per-call `route` to `local` when the request says local, private,
|
Use `image_generate_local` when the request says local, private, on my
|
||||||
on my hardware, or FLUX; set it to `hosted` when the request says OpenAI,
|
hardware, or FLUX. Use `image_generate_hosted` when the request says
|
||||||
hosted, GPT Image, or highest hosted quality; otherwise set it to `auto`.
|
OpenAI, hosted, GPT Image, or highest hosted quality. Otherwise use the
|
||||||
AUTO tries GPT Image 2 High first and falls back to local FLUX when the
|
standard `image_generate` tool in AUTO mode; AUTO tries GPT Image 2 High
|
||||||
hosted route fails or refuses. Honor requested aspect ratio
|
first and falls back to local FLUX when the hosted route fails or refuses.
|
||||||
|
Never use standard `image_generate` or the hosted tool for an explicitly
|
||||||
|
local request. Honor requested aspect ratio
|
||||||
and use uploaded images as references when provided. Return the generated
|
and use uploaded images as references when provided. Return the generated
|
||||||
image inline so the WebUI offers its normal preview and download controls.
|
image inline so the WebUI offers its normal preview and download controls.
|
||||||
The local FLUX route is already provisioned: never tell a user to install
|
The local FLUX route is already provisioned: never tell a user to install
|
||||||
|
|||||||
@ -28,7 +28,7 @@ spec:
|
|||||||
ai.bstein.dev/role: isolated-user-chat
|
ai.bstein.dev/role: isolated-user-chat
|
||||||
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
||||||
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
||||||
ai.bstein.dev/config-rev: "20260811-route-aware-image-tool"
|
ai.bstein.dev/config-rev: "20260811-explicit-image-route-tools"
|
||||||
vault.hashicorp.com/agent-inject: "true"
|
vault.hashicorp.com/agent-inject: "true"
|
||||||
vault.hashicorp.com/role: hermes-chat
|
vault.hashicorp.com/role: hermes-chat
|
||||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||||
|
|||||||
@ -48,37 +48,13 @@ MODELS = {
|
|||||||
}
|
}
|
||||||
MAX_INPUT_BYTES = 25 << 20
|
MAX_INPUT_BYTES = 25 << 20
|
||||||
|
|
||||||
IMAGE_GENERATE_SCHEMA = {
|
IMAGE_GENERATE_PARAMETERS = {
|
||||||
"name": "image_generate",
|
|
||||||
"description": (
|
|
||||||
"Generate or edit an image with the private Atlas image studio. The "
|
|
||||||
"backend is selectable on every call: route='local' uses the local "
|
|
||||||
"FLUX model on Atlas hardware, route='hosted' uses GPT Image at its "
|
|
||||||
"highest quality, and route='auto' tries hosted first with local "
|
|
||||||
"fallback. You MUST honor the current user's requested route. Words "
|
|
||||||
"such as local, private, FLUX, or on my hardware mean route='local'; "
|
|
||||||
"OpenAI, GPT Image, or hosted mean route='hosted'. Use route='auto' "
|
|
||||||
"when the user does not express a preference. The backend is already "
|
|
||||||
"provisioned; never ask for an endpoint or tell the user to install "
|
|
||||||
"Diffusers or ComfyUI. Return the generated image path to the user "
|
|
||||||
"with the platform's normal file-delivery convention."
|
|
||||||
),
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"prompt": {
|
"prompt": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "Detailed description of the image or requested edit.",
|
"description": "Detailed description of the image or requested edit.",
|
||||||
},
|
},
|
||||||
"route": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["auto", "local", "hosted"],
|
|
||||||
"default": "auto",
|
|
||||||
"description": (
|
|
||||||
"Image compute route for this call. Honor an explicit user "
|
|
||||||
"preference; otherwise use auto."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"aspect_ratio": {
|
"aspect_ratio": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": ["landscape", "square", "portrait"],
|
"enum": ["landscape", "square", "portrait"],
|
||||||
@ -96,7 +72,30 @@ IMAGE_GENERATE_SCHEMA = {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
"required": ["prompt"],
|
"required": ["prompt"],
|
||||||
},
|
}
|
||||||
|
|
||||||
|
LOCAL_IMAGE_SCHEMA = {
|
||||||
|
"name": "image_generate_local",
|
||||||
|
"description": (
|
||||||
|
"Generate or edit an image only with the private local FLUX model on "
|
||||||
|
"Atlas hardware. Use this tool whenever the user says local, private, "
|
||||||
|
"FLUX, on my hardware, or otherwise explicitly rejects a hosted image "
|
||||||
|
"provider. Never substitute the generic image_generate tool for an "
|
||||||
|
"explicit local request. The backend is already provisioned; never ask "
|
||||||
|
"for an endpoint or tell the user to install Diffusers or ComfyUI."
|
||||||
|
),
|
||||||
|
"parameters": IMAGE_GENERATE_PARAMETERS,
|
||||||
|
}
|
||||||
|
|
||||||
|
HOSTED_IMAGE_SCHEMA = {
|
||||||
|
"name": "image_generate_hosted",
|
||||||
|
"description": (
|
||||||
|
"Generate or edit an image only with hosted OpenAI GPT Image at the "
|
||||||
|
"highest configured quality. Use this tool when the user explicitly "
|
||||||
|
"asks for OpenAI, GPT Image, or hosted image generation. Do not use it "
|
||||||
|
"when the user explicitly requests local or private generation."
|
||||||
|
),
|
||||||
|
"parameters": IMAGE_GENERATE_PARAMETERS,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -295,8 +294,8 @@ class AtlasBrokerImageProvider(ImageGenProvider):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _handle_image_generate(args: dict[str, Any], **_kwargs: Any) -> str:
|
def _handle_image_generate(args: dict[str, Any], route: str) -> str:
|
||||||
"""Dispatch one image request while honoring the per-call route."""
|
"""Dispatch one image request through a fixed route."""
|
||||||
prompt = str(args.get("prompt") or "").strip()
|
prompt = str(args.get("prompt") or "").strip()
|
||||||
if not prompt:
|
if not prompt:
|
||||||
return json.dumps(
|
return json.dumps(
|
||||||
@ -307,17 +306,9 @@ def _handle_image_generate(args: dict[str, Any], **_kwargs: Any) -> str:
|
|||||||
"error_type": "invalid_argument",
|
"error_type": "invalid_argument",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
route = str(args.get("route") or "auto").strip().lower()
|
|
||||||
model = ROUTE_MODELS.get(route)
|
model = ROUTE_MODELS.get(route)
|
||||||
if model is None:
|
if model is None: # pragma: no cover - routes are fixed by the wrappers
|
||||||
return json.dumps(
|
raise ValueError(f"unsupported image route: {route}")
|
||||||
{
|
|
||||||
"success": False,
|
|
||||||
"image": None,
|
|
||||||
"error": "route must be auto, local, or hosted",
|
|
||||||
"error_type": "invalid_argument",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
provider = AtlasBrokerImageProvider()
|
provider = AtlasBrokerImageProvider()
|
||||||
result = provider.generate(
|
result = provider.generate(
|
||||||
prompt,
|
prompt,
|
||||||
@ -331,23 +322,43 @@ def _handle_image_generate(args: dict[str, Any], **_kwargs: Any) -> str:
|
|||||||
return json.dumps(result, ensure_ascii=False)
|
return json.dumps(result, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_local_image(args: dict[str, Any], **_kwargs: Any) -> str:
|
||||||
|
"""Generate an image without allowing hosted provider substitution."""
|
||||||
|
return _handle_image_generate(args, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_hosted_image(args: dict[str, Any], **_kwargs: Any) -> str:
|
||||||
|
"""Generate an image through the highest-quality hosted route."""
|
||||||
|
return _handle_image_generate(args, "hosted")
|
||||||
|
|
||||||
|
|
||||||
def _image_tool_available() -> bool:
|
def _image_tool_available() -> bool:
|
||||||
"""Expose the tool when at least one broker route is healthy."""
|
"""Expose the tool when at least one broker route is healthy."""
|
||||||
return AtlasBrokerImageProvider().is_available()
|
return AtlasBrokerImageProvider().is_available()
|
||||||
|
|
||||||
|
|
||||||
def register(ctx: Any) -> None:
|
def register(ctx: Any) -> None:
|
||||||
"""Register the broker and its route-aware image tool contract."""
|
"""Register the broker and explicit local/hosted image tools."""
|
||||||
ctx.register_image_gen_provider(AtlasBrokerImageProvider())
|
ctx.register_image_gen_provider(AtlasBrokerImageProvider())
|
||||||
ctx.register_tool(
|
ctx.register_tool(
|
||||||
name="image_generate",
|
name="image_generate_local",
|
||||||
toolset="image_gen",
|
toolset="image_gen",
|
||||||
schema=IMAGE_GENERATE_SCHEMA,
|
schema=LOCAL_IMAGE_SCHEMA,
|
||||||
handler=_handle_image_generate,
|
handler=_handle_local_image,
|
||||||
check_fn=_image_tool_available,
|
check_fn=_image_tool_available,
|
||||||
requires_env=[],
|
requires_env=[],
|
||||||
is_async=False,
|
is_async=False,
|
||||||
description=IMAGE_GENERATE_SCHEMA["description"],
|
description=LOCAL_IMAGE_SCHEMA["description"],
|
||||||
|
emoji="🎨",
|
||||||
|
)
|
||||||
|
ctx.register_tool(
|
||||||
|
name="image_generate_hosted",
|
||||||
|
toolset="image_gen",
|
||||||
|
schema=HOSTED_IMAGE_SCHEMA,
|
||||||
|
handler=_handle_hosted_image,
|
||||||
|
check_fn=_image_tool_available,
|
||||||
|
requires_env=[],
|
||||||
|
is_async=False,
|
||||||
|
description=HOSTED_IMAGE_SCHEMA["description"],
|
||||||
emoji="🎨",
|
emoji="🎨",
|
||||||
override=True,
|
|
||||||
)
|
)
|
||||||
|
|||||||
@ -319,7 +319,8 @@ def test_chat_image_generation_uses_private_owner_broker():
|
|||||||
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
||||||
assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"]
|
assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"]
|
||||||
assert "local FLUX waits" in configmap["data"]["SOUL.md"]
|
assert "local FLUX waits" in configmap["data"]["SOUL.md"]
|
||||||
assert "Set its per-call `route` to `local`" in configmap["data"]["SOUL.md"]
|
assert "Use `image_generate_local`" in configmap["data"]["SOUL.md"]
|
||||||
|
assert "Use `image_generate_hosted`" in configmap["data"]["SOUL.md"]
|
||||||
assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"]
|
assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"]
|
||||||
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
||||||
assert config["image_gen"] == {
|
assert config["image_gen"] == {
|
||||||
@ -341,11 +342,11 @@ def test_chat_image_generation_uses_private_owner_broker():
|
|||||||
assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.")
|
assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.")
|
||||||
|
|
||||||
plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text()
|
plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text()
|
||||||
assert '"enum": ["auto", "local", "hosted"]' in plugin
|
|
||||||
assert '"local": "flux-2-klein-4b-local"' in plugin
|
assert '"local": "flux-2-klein-4b-local"' in plugin
|
||||||
assert '"hosted": "gpt-image-2-high"' in plugin
|
assert '"hosted": "gpt-image-2-high"' in plugin
|
||||||
assert 'name="image_generate"' in plugin
|
assert 'name="image_generate_local"' in plugin
|
||||||
assert "override=True" in plugin
|
assert 'name="image_generate_hosted"' in plugin
|
||||||
|
assert "override=True" not in plugin
|
||||||
|
|
||||||
agent = _documents(HERMES / "agent-deployment.yaml")[0]
|
agent = _documents(HERMES / "agent-deployment.yaml")[0]
|
||||||
containers = agent["spec"]["template"]["spec"]["containers"]
|
containers = agent["spec"]["template"]["spec"]["containers"]
|
||||||
@ -439,7 +440,7 @@ def test_chat_reasoning_uses_private_owner_codex_broker():
|
|||||||
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
||||||
assert statefulset["spec"]["template"]["metadata"]["annotations"][
|
assert statefulset["spec"]["template"]["metadata"]["annotations"][
|
||||||
"ai.bstein.dev/config-rev"
|
"ai.bstein.dev/config-rev"
|
||||||
] == "20260811-route-aware-image-tool"
|
] == "20260811-explicit-image-route-tools"
|
||||||
hermes = next(
|
hermes = next(
|
||||||
item
|
item
|
||||||
for item in statefulset["spec"]["template"]["spec"]["containers"]
|
for item in statefulset["spec"]["template"]["spec"]["containers"]
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user