hermes(chat): route image generation per request
All checks were successful
Tests / Declarative: Post Actions passed: 247
All checks were successful
Tests / Declarative: Post Actions passed: 247
This commit is contained in:
parent
0ded081f7c
commit
85e4102ba6
@ -117,15 +117,16 @@ data:
|
|||||||
transport services only; they do not select or replace the answering model.
|
transport services only; they do not select or replace the answering model.
|
||||||
|
|
||||||
When a user asks to create or edit an image, use the image generation tool.
|
When a user asks to create or edit an image, use the image generation tool.
|
||||||
Use `atlas-image-auto-high` unless the user selects a route: requests that
|
Set its per-call `route` to `local` when the request says local, private,
|
||||||
say local, private, on my hardware, or FLUX use `flux-2-klein-4b-local`;
|
on my hardware, or FLUX; set it to `hosted` when the request says OpenAI,
|
||||||
requests that say OpenAI, hosted, GPT Image, or highest hosted quality use
|
hosted, GPT Image, or highest hosted quality; otherwise set it to `auto`.
|
||||||
`gpt-image-2-high`. AUTO tries GPT Image 2 High first and falls back to local
|
AUTO tries GPT Image 2 High first and falls back to local FLUX when the
|
||||||
FLUX when the hosted route fails or refuses. Honor requested aspect ratio
|
hosted route fails or refuses. Honor requested aspect ratio
|
||||||
and use uploaded images as references when provided. Return the generated
|
and use uploaded images as references when provided. Return the generated
|
||||||
image inline so the WebUI offers its normal preview and download controls.
|
image inline so the WebUI offers its normal preview and download controls.
|
||||||
The local FLUX route is already provisioned: never tell a user to install
|
The local FLUX route is already provisioned: never tell a user to install
|
||||||
Diffusers, download a checkpoint, or write a Python generation script. If
|
Diffusers, download a checkpoint, write a Python generation script, find a
|
||||||
|
ComfyUI endpoint, or provide any service address. If
|
||||||
local rendering is unavailable because the shared desktop/Wolf lane owns
|
local rendering is unavailable because the shared desktop/Wolf lane owns
|
||||||
titan-24, say so plainly and offer AUTO or OpenAI. The desktop and Wolf may
|
titan-24, say so plainly and offer AUTO or OpenAI. The desktop and Wolf may
|
||||||
share their interactive reservation; local FLUX waits until that reservation
|
share their interactive reservation; local FLUX waits until that reservation
|
||||||
|
|||||||
@ -28,7 +28,7 @@ spec:
|
|||||||
ai.bstein.dev/role: isolated-user-chat
|
ai.bstein.dev/role: isolated-user-chat
|
||||||
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject
|
||||||
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides
|
||||||
ai.bstein.dev/config-rev: "20260811-desktop-image-lane"
|
ai.bstein.dev/config-rev: "20260811-route-aware-image-tool"
|
||||||
vault.hashicorp.com/agent-inject: "true"
|
vault.hashicorp.com/agent-inject: "true"
|
||||||
vault.hashicorp.com/role: hermes-chat
|
vault.hashicorp.com/role: hermes-chat
|
||||||
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens
|
||||||
|
|||||||
@ -26,6 +26,11 @@ BROKER_URL = os.environ.get(
|
|||||||
"http://hermes-image-broker.hermes.svc.cluster.local:9002",
|
"http://hermes-image-broker.hermes.svc.cluster.local:9002",
|
||||||
).rstrip("/")
|
).rstrip("/")
|
||||||
DEFAULT_MODEL = "atlas-image-auto-high"
|
DEFAULT_MODEL = "atlas-image-auto-high"
|
||||||
|
ROUTE_MODELS = {
|
||||||
|
"auto": "atlas-image-auto-high",
|
||||||
|
"hosted": "gpt-image-2-high",
|
||||||
|
"local": "flux-2-klein-4b-local",
|
||||||
|
}
|
||||||
MODELS = {
|
MODELS = {
|
||||||
"atlas-image-auto-high": (
|
"atlas-image-auto-high": (
|
||||||
"Auto (GPT Image quality, local fallback)",
|
"Auto (GPT Image quality, local fallback)",
|
||||||
@ -43,6 +48,57 @@ MODELS = {
|
|||||||
}
|
}
|
||||||
MAX_INPUT_BYTES = 25 << 20
|
MAX_INPUT_BYTES = 25 << 20
|
||||||
|
|
||||||
|
IMAGE_GENERATE_SCHEMA = {
|
||||||
|
"name": "image_generate",
|
||||||
|
"description": (
|
||||||
|
"Generate or edit an image with the private Atlas image studio. The "
|
||||||
|
"backend is selectable on every call: route='local' uses the local "
|
||||||
|
"FLUX model on Atlas hardware, route='hosted' uses GPT Image at its "
|
||||||
|
"highest quality, and route='auto' tries hosted first with local "
|
||||||
|
"fallback. You MUST honor the current user's requested route. Words "
|
||||||
|
"such as local, private, FLUX, or on my hardware mean route='local'; "
|
||||||
|
"OpenAI, GPT Image, or hosted mean route='hosted'. Use route='auto' "
|
||||||
|
"when the user does not express a preference. The backend is already "
|
||||||
|
"provisioned; never ask for an endpoint or tell the user to install "
|
||||||
|
"Diffusers or ComfyUI. Return the generated image path to the user "
|
||||||
|
"with the platform's normal file-delivery convention."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"prompt": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Detailed description of the image or requested edit.",
|
||||||
|
},
|
||||||
|
"route": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["auto", "local", "hosted"],
|
||||||
|
"default": "auto",
|
||||||
|
"description": (
|
||||||
|
"Image compute route for this call. Honor an explicit user "
|
||||||
|
"preference; otherwise use auto."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"aspect_ratio": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["landscape", "square", "portrait"],
|
||||||
|
"default": DEFAULT_ASPECT_RATIO,
|
||||||
|
"description": "Requested output aspect ratio.",
|
||||||
|
},
|
||||||
|
"image_url": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Optional uploaded image URL or private local path to edit.",
|
||||||
|
},
|
||||||
|
"reference_image_urls": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "string"},
|
||||||
|
"description": "Optional reference images for the generation or edit.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": ["prompt"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _broker_key() -> str:
|
def _broker_key() -> str:
|
||||||
"""Load the internal relay key from process env or the private .env."""
|
"""Load the internal relay key from process env or the private .env."""
|
||||||
@ -59,7 +115,9 @@ def _broker_key() -> str:
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
def _request(path: str, payload: dict[str, Any] | None, timeout: float) -> dict[str, Any]:
|
def _request(
|
||||||
|
path: str, payload: dict[str, Any] | None, timeout: float
|
||||||
|
) -> dict[str, Any]:
|
||||||
"""Call the internal broker with a bounded authenticated JSON request."""
|
"""Call the internal broker with a bounded authenticated JSON request."""
|
||||||
key = _broker_key()
|
key = _broker_key()
|
||||||
if not key:
|
if not key:
|
||||||
@ -144,7 +202,9 @@ class AtlasBrokerImageProvider(ImageGenProvider):
|
|||||||
"id": model,
|
"id": model,
|
||||||
"display": display,
|
"display": display,
|
||||||
"speed": "~2–15min" if local else "~30s–3min",
|
"speed": "~2–15min" if local else "~30s–3min",
|
||||||
"strengths": strengths[0] if strengths else "OpenAI GPT Image 2 quality",
|
"strengths": strengths[0]
|
||||||
|
if strengths
|
||||||
|
else "OpenAI GPT Image 2 quality",
|
||||||
"price": "local compute" if local else "included account capacity",
|
"price": "local compute" if local else "included account capacity",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@ -181,7 +241,9 @@ class AtlasBrokerImageProvider(ImageGenProvider):
|
|||||||
primary = _local_image_data_url(image_url) if image_url else None
|
primary = _local_image_data_url(image_url) if image_url else None
|
||||||
references = [
|
references = [
|
||||||
_local_image_data_url(item)
|
_local_image_data_url(item)
|
||||||
for item in (normalize_reference_images(reference_image_urls) or [])[:16]
|
for item in (normalize_reference_images(reference_image_urls) or [])[
|
||||||
|
:16
|
||||||
|
]
|
||||||
]
|
]
|
||||||
response = _request(
|
response = _request(
|
||||||
"/v1/images/generations",
|
"/v1/images/generations",
|
||||||
@ -203,9 +265,7 @@ class AtlasBrokerImageProvider(ImageGenProvider):
|
|||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
aspect_ratio=aspect,
|
aspect_ratio=aspect,
|
||||||
)
|
)
|
||||||
image = save_b64_image(
|
image = save_b64_image(str(response["image_b64"]), prefix=f"atlas_{model}")
|
||||||
str(response["image_b64"]), prefix=f"atlas_{model}"
|
|
||||||
)
|
|
||||||
return success_response(
|
return success_response(
|
||||||
image=str(image),
|
image=str(image),
|
||||||
model=str(response.get("model") or model),
|
model=str(response.get("model") or model),
|
||||||
@ -235,6 +295,59 @@ class AtlasBrokerImageProvider(ImageGenProvider):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_image_generate(args: dict[str, Any], **_kwargs: Any) -> str:
|
||||||
|
"""Dispatch one image request while honoring the per-call route."""
|
||||||
|
prompt = str(args.get("prompt") or "").strip()
|
||||||
|
if not prompt:
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"success": False,
|
||||||
|
"image": None,
|
||||||
|
"error": "prompt is required for image generation",
|
||||||
|
"error_type": "invalid_argument",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
route = str(args.get("route") or "auto").strip().lower()
|
||||||
|
model = ROUTE_MODELS.get(route)
|
||||||
|
if model is None:
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"success": False,
|
||||||
|
"image": None,
|
||||||
|
"error": "route must be auto, local, or hosted",
|
||||||
|
"error_type": "invalid_argument",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
provider = AtlasBrokerImageProvider()
|
||||||
|
result = provider.generate(
|
||||||
|
prompt,
|
||||||
|
str(args.get("aspect_ratio") or DEFAULT_ASPECT_RATIO),
|
||||||
|
model=model,
|
||||||
|
image_url=args.get("image_url"),
|
||||||
|
reference_image_urls=args.get("reference_image_urls"),
|
||||||
|
)
|
||||||
|
if isinstance(result, dict):
|
||||||
|
result.setdefault("requested_route", route)
|
||||||
|
return json.dumps(result, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _image_tool_available() -> bool:
|
||||||
|
"""Expose the tool when at least one broker route is healthy."""
|
||||||
|
return AtlasBrokerImageProvider().is_available()
|
||||||
|
|
||||||
|
|
||||||
def register(ctx: Any) -> None:
|
def register(ctx: Any) -> None:
|
||||||
"""Register the broker as a normal Hermes image backend."""
|
"""Register the broker and its route-aware image tool contract."""
|
||||||
ctx.register_image_gen_provider(AtlasBrokerImageProvider())
|
ctx.register_image_gen_provider(AtlasBrokerImageProvider())
|
||||||
|
ctx.register_tool(
|
||||||
|
name="image_generate",
|
||||||
|
toolset="image_gen",
|
||||||
|
schema=IMAGE_GENERATE_SCHEMA,
|
||||||
|
handler=_handle_image_generate,
|
||||||
|
check_fn=_image_tool_available,
|
||||||
|
requires_env=[],
|
||||||
|
is_async=False,
|
||||||
|
description=IMAGE_GENERATE_SCHEMA["description"],
|
||||||
|
emoji="🎨",
|
||||||
|
override=True,
|
||||||
|
)
|
||||||
|
|||||||
@ -319,6 +319,8 @@ def test_chat_image_generation_uses_private_owner_broker():
|
|||||||
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
configmap = _documents(HERMES / "chat-configmap.yaml")[0]
|
||||||
assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"]
|
assert "shared desktop/Wolf lane" in configmap["data"]["SOUL.md"]
|
||||||
assert "local FLUX waits" in configmap["data"]["SOUL.md"]
|
assert "local FLUX waits" in configmap["data"]["SOUL.md"]
|
||||||
|
assert "Set its per-call `route` to `local`" in configmap["data"]["SOUL.md"]
|
||||||
|
assert "ComfyUI endpoint" in configmap["data"]["SOUL.md"]
|
||||||
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
config = yaml.safe_load(configmap["data"]["config.yaml"])
|
||||||
assert config["image_gen"] == {
|
assert config["image_gen"] == {
|
||||||
"provider": "atlas-broker",
|
"provider": "atlas-broker",
|
||||||
@ -338,6 +340,13 @@ def test_chat_image_generation_uses_private_owner_broker():
|
|||||||
env = {item["name"]: item["value"] for item in hermes["env"]}
|
env = {item["name"]: item["value"] for item in hermes["env"]}
|
||||||
assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.")
|
assert env["HERMES_IMAGE_BROKER_URL"].startswith("http://hermes-image-broker.")
|
||||||
|
|
||||||
|
plugin = (HERMES / "plugins" / "image-gen-broker" / "__init__.py").read_text()
|
||||||
|
assert '"enum": ["auto", "local", "hosted"]' in plugin
|
||||||
|
assert '"local": "flux-2-klein-4b-local"' in plugin
|
||||||
|
assert '"hosted": "gpt-image-2-high"' in plugin
|
||||||
|
assert 'name="image_generate"' in plugin
|
||||||
|
assert "override=True" in plugin
|
||||||
|
|
||||||
agent = _documents(HERMES / "agent-deployment.yaml")[0]
|
agent = _documents(HERMES / "agent-deployment.yaml")[0]
|
||||||
containers = agent["spec"]["template"]["spec"]["containers"]
|
containers = agent["spec"]["template"]["spec"]["containers"]
|
||||||
broker = next(item for item in containers if item["name"] == "image-broker")
|
broker = next(item for item in containers if item["name"] == "image-broker")
|
||||||
@ -430,7 +439,7 @@ def test_chat_reasoning_uses_private_owner_codex_broker():
|
|||||||
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
statefulset = _documents(HERMES / "chat-statefulset.yaml")[0]
|
||||||
assert statefulset["spec"]["template"]["metadata"]["annotations"][
|
assert statefulset["spec"]["template"]["metadata"]["annotations"][
|
||||||
"ai.bstein.dev/config-rev"
|
"ai.bstein.dev/config-rev"
|
||||||
] == "20260811-desktop-image-lane"
|
] == "20260811-route-aware-image-tool"
|
||||||
hermes = next(
|
hermes = next(
|
||||||
item
|
item
|
||||||
for item in statefulset["spec"]["template"]["spec"]["containers"]
|
for item in statefulset["spec"]["template"]["spec"]["containers"]
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user