hermes: enforce AUTO routing quality floors
All checks were successful
Tests / Declarative: Post Actions passed: 244
All checks were successful
Tests / Declarative: Post Actions passed: 244
This commit is contained in:
parent
d81778013f
commit
a50fbb63ab
@ -195,19 +195,39 @@ data:
|
|||||||
tool_calling = true
|
tool_calling = true
|
||||||
reasoning = true
|
reasoning = true
|
||||||
prompt = """
|
prompt = """
|
||||||
You are the routing authority for a private family assistant. Choose exactly
|
You are the deterministic routing authority for one private family-assistant
|
||||||
one configured target for this model-call boundary. Mildly favor fast and
|
model-call boundary. Mildly favor speed, but apply these rules in order.
|
||||||
economical answers, but never trade away correctness for difficult,
|
|
||||||
ambiguous, safety-sensitive, tool-heavy, or consequential work. Use local
|
1. Set a mandatory minimum effort floor from the whole current objective and
|
||||||
Qwen only for simple low-risk conversation, formatting, or continuity.
|
recent context: xhigh for critical security work, risky production
|
||||||
Prefer Codex for implementation, debugging, tests, and direct repository
|
migrations, destructive or data-loss risk, or consequential independent
|
||||||
work. Prefer Claude for architecture, ambiguity, long-context synthesis,
|
final/release review; high for difficult debugging, consequential
|
||||||
risk analysis, and independent review. Interpret requests such as "answer
|
implementation, adversarial review, major architectural tradeoffs, or
|
||||||
quickly", "think hard", or "use Claude" semantically. An explicit user
|
uncertain production work; medium for ordinary implementation, tool use,
|
||||||
provider, model, or depth instruction wins unless it would undercut a clear
|
analysis, or bounded architecture; low only for simple conversation,
|
||||||
safety floor. Every request is a fresh boundary; classify the actual current
|
formatting, lookup, or mechanical reversible work. Never choose below the
|
||||||
objective and recent context, including referential instructions like
|
floor and never exceed xhigh.
|
||||||
"continue" or "do it". Never select effort above xhigh.
|
|
||||||
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
|
request to raise capability by at least one tier when the safety floor is
|
||||||
|
lower. Treat "answer quickly" and equivalent intent as permission to choose
|
||||||
|
the fastest target at the safety floor, never below it. An explicit
|
||||||
|
available provider or model wins at the safety floor.
|
||||||
|
|
||||||
|
3. Use local Qwen only at low or medium for low-risk conversation,
|
||||||
|
formatting, lookup, and continuity. Prefer Codex for implementation,
|
||||||
|
debugging, tests, commands, and repository work. Prefer Claude for
|
||||||
|
architecture, ambiguity, synthesis, risk analysis, adversarial analysis,
|
||||||
|
and independent review. Anthropic means Claude; OpenAI means Codex. Never
|
||||||
|
select a provider stated to be unavailable, failed, exhausted, rate-limited,
|
||||||
|
or out of capacity; use the other provider at the same floor.
|
||||||
|
|
||||||
|
4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium,
|
||||||
|
high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low,
|
||||||
|
medium=claude_sonnet_medium, high=claude_sonnet_high,
|
||||||
|
xhigh=claude_opus_xhigh; local low=local_qwen_low and local
|
||||||
|
medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue"
|
||||||
|
or "do it" from recent context.
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","local_qwen_low","local_qwen_medium","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","local_qwen_low","local_qwen_medium","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
@ -231,18 +251,40 @@ data:
|
|||||||
tool_calling = true
|
tool_calling = true
|
||||||
reasoning = true
|
reasoning = true
|
||||||
prompt = """
|
prompt = """
|
||||||
You are the routing authority for a private general assistant. Choose
|
You are the deterministic routing authority for one private general-assistant
|
||||||
exactly one configured target for this model-call boundary. Balance latency,
|
model-call boundary. Balance speed and intelligence, then apply these rules
|
||||||
cost, and intelligence while giving difficult, uncertain, tool-heavy, or
|
in order.
|
||||||
consequential work enough capability. Use local Qwen only for simple
|
|
||||||
low-risk conversation, formatting, or continuity. Prefer Codex for
|
1. Set a mandatory minimum effort floor from the whole current objective and
|
||||||
implementation, debugging, tests, and direct repository work. Prefer Claude
|
recent context: xhigh for critical security work, risky production
|
||||||
for architecture, ambiguity, long-context synthesis, risk analysis, and
|
migrations, destructive or data-loss risk, or consequential independent
|
||||||
independent review. Interpret speed/depth and provider requests
|
final/release review; high for difficult debugging, consequential
|
||||||
semantically; explicit user intent wins unless it undercuts a clear safety
|
implementation, adversarial review, major architectural tradeoffs, or
|
||||||
floor. Classify the actual current objective and recent context, including
|
uncertain production work; medium for ordinary implementation, tool use,
|
||||||
referential instructions such as "continue" or "do it". Never select effort
|
analysis, or bounded architecture; low only for simple conversation,
|
||||||
above xhigh.
|
formatting, lookup, or mechanical reversible work. Never choose below the
|
||||||
|
floor and never exceed xhigh.
|
||||||
|
|
||||||
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
|
request to raise capability by at least one tier when the safety floor is
|
||||||
|
lower. Treat "answer quickly" and equivalent intent as permission to choose
|
||||||
|
the fastest target at the safety floor, never below it. An explicit
|
||||||
|
available provider or model wins at the safety floor.
|
||||||
|
|
||||||
|
3. Use local Qwen only at low or medium for low-risk conversation,
|
||||||
|
formatting, lookup, and continuity. Prefer Codex for implementation,
|
||||||
|
debugging, tests, commands, and repository work. Prefer Claude for
|
||||||
|
architecture, ambiguity, synthesis, risk analysis, adversarial analysis,
|
||||||
|
and independent review. Anthropic means Claude; OpenAI means Codex. Never
|
||||||
|
select a provider stated to be unavailable, failed, exhausted, rate-limited,
|
||||||
|
or out of capacity; use the other provider at the same floor.
|
||||||
|
|
||||||
|
4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium,
|
||||||
|
high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low,
|
||||||
|
medium=claude_sonnet_medium, high=claude_sonnet_high,
|
||||||
|
xhigh=claude_opus_xhigh; local low=local_qwen_low and local
|
||||||
|
medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue"
|
||||||
|
or "do it" from recent context.
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","local_qwen_low","local_qwen_medium","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_terra_medium","claude_sonnet_medium","codex_luna_low","claude_haiku_low","local_qwen_low","local_qwen_medium","codex_terra_low","codex_terra_high","codex_sol_medium","codex_sol_high","codex_sol_xhigh","claude_sonnet_high","claude_opus_high","claude_opus_xhigh"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
@ -266,17 +308,40 @@ data:
|
|||||||
tool_calling = true
|
tool_calling = true
|
||||||
reasoning = true
|
reasoning = true
|
||||||
prompt = """
|
prompt = """
|
||||||
You are the routing authority for operations triage. Choose exactly one
|
You are the deterministic routing authority for one operations-triage
|
||||||
configured target for this model-call boundary. Favor evidence, diagnosis,
|
model-call boundary. Favor evidence and intelligence, then apply these rules
|
||||||
uncertainty handling, and correctness over marginal latency savings. Cheap
|
in order.
|
||||||
routes are appropriate only for genuinely mechanical low-risk steps. Prefer
|
|
||||||
Codex for implementation, debugging, tests, and repository work; prefer
|
1. Set a mandatory minimum effort floor from the whole current objective,
|
||||||
Claude for diagnosis, architecture, ambiguity, synthesis, risk analysis,
|
recent context, and tool evidence: xhigh for critical security incidents,
|
||||||
and independent review. Interpret requests for speed, depth, or a provider
|
risky production migrations, destructive or data-loss risk, or
|
||||||
semantically. Explicit user intent wins unless it violates a safety floor.
|
consequential independent final/release review; high for difficult
|
||||||
Re-evaluate every current boundary using recent tool evidence and resolve
|
diagnosis or debugging, consequential implementation, adversarial review,
|
||||||
referential instructions such as "continue" or "do it" from that context.
|
major architectural tradeoffs, or uncertain production work; medium for
|
||||||
Never select effort above xhigh.
|
ordinary diagnosis, implementation, tool use, analysis, or bounded
|
||||||
|
architecture; low only for simple notification summaries, lookup, or
|
||||||
|
mechanical reversible work. Never choose below the floor or above xhigh.
|
||||||
|
|
||||||
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
|
request to raise capability by at least one tier when the safety floor is
|
||||||
|
lower. Treat "answer quickly" and equivalent intent as permission to choose
|
||||||
|
the fastest target at the safety floor, never below it. An explicit
|
||||||
|
available provider or model wins at the safety floor.
|
||||||
|
|
||||||
|
3. Use local Qwen only at low or medium for low-risk summaries, formatting,
|
||||||
|
lookup, and continuity. Prefer Codex for implementation, debugging, tests,
|
||||||
|
commands, and repository work. Prefer Claude for diagnosis, architecture,
|
||||||
|
ambiguity, synthesis, risk analysis, adversarial analysis, and independent
|
||||||
|
review. Anthropic means Claude; OpenAI means Codex. Never select a provider
|
||||||
|
stated to be unavailable, failed, exhausted, rate-limited, or out of
|
||||||
|
capacity; use the other provider at the same floor.
|
||||||
|
|
||||||
|
4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium,
|
||||||
|
high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low,
|
||||||
|
medium=claude_sonnet_medium, high=claude_sonnet_high,
|
||||||
|
xhigh=claude_opus_xhigh; local low=local_qwen_low and local
|
||||||
|
medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue"
|
||||||
|
or "do it" from recent context.
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_opus_xhigh","codex_luna_low","claude_haiku_low","local_qwen_medium","local_qwen_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["claude_sonnet_high","codex_sol_high","codex_terra_medium","claude_sonnet_medium","codex_terra_high","codex_sol_medium","claude_opus_high","codex_sol_xhigh","claude_opus_xhigh","codex_luna_low","claude_haiku_low","local_qwen_medium","local_qwen_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
@ -300,19 +365,40 @@ data:
|
|||||||
tool_calling = true
|
tool_calling = true
|
||||||
reasoning = true
|
reasoning = true
|
||||||
prompt = """
|
prompt = """
|
||||||
You are the routing authority for an owner-only engineering agent. Choose
|
You are the deterministic routing authority for one owner-only engineering
|
||||||
exactly one configured target for this model-call boundary. Strongly favor
|
model-call boundary. Strongly favor intelligence, verification, and task
|
||||||
correctness, verification, and task completion; do not waste premium
|
completion, then apply these rules in order.
|
||||||
capacity on truly mechanical steps. Prefer Codex for implementation,
|
|
||||||
debugging, tests, and direct repository changes. Prefer Claude for
|
1. Set a mandatory minimum effort floor from the whole current objective,
|
||||||
architecture, difficult ambiguity, long-context synthesis, risk analysis,
|
recent context, and tool evidence: xhigh for critical security work, risky
|
||||||
and independent review. Production changes, security, migrations, data-loss
|
production migrations, destructive or data-loss risk, or consequential
|
||||||
risk, destructive work, and critical final review require high or xhigh.
|
independent final/release review; high for difficult debugging,
|
||||||
Interpret user requests for speed, deeper thought, or a specific provider
|
consequential implementation, adversarial review, major architectural
|
||||||
semantically. Explicit user intent wins unless it undercuts the safety
|
tradeoffs, or uncertain production work; medium for ordinary
|
||||||
floor. Re-evaluate every current boundary from the objective and recent tool
|
implementation, tests, tool use, analysis, or bounded architecture; low
|
||||||
evidence, including referential instructions such as "continue" or "do
|
only for lookup or truly mechanical reversible work. Never choose below the
|
||||||
it". Never select effort above xhigh.
|
floor or above xhigh.
|
||||||
|
|
||||||
|
2. Treat "think hard", "deeply", "carefully", and equivalent intent as a
|
||||||
|
request to raise capability by at least one tier when the safety floor is
|
||||||
|
lower. Treat "answer quickly" and equivalent intent as permission to choose
|
||||||
|
the fastest target at the safety floor, never below it. An explicit
|
||||||
|
available provider or model wins at the safety floor.
|
||||||
|
|
||||||
|
3. Use local Qwen only at low or medium for low-risk lookup, formatting, and
|
||||||
|
continuity. Prefer Codex for implementation, debugging, tests, commands,
|
||||||
|
and repository changes. Prefer Claude for architecture, ambiguity,
|
||||||
|
synthesis, risk analysis, adversarial analysis, and independent review.
|
||||||
|
Anthropic means Claude; OpenAI means Codex. Never select a provider stated
|
||||||
|
to be unavailable, failed, exhausted, rate-limited, or out of capacity; use
|
||||||
|
the other provider at the same floor.
|
||||||
|
|
||||||
|
4. Map exactly: Codex low=codex_luna_low, medium=codex_terra_medium,
|
||||||
|
high=codex_sol_high, xhigh=codex_sol_xhigh; Claude low=claude_haiku_low,
|
||||||
|
medium=claude_sonnet_medium, high=claude_sonnet_high,
|
||||||
|
xhigh=claude_opus_xhigh; local low=local_qwen_low and local
|
||||||
|
medium=local_qwen_medium. Re-evaluate every boundary and resolve "continue"
|
||||||
|
or "do it" from recent context.
|
||||||
"""
|
"""
|
||||||
response_schema = '''
|
response_schema = '''
|
||||||
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","codex_terra_medium","claude_sonnet_medium","codex_sol_medium","codex_luna_low","claude_haiku_low","local_qwen_medium","local_qwen_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
{"type":"object","properties":{"decision":{"type":"object","properties":{"target":{"type":"string","enum":["codex_sol_high","claude_opus_high","claude_sonnet_high","codex_terra_high","codex_sol_xhigh","claude_opus_xhigh","codex_terra_medium","claude_sonnet_medium","codex_sol_medium","codex_luna_low","claude_haiku_low","local_qwen_medium","local_qwen_low","codex_terra_low"]}},"required":["target"],"additionalProperties":false}},"required":["decision"],"additionalProperties":false}
|
||||||
|
|||||||
@ -10,7 +10,10 @@ spec:
|
|||||||
replicas: 1
|
replicas: 1
|
||||||
revisionHistoryLimit: 2
|
revisionHistoryLimit: 2
|
||||||
strategy:
|
strategy:
|
||||||
type: Recreate
|
type: RollingUpdate
|
||||||
|
rollingUpdate:
|
||||||
|
maxSurge: 1
|
||||||
|
maxUnavailable: 0
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: hermes-switchyard
|
app: hermes-switchyard
|
||||||
@ -19,7 +22,7 @@ spec:
|
|||||||
labels:
|
labels:
|
||||||
app: hermes-switchyard
|
app: hermes-switchyard
|
||||||
annotations:
|
annotations:
|
||||||
ai.bstein.dev/config-rev: "20260811-switchyard-authority-v9"
|
ai.bstein.dev/config-rev: "20260811-switchyard-authority-v10"
|
||||||
prometheus.io/scrape: "true"
|
prometheus.io/scrape: "true"
|
||||||
prometheus.io/port: "9005"
|
prometheus.io/port: "9005"
|
||||||
prometheus.io/path: /metrics
|
prometheus.io/path: /metrics
|
||||||
|
|||||||
@ -773,6 +773,10 @@ def test_local_flux_runtime_and_gpu_handoff_are_flux_managed():
|
|||||||
assert "Claude xhigh=worker_claude_opus_xhigh" in switchyard
|
assert "Claude xhigh=worker_claude_opus_xhigh" in switchyard
|
||||||
assert "Anthropic and Claude name the same provider" in switchyard
|
assert "Anthropic and Claude name the same provider" in switchyard
|
||||||
assert "OpenAI and Codex name the same provider" in switchyard
|
assert "OpenAI and Codex name the same provider" in switchyard
|
||||||
|
assert switchyard.count("Codex low=codex_luna_low") == 4
|
||||||
|
assert switchyard.count("Claude low=claude_haiku_low") == 4
|
||||||
|
assert switchyard.count('Treat "think hard"') == 4
|
||||||
|
assert switchyard.count("Never choose below the") >= 5
|
||||||
model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][
|
model_gate = _documents(HERMES / "model-gate-configmap.yaml")[0]["data"][
|
||||||
"model_gate.py"
|
"model_gate.py"
|
||||||
]
|
]
|
||||||
|
|||||||
@ -887,6 +887,11 @@ def test_switchyard_active_state_uses_a_relocatable_rwx_claim():
|
|||||||
assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"]
|
assert active_claim["spec"]["accessModes"] == ["ReadWriteMany"]
|
||||||
|
|
||||||
deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text())
|
deployment = yaml.safe_load((HERMES / "switchyard-deployment.yaml").read_text())
|
||||||
|
strategy = deployment["spec"]["strategy"]
|
||||||
|
assert strategy == {
|
||||||
|
"type": "RollingUpdate",
|
||||||
|
"rollingUpdate": {"maxSurge": 1, "maxUnavailable": 0},
|
||||||
|
}
|
||||||
pod = deployment["spec"]["template"]["spec"]
|
pod = deployment["spec"]["template"]["spec"]
|
||||||
state = next(item for item in pod["volumes"] if item["name"] == "state")
|
state = next(item for item in pod["volumes"] if item["name"] == "state")
|
||||||
assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][
|
assert state["persistentVolumeClaim"]["claimName"] == active_claim["metadata"][
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user