gpu(titan-24): add Hermes local-first fallback handoff

This commit is contained in:
jenkins 2026-08-01 22:38:07 -03:00
parent d58aa8e7e8
commit be91902a01
17 changed files with 546 additions and 21 deletions

View File

@ -18,6 +18,10 @@ spec:
wait: true
timeout: 45m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-model-gate
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-ollama

View File

@ -68,7 +68,7 @@ spec:
args:
- >-
. /vault/secrets/portal-env.sh
&& exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 600 app:app
&& exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 1200 app:app
env:
- name: AI_CHAT_API
value: http://ollama.ai.svc.cluster.local:11434
@ -119,6 +119,8 @@ spec:
value: http://ariadne.maintenance.svc.cluster.local
- name: ARIADNE_TIMEOUT_SEC
value: "10"
- name: ARIADNE_GAME_MODE_TIMEOUT_SEC
value: "900"
- name: ACCOUNT_ALLOWED_GROUPS
value: ""
- name: HTTP_CHECK_TIMEOUT_SEC

View File

@ -0,0 +1,30 @@
# services/hermes/ariadne-handoff-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ariadne-gpu-handoff
namespace: hermes
rules:
- apiGroups: ["coordination.k8s.io"]
resources:
- leases
resourceNames:
- titan-24-gpu-owner
verbs:
- get
- patch
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ariadne-gpu-handoff
namespace: hermes
subjects:
- kind: ServiceAccount
name: ariadne
namespace: maintenance
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ariadne-gpu-handoff

View File

@ -10,11 +10,24 @@ data:
config.yaml: |
model:
provider: custom
default: qwen2.5:7b-instruct-q4_0
model: qwen2.5:7b-instruct-q4_0
base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1
default: gpt-oss:20b
model: gpt-oss:20b
context_length: 64000
base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
api_key: ollama
fallback_providers:
- provider: openai-codex
model: gpt-5.4
agent:
api_max_retries: 1
skills:
creation_nudge_interval: 15
external_dirs:
- /opt/data/workspace/skills
terminal:
backend: local
cwd: /opt/data/workspace

View File

@ -20,10 +20,10 @@ spec:
labels:
app: hermes
annotations:
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
ai.bstein.dev/model: gpt-oss:20b with openai-codex fallback
ai.bstein.dev/role: testing-triage
ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
ai.bstein.dev/config-rev: "20260721-hermes-rollout-deadline"
ai.bstein.dev/config-rev: "20260801-gpu-owner-fallback"
spec:
serviceAccountName: hermes-triage
automountServiceAccountToken: true
@ -190,6 +190,9 @@ spec:
mountPath: /opt/data
- name: tools
mountPath: /opt/data/home/.local/bin
- name: triage-skill
mountPath: /opt/data/workspace/skills/triage-titan-test-failures
readOnly: true
readinessProbe:
httpGet:
path: /api/status
@ -220,3 +223,11 @@ spec:
name: hermes-config
- name: tools
emptyDir: {}
- name: triage-skill
configMap:
name: hermes-triage-skill
items:
- key: SKILL.md
path: SKILL.md
- key: openai.yaml
path: agents/openai.yaml

View File

@ -7,8 +7,23 @@ resources:
- configmap.yaml
- rbac.yaml
- pvc.yaml
- model-gate-rbac.yaml
- ariadne-handoff-rbac.yaml
- model-gate-state.yaml
- model-gate-configmap.yaml
- model-gate-deployment.yaml
- networkpolicy.yaml
- ollama-deployment.yaml
- deployment.yaml
- service.yaml
- agent-certificate.yaml
- agent-ingress.yaml
configMapGenerator:
- name: hermes-triage-skill
namespace: hermes
files:
- SKILL.md=skills/triage-titan-test-failures/SKILL.md
- openai.yaml=skills/triage-titan-test-failures/agents/openai.yaml
options:
disableNameSuffixHash: true

View File

@ -0,0 +1,151 @@
# services/hermes/model-gate-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: hermes-model-gate
namespace: hermes
data:
model_gate.py: |
#!/usr/bin/env python3
"""Fail-closed proxy that admits local inference only while Hermes owns titan-24."""
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import ssl
import threading
import time
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0")
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/")
LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes")
LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner")
CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1"))
API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443")
TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
LEASE_URL = (
f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/"
f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}"
)
_cache_lock = threading.Lock()
_cached_owner = "unavailable"
_cached_at = 0.0
def _lease_owner() -> str:
"""Return the current owner, failing closed when Kubernetes is unavailable."""
global _cached_at, _cached_owner
now = time.monotonic()
with _cache_lock:
if now - _cached_at < CACHE_TTL_SEC:
return _cached_owner
try:
token = TOKEN_PATH.read_text(encoding="utf-8").strip()
request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"})
context = ssl.create_default_context(cafile=str(CA_PATH))
with urlopen(request, timeout=3, context=context) as response:
payload = json.load(response)
owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip()
except Exception:
owner = "unavailable"
_cached_owner = owner
_cached_at = now
return owner
class Handler(BaseHTTPRequestHandler):
"""Proxy local model traffic while exposing health and ownership status."""
protocol_version = "HTTP/1.1"
def _json(self, status: int, payload: dict) -> None:
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def _local_allowed(self) -> tuple[bool, str]:
owner = _lease_owner()
return owner == "hermes", owner
def _proxy(self) -> None:
allowed, owner = self._local_allowed()
if not allowed:
self._json(
503,
{
"error": {
"message": f"local GPU inference unavailable while titan-24 owner is {owner}",
"type": "server_error",
},
"gpu_owner": owner,
"fallback_required": True,
},
)
return
length = int(self.headers.get("Content-Length", "0") or 0)
body = self.rfile.read(length) if length else None
headers = {"Content-Type": self.headers.get("Content-Type", "application/json")}
if self.headers.get("Accept"):
headers["Accept"] = self.headers["Accept"]
request = Request(f"{UPSTREAM_URL}{self.path}", data=body, headers=headers, method=self.command)
try:
response = urlopen(request, timeout=1800)
except HTTPError as exc:
response = exc
except (TimeoutError, URLError) as exc:
self._json(503, {"error": {"message": f"local model upstream unavailable: {exc}", "type": "server_error"}})
return
self.send_response(response.status)
content_type = response.headers.get("Content-Type")
if content_type:
self.send_header("Content-Type", content_type)
content_length = response.headers.get("Content-Length")
if content_length:
self.send_header("Content-Length", content_length)
else:
self.send_header("Connection", "close")
self.close_connection = True
self.send_header("Cache-Control", "no-store")
self.end_headers()
while True:
chunk = response.read(65536)
if not chunk:
break
self.wfile.write(chunk)
self.wfile.flush()
response.close()
def do_GET(self) -> None:
if self.path == "/healthz":
self._json(200, {"status": "ok"})
return
if self.path == "/gate/status":
allowed, owner = self._local_allowed()
self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed})
return
self._proxy()
def do_POST(self) -> None:
self._proxy()
def log_message(self, format_string: str, *args) -> None:
print(f"model-gate {self.address_string()} {format_string % args}", flush=True)
if __name__ == "__main__":
ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever()

View File

@ -0,0 +1,124 @@
# services/hermes/model-gate-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-model-gate
namespace: hermes
labels:
app: hermes-model-gate
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: hermes-model-gate
template:
metadata:
labels:
app: hermes-model-gate
spec:
serviceAccountName: hermes-model-gate
securityContext:
seccompProfile:
type: RuntimeDefault
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values:
- arm64
- key: node-role.kubernetes.io/worker
operator: In
values:
- "true"
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-19
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 90
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
containers:
- name: model-gate
image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
imagePullPolicy: IfNotPresent
command:
- python
- /opt/model-gate/model_gate.py
ports:
- name: http
containerPort: 8080
env:
- name: UPSTREAM_URL
value: http://hermes-ollama.hermes.svc.cluster.local:11434
- name: LEASE_NAMESPACE
value: hermes
- name: LEASE_NAME
value: titan-24-gpu-owner
readinessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 3
periodSeconds: 10
livenessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 10
periodSeconds: 30
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65532
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: script
mountPath: /opt/model-gate
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
- name: script
configMap:
name: hermes-model-gate
defaultMode: 0555
- name: tmp
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: hermes-model-gate
namespace: hermes
labels:
app: hermes-model-gate
spec:
type: ClusterIP
selector:
app: hermes-model-gate
ports:
- name: http
port: 11434
targetPort: http

View File

@ -0,0 +1,34 @@
# services/hermes/model-gate-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: hermes-model-gate
namespace: hermes
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: hermes-model-gate
namespace: hermes
rules:
- apiGroups: ["coordination.k8s.io"]
resources:
- leases
resourceNames:
- titan-24-gpu-owner
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: hermes-model-gate
namespace: hermes
subjects:
- kind: ServiceAccount
name: hermes-model-gate
namespace: hermes
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: hermes-model-gate

View File

@ -0,0 +1,10 @@
# services/hermes/model-gate-state.yaml
apiVersion: coordination.k8s.io/v1
kind: Lease
metadata:
name: titan-24-gpu-owner
namespace: hermes
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
holderIdentity: hermes

View File

@ -0,0 +1,30 @@
# services/hermes/networkpolicy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-ollama-ingress
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-ollama
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
app: hermes-model-gate
ports:
- protocol: TCP
port: 11434
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: maintenance
podSelector:
matchLabels:
app: ariadne
ports:
- protocol: TCP
port: 11434

View File

@ -18,8 +18,8 @@ spec:
labels:
app: hermes-ollama
annotations:
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
ai.bstein.dev/gpu: accelerator MVP lane (titan-24)
ai.bstein.dev/model: gpt-oss:20b
ai.bstein.dev/gpu: titan-24 local-first lane
spec:
runtimeClassName: nvidia
affinity:
@ -33,7 +33,8 @@ spec:
- titan-24
volumes:
- name: models
emptyDir: {}
persistentVolumeClaim:
claimName: hermes-models
initContainers:
- name: warm-model
image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
@ -44,7 +45,7 @@ spec:
- name: OLLAMA_MODELS
value: /root/.ollama
- name: OLLAMA_MODEL
value: qwen2.5:7b-instruct-q4_0
value: gpt-oss:20b
- name: NVIDIA_VISIBLE_DEVICES
value: all
- name: NVIDIA_DRIVER_CAPABILITIES
@ -68,7 +69,7 @@ spec:
nvidia.com/gpu.shared: 1
limits:
cpu: "4"
memory: 10Gi
memory: 16Gi
nvidia.com/gpu.shared: 1
containers:
- name: ollama
@ -82,6 +83,16 @@ spec:
value: 0.0.0.0
- name: OLLAMA_KEEP_ALIVE
value: 6h
- name: OLLAMA_CONTEXT_LENGTH
value: "64000"
- name: OLLAMA_FLASH_ATTENTION
value: "1"
- name: OLLAMA_KV_CACHE_TYPE
value: q8_0
- name: OLLAMA_MAX_LOADED_MODELS
value: "1"
- name: OLLAMA_NUM_PARALLEL
value: "1"
- name: OLLAMA_MODELS
value: /root/.ollama
- name: NVIDIA_VISIBLE_DEVICES
@ -100,10 +111,10 @@ spec:
timeoutSeconds: 5
resources:
requests:
cpu: "2"
memory: 8Gi
cpu: "8"
memory: 24Gi
nvidia.com/gpu.shared: 1
limits:
cpu: "6"
memory: 12Gi
cpu: "16"
memory: 40Gi
nvidia.com/gpu.shared: 1

View File

@ -13,3 +13,18 @@ spec:
resources:
requests:
storage: 4Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: hermes-models
namespace: hermes
labels:
app: hermes-ollama
spec:
accessModes:
- ReadWriteOnce
storageClassName: local-path
resources:
requests:
storage: 30Gi

View File

@ -0,0 +1,59 @@
---
name: triage-titan-test-failures
description: Diagnose Titan CI test failures and environment regressions from Ariadne evidence, Jenkins artifacts, Flux state, Kubernetes health, Pushgateway quality metrics, and Grafana context. Use for failed or flaky in-scope suites, suspected cluster-caused test failures, release-quality questions, or requests for a supervised read-only triage summary and repo-side next steps.
---
# Triage Titan Test Failures
Follow the established Titan evidence path. Keep the investigation read-only and distinguish observed facts from inference.
## Collect the canonical evidence
1. Read the latest Ariadne diagnosis:
```sh
curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"
```
2. Read the deterministic bundle when the diagnosis is missing, stale, unavailable, or lacks evidence:
```sh
curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"
```
3. Ask for human approval before triggering a fresh collection or diagnosis. Use these only after approval:
```sh
curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/collect"
curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"
```
Treat Ariadne's bundle as the evidence source of truth. A local-model diagnosis may be unavailable while Wolf owns titan-24; continue from the stored bundle using the active Hermes fallback model.
## Narrow the failure
Work in this order:
1. Confirm the failed suite and build are in the canonical scope: `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `metis`, `pegasus`, `soteria`, or `titan_iac`.
2. Identify the first failed gate in the enforced order: `style`, `loc`, `coverage`, `tests`, `gate_glue`, `sonarqube`, `supply_chain`.
3. Correlate the build timestamp with retained Jenkins logs/artifacts, recent Git commits, and Flux revisions.
4. Check whether Kubernetes health, node pressure, image pulls, storage, DNS, or a shared dependency explains the failure better than a repo regression.
5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry.
6. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause.
Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands.
## Produce the triage result
Return these sections:
- `Finding`: one sentence naming the most likely failure class.
- `Confidence`: low, medium, or high, with the reason.
- `Evidence`: the smallest set of concrete timestamps, build IDs, artifact paths, commits, metrics, pods, nodes, or Flux revisions that support the finding.
- `Likely cause`: explain the causal chain and label inference as inference.
- `Blast radius`: affected suites, services, branches, or environments.
- `Next checks`: ordered read-only checks with exact commands or URLs.
- `Repo-side fix`: the smallest Flux/IaC or application change, or `none yet` when evidence is insufficient.
- `Approval required`: call out every step that would modify files, infrastructure, credentials, test environments, or external systems.
Never present a proposed fix as applied. Prefer a concise evidence-backed answer over a general log summary.

View File

@ -0,0 +1,4 @@
interface:
display_name: "Triage Titan Test Failures"
short_description: "Analyze evidence for supervised test-failure triage"
default_prompt: "Use $triage-titan-test-failures to diagnose the latest Titan test failure from Ariadne evidence."

View File

@ -285,8 +285,21 @@ spec:
- name: GAME_MODE_NODE_NAME
value: titan-24
- name: GAME_MODE_DISPLACE_WORKLOADS
value: >-
[{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}]
value: "[]"
- name: GAME_MODE_LEASE_NAMESPACE
value: hermes
- name: GAME_MODE_LEASE_NAME
value: titan-24-gpu-owner
- name: GAME_MODE_OLLAMA_URL
value: http://hermes-ollama.hermes.svc.cluster.local:11434
- name: GAME_MODE_OLLAMA_MODEL
value: gpt-oss:20b
- name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC
value: "900"
- name: GAME_MODE_TRANSITION_TIMEOUT_SEC
value: "900"
- name: GAME_MODE_POLL_INTERVAL_SEC
value: "1"
- name: WOLF_OIDC_CLIENT_ID
value: wolf
- name: WOLF_OIDC_BASE_URL
@ -444,11 +457,11 @@ spec:
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
value: "*/15 * * * *"
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
value: http://hermes-ollama.hermes.svc.cluster.local:11434
value: http://hermes-model-gate.hermes.svc.cluster.local:11434
- name: ARIADNE_TESTING_TRIAGE_MODEL
value: qwen2.5:7b-instruct-q4_0
value: gpt-oss:20b
- name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC
value: "180"
value: "900"
- name: JENKINS_WORKSPACE_NAMESPACE
value: jenkins
- name: JENKINS_WORKSPACE_PVC_PREFIX

View File

@ -78,7 +78,6 @@ rules:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding