gpu(titan-24): add Hermes local-first fallback handoff
This commit is contained in:
parent
d58aa8e7e8
commit
be91902a01
@ -18,6 +18,10 @@ spec:
|
|||||||
wait: true
|
wait: true
|
||||||
timeout: 45m
|
timeout: 45m
|
||||||
healthChecks:
|
healthChecks:
|
||||||
|
- apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
name: hermes-model-gate
|
||||||
|
namespace: hermes
|
||||||
- apiVersion: apps/v1
|
- apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
name: hermes-ollama
|
name: hermes-ollama
|
||||||
|
|||||||
@ -68,7 +68,7 @@ spec:
|
|||||||
args:
|
args:
|
||||||
- >-
|
- >-
|
||||||
. /vault/secrets/portal-env.sh
|
. /vault/secrets/portal-env.sh
|
||||||
&& exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 600 app:app
|
&& exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 1200 app:app
|
||||||
env:
|
env:
|
||||||
- name: AI_CHAT_API
|
- name: AI_CHAT_API
|
||||||
value: http://ollama.ai.svc.cluster.local:11434
|
value: http://ollama.ai.svc.cluster.local:11434
|
||||||
@ -119,6 +119,8 @@ spec:
|
|||||||
value: http://ariadne.maintenance.svc.cluster.local
|
value: http://ariadne.maintenance.svc.cluster.local
|
||||||
- name: ARIADNE_TIMEOUT_SEC
|
- name: ARIADNE_TIMEOUT_SEC
|
||||||
value: "10"
|
value: "10"
|
||||||
|
- name: ARIADNE_GAME_MODE_TIMEOUT_SEC
|
||||||
|
value: "900"
|
||||||
- name: ACCOUNT_ALLOWED_GROUPS
|
- name: ACCOUNT_ALLOWED_GROUPS
|
||||||
value: ""
|
value: ""
|
||||||
- name: HTTP_CHECK_TIMEOUT_SEC
|
- name: HTTP_CHECK_TIMEOUT_SEC
|
||||||
|
|||||||
30
services/hermes/ariadne-handoff-rbac.yaml
Normal file
30
services/hermes/ariadne-handoff-rbac.yaml
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
# services/hermes/ariadne-handoff-rbac.yaml
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: ariadne-gpu-handoff
|
||||||
|
namespace: hermes
|
||||||
|
rules:
|
||||||
|
- apiGroups: ["coordination.k8s.io"]
|
||||||
|
resources:
|
||||||
|
- leases
|
||||||
|
resourceNames:
|
||||||
|
- titan-24-gpu-owner
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: ariadne-gpu-handoff
|
||||||
|
namespace: hermes
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: ariadne
|
||||||
|
namespace: maintenance
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: ariadne-gpu-handoff
|
||||||
@ -10,11 +10,24 @@ data:
|
|||||||
config.yaml: |
|
config.yaml: |
|
||||||
model:
|
model:
|
||||||
provider: custom
|
provider: custom
|
||||||
default: qwen2.5:7b-instruct-q4_0
|
default: gpt-oss:20b
|
||||||
model: qwen2.5:7b-instruct-q4_0
|
model: gpt-oss:20b
|
||||||
base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1
|
context_length: 64000
|
||||||
|
base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
|
||||||
api_key: ollama
|
api_key: ollama
|
||||||
|
|
||||||
|
fallback_providers:
|
||||||
|
- provider: openai-codex
|
||||||
|
model: gpt-5.4
|
||||||
|
|
||||||
|
agent:
|
||||||
|
api_max_retries: 1
|
||||||
|
|
||||||
|
skills:
|
||||||
|
creation_nudge_interval: 15
|
||||||
|
external_dirs:
|
||||||
|
- /opt/data/workspace/skills
|
||||||
|
|
||||||
terminal:
|
terminal:
|
||||||
backend: local
|
backend: local
|
||||||
cwd: /opt/data/workspace
|
cwd: /opt/data/workspace
|
||||||
|
|||||||
@ -20,10 +20,10 @@ spec:
|
|||||||
labels:
|
labels:
|
||||||
app: hermes
|
app: hermes
|
||||||
annotations:
|
annotations:
|
||||||
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
|
ai.bstein.dev/model: gpt-oss:20b with openai-codex fallback
|
||||||
ai.bstein.dev/role: testing-triage
|
ai.bstein.dev/role: testing-triage
|
||||||
ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
|
ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
|
||||||
ai.bstein.dev/config-rev: "20260721-hermes-rollout-deadline"
|
ai.bstein.dev/config-rev: "20260801-gpu-owner-fallback"
|
||||||
spec:
|
spec:
|
||||||
serviceAccountName: hermes-triage
|
serviceAccountName: hermes-triage
|
||||||
automountServiceAccountToken: true
|
automountServiceAccountToken: true
|
||||||
@ -190,6 +190,9 @@ spec:
|
|||||||
mountPath: /opt/data
|
mountPath: /opt/data
|
||||||
- name: tools
|
- name: tools
|
||||||
mountPath: /opt/data/home/.local/bin
|
mountPath: /opt/data/home/.local/bin
|
||||||
|
- name: triage-skill
|
||||||
|
mountPath: /opt/data/workspace/skills/triage-titan-test-failures
|
||||||
|
readOnly: true
|
||||||
readinessProbe:
|
readinessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /api/status
|
path: /api/status
|
||||||
@ -220,3 +223,11 @@ spec:
|
|||||||
name: hermes-config
|
name: hermes-config
|
||||||
- name: tools
|
- name: tools
|
||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
|
- name: triage-skill
|
||||||
|
configMap:
|
||||||
|
name: hermes-triage-skill
|
||||||
|
items:
|
||||||
|
- key: SKILL.md
|
||||||
|
path: SKILL.md
|
||||||
|
- key: openai.yaml
|
||||||
|
path: agents/openai.yaml
|
||||||
|
|||||||
@ -7,8 +7,23 @@ resources:
|
|||||||
- configmap.yaml
|
- configmap.yaml
|
||||||
- rbac.yaml
|
- rbac.yaml
|
||||||
- pvc.yaml
|
- pvc.yaml
|
||||||
|
- model-gate-rbac.yaml
|
||||||
|
- ariadne-handoff-rbac.yaml
|
||||||
|
- model-gate-state.yaml
|
||||||
|
- model-gate-configmap.yaml
|
||||||
|
- model-gate-deployment.yaml
|
||||||
|
- networkpolicy.yaml
|
||||||
- ollama-deployment.yaml
|
- ollama-deployment.yaml
|
||||||
- deployment.yaml
|
- deployment.yaml
|
||||||
- service.yaml
|
- service.yaml
|
||||||
- agent-certificate.yaml
|
- agent-certificate.yaml
|
||||||
- agent-ingress.yaml
|
- agent-ingress.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: hermes-triage-skill
|
||||||
|
namespace: hermes
|
||||||
|
files:
|
||||||
|
- SKILL.md=skills/triage-titan-test-failures/SKILL.md
|
||||||
|
- openai.yaml=skills/triage-titan-test-failures/agents/openai.yaml
|
||||||
|
options:
|
||||||
|
disableNameSuffixHash: true
|
||||||
|
|||||||
151
services/hermes/model-gate-configmap.yaml
Normal file
151
services/hermes/model-gate-configmap.yaml
Normal file
@ -0,0 +1,151 @@
|
|||||||
|
# services/hermes/model-gate-configmap.yaml
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: hermes-model-gate
|
||||||
|
namespace: hermes
|
||||||
|
data:
|
||||||
|
model_gate.py: |
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed proxy that admits local inference only while Hermes owns titan-24."""
|
||||||
|
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import ssl
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from urllib.error import HTTPError, URLError
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
|
||||||
|
LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0")
|
||||||
|
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
|
||||||
|
UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/")
|
||||||
|
LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes")
|
||||||
|
LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner")
|
||||||
|
CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1"))
|
||||||
|
API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
|
||||||
|
API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443")
|
||||||
|
TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
|
||||||
|
CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
|
||||||
|
LEASE_URL = (
|
||||||
|
f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/"
|
||||||
|
f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}"
|
||||||
|
)
|
||||||
|
|
||||||
|
_cache_lock = threading.Lock()
|
||||||
|
_cached_owner = "unavailable"
|
||||||
|
_cached_at = 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _lease_owner() -> str:
|
||||||
|
"""Return the current owner, failing closed when Kubernetes is unavailable."""
|
||||||
|
|
||||||
|
global _cached_at, _cached_owner
|
||||||
|
now = time.monotonic()
|
||||||
|
with _cache_lock:
|
||||||
|
if now - _cached_at < CACHE_TTL_SEC:
|
||||||
|
return _cached_owner
|
||||||
|
try:
|
||||||
|
token = TOKEN_PATH.read_text(encoding="utf-8").strip()
|
||||||
|
request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"})
|
||||||
|
context = ssl.create_default_context(cafile=str(CA_PATH))
|
||||||
|
with urlopen(request, timeout=3, context=context) as response:
|
||||||
|
payload = json.load(response)
|
||||||
|
owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip()
|
||||||
|
except Exception:
|
||||||
|
owner = "unavailable"
|
||||||
|
_cached_owner = owner
|
||||||
|
_cached_at = now
|
||||||
|
return owner
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
"""Proxy local model traffic while exposing health and ownership status."""
|
||||||
|
|
||||||
|
protocol_version = "HTTP/1.1"
|
||||||
|
|
||||||
|
def _json(self, status: int, payload: dict) -> None:
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def _local_allowed(self) -> tuple[bool, str]:
|
||||||
|
owner = _lease_owner()
|
||||||
|
return owner == "hermes", owner
|
||||||
|
|
||||||
|
def _proxy(self) -> None:
|
||||||
|
allowed, owner = self._local_allowed()
|
||||||
|
if not allowed:
|
||||||
|
self._json(
|
||||||
|
503,
|
||||||
|
{
|
||||||
|
"error": {
|
||||||
|
"message": f"local GPU inference unavailable while titan-24 owner is {owner}",
|
||||||
|
"type": "server_error",
|
||||||
|
},
|
||||||
|
"gpu_owner": owner,
|
||||||
|
"fallback_required": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||||
|
body = self.rfile.read(length) if length else None
|
||||||
|
headers = {"Content-Type": self.headers.get("Content-Type", "application/json")}
|
||||||
|
if self.headers.get("Accept"):
|
||||||
|
headers["Accept"] = self.headers["Accept"]
|
||||||
|
request = Request(f"{UPSTREAM_URL}{self.path}", data=body, headers=headers, method=self.command)
|
||||||
|
try:
|
||||||
|
response = urlopen(request, timeout=1800)
|
||||||
|
except HTTPError as exc:
|
||||||
|
response = exc
|
||||||
|
except (TimeoutError, URLError) as exc:
|
||||||
|
self._json(503, {"error": {"message": f"local model upstream unavailable: {exc}", "type": "server_error"}})
|
||||||
|
return
|
||||||
|
|
||||||
|
self.send_response(response.status)
|
||||||
|
content_type = response.headers.get("Content-Type")
|
||||||
|
if content_type:
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
content_length = response.headers.get("Content-Length")
|
||||||
|
if content_length:
|
||||||
|
self.send_header("Content-Length", content_length)
|
||||||
|
else:
|
||||||
|
self.send_header("Connection", "close")
|
||||||
|
self.close_connection = True
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.end_headers()
|
||||||
|
while True:
|
||||||
|
chunk = response.read(65536)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
self.wfile.write(chunk)
|
||||||
|
self.wfile.flush()
|
||||||
|
response.close()
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/healthz":
|
||||||
|
self._json(200, {"status": "ok"})
|
||||||
|
return
|
||||||
|
if self.path == "/gate/status":
|
||||||
|
allowed, owner = self._local_allowed()
|
||||||
|
self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed})
|
||||||
|
return
|
||||||
|
self._proxy()
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
self._proxy()
|
||||||
|
|
||||||
|
def log_message(self, format_string: str, *args) -> None:
|
||||||
|
print(f"model-gate {self.address_string()} {format_string % args}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever()
|
||||||
124
services/hermes/model-gate-deployment.yaml
Normal file
124
services/hermes/model-gate-deployment.yaml
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
# services/hermes/model-gate-deployment.yaml
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: hermes-model-gate
|
||||||
|
namespace: hermes
|
||||||
|
labels:
|
||||||
|
app: hermes-model-gate
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
revisionHistoryLimit: 2
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: hermes-model-gate
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: hermes-model-gate
|
||||||
|
spec:
|
||||||
|
serviceAccountName: hermes-model-gate
|
||||||
|
securityContext:
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
affinity:
|
||||||
|
nodeAffinity:
|
||||||
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
nodeSelectorTerms:
|
||||||
|
- matchExpressions:
|
||||||
|
- key: kubernetes.io/arch
|
||||||
|
operator: In
|
||||||
|
values:
|
||||||
|
- arm64
|
||||||
|
- key: node-role.kubernetes.io/worker
|
||||||
|
operator: In
|
||||||
|
values:
|
||||||
|
- "true"
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- titan-13
|
||||||
|
- titan-15
|
||||||
|
- titan-17
|
||||||
|
- titan-19
|
||||||
|
preferredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
- weight: 90
|
||||||
|
preference:
|
||||||
|
matchExpressions:
|
||||||
|
- key: hardware
|
||||||
|
operator: In
|
||||||
|
values:
|
||||||
|
- rpi5
|
||||||
|
containers:
|
||||||
|
- name: model-gate
|
||||||
|
image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
command:
|
||||||
|
- python
|
||||||
|
- /opt/model-gate/model_gate.py
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8080
|
||||||
|
env:
|
||||||
|
- name: UPSTREAM_URL
|
||||||
|
value: http://hermes-ollama.hermes.svc.cluster.local:11434
|
||||||
|
- name: LEASE_NAMESPACE
|
||||||
|
value: hermes
|
||||||
|
- name: LEASE_NAME
|
||||||
|
value: titan-24-gpu-owner
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 3
|
||||||
|
periodSeconds: 10
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 30
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 65532
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 25m
|
||||||
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 128Mi
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /opt/model-gate
|
||||||
|
readOnly: true
|
||||||
|
- name: tmp
|
||||||
|
mountPath: /tmp
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: hermes-model-gate
|
||||||
|
defaultMode: 0555
|
||||||
|
- name: tmp
|
||||||
|
emptyDir: {}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: hermes-model-gate
|
||||||
|
namespace: hermes
|
||||||
|
labels:
|
||||||
|
app: hermes-model-gate
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: hermes-model-gate
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 11434
|
||||||
|
targetPort: http
|
||||||
34
services/hermes/model-gate-rbac.yaml
Normal file
34
services/hermes/model-gate-rbac.yaml
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
# services/hermes/model-gate-rbac.yaml
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: hermes-model-gate
|
||||||
|
namespace: hermes
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: hermes-model-gate
|
||||||
|
namespace: hermes
|
||||||
|
rules:
|
||||||
|
- apiGroups: ["coordination.k8s.io"]
|
||||||
|
resources:
|
||||||
|
- leases
|
||||||
|
resourceNames:
|
||||||
|
- titan-24-gpu-owner
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: hermes-model-gate
|
||||||
|
namespace: hermes
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: hermes-model-gate
|
||||||
|
namespace: hermes
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
name: hermes-model-gate
|
||||||
10
services/hermes/model-gate-state.yaml
Normal file
10
services/hermes/model-gate-state.yaml
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
# services/hermes/model-gate-state.yaml
|
||||||
|
apiVersion: coordination.k8s.io/v1
|
||||||
|
kind: Lease
|
||||||
|
metadata:
|
||||||
|
name: titan-24-gpu-owner
|
||||||
|
namespace: hermes
|
||||||
|
annotations:
|
||||||
|
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||||
|
spec:
|
||||||
|
holderIdentity: hermes
|
||||||
30
services/hermes/networkpolicy.yaml
Normal file
30
services/hermes/networkpolicy.yaml
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
# services/hermes/networkpolicy.yaml
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: hermes-ollama-ingress
|
||||||
|
namespace: hermes
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: hermes-ollama
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: hermes-model-gate
|
||||||
|
ports:
|
||||||
|
- protocol: TCP
|
||||||
|
port: 11434
|
||||||
|
- from:
|
||||||
|
- namespaceSelector:
|
||||||
|
matchLabels:
|
||||||
|
kubernetes.io/metadata.name: maintenance
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app: ariadne
|
||||||
|
ports:
|
||||||
|
- protocol: TCP
|
||||||
|
port: 11434
|
||||||
@ -18,8 +18,8 @@ spec:
|
|||||||
labels:
|
labels:
|
||||||
app: hermes-ollama
|
app: hermes-ollama
|
||||||
annotations:
|
annotations:
|
||||||
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
|
ai.bstein.dev/model: gpt-oss:20b
|
||||||
ai.bstein.dev/gpu: accelerator MVP lane (titan-24)
|
ai.bstein.dev/gpu: titan-24 local-first lane
|
||||||
spec:
|
spec:
|
||||||
runtimeClassName: nvidia
|
runtimeClassName: nvidia
|
||||||
affinity:
|
affinity:
|
||||||
@ -33,7 +33,8 @@ spec:
|
|||||||
- titan-24
|
- titan-24
|
||||||
volumes:
|
volumes:
|
||||||
- name: models
|
- name: models
|
||||||
emptyDir: {}
|
persistentVolumeClaim:
|
||||||
|
claimName: hermes-models
|
||||||
initContainers:
|
initContainers:
|
||||||
- name: warm-model
|
- name: warm-model
|
||||||
image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
|
image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
|
||||||
@ -44,7 +45,7 @@ spec:
|
|||||||
- name: OLLAMA_MODELS
|
- name: OLLAMA_MODELS
|
||||||
value: /root/.ollama
|
value: /root/.ollama
|
||||||
- name: OLLAMA_MODEL
|
- name: OLLAMA_MODEL
|
||||||
value: qwen2.5:7b-instruct-q4_0
|
value: gpt-oss:20b
|
||||||
- name: NVIDIA_VISIBLE_DEVICES
|
- name: NVIDIA_VISIBLE_DEVICES
|
||||||
value: all
|
value: all
|
||||||
- name: NVIDIA_DRIVER_CAPABILITIES
|
- name: NVIDIA_DRIVER_CAPABILITIES
|
||||||
@ -68,7 +69,7 @@ spec:
|
|||||||
nvidia.com/gpu.shared: 1
|
nvidia.com/gpu.shared: 1
|
||||||
limits:
|
limits:
|
||||||
cpu: "4"
|
cpu: "4"
|
||||||
memory: 10Gi
|
memory: 16Gi
|
||||||
nvidia.com/gpu.shared: 1
|
nvidia.com/gpu.shared: 1
|
||||||
containers:
|
containers:
|
||||||
- name: ollama
|
- name: ollama
|
||||||
@ -82,6 +83,16 @@ spec:
|
|||||||
value: 0.0.0.0
|
value: 0.0.0.0
|
||||||
- name: OLLAMA_KEEP_ALIVE
|
- name: OLLAMA_KEEP_ALIVE
|
||||||
value: 6h
|
value: 6h
|
||||||
|
- name: OLLAMA_CONTEXT_LENGTH
|
||||||
|
value: "64000"
|
||||||
|
- name: OLLAMA_FLASH_ATTENTION
|
||||||
|
value: "1"
|
||||||
|
- name: OLLAMA_KV_CACHE_TYPE
|
||||||
|
value: q8_0
|
||||||
|
- name: OLLAMA_MAX_LOADED_MODELS
|
||||||
|
value: "1"
|
||||||
|
- name: OLLAMA_NUM_PARALLEL
|
||||||
|
value: "1"
|
||||||
- name: OLLAMA_MODELS
|
- name: OLLAMA_MODELS
|
||||||
value: /root/.ollama
|
value: /root/.ollama
|
||||||
- name: NVIDIA_VISIBLE_DEVICES
|
- name: NVIDIA_VISIBLE_DEVICES
|
||||||
@ -100,10 +111,10 @@ spec:
|
|||||||
timeoutSeconds: 5
|
timeoutSeconds: 5
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: "2"
|
cpu: "8"
|
||||||
memory: 8Gi
|
memory: 24Gi
|
||||||
nvidia.com/gpu.shared: 1
|
nvidia.com/gpu.shared: 1
|
||||||
limits:
|
limits:
|
||||||
cpu: "6"
|
cpu: "16"
|
||||||
memory: 12Gi
|
memory: 40Gi
|
||||||
nvidia.com/gpu.shared: 1
|
nvidia.com/gpu.shared: 1
|
||||||
|
|||||||
@ -13,3 +13,18 @@ spec:
|
|||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
storage: 4Gi
|
storage: 4Gi
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: hermes-models
|
||||||
|
namespace: hermes
|
||||||
|
labels:
|
||||||
|
app: hermes-ollama
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: local-path
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 30Gi
|
||||||
|
|||||||
59
services/hermes/skills/triage-titan-test-failures/SKILL.md
Normal file
59
services/hermes/skills/triage-titan-test-failures/SKILL.md
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
---
|
||||||
|
name: triage-titan-test-failures
|
||||||
|
description: Diagnose Titan CI test failures and environment regressions from Ariadne evidence, Jenkins artifacts, Flux state, Kubernetes health, Pushgateway quality metrics, and Grafana context. Use for failed or flaky in-scope suites, suspected cluster-caused test failures, release-quality questions, or requests for a supervised read-only triage summary and repo-side next steps.
|
||||||
|
---
|
||||||
|
|
||||||
|
# Triage Titan Test Failures
|
||||||
|
|
||||||
|
Follow the established Titan evidence path. Keep the investigation read-only and distinguish observed facts from inference.
|
||||||
|
|
||||||
|
## Collect the canonical evidence
|
||||||
|
|
||||||
|
1. Read the latest Ariadne diagnosis:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Read the deterministic bundle when the diagnosis is missing, stale, unavailable, or lacks evidence:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Ask for human approval before triggering a fresh collection or diagnosis. Use these only after approval:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/collect"
|
||||||
|
curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"
|
||||||
|
```
|
||||||
|
|
||||||
|
Treat Ariadne's bundle as the evidence source of truth. A local-model diagnosis may be unavailable while Wolf owns titan-24; continue from the stored bundle using the active Hermes fallback model.
|
||||||
|
|
||||||
|
## Narrow the failure
|
||||||
|
|
||||||
|
Work in this order:
|
||||||
|
|
||||||
|
1. Confirm the failed suite and build are in the canonical scope: `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `metis`, `pegasus`, `soteria`, or `titan_iac`.
|
||||||
|
2. Identify the first failed gate in the enforced order: `style`, `loc`, `coverage`, `tests`, `gate_glue`, `sonarqube`, `supply_chain`.
|
||||||
|
3. Correlate the build timestamp with retained Jenkins logs/artifacts, recent Git commits, and Flux revisions.
|
||||||
|
4. Check whether Kubernetes health, node pressure, image pulls, storage, DNS, or a shared dependency explains the failure better than a repo regression.
|
||||||
|
5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry.
|
||||||
|
6. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause.
|
||||||
|
|
||||||
|
Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands.
|
||||||
|
|
||||||
|
## Produce the triage result
|
||||||
|
|
||||||
|
Return these sections:
|
||||||
|
|
||||||
|
- `Finding`: one sentence naming the most likely failure class.
|
||||||
|
- `Confidence`: low, medium, or high, with the reason.
|
||||||
|
- `Evidence`: the smallest set of concrete timestamps, build IDs, artifact paths, commits, metrics, pods, nodes, or Flux revisions that support the finding.
|
||||||
|
- `Likely cause`: explain the causal chain and label inference as inference.
|
||||||
|
- `Blast radius`: affected suites, services, branches, or environments.
|
||||||
|
- `Next checks`: ordered read-only checks with exact commands or URLs.
|
||||||
|
- `Repo-side fix`: the smallest Flux/IaC or application change, or `none yet` when evidence is insufficient.
|
||||||
|
- `Approval required`: call out every step that would modify files, infrastructure, credentials, test environments, or external systems.
|
||||||
|
|
||||||
|
Never present a proposed fix as applied. Prefer a concise evidence-backed answer over a general log summary.
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
interface:
|
||||||
|
display_name: "Triage Titan Test Failures"
|
||||||
|
short_description: "Analyze evidence for supervised test-failure triage"
|
||||||
|
default_prompt: "Use $triage-titan-test-failures to diagnose the latest Titan test failure from Ariadne evidence."
|
||||||
@ -285,8 +285,21 @@ spec:
|
|||||||
- name: GAME_MODE_NODE_NAME
|
- name: GAME_MODE_NODE_NAME
|
||||||
value: titan-24
|
value: titan-24
|
||||||
- name: GAME_MODE_DISPLACE_WORKLOADS
|
- name: GAME_MODE_DISPLACE_WORKLOADS
|
||||||
value: >-
|
value: "[]"
|
||||||
[{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}]
|
- name: GAME_MODE_LEASE_NAMESPACE
|
||||||
|
value: hermes
|
||||||
|
- name: GAME_MODE_LEASE_NAME
|
||||||
|
value: titan-24-gpu-owner
|
||||||
|
- name: GAME_MODE_OLLAMA_URL
|
||||||
|
value: http://hermes-ollama.hermes.svc.cluster.local:11434
|
||||||
|
- name: GAME_MODE_OLLAMA_MODEL
|
||||||
|
value: gpt-oss:20b
|
||||||
|
- name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC
|
||||||
|
value: "900"
|
||||||
|
- name: GAME_MODE_TRANSITION_TIMEOUT_SEC
|
||||||
|
value: "900"
|
||||||
|
- name: GAME_MODE_POLL_INTERVAL_SEC
|
||||||
|
value: "1"
|
||||||
- name: WOLF_OIDC_CLIENT_ID
|
- name: WOLF_OIDC_CLIENT_ID
|
||||||
value: wolf
|
value: wolf
|
||||||
- name: WOLF_OIDC_BASE_URL
|
- name: WOLF_OIDC_BASE_URL
|
||||||
@ -444,11 +457,11 @@ spec:
|
|||||||
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
|
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
|
||||||
value: "*/15 * * * *"
|
value: "*/15 * * * *"
|
||||||
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
|
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
|
||||||
value: http://hermes-ollama.hermes.svc.cluster.local:11434
|
value: http://hermes-model-gate.hermes.svc.cluster.local:11434
|
||||||
- name: ARIADNE_TESTING_TRIAGE_MODEL
|
- name: ARIADNE_TESTING_TRIAGE_MODEL
|
||||||
value: qwen2.5:7b-instruct-q4_0
|
value: gpt-oss:20b
|
||||||
- name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC
|
- name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC
|
||||||
value: "180"
|
value: "900"
|
||||||
- name: JENKINS_WORKSPACE_NAMESPACE
|
- name: JENKINS_WORKSPACE_NAMESPACE
|
||||||
value: jenkins
|
value: jenkins
|
||||||
- name: JENKINS_WORKSPACE_PVC_PREFIX
|
- name: JENKINS_WORKSPACE_PVC_PREFIX
|
||||||
|
|||||||
@ -78,7 +78,6 @@ rules:
|
|||||||
- get
|
- get
|
||||||
- list
|
- list
|
||||||
- watch
|
- watch
|
||||||
|
|
||||||
---
|
---
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
kind: ClusterRoleBinding
|
kind: ClusterRoleBinding
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user