gpu(titan-24): add Hermes local-first fallback handoff
This commit is contained in:
parent
d58aa8e7e8
commit
be91902a01
@ -18,6 +18,10 @@ spec:
|
||||
wait: true
|
||||
timeout: 45m
|
||||
healthChecks:
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: hermes-ollama
|
||||
|
||||
@ -68,7 +68,7 @@ spec:
|
||||
args:
|
||||
- >-
|
||||
. /vault/secrets/portal-env.sh
|
||||
&& exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 600 app:app
|
||||
&& exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 1200 app:app
|
||||
env:
|
||||
- name: AI_CHAT_API
|
||||
value: http://ollama.ai.svc.cluster.local:11434
|
||||
@ -119,6 +119,8 @@ spec:
|
||||
value: http://ariadne.maintenance.svc.cluster.local
|
||||
- name: ARIADNE_TIMEOUT_SEC
|
||||
value: "10"
|
||||
- name: ARIADNE_GAME_MODE_TIMEOUT_SEC
|
||||
value: "900"
|
||||
- name: ACCOUNT_ALLOWED_GROUPS
|
||||
value: ""
|
||||
- name: HTTP_CHECK_TIMEOUT_SEC
|
||||
|
||||
30
services/hermes/ariadne-handoff-rbac.yaml
Normal file
30
services/hermes/ariadne-handoff-rbac.yaml
Normal file
@ -0,0 +1,30 @@
|
||||
# services/hermes/ariadne-handoff-rbac.yaml
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: ariadne-gpu-handoff
|
||||
namespace: hermes
|
||||
rules:
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources:
|
||||
- leases
|
||||
resourceNames:
|
||||
- titan-24-gpu-owner
|
||||
verbs:
|
||||
- get
|
||||
- patch
|
||||
- update
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: ariadne-gpu-handoff
|
||||
namespace: hermes
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: ariadne
|
||||
namespace: maintenance
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: ariadne-gpu-handoff
|
||||
@ -10,11 +10,24 @@ data:
|
||||
config.yaml: |
|
||||
model:
|
||||
provider: custom
|
||||
default: qwen2.5:7b-instruct-q4_0
|
||||
model: qwen2.5:7b-instruct-q4_0
|
||||
base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1
|
||||
default: gpt-oss:20b
|
||||
model: gpt-oss:20b
|
||||
context_length: 64000
|
||||
base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
|
||||
api_key: ollama
|
||||
|
||||
fallback_providers:
|
||||
- provider: openai-codex
|
||||
model: gpt-5.4
|
||||
|
||||
agent:
|
||||
api_max_retries: 1
|
||||
|
||||
skills:
|
||||
creation_nudge_interval: 15
|
||||
external_dirs:
|
||||
- /opt/data/workspace/skills
|
||||
|
||||
terminal:
|
||||
backend: local
|
||||
cwd: /opt/data/workspace
|
||||
|
||||
@ -20,10 +20,10 @@ spec:
|
||||
labels:
|
||||
app: hermes
|
||||
annotations:
|
||||
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
|
||||
ai.bstein.dev/model: gpt-oss:20b with openai-codex fallback
|
||||
ai.bstein.dev/role: testing-triage
|
||||
ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
|
||||
ai.bstein.dev/config-rev: "20260721-hermes-rollout-deadline"
|
||||
ai.bstein.dev/config-rev: "20260801-gpu-owner-fallback"
|
||||
spec:
|
||||
serviceAccountName: hermes-triage
|
||||
automountServiceAccountToken: true
|
||||
@ -190,6 +190,9 @@ spec:
|
||||
mountPath: /opt/data
|
||||
- name: tools
|
||||
mountPath: /opt/data/home/.local/bin
|
||||
- name: triage-skill
|
||||
mountPath: /opt/data/workspace/skills/triage-titan-test-failures
|
||||
readOnly: true
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/status
|
||||
@ -220,3 +223,11 @@ spec:
|
||||
name: hermes-config
|
||||
- name: tools
|
||||
emptyDir: {}
|
||||
- name: triage-skill
|
||||
configMap:
|
||||
name: hermes-triage-skill
|
||||
items:
|
||||
- key: SKILL.md
|
||||
path: SKILL.md
|
||||
- key: openai.yaml
|
||||
path: agents/openai.yaml
|
||||
|
||||
@ -7,8 +7,23 @@ resources:
|
||||
- configmap.yaml
|
||||
- rbac.yaml
|
||||
- pvc.yaml
|
||||
- model-gate-rbac.yaml
|
||||
- ariadne-handoff-rbac.yaml
|
||||
- model-gate-state.yaml
|
||||
- model-gate-configmap.yaml
|
||||
- model-gate-deployment.yaml
|
||||
- networkpolicy.yaml
|
||||
- ollama-deployment.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- agent-certificate.yaml
|
||||
- agent-ingress.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: hermes-triage-skill
|
||||
namespace: hermes
|
||||
files:
|
||||
- SKILL.md=skills/triage-titan-test-failures/SKILL.md
|
||||
- openai.yaml=skills/triage-titan-test-failures/agents/openai.yaml
|
||||
options:
|
||||
disableNameSuffixHash: true
|
||||
|
||||
151
services/hermes/model-gate-configmap.yaml
Normal file
151
services/hermes/model-gate-configmap.yaml
Normal file
@ -0,0 +1,151 @@
|
||||
# services/hermes/model-gate-configmap.yaml
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
data:
|
||||
model_gate.py: |
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed proxy that admits local inference only while Hermes owns titan-24."""
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0")
|
||||
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
|
||||
UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/")
|
||||
LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes")
|
||||
LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner")
|
||||
CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1"))
|
||||
API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
|
||||
API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443")
|
||||
TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
|
||||
CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
|
||||
LEASE_URL = (
|
||||
f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/"
|
||||
f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}"
|
||||
)
|
||||
|
||||
_cache_lock = threading.Lock()
|
||||
_cached_owner = "unavailable"
|
||||
_cached_at = 0.0
|
||||
|
||||
|
||||
def _lease_owner() -> str:
|
||||
"""Return the current owner, failing closed when Kubernetes is unavailable."""
|
||||
|
||||
global _cached_at, _cached_owner
|
||||
now = time.monotonic()
|
||||
with _cache_lock:
|
||||
if now - _cached_at < CACHE_TTL_SEC:
|
||||
return _cached_owner
|
||||
try:
|
||||
token = TOKEN_PATH.read_text(encoding="utf-8").strip()
|
||||
request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"})
|
||||
context = ssl.create_default_context(cafile=str(CA_PATH))
|
||||
with urlopen(request, timeout=3, context=context) as response:
|
||||
payload = json.load(response)
|
||||
owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip()
|
||||
except Exception:
|
||||
owner = "unavailable"
|
||||
_cached_owner = owner
|
||||
_cached_at = now
|
||||
return owner
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
"""Proxy local model traffic while exposing health and ownership status."""
|
||||
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def _json(self, status: int, payload: dict) -> None:
|
||||
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _local_allowed(self) -> tuple[bool, str]:
|
||||
owner = _lease_owner()
|
||||
return owner == "hermes", owner
|
||||
|
||||
def _proxy(self) -> None:
|
||||
allowed, owner = self._local_allowed()
|
||||
if not allowed:
|
||||
self._json(
|
||||
503,
|
||||
{
|
||||
"error": {
|
||||
"message": f"local GPU inference unavailable while titan-24 owner is {owner}",
|
||||
"type": "server_error",
|
||||
},
|
||||
"gpu_owner": owner,
|
||||
"fallback_required": True,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||
body = self.rfile.read(length) if length else None
|
||||
headers = {"Content-Type": self.headers.get("Content-Type", "application/json")}
|
||||
if self.headers.get("Accept"):
|
||||
headers["Accept"] = self.headers["Accept"]
|
||||
request = Request(f"{UPSTREAM_URL}{self.path}", data=body, headers=headers, method=self.command)
|
||||
try:
|
||||
response = urlopen(request, timeout=1800)
|
||||
except HTTPError as exc:
|
||||
response = exc
|
||||
except (TimeoutError, URLError) as exc:
|
||||
self._json(503, {"error": {"message": f"local model upstream unavailable: {exc}", "type": "server_error"}})
|
||||
return
|
||||
|
||||
self.send_response(response.status)
|
||||
content_type = response.headers.get("Content-Type")
|
||||
if content_type:
|
||||
self.send_header("Content-Type", content_type)
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
self.send_header("Content-Length", content_length)
|
||||
else:
|
||||
self.send_header("Connection", "close")
|
||||
self.close_connection = True
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
while True:
|
||||
chunk = response.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
self.wfile.write(chunk)
|
||||
self.wfile.flush()
|
||||
response.close()
|
||||
|
||||
def do_GET(self) -> None:
|
||||
if self.path == "/healthz":
|
||||
self._json(200, {"status": "ok"})
|
||||
return
|
||||
if self.path == "/gate/status":
|
||||
allowed, owner = self._local_allowed()
|
||||
self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed})
|
||||
return
|
||||
self._proxy()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
self._proxy()
|
||||
|
||||
def log_message(self, format_string: str, *args) -> None:
|
||||
print(f"model-gate {self.address_string()} {format_string % args}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever()
|
||||
124
services/hermes/model-gate-deployment.yaml
Normal file
124
services/hermes/model-gate-deployment.yaml
Normal file
@ -0,0 +1,124 @@
|
||||
# services/hermes/model-gate-deployment.yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
labels:
|
||||
app: hermes-model-gate
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: hermes-model-gate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: hermes-model-gate
|
||||
spec:
|
||||
serviceAccountName: hermes-model-gate
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- arm64
|
||||
- key: node-role.kubernetes.io/worker
|
||||
operator: In
|
||||
values:
|
||||
- "true"
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-13
|
||||
- titan-15
|
||||
- titan-17
|
||||
- titan-19
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 90
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: hardware
|
||||
operator: In
|
||||
values:
|
||||
- rpi5
|
||||
containers:
|
||||
- name: model-gate
|
||||
image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- python
|
||||
- /opt/model-gate/model_gate.py
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
env:
|
||||
- name: UPSTREAM_URL
|
||||
value: http://hermes-ollama.hermes.svc.cluster.local:11434
|
||||
- name: LEASE_NAMESPACE
|
||||
value: hermes
|
||||
- name: LEASE_NAME
|
||||
value: titan-24-gpu-owner
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 128Mi
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /opt/model-gate
|
||||
readOnly: true
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: hermes-model-gate
|
||||
defaultMode: 0555
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
labels:
|
||||
app: hermes-model-gate
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: hermes-model-gate
|
||||
ports:
|
||||
- name: http
|
||||
port: 11434
|
||||
targetPort: http
|
||||
34
services/hermes/model-gate-rbac.yaml
Normal file
34
services/hermes/model-gate-rbac.yaml
Normal file
@ -0,0 +1,34 @@
|
||||
# services/hermes/model-gate-rbac.yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
rules:
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources:
|
||||
- leases
|
||||
resourceNames:
|
||||
- titan-24-gpu-owner
|
||||
verbs:
|
||||
- get
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: hermes-model-gate
|
||||
namespace: hermes
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: hermes-model-gate
|
||||
10
services/hermes/model-gate-state.yaml
Normal file
10
services/hermes/model-gate-state.yaml
Normal file
@ -0,0 +1,10 @@
|
||||
# services/hermes/model-gate-state.yaml
|
||||
apiVersion: coordination.k8s.io/v1
|
||||
kind: Lease
|
||||
metadata:
|
||||
name: titan-24-gpu-owner
|
||||
namespace: hermes
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
|
||||
spec:
|
||||
holderIdentity: hermes
|
||||
30
services/hermes/networkpolicy.yaml
Normal file
30
services/hermes/networkpolicy.yaml
Normal file
@ -0,0 +1,30 @@
|
||||
# services/hermes/networkpolicy.yaml
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: hermes-ollama-ingress
|
||||
namespace: hermes
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: hermes-ollama
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: hermes-model-gate
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 11434
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: maintenance
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: ariadne
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 11434
|
||||
@ -18,8 +18,8 @@ spec:
|
||||
labels:
|
||||
app: hermes-ollama
|
||||
annotations:
|
||||
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
|
||||
ai.bstein.dev/gpu: accelerator MVP lane (titan-24)
|
||||
ai.bstein.dev/model: gpt-oss:20b
|
||||
ai.bstein.dev/gpu: titan-24 local-first lane
|
||||
spec:
|
||||
runtimeClassName: nvidia
|
||||
affinity:
|
||||
@ -33,7 +33,8 @@ spec:
|
||||
- titan-24
|
||||
volumes:
|
||||
- name: models
|
||||
emptyDir: {}
|
||||
persistentVolumeClaim:
|
||||
claimName: hermes-models
|
||||
initContainers:
|
||||
- name: warm-model
|
||||
image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
|
||||
@ -44,7 +45,7 @@ spec:
|
||||
- name: OLLAMA_MODELS
|
||||
value: /root/.ollama
|
||||
- name: OLLAMA_MODEL
|
||||
value: qwen2.5:7b-instruct-q4_0
|
||||
value: gpt-oss:20b
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: all
|
||||
- name: NVIDIA_DRIVER_CAPABILITIES
|
||||
@ -68,7 +69,7 @@ spec:
|
||||
nvidia.com/gpu.shared: 1
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 10Gi
|
||||
memory: 16Gi
|
||||
nvidia.com/gpu.shared: 1
|
||||
containers:
|
||||
- name: ollama
|
||||
@ -82,6 +83,16 @@ spec:
|
||||
value: 0.0.0.0
|
||||
- name: OLLAMA_KEEP_ALIVE
|
||||
value: 6h
|
||||
- name: OLLAMA_CONTEXT_LENGTH
|
||||
value: "64000"
|
||||
- name: OLLAMA_FLASH_ATTENTION
|
||||
value: "1"
|
||||
- name: OLLAMA_KV_CACHE_TYPE
|
||||
value: q8_0
|
||||
- name: OLLAMA_MAX_LOADED_MODELS
|
||||
value: "1"
|
||||
- name: OLLAMA_NUM_PARALLEL
|
||||
value: "1"
|
||||
- name: OLLAMA_MODELS
|
||||
value: /root/.ollama
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
@ -100,10 +111,10 @@ spec:
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
cpu: "8"
|
||||
memory: 24Gi
|
||||
nvidia.com/gpu.shared: 1
|
||||
limits:
|
||||
cpu: "6"
|
||||
memory: 12Gi
|
||||
cpu: "16"
|
||||
memory: 40Gi
|
||||
nvidia.com/gpu.shared: 1
|
||||
|
||||
@ -13,3 +13,18 @@ spec:
|
||||
resources:
|
||||
requests:
|
||||
storage: 4Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: hermes-models
|
||||
namespace: hermes
|
||||
labels:
|
||||
app: hermes-ollama
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: local-path
|
||||
resources:
|
||||
requests:
|
||||
storage: 30Gi
|
||||
|
||||
59
services/hermes/skills/triage-titan-test-failures/SKILL.md
Normal file
59
services/hermes/skills/triage-titan-test-failures/SKILL.md
Normal file
@ -0,0 +1,59 @@
|
||||
---
|
||||
name: triage-titan-test-failures
|
||||
description: Diagnose Titan CI test failures and environment regressions from Ariadne evidence, Jenkins artifacts, Flux state, Kubernetes health, Pushgateway quality metrics, and Grafana context. Use for failed or flaky in-scope suites, suspected cluster-caused test failures, release-quality questions, or requests for a supervised read-only triage summary and repo-side next steps.
|
||||
---
|
||||
|
||||
# Triage Titan Test Failures
|
||||
|
||||
Follow the established Titan evidence path. Keep the investigation read-only and distinguish observed facts from inference.
|
||||
|
||||
## Collect the canonical evidence
|
||||
|
||||
1. Read the latest Ariadne diagnosis:
|
||||
|
||||
```sh
|
||||
curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"
|
||||
```
|
||||
|
||||
2. Read the deterministic bundle when the diagnosis is missing, stale, unavailable, or lacks evidence:
|
||||
|
||||
```sh
|
||||
curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"
|
||||
```
|
||||
|
||||
3. Ask for human approval before triggering a fresh collection or diagnosis. Use these only after approval:
|
||||
|
||||
```sh
|
||||
curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/collect"
|
||||
curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"
|
||||
```
|
||||
|
||||
Treat Ariadne's bundle as the evidence source of truth. A local-model diagnosis may be unavailable while Wolf owns titan-24; continue from the stored bundle using the active Hermes fallback model.
|
||||
|
||||
## Narrow the failure
|
||||
|
||||
Work in this order:
|
||||
|
||||
1. Confirm the failed suite and build are in the canonical scope: `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `metis`, `pegasus`, `soteria`, or `titan_iac`.
|
||||
2. Identify the first failed gate in the enforced order: `style`, `loc`, `coverage`, `tests`, `gate_glue`, `sonarqube`, `supply_chain`.
|
||||
3. Correlate the build timestamp with retained Jenkins logs/artifacts, recent Git commits, and Flux revisions.
|
||||
4. Check whether Kubernetes health, node pressure, image pulls, storage, DNS, or a shared dependency explains the failure better than a repo regression.
|
||||
5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry.
|
||||
6. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause.
|
||||
|
||||
Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands.
|
||||
|
||||
## Produce the triage result
|
||||
|
||||
Return these sections:
|
||||
|
||||
- `Finding`: one sentence naming the most likely failure class.
|
||||
- `Confidence`: low, medium, or high, with the reason.
|
||||
- `Evidence`: the smallest set of concrete timestamps, build IDs, artifact paths, commits, metrics, pods, nodes, or Flux revisions that support the finding.
|
||||
- `Likely cause`: explain the causal chain and label inference as inference.
|
||||
- `Blast radius`: affected suites, services, branches, or environments.
|
||||
- `Next checks`: ordered read-only checks with exact commands or URLs.
|
||||
- `Repo-side fix`: the smallest Flux/IaC or application change, or `none yet` when evidence is insufficient.
|
||||
- `Approval required`: call out every step that would modify files, infrastructure, credentials, test environments, or external systems.
|
||||
|
||||
Never present a proposed fix as applied. Prefer a concise evidence-backed answer over a general log summary.
|
||||
@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "Triage Titan Test Failures"
|
||||
short_description: "Analyze evidence for supervised test-failure triage"
|
||||
default_prompt: "Use $triage-titan-test-failures to diagnose the latest Titan test failure from Ariadne evidence."
|
||||
@ -285,8 +285,21 @@ spec:
|
||||
- name: GAME_MODE_NODE_NAME
|
||||
value: titan-24
|
||||
- name: GAME_MODE_DISPLACE_WORKLOADS
|
||||
value: >-
|
||||
[{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}]
|
||||
value: "[]"
|
||||
- name: GAME_MODE_LEASE_NAMESPACE
|
||||
value: hermes
|
||||
- name: GAME_MODE_LEASE_NAME
|
||||
value: titan-24-gpu-owner
|
||||
- name: GAME_MODE_OLLAMA_URL
|
||||
value: http://hermes-ollama.hermes.svc.cluster.local:11434
|
||||
- name: GAME_MODE_OLLAMA_MODEL
|
||||
value: gpt-oss:20b
|
||||
- name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC
|
||||
value: "900"
|
||||
- name: GAME_MODE_TRANSITION_TIMEOUT_SEC
|
||||
value: "900"
|
||||
- name: GAME_MODE_POLL_INTERVAL_SEC
|
||||
value: "1"
|
||||
- name: WOLF_OIDC_CLIENT_ID
|
||||
value: wolf
|
||||
- name: WOLF_OIDC_BASE_URL
|
||||
@ -444,11 +457,11 @@ spec:
|
||||
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
|
||||
value: "*/15 * * * *"
|
||||
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
|
||||
value: http://hermes-ollama.hermes.svc.cluster.local:11434
|
||||
value: http://hermes-model-gate.hermes.svc.cluster.local:11434
|
||||
- name: ARIADNE_TESTING_TRIAGE_MODEL
|
||||
value: qwen2.5:7b-instruct-q4_0
|
||||
value: gpt-oss:20b
|
||||
- name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC
|
||||
value: "180"
|
||||
value: "900"
|
||||
- name: JENKINS_WORKSPACE_NAMESPACE
|
||||
value: jenkins
|
||||
- name: JENKINS_WORKSPACE_PVC_PREFIX
|
||||
|
||||
@ -78,7 +78,6 @@ rules:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user