From be91902a018a5fb90fbd8ba7cf2850a7b48bd9e3 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sat, 1 Aug 2026 22:38:07 -0300 Subject: [PATCH] gpu(titan-24): add Hermes local-first fallback handoff --- .../applications/hermes/kustomization.yaml | 4 + .../bstein-dev-home/backend-deployment.yaml | 4 +- services/hermes/ariadne-handoff-rbac.yaml | 30 ++++ services/hermes/configmap.yaml | 19 ++- services/hermes/deployment.yaml | 15 +- services/hermes/kustomization.yaml | 15 ++ services/hermes/model-gate-configmap.yaml | 151 ++++++++++++++++++ services/hermes/model-gate-deployment.yaml | 124 ++++++++++++++ services/hermes/model-gate-rbac.yaml | 34 ++++ services/hermes/model-gate-state.yaml | 10 ++ services/hermes/networkpolicy.yaml | 30 ++++ services/hermes/ollama-deployment.yaml | 29 ++-- services/hermes/pvc.yaml | 15 ++ .../triage-titan-test-failures/SKILL.md | 59 +++++++ .../agents/openai.yaml | 4 + services/maintenance/ariadne-deployment.yaml | 23 ++- services/maintenance/ariadne-rbac.yaml | 1 - 17 files changed, 546 insertions(+), 21 deletions(-) create mode 100644 services/hermes/ariadne-handoff-rbac.yaml create mode 100644 services/hermes/model-gate-configmap.yaml create mode 100644 services/hermes/model-gate-deployment.yaml create mode 100644 services/hermes/model-gate-rbac.yaml create mode 100644 services/hermes/model-gate-state.yaml create mode 100644 services/hermes/networkpolicy.yaml create mode 100644 services/hermes/skills/triage-titan-test-failures/SKILL.md create mode 100644 services/hermes/skills/triage-titan-test-failures/agents/openai.yaml diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 6a16b1b51..ec31a55c0 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -18,6 +18,10 @@ spec: wait: true timeout: 45m healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-model-gate + namespace: hermes - apiVersion: apps/v1 kind: Deployment name: hermes-ollama diff --git a/services/bstein-dev-home/backend-deployment.yaml b/services/bstein-dev-home/backend-deployment.yaml index fbaa425ee..b39244ee4 100644 --- a/services/bstein-dev-home/backend-deployment.yaml +++ b/services/bstein-dev-home/backend-deployment.yaml @@ -68,7 +68,7 @@ spec: args: - >- . /vault/secrets/portal-env.sh - && exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 600 app:app + && exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 1200 app:app env: - name: AI_CHAT_API value: http://ollama.ai.svc.cluster.local:11434 @@ -119,6 +119,8 @@ spec: value: http://ariadne.maintenance.svc.cluster.local - name: ARIADNE_TIMEOUT_SEC value: "10" + - name: ARIADNE_GAME_MODE_TIMEOUT_SEC + value: "900" - name: ACCOUNT_ALLOWED_GROUPS value: "" - name: HTTP_CHECK_TIMEOUT_SEC diff --git a/services/hermes/ariadne-handoff-rbac.yaml b/services/hermes/ariadne-handoff-rbac.yaml new file mode 100644 index 000000000..a8da16cc7 --- /dev/null +++ b/services/hermes/ariadne-handoff-rbac.yaml @@ -0,0 +1,30 @@ +# services/hermes/ariadne-handoff-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ariadne-gpu-handoff + namespace: hermes +rules: + - apiGroups: ["coordination.k8s.io"] + resources: + - leases + resourceNames: + - titan-24-gpu-owner + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ariadne-gpu-handoff + namespace: hermes +subjects: + - kind: ServiceAccount + name: ariadne + namespace: maintenance +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ariadne-gpu-handoff diff --git a/services/hermes/configmap.yaml b/services/hermes/configmap.yaml index e751f45a2..af663e113 100644 --- a/services/hermes/configmap.yaml +++ b/services/hermes/configmap.yaml @@ -10,11 +10,24 @@ data: config.yaml: | model: provider: custom - default: qwen2.5:7b-instruct-q4_0 - model: qwen2.5:7b-instruct-q4_0 - base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1 + default: gpt-oss:20b + model: gpt-oss:20b + context_length: 64000 + base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 api_key: ollama + fallback_providers: + - provider: openai-codex + model: gpt-5.4 + + agent: + api_max_retries: 1 + + skills: + creation_nudge_interval: 15 + external_dirs: + - /opt/data/workspace/skills + terminal: backend: local cwd: /opt/data/workspace diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 75437fe6d..b7035a298 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -20,10 +20,10 @@ spec: labels: app: hermes annotations: - ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0 + ai.bstein.dev/model: gpt-oss:20b with openai-codex fallback ai.bstein.dev/role: testing-triage ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred) - ai.bstein.dev/config-rev: "20260721-hermes-rollout-deadline" + ai.bstein.dev/config-rev: "20260801-gpu-owner-fallback" spec: serviceAccountName: hermes-triage automountServiceAccountToken: true @@ -190,6 +190,9 @@ spec: mountPath: /opt/data - name: tools mountPath: /opt/data/home/.local/bin + - name: triage-skill + mountPath: /opt/data/workspace/skills/triage-titan-test-failures + readOnly: true readinessProbe: httpGet: path: /api/status @@ -220,3 +223,11 @@ spec: name: hermes-config - name: tools emptyDir: {} + - name: triage-skill + configMap: + name: hermes-triage-skill + items: + - key: SKILL.md + path: SKILL.md + - key: openai.yaml + path: agents/openai.yaml diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 96e642e4a..7287255d9 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -7,8 +7,23 @@ resources: - configmap.yaml - rbac.yaml - pvc.yaml + - model-gate-rbac.yaml + - ariadne-handoff-rbac.yaml + - model-gate-state.yaml + - model-gate-configmap.yaml + - model-gate-deployment.yaml + - networkpolicy.yaml - ollama-deployment.yaml - deployment.yaml - service.yaml - agent-certificate.yaml - agent-ingress.yaml + +configMapGenerator: + - name: hermes-triage-skill + namespace: hermes + files: + - SKILL.md=skills/triage-titan-test-failures/SKILL.md + - openai.yaml=skills/triage-titan-test-failures/agents/openai.yaml + options: + disableNameSuffixHash: true diff --git a/services/hermes/model-gate-configmap.yaml b/services/hermes/model-gate-configmap.yaml new file mode 100644 index 000000000..cf179d84d --- /dev/null +++ b/services/hermes/model-gate-configmap.yaml @@ -0,0 +1,151 @@ +# services/hermes/model-gate-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-model-gate + namespace: hermes +data: + model_gate.py: | + #!/usr/bin/env python3 + """Fail-closed proxy that admits local inference only while Hermes owns titan-24.""" + + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + import json + import os + from pathlib import Path + import ssl + import threading + import time + from urllib.error import HTTPError, URLError + from urllib.request import Request, urlopen + + + LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0") + LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080")) + UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/") + LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes") + LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner") + CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1")) + API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc") + API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443") + TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token") + CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + LEASE_URL = ( + f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/" + f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}" + ) + + _cache_lock = threading.Lock() + _cached_owner = "unavailable" + _cached_at = 0.0 + + + def _lease_owner() -> str: + """Return the current owner, failing closed when Kubernetes is unavailable.""" + + global _cached_at, _cached_owner + now = time.monotonic() + with _cache_lock: + if now - _cached_at < CACHE_TTL_SEC: + return _cached_owner + try: + token = TOKEN_PATH.read_text(encoding="utf-8").strip() + request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"}) + context = ssl.create_default_context(cafile=str(CA_PATH)) + with urlopen(request, timeout=3, context=context) as response: + payload = json.load(response) + owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip() + except Exception: + owner = "unavailable" + _cached_owner = owner + _cached_at = now + return owner + + + class Handler(BaseHTTPRequestHandler): + """Proxy local model traffic while exposing health and ownership status.""" + + protocol_version = "HTTP/1.1" + + def _json(self, status: int, payload: dict) -> None: + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _local_allowed(self) -> tuple[bool, str]: + owner = _lease_owner() + return owner == "hermes", owner + + def _proxy(self) -> None: + allowed, owner = self._local_allowed() + if not allowed: + self._json( + 503, + { + "error": { + "message": f"local GPU inference unavailable while titan-24 owner is {owner}", + "type": "server_error", + }, + "gpu_owner": owner, + "fallback_required": True, + }, + ) + return + + length = int(self.headers.get("Content-Length", "0") or 0) + body = self.rfile.read(length) if length else None + headers = {"Content-Type": self.headers.get("Content-Type", "application/json")} + if self.headers.get("Accept"): + headers["Accept"] = self.headers["Accept"] + request = Request(f"{UPSTREAM_URL}{self.path}", data=body, headers=headers, method=self.command) + try: + response = urlopen(request, timeout=1800) + except HTTPError as exc: + response = exc + except (TimeoutError, URLError) as exc: + self._json(503, {"error": {"message": f"local model upstream unavailable: {exc}", "type": "server_error"}}) + return + + self.send_response(response.status) + content_type = response.headers.get("Content-Type") + if content_type: + self.send_header("Content-Type", content_type) + content_length = response.headers.get("Content-Length") + if content_length: + self.send_header("Content-Length", content_length) + else: + self.send_header("Connection", "close") + self.close_connection = True + self.send_header("Cache-Control", "no-store") + self.end_headers() + while True: + chunk = response.read(65536) + if not chunk: + break + self.wfile.write(chunk) + self.wfile.flush() + response.close() + + def do_GET(self) -> None: + if self.path == "/healthz": + self._json(200, {"status": "ok"}) + return + if self.path == "/gate/status": + allowed, owner = self._local_allowed() + self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed}) + return + self._proxy() + + def do_POST(self) -> None: + self._proxy() + + def log_message(self, format_string: str, *args) -> None: + print(f"model-gate {self.address_string()} {format_string % args}", flush=True) + + + if __name__ == "__main__": + ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever() diff --git a/services/hermes/model-gate-deployment.yaml b/services/hermes/model-gate-deployment.yaml new file mode 100644 index 000000000..2fc83aa70 --- /dev/null +++ b/services/hermes/model-gate-deployment.yaml @@ -0,0 +1,124 @@ +# services/hermes/model-gate-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-model-gate + namespace: hermes + labels: + app: hermes-model-gate +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-model-gate + template: + metadata: + labels: + app: hermes-model-gate + spec: + serviceAccountName: hermes-model-gate + securityContext: + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-13 + - titan-15 + - titan-17 + - titan-19 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 90 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + containers: + - name: model-gate + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + imagePullPolicy: IfNotPresent + command: + - python + - /opt/model-gate/model_gate.py + ports: + - name: http + containerPort: 8080 + env: + - name: UPSTREAM_URL + value: http://hermes-ollama.hermes.svc.cluster.local:11434 + - name: LEASE_NAMESPACE + value: hermes + - name: LEASE_NAME + value: titan-24-gpu-owner + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + volumeMounts: + - name: script + mountPath: /opt/model-gate + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: script + configMap: + name: hermes-model-gate + defaultMode: 0555 + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-model-gate + namespace: hermes + labels: + app: hermes-model-gate +spec: + type: ClusterIP + selector: + app: hermes-model-gate + ports: + - name: http + port: 11434 + targetPort: http diff --git a/services/hermes/model-gate-rbac.yaml b/services/hermes/model-gate-rbac.yaml new file mode 100644 index 000000000..1c56556ac --- /dev/null +++ b/services/hermes/model-gate-rbac.yaml @@ -0,0 +1,34 @@ +# services/hermes/model-gate-rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-model-gate + namespace: hermes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: hermes-model-gate + namespace: hermes +rules: + - apiGroups: ["coordination.k8s.io"] + resources: + - leases + resourceNames: + - titan-24-gpu-owner + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hermes-model-gate + namespace: hermes +subjects: + - kind: ServiceAccount + name: hermes-model-gate + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hermes-model-gate diff --git a/services/hermes/model-gate-state.yaml b/services/hermes/model-gate-state.yaml new file mode 100644 index 000000000..756591187 --- /dev/null +++ b/services/hermes/model-gate-state.yaml @@ -0,0 +1,10 @@ +# services/hermes/model-gate-state.yaml +apiVersion: coordination.k8s.io/v1 +kind: Lease +metadata: + name: titan-24-gpu-owner + namespace: hermes + annotations: + kustomize.toolkit.fluxcd.io/ssa: IfNotPresent +spec: + holderIdentity: hermes diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml new file mode 100644 index 000000000..57bfa577f --- /dev/null +++ b/services/hermes/networkpolicy.yaml @@ -0,0 +1,30 @@ +# services/hermes/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-ollama-ingress + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-ollama + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-model-gate + ports: + - protocol: TCP + port: 11434 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: maintenance + podSelector: + matchLabels: + app: ariadne + ports: + - protocol: TCP + port: 11434 diff --git a/services/hermes/ollama-deployment.yaml b/services/hermes/ollama-deployment.yaml index 946aa7b12..64a351f59 100644 --- a/services/hermes/ollama-deployment.yaml +++ b/services/hermes/ollama-deployment.yaml @@ -18,8 +18,8 @@ spec: labels: app: hermes-ollama annotations: - ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0 - ai.bstein.dev/gpu: accelerator MVP lane (titan-24) + ai.bstein.dev/model: gpt-oss:20b + ai.bstein.dev/gpu: titan-24 local-first lane spec: runtimeClassName: nvidia affinity: @@ -33,7 +33,8 @@ spec: - titan-24 volumes: - name: models - emptyDir: {} + persistentVolumeClaim: + claimName: hermes-models initContainers: - name: warm-model image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d @@ -44,7 +45,7 @@ spec: - name: OLLAMA_MODELS value: /root/.ollama - name: OLLAMA_MODEL - value: qwen2.5:7b-instruct-q4_0 + value: gpt-oss:20b - name: NVIDIA_VISIBLE_DEVICES value: all - name: NVIDIA_DRIVER_CAPABILITIES @@ -68,7 +69,7 @@ spec: nvidia.com/gpu.shared: 1 limits: cpu: "4" - memory: 10Gi + memory: 16Gi nvidia.com/gpu.shared: 1 containers: - name: ollama @@ -82,6 +83,16 @@ spec: value: 0.0.0.0 - name: OLLAMA_KEEP_ALIVE value: 6h + - name: OLLAMA_CONTEXT_LENGTH + value: "64000" + - name: OLLAMA_FLASH_ATTENTION + value: "1" + - name: OLLAMA_KV_CACHE_TYPE + value: q8_0 + - name: OLLAMA_MAX_LOADED_MODELS + value: "1" + - name: OLLAMA_NUM_PARALLEL + value: "1" - name: OLLAMA_MODELS value: /root/.ollama - name: NVIDIA_VISIBLE_DEVICES @@ -100,10 +111,10 @@ spec: timeoutSeconds: 5 resources: requests: - cpu: "2" - memory: 8Gi + cpu: "8" + memory: 24Gi nvidia.com/gpu.shared: 1 limits: - cpu: "6" - memory: 12Gi + cpu: "16" + memory: 40Gi nvidia.com/gpu.shared: 1 diff --git a/services/hermes/pvc.yaml b/services/hermes/pvc.yaml index 85812f5ef..9ff1830fb 100644 --- a/services/hermes/pvc.yaml +++ b/services/hermes/pvc.yaml @@ -13,3 +13,18 @@ spec: resources: requests: storage: 4Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hermes-models + namespace: hermes + labels: + app: hermes-ollama +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 30Gi diff --git a/services/hermes/skills/triage-titan-test-failures/SKILL.md b/services/hermes/skills/triage-titan-test-failures/SKILL.md new file mode 100644 index 000000000..a5c4eb6d2 --- /dev/null +++ b/services/hermes/skills/triage-titan-test-failures/SKILL.md @@ -0,0 +1,59 @@ +--- +name: triage-titan-test-failures +description: Diagnose Titan CI test failures and environment regressions from Ariadne evidence, Jenkins artifacts, Flux state, Kubernetes health, Pushgateway quality metrics, and Grafana context. Use for failed or flaky in-scope suites, suspected cluster-caused test failures, release-quality questions, or requests for a supervised read-only triage summary and repo-side next steps. +--- + +# Triage Titan Test Failures + +Follow the established Titan evidence path. Keep the investigation read-only and distinguish observed facts from inference. + +## Collect the canonical evidence + +1. Read the latest Ariadne diagnosis: + + ```sh + curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest" + ``` + +2. Read the deterministic bundle when the diagnosis is missing, stale, unavailable, or lacks evidence: + + ```sh + curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest" + ``` + +3. Ask for human approval before triggering a fresh collection or diagnosis. Use these only after approval: + + ```sh + curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/collect" + curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run" + ``` + +Treat Ariadne's bundle as the evidence source of truth. A local-model diagnosis may be unavailable while Wolf owns titan-24; continue from the stored bundle using the active Hermes fallback model. + +## Narrow the failure + +Work in this order: + +1. Confirm the failed suite and build are in the canonical scope: `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `metis`, `pegasus`, `soteria`, or `titan_iac`. +2. Identify the first failed gate in the enforced order: `style`, `loc`, `coverage`, `tests`, `gate_glue`, `sonarqube`, `supply_chain`. +3. Correlate the build timestamp with retained Jenkins logs/artifacts, recent Git commits, and Flux revisions. +4. Check whether Kubernetes health, node pressure, image pulls, storage, DNS, or a shared dependency explains the failure better than a repo regression. +5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry. +6. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause. + +Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands. + +## Produce the triage result + +Return these sections: + +- `Finding`: one sentence naming the most likely failure class. +- `Confidence`: low, medium, or high, with the reason. +- `Evidence`: the smallest set of concrete timestamps, build IDs, artifact paths, commits, metrics, pods, nodes, or Flux revisions that support the finding. +- `Likely cause`: explain the causal chain and label inference as inference. +- `Blast radius`: affected suites, services, branches, or environments. +- `Next checks`: ordered read-only checks with exact commands or URLs. +- `Repo-side fix`: the smallest Flux/IaC or application change, or `none yet` when evidence is insufficient. +- `Approval required`: call out every step that would modify files, infrastructure, credentials, test environments, or external systems. + +Never present a proposed fix as applied. Prefer a concise evidence-backed answer over a general log summary. diff --git a/services/hermes/skills/triage-titan-test-failures/agents/openai.yaml b/services/hermes/skills/triage-titan-test-failures/agents/openai.yaml new file mode 100644 index 000000000..f29a6d9b8 --- /dev/null +++ b/services/hermes/skills/triage-titan-test-failures/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Triage Titan Test Failures" + short_description: "Analyze evidence for supervised test-failure triage" + default_prompt: "Use $triage-titan-test-failures to diagnose the latest Titan test failure from Ariadne evidence." diff --git a/services/maintenance/ariadne-deployment.yaml b/services/maintenance/ariadne-deployment.yaml index f39ddba3a..6b57cf858 100644 --- a/services/maintenance/ariadne-deployment.yaml +++ b/services/maintenance/ariadne-deployment.yaml @@ -285,8 +285,21 @@ spec: - name: GAME_MODE_NODE_NAME value: titan-24 - name: GAME_MODE_DISPLACE_WORKLOADS - value: >- - [{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}] + value: "[]" + - name: GAME_MODE_LEASE_NAMESPACE + value: hermes + - name: GAME_MODE_LEASE_NAME + value: titan-24-gpu-owner + - name: GAME_MODE_OLLAMA_URL + value: http://hermes-ollama.hermes.svc.cluster.local:11434 + - name: GAME_MODE_OLLAMA_MODEL + value: gpt-oss:20b + - name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC + value: "900" + - name: GAME_MODE_TRANSITION_TIMEOUT_SEC + value: "900" + - name: GAME_MODE_POLL_INTERVAL_SEC + value: "1" - name: WOLF_OIDC_CLIENT_ID value: wolf - name: WOLF_OIDC_BASE_URL @@ -444,11 +457,11 @@ spec: - name: ARIADNE_SCHEDULE_TESTING_TRIAGE value: "*/15 * * * *" - name: ARIADNE_TESTING_TRIAGE_MODEL_URL - value: http://hermes-ollama.hermes.svc.cluster.local:11434 + value: http://hermes-model-gate.hermes.svc.cluster.local:11434 - name: ARIADNE_TESTING_TRIAGE_MODEL - value: qwen2.5:7b-instruct-q4_0 + value: gpt-oss:20b - name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC - value: "180" + value: "900" - name: JENKINS_WORKSPACE_NAMESPACE value: jenkins - name: JENKINS_WORKSPACE_PVC_PREFIX diff --git a/services/maintenance/ariadne-rbac.yaml b/services/maintenance/ariadne-rbac.yaml index 73c5e3b55..cdc455835 100644 --- a/services/maintenance/ariadne-rbac.yaml +++ b/services/maintenance/ariadne-rbac.yaml @@ -78,7 +78,6 @@ rules: - get - list - watch - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding