Deploy Cassandra 0.8.72

clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml

@@ -1,28 +0,0 @@
-# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: hermes-chat
-  namespace: flux-system
-spec:
-  interval: 10m
-  path: ./services/hermes-chat
-  targetNamespace: hermes-chat
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-    namespace: flux-system
-  wait: true
-  timeout: 30m
-  healthChecks:
-    - apiVersion: apps/v1
-      kind: Deployment
-      name: hermes-chat
-      namespace: hermes-chat
-  dependsOn:
-    - name: cert-manager
-    - name: core
-    - name: hermes
-    - name: keycloak
-    - name: longhorn

clusters/atlas/flux-system/applications/hermes/kustomization.yaml

@@ -18,10 +18,6 @@ spec:
   wait: true
   timeout: 45m
   healthChecks:
-    - apiVersion: apps/v1
-      kind: Deployment
-      name: hermes-model-gate
-      namespace: hermes
     - apiVersion: apps/v1
       kind: Deployment
       name: hermes-ollama
@@ -30,10 +26,6 @@ spec:
       kind: Deployment
       name: hermes
       namespace: hermes
-    - apiVersion: apps/v1
-      kind: Deployment
-      name: oauth2-proxy-hermes
-      namespace: hermes
   dependsOn:
     - name: cert-manager
     - name: core

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -28,7 +28,6 @@ resources:
   - ai-llm/kustomization.yaml
   - openclaw/kustomization.yaml
   - hermes/kustomization.yaml
-  - hermes-chat/kustomization.yaml
   - game-stream/kustomization.yaml
   - cassandra-auth/kustomization.yaml
   - cassandra/kustomization.yaml

dockerfiles/Dockerfile.hermes-agent

@@ -1,122 +0,0 @@
-# syntax=docker/dockerfile:1
-# dockerfiles/Dockerfile.hermes-agent
-FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
-
-USER root
-
-# Keep dashboard chat sockets tied to the intended React mount and conversation.
-# A resumed conversation needs a different PTY attachment key from a fresh chat;
-# reconnects to that same conversation must keep using the same key.
-RUN node <<'NODE'
-const fs = require("node:fs");
-const path = "/opt/hermes/web/src/pages/ChatPage.tsx";
-let source = fs.readFileSync(path, "utf8");
-const socketBefore = [
-  '      const url = await api.buildWsUrl("/api/pty", params);',
-  '      const ws = new WebSocket(url);',
-].join("\n");
-const socketAfter = [
-  '      const url = await api.buildWsUrl("/api/pty", params);',
-  '      if (unmounting) return;',
-  '      const ws = new WebSocket(url);',
-].join("\n");
-const attachBefore = '      params.attach = ptyAttachToken(forceFresh);';
-const attachAfter = [
-  '      const attachScope = resumeParam',
-  '        ? `resume:${resumeParam}:${scopedProfile ?? ""}`',
-  '        : `fresh:${scopedProfile ?? ""}`;',
-  '      params.attach = `${ptyAttachToken(forceFresh)}:${attachScope}`;',
-].join("\n");
-
-if (!source.includes(socketBefore)) {
-  throw new Error("Hermes ChatPage WebSocket patch context changed");
-}
-if (!source.includes(attachBefore)) {
-  throw new Error("Hermes ChatPage PTY attachment patch context changed");
-}
-source = source.replace(socketBefore, socketAfter);
-source = source.replace(attachBefore, attachAfter);
-fs.writeFileSync(path, source);
-NODE
-
-# The upstream OIDC gate authenticates users but deliberately treats the
-# dashboard as one shared workstation. Allow a deployment to narrow that
-# workstation to explicit OIDC subjects. Enforce this after normal provider
-# verification so a denied account is a 403, not a misleading provider 503.
-RUN python - <<'PY'
-from pathlib import Path
-
-path = Path("/opt/hermes/hermes_cli/dashboard_auth/middleware.py")
-source = path.read_text()
-helper_before = '''def _client_ip(request: Request) -> str:
-    fwd = request.headers.get("x-forwarded-for", "")
-    if fwd:
-        return fwd.split(",")[0].strip()
-    return request.client.host if request.client else ""
-
-
-'''
-helper_after = helper_before + '''def _dashboard_user_allowed(session) -> bool:
-    """Apply an optional deployment-level OIDC-subject allowlist."""
-    import os
-
-    allowed = {
-        value.strip()
-        for value in os.environ.get(
-            "HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS", ""
-        ).split(",")
-        if value.strip()
-    }
-    return not allowed or session.user_id in allowed
-
-
-def _user_forbidden_response() -> Response:
-    """Return an authorization failure without exposing identities."""
-    return JSONResponse(
-        {
-            "error": "forbidden",
-            "detail": "This Atlas account is not authorized for this dashboard.",
-        },
-        status_code=403,
-    )
-
-
-'''
-refresh_before = '''            new_session, refreshing_provider = refreshed
-            request.state.session = new_session
-            response = await call_next(request)
-'''
-refresh_after = '''            new_session, refreshing_provider = refreshed
-            if not _dashboard_user_allowed(new_session):
-                return _user_forbidden_response()
-            request.state.session = new_session
-            response = await call_next(request)
-'''
-final_before = '''    request.state.session = session
-    return await call_next(request)
-'''
-final_after = '''    if not _dashboard_user_allowed(session):
-        return _user_forbidden_response()
-    request.state.session = session
-    return await call_next(request)
-'''
-for before, after, label in (
-    (helper_before, helper_after, "allowlist helper"),
-    (refresh_before, refresh_after, "refreshed session"),
-    (final_before, final_after, "verified session"),
-):
-    if before not in source:
-        raise SystemExit(f"Hermes dashboard auth {label} patch context changed")
-    source = source.replace(before, after, 1)
-path.write_text(source)
-PY
-
-COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
-
-RUN cd /opt/hermes/web \
-    && npm run build \
-    && grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \
-    && grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \
-    && grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \
-        /opt/hermes/hermes_cli/dashboard_auth/middleware.py \
-    && chmod 0755 /opt/hermes/bin/hermes-session-migrate

dockerfiles/hermes-session-migrate.py

@@ -1,101 +0,0 @@
-#!/usr/bin/env python3
-"""Copy selected legacy Hermes sessions into an isolated user home."""
-
-from __future__ import annotations
-
-import os
-from pathlib import Path
-import shutil
-import sqlite3
-
-from hermes_state import SessionDB
-
-
-def _copy_if_missing(source: Path, target: Path) -> None:
-    """Copy one credential/config file without overwriting user state."""
-    if source.is_file() and not target.exists():
-        target.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copy2(source, target)
-
-
-def _copy_session(
-    source: sqlite3.Connection,
-    target: sqlite3.Connection,
-    session_id: str,
-    user_id: str,
-) -> bool:
-    """Copy a session and its messages, preserving original timestamps."""
-    source.row_factory = sqlite3.Row
-    session = source.execute(
-        "SELECT * FROM sessions WHERE id = ?", (session_id,)
-    ).fetchone()
-    if session is None:
-        return False
-
-    columns = list(session.keys())
-    values = [user_id if column == "user_id" else session[column] for column in columns]
-    placeholders = ", ".join("?" for _ in columns)
-    target.execute(
-        f"INSERT OR IGNORE INTO sessions ({', '.join(columns)}) "
-        f"VALUES ({placeholders})",
-        values,
-    )
-
-    message_columns = [
-        row[1] for row in target.execute("PRAGMA table_info(messages)").fetchall()
-    ]
-    selected_columns = [
-        column
-        for column in message_columns
-        if column in {
-            row[1]
-            for row in source.execute("PRAGMA table_info(messages)").fetchall()
-        }
-    ]
-    column_sql = ", ".join(selected_columns)
-    target.execute(
-        f"INSERT OR IGNORE INTO messages ({column_sql}) "
-        f"SELECT {column_sql} FROM source_db.messages WHERE session_id = ?",
-        (session_id,),
-    )
-    return True
-
-
-def main() -> None:
-    """Create the isolated home and perform the idempotent session copy."""
-    source_home = Path(os.environ["HERMES_MIGRATE_SOURCE_HOME"])
-    target_home = Path(os.environ["HERMES_HOME"])
-    user_id = os.environ["HERMES_MIGRATE_USER_ID"].strip()
-    session_ids = [
-        value.strip()
-        for value in os.environ.get("HERMES_MIGRATE_SESSION_IDS", "").split(",")
-        if value.strip()
-    ]
-
-    target_home.mkdir(parents=True, exist_ok=True)
-    (target_home / "home" / ".local" / "bin").mkdir(parents=True, exist_ok=True)
-    (target_home / "workspace" / "skills").mkdir(parents=True, exist_ok=True)
-    (target_home / "logs").mkdir(parents=True, exist_ok=True)
-    for filename in (".env", "auth.json"):
-        _copy_if_missing(source_home / filename, target_home / filename)
-
-    source_db = source_home / "state.db"
-    target_db = target_home / "state.db"
-    SessionDB(db_path=target_db).close()
-    copied = 0
-    if source_db.is_file() and session_ids:
-        source = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)
-        target = sqlite3.connect(target_db)
-        target.execute("ATTACH DATABASE ? AS source_db", (str(source_db),))
-        try:
-            with target:
-                for session_id in session_ids:
-                    copied += int(_copy_session(source, target, session_id, user_id))
-        finally:
-            target.close()
-            source.close()
-    print(f"isolated Hermes home ready; migrated_sessions={copied}")
-
-
-if __name__ == "__main__":
-    main()

infrastructure/traefik/deployment.yaml

@@ -4,6 +4,8 @@ items:
 - apiVersion: apps/v1
   kind: Deployment
   metadata:
+    annotations:
+      deployment.kubernetes.io/revision: "4"
     name: traefik
     namespace: traefik
   spec:
@@ -71,14 +73,6 @@ items:
           node-role.kubernetes.io/worker: "true"
         affinity:
           nodeAffinity:
-            requiredDuringSchedulingIgnoredDuringExecution:
-              nodeSelectorTerms:
-              - matchExpressions:
-                - key: hardware
-                  operator: In
-                  values:
-                  - rpi5
-                  - rpi4
             preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
               preference:
@@ -109,12 +103,6 @@ items:
                   operator: In
                   values:
                   - rpi4
-          podAntiAffinity:
-            requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchLabels:
-                  app: traefik
-              topologyKey: kubernetes.io/hostname
         restartPolicy: Always
         schedulerName: default-scheduler
         serviceAccount: atlas-traefik-ingress-controller

scripts/dashboards_render_atlas.py

@@ -110,6 +110,8 @@ WORKER_NODES = [
     "titan-06",
     "titan-07",
     "titan-08",
+    "titan-09",
+    "titan-10",
     "titan-11",
     "titan-20",
     "titan-21",
@@ -117,6 +119,7 @@ WORKER_NODES = [
     "titan-13",
     "titan-14",
     "titan-15",
+    "titan-16",
     "titan-17",
     "titan-18",
     "titan-19",
@@ -181,8 +184,7 @@ def scoped_node_expr(base, scope=""):
 
 def node_cpu_expr(scope=""):
     idle = 'avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))'
-    # Scrape stalls can briefly report an impossible idle rate after recovery.
-    base = f"clamp_max(clamp_min((1 - {idle}) * 100, 0), 100)"
+    base = f"(1 - {idle}) * 100"
     return scoped_node_expr(base, scope)
 
 
@@ -282,14 +284,8 @@ def dcgm_gpu_util_by_node():
     )
 
 
-def nvidia_gpu_util_by_node():
-    return "max by (node) (nvidia_gpu_device_utilization_percent)"
-
-
 def gpu_util_by_node():
-    process_exporter = nvidia_gpu_util_by_node()
-    dcgm_fallback = f"({dcgm_gpu_util_by_node()}) unless on(node) ({process_exporter})"
-    return f"{process_exporter} or {dcgm_fallback} or {jetson_gpu_util_by_node()}"
+    return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}"
 
 
 def gpu_util_by_hostname():
@@ -299,11 +295,16 @@ def gpu_util_by_hostname():
 GPU_RESOURCE_REGEX = "nvidia(_com_|[.]com/)gpu.*"
 
 
+def gpu_node_labels():
+    return f'max by (node) (kube_node_status_allocatable{{resource=~"{GPU_RESOURCE_REGEX}"}} > bool 0)'
+
+
 def gpu_requests_by_namespace_node(scope_var):
     return (
         "sum by (namespace,node) ("
         f'kube_pod_container_resource_requests{{resource=~"{GPU_RESOURCE_REGEX}",{scope_var}}} '
         "* on(namespace,pod) group_left(node) kube_pod_info "
+        f"* on(node) group_left() ({gpu_node_labels()})"
         ")"
     )
 
@@ -325,8 +326,7 @@ def gpu_usage_by_namespace(scope_var):
 
 def jetson_gpu_usage_by_namespace(scope_var):
     requests_by_ns = gpu_requests_by_namespace_node(scope_var)
-    all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
-    total_by_node = f"sum by (node) ({all_requests})"
+    total_by_node = f"sum by (node) ({requests_by_ns})"
     return (
         "sum by (namespace) ("
         f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) "
@@ -335,12 +335,6 @@ def jetson_gpu_usage_by_namespace(scope_var):
     )
 
 
-def jetson_gpu_requested_nodes():
-    all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
-    requested = f"(sum by (node) ({all_requests}) > 0)"
-    return f"({requested}) and on(node) ({jetson_gpu_util_by_node()})"
-
-
 def namespace_share_expr(resource_expr):
     total = f"clamp_min(sum( {resource_expr} ), 1)"
     return f"100 * ( {resource_expr} ) / {total}"
@@ -390,29 +384,14 @@ def gpu_total_devices_expr():
 
 
 def unattributed_gpu_usage():
-    unresolved = (
-        f"({legacy_gpu_util_without_process_exporter()}) "
-        f"unless on(node) ({jetson_gpu_requested_nodes()})"
-    )
-    legacy_total = f"(sum({unresolved}) or on() vector(0))"
+    legacy_total = f"(sum({legacy_gpu_util_without_process_exporter()}) or on() vector(0))"
     return (
         f'label_replace(({legacy_total} > 0), "namespace", "unattributed", "", "")'
     )
 
 
 def gpu_utilization_raw(scope_var):
-    nvidia = (
-        'label_replace('
-        f'{nvidia_process_gpu_usage_by_namespace(scope_var)}, '
-        '"gpu_source", "nvidia", "", "")'
-    )
-    jetson = (
-        'label_replace('
-        f'(({jetson_gpu_usage_by_namespace(scope_var)}) > 0), '
-        '"gpu_source", "jetson", "", "")'
-    )
-    attributed = f"sum by (namespace) (({nvidia}) or ({jetson}))"
-    return f"({attributed}) or ({unattributed_gpu_usage()})"
+    return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
 
 
 def gpu_pool_used_expr(scope_var):
@@ -432,20 +411,13 @@ def namespace_gpu_share_expr(scope_var):
 
 
 PROBLEM_PODS_EXPR = (
-    '((sum(max by(namespace,pod) ('
-    '(kube_pod_status_phase{phase="Pending",namespace!~"veles"} == 1) '
-    'and on(namespace,pod) ((time() - kube_pod_created{namespace!~"veles"}) > 900)'
-    ')) or on() vector(0)) + (sum(max by(namespace,pod) ('
-    '(kube_pod_status_phase{phase=~"Failed|Unknown",namespace!~"veles"} == 1) '
-    'unless on(namespace,pod) kube_pod_owner{owner_kind="Job"}'
-    ')) or on() vector(0)))'
+    'sum(max by (namespace,pod) (kube_pod_status_phase{phase!~"Running|Succeeded"})) '
+    "or on() vector(0)"
 )
 CRASHLOOP_EXPR = (
-    'sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason'
-    '{namespace!~"veles",reason=~"CrashLoopBackOff|ImagePullBackOff"} '
-    'and on(namespace,pod) '
-    '((time() - kube_pod_created{namespace!~"veles"}) > 900))) '
-    'or on() vector(0)'
+    'sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason'
+    '{reason=~"CrashLoopBackOff|ImagePullBackOff"})) '
+    "or on() vector(0)"
 )
 STUCK_TERMINATING_EXPR = (
     'sum(max by (namespace,pod) ('
@@ -456,24 +428,21 @@ STUCK_TERMINATING_EXPR = (
 )
 UPTIME_WINDOW = "365d"
 # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series.
-UPTIME_RECORDING_METRIC = (
-    f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="request-v4"}}'
+UPTIME_RECORDING_METRIC = f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas"}}'
+UPTIME_RECORDING_EXPR = f"last_over_time({UPTIME_RECORDING_METRIC}[24h])"
+TRAEFIK_READY_EXPR = (
+    "("
+    'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
+    " / clamp_min("
+    'sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)'
+    ")"
 )
-AVAILABILITY_REQUESTS_1H_EXPR = (
-    'sum(increase(traefik_entrypoint_requests_total{'
-    'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))'
+CONTROL_READY_FRACTION_EXPR = (
+    f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
+    f" / {CONTROL_TOTAL})"
 )
-AVAILABILITY_FAILURES_1H_EXPR = (
-    'sum(increase(traefik_entrypoint_requests_total{'
-    'entrypoint="websecure",protocol="http",code=~"5.."}[1h]))'
-)
-UPTIME_LIVE_FALLBACK_EXPR = (
-    f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / "
-    f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
-)
-UPTIME_RECORDING_EXPR = (
-    f"(last_over_time({UPTIME_RECORDING_METRIC}[48h]) "
-    f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
+UPTIME_AVAIL_EXPR = (
+    f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
 )
 
 # Tie-breaker to deterministically pick one node per namespace when shares tie.
@@ -1901,9 +1870,9 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.",
     "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.",
     "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.",
-    "Atlas Availability (365d)": "Request-weighted Atlas ingress availability; every server-side 5xx response counts as a failed request.",
-    "Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.",
-    "CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.",
+    "Atlas Availability (365d)": "Rolling one-year Atlas availability; higher is better, below target means users saw downtime.",
+    "Problem Pods": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.",
+    "CrashLoop / ImagePull": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.",
     "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.",
     "Hottest node: CPU": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.",
     "Hottest node: RAM": "Highest worker memory use right now; lower is safer, high values risk evictions.",
@@ -1941,7 +1910,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
     "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
     "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
-    "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
+    "Namespace GPU Utilization": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.",
     "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
     "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
     "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",
@@ -2142,7 +2111,7 @@ def build_overview():
             "decimals": 4,
             "text_mode": "value",
             "instant": True,
-            "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists.",
+            "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data.",
         },
         {
             "id": 4,
@@ -2366,10 +2335,10 @@ def build_overview():
     }
     overview_avg_coverage = f"(avg(({QUALITY_GATE_COVERAGE_BY_SUITE})) or on() vector(0))"
     overview_category_health = (
-        f'(avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{'
+        f'avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{'
         f'suite=~"{PLATFORM_TEST_SUITE_CANONICAL_MATCHER}",branch!="",branch=~"main|master|origin/main|origin/master",'
         f'category=~"{PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"'
-        '})) or label_set(vector(0), "category", "none")'
+        "})"
     )
     for panel_id, title, draw_expr, runtime_expr, y_pos in [
         (40, "Pyrphoros UPS Current", ANANKE_UPS_DRAW_WATTS_DB, ANANKE_UPS_RUNTIME_DB, 7),
@@ -5521,13 +5490,12 @@ def build_gpu_dashboard():
     panels.append(
         table_panel(
             4,
-            "GPU Processes by Pod",
-            'topk(10, sum by (namespace,pod,node,process) '
-            '(nvidia_process_gpu_sm_util_percent{pod!="host"}) > 0)',
+            "GPU Pods Reporting Device Util",
+            'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))',
             {"h": 8, "w": 12, "x": 12, "y": 8},
             unit="percent",
             transformations=[{"id": "labelsToFields", "options": {}}],
-            description="NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization.",
+            description="DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.",
         )
     )
     return {

scripts/tests/test_availability_cleanup.py

@@ -1,28 +0,0 @@
-"""Safety tests for the legacy Atlas availability cleanup."""
-
-import importlib.util
-from pathlib import Path
-
-
-def load_module():
-    """Load the service-owned cleanup module without packaging it."""
-    path = (
-        Path(__file__).resolve().parents[2]
-        / "services/monitoring/scripts/availability_cleanup.py"
-    )
-    spec = importlib.util.spec_from_file_location("availability_cleanup", path)
-    module = importlib.util.module_from_spec(spec)
-    assert spec.loader is not None
-    spec.loader.exec_module(module)
-    return module
-
-
-def test_cleanup_selector_cannot_match_request_v4() -> None:
-    """Constrain deletion to obsolete annual Atlas definitions."""
-    mod = load_module()
-
-    assert '__name__="atlas:availability:ratio_365d"' in mod.LEGACY_SELECTOR
-    assert 'scope="atlas"' in mod.LEGACY_SELECTOR
-    assert 'definition!="request-v4"' in mod.LEGACY_SELECTOR
-    assert 'definition="request-v4"' in mod.PROTECTED_SELECTOR
-    assert "ratio_1h" not in mod.LEGACY_SELECTOR

scripts/tests/test_availability_rollup.py

@@ -1,52 +0,0 @@
-"""Unit tests for the Atlas availability publisher."""
-
-import importlib.util
-import json
-from pathlib import Path
-
-import pytest
-
-
-def load_module():
-    """Load the service-owned rollup module without packaging it."""
-    path = (
-        Path(__file__).resolve().parents[2]
-        / "services/monitoring/scripts/availability_rollup.py"
-    )
-    spec = importlib.util.spec_from_file_location("availability_rollup", path)
-    module = importlib.util.module_from_spec(spec)
-    assert spec.loader is not None
-    spec.loader.exec_module(module)
-    return module
-
-
-def test_parse_export_deduplicates_replay_boundaries() -> None:
-    """Keep only the final value when replay chunks share a timestamp."""
-    mod = load_module()
-    lines = [
-        (json.dumps({"timestamps": [1000, 2000], "values": [2, 3]}) + "\n").encode(),
-        (json.dumps({"timestamps": [2000, 3000], "values": [3, 5]}) + "\n").encode(),
-    ]
-
-    assert mod.parse_export(lines) == {1000: 2.0, 2000: 3.0, 3000: 5.0}
-
-
-def test_calculate_availability_uses_all_server_failures() -> None:
-    """Calculate one bounded successful-request ratio."""
-    mod = load_module()
-
-    assert mod.calculate_availability(1000, 2) == pytest.approx(0.998)
-    with pytest.raises(ValueError):
-        mod.calculate_availability(0, 0)
-    with pytest.raises(ValueError):
-        mod.calculate_availability(100, -1)
-
-
-def test_render_metric_publishes_only_the_request_v4_series() -> None:
-    """Render the single series selected by the Grafana panel."""
-    mod = load_module()
-
-    assert mod.render_metric(0.9995, 1234) == (
-        'atlas:availability:ratio_365d{definition="request-v4",scope="atlas",'
-        'rollup="yearly"} 0.999500000000 1234\n'
-    )

scripts/tests/test_dashboards_render_atlas.py

@@ -47,8 +47,6 @@ def test_node_filter_and_expr_helpers():
     cpu_expr = mod.node_cpu_expr("titan-.*")
     mem_expr = mod.node_mem_expr("titan-.*")
     assert "node_cpu_seconds_total" in cpu_expr
-    assert "clamp_max(clamp_min(" in cpu_expr
-    assert "* 100, 0), 100)" in cpu_expr
     assert "node_memory_MemAvailable_bytes" in mem_expr
 
 
@@ -58,23 +56,10 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
     panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
 
     assert panel["title"] == "Atlas Availability (365d)"
-    availability_expr = panel["targets"][0]["expr"]
-    assert (
-        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
-        in availability_expr
-    )
-    assert 'code=~"5.."' in availability_expr
-    assert 'code=~"[1-5].."' in availability_expr
-    assert "atlas:availability:failures_1d" not in availability_expr
-    assert "atlas:availability:requests_1d" not in availability_expr
-    assert "sum_over_time" not in availability_expr
-    assert "kube_node_status_condition" not in availability_expr
-    assert "kube_deployment_status_replicas_available" not in availability_expr
+    assert panel["targets"][0]["expr"] == 'last_over_time(atlas:availability:ratio_365d{scope="atlas"}[24h])'
     assert panel["targets"][0]["instant"] is True
-    assert "Every server-side 5xx" in panel["description"]
-    assert "Replica counts, Grafana health" in panel["description"]
-    assert "daily rollup job publishes one annual sample" in panel["description"]
-    assert "keeps it for up to 48 hours" in panel["description"]
+    assert "precomputed" in panel["description"]
+    assert "last successful rollup for up to 24h" in panel["description"]
 
 
 def test_overview_uses_readable_quality_power_and_gitops_panels():
@@ -163,7 +148,6 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
     assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
     assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
-    assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
     assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
 
     pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
@@ -173,41 +157,14 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
     assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
     assert "nvidia_gpu_device_utilization_percent" in gpu_expr
-    assert "sum_over_time" not in gpu_expr
-    assert "count_over_time" not in gpu_expr
-    assert "avg_over_time" not in gpu_expr
-    assert "$__range" not in gpu_expr
     assert "sum by (namespace)" in gpu_expr
     assert 'namespace", "shared"' not in gpu_expr
-    assert "kube_pod_container_resource_requests" in gpu_expr
-    assert mod.GPU_RESOURCE_REGEX in gpu_expr
-    assert '"gpu_source", "nvidia"' in gpu_expr
-    assert '"gpu_source", "jetson"' in gpu_expr
+    assert "kube_node_labels" not in gpu_expr
     assert "100 *" in gpu_expr
     assert "100 -" not in gpu_expr
     assert 'namespace", "unattributed"' in gpu_expr
     assert 'namespace", "idle"' in gpu_expr
     assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
-    assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"]
-    assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"]
-    assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
-
-
-def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
-    mod = load_module()
-    dashboard = mod.build_gpu_dashboard()
-    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
-
-    node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
-    assert "nvidia_gpu_device_utilization_percent" in node_expr
-    assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
-    assert "jetson_gr3d_freq_percent" in node_expr
-    assert "unless on(node)" in node_expr
-    assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
-
-    process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
-    assert "nvidia_process_gpu_sm_util_percent" in process_expr
-    assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
 
 
 def test_overview_and_testing_panels_all_have_concise_descriptions():
@@ -324,7 +281,7 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
     suite_freshness_expr = panels_by_title["Suite Freshness (24h)"]["targets"][0]["expr"]
     assert "platform_quality:suite_runs:increase_24h" in suite_freshness_expr
     assert "max_over_time(platform_quality_gate_runs_total" not in suite_freshness_expr
-    assert "[7d:1h]" in panels_by_title["CI Run Success Rate (7d)"]["targets"][0]["expr"]
+    assert "[30d:15m]" in panels_by_title["CI Run Success Rate (30d)"]["targets"][0]["expr"]
     assert panels_by_title["Latest Gate Health by Suite"]["gridPos"]["w"] == 6
     assert panels_by_title["CI Run Success by Suite (24h)"]["gridPos"]["w"] == 6
     assert panels_by_title["Coverage by Suite (Latest, gate 95)"]["gridPos"] == {"h": 7, "w": 6, "x": 12, "y": 4}
@@ -343,7 +300,7 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
 
     rolling_panel = panels_by_title["CI Run Success by Suite (7d rolling)"]
     assert rolling_panel["type"] == "state-timeline"
-    assert "[7d:1h]" in rolling_panel["targets"][0]["expr"]
+    assert "[7d:1m]" in rolling_panel["targets"][0]["expr"]
     category_panel = panels_by_title["Test Category Health History"]
     assert category_panel["type"] == "state-timeline"
     assert "category" in category_panel["targets"][0]["expr"]
@@ -501,12 +458,12 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
     assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"]
     assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"]
 
-    branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
-    recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]
+    branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (30d)"]
+    recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (30d)"]
     assert branch_panel["gridPos"]["x"] == 12
     assert recent_branch_panel["gridPos"]["x"] == 18
-    assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"]
-    assert "[7d:1h]" in branch_panel["targets"][0]["expr"]
+    assert "[30d:15m]" in recent_branch_panel["targets"][0]["expr"]
+    assert "[30d:15m]" in branch_panel["targets"][0]["expr"]
     assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent"
     assert "unless on(suite)" in branch_panel["targets"][0]["expr"]
     assert "> bool 0" in branch_panel["targets"][0]["expr"]

scripts/tests/test_monitoring_query_capacity.py

@@ -1,176 +0,0 @@
-"""Protect the monitoring backend from known query-starvation regressions."""
-
-from pathlib import Path
-
-import yaml
-
-
-REPO_ROOT = Path(__file__).resolve().parents[2]
-
-
-def _documents(path: Path) -> list[dict]:
-    """Load every non-empty YAML document from a repository manifest."""
-    return [document for document in yaml.safe_load_all(path.read_text()) if document]
-
-
-def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
-    """Keep search capacity and pod resources above the proven failure floor."""
-    manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml")
-    release = next(
-        manifest
-        for manifest in manifests
-        if manifest.get("kind") == "HelmRelease"
-        and manifest.get("metadata", {}).get("name") == "victoria-metrics-single"
-    )
-    server = release["spec"]["values"]["server"]
-
-    assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4
-    assert server["extraArgs"]["search.maxQueryDuration"] == "1m"
-    assert server["extraArgs"]["search.maxQueueDuration"] == "30s"
-    assert server["resources"]["requests"]["memory"] == "2Gi"
-    assert server["resources"]["limits"]["cpu"] == "2"
-    assert server["resources"]["limits"]["memory"] == "4Gi"
-
-    required_terms = server["affinity"]["nodeAffinity"][
-        "requiredDuringSchedulingIgnoredDuringExecution"
-    ]["nodeSelectorTerms"]
-    hostname_rule = next(
-        expression
-        for term in required_terms
-        for expression in term["matchExpressions"]
-        if expression["key"] == "kubernetes.io/hostname"
-    )
-    assert hostname_rule["operator"] == "NotIn"
-    assert {"titan-14", "titan-18"} <= set(hostname_rule["values"])
-
-
-def test_yearly_availability_is_published_outside_the_query_pool() -> None:
-    """Keep all long-range availability work out of Grafana and MetricsQL."""
-    manifest = _documents(
-        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
-    )[0]
-    groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
-    rules = [rule for group in groups for rule in group["rules"]]
-    manifests = _documents(
-        REPO_ROOT / "services/monitoring/availability-rollup-cronjob.yaml"
-    )
-    cronjob = next(manifest for manifest in manifests if manifest["kind"] == "CronJob")
-    source = (
-        REPO_ROOT / "services/monitoring/scripts/availability_rollup.py"
-    ).read_text()
-
-    assert all(rule["record"] != "atlas:availability:ratio_365d" for rule in rules)
-    assert "atlas:availability:requests_1d" in source
-    assert "atlas:availability:failures_1d" in source
-    assert "/api/v1/export" in source
-    assert "/api/v1/import/prometheus" in source
-    assert cronjob["spec"]["schedule"] == "10 0 * * *"
-    assert cronjob["spec"]["concurrencyPolicy"] == "Forbid"
-
-
-def test_daily_availability_rollups_use_the_same_request_sli() -> None:
-    """Keep annual availability cheap without changing its request definition."""
-    manifest = _documents(
-        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
-    )[0]
-    groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
-    daily_group = next(
-        group for group in groups if group["name"] == "atlas.availability.rollup"
-    )
-    daily = {rule["record"]: rule for rule in daily_group["rules"]}
-
-    assert daily_group["interval"] == "1d"
-    assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"]
-    assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"]
-    assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values())
-
-
-def test_availability_uses_request_failures_instead_of_replica_capacity() -> None:
-    """Measure HTTP outcomes without treating redundant replica loss as downtime."""
-    manifest = _documents(
-        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
-    )[0]
-    groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
-    rules = [rule for group in groups for rule in group["rules"]]
-    requests = next(
-        rule
-        for rule in rules
-        if rule["record"] == "atlas:availability:requests_1h"
-    )
-    failures = next(
-        rule
-        for rule in rules
-        if rule["record"] == "atlas:availability:failures_1h"
-    )
-
-    assert 'code=~"[1-5].."' in requests["expr"]
-    assert 'code=~"5.."' in failures["expr"]
-    assert "traefik_entrypoint_requests_total" in requests["expr"]
-    assert "traefik_entrypoint_requests_total" in failures["expr"]
-    assert "kube_node_status_condition" not in repr(rules)
-    assert "kube_deployment_status_replicas_available" not in repr(rules)
-    assert requests["labels"]["definition"] == "request-v4"
-    assert failures["labels"]["definition"] == "request-v4"
-
-
-def test_availability_backfill_replays_the_same_request_sli() -> None:
-    """Backfill retained history with the exact rules used for future samples."""
-    manifests = _documents(
-        REPO_ROOT / "services/monitoring/availability-backfill-v4-job.yaml"
-    )
-    config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
-    job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
-    backfill = yaml.safe_load(config["data"]["atlas-request-history.yaml"])
-    expressions = {
-        rule["record"]: rule["expr"]
-        for group in backfill["groups"]
-        for rule in group["rules"]
-    }
-
-    assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1h"]
-    assert 'code=~"5.."' in expressions["atlas:availability:failures_1h"]
-    args = job["spec"]["template"]["spec"]["containers"][0]["args"]
-    assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args
-    assert "-replay.timeTo=2026-08-04T23:00:00Z" in args
-    assert "-replay.maxDatapointsPerQuery=48" in args
-
-
-def test_daily_availability_backfill_stays_below_query_timeout() -> None:
-    """Replay daily buckets in small chunks so raw history never starves Grafana."""
-    manifests = _documents(
-        REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml"
-    )
-    config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
-    job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
-    backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"])
-    group = backfill["groups"][0]
-    expressions = {rule["record"]: rule["expr"] for rule in group["rules"]}
-
-    assert group["interval"] == "1d"
-    assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"]
-    assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"]
-    args = job["spec"]["template"]["spec"]["containers"][0]["args"]
-    assert "-replay.maxDatapointsPerQuery=4" in args
-
-
-def test_quality_rollups_do_not_run_every_minute() -> None:
-    """Keep high-cardinality quality rollups below the backend saturation cadence."""
-    manifest = _documents(
-        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
-    )[0]
-    quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0]
-
-    assert quality["interval"] == "5m"
-
-
-def test_vmalert_reloads_updated_rule_files() -> None:
-    """Make Flux ConfigMap updates take effect without manual pod revision bumps."""
-    manifests = _documents(
-        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
-    )
-    deployment = next(
-        manifest for manifest in manifests if manifest.get("kind") == "Deployment"
-    )
-    args = deployment["spec"]["template"]["spec"]["containers"][0]["args"]
-
-    assert "-configCheckInterval=30s" in args

scripts/tests/test_nvidia_process_exporter.py

@@ -1,99 +0,0 @@
-import importlib.util
-from pathlib import Path
-import sys
-import types
-
-import pytest
-
-
-ROOT = Path(__file__).resolve().parents[2]
-EXPORTER_PATH = ROOT / "services" / "monitoring" / "scripts" / "nvidia_process_exporter.py"
-
-
-def load_exporter(monkeypatch):
-    """Load the exporter without requiring an NVIDIA driver on the test host."""
-
-    pynvml = types.ModuleType("pynvml")
-
-    class NVMLError(Exception):
-        pass
-
-    class NVMLErrorNotFound(NVMLError):
-        pass
-
-    class NVMLErrorNotSupported(NVMLError):
-        pass
-
-    pynvml.NVMLError = NVMLError
-    pynvml.NVMLError_NotFound = NVMLErrorNotFound
-    pynvml.NVMLError_NotSupported = NVMLErrorNotSupported
-    for name in (
-        "nvmlDeviceGetComputeRunningProcesses_v3",
-        "nvmlDeviceGetCount",
-        "nvmlDeviceGetGraphicsRunningProcesses_v3",
-        "nvmlDeviceGetHandleByIndex",
-        "nvmlDeviceGetName",
-        "nvmlDeviceGetProcessUtilization",
-        "nvmlDeviceGetUUID",
-        "nvmlDeviceGetUtilizationRates",
-        "nvmlInit",
-    ):
-        setattr(pynvml, name, lambda *args, **kwargs: None)
-
-    monkeypatch.setitem(sys.modules, "pynvml", pynvml)
-    spec = importlib.util.spec_from_file_location("nvidia_process_exporter_test", EXPORTER_PATH)
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-def test_process_sample_window_uses_nvml_microseconds(monkeypatch):
-    exporter = load_exporter(monkeypatch)
-    observed = []
-    monkeypatch.setattr(exporter.time, "time", lambda: 1_700_000_000.0)
-    monkeypatch.setattr(
-        exporter,
-        "nvmlDeviceGetProcessUtilization",
-        lambda handle, since: observed.append(since) or [],
-    )
-
-    samples, supported = exporter.process_utilization_samples(object())
-
-    assert samples == {}
-    assert supported == 1
-    assert observed == [1_700_000_000_000_000 - 30_000_000]
-
-
-def test_namespace_attribution_scales_to_current_device_total(monkeypatch):
-    exporter = load_exporter(monkeypatch)
-
-    result = exporter.reconcile_namespace_utilization(
-        {"game-stream": 20, "hermes": 10},
-        device_util=3,
-    )
-
-    assert sum(result.values()) == pytest.approx(3)
-    assert result["game-stream"] == pytest.approx(2)
-    assert result["hermes"] == pytest.approx(1)
-
-
-def test_namespace_attribution_assigns_unexplained_compute_to_host(monkeypatch):
-    exporter = load_exporter(monkeypatch)
-
-    result = exporter.reconcile_namespace_utilization(
-        {"hermes": 1},
-        device_util=3,
-    )
-
-    assert result == {"hermes": 1, "host": 2}
-
-
-def test_zero_device_utilization_clears_stale_process_samples(monkeypatch):
-    exporter = load_exporter(monkeypatch)
-
-    result = exporter.reconcile_namespace_utilization(
-        {"hermes": 40, "game-stream": 5},
-        device_util=0,
-    )
-
-    assert result == {"hermes": 0, "game-stream": 0}

services/bstein-dev-home/backend-deployment.yaml

@@ -68,7 +68,7 @@ spec:
           args:
             - >-
               . /vault/secrets/portal-env.sh
-              && exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 1200 app:app
+              && exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 600 app:app
           env:
             - name: AI_CHAT_API
               value: http://ollama.ai.svc.cluster.local:11434
@@ -119,8 +119,6 @@ spec:
               value: http://ariadne.maintenance.svc.cluster.local
             - name: ARIADNE_TIMEOUT_SEC
               value: "10"
-            - name: ARIADNE_GAME_MODE_TIMEOUT_SEC
-              value: "900"
             - name: ACCOUNT_ALLOWED_GROUPS
               value: ""
             - name: HTTP_CHECK_TIMEOUT_SEC

services/bstein-dev-home/kustomization.yaml

@@ -20,9 +20,9 @@ resources:
   - ingress.yaml
 images:
   - name: registry.bstein.dev/bstein/bstein-dev-home-frontend
-    newTag: 0.1.1-441 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
+    newTag: 0.1.1-428 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
   - name: registry.bstein.dev/bstein/bstein-dev-home-backend
-    newTag: 0.1.1-441 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
+    newTag: 0.1.1-428 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
 configMapGenerator:
   - name: chat-ai-gateway
     namespace: bstein-dev-home

services/cassandra/backend-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       labels:
         app: cassandra-backend
       annotations:
-        cassandra.bstein.dev/redeploy: "2026-08-04-0.8.150-lane-semantics"
+        cassandra.bstein.dev/redeploy: "2026-07-29-0.8.72-lab-detail-first-paint"
         vault.hashicorp.com/agent-inject: "true"
         vault.hashicorp.com/agent-init-first: "true"
         vault.hashicorp.com/agent-pre-populate-only: "true"
@@ -66,7 +66,7 @@ spec:
           type: RuntimeDefault
       initContainers:
         - name: database-init
-          image: registry.bstein.dev/cassandra/cassandra-backend:0.9.16 # {"$imagepolicy": "cassandra:cassandra-backend"}
+          image: registry.bstein.dev/cassandra/cassandra-backend:0.8.72 # {"$imagepolicy": "cassandra:cassandra-backend"}
           imagePullPolicy: Always
           command: ["/bin/sh", "-c"]
           args:
@@ -88,7 +88,7 @@ spec:
               memory: 512Mi
       containers:
         - name: backend
-          image: registry.bstein.dev/cassandra/cassandra-backend:0.9.16 # {"$imagepolicy": "cassandra:cassandra-backend"}
+          image: registry.bstein.dev/cassandra/cassandra-backend:0.8.72 # {"$imagepolicy": "cassandra:cassandra-backend"}
           imagePullPolicy: Always
           command: ["/bin/sh", "-c"]
           args:
@@ -104,9 +104,9 @@ spec:
               protocol: TCP
           env:
             - name: CASSANDRA_SIM_IMAGE
-              value: registry.bstein.dev/cassandra/cassandra-sim-worker:0.9.16 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
+              value: registry.bstein.dev/cassandra/cassandra-sim-worker:0.8.72 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
             - name: CASSANDRA_GENERATOR_WORKER_IMAGE
-              value: registry.bstein.dev/cassandra/cassandra-generator-worker:0.9.16 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
+              value: registry.bstein.dev/cassandra/cassandra-generator-worker:0.8.72 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
           envFrom:
             - configMapRef:
                 name: cassandra-config

services/cassandra/configmap.yaml

@@ -19,11 +19,10 @@ data:
   CASSANDRA_OIDC_ADMIN_GROUPS: admin
   CASSANDRA_NAMESPACE: cassandra
   CASSANDRA_OWNER_USER_ID_ALIASES: veles-dev=d357688e-e3e4-4f18-9491-a4ed5622deb6;a21a286c-6c82-4180-9243-7c916442ecf0=d357688e-e3e4-4f18-9491-a4ed5622deb6
-  CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.9.16 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
-  CASSANDRA_GENERATOR_WORKER_IMAGE: registry.bstein.dev/cassandra/cassandra-generator-worker:0.9.16 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
-  CASSANDRA_SIM_WORKER_VERSION: "0.8.150"
-  CASSANDRA_GENERATOR_WORKER_VERSION: "0.8.150"
-  CASSANDRA_STRATEGY_SPEC_HINT_INTERPRETATION_TIMEOUT: "420"
+  CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.8.72 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
+  CASSANDRA_GENERATOR_WORKER_IMAGE: registry.bstein.dev/cassandra/cassandra-generator-worker:0.8.72 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
+  CASSANDRA_SIM_WORKER_VERSION: "0.8.72"
+  CASSANDRA_GENERATOR_WORKER_VERSION: "0.8.72"
   CASSANDRA_SIM_SERVICE_ACCOUNT: cassandra-sim
   CASSANDRA_GENERATOR_WORKER_SERVICE_ACCOUNT: cassandra-generator
   CASSANDRA_SIM_PRIORITY_CLASS: cassandra-sim
@@ -33,7 +32,7 @@ data:
   CASSANDRA_SIM_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule
   CASSANDRA_GENERATOR_WORKER_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule
   CASSANDRA_SIM_ACTIVE_DEADLINE_SECONDS: "7200"
-  CASSANDRA_GENERATOR_WORKER_ACTIVE_DEADLINE_SECONDS: "14400"
+  CASSANDRA_GENERATOR_WORKER_ACTIVE_DEADLINE_SECONDS: "1800"
   CASSANDRA_SIM_TTL_SECONDS: "3600"
   CASSANDRA_GENERATOR_WORKER_TTL_SECONDS: "600"
   CASSANDRA_SIM_CPU_REQUEST: 500m

services/cassandra/frontend-deployment.yaml

@@ -48,7 +48,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: frontend
-          image: registry.bstein.dev/cassandra/cassandra-frontend:0.9.16 # {"$imagepolicy": "cassandra:cassandra-frontend"}
+          image: registry.bstein.dev/cassandra/cassandra-frontend:0.8.72 # {"$imagepolicy": "cassandra:cassandra-frontend"}
           imagePullPolicy: Always
           ports:
             - name: http

services/cassandra/retention-cronjob.yaml

@@ -38,7 +38,7 @@ spec:
               type: RuntimeDefault
           containers:
             - name: retention
-              image: registry.bstein.dev/cassandra/cassandra-backend:0.9.16 # {"$imagepolicy": "cassandra:cassandra-backend"}
+              image: registry.bstein.dev/cassandra/cassandra-backend:0.8.72 # {"$imagepolicy": "cassandra:cassandra-backend"}
               imagePullPolicy: IfNotPresent
               command: ["cassandra-hygiene"]
               args: ["--prune-artifacts"]

services/hermes-chat/certificate.yaml

@@ -1,13 +0,0 @@
-# services/hermes-chat/certificate.yaml
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: chat-tls
-  namespace: hermes-chat
-spec:
-  secretName: chat-tls
-  issuerRef:
-    kind: ClusterIssuer
-    name: letsencrypt
-  dnsNames:
-    - chat.bstein.dev

services/hermes-chat/configmap.yaml

@@ -1,153 +0,0 @@
-# services/hermes-chat/configmap.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hermes-chat-config
-  namespace: hermes-chat
-  labels:
-    app: hermes-chat
-data:
-  config.yaml: |
-    model:
-      provider: openai-codex
-      default: gpt-5.6-terra
-      model: gpt-5.6-terra
-
-    fallback_providers:
-      - provider: custom
-        model: gpt-oss:20b
-        base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
-        api_key: ollama
-
-    agent:
-      api_max_retries: 1
-
-    platform_toolsets:
-      cli:
-        - clarify
-        - file
-        - terminal
-        - web
-      api_server:
-        - clarify
-        - file
-        - terminal
-        - web
-
-    skills:
-      creation_nudge_interval: 15
-      external_dirs:
-        - /opt/data/workspace/skills
-
-    terminal:
-      backend: local
-      cwd: /opt/data/workspace
-      timeout: 180
-      home_mode: profile
-
-    approvals:
-      mode: manual
-      deny:
-        - "*kubectl apply*"
-        - "*kubectl create*"
-        - "*kubectl delete*"
-        - "*kubectl edit*"
-        - "*kubectl patch*"
-        - "*kubectl replace*"
-        - "*kubectl scale*"
-        - "*kubectl set*"
-        - "*kubectl label*"
-        - "*kubectl annotate*"
-        - "*kubectl cordon*"
-        - "*kubectl uncordon*"
-        - "*kubectl drain*"
-        - "*kubectl rollout restart*"
-        - "*kubectl rollout undo*"
-        - "*kubectl exec*"
-        - "*kubectl attach*"
-        - "*kubectl cp*"
-        - "*kubectl debug*"
-        - "*kubectl expose*"
-        - "*kubectl port-forward*"
-        - "*kubectl proxy*"
-        - "*kubectl run*"
-        - "*kubectl get secret*"
-        - "*kubectl describe secret*"
-        - "*flux reconcile*"
-        - "*flux suspend*"
-        - "*flux resume*"
-        - "*vault *"
-        - "*k3s *"
-        - "*crictl *"
-        - "*ctr *"
-        - "*169.254.169.254*"
-        - "*kubernetes.default*"
-        - "*/var/run/secrets/kubernetes.io*"
-
-    dashboard:
-      public_url: https://chat.bstein.dev
-      oauth:
-        provider: self-hosted
-        self_hosted:
-          issuer: https://sso.bstein.dev/realms/atlas
-          client_id: hermes-chat-dashboard
-          scopes: openid profile email groups
-
-    display:
-      compact: true
-      tool_progress: all
-      interim_assistant_messages: true
-      long_running_notifications: true
-
-    tool_loop_guardrails:
-      warnings_enabled: true
-      hard_stop_enabled: true
-      warn_after:
-        exact_failure: 2
-        same_tool_failure: 3
-        idempotent_no_progress: 2
-      hard_stop_after:
-        exact_failure: 5
-        same_tool_failure: 8
-        idempotent_no_progress: 5
-
-    updates:
-      pre_update_backup: quick
-      backup_keep: 3
-      non_interactive_local_changes: stash
-  SOUL.md: |
-    You are a personal AI assistant and researcher for Atlas users. Be useful,
-    curious, careful, and direct. Help with research, writing, planning,
-    learning, files, personal automation, and creative work. Use tools when
-    they materially improve the answer and make reusable skills when a stable
-    workflow is worth keeping.
-
-    Your environment is a private consumer sandbox. Work in the mounted
-    workspace and with public Internet services the user intentionally asks
-    you to use. You may inspect the Titan cluster through the dedicated
-    read-only Kubernetes identity to explain health and status. You do not
-    administer it: never attempt mutations, secret access, exec, attach,
-    port-forwarding, private-service access, or cloud metadata access.
-  AGENTS.md: |
-    # Personal Hermes workspace
-
-    This Hermes instance is a personal chat, research, and automation
-    environment. User-created files and skills belong under `/opt/data/workspace`.
-
-    You may:
-
-    - research public Internet sources and cite them
-    - create, read, and edit files in the workspace
-    - create and improve reusable skills
-    - configure user-owned channels and provider integrations
-    - run ordinary local commands needed for the user's task
-    - inspect non-secret Kubernetes resources and pod logs with read-only
-      `kubectl get`, `describe`, and `logs` commands
-
-    This container is not an infrastructure administration environment. The
-    Kubernetes identity is observation-only and cannot read Secrets or use pod
-    exec, attach, or port-forwarding. Do not attempt cluster mutations, private
-    service access, node LAN access, metadata services, Vault, container
-    runtimes, or the operator Hermes instance. If a request needs an action,
-    explain the evidence and say Brad must perform it from the operator
-    instance at `agent.bstein.dev`.

services/hermes-chat/deployment.yaml

@@ -1,259 +0,0 @@
-# services/hermes-chat/deployment.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: hermes-chat
-  namespace: hermes-chat
-  labels:
-    app: hermes-chat
-spec:
-  replicas: 1
-  revisionHistoryLimit: 2
-  progressDeadlineSeconds: 1800
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: hermes-chat
-  template:
-    metadata:
-      labels:
-        app: hermes-chat
-      annotations:
-        ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback
-        ai.bstein.dev/role: personal-chat-research
-        ai.bstein.dev/isolation: observer-only Kubernetes RBAC, private-service egress denied
-        ai.bstein.dev/session-reset: "20260802-consumer-pty-recovery"
-        ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation
-        ai.bstein.dev/user-boundary: one OIDC subject with an isolated PVC subdirectory
-    spec:
-      serviceAccountName: hermes-chat
-      automountServiceAccountToken: true
-      securityContext:
-        fsGroup: 10000
-        fsGroupChangePolicy: OnRootMismatch
-        seccompProfile:
-          type: RuntimeDefault
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - arm64
-                  - key: node-role.kubernetes.io/worker
-                    operator: In
-                    values:
-                      - "true"
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-13
-                      - titan-15
-                      - titan-17
-                      - titan-18
-                      - titan-19
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 100
-              preference:
-                matchExpressions:
-                  - key: atlas.bstein.dev/spillover
-                    operator: DoesNotExist
-            - weight: 90
-              preference:
-                matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi5
-            - weight: 50
-              preference:
-                matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi4
-      initContainers:
-        - name: migrate-user-sessions
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
-          imagePullPolicy: IfNotPresent
-          command:
-            - /opt/hermes/bin/hermes-session-migrate
-          env:
-            - name: HERMES_HOME
-              value: /storage/users/4794ab8284a14d421eaeea3e
-            - name: HERMES_MIGRATE_SOURCE_HOME
-              value: /storage
-            - name: HERMES_MIGRATE_USER_ID
-              value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc
-            - name: HERMES_MIGRATE_SESSION_IDS
-              value: 20260802_182723_9f53f1,20260802_173527_9dc51b
-          securityContext:
-            allowPrivilegeEscalation: false
-            runAsUser: 10000
-            runAsGroup: 10000
-            seccompProfile:
-              type: RuntimeDefault
-          volumeMounts:
-            - name: home
-              mountPath: /storage
-          resources:
-            requests:
-              cpu: 25m
-              memory: 64Mi
-            limits:
-              cpu: 250m
-              memory: 256Mi
-        - name: init-config
-          image: busybox:1.37
-          imagePullPolicy: IfNotPresent
-          command:
-            - sh
-            - -c
-            - |
-              set -eu
-              user_home=/storage/users/4794ab8284a14d421eaeea3e
-              mkdir -p "${user_home}/workspace/skills" "${user_home}/home/.local/bin" "${user_home}/logs"
-              cp /config/config.yaml "${user_home}/config.yaml"
-              cp /config/SOUL.md "${user_home}/SOUL.md"
-              cp /config/AGENTS.md "${user_home}/workspace/AGENTS.md"
-              touch "${user_home}/.env"
-              if ! grep -q '^API_SERVER_KEY=' "${user_home}/.env"; then
-                api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
-                printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> "${user_home}/.env"
-              fi
-              chmod 0600 "${user_home}/.env"
-              chown -R 10000:10000 "${user_home}"
-          securityContext:
-            allowPrivilegeEscalation: false
-            runAsUser: 0
-            runAsGroup: 0
-            seccompProfile:
-              type: RuntimeDefault
-          volumeMounts:
-            - name: home
-              mountPath: /storage
-            - name: config
-              mountPath: /config
-              readOnly: true
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 100m
-              memory: 64Mi
-        - name: install-kubectl
-          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
-          imagePullPolicy: IfNotPresent
-          command:
-            - /bin/sh
-            - -c
-            - |
-              set -eu
-              cp "$(command -v kubectl)" /tools/kubectl
-              chmod 0755 /tools/kubectl
-              chown 10000:10000 /tools/kubectl
-          securityContext:
-            allowPrivilegeEscalation: false
-            runAsUser: 0
-            runAsGroup: 0
-            seccompProfile:
-              type: RuntimeDefault
-          volumeMounts:
-            - name: tools
-              mountPath: /tools
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 100m
-              memory: 64Mi
-      containers:
-        - name: hermes-chat
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
-          imagePullPolicy: IfNotPresent
-          args:
-            - gateway
-            - run
-          ports:
-            - name: dashboard
-              containerPort: 9119
-              protocol: TCP
-          env:
-            - name: HERMES_HOME
-              value: /opt/data
-            - name: HOME
-              value: /opt/data/home
-            - name: PATH
-              value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-            - name: HERMES_DASHBOARD
-              value: "1"
-            - name: HERMES_DASHBOARD_HOST
-              value: 0.0.0.0
-            - name: HERMES_DASHBOARD_PORT
-              value: "9119"
-            - name: HERMES_DASHBOARD_PUBLIC_URL
-              value: https://chat.bstein.dev
-            - name: HERMES_DASHBOARD_OIDC_ISSUER
-              value: https://sso.bstein.dev/realms/atlas
-            - name: HERMES_DASHBOARD_OIDC_CLIENT_ID
-              value: hermes-chat-dashboard
-            - name: HERMES_DASHBOARD_OIDC_SCOPES
-              value: openid profile email groups
-            - name: HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS
-              value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc
-            - name: API_SERVER_ENABLED
-              value: "true"
-            - name: API_SERVER_HOST
-              value: 0.0.0.0
-            - name: API_SERVER_PORT
-              value: "8642"
-            - name: API_SERVER_CORS_ORIGINS
-              value: https://chat.bstein.dev
-          volumeMounts:
-            - name: home
-              mountPath: /opt/data
-              subPath: users/4794ab8284a14d421eaeea3e
-            - name: tools
-              mountPath: /usr/local/bin/kubectl
-              subPath: kubectl
-          readinessProbe:
-            httpGet:
-              path: /api/status
-              port: dashboard
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            timeoutSeconds: 5
-          livenessProbe:
-            httpGet:
-              path: /api/status
-              port: dashboard
-            initialDelaySeconds: 90
-            periodSeconds: 30
-            timeoutSeconds: 10
-          securityContext:
-            allowPrivilegeEscalation: false
-            seccompProfile:
-              type: RuntimeDefault
-          resources:
-            requests:
-              cpu: 100m
-              memory: 768Mi
-              ephemeral-storage: 256Mi
-            limits:
-              cpu: "1"
-              memory: 2Gi
-              ephemeral-storage: 1Gi
-      volumes:
-        - name: home
-          persistentVolumeClaim:
-            claimName: hermes-chat-home
-        - name: config
-          configMap:
-            name: hermes-chat-config
-        - name: tools
-          emptyDir: {}

services/hermes-chat/ingress.yaml

@@ -1,27 +0,0 @@
-# services/hermes-chat/ingress.yaml
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: chat
-  namespace: hermes-chat
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt
-    traefik.ingress.kubernetes.io/router.entrypoints: websecure
-    traefik.ingress.kubernetes.io/router.tls: "true"
-spec:
-  ingressClassName: traefik
-  tls:
-    - hosts:
-        - chat.bstein.dev
-      secretName: chat-tls
-  rules:
-    - host: chat.bstein.dev
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: hermes-chat
-                port:
-                  name: dashboard

services/hermes-chat/kustomization.yaml

@@ -1,15 +0,0 @@
-# services/hermes-chat/kustomization.yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: hermes-chat
-resources:
-  - namespace.yaml
-  - serviceaccount.yaml
-  - rbac.yaml
-  - configmap.yaml
-  - pvc.yaml
-  - deployment.yaml
-  - service.yaml
-  - networkpolicy.yaml
-  - certificate.yaml
-  - ingress.yaml

services/hermes-chat/namespace.yaml

@@ -1,7 +0,0 @@
-# services/hermes-chat/namespace.yaml
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: hermes-chat
-  labels:
-    app.kubernetes.io/name: hermes-chat

services/hermes-chat/networkpolicy.yaml

@@ -1,87 +0,0 @@
-# services/hermes-chat/networkpolicy.yaml
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: hermes-chat-isolation
-  namespace: hermes-chat
-spec:
-  podSelector:
-    matchLabels:
-      app: hermes-chat
-  policyTypes:
-    - Ingress
-    - Egress
-  ingress:
-    - from:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: traefik
-          podSelector:
-            matchLabels:
-              app.kubernetes.io/name: traefik
-      ports:
-        - protocol: TCP
-          port: 9119
-  egress:
-    - to:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: kube-system
-          podSelector:
-            matchLabels:
-              k8s-app: kube-dns
-      ports:
-        - protocol: UDP
-          port: 53
-        - protocol: TCP
-          port: 53
-    - to:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: hermes
-          podSelector:
-            matchLabels:
-              app: hermes-model-gate
-      ports:
-        - protocol: TCP
-          port: 8080
-    # Atlas OIDC uses the public sso.bstein.dev issuer, which resolves to the
-    # in-cluster Traefik load balancer for pods on the Atlas network.
-    - to:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: traefik
-          podSelector:
-            matchLabels:
-              app: traefik
-      ports:
-        - protocol: TCP
-          port: 443
-    - to:
-        - ipBlock:
-            cidr: 10.43.0.1/32
-      ports:
-        - protocol: TCP
-          port: 443
-    # K3s applies egress policy after Service DNAT, so admit only the fixed API
-    # server endpoints as well as the kubernetes Service IP above.
-    - to:
-        - ipBlock:
-            cidr: 192.168.22.11/32
-        - ipBlock:
-            cidr: 192.168.22.12/32
-        - ipBlock:
-            cidr: 192.168.22.13/32
-      ports:
-        - protocol: TCP
-          port: 6443
-    - to:
-        - ipBlock:
-            cidr: 0.0.0.0/0
-            except:
-              - 10.0.0.0/8
-              - 100.64.0.0/10
-              - 127.0.0.0/8
-              - 169.254.0.0/16
-              - 172.16.0.0/12
-              - 192.168.0.0/16

services/hermes-chat/pvc.yaml

@@ -1,15 +0,0 @@
-# services/hermes-chat/pvc.yaml
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: hermes-chat-home
-  namespace: hermes-chat
-  labels:
-    app: hermes-chat
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: astreae
-  resources:
-    requests:
-      storage: 2Gi

services/hermes-chat/rbac.yaml

@@ -1,62 +0,0 @@
-# services/hermes-chat/rbac.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: hermes-chat-observer
-rules:
-  - apiGroups: [""]
-    resources:
-      - endpoints
-      - events
-      - namespaces
-      - nodes
-      - persistentvolumeclaims
-      - persistentvolumes
-      - pods
-      - pods/log
-      - replicationcontrollers
-      - services
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["apps"]
-    resources:
-      - daemonsets
-      - deployments
-      - replicasets
-      - statefulsets
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["batch"]
-    resources:
-      - cronjobs
-      - jobs
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["networking.k8s.io"]
-    resources:
-      - ingresses
-      - networkpolicies
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["helm.toolkit.fluxcd.io"]
-    resources:
-      - helmreleases
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["kustomize.toolkit.fluxcd.io"]
-    resources:
-      - kustomizations
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["source.toolkit.fluxcd.io"]
-    resources:
-      - gitrepositories
-      - helmrepositories
-    verbs: ["get", "list", "watch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: hermes-chat-observer
-subjects:
-  - kind: ServiceAccount
-    name: hermes-chat
-    namespace: hermes-chat
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: hermes-chat-observer

services/hermes-chat/service.yaml

@@ -1,17 +0,0 @@
-# services/hermes-chat/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: hermes-chat
-  namespace: hermes-chat
-  labels:
-    app: hermes-chat
-spec:
-  type: ClusterIP
-  selector:
-    app: hermes-chat
-  ports:
-    - name: dashboard
-      port: 9119
-      targetPort: dashboard
-      protocol: TCP

services/hermes-chat/serviceaccount.yaml

@@ -1,7 +0,0 @@
-# services/hermes-chat/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: hermes-chat
-  namespace: hermes-chat
-automountServiceAccountToken: true

services/hermes/NOTES.md

@@ -1,239 +0,0 @@
-# Hermes on Atlas: operator guide
-
-This is the mental model and demonstration script for the operator instance at
-`agent.bstein.dev`. Read it once, then prove each section in the live UI. The
-consumer instance at `chat.bstein.dev` is intentionally separate and is not the
-place to perform infrastructure triage.
-
-## The one-sentence explanation
-
-Hermes is the persistent agent runtime and control surface; Codex or the local
-GPU model supplies reasoning, while Hermes supplies sessions, tools, skills,
-approval rules, identity, and the repeatable procedure that connects the model
-to Atlas evidence.
-
-Hermes is not the model. Replacing `gpt-5.6-terra` with `gpt-oss:20b` changes the
-reasoning engine, but it does not replace the Hermes sessions, tools, skills,
-files, permission boundary, or workflow.
-
-## Request and evidence path
-
-```text
-browser
-  -> Keycloak login
-  -> oauth2-proxy exact-user check
-  -> operator Hermes pod and persistent workspace
-  -> SOUL.md + AGENTS.md + selected skill
-  -> Codex primary model
-       -> read-only terminal/web/file tools
-       -> Ariadne deterministic evidence
-       -> retained Jenkins logs and artifacts
-       -> Gitea commits and Flux state
-       -> Kubernetes workloads, events, logs, and dependencies
-       -> Pushgateway data queried through VictoriaMetrics
-       -> Grafana query and alert context
-  -> structured finding and proposed repo-side change
-  -> human review and approval
-  -> Git/Flux delivery outside Hermes
-
-provider failure
-  -> hermes-model-gate
-       -> Hermes owns titan-24: Ollama serves gpt-oss:20b
-       -> Wolf owns titan-24: gate returns 503; local inference yields
-```
-
-The operator agent pod runs on an ARM worker and does not reserve the GPU.
-Ollama is the component on `titan-24` that owns GPU memory. The Lease
-`hermes/titan-24-gpu-owner` controls whether the model gate admits local
-inference. Normal operator conversations use independently authenticated Codex
-first, so Wolf ownership does not need to block the web agent.
-
-## Who does what
-
-| Component | Responsibility | What it does not prove |
-| --- | --- | --- |
-| Hermes | Maintains the conversation, chooses tools/skills, reasons over evidence, saves reports, and enforces the approval experience | It is not the inference model and does not automatically have cluster-admin access |
-| Ariadne | Collects and normalizes deterministic cluster, Jenkins, and quality evidence into a timestamped bundle; may optionally run a local diagnosis | Its diagnosis is not authoritative when stale, empty, or contradicted by the bundle |
-| Jenkins | Runs the suites and retains console/artifact evidence | A final `exit 1` alone does not identify the first failed gate |
-| Pushgateway | Receives suite/build quality telemetry | A cumulative counter is not the current build result |
-| VictoriaMetrics | Stores and answers PromQL for quality and environment telemetry | A Grafana color is not direct proof of root cause |
-| Grafana | Visualizes and alerts on VictoriaMetrics data | A red panel must be traced to its query, labels, time range, and raw series |
-| Flux | Applies reviewed Git state to the cluster | A recent reconciliation is correlation until the changed path matches the failure |
-| Kubernetes | Supplies workload, event, log, node, storage, and dependency evidence | An unrelated unhealthy pod is not proof that CI failed because of the cluster |
-
-Direct Jenkins artifact requests currently require authorization and can return
-HTTP 403. That does not break the workflow: Ariadne retains selected Jenkins
-console tails and named artifact contents in its deterministic bundle. A report
-must say `retained Ariadne evidence` when that fallback is used; it must not
-pretend direct Jenkins access succeeded.
-
-## The actual supervised triage algorithm
-
-1. Classify the request as test/build triage, service health, or alert tuning.
-2. Read the latest Ariadne diagnosis and deterministic bundle with HTTP GET.
-3. Compare their timestamps. The deterministic bundle remains authoritative.
-4. Select a terminal failed build. Keep running or unknown builds out of the
-   terminal-failure list.
-5. Identify the first enforced failed gate in this order:
-   `style -> loc -> coverage -> tests -> gate_glue -> sonarqube -> supply_chain`.
-6. Cite the smallest decisive Jenkins log/artifact evidence. State whether it
-   came directly from Jenkins or from Ariadne retention.
-7. Query the matching build/check/test series in VictoriaMetrics. Distinguish a
-   current gauge from a cumulative counter and ignore zero-valued failure
-   series.
-8. Correlate recent Gitea commits and the relevant Flux revision by timestamp
-   and affected path. Recency alone is not causation.
-9. Check only the Kubernetes resources capable of explaining that failure.
-   Classify observations as direct, contributing candidate, or background.
-10. Return `Finding`, `Confidence`, `Evidence`, `Likely cause`, `Blast radius`,
-    `Next checks`, `Repo-side fix`, and `Approval required`.
-11. Stop at a proposal. A human reviews the patch or action; Git and Flux remain
-    the delivery path.
-
-## What Hermes can read and what it cannot do
-
-The live `hermes-triage` ServiceAccount can get/list/watch ordinary workload,
-log, event, ingress, storage, Flux, and image-automation metadata. It cannot
-read Kubernetes Secret values, create exec sessions, patch Deployments, or
-patch Flux Kustomizations. The configured command deny list also blocks common
-Kubernetes and Flux mutations.
-
-The security boundary is layered:
-
-1. Keycloak authenticates the person.
-2. oauth2-proxy restricts the operator surface to Brad.
-3. The operator has a separate namespace, PVC, configuration, and ServiceAccount.
-4. Kubernetes RBAC is the hard API authorization boundary.
-5. NetworkPolicy limits reachable paths where configured.
-6. Hermes approvals and instructions provide a user-facing safety layer.
-
-A skill is procedure and context, not a permission grant. Adding a sentence to
-a skill cannot bypass Kubernetes RBAC.
-
-## What “Hermes learns the workflow” means
-
-It does not silently retrain model weights on cluster data. Learning here means
-that a successful repeated procedure is written as a reusable skill on the
-operator PVC or versioned in Git. The skill describes when it should trigger,
-the evidence order, interpretation rules, output contract, and safety boundary.
-
-The current workflow has a versioned top-level skill and persisted specialist
-skills for:
-
-- orchestration and evidence reporting;
-- retained Jenkins evidence;
-- quality metrics;
-- Gitea/Flux correlation;
-- Kubernetes failure classification;
-- Grafana metric provenance;
-- Soteria backup health;
-- approval-required actions.
-
-The model still evaluates fresh variable evidence on every incident. The skill
-makes the process repeatable; it does not freeze the answer.
-
-## The four operator skills to remember
-
-- `triage-titan-test-failures`: CI, tests, builds, quality gates, and suspected
-  test-environment regressions.
-- `triage-atlas-service-health`: active service or cluster incidents and red
-  high-level health panels.
-- `tune-atlas-alerts`: noisy alerts, impossible metrics, bad PromQL, and
-  generator-owned Grafana corrections.
-- `master-hermes-on-atlas`: hands-on training, assessments, architecture, and
-  claim audits.
-
-Do not start by choosing raw tools. State the operational question and let the
-skill route to the narrow evidence source.
-
-## Two live, repeatable proof cases
-
-### Proof 6: Soteria build 272
-
-- Terminal result: failure.
-- Tests: 318 passed, zero failed.
-- Local coverage: 96.195 percent.
-- First enforced failure: SonarQube `new_coverage=0.0` against threshold 80.
-- Runtime: Soteria remained Ready.
-- Correct conclusion: a Sonar new-code policy/reporting problem, not a test or
-  Kubernetes capacity failure.
-- Correct restraint: do not weaken the policy until scanner import and baseline
-  evidence identifies whether the issue is configuration or legitimate new
-  uncovered code.
-
-### Proof 7: Ananke build 242
-
-- Terminal result: failure.
-- Local coverage gate: 61.8 percent and failed.
-- Other checks: tests, SonarQube, supply-chain, LOC, docs naming, and gate glue
-  were healthy.
-- Sonar new-code coverage: 93.1 percent, which is a different scope.
-- Correct conclusion: local coverage input/scope caused the enforced failure;
-  do not call the Sonar number contradictory without comparing inputs.
-- Correct restraint: do not lower the threshold until the retained local gate
-  inputs are readable.
-
-The full sessions and redacted Markdown exports are in Sessions and
-`Files/triage-proof`.
-
-## Five-minute demonstration
-
-1. Open Models and show `openai-codex/gpt-5.6-terra` as primary and
-   `gpt-oss:20b` as local fallback.
-2. Open Skills and show the four operator skills plus the persisted component
-   skills.
-3. Open `Proof 6 - Soteria 272 Sonar new coverage triage` in Sessions.
-4. Point out the timestamped bundle, exact build/artifact evidence, metric
-   corroboration, healthy runtime, fact/inference separation, and approval
-   boundary.
-5. Open Proof 7 and explain why local coverage and Sonar new-code coverage can
-   differ without either number being fabricated.
-6. End by showing that Hermes proposes a repo-side correction but cannot patch
-   the Deployment or Flux Kustomization with its ServiceAccount.
-
-Use this short explanation:
-
-> Hermes is the persistent, permissioned workflow layer around the model. In
-> this cluster Ariadne collects deterministic evidence, Hermes correlates it
-> with retained Jenkins artifacts, Git/Flux state, Kubernetes health, and
-> quality metrics, and then returns a supervised proposal. Repeated procedures
-> become explicit skills. Kubernetes RBAC prevents the agent from turning a
-> diagnosis into an unreviewed infrastructure change.
-
-## Prompts that exercise the real system
-
-- `Use $triage-titan-test-failures. Triage the worst current terminal CI failure and link every decisive piece of evidence.`
-- `Use $triage-titan-test-failures. Triage Ananke build 242. Separate local coverage from Sonar new-code coverage.`
-- `Use $triage-atlas-service-health. Explain the VictoriaMetrics outage from current state and retained events; separate recovered impact from current impact.`
-- `Use $tune-atlas-alerts. Trace one currently firing alert to its generated source and raw PromQL, but do not edit it.`
-- `Use $master-hermes-on-atlas. Assess me on the request path and permission boundary. One question at a time.`
-
-## Honest limits
-
-- Hermes does not currently apply production or cluster changes autonomously.
-- Direct Jenkins console/artifact access may be forbidden; Ariadne retention is
-  the current fallback and must be named as provenance.
-- Local `gpt-oss:20b` is useful for bounded work but is slower and less reliable
-  for multi-source triage than Codex.
-- The current Ariadne local diagnosis can return an empty model response. That
-  does not invalidate the deterministic bundle or prevent Codex-backed Hermes
-  from triaging it.
-- A single successful case demonstrates capability, not mastery. Mastery means
-  leading different real incidents, improving the skill after failures, and
-  teaching the architecture without prompts.
-
-## Your shortest path to fluency
-
-1. Explain the request diagram without looking.
-2. Explain why Ariadne and Hermes are separate components.
-3. Reproduce Proof 6 from the raw bundle and metrics.
-4. Reproduce Proof 7 and explain the two coverage scopes.
-5. Produce the live RBAC allow/deny matrix with `kubectl auth can-i`.
-6. Create one small writable skill from a repeated sub-workflow, test a trigger
-   and non-trigger case, then improve it.
-7. Lead one new incident while Hermes coaches rather than answers for you.
-8. Teach the five-minute demonstration to another person.
-
-At that point the interview claims are demonstrable. Continue repeating real
-incidents before describing yourself as fully autonomous or the system as
-self-correcting.

services/hermes/agent-ingress.yaml

@@ -21,6 +21,6 @@ spec:
             pathType: Prefix
             backend:
               service:
-                name: oauth2-proxy-hermes
+                name: hermes
                 port:
-                  name: http
+                  name: dashboard

services/hermes/ariadne-handoff-rbac.yaml

@@ -1,30 +0,0 @@
-# services/hermes/ariadne-handoff-rbac.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: ariadne-gpu-handoff
-  namespace: hermes
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources:
-      - leases
-    resourceNames:
-      - titan-24-gpu-owner
-    verbs:
-      - get
-      - patch
-      - update
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: ariadne-gpu-handoff
-  namespace: hermes
-subjects:
-  - kind: ServiceAccount
-    name: ariadne
-    namespace: maintenance
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: ariadne-gpu-handoff

services/hermes/configmap.yaml

@@ -9,41 +9,11 @@ metadata:
 data:
   config.yaml: |
     model:
-      provider: openai-codex
-      default: gpt-5.6-terra
-      model: gpt-5.6-terra
-
-    fallback_providers:
-      - provider: custom
-        model: gpt-oss:20b
-        base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
-        api_key: ollama
-
-    agent:
-      api_max_retries: 1
-
-    platform_toolsets:
-      cli:
-        - clarify
-        - file
-        - session_search
-        - skills
-        - terminal
-        - todo
-        - web
-      api_server:
-        - clarify
-        - file
-        - session_search
-        - skills
-        - terminal
-        - todo
-        - web
-
-    skills:
-      creation_nudge_interval: 15
-      external_dirs:
-        - /opt/data/workspace/skills
+      provider: custom
+      default: qwen2.5:7b-instruct-q4_0
+      model: qwen2.5:7b-instruct-q4_0
+      base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1
+      api_key: ollama
 
     terminal:
       backend: local
@@ -52,7 +22,7 @@ data:
       home_mode: auto
 
     approvals:
-      mode: smart
+      mode: manual
       deny:
         - "*kubectl apply*"
         - "*kubectl delete*"
@@ -110,12 +80,6 @@ data:
     read-only state. Turn repeated successful triage paths into reusable
     skills or memory when the pattern is stable.
 
-    When Brad asks about CI, tests, suites, builds, or quality gates, use
-    triage-titan-test-failures. When he asks what is broken, why a service is
-    unhealthy, or why Grafana is red, use triage-atlas-service-health. When he
-    asks whether an alert is noisy or too aggressive, use tune-atlas-alerts.
-    Do not make him choose a tool, page, or evidence source first.
-
     Stay Flux-first. Do not mutate the cluster directly. Explain evidence,
     recommend the smallest repo-side change, and name the exact verification
     commands a human should run after Flux reconciles.
@@ -125,11 +89,6 @@ data:
     You are Hermes running inside the Titan Kubernetes cluster as a read-only
     testing and operations triage assistant.
 
-    Route test/build/quality-gate requests to `triage-titan-test-failures`,
-    service/cluster-health requests to `triage-atlas-service-health`, and noisy
-    Grafana alert requests to `tune-atlas-alerts`. Ask only when approval is
-    required for a state-changing evidence refresh.
-
     Ariadne owns deterministic evidence collection and local diagnosis. Start
     every testing triage by reading:
 
@@ -163,181 +122,3 @@ data:
     Do not run mutating commands such as `kubectl apply`, `delete`, `scale`,
     `patch`, `cordon`, `uncordon`, `drain`, or `rollout restart`. Do not read
     Kubernetes Secret values. Draft repo changes or operator steps instead.
-  START-HERE.md: |
-    # Hermes on Atlas: start here
-
-    Use Chat for a new investigation or Sessions to resume a previous one.
-    The shortest useful prompts are:
-
-    - `Triage the latest test failure.`
-    - `What is broken in the cluster right now?`
-    - `Why is Grafana red, and which alerts are noise?`
-    - `Triage <suite> build <number> and save the report.`
-
-    Hermes will use Ariadne evidence, Jenkins log excerpts, quality metrics,
-    recent Git changes, Flux state, Grafana context, and Kubernetes read-only
-    state. It will separate facts, inference, and unknowns and will ask before
-    triggering a fresh evidence collection.
-
-    Hermes now has three focused operator skills:
-
-    1. `triage-titan-test-failures` for all ten custom CI suites.
-    2. `triage-atlas-service-health` for any live namespace or service.
-    3. `tune-atlas-alerts` for evidence-backed alert and dashboard corrections.
-
-    Validated CI examples are available under Sessions, including:
-
-    1. `Proof 6 - Soteria 272 Sonar new coverage triage`
-    2. `Proof 7 - Ananke 242 local coverage triage`
-
-    Their saved reports are in Files under `triage-proof/`. Read
-    `HERMES-OPERATOR-RUNBOOK.md` for the system mental model and five-minute
-    demonstration.
-
-    Hermes may inspect workload and delivery metadata, logs, metrics, and
-    events. Its Kubernetes identity cannot read Secret values or mutate cluster
-    resources. Apply fixes through the titan-iac GitOps workflow after review.
-  HERMES-CAPABILITIES.md: |
-    # What Hermes can do on Atlas
-
-    ## Working now
-
-    - Triage all custom CI suites: Ananke, Ariadne, Atlasbot, bstein_home,
-      data_prepper, Lesavka, Metis, Pegasus, Soteria, and titan_iac.
-    - Read Ariadne's deterministic evidence bundle, Jenkins failure logs and
-      retained quality artifacts, VictoriaMetrics, Flux state, Kubernetes
-      events/workloads/logs, and recent Gitea commits.
-    - Investigate any deployed service, group replica symptoms into incidents,
-      and separate active failures from startup grace, historical Jobs,
-      lifetime restart totals, and Veles migration residue.
-    - Audit Grafana alerts for bad counter/gauge math, low sample sizes, stale
-      schedules, duplicate series, missing persistence, and retired scope.
-    - Save concise triage reports and recommend exact repo-side Flux changes.
-
-    ## Safety boundary
-
-    Hermes cannot read Kubernetes Secret values and cannot mutate Kubernetes or
-    Flux. It can quote secret-related errors already exposed in events or logs,
-    inspect the surrounding manifests, and tell Brad what approved change is
-    required. This keeps cluster operation supervised.
-
-    ## Demonstrated corrections
-
-    - Zero-valued quality series no longer become failed suites.
-    - Running Jenkins builds remain in-progress instead of failed.
-    - Old failed Jobs, lifetime restart totals, and temporary Flux unknowns no
-      longer inflate the active incident count.
-    - Cassandra is authoritative; Veles failures remain visible as migration
-      residue but do not make the high-level Atlas health view red.
-    - Worker readiness uses the actual 18 Kubernetes workers.
-    - Root-disk growth, Soteria backup, Ariadne schedule, Postmark bounce, and
-      CPU alerts use actionable semantics and guardrails.
-
-    The evidence and verification for these changes are in
-    `triage-proof/ATLAS-TRIAGE-PROOFS.md`.
-
-    ## Good next prompts
-
-    - `Use $triage-atlas-service-health. What is broken right now?`
-    - `Use $triage-titan-test-failures. Triage the worst current suite.`
-    - `Use $tune-atlas-alerts. Audit alerts fired in the last 24 hours.`
-    - `Save this investigation as a reusable skill after I approve the pattern.`
-  ATLAS-TRIAGE-PROOFS.md: |
-    # Atlas triage proofs
-
-    This file is the short evidence trail for what Hermes can do today. The
-    operator skills remain read-only: fixes are reviewed and delivered through
-    Git and Flux.
-
-    ## Proof 1: Cassandra secret-sync recovery
-
-    - Finding: `cassandra-vault-sync` was stuck because the expected
-      `VELES_BYOK_ENCRYPTION_KEY` migration input was absent from Cassandra's
-      Vault path.
-    - Action: the existing Flux-tracked, suspended one-shot migration Job was
-      enabled, completed successfully, and immediately returned to suspended.
-    - Verification: Cassandra frontend, backend, PostgreSQL, and Vault sync all
-      became Ready. No Kubernetes Secret value was read or committed.
-    - Why this matters: Hermes can correlate workload state, logs, Vault policy
-      manifests, and migration ownership without confusing Veles residue with
-      a current Veles outage.
-
-    ## Proof 2: Grafana alert-noise correction
-
-    - Removed retired Veles pods and completed Jobs from high-level incident
-      counts while preserving them on drill-down dashboards.
-    - Added 15-minute persistence to Pending, CrashLoopBackOff, and image-pull
-      summaries.
-    - Replaced invalid `increase()` use on gauges, bounded CPU percentages,
-      required meaningful Postmark sample sizes, and scoped Soteria/Ariadne
-      alerts to configured current work.
-    - Live verification on 2026-08-03: `Problem Pods=0`,
-      `CrashLoop/ImagePull=0`, and `Workers Ready=18/18`.
-
-    ## Proof 3: all-suite testing triage
-
-    - Ariadne build 384 passed 676 tests and the exact coverage contract:
-      82 source files at or above 95%.
-    - Deployed image: `registry.bstein.dev/bstein/ariadne:0.1.0-384`.
-    - The deterministic bundle covers Ananke, Ariadne, Atlasbot, bstein_home,
-      data_prepper, Lesavka, Metis, Pegasus, Soteria, and titan_iac.
-    - Zero-valued quality metrics are healthy, running Jenkins builds are
-      in-progress, old failed Jobs are historical, and Veles objects are
-      migration residue. They no longer inflate the active failure set.
-    - Failed builds still include direct Jenkins console and retained artifact
-      links so Hermes can identify the first failed gate and smallest repo fix.
-
-    ## Proof 4: Cassandra generator failure classification
-
-    - Finding: the Cassandra serving path is healthy, but generation Job
-      `cassandra-generator-3ce198971b` failed.
-    - Evidence: the primary OpenAI request returned HTTP 429 because its credit
-      balance was exhausted. Codex CLI fallback ran, but the strict promotion
-      gate still rejected unresolved high-impact hint claims and blocked the
-      semantic scenarios.
-    - Impact: one generation capability/request failed; this is not a Cassandra
-      registry outage and not a Veles outage.
-    - Next action: restore provider credits or continue through Codex fallback,
-      then resolve the reported hint claims before retrying. This needs operator
-      approval because it can spend money or rerun an expensive generation.
-
-    ## Proof 5: live Hermes Soteria triage
-
-    - Hermes read the fresh 2026-08-03 Ariadne bundle without an approval
-      timeout and chose Soteria build 270 as the worst terminal failure.
-    - It proved that all 318 tests passed and local coverage was 96.195%, then
-      isolated the enforced failure to SonarQube reporting new-code coverage
-      as 0.0%. It did not blame the optional supply-chain advisory.
-    - It kept running titan-iac, Data Prepper, and Lesavka builds out of the
-      terminal-failure list and requested approval before any rerun or edit.
-
-    ## Proof 6: Soteria build 272 Sonar new-coverage triage
-
-    - Hermes read the fresh 2026-08-04 Ariadne bundle and selected terminal
-      Soteria build 272.
-    - It proved that all 318 tests passed and local coverage was 96.195%, then
-      isolated the enforced failure to SonarQube `new_coverage=0.0` against the
-      threshold of 80.
-    - It corroborated the retained Jenkins evidence with current quality
-      metrics and healthy Soteria/Flux state, and did not blame Kubernetes.
-    - It proposed no change until scanner import and baseline evidence can
-      distinguish configuration error from genuinely uncovered new code.
-
-    ## Proof 7: Ananke build 242 local-coverage triage
-
-    - Hermes selected terminal Ananke build 242 and isolated the first failure
-      to the local coverage gate at 61.8%.
-    - It showed that tests, SonarQube, supply chain, LOC, docs naming, gate glue,
-      Flux, and cluster readiness were healthy.
-    - It distinguished the local 61.8% scope from SonarQube's 93.1% new-code
-      scope instead of treating the numbers as interchangeable.
-    - Direct Jenkins artifact reads returned HTTP 403, so it explicitly named
-      Ariadne's retained bundle as evidence provenance and stopped short of an
-      unsupported fix.
-
-    ## Use Hermes next
-
-    - `What is broken in the cluster right now?`
-    - `Triage the worst current CI suite and link the evidence.`
-    - `Audit Grafana alerts from the last 24 hours and separate incidents from noise.`
-    - `Explain the Cassandra generator failure and give me the smallest safe next step.`

services/hermes/deployment.yaml

@@ -20,11 +20,10 @@ spec:
       labels:
         app: hermes
       annotations:
-        ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation
-        ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback
+        ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
         ai.bstein.dev/role: testing-triage
         ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
-        ai.bstein.dev/config-rev: "20260804-root-operator-docs"
+        ai.bstein.dev/config-rev: "20260721-hermes-rollout-deadline"
     spec:
       serviceAccountName: hermes-triage
       automountServiceAccountToken: true
@@ -51,7 +50,6 @@ spec:
                       - titan-13
                       - titan-15
                       - titan-17
-                      - titan-18
                       - titan-19
           preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
@@ -82,18 +80,10 @@ spec:
             - -c
             - |
               set -eu
-              mkdir -p /opt/data/workspace/triage-proof /opt/data/home/.local/bin /opt/data/logs
+              mkdir -p /opt/data/workspace /opt/data/home/.local/bin /opt/data/logs
               cp /config/config.yaml /opt/data/config.yaml
               cp /config/SOUL.md /opt/data/SOUL.md
               cp /config/AGENTS.md /opt/data/workspace/AGENTS.md
-              cp /config/START-HERE.md /opt/data/workspace/START-HERE.md
-              cp /config/HERMES-CAPABILITIES.md /opt/data/workspace/HERMES-CAPABILITIES.md
-              cp /guide/OPERATOR-RUNBOOK.md /opt/data/workspace/HERMES-OPERATOR-RUNBOOK.md
-              cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/workspace/triage-proof/ATLAS-TRIAGE-PROOFS.md
-              cp /config/START-HERE.md /opt/data/START-HERE.md
-              cp /config/HERMES-CAPABILITIES.md /opt/data/HERMES-CAPABILITIES.md
-              cp /guide/OPERATOR-RUNBOOK.md /opt/data/HERMES-OPERATOR-RUNBOOK.md
-              cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/ATLAS-TRIAGE-PROOFS.md
               touch /opt/data/.env
               if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then
                 api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
@@ -109,8 +99,8 @@ spec:
               mountPath: /opt/data
             - name: config
               mountPath: /config
-            - name: operator-guide
-              mountPath: /guide
+            - name: tools
+              mountPath: /opt/data/home/.local/bin
           resources:
             requests:
               cpu: 25m
@@ -144,7 +134,7 @@ spec:
               memory: 64Mi
       containers:
         - name: hermes
-          image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
+          image: nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
           imagePullPolicy: IfNotPresent
           args:
             - gateway
@@ -199,20 +189,7 @@ spec:
             - name: home
               mountPath: /opt/data
             - name: tools
-              mountPath: /usr/local/bin/kubectl
-              subPath: kubectl
-            - name: triage-skill
-              mountPath: /opt/data/workspace/skills/triage-titan-test-failures
-              readOnly: true
-            - name: mastery-skill
-              mountPath: /opt/data/workspace/skills/master-hermes-on-atlas
-              readOnly: true
-            - name: service-health-skill
-              mountPath: /opt/data/workspace/skills/triage-atlas-service-health
-              readOnly: true
-            - name: alert-tuning-skill
-              mountPath: /opt/data/workspace/skills/tune-atlas-alerts
-              readOnly: true
+              mountPath: /opt/data/home/.local/bin
           readinessProbe:
             httpGet:
               path: /api/status
@@ -241,54 +218,5 @@ spec:
         - name: config
           configMap:
             name: hermes-config
-        - name: operator-guide
-          configMap:
-            name: hermes-operator-guide
         - name: tools
           emptyDir: {}
-        - name: triage-skill
-          configMap:
-            name: hermes-triage-skill
-            items:
-              - key: SKILL.md
-                path: SKILL.md
-              - key: openai.yaml
-                path: agents/openai.yaml
-        - name: mastery-skill
-          configMap:
-            name: hermes-mastery-skill
-            items:
-              - key: SKILL.md
-                path: SKILL.md
-              - key: openai.yaml
-                path: agents/openai.yaml
-              - key: architecture.md
-                path: references/architecture.md
-              - key: curriculum.md
-                path: references/curriculum.md
-              - key: incident-drills.md
-                path: references/incident-drills.md
-              - key: mastery-rubric.md
-                path: references/mastery-rubric.md
-              - key: two-hour-proof-sprint.md
-                path: references/two-hour-proof-sprint.md
-        - name: service-health-skill
-          configMap:
-            name: hermes-service-health-skill
-            items:
-              - key: SKILL.md
-                path: SKILL.md
-              - key: openai.yaml
-                path: agents/openai.yaml
-              - key: service-map.md
-                path: references/service-map.md
-        - name: alert-tuning-skill
-          configMap:
-            name: hermes-alert-tuning-skill
-            items:
-              - key: SKILL.md
-                path: SKILL.md
-              - key: openai.yaml
-                path: agents/openai.yaml
-              - key: alert-review.md
-                path: references/alert-review.md

services/hermes/kustomization.yaml

@@ -4,62 +4,11 @@ kind: Kustomization
 namespace: hermes
 resources:
   - namespace.yaml
-  - vault-serviceaccount.yaml
   - configmap.yaml
   - rbac.yaml
   - pvc.yaml
-  - model-gate-rbac.yaml
-  - ariadne-handoff-rbac.yaml
-  - model-gate-state.yaml
-  - model-gate-configmap.yaml
-  - model-gate-deployment.yaml
-  - networkpolicy.yaml
   - ollama-deployment.yaml
   - deployment.yaml
   - service.yaml
-  - oauth2-proxy.yaml
   - agent-certificate.yaml
   - agent-ingress.yaml
-
-configMapGenerator:
-  - name: hermes-operator-guide
-    namespace: hermes
-    files:
-      - OPERATOR-RUNBOOK.md=NOTES.md
-    options:
-      disableNameSuffixHash: true
-  - name: hermes-triage-skill
-    namespace: hermes
-    files:
-      - SKILL.md=skills/triage-titan-test-failures/SKILL.md
-      - openai.yaml=skills/triage-titan-test-failures/agents/openai.yaml
-    options:
-      disableNameSuffixHash: true
-  - name: hermes-mastery-skill
-    namespace: hermes
-    files:
-      - SKILL.md=skills/master-hermes-on-atlas/SKILL.md
-      - openai.yaml=skills/master-hermes-on-atlas/agents/openai.yaml
-      - architecture.md=skills/master-hermes-on-atlas/references/architecture.md
-      - curriculum.md=skills/master-hermes-on-atlas/references/curriculum.md
-      - incident-drills.md=skills/master-hermes-on-atlas/references/incident-drills.md
-      - mastery-rubric.md=skills/master-hermes-on-atlas/references/mastery-rubric.md
-      - two-hour-proof-sprint.md=skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md
-    options:
-      disableNameSuffixHash: true
-  - name: hermes-service-health-skill
-    namespace: hermes
-    files:
-      - SKILL.md=skills/triage-atlas-service-health/SKILL.md
-      - openai.yaml=skills/triage-atlas-service-health/agents/openai.yaml
-      - service-map.md=skills/triage-atlas-service-health/references/service-map.md
-    options:
-      disableNameSuffixHash: true
-  - name: hermes-alert-tuning-skill
-    namespace: hermes
-    files:
-      - SKILL.md=skills/tune-atlas-alerts/SKILL.md
-      - openai.yaml=skills/tune-atlas-alerts/agents/openai.yaml
-      - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md
-    options:
-      disableNameSuffixHash: true

services/hermes/model-gate-configmap.yaml

@@ -1,151 +0,0 @@
-# services/hermes/model-gate-configmap.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hermes-model-gate
-  namespace: hermes
-data:
-  model_gate.py: |
-    #!/usr/bin/env python3
-    """Fail-closed proxy that admits local inference only while Hermes owns titan-24."""
-
-    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-    import json
-    import os
-    from pathlib import Path
-    import ssl
-    import threading
-    import time
-    from urllib.error import HTTPError, URLError
-    from urllib.request import Request, urlopen
-
-
-    LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0")
-    LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
-    UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/")
-    LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes")
-    LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner")
-    CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1"))
-    API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
-    API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443")
-    TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
-    CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
-    LEASE_URL = (
-        f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/"
-        f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}"
-    )
-
-    _cache_lock = threading.Lock()
-    _cached_owner = "unavailable"
-    _cached_at = 0.0
-
-
-    def _lease_owner() -> str:
-      """Return the current owner, failing closed when Kubernetes is unavailable."""
-
-      global _cached_at, _cached_owner
-      now = time.monotonic()
-      with _cache_lock:
-        if now - _cached_at < CACHE_TTL_SEC:
-          return _cached_owner
-        try:
-          token = TOKEN_PATH.read_text(encoding="utf-8").strip()
-          request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"})
-          context = ssl.create_default_context(cafile=str(CA_PATH))
-          with urlopen(request, timeout=3, context=context) as response:
-            payload = json.load(response)
-          owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip()
-        except Exception:
-          owner = "unavailable"
-        _cached_owner = owner
-        _cached_at = now
-        return owner
-
-
-    class Handler(BaseHTTPRequestHandler):
-      """Proxy local model traffic while exposing health and ownership status."""
-
-      protocol_version = "HTTP/1.1"
-
-      def _json(self, status: int, payload: dict) -> None:
-        body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
-        self.send_response(status)
-        self.send_header("Content-Type", "application/json")
-        self.send_header("Content-Length", str(len(body)))
-        self.send_header("Cache-Control", "no-store")
-        self.end_headers()
-        self.wfile.write(body)
-
-      def _local_allowed(self) -> tuple[bool, str]:
-        owner = _lease_owner()
-        return owner == "hermes", owner
-
-      def _proxy(self) -> None:
-        allowed, owner = self._local_allowed()
-        if not allowed:
-          self._json(
-              503,
-              {
-                  "error": {
-                      "message": f"local GPU inference unavailable while titan-24 owner is {owner}",
-                      "type": "server_error",
-                  },
-                  "gpu_owner": owner,
-                  "fallback_required": True,
-              },
-          )
-          return
-
-        length = int(self.headers.get("Content-Length", "0") or 0)
-        body = self.rfile.read(length) if length else None
-        headers = {"Content-Type": self.headers.get("Content-Type", "application/json")}
-        if self.headers.get("Accept"):
-          headers["Accept"] = self.headers["Accept"]
-        request = Request(f"{UPSTREAM_URL}{self.path}", data=body, headers=headers, method=self.command)
-        try:
-          response = urlopen(request, timeout=1800)
-        except HTTPError as exc:
-          response = exc
-        except (TimeoutError, URLError) as exc:
-          self._json(503, {"error": {"message": f"local model upstream unavailable: {exc}", "type": "server_error"}})
-          return
-
-        self.send_response(response.status)
-        content_type = response.headers.get("Content-Type")
-        if content_type:
-          self.send_header("Content-Type", content_type)
-        content_length = response.headers.get("Content-Length")
-        if content_length:
-          self.send_header("Content-Length", content_length)
-        else:
-          self.send_header("Connection", "close")
-          self.close_connection = True
-        self.send_header("Cache-Control", "no-store")
-        self.end_headers()
-        while True:
-          chunk = response.read(65536)
-          if not chunk:
-            break
-          self.wfile.write(chunk)
-          self.wfile.flush()
-        response.close()
-
-      def do_GET(self) -> None:
-        if self.path == "/healthz":
-          self._json(200, {"status": "ok"})
-          return
-        if self.path == "/gate/status":
-          allowed, owner = self._local_allowed()
-          self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed})
-          return
-        self._proxy()
-
-      def do_POST(self) -> None:
-        self._proxy()
-
-      def log_message(self, format_string: str, *args) -> None:
-        print(f"model-gate {self.address_string()} {format_string % args}", flush=True)
-
-
-    if __name__ == "__main__":
-      ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever()

services/hermes/model-gate-deployment.yaml

@@ -1,125 +0,0 @@
-# services/hermes/model-gate-deployment.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: hermes-model-gate
-  namespace: hermes
-  labels:
-    app: hermes-model-gate
-spec:
-  replicas: 1
-  revisionHistoryLimit: 2
-  selector:
-    matchLabels:
-      app: hermes-model-gate
-  template:
-    metadata:
-      labels:
-        app: hermes-model-gate
-    spec:
-      serviceAccountName: hermes-model-gate
-      securityContext:
-        seccompProfile:
-          type: RuntimeDefault
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - arm64
-                  - key: node-role.kubernetes.io/worker
-                    operator: In
-                    values:
-                      - "true"
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-04
-                      - titan-13
-                      - titan-15
-                      - titan-17
-                      - titan-19
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 90
-              preference:
-                matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi5
-      containers:
-        - name: model-gate
-          image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
-          imagePullPolicy: IfNotPresent
-          command:
-            - python
-            - /opt/model-gate/model_gate.py
-          ports:
-            - name: http
-              containerPort: 8080
-          env:
-            - name: UPSTREAM_URL
-              value: http://hermes-ollama.hermes.svc.cluster.local:11434
-            - name: LEASE_NAMESPACE
-              value: hermes
-            - name: LEASE_NAME
-              value: titan-24-gpu-owner
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: http
-            initialDelaySeconds: 3
-            periodSeconds: 10
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: http
-            initialDelaySeconds: 10
-            periodSeconds: 30
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-            readOnlyRootFilesystem: true
-            runAsNonRoot: true
-            runAsUser: 65532
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 250m
-              memory: 128Mi
-          volumeMounts:
-            - name: script
-              mountPath: /opt/model-gate
-              readOnly: true
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: script
-          configMap:
-            name: hermes-model-gate
-            defaultMode: 0555
-        - name: tmp
-          emptyDir: {}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: hermes-model-gate
-  namespace: hermes
-  labels:
-    app: hermes-model-gate
-spec:
-  type: ClusterIP
-  selector:
-    app: hermes-model-gate
-  ports:
-    - name: http
-      port: 11434
-      targetPort: http

services/hermes/model-gate-rbac.yaml

@@ -1,34 +0,0 @@
-# services/hermes/model-gate-rbac.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: hermes-model-gate
-  namespace: hermes
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: hermes-model-gate
-  namespace: hermes
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources:
-      - leases
-    resourceNames:
-      - titan-24-gpu-owner
-    verbs:
-      - get
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: hermes-model-gate
-  namespace: hermes
-subjects:
-  - kind: ServiceAccount
-    name: hermes-model-gate
-    namespace: hermes
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: hermes-model-gate

services/hermes/model-gate-state.yaml

@@ -1,10 +0,0 @@
-# services/hermes/model-gate-state.yaml
-apiVersion: coordination.k8s.io/v1
-kind: Lease
-metadata:
-  name: titan-24-gpu-owner
-  namespace: hermes
-  annotations:
-    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
-spec:
-  holderIdentity: hermes

services/hermes/networkpolicy.yaml

@@ -1,30 +0,0 @@
-# services/hermes/networkpolicy.yaml
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: hermes-ollama-ingress
-  namespace: hermes
-spec:
-  podSelector:
-    matchLabels:
-      app: hermes-ollama
-  policyTypes:
-    - Ingress
-  ingress:
-    - from:
-        - podSelector:
-            matchLabels:
-              app: hermes-model-gate
-      ports:
-        - protocol: TCP
-          port: 11434
-    - from:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: maintenance
-          podSelector:
-            matchLabels:
-              app: ariadne
-      ports:
-        - protocol: TCP
-          port: 11434

services/hermes/oauth2-proxy.yaml

@@ -1,161 +0,0 @@
-# services/hermes/oauth2-proxy.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hermes-operator-allowlist
-  namespace: hermes
-data:
-  allowed-emails: |
-    brad@bstein.dev
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: oauth2-proxy-hermes
-  namespace: hermes
-  labels:
-    app: oauth2-proxy-hermes
-spec:
-  selector:
-    app: oauth2-proxy-hermes
-  ports:
-    - name: http
-      port: 80
-      targetPort: http
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: oauth2-proxy-hermes
-  namespace: hermes
-  labels:
-    app: oauth2-proxy-hermes
-spec:
-  replicas: 1
-  revisionHistoryLimit: 2
-  selector:
-    matchLabels:
-      app: oauth2-proxy-hermes
-  template:
-    metadata:
-      labels:
-        app: oauth2-proxy-hermes
-      annotations:
-        vault.hashicorp.com/agent-inject: "true"
-        vault.hashicorp.com/agent-pre-populate-only: "true"
-        vault.hashicorp.com/role: hermes
-        vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/operator-oidc
-        vault.hashicorp.com/agent-inject-template-oidc-config: |
-          {{- with secret "kv/data/atlas/hermes/operator-oidc" -}}
-          client_id = "{{ .Data.data.client_id }}"
-          client_secret = "{{ .Data.data.client_secret }}"
-          cookie_secret = "{{ .Data.data.cookie_secret }}"
-          {{- end -}}
-    spec:
-      serviceAccountName: hermes-vault
-      automountServiceAccountToken: true
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - arm64
-                  - key: node-role.kubernetes.io/worker
-                    operator: In
-                    values:
-                      - "true"
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-13
-                      - titan-15
-                      - titan-17
-                      - titan-18
-                      - titan-19
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 90
-              preference:
-                matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi5
-            - weight: 50
-              preference:
-                matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi4
-      containers:
-        - name: oauth2-proxy
-          image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0@sha256:dcb6ff8dd21bf3058f6a22c6fa385fa5b897a9cd3914c88a2cc2bb0a85f8065d
-          imagePullPolicy: IfNotPresent
-          args:
-            - --provider=oidc
-            - --config=/vault/secrets/oidc-config
-            - --redirect-url=https://agent.bstein.dev/oauth2/callback
-            - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas
-            - --code-challenge-method=S256
-            - --scope=openid profile email
-            - --email-domain=*
-            - --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails
-            - --set-xauthrequest=true
-            - --session-cookie-minimal=true
-            - --cookie-secure=true
-            - --cookie-samesite=lax
-            - --cookie-refresh=0
-            - --cookie-expire=8h
-            - --upstream=http://hermes.hermes.svc.cluster.local:9119
-            - --http-address=0.0.0.0:4180
-            - --skip-provider-button=true
-            - --skip-jwt-bearer-tokens=true
-            - --cookie-domain=agent.bstein.dev
-            - --reverse-proxy=true
-          ports:
-            - name: http
-              containerPort: 4180
-          readinessProbe:
-            httpGet:
-              path: /ping
-              port: http
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          livenessProbe:
-            httpGet:
-              path: /ping
-              port: http
-            initialDelaySeconds: 20
-            periodSeconds: 20
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-            readOnlyRootFilesystem: true
-            runAsNonRoot: true
-            seccompProfile:
-              type: RuntimeDefault
-          resources:
-            requests:
-              cpu: 25m
-              memory: 64Mi
-            limits:
-              cpu: 250m
-              memory: 256Mi
-          volumeMounts:
-            - name: allowlist
-              mountPath: /etc/oauth2-proxy
-              readOnly: true
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: allowlist
-          configMap:
-            name: hermes-operator-allowlist
-        - name: tmp
-          emptyDir:
-            sizeLimit: 64Mi

services/hermes/ollama-deployment.yaml

@@ -18,8 +18,8 @@ spec:
       labels:
         app: hermes-ollama
       annotations:
-        ai.bstein.dev/model: gpt-oss:20b
-        ai.bstein.dev/gpu: titan-24 local-first lane
+        ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
+        ai.bstein.dev/gpu: accelerator MVP lane (titan-24)
     spec:
       runtimeClassName: nvidia
       affinity:
@@ -33,8 +33,7 @@ spec:
                       - titan-24
       volumes:
         - name: models
-          persistentVolumeClaim:
-            claimName: hermes-models
+          emptyDir: {}
       initContainers:
         - name: warm-model
           image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
@@ -45,7 +44,7 @@ spec:
             - name: OLLAMA_MODELS
               value: /root/.ollama
             - name: OLLAMA_MODEL
-              value: gpt-oss:20b
+              value: qwen2.5:7b-instruct-q4_0
             - name: NVIDIA_VISIBLE_DEVICES
               value: all
             - name: NVIDIA_DRIVER_CAPABILITIES
@@ -69,7 +68,7 @@ spec:
               nvidia.com/gpu.shared: 1
             limits:
               cpu: "4"
-              memory: 16Gi
+              memory: 10Gi
               nvidia.com/gpu.shared: 1
       containers:
         - name: ollama
@@ -83,16 +82,6 @@ spec:
               value: 0.0.0.0
             - name: OLLAMA_KEEP_ALIVE
               value: 6h
-            - name: OLLAMA_CONTEXT_LENGTH
-              value: "64000"
-            - name: OLLAMA_FLASH_ATTENTION
-              value: "1"
-            - name: OLLAMA_KV_CACHE_TYPE
-              value: q8_0
-            - name: OLLAMA_MAX_LOADED_MODELS
-              value: "1"
-            - name: OLLAMA_NUM_PARALLEL
-              value: "1"
             - name: OLLAMA_MODELS
               value: /root/.ollama
             - name: NVIDIA_VISIBLE_DEVICES
@@ -111,10 +100,10 @@ spec:
             timeoutSeconds: 5
           resources:
             requests:
-              cpu: "8"
-              memory: 24Gi
+              cpu: "2"
+              memory: 8Gi
               nvidia.com/gpu.shared: 1
             limits:
-              cpu: "16"
-              memory: 40Gi
+              cpu: "6"
+              memory: 12Gi
               nvidia.com/gpu.shared: 1

services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js

@@ -1,88 +0,0 @@
-(function () {
-  "use strict";
-
-  var pluginName = "atlas-operator-guide";
-  var storageKey = "atlas.operator-guide.hidden.v1";
-  var prompt = [
-    "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint.",
-    "I will perform every setup and evidence step myself.",
-    "Start with step 1 only, require live proof, and stop at every mutation or approval boundary.",
-    "First confirm this session is using openai-codex/gpt-5.6-terra rather than the local fallback."
-  ].join(" ");
-
-  function OperatorGuide() {
-    var sdk = window.__HERMES_PLUGIN_SDK__;
-    var React = sdk.React;
-    var state = sdk.hooks.useState(function () {
-      return window.localStorage.getItem(storageKey) === "1";
-    });
-    var hidden = state[0];
-    var setHidden = state[1];
-    var copiedState = sdk.hooks.useState(false);
-    var copied = copiedState[0];
-    var setCopied = copiedState[1];
-
-    if (hidden) {
-      return React.createElement(
-        "button",
-        {
-          className: "atlas-guide-restore",
-          onClick: function () {
-            window.localStorage.removeItem(storageKey);
-            setHidden(false);
-          },
-          type: "button"
-        },
-        "Show Atlas operator walkthrough"
-      );
-    }
-
-    function copyPrompt() {
-      navigator.clipboard.writeText(prompt).then(function () {
-        setCopied(true);
-        window.setTimeout(function () { setCopied(false); }, 2500);
-      });
-    }
-
-    return React.createElement(
-      "section",
-      { className: "atlas-guide-card", role: "region", "aria-label": "Atlas operator walkthrough" },
-      React.createElement(
-        "div",
-        { className: "atlas-guide-heading" },
-        React.createElement("div", null,
-          React.createElement("strong", null, "Start here: two-hour Hermes proof sprint"),
-          React.createElement("p", null, "You do the setup and evidence work. Hermes coaches, checks, and stops at approval boundaries.")
-        ),
-        React.createElement(
-          "button",
-          {
-            className: "atlas-guide-hide",
-            onClick: function () {
-              window.localStorage.setItem(storageKey, "1");
-              setHidden(true);
-            },
-            type: "button"
-          },
-          "Hide"
-        )
-      ),
-      React.createElement(
-        "ol",
-        { className: "atlas-guide-steps" },
-        React.createElement("li", null, "Confirm Models shows openai-codex/gpt-5.6-terra for the coaching session."),
-        React.createElement("li", null, "Copy the sprint prompt, paste it into Chat, and press Enter."),
-        React.createElement("li", null, "Complete one live triage and build one writable skill yourself.")
-      ),
-      React.createElement(
-        "div",
-        { className: "atlas-guide-actions" },
-        React.createElement("a", { className: "atlas-guide-button", href: "/models" }, "Open Models"),
-        React.createElement("button", { className: "atlas-guide-button", onClick: copyPrompt, type: "button" }, copied ? "Prompt copied" : "Copy sprint prompt"),
-        React.createElement("a", { className: "atlas-guide-link", href: "/skills" }, "Inspect skills")
-      )
-    );
-  }
-
-  window.__HERMES_PLUGINS__.registerSlot(pluginName, "chat:top", OperatorGuide);
-}());

services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css

@@ -1,64 +0,0 @@
-.atlas-guide-card {
-  border: 1px solid rgba(247, 201, 72, 0.7);
-  background: rgba(247, 201, 72, 0.08);
-  color: inherit;
-  padding: 0.75rem 1rem;
-  font-family: "Mondwest", monospace;
-}
-
-.atlas-guide-heading {
-  display: flex;
-  align-items: flex-start;
-  justify-content: space-between;
-  gap: 1rem;
-}
-
-.atlas-guide-heading strong {
-  color: #f7c948;
-  font-family: "Rules Expanded", sans-serif;
-  letter-spacing: 0.04em;
-  text-transform: uppercase;
-}
-
-.atlas-guide-heading p {
-  margin: 0.25rem 0 0;
-  opacity: 0.8;
-}
-
-.atlas-guide-steps {
-  margin: 0.6rem 0;
-  padding-left: 1.25rem;
-}
-
-.atlas-guide-actions {
-  display: flex;
-  flex-wrap: wrap;
-  align-items: center;
-  gap: 0.5rem;
-}
-
-.atlas-guide-button,
-.atlas-guide-hide,
-.atlas-guide-restore {
-  border: 1px solid rgba(247, 201, 72, 0.7);
-  background: transparent;
-  color: #f7c948;
-  cursor: pointer;
-  padding: 0.35rem 0.6rem;
-  text-decoration: none;
-}
-
-.atlas-guide-button:hover,
-.atlas-guide-hide:hover,
-.atlas-guide-restore:hover {
-  background: rgba(247, 201, 72, 0.14);
-}
-
-.atlas-guide-hide,
-.atlas-guide-restore {
-  font-size: 0.75rem;
-}
-
-.atlas-guide-link {
-  color: #f7c948;
-}

services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json

@@ -1,16 +0,0 @@
-{
-  "name": "atlas-operator-guide",
-  "label": "Atlas Operator Guide",
-  "description": "Hands-on onboarding for supervised Atlas triage and Hermes skill building.",
-  "icon": "Sparkles",
-  "version": "1.0.0",
-  "tab": {
-    "path": "/atlas-operator-guide",
-    "hidden": true
-  },
-  "slots": [
-    "chat:top"
-  ],
-  "entry": "dist/index.js",
-  "css": "dist/style.css"
-}

services/hermes/pvc.yaml

@@ -13,18 +13,3 @@ spec:
   resources:
     requests:
       storage: 4Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: hermes-models
-  namespace: hermes
-  labels:
-    app: hermes-ollama
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: local-path
-  resources:
-    requests:
-      storage: 30Gi

services/hermes/rbac.yaml

@@ -22,7 +22,6 @@ rules:
       - pods
       - pods/log
       - replicationcontrollers
-      - serviceaccounts
       - services
     verbs: ["get", "list", "watch"]
   - apiGroups: ["apps"]
@@ -55,12 +54,6 @@ rules:
       - gitrepositories
       - helmrepositories
     verbs: ["get", "list", "watch"]
-  - apiGroups: ["image.toolkit.fluxcd.io"]
-    resources:
-      - imagepolicies
-      - imagerepositories
-      - imageupdateautomations
-    verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -74,3 +67,4 @@ subjects:
   - kind: ServiceAccount
     name: hermes-triage
     namespace: hermes
+

services/hermes/skills/master-hermes-on-atlas/SKILL.md

@@ -1,84 +0,0 @@
----
-name: master-hermes-on-atlas
-description: Coach Brad through hands-on mastery of the live Atlas Hermes deployment, including its web and CLI interfaces, local GPU inference and Codex fallback, tools, skills, sessions, profiles, channels, security boundaries, and evidence-driven test-failure triage. Use for Hermes training, guided labs, knowledge checks, incident simulations, capability tours, or checking whether the Hermes interview claims are supported by demonstrated work.
----
-
-# Master Hermes on Atlas
-
-Act as a demanding, practical coach. Teach the deployed system, not a generic
-Hermes installation. Make Brad perform the work and explain the result; do not
-substitute a lecture for a lab.
-
-## Select the training model
-
-Use `openai-codex/gpt-5.6-terra` for assessments, multi-reference labs, incident
-grading, and skill evaluation. If the active provider is the local
-`gpt-oss:20b`, stop before reading the reference files and ask Brad to select
-Codex in Models or start a Codex-backed session. Continue locally only when Brad
-explicitly asks to evaluate the local model itself or requests a lightweight
-single-question review.
-
-## Start or resume training
-
-1. Read `references/architecture.md` before teaching deployment-specific facts.
-2. Read `references/curriculum.md` to select the next lab.
-3. When Brad asks for the two-hour proof sprint, read
-   `references/two-hour-proof-sprint.md` and run it in order. Brad performs
-   every required UI action and evidence check; coach and verify without doing
-   the learning-critical steps for him.
-4. Ask whether to assess, resume, or choose a lab. If no progress record exists,
-   default to a five-question assessment followed by the first weak area.
-5. Give one bounded task at a time. State the goal, safety boundary, exact
-   success evidence, and at most one initial hint.
-6. Wait for Brad's answer or observed command output before revealing the
-   explanation.
-7. Grade with `references/mastery-rubric.md`. Separate demonstrated ability
-   from verbal familiarity.
-8. Offer a concise progress update. Write it to
-   `/opt/data/workspace/hermes-training/progress.md` only after Brad explicitly
-   approves the file change.
-
-## Run live exercises safely
-
-- Use read-only commands by default: `kubectl get`, `describe`, `logs`, `auth
-  can-i`, HTTP GET, `hermes status`, and Hermes list/status commands.
-- Never read Kubernetes Secret values. Never use Vault reads as a training
-  shortcut. Redact bearer tokens, cookies, device codes, and credentials.
-- Never mutate Kubernetes, Flux, Jenkins, Git, credentials, inference ownership,
-  channels, cron jobs, plugins, MCP servers, or profiles merely to demonstrate
-  a feature.
-- Draft state-changing commands and explain their effect. Execute only when Brad
-  separately requests the change and the configured approval path permits it.
-- Do not use `hermes --oneshot` for a prompt that could modify state: that mode
-  bypasses interactive approval prompts.
-- Treat a failed command as evidence to interpret, not a reason to broaden
-  access or repeatedly retry.
-
-## Teach evidence-driven triage
-
-For a live test-failure lab, also use `$triage-titan-test-failures`. Require the
-student to distinguish:
-
-- observed fact from inference;
-- stale evidence from current state;
-- application regression from environment failure;
-- a read-only next check from a proposed mutation;
-- a generic suggestion from the smallest Flux-tracked repo-side fix.
-
-For offline practice, read `references/incident-drills.md` and present only the
-student packet for one incident. Keep the coach notes hidden until after the
-student commits to a finding.
-
-## Validate claims honestly
-
-Use the claim audit in `references/mastery-rubric.md`. Current configuration and
-live demonstrations support current-use claims. The retained Git history and
-bound OpenClaw PVC support the deployment/replacement history; require Brad to
-inspect that evidence and distinguish it from proof of every past interaction.
-
-End every session with:
-
-- what Brad demonstrated;
-- what remains unproven;
-- the next smallest lab;
-- any safety or operational issue discovered.

services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml

@@ -1,4 +0,0 @@
-interface:
-  display_name: "Master Hermes on Atlas"
-  short_description: "Practice Hermes safely on the live Atlas stack"
-  default_prompt: "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint. I will perform every setup and evidence step myself. Start with step 1 only, require live proof, and stop at every mutation or approval boundary. First confirm this session is using openai-codex/gpt-5.6-terra rather than the local fallback."

services/hermes/skills/master-hermes-on-atlas/references/architecture.md

@@ -1,117 +0,0 @@
-# Live Atlas Hermes architecture
-
-Use this reference for deployment-specific facts. Re-check live state before
-asserting health, placement, ownership, or current model availability.
-
-## Two isolated agent instances
-
-| Surface | Purpose | Identity boundary | State and permissions |
-| --- | --- | --- | --- |
-| `agent.bstein.dev` | Brad's operator and triage lab | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace, its own PVC and service account; read-only cluster triage plus approved internal evidence endpoints |
-| `chat.bstein.dev` | Consumer chat and research | Keycloak login | `hermes-chat` namespace, separate PVC/config/auth/sessions/skills; read-only cluster observer RBAC and no access to Secrets or Kubernetes mutation APIs |
-
-The instances do not share conversation state, credentials, profiles, skills
-created on their PVCs, or Kubernetes identities. They share only the inference
-service described below. Consumer users cannot reach the operator dashboard
-through the chat Service or Ingress.
-
-## Inference path
-
-```text
-operator Hermes ───> independently authenticated openai-codex/gpt-5.6-terra
-consumer Hermes ───> independently authenticated openai-codex/gpt-5.6-terra
-       │
-       └─ provider error or manual selection ─> hermes-model-gate
-                                                │
-                                                ├─ Hermes owns GPU ─> Ollama gpt-oss:20b
-                                                └─ Wolf owns GPU ───> HTTP 503
-```
-
-- The configured context length is 64,000 tokens.
-- `hermes-model-gate` is the stable OpenAI-compatible endpoint.
-- A coordination Lease stores whether Hermes or Wolf/Moonlight owns the GPU.
-  Ariadne has narrowly scoped permission to update that Lease during handoff.
-- Wolf ownership makes the local gate return a deliberate unavailable response.
-  Normal interactive chat remains on its Codex primary and does not consume
-  titan-24 GPU resources.
-- The two agent pods do not consume titan-24 GPU memory. Ollama does. Agent pods
-  are ARM gateway workloads with persistent state on separate PVCs.
-- Codex credentials are deliberately per-instance. Never copy Brad's Codex
-  credential store into the consumer PVC.
-
-Verify rather than memorize:
-
-```sh
-hermes status
-hermes fallback list
-kubectl -n hermes get lease titan-24-gpu-owner -o yaml
-kubectl -n hermes get deploy hermes hermes-model-gate hermes-ollama -o wide
-kubectl -n hermes-chat get deploy hermes-chat -o wide
-```
-
-Do not change the GPU owner as part of a lesson.
-
-## Retained OpenClaw replacement evidence
-
-OpenClaw is not part of the current inference path, but its deployment history
-is retained:
-
-- `1bc58e10` added the OpenClaw testing-triage workspace, workload, model,
-  ingress, RBAC, and storage.
-- follow-up commits wired local diagnosis, OIDC, persistence, and the gateway.
-- `97ecb36b` replaced OpenClaw with Hermes and moved the agent-facing resources.
-- `bb5e286f` decoupled Hermes from the retired OpenClaw Flux dependency.
-- the `openclaw-home` PVC remains bound as retained state while no OpenClaw
-  workload is running.
-
-Have Brad verify the commits and live resource shape. This establishes the
-cluster deployment and replacement sequence; Brad's direct experience remains
-the evidence for how he personally used the old agent.
-
-## Operator evidence path
-
-The operator role is designed for supervised test-failure triage:
-
-1. Ariadne deterministic bundle and optional diagnosis.
-2. Retained Jenkins logs and artifacts.
-3. Recent Git changes and Flux revision/state.
-4. Kubernetes workload, node, event, storage, DNS, and dependency health.
-5. Pushgateway quality metrics and VictoriaMetrics/Grafana context.
-6. A concise finding, confidence, evidence, blast radius, next checks,
-   repo-side fix, and approval-required section.
-
-The versioned `$triage-titan-test-failures` skill encodes this workflow. The
-operator has internal URLs for Ariadne, Jenkins, and VictoriaMetrics. The
-consumer instance intentionally does not.
-
-## Security layers and their limits
-
-Security is layered rather than delegated to a prompt:
-
-1. Keycloak authenticates the human.
-2. The operator proxy limits `agent.bstein.dev` to Brad's exact email.
-3. Separate namespaces, PVCs, service accounts, configs, and ingresses isolate
-   operator and consumer state.
-4. Kubernetes RBAC grants the consumer get/list/watch only and excludes Secrets,
-   exec, attach, port-forward, and mutation.
-5. NetworkPolicy permits the consumer to reach DNS, the shared model gate, the
-   Kubernetes API, and public IPv4 while blocking arbitrary private service
-   access.
-6. Hermes deny patterns and instructions add user-facing guardrails.
-
-Prompt rules and shell deny patterns are not the primary security boundary.
-RBAC, network policy, credential separation, and routing isolation are. The
-consumer can still disclose non-secret cluster metadata it is authorized to
-read; that is an intentional requirement and should be described honestly.
-
-## CLI-to-web mental model
-
-The pinned Hermes build exposes chat, models and fallback, auth, sessions,
-profiles, skills and bundles, plugins, tools, MCP, gateway/channels, pairing,
-webhooks, cron, kanban, projects, hooks, memory/journey, logs, security, backup,
-and diagnostics. The web interface is a control surface over many of these
-same persisted capabilities.
-
-Use `<command> --help` and the web Documentation page for the installed version.
-Do not rely on screenshots or online docs from a different release when a live
-command can settle the question.

services/hermes/skills/master-hermes-on-atlas/references/curriculum.md

@@ -1,131 +0,0 @@
-# Hands-on curriculum
-
-Complete labs by evidence, not elapsed time. A focused pass can establish
-operational competence in several days; mastery requires repeating real triage
-and recovery work over multiple incidents.
-
-Run the curriculum on `openai-codex/gpt-5.6-terra`. Use the local model only for a
-deliberate comparison lab; it is not the default coach for multi-reference work.
-
-## Phase 1: orientation and control
-
-### Lab 1 — Map the system
-
-From the web UI, identify Chat, Sessions, Files, Models, Logs, Cron, Skills,
-Plugins, MCP, Channels, Webhooks, Pairing, Profiles, Config, Keys, System, and
-Documentation. Explain which state belongs to the operator PVC and which
-components are shared. Verify three claims with read-only CLI output.
-
-Success evidence: a correct diagram or written request path from browser to
-agent to model, including the fallback branch and the consumer boundary.
-
-### Lab 2 — Models, context, and fallback
-
-Inspect `hermes status`, `hermes fallback list`, deployment placement, and GPU
-owner state. Explain why a 32K model was rejected, why normal Codex-backed chat
-remains available when Wolf owns the GPU, and when the local fallback can run.
-
-Success evidence: predict outcomes for local healthy, local slow, gate 503,
-invalid local response, and expired Codex authorization without changing state.
-
-### Lab 3 — Sessions, files, profiles, and logs
-
-Create a named training session and a harmless workspace note through the UI,
-then find the corresponding session/file/log surfaces. Inspect profile and
-backup help without creating a profile or backup.
-
-Success evidence: explain persistence, what survives a pod replacement, what is
-instance-local, and how to recover a lost conversation without exposing auth.
-
-## Phase 2: tools and safe autonomy
-
-### Lab 4 — Tools versus skills versus MCP
-
-Use `hermes tools`, `hermes skills`, `hermes plugins`, and `hermes mcp` help or
-list/status views. Classify each as executable capability, procedure/context,
-packaged extension, or external protocol integration. Explain why a skill is
-not a security boundary.
-
-Success evidence: choose the right mechanism for three examples: repeatable
-Titan triage, a read-only external API, and a scheduled notification.
-
-### Lab 5 — Prove the permission boundary
-
-Use `kubectl auth can-i` as both service accounts. Inspect the relevant
-ClusterRoles and NetworkPolicies. Do not attempt mutations.
-
-Success evidence: an allow/deny matrix covering pods, logs, Secrets, exec,
-deployment patch, Flux reads, Flux reconcile, the model gate, internal services,
-and public web research.
-
-### Lab 6 — Channels, pairing, webhooks, and cron
-
-Inspect the configured state and help for gateway/channels, pairing, webhooks,
-and cron. Design one safe notification workflow and one unsafe workflow. Do not
-register a channel, create a webhook, or schedule a job during the lab.
-
-Success evidence: identify the identity, secret, audience, tool policy, failure
-mode, audit trail, and revocation path for the proposed integration.
-
-## Phase 3: the interview workflow
-
-### Lab 7 — Deterministic live triage
-
-Invoke `$triage-titan-test-failures`. Read the current Ariadne diagnosis and
-bundle, then correlate one suite with Jenkins, Flux/Git, cluster health, and
-quality metrics. Keep every collection step read-only.
-
-Success evidence: the required structured triage result with timestamps and no
-invented evidence.
-
-### Lab 8 — Incident simulations
-
-Use one student packet from `incident-drills.md`. Diagnose it before receiving
-coach notes. Repeat until three cases score at least `Independent`.
-
-Success evidence: correct failure class, causal chain, next checks, minimal
-repo-side correction, and explicit approval boundary.
-
-### Lab 9 — Build and evaluate a reusable skill
-
-Identify a stable repeated workflow, create a small skill in the operator's
-writable skills directory, inspect it, and test it against a fresh incident.
-Do not alter the Flux-mounted training or triage skills.
-
-Success evidence: precise trigger description, concise procedure, progressive
-disclosure where useful, one passing case, one adversarial case, and a stated
-permission boundary.
-
-### Lab 10 — Model outage drill
-
-Use retained logs/current status or a separately approved maintenance window.
-Do not take the live model down for training. Explain the expected trace from
-model gate to fallback, recognize an auth failure, and name the restoration
-checks.
-
-Success evidence: correctly distinguish model unavailability, context rejection,
-provider auth failure, and agent failure.
-
-## Phase 4: independent operation
-
-### Lab 11 — Supervised real incident
-
-Lead a real failure triage from intake to an approved repo-side proposal. A
-human applies any change through Git/Flux. Verify the result read-only.
-
-### Lab 12 — Teach it back and audit the claims
-
-Explain the complete architecture and demonstrate the workflow without hints.
-Audit each sentence of the interview answer using the claim table in
-`mastery-rubric.md`. Weaken or qualify anything not supported by evidence.
-
-## Suggested pace
-
-- Day 1: Labs 1–3.
-- Days 2–3: Labs 4–6.
-- Days 4–7: Labs 7–10.
-- Weeks 2–4: repeat Labs 7–11 on real incidents; improve one skill from observed
-  failures; finish with Lab 12.
-
-Do not advance solely because a day elapsed. Re-run any lab graded below
-`Independent`.

services/hermes/skills/master-hermes-on-atlas/references/incident-drills.md

@@ -1,95 +0,0 @@
-# Incident drills
-
-Present only one `Student packet` before the student answers. Use the coach
-notes afterward to grade the failure classification and reasoning. These cases
-are based on retained Atlas rollout observations; treat them as training
-fixtures, not proof of current live state.
-
-## Case A — Metrics disappear after a Soteria run
-
-### Student packet
-
-- The Soteria test stages ran.
-- The post-stage log ends with:
-  `Syntax error: end of file unexpected (expecting "fi")`.
-- The canonical Soteria Pushgateway series did not refresh.
-- Other suites continued publishing.
-
-Ask for: finding, confidence, evidence, likely cause, blast radius, read-only
-next checks, repo-side fix, and approval-required actions.
-
-### Coach notes
-
-The primary class is pipeline glue/shell syntax, not a Pushgateway outage and
-not a product-test failure. Inspect the post block and the exact shell parsed by
-`/bin/sh`; confirm the last successful metric timestamp. The smallest likely
-fix is closing/correcting the conditional in the Jenkinsfile, then a normal SCM
-run and read-only metric verification. Editing, pushing, triggering, or manual
-metric backfill requires explicit approval.
-
-## Case B — UI tests fail before metrics publish
-
-### Student packet
-
-- `bstein-dev-home` requires Playwright `1.59.1`.
-- The CI test container contains Playwright `1.51.0`.
-- The frontend test stage hard-fails.
-- No fresh canonical quality metrics appear for the run.
-
-### Coach notes
-
-Separate the direct test-environment mismatch from the telemetry-control-flow
-defect. The version mismatch explains the test failure; early pipeline abort
-explains missing metrics. A complete proposal aligns the image version and
-persists stage return codes so publishing still runs while the final gate still
-fails. Do not call this a Kubernetes capacity problem without node/pod evidence.
-
-## Case C — Promotion exits 127
-
-### Student packet
-
-- Titan IaC validation and tests finish.
-- The Promote stage uses `python:3.12-slim`.
-- The stage returns exit code `127` at its first Git command.
-- The repository and remote are reachable from other jobs.
-
-### Coach notes
-
-Exit 127 means the invoked command is unavailable. Confirm the exact log line
-and image contents; the likely issue is that the slim image lacks `git`, not bad
-Git credentials. The minimal repo-side fix installs `git` and CA certificates
-in the job's dependency setup or uses a suitable pinned runner image.
-
-## Case D — Data Preppers cannot publish an image
-
-### Student packet
-
-- Tests and local image build complete.
-- Push to `registry.bstein.dev/streaming/data-prepper:2.8.0` is denied.
-- The configured Jenkins credential belongs to a different Harbor project.
-- A Sonar evidence GET also returns `401` in the same run.
-
-### Coach notes
-
-Do not collapse independent authentication failures into a generic network
-failure. Verify the Harbor repository/project and credential ID without reading
-the secret value; separately verify the Sonar token injection path and API
-scope. The likely image fix is selecting a streaming-scoped robot credential.
-Treat the Sonar 401 as a second issue unless evidence proves a shared cause.
-
-## Case E — Hermes local inference fails during game streaming
-
-### Student packet
-
-- Wolf owns titan-24 according to the GPU-owner state.
-- `hermes-model-gate` is ready but returns a deliberate unavailable response.
-- Both Hermes dashboards remain healthy.
-- One instance answers through Codex; the other reports missing provider auth.
-
-### Coach notes
-
-This is expected resource arbitration plus an instance-specific fallback auth
-gap. Do not restart the agent or claim the GPU is underprovisioned. Confirm the
-gate response, fallback chain, and each instance's auth status without exposing
-tokens. Authorize the affected instance with its intended user's account. The
-separate credential stores are a security property, not configuration drift.

services/hermes/skills/master-hermes-on-atlas/references/mastery-rubric.md

@@ -1,75 +0,0 @@
-# Mastery and claim rubric
-
-## Performance levels
-
-| Level | Evidence |
-| --- | --- |
-| Exposed | Recognizes terms but needs the path and commands supplied. |
-| Assisted | Completes a lab with hints and can explain the result afterward. |
-| Independent | Selects safe tools, gathers fresh evidence, and reaches a defensible result without hints. |
-| Mastery | Handles ambiguity and failure, teaches the architecture, improves the reusable workflow, and preserves security boundaries. |
-
-Score each domain separately:
-
-1. Architecture and request routing.
-2. Models, context, GPU ownership, and fallback.
-3. Sessions, files, profiles, logs, and recovery.
-4. Tools, skills, plugins, MCP, channels, webhooks, and cron.
-5. Identity, RBAC, network, credentials, approvals, and auditability.
-6. Jenkins/Flux/Kubernetes/metrics evidence correlation.
-7. Clear triage findings and minimal repo-side proposals.
-8. Skill creation, evaluation, and iteration.
-
-Never award `Independent` from a verbal answer alone when the lab calls for live
-or fixture evidence. Never award `Mastery` until Brad has led at least two
-different real incidents and improved a workflow based on what failed.
-
-## Interview claim audit
-
-| Claim | What supports it | Minimum demonstration |
-| --- | --- | --- |
-| Hermes runs in the Kubernetes cluster | Flux manifests and live workloads | Trace browser, agent, model gate, Ollama, PVC, and service account. |
-| Hermes fits existing automation | Operator config, internal evidence URLs, triage skill | Complete a triage using Ariadne, Jenkins, Git/Flux, cluster health, and metrics. |
-| Hermes follows Brad's triage path | `$triage-titan-test-failures` procedure | Lead two evidence-backed cases with facts separated from inference. |
-| Repeated workflows become reusable skills | Versioned triage/training skills plus writable user skill area | Build, test, and improve one skill from a repeated real workflow. |
-| Access starts read-only and scoped | RBAC, NetworkPolicy, separate identities and state | Produce and explain the allow/deny matrix from live authorization checks. |
-| Human approval protects changes | Approval config, deny patterns, Flux workflow | Identify every mutation in a proposed incident response and stop before it. |
-| Local GPU gracefully yields to gaming | Owner state, model gate, fallback | Explain or observe a Wolf ownership window and verify continued fallback service. |
-| OpenClaw previously ran and was replaced | `1bc58e10` deploys the OpenClaw triage stack; follow-up commits add gateway/OIDC/persistence; `97ecb36b` replaces it with Hermes; `bb5e286f` retires the dependency; the bound `openclaw-home` PVC remains without a workload | Inspect the commits and live Flux/PVC state, then explain what they prove versus what depends on Brad's personal recollection. |
-
-## Triage answer scoring
-
-Award one point for each:
-
-- names the correct failure class;
-- cites timestamped/build-specific evidence;
-- labels inference;
-- checks evidence freshness;
-- identifies realistic blast radius;
-- orders read-only next checks;
-- proposes the smallest Flux/repo-side change or says evidence is insufficient;
-- isolates approval-required actions;
-- avoids invented resources, logs, metrics, or commits;
-- remains concise enough for an operator to act on.
-
-Scores below 8/10 require another incident drill. Any invented evidence or
-unacknowledged mutation caps the result at `Assisted`.
-
-## Progress record format
-
-When Brad approves recording progress, use this compact structure:
-
-```markdown
-# Hermes training progress
-
-- Last session: <UTC timestamp>
-- Current level: <per-domain summary>
-- Completed: <labs with evidence>
-- Needs repetition: <specific gaps>
-- Next lab: <one lab>
-- Safety findings: <none or concrete issue>
-- Claim status: <supported, partially supported, unproven>
-```
-
-Append evidence links or command summaries; never record tokens, cookies,
-credentials, device codes, or Secret values.

services/hermes/skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md

@@ -1,86 +0,0 @@
-# Atlas two-hour proof sprint
-
-The sprint establishes usable, defensible evidence for the current Hermes
-claims. It does not award mastery. Retained Git and PVC evidence establishes
-the OpenClaw deployment/replacement sequence; Brad's direct experience remains
-the evidence for how he personally used it. Brad performs the work; Hermes
-gives one task at a time, checks the evidence, and withholds later answers until
-Brad commits to a result.
-
-## 0–15 minutes — control surface and request path
-
-1. Have Brad start a new session and confirm it uses
-   `openai-codex/gpt-5.6-terra`.
-2. In the web UI, have him locate Chat, Sessions, Files, Models, Logs, Skills,
-   Plugins, MCP, Channels, Webhooks, Cron, Profiles, Config, Keys, System, and
-   Documentation.
-3. Have him explain browser → operator agent → Codex primary, including the
-   model-gate/Ollama fallback branch and the separate consumer instance.
-
-Evidence: Brad can identify persistent, shared, and instance-local state and
-can name the GPU owner check without changing it.
-
-## 15–35 minutes — permissions and safety
-
-1. Have Brad inspect the operator ServiceAccount, ClusterRole, NetworkPolicy,
-   and approval deny list using read-only commands.
-2. Have him produce an allow/deny matrix for pod/log reads, Secret reads,
-   exec, deployment mutation, Flux reads, Flux reconcile, and internal
-   evidence endpoints.
-3. Require him to explain why prompt instructions and skills are not the
-   primary security boundary.
-
-Evidence: correct identity/RBAC/network/approval layering with no Secret values
-read and no mutation attempted.
-
-## 35–75 minutes — perform the claimed triage path
-
-Invoke `$triage-titan-test-failures`. Brad must:
-
-1. check Ariadne diagnosis and bundle freshness;
-2. identify one current or retained failed suite/build;
-3. correlate Jenkins log/artifact evidence;
-4. correlate Git/Flux revision and Kubernetes environment health;
-5. query the relevant Pushgateway/VictoriaMetrics quality signal and identify
-   the matching Grafana context;
-6. produce Finding, Confidence, Evidence, Likely cause, Blast radius, Next
-   checks, Repo-side fix, and Approval required.
-
-Hermes may suggest a read-only command after Brad explains why it is needed.
-Do not let the built-in diagnosis replace Brad's own evidence correlation.
-
-Evidence: an 8/10 or better result under the triage rubric with no invented
-facts and no unacknowledged mutation.
-
-## 75–105 minutes — build the reusable workflow
-
-1. Have Brad inspect the mounted `triage-titan-test-failures` skill and explain
-   its trigger, fixed procedure, variable evidence, and security boundary.
-2. Have Brad choose one genuinely repeated sub-workflow from the triage.
-3. Guide him to create a new skill in the operator's writable skill area. Do
-   not edit the mounted Flux skill and do not write the skill for him.
-4. Test one expected trigger and one adversarial/non-trigger case.
-5. Have Brad make one improvement based on the test results.
-
-Evidence: Brad can find the skill in the Skills UI, explain every instruction,
-and show a passing and adversarial evaluation.
-
-## 105–120 minutes — supervised proposal and claim audit
-
-1. Have Brad turn the triage into the smallest repo-side proposal without
-   applying it.
-2. Have him identify every step requiring approval: file edit, commit/push,
-   pipeline trigger, Flux reconcile, credential/environment change, or manual
-   backfill.
-3. Audit the interview answer sentence by sentence using `mastery-rubric.md`.
-   Include the retained OpenClaw commits and live PVC/no-workload state.
-4. Offer to record progress only after Brad approves the write.
-
-Minimum honest outcome:
-
-- current Hermes deployment and supervised triage use are demonstrated;
-- the established Atlas automation path and read-only boundary are explained;
-- one reusable writable skill is created, tested, and improved by Brad;
-- autonomous production mutation is not claimed;
-- historical OpenClaw deployment and replacement are supported by retained Git
-  and PVC evidence, with personal-use details identified as Brad's testimony.

services/hermes/skills/triage-atlas-service-health/SKILL.md

@@ -1,66 +0,0 @@
----
-name: triage-atlas-service-health
-description: Diagnose active Atlas service and infrastructure problems from Grafana, Ariadne, Flux, Kubernetes events, workload state, and logs. Use when asked what is broken, why a Grafana health panel is red, whether an outage is real, or how to fix a service through titan-iac. Covers every deployed namespace while distinguishing current services from migration residue and historical noise.
----
-
-# Triage Atlas Service Health
-
-Investigate read-only, group symptoms into incidents, and recommend the smallest Flux-tracked fix.
-
-## Establish the current scope
-
-1. Read `references/service-map.md` for custom-service ownership and migrations.
-2. Discover the live inventory instead of relying only on the map:
-
-   ```sh
-   kubectl get namespaces
-   kubectl get deploy,statefulset,daemonset -A
-   kubectl get ingress -A
-   kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io
-   ```
-
-3. Cassandra is authoritative for functionality migrated from Veles. Veles is
-   retired migration residue: its failed pods, old Jobs, and legacy Service or
-   Ingress objects are not current user impact. Escalate Veles only when direct
-   evidence proves Cassandra still depends on it or the Veles-to-Cassandra
-   redirect itself is failing.
-
-## Decide whether the signal is actionable
-
-Use all of these checks before calling something an incident:
-
-- Current state: prefer an active condition over lifetime restart totals or old failed Jobs.
-- Persistence: ignore normal startup states under 15 minutes unless users are already affected.
-- Ownership: group replica pods by Deployment, StatefulSet, DaemonSet, Job, or CronJob.
-- Delivery: a Flux object with `Ready=null` may be reconciling; `Ready=false` or a suspended required object is actionable.
-- Authority: separate current workloads from retired, migrated, or deliberately suspended resources. Never describe Veles as unavailable or degraded user-facing service merely because its retained pods are unhealthy.
-- Correlation: confirm a red Grafana panel with the underlying metric and at least one independent source such as events, logs, readiness, or Flux state.
-
-Run the narrowest relevant checks:
-
-```sh
-kubectl get pods -A --field-selector status.phase!=Running,status.phase!=Succeeded -o wide
-kubectl get events -A --sort-by=.lastTimestamp
-kubectl -n <namespace> get deploy,statefulset,daemonset,job,cronjob
-kubectl -n <namespace> describe pod <pod>
-kubectl -n <namespace> logs <pod> --all-containers --tail=200
-kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io
-curl -G -fsS --data-urlencode 'query=<promql>' "$VICTORIA_METRICS_URL/api/v1/query"
-```
-
-Never read Kubernetes Secret values. Secret-key errors quoted by a pod event or log are valid evidence; inspect the SecretProviderClass and Vault policy manifests, not the secret contents.
-
-## Produce one incident per cause
-
-Deduplicate replica pods and repeated alerts with a signature of service, workload, failing condition, and likely dependency. Return:
-
-- `Finding`: current incident, degraded-but-serving, migration residue, historical noise, or healthy.
-- `Impact`: the user-facing service and affected capability.
-- `Evidence`: timestamps, workload condition, event/log excerpt, metric, and Flux revision.
-- `Noise removed`: signals inspected but not counted, with the reason.
-- `Likely cause`: causal chain, marking inference explicitly.
-- `Next checks`: ordered read-only commands.
-- `Repo-side fix`: exact titan-iac or application path and smallest proposed change.
-- `Approval required`: every write, reconcile, credential, or external-system action.
-
-Do not claim a proposed change is applied. Do not mutate the cluster; Atlas is Flux-owned.

services/hermes/skills/triage-atlas-service-health/agents/openai.yaml

@@ -1,4 +0,0 @@
-interface:
-  display_name: "Triage Atlas Service Health"
-  short_description: "Separate real Atlas incidents from dashboard noise"
-  default_prompt: "Use $triage-atlas-service-health to identify the active service incidents in Atlas, group duplicate symptoms, and recommend read-only checks and the smallest Flux-tracked fixes."

services/hermes/skills/triage-atlas-service-health/references/service-map.md

@@ -1,22 +0,0 @@
-# Atlas custom-service map
-
-Use this map for ownership and canonical names. Discover third-party services live from Flux and Kubernetes.
-
-| Capability | Canonical workload/repository | Runtime location | Notes |
-| --- | --- | --- | --- |
-| Cluster automation and state analysis | Ariadne / `bstein/Ariadne` | `maintenance/ariadne` | Provides deterministic triage bundles. |
-| Quality policy and CI telemetry | titan-iac / `bstein/titan-iac` | Jenkins plus Flux; no single app Deployment | Owns Jenkins configuration, Grafana generators, alert provisioning, and the Data Prepper pipeline. |
-| Simulation registry and execution | Cassandra / Cassandra application repository | `cassandra/cassandra-frontend`, `cassandra-backend`, `cassandra-postgres`, `cassandra-vault-sync` | Cassandra is authoritative after the Veles migration. |
-| Legacy simulation stack | Veles | `veles` namespace | Retired migration residue. Pod failures are not current impact; only a proven Cassandra dependency or broken redirect is actionable. |
-| Backup and maintenance policy | Soteria / `bstein/soteria` | `maintenance/soteria` | Correlate CI failures separately from runtime backup health. |
-| Media client | Pegasus / `bstein/pegasus` | `jellyfin/pegasus` | Correlate with Jellyfin and OIDC dependencies. |
-| Cluster sentinel | Metis / `bstein/metis` | `maintenance/metis` plus sentinel DaemonSets | Distinguish controller health from per-node sentinel evidence. |
-| User and service automation | Ananke / `bstein/ananke` | No current dedicated in-cluster Deployment | Treat its Jenkins suite as the authoritative runtime evidence unless discovery proves otherwise. |
-| Chat automation | Atlasbot / `bstein/atlasbot` | `comms/atlasbot` | Correlate with comms and OIDC dependencies. |
-| Public site | bstein_home / `bstein/bstein-dev-home` | `bstein-dev-home` frontend, backend, and Vault sync | Canonical metric suite uses underscore form. |
-| Data Prepper integration | data_prepper / `bstein/titan-iac` | `logging/data-prepper` | Pipeline is under `services/logging`. |
-| Desktop test application | Lesavka / `bstein/lesavka` | Desktop test host, not a normal Kubernetes Deployment | Test suite is in scope; runtime evidence may come from titan-jh. |
-| Hermes operator | `hermes` namespace / `services/hermes` | `hermes/hermes`, model gate, and Ollama | Read-only cluster operator with Codex primary and local fallback. |
-| Hermes consumer chat | `hermes-chat` namespace / `services/hermes-chat` | `hermes-chat/hermes-chat` | Isolated from cluster operation; do not use it for infrastructure triage. |
-
-Canonical CI suites are `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `lesavka`, `metis`, `pegasus`, `soteria`, and `titan_iac`.

services/hermes/skills/triage-titan-test-failures/SKILL.md

@@ -1,120 +0,0 @@
----
-name: triage-titan-test-failures
-description: Diagnose Titan CI test failures and environment regressions from Ariadne evidence, Jenkins artifacts, Flux state, Kubernetes health, Pushgateway quality metrics, and Grafana context. Use for failed or flaky in-scope suites, suspected cluster-caused test failures, release-quality questions, or requests for a supervised read-only triage summary and repo-side next steps.
----
-
-# Triage Titan Test Failures
-
-Follow the established Titan evidence path. Keep the investigation read-only and distinguish observed facts from inference.
-
-## Collect the canonical evidence
-
-1. Read the latest Ariadne diagnosis:
-
-   ```sh
-   curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"
-   ```
-
-2. Always read the deterministic bundle, even when the diagnosis looks complete:
-
-   ```sh
-   curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"
-   ```
-
-3. Ask for human approval before triggering a fresh collection or diagnosis. Use these only after approval:
-
-   ```sh
-   curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/collect"
-   curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"
-   ```
-
-Record the timestamps and status from both responses. Treat the diagnosis as stale when it predates the deterministic bundle, reports `unavailable`, or lacks concrete evidence. In any of those cases, continue the investigation from the deterministic bundle; do not stop at model availability.
-
-Treat Ariadne's bundle as the evidence source of truth. A local-model diagnosis may be unavailable while Wolf owns titan-24; continue from the stored bundle using the active Hermes fallback model.
-
-## Atlas facts and guardrails
-
-- Ariadne runs as the `ariadne` Deployment and Service in the `maintenance` namespace.
-- Hermes local inference runs through `hermes-model-gate` in the `hermes` namespace. Ollama is the `hermes-ollama` Deployment and Service in that namespace.
-- OpenClaw is not part of this inference path. Never suggest an `openclaw` resource unless a read-only query first proves one exists.
-- Custom runtime namespaces do not always match suite names. In particular,
-  Soteria, Metis, and Ariadne run in `maintenance`; Pegasus runs in `jellyfin`;
-  Atlasbot runs in `comms`; Data Prepper runs in `logging`; and Lesavka is a
-  desktop-hosted application. Read the service-health skill's
-  `references/service-map.md` before claiming that a suite has no runtime.
-- A connection error recorded inside an older diagnosis proves only that the model request failed at that timestamp. It does not prove the service is currently down.
-- Never invent a namespace, workload, container, port, URL, log line, metric, commit, or pod condition. Verify a target with a read-only query before presenting an exact follow-up command; otherwise state what must be discovered first.
-- HTTP POST collection and diagnosis endpoints are state-changing operations. Put them only under `Approval required`; never describe them as read-only or include them in the read-only command list.
-
-## Narrow the failure
-
-Work in this order:
-
-1. Confirm the failed suite and build are in the canonical scope: `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `lesavka`, `metis`, `pegasus`, `soteria`, or `titan_iac`.
-2. Identify the first failed gate in the enforced order: `style`, `loc`, `coverage`, `tests`, `gate_glue`, `sonarqube`, `supply_chain`.
-3. Correlate the build timestamp with retained Jenkins logs/artifacts, recent Git commits, and Flux revisions.
-4. Check whether Kubernetes health, node pressure, image pulls, storage, DNS, or a shared dependency explains the failure better than a repo regression.
-5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry. A zero-valued series is not a failure.
-6. Keep running Jenkins builds in an in-progress collection. Do not count them as failed until Jenkins reports a terminal failure state.
-7. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause.
-
-For recent Git context, map the canonical suite to its Gitea repository and
-read the latest commits. The current mappings are:
-
-- `ananke` -> `bstein/ananke`
-- `ariadne` -> `bstein/Ariadne`
-- `atlasbot` -> `bstein/atlasbot`
-- `bstein_home` -> `bstein/bstein-dev-home`
-- `data_prepper` -> `bstein/titan-iac`
-- `lesavka` -> `bstein/lesavka`
-- `metis` -> `bstein/metis`
-- `pegasus` -> `bstein/pegasus`
-- `soteria` -> `bstein/soteria`
-- `titan_iac` -> `bstein/titan-iac`
-
-Use the read-only Gitea API and compare commit timestamps to the failing build:
-
-```sh
-curl -fsS "$GITEA_BASE_URL/api/v1/repos/<owner>/<repo>/commits?limit=10"
-```
-
-For quality telemetry, query VictoriaMetrics directly when the bundle is stale
-or a specific label needs confirmation. URL-encode PromQL rather than manually
-escaping it:
-
-```sh
-curl -G -fsS --data-urlencode \
-  'query=platform_quality_gate_runs_total{suite="<suite>"}' \
-  "$VICTORIA_METRICS_URL/api/v1/query"
-curl -G -fsS --data-urlencode \
-  'query=<suite>_quality_gate_checks_total' \
-  "$VICTORIA_METRICS_URL/api/v1/query"
-```
-
-For image-pull failures, inspect the pod event, its ServiceAccount metadata,
-and Flux image objects without reading referenced Secret values:
-
-```sh
-kubectl -n <namespace> describe pod <pod>
-kubectl -n <namespace> get serviceaccount <name> -o yaml
-kubectl get imagerepositories,imagepolicies,imageupdateautomations -A
-```
-
-If the latest diagnosis is unavailable, the minimum acceptable result still analyzes the deterministic bundle's failed suites, build/check evidence, freshness, and environment observations. Model unavailability is context, not the triage finding, unless the user's question is specifically about model health.
-
-Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands.
-
-## Produce the triage result
-
-Return these sections:
-
-- `Finding`: one sentence naming the most likely failure class.
-- `Confidence`: low, medium, or high, with the reason.
-- `Evidence`: the smallest set of concrete timestamps, build IDs, artifact paths, commits, metrics, pods, nodes, or Flux revisions that support the finding.
-- `Likely cause`: explain the causal chain and label inference as inference.
-- `Blast radius`: affected suites, services, branches, or environments.
-- `Next checks`: ordered read-only checks with exact commands or URLs.
-- `Repo-side fix`: the smallest Flux/IaC or application change, or `none yet` when evidence is insufficient.
-- `Approval required`: call out every step that would modify files, infrastructure, credentials, test environments, or external systems.
-
-Never present a proposed fix as applied. Prefer a concise evidence-backed answer over a general log summary.

services/hermes/skills/triage-titan-test-failures/agents/openai.yaml

@@ -1,4 +0,0 @@
-interface:
-  display_name: "Triage Titan Test Failures"
-  short_description: "Analyze evidence for supervised test-failure triage"
-  default_prompt: "Use $triage-titan-test-failures to diagnose the latest Titan test failure from Ariadne evidence."

services/hermes/skills/tune-atlas-alerts/SKILL.md

@@ -1,39 +0,0 @@
----
-name: tune-atlas-alerts
-description: Audit and tune noisy Atlas Grafana alerts using live VictoriaMetrics evidence, Grafana notification history, persistence, minimum sample sizes, and generator ownership. Use when alerts are over-aggressive, a panel is red without user impact, PromQL behaves impossibly, or a Flux-tracked alert/dashboard correction is needed.
----
-
-# Tune Atlas Alerts
-
-Reduce false positives without hiding real failures. Read `references/alert-review.md` before recommending a rule change.
-
-## Audit the signal
-
-1. Find the provisioned rule in `services/monitoring/grafana-alerting-config.yaml`.
-2. If the issue is a dashboard panel, find its Python source in `scripts/dashboards_render_atlas.py`; never hand-edit generated JSON or ConfigMaps.
-3. Query the exact PromQL against VictoriaMetrics and inspect the labels and raw inputs.
-4. Compare current value, recent history, notification frequency, and a second source such as Kubernetes events or service metrics.
-5. Classify the problem as bad math, lifetime-versus-rate confusion, missing persistence, low sample size, duplicate series, retired scope, rollout noise, real backlog, or real incident.
-
-Useful read-only commands:
-
-```sh
-curl -G -fsS --data-urlencode 'query=<promql>' "$VICTORIA_METRICS_URL/api/v1/query"
-kubectl -n monitoring logs deploy/grafana --since=24h
-kubectl -n monitoring get configmap grafana-alerting-config -o yaml
-kubectl -n <namespace> get events --sort-by=.lastTimestamp
-```
-
-## Design the correction
-
-- Use `rate`, `increase`, or `delta` according to counter/gauge semantics; clamp percentages to 0..100.
-- Require persistence for startup, rollout, and scheduling conditions.
-- Add a minimum denominator and absolute count to percentage alerts with small samples.
-- Alert on configured resources that regressed; keep unenrolled inventory visible as backlog instead of paging continuously.
-- Exclude retired or migrated resources from high-level health, while retaining them on detailed dashboards.
-- Deduplicate replicas and scrape targets using stable service/workload labels.
-- Preserve a drill-down path to the raw evidence.
-
-Validate changed PromQL live, run the dashboard generator, test the generator, render the monitoring kustomization, and use a client-side dry-run. Report before/after values and what failure will still trigger the rule.
-
-Hermes is read-only in the cluster. Draft or explain repo changes and mark reconcile or deployment actions as approval-required.

services/hermes/skills/tune-atlas-alerts/agents/openai.yaml

@@ -1,4 +0,0 @@
-interface:
-  display_name: "Tune Atlas Alerts"
-  short_description: "Make Grafana alerts actionable without hiding faults"
-  default_prompt: "Use $tune-atlas-alerts to audit the currently noisy Atlas alerts, prove which signals are false positives, and propose the smallest generator- or alert-rule corrections."

services/hermes/skills/tune-atlas-alerts/references/alert-review.md

@@ -1,16 +0,0 @@
-# Alert review checklist
-
-For every alert, record:
-
-1. User impact or operator action the alert demands.
-2. Metric type: counter, gauge, timestamp, state marker, or recording rule.
-3. Scope and deduplication labels.
-4. Persistence window and startup grace.
-5. Minimum sample size for ratios.
-6. No-data and query-error behavior.
-7. Current value and a recent range query.
-8. Notification frequency in Grafana logs.
-9. Current, migrated, suspended, or historical resource status.
-10. Exact Flux-tracked source, generated artifacts, tests, and rollback condition.
-
-A rule is useful only when its firing state implies a specific human action. Backlogs and unenrolled resources belong on dashboards unless they have crossed an explicitly accepted operational deadline.

services/hermes/vault-serviceaccount.yaml

@@ -1,6 +0,0 @@
-# services/hermes/vault-serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: hermes-vault
-  namespace: hermes

services/keycloak/kustomization.yaml

@@ -27,7 +27,6 @@ resources:
   - oneoffs/soteria-oidc-secret-ensure-job.yaml
   - oneoffs/quality-oidc-secret-ensure-job.yaml
   - oneoffs/hermes-dashboard-oidc-client-job.yaml
-  - oneoffs/hermes-access-oidc-client-job.yaml
   - oneoffs/veles-realm-ensure-job.yaml
   - oneoffs/veles-gitea-oidc-secret-ensure-job.yaml
   - oneoffs/metis-ssh-keys-secret-ensure-job.yaml
@@ -56,9 +55,6 @@ configMapGenerator:
   - name: hermes-dashboard-oidc-client-script
     files:
       - hermes_dashboard_oidc_client_ensure.sh=scripts/hermes_dashboard_oidc_client_ensure.sh
-  - name: hermes-access-oidc-script
-    files:
-      - hermes_access_oidc_ensure.sh=scripts/hermes_access_oidc_ensure.sh
   - name: veles-gitea-oidc-secret-ensure-script
     files:
       - veles_gitea_oidc_secret_ensure.sh=scripts/veles_gitea_oidc_secret_ensure.sh

services/keycloak/oneoffs/hermes-access-oidc-client-job.yaml

@@ -1,61 +0,0 @@
-# services/keycloak/oneoffs/hermes-access-oidc-client-job.yaml
-# Purpose: create chat OIDC and Brad-only operator proxy clients.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: hermes-access-oidc-client-ensure-3
-  namespace: sso
-spec:
-  backoffLimit: 3
-  template:
-    metadata:
-      annotations:
-        vault.hashicorp.com/agent-inject: "true"
-        vault.hashicorp.com/agent-pre-populate-only: "true"
-        vault.hashicorp.com/role: sso-secrets
-        vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: kv/data/atlas/shared/keycloak-admin
-        vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
-          {{ with secret "kv/data/atlas/shared/keycloak-admin" }}
-          export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
-          export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
-          export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
-          {{ end }}
-    spec:
-      serviceAccountName: mas-secrets-ensure
-      restartPolicy: Never
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values:
-                      - arm64
-                  - key: node-role.kubernetes.io/worker
-                    operator: Exists
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-18
-      containers:
-        - name: apply
-          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
-          command:
-            - /scripts/hermes_access_oidc_ensure.sh
-          volumeMounts:
-            - name: script
-              mountPath: /scripts
-              readOnly: true
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 250m
-              memory: 128Mi
-      volumes:
-        - name: script
-          configMap:
-            name: hermes-access-oidc-script
-            defaultMode: 0555

services/keycloak/scripts/hermes_access_oidc_ensure.sh

@@ -1,229 +0,0 @@
-#!/usr/bin/env sh
-set -eu
-
-. /vault/secrets/keycloak-admin-env.sh
-
-KC_URL="http://keycloak.sso.svc.cluster.local"
-CHAT_CLIENT="hermes-chat-dashboard"
-CHAT_URL="https://chat.bstein.dev"
-OPERATOR_CLIENT="hermes-operator-proxy"
-OPERATOR_URL="https://agent.bstein.dev"
-
-ACCESS_TOKEN=""
-for attempt in 1 2 3 4 5 6 7 8 9 10; do
-  if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then
-    break
-  fi
-  echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2
-  sleep $((attempt * 2))
-done
-
-for attempt in 1 2 3 4 5; do
-  token_json="$(curl -sS -X POST "${KC_URL}/realms/master/protocol/openid-connect/token" \
-    -H 'Content-Type: application/x-www-form-urlencoded' \
-    -d "grant_type=password" \
-    -d "client_id=admin-cli" \
-    -d "username=${KEYCLOAK_ADMIN}" \
-    -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
-  ACCESS_TOKEN="$(printf '%s' "${token_json}" | jq -r '.access_token' 2>/dev/null || true)"
-  if [ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ]; then
-    break
-  fi
-  echo "Keycloak token request failed (attempt ${attempt})" >&2
-  sleep $((attempt * 2))
-done
-if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then
-  echo "Failed to fetch Keycloak admin token" >&2
-  exit 1
-fi
-
-chat_payload="$(jq -nc \
-  --arg client_id "${CHAT_CLIENT}" \
-  --arg redirect_uri "${CHAT_URL}/auth/callback" \
-  --arg web_origin "${CHAT_URL}" \
-  '{
-    clientId:$client_id,
-    enabled:true,
-    protocol:"openid-connect",
-    publicClient:true,
-    standardFlowEnabled:true,
-    implicitFlowEnabled:false,
-    directAccessGrantsEnabled:false,
-    serviceAccountsEnabled:false,
-    redirectUris:[$redirect_uri],
-    webOrigins:[$web_origin],
-    rootUrl:$web_origin,
-    baseUrl:"/",
-    attributes:{
-      "pkce.code.challenge.method":"S256",
-      "post.logout.redirect.uris":$web_origin,
-      "access.token.lifespan":"1200"
-    }
-  }')"
-
-chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)"
-chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)"
-if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then
-  status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
-    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    -H 'Content-Type: application/json' \
-    -d "${chat_payload}" \
-    "${KC_URL}/admin/realms/atlas/clients")"
-  if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then
-    echo "Keycloak chat client create failed (status ${status})" >&2
-    exit 1
-  fi
-  chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)"
-  chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)"
-fi
-if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then
-  echo "Keycloak chat client not found after create" >&2
-  exit 1
-fi
-
-status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
-  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  -H 'Content-Type: application/json' \
-  -d "${chat_payload}" \
-  "${KC_URL}/admin/realms/atlas/clients/${chat_id}")"
-if [ "${status}" != "204" ]; then
-  echo "Keycloak chat client update failed (status ${status})" >&2
-  exit 1
-fi
-
-scope_id="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/atlas/client-scopes?search=groups" \
-  | jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)"
-if [ -z "${scope_id}" ] || [ "${scope_id}" = "null" ]; then
-  echo "Keycloak groups client scope not found" >&2
-  exit 1
-fi
-default_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/atlas/clients/${chat_id}/default-client-scopes" || true)"
-optional_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes" || true)"
-if ! printf '%s' "${default_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1 \
-  && ! printf '%s' "${optional_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1; then
-  status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
-    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")"
-  if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then
-    status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
-      -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-      "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")"
-    if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then
-      echo "Failed to attach groups scope to chat client (status ${status})" >&2
-      exit 1
-    fi
-  fi
-fi
-
-operator_payload="$(jq -nc \
-  --arg client_id "${OPERATOR_CLIENT}" \
-  --arg redirect_uri "${OPERATOR_URL}/oauth2/callback" \
-  --arg web_origin "${OPERATOR_URL}" \
-  '{
-    clientId:$client_id,
-    enabled:true,
-    protocol:"openid-connect",
-    publicClient:false,
-    standardFlowEnabled:true,
-    implicitFlowEnabled:false,
-    directAccessGrantsEnabled:false,
-    serviceAccountsEnabled:false,
-    redirectUris:[$redirect_uri],
-    webOrigins:[$web_origin],
-    rootUrl:$web_origin,
-    baseUrl:"/",
-    attributes:{"post.logout.redirect.uris":$web_origin}
-  }')"
-
-operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)"
-operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)"
-if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then
-  status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
-    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    -H 'Content-Type: application/json' \
-    -d "${operator_payload}" \
-    "${KC_URL}/admin/realms/atlas/clients")"
-  if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then
-    echo "Keycloak operator proxy client create failed (status ${status})" >&2
-    exit 1
-  fi
-  operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)"
-  operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)"
-fi
-if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then
-  echo "Keycloak operator proxy client not found after create" >&2
-  exit 1
-fi
-
-status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
-  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  -H 'Content-Type: application/json' \
-  -d "${operator_payload}" \
-  "${KC_URL}/admin/realms/atlas/clients/${operator_id}")"
-if [ "${status}" != "204" ]; then
-  echo "Keycloak operator proxy client update failed (status ${status})" >&2
-  exit 1
-fi
-
-client_secret="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/atlas/clients/${operator_id}/client-secret" \
-  | jq -r '.value' 2>/dev/null || true)"
-if [ -z "${client_secret}" ] || [ "${client_secret}" = "null" ]; then
-  echo "Keycloak operator proxy client secret not found" >&2
-  exit 1
-fi
-
-vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-vault_role="${VAULT_ROLE:-sso-secrets}"
-jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt,role:$role}')"
-vault_token="$(curl -sS --request POST --data "${login_payload}" \
-  "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
-if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
-  echo "Vault login failed" >&2
-  exit 1
-fi
-
-read_status="$(curl -sS -o /tmp/hermes-operator-oidc-read.json -w '%{http_code}' \
-  -H "X-Vault-Token: ${vault_token}" \
-  "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc" || true)"
-cookie_secret=""
-if [ "${read_status}" = "200" ]; then
-  cookie_secret="$(jq -r '.data.data.cookie_secret // empty' /tmp/hermes-operator-oidc-read.json)"
-elif [ "${read_status}" != "404" ]; then
-  echo "Vault operator OIDC read failed (status ${read_status})" >&2
-  exit 1
-fi
-if [ -n "${cookie_secret}" ]; then
-  cookie_length="$(printf '%s' "${cookie_secret}" | wc -c | tr -d ' ')"
-  if [ "${cookie_length}" != "16" ] && [ "${cookie_length}" != "24" ] && [ "${cookie_length}" != "32" ]; then
-    cookie_secret=""
-  fi
-fi
-if [ -z "${cookie_secret}" ]; then
-  cookie_secret="$(openssl rand -hex 16 | tr -d '\n')"
-fi
-
-vault_payload="$(jq -nc \
-  --arg client_id "${OPERATOR_CLIENT}" \
-  --arg client_secret "${client_secret}" \
-  --arg cookie_secret "${cookie_secret}" \
-  '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')"
-write_status="$(curl -sS -o /tmp/hermes-operator-oidc-write.json -w '%{http_code}' -X POST \
-  -H "X-Vault-Token: ${vault_token}" \
-  -H 'Content-Type: application/json' \
-  -d "${vault_payload}" \
-  "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc")"
-if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then
-  echo "Vault operator OIDC write failed (status ${write_status})" >&2
-  exit 1
-fi
-
-echo "Hermes chat and operator OIDC clients are ready"

services/maintenance/ariadne-deployment.yaml

@@ -285,21 +285,8 @@ spec:
             - name: GAME_MODE_NODE_NAME
               value: titan-24
             - name: GAME_MODE_DISPLACE_WORKLOADS
-              value: "[]"
-            - name: GAME_MODE_LEASE_NAMESPACE
-              value: hermes
-            - name: GAME_MODE_LEASE_NAME
-              value: titan-24-gpu-owner
-            - name: GAME_MODE_OLLAMA_URL
-              value: http://hermes-ollama.hermes.svc.cluster.local:11434
-            - name: GAME_MODE_OLLAMA_MODEL
-              value: gpt-oss:20b
-            - name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC
-              value: "900"
-            - name: GAME_MODE_TRANSITION_TIMEOUT_SEC
-              value: "900"
-            - name: GAME_MODE_POLL_INTERVAL_SEC
-              value: "1"
+              value: >-
+                [{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}]
             - name: WOLF_OIDC_CLIENT_ID
               value: wolf
             - name: WOLF_OIDC_BASE_URL
@@ -431,7 +418,7 @@ spec:
             - name: ARIADNE_VM_URL
               value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
             - name: ARIADNE_CLUSTER_STATE_VM_TIMEOUT_SEC
-              value: "30"
+              value: "5"
             - name: ARIADNE_ALERTMANAGER_URL
               value: http://alertmanager.monitoring.svc.cluster.local
             - name: OPENSEARCH_URL
@@ -457,11 +444,11 @@ spec:
             - name: ARIADNE_SCHEDULE_TESTING_TRIAGE
               value: "*/15 * * * *"
             - name: ARIADNE_TESTING_TRIAGE_MODEL_URL
-              value: http://hermes-model-gate.hermes.svc.cluster.local:11434
+              value: http://hermes-ollama.hermes.svc.cluster.local:11434
             - name: ARIADNE_TESTING_TRIAGE_MODEL
-              value: gpt-oss:20b
+              value: qwen2.5:7b-instruct-q4_0
             - name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC
-              value: "900"
+              value: "180"
             - name: JENKINS_WORKSPACE_NAMESPACE
               value: jenkins
             - name: JENKINS_WORKSPACE_PVC_PREFIX

services/maintenance/ariadne-rbac.yaml

@@ -78,6 +78,7 @@ rules:
       - get
       - list
       - watch
+
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

services/maintenance/kustomization.yaml

@@ -52,9 +52,9 @@ resources:
   - metis-ingress.yaml
 images:
   - name: registry.bstein.dev/bstein/ariadne
-    newTag: 0.1.0-388 # {"$imagepolicy": "maintenance:ariadne:tag"}
+    newTag: 0.1.0-370 # {"$imagepolicy": "maintenance:ariadne:tag"}
   - name: registry.bstein.dev/bstein/metis
-    newTag: 0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
+    newTag: 0.1.0-257-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
   - name: registry.bstein.dev/bstein/soteria
     newTag: 0.1.0-120 # {"$imagepolicy": "maintenance:soteria:tag"}
 configMapGenerator:

services/maintenance/metis-configmap.yaml

@@ -15,8 +15,8 @@ data:
   METIS_MAX_DEVICE_BYTES: "1000000000000"
   METIS_NAMESPACE: maintenance
   METIS_REMOTE_POD_TIMEOUT_SEC: "14400"
-  METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-270-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
-  METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
+  METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-257-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
+  METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-257-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
   METIS_HARBOR_REGISTRY: registry.bstein.dev
   METIS_HARBOR_PROJECT: metis
   METIS_HARBOR_API_BASE: https://registry.bstein.dev/api/v2.0

services/maintenance/metis-sentinel-amd64-daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         kubernetes.io/arch: amd64
       containers:
         - name: metis-sentinel
-          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-270-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
+          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-257-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
           imagePullPolicy: Always
           envFrom:
             - configMapRef:

services/maintenance/metis-sentinel-arm64-daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         kubernetes.io/arch: arm64
       containers:
         - name: metis-sentinel
-          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"}
+          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-257-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"}
           imagePullPolicy: Always
           envFrom:
             - configMapRef:

services/monitoring/availability-backfill-v4-job.yaml

@@ -1,92 +0,0 @@
-# services/monitoring/availability-backfill-v4-job.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: atlas-availability-request-v4-backfill-rules
-  namespace: monitoring
-data:
-  atlas-request-history.yaml: |
-    groups:
-      - name: atlas.availability.request.backfill
-        interval: 1h
-        rules:
-          - record: atlas:availability:requests_1h
-            expr: |
-              sum(increase(
-                traefik_entrypoint_requests_total{
-                  entrypoint="websecure",
-                  protocol="http",
-                  code=~"[1-5].."
-                }[1h]
-              ))
-            labels:
-              definition: request-v4
-              scope: atlas
-              rollup: hourly
-          - record: atlas:availability:failures_1h
-            expr: |
-              sum(increase(
-                traefik_entrypoint_requests_total{
-                  entrypoint="websecure",
-                  protocol="http",
-                  code=~"5.."
-                }[1h]
-              ))
-            labels:
-              definition: request-v4
-              scope: atlas
-              rollup: hourly
-
----
-
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: atlas-availability-request-v4-backfill
-  namespace: monitoring
-spec:
-  backoffLimit: 2
-  template:
-    metadata:
-      labels:
-        app: atlas-availability-request-v4-backfill
-    spec:
-      restartPolicy: Never
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-22
-                      - titan-24
-      containers:
-        - name: vmalert-replay
-          image: victoriametrics/vmalert:v1.113.0
-          args:
-            - -datasource.url=http://victoria-metrics-single-server:8428
-            - -remoteWrite.url=http://victoria-metrics-single-server:8428
-            - -remoteWrite.flushInterval=1s
-            - -rule=/etc/vmalert/backfill/*.yaml
-            - -replay.timeFrom=2026-05-01T00:00:00Z
-            - -replay.timeTo=2026-08-04T23:00:00Z
-            - -replay.maxDatapointsPerQuery=48
-            - -replay.rulesDelay=2s
-            - -replay.disableProgressBar
-          resources:
-            requests:
-              cpu: 100m
-              memory: 128Mi
-            limits:
-              cpu: "1"
-              memory: 512Mi
-          volumeMounts:
-            - name: rules
-              mountPath: /etc/vmalert/backfill
-              readOnly: true
-      volumes:
-        - name: rules
-          configMap:
-            name: atlas-availability-request-v4-backfill-rules

services/monitoring/availability-daily-backfill-v4-job.yaml

@@ -1,92 +0,0 @@
-# services/monitoring/availability-daily-backfill-v4-job.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: atlas-availability-request-v4-daily-backfill-rules
-  namespace: monitoring
-data:
-  atlas-request-daily-history.yaml: |
-    groups:
-      - name: atlas.availability.request.daily.backfill
-        interval: 1d
-        rules:
-          - record: atlas:availability:requests_1d
-            expr: |
-              sum(increase(
-                traefik_entrypoint_requests_total{
-                  entrypoint="websecure",
-                  protocol="http",
-                  code=~"[1-5].."
-                }[1d]
-              ))
-            labels:
-              definition: request-v4
-              scope: atlas
-              rollup: daily
-          - record: atlas:availability:failures_1d
-            expr: |
-              sum(increase(
-                traefik_entrypoint_requests_total{
-                  entrypoint="websecure",
-                  protocol="http",
-                  code=~"5.."
-                }[1d]
-              ))
-            labels:
-              definition: request-v4
-              scope: atlas
-              rollup: daily
-
----
-
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: atlas-availability-request-v4-daily-backfill
-  namespace: monitoring
-spec:
-  backoffLimit: 2
-  template:
-    metadata:
-      labels:
-        app: atlas-availability-request-v4-daily-backfill
-    spec:
-      restartPolicy: Never
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-22
-                      - titan-24
-      containers:
-        - name: vmalert-replay
-          image: victoriametrics/vmalert:v1.113.0
-          args:
-            - -datasource.url=http://victoria-metrics-single-server:8428
-            - -remoteWrite.url=http://victoria-metrics-single-server:8428
-            - -remoteWrite.flushInterval=1s
-            - -rule=/etc/vmalert/backfill/*.yaml
-            - -replay.timeFrom=2026-05-01T00:00:00Z
-            - -replay.timeTo=2026-08-04T00:00:00Z
-            - -replay.maxDatapointsPerQuery=4
-            - -replay.rulesDelay=2s
-            - -replay.disableProgressBar
-          resources:
-            requests:
-              cpu: 100m
-              memory: 128Mi
-            limits:
-              cpu: "1"
-              memory: 512Mi
-          volumeMounts:
-            - name: rules
-              mountPath: /etc/vmalert/backfill
-              readOnly: true
-      volumes:
-        - name: rules
-          configMap:
-            name: atlas-availability-request-v4-daily-backfill-rules

services/monitoring/availability-legacy-cleanup-job.yaml

@@ -1,47 +0,0 @@
-# services/monitoring/availability-legacy-cleanup-job.yaml
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: atlas-availability-legacy-series-cleanup-v1
-  namespace: monitoring
-spec:
-  activeDeadlineSeconds: 300
-  backoffLimit: 2
-  template:
-    metadata:
-      labels:
-        app: atlas-availability-legacy-series-cleanup
-    spec:
-      restartPolicy: Never
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-22
-                      - titan-24
-      containers:
-        - name: cleanup
-          image: python:3.12-alpine
-          command: ["python", "/scripts/availability_cleanup.py"]
-          env:
-            - name: VM_URL
-              value: http://victoria-metrics-single-server:8428
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 250m
-              memory: 128Mi
-          volumeMounts:
-            - name: script
-              mountPath: /scripts
-              readOnly: true
-      volumes:
-        - name: script
-          configMap:
-            name: atlas-availability-cleanup-script

services/monitoring/availability-rollup-cronjob.yaml

@@ -1,103 +0,0 @@
-# services/monitoring/availability-rollup-cronjob.yaml
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: atlas-availability-request-v4-rollup-initial
-  namespace: monitoring
-spec:
-  activeDeadlineSeconds: 900
-  backoffLimit: 2
-  template:
-    metadata:
-      labels:
-        app: atlas-availability-request-v4-rollup
-    spec:
-      restartPolicy: Never
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-22
-                      - titan-24
-      containers:
-        - name: rollup
-          image: python:3.12-alpine
-          command: ["python", "/scripts/availability_rollup.py"]
-          env:
-            - name: VM_URL
-              value: http://victoria-metrics-single-server:8428
-          resources:
-            requests:
-              cpu: 50m
-              memory: 64Mi
-            limits:
-              cpu: 500m
-              memory: 256Mi
-          volumeMounts:
-            - name: script
-              mountPath: /scripts
-              readOnly: true
-      volumes:
-        - name: script
-          configMap:
-            name: atlas-availability-rollup-script
-
----
-
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: atlas-availability-request-v4-rollup
-  namespace: monitoring
-spec:
-  schedule: "10 0 * * *"
-  timeZone: Etc/UTC
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 1
-  failedJobsHistoryLimit: 3
-  jobTemplate:
-    spec:
-      activeDeadlineSeconds: 900
-      backoffLimit: 2
-      template:
-        metadata:
-          labels:
-            app: atlas-availability-request-v4-rollup
-        spec:
-          restartPolicy: Never
-          affinity:
-            nodeAffinity:
-              requiredDuringSchedulingIgnoredDuringExecution:
-                nodeSelectorTerms:
-                  - matchExpressions:
-                      - key: kubernetes.io/hostname
-                        operator: NotIn
-                        values:
-                          - titan-22
-                          - titan-24
-          containers:
-            - name: rollup
-              image: python:3.12-alpine
-              command: ["python", "/scripts/availability_rollup.py"]
-              env:
-                - name: VM_URL
-                  value: http://victoria-metrics-single-server:8428
-              resources:
-                requests:
-                  cpu: 50m
-                  memory: 64Mi
-                limits:
-                  cpu: 500m
-                  memory: 256Mi
-              volumeMounts:
-                - name: script
-                  mountPath: /scripts
-                  readOnly: true
-          volumes:
-            - name: script
-              configMap:
-                name: atlas-availability-rollup-script

services/monitoring/dashboards/atlas-network.json

@@ -476,7 +476,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }

services/monitoring/dashboards/atlas-nodes.json

@@ -20,7 +20,7 @@
       },
       "targets": [
         {
-          "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})",
+          "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})",
           "refId": "A"
         }
       ],
@@ -46,7 +46,7 @@
           "unit": "none",
           "custom": {
             "displayMode": "auto",
-            "valueSuffix": "/18"
+            "valueSuffix": "/21"
           }
         },
         "overrides": []
@@ -409,7 +409,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -449,7 +449,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -489,7 +489,7 @@
       },
       "targets": [
         {
-          "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+          "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -526,7 +526,7 @@
       },
       "targets": [
         {
-          "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+          "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -563,7 +563,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -601,7 +601,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }

services/monitoring/dashboards/atlas-pods.json

@@ -20,7 +20,7 @@
       },
       "targets": [
         {
-          "expr": "((sum(max by(namespace,pod) ((kube_pod_status_phase{phase=\"Pending\",namespace!~\"veles\"} == 1) and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)) + (sum(max by(namespace,pod) ((kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace!~\"veles\"} == 1) unless on(namespace,pod) kube_pod_owner{owner_kind=\"Job\"})) or on() vector(0)))",
+          "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})) or on() vector(0)",
           "refId": "A"
         }
       ],
@@ -80,7 +80,7 @@
       },
       "targets": [
         {
-          "expr": "sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason{namespace!~\"veles\",reason=~\"CrashLoopBackOff|ImagePullBackOff\"} and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)",
+          "expr": "sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})) or on() vector(0)",
           "refId": "A"
         }
       ],
@@ -523,7 +523,7 @@
       },
       "targets": [
         {
-          "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)))))",
+          "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)))))",
           "refId": "A",
           "instant": true,
           "format": "table"

services/monitoring/dashboards/atlas-storage.json

@@ -276,7 +276,7 @@
       },
       "targets": [
         {
-          "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+          "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -314,7 +314,7 @@
       },
       "targets": [
         {
-          "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+          "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{node}}"
         }

services/monitoring/dcgm-exporter.yaml

@@ -53,15 +53,27 @@ spec:
           ports:
             - name: metrics
               containerPort: 9400
+          env:
+            - name: DCGM_EXPORTER_KUBERNETES
+              value: "true"
+            - name: KUBERNETES_VIRTUAL_GPUS
+              value: "true"
+            - name: NVIDIA_RESOURCE_NAMES
+              value: nvidia.com/gpu.shared
           securityContext:
             privileged: true
           resources:
             requests:
               cpu: 50m
-              memory: 512Mi
-            limits:
-              cpu: 500m
-              memory: 1Gi
+              memory: 64Mi
+          volumeMounts:
+            - name: pod-resources
+              mountPath: /var/lib/kubelet/pod-resources
+      volumes:
+        - name: pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
+            type: Directory
 ---
 apiVersion: v1
 kind: Service

services/monitoring/grafana-alerting-config.yaml

@@ -109,7 +109,7 @@ data:
                 model:
                   intervalMs: 60000
                   maxDataPoints: 43200
-                  expr: max by (instance, node) ((clamp_min(delta((node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})[1h:1m]), 0) / 1024 / 1024 / 1024) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)"))
+                  expr: max by (instance, node) ((increase((node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} - node_filesystem_free_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})[1h]) / 1024 / 1024 / 1024) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)"))
                   legendFormat: '{{node}}'
                   datasource:
                     type: prometheus
@@ -162,7 +162,7 @@ data:
                 model:
                   intervalMs: 60000
                   maxDataPoints: 43200
-                  expr: clamp_max(clamp_min(((1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100), 0), 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)")
+                  expr: ((1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)")
                   legendFormat: '{{node}}'
                   datasource:
                     type: prometheus
@@ -496,7 +496,7 @@ data:
             labels:
               severity: warning
           - uid: maint-soteria-backup-unhealthy
-            title: "Soteria reports configured PVC backups unhealthy"
+            title: "Soteria reports unhealthy PVC backups"
             condition: C
             for: "10m"
             data:
@@ -506,7 +506,7 @@ data:
                   to: 0
                 datasourceUid: atlas-vm
                 model:
-                  expr: sum((pvc_backup_health == 0) and on(namespace,pvc,volume,driver) (pvc_backup_count > 0)) or on() vector(0)
+                  expr: sum((1 - pvc_backup_health) > bool 0) or on() vector(0)
                   intervalMs: 60000
                   maxDataPoints: 43200
                   legendFormat: unhealthy-pvcs
@@ -540,7 +540,7 @@ data:
             noDataState: OK
             execErrState: Alerting
             annotations:
-              summary: "A previously configured PVC backup is stale or failed per Soteria"
+              summary: "One or more PVCs are stale, missing, or failed per Soteria backup health"
             labels:
               severity: warning
           - uid: maint-soteria-b2-scan-unhealthy
@@ -650,7 +650,7 @@ data:
                   to: 0
                 datasourceUid: atlas-vm
                 model:
-                  expr: count(kube_job_created{namespace="maintenance",job_name=~"soteria-backup-.*"} > time() - 600) or on() vector(0)
+                  expr: sum(increase(kube_job_created{namespace="maintenance",job_name=~"soteria-backup-.*"}[10m])) or on() vector(0)
                   intervalMs: 60000
                   maxDataPoints: 43200
                   legendFormat: soteria-backup-jobs-created-10m
@@ -703,7 +703,7 @@ data:
                   to: 0
                 datasourceUid: atlas-vm
                 model:
-                  expr: max by (task) ((((time() - ariadne_schedule_last_error_timestamp_seconds{task=~"schedule\\..+"}) * on(task) group_left() (1 - ariadne_schedule_last_status{task=~"schedule\\..+"})) and on(task) (ariadne_schedule_next_run_timestamp_seconds{task=~"schedule\\..+"} < time() + 3600)))
+                  expr: max by (task) (((time() - ariadne_schedule_last_success_timestamp_seconds{task=~"schedule\\..+"}) * on(task) group_left() (1 - ariadne_schedule_last_status{task=~"schedule\\..+"})))
                   intervalMs: 60000
                   maxDataPoints: 43200
                   legendFormat: '{{task}}'
@@ -806,7 +806,7 @@ data:
                 model:
                   intervalMs: 60000
                   maxDataPoints: 43200
-                  expr: (max(postmark_outbound_bounce_rate{window="1d"}) and on() (max(postmark_outbound_sent{window="1d"}) >= 50) and on() (max(postmark_outbound_bounced{window="1d"}) >= 3)) or on() vector(0)
+                  expr: max(postmark_outbound_bounce_rate{window="1d"}) or on() vector(0)
                   legendFormat: bounce 1d
                   datasource:
                     type: prometheus
@@ -838,7 +838,7 @@ data:
             noDataState: OK
             execErrState: Error
             annotations:
-              summary: "Postmark 1d bounce rate >5% with at least 50 sent and 3 bounced"
+              summary: "Postmark 1d bounce rate >5%"
             labels:
               severity: warning
           - uid: postmark-api-down

services/monitoring/grafana-dashboard-network.yaml

@@ -485,7 +485,7 @@ data:
           },
           "targets": [
             {
-              "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+              "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
               "refId": "A",
               "legendFormat": "{{node}}"
             }

services/monitoring/grafana-dashboard-nodes.yaml

@@ -29,7 +29,7 @@ data:
           },
           "targets": [
             {
-              "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})",
+              "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})",
               "refId": "A"
             }
           ],
@@ -55,7 +55,7 @@ data:
               "unit": "none",
               "custom": {
                 "displayMode": "auto",
-                "valueSuffix": "/18"
+                "valueSuffix": "/21"
               }
             },
             "overrides": []
@@ -418,7 +418,7 @@ data:
           },
           "targets": [
             {
-              "expr": "avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+              "expr": "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
               "refId": "A",
               "legendFormat": "{{node}}"
             }
@@ -458,7 +458,7 @@ data:
           },
           "targets": [
             {
-              "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+              "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
               "refId": "A",
               "legendFormat": "{{node}}"
             }
@@ -498,7 +498,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+              "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
               "refId": "A",
               "legendFormat": "{{node}}"
             }
@@ -535,7 +535,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+              "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
               "refId": "A",
               "legendFormat": "{{node}}"
             }
@@ -572,7 +572,7 @@ data:
           },
           "targets": [
             {
-              "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+              "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
               "refId": "A",
               "legendFormat": "{{node}}"
             }
@@ -610,7 +610,7 @@ data:
           },
           "targets": [
             {
-              "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+              "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
               "refId": "A",
               "legendFormat": "{{node}}"
             }

services/monitoring/grafana-dashboard-pods.yaml

@@ -29,7 +29,7 @@ data:
           },
           "targets": [
             {
-              "expr": "((sum(max by(namespace,pod) ((kube_pod_status_phase{phase=\"Pending\",namespace!~\"veles\"} == 1) and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)) + (sum(max by(namespace,pod) ((kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace!~\"veles\"} == 1) unless on(namespace,pod) kube_pod_owner{owner_kind=\"Job\"})) or on() vector(0)))",
+              "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})) or on() vector(0)",
               "refId": "A"
             }
           ],
@@ -89,7 +89,7 @@ data:
           },
           "targets": [
             {
-              "expr": "sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason{namespace!~\"veles\",reason=~\"CrashLoopBackOff|ImagePullBackOff\"} and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)",
+              "expr": "sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})) or on() vector(0)",
               "refId": "A"
             }
           ],
@@ -532,7 +532,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)))))",
+              "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)))))",
               "refId": "A",
               "instant": true,
               "format": "table"

services/monitoring/grafana-dashboard-storage.yaml

@@ -285,7 +285,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+              "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
               "refId": "A",
               "legendFormat": "{{node}}"
             }
@@ -323,7 +323,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+              "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
               "refId": "A",
               "legendFormat": "{{node}}"
             }

services/monitoring/helmrelease.yaml

@@ -85,20 +85,17 @@ spec:
       extraArgs:
         retentionPeriod: "1y"   # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11}
         promscrape.configCheckInterval: "30s"
-        search.maxConcurrentRequests: "4"
-        search.maxQueryDuration: "1m"
-        search.maxQueueDuration: "30s"
 
       persistentVolume:
         enabled: true
         size: 100Gi
       resources:
         requests:
-          cpu: 500m
-          memory: 2Gi
+          cpu: 250m
+          memory: 1Gi
         limits:
-          cpu: "2"
-          memory: 4Gi
+          cpu: "1"
+          memory: 2Gi
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -111,34 +108,10 @@ spec:
                   - key: kubernetes.io/hostname
                     operator: NotIn
                     values:
-                      # Longhorn engine-image probes repeatedly fail on these
-                      # attachment hosts and can surface volume I/O errors.
-                      - titan-14
-                      - titan-18
                       - titan-20
                       - titan-21
                       - titan-22
                       - titan-24
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 100
-              preference:
-                matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi5
-            - weight: 50
-              preference:
-                matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi4
-            - weight: 25
-              preference:
-                matchExpressions:
-                  - key: node-role.kubernetes.io/storage-backbone
-                    operator: Exists
 
       # Enable built-in Kubernetes scraping
       scrape:
@@ -370,7 +343,7 @@ spec:
       vault.hashicorp.com/agent-requests-mem: "64Mi"
       vault.hashicorp.com/agent-limits-cpu: "250m"
       vault.hashicorp.com/agent-limits-mem: "128Mi"
-      monitoring.bstein.dev/restart-rev: "13"
+      monitoring.bstein.dev/restart-rev: "12"
       vault.hashicorp.com/agent-inject-secret-grafana-env.sh: "kv/data/atlas/monitoring/grafana-admin"
       vault.hashicorp.com/agent-inject-template-grafana-env.sh: |
         {{ with secret "kv/data/atlas/monitoring/grafana-admin" }}
@@ -488,8 +461,6 @@ spec:
         hide_version: true
       users:
         default_theme: dark
-      date_formats:
-        default_timezone: America/Chicago
     ingress:
       enabled: true
       ingressClassName: traefik

services/monitoring/kustomization.yaml

@@ -19,10 +19,6 @@ resources:
   - grafana-dashboard-mail.yaml
   - grafana-dashboard-testing.yaml
   - vmalert-atlas-availability.yaml
-  - availability-backfill-v4-job.yaml
-  - availability-daily-backfill-v4-job.yaml
-  - availability-rollup-cronjob.yaml
-  - availability-legacy-cleanup-job.yaml
   - dcgm-exporter.yaml
   - nvidia-process-exporter.yaml
   - jetson-tegrastats-exporter.yaml
@@ -70,15 +66,3 @@ configMapGenerator:
       - platform_quality_suite_probe.sh=scripts/platform_quality_suite_probe.sh
     options:
       disableNameSuffixHash: true
-  - name: atlas-availability-rollup-script
-    namespace: monitoring
-    files:
-      - availability_rollup.py=scripts/availability_rollup.py
-    options:
-      disableNameSuffixHash: true
-  - name: atlas-availability-cleanup-script
-    namespace: monitoring
-    files:
-      - availability_cleanup.py=scripts/availability_cleanup.py
-    options:
-      disableNameSuffixHash: true

services/monitoring/nvidia-process-exporter.yaml

@@ -49,7 +49,6 @@ spec:
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "9401"
-        monitoring.bstein.dev/restart-rev: "20260802-current-attribution"
     spec:
       serviceAccountName: nvidia-process-exporter
       imagePullSecrets:

services/monitoring/scripts/availability_cleanup.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-"""Remove obsolete Atlas annual-availability series after the request-v4 migration."""
-
-from __future__ import annotations
-
-import json
-import os
-import time
-from datetime import datetime, timedelta, timezone
-from urllib.parse import urlencode
-from urllib.request import Request, urlopen
-
-
-VM_URL = os.environ.get(
-    "VM_URL", "http://victoria-metrics-single-server:8428"
-).rstrip("/")
-LEGACY_SELECTOR = (
-    '{__name__="atlas:availability:ratio_365d",scope="atlas",'
-    'definition!="request-v4"}'
-)
-PROTECTED_SELECTOR = (
-    '{__name__="atlas:availability:ratio_365d",scope="atlas",'
-    'definition="request-v4"}'
-)
-
-
-def list_series(selector: str) -> list[dict[str, str]]:
-    """List recently visible series matching an exact safety selector."""
-    end = datetime.now(timezone.utc)
-    start = end - timedelta(hours=48)
-    query = urlencode(
-        {
-            "match[]": selector,
-            "start": start.isoformat().replace("+00:00", "Z"),
-            "end": end.isoformat().replace("+00:00", "Z"),
-        }
-    )
-    with urlopen(f"{VM_URL}/api/v1/series?{query}", timeout=60) as response:
-        payload = json.load(response)
-    if payload.get("status") != "success":
-        raise RuntimeError(f"VictoriaMetrics series lookup failed: {payload}")
-    return payload["data"]
-
-
-def delete_legacy_series() -> None:
-    """Delete only annual Atlas series outside the protected request-v4 label set."""
-    query = urlencode({"match[]": LEGACY_SELECTOR})
-    request = Request(
-        f"{VM_URL}/api/v1/admin/tsdb/delete_series?{query}",
-        data=b"",
-        method="POST",
-    )
-    with urlopen(request, timeout=60) as response:
-        if response.status not in {200, 204}:
-            raise RuntimeError(f"VictoriaMetrics deletion returned HTTP {response.status}")
-
-
-def main() -> None:
-    """Guard the new series, delete retired definitions, and verify convergence."""
-    protected_before = list_series(PROTECTED_SELECTOR)
-    if len(protected_before) != 1:
-        raise RuntimeError(
-            f"expected exactly one protected request-v4 series, found {len(protected_before)}"
-        )
-    legacy_before = list_series(LEGACY_SELECTOR)
-    if legacy_before:
-        delete_legacy_series()
-
-    legacy_after = legacy_before
-    for _ in range(10):
-        legacy_after = list_series(LEGACY_SELECTOR)
-        if not legacy_after:
-            break
-        time.sleep(1)
-    if legacy_after:
-        raise RuntimeError(f"legacy availability series remain: {legacy_after}")
-    if len(list_series(PROTECTED_SELECTOR)) != 1:
-        raise RuntimeError("protected request-v4 series was not preserved")
-
-    print(
-        json.dumps(
-            {
-                "deleted_legacy_series": len(legacy_before),
-                "protected_series": len(protected_before),
-                "selector": LEGACY_SELECTOR,
-            },
-            sort_keys=True,
-        )
-    )
-
-
-if __name__ == "__main__":
-    main()

services/monitoring/scripts/availability_rollup.py

@@ -1,107 +0,0 @@
-#!/usr/bin/env python3
-"""Publish Atlas request availability from deduplicated daily rollups."""
-
-from __future__ import annotations
-
-import json
-import os
-import time
-from datetime import datetime, timedelta, timezone
-from typing import Iterable
-from urllib.parse import urlencode
-from urllib.request import Request, urlopen
-
-
-VM_URL = os.environ.get(
-    "VM_URL", "http://victoria-metrics-single-server:8428"
-).rstrip("/")
-SCOPE = "atlas"
-DEFINITION = "request-v4"
-REQUESTS_METRIC = "atlas:availability:requests_1d"
-FAILURES_METRIC = "atlas:availability:failures_1d"
-OUTPUT_METRIC = "atlas:availability:ratio_365d"
-WINDOW_DAYS = 365
-
-
-def parse_export(lines: Iterable[bytes]) -> dict[int, float]:
-    """Return the last exported value for each timestamp."""
-    points: dict[int, float] = {}
-    for raw_line in lines:
-        if not raw_line.strip():
-            continue
-        series = json.loads(raw_line)
-        for timestamp, value in zip(series["timestamps"], series["values"], strict=True):
-            points[int(timestamp)] = float(value)
-    return points
-
-
-def fetch_rollup(metric: str, start: datetime, end: datetime) -> dict[int, float]:
-    """Stream one compact rollup series from VictoriaMetrics."""
-    matcher = (
-        f'{{__name__="{metric}",scope="{SCOPE}",definition="{DEFINITION}"}}'
-    )
-    query = urlencode(
-        {
-            "match[]": matcher,
-            "start": start.isoformat().replace("+00:00", "Z"),
-            "end": end.isoformat().replace("+00:00", "Z"),
-        }
-    )
-    with urlopen(f"{VM_URL}/api/v1/export?{query}", timeout=600) as response:
-        return parse_export(response)
-
-
-def calculate_availability(requests: float, failures: float) -> float:
-    """Calculate the bounded successful-request ratio."""
-    if requests <= 0:
-        raise ValueError("availability requires at least one observed request")
-    if failures < 0:
-        raise ValueError("failed request count cannot be negative")
-    return max(0.0, min(1.0, 1.0 - (failures / requests)))
-
-
-def render_metric(value: float, timestamp_ms: int) -> str:
-    """Render one VictoriaMetrics Prometheus-import sample."""
-    return (
-        f'{OUTPUT_METRIC}{{definition="{DEFINITION}",scope="{SCOPE}",'
-        f'rollup="yearly"}} {value:.12f} {timestamp_ms}\n'
-    )
-
-
-def publish(value: float, timestamp_ms: int) -> None:
-    """Write the calculated annual ratio to VictoriaMetrics."""
-    request = Request(
-        f"{VM_URL}/api/v1/import/prometheus",
-        data=render_metric(value, timestamp_ms).encode(),
-        headers={"Content-Type": "text/plain"},
-        method="POST",
-    )
-    with urlopen(request, timeout=30) as response:
-        if response.status not in {200, 204}:
-            raise RuntimeError(f"VictoriaMetrics import returned HTTP {response.status}")
-
-
-def main() -> None:
-    """Rebuild and publish the rolling request-availability sample."""
-    end = datetime.now(timezone.utc)
-    start = end - timedelta(days=WINDOW_DAYS)
-    requests = sum(fetch_rollup(REQUESTS_METRIC, start, end).values())
-    failures = sum(fetch_rollup(FAILURES_METRIC, start, end).values())
-    availability = calculate_availability(requests, failures)
-    timestamp_ms = time.time_ns() // 1_000_000
-    publish(availability, timestamp_ms)
-    print(
-        json.dumps(
-            {
-                "requests": requests,
-                "failures": failures,
-                "availability_percent": availability * 100,
-                "timestamp_ms": timestamp_ms,
-            },
-            sort_keys=True,
-        )
-    )
-
-
-if __name__ == "__main__":
-    main()

services/monitoring/scripts/nvidia_process_exporter.py

@@ -134,8 +134,7 @@ def running_process_memory(handle):
 
 def process_utilization_samples(handle):
     try:
-        # NVML process sample timestamps are microseconds since the epoch.
-        since = int(time.time() * 1_000_000) - (SAMPLE_WINDOW_MS * 1000)
+        since = int(time.time() * 1000) - SAMPLE_WINDOW_MS
         samples = nvmlDeviceGetProcessUtilization(handle, since)
     except NVMLError_NotFound:
         return {}, 1
@@ -157,32 +156,6 @@ def process_utilization_samples(handle):
     return by_pid, 1
 
 
-def reconcile_namespace_utilization(namespace_sm, device_util):
-    """Make namespace attribution add up to the device's current utilization."""
-
-    current = max(float(device_util), 0.0)
-    reconciled = {
-        namespace: max(float(value), 0.0)
-        for namespace, value in namespace_sm.items()
-    }
-    attributed = sum(reconciled.values())
-
-    if current == 0:
-        return {namespace: 0.0 for namespace in reconciled}
-
-    # Per-process NVML samples and the device gauge are collected on slightly
-    # different intervals. Scale a stale/overlapping process sum down to the
-    # device's current total while preserving the namespace proportions.
-    if attributed > current and attributed > 0:
-        scale = current / attributed
-        return {namespace: value * scale for namespace, value in reconciled.items()}
-
-    residual = current - attributed
-    if residual > 0.1:
-        reconciled["host"] = reconciled.get("host", 0.0) + residual
-    return reconciled
-
-
 def collect_metrics():
     nvmlInit()
     pods = load_pods()
@@ -231,7 +204,10 @@ def collect_metrics():
             lines.append(metric_line("nvidia_process_gpu_sm_util_percent", labels, sm_util))
             lines.append(metric_line("nvidia_process_gpu_memory_used_bytes", labels, int(proc_info["memory"])))
 
-        namespace_sm = reconcile_namespace_utilization(namespace_sm, device_util)
+        attributed = sum(namespace_sm.values())
+        residual = max(device_util - attributed, 0.0)
+        if residual > 0.1:
+            namespace_sm["host"] = namespace_sm.get("host", 0.0) + residual
 
         for namespace, value in sorted(namespace_sm.items()):
             labels = {**base, "namespace": namespace, "pod": "__namespace_total__"}

services/monitoring/vmalert-atlas-availability.yaml

@@ -7,70 +7,142 @@ metadata:
 data:
   atlas-availability.yaml: |
     groups:
-      - name: atlas.availability.gateway
-        interval: 1h
-        eval_offset: 59m
+      - name: atlas.availability
+        interval: 15m
+        eval_offset: 14m
         rules:
-          - record: atlas:availability:requests_1h
+          - record: atlas:availability:ratio_1h
             expr: |
-              sum(increase(
-                traefik_entrypoint_requests_total{
-                  entrypoint="websecure",
-                  protocol="http",
-                  code=~"[1-5].."
-                }[1h]
-              ))
+              avg_over_time((
+                min(
+                  (
+                    sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
+                    / 3
+                  ),
+                  (
+                    sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
+                    / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
+                  )
+                )
+              )[1h:5m])
             labels:
-              definition: request-v4
               scope: atlas
               rollup: hourly
-          - record: atlas:availability:failures_1h
+          - record: atlas:availability:ratio_365d
             expr: |
-              sum(increase(
-                traefik_entrypoint_requests_total{
-                  entrypoint="websecure",
-                  protocol="http",
-                  code=~"5.."
-                }[1h]
-              ))
+              clamp_max((
+                (
+                  sum(sum_over_time((
+                    min(
+                      (
+                        sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
+                        / 3
+                      ),
+                      (
+                        sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
+                        / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
+                      )
+                    )
+                  )[365d:1h]))
+                  or on() vector(0)
+                )
+                +
+                clamp_min(
+                  8761
+                  -
+                  (
+                    clamp_min(
+                      floor(
+                        (
+                          time()
+                          -
+                          (
+                            min(min_over_time(timestamp(
+                              min(
+                                (
+                                  sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
+                                  / 3
+                                ),
+                                (
+                                  sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
+                                  / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
+                                )
+                              )
+                            )[365d:1h]))
+                            or on() vector(time() + 3600)
+                          )
+                        )
+                        / 3600
+                      )
+                      + 1,
+                      0
+                    )
+                  ),
+                  0
+                )
+              )
+              /
+              clamp_min(
+                (
+                  (
+                    sum(count_over_time((
+                      min(
+                        (
+                          sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
+                          / 3
+                        ),
+                        (
+                          sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
+                          / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
+                        )
+                      )
+                    )[365d:1h]))
+                    or on() vector(0)
+                  )
+                  +
+                  clamp_min(
+                    8761
+                    -
+                    (
+                      clamp_min(
+                        floor(
+                          (
+                            time()
+                            -
+                            (
+                              min(min_over_time(timestamp(
+                                min(
+                                  (
+                                    sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"})
+                                    / 3
+                                  ),
+                                  (
+                                    sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})
+                                    / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)
+                                  )
+                                )
+                              )[365d:1h]))
+                              or on() vector(time() + 3600)
+                            )
+                          )
+                          / 3600
+                        )
+                        + 1,
+                        0
+                      )
+                    ),
+                    0
+                  )
+                ),
+                1
+              ), 1)
             labels:
-              definition: request-v4
               scope: atlas
-              rollup: hourly
-      - name: atlas.availability.rollup
-        interval: 1d
-        eval_offset: 23h59m
-        rules:
-          - record: atlas:availability:requests_1d
-            expr: |
-              sum(increase(
-                traefik_entrypoint_requests_total{
-                  entrypoint="websecure",
-                  protocol="http",
-                  code=~"[1-5].."
-                }[1d]
-              ))
-            labels:
-              definition: request-v4
-              scope: atlas
-              rollup: daily
-          - record: atlas:availability:failures_1d
-            expr: |
-              sum(increase(
-                traefik_entrypoint_requests_total{
-                  entrypoint="websecure",
-                  protocol="http",
-                  code=~"5.."
-                }[1d]
-              ))
-            labels:
-              definition: request-v4
-              scope: atlas
-              rollup: daily
+              rollup: yearly
   platform-quality.yaml: |
     groups:
       - name: platform.quality
-        interval: 5m
+        interval: 1m
         rules:
           - record: platform_quality:test_case_status:count_1h
             expr: |
@@ -347,7 +419,6 @@ spec:
             - -datasource.queryStep=1h
             - -remoteWrite.url=http://victoria-metrics-single-server:8428
             - -rule=/etc/vmalert/rules/*.yaml
-            - -configCheckInterval=30s
             - -evaluationInterval=15m
             - -httpListenAddr=:8880
           ports:

services/vault/hermes-auth-role-bootstrap-job.yaml

@@ -1,54 +0,0 @@
-# services/vault/hermes-auth-role-bootstrap-job.yaml
-# Purpose: apply the Vault read/write boundaries needed by Hermes operator OIDC.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: vault-k8s-auth-hermes-1
-  namespace: vault
-spec:
-  backoffLimit: 2
-  template:
-    spec:
-      serviceAccountName: vault-admin
-      restartPolicy: Never
-      nodeSelector:
-        kubernetes.io/arch: arm64
-        node-role.kubernetes.io/worker: "true"
-      containers:
-        - name: configure-k8s-auth
-          image: hashicorp/vault:1.21.4
-          imagePullPolicy: IfNotPresent
-          command:
-            - sh
-            - /scripts/vault_k8s_auth_configure.sh
-          env:
-            - name: VAULT_ADDR
-              value: http://10.43.57.249:8200
-            - name: VAULT_K8S_ROLE
-              value: vault-admin
-            - name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE
-              value: /var/run/secrets/vault-token-reviewer/token
-            - name: VAULT_K8S_ROLE_TTL
-              value: 1h
-          volumeMounts:
-            - name: k8s-auth-config-script
-              mountPath: /scripts
-              readOnly: true
-            - name: token-reviewer
-              mountPath: /var/run/secrets/vault-token-reviewer
-              readOnly: true
-          resources:
-            requests:
-              cpu: 25m
-              memory: 32Mi
-            limits:
-              cpu: 250m
-              memory: 128Mi
-      volumes:
-        - name: k8s-auth-config-script
-          configMap:
-            name: vault-k8s-auth-config-script
-            defaultMode: 0555
-        - name: token-reviewer
-          secret:
-            secretName: vault-admin-token-reviewer

services/vault/kustomization.yaml

@@ -11,7 +11,6 @@ resources:
   - configmap.yaml
   - statefulset.yaml
   - k8s-auth-config-cronjob.yaml
-  - hermes-auth-role-bootstrap-job.yaml
   - oidc-config-cronjob.yaml
   - service.yaml
   - certificate.yaml

services/vault/scripts/vault_k8s_auth_configure.sh

@@ -253,8 +253,6 @@ write_policy_and_role "health" "health" "health-vault-sync" \
   "health/*" ""
 write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \
   "game-stream/*" ""
-write_policy_and_role "hermes" "hermes" "hermes-vault" \
-  "hermes/operator-oidc" ""
 write_policy_and_role "veles" "veles" "veles-backend,veles-generator,veles-postgres,veles-vault-sync" \
   "veles/* shared/harbor-pull shared/postmark-relay" ""
 write_policy_and_role "veles-sim" "veles" "veles-sim" \
@@ -298,7 +296,7 @@ write_policy_and_role "vault" "vault" "vault" \
 
 write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
   "shared/keycloak-admin shared/postmark-relay maintenance/metis-ssh-keys" \
-  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc hermes/operator-oidc" \
+  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc" \
   '
 path "kv/data/atlas/nodes/*" {
   capabilities = ["create", "update", "read"]