Merge origin/main into layout migration

Reconciles the services layout migration with ~991 upstream commits:
- Remote content wins for cassandra/cassandra-auth, monitoring dashboards,
  vmalert availability rules, veles, vault auth script, dashboard render
  script and tests (request-v4 availability definition)
- Layout paths win for structure: keycloak/bstein-dev-home job dirs use
  bootstrap-jobs/validation-jobs; cassandra realm jobs live in
  cassandra-auth (removed keycloak duplicates)
- Union: applications CR list gains hermes-chat and cassandra
  image-automation
- Fixed post-migration paths in dashboard test module loader and
  hermes-access job header

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml

@@ -0,0 +1,28 @@
+# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-chat
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes-chat
+  targetNamespace: hermes-chat
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 30m
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: hermes-chat
+      namespace: hermes-chat
+  dependsOn:
+    - name: cert-manager
+    - name: core
+    - name: hermes
+    - name: keycloak
+    - name: longhorn

clusters/atlas/flux-system/applications/hermes/kustomization.yaml

@@ -18,6 +18,10 @@ spec:
   wait: true
   timeout: 45m
   healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: hermes-model-gate
+      namespace: hermes
     - apiVersion: apps/v1
       kind: Deployment
       name: hermes-ollama
@@ -26,6 +30,10 @@ spec:
       kind: Deployment
       name: hermes
       namespace: hermes
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: oauth2-proxy-hermes
+      namespace: hermes
   dependsOn:
     - name: cert-manager
     - name: core

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -28,9 +28,11 @@ resources:
   - ai-llm/kustomization.yaml
   - openclaw/kustomization.yaml
   - hermes/kustomization.yaml
+  - hermes-chat/kustomization.yaml
   - game-stream/kustomization.yaml
   - cassandra-auth/kustomization.yaml
   - cassandra/kustomization.yaml
+  - cassandra/image-automation.yaml
   - veles/kustomization.yaml
   - typhon/kustomization.yaml
   - nextcloud/kustomization.yaml

dockerfiles/Dockerfile.hermes-agent

@@ -0,0 +1,122 @@
+# syntax=docker/dockerfile:1
+# dockerfiles/Dockerfile.hermes-agent
+FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
+
+USER root
+
+# Keep dashboard chat sockets tied to the intended React mount and conversation.
+# A resumed conversation needs a different PTY attachment key from a fresh chat;
+# reconnects to that same conversation must keep using the same key.
+RUN node <<'NODE'
+const fs = require("node:fs");
+const path = "/opt/hermes/web/src/pages/ChatPage.tsx";
+let source = fs.readFileSync(path, "utf8");
+const socketBefore = [
+  '      const url = await api.buildWsUrl("/api/pty", params);',
+  '      const ws = new WebSocket(url);',
+].join("\n");
+const socketAfter = [
+  '      const url = await api.buildWsUrl("/api/pty", params);',
+  '      if (unmounting) return;',
+  '      const ws = new WebSocket(url);',
+].join("\n");
+const attachBefore = '      params.attach = ptyAttachToken(forceFresh);';
+const attachAfter = [
+  '      const attachScope = resumeParam',
+  '        ? `resume:${resumeParam}:${scopedProfile ?? ""}`',
+  '        : `fresh:${scopedProfile ?? ""}`;',
+  '      params.attach = `${ptyAttachToken(forceFresh)}:${attachScope}`;',
+].join("\n");
+
+if (!source.includes(socketBefore)) {
+  throw new Error("Hermes ChatPage WebSocket patch context changed");
+}
+if (!source.includes(attachBefore)) {
+  throw new Error("Hermes ChatPage PTY attachment patch context changed");
+}
+source = source.replace(socketBefore, socketAfter);
+source = source.replace(attachBefore, attachAfter);
+fs.writeFileSync(path, source);
+NODE
+
+# The upstream OIDC gate authenticates users but deliberately treats the
+# dashboard as one shared workstation. Allow a deployment to narrow that
+# workstation to explicit OIDC subjects. Enforce this after normal provider
+# verification so a denied account is a 403, not a misleading provider 503.
+RUN python - <<'PY'
+from pathlib import Path
+
+path = Path("/opt/hermes/hermes_cli/dashboard_auth/middleware.py")
+source = path.read_text()
+helper_before = '''def _client_ip(request: Request) -> str:
+    fwd = request.headers.get("x-forwarded-for", "")
+    if fwd:
+        return fwd.split(",")[0].strip()
+    return request.client.host if request.client else ""
+
+
+'''
+helper_after = helper_before + '''def _dashboard_user_allowed(session) -> bool:
+    """Apply an optional deployment-level OIDC-subject allowlist."""
+    import os
+
+    allowed = {
+        value.strip()
+        for value in os.environ.get(
+            "HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS", ""
+        ).split(",")
+        if value.strip()
+    }
+    return not allowed or session.user_id in allowed
+
+
+def _user_forbidden_response() -> Response:
+    """Return an authorization failure without exposing identities."""
+    return JSONResponse(
+        {
+            "error": "forbidden",
+            "detail": "This Atlas account is not authorized for this dashboard.",
+        },
+        status_code=403,
+    )
+
+
+'''
+refresh_before = '''            new_session, refreshing_provider = refreshed
+            request.state.session = new_session
+            response = await call_next(request)
+'''
+refresh_after = '''            new_session, refreshing_provider = refreshed
+            if not _dashboard_user_allowed(new_session):
+                return _user_forbidden_response()
+            request.state.session = new_session
+            response = await call_next(request)
+'''
+final_before = '''    request.state.session = session
+    return await call_next(request)
+'''
+final_after = '''    if not _dashboard_user_allowed(session):
+        return _user_forbidden_response()
+    request.state.session = session
+    return await call_next(request)
+'''
+for before, after, label in (
+    (helper_before, helper_after, "allowlist helper"),
+    (refresh_before, refresh_after, "refreshed session"),
+    (final_before, final_after, "verified session"),
+):
+    if before not in source:
+        raise SystemExit(f"Hermes dashboard auth {label} patch context changed")
+    source = source.replace(before, after, 1)
+path.write_text(source)
+PY
+
+COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
+
+RUN cd /opt/hermes/web \
+    && npm run build \
+    && grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \
+    && grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \
+    && grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \
+        /opt/hermes/hermes_cli/dashboard_auth/middleware.py \
+    && chmod 0755 /opt/hermes/bin/hermes-session-migrate

dockerfiles/hermes-session-migrate.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Copy selected legacy Hermes sessions into an isolated user home."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import shutil
+import sqlite3
+
+from hermes_state import SessionDB
+
+
+def _copy_if_missing(source: Path, target: Path) -> None:
+    """Copy one credential/config file without overwriting user state."""
+    if source.is_file() and not target.exists():
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, target)
+
+
+def _copy_session(
+    source: sqlite3.Connection,
+    target: sqlite3.Connection,
+    session_id: str,
+    user_id: str,
+) -> bool:
+    """Copy a session and its messages, preserving original timestamps."""
+    source.row_factory = sqlite3.Row
+    session = source.execute(
+        "SELECT * FROM sessions WHERE id = ?", (session_id,)
+    ).fetchone()
+    if session is None:
+        return False
+
+    columns = list(session.keys())
+    values = [user_id if column == "user_id" else session[column] for column in columns]
+    placeholders = ", ".join("?" for _ in columns)
+    target.execute(
+        f"INSERT OR IGNORE INTO sessions ({', '.join(columns)}) "
+        f"VALUES ({placeholders})",
+        values,
+    )
+
+    message_columns = [
+        row[1] for row in target.execute("PRAGMA table_info(messages)").fetchall()
+    ]
+    selected_columns = [
+        column
+        for column in message_columns
+        if column in {
+            row[1]
+            for row in source.execute("PRAGMA table_info(messages)").fetchall()
+        }
+    ]
+    column_sql = ", ".join(selected_columns)
+    target.execute(
+        f"INSERT OR IGNORE INTO messages ({column_sql}) "
+        f"SELECT {column_sql} FROM source_db.messages WHERE session_id = ?",
+        (session_id,),
+    )
+    return True
+
+
+def main() -> None:
+    """Create the isolated home and perform the idempotent session copy."""
+    source_home = Path(os.environ["HERMES_MIGRATE_SOURCE_HOME"])
+    target_home = Path(os.environ["HERMES_HOME"])
+    user_id = os.environ["HERMES_MIGRATE_USER_ID"].strip()
+    session_ids = [
+        value.strip()
+        for value in os.environ.get("HERMES_MIGRATE_SESSION_IDS", "").split(",")
+        if value.strip()
+    ]
+
+    target_home.mkdir(parents=True, exist_ok=True)
+    (target_home / "home" / ".local" / "bin").mkdir(parents=True, exist_ok=True)
+    (target_home / "workspace" / "skills").mkdir(parents=True, exist_ok=True)
+    (target_home / "logs").mkdir(parents=True, exist_ok=True)
+    for filename in (".env", "auth.json"):
+        _copy_if_missing(source_home / filename, target_home / filename)
+
+    source_db = source_home / "state.db"
+    target_db = target_home / "state.db"
+    SessionDB(db_path=target_db).close()
+    copied = 0
+    if source_db.is_file() and session_ids:
+        source = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)
+        target = sqlite3.connect(target_db)
+        target.execute("ATTACH DATABASE ? AS source_db", (str(source_db),))
+        try:
+            with target:
+                for session_id in session_ids:
+                    copied += int(_copy_session(source, target, session_id, user_id))
+        finally:
+            target.close()
+            source.close()
+    print(f"isolated Hermes home ready; migrated_sessions={copied}")
+
+
+if __name__ == "__main__":
+    main()

infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: longhorn-disk-tags-ensure-3
+  name: longhorn-disk-tags-ensure-4
   namespace: longhorn-system
 spec:
   backoffLimit: 0

infrastructure/traefik/deployment.yaml

@@ -4,8 +4,6 @@ items:
 - apiVersion: apps/v1
   kind: Deployment
   metadata:
-    annotations:
-      deployment.kubernetes.io/revision: "4"
     name: traefik
     namespace: traefik
   spec:
@@ -73,6 +71,14 @@ items:
           node-role.kubernetes.io/worker: "true"
         affinity:
           nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                  - rpi5
+                  - rpi4
             preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
               preference:
@@ -103,6 +109,12 @@ items:
                   operator: In
                   values:
                   - rpi4
+          podAntiAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: traefik
+              topologyKey: kubernetes.io/hostname
         restartPolicy: Always
         schedulerName: default-scheduler
         serviceAccount: atlas-traefik-ingress-controller

scripts/render/dashboards_render_atlas.py

@@ -2,8 +2,8 @@
 """Generate Atlas Grafana dashboards and render them into ConfigMaps.
 
 Usage:
-  scripts/render/dashboards_render_atlas.py --build   # rebuild JSON + ConfigMaps
+  scripts/dashboards_render_atlas.py --build   # rebuild JSON + ConfigMaps
-  scripts/render/dashboards_render_atlas.py           # re-render ConfigMaps from JSON
+  scripts/dashboards_render_atlas.py           # re-render ConfigMaps from JSON
 """
 
 import argparse
@@ -16,14 +16,7 @@ from pathlib import Path
 # Paths, folders, and shared metadata
 # ---------------------------------------------------------------------------
 
-def repo_root() -> Path:
+ROOT = Path(__file__).resolve().parents[1]
-    for path in Path(__file__).resolve().parents:
-        if (path / "clusters").is_dir() and (path / "services").is_dir():
-            return path
-    raise RuntimeError("could not locate repository root")  # pragma: no cover
-
-
-ROOT = repo_root()
 DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards"
 CONFIG_TEMPLATE = textwrap.dedent(
     """# {relative_path}
@@ -117,8 +110,6 @@ WORKER_NODES = [
     "titan-06",
     "titan-07",
     "titan-08",
-    "titan-09",
-    "titan-10",
     "titan-11",
     "titan-20",
     "titan-21",
@@ -126,7 +117,6 @@ WORKER_NODES = [
     "titan-13",
     "titan-14",
     "titan-15",
-    "titan-16",
     "titan-17",
     "titan-18",
     "titan-19",
@@ -191,7 +181,8 @@ def scoped_node_expr(base, scope=""):
 
 def node_cpu_expr(scope=""):
     idle = 'avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))'
-    base = f"(1 - {idle}) * 100"
+    # Scrape stalls can briefly report an impossible idle rate after recovery.
+    base = f"clamp_max(clamp_min((1 - {idle}) * 100, 0), 100)"
     return scoped_node_expr(base, scope)
 
 
@@ -291,8 +282,14 @@ def dcgm_gpu_util_by_node():
     )
 
 
+def nvidia_gpu_util_by_node():
+    return "max by (node) (nvidia_gpu_device_utilization_percent)"
+
+
 def gpu_util_by_node():
-    return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}"
+    process_exporter = nvidia_gpu_util_by_node()
+    dcgm_fallback = f"({dcgm_gpu_util_by_node()}) unless on(node) ({process_exporter})"
+    return f"{process_exporter} or {dcgm_fallback} or {jetson_gpu_util_by_node()}"
 
 
 def gpu_util_by_hostname():
@@ -302,16 +299,11 @@ def gpu_util_by_hostname():
 GPU_RESOURCE_REGEX = "nvidia(_com_|[.]com/)gpu.*"
 
 
-def gpu_node_labels():
-    return f'max by (node) (kube_node_status_allocatable{{resource=~"{GPU_RESOURCE_REGEX}"}} > bool 0)'
-
-
 def gpu_requests_by_namespace_node(scope_var):
     return (
         "sum by (namespace,node) ("
         f'kube_pod_container_resource_requests{{resource=~"{GPU_RESOURCE_REGEX}",{scope_var}}} '
         "* on(namespace,pod) group_left(node) kube_pod_info "
-        f"* on(node) group_left() ({gpu_node_labels()})"
         ")"
     )
 
@@ -333,7 +325,8 @@ def gpu_usage_by_namespace(scope_var):
 
 def jetson_gpu_usage_by_namespace(scope_var):
     requests_by_ns = gpu_requests_by_namespace_node(scope_var)
-    total_by_node = f"sum by (node) ({requests_by_ns})"
+    all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
+    total_by_node = f"sum by (node) ({all_requests})"
     return (
         "sum by (namespace) ("
         f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) "
@@ -342,6 +335,12 @@ def jetson_gpu_usage_by_namespace(scope_var):
     )
 
 
+def jetson_gpu_requested_nodes():
+    all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
+    requested = f"(sum by (node) ({all_requests}) > 0)"
+    return f"({requested}) and on(node) ({jetson_gpu_util_by_node()})"
+
+
 def namespace_share_expr(resource_expr):
     total = f"clamp_min(sum( {resource_expr} ), 1)"
     return f"100 * ( {resource_expr} ) / {total}"
@@ -391,14 +390,29 @@ def gpu_total_devices_expr():
 
 
 def unattributed_gpu_usage():
-    legacy_total = f"(sum({legacy_gpu_util_without_process_exporter()}) or on() vector(0))"
+    unresolved = (
+        f"({legacy_gpu_util_without_process_exporter()}) "
+        f"unless on(node) ({jetson_gpu_requested_nodes()})"
+    )
+    legacy_total = f"(sum({unresolved}) or on() vector(0))"
     return (
         f'label_replace(({legacy_total} > 0), "namespace", "unattributed", "", "")'
     )
 
 
 def gpu_utilization_raw(scope_var):
-    return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
+    nvidia = (
+        'label_replace('
+        f'{nvidia_process_gpu_usage_by_namespace(scope_var)}, '
+        '"gpu_source", "nvidia", "", "")'
+    )
+    jetson = (
+        'label_replace('
+        f'(({jetson_gpu_usage_by_namespace(scope_var)}) > 0), '
+        '"gpu_source", "jetson", "", "")'
+    )
+    attributed = f"sum by (namespace) (({nvidia}) or ({jetson}))"
+    return f"({attributed}) or ({unattributed_gpu_usage()})"
 
 
 def gpu_pool_used_expr(scope_var):
@@ -418,13 +432,20 @@ def namespace_gpu_share_expr(scope_var):
 
 
 PROBLEM_PODS_EXPR = (
-    'sum(max by (namespace,pod) (kube_pod_status_phase{phase!~"Running|Succeeded"})) '
+    '((sum(max by(namespace,pod) ('
-    "or on() vector(0)"
+    '(kube_pod_status_phase{phase="Pending",namespace!~"veles"} == 1) '
+    'and on(namespace,pod) ((time() - kube_pod_created{namespace!~"veles"}) > 900)'
+    ')) or on() vector(0)) + (sum(max by(namespace,pod) ('
+    '(kube_pod_status_phase{phase=~"Failed|Unknown",namespace!~"veles"} == 1) '
+    'unless on(namespace,pod) kube_pod_owner{owner_kind="Job"}'
+    ')) or on() vector(0)))'
 )
 CRASHLOOP_EXPR = (
-    'sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason'
+    'sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason'
-    '{reason=~"CrashLoopBackOff|ImagePullBackOff"})) '
+    '{namespace!~"veles",reason=~"CrashLoopBackOff|ImagePullBackOff"} '
-    "or on() vector(0)"
+    'and on(namespace,pod) '
+    '((time() - kube_pod_created{namespace!~"veles"}) > 900))) '
+    'or on() vector(0)'
 )
 STUCK_TERMINATING_EXPR = (
     'sum(max by (namespace,pod) ('
@@ -436,23 +457,22 @@ STUCK_TERMINATING_EXPR = (
 UPTIME_WINDOW = "365d"
 # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series.
 UPTIME_RECORDING_METRIC = (
-    f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}'
+    f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="request-v4"}}'
 )
-TRAEFIK_READY_EXPR = (
+AVAILABILITY_REQUESTS_1H_EXPR = (
-    "("
+    'sum(increase(traefik_entrypoint_requests_total{'
-    'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
+    'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))'
-    " > bool 0)"
 )
-CONTROL_READY_FRACTION_EXPR = (
+AVAILABILITY_FAILURES_1H_EXPR = (
-    f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
+    'sum(increase(traefik_entrypoint_requests_total{'
-    " >= bool 2)"
+    'entrypoint="websecure",protocol="http",code=~"5.."}[1h]))'
 )
-UPTIME_AVAIL_EXPR = (
+UPTIME_LIVE_FALLBACK_EXPR = (
-    f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
+    f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / "
+    f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
 )
-UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])"
 UPTIME_RECORDING_EXPR = (
-    f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
+    f"(last_over_time({UPTIME_RECORDING_METRIC}[48h]) "
     f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
 )
 
@@ -631,7 +651,7 @@ PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX = (
     "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube"
 )
 PLATFORM_TEST_STANDARD_CHECK_REGEX = (
-    "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|semgrep|supply_chain"
+    "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|supply_chain"
 )
 PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_SUITES = [
     "ariadne",
@@ -1881,9 +1901,9 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.",
     "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.",
     "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.",
-    "Atlas Serving Availability": "Observed availability with control-plane quorum and at least one Traefik replica serving; partial replica capacity remains available.",
+    "Atlas Availability (365d)": "Request-weighted Atlas ingress availability; every server-side 5xx response counts as a failed request.",
-    "Problem Pods": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.",
+    "Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.",
-    "CrashLoop / ImagePull": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.",
+    "CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.",
     "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.",
     "Hottest node: CPU": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.",
     "Hottest node: RAM": "Highest worker memory use right now; lower is safer, high values risk evictions.",
@@ -1921,7 +1941,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
     "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
     "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
-    "Namespace GPU Utilization": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.",
+    "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
     "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
     "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
     "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",
@@ -1963,7 +1983,6 @@ TESTING_PANEL_DESCRIPTIONS = {
     "Style Failure Rate": "Percent of style checks currently failing; blue means style/docs gates pass.",
     "Gate Glue Failure Rate": "Percent of metric-contract checks failing; blue means dashboard telemetry is trustworthy.",
     "SonarQube Failure Rate": "Percent of Sonar checks failing; blue means Sonar quality gates pass.",
-    "Semgrep Failure Rate": "Percent of Semgrep checks failing; blue means SAST rule scans pass.",
     "Supply Chain Failure Rate": "Percent of supply-chain checks failing; blue means artifact/image checks pass.",
     "Check Healthy Rates By Suite": "Healthy percent by check family; blue means all selected checks are good.",
     "Tests Healthy Rate": "Percent of test checks passing or not applicable; higher is better.",
@@ -1972,7 +1991,6 @@ TESTING_PANEL_DESCRIPTIONS = {
     "Style Healthy Rate": "Percent of style checks passing or not applicable; higher is better.",
     "Gate Glue Healthy Rate": "Percent of telemetry-contract checks passing; higher means cleaner reporting.",
     "SonarQube Healthy Rate": "Percent of Sonar checks passing or not applicable; higher is better.",
-    "Semgrep Healthy Rate": "Percent of Semgrep checks passing or not applicable; higher is better.",
     "Supply Chain Healthy Rate": "Percent of supply-chain checks passing or not applicable; higher is better.",
     "Test Drilldowns And Problem Tests": "Test-case detail for finding which tests are hurting reliability.",
     "Problematic Tests Over Time (Top failures)": "Current outlier tests by rolling 24h failures; tests need repeat failures to stay visible.",
@@ -1989,13 +2007,6 @@ TESTING_PANEL_DESCRIPTIONS = {
     "Recent Branch Evidence by Suite (7d)": "Branches with recent CI evidence; unexpected branches can mean drift or stale work.",
     "Primary Branch Clean by Suite (7d)": "Percent clean of non-primary branch evidence; 100% means only main/master is reporting.",
     "SonarQube Project Health": "SonarQube availability, projects, fetch errors, and gate status.",
-    "Public ZAP Baseline": "OWASP ZAP passive baseline results for externally addressed Atlas domains.",
-    "ZAP Targets Scanned": "Number of public hostnames with a latest OWASP ZAP baseline scan sample.",
-    "ZAP High/Medium Alerts": "Current high and medium ZAP alert instance count across public targets.",
-    "ZAP Scan Errors": "Targets whose latest ZAP baseline failed or did not produce a usable report.",
-    "ZAP Alerts by Risk": "Current ZAP alert distribution by risk level.",
-    "ZAP Target Health": "Latest ZAP health per hostname from passive baseline scans.",
-    "ZAP Alerts by Host": "Current high, medium, and low ZAP alert counts by public hostname.",
     "SonarQube API Up": "Whether the SonarQube exporter can reach SonarQube; 1 is good.",
     "Sonar Projects (Selected)": "Selected SonarQube project count; zero means Sonar is not tracking that suite.",
     "Sonar Gate Fetch Errors": "Sonar exporter fetch errors; zero is good because stale Sonar data misleads.",
@@ -2123,7 +2134,7 @@ def build_overview():
         },
         {
             "id": 27,
-            "title": "Atlas Serving Availability",
+            "title": "Atlas Availability (365d)",
             "expr": UPTIME_PERCENT_EXPR,
             "kind": "stat",
             "thresholds": UPTIME_PERCENT_THRESHOLDS,
@@ -2131,7 +2142,7 @@ def build_overview():
             "decimals": 4,
             "text_mode": "value",
             "instant": True,
-            "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.",
+            "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists.",
         },
         {
             "id": 4,
@@ -4183,7 +4194,6 @@ def build_jobs_dashboard():
     check_regex_style = "docs|naming|hygiene|lint|docs_naming|style"
     check_regex_gate_glue = "gate|glue|gate_glue"
     check_regex_sonarqube = "sonarqube|sonar"
-    check_regex_semgrep = "semgrep|sast"
     check_regex_supply_chain = "ironbank|supply_chain|image_compliance|artifact_security"
 
     def _check_state_percent_series(regex: str, failed: bool) -> str:
@@ -4227,30 +4237,6 @@ def build_jobs_dashboard():
         f"topk by (suite) (1, ({rollup_failed_tests_history}))"
     )
     worst_test_per_suite = worst_test_per_suite_core
-    zap_target_health = (
-        "(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) "
-        "or on() vector(0))"
-    )
-    zap_targets_scanned = (
-        '(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job="platform-security-zap"})) '
-        "or on() vector(0))"
-    )
-    zap_high_medium_alerts = (
-        '(sum(platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk=~"high|medium"}) '
-        "or on() vector(0))"
-    )
-    zap_scan_errors = (
-        '(sum(platform_zap_baseline_scan_status{exported_job="platform-security-zap",status=~"fail|error"}) '
-        "or on() vector(0))"
-    )
-    zap_alerts_by_risk = (
-        'sum by (risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk!=""}) '
-        "or on() vector(0)"
-    )
-    zap_alerts_by_host_risk = (
-        'sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",'
-        'risk=~"high|medium|low"}) or on() vector(0)'
-    )
 
     def _selected_status_volume(status: str) -> str:
         return (
@@ -4596,7 +4582,6 @@ def build_jobs_dashboard():
         ("Style", check_regex_style),
         ("Gate Glue", check_regex_gate_glue),
         ("SonarQube", check_regex_sonarqube),
-        ("Semgrep", check_regex_semgrep),
         ("Supply Chain", check_regex_supply_chain),
     ]
 
@@ -4620,12 +4605,12 @@ def build_jobs_dashboard():
             )
             panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
             panels.append(panel)
-        for index, (label, regex) in enumerate(check_dimensions[4:8]):
+        for index, (label, regex) in enumerate(check_dimensions[4:]):
             panel = state_timeline_panel(
                 start_id + 4 + index,
                 f"{label} {title_prefix}",
                 _check_state_percent_series(regex, failed),
-                {"h": 7, "w": 6, "x": index * 6, "y": y + 7},
+                {"h": 7, "w": 8, "x": index * 8, "y": y + 7},
                 thresholds=trend_thresholds,
                 description=trend_description,
             )
@@ -4633,7 +4618,7 @@ def build_jobs_dashboard():
             panels.append(panel)
 
     _append_check_trends(130, "Failure Rate", True, 29)
-    _append_check_trends(160, "Healthy Rate", False, 43)
+    _append_check_trends(138, "Healthy Rate", False, 43)
     panels.append(
         state_timeline_panel(
             145,
@@ -4889,79 +4874,6 @@ def build_jobs_dashboard():
     )
     sonar_gate_project_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
     panels.append(sonar_gate_project_panel)
-    panels.append(
-        stat_panel(
-            168,
-            "ZAP Targets Scanned",
-            zap_targets_scanned,
-            {"h": 6, "w": 4, "x": 0, "y": 117},
-            unit="none",
-            instant=True,
-            thresholds=missing_thresholds,
-        )
-    )
-    panels.append(
-        stat_panel(
-            169,
-            "ZAP High/Medium Alerts",
-            zap_high_medium_alerts,
-            {"h": 6, "w": 4, "x": 4, "y": 117},
-            unit="none",
-            instant=True,
-            thresholds=failures_thresholds,
-        )
-    )
-    panels.append(
-        stat_panel(
-            170,
-            "ZAP Scan Errors",
-            zap_scan_errors,
-            {"h": 6, "w": 4, "x": 8, "y": 117},
-            unit="none",
-            instant=True,
-            thresholds=failures_thresholds,
-        )
-    )
-    zap_risk_panel = pie_panel(
-        171,
-        "ZAP Alerts by Risk",
-        zap_alerts_by_risk,
-        {"h": 6, "w": 4, "x": 12, "y": 117},
-    )
-    zap_risk_panel["targets"][0]["legendFormat"] = "{{risk}}"
-    panels.append(zap_risk_panel)
-    zap_target_panel = state_timeline_panel(
-        172,
-        "ZAP Target Health",
-        zap_target_health,
-        {"h": 6, "w": 8, "x": 16, "y": 117},
-        thresholds=success_thresholds,
-        unit="percent",
-        min_value=0,
-        max_value=100,
-        legend="{{host}}",
-        description=(
-            "Latest OWASP ZAP passive baseline health per public hostname. "
-            "High alerts and scan errors drop a target to zero; medium/low alerts mark warning health."
-        ),
-    )
-    zap_target_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
-    panels.append(zap_target_panel)
-    panels.append(
-        bargauge_panel(
-            173,
-            "ZAP Alerts by Host",
-            zap_alerts_by_host_risk,
-            {"h": 7, "w": 24, "x": 0, "y": 123},
-            unit="none",
-            instant=True,
-            legend="{{host}} · {{risk}}",
-            sort_order="desc",
-            thresholds=failures_thresholds,
-            decimals=0,
-            limit=20,
-        )
-    )
     panels.append(
         bargauge_panel(
             148,
@@ -5071,12 +4983,6 @@ def build_jobs_dashboard():
         33: {"h": 6, "w": 4, "x": 8, "y": 111},
         34: {"h": 6, "w": 4, "x": 12, "y": 111},
         35: {"h": 6, "w": 8, "x": 16, "y": 111},
-        168: {"h": 6, "w": 4, "x": 0, "y": 117},
-        169: {"h": 6, "w": 4, "x": 4, "y": 117},
-        170: {"h": 6, "w": 4, "x": 8, "y": 117},
-        171: {"h": 6, "w": 4, "x": 12, "y": 117},
-        172: {"h": 6, "w": 8, "x": 16, "y": 117},
-        173: {"h": 7, "w": 24, "x": 0, "y": 123},
     }
     for panel_id, grid in row_layout.items():
         panel_by_id[panel_id]["gridPos"] = grid
@@ -5088,13 +4994,13 @@ def build_jobs_dashboard():
                 501,
                 "Check Failure Rates By Suite",
                 12,
-                panels=children([130, 131, 132, 133, 134, 135, 136, 137]),
+                panels=children([130, 131, 132, 133, 134, 135, 136]),
             ),
             row_panel(
                 502,
                 "Check Healthy Rates By Suite",
                 13,
-                panels=children([160, 161, 162, 163, 164, 165, 166, 167]),
+                panels=children([138, 139, 140, 141, 142, 143, 144]),
             ),
             row_panel(
                 503,
@@ -5114,12 +5020,6 @@ def build_jobs_dashboard():
                 16,
                 panels=children([31, 32, 33, 34, 35]),
             ),
-            row_panel(
-                506,
-                "Public ZAP Baseline",
-                17,
-                panels=children([168, 169, 170, 171, 172, 173]),
-            ),
         ]
     )
     panels = compact_panels
@@ -5621,12 +5521,13 @@ def build_gpu_dashboard():
     panels.append(
         table_panel(
             4,
-            "GPU Pods Reporting Device Util",
+            "GPU Processes by Pod",
-            'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))',
+            'topk(10, sum by (namespace,pod,node,process) '
+            '(nvidia_process_gpu_sm_util_percent{pod!="host"}) > 0)',
             {"h": 8, "w": 12, "x": 12, "y": 8},
             unit="percent",
             transformations=[{"id": "labelsToFields", "options": {}}],
-            description="DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.",
+            description="NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization.",
         )
     )
     return {

scripts/tests/test_availability_cleanup.py

@@ -0,0 +1,28 @@
+"""Safety tests for the legacy Atlas availability cleanup."""
+
+import importlib.util
+from pathlib import Path
+
+
+def load_module():
+    """Load the service-owned cleanup module without packaging it."""
+    path = (
+        Path(__file__).resolve().parents[2]
+        / "services/monitoring/scripts/availability_cleanup.py"
+    )
+    spec = importlib.util.spec_from_file_location("availability_cleanup", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_cleanup_selector_cannot_match_request_v4() -> None:
+    """Constrain deletion to obsolete annual Atlas definitions."""
+    mod = load_module()
+
+    assert '__name__="atlas:availability:ratio_365d"' in mod.LEGACY_SELECTOR
+    assert 'scope="atlas"' in mod.LEGACY_SELECTOR
+    assert 'definition!="request-v4"' in mod.LEGACY_SELECTOR
+    assert 'definition="request-v4"' in mod.PROTECTED_SELECTOR
+    assert "ratio_1h" not in mod.LEGACY_SELECTOR

scripts/tests/test_availability_rollup.py

@@ -0,0 +1,52 @@
+"""Unit tests for the Atlas availability publisher."""
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+
+def load_module():
+    """Load the service-owned rollup module without packaging it."""
+    path = (
+        Path(__file__).resolve().parents[2]
+        / "services/monitoring/scripts/availability_rollup.py"
+    )
+    spec = importlib.util.spec_from_file_location("availability_rollup", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_parse_export_deduplicates_replay_boundaries() -> None:
+    """Keep only the final value when replay chunks share a timestamp."""
+    mod = load_module()
+    lines = [
+        (json.dumps({"timestamps": [1000, 2000], "values": [2, 3]}) + "\n").encode(),
+        (json.dumps({"timestamps": [2000, 3000], "values": [3, 5]}) + "\n").encode(),
+    ]
+
+    assert mod.parse_export(lines) == {1000: 2.0, 2000: 3.0, 3000: 5.0}
+
+
+def test_calculate_availability_uses_all_server_failures() -> None:
+    """Calculate one bounded successful-request ratio."""
+    mod = load_module()
+
+    assert mod.calculate_availability(1000, 2) == pytest.approx(0.998)
+    with pytest.raises(ValueError):
+        mod.calculate_availability(0, 0)
+    with pytest.raises(ValueError):
+        mod.calculate_availability(100, -1)
+
+
+def test_render_metric_publishes_only_the_request_v4_series() -> None:
+    """Render the single series selected by the Grafana panel."""
+    mod = load_module()
+
+    assert mod.render_metric(0.9995, 1234) == (
+        'atlas:availability:ratio_365d{definition="request-v4",scope="atlas",'
+        'rollup="yearly"} 0.999500000000 1234\n'
+    )

scripts/tests/test_dashboards_render_atlas.py

@@ -3,10 +3,8 @@ import pathlib
 
 
 def load_module():
-    path = pathlib.Path(__file__).resolve().parents[1] / "render/dashboards_render_atlas.py"
+    path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
-    spec = importlib.util.spec_from_file_location(
+    spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
-        "scripts.render.dashboards_render_atlas", path
-    )
     module = importlib.util.module_from_spec(spec)
     assert spec.loader is not None
     spec.loader.exec_module(module)
@@ -49,6 +47,8 @@ def test_node_filter_and_expr_helpers():
     cpu_expr = mod.node_cpu_expr("titan-.*")
     mem_expr = mod.node_mem_expr("titan-.*")
     assert "node_cpu_seconds_total" in cpu_expr
+    assert "clamp_max(clamp_min(" in cpu_expr
+    assert "* 100, 0), 100)" in cpu_expr
     assert "node_memory_MemAvailable_bytes" in mem_expr
 
 
@@ -57,20 +57,24 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
     dashboard = mod.build_overview()
     panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
 
-    assert panel["title"] == "Atlas Serving Availability"
+    assert panel["title"] == "Atlas Availability (365d)"
     availability_expr = panel["targets"][0]["expr"]
     assert (
-        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])'
+        'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
         in availability_expr
     )
-    assert ">= bool 2" in availability_expr
+    assert 'code=~"5.."' in availability_expr
-    assert "> bool 0" in availability_expr
+    assert 'code=~"[1-5].."' in availability_expr
-    assert "[1h:5m]" in availability_expr
+    assert "atlas:availability:failures_1d" not in availability_expr
+    assert "atlas:availability:requests_1d" not in availability_expr
+    assert "sum_over_time" not in availability_expr
+    assert "kube_node_status_condition" not in availability_expr
+    assert "kube_deployment_status_replicas_available" not in availability_expr
     assert panel["targets"][0]["instant"] is True
-    assert "at least one Traefik replica serving" in panel["description"]
+    assert "Every server-side 5xx" in panel["description"]
-    assert "partial-capacity samples remain available" in panel["description"]
+    assert "Replica counts, Grafana health" in panel["description"]
-    assert "monitoring gaps are excluded" in panel["description"]
+    assert "daily rollup job publishes one annual sample" in panel["description"]
-    assert "falls back to the live binary serving state" in panel["description"]
+    assert "keeps it for up to 48 hours" in panel["description"]
 
 
 def test_overview_uses_readable_quality_power_and_gitops_panels():
@@ -169,14 +173,41 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
     assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
     assert "nvidia_gpu_device_utilization_percent" in gpu_expr
+    assert "sum_over_time" not in gpu_expr
+    assert "count_over_time" not in gpu_expr
+    assert "avg_over_time" not in gpu_expr
+    assert "$__range" not in gpu_expr
     assert "sum by (namespace)" in gpu_expr
     assert 'namespace", "shared"' not in gpu_expr
-    assert "kube_node_labels" not in gpu_expr
+    assert "kube_pod_container_resource_requests" in gpu_expr
+    assert mod.GPU_RESOURCE_REGEX in gpu_expr
+    assert '"gpu_source", "nvidia"' in gpu_expr
+    assert '"gpu_source", "jetson"' in gpu_expr
     assert "100 *" in gpu_expr
     assert "100 -" not in gpu_expr
     assert 'namespace", "unattributed"' in gpu_expr
     assert 'namespace", "idle"' in gpu_expr
     assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
+    assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
+
+
+def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
+    mod = load_module()
+    dashboard = mod.build_gpu_dashboard()
+    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
+
+    node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
+    assert "nvidia_gpu_device_utilization_percent" in node_expr
+    assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
+    assert "jetson_gr3d_freq_percent" in node_expr
+    assert "unless on(node)" in node_expr
+    assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
+
+    process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
+    assert "nvidia_process_gpu_sm_util_percent" in process_expr
+    assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
 
 
 def test_overview_and_testing_panels_all_have_concise_descriptions():
@@ -396,7 +427,7 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
         for child in row.get("panels", [])
     }
 
-    assert len(panels) == 19
+    assert len(panels) == 18
     assert len(visible_query_panels) == 12
     assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
     assert all(
@@ -410,18 +441,15 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
         "Test Drilldowns And Problem Tests",
         "Telemetry Completeness And Branches",
         "SonarQube Project Health",
-        "Public ZAP Baseline",
     ]
     assert all(row["collapsed"] for row in rows)
 
     assert "Coverage Failure Rate" in nested_panels_by_title
-    assert "Semgrep Failure Rate" in nested_panels_by_title
     assert "Supply Chain Healthy Rate" in nested_panels_by_title
     assert "Test Category Health History" in nested_panels_by_title
     assert "Selected Test Pass Rate History" in nested_panels_by_title
     assert "Coverage Metrics Present by Suite" in nested_panels_by_title
     assert "SonarQube API Up" in nested_panels_by_title
-    assert "ZAP Target Health" in nested_panels_by_title
 
     failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
     assert failure_rate_panel["type"] == "state-timeline"
@@ -472,15 +500,6 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
     assert sonar_health_panel["type"] == "state-timeline"
     assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"]
     assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"]
-    semgrep_panel = nested_panels_by_title["Semgrep Healthy Rate"]
-    assert semgrep_panel["type"] == "state-timeline"
-    assert 'check=~"semgrep|sast"' in semgrep_panel["targets"][0]["expr"]
-
-    zap_health_panel = nested_panels_by_title["ZAP Target Health"]
-    assert zap_health_panel["type"] == "state-timeline"
-    assert "platform_zap_baseline_target_health_percent" in zap_health_panel["targets"][0]["expr"]
-    zap_alerts_panel = nested_panels_by_title["ZAP Alerts by Host"]
-    assert "platform_zap_baseline_alerts_total" in zap_alerts_panel["targets"][0]["expr"]
 
     branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
     recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]

scripts/tests/test_monitoring_query_capacity.py

@@ -44,43 +44,113 @@ def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
     assert {"titan-14", "titan-18"} <= set(hostname_rule["values"])
 
 
-def test_yearly_availability_reuses_the_hourly_rollup() -> None:
+def test_yearly_availability_is_published_outside_the_query_pool() -> None:
-    """Prevent the yearly rule from rescanning raw cluster metrics for 365 days."""
+    """Keep all long-range availability work out of Grafana and MetricsQL."""
     manifest = _documents(
         REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
     )[0]
-    rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
+    groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
-    yearly = next(
+    rules = [rule for group in groups for rule in group["rules"]]
-        rule
+    manifests = _documents(
-        for rule in rules["rules"]
+        REPO_ROOT / "services/monitoring/availability-rollup-cronjob.yaml"
-        if rule["record"] == "atlas:availability:ratio_365d"
     )
+    cronjob = next(manifest for manifest in manifests if manifest["kind"] == "CronJob")
+    source = (
+        REPO_ROOT / "services/monitoring/scripts/availability_rollup.py"
+    ).read_text()
 
-    assert "atlas:availability:ratio_1h" in yearly["expr"]
+    assert all(rule["record"] != "atlas:availability:ratio_365d" for rule in rules)
-    assert 'definition="serving-v2"' in yearly["expr"]
+    assert "atlas:availability:requests_1d" in source
-    assert 'definition=""' in yearly["expr"]
+    assert "atlas:availability:failures_1d" in source
-    assert "share_gt_over_time" in yearly["expr"]
+    assert "/api/v1/export" in source
-    assert "[365d:15m]" in yearly["expr"]
+    assert "/api/v1/import/prometheus" in source
-    assert "kube_node_status_condition" not in yearly["expr"]
+    assert cronjob["spec"]["schedule"] == "10 0 * * *"
-    assert "[365d:1h]" not in yearly["expr"]
+    assert cronjob["spec"]["concurrencyPolicy"] == "Forbid"
 
 
-def test_availability_counts_serving_state_instead_of_replica_capacity() -> None:
+def test_daily_availability_rollups_use_the_same_request_sli() -> None:
-    """Treat quorum and any serving ingress replica as available binary states."""
+    """Keep annual availability cheap without changing its request definition."""
     manifest = _documents(
         REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
     )[0]
-    rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
+    groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
-    hourly = next(
+    daily_group = next(
+        group for group in groups if group["name"] == "atlas.availability.rollup"
+    )
+    daily = {rule["record"]: rule for rule in daily_group["rules"]}
+
+    assert daily_group["interval"] == "1d"
+    assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"]
+    assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"]
+    assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values())
+
+
+def test_availability_uses_request_failures_instead_of_replica_capacity() -> None:
+    """Measure HTTP outcomes without treating redundant replica loss as downtime."""
+    manifest = _documents(
+        REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
+    )[0]
+    groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
+    rules = [rule for group in groups for rule in group["rules"]]
+    requests = next(
         rule
-        for rule in rules["rules"]
+        for rule in rules
-        if rule["record"] == "atlas:availability:ratio_1h"
+        if rule["record"] == "atlas:availability:requests_1h"
+    )
+    failures = next(
+        rule
+        for rule in rules
+        if rule["record"] == "atlas:availability:failures_1h"
     )
 
-    assert ">= bool 2" in hourly["expr"]
+    assert 'code=~"[1-5].."' in requests["expr"]
-    assert "> bool 0" in hourly["expr"]
+    assert 'code=~"5.."' in failures["expr"]
-    assert "/ 3" not in hourly["expr"]
+    assert "traefik_entrypoint_requests_total" in requests["expr"]
-    assert hourly["labels"]["definition"] == "serving-v2"
+    assert "traefik_entrypoint_requests_total" in failures["expr"]
+    assert "kube_node_status_condition" not in repr(rules)
+    assert "kube_deployment_status_replicas_available" not in repr(rules)
+    assert requests["labels"]["definition"] == "request-v4"
+    assert failures["labels"]["definition"] == "request-v4"
+
+
+def test_availability_backfill_replays_the_same_request_sli() -> None:
+    """Backfill retained history with the exact rules used for future samples."""
+    manifests = _documents(
+        REPO_ROOT / "services/monitoring/availability-backfill-v4-job.yaml"
+    )
+    config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
+    job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
+    backfill = yaml.safe_load(config["data"]["atlas-request-history.yaml"])
+    expressions = {
+        rule["record"]: rule["expr"]
+        for group in backfill["groups"]
+        for rule in group["rules"]
+    }
+
+    assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1h"]
+    assert 'code=~"5.."' in expressions["atlas:availability:failures_1h"]
+    args = job["spec"]["template"]["spec"]["containers"][0]["args"]
+    assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args
+    assert "-replay.timeTo=2026-08-04T23:00:00Z" in args
+    assert "-replay.maxDatapointsPerQuery=48" in args
+
+
+def test_daily_availability_backfill_stays_below_query_timeout() -> None:
+    """Replay daily buckets in small chunks so raw history never starves Grafana."""
+    manifests = _documents(
+        REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml"
+    )
+    config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
+    job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
+    backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"])
+    group = backfill["groups"][0]
+    expressions = {rule["record"]: rule["expr"] for rule in group["rules"]}
+
+    assert group["interval"] == "1d"
+    assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"]
+    assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"]
+    args = job["spec"]["template"]["spec"]["containers"][0]["args"]
+    assert "-replay.maxDatapointsPerQuery=4" in args
 
 
 def test_quality_rollups_do_not_run_every_minute() -> None:

scripts/tests/test_nvidia_process_exporter.py

@@ -0,0 +1,99 @@
+import importlib.util
+from pathlib import Path
+import sys
+import types
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+EXPORTER_PATH = ROOT / "services" / "monitoring" / "scripts" / "nvidia_process_exporter.py"
+
+
+def load_exporter(monkeypatch):
+    """Load the exporter without requiring an NVIDIA driver on the test host."""
+
+    pynvml = types.ModuleType("pynvml")
+
+    class NVMLError(Exception):
+        pass
+
+    class NVMLErrorNotFound(NVMLError):
+        pass
+
+    class NVMLErrorNotSupported(NVMLError):
+        pass
+
+    pynvml.NVMLError = NVMLError
+    pynvml.NVMLError_NotFound = NVMLErrorNotFound
+    pynvml.NVMLError_NotSupported = NVMLErrorNotSupported
+    for name in (
+        "nvmlDeviceGetComputeRunningProcesses_v3",
+        "nvmlDeviceGetCount",
+        "nvmlDeviceGetGraphicsRunningProcesses_v3",
+        "nvmlDeviceGetHandleByIndex",
+        "nvmlDeviceGetName",
+        "nvmlDeviceGetProcessUtilization",
+        "nvmlDeviceGetUUID",
+        "nvmlDeviceGetUtilizationRates",
+        "nvmlInit",
+    ):
+        setattr(pynvml, name, lambda *args, **kwargs: None)
+
+    monkeypatch.setitem(sys.modules, "pynvml", pynvml)
+    spec = importlib.util.spec_from_file_location("nvidia_process_exporter_test", EXPORTER_PATH)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_process_sample_window_uses_nvml_microseconds(monkeypatch):
+    exporter = load_exporter(monkeypatch)
+    observed = []
+    monkeypatch.setattr(exporter.time, "time", lambda: 1_700_000_000.0)
+    monkeypatch.setattr(
+        exporter,
+        "nvmlDeviceGetProcessUtilization",
+        lambda handle, since: observed.append(since) or [],
+    )
+
+    samples, supported = exporter.process_utilization_samples(object())
+
+    assert samples == {}
+    assert supported == 1
+    assert observed == [1_700_000_000_000_000 - 30_000_000]
+
+
+def test_namespace_attribution_scales_to_current_device_total(monkeypatch):
+    exporter = load_exporter(monkeypatch)
+
+    result = exporter.reconcile_namespace_utilization(
+        {"game-stream": 20, "hermes": 10},
+        device_util=3,
+    )
+
+    assert sum(result.values()) == pytest.approx(3)
+    assert result["game-stream"] == pytest.approx(2)
+    assert result["hermes"] == pytest.approx(1)
+
+
+def test_namespace_attribution_assigns_unexplained_compute_to_host(monkeypatch):
+    exporter = load_exporter(monkeypatch)
+
+    result = exporter.reconcile_namespace_utilization(
+        {"hermes": 1},
+        device_util=3,
+    )
+
+    assert result == {"hermes": 1, "host": 2}
+
+
+def test_zero_device_utilization_clears_stale_process_samples(monkeypatch):
+    exporter = load_exporter(monkeypatch)
+
+    result = exporter.reconcile_namespace_utilization(
+        {"hermes": 40, "game-stream": 5},
+        device_util=0,
+    )
+
+    assert result == {"hermes": 0, "game-stream": 0}

services/bstein-dev-home/backend-deployment.yaml

@@ -68,7 +68,8 @@ spec:
           args:
             - >-
               . /vault/secrets/portal-env.sh
-              && exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 600 app:app
+              && export SMTP_HOST=""
+              && exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 1200 app:app
           env:
             - name: AI_CHAT_API
               value: http://ollama.ai.svc.cluster.local:11434
@@ -119,6 +120,8 @@ spec:
               value: http://ariadne.maintenance.svc.cluster.local
             - name: ARIADNE_TIMEOUT_SEC
               value: "10"
+            - name: ARIADNE_GAME_MODE_TIMEOUT_SEC
+              value: "900"
             - name: ACCOUNT_ALLOWED_GROUPS
               value: ""
             - name: HTTP_CHECK_TIMEOUT_SEC
@@ -139,6 +142,8 @@ spec:
               value: "10"
             - name: PORTAL_RUN_MIGRATIONS
               value: "false"
+            - name: ACCESS_REQUEST_ENABLED
+              value: "false"
             - name: ACCESS_REQUEST_SUBMIT_RATE_LIMIT
               value: "30"
             - name: ACCESS_REQUEST_SUBMIT_RATE_WINDOW_SEC

services/bstein-dev-home/kustomization.yaml

@@ -20,9 +20,9 @@ resources:
   - ingress.yaml
 images:
   - name: registry.bstein.dev/bstein/bstein-dev-home-frontend
-    newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
+    newTag: 0.1.1-441 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
   - name: registry.bstein.dev/bstein/bstein-dev-home-backend
-    newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
+    newTag: 0.1.1-441 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
 configMapGenerator:
   - name: chat-ai-gateway
     namespace: bstein-dev-home

services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml

@@ -0,0 +1,397 @@
+# services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml
+# One-off job for sso/cassandra-ldap-federation-3.
+# Purpose: attach the Cassandra Keycloak realm to the shared OpenLDAP directory.
+# Run unsuspended for migration, then suspend/replace with a new job name for future runs.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cassandra-ldap-federation-3
+  namespace: sso
+spec:
+  ttlSecondsAfterFinished: 3600
+  suspend: true
+  backoffLimit: 2
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/role: "sso"
+        vault.hashicorp.com/agent-inject-secret-keycloak-env.sh: "kv/data/atlas/shared/keycloak-admin"
+        vault.hashicorp.com/agent-inject-template-keycloak-env.sh: |
+          {{ with secret "kv/data/atlas/shared/keycloak-admin" }}
+          export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
+          export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
+          export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
+          {{ end }}
+          {{ with secret "kv/data/atlas/sso/openldap-admin" }}
+          export LDAP_ADMIN_PASSWORD="{{ .Data.data.LDAP_ADMIN_PASSWORD }}"
+          export LDAP_CONFIG_PASSWORD="{{ .Data.data.LDAP_CONFIG_PASSWORD }}"
+          export LDAP_BIND_PASSWORD="${LDAP_ADMIN_PASSWORD}"
+          {{ end }}
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: ["rpi5","rpi4"]
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+      restartPolicy: OnFailure
+      serviceAccountName: sso-vault
+      containers:
+        - name: configure
+          image: python:3.11-alpine
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: KEYCLOAK_SERVER
+              value: http://keycloak.sso.svc.cluster.local
+            - name: KEYCLOAK_REALM
+              value: cassandra
+            - name: LDAP_URL
+              value: ldap://openldap.sso.svc.cluster.local:389
+            - name: LDAP_BIND_DN
+              value: cn=admin,dc=bstein,dc=dev
+            - name: LDAP_USERS_DN
+              value: ou=users,dc=bstein,dc=dev
+            - name: LDAP_GROUPS_DN
+              value: ou=groups,dc=bstein,dc=dev
+            - name: VERIFY_USERS
+              value: veles-dev,cassandra-dev,daniel-test,viktor-test
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              . /vault/secrets/keycloak-env.sh
+              python - <<'PY'
+              import json
+              import os
+              import time
+              import urllib.error
+              import urllib.parse
+              import urllib.request
+
+              base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/")
+              realm = os.environ["KEYCLOAK_REALM"]
+              admin_user = os.environ["KEYCLOAK_ADMIN_USER"]
+              admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"]
+              ldap_url = os.environ["LDAP_URL"]
+              ldap_bind_dn = os.environ["LDAP_BIND_DN"]
+              ldap_bind_password = os.environ["LDAP_BIND_PASSWORD"]
+              ldap_users_dn = os.environ["LDAP_USERS_DN"]
+              ldap_groups_dn = os.environ["LDAP_GROUPS_DN"]
+
+              def request(method, url, token=None, payload=None, timeout=30):
+                  data = None
+                  headers = {}
+                  if token:
+                      headers["Authorization"] = f"Bearer {token}"
+                  if payload is not None:
+                      data = json.dumps(payload).encode()
+                      headers["Content-Type"] = "application/json"
+                  req = urllib.request.Request(url, data=data, headers=headers, method=method)
+                  try:
+                      with urllib.request.urlopen(req, timeout=timeout) as resp:
+                          body = resp.read()
+                          if not body:
+                              return resp.status, None, dict(resp.headers)
+                          return resp.status, json.loads(body.decode()), dict(resp.headers)
+                  except urllib.error.HTTPError as exc:
+                      raw = exc.read()
+                      body = None
+                      if raw:
+                          try:
+                              body = json.loads(raw.decode())
+                          except Exception:
+                              body = {"raw": raw.decode(errors="replace")}
+                      return exc.code, body, dict(exc.headers)
+
+              def get_token():
+                  data = urllib.parse.urlencode(
+                      {
+                          "grant_type": "password",
+                          "client_id": "admin-cli",
+                          "username": admin_user,
+                          "password": admin_password,
+                      }
+                  ).encode()
+                  req = urllib.request.Request(
+                      f"{base_url}/realms/master/protocol/openid-connect/token",
+                      data=data,
+                      headers={"Content-Type": "application/x-www-form-urlencoded"},
+                      method="POST",
+                  )
+                  with urllib.request.urlopen(req, timeout=30) as resp:
+                      return json.loads(resp.read().decode())["access_token"]
+
+              token = None
+              for attempt in range(1, 31):
+                  try:
+                      token = get_token()
+                      break
+                  except Exception as exc:
+                      if attempt == 30:
+                          raise SystemExit(f"Keycloak token request failed: {exc}")
+                      time.sleep(2)
+              assert token
+
+              status, realm_rep, _ = request("GET", f"{base_url}/admin/realms/{realm}", token)
+              if status != 200 or not isinstance(realm_rep, dict) or not realm_rep.get("id"):
+                  raise SystemExit(f"Unable to resolve realm id for {realm}: status={status}")
+              realm_id = realm_rep["id"]
+
+              status, components, _ = request(
+                  "GET",
+                  f"{base_url}/admin/realms/{realm}/components",
+                  token,
+              )
+              if status != 200:
+                  raise SystemExit(f"Unable to list realm components: status={status}")
+              components = components or []
+
+              for component in components:
+                  if component.get("providerId") != "ldap":
+                      continue
+                  if component.get("providerType") != "org.keycloak.storage.UserStorageProvider":
+                      continue
+                  if component.get("parentId") == realm_id:
+                      continue
+                  cid = component.get("id")
+                  if not cid:
+                      continue
+                  status, payload, _ = request(
+                      "GET",
+                      f"{base_url}/admin/realms/{realm}/components/{cid}",
+                      token,
+                  )
+                  if status != 200 or not isinstance(payload, dict):
+                      raise SystemExit(f"Unable to fetch LDAP component {cid}: status={status}")
+                  payload["parentId"] = realm_id
+                  status, _, _ = request(
+                      "PUT",
+                      f"{base_url}/admin/realms/{realm}/components/{cid}",
+                      token,
+                      payload,
+                  )
+                  if status not in (200, 204):
+                      raise SystemExit(f"Unable to repair LDAP component {cid}: status={status}")
+
+              status, storage_components, _ = request(
+                  "GET",
+                  f"{base_url}/admin/realms/{realm}/components?type=org.keycloak.storage.UserStorageProvider",
+                  token,
+              )
+              if status != 200:
+                  raise SystemExit(f"Unable to list user-storage components: status={status}")
+              storage_components = storage_components or []
+
+              ldap_components = [
+                  c
+                  for c in storage_components
+                  if c.get("providerId") == "ldap" and c.get("id")
+              ]
+              candidates = []
+              for component in ldap_components:
+                  config = component.get("config") or {}
+                  if component.get("name") in ("openldap", "ldap") and (config.get("connectionUrl") or [None])[0] == ldap_url:
+                      candidates.append(component)
+              if not candidates:
+                  candidates = [c for c in ldap_components if c.get("name") in ("openldap", "ldap")]
+              candidates.sort(key=lambda item: item.get("id", ""))
+              ldap_component = candidates[0] if candidates else None
+              ldap_component_id = ldap_component.get("id") if ldap_component else None
+
+              desired = {
+                  "name": "openldap",
+                  "providerId": "ldap",
+                  "providerType": "org.keycloak.storage.UserStorageProvider",
+                  "parentId": realm_id,
+                  "config": {
+                      "enabled": ["true"],
+                      "priority": ["0"],
+                      "importEnabled": ["true"],
+                      "editMode": ["WRITABLE"],
+                      "syncRegistrations": ["true"],
+                      "vendor": ["other"],
+                      "connectionUrl": [ldap_url],
+                      "bindDn": [ldap_bind_dn],
+                      "bindCredential": [ldap_bind_password],
+                      "authType": ["simple"],
+                      "usersDn": [ldap_users_dn],
+                      "searchScope": ["1"],
+                      "pagination": ["true"],
+                      "usernameLDAPAttribute": ["uid"],
+                      "rdnLDAPAttribute": ["uid"],
+                      "uuidLDAPAttribute": ["entryUUID"],
+                      "userObjectClasses": ["inetOrgPerson, organizationalPerson, person, top"],
+                      "trustEmail": ["true"],
+                      "useTruststoreSpi": ["never"],
+                      "connectionPooling": ["true"],
+                      "cachePolicy": ["DEFAULT"],
+                      "useKerberosForPasswordAuthentication": ["false"],
+                      "allowKerberosAuthentication": ["false"],
+                  },
+              }
+
+              if ldap_component_id:
+                  desired["id"] = ldap_component_id
+                  print(f"Updating Cassandra LDAP federation provider: {ldap_component_id}")
+                  status, _, _ = request(
+                      "PUT",
+                      f"{base_url}/admin/realms/{realm}/components/{ldap_component_id}",
+                      token,
+                      desired,
+                  )
+                  if status not in (200, 204):
+                      raise SystemExit(f"LDAP provider update failed: status={status}")
+              else:
+                  print("Creating Cassandra LDAP federation provider")
+                  status, _, headers = request(
+                      "POST",
+                      f"{base_url}/admin/realms/{realm}/components",
+                      token,
+                      desired,
+                  )
+                  if status not in (201, 204):
+                      raise SystemExit(f"LDAP provider create failed: status={status}")
+                  location = headers.get("Location", "")
+                  ldap_component_id = location.rstrip("/").split("/")[-1] if location else None
+
+              if not ldap_component_id:
+                  raise SystemExit("Unable to determine Cassandra LDAP provider id")
+
+              status, mapper_components, _ = request(
+                  "GET",
+                  f"{base_url}/admin/realms/{realm}/components?type=org.keycloak.storage.ldap.mappers.LDAPStorageMapper",
+                  token,
+              )
+              if status != 200:
+                  raise SystemExit(f"Unable to list LDAP mappers: status={status}")
+              mapper_components = mapper_components or []
+
+              def upsert_mapper(name, provider_id, config):
+                  existing = next(
+                      (
+                          c
+                          for c in mapper_components
+                          if c.get("name") == name and c.get("parentId") == ldap_component_id
+                      ),
+                      None,
+                  )
+                  payload = {
+                      "name": name,
+                      "providerId": provider_id,
+                      "providerType": "org.keycloak.storage.ldap.mappers.LDAPStorageMapper",
+                      "parentId": ldap_component_id,
+                      "config": config,
+                  }
+                  if existing:
+                      payload["id"] = existing["id"]
+                      status, _, _ = request(
+                          "PUT",
+                          f"{base_url}/admin/realms/{realm}/components/{existing['id']}",
+                          token,
+                          payload,
+                      )
+                      action = "Updated"
+                  else:
+                      status, _, _ = request(
+                          "POST",
+                          f"{base_url}/admin/realms/{realm}/components",
+                          token,
+                          payload,
+                      )
+                      action = "Created"
+                  if status not in (200, 201, 204):
+                      raise SystemExit(f"Mapper {name} upsert failed: status={status}")
+                  print(f"{action} Cassandra LDAP mapper: {name}")
+
+              upsert_mapper(
+                  "openldap-groups",
+                  "group-ldap-mapper",
+                  {
+                      "groups.dn": [ldap_groups_dn],
+                      "group.name.ldap.attribute": ["cn"],
+                      "group.object.classes": ["groupOfNames"],
+                      "membership.ldap.attribute": ["member"],
+                      "membership.attribute.type": ["DN"],
+                      "mode": ["LDAP_ONLY"],
+                      "user.roles.retrieve.strategy": ["LOAD_GROUPS_BY_MEMBER_ATTRIBUTE"],
+                      "preserve.group.inheritance": ["true"],
+                  },
+              )
+              for name, ldap_attr, user_attr in (
+                  ("openldap-email", "mail", "email"),
+                  ("openldap-first-name", "givenName", "firstName"),
+                  ("openldap-last-name", "sn", "lastName"),
+              ):
+                  upsert_mapper(
+                      name,
+                      "user-attribute-ldap-mapper",
+                      {
+                          "ldap.attribute": [ldap_attr],
+                          "user.model.attribute": [user_attr],
+                          "read.only": ["false"],
+                          "always.read.value.from.ldap": ["false"],
+                          "is.mandatory.in.ldap": ["false"],
+                      },
+                  )
+
+              status, sync_body, _ = request(
+                  "POST",
+                  f"{base_url}/admin/realms/{realm}/user-storage/{ldap_component_id}/sync?action=triggerFullSync",
+                  token,
+                  timeout=120,
+              )
+              if status in (200, 201, 204):
+                  print(f"Cassandra LDAP full sync requested: {sync_body}")
+              else:
+                  print(f"WARNING: Cassandra LDAP full sync returned status={status} body={sync_body}")
+
+              for username in (item.strip() for item in os.environ.get("VERIFY_USERS", "").split(",")):
+                  if not username:
+                      continue
+                  encoded = urllib.parse.quote(username)
+                  status, users, _ = request(
+                      "GET",
+                      f"{base_url}/admin/realms/{realm}/users?username={encoded}&exact=true",
+                      token,
+                  )
+                  found = status == 200 and any(
+                      str(user.get("username") or "").casefold() == username.casefold()
+                      for user in users or []
+                  )
+                  print(f"Cassandra LDAP user check {username}: {'found' if found else 'missing'}")
+
+              for audit_realm in ("veles", realm):
+                  status, count, _ = request(
+                      "GET",
+                      f"{base_url}/admin/realms/{audit_realm}/users/count",
+                      token,
+                  )
+                  if status == 200:
+                      print(f"SSO audit realm {audit_realm}: user_count={count}")
+                  for term in (item.strip() for item in os.environ.get("VERIFY_USERS", "").split(",")):
+                      if not term:
+                          continue
+                      encoded = urllib.parse.quote(term)
+                      status, users, _ = request(
+                          "GET",
+                          f"{base_url}/admin/realms/{audit_realm}/users?search={encoded}",
+                          token,
+                      )
+                      users = users if status == 200 and isinstance(users, list) else []
+                      matches = []
+                      for user in users:
+                          username = str(user.get("username") or "")
+                          email = str(user.get("email") or "")
+                          if term.casefold() in username.casefold() or term.casefold() in email.casefold():
+                              matches.append(username if not email else f"{username}<{email}>")
+                      joined = ", ".join(matches[:8]) if matches else "none"
+                      print(f"SSO audit realm {audit_realm} search {term}: {joined}")
+
+              print("Cassandra LDAP federation ready")
+              PY

services/cassandra-auth/bootstrap-jobs/cassandra-realm-ensure-job.yaml

@@ -1,11 +1,11 @@
 # services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml
-# One-off job for sso/cassandra-realm-ensure-1.
+# One-off job for sso/cassandra-realm-ensure-4.
 # Purpose: create the Cassandra realm, groups, OIDC client, SMTP settings, and Vault client secret.
 # Keep suspended until Cassandra Vault paths/policies have reconciled, then unsuspend once.
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: cassandra-realm-ensure-1
+  name: cassandra-realm-ensure-4
   namespace: sso
 spec:
   suspend: true
@@ -70,7 +70,7 @@ spec:
             - name: KEYCLOAK_SMTP_FROM_NAME
               value: Cassandra
             - name: CASSANDRA_PUBLIC_REPLAY_USERS
-              value: cassandra-dev,daniel-test,viktor-test
+              value: veles-dev,daniel-test,viktor-test
           command: ["/bin/sh", "-c"]
           args:
             - |
@@ -260,15 +260,35 @@ spec:
                   if status not in (200, 204):
                       raise SystemExit(f"User group update failed for {username}: status={status} body={body}")
 
-              ensure_group("alpha")
+              alpha_group = ensure_group("alpha")
               ensure_group("admin")
-              ensure_group(os.environ["CASSANDRA_TESTER_GROUP"])
+              tester_group = ensure_group(os.environ["CASSANDRA_TESTER_GROUP"])
               public_replay_group = ensure_group("public-replay")
+              pilot_groups = {
+                  "pilot-skill-proficient": ensure_group("pilot-skill-proficient"),
+                  "pilot-skill-adept": ensure_group("pilot-skill-adept"),
+                  "pilot-skill-veteran": ensure_group("pilot-skill-veteran"),
+                  "pilot-skill-expert": ensure_group("pilot-skill-expert"),
+                  "pilot-skill-master": ensure_group("pilot-skill-master"),
+                  "all-pilot-skill-level": ensure_group("all-pilot-skill-level"),
+                  "codex-cli-fallback": ensure_group("codex-cli-fallback"),
+              }
               for username in os.environ.get("CASSANDRA_PUBLIC_REPLAY_USERS", "").split(","):
                   username = username.strip()
                   if username:
                       ensure_user_group(username, public_replay_group)
 
+              ensure_user_group("veles-dev", tester_group)
+              ensure_user_group("veles-dev", pilot_groups["all-pilot-skill-level"])
+              ensure_user_group("veles-dev", pilot_groups["codex-cli-fallback"])
+              for username in ("daniel-test", "viktor-test"):
+                  ensure_user_group(username, alpha_group)
+                  ensure_user_group(username, tester_group)
+                  ensure_user_group(username, pilot_groups["pilot-skill-proficient"])
+                  ensure_user_group(username, pilot_groups["pilot-skill-adept"])
+                  ensure_user_group(username, pilot_groups["pilot-skill-veteran"])
+                  ensure_user_group(username, pilot_groups["codex-cli-fallback"])
+
               status, clients = request(
                   "GET",
                   f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}",

services/cassandra-auth/kustomization.yaml

@@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: sso
 resources:
+  - bootstrap-jobs/cassandra-ldap-federation-job.yaml
   - bootstrap-jobs/cassandra-realm-ensure-job.yaml
   - bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
 configMapGenerator:

services/cassandra/backend-deployment.yaml

@@ -22,7 +22,7 @@ spec:
       labels:
         app: cassandra-backend
       annotations:
-        cassandra.bstein.dev/redeploy: "2026-07-28-0.8.51-jobs-compact"
+        cassandra.bstein.dev/redeploy: "2026-08-04-0.8.150-lane-semantics"
         vault.hashicorp.com/agent-inject: "true"
         vault.hashicorp.com/agent-init-first: "true"
         vault.hashicorp.com/agent-pre-populate-only: "true"
@@ -66,7 +66,7 @@ spec:
           type: RuntimeDefault
       initContainers:
         - name: database-init
-          image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"}
+          image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"}
           imagePullPolicy: Always
           command: ["/bin/sh", "-c"]
           args:
@@ -88,7 +88,7 @@ spec:
               memory: 512Mi
       containers:
         - name: backend
-          image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"}
+          image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"}
           imagePullPolicy: Always
           command: ["/bin/sh", "-c"]
           args:
@@ -102,6 +102,11 @@ spec:
             - name: http
               containerPort: 8796
               protocol: TCP
+          env:
+            - name: CASSANDRA_SIM_IMAGE
+              value: registry.bstein.dev/cassandra/cassandra-sim-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
+            - name: CASSANDRA_GENERATOR_WORKER_IMAGE
+              value: registry.bstein.dev/cassandra/cassandra-generator-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
           envFrom:
             - configMapRef:
                 name: cassandra-config

services/cassandra/bootstrap-jobs/cassandra-secrets-ensure-job.yaml

@@ -5,7 +5,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: cassandra-secrets-ensure-2
+  name: cassandra-secrets-ensure-3
   namespace: cassandra
 spec:
   suspend: true

services/cassandra/configmap.yaml

@@ -18,20 +18,34 @@ data:
   CASSANDRA_OIDC_REQUIRED_GROUPS: alpha,cassandra-tester,admin
   CASSANDRA_OIDC_ADMIN_GROUPS: admin
   CASSANDRA_NAMESPACE: cassandra
-  CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.8.51 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
+  CASSANDRA_OWNER_USER_ID_ALIASES: veles-dev=d357688e-e3e4-4f18-9491-a4ed5622deb6;a21a286c-6c82-4180-9243-7c916442ecf0=d357688e-e3e4-4f18-9491-a4ed5622deb6
-  CASSANDRA_SIM_WORKER_VERSION: "0.8.51"
+  CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
-  CASSANDRA_GENERATOR_WORKER_VERSION: "0.8.51"
+  CASSANDRA_GENERATOR_WORKER_IMAGE: registry.bstein.dev/cassandra/cassandra-generator-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
+  CASSANDRA_SIM_WORKER_VERSION: "0.8.150"
+  CASSANDRA_GENERATOR_WORKER_VERSION: "0.8.150"
+  CASSANDRA_STRATEGY_SPEC_HINT_INTERPRETATION_TIMEOUT: "420"
   CASSANDRA_SIM_SERVICE_ACCOUNT: cassandra-sim
+  CASSANDRA_GENERATOR_WORKER_SERVICE_ACCOUNT: cassandra-generator
   CASSANDRA_SIM_PRIORITY_CLASS: cassandra-sim
+  CASSANDRA_GENERATOR_WORKER_PRIORITY_CLASS: cassandra-core
   CASSANDRA_SIM_NODE_SELECTOR: cassandra.bstein.dev/node-pool=oceanus,kubernetes.io/arch=amd64
+  CASSANDRA_GENERATOR_WORKER_NODE_SELECTOR: cassandra.bstein.dev/node-pool=oceanus,kubernetes.io/arch=amd64
   CASSANDRA_SIM_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule
+  CASSANDRA_GENERATOR_WORKER_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule
   CASSANDRA_SIM_ACTIVE_DEADLINE_SECONDS: "7200"
+  CASSANDRA_GENERATOR_WORKER_ACTIVE_DEADLINE_SECONDS: "14400"
   CASSANDRA_SIM_TTL_SECONDS: "3600"
+  CASSANDRA_GENERATOR_WORKER_TTL_SECONDS: "600"
   CASSANDRA_SIM_CPU_REQUEST: 500m
   CASSANDRA_SIM_CPU_LIMIT: "2"
   CASSANDRA_SIM_MEMORY_REQUEST: 1Gi
   CASSANDRA_SIM_MEMORY_LIMIT: 4Gi
+  CASSANDRA_GENERATOR_WORKER_CPU_REQUEST: 100m
+  CASSANDRA_GENERATOR_WORKER_CPU_LIMIT: "1"
+  CASSANDRA_GENERATOR_WORKER_MEMORY_REQUEST: 512Mi
+  CASSANDRA_GENERATOR_WORKER_MEMORY_LIMIT: 2Gi
   CASSANDRA_SIM_FS_GROUP: "10001"
+  CASSANDRA_GENERATOR_WORKER_FS_GROUP: "10001"
   CASSANDRA_MAX_ACTIVE_SIMS_PER_USER: "32"
   CASSANDRA_MAX_ACTIVE_SIMS_GLOBAL: "32"
   CASSANDRA_RETENTION_DAYS: "30"
@@ -43,3 +57,6 @@ data:
   CASSANDRA_SIM_CODEX_CLI_SECRET_PROVIDER_CLASS: cassandra-codex-cli
   CASSANDRA_SIM_CODEX_CLI_MOUNT_PATH: /var/run/cassandra-codex-cli
   CASSANDRA_SIM_CODEX_CLI_HOME: /home/cassandra/.codex
+  CASSANDRA_GENERATOR_WORKER_CODEX_CLI_SECRET_PROVIDER_CLASS: cassandra-generator-codex-cli
+  CASSANDRA_GENERATOR_WORKER_CODEX_CLI_MOUNT_PATH: /var/run/cassandra-codex-cli
+  CASSANDRA_GENERATOR_WORKER_CODEX_CLI_HOME: /home/cassandra/.codex

services/cassandra/frontend-deployment.yaml

@@ -48,7 +48,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: frontend
-          image: registry.bstein.dev/cassandra/cassandra-frontend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-frontend"}
+          image: registry.bstein.dev/cassandra/cassandra-frontend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-frontend"}
           imagePullPolicy: Always
           ports:
             - name: http

services/cassandra/kustomization.yaml

@@ -16,7 +16,6 @@ resources:
   - postgres-statefulset.yaml
   - services.yaml
   - backend-deployment.yaml
-  - generator-deployment.yaml
   - frontend-deployment.yaml
   - retention-cronjob.yaml
   - image.yaml
@@ -24,3 +23,4 @@ resources:
   - bootstrap-jobs/cassandra-secrets-ensure-job.yaml
   - migration-jobs/db-copy-veles-to-cassandra-job.yaml
   - migration-jobs/artifact-copy-veles-to-cassandra-job.yaml
+  - migration-jobs/artifact-permissions-job.yaml

services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml

@@ -3,7 +3,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: cassandra-artifact-copy-from-veles-1
+  name: cassandra-artifact-copy-from-veles-2
   namespace: cassandra
 spec:
   suspend: true
@@ -33,6 +33,8 @@ spec:
               value: /dest
             - name: ALLOW_NONEMPTY_DEST
               value: "0"
+            - name: RESET_DESTINATION
+              value: "1"
           command: ["python", "-c"]
           args:
             - |
@@ -42,12 +44,15 @@ spec:
               import hashlib
               import json
               import os
+              import shutil
               import tarfile
+              import time
               import urllib.request
 
               SOURCE_URL = os.environ["SOURCE_URL"].rstrip("/")
               DEST = Path(os.environ.get("DEST_ROOT", "/dest")).resolve()
               ALLOW_NONEMPTY = os.environ.get("ALLOW_NONEMPTY_DEST", "") in {"1", "true", "yes"}
+              RESET_DESTINATION = os.environ.get("RESET_DESTINATION", "") in {"1", "true", "yes"}
               EXCLUDE_TOP = {"lost+found", ".cassandra-artifact-migration"}
               REPORT_DIR = DEST / ".cassandra-artifact-migration"
 
@@ -57,6 +62,13 @@ spec:
               def assert_safe_destination() -> None:
                   DEST.mkdir(parents=True, exist_ok=True)
                   entries = visible_entries()
+                  if entries and RESET_DESTINATION:
+                      for entry in entries:
+                          if entry.is_dir():
+                              shutil.rmtree(entry)
+                          else:
+                              entry.unlink()
+                      entries = visible_entries()
                   if entries and not ALLOW_NONEMPTY:
                       names = ", ".join(entry.name for entry in entries[:10])
                       raise SystemExit(f"destination is not empty; refusing to merge into existing data: {names}")
@@ -66,6 +78,16 @@ spec:
                   with urllib.request.urlopen(f"{SOURCE_URL}/manifest.json", timeout=120) as response:
                       return json.loads(response.read().decode())
 
+              def wait_for_manifest() -> dict[str, object]:
+                  last_error = ""
+                  for _attempt in range(90):
+                      try:
+                          return fetch_manifest()
+                      except Exception as exc:
+                          last_error = str(exc)
+                          time.sleep(5)
+                  raise SystemExit(f"source artifact exporter did not become ready: {last_error}")
+
               def safe_extract() -> None:
                   root_text = str(DEST)
                   with urllib.request.urlopen(f"{SOURCE_URL}/archive.tar", timeout=600) as response:
@@ -116,7 +138,7 @@ spec:
                   }
 
               assert_safe_destination()
-              source = fetch_manifest()
+              source = wait_for_manifest()
               (REPORT_DIR / "source_manifest.json").write_text(json.dumps(source, indent=2, sort_keys=True))
               safe_extract()
               dest = destination_manifest()

services/cassandra/migration-jobs/artifact-permissions-job.yaml

@@ -0,0 +1,42 @@
+# services/cassandra/migration-jobs/artifact-permissions-job.yaml
+# One-off migration repair after copying artifacts from the Veles PVC.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cassandra-artifact-permissions-2
+  namespace: cassandra
+spec:
+  suspend: true
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 86400
+  template:
+    spec:
+      serviceAccountName: cassandra-artifact-migration
+      automountServiceAccountToken: false
+      restartPolicy: Never
+      nodeSelector:
+        cassandra.bstein.dev/node-pool: oceanus
+      tolerations:
+        - key: veles.bstein.dev/simulation
+          operator: Equal
+          value: "true"
+          effect: NoSchedule
+      containers:
+        - name: permissions
+          image: alpine:3.20
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -eu
+              chown -R 10001:10001 /data/cassandra-artifacts
+              find /data/cassandra-artifacts -type d -exec chmod 2775 {} +
+              find /data/cassandra-artifacts -type f -exec chmod 0664 {} +
+              printf 'artifact permission repair complete\n'
+          volumeMounts:
+            - name: artifacts
+              mountPath: /data/cassandra-artifacts
+      volumes:
+        - name: artifacts
+          persistentVolumeClaim:
+            claimName: cassandra-artifacts

services/cassandra/migration-jobs/db-copy-veles-to-cassandra-job.yaml

@@ -5,7 +5,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: cassandra-db-copy-from-veles-1
+  name: cassandra-db-copy-from-veles-3
   namespace: cassandra
 spec:
   suspend: true
@@ -68,7 +68,9 @@ spec:
               pg_dump --format=custom --no-owner --no-acl \
                 --dbname="${SOURCE_DATABASE_URL}" \
                 --file="${dump_path}"
-              pg_restore --clean --if-exists --no-owner --no-acl \
+              psql -X --set=ON_ERROR_STOP=1 --dbname="${TARGET_DATABASE_URL}" \
+                --command="DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO cassandra; GRANT ALL ON SCHEMA public TO public;"
+              pg_restore --no-owner --no-acl \
                 --dbname="${TARGET_DATABASE_URL}" \
                 "${dump_path}"
 

services/cassandra/retention-cronjob.yaml

@@ -38,7 +38,7 @@ spec:
               type: RuntimeDefault
           containers:
             - name: retention
-              image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"}
+              image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"}
               imagePullPolicy: IfNotPresent
               command: ["cassandra-hygiene"]
               args: ["--prune-artifacts"]

services/cassandra/secretproviderclass.yaml

@@ -73,3 +73,19 @@ spec:
         secretPath: "kv/data/atlas/cassandra/codex-cli"
         secretKey: "auth.json"
         filePermission: 292
+---
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: cassandra-generator-codex-cli
+  namespace: cassandra
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.vault.svc.cluster.local:8200"
+    roleName: "cassandra"
+    objects: |
+      - objectName: "auth.json"
+        secretPath: "kv/data/atlas/cassandra/codex-cli"
+        secretKey: "auth.json"
+        filePermission: 292

services/harbor/bootstrap-jobs/cassandra-registry-ensure-job.yaml

@@ -0,0 +1,138 @@
+# services/harbor/bootstrap-jobs/cassandra-registry-ensure-job.yaml
+# One-off repair/audit job for the Cassandra Harbor migration.
+# Keep suspended unless the Harbor project or robot scopes need to be recreated.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cassandra-registry-ensure-1
+  namespace: harbor
+spec:
+  suspend: true
+  backoffLimit: 1
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/role: "harbor"
+        vault.hashicorp.com/agent-inject-secret-harbor-admin.sh: "kv/data/atlas/harbor/harbor-core"
+        vault.hashicorp.com/agent-inject-template-harbor-admin.sh: |
+          {{- with secret "kv/data/atlas/harbor/harbor-core" }}
+          export HARBOR_ADMIN_PASSWORD="{{ .Data.data.harbor_admin_password }}"
+          {{- end }}
+    spec:
+      restartPolicy: Never
+      serviceAccountName: harbor-vault-sync
+      containers:
+        - name: ensure
+          image: python:3.11-alpine
+          command: ["python", "-c"]
+          args:
+            - |
+              import base64
+              import json
+              import os
+              import urllib.error
+              import urllib.request
+
+              secret_path = "/vault/secrets/harbor-admin.sh"
+              env = {}
+              with open(secret_path, "r", encoding="utf-8") as handle:
+                  for line in handle:
+                      line = line.strip()
+                      if not line.startswith("export "):
+                          continue
+                      key, value = line[len("export ") :].split("=", 1)
+                      env[key] = value.strip('"')
+
+              password = env["HARBOR_ADMIN_PASSWORD"]
+              api = os.environ.get(
+                  "HARBOR_API",
+                  "http://harbor-core.harbor.svc.cluster.local/api/v2.0",
+              ).rstrip("/")
+              auth = base64.b64encode(f"admin:{password}".encode("utf-8")).decode("ascii")
+
+              def request(method, path, payload=None, ok=(200,)):
+                  data = None
+                  headers = {"Authorization": f"Basic {auth}"}
+                  if payload is not None:
+                      data = json.dumps(payload).encode("utf-8")
+                      headers["Content-Type"] = "application/json"
+                  req = urllib.request.Request(
+                      f"{api}{path}",
+                      data=data,
+                      method=method,
+                      headers=headers,
+                  )
+                  try:
+                      with urllib.request.urlopen(req, timeout=30) as response:
+                          body = response.read()
+                          if response.status not in ok:
+                              raise RuntimeError(f"{method} {path} returned {response.status}")
+                          if not body:
+                              return None
+                          return json.loads(body.decode("utf-8"))
+                  except urllib.error.HTTPError as exc:
+                      detail = exc.read().decode("utf-8", errors="replace")
+                      if exc.code in ok:
+                          return None
+                      raise RuntimeError(f"{method} {path} returned {exc.code}: {detail}") from exc
+
+              projects = request("GET", "/projects?name=cassandra")
+              if not projects:
+                  request(
+                      "POST",
+                      "/projects",
+                      {
+                          "project_name": "cassandra",
+                          "metadata": {
+                              "public": "false",
+                              "auto_scan": "false",
+                              "enable_content_trust": "false",
+                              "prevent_vul": "false",
+                              "reuse_sys_cve_allowlist": "true",
+                          },
+                      },
+                      ok=(201, 409),
+                  )
+
+              def ensure_permission(robot_name, access, description=None):
+                  robots = request("GET", "/robots?page_size=100")
+                  matches = [robot for robot in robots if robot.get("name") == robot_name]
+                  if not matches:
+                      raise RuntimeError(f"Harbor robot {robot_name} not found")
+                  robot = request("GET", f"/robots/{matches[0]['id']}")
+                  permissions = list(robot.get("permissions") or [])
+                  if not any(
+                      item.get("kind") == "project" and item.get("namespace") == "cassandra"
+                      for item in permissions
+                  ):
+                      permissions.append(
+                          {"kind": "project", "namespace": "cassandra", "access": access}
+                      )
+                  payload = {
+                      "name": robot["name"],
+                      "level": robot["level"],
+                      "disable": bool(robot.get("disable", False)),
+                      "duration": int(robot.get("duration", -1)),
+                      "permissions": permissions,
+                  }
+                  if description:
+                      payload["description"] = description
+                  elif robot.get("description"):
+                      payload["description"] = robot["description"]
+                  request("PUT", f"/robots/{robot['id']}", payload, ok=(200,))
+
+              ensure_permission(
+                  "robot$pull-atlas",
+                  [{"resource": "repository", "action": "pull"}],
+              )
+              ensure_permission(
+                  "robot$jenkins-pipelines",
+                  [
+                      {"resource": "repository", "action": "pull"},
+                      {"resource": "repository", "action": "push"},
+                  ],
+                  "Jenkins image push robot for bstein, veles, and cassandra pipelines",
+              )
+              print("Cassandra Harbor project and robot scopes are present")

services/harbor/kustomization.yaml

@@ -12,6 +12,7 @@ resources:
   - certificate.yaml
   - helmrelease.yaml
   - vault-sync-deployment.yaml
+  - bootstrap-jobs/cassandra-registry-ensure-job.yaml
   - image.yaml
 configMapGenerator:
   - name: harbor-vault-entrypoint

services/harbor/pvc.yaml

@@ -8,7 +8,7 @@ spec:
   accessModes: [ "ReadWriteOnce" ]
   resources:
     requests:
-      storage: 450Gi
+      storage: 600Gi
   storageClassName: astreae
 ---
 apiVersion: v1

services/hermes-chat/certificate.yaml

@@ -0,0 +1,13 @@
+# services/hermes-chat/certificate.yaml
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: chat-tls
+  namespace: hermes-chat
+spec:
+  secretName: chat-tls
+  issuerRef:
+    kind: ClusterIssuer
+    name: letsencrypt
+  dnsNames:
+    - chat.bstein.dev

services/hermes-chat/configmap.yaml

@@ -0,0 +1,153 @@
+# services/hermes-chat/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hermes-chat-config
+  namespace: hermes-chat
+  labels:
+    app: hermes-chat
+data:
+  config.yaml: |
+    model:
+      provider: openai-codex
+      default: gpt-5.6-terra
+      model: gpt-5.6-terra
+
+    fallback_providers:
+      - provider: custom
+        model: gpt-oss:20b
+        base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
+        api_key: ollama
+
+    agent:
+      api_max_retries: 1
+
+    platform_toolsets:
+      cli:
+        - clarify
+        - file
+        - terminal
+        - web
+      api_server:
+        - clarify
+        - file
+        - terminal
+        - web
+
+    skills:
+      creation_nudge_interval: 15
+      external_dirs:
+        - /opt/data/workspace/skills
+
+    terminal:
+      backend: local
+      cwd: /opt/data/workspace
+      timeout: 180
+      home_mode: profile
+
+    approvals:
+      mode: manual
+      deny:
+        - "*kubectl apply*"
+        - "*kubectl create*"
+        - "*kubectl delete*"
+        - "*kubectl edit*"
+        - "*kubectl patch*"
+        - "*kubectl replace*"
+        - "*kubectl scale*"
+        - "*kubectl set*"
+        - "*kubectl label*"
+        - "*kubectl annotate*"
+        - "*kubectl cordon*"
+        - "*kubectl uncordon*"
+        - "*kubectl drain*"
+        - "*kubectl rollout restart*"
+        - "*kubectl rollout undo*"
+        - "*kubectl exec*"
+        - "*kubectl attach*"
+        - "*kubectl cp*"
+        - "*kubectl debug*"
+        - "*kubectl expose*"
+        - "*kubectl port-forward*"
+        - "*kubectl proxy*"
+        - "*kubectl run*"
+        - "*kubectl get secret*"
+        - "*kubectl describe secret*"
+        - "*flux reconcile*"
+        - "*flux suspend*"
+        - "*flux resume*"
+        - "*vault *"
+        - "*k3s *"
+        - "*crictl *"
+        - "*ctr *"
+        - "*169.254.169.254*"
+        - "*kubernetes.default*"
+        - "*/var/run/secrets/kubernetes.io*"
+
+    dashboard:
+      public_url: https://chat.bstein.dev
+      oauth:
+        provider: self-hosted
+        self_hosted:
+          issuer: https://sso.bstein.dev/realms/atlas
+          client_id: hermes-chat-dashboard
+          scopes: openid profile email groups
+
+    display:
+      compact: true
+      tool_progress: all
+      interim_assistant_messages: true
+      long_running_notifications: true
+
+    tool_loop_guardrails:
+      warnings_enabled: true
+      hard_stop_enabled: true
+      warn_after:
+        exact_failure: 2
+        same_tool_failure: 3
+        idempotent_no_progress: 2
+      hard_stop_after:
+        exact_failure: 5
+        same_tool_failure: 8
+        idempotent_no_progress: 5
+
+    updates:
+      pre_update_backup: quick
+      backup_keep: 3
+      non_interactive_local_changes: stash
+  SOUL.md: |
+    You are a personal AI assistant and researcher for Atlas users. Be useful,
+    curious, careful, and direct. Help with research, writing, planning,
+    learning, files, personal automation, and creative work. Use tools when
+    they materially improve the answer and make reusable skills when a stable
+    workflow is worth keeping.
+
+    Your environment is a private consumer sandbox. Work in the mounted
+    workspace and with public Internet services the user intentionally asks
+    you to use. You may inspect the Titan cluster through the dedicated
+    read-only Kubernetes identity to explain health and status. You do not
+    administer it: never attempt mutations, secret access, exec, attach,
+    port-forwarding, private-service access, or cloud metadata access.
+  AGENTS.md: |
+    # Personal Hermes workspace
+
+    This Hermes instance is a personal chat, research, and automation
+    environment. User-created files and skills belong under `/opt/data/workspace`.
+
+    You may:
+
+    - research public Internet sources and cite them
+    - create, read, and edit files in the workspace
+    - create and improve reusable skills
+    - configure user-owned channels and provider integrations
+    - run ordinary local commands needed for the user's task
+    - inspect non-secret Kubernetes resources and pod logs with read-only
+      `kubectl get`, `describe`, and `logs` commands
+
+    This container is not an infrastructure administration environment. The
+    Kubernetes identity is observation-only and cannot read Secrets or use pod
+    exec, attach, or port-forwarding. Do not attempt cluster mutations, private
+    service access, node LAN access, metadata services, Vault, container
+    runtimes, or the operator Hermes instance. If a request needs an action,
+    explain the evidence and say Brad must perform it from the operator
+    instance at `agent.bstein.dev`.

services/hermes-chat/deployment.yaml

@@ -0,0 +1,259 @@
+# services/hermes-chat/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-chat
+  namespace: hermes-chat
+  labels:
+    app: hermes-chat
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  progressDeadlineSeconds: 1800
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: hermes-chat
+  template:
+    metadata:
+      labels:
+        app: hermes-chat
+      annotations:
+        ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback
+        ai.bstein.dev/role: personal-chat-research
+        ai.bstein.dev/isolation: observer-only Kubernetes RBAC, private-service egress denied
+        ai.bstein.dev/session-reset: "20260802-consumer-pty-recovery"
+        ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation
+        ai.bstein.dev/user-boundary: one OIDC subject with an isolated PVC subdirectory
+    spec:
+      serviceAccountName: hermes-chat
+      automountServiceAccountToken: true
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values:
+                      - arm64
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values:
+                      - "true"
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-13
+                      - titan-15
+                      - titan-17
+                      - titan-18
+                      - titan-19
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: atlas.bstein.dev/spillover
+                    operator: DoesNotExist
+            - weight: 90
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi4
+      initContainers:
+        - name: migrate-user-sessions
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
+          imagePullPolicy: IfNotPresent
+          command:
+            - /opt/hermes/bin/hermes-session-migrate
+          env:
+            - name: HERMES_HOME
+              value: /storage/users/4794ab8284a14d421eaeea3e
+            - name: HERMES_MIGRATE_SOURCE_HOME
+              value: /storage
+            - name: HERMES_MIGRATE_USER_ID
+              value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc
+            - name: HERMES_MIGRATE_SESSION_IDS
+              value: 20260802_182723_9f53f1,20260802_173527_9dc51b
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 10000
+            runAsGroup: 10000
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: home
+              mountPath: /storage
+          resources:
+            requests:
+              cpu: 25m
+              memory: 64Mi
+            limits:
+              cpu: 250m
+              memory: 256Mi
+        - name: init-config
+          image: busybox:1.37
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - |
+              set -eu
+              user_home=/storage/users/4794ab8284a14d421eaeea3e
+              mkdir -p "${user_home}/workspace/skills" "${user_home}/home/.local/bin" "${user_home}/logs"
+              cp /config/config.yaml "${user_home}/config.yaml"
+              cp /config/SOUL.md "${user_home}/SOUL.md"
+              cp /config/AGENTS.md "${user_home}/workspace/AGENTS.md"
+              touch "${user_home}/.env"
+              if ! grep -q '^API_SERVER_KEY=' "${user_home}/.env"; then
+                api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
+                printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> "${user_home}/.env"
+              fi
+              chmod 0600 "${user_home}/.env"
+              chown -R 10000:10000 "${user_home}"
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: home
+              mountPath: /storage
+            - name: config
+              mountPath: /config
+              readOnly: true
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+        - name: install-kubectl
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          imagePullPolicy: IfNotPresent
+          command:
+            - /bin/sh
+            - -c
+            - |
+              set -eu
+              cp "$(command -v kubectl)" /tools/kubectl
+              chmod 0755 /tools/kubectl
+              chown 10000:10000 /tools/kubectl
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 0
+            runAsGroup: 0
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: tools
+              mountPath: /tools
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+      containers:
+        - name: hermes-chat
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
+          imagePullPolicy: IfNotPresent
+          args:
+            - gateway
+            - run
+          ports:
+            - name: dashboard
+              containerPort: 9119
+              protocol: TCP
+          env:
+            - name: HERMES_HOME
+              value: /opt/data
+            - name: HOME
+              value: /opt/data/home
+            - name: PATH
+              value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+            - name: HERMES_DASHBOARD
+              value: "1"
+            - name: HERMES_DASHBOARD_HOST
+              value: 0.0.0.0
+            - name: HERMES_DASHBOARD_PORT
+              value: "9119"
+            - name: HERMES_DASHBOARD_PUBLIC_URL
+              value: https://chat.bstein.dev
+            - name: HERMES_DASHBOARD_OIDC_ISSUER
+              value: https://sso.bstein.dev/realms/atlas
+            - name: HERMES_DASHBOARD_OIDC_CLIENT_ID
+              value: hermes-chat-dashboard
+            - name: HERMES_DASHBOARD_OIDC_SCOPES
+              value: openid profile email groups
+            - name: HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS
+              value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc
+            - name: API_SERVER_ENABLED
+              value: "true"
+            - name: API_SERVER_HOST
+              value: 0.0.0.0
+            - name: API_SERVER_PORT
+              value: "8642"
+            - name: API_SERVER_CORS_ORIGINS
+              value: https://chat.bstein.dev
+          volumeMounts:
+            - name: home
+              mountPath: /opt/data
+              subPath: users/4794ab8284a14d421eaeea3e
+            - name: tools
+              mountPath: /usr/local/bin/kubectl
+              subPath: kubectl
+          readinessProbe:
+            httpGet:
+              path: /api/status
+              port: dashboard
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /api/status
+              port: dashboard
+            initialDelaySeconds: 90
+            periodSeconds: 30
+            timeoutSeconds: 10
+          securityContext:
+            allowPrivilegeEscalation: false
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests:
+              cpu: 100m
+              memory: 768Mi
+              ephemeral-storage: 256Mi
+            limits:
+              cpu: "1"
+              memory: 2Gi
+              ephemeral-storage: 1Gi
+      volumes:
+        - name: home
+          persistentVolumeClaim:
+            claimName: hermes-chat-home
+        - name: config
+          configMap:
+            name: hermes-chat-config
+        - name: tools
+          emptyDir: {}

services/hermes-chat/ingress.yaml

@@ -0,0 +1,27 @@
+# services/hermes-chat/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: chat
+  namespace: hermes-chat
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - chat.bstein.dev
+      secretName: chat-tls
+  rules:
+    - host: chat.bstein.dev
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: hermes-chat
+                port:
+                  name: dashboard

services/hermes-chat/kustomization.yaml

@@ -0,0 +1,15 @@
+# services/hermes-chat/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: hermes-chat
+resources:
+  - namespace.yaml
+  - serviceaccount.yaml
+  - rbac.yaml
+  - configmap.yaml
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  - networkpolicy.yaml
+  - certificate.yaml
+  - ingress.yaml

services/hermes-chat/namespace.yaml

@@ -0,0 +1,7 @@
+# services/hermes-chat/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: hermes-chat
+  labels:
+    app.kubernetes.io/name: hermes-chat

services/hermes-chat/networkpolicy.yaml

@@ -0,0 +1,87 @@
+# services/hermes-chat/networkpolicy.yaml
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-chat-isolation
+  namespace: hermes-chat
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-chat
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: traefik
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
+      ports:
+        - protocol: TCP
+          port: 9119
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: hermes
+          podSelector:
+            matchLabels:
+              app: hermes-model-gate
+      ports:
+        - protocol: TCP
+          port: 8080
+    # Atlas OIDC uses the public sso.bstein.dev issuer, which resolves to the
+    # in-cluster Traefik load balancer for pods on the Atlas network.
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: traefik
+          podSelector:
+            matchLabels:
+              app: traefik
+      ports:
+        - protocol: TCP
+          port: 443
+    - to:
+        - ipBlock:
+            cidr: 10.43.0.1/32
+      ports:
+        - protocol: TCP
+          port: 443
+    # K3s applies egress policy after Service DNAT, so admit only the fixed API
+    # server endpoints as well as the kubernetes Service IP above.
+    - to:
+        - ipBlock:
+            cidr: 192.168.22.11/32
+        - ipBlock:
+            cidr: 192.168.22.12/32
+        - ipBlock:
+            cidr: 192.168.22.13/32
+      ports:
+        - protocol: TCP
+          port: 6443
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 10.0.0.0/8
+              - 100.64.0.0/10
+              - 127.0.0.0/8
+              - 169.254.0.0/16
+              - 172.16.0.0/12
+              - 192.168.0.0/16

services/hermes-chat/pvc.yaml

@@ -0,0 +1,15 @@
+# services/hermes-chat/pvc.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: hermes-chat-home
+  namespace: hermes-chat
+  labels:
+    app: hermes-chat
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: astreae
+  resources:
+    requests:
+      storage: 2Gi

services/hermes-chat/rbac.yaml

@@ -0,0 +1,62 @@
+# services/hermes-chat/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: hermes-chat-observer
+rules:
+  - apiGroups: [""]
+    resources:
+      - endpoints
+      - events
+      - namespaces
+      - nodes
+      - persistentvolumeclaims
+      - persistentvolumes
+      - pods
+      - pods/log
+      - replicationcontrollers
+      - services
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources:
+      - daemonsets
+      - deployments
+      - replicasets
+      - statefulsets
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["batch"]
+    resources:
+      - cronjobs
+      - jobs
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["networking.k8s.io"]
+    resources:
+      - ingresses
+      - networkpolicies
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources:
+      - helmreleases
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["kustomize.toolkit.fluxcd.io"]
+    resources:
+      - kustomizations
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["source.toolkit.fluxcd.io"]
+    resources:
+      - gitrepositories
+      - helmrepositories
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: hermes-chat-observer
+subjects:
+  - kind: ServiceAccount
+    name: hermes-chat
+    namespace: hermes-chat
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: hermes-chat-observer

services/hermes-chat/service.yaml

@@ -0,0 +1,17 @@
+# services/hermes-chat/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: hermes-chat
+  namespace: hermes-chat
+  labels:
+    app: hermes-chat
+spec:
+  type: ClusterIP
+  selector:
+    app: hermes-chat
+  ports:
+    - name: dashboard
+      port: 9119
+      targetPort: dashboard
+      protocol: TCP

services/hermes-chat/serviceaccount.yaml

@@ -0,0 +1,7 @@
+# services/hermes-chat/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-chat
+  namespace: hermes-chat
+automountServiceAccountToken: true

services/hermes/NOTES.md

@@ -0,0 +1,239 @@
+# Hermes on Atlas: operator guide
+
+This is the mental model and demonstration script for the operator instance at
+`agent.bstein.dev`. Read it once, then prove each section in the live UI. The
+consumer instance at `chat.bstein.dev` is intentionally separate and is not the
+place to perform infrastructure triage.
+
+## The one-sentence explanation
+
+Hermes is the persistent agent runtime and control surface; Codex or the local
+GPU model supplies reasoning, while Hermes supplies sessions, tools, skills,
+approval rules, identity, and the repeatable procedure that connects the model
+to Atlas evidence.
+
+Hermes is not the model. Replacing `gpt-5.6-terra` with `gpt-oss:20b` changes the
+reasoning engine, but it does not replace the Hermes sessions, tools, skills,
+files, permission boundary, or workflow.
+
+## Request and evidence path
+
+```text
+browser
+  -> Keycloak login
+  -> oauth2-proxy exact-user check
+  -> operator Hermes pod and persistent workspace
+  -> SOUL.md + AGENTS.md + selected skill
+  -> Codex primary model
+       -> read-only terminal/web/file tools
+       -> Ariadne deterministic evidence
+       -> retained Jenkins logs and artifacts
+       -> Gitea commits and Flux state
+       -> Kubernetes workloads, events, logs, and dependencies
+       -> Pushgateway data queried through VictoriaMetrics
+       -> Grafana query and alert context
+  -> structured finding and proposed repo-side change
+  -> human review and approval
+  -> Git/Flux delivery outside Hermes
+
+provider failure
+  -> hermes-model-gate
+       -> Hermes owns titan-24: Ollama serves gpt-oss:20b
+       -> Wolf owns titan-24: gate returns 503; local inference yields
+```
+
+The operator agent pod runs on an ARM worker and does not reserve the GPU.
+Ollama is the component on `titan-24` that owns GPU memory. The Lease
+`hermes/titan-24-gpu-owner` controls whether the model gate admits local
+inference. Normal operator conversations use independently authenticated Codex
+first, so Wolf ownership does not need to block the web agent.
+
+## Who does what
+
+| Component | Responsibility | What it does not prove |
+| --- | --- | --- |
+| Hermes | Maintains the conversation, chooses tools/skills, reasons over evidence, saves reports, and enforces the approval experience | It is not the inference model and does not automatically have cluster-admin access |
+| Ariadne | Collects and normalizes deterministic cluster, Jenkins, and quality evidence into a timestamped bundle; may optionally run a local diagnosis | Its diagnosis is not authoritative when stale, empty, or contradicted by the bundle |
+| Jenkins | Runs the suites and retains console/artifact evidence | A final `exit 1` alone does not identify the first failed gate |
+| Pushgateway | Receives suite/build quality telemetry | A cumulative counter is not the current build result |
+| VictoriaMetrics | Stores and answers PromQL for quality and environment telemetry | A Grafana color is not direct proof of root cause |
+| Grafana | Visualizes and alerts on VictoriaMetrics data | A red panel must be traced to its query, labels, time range, and raw series |
+| Flux | Applies reviewed Git state to the cluster | A recent reconciliation is correlation until the changed path matches the failure |
+| Kubernetes | Supplies workload, event, log, node, storage, and dependency evidence | An unrelated unhealthy pod is not proof that CI failed because of the cluster |
+
+Direct Jenkins artifact requests currently require authorization and can return
+HTTP 403. That does not break the workflow: Ariadne retains selected Jenkins
+console tails and named artifact contents in its deterministic bundle. A report
+must say `retained Ariadne evidence` when that fallback is used; it must not
+pretend direct Jenkins access succeeded.
+
+## The actual supervised triage algorithm
+
+1. Classify the request as test/build triage, service health, or alert tuning.
+2. Read the latest Ariadne diagnosis and deterministic bundle with HTTP GET.
+3. Compare their timestamps. The deterministic bundle remains authoritative.
+4. Select a terminal failed build. Keep running or unknown builds out of the
+   terminal-failure list.
+5. Identify the first enforced failed gate in this order:
+   `style -> loc -> coverage -> tests -> gate_glue -> sonarqube -> supply_chain`.
+6. Cite the smallest decisive Jenkins log/artifact evidence. State whether it
+   came directly from Jenkins or from Ariadne retention.
+7. Query the matching build/check/test series in VictoriaMetrics. Distinguish a
+   current gauge from a cumulative counter and ignore zero-valued failure
+   series.
+8. Correlate recent Gitea commits and the relevant Flux revision by timestamp
+   and affected path. Recency alone is not causation.
+9. Check only the Kubernetes resources capable of explaining that failure.
+   Classify observations as direct, contributing candidate, or background.
+10. Return `Finding`, `Confidence`, `Evidence`, `Likely cause`, `Blast radius`,
+    `Next checks`, `Repo-side fix`, and `Approval required`.
+11. Stop at a proposal. A human reviews the patch or action; Git and Flux remain
+    the delivery path.
+
+## What Hermes can read and what it cannot do
+
+The live `hermes-triage` ServiceAccount can get/list/watch ordinary workload,
+log, event, ingress, storage, Flux, and image-automation metadata. It cannot
+read Kubernetes Secret values, create exec sessions, patch Deployments, or
+patch Flux Kustomizations. The configured command deny list also blocks common
+Kubernetes and Flux mutations.
+
+The security boundary is layered:
+
+1. Keycloak authenticates the person.
+2. oauth2-proxy restricts the operator surface to Brad.
+3. The operator has a separate namespace, PVC, configuration, and ServiceAccount.
+4. Kubernetes RBAC is the hard API authorization boundary.
+5. NetworkPolicy limits reachable paths where configured.
+6. Hermes approvals and instructions provide a user-facing safety layer.
+
+A skill is procedure and context, not a permission grant. Adding a sentence to
+a skill cannot bypass Kubernetes RBAC.
+
+## What “Hermes learns the workflow” means
+
+It does not silently retrain model weights on cluster data. Learning here means
+that a successful repeated procedure is written as a reusable skill on the
+operator PVC or versioned in Git. The skill describes when it should trigger,
+the evidence order, interpretation rules, output contract, and safety boundary.
+
+The current workflow has a versioned top-level skill and persisted specialist
+skills for:
+
+- orchestration and evidence reporting;
+- retained Jenkins evidence;
+- quality metrics;
+- Gitea/Flux correlation;
+- Kubernetes failure classification;
+- Grafana metric provenance;
+- Soteria backup health;
+- approval-required actions.
+
+The model still evaluates fresh variable evidence on every incident. The skill
+makes the process repeatable; it does not freeze the answer.
+
+## The four operator skills to remember
+
+- `triage-titan-test-failures`: CI, tests, builds, quality gates, and suspected
+  test-environment regressions.
+- `triage-atlas-service-health`: active service or cluster incidents and red
+  high-level health panels.
+- `tune-atlas-alerts`: noisy alerts, impossible metrics, bad PromQL, and
+  generator-owned Grafana corrections.
+- `master-hermes-on-atlas`: hands-on training, assessments, architecture, and
+  claim audits.
+
+Do not start by choosing raw tools. State the operational question and let the
+skill route to the narrow evidence source.
+
+## Two live, repeatable proof cases
+
+### Proof 6: Soteria build 272
+
+- Terminal result: failure.
+- Tests: 318 passed, zero failed.
+- Local coverage: 96.195 percent.
+- First enforced failure: SonarQube `new_coverage=0.0` against threshold 80.
+- Runtime: Soteria remained Ready.
+- Correct conclusion: a Sonar new-code policy/reporting problem, not a test or
+  Kubernetes capacity failure.
+- Correct restraint: do not weaken the policy until scanner import and baseline
+  evidence identifies whether the issue is configuration or legitimate new
+  uncovered code.
+
+### Proof 7: Ananke build 242
+
+- Terminal result: failure.
+- Local coverage gate: 61.8 percent and failed.
+- Other checks: tests, SonarQube, supply-chain, LOC, docs naming, and gate glue
+  were healthy.
+- Sonar new-code coverage: 93.1 percent, which is a different scope.
+- Correct conclusion: local coverage input/scope caused the enforced failure;
+  do not call the Sonar number contradictory without comparing inputs.
+- Correct restraint: do not lower the threshold until the retained local gate
+  inputs are readable.
+
+The full sessions and redacted Markdown exports are in Sessions and
+`Files/triage-proof`.
+
+## Five-minute demonstration
+
+1. Open Models and show `openai-codex/gpt-5.6-terra` as primary and
+   `gpt-oss:20b` as local fallback.
+2. Open Skills and show the four operator skills plus the persisted component
+   skills.
+3. Open `Proof 6 - Soteria 272 Sonar new coverage triage` in Sessions.
+4. Point out the timestamped bundle, exact build/artifact evidence, metric
+   corroboration, healthy runtime, fact/inference separation, and approval
+   boundary.
+5. Open Proof 7 and explain why local coverage and Sonar new-code coverage can
+   differ without either number being fabricated.
+6. End by showing that Hermes proposes a repo-side correction but cannot patch
+   the Deployment or Flux Kustomization with its ServiceAccount.
+
+Use this short explanation:
+
+> Hermes is the persistent, permissioned workflow layer around the model. In
+> this cluster Ariadne collects deterministic evidence, Hermes correlates it
+> with retained Jenkins artifacts, Git/Flux state, Kubernetes health, and
+> quality metrics, and then returns a supervised proposal. Repeated procedures
+> become explicit skills. Kubernetes RBAC prevents the agent from turning a
+> diagnosis into an unreviewed infrastructure change.
+
+## Prompts that exercise the real system
+
+- `Use $triage-titan-test-failures. Triage the worst current terminal CI failure and link every decisive piece of evidence.`
+- `Use $triage-titan-test-failures. Triage Ananke build 242. Separate local coverage from Sonar new-code coverage.`
+- `Use $triage-atlas-service-health. Explain the VictoriaMetrics outage from current state and retained events; separate recovered impact from current impact.`
+- `Use $tune-atlas-alerts. Trace one currently firing alert to its generated source and raw PromQL, but do not edit it.`
+- `Use $master-hermes-on-atlas. Assess me on the request path and permission boundary. One question at a time.`
+
+## Honest limits
+
+- Hermes does not currently apply production or cluster changes autonomously.
+- Direct Jenkins console/artifact access may be forbidden; Ariadne retention is
+  the current fallback and must be named as provenance.
+- Local `gpt-oss:20b` is useful for bounded work but is slower and less reliable
+  for multi-source triage than Codex.
+- The current Ariadne local diagnosis can return an empty model response. That
+  does not invalidate the deterministic bundle or prevent Codex-backed Hermes
+  from triaging it.
+- A single successful case demonstrates capability, not mastery. Mastery means
+  leading different real incidents, improving the skill after failures, and
+  teaching the architecture without prompts.
+
+## Your shortest path to fluency
+
+1. Explain the request diagram without looking.
+2. Explain why Ariadne and Hermes are separate components.
+3. Reproduce Proof 6 from the raw bundle and metrics.
+4. Reproduce Proof 7 and explain the two coverage scopes.
+5. Produce the live RBAC allow/deny matrix with `kubectl auth can-i`.
+6. Create one small writable skill from a repeated sub-workflow, test a trigger
+   and non-trigger case, then improve it.
+7. Lead one new incident while Hermes coaches rather than answers for you.
+8. Teach the five-minute demonstration to another person.
+
+At that point the interview claims are demonstrable. Continue repeating real
+incidents before describing yourself as fully autonomous or the system as
+self-correcting.

services/hermes/agent-ingress.yaml

@@ -21,6 +21,6 @@ spec:
             pathType: Prefix
             backend:
               service:
-                name: hermes
+                name: oauth2-proxy-hermes
                 port:
-                  name: dashboard
+                  name: http

services/hermes/ariadne-handoff-rbac.yaml

@@ -0,0 +1,30 @@
+# services/hermes/ariadne-handoff-rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ariadne-gpu-handoff
+  namespace: hermes
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources:
+      - leases
+    resourceNames:
+      - titan-24-gpu-owner
+    verbs:
+      - get
+      - patch
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ariadne-gpu-handoff
+  namespace: hermes
+subjects:
+  - kind: ServiceAccount
+    name: ariadne
+    namespace: maintenance
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ariadne-gpu-handoff

services/hermes/configmap.yaml

@@ -9,11 +9,41 @@ metadata:
 data:
   config.yaml: |
     model:
-      provider: custom
+      provider: openai-codex
-      default: qwen2.5:7b-instruct-q4_0
+      default: gpt-5.6-terra
-      model: qwen2.5:7b-instruct-q4_0
+      model: gpt-5.6-terra
-      base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1
+
-      api_key: ollama
+    fallback_providers:
+      - provider: custom
+        model: gpt-oss:20b
+        base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
+        api_key: ollama
+
+    agent:
+      api_max_retries: 1
+
+    platform_toolsets:
+      cli:
+        - clarify
+        - file
+        - session_search
+        - skills
+        - terminal
+        - todo
+        - web
+      api_server:
+        - clarify
+        - file
+        - session_search
+        - skills
+        - terminal
+        - todo
+        - web
+
+    skills:
+      creation_nudge_interval: 15
+      external_dirs:
+        - /opt/data/workspace/skills
 
     terminal:
       backend: local
@@ -22,7 +52,7 @@ data:
       home_mode: auto
 
     approvals:
-      mode: manual
+      mode: smart
       deny:
         - "*kubectl apply*"
         - "*kubectl delete*"
@@ -80,6 +110,12 @@ data:
     read-only state. Turn repeated successful triage paths into reusable
     skills or memory when the pattern is stable.
 
+    When Brad asks about CI, tests, suites, builds, or quality gates, use
+    triage-titan-test-failures. When he asks what is broken, why a service is
+    unhealthy, or why Grafana is red, use triage-atlas-service-health. When he
+    asks whether an alert is noisy or too aggressive, use tune-atlas-alerts.
+    Do not make him choose a tool, page, or evidence source first.
+
     Stay Flux-first. Do not mutate the cluster directly. Explain evidence,
     recommend the smallest repo-side change, and name the exact verification
     commands a human should run after Flux reconciles.
@@ -89,6 +125,11 @@ data:
     You are Hermes running inside the Titan Kubernetes cluster as a read-only
     testing and operations triage assistant.
 
+    Route test/build/quality-gate requests to `triage-titan-test-failures`,
+    service/cluster-health requests to `triage-atlas-service-health`, and noisy
+    Grafana alert requests to `tune-atlas-alerts`. Ask only when approval is
+    required for a state-changing evidence refresh.
+
     Ariadne owns deterministic evidence collection and local diagnosis. Start
     every testing triage by reading:
 
@@ -122,3 +163,181 @@ data:
     Do not run mutating commands such as `kubectl apply`, `delete`, `scale`,
     `patch`, `cordon`, `uncordon`, `drain`, or `rollout restart`. Do not read
     Kubernetes Secret values. Draft repo changes or operator steps instead.
+  START-HERE.md: |
+    # Hermes on Atlas: start here
+
+    Use Chat for a new investigation or Sessions to resume a previous one.
+    The shortest useful prompts are:
+
+    - `Triage the latest test failure.`
+    - `What is broken in the cluster right now?`
+    - `Why is Grafana red, and which alerts are noise?`
+    - `Triage <suite> build <number> and save the report.`
+
+    Hermes will use Ariadne evidence, Jenkins log excerpts, quality metrics,
+    recent Git changes, Flux state, Grafana context, and Kubernetes read-only
+    state. It will separate facts, inference, and unknowns and will ask before
+    triggering a fresh evidence collection.
+
+    Hermes now has three focused operator skills:
+
+    1. `triage-titan-test-failures` for all ten custom CI suites.
+    2. `triage-atlas-service-health` for any live namespace or service.
+    3. `tune-atlas-alerts` for evidence-backed alert and dashboard corrections.
+
+    Validated CI examples are available under Sessions, including:
+
+    1. `Proof 6 - Soteria 272 Sonar new coverage triage`
+    2. `Proof 7 - Ananke 242 local coverage triage`
+
+    Their saved reports are in Files under `triage-proof/`. Read
+    `HERMES-OPERATOR-RUNBOOK.md` for the system mental model and five-minute
+    demonstration.
+
+    Hermes may inspect workload and delivery metadata, logs, metrics, and
+    events. Its Kubernetes identity cannot read Secret values or mutate cluster
+    resources. Apply fixes through the titan-iac GitOps workflow after review.
+  HERMES-CAPABILITIES.md: |
+    # What Hermes can do on Atlas
+
+    ## Working now
+
+    - Triage all custom CI suites: Ananke, Ariadne, Atlasbot, bstein_home,
+      data_prepper, Lesavka, Metis, Pegasus, Soteria, and titan_iac.
+    - Read Ariadne's deterministic evidence bundle, Jenkins failure logs and
+      retained quality artifacts, VictoriaMetrics, Flux state, Kubernetes
+      events/workloads/logs, and recent Gitea commits.
+    - Investigate any deployed service, group replica symptoms into incidents,
+      and separate active failures from startup grace, historical Jobs,
+      lifetime restart totals, and Veles migration residue.
+    - Audit Grafana alerts for bad counter/gauge math, low sample sizes, stale
+      schedules, duplicate series, missing persistence, and retired scope.
+    - Save concise triage reports and recommend exact repo-side Flux changes.
+
+    ## Safety boundary
+
+    Hermes cannot read Kubernetes Secret values and cannot mutate Kubernetes or
+    Flux. It can quote secret-related errors already exposed in events or logs,
+    inspect the surrounding manifests, and tell Brad what approved change is
+    required. This keeps cluster operation supervised.
+
+    ## Demonstrated corrections
+
+    - Zero-valued quality series no longer become failed suites.
+    - Running Jenkins builds remain in-progress instead of failed.
+    - Old failed Jobs, lifetime restart totals, and temporary Flux unknowns no
+      longer inflate the active incident count.
+    - Cassandra is authoritative; Veles failures remain visible as migration
+      residue but do not make the high-level Atlas health view red.
+    - Worker readiness uses the actual 18 Kubernetes workers.
+    - Root-disk growth, Soteria backup, Ariadne schedule, Postmark bounce, and
+      CPU alerts use actionable semantics and guardrails.
+
+    The evidence and verification for these changes are in
+    `triage-proof/ATLAS-TRIAGE-PROOFS.md`.
+
+    ## Good next prompts
+
+    - `Use $triage-atlas-service-health. What is broken right now?`
+    - `Use $triage-titan-test-failures. Triage the worst current suite.`
+    - `Use $tune-atlas-alerts. Audit alerts fired in the last 24 hours.`
+    - `Save this investigation as a reusable skill after I approve the pattern.`
+  ATLAS-TRIAGE-PROOFS.md: |
+    # Atlas triage proofs
+
+    This file is the short evidence trail for what Hermes can do today. The
+    operator skills remain read-only: fixes are reviewed and delivered through
+    Git and Flux.
+
+    ## Proof 1: Cassandra secret-sync recovery
+
+    - Finding: `cassandra-vault-sync` was stuck because the expected
+      `VELES_BYOK_ENCRYPTION_KEY` migration input was absent from Cassandra's
+      Vault path.
+    - Action: the existing Flux-tracked, suspended one-shot migration Job was
+      enabled, completed successfully, and immediately returned to suspended.
+    - Verification: Cassandra frontend, backend, PostgreSQL, and Vault sync all
+      became Ready. No Kubernetes Secret value was read or committed.
+    - Why this matters: Hermes can correlate workload state, logs, Vault policy
+      manifests, and migration ownership without confusing Veles residue with
+      a current Veles outage.
+
+    ## Proof 2: Grafana alert-noise correction
+
+    - Removed retired Veles pods and completed Jobs from high-level incident
+      counts while preserving them on drill-down dashboards.
+    - Added 15-minute persistence to Pending, CrashLoopBackOff, and image-pull
+      summaries.
+    - Replaced invalid `increase()` use on gauges, bounded CPU percentages,
+      required meaningful Postmark sample sizes, and scoped Soteria/Ariadne
+      alerts to configured current work.
+    - Live verification on 2026-08-03: `Problem Pods=0`,
+      `CrashLoop/ImagePull=0`, and `Workers Ready=18/18`.
+
+    ## Proof 3: all-suite testing triage
+
+    - Ariadne build 384 passed 676 tests and the exact coverage contract:
+      82 source files at or above 95%.
+    - Deployed image: `registry.bstein.dev/bstein/ariadne:0.1.0-384`.
+    - The deterministic bundle covers Ananke, Ariadne, Atlasbot, bstein_home,
+      data_prepper, Lesavka, Metis, Pegasus, Soteria, and titan_iac.
+    - Zero-valued quality metrics are healthy, running Jenkins builds are
+      in-progress, old failed Jobs are historical, and Veles objects are
+      migration residue. They no longer inflate the active failure set.
+    - Failed builds still include direct Jenkins console and retained artifact
+      links so Hermes can identify the first failed gate and smallest repo fix.
+
+    ## Proof 4: Cassandra generator failure classification
+
+    - Finding: the Cassandra serving path is healthy, but generation Job
+      `cassandra-generator-3ce198971b` failed.
+    - Evidence: the primary OpenAI request returned HTTP 429 because its credit
+      balance was exhausted. Codex CLI fallback ran, but the strict promotion
+      gate still rejected unresolved high-impact hint claims and blocked the
+      semantic scenarios.
+    - Impact: one generation capability/request failed; this is not a Cassandra
+      registry outage and not a Veles outage.
+    - Next action: restore provider credits or continue through Codex fallback,
+      then resolve the reported hint claims before retrying. This needs operator
+      approval because it can spend money or rerun an expensive generation.
+
+    ## Proof 5: live Hermes Soteria triage
+
+    - Hermes read the fresh 2026-08-03 Ariadne bundle without an approval
+      timeout and chose Soteria build 270 as the worst terminal failure.
+    - It proved that all 318 tests passed and local coverage was 96.195%, then
+      isolated the enforced failure to SonarQube reporting new-code coverage
+      as 0.0%. It did not blame the optional supply-chain advisory.
+    - It kept running titan-iac, Data Prepper, and Lesavka builds out of the
+      terminal-failure list and requested approval before any rerun or edit.
+
+    ## Proof 6: Soteria build 272 Sonar new-coverage triage
+
+    - Hermes read the fresh 2026-08-04 Ariadne bundle and selected terminal
+      Soteria build 272.
+    - It proved that all 318 tests passed and local coverage was 96.195%, then
+      isolated the enforced failure to SonarQube `new_coverage=0.0` against the
+      threshold of 80.
+    - It corroborated the retained Jenkins evidence with current quality
+      metrics and healthy Soteria/Flux state, and did not blame Kubernetes.
+    - It proposed no change until scanner import and baseline evidence can
+      distinguish configuration error from genuinely uncovered new code.
+
+    ## Proof 7: Ananke build 242 local-coverage triage
+
+    - Hermes selected terminal Ananke build 242 and isolated the first failure
+      to the local coverage gate at 61.8%.
+    - It showed that tests, SonarQube, supply chain, LOC, docs naming, gate glue,
+      Flux, and cluster readiness were healthy.
+    - It distinguished the local 61.8% scope from SonarQube's 93.1% new-code
+      scope instead of treating the numbers as interchangeable.
+    - Direct Jenkins artifact reads returned HTTP 403, so it explicitly named
+      Ariadne's retained bundle as evidence provenance and stopped short of an
+      unsupported fix.
+
+    ## Use Hermes next
+
+    - `What is broken in the cluster right now?`
+    - `Triage the worst current CI suite and link the evidence.`
+    - `Audit Grafana alerts from the last 24 hours and separate incidents from noise.`
+    - `Explain the Cassandra generator failure and give me the smallest safe next step.`

services/hermes/deployment.yaml

@@ -20,10 +20,11 @@ spec:
       labels:
         app: hermes
       annotations:
-        ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
+        ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation
+        ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback
         ai.bstein.dev/role: testing-triage
         ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
-        ai.bstein.dev/config-rev: "20260721-hermes-rollout-deadline"
+        ai.bstein.dev/config-rev: "20260804-root-operator-docs"
     spec:
       serviceAccountName: hermes-triage
       automountServiceAccountToken: true
@@ -50,6 +51,7 @@ spec:
                       - titan-13
                       - titan-15
                       - titan-17
+                      - titan-18
                       - titan-19
           preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
@@ -80,10 +82,18 @@ spec:
             - -c
             - |
               set -eu
-              mkdir -p /opt/data/workspace /opt/data/home/.local/bin /opt/data/logs
+              mkdir -p /opt/data/workspace/triage-proof /opt/data/home/.local/bin /opt/data/logs
               cp /config/config.yaml /opt/data/config.yaml
               cp /config/SOUL.md /opt/data/SOUL.md
               cp /config/AGENTS.md /opt/data/workspace/AGENTS.md
+              cp /config/START-HERE.md /opt/data/workspace/START-HERE.md
+              cp /config/HERMES-CAPABILITIES.md /opt/data/workspace/HERMES-CAPABILITIES.md
+              cp /guide/OPERATOR-RUNBOOK.md /opt/data/workspace/HERMES-OPERATOR-RUNBOOK.md
+              cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/workspace/triage-proof/ATLAS-TRIAGE-PROOFS.md
+              cp /config/START-HERE.md /opt/data/START-HERE.md
+              cp /config/HERMES-CAPABILITIES.md /opt/data/HERMES-CAPABILITIES.md
+              cp /guide/OPERATOR-RUNBOOK.md /opt/data/HERMES-OPERATOR-RUNBOOK.md
+              cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/ATLAS-TRIAGE-PROOFS.md
               touch /opt/data/.env
               if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then
                 api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
@@ -99,8 +109,8 @@ spec:
               mountPath: /opt/data
             - name: config
               mountPath: /config
-            - name: tools
+            - name: operator-guide
-              mountPath: /opt/data/home/.local/bin
+              mountPath: /guide
           resources:
             requests:
               cpu: 25m
@@ -134,7 +144,7 @@ spec:
               memory: 64Mi
       containers:
         - name: hermes
-          image: nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
+          image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
           imagePullPolicy: IfNotPresent
           args:
             - gateway
@@ -189,7 +199,20 @@ spec:
             - name: home
               mountPath: /opt/data
             - name: tools
-              mountPath: /opt/data/home/.local/bin
+              mountPath: /usr/local/bin/kubectl
+              subPath: kubectl
+            - name: triage-skill
+              mountPath: /opt/data/workspace/skills/triage-titan-test-failures
+              readOnly: true
+            - name: mastery-skill
+              mountPath: /opt/data/workspace/skills/master-hermes-on-atlas
+              readOnly: true
+            - name: service-health-skill
+              mountPath: /opt/data/workspace/skills/triage-atlas-service-health
+              readOnly: true
+            - name: alert-tuning-skill
+              mountPath: /opt/data/workspace/skills/tune-atlas-alerts
+              readOnly: true
           readinessProbe:
             httpGet:
               path: /api/status
@@ -218,5 +241,54 @@ spec:
         - name: config
           configMap:
             name: hermes-config
+        - name: operator-guide
+          configMap:
+            name: hermes-operator-guide
         - name: tools
           emptyDir: {}
+        - name: triage-skill
+          configMap:
+            name: hermes-triage-skill
+            items:
+              - key: SKILL.md
+                path: SKILL.md
+              - key: openai.yaml
+                path: agents/openai.yaml
+        - name: mastery-skill
+          configMap:
+            name: hermes-mastery-skill
+            items:
+              - key: SKILL.md
+                path: SKILL.md
+              - key: openai.yaml
+                path: agents/openai.yaml
+              - key: architecture.md
+                path: references/architecture.md
+              - key: curriculum.md
+                path: references/curriculum.md
+              - key: incident-drills.md
+                path: references/incident-drills.md
+              - key: mastery-rubric.md
+                path: references/mastery-rubric.md
+              - key: two-hour-proof-sprint.md
+                path: references/two-hour-proof-sprint.md
+        - name: service-health-skill
+          configMap:
+            name: hermes-service-health-skill
+            items:
+              - key: SKILL.md
+                path: SKILL.md
+              - key: openai.yaml
+                path: agents/openai.yaml
+              - key: service-map.md
+                path: references/service-map.md
+        - name: alert-tuning-skill
+          configMap:
+            name: hermes-alert-tuning-skill
+            items:
+              - key: SKILL.md
+                path: SKILL.md
+              - key: openai.yaml
+                path: agents/openai.yaml
+              - key: alert-review.md
+                path: references/alert-review.md

services/hermes/kustomization.yaml

@@ -4,11 +4,62 @@ kind: Kustomization
 namespace: hermes
 resources:
   - namespace.yaml
+  - vault-serviceaccount.yaml
   - configmap.yaml
   - rbac.yaml
   - pvc.yaml
+  - model-gate-rbac.yaml
+  - ariadne-handoff-rbac.yaml
+  - model-gate-state.yaml
+  - model-gate-configmap.yaml
+  - model-gate-deployment.yaml
+  - networkpolicy.yaml
   - ollama-deployment.yaml
   - deployment.yaml
   - service.yaml
+  - oauth2-proxy.yaml
   - agent-certificate.yaml
   - agent-ingress.yaml
+
+configMapGenerator:
+  - name: hermes-operator-guide
+    namespace: hermes
+    files:
+      - OPERATOR-RUNBOOK.md=NOTES.md
+    options:
+      disableNameSuffixHash: true
+  - name: hermes-triage-skill
+    namespace: hermes
+    files:
+      - SKILL.md=skills/triage-titan-test-failures/SKILL.md
+      - openai.yaml=skills/triage-titan-test-failures/agents/openai.yaml
+    options:
+      disableNameSuffixHash: true
+  - name: hermes-mastery-skill
+    namespace: hermes
+    files:
+      - SKILL.md=skills/master-hermes-on-atlas/SKILL.md
+      - openai.yaml=skills/master-hermes-on-atlas/agents/openai.yaml
+      - architecture.md=skills/master-hermes-on-atlas/references/architecture.md
+      - curriculum.md=skills/master-hermes-on-atlas/references/curriculum.md
+      - incident-drills.md=skills/master-hermes-on-atlas/references/incident-drills.md
+      - mastery-rubric.md=skills/master-hermes-on-atlas/references/mastery-rubric.md
+      - two-hour-proof-sprint.md=skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md
+    options:
+      disableNameSuffixHash: true
+  - name: hermes-service-health-skill
+    namespace: hermes
+    files:
+      - SKILL.md=skills/triage-atlas-service-health/SKILL.md
+      - openai.yaml=skills/triage-atlas-service-health/agents/openai.yaml
+      - service-map.md=skills/triage-atlas-service-health/references/service-map.md
+    options:
+      disableNameSuffixHash: true
+  - name: hermes-alert-tuning-skill
+    namespace: hermes
+    files:
+      - SKILL.md=skills/tune-atlas-alerts/SKILL.md
+      - openai.yaml=skills/tune-atlas-alerts/agents/openai.yaml
+      - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md
+    options:
+      disableNameSuffixHash: true

services/hermes/model-gate-configmap.yaml

@@ -0,0 +1,151 @@
+# services/hermes/model-gate-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hermes-model-gate
+  namespace: hermes
+data:
+  model_gate.py: |
+    #!/usr/bin/env python3
+    """Fail-closed proxy that admits local inference only while Hermes owns titan-24."""
+
+    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+    import json
+    import os
+    from pathlib import Path
+    import ssl
+    import threading
+    import time
+    from urllib.error import HTTPError, URLError
+    from urllib.request import Request, urlopen
+
+
+    LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0")
+    LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
+    UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/")
+    LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes")
+    LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner")
+    CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1"))
+    API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
+    API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443")
+    TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
+    CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+    LEASE_URL = (
+        f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/"
+        f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}"
+    )
+
+    _cache_lock = threading.Lock()
+    _cached_owner = "unavailable"
+    _cached_at = 0.0
+
+
+    def _lease_owner() -> str:
+      """Return the current owner, failing closed when Kubernetes is unavailable."""
+
+      global _cached_at, _cached_owner
+      now = time.monotonic()
+      with _cache_lock:
+        if now - _cached_at < CACHE_TTL_SEC:
+          return _cached_owner
+        try:
+          token = TOKEN_PATH.read_text(encoding="utf-8").strip()
+          request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"})
+          context = ssl.create_default_context(cafile=str(CA_PATH))
+          with urlopen(request, timeout=3, context=context) as response:
+            payload = json.load(response)
+          owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip()
+        except Exception:
+          owner = "unavailable"
+        _cached_owner = owner
+        _cached_at = now
+        return owner
+
+
+    class Handler(BaseHTTPRequestHandler):
+      """Proxy local model traffic while exposing health and ownership status."""
+
+      protocol_version = "HTTP/1.1"
+
+      def _json(self, status: int, payload: dict) -> None:
+        body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        self.wfile.write(body)
+
+      def _local_allowed(self) -> tuple[bool, str]:
+        owner = _lease_owner()
+        return owner == "hermes", owner
+
+      def _proxy(self) -> None:
+        allowed, owner = self._local_allowed()
+        if not allowed:
+          self._json(
+              503,
+              {
+                  "error": {
+                      "message": f"local GPU inference unavailable while titan-24 owner is {owner}",
+                      "type": "server_error",
+                  },
+                  "gpu_owner": owner,
+                  "fallback_required": True,
+              },
+          )
+          return
+
+        length = int(self.headers.get("Content-Length", "0") or 0)
+        body = self.rfile.read(length) if length else None
+        headers = {"Content-Type": self.headers.get("Content-Type", "application/json")}
+        if self.headers.get("Accept"):
+          headers["Accept"] = self.headers["Accept"]
+        request = Request(f"{UPSTREAM_URL}{self.path}", data=body, headers=headers, method=self.command)
+        try:
+          response = urlopen(request, timeout=1800)
+        except HTTPError as exc:
+          response = exc
+        except (TimeoutError, URLError) as exc:
+          self._json(503, {"error": {"message": f"local model upstream unavailable: {exc}", "type": "server_error"}})
+          return
+
+        self.send_response(response.status)
+        content_type = response.headers.get("Content-Type")
+        if content_type:
+          self.send_header("Content-Type", content_type)
+        content_length = response.headers.get("Content-Length")
+        if content_length:
+          self.send_header("Content-Length", content_length)
+        else:
+          self.send_header("Connection", "close")
+          self.close_connection = True
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        while True:
+          chunk = response.read(65536)
+          if not chunk:
+            break
+          self.wfile.write(chunk)
+          self.wfile.flush()
+        response.close()
+
+      def do_GET(self) -> None:
+        if self.path == "/healthz":
+          self._json(200, {"status": "ok"})
+          return
+        if self.path == "/gate/status":
+          allowed, owner = self._local_allowed()
+          self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed})
+          return
+        self._proxy()
+
+      def do_POST(self) -> None:
+        self._proxy()
+
+      def log_message(self, format_string: str, *args) -> None:
+        print(f"model-gate {self.address_string()} {format_string % args}", flush=True)
+
+
+    if __name__ == "__main__":
+      ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever()

services/hermes/model-gate-deployment.yaml

@@ -0,0 +1,125 @@
+# services/hermes/model-gate-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-model-gate
+  namespace: hermes
+  labels:
+    app: hermes-model-gate
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: hermes-model-gate
+  template:
+    metadata:
+      labels:
+        app: hermes-model-gate
+    spec:
+      serviceAccountName: hermes-model-gate
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values:
+                      - arm64
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values:
+                      - "true"
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-04
+                      - titan-13
+                      - titan-15
+                      - titan-17
+                      - titan-19
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 90
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+      containers:
+        - name: model-gate
+          image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
+          imagePullPolicy: IfNotPresent
+          command:
+            - python
+            - /opt/model-gate/model_gate.py
+          ports:
+            - name: http
+              containerPort: 8080
+          env:
+            - name: UPSTREAM_URL
+              value: http://hermes-ollama.hermes.svc.cluster.local:11434
+            - name: LEASE_NAMESPACE
+              value: hermes
+            - name: LEASE_NAME
+              value: titan-24-gpu-owner
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 65532
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 250m
+              memory: 128Mi
+          volumeMounts:
+            - name: script
+              mountPath: /opt/model-gate
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: script
+          configMap:
+            name: hermes-model-gate
+            defaultMode: 0555
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hermes-model-gate
+  namespace: hermes
+  labels:
+    app: hermes-model-gate
+spec:
+  type: ClusterIP
+  selector:
+    app: hermes-model-gate
+  ports:
+    - name: http
+      port: 11434
+      targetPort: http

services/hermes/model-gate-rbac.yaml

@@ -0,0 +1,34 @@
+# services/hermes/model-gate-rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-model-gate
+  namespace: hermes
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: hermes-model-gate
+  namespace: hermes
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources:
+      - leases
+    resourceNames:
+      - titan-24-gpu-owner
+    verbs:
+      - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: hermes-model-gate
+  namespace: hermes
+subjects:
+  - kind: ServiceAccount
+    name: hermes-model-gate
+    namespace: hermes
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: hermes-model-gate

services/hermes/model-gate-state.yaml

@@ -0,0 +1,10 @@
+# services/hermes/model-gate-state.yaml
+apiVersion: coordination.k8s.io/v1
+kind: Lease
+metadata:
+  name: titan-24-gpu-owner
+  namespace: hermes
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  holderIdentity: hermes

services/hermes/networkpolicy.yaml

@@ -0,0 +1,30 @@
+# services/hermes/networkpolicy.yaml
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-ollama-ingress
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-ollama
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app: hermes-model-gate
+      ports:
+        - protocol: TCP
+          port: 11434
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: maintenance
+          podSelector:
+            matchLabels:
+              app: ariadne
+      ports:
+        - protocol: TCP
+          port: 11434

services/hermes/oauth2-proxy.yaml

@@ -0,0 +1,161 @@
+# services/hermes/oauth2-proxy.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hermes-operator-allowlist
+  namespace: hermes
+data:
+  allowed-emails: |
+    brad@bstein.dev
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: oauth2-proxy-hermes
+  namespace: hermes
+  labels:
+    app: oauth2-proxy-hermes
+spec:
+  selector:
+    app: oauth2-proxy-hermes
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: oauth2-proxy-hermes
+  namespace: hermes
+  labels:
+    app: oauth2-proxy-hermes
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: oauth2-proxy-hermes
+  template:
+    metadata:
+      labels:
+        app: oauth2-proxy-hermes
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/role: hermes
+        vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/operator-oidc
+        vault.hashicorp.com/agent-inject-template-oidc-config: |
+          {{- with secret "kv/data/atlas/hermes/operator-oidc" -}}
+          client_id = "{{ .Data.data.client_id }}"
+          client_secret = "{{ .Data.data.client_secret }}"
+          cookie_secret = "{{ .Data.data.cookie_secret }}"
+          {{- end -}}
+    spec:
+      serviceAccountName: hermes-vault
+      automountServiceAccountToken: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values:
+                      - arm64
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values:
+                      - "true"
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-13
+                      - titan-15
+                      - titan-17
+                      - titan-18
+                      - titan-19
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 90
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi4
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0@sha256:dcb6ff8dd21bf3058f6a22c6fa385fa5b897a9cd3914c88a2cc2bb0a85f8065d
+          imagePullPolicy: IfNotPresent
+          args:
+            - --provider=oidc
+            - --config=/vault/secrets/oidc-config
+            - --redirect-url=https://agent.bstein.dev/oauth2/callback
+            - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas
+            - --code-challenge-method=S256
+            - --scope=openid profile email
+            - --email-domain=*
+            - --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails
+            - --set-xauthrequest=true
+            - --session-cookie-minimal=true
+            - --cookie-secure=true
+            - --cookie-samesite=lax
+            - --cookie-refresh=0
+            - --cookie-expire=8h
+            - --upstream=http://hermes.hermes.svc.cluster.local:9119
+            - --http-address=0.0.0.0:4180
+            - --skip-provider-button=true
+            - --skip-jwt-bearer-tokens=true
+            - --cookie-domain=agent.bstein.dev
+            - --reverse-proxy=true
+          ports:
+            - name: http
+              containerPort: 4180
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            initialDelaySeconds: 20
+            periodSeconds: 20
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests:
+              cpu: 25m
+              memory: 64Mi
+            limits:
+              cpu: 250m
+              memory: 256Mi
+          volumeMounts:
+            - name: allowlist
+              mountPath: /etc/oauth2-proxy
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: allowlist
+          configMap:
+            name: hermes-operator-allowlist
+        - name: tmp
+          emptyDir:
+            sizeLimit: 64Mi

services/hermes/ollama-deployment.yaml

@@ -18,8 +18,8 @@ spec:
       labels:
         app: hermes-ollama
       annotations:
-        ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
+        ai.bstein.dev/model: gpt-oss:20b
-        ai.bstein.dev/gpu: accelerator MVP lane (titan-24)
+        ai.bstein.dev/gpu: titan-24 local-first lane
     spec:
       runtimeClassName: nvidia
       affinity:
@@ -33,7 +33,8 @@ spec:
                       - titan-24
       volumes:
         - name: models
-          emptyDir: {}
+          persistentVolumeClaim:
+            claimName: hermes-models
       initContainers:
         - name: warm-model
           image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
@@ -44,7 +45,7 @@ spec:
             - name: OLLAMA_MODELS
               value: /root/.ollama
             - name: OLLAMA_MODEL
-              value: qwen2.5:7b-instruct-q4_0
+              value: gpt-oss:20b
             - name: NVIDIA_VISIBLE_DEVICES
               value: all
             - name: NVIDIA_DRIVER_CAPABILITIES
@@ -68,7 +69,7 @@ spec:
               nvidia.com/gpu.shared: 1
             limits:
               cpu: "4"
-              memory: 10Gi
+              memory: 16Gi
               nvidia.com/gpu.shared: 1
       containers:
         - name: ollama
@@ -82,6 +83,16 @@ spec:
               value: 0.0.0.0
             - name: OLLAMA_KEEP_ALIVE
               value: 6h
+            - name: OLLAMA_CONTEXT_LENGTH
+              value: "64000"
+            - name: OLLAMA_FLASH_ATTENTION
+              value: "1"
+            - name: OLLAMA_KV_CACHE_TYPE
+              value: q8_0
+            - name: OLLAMA_MAX_LOADED_MODELS
+              value: "1"
+            - name: OLLAMA_NUM_PARALLEL
+              value: "1"
             - name: OLLAMA_MODELS
               value: /root/.ollama
             - name: NVIDIA_VISIBLE_DEVICES
@@ -100,10 +111,10 @@ spec:
             timeoutSeconds: 5
           resources:
             requests:
-              cpu: "2"
+              cpu: "8"
-              memory: 8Gi
+              memory: 24Gi
               nvidia.com/gpu.shared: 1
             limits:
-              cpu: "6"
+              cpu: "16"
-              memory: 12Gi
+              memory: 40Gi
               nvidia.com/gpu.shared: 1

services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js

@@ -0,0 +1,88 @@
+(function () {
+  "use strict";
+
+  var pluginName = "atlas-operator-guide";
+  var storageKey = "atlas.operator-guide.hidden.v1";
+  var prompt = [
+    "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint.",
+    "I will perform every setup and evidence step myself.",
+    "Start with step 1 only, require live proof, and stop at every mutation or approval boundary.",
+    "First confirm this session is using openai-codex/gpt-5.6-terra rather than the local fallback."
+  ].join(" ");
+
+  function OperatorGuide() {
+    var sdk = window.__HERMES_PLUGIN_SDK__;
+    var React = sdk.React;
+    var state = sdk.hooks.useState(function () {
+      return window.localStorage.getItem(storageKey) === "1";
+    });
+    var hidden = state[0];
+    var setHidden = state[1];
+    var copiedState = sdk.hooks.useState(false);
+    var copied = copiedState[0];
+    var setCopied = copiedState[1];
+
+    if (hidden) {
+      return React.createElement(
+        "button",
+        {
+          className: "atlas-guide-restore",
+          onClick: function () {
+            window.localStorage.removeItem(storageKey);
+            setHidden(false);
+          },
+          type: "button"
+        },
+        "Show Atlas operator walkthrough"
+      );
+    }
+
+    function copyPrompt() {
+      navigator.clipboard.writeText(prompt).then(function () {
+        setCopied(true);
+        window.setTimeout(function () { setCopied(false); }, 2500);
+      });
+    }
+
+    return React.createElement(
+      "section",
+      { className: "atlas-guide-card", role: "region", "aria-label": "Atlas operator walkthrough" },
+      React.createElement(
+        "div",
+        { className: "atlas-guide-heading" },
+        React.createElement("div", null,
+          React.createElement("strong", null, "Start here: two-hour Hermes proof sprint"),
+          React.createElement("p", null, "You do the setup and evidence work. Hermes coaches, checks, and stops at approval boundaries.")
+        ),
+        React.createElement(
+          "button",
+          {
+            className: "atlas-guide-hide",
+            onClick: function () {
+              window.localStorage.setItem(storageKey, "1");
+              setHidden(true);
+            },
+            type: "button"
+          },
+          "Hide"
+        )
+      ),
+      React.createElement(
+        "ol",
+        { className: "atlas-guide-steps" },
+        React.createElement("li", null, "Confirm Models shows openai-codex/gpt-5.6-terra for the coaching session."),
+        React.createElement("li", null, "Copy the sprint prompt, paste it into Chat, and press Enter."),
+        React.createElement("li", null, "Complete one live triage and build one writable skill yourself.")
+      ),
+      React.createElement(
+        "div",
+        { className: "atlas-guide-actions" },
+        React.createElement("a", { className: "atlas-guide-button", href: "/models" }, "Open Models"),
+        React.createElement("button", { className: "atlas-guide-button", onClick: copyPrompt, type: "button" }, copied ? "Prompt copied" : "Copy sprint prompt"),
+        React.createElement("a", { className: "atlas-guide-link", href: "/skills" }, "Inspect skills")
+      )
+    );
+  }
+
+  window.__HERMES_PLUGINS__.registerSlot(pluginName, "chat:top", OperatorGuide);
+}());

services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css

@@ -0,0 +1,64 @@
+.atlas-guide-card {
+  border: 1px solid rgba(247, 201, 72, 0.7);
+  background: rgba(247, 201, 72, 0.08);
+  color: inherit;
+  padding: 0.75rem 1rem;
+  font-family: "Mondwest", monospace;
+}
+
+.atlas-guide-heading {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 1rem;
+}
+
+.atlas-guide-heading strong {
+  color: #f7c948;
+  font-family: "Rules Expanded", sans-serif;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+}
+
+.atlas-guide-heading p {
+  margin: 0.25rem 0 0;
+  opacity: 0.8;
+}
+
+.atlas-guide-steps {
+  margin: 0.6rem 0;
+  padding-left: 1.25rem;
+}
+
+.atlas-guide-actions {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 0.5rem;
+}
+
+.atlas-guide-button,
+.atlas-guide-hide,
+.atlas-guide-restore {
+  border: 1px solid rgba(247, 201, 72, 0.7);
+  background: transparent;
+  color: #f7c948;
+  cursor: pointer;
+  padding: 0.35rem 0.6rem;
+  text-decoration: none;
+}
+
+.atlas-guide-button:hover,
+.atlas-guide-hide:hover,
+.atlas-guide-restore:hover {
+  background: rgba(247, 201, 72, 0.14);
+}
+
+.atlas-guide-hide,
+.atlas-guide-restore {
+  font-size: 0.75rem;
+}
+
+.atlas-guide-link {
+  color: #f7c948;
+}

services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json

@@ -0,0 +1,16 @@
+{
+  "name": "atlas-operator-guide",
+  "label": "Atlas Operator Guide",
+  "description": "Hands-on onboarding for supervised Atlas triage and Hermes skill building.",
+  "icon": "Sparkles",
+  "version": "1.0.0",
+  "tab": {
+    "path": "/atlas-operator-guide",
+    "hidden": true
+  },
+  "slots": [
+    "chat:top"
+  ],
+  "entry": "dist/index.js",
+  "css": "dist/style.css"
+}

services/hermes/pvc.yaml

@@ -13,3 +13,18 @@ spec:
   resources:
     requests:
       storage: 4Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: hermes-models
+  namespace: hermes
+  labels:
+    app: hermes-ollama
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 30Gi

services/hermes/rbac.yaml

@@ -22,6 +22,7 @@ rules:
       - pods
       - pods/log
       - replicationcontrollers
+      - serviceaccounts
       - services
     verbs: ["get", "list", "watch"]
   - apiGroups: ["apps"]
@@ -54,6 +55,12 @@ rules:
       - gitrepositories
       - helmrepositories
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["image.toolkit.fluxcd.io"]
+    resources:
+      - imagepolicies
+      - imagerepositories
+      - imageupdateautomations
+    verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -67,4 +74,3 @@ subjects:
   - kind: ServiceAccount
     name: hermes-triage
     namespace: hermes
-

services/hermes/skills/master-hermes-on-atlas/SKILL.md

@@ -0,0 +1,84 @@
+---
+name: master-hermes-on-atlas
+description: Coach Brad through hands-on mastery of the live Atlas Hermes deployment, including its web and CLI interfaces, local GPU inference and Codex fallback, tools, skills, sessions, profiles, channels, security boundaries, and evidence-driven test-failure triage. Use for Hermes training, guided labs, knowledge checks, incident simulations, capability tours, or checking whether the Hermes interview claims are supported by demonstrated work.
+---
+
+# Master Hermes on Atlas
+
+Act as a demanding, practical coach. Teach the deployed system, not a generic
+Hermes installation. Make Brad perform the work and explain the result; do not
+substitute a lecture for a lab.
+
+## Select the training model
+
+Use `openai-codex/gpt-5.6-terra` for assessments, multi-reference labs, incident
+grading, and skill evaluation. If the active provider is the local
+`gpt-oss:20b`, stop before reading the reference files and ask Brad to select
+Codex in Models or start a Codex-backed session. Continue locally only when Brad
+explicitly asks to evaluate the local model itself or requests a lightweight
+single-question review.
+
+## Start or resume training
+
+1. Read `references/architecture.md` before teaching deployment-specific facts.
+2. Read `references/curriculum.md` to select the next lab.
+3. When Brad asks for the two-hour proof sprint, read
+   `references/two-hour-proof-sprint.md` and run it in order. Brad performs
+   every required UI action and evidence check; coach and verify without doing
+   the learning-critical steps for him.
+4. Ask whether to assess, resume, or choose a lab. If no progress record exists,
+   default to a five-question assessment followed by the first weak area.
+5. Give one bounded task at a time. State the goal, safety boundary, exact
+   success evidence, and at most one initial hint.
+6. Wait for Brad's answer or observed command output before revealing the
+   explanation.
+7. Grade with `references/mastery-rubric.md`. Separate demonstrated ability
+   from verbal familiarity.
+8. Offer a concise progress update. Write it to
+   `/opt/data/workspace/hermes-training/progress.md` only after Brad explicitly
+   approves the file change.
+
+## Run live exercises safely
+
+- Use read-only commands by default: `kubectl get`, `describe`, `logs`, `auth
+  can-i`, HTTP GET, `hermes status`, and Hermes list/status commands.
+- Never read Kubernetes Secret values. Never use Vault reads as a training
+  shortcut. Redact bearer tokens, cookies, device codes, and credentials.
+- Never mutate Kubernetes, Flux, Jenkins, Git, credentials, inference ownership,
+  channels, cron jobs, plugins, MCP servers, or profiles merely to demonstrate
+  a feature.
+- Draft state-changing commands and explain their effect. Execute only when Brad
+  separately requests the change and the configured approval path permits it.
+- Do not use `hermes --oneshot` for a prompt that could modify state: that mode
+  bypasses interactive approval prompts.
+- Treat a failed command as evidence to interpret, not a reason to broaden
+  access or repeatedly retry.
+
+## Teach evidence-driven triage
+
+For a live test-failure lab, also use `$triage-titan-test-failures`. Require the
+student to distinguish:
+
+- observed fact from inference;
+- stale evidence from current state;
+- application regression from environment failure;
+- a read-only next check from a proposed mutation;
+- a generic suggestion from the smallest Flux-tracked repo-side fix.
+
+For offline practice, read `references/incident-drills.md` and present only the
+student packet for one incident. Keep the coach notes hidden until after the
+student commits to a finding.
+
+## Validate claims honestly
+
+Use the claim audit in `references/mastery-rubric.md`. Current configuration and
+live demonstrations support current-use claims. The retained Git history and
+bound OpenClaw PVC support the deployment/replacement history; require Brad to
+inspect that evidence and distinguish it from proof of every past interaction.
+
+End every session with:
+
+- what Brad demonstrated;
+- what remains unproven;
+- the next smallest lab;
+- any safety or operational issue discovered.

services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Master Hermes on Atlas"
+  short_description: "Practice Hermes safely on the live Atlas stack"
+  default_prompt: "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint. I will perform every setup and evidence step myself. Start with step 1 only, require live proof, and stop at every mutation or approval boundary. First confirm this session is using openai-codex/gpt-5.6-terra rather than the local fallback."

services/hermes/skills/master-hermes-on-atlas/references/architecture.md

@@ -0,0 +1,117 @@
+# Live Atlas Hermes architecture
+
+Use this reference for deployment-specific facts. Re-check live state before
+asserting health, placement, ownership, or current model availability.
+
+## Two isolated agent instances
+
+| Surface | Purpose | Identity boundary | State and permissions |
+| --- | --- | --- | --- |
+| `agent.bstein.dev` | Brad's operator and triage lab | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace, its own PVC and service account; read-only cluster triage plus approved internal evidence endpoints |
+| `chat.bstein.dev` | Consumer chat and research | Keycloak login | `hermes-chat` namespace, separate PVC/config/auth/sessions/skills; read-only cluster observer RBAC and no access to Secrets or Kubernetes mutation APIs |
+
+The instances do not share conversation state, credentials, profiles, skills
+created on their PVCs, or Kubernetes identities. They share only the inference
+service described below. Consumer users cannot reach the operator dashboard
+through the chat Service or Ingress.
+
+## Inference path
+
+```text
+operator Hermes ───> independently authenticated openai-codex/gpt-5.6-terra
+consumer Hermes ───> independently authenticated openai-codex/gpt-5.6-terra
+       │
+       └─ provider error or manual selection ─> hermes-model-gate
+                                                │
+                                                ├─ Hermes owns GPU ─> Ollama gpt-oss:20b
+                                                └─ Wolf owns GPU ───> HTTP 503
+```
+
+- The configured context length is 64,000 tokens.
+- `hermes-model-gate` is the stable OpenAI-compatible endpoint.
+- A coordination Lease stores whether Hermes or Wolf/Moonlight owns the GPU.
+  Ariadne has narrowly scoped permission to update that Lease during handoff.
+- Wolf ownership makes the local gate return a deliberate unavailable response.
+  Normal interactive chat remains on its Codex primary and does not consume
+  titan-24 GPU resources.
+- The two agent pods do not consume titan-24 GPU memory. Ollama does. Agent pods
+  are ARM gateway workloads with persistent state on separate PVCs.
+- Codex credentials are deliberately per-instance. Never copy Brad's Codex
+  credential store into the consumer PVC.
+
+Verify rather than memorize:
+
+```sh
+hermes status
+hermes fallback list
+kubectl -n hermes get lease titan-24-gpu-owner -o yaml
+kubectl -n hermes get deploy hermes hermes-model-gate hermes-ollama -o wide
+kubectl -n hermes-chat get deploy hermes-chat -o wide
+```
+
+Do not change the GPU owner as part of a lesson.
+
+## Retained OpenClaw replacement evidence
+
+OpenClaw is not part of the current inference path, but its deployment history
+is retained:
+
+- `1bc58e10` added the OpenClaw testing-triage workspace, workload, model,
+  ingress, RBAC, and storage.
+- follow-up commits wired local diagnosis, OIDC, persistence, and the gateway.
+- `97ecb36b` replaced OpenClaw with Hermes and moved the agent-facing resources.
+- `bb5e286f` decoupled Hermes from the retired OpenClaw Flux dependency.
+- the `openclaw-home` PVC remains bound as retained state while no OpenClaw
+  workload is running.
+
+Have Brad verify the commits and live resource shape. This establishes the
+cluster deployment and replacement sequence; Brad's direct experience remains
+the evidence for how he personally used the old agent.
+
+## Operator evidence path
+
+The operator role is designed for supervised test-failure triage:
+
+1. Ariadne deterministic bundle and optional diagnosis.
+2. Retained Jenkins logs and artifacts.
+3. Recent Git changes and Flux revision/state.
+4. Kubernetes workload, node, event, storage, DNS, and dependency health.
+5. Pushgateway quality metrics and VictoriaMetrics/Grafana context.
+6. A concise finding, confidence, evidence, blast radius, next checks,
+   repo-side fix, and approval-required section.
+
+The versioned `$triage-titan-test-failures` skill encodes this workflow. The
+operator has internal URLs for Ariadne, Jenkins, and VictoriaMetrics. The
+consumer instance intentionally does not.
+
+## Security layers and their limits
+
+Security is layered rather than delegated to a prompt:
+
+1. Keycloak authenticates the human.
+2. The operator proxy limits `agent.bstein.dev` to Brad's exact email.
+3. Separate namespaces, PVCs, service accounts, configs, and ingresses isolate
+   operator and consumer state.
+4. Kubernetes RBAC grants the consumer get/list/watch only and excludes Secrets,
+   exec, attach, port-forward, and mutation.
+5. NetworkPolicy permits the consumer to reach DNS, the shared model gate, the
+   Kubernetes API, and public IPv4 while blocking arbitrary private service
+   access.
+6. Hermes deny patterns and instructions add user-facing guardrails.
+
+Prompt rules and shell deny patterns are not the primary security boundary.
+RBAC, network policy, credential separation, and routing isolation are. The
+consumer can still disclose non-secret cluster metadata it is authorized to
+read; that is an intentional requirement and should be described honestly.
+
+## CLI-to-web mental model
+
+The pinned Hermes build exposes chat, models and fallback, auth, sessions,
+profiles, skills and bundles, plugins, tools, MCP, gateway/channels, pairing,
+webhooks, cron, kanban, projects, hooks, memory/journey, logs, security, backup,
+and diagnostics. The web interface is a control surface over many of these
+same persisted capabilities.
+
+Use `<command> --help` and the web Documentation page for the installed version.
+Do not rely on screenshots or online docs from a different release when a live
+command can settle the question.

services/hermes/skills/master-hermes-on-atlas/references/curriculum.md

@@ -0,0 +1,131 @@
+# Hands-on curriculum
+
+Complete labs by evidence, not elapsed time. A focused pass can establish
+operational competence in several days; mastery requires repeating real triage
+and recovery work over multiple incidents.
+
+Run the curriculum on `openai-codex/gpt-5.6-terra`. Use the local model only for a
+deliberate comparison lab; it is not the default coach for multi-reference work.
+
+## Phase 1: orientation and control
+
+### Lab 1 — Map the system
+
+From the web UI, identify Chat, Sessions, Files, Models, Logs, Cron, Skills,
+Plugins, MCP, Channels, Webhooks, Pairing, Profiles, Config, Keys, System, and
+Documentation. Explain which state belongs to the operator PVC and which
+components are shared. Verify three claims with read-only CLI output.
+
+Success evidence: a correct diagram or written request path from browser to
+agent to model, including the fallback branch and the consumer boundary.
+
+### Lab 2 — Models, context, and fallback
+
+Inspect `hermes status`, `hermes fallback list`, deployment placement, and GPU
+owner state. Explain why a 32K model was rejected, why normal Codex-backed chat
+remains available when Wolf owns the GPU, and when the local fallback can run.
+
+Success evidence: predict outcomes for local healthy, local slow, gate 503,
+invalid local response, and expired Codex authorization without changing state.
+
+### Lab 3 — Sessions, files, profiles, and logs
+
+Create a named training session and a harmless workspace note through the UI,
+then find the corresponding session/file/log surfaces. Inspect profile and
+backup help without creating a profile or backup.
+
+Success evidence: explain persistence, what survives a pod replacement, what is
+instance-local, and how to recover a lost conversation without exposing auth.
+
+## Phase 2: tools and safe autonomy
+
+### Lab 4 — Tools versus skills versus MCP
+
+Use `hermes tools`, `hermes skills`, `hermes plugins`, and `hermes mcp` help or
+list/status views. Classify each as executable capability, procedure/context,
+packaged extension, or external protocol integration. Explain why a skill is
+not a security boundary.
+
+Success evidence: choose the right mechanism for three examples: repeatable
+Titan triage, a read-only external API, and a scheduled notification.
+
+### Lab 5 — Prove the permission boundary
+
+Use `kubectl auth can-i` as both service accounts. Inspect the relevant
+ClusterRoles and NetworkPolicies. Do not attempt mutations.
+
+Success evidence: an allow/deny matrix covering pods, logs, Secrets, exec,
+deployment patch, Flux reads, Flux reconcile, the model gate, internal services,
+and public web research.
+
+### Lab 6 — Channels, pairing, webhooks, and cron
+
+Inspect the configured state and help for gateway/channels, pairing, webhooks,
+and cron. Design one safe notification workflow and one unsafe workflow. Do not
+register a channel, create a webhook, or schedule a job during the lab.
+
+Success evidence: identify the identity, secret, audience, tool policy, failure
+mode, audit trail, and revocation path for the proposed integration.
+
+## Phase 3: the interview workflow
+
+### Lab 7 — Deterministic live triage
+
+Invoke `$triage-titan-test-failures`. Read the current Ariadne diagnosis and
+bundle, then correlate one suite with Jenkins, Flux/Git, cluster health, and
+quality metrics. Keep every collection step read-only.
+
+Success evidence: the required structured triage result with timestamps and no
+invented evidence.
+
+### Lab 8 — Incident simulations
+
+Use one student packet from `incident-drills.md`. Diagnose it before receiving
+coach notes. Repeat until three cases score at least `Independent`.
+
+Success evidence: correct failure class, causal chain, next checks, minimal
+repo-side correction, and explicit approval boundary.
+
+### Lab 9 — Build and evaluate a reusable skill
+
+Identify a stable repeated workflow, create a small skill in the operator's
+writable skills directory, inspect it, and test it against a fresh incident.
+Do not alter the Flux-mounted training or triage skills.
+
+Success evidence: precise trigger description, concise procedure, progressive
+disclosure where useful, one passing case, one adversarial case, and a stated
+permission boundary.
+
+### Lab 10 — Model outage drill
+
+Use retained logs/current status or a separately approved maintenance window.
+Do not take the live model down for training. Explain the expected trace from
+model gate to fallback, recognize an auth failure, and name the restoration
+checks.
+
+Success evidence: correctly distinguish model unavailability, context rejection,
+provider auth failure, and agent failure.
+
+## Phase 4: independent operation
+
+### Lab 11 — Supervised real incident
+
+Lead a real failure triage from intake to an approved repo-side proposal. A
+human applies any change through Git/Flux. Verify the result read-only.
+
+### Lab 12 — Teach it back and audit the claims
+
+Explain the complete architecture and demonstrate the workflow without hints.
+Audit each sentence of the interview answer using the claim table in
+`mastery-rubric.md`. Weaken or qualify anything not supported by evidence.
+
+## Suggested pace
+
+- Day 1: Labs 1–3.
+- Days 2–3: Labs 4–6.
+- Days 4–7: Labs 7–10.
+- Weeks 2–4: repeat Labs 7–11 on real incidents; improve one skill from observed
+  failures; finish with Lab 12.
+
+Do not advance solely because a day elapsed. Re-run any lab graded below
+`Independent`.

services/hermes/skills/master-hermes-on-atlas/references/incident-drills.md

@@ -0,0 +1,95 @@
+# Incident drills
+
+Present only one `Student packet` before the student answers. Use the coach
+notes afterward to grade the failure classification and reasoning. These cases
+are based on retained Atlas rollout observations; treat them as training
+fixtures, not proof of current live state.
+
+## Case A — Metrics disappear after a Soteria run
+
+### Student packet
+
+- The Soteria test stages ran.
+- The post-stage log ends with:
+  `Syntax error: end of file unexpected (expecting "fi")`.
+- The canonical Soteria Pushgateway series did not refresh.
+- Other suites continued publishing.
+
+Ask for: finding, confidence, evidence, likely cause, blast radius, read-only
+next checks, repo-side fix, and approval-required actions.
+
+### Coach notes
+
+The primary class is pipeline glue/shell syntax, not a Pushgateway outage and
+not a product-test failure. Inspect the post block and the exact shell parsed by
+`/bin/sh`; confirm the last successful metric timestamp. The smallest likely
+fix is closing/correcting the conditional in the Jenkinsfile, then a normal SCM
+run and read-only metric verification. Editing, pushing, triggering, or manual
+metric backfill requires explicit approval.
+
+## Case B — UI tests fail before metrics publish
+
+### Student packet
+
+- `bstein-dev-home` requires Playwright `1.59.1`.
+- The CI test container contains Playwright `1.51.0`.
+- The frontend test stage hard-fails.
+- No fresh canonical quality metrics appear for the run.
+
+### Coach notes
+
+Separate the direct test-environment mismatch from the telemetry-control-flow
+defect. The version mismatch explains the test failure; early pipeline abort
+explains missing metrics. A complete proposal aligns the image version and
+persists stage return codes so publishing still runs while the final gate still
+fails. Do not call this a Kubernetes capacity problem without node/pod evidence.
+
+## Case C — Promotion exits 127
+
+### Student packet
+
+- Titan IaC validation and tests finish.
+- The Promote stage uses `python:3.12-slim`.
+- The stage returns exit code `127` at its first Git command.
+- The repository and remote are reachable from other jobs.
+
+### Coach notes
+
+Exit 127 means the invoked command is unavailable. Confirm the exact log line
+and image contents; the likely issue is that the slim image lacks `git`, not bad
+Git credentials. The minimal repo-side fix installs `git` and CA certificates
+in the job's dependency setup or uses a suitable pinned runner image.
+
+## Case D — Data Preppers cannot publish an image
+
+### Student packet
+
+- Tests and local image build complete.
+- Push to `registry.bstein.dev/streaming/data-prepper:2.8.0` is denied.
+- The configured Jenkins credential belongs to a different Harbor project.
+- A Sonar evidence GET also returns `401` in the same run.
+
+### Coach notes
+
+Do not collapse independent authentication failures into a generic network
+failure. Verify the Harbor repository/project and credential ID without reading
+the secret value; separately verify the Sonar token injection path and API
+scope. The likely image fix is selecting a streaming-scoped robot credential.
+Treat the Sonar 401 as a second issue unless evidence proves a shared cause.
+
+## Case E — Hermes local inference fails during game streaming
+
+### Student packet
+
+- Wolf owns titan-24 according to the GPU-owner state.
+- `hermes-model-gate` is ready but returns a deliberate unavailable response.
+- Both Hermes dashboards remain healthy.
+- One instance answers through Codex; the other reports missing provider auth.
+
+### Coach notes
+
+This is expected resource arbitration plus an instance-specific fallback auth
+gap. Do not restart the agent or claim the GPU is underprovisioned. Confirm the
+gate response, fallback chain, and each instance's auth status without exposing
+tokens. Authorize the affected instance with its intended user's account. The
+separate credential stores are a security property, not configuration drift.

services/hermes/skills/master-hermes-on-atlas/references/mastery-rubric.md

@@ -0,0 +1,75 @@
+# Mastery and claim rubric
+
+## Performance levels
+
+| Level | Evidence |
+| --- | --- |
+| Exposed | Recognizes terms but needs the path and commands supplied. |
+| Assisted | Completes a lab with hints and can explain the result afterward. |
+| Independent | Selects safe tools, gathers fresh evidence, and reaches a defensible result without hints. |
+| Mastery | Handles ambiguity and failure, teaches the architecture, improves the reusable workflow, and preserves security boundaries. |
+
+Score each domain separately:
+
+1. Architecture and request routing.
+2. Models, context, GPU ownership, and fallback.
+3. Sessions, files, profiles, logs, and recovery.
+4. Tools, skills, plugins, MCP, channels, webhooks, and cron.
+5. Identity, RBAC, network, credentials, approvals, and auditability.
+6. Jenkins/Flux/Kubernetes/metrics evidence correlation.
+7. Clear triage findings and minimal repo-side proposals.
+8. Skill creation, evaluation, and iteration.
+
+Never award `Independent` from a verbal answer alone when the lab calls for live
+or fixture evidence. Never award `Mastery` until Brad has led at least two
+different real incidents and improved a workflow based on what failed.
+
+## Interview claim audit
+
+| Claim | What supports it | Minimum demonstration |
+| --- | --- | --- |
+| Hermes runs in the Kubernetes cluster | Flux manifests and live workloads | Trace browser, agent, model gate, Ollama, PVC, and service account. |
+| Hermes fits existing automation | Operator config, internal evidence URLs, triage skill | Complete a triage using Ariadne, Jenkins, Git/Flux, cluster health, and metrics. |
+| Hermes follows Brad's triage path | `$triage-titan-test-failures` procedure | Lead two evidence-backed cases with facts separated from inference. |
+| Repeated workflows become reusable skills | Versioned triage/training skills plus writable user skill area | Build, test, and improve one skill from a repeated real workflow. |
+| Access starts read-only and scoped | RBAC, NetworkPolicy, separate identities and state | Produce and explain the allow/deny matrix from live authorization checks. |
+| Human approval protects changes | Approval config, deny patterns, Flux workflow | Identify every mutation in a proposed incident response and stop before it. |
+| Local GPU gracefully yields to gaming | Owner state, model gate, fallback | Explain or observe a Wolf ownership window and verify continued fallback service. |
+| OpenClaw previously ran and was replaced | `1bc58e10` deploys the OpenClaw triage stack; follow-up commits add gateway/OIDC/persistence; `97ecb36b` replaces it with Hermes; `bb5e286f` retires the dependency; the bound `openclaw-home` PVC remains without a workload | Inspect the commits and live Flux/PVC state, then explain what they prove versus what depends on Brad's personal recollection. |
+
+## Triage answer scoring
+
+Award one point for each:
+
+- names the correct failure class;
+- cites timestamped/build-specific evidence;
+- labels inference;
+- checks evidence freshness;
+- identifies realistic blast radius;
+- orders read-only next checks;
+- proposes the smallest Flux/repo-side change or says evidence is insufficient;
+- isolates approval-required actions;
+- avoids invented resources, logs, metrics, or commits;
+- remains concise enough for an operator to act on.
+
+Scores below 8/10 require another incident drill. Any invented evidence or
+unacknowledged mutation caps the result at `Assisted`.
+
+## Progress record format
+
+When Brad approves recording progress, use this compact structure:
+
+```markdown
+# Hermes training progress
+
+- Last session: <UTC timestamp>
+- Current level: <per-domain summary>
+- Completed: <labs with evidence>
+- Needs repetition: <specific gaps>
+- Next lab: <one lab>
+- Safety findings: <none or concrete issue>
+- Claim status: <supported, partially supported, unproven>
+```
+
+Append evidence links or command summaries; never record tokens, cookies,
+credentials, device codes, or Secret values.

services/hermes/skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md

@@ -0,0 +1,86 @@
+# Atlas two-hour proof sprint
+
+The sprint establishes usable, defensible evidence for the current Hermes
+claims. It does not award mastery. Retained Git and PVC evidence establishes
+the OpenClaw deployment/replacement sequence; Brad's direct experience remains
+the evidence for how he personally used it. Brad performs the work; Hermes
+gives one task at a time, checks the evidence, and withholds later answers until
+Brad commits to a result.
+
+## 0–15 minutes — control surface and request path
+
+1. Have Brad start a new session and confirm it uses
+   `openai-codex/gpt-5.6-terra`.
+2. In the web UI, have him locate Chat, Sessions, Files, Models, Logs, Skills,
+   Plugins, MCP, Channels, Webhooks, Cron, Profiles, Config, Keys, System, and
+   Documentation.
+3. Have him explain browser → operator agent → Codex primary, including the
+   model-gate/Ollama fallback branch and the separate consumer instance.
+
+Evidence: Brad can identify persistent, shared, and instance-local state and
+can name the GPU owner check without changing it.
+
+## 15–35 minutes — permissions and safety
+
+1. Have Brad inspect the operator ServiceAccount, ClusterRole, NetworkPolicy,
+   and approval deny list using read-only commands.
+2. Have him produce an allow/deny matrix for pod/log reads, Secret reads,
+   exec, deployment mutation, Flux reads, Flux reconcile, and internal
+   evidence endpoints.
+3. Require him to explain why prompt instructions and skills are not the
+   primary security boundary.
+
+Evidence: correct identity/RBAC/network/approval layering with no Secret values
+read and no mutation attempted.
+
+## 35–75 minutes — perform the claimed triage path
+
+Invoke `$triage-titan-test-failures`. Brad must:
+
+1. check Ariadne diagnosis and bundle freshness;
+2. identify one current or retained failed suite/build;
+3. correlate Jenkins log/artifact evidence;
+4. correlate Git/Flux revision and Kubernetes environment health;
+5. query the relevant Pushgateway/VictoriaMetrics quality signal and identify
+   the matching Grafana context;
+6. produce Finding, Confidence, Evidence, Likely cause, Blast radius, Next
+   checks, Repo-side fix, and Approval required.
+
+Hermes may suggest a read-only command after Brad explains why it is needed.
+Do not let the built-in diagnosis replace Brad's own evidence correlation.
+
+Evidence: an 8/10 or better result under the triage rubric with no invented
+facts and no unacknowledged mutation.
+
+## 75–105 minutes — build the reusable workflow
+
+1. Have Brad inspect the mounted `triage-titan-test-failures` skill and explain
+   its trigger, fixed procedure, variable evidence, and security boundary.
+2. Have Brad choose one genuinely repeated sub-workflow from the triage.
+3. Guide him to create a new skill in the operator's writable skill area. Do
+   not edit the mounted Flux skill and do not write the skill for him.
+4. Test one expected trigger and one adversarial/non-trigger case.
+5. Have Brad make one improvement based on the test results.
+
+Evidence: Brad can find the skill in the Skills UI, explain every instruction,
+and show a passing and adversarial evaluation.
+
+## 105–120 minutes — supervised proposal and claim audit
+
+1. Have Brad turn the triage into the smallest repo-side proposal without
+   applying it.
+2. Have him identify every step requiring approval: file edit, commit/push,
+   pipeline trigger, Flux reconcile, credential/environment change, or manual
+   backfill.
+3. Audit the interview answer sentence by sentence using `mastery-rubric.md`.
+   Include the retained OpenClaw commits and live PVC/no-workload state.
+4. Offer to record progress only after Brad approves the write.
+
+Minimum honest outcome:
+
+- current Hermes deployment and supervised triage use are demonstrated;
+- the established Atlas automation path and read-only boundary are explained;
+- one reusable writable skill is created, tested, and improved by Brad;
+- autonomous production mutation is not claimed;
+- historical OpenClaw deployment and replacement are supported by retained Git
+  and PVC evidence, with personal-use details identified as Brad's testimony.

services/hermes/skills/triage-atlas-service-health/SKILL.md

@@ -0,0 +1,66 @@
+---
+name: triage-atlas-service-health
+description: Diagnose active Atlas service and infrastructure problems from Grafana, Ariadne, Flux, Kubernetes events, workload state, and logs. Use when asked what is broken, why a Grafana health panel is red, whether an outage is real, or how to fix a service through titan-iac. Covers every deployed namespace while distinguishing current services from migration residue and historical noise.
+---
+
+# Triage Atlas Service Health
+
+Investigate read-only, group symptoms into incidents, and recommend the smallest Flux-tracked fix.
+
+## Establish the current scope
+
+1. Read `references/service-map.md` for custom-service ownership and migrations.
+2. Discover the live inventory instead of relying only on the map:
+
+   ```sh
+   kubectl get namespaces
+   kubectl get deploy,statefulset,daemonset -A
+   kubectl get ingress -A
+   kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io
+   ```
+
+3. Cassandra is authoritative for functionality migrated from Veles. Veles is
+   retired migration residue: its failed pods, old Jobs, and legacy Service or
+   Ingress objects are not current user impact. Escalate Veles only when direct
+   evidence proves Cassandra still depends on it or the Veles-to-Cassandra
+   redirect itself is failing.
+
+## Decide whether the signal is actionable
+
+Use all of these checks before calling something an incident:
+
+- Current state: prefer an active condition over lifetime restart totals or old failed Jobs.
+- Persistence: ignore normal startup states under 15 minutes unless users are already affected.
+- Ownership: group replica pods by Deployment, StatefulSet, DaemonSet, Job, or CronJob.
+- Delivery: a Flux object with `Ready=null` may be reconciling; `Ready=false` or a suspended required object is actionable.
+- Authority: separate current workloads from retired, migrated, or deliberately suspended resources. Never describe Veles as unavailable or degraded user-facing service merely because its retained pods are unhealthy.
+- Correlation: confirm a red Grafana panel with the underlying metric and at least one independent source such as events, logs, readiness, or Flux state.
+
+Run the narrowest relevant checks:
+
+```sh
+kubectl get pods -A --field-selector status.phase!=Running,status.phase!=Succeeded -o wide
+kubectl get events -A --sort-by=.lastTimestamp
+kubectl -n <namespace> get deploy,statefulset,daemonset,job,cronjob
+kubectl -n <namespace> describe pod <pod>
+kubectl -n <namespace> logs <pod> --all-containers --tail=200
+kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io
+curl -G -fsS --data-urlencode 'query=<promql>' "$VICTORIA_METRICS_URL/api/v1/query"
+```
+
+Never read Kubernetes Secret values. Secret-key errors quoted by a pod event or log are valid evidence; inspect the SecretProviderClass and Vault policy manifests, not the secret contents.
+
+## Produce one incident per cause
+
+Deduplicate replica pods and repeated alerts with a signature of service, workload, failing condition, and likely dependency. Return:
+
+- `Finding`: current incident, degraded-but-serving, migration residue, historical noise, or healthy.
+- `Impact`: the user-facing service and affected capability.
+- `Evidence`: timestamps, workload condition, event/log excerpt, metric, and Flux revision.
+- `Noise removed`: signals inspected but not counted, with the reason.
+- `Likely cause`: causal chain, marking inference explicitly.
+- `Next checks`: ordered read-only commands.
+- `Repo-side fix`: exact titan-iac or application path and smallest proposed change.
+- `Approval required`: every write, reconcile, credential, or external-system action.
+
+Do not claim a proposed change is applied. Do not mutate the cluster; Atlas is Flux-owned.

services/hermes/skills/triage-atlas-service-health/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Triage Atlas Service Health"
+  short_description: "Separate real Atlas incidents from dashboard noise"
+  default_prompt: "Use $triage-atlas-service-health to identify the active service incidents in Atlas, group duplicate symptoms, and recommend read-only checks and the smallest Flux-tracked fixes."

services/hermes/skills/triage-atlas-service-health/references/service-map.md

@@ -0,0 +1,22 @@
+# Atlas custom-service map
+
+Use this map for ownership and canonical names. Discover third-party services live from Flux and Kubernetes.
+
+| Capability | Canonical workload/repository | Runtime location | Notes |
+| --- | --- | --- | --- |
+| Cluster automation and state analysis | Ariadne / `bstein/Ariadne` | `maintenance/ariadne` | Provides deterministic triage bundles. |
+| Quality policy and CI telemetry | titan-iac / `bstein/titan-iac` | Jenkins plus Flux; no single app Deployment | Owns Jenkins configuration, Grafana generators, alert provisioning, and the Data Prepper pipeline. |
+| Simulation registry and execution | Cassandra / Cassandra application repository | `cassandra/cassandra-frontend`, `cassandra-backend`, `cassandra-postgres`, `cassandra-vault-sync` | Cassandra is authoritative after the Veles migration. |
+| Legacy simulation stack | Veles | `veles` namespace | Retired migration residue. Pod failures are not current impact; only a proven Cassandra dependency or broken redirect is actionable. |
+| Backup and maintenance policy | Soteria / `bstein/soteria` | `maintenance/soteria` | Correlate CI failures separately from runtime backup health. |
+| Media client | Pegasus / `bstein/pegasus` | `jellyfin/pegasus` | Correlate with Jellyfin and OIDC dependencies. |
+| Cluster sentinel | Metis / `bstein/metis` | `maintenance/metis` plus sentinel DaemonSets | Distinguish controller health from per-node sentinel evidence. |
+| User and service automation | Ananke / `bstein/ananke` | No current dedicated in-cluster Deployment | Treat its Jenkins suite as the authoritative runtime evidence unless discovery proves otherwise. |
+| Chat automation | Atlasbot / `bstein/atlasbot` | `comms/atlasbot` | Correlate with comms and OIDC dependencies. |
+| Public site | bstein_home / `bstein/bstein-dev-home` | `bstein-dev-home` frontend, backend, and Vault sync | Canonical metric suite uses underscore form. |
+| Data Prepper integration | data_prepper / `bstein/titan-iac` | `logging/data-prepper` | Pipeline is under `services/logging`. |
+| Desktop test application | Lesavka / `bstein/lesavka` | Desktop test host, not a normal Kubernetes Deployment | Test suite is in scope; runtime evidence may come from titan-jh. |
+| Hermes operator | `hermes` namespace / `services/hermes` | `hermes/hermes`, model gate, and Ollama | Read-only cluster operator with Codex primary and local fallback. |
+| Hermes consumer chat | `hermes-chat` namespace / `services/hermes-chat` | `hermes-chat/hermes-chat` | Isolated from cluster operation; do not use it for infrastructure triage. |
+
+Canonical CI suites are `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `lesavka`, `metis`, `pegasus`, `soteria`, and `titan_iac`.

services/hermes/skills/triage-titan-test-failures/SKILL.md

@@ -0,0 +1,120 @@
+---
+name: triage-titan-test-failures
+description: Diagnose Titan CI test failures and environment regressions from Ariadne evidence, Jenkins artifacts, Flux state, Kubernetes health, Pushgateway quality metrics, and Grafana context. Use for failed or flaky in-scope suites, suspected cluster-caused test failures, release-quality questions, or requests for a supervised read-only triage summary and repo-side next steps.
+---
+
+# Triage Titan Test Failures
+
+Follow the established Titan evidence path. Keep the investigation read-only and distinguish observed facts from inference.
+
+## Collect the canonical evidence
+
+1. Read the latest Ariadne diagnosis:
+
+   ```sh
+   curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"
+   ```
+
+2. Always read the deterministic bundle, even when the diagnosis looks complete:
+
+   ```sh
+   curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"
+   ```
+
+3. Ask for human approval before triggering a fresh collection or diagnosis. Use these only after approval:
+
+   ```sh
+   curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/collect"
+   curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"
+   ```
+
+Record the timestamps and status from both responses. Treat the diagnosis as stale when it predates the deterministic bundle, reports `unavailable`, or lacks concrete evidence. In any of those cases, continue the investigation from the deterministic bundle; do not stop at model availability.
+
+Treat Ariadne's bundle as the evidence source of truth. A local-model diagnosis may be unavailable while Wolf owns titan-24; continue from the stored bundle using the active Hermes fallback model.
+
+## Atlas facts and guardrails
+
+- Ariadne runs as the `ariadne` Deployment and Service in the `maintenance` namespace.
+- Hermes local inference runs through `hermes-model-gate` in the `hermes` namespace. Ollama is the `hermes-ollama` Deployment and Service in that namespace.
+- OpenClaw is not part of this inference path. Never suggest an `openclaw` resource unless a read-only query first proves one exists.
+- Custom runtime namespaces do not always match suite names. In particular,
+  Soteria, Metis, and Ariadne run in `maintenance`; Pegasus runs in `jellyfin`;
+  Atlasbot runs in `comms`; Data Prepper runs in `logging`; and Lesavka is a
+  desktop-hosted application. Read the service-health skill's
+  `references/service-map.md` before claiming that a suite has no runtime.
+- A connection error recorded inside an older diagnosis proves only that the model request failed at that timestamp. It does not prove the service is currently down.
+- Never invent a namespace, workload, container, port, URL, log line, metric, commit, or pod condition. Verify a target with a read-only query before presenting an exact follow-up command; otherwise state what must be discovered first.
+- HTTP POST collection and diagnosis endpoints are state-changing operations. Put them only under `Approval required`; never describe them as read-only or include them in the read-only command list.
+
+## Narrow the failure
+
+Work in this order:
+
+1. Confirm the failed suite and build are in the canonical scope: `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `lesavka`, `metis`, `pegasus`, `soteria`, or `titan_iac`.
+2. Identify the first failed gate in the enforced order: `style`, `loc`, `coverage`, `tests`, `gate_glue`, `sonarqube`, `supply_chain`.
+3. Correlate the build timestamp with retained Jenkins logs/artifacts, recent Git commits, and Flux revisions.
+4. Check whether Kubernetes health, node pressure, image pulls, storage, DNS, or a shared dependency explains the failure better than a repo regression.
+5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry. A zero-valued series is not a failure.
+6. Keep running Jenkins builds in an in-progress collection. Do not count them as failed until Jenkins reports a terminal failure state.
+7. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause.
+
+For recent Git context, map the canonical suite to its Gitea repository and
+read the latest commits. The current mappings are:
+
+- `ananke` -> `bstein/ananke`
+- `ariadne` -> `bstein/Ariadne`
+- `atlasbot` -> `bstein/atlasbot`
+- `bstein_home` -> `bstein/bstein-dev-home`
+- `data_prepper` -> `bstein/titan-iac`
+- `lesavka` -> `bstein/lesavka`
+- `metis` -> `bstein/metis`
+- `pegasus` -> `bstein/pegasus`
+- `soteria` -> `bstein/soteria`
+- `titan_iac` -> `bstein/titan-iac`
+
+Use the read-only Gitea API and compare commit timestamps to the failing build:
+
+```sh
+curl -fsS "$GITEA_BASE_URL/api/v1/repos/<owner>/<repo>/commits?limit=10"
+```
+
+For quality telemetry, query VictoriaMetrics directly when the bundle is stale
+or a specific label needs confirmation. URL-encode PromQL rather than manually
+escaping it:
+
+```sh
+curl -G -fsS --data-urlencode \
+  'query=platform_quality_gate_runs_total{suite="<suite>"}' \
+  "$VICTORIA_METRICS_URL/api/v1/query"
+curl -G -fsS --data-urlencode \
+  'query=<suite>_quality_gate_checks_total' \
+  "$VICTORIA_METRICS_URL/api/v1/query"
+```
+
+For image-pull failures, inspect the pod event, its ServiceAccount metadata,
+and Flux image objects without reading referenced Secret values:
+
+```sh
+kubectl -n <namespace> describe pod <pod>
+kubectl -n <namespace> get serviceaccount <name> -o yaml
+kubectl get imagerepositories,imagepolicies,imageupdateautomations -A
+```
+
+If the latest diagnosis is unavailable, the minimum acceptable result still analyzes the deterministic bundle's failed suites, build/check evidence, freshness, and environment observations. Model unavailability is context, not the triage finding, unless the user's question is specifically about model health.
+
+Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands.
+
+## Produce the triage result
+
+Return these sections:
+
+- `Finding`: one sentence naming the most likely failure class.
+- `Confidence`: low, medium, or high, with the reason.
+- `Evidence`: the smallest set of concrete timestamps, build IDs, artifact paths, commits, metrics, pods, nodes, or Flux revisions that support the finding.
+- `Likely cause`: explain the causal chain and label inference as inference.
+- `Blast radius`: affected suites, services, branches, or environments.
+- `Next checks`: ordered read-only checks with exact commands or URLs.
+- `Repo-side fix`: the smallest Flux/IaC or application change, or `none yet` when evidence is insufficient.
+- `Approval required`: call out every step that would modify files, infrastructure, credentials, test environments, or external systems.
+
+Never present a proposed fix as applied. Prefer a concise evidence-backed answer over a general log summary.

services/hermes/skills/triage-titan-test-failures/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Triage Titan Test Failures"
+  short_description: "Analyze evidence for supervised test-failure triage"
+  default_prompt: "Use $triage-titan-test-failures to diagnose the latest Titan test failure from Ariadne evidence."

services/hermes/skills/tune-atlas-alerts/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: tune-atlas-alerts
+description: Audit and tune noisy Atlas Grafana alerts using live VictoriaMetrics evidence, Grafana notification history, persistence, minimum sample sizes, and generator ownership. Use when alerts are over-aggressive, a panel is red without user impact, PromQL behaves impossibly, or a Flux-tracked alert/dashboard correction is needed.
+---
+
+# Tune Atlas Alerts
+
+Reduce false positives without hiding real failures. Read `references/alert-review.md` before recommending a rule change.
+
+## Audit the signal
+
+1. Find the provisioned rule in `services/monitoring/grafana-alerting-config.yaml`.
+2. If the issue is a dashboard panel, find its Python source in `scripts/dashboards_render_atlas.py`; never hand-edit generated JSON or ConfigMaps.
+3. Query the exact PromQL against VictoriaMetrics and inspect the labels and raw inputs.
+4. Compare current value, recent history, notification frequency, and a second source such as Kubernetes events or service metrics.
+5. Classify the problem as bad math, lifetime-versus-rate confusion, missing persistence, low sample size, duplicate series, retired scope, rollout noise, real backlog, or real incident.
+
+Useful read-only commands:
+
+```sh
+curl -G -fsS --data-urlencode 'query=<promql>' "$VICTORIA_METRICS_URL/api/v1/query"
+kubectl -n monitoring logs deploy/grafana --since=24h
+kubectl -n monitoring get configmap grafana-alerting-config -o yaml
+kubectl -n <namespace> get events --sort-by=.lastTimestamp
+```
+
+## Design the correction
+
+- Use `rate`, `increase`, or `delta` according to counter/gauge semantics; clamp percentages to 0..100.
+- Require persistence for startup, rollout, and scheduling conditions.
+- Add a minimum denominator and absolute count to percentage alerts with small samples.
+- Alert on configured resources that regressed; keep unenrolled inventory visible as backlog instead of paging continuously.
+- Exclude retired or migrated resources from high-level health, while retaining them on detailed dashboards.
+- Deduplicate replicas and scrape targets using stable service/workload labels.
+- Preserve a drill-down path to the raw evidence.
+
+Validate changed PromQL live, run the dashboard generator, test the generator, render the monitoring kustomization, and use a client-side dry-run. Report before/after values and what failure will still trigger the rule.
+
+Hermes is read-only in the cluster. Draft or explain repo changes and mark reconcile or deployment actions as approval-required.

services/hermes/skills/tune-atlas-alerts/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Tune Atlas Alerts"
+  short_description: "Make Grafana alerts actionable without hiding faults"
+  default_prompt: "Use $tune-atlas-alerts to audit the currently noisy Atlas alerts, prove which signals are false positives, and propose the smallest generator- or alert-rule corrections."

services/hermes/skills/tune-atlas-alerts/references/alert-review.md

@@ -0,0 +1,16 @@
+# Alert review checklist
+
+For every alert, record:
+
+1. User impact or operator action the alert demands.
+2. Metric type: counter, gauge, timestamp, state marker, or recording rule.
+3. Scope and deduplication labels.
+4. Persistence window and startup grace.
+5. Minimum sample size for ratios.
+6. No-data and query-error behavior.
+7. Current value and a recent range query.
+8. Notification frequency in Grafana logs.
+9. Current, migrated, suspended, or historical resource status.
+10. Exact Flux-tracked source, generated artifacts, tests, and rollback condition.
+
+A rule is useful only when its firing state implies a specific human action. Backlogs and unenrolled resources belong on dashboards unless they have crossed an explicitly accepted operational deadline.

services/hermes/vault-serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/hermes/vault-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-vault
+  namespace: hermes

services/jenkins/configmap-jcasc.yaml

@@ -429,15 +429,15 @@ data:
               }
             }
           }
-          pipelineJob('veles') {
+          pipelineJob('cassandra') {
             disabled(false)
-            description('Staged Veles alpha image pipeline. Backend/frontend should build linux/amd64 and linux/arm64; sim-worker may begin amd64-only if Forge dependencies require it.')
+            description('Staged Cassandra alpha image pipeline. Backend/frontend should build linux/amd64 and linux/arm64; sim-worker may begin amd64-only if Forge dependencies require it.')
             definition {
               cpsScm {
                 scm {
                   git {
                     remote {
-                      url('https://scm.bstein.dev/bstein/veles.git')
+                      url('https://scm.bstein.dev/bstein/cassandra.git')
                       credentials('gitea-pat')
                     }
                     branches('*/main')

services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml

@@ -1,401 +0,0 @@
-# services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml
-# One-off job for sso/cassandra-realm-ensure-1.
-# Purpose: create the Cassandra realm, groups, OIDC client, SMTP settings, and Vault client secret.
-# Keep suspended until Cassandra Vault paths/policies have reconciled, then unsuspend once.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: cassandra-realm-ensure-1
-  namespace: sso
-spec:
-  suspend: true
-  backoffLimit: 0
-  ttlSecondsAfterFinished: 3600
-  template:
-    metadata:
-      annotations:
-        vault.hashicorp.com/agent-inject: "true"
-        vault.hashicorp.com/agent-pre-populate-only: "true"
-        vault.hashicorp.com/role: "sso-secrets"
-        vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
-        vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
-          {{ with secret "kv/data/atlas/shared/keycloak-admin" }}
-          export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
-          export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
-          export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
-          {{ end }}
-          {{ with secret "kv/data/atlas/shared/postmark-relay" }}
-          export KEYCLOAK_SMTP_USER="{{ index .Data.data "apikey" }}"
-          export KEYCLOAK_SMTP_PASSWORD="{{ index .Data.data "apikey" }}"
-          {{ end }}
-    spec:
-      serviceAccountName: mas-secrets-ensure
-      restartPolicy: Never
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/worker
-                    operator: Exists
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 100
-              preference:
-                matchExpressions:
-                  - key: kubernetes.io/arch
-                    operator: In
-                    values: ["arm64"]
-      containers:
-        - name: configure
-          image: python:3.11-alpine
-          env:
-            - name: KEYCLOAK_SERVER
-              value: http://keycloak.sso.svc.cluster.local
-            - name: KEYCLOAK_REALM
-              value: cassandra
-            - name: KEYCLOAK_CLIENT_ID
-              value: cassandra-web
-            - name: KEYCLOAK_PUBLIC_ISSUER
-              value: https://sso.bstein.dev/realms/cassandra
-            - name: CASSANDRA_BASE_URL
-              value: https://cassandra.bstein.dev
-            - name: CASSANDRA_TESTER_GROUP
-              value: cassandra-tester
-            - name: KEYCLOAK_SMTP_HOST
-              value: mail.bstein.dev
-            - name: KEYCLOAK_SMTP_PORT
-              value: "587"
-            - name: KEYCLOAK_SMTP_FROM
-              value: no-reply-cassandra@bstein.dev
-            - name: KEYCLOAK_SMTP_FROM_NAME
-              value: Cassandra
-            - name: CASSANDRA_PUBLIC_REPLAY_USERS
-              value: cassandra-dev,daniel-test,viktor-test
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              set -eu
-              . /vault/secrets/keycloak-admin-env.sh
-              python - <<'PY'
-              import json
-              import os
-              import time
-              import urllib.error
-              import urllib.parse
-              import urllib.request
-
-              base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/")
-              realm = os.environ["KEYCLOAK_REALM"]
-              client_id = os.environ["KEYCLOAK_CLIENT_ID"]
-              issuer = os.environ["KEYCLOAK_PUBLIC_ISSUER"]
-              app_base_url = os.environ["CASSANDRA_BASE_URL"].rstrip("/")
-              admin_user = os.environ["KEYCLOAK_ADMIN_USER"]
-              admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"]
-
-              def request(method, url, token=None, payload=None, headers=None, timeout=30):
-                  data = None
-                  req_headers = headers.copy() if headers else {}
-                  if token:
-                      req_headers["Authorization"] = f"Bearer {token}"
-                  if payload is not None:
-                      data = json.dumps(payload).encode()
-                      req_headers["Content-Type"] = "application/json"
-                  req = urllib.request.Request(url, data=data, headers=req_headers, method=method)
-                  try:
-                      with urllib.request.urlopen(req, timeout=timeout) as resp:
-                          body = resp.read()
-                          if not body:
-                              return resp.status, None
-                          return resp.status, json.loads(body.decode())
-                  except urllib.error.HTTPError as exc:
-                      raw = exc.read()
-                      if not raw:
-                          return exc.code, None
-                      try:
-                          return exc.code, json.loads(raw.decode())
-                      except Exception:
-                          return exc.code, {"raw": raw.decode(errors="replace")}
-
-              token_body = None
-              form = urllib.parse.urlencode(
-                  {
-                      "grant_type": "password",
-                      "client_id": "admin-cli",
-                      "username": admin_user,
-                      "password": admin_password,
-                  }
-              ).encode()
-              for attempt in range(1, 11):
-                  req = urllib.request.Request(
-                      f"{base_url}/realms/master/protocol/openid-connect/token",
-                      data=form,
-                      headers={"Content-Type": "application/x-www-form-urlencoded"},
-                      method="POST",
-                  )
-                  try:
-                      with urllib.request.urlopen(req, timeout=10) as resp:
-                          token_body = json.loads(resp.read().decode())
-                          break
-                  except urllib.error.URLError as exc:
-                      if attempt == 10:
-                          raise SystemExit(f"Keycloak token request failed after retries: {exc}")
-                      time.sleep(attempt * 2)
-              token = token_body["access_token"]
-
-              smtp = {
-                  "host": os.environ["KEYCLOAK_SMTP_HOST"],
-                  "port": os.environ["KEYCLOAK_SMTP_PORT"],
-                  "from": os.environ["KEYCLOAK_SMTP_FROM"],
-                  "fromDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"],
-                  "replyTo": os.environ["KEYCLOAK_SMTP_FROM"],
-                  "replyToDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"],
-                  "user": os.environ["KEYCLOAK_SMTP_USER"],
-                  "password": os.environ["KEYCLOAK_SMTP_PASSWORD"],
-                  "auth": "true",
-                  "starttls": "true",
-                  "ssl": "false",
-              }
-
-              status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token)
-              if status == 404:
-                  create_payload = {
-                      "realm": realm,
-                      "enabled": True,
-                      "registrationAllowed": False,
-                      "resetPasswordAllowed": True,
-                      "verifyEmail": True,
-                      "loginWithEmailAllowed": True,
-                      "duplicateEmailsAllowed": False,
-                      "smtpServer": smtp,
-                  }
-                  status, body = request("POST", f"{base_url}/admin/realms", token, create_payload)
-                  if status not in (201, 204, 409):
-                      raise SystemExit(f"Realm create failed: status={status} body={body}")
-                  status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token)
-              if status != 200 or not isinstance(realm_rep, dict):
-                  raise SystemExit(f"Realm fetch failed: status={status}")
-
-              realm_rep.update(
-                  {
-                      "enabled": True,
-                      "registrationAllowed": False,
-                      "resetPasswordAllowed": True,
-                      "verifyEmail": True,
-                      "loginWithEmailAllowed": True,
-                      "duplicateEmailsAllowed": False,
-                      "smtpServer": smtp,
-                  }
-              )
-              status, body = request("PUT", f"{base_url}/admin/realms/{realm}", token, realm_rep)
-              if status not in (200, 204):
-                  raise SystemExit(f"Realm update failed: status={status} body={body}")
-
-              def find_group(name):
-                  status, groups = request(
-                      "GET",
-                      f"{base_url}/admin/realms/{realm}/groups?search={urllib.parse.quote(name)}",
-                      token,
-                  )
-                  if status != 200:
-                      raise SystemExit(f"Group search failed for {name}: status={status}")
-                  for group in groups or []:
-                      if group.get("name") == name:
-                          return group
-                  return None
-
-              def ensure_group(name):
-                  group = find_group(name)
-                  if group:
-                      return group
-                  status, body = request("POST", f"{base_url}/admin/realms/{realm}/groups", token, {"name": name})
-                  if status not in (201, 204, 409):
-                      raise SystemExit(f"Group create failed for {name}: status={status} body={body}")
-                  group = find_group(name)
-                  if not group:
-                      raise SystemExit(f"Group missing after create for {name}")
-                  return group
-
-              def find_user(username):
-                  encoded = urllib.parse.quote(username)
-                  for query in (f"username={encoded}&exact=true", f"email={encoded}&exact=true", f"search={encoded}"):
-                      status, users = request(
-                          "GET",
-                          f"{base_url}/admin/realms/{realm}/users?{query}",
-                          token,
-                      )
-                      if status != 200:
-                          raise SystemExit(f"User search failed for {username}: status={status}")
-                      for user in users or []:
-                          candidates = {
-                              str(user.get("username") or "").casefold(),
-                              str(user.get("email") or "").casefold(),
-                          }
-                          if username.casefold() in candidates:
-                              return user
-                  return None
-
-              def ensure_user_group(username, group):
-                  user = find_user(username)
-                  if not user:
-                      print(f"WARNING: Cassandra public replay user {username!r} not found; skipping membership")
-                      return
-                  user_id = user.get("id")
-                  group_id = group.get("id")
-                  if not user_id or not group_id:
-                      raise SystemExit(f"Missing id for user/group membership: user={username}")
-                  status, memberships = request(
-                      "GET",
-                      f"{base_url}/admin/realms/{realm}/users/{user_id}/groups",
-                      token,
-                  )
-                  if status != 200:
-                      raise SystemExit(f"User group lookup failed for {username}: status={status}")
-                  if any(item.get("id") == group_id for item in memberships or []):
-                      return
-                  status, body = request(
-                      "PUT",
-                      f"{base_url}/admin/realms/{realm}/users/{user_id}/groups/{group_id}",
-                      token,
-                  )
-                  if status not in (200, 204):
-                      raise SystemExit(f"User group update failed for {username}: status={status} body={body}")
-
-              ensure_group("alpha")
-              ensure_group("admin")
-              ensure_group(os.environ["CASSANDRA_TESTER_GROUP"])
-              public_replay_group = ensure_group("public-replay")
-              for username in os.environ.get("CASSANDRA_PUBLIC_REPLAY_USERS", "").split(","):
-                  username = username.strip()
-                  if username:
-                      ensure_user_group(username, public_replay_group)
-
-              status, clients = request(
-                  "GET",
-                  f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}",
-                  token,
-              )
-              if status != 200:
-                  raise SystemExit(f"Client lookup failed: status={status}")
-              client_uuid = clients[0]["id"] if clients else None
-              client_payload = {
-                  "clientId": client_id,
-                  "enabled": True,
-                  "protocol": "openid-connect",
-                  "publicClient": False,
-                  "standardFlowEnabled": True,
-                  "implicitFlowEnabled": False,
-                  "directAccessGrantsEnabled": False,
-                  "serviceAccountsEnabled": False,
-                  "redirectUris": [f"{app_base_url}/*"],
-                  "webOrigins": [app_base_url],
-                  "rootUrl": app_base_url,
-                  "baseUrl": "/",
-                  "attributes": {
-                      "pkce.code.challenge.method": "S256",
-                      "post.logout.redirect.uris": f"{app_base_url}/*",
-                  },
-              }
-              if not client_uuid:
-                  status, body = request("POST", f"{base_url}/admin/realms/{realm}/clients", token, client_payload)
-                  if status not in (201, 204, 409):
-                      raise SystemExit(f"Client create failed: status={status} body={body}")
-                  status, clients = request(
-                      "GET",
-                      f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}",
-                      token,
-                  )
-                  client_uuid = clients[0]["id"] if clients else None
-              if not client_uuid:
-                  raise SystemExit("Client cassandra-web not found after create")
-              status, body = request(
-                  "PUT",
-                  f"{base_url}/admin/realms/{realm}/clients/{client_uuid}",
-                  token,
-                  client_payload,
-              )
-              if status not in (200, 204):
-                  raise SystemExit(f"Client update failed: status={status} body={body}")
-
-              mapper_payload = {
-                  "name": "groups",
-                  "protocol": "openid-connect",
-                  "protocolMapper": "oidc-group-membership-mapper",
-                  "consentRequired": False,
-                  "config": {
-                      "full.path": "false",
-                      "id.token.claim": "true",
-                      "access.token.claim": "true",
-                      "userinfo.token.claim": "true",
-                      "claim.name": "groups",
-                      "jsonType.label": "String",
-                  },
-              }
-              status, mappers = request(
-                  "GET",
-                  f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models",
-                  token,
-              )
-              if status != 200:
-                  raise SystemExit(f"Mapper lookup failed: status={status}")
-              mapper_id = next((mapper.get("id") for mapper in mappers or [] if mapper.get("name") == "groups"), None)
-              if mapper_id:
-                  mapper_update_payload = dict(mapper_payload)
-                  mapper_update_payload["id"] = mapper_id
-                  status, body = request(
-                      "PUT",
-                      f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models/{mapper_id}",
-                      token,
-                      mapper_update_payload,
-                  )
-              else:
-                  status, body = request(
-                      "POST",
-                      f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models",
-                      token,
-                      mapper_payload,
-                  )
-              if status not in (200, 201, 204):
-                  raise SystemExit(f"Mapper ensure failed: status={status} body={body}")
-
-              status, secret = request(
-                  "GET",
-                  f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/client-secret",
-                  token,
-              )
-              client_secret = (secret or {}).get("value")
-              if status != 200 or not client_secret:
-                  raise SystemExit(f"Client secret fetch failed: status={status}")
-
-              vault_addr = os.environ.get("VAULT_ADDR", "http://vault.vault.svc.cluster.local:8200")
-              jwt = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip()
-              login_payload = json.dumps({"jwt": jwt, "role": os.environ.get("VAULT_ROLE", "sso-secrets")}).encode()
-              req = urllib.request.Request(
-                  f"{vault_addr}/v1/auth/kubernetes/login",
-                  data=login_payload,
-                  headers={"Content-Type": "application/json"},
-                  method="POST",
-              )
-              with urllib.request.urlopen(req, timeout=20) as resp:
-                  vault_token = json.loads(resp.read().decode())["auth"]["client_token"]
-
-              payload = {
-                  "data": {
-                      "client_id": client_id,
-                      "client_secret": client_secret,
-                      "issuer": issuer,
-                      "realm": realm,
-                      "required_groups": "alpha,admin",
-                  }
-              }
-              req = urllib.request.Request(
-                  f"{vault_addr}/v1/kv/data/atlas/cassandra/cassandra-oidc",
-                  data=json.dumps(payload).encode(),
-                  headers={"X-Vault-Token": vault_token, "Content-Type": "application/json"},
-                  method="POST",
-              )
-              with urllib.request.urlopen(req, timeout=20) as resp:
-                  if resp.status not in (200, 204):
-                      raise SystemExit(f"Vault write returned {resp.status}")
-
-              print("Cassandra Keycloak realm/client ready")
-              PY

services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml

@@ -1,25 +1,19 @@
-# services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
+# services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml
-# One-off job for sso/cassandra-gitea-oidc-secret-ensure-1.
+# Purpose: create chat OIDC and Brad-only operator proxy clients.
-# Purpose: create/update the Keycloak cassandra realm Gitea OIDC client, ensure the
-# cassandra-tester group exists, and write the client secret to Vault for Gitea.
-# Run by setting spec.suspend to false, reconcile, then set it back to true.
-# Safe to delete the finished Job/pod; it should not run continuously.
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: cassandra-gitea-oidc-secret-ensure-1
+  name: hermes-access-oidc-client-ensure-3
   namespace: sso
 spec:
-  suspend: true
+  backoffLimit: 3
-  backoffLimit: 0
-  ttlSecondsAfterFinished: 3600
   template:
     metadata:
       annotations:
         vault.hashicorp.com/agent-inject: "true"
         vault.hashicorp.com/agent-pre-populate-only: "true"
-        vault.hashicorp.com/role: "sso-secrets"
+        vault.hashicorp.com/role: sso-secrets
-        vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
+        vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: kv/data/atlas/shared/keycloak-admin
         vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
           {{ with secret "kv/data/atlas/shared/keycloak-admin" }}
           export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
@@ -29,11 +23,6 @@ spec:
     spec:
       serviceAccountName: mas-secrets-ensure
       restartPolicy: Never
-      volumes:
-        - name: cassandra-gitea-oidc-secret-ensure-script
-          configMap:
-            name: cassandra-gitea-oidc-secret-ensure-script
-            defaultMode: 0555
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -41,14 +30,32 @@ spec:
               - matchExpressions:
                   - key: kubernetes.io/arch
                     operator: In
-                    values: ["arm64"]
+                    values:
+                      - arm64
                   - key: node-role.kubernetes.io/worker
                     operator: Exists
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-18
       containers:
         - name: apply
           image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
-          command: ["/scripts/cassandra_gitea_oidc_secret_ensure.sh"]
+          command:
+            - /scripts/hermes_access_oidc_ensure.sh
           volumeMounts:
-            - name: cassandra-gitea-oidc-secret-ensure-script
+            - name: script
               mountPath: /scripts
               readOnly: true
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 250m
+              memory: 128Mi
+      volumes:
+        - name: script
+          configMap:
+            name: hermes-access-oidc-script
+            defaultMode: 0555

services/keycloak/kustomization.yaml

@@ -27,8 +27,7 @@ resources:
   - bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml
   - bootstrap-jobs/quality-oidc-secret-ensure-job.yaml
   - bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml
-  - bootstrap-jobs/cassandra-realm-ensure-job.yaml
+  - bootstrap-jobs/hermes-access-oidc-client-job.yaml
-  - bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
   - bootstrap-jobs/veles-realm-ensure-job.yaml
   - bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml
   - bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml
@@ -57,9 +56,9 @@ configMapGenerator:
   - name: hermes-dashboard-oidc-client-script
     files:
       - hermes_dashboard_oidc_client_ensure.sh=scripts/hermes_dashboard_oidc_client_ensure.sh
-  - name: cassandra-gitea-oidc-secret-ensure-script
+  - name: hermes-access-oidc-script
     files:
-      - cassandra_gitea_oidc_secret_ensure.sh=scripts/cassandra_gitea_oidc_secret_ensure.sh
+      - hermes_access_oidc_ensure.sh=scripts/hermes_access_oidc_ensure.sh
   - name: veles-gitea-oidc-secret-ensure-script
     files:
       - veles_gitea_oidc_secret_ensure.sh=scripts/veles_gitea_oidc_secret_ensure.sh

services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh

@@ -1,157 +0,0 @@
-#!/usr/bin/env sh
-set -euo pipefail
-
-. /vault/secrets/keycloak-admin-env.sh
-
-KC_URL="${KEYCLOAK_SERVER:-http://keycloak.sso.svc.cluster.local}"
-REALM="${KEYCLOAK_REALM:-cassandra}"
-CLIENT_ID="${KEYCLOAK_CLIENT_ID:-gitea}"
-PUBLIC_BASE_URL="${GITEA_PUBLIC_BASE_URL:-https://scm.bstein.dev}"
-AUTH_SOURCE_NAME="${GITEA_AUTH_SOURCE_NAME:-cassandra}"
-TESTER_GROUP="${CASSANDRA_GITEA_TESTER_GROUP:-cassandra-tester}"
-VAULT_SECRET_PATH="${VAULT_SECRET_PATH:-kv/data/atlas/gitea/gitea-cassandra-oidc}"
-
-ACCESS_TOKEN=""
-for attempt in 1 2 3 4 5 6 7 8 9 10; do
-  if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then
-    break
-  fi
-  echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2
-  sleep $((attempt * 2))
-done
-for attempt in 1 2 3 4 5; do
-  TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \
-    -H 'Content-Type: application/x-www-form-urlencoded' \
-    -d "grant_type=password" \
-    -d "client_id=admin-cli" \
-    -d "username=${KEYCLOAK_ADMIN}" \
-    -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
-  ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)"
-  if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then
-    break
-  fi
-  echo "Keycloak token request failed (attempt ${attempt})" >&2
-  sleep $((attempt * 2))
-done
-if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
-  echo "Failed to fetch Keycloak admin token" >&2
-  exit 1
-fi
-
-ensure_group() {
-  group_name="$1"
-  groups="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    "${KC_URL}/admin/realms/${REALM}/groups?search=$(printf '%s' "${group_name}" | jq -sRr @uri)" || true)"
-  if echo "$groups" | jq -e --arg name "$group_name" '.[]? | select(.name == $name)' >/dev/null 2>&1; then
-    return
-  fi
-  status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    -H 'Content-Type: application/json' \
-    -d "$(jq -nc --arg name "$group_name" '{name:$name}')" \
-    "${KC_URL}/admin/realms/${REALM}/groups")"
-  if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then
-    echo "Keycloak group create failed for ${group_name} (status ${status})" >&2
-    exit 1
-  fi
-}
-
-ensure_group "${TESTER_GROUP}"
-
-CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)"
-CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
-
-client_payload="$(jq -nc \
-  --arg client_id "${CLIENT_ID}" \
-  --arg root_url "${PUBLIC_BASE_URL}" \
-  --arg callback "${PUBLIC_BASE_URL}/user/oauth2/${AUTH_SOURCE_NAME}/callback" \
-  '{clientId:$client_id,enabled:true,protocol:"openid-connect",publicClient:false,standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false,serviceAccountsEnabled:false,redirectUris:[$callback],webOrigins:[$root_url],rootUrl:$root_url,baseUrl:"/",attributes:{"pkce.code.challenge.method":"S256","post.logout.redirect.uris":($root_url + "/*")}}')"
-
-if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then
-  status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    -H 'Content-Type: application/json' \
-    -d "${client_payload}" \
-    "${KC_URL}/admin/realms/${REALM}/clients")"
-  if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then
-    echo "Keycloak client create failed for ${CLIENT_ID} (status ${status})" >&2
-    exit 1
-  fi
-  CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    "${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)"
-  CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
-fi
-
-if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then
-  echo "Keycloak client ${CLIENT_ID} not found after create" >&2
-  exit 1
-fi
-
-status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
-  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  -H 'Content-Type: application/json' \
-  -d "${client_payload}" \
-  "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}")"
-if [ "$status" != "200" ] && [ "$status" != "204" ]; then
-  echo "Keycloak client update failed for ${CLIENT_ID} (status ${status})" >&2
-  exit 1
-fi
-
-mapper_payload="$(jq -nc \
-  '{name:"groups",protocol:"openid-connect",protocolMapper:"oidc-group-membership-mapper",consentRequired:false,config:{"full.path":"false","id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","claim.name":"groups","jsonType.label":"String"}}')"
-mappers="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models" || true)"
-MAPPER_ID="$(echo "$mappers" | jq -r '.[]? | select(.name == "groups") | .id' | head -n1 || true)"
-if [ -n "$MAPPER_ID" ] && [ "$MAPPER_ID" != "null" ]; then
-  status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
-    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    -H 'Content-Type: application/json' \
-    -d "${mapper_payload}" \
-    "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models/${MAPPER_ID}")"
-else
-  status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-    -H 'Content-Type: application/json' \
-    -d "${mapper_payload}" \
-    "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models")"
-fi
-if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then
-  echo "Keycloak groups mapper ensure failed (status ${status})" >&2
-  exit 1
-fi
-
-CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-  "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/client-secret" | jq -r '.value' 2>/dev/null || true)"
-if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
-  echo "Keycloak client secret not found for ${CLIENT_ID}" >&2
-  exit 1
-fi
-
-vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
-vault_role="${VAULT_ROLE:-sso-secrets}"
-jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
-vault_token="$(curl -sS --request POST --data "${login_payload}" \
-  "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
-if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
-  echo "vault login failed" >&2
-  exit 1
-fi
-
-payload="$(jq -nc \
-  --arg client_id "${CLIENT_ID}" \
-  --arg client_secret "${CLIENT_SECRET}" \
-  --arg discovery_url "${PUBLIC_BASE_URL}" \
-  --arg issuer "https://sso.bstein.dev/realms/${REALM}" \
-  --arg auto_discovery_url "https://sso.bstein.dev/realms/${REALM}/.well-known/openid-configuration" \
-  --arg auth_source_name "${AUTH_SOURCE_NAME}" \
-  --arg tester_group "${TESTER_GROUP}" \
-  --arg group_team_map "{\"${TESTER_GROUP}\":{\"cassandra-alpha\":[\"testers\"]}}" \
-  '{data:{client_id:$client_id,client_secret:$client_secret,issuer:$issuer,openid_auto_discovery_url:$auto_discovery_url,auth_source_name:$auth_source_name,required_claim_name:"groups",required_claim_value:$tester_group,group_claim_name:"groups",restricted_group:$tester_group,group_team_map:$group_team_map,gitea_base_url:$discovery_url}}')"
-
-curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-  -H 'Content-Type: application/json' \
-  -d "${payload}" "${vault_addr}/v1/${VAULT_SECRET_PATH}" >/dev/null
-
-echo "Cassandra realm Gitea OIDC client ready"

services/keycloak/scripts/hermes_access_oidc_ensure.sh

@@ -0,0 +1,229 @@
+#!/usr/bin/env sh
+set -eu
+
+. /vault/secrets/keycloak-admin-env.sh
+
+KC_URL="http://keycloak.sso.svc.cluster.local"
+CHAT_CLIENT="hermes-chat-dashboard"
+CHAT_URL="https://chat.bstein.dev"
+OPERATOR_CLIENT="hermes-operator-proxy"
+OPERATOR_URL="https://agent.bstein.dev"
+
+ACCESS_TOKEN=""
+for attempt in 1 2 3 4 5 6 7 8 9 10; do
+  if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then
+    break
+  fi
+  echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2
+  sleep $((attempt * 2))
+done
+
+for attempt in 1 2 3 4 5; do
+  token_json="$(curl -sS -X POST "${KC_URL}/realms/master/protocol/openid-connect/token" \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d "grant_type=password" \
+    -d "client_id=admin-cli" \
+    -d "username=${KEYCLOAK_ADMIN}" \
+    -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
+  ACCESS_TOKEN="$(printf '%s' "${token_json}" | jq -r '.access_token' 2>/dev/null || true)"
+  if [ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ]; then
+    break
+  fi
+  echo "Keycloak token request failed (attempt ${attempt})" >&2
+  sleep $((attempt * 2))
+done
+if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then
+  echo "Failed to fetch Keycloak admin token" >&2
+  exit 1
+fi
+
+chat_payload="$(jq -nc \
+  --arg client_id "${CHAT_CLIENT}" \
+  --arg redirect_uri "${CHAT_URL}/auth/callback" \
+  --arg web_origin "${CHAT_URL}" \
+  '{
+    clientId:$client_id,
+    enabled:true,
+    protocol:"openid-connect",
+    publicClient:true,
+    standardFlowEnabled:true,
+    implicitFlowEnabled:false,
+    directAccessGrantsEnabled:false,
+    serviceAccountsEnabled:false,
+    redirectUris:[$redirect_uri],
+    webOrigins:[$web_origin],
+    rootUrl:$web_origin,
+    baseUrl:"/",
+    attributes:{
+      "pkce.code.challenge.method":"S256",
+      "post.logout.redirect.uris":$web_origin,
+      "access.token.lifespan":"1200"
+    }
+  }')"
+
+chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)"
+chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)"
+if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then
+  status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
+    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    -H 'Content-Type: application/json' \
+    -d "${chat_payload}" \
+    "${KC_URL}/admin/realms/atlas/clients")"
+  if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then
+    echo "Keycloak chat client create failed (status ${status})" >&2
+    exit 1
+  fi
+  chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)"
+  chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)"
+fi
+if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then
+  echo "Keycloak chat client not found after create" >&2
+  exit 1
+fi
+
+status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
+  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  -H 'Content-Type: application/json' \
+  -d "${chat_payload}" \
+  "${KC_URL}/admin/realms/atlas/clients/${chat_id}")"
+if [ "${status}" != "204" ]; then
+  echo "Keycloak chat client update failed (status ${status})" >&2
+  exit 1
+fi
+
+scope_id="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "${KC_URL}/admin/realms/atlas/client-scopes?search=groups" \
+  | jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)"
+if [ -z "${scope_id}" ] || [ "${scope_id}" = "null" ]; then
+  echo "Keycloak groups client scope not found" >&2
+  exit 1
+fi
+default_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "${KC_URL}/admin/realms/atlas/clients/${chat_id}/default-client-scopes" || true)"
+optional_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes" || true)"
+if ! printf '%s' "${default_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1 \
+  && ! printf '%s' "${optional_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1; then
+  status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
+    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")"
+  if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then
+    status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
+      -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+      "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")"
+    if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then
+      echo "Failed to attach groups scope to chat client (status ${status})" >&2
+      exit 1
+    fi
+  fi
+fi
+
+operator_payload="$(jq -nc \
+  --arg client_id "${OPERATOR_CLIENT}" \
+  --arg redirect_uri "${OPERATOR_URL}/oauth2/callback" \
+  --arg web_origin "${OPERATOR_URL}" \
+  '{
+    clientId:$client_id,
+    enabled:true,
+    protocol:"openid-connect",
+    publicClient:false,
+    standardFlowEnabled:true,
+    implicitFlowEnabled:false,
+    directAccessGrantsEnabled:false,
+    serviceAccountsEnabled:false,
+    redirectUris:[$redirect_uri],
+    webOrigins:[$web_origin],
+    rootUrl:$web_origin,
+    baseUrl:"/",
+    attributes:{"post.logout.redirect.uris":$web_origin}
+  }')"
+
+operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)"
+operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)"
+if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then
+  status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
+    -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    -H 'Content-Type: application/json' \
+    -d "${operator_payload}" \
+    "${KC_URL}/admin/realms/atlas/clients")"
+  if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then
+    echo "Keycloak operator proxy client create failed (status ${status})" >&2
+    exit 1
+  fi
+  operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+    "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)"
+  operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)"
+fi
+if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then
+  echo "Keycloak operator proxy client not found after create" >&2
+  exit 1
+fi
+
+status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
+  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  -H 'Content-Type: application/json' \
+  -d "${operator_payload}" \
+  "${KC_URL}/admin/realms/atlas/clients/${operator_id}")"
+if [ "${status}" != "204" ]; then
+  echo "Keycloak operator proxy client update failed (status ${status})" >&2
+  exit 1
+fi
+
+client_secret="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+  "${KC_URL}/admin/realms/atlas/clients/${operator_id}/client-secret" \
+  | jq -r '.value' 2>/dev/null || true)"
+if [ -z "${client_secret}" ] || [ "${client_secret}" = "null" ]; then
+  echo "Keycloak operator proxy client secret not found" >&2
+  exit 1
+fi
+
+vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
+vault_role="${VAULT_ROLE:-sso-secrets}"
+jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt,role:$role}')"
+vault_token="$(curl -sS --request POST --data "${login_payload}" \
+  "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+  echo "Vault login failed" >&2
+  exit 1
+fi
+
+read_status="$(curl -sS -o /tmp/hermes-operator-oidc-read.json -w '%{http_code}' \
+  -H "X-Vault-Token: ${vault_token}" \
+  "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc" || true)"
+cookie_secret=""
+if [ "${read_status}" = "200" ]; then
+  cookie_secret="$(jq -r '.data.data.cookie_secret // empty' /tmp/hermes-operator-oidc-read.json)"
+elif [ "${read_status}" != "404" ]; then
+  echo "Vault operator OIDC read failed (status ${read_status})" >&2
+  exit 1
+fi
+if [ -n "${cookie_secret}" ]; then
+  cookie_length="$(printf '%s' "${cookie_secret}" | wc -c | tr -d ' ')"
+  if [ "${cookie_length}" != "16" ] && [ "${cookie_length}" != "24" ] && [ "${cookie_length}" != "32" ]; then
+    cookie_secret=""
+  fi
+fi
+if [ -z "${cookie_secret}" ]; then
+  cookie_secret="$(openssl rand -hex 16 | tr -d '\n')"
+fi
+
+vault_payload="$(jq -nc \
+  --arg client_id "${OPERATOR_CLIENT}" \
+  --arg client_secret "${client_secret}" \
+  --arg cookie_secret "${cookie_secret}" \
+  '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')"
+write_status="$(curl -sS -o /tmp/hermes-operator-oidc-write.json -w '%{http_code}' -X POST \
+  -H "X-Vault-Token: ${vault_token}" \
+  -H 'Content-Type: application/json' \
+  -d "${vault_payload}" \
+  "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc")"
+if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then
+  echo "Vault operator OIDC write failed (status ${write_status})" >&2
+  exit 1
+fi
+
+echo "Hermes chat and operator OIDC clients are ready"

services/maintenance/apps/ariadne-deployment.yaml

@@ -285,8 +285,21 @@ spec:
             - name: GAME_MODE_NODE_NAME
               value: titan-24
             - name: GAME_MODE_DISPLACE_WORKLOADS
-              value: >-
+              value: "[]"
-                [{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}]
+            - name: GAME_MODE_LEASE_NAMESPACE
+              value: hermes
+            - name: GAME_MODE_LEASE_NAME
+              value: titan-24-gpu-owner
+            - name: GAME_MODE_OLLAMA_URL
+              value: http://hermes-ollama.hermes.svc.cluster.local:11434
+            - name: GAME_MODE_OLLAMA_MODEL
+              value: gpt-oss:20b
+            - name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC
+              value: "900"
+            - name: GAME_MODE_TRANSITION_TIMEOUT_SEC
+              value: "900"
+            - name: GAME_MODE_POLL_INTERVAL_SEC
+              value: "1"
             - name: WOLF_OIDC_CLIENT_ID
               value: wolf
             - name: WOLF_OIDC_BASE_URL
@@ -418,7 +431,7 @@ spec:
             - name: ARIADNE_VM_URL
               value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
             - name: ARIADNE_CLUSTER_STATE_VM_TIMEOUT_SEC
-              value: "5"
+              value: "30"
             - name: ARIADNE_ALERTMANAGER_URL
               value: http://alertmanager.monitoring.svc.cluster.local
             - name: OPENSEARCH_URL
@@ -444,11 +457,11 @@ spec:
             - name: ARIADNE_SCHEDULE_TESTING_TRIAGE
               value: "*/15 * * * *"
             - name: ARIADNE_TESTING_TRIAGE_MODEL_URL
-              value: http://hermes-ollama.hermes.svc.cluster.local:11434
+              value: http://hermes-model-gate.hermes.svc.cluster.local:11434
             - name: ARIADNE_TESTING_TRIAGE_MODEL
-              value: qwen2.5:7b-instruct-q4_0
+              value: gpt-oss:20b
             - name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC
-              value: "180"
+              value: "900"
             - name: JENKINS_WORKSPACE_NAMESPACE
               value: jenkins
             - name: JENKINS_WORKSPACE_PVC_PREFIX

services/maintenance/apps/ariadne-rbac.yaml

@@ -78,7 +78,6 @@ rules:
       - get
       - list
       - watch
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

services/maintenance/apps/metis-configmap.yaml

@@ -15,8 +15,8 @@ data:
   METIS_MAX_DEVICE_BYTES: "1000000000000"
   METIS_NAMESPACE: maintenance
   METIS_REMOTE_POD_TIMEOUT_SEC: "14400"
-  METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-243-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
+  METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-270-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
-  METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
+  METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
   METIS_HARBOR_REGISTRY: registry.bstein.dev
   METIS_HARBOR_PROJECT: metis
   METIS_HARBOR_API_BASE: https://registry.bstein.dev/api/v2.0

services/maintenance/kustomization.yaml

@@ -11,8 +11,8 @@ resources:
   - networking
 images:
   - name: registry.bstein.dev/bstein/ariadne
-    newTag: 0.1.0-356 # {"$imagepolicy": "maintenance:ariadne:tag"}
+    newTag: 0.1.0-389 # {"$imagepolicy": "maintenance:ariadne:tag"}
   - name: registry.bstein.dev/bstein/metis
-    newTag: 0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
+    newTag: 0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
   - name: registry.bstein.dev/bstein/soteria
     newTag: 0.1.0-120 # {"$imagepolicy": "maintenance:soteria:tag"}

services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         kubernetes.io/arch: amd64
       containers:
         - name: metis-sentinel
-          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
+          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-270-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
           imagePullPolicy: Always
           envFrom:
             - configMapRef:

services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         kubernetes.io/arch: arm64
       containers:
         - name: metis-sentinel
-          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"}
+          image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"}
           imagePullPolicy: Always
           envFrom:
             - configMapRef:

services/maintenance/node-ops/node-image-sweeper-daemonset.yaml

@@ -25,6 +25,14 @@ spec:
         - key: node-role.kubernetes.io/master
           operator: Exists
           effect: NoSchedule
+        - key: veles.bstein.dev/simulation
+          operator: Equal
+          value: "true"
+          effect: NoSchedule
+        - key: cassandra.bstein.dev/simulation
+          operator: Equal
+          value: "true"
+          effect: NoSchedule
       nodeSelector:
         kubernetes.io/os: linux
       containers:

services/monitoring/availability-backfill-v4-job.yaml

@@ -0,0 +1,92 @@
+# services/monitoring/availability-backfill-v4-job.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atlas-availability-request-v4-backfill-rules
+  namespace: monitoring
+data:
+  atlas-request-history.yaml: |
+    groups:
+      - name: atlas.availability.request.backfill
+        interval: 1h
+        rules:
+          - record: atlas:availability:requests_1h
+            expr: |
+              sum(increase(
+                traefik_entrypoint_requests_total{
+                  entrypoint="websecure",
+                  protocol="http",
+                  code=~"[1-5].."
+                }[1h]
+              ))
+            labels:
+              definition: request-v4
+              scope: atlas
+              rollup: hourly
+          - record: atlas:availability:failures_1h
+            expr: |
+              sum(increase(
+                traefik_entrypoint_requests_total{
+                  entrypoint="websecure",
+                  protocol="http",
+                  code=~"5.."
+                }[1h]
+              ))
+            labels:
+              definition: request-v4
+              scope: atlas
+              rollup: hourly
+
+---
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: atlas-availability-request-v4-backfill
+  namespace: monitoring
+spec:
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        app: atlas-availability-request-v4-backfill
+    spec:
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-22
+                      - titan-24
+      containers:
+        - name: vmalert-replay
+          image: victoriametrics/vmalert:v1.113.0
+          args:
+            - -datasource.url=http://victoria-metrics-single-server:8428
+            - -remoteWrite.url=http://victoria-metrics-single-server:8428
+            - -remoteWrite.flushInterval=1s
+            - -rule=/etc/vmalert/backfill/*.yaml
+            - -replay.timeFrom=2026-05-01T00:00:00Z
+            - -replay.timeTo=2026-08-04T23:00:00Z
+            - -replay.maxDatapointsPerQuery=48
+            - -replay.rulesDelay=2s
+            - -replay.disableProgressBar
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: "1"
+              memory: 512Mi
+          volumeMounts:
+            - name: rules
+              mountPath: /etc/vmalert/backfill
+              readOnly: true
+      volumes:
+        - name: rules
+          configMap:
+            name: atlas-availability-request-v4-backfill-rules

services/monitoring/availability-daily-backfill-v4-job.yaml

@@ -0,0 +1,92 @@
+# services/monitoring/availability-daily-backfill-v4-job.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atlas-availability-request-v4-daily-backfill-rules
+  namespace: monitoring
+data:
+  atlas-request-daily-history.yaml: |
+    groups:
+      - name: atlas.availability.request.daily.backfill
+        interval: 1d
+        rules:
+          - record: atlas:availability:requests_1d
+            expr: |
+              sum(increase(
+                traefik_entrypoint_requests_total{
+                  entrypoint="websecure",
+                  protocol="http",
+                  code=~"[1-5].."
+                }[1d]
+              ))
+            labels:
+              definition: request-v4
+              scope: atlas
+              rollup: daily
+          - record: atlas:availability:failures_1d
+            expr: |
+              sum(increase(
+                traefik_entrypoint_requests_total{
+                  entrypoint="websecure",
+                  protocol="http",
+                  code=~"5.."
+                }[1d]
+              ))
+            labels:
+              definition: request-v4
+              scope: atlas
+              rollup: daily
+
+---
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: atlas-availability-request-v4-daily-backfill
+  namespace: monitoring
+spec:
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        app: atlas-availability-request-v4-daily-backfill
+    spec:
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-22
+                      - titan-24
+      containers:
+        - name: vmalert-replay
+          image: victoriametrics/vmalert:v1.113.0
+          args:
+            - -datasource.url=http://victoria-metrics-single-server:8428
+            - -remoteWrite.url=http://victoria-metrics-single-server:8428
+            - -remoteWrite.flushInterval=1s
+            - -rule=/etc/vmalert/backfill/*.yaml
+            - -replay.timeFrom=2026-05-01T00:00:00Z
+            - -replay.timeTo=2026-08-04T00:00:00Z
+            - -replay.maxDatapointsPerQuery=4
+            - -replay.rulesDelay=2s
+            - -replay.disableProgressBar
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: "1"
+              memory: 512Mi
+          volumeMounts:
+            - name: rules
+              mountPath: /etc/vmalert/backfill
+              readOnly: true
+      volumes:
+        - name: rules
+          configMap:
+            name: atlas-availability-request-v4-daily-backfill-rules

services/monitoring/availability-legacy-cleanup-job.yaml

@@ -0,0 +1,47 @@
+# services/monitoring/availability-legacy-cleanup-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: atlas-availability-legacy-series-cleanup-v1
+  namespace: monitoring
+spec:
+  activeDeadlineSeconds: 300
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        app: atlas-availability-legacy-series-cleanup
+    spec:
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-22
+                      - titan-24
+      containers:
+        - name: cleanup
+          image: python:3.12-alpine
+          command: ["python", "/scripts/availability_cleanup.py"]
+          env:
+            - name: VM_URL
+              value: http://victoria-metrics-single-server:8428
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 250m
+              memory: 128Mi
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: script
+          configMap:
+            name: atlas-availability-cleanup-script

services/monitoring/availability-rollup-cronjob.yaml

@@ -0,0 +1,103 @@
+# services/monitoring/availability-rollup-cronjob.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: atlas-availability-request-v4-rollup-initial
+  namespace: monitoring
+spec:
+  activeDeadlineSeconds: 900
+  backoffLimit: 2
+  template:
+    metadata:
+      labels:
+        app: atlas-availability-request-v4-rollup
+    spec:
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-22
+                      - titan-24
+      containers:
+        - name: rollup
+          image: python:3.12-alpine
+          command: ["python", "/scripts/availability_rollup.py"]
+          env:
+            - name: VM_URL
+              value: http://victoria-metrics-single-server:8428
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: script
+          configMap:
+            name: atlas-availability-rollup-script
+
+---
+
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: atlas-availability-request-v4-rollup
+  namespace: monitoring
+spec:
+  schedule: "10 0 * * *"
+  timeZone: Etc/UTC
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 900
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app: atlas-availability-request-v4-rollup
+        spec:
+          restartPolicy: Never
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: kubernetes.io/hostname
+                        operator: NotIn
+                        values:
+                          - titan-22
+                          - titan-24
+          containers:
+            - name: rollup
+              image: python:3.12-alpine
+              command: ["python", "/scripts/availability_rollup.py"]
+              env:
+                - name: VM_URL
+                  value: http://victoria-metrics-single-server:8428
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: 500m
+                  memory: 256Mi
+              volumeMounts:
+                - name: script
+                  mountPath: /scripts
+                  readOnly: true
+          volumes:
+            - name: script
+              configMap:
+                name: atlas-availability-rollup-script

services/monitoring/dashboards/atlas-network.json

@@ -476,7 +476,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }

services/monitoring/dashboards/atlas-nodes.json

@@ -20,7 +20,7 @@
       },
       "targets": [
         {
-          "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})",
+          "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})",
           "refId": "A"
         }
       ],
@@ -46,7 +46,7 @@
           "unit": "none",
           "custom": {
             "displayMode": "auto",
-            "valueSuffix": "/21"
+            "valueSuffix": "/18"
           }
         },
         "overrides": []
@@ -409,7 +409,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -449,7 +449,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -489,7 +489,7 @@
       },
       "targets": [
         {
-          "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+          "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -526,7 +526,7 @@
       },
       "targets": [
         {
-          "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+          "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -563,7 +563,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -601,7 +601,7 @@
       },
       "targets": [
         {
-          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }

services/monitoring/dashboards/atlas-pods.json

@@ -20,7 +20,7 @@
       },
       "targets": [
         {
-          "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})) or on() vector(0)",
+          "expr": "((sum(max by(namespace,pod) ((kube_pod_status_phase{phase=\"Pending\",namespace!~\"veles\"} == 1) and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)) + (sum(max by(namespace,pod) ((kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace!~\"veles\"} == 1) unless on(namespace,pod) kube_pod_owner{owner_kind=\"Job\"})) or on() vector(0)))",
           "refId": "A"
         }
       ],
@@ -80,7 +80,7 @@
       },
       "targets": [
         {
-          "expr": "sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})) or on() vector(0)",
+          "expr": "sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason{namespace!~\"veles\",reason=~\"CrashLoopBackOff|ImagePullBackOff\"} and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)",
           "refId": "A"
         }
       ],
@@ -523,7 +523,7 @@
       },
       "targets": [
         {
-          "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)))))",
+          "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)))))",
           "refId": "A",
           "instant": true,
           "format": "table"

services/monitoring/dashboards/atlas-storage.json

@@ -276,7 +276,7 @@
       },
       "targets": [
         {
-          "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+          "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
@@ -314,7 +314,7 @@
       },
       "targets": [
         {
-          "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
+          "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{node}}"
         }

services/monitoring/dcgm-exporter.yaml

@@ -53,27 +53,15 @@ spec:
           ports:
             - name: metrics
               containerPort: 9400
-          env:
-            - name: DCGM_EXPORTER_KUBERNETES
-              value: "true"
-            - name: KUBERNETES_VIRTUAL_GPUS
-              value: "true"
-            - name: NVIDIA_RESOURCE_NAMES
-              value: nvidia.com/gpu.shared
           securityContext:
             privileged: true
           resources:
             requests:
               cpu: 50m
-              memory: 64Mi
+              memory: 512Mi
-          volumeMounts:
+            limits:
-            - name: pod-resources
+              cpu: 500m
-              mountPath: /var/lib/kubelet/pod-resources
+              memory: 1Gi
-      volumes:
-        - name: pod-resources
-          hostPath:
-            path: /var/lib/kubelet/pod-resources
-            type: Directory
 ---
 apiVersion: v1
 kind: Service