titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

Merge origin/main into layout migration

Browse Source 
Reconciles the services layout migration with ~991 upstream commits:
- Remote content wins for cassandra/cassandra-auth, monitoring dashboards,
  vmalert availability rules, veles, vault auth script, dashboard render
  script and tests (request-v4 availability definition)
- Layout paths win for structure: keycloak/bstein-dev-home job dirs use
  bootstrap-jobs/validation-jobs; cassandra realm jobs live in
  cassandra-auth (removed keycloak duplicates)
- Union: applications CR list gains hermes-chat and cassandra
  image-automation
- Fixed post-migration paths in dashboard test module loader and
  hermes-access job header

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jenkins 2026-08-05 16:26:36 -03:00
commit c4b4e8a404
121 changed files with 5958 additions and 1026 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml Normal file
View File

@ -0,0 +1,28 @@
# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-chat
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-chat
targetNamespace: hermes-chat
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 30m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-chat
namespace: hermes-chat
dependsOn:
- name: cert-manager
- name: core
- name: hermes
- name: keycloak
- name: longhorn

8
clusters/atlas/flux-system/applications/hermes/kustomization.yaml
View File

@ -18,6 +18,10 @@ spec:
wait: true
timeout: 45m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-model-gate
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-ollama
@ -26,6 +30,10 @@ spec:
kind: Deployment
name: hermes
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-hermes
namespace: hermes
dependsOn:
- name: cert-manager
- name: core

2
clusters/atlas/flux-system/applications/kustomization.yaml
View File

@ -28,9 +28,11 @@ resources:
- ai-llm/kustomization.yaml
- openclaw/kustomization.yaml
- hermes/kustomization.yaml
- hermes-chat/kustomization.yaml
- game-stream/kustomization.yaml
- cassandra-auth/kustomization.yaml
- cassandra/kustomization.yaml
- cassandra/image-automation.yaml
- veles/kustomization.yaml
- typhon/kustomization.yaml
- nextcloud/kustomization.yaml

122
dockerfiles/Dockerfile.hermes-agent Normal file
View File

@ -0,0 +1,122 @@
# syntax=docker/dockerfile:1
# dockerfiles/Dockerfile.hermes-agent
FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
USER root
# Keep dashboard chat sockets tied to the intended React mount and conversation.
# A resumed conversation needs a different PTY attachment key from a fresh chat;
# reconnects to that same conversation must keep using the same key.
RUN node <<'NODE'
const fs = require("node:fs");
const path = "/opt/hermes/web/src/pages/ChatPage.tsx";
let source = fs.readFileSync(path, "utf8");
const socketBefore = [
' const url = await api.buildWsUrl("/api/pty", params);',
' const ws = new WebSocket(url);',
].join("\n");
const socketAfter = [
' const url = await api.buildWsUrl("/api/pty", params);',
' if (unmounting) return;',
' const ws = new WebSocket(url);',
].join("\n");
const attachBefore = ' params.attach = ptyAttachToken(forceFresh);';
const attachAfter = [
' const attachScope = resumeParam',
' ? `resume:${resumeParam}:${scopedProfile ?? ""}`',
' : `fresh:${scopedProfile ?? ""}`;',
' params.attach = `${ptyAttachToken(forceFresh)}:${attachScope}`;',
].join("\n");
if (!source.includes(socketBefore)) {
throw new Error("Hermes ChatPage WebSocket patch context changed");
}
if (!source.includes(attachBefore)) {
throw new Error("Hermes ChatPage PTY attachment patch context changed");
}
source = source.replace(socketBefore, socketAfter);
source = source.replace(attachBefore, attachAfter);
fs.writeFileSync(path, source);
NODE
# The upstream OIDC gate authenticates users but deliberately treats the
# dashboard as one shared workstation. Allow a deployment to narrow that
# workstation to explicit OIDC subjects. Enforce this after normal provider
# verification so a denied account is a 403, not a misleading provider 503.
RUN python - <<'PY'
from pathlib import Path
path = Path("/opt/hermes/hermes_cli/dashboard_auth/middleware.py")
source = path.read_text()
helper_before = '''def _client_ip(request: Request) -> str:
fwd = request.headers.get("x-forwarded-for", "")
if fwd:
return fwd.split(",")[0].strip()
return request.client.host if request.client else ""
'''
helper_after = helper_before + '''def _dashboard_user_allowed(session) -> bool:
"""Apply an optional deployment-level OIDC-subject allowlist."""
import os
allowed = {
value.strip()
for value in os.environ.get(
"HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS", ""
).split(",")
if value.strip()
}
return not allowed or session.user_id in allowed
def _user_forbidden_response() -> Response:
"""Return an authorization failure without exposing identities."""
return JSONResponse(
{
"error": "forbidden",
"detail": "This Atlas account is not authorized for this dashboard.",
},
status_code=403,
)
'''
refresh_before = ''' new_session, refreshing_provider = refreshed
request.state.session = new_session
response = await call_next(request)
'''
refresh_after = ''' new_session, refreshing_provider = refreshed
if not _dashboard_user_allowed(new_session):
return _user_forbidden_response()
request.state.session = new_session
response = await call_next(request)
'''
final_before = ''' request.state.session = session
return await call_next(request)
'''
final_after = ''' if not _dashboard_user_allowed(session):
return _user_forbidden_response()
request.state.session = session
return await call_next(request)
'''
for before, after, label in (
(helper_before, helper_after, "allowlist helper"),
(refresh_before, refresh_after, "refreshed session"),
(final_before, final_after, "verified session"),
):
if before not in source:
raise SystemExit(f"Hermes dashboard auth {label} patch context changed")
source = source.replace(before, after, 1)
path.write_text(source)
PY
COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
RUN cd /opt/hermes/web \
&& npm run build \
&& grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \
&& grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \
&& grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \
/opt/hermes/hermes_cli/dashboard_auth/middleware.py \
&& chmod 0755 /opt/hermes/bin/hermes-session-migrate

101
dockerfiles/hermes-session-migrate.py Normal file
View File

@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Copy selected legacy Hermes sessions into an isolated user home."""
from __future__ import annotations
import os
from pathlib import Path
import shutil
import sqlite3
from hermes_state import SessionDB
def _copy_if_missing(source: Path, target: Path) -> None:
"""Copy one credential/config file without overwriting user state."""
if source.is_file() and not target.exists():
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, target)
def _copy_session(
source: sqlite3.Connection,
target: sqlite3.Connection,
session_id: str,
user_id: str,
) -> bool:
"""Copy a session and its messages, preserving original timestamps."""
source.row_factory = sqlite3.Row
session = source.execute(
"SELECT * FROM sessions WHERE id = ?", (session_id,)
).fetchone()
if session is None:
return False
columns = list(session.keys())
values = [user_id if column == "user_id" else session[column] for column in columns]
placeholders = ", ".join("?" for _ in columns)
target.execute(
f"INSERT OR IGNORE INTO sessions ({', '.join(columns)}) "
f"VALUES ({placeholders})",
values,
)
message_columns = [
row[1] for row in target.execute("PRAGMA table_info(messages)").fetchall()
]
selected_columns = [
column
for column in message_columns
if column in {
row[1]
for row in source.execute("PRAGMA table_info(messages)").fetchall()
}
]
column_sql = ", ".join(selected_columns)
target.execute(
f"INSERT OR IGNORE INTO messages ({column_sql}) "
f"SELECT {column_sql} FROM source_db.messages WHERE session_id = ?",
(session_id,),
)
return True
def main() -> None:
"""Create the isolated home and perform the idempotent session copy."""
source_home = Path(os.environ["HERMES_MIGRATE_SOURCE_HOME"])
target_home = Path(os.environ["HERMES_HOME"])
user_id = os.environ["HERMES_MIGRATE_USER_ID"].strip()
session_ids = [
value.strip()
for value in os.environ.get("HERMES_MIGRATE_SESSION_IDS", "").split(",")
if value.strip()
]
target_home.mkdir(parents=True, exist_ok=True)
(target_home / "home" / ".local" / "bin").mkdir(parents=True, exist_ok=True)
(target_home / "workspace" / "skills").mkdir(parents=True, exist_ok=True)
(target_home / "logs").mkdir(parents=True, exist_ok=True)
for filename in (".env", "auth.json"):
_copy_if_missing(source_home / filename, target_home / filename)
source_db = source_home / "state.db"
target_db = target_home / "state.db"
SessionDB(db_path=target_db).close()
copied = 0
if source_db.is_file() and session_ids:
source = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)
target = sqlite3.connect(target_db)
target.execute("ATTACH DATABASE ? AS source_db", (str(source_db),))
try:
with target:
for session_id in session_ids:
copied += int(_copy_session(source, target, session_id, user_id))
finally:
target.close()
source.close()
print(f"isolated Hermes home ready; migrated_sessions={copied}")
if __name__ == "__main__":
main()

2
infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml
View File

@ -2,7 +2,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: longhorn-disk-tags-ensure-3
name: longhorn-disk-tags-ensure-4
namespace: longhorn-system
spec:
backoffLimit: 0

16
infrastructure/traefik/deployment.yaml
View File

@ -4,8 +4,6 @@ items:
- apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "4"
name: traefik
namespace: traefik
spec:
@ -73,6 +71,14 @@ items:
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: hardware
operator: In
values:
- rpi5
- rpi4
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
@ -103,6 +109,12 @@ items:
operator: In
values:
- rpi4
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app: traefik
topologyKey: kubernetes.io/hostname
restartPolicy: Always
schedulerName: default-scheduler
serviceAccount: atlas-traefik-ingress-controller

251
scripts/render/dashboards_render_atlas.py
View File

@ -2,8 +2,8 @@
"""Generate Atlas Grafana dashboards and render them into ConfigMaps.
Usage:
scripts/render/dashboards_render_atlas.py --build # rebuild JSON + ConfigMaps
scripts/render/dashboards_render_atlas.py # re-render ConfigMaps from JSON
scripts/dashboards_render_atlas.py --build # rebuild JSON + ConfigMaps
scripts/dashboards_render_atlas.py # re-render ConfigMaps from JSON
"""
import argparse
@ -16,14 +16,7 @@ from pathlib import Path
# Paths, folders, and shared metadata
# ---------------------------------------------------------------------------
def repo_root() -> Path:
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
ROOT = repo_root()
ROOT = Path(__file__).resolve().parents[1]
DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards"
CONFIG_TEMPLATE = textwrap.dedent(
"""# {relative_path}
@ -117,8 +110,6 @@ WORKER_NODES = [
"titan-06",
"titan-07",
"titan-08",
"titan-09",
"titan-10",
"titan-11",
"titan-20",
"titan-21",
@ -126,7 +117,6 @@ WORKER_NODES = [
"titan-13",
"titan-14",
"titan-15",
"titan-16",
"titan-17",
"titan-18",
"titan-19",
@ -191,7 +181,8 @@ def scoped_node_expr(base, scope=""):
def node_cpu_expr(scope=""):
idle = 'avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))'
base = f"(1 - {idle}) * 100"
# Scrape stalls can briefly report an impossible idle rate after recovery.
base = f"clamp_max(clamp_min((1 - {idle}) * 100, 0), 100)"
return scoped_node_expr(base, scope)
@ -291,8 +282,14 @@ def dcgm_gpu_util_by_node():
)
def nvidia_gpu_util_by_node():
return "max by (node) (nvidia_gpu_device_utilization_percent)"
def gpu_util_by_node():
return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}"
process_exporter = nvidia_gpu_util_by_node()
dcgm_fallback = f"({dcgm_gpu_util_by_node()}) unless on(node) ({process_exporter})"
return f"{process_exporter} or {dcgm_fallback} or {jetson_gpu_util_by_node()}"
def gpu_util_by_hostname():
@ -302,16 +299,11 @@ def gpu_util_by_hostname():
GPU_RESOURCE_REGEX = "nvidia(_com_|[.]com/)gpu.*"
def gpu_node_labels():
return f'max by (node) (kube_node_status_allocatable{{resource=~"{GPU_RESOURCE_REGEX}"}} > bool 0)'
def gpu_requests_by_namespace_node(scope_var):
return (
"sum by (namespace,node) ("
f'kube_pod_container_resource_requests{{resource=~"{GPU_RESOURCE_REGEX}",{scope_var}}} '
"* on(namespace,pod) group_left(node) kube_pod_info "
f"* on(node) group_left() ({gpu_node_labels()})"
")"
)
@ -333,7 +325,8 @@ def gpu_usage_by_namespace(scope_var):
def jetson_gpu_usage_by_namespace(scope_var):
requests_by_ns = gpu_requests_by_namespace_node(scope_var)
total_by_node = f"sum by (node) ({requests_by_ns})"
all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
total_by_node = f"sum by (node) ({all_requests})"
return (
"sum by (namespace) ("
f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) "
@ -342,6 +335,12 @@ def jetson_gpu_usage_by_namespace(scope_var):
)
def jetson_gpu_requested_nodes():
all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
requested = f"(sum by (node) ({all_requests}) > 0)"
return f"({requested}) and on(node) ({jetson_gpu_util_by_node()})"
def namespace_share_expr(resource_expr):
total = f"clamp_min(sum( {resource_expr} ), 1)"
return f"100 * ( {resource_expr} ) / {total}"
@ -391,14 +390,29 @@ def gpu_total_devices_expr():
def unattributed_gpu_usage():
legacy_total = f"(sum({legacy_gpu_util_without_process_exporter()}) or on() vector(0))"
unresolved = (
f"({legacy_gpu_util_without_process_exporter()}) "
f"unless on(node) ({jetson_gpu_requested_nodes()})"
)
legacy_total = f"(sum({unresolved}) or on() vector(0))"
return (
f'label_replace(({legacy_total} > 0), "namespace", "unattributed", "", "")'
)
def gpu_utilization_raw(scope_var):
return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
nvidia = (
'label_replace('
f'{nvidia_process_gpu_usage_by_namespace(scope_var)}, '
'"gpu_source", "nvidia", "", "")'
)
jetson = (
'label_replace('
f'(({jetson_gpu_usage_by_namespace(scope_var)}) > 0), '
'"gpu_source", "jetson", "", "")'
)
attributed = f"sum by (namespace) (({nvidia}) or ({jetson}))"
return f"({attributed}) or ({unattributed_gpu_usage()})"
def gpu_pool_used_expr(scope_var):
@ -418,13 +432,20 @@ def namespace_gpu_share_expr(scope_var):
PROBLEM_PODS_EXPR = (
'sum(max by (namespace,pod) (kube_pod_status_phase{phase!~"Running|Succeeded"})) '
"or on() vector(0)"
'((sum(max by(namespace,pod) ('
'(kube_pod_status_phase{phase="Pending",namespace!~"veles"} == 1) '
'and on(namespace,pod) ((time() - kube_pod_created{namespace!~"veles"}) > 900)'
')) or on() vector(0)) + (sum(max by(namespace,pod) ('
'(kube_pod_status_phase{phase=~"Failed|Unknown",namespace!~"veles"} == 1) '
'unless on(namespace,pod) kube_pod_owner{owner_kind="Job"}'
')) or on() vector(0)))'
)
CRASHLOOP_EXPR = (
'sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason'
'{reason=~"CrashLoopBackOff|ImagePullBackOff"})) '
"or on() vector(0)"
'sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason'
'{namespace!~"veles",reason=~"CrashLoopBackOff|ImagePullBackOff"} '
'and on(namespace,pod) '
'((time() - kube_pod_created{namespace!~"veles"}) > 900))) '
'or on() vector(0)'
)
STUCK_TERMINATING_EXPR = (
'sum(max by (namespace,pod) ('
@ -436,23 +457,22 @@ STUCK_TERMINATING_EXPR = (
UPTIME_WINDOW = "365d"
# vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series.
UPTIME_RECORDING_METRIC = (
f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}'
f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="request-v4"}}'
)
TRAEFIK_READY_EXPR = (
"("
'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
" > bool 0)"
AVAILABILITY_REQUESTS_1H_EXPR = (
'sum(increase(traefik_entrypoint_requests_total{'
'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))'
)
CONTROL_READY_FRACTION_EXPR = (
f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
" >= bool 2)"
AVAILABILITY_FAILURES_1H_EXPR = (
'sum(increase(traefik_entrypoint_requests_total{'
'entrypoint="websecure",protocol="http",code=~"5.."}[1h]))'
)
UPTIME_AVAIL_EXPR = (
f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
UPTIME_LIVE_FALLBACK_EXPR = (
f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / "
f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
)
UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])"
UPTIME_RECORDING_EXPR = (
f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
f"(last_over_time({UPTIME_RECORDING_METRIC}[48h]) "
f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
)
@ -631,7 +651,7 @@ PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX = (
"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube"
)
PLATFORM_TEST_STANDARD_CHECK_REGEX = (
"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|semgrep|supply_chain"
"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|supply_chain"
)
PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_SUITES = [
"ariadne",
@ -1881,9 +1901,9 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
"Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.",
"Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.",
"Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.",
"Atlas Serving Availability": "Observed availability with control-plane quorum and at least one Traefik replica serving; partial replica capacity remains available.",
"Problem Pods": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.",
"CrashLoop / ImagePull": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.",
"Atlas Availability (365d)": "Request-weighted Atlas ingress availability; every server-side 5xx response counts as a failed request.",
"Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.",
"CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.",
"Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.",
"Hottest node: CPU": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.",
"Hottest node: RAM": "Highest worker memory use right now; lower is safer, high values risk evictions.",
@ -1921,7 +1941,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
"Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
"Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
"Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
"Namespace GPU Utilization": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.",
"Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
"Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
"Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
"Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",
@ -1963,7 +1983,6 @@ TESTING_PANEL_DESCRIPTIONS = {
"Style Failure Rate": "Percent of style checks currently failing; blue means style/docs gates pass.",
"Gate Glue Failure Rate": "Percent of metric-contract checks failing; blue means dashboard telemetry is trustworthy.",
"SonarQube Failure Rate": "Percent of Sonar checks failing; blue means Sonar quality gates pass.",
"Semgrep Failure Rate": "Percent of Semgrep checks failing; blue means SAST rule scans pass.",
"Supply Chain Failure Rate": "Percent of supply-chain checks failing; blue means artifact/image checks pass.",
"Check Healthy Rates By Suite": "Healthy percent by check family; blue means all selected checks are good.",
"Tests Healthy Rate": "Percent of test checks passing or not applicable; higher is better.",
@ -1972,7 +1991,6 @@ TESTING_PANEL_DESCRIPTIONS = {
"Style Healthy Rate": "Percent of style checks passing or not applicable; higher is better.",
"Gate Glue Healthy Rate": "Percent of telemetry-contract checks passing; higher means cleaner reporting.",
"SonarQube Healthy Rate": "Percent of Sonar checks passing or not applicable; higher is better.",
"Semgrep Healthy Rate": "Percent of Semgrep checks passing or not applicable; higher is better.",
"Supply Chain Healthy Rate": "Percent of supply-chain checks passing or not applicable; higher is better.",
"Test Drilldowns And Problem Tests": "Test-case detail for finding which tests are hurting reliability.",
"Problematic Tests Over Time (Top failures)": "Current outlier tests by rolling 24h failures; tests need repeat failures to stay visible.",
@ -1989,13 +2007,6 @@ TESTING_PANEL_DESCRIPTIONS = {
"Recent Branch Evidence by Suite (7d)": "Branches with recent CI evidence; unexpected branches can mean drift or stale work.",
"Primary Branch Clean by Suite (7d)": "Percent clean of non-primary branch evidence; 100% means only main/master is reporting.",
"SonarQube Project Health": "SonarQube availability, projects, fetch errors, and gate status.",
"Public ZAP Baseline": "OWASP ZAP passive baseline results for externally addressed Atlas domains.",
"ZAP Targets Scanned": "Number of public hostnames with a latest OWASP ZAP baseline scan sample.",
"ZAP High/Medium Alerts": "Current high and medium ZAP alert instance count across public targets.",
"ZAP Scan Errors": "Targets whose latest ZAP baseline failed or did not produce a usable report.",
"ZAP Alerts by Risk": "Current ZAP alert distribution by risk level.",
"ZAP Target Health": "Latest ZAP health per hostname from passive baseline scans.",
"ZAP Alerts by Host": "Current high, medium, and low ZAP alert counts by public hostname.",
"SonarQube API Up": "Whether the SonarQube exporter can reach SonarQube; 1 is good.",
"Sonar Projects (Selected)": "Selected SonarQube project count; zero means Sonar is not tracking that suite.",
"Sonar Gate Fetch Errors": "Sonar exporter fetch errors; zero is good because stale Sonar data misleads.",
@ -2123,7 +2134,7 @@ def build_overview():
},
{
"id": 27,
"title": "Atlas Serving Availability",
"title": "Atlas Availability (365d)",
"expr": UPTIME_PERCENT_EXPR,
"kind": "stat",
"thresholds": UPTIME_PERCENT_THRESHOLDS,
@ -2131,7 +2142,7 @@ def build_overview():
"decimals": 4,
"text_mode": "value",
"instant": True,
"description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.",
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists.",
},
{
"id": 4,
@ -4183,7 +4194,6 @@ def build_jobs_dashboard():
check_regex_style = "docs|naming|hygiene|lint|docs_naming|style"
check_regex_gate_glue = "gate|glue|gate_glue"
check_regex_sonarqube = "sonarqube|sonar"
check_regex_semgrep = "semgrep|sast"
check_regex_supply_chain = "ironbank|supply_chain|image_compliance|artifact_security"
def _check_state_percent_series(regex: str, failed: bool) -> str:
@ -4227,30 +4237,6 @@ def build_jobs_dashboard():
f"topk by (suite) (1, ({rollup_failed_tests_history}))"
)
worst_test_per_suite = worst_test_per_suite_core
zap_target_health = (
"(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) "
"or on() vector(0))"
)
zap_targets_scanned = (
'(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job="platform-security-zap"})) '
"or on() vector(0))"
)
zap_high_medium_alerts = (
'(sum(platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk=~"high|medium"}) '
"or on() vector(0))"
)
zap_scan_errors = (
'(sum(platform_zap_baseline_scan_status{exported_job="platform-security-zap",status=~"fail|error"}) '
"or on() vector(0))"
)
zap_alerts_by_risk = (
'sum by (risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk!=""}) '
"or on() vector(0)"
)
zap_alerts_by_host_risk = (
'sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",'
'risk=~"high|medium|low"}) or on() vector(0)'
)
def _selected_status_volume(status: str) -> str:
return (
@ -4596,7 +4582,6 @@ def build_jobs_dashboard():
("Style", check_regex_style),
("Gate Glue", check_regex_gate_glue),
("SonarQube", check_regex_sonarqube),
("Semgrep", check_regex_semgrep),
("Supply Chain", check_regex_supply_chain),
]
@ -4620,12 +4605,12 @@ def build_jobs_dashboard():
)
panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
panels.append(panel)
for index, (label, regex) in enumerate(check_dimensions[4:8]):
for index, (label, regex) in enumerate(check_dimensions[4:]):
panel = state_timeline_panel(
start_id + 4 + index,
f"{label} {title_prefix}",
_check_state_percent_series(regex, failed),
{"h": 7, "w": 6, "x": index * 6, "y": y + 7},
{"h": 7, "w": 8, "x": index * 8, "y": y + 7},
thresholds=trend_thresholds,
description=trend_description,
)
@ -4633,7 +4618,7 @@ def build_jobs_dashboard():
panels.append(panel)
_append_check_trends(130, "Failure Rate", True, 29)
_append_check_trends(160, "Healthy Rate", False, 43)
_append_check_trends(138, "Healthy Rate", False, 43)
panels.append(
state_timeline_panel(
145,
@ -4889,79 +4874,6 @@ def build_jobs_dashboard():
)
sonar_gate_project_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
panels.append(sonar_gate_project_panel)
panels.append(
stat_panel(
168,
"ZAP Targets Scanned",
zap_targets_scanned,
{"h": 6, "w": 4, "x": 0, "y": 117},
unit="none",
instant=True,
thresholds=missing_thresholds,
)
)
panels.append(
stat_panel(
169,
"ZAP High/Medium Alerts",
zap_high_medium_alerts,
{"h": 6, "w": 4, "x": 4, "y": 117},
unit="none",
instant=True,
thresholds=failures_thresholds,
)
)
panels.append(
stat_panel(
170,
"ZAP Scan Errors",
zap_scan_errors,
{"h": 6, "w": 4, "x": 8, "y": 117},
unit="none",
instant=True,
thresholds=failures_thresholds,
)
)
zap_risk_panel = pie_panel(
171,
"ZAP Alerts by Risk",
zap_alerts_by_risk,
{"h": 6, "w": 4, "x": 12, "y": 117},
)
zap_risk_panel["targets"][0]["legendFormat"] = "{{risk}}"
panels.append(zap_risk_panel)
zap_target_panel = state_timeline_panel(
172,
"ZAP Target Health",
zap_target_health,
{"h": 6, "w": 8, "x": 16, "y": 117},
thresholds=success_thresholds,
unit="percent",
min_value=0,
max_value=100,
legend="{{host}}",
description=(
"Latest OWASP ZAP passive baseline health per public hostname. "
"High alerts and scan errors drop a target to zero; medium/low alerts mark warning health."
),
)
zap_target_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
panels.append(zap_target_panel)
panels.append(
bargauge_panel(
173,
"ZAP Alerts by Host",
zap_alerts_by_host_risk,
{"h": 7, "w": 24, "x": 0, "y": 123},
unit="none",
instant=True,
legend="{{host}} · {{risk}}",
sort_order="desc",
thresholds=failures_thresholds,
decimals=0,
limit=20,
)
)
panels.append(
bargauge_panel(
148,
@ -5071,12 +4983,6 @@ def build_jobs_dashboard():
33: {"h": 6, "w": 4, "x": 8, "y": 111},
34: {"h": 6, "w": 4, "x": 12, "y": 111},
35: {"h": 6, "w": 8, "x": 16, "y": 111},
168: {"h": 6, "w": 4, "x": 0, "y": 117},
169: {"h": 6, "w": 4, "x": 4, "y": 117},
170: {"h": 6, "w": 4, "x": 8, "y": 117},
171: {"h": 6, "w": 4, "x": 12, "y": 117},
172: {"h": 6, "w": 8, "x": 16, "y": 117},
173: {"h": 7, "w": 24, "x": 0, "y": 123},
}
for panel_id, grid in row_layout.items():
panel_by_id[panel_id]["gridPos"] = grid
@ -5088,13 +4994,13 @@ def build_jobs_dashboard():
501,
"Check Failure Rates By Suite",
12,
panels=children([130, 131, 132, 133, 134, 135, 136, 137]),
panels=children([130, 131, 132, 133, 134, 135, 136]),
),
row_panel(
502,
"Check Healthy Rates By Suite",
13,
panels=children([160, 161, 162, 163, 164, 165, 166, 167]),
panels=children([138, 139, 140, 141, 142, 143, 144]),
),
row_panel(
503,
@ -5114,12 +5020,6 @@ def build_jobs_dashboard():
16,
panels=children([31, 32, 33, 34, 35]),
),
row_panel(
506,
"Public ZAP Baseline",
17,
panels=children([168, 169, 170, 171, 172, 173]),
),
]
)
panels = compact_panels
@ -5621,12 +5521,13 @@ def build_gpu_dashboard():
panels.append(
table_panel(
4,
"GPU Pods Reporting Device Util",
'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))',
"GPU Processes by Pod",
'topk(10, sum by (namespace,pod,node,process) '
'(nvidia_process_gpu_sm_util_percent{pod!="host"}) > 0)',
{"h": 8, "w": 12, "x": 12, "y": 8},
unit="percent",
transformations=[{"id": "labelsToFields", "options": {}}],
description="DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.",
description="NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization.",
)
)
return {

28
scripts/tests/test_availability_cleanup.py Normal file
View File

@ -0,0 +1,28 @@
"""Safety tests for the legacy Atlas availability cleanup."""
import importlib.util
from pathlib import Path
def load_module():
"""Load the service-owned cleanup module without packaging it."""
path = (
Path(__file__).resolve().parents[2]
/ "services/monitoring/scripts/availability_cleanup.py"
)
spec = importlib.util.spec_from_file_location("availability_cleanup", path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
return module
def test_cleanup_selector_cannot_match_request_v4() -> None:
"""Constrain deletion to obsolete annual Atlas definitions."""
mod = load_module()
assert '__name__="atlas:availability:ratio_365d"' in mod.LEGACY_SELECTOR
assert 'scope="atlas"' in mod.LEGACY_SELECTOR
assert 'definition!="request-v4"' in mod.LEGACY_SELECTOR
assert 'definition="request-v4"' in mod.PROTECTED_SELECTOR
assert "ratio_1h" not in mod.LEGACY_SELECTOR

52
scripts/tests/test_availability_rollup.py Normal file
View File

@ -0,0 +1,52 @@
"""Unit tests for the Atlas availability publisher."""
import importlib.util
import json
from pathlib import Path
import pytest
def load_module():
"""Load the service-owned rollup module without packaging it."""
path = (
Path(__file__).resolve().parents[2]
/ "services/monitoring/scripts/availability_rollup.py"
)
spec = importlib.util.spec_from_file_location("availability_rollup", path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
return module
def test_parse_export_deduplicates_replay_boundaries() -> None:
"""Keep only the final value when replay chunks share a timestamp."""
mod = load_module()
lines = [
(json.dumps({"timestamps": [1000, 2000], "values": [2, 3]}) + "\n").encode(),
(json.dumps({"timestamps": [2000, 3000], "values": [3, 5]}) + "\n").encode(),
]
assert mod.parse_export(lines) == {1000: 2.0, 2000: 3.0, 3000: 5.0}
def test_calculate_availability_uses_all_server_failures() -> None:
"""Calculate one bounded successful-request ratio."""
mod = load_module()
assert mod.calculate_availability(1000, 2) == pytest.approx(0.998)
with pytest.raises(ValueError):
mod.calculate_availability(0, 0)
with pytest.raises(ValueError):
mod.calculate_availability(100, -1)
def test_render_metric_publishes_only_the_request_v4_series() -> None:
"""Render the single series selected by the Grafana panel."""
mod = load_module()
assert mod.render_metric(0.9995, 1234) == (
'atlas:availability:ratio_365d{definition="request-v4",scope="atlas",'
'rollup="yearly"} 0.999500000000 1234\n'
)

73
scripts/tests/test_dashboards_render_atlas.py
View File

@ -3,10 +3,8 @@ import pathlib
def load_module():
path = pathlib.Path(__file__).resolve().parents[1] / "render/dashboards_render_atlas.py"
spec = importlib.util.spec_from_file_location(
"scripts.render.dashboards_render_atlas", path
)
path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py"
spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
@ -49,6 +47,8 @@ def test_node_filter_and_expr_helpers():
cpu_expr = mod.node_cpu_expr("titan-.*")
mem_expr = mod.node_mem_expr("titan-.*")
assert "node_cpu_seconds_total" in cpu_expr
assert "clamp_max(clamp_min(" in cpu_expr
assert "* 100, 0), 100)" in cpu_expr
assert "node_memory_MemAvailable_bytes" in mem_expr
@ -57,20 +57,24 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
dashboard = mod.build_overview()
panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
assert panel["title"] == "Atlas Serving Availability"
assert panel["title"] == "Atlas Availability (365d)"
availability_expr = panel["targets"][0]["expr"]
assert (
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])'
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])'
in availability_expr
)
assert ">= bool 2" in availability_expr
assert "> bool 0" in availability_expr
assert "[1h:5m]" in availability_expr
assert 'code=~"5.."' in availability_expr
assert 'code=~"[1-5].."' in availability_expr
assert "atlas:availability:failures_1d" not in availability_expr
assert "atlas:availability:requests_1d" not in availability_expr
assert "sum_over_time" not in availability_expr
assert "kube_node_status_condition" not in availability_expr
assert "kube_deployment_status_replicas_available" not in availability_expr
assert panel["targets"][0]["instant"] is True
assert "at least one Traefik replica serving" in panel["description"]
assert "partial-capacity samples remain available" in panel["description"]
assert "monitoring gaps are excluded" in panel["description"]
assert "falls back to the live binary serving state" in panel["description"]
assert "Every server-side 5xx" in panel["description"]
assert "Replica counts, Grafana health" in panel["description"]
assert "daily rollup job publishes one annual sample" in panel["description"]
assert "keeps it for up to 48 hours" in panel["description"]
def test_overview_uses_readable_quality_power_and_gitops_panels():
@ -169,14 +173,41 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
assert "sum_over_time" not in gpu_expr
assert "count_over_time" not in gpu_expr
assert "avg_over_time" not in gpu_expr
assert "$__range" not in gpu_expr
assert "sum by (namespace)" in gpu_expr
assert 'namespace", "shared"' not in gpu_expr
assert "kube_node_labels" not in gpu_expr
assert "kube_pod_container_resource_requests" in gpu_expr
assert mod.GPU_RESOURCE_REGEX in gpu_expr
assert '"gpu_source", "nvidia"' in gpu_expr
assert '"gpu_source", "jetson"' in gpu_expr
assert "100 *" in gpu_expr
assert "100 -" not in gpu_expr
assert 'namespace", "unattributed"' in gpu_expr
assert 'namespace", "idle"' in gpu_expr
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
mod = load_module()
dashboard = mod.build_gpu_dashboard()
panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
assert "nvidia_gpu_device_utilization_percent" in node_expr
assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
assert "jetson_gr3d_freq_percent" in node_expr
assert "unless on(node)" in node_expr
assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
assert "nvidia_process_gpu_sm_util_percent" in process_expr
assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
def test_overview_and_testing_panels_all_have_concise_descriptions():
@ -396,7 +427,7 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
for child in row.get("panels", [])
}
assert len(panels) == 19
assert len(panels) == 18
assert len(visible_query_panels) == 12
assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
assert all(
@ -410,18 +441,15 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
"Test Drilldowns And Problem Tests",
"Telemetry Completeness And Branches",
"SonarQube Project Health",
"Public ZAP Baseline",
]
assert all(row["collapsed"] for row in rows)
assert "Coverage Failure Rate" in nested_panels_by_title
assert "Semgrep Failure Rate" in nested_panels_by_title
assert "Supply Chain Healthy Rate" in nested_panels_by_title
assert "Test Category Health History" in nested_panels_by_title
assert "Selected Test Pass Rate History" in nested_panels_by_title
assert "Coverage Metrics Present by Suite" in nested_panels_by_title
assert "SonarQube API Up" in nested_panels_by_title
assert "ZAP Target Health" in nested_panels_by_title
failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
assert failure_rate_panel["type"] == "state-timeline"
@ -472,15 +500,6 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
assert sonar_health_panel["type"] == "state-timeline"
assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"]
assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"]
semgrep_panel = nested_panels_by_title["Semgrep Healthy Rate"]
assert semgrep_panel["type"] == "state-timeline"
assert 'check=~"semgrep|sast"' in semgrep_panel["targets"][0]["expr"]
zap_health_panel = nested_panels_by_title["ZAP Target Health"]
assert zap_health_panel["type"] == "state-timeline"
assert "platform_zap_baseline_target_health_percent" in zap_health_panel["targets"][0]["expr"]
zap_alerts_panel = nested_panels_by_title["ZAP Alerts by Host"]
assert "platform_zap_baseline_alerts_total" in zap_alerts_panel["targets"][0]["expr"]
branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]

118
scripts/tests/test_monitoring_query_capacity.py
View File

@ -44,43 +44,113 @@ def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
assert {"titan-14", "titan-18"} <= set(hostname_rule["values"])
def test_yearly_availability_reuses_the_hourly_rollup() -> None:
"""Prevent the yearly rule from rescanning raw cluster metrics for 365 days."""
def test_yearly_availability_is_published_outside_the_query_pool() -> None:
"""Keep all long-range availability work out of Grafana and MetricsQL."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
yearly = next(
rule
for rule in rules["rules"]
if rule["record"] == "atlas:availability:ratio_365d"
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
rules = [rule for group in groups for rule in group["rules"]]
manifests = _documents(
REPO_ROOT / "services/monitoring/availability-rollup-cronjob.yaml"
)
cronjob = next(manifest for manifest in manifests if manifest["kind"] == "CronJob")
source = (
REPO_ROOT / "services/monitoring/scripts/availability_rollup.py"
).read_text()
assert "atlas:availability:ratio_1h" in yearly["expr"]
assert 'definition="serving-v2"' in yearly["expr"]
assert 'definition=""' in yearly["expr"]
assert "share_gt_over_time" in yearly["expr"]
assert "[365d:15m]" in yearly["expr"]
assert "kube_node_status_condition" not in yearly["expr"]
assert "[365d:1h]" not in yearly["expr"]
assert all(rule["record"] != "atlas:availability:ratio_365d" for rule in rules)
assert "atlas:availability:requests_1d" in source
assert "atlas:availability:failures_1d" in source
assert "/api/v1/export" in source
assert "/api/v1/import/prometheus" in source
assert cronjob["spec"]["schedule"] == "10 0 * * *"
assert cronjob["spec"]["concurrencyPolicy"] == "Forbid"
def test_availability_counts_serving_state_instead_of_replica_capacity() -> None:
"""Treat quorum and any serving ingress replica as available binary states."""
def test_daily_availability_rollups_use_the_same_request_sli() -> None:
"""Keep annual availability cheap without changing its request definition."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
hourly = next(
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
daily_group = next(
group for group in groups if group["name"] == "atlas.availability.rollup"
)
daily = {rule["record"]: rule for rule in daily_group["rules"]}
assert daily_group["interval"] == "1d"
assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"]
assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"]
assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values())
def test_availability_uses_request_failures_instead_of_replica_capacity() -> None:
"""Measure HTTP outcomes without treating redundant replica loss as downtime."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"]
rules = [rule for group in groups for rule in group["rules"]]
requests = next(
rule
for rule in rules["rules"]
if rule["record"] == "atlas:availability:ratio_1h"
for rule in rules
if rule["record"] == "atlas:availability:requests_1h"
)
failures = next(
rule
for rule in rules
if rule["record"] == "atlas:availability:failures_1h"
)
assert ">= bool 2" in hourly["expr"]
assert "> bool 0" in hourly["expr"]
assert "/ 3" not in hourly["expr"]
assert hourly["labels"]["definition"] == "serving-v2"
assert 'code=~"[1-5].."' in requests["expr"]
assert 'code=~"5.."' in failures["expr"]
assert "traefik_entrypoint_requests_total" in requests["expr"]
assert "traefik_entrypoint_requests_total" in failures["expr"]
assert "kube_node_status_condition" not in repr(rules)
assert "kube_deployment_status_replicas_available" not in repr(rules)
assert requests["labels"]["definition"] == "request-v4"
assert failures["labels"]["definition"] == "request-v4"
def test_availability_backfill_replays_the_same_request_sli() -> None:
"""Backfill retained history with the exact rules used for future samples."""
manifests = _documents(
REPO_ROOT / "services/monitoring/availability-backfill-v4-job.yaml"
)
config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
backfill = yaml.safe_load(config["data"]["atlas-request-history.yaml"])
expressions = {
rule["record"]: rule["expr"]
for group in backfill["groups"]
for rule in group["rules"]
}
assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1h"]
assert 'code=~"5.."' in expressions["atlas:availability:failures_1h"]
args = job["spec"]["template"]["spec"]["containers"][0]["args"]
assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args
assert "-replay.timeTo=2026-08-04T23:00:00Z" in args
assert "-replay.maxDatapointsPerQuery=48" in args
def test_daily_availability_backfill_stays_below_query_timeout() -> None:
"""Replay daily buckets in small chunks so raw history never starves Grafana."""
manifests = _documents(
REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml"
)
config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap")
job = next(manifest for manifest in manifests if manifest["kind"] == "Job")
backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"])
group = backfill["groups"][0]
expressions = {rule["record"]: rule["expr"] for rule in group["rules"]}
assert group["interval"] == "1d"
assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"]
assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"]
args = job["spec"]["template"]["spec"]["containers"][0]["args"]
assert "-replay.maxDatapointsPerQuery=4" in args
def test_quality_rollups_do_not_run_every_minute() -> None:

99
scripts/tests/test_nvidia_process_exporter.py Normal file
View File

@ -0,0 +1,99 @@
import importlib.util
from pathlib import Path
import sys
import types
import pytest
ROOT = Path(__file__).resolve().parents[2]
EXPORTER_PATH = ROOT / "services" / "monitoring" / "scripts" / "nvidia_process_exporter.py"
def load_exporter(monkeypatch):
"""Load the exporter without requiring an NVIDIA driver on the test host."""
pynvml = types.ModuleType("pynvml")
class NVMLError(Exception):
pass
class NVMLErrorNotFound(NVMLError):
pass
class NVMLErrorNotSupported(NVMLError):
pass
pynvml.NVMLError = NVMLError
pynvml.NVMLError_NotFound = NVMLErrorNotFound
pynvml.NVMLError_NotSupported = NVMLErrorNotSupported
for name in (
"nvmlDeviceGetComputeRunningProcesses_v3",
"nvmlDeviceGetCount",
"nvmlDeviceGetGraphicsRunningProcesses_v3",
"nvmlDeviceGetHandleByIndex",
"nvmlDeviceGetName",
"nvmlDeviceGetProcessUtilization",
"nvmlDeviceGetUUID",
"nvmlDeviceGetUtilizationRates",
"nvmlInit",
):
setattr(pynvml, name, lambda *args, **kwargs: None)
monkeypatch.setitem(sys.modules, "pynvml", pynvml)
spec = importlib.util.spec_from_file_location("nvidia_process_exporter_test", EXPORTER_PATH)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_process_sample_window_uses_nvml_microseconds(monkeypatch):
exporter = load_exporter(monkeypatch)
observed = []
monkeypatch.setattr(exporter.time, "time", lambda: 1_700_000_000.0)
monkeypatch.setattr(
exporter,
"nvmlDeviceGetProcessUtilization",
lambda handle, since: observed.append(since) or [],
)
samples, supported = exporter.process_utilization_samples(object())
assert samples == {}
assert supported == 1
assert observed == [1_700_000_000_000_000 - 30_000_000]
def test_namespace_attribution_scales_to_current_device_total(monkeypatch):
exporter = load_exporter(monkeypatch)
result = exporter.reconcile_namespace_utilization(
{"game-stream": 20, "hermes": 10},
device_util=3,
)
assert sum(result.values()) == pytest.approx(3)
assert result["game-stream"] == pytest.approx(2)
assert result["hermes"] == pytest.approx(1)
def test_namespace_attribution_assigns_unexplained_compute_to_host(monkeypatch):
exporter = load_exporter(monkeypatch)
result = exporter.reconcile_namespace_utilization(
{"hermes": 1},
device_util=3,
)
assert result == {"hermes": 1, "host": 2}
def test_zero_device_utilization_clears_stale_process_samples(monkeypatch):
exporter = load_exporter(monkeypatch)
result = exporter.reconcile_namespace_utilization(
{"hermes": 40, "game-stream": 5},
device_util=0,
)
assert result == {"hermes": 0, "game-stream": 0}

7
services/bstein-dev-home/backend-deployment.yaml
View File

@ -68,7 +68,8 @@ spec:
args:
- >-
. /vault/secrets/portal-env.sh
&& exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 600 app:app
&& export SMTP_HOST=""
&& exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 1200 app:app
env:
- name: AI_CHAT_API
value: http://ollama.ai.svc.cluster.local:11434
@ -119,6 +120,8 @@ spec:
value: http://ariadne.maintenance.svc.cluster.local
- name: ARIADNE_TIMEOUT_SEC
value: "10"
- name: ARIADNE_GAME_MODE_TIMEOUT_SEC
value: "900"
- name: ACCOUNT_ALLOWED_GROUPS
value: ""
- name: HTTP_CHECK_TIMEOUT_SEC
@ -139,6 +142,8 @@ spec:
value: "10"
- name: PORTAL_RUN_MIGRATIONS
value: "false"
- name: ACCESS_REQUEST_ENABLED
value: "false"
- name: ACCESS_REQUEST_SUBMIT_RATE_LIMIT
value: "30"
- name: ACCESS_REQUEST_SUBMIT_RATE_WINDOW_SEC

4
services/bstein-dev-home/kustomization.yaml
View File

@ -20,9 +20,9 @@ resources:
- ingress.yaml
images:
- name: registry.bstein.dev/bstein/bstein-dev-home-frontend
newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
newTag: 0.1.1-441 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
- name: registry.bstein.dev/bstein/bstein-dev-home-backend
newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
newTag: 0.1.1-441 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
configMapGenerator:
- name: chat-ai-gateway
namespace: bstein-dev-home

397
services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml Normal file
View File

@ -0,0 +1,397 @@
# services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml
# One-off job for sso/cassandra-ldap-federation-3.
# Purpose: attach the Cassandra Keycloak realm to the shared OpenLDAP directory.
# Run unsuspended for migration, then suspend/replace with a new job name for future runs.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-ldap-federation-3
namespace: sso
spec:
ttlSecondsAfterFinished: 3600
suspend: true
backoffLimit: 2
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "sso"
vault.hashicorp.com/agent-inject-secret-keycloak-env.sh: "kv/data/atlas/shared/keycloak-admin"
vault.hashicorp.com/agent-inject-template-keycloak-env.sh: |
{{ with secret "kv/data/atlas/shared/keycloak-admin" }}
export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
{{ end }}
{{ with secret "kv/data/atlas/sso/openldap-admin" }}
export LDAP_ADMIN_PASSWORD="{{ .Data.data.LDAP_ADMIN_PASSWORD }}"
export LDAP_CONFIG_PASSWORD="{{ .Data.data.LDAP_CONFIG_PASSWORD }}"
export LDAP_BIND_PASSWORD="${LDAP_ADMIN_PASSWORD}"
{{ end }}
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: hardware
operator: In
values: ["rpi5","rpi4"]
- key: node-role.kubernetes.io/worker
operator: Exists
restartPolicy: OnFailure
serviceAccountName: sso-vault
containers:
- name: configure
image: python:3.11-alpine
imagePullPolicy: IfNotPresent
env:
- name: KEYCLOAK_SERVER
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: cassandra
- name: LDAP_URL
value: ldap://openldap.sso.svc.cluster.local:389
- name: LDAP_BIND_DN
value: cn=admin,dc=bstein,dc=dev
- name: LDAP_USERS_DN
value: ou=users,dc=bstein,dc=dev
- name: LDAP_GROUPS_DN
value: ou=groups,dc=bstein,dc=dev
- name: VERIFY_USERS
value: veles-dev,cassandra-dev,daniel-test,viktor-test
command: ["/bin/sh", "-c"]
args:
- |
set -euo pipefail
. /vault/secrets/keycloak-env.sh
python - <<'PY'
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/")
realm = os.environ["KEYCLOAK_REALM"]
admin_user = os.environ["KEYCLOAK_ADMIN_USER"]
admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"]
ldap_url = os.environ["LDAP_URL"]
ldap_bind_dn = os.environ["LDAP_BIND_DN"]
ldap_bind_password = os.environ["LDAP_BIND_PASSWORD"]
ldap_users_dn = os.environ["LDAP_USERS_DN"]
ldap_groups_dn = os.environ["LDAP_GROUPS_DN"]
def request(method, url, token=None, payload=None, timeout=30):
data = None
headers = {}
if token:
headers["Authorization"] = f"Bearer {token}"
if payload is not None:
data = json.dumps(payload).encode()
headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
body = resp.read()
if not body:
return resp.status, None, dict(resp.headers)
return resp.status, json.loads(body.decode()), dict(resp.headers)
except urllib.error.HTTPError as exc:
raw = exc.read()
body = None
if raw:
try:
body = json.loads(raw.decode())
except Exception:
body = {"raw": raw.decode(errors="replace")}
return exc.code, body, dict(exc.headers)
def get_token():
data = urllib.parse.urlencode(
{
"grant_type": "password",
"client_id": "admin-cli",
"username": admin_user,
"password": admin_password,
}
).encode()
req = urllib.request.Request(
f"{base_url}/realms/master/protocol/openid-connect/token",
data=data,
headers={"Content-Type": "application/x-www-form-urlencoded"},
method="POST",
)
with urllib.request.urlopen(req, timeout=30) as resp:
return json.loads(resp.read().decode())["access_token"]
token = None
for attempt in range(1, 31):
try:
token = get_token()
break
except Exception as exc:
if attempt == 30:
raise SystemExit(f"Keycloak token request failed: {exc}")
time.sleep(2)
assert token
status, realm_rep, _ = request("GET", f"{base_url}/admin/realms/{realm}", token)
if status != 200 or not isinstance(realm_rep, dict) or not realm_rep.get("id"):
raise SystemExit(f"Unable to resolve realm id for {realm}: status={status}")
realm_id = realm_rep["id"]
status, components, _ = request(
"GET",
f"{base_url}/admin/realms/{realm}/components",
token,
)
if status != 200:
raise SystemExit(f"Unable to list realm components: status={status}")
components = components or []
for component in components:
if component.get("providerId") != "ldap":
continue
if component.get("providerType") != "org.keycloak.storage.UserStorageProvider":
continue
if component.get("parentId") == realm_id:
continue
cid = component.get("id")
if not cid:
continue
status, payload, _ = request(
"GET",
f"{base_url}/admin/realms/{realm}/components/{cid}",
token,
)
if status != 200 or not isinstance(payload, dict):
raise SystemExit(f"Unable to fetch LDAP component {cid}: status={status}")
payload["parentId"] = realm_id
status, _, _ = request(
"PUT",
f"{base_url}/admin/realms/{realm}/components/{cid}",
token,
payload,
)
if status not in (200, 204):
raise SystemExit(f"Unable to repair LDAP component {cid}: status={status}")
status, storage_components, _ = request(
"GET",
f"{base_url}/admin/realms/{realm}/components?type=org.keycloak.storage.UserStorageProvider",
token,
)
if status != 200:
raise SystemExit(f"Unable to list user-storage components: status={status}")
storage_components = storage_components or []
ldap_components = [
c
for c in storage_components
if c.get("providerId") == "ldap" and c.get("id")
]
candidates = []
for component in ldap_components:
config = component.get("config") or {}
if component.get("name") in ("openldap", "ldap") and (config.get("connectionUrl") or [None])[0] == ldap_url:
candidates.append(component)
if not candidates:
candidates = [c for c in ldap_components if c.get("name") in ("openldap", "ldap")]
candidates.sort(key=lambda item: item.get("id", ""))
ldap_component = candidates[0] if candidates else None
ldap_component_id = ldap_component.get("id") if ldap_component else None
desired = {
"name": "openldap",
"providerId": "ldap",
"providerType": "org.keycloak.storage.UserStorageProvider",
"parentId": realm_id,
"config": {
"enabled": ["true"],
"priority": ["0"],
"importEnabled": ["true"],
"editMode": ["WRITABLE"],
"syncRegistrations": ["true"],
"vendor": ["other"],
"connectionUrl": [ldap_url],
"bindDn": [ldap_bind_dn],
"bindCredential": [ldap_bind_password],
"authType": ["simple"],
"usersDn": [ldap_users_dn],
"searchScope": ["1"],
"pagination": ["true"],
"usernameLDAPAttribute": ["uid"],
"rdnLDAPAttribute": ["uid"],
"uuidLDAPAttribute": ["entryUUID"],
"userObjectClasses": ["inetOrgPerson, organizationalPerson, person, top"],
"trustEmail": ["true"],
"useTruststoreSpi": ["never"],
"connectionPooling": ["true"],
"cachePolicy": ["DEFAULT"],
"useKerberosForPasswordAuthentication": ["false"],
"allowKerberosAuthentication": ["false"],
},
}
if ldap_component_id:
desired["id"] = ldap_component_id
print(f"Updating Cassandra LDAP federation provider: {ldap_component_id}")
status, _, _ = request(
"PUT",
f"{base_url}/admin/realms/{realm}/components/{ldap_component_id}",
token,
desired,
)
if status not in (200, 204):
raise SystemExit(f"LDAP provider update failed: status={status}")
else:
print("Creating Cassandra LDAP federation provider")
status, _, headers = request(
"POST",
f"{base_url}/admin/realms/{realm}/components",
token,
desired,
)
if status not in (201, 204):
raise SystemExit(f"LDAP provider create failed: status={status}")
location = headers.get("Location", "")
ldap_component_id = location.rstrip("/").split("/")[-1] if location else None
if not ldap_component_id:
raise SystemExit("Unable to determine Cassandra LDAP provider id")
status, mapper_components, _ = request(
"GET",
f"{base_url}/admin/realms/{realm}/components?type=org.keycloak.storage.ldap.mappers.LDAPStorageMapper",
token,
)
if status != 200:
raise SystemExit(f"Unable to list LDAP mappers: status={status}")
mapper_components = mapper_components or []
def upsert_mapper(name, provider_id, config):
existing = next(
(
c
for c in mapper_components
if c.get("name") == name and c.get("parentId") == ldap_component_id
),
None,
)
payload = {
"name": name,
"providerId": provider_id,
"providerType": "org.keycloak.storage.ldap.mappers.LDAPStorageMapper",
"parentId": ldap_component_id,
"config": config,
}
if existing:
payload["id"] = existing["id"]
status, _, _ = request(
"PUT",
f"{base_url}/admin/realms/{realm}/components/{existing['id']}",
token,
payload,
)
action = "Updated"
else:
status, _, _ = request(
"POST",
f"{base_url}/admin/realms/{realm}/components",
token,
payload,
)
action = "Created"
if status not in (200, 201, 204):
raise SystemExit(f"Mapper {name} upsert failed: status={status}")
print(f"{action} Cassandra LDAP mapper: {name}")
upsert_mapper(
"openldap-groups",
"group-ldap-mapper",
{
"groups.dn": [ldap_groups_dn],
"group.name.ldap.attribute": ["cn"],
"group.object.classes": ["groupOfNames"],
"membership.ldap.attribute": ["member"],
"membership.attribute.type": ["DN"],
"mode": ["LDAP_ONLY"],
"user.roles.retrieve.strategy": ["LOAD_GROUPS_BY_MEMBER_ATTRIBUTE"],
"preserve.group.inheritance": ["true"],
},
)
for name, ldap_attr, user_attr in (
("openldap-email", "mail", "email"),
("openldap-first-name", "givenName", "firstName"),
("openldap-last-name", "sn", "lastName"),
):
upsert_mapper(
name,
"user-attribute-ldap-mapper",
{
"ldap.attribute": [ldap_attr],
"user.model.attribute": [user_attr],
"read.only": ["false"],
"always.read.value.from.ldap": ["false"],
"is.mandatory.in.ldap": ["false"],
},
)
status, sync_body, _ = request(
"POST",
f"{base_url}/admin/realms/{realm}/user-storage/{ldap_component_id}/sync?action=triggerFullSync",
token,
timeout=120,
)
if status in (200, 201, 204):
print(f"Cassandra LDAP full sync requested: {sync_body}")
else:
print(f"WARNING: Cassandra LDAP full sync returned status={status} body={sync_body}")
for username in (item.strip() for item in os.environ.get("VERIFY_USERS", "").split(",")):
if not username:
continue
encoded = urllib.parse.quote(username)
status, users, _ = request(
"GET",
f"{base_url}/admin/realms/{realm}/users?username={encoded}&exact=true",
token,
)
found = status == 200 and any(
str(user.get("username") or "").casefold() == username.casefold()
for user in users or []
)
print(f"Cassandra LDAP user check {username}: {'found' if found else 'missing'}")
for audit_realm in ("veles", realm):
status, count, _ = request(
"GET",
f"{base_url}/admin/realms/{audit_realm}/users/count",
token,
)
if status == 200:
print(f"SSO audit realm {audit_realm}: user_count={count}")
for term in (item.strip() for item in os.environ.get("VERIFY_USERS", "").split(",")):
if not term:
continue
encoded = urllib.parse.quote(term)
status, users, _ = request(
"GET",
f"{base_url}/admin/realms/{audit_realm}/users?search={encoded}",
token,
)
users = users if status == 200 and isinstance(users, list) else []
matches = []
for user in users:
username = str(user.get("username") or "")
email = str(user.get("email") or "")
if term.casefold() in username.casefold() or term.casefold() in email.casefold():
matches.append(username if not email else f"{username}<{email}>")
joined = ", ".join(matches[:8]) if matches else "none"
print(f"SSO audit realm {audit_realm} search {term}: {joined}")
print("Cassandra LDAP federation ready")
PY

30
services/cassandra-auth/bootstrap-jobs/cassandra-realm-ensure-job.yaml
View File

@ -1,11 +1,11 @@
# services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml
# One-off job for sso/cassandra-realm-ensure-1.
# One-off job for sso/cassandra-realm-ensure-4.
# Purpose: create the Cassandra realm, groups, OIDC client, SMTP settings, and Vault client secret.
# Keep suspended until Cassandra Vault paths/policies have reconciled, then unsuspend once.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-realm-ensure-1
name: cassandra-realm-ensure-4
namespace: sso
spec:
suspend: true
@ -70,7 +70,7 @@ spec:
- name: KEYCLOAK_SMTP_FROM_NAME
value: Cassandra
- name: CASSANDRA_PUBLIC_REPLAY_USERS
value: cassandra-dev,daniel-test,viktor-test
value: veles-dev,daniel-test,viktor-test
command: ["/bin/sh", "-c"]
args:
- |
@ -260,15 +260,35 @@ spec:
if status not in (200, 204):
raise SystemExit(f"User group update failed for {username}: status={status} body={body}")
ensure_group("alpha")
alpha_group = ensure_group("alpha")
ensure_group("admin")
ensure_group(os.environ["CASSANDRA_TESTER_GROUP"])
tester_group = ensure_group(os.environ["CASSANDRA_TESTER_GROUP"])
public_replay_group = ensure_group("public-replay")
pilot_groups = {
"pilot-skill-proficient": ensure_group("pilot-skill-proficient"),
"pilot-skill-adept": ensure_group("pilot-skill-adept"),
"pilot-skill-veteran": ensure_group("pilot-skill-veteran"),
"pilot-skill-expert": ensure_group("pilot-skill-expert"),
"pilot-skill-master": ensure_group("pilot-skill-master"),
"all-pilot-skill-level": ensure_group("all-pilot-skill-level"),
"codex-cli-fallback": ensure_group("codex-cli-fallback"),
}
for username in os.environ.get("CASSANDRA_PUBLIC_REPLAY_USERS", "").split(","):
username = username.strip()
if username:
ensure_user_group(username, public_replay_group)
ensure_user_group("veles-dev", tester_group)
ensure_user_group("veles-dev", pilot_groups["all-pilot-skill-level"])
ensure_user_group("veles-dev", pilot_groups["codex-cli-fallback"])
for username in ("daniel-test", "viktor-test"):
ensure_user_group(username, alpha_group)
ensure_user_group(username, tester_group)
ensure_user_group(username, pilot_groups["pilot-skill-proficient"])
ensure_user_group(username, pilot_groups["pilot-skill-adept"])
ensure_user_group(username, pilot_groups["pilot-skill-veteran"])
ensure_user_group(username, pilot_groups["codex-cli-fallback"])
status, clients = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}",

1
services/cassandra-auth/kustomization.yaml
View File

@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: sso
resources:
- bootstrap-jobs/cassandra-ldap-federation-job.yaml
- bootstrap-jobs/cassandra-realm-ensure-job.yaml
- bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
configMapGenerator:

11
services/cassandra/backend-deployment.yaml
View File

@ -22,7 +22,7 @@ spec:
labels:
app: cassandra-backend
annotations:
cassandra.bstein.dev/redeploy: "2026-07-28-0.8.51-jobs-compact"
cassandra.bstein.dev/redeploy: "2026-08-04-0.8.150-lane-semantics"
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
@ -66,7 +66,7 @@ spec:
type: RuntimeDefault
initContainers:
- name: database-init
image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"}
image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"}
imagePullPolicy: Always
command: ["/bin/sh", "-c"]
args:
@ -88,7 +88,7 @@ spec:
memory: 512Mi
containers:
- name: backend
image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"}
image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"}
imagePullPolicy: Always
command: ["/bin/sh", "-c"]
args:
@ -102,6 +102,11 @@ spec:
- name: http
containerPort: 8796
protocol: TCP
env:
- name: CASSANDRA_SIM_IMAGE
value: registry.bstein.dev/cassandra/cassandra-sim-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
- name: CASSANDRA_GENERATOR_WORKER_IMAGE
value: registry.bstein.dev/cassandra/cassandra-generator-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
envFrom:
- configMapRef:
name: cassandra-config

2
services/cassandra/bootstrap-jobs/cassandra-secrets-ensure-job.yaml
View File

@ -5,7 +5,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-secrets-ensure-2
name: cassandra-secrets-ensure-3
namespace: cassandra
spec:
suspend: true

23
services/cassandra/configmap.yaml
View File

@ -18,20 +18,34 @@ data:
CASSANDRA_OIDC_REQUIRED_GROUPS: alpha,cassandra-tester,admin
CASSANDRA_OIDC_ADMIN_GROUPS: admin
CASSANDRA_NAMESPACE: cassandra
CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.8.51 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
CASSANDRA_SIM_WORKER_VERSION: "0.8.51"
CASSANDRA_GENERATOR_WORKER_VERSION: "0.8.51"
CASSANDRA_OWNER_USER_ID_ALIASES: veles-dev=d357688e-e3e4-4f18-9491-a4ed5622deb6;a21a286c-6c82-4180-9243-7c916442ecf0=d357688e-e3e4-4f18-9491-a4ed5622deb6
CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
CASSANDRA_GENERATOR_WORKER_IMAGE: registry.bstein.dev/cassandra/cassandra-generator-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
CASSANDRA_SIM_WORKER_VERSION: "0.8.150"
CASSANDRA_GENERATOR_WORKER_VERSION: "0.8.150"
CASSANDRA_STRATEGY_SPEC_HINT_INTERPRETATION_TIMEOUT: "420"
CASSANDRA_SIM_SERVICE_ACCOUNT: cassandra-sim
CASSANDRA_GENERATOR_WORKER_SERVICE_ACCOUNT: cassandra-generator
CASSANDRA_SIM_PRIORITY_CLASS: cassandra-sim
CASSANDRA_GENERATOR_WORKER_PRIORITY_CLASS: cassandra-core
CASSANDRA_SIM_NODE_SELECTOR: cassandra.bstein.dev/node-pool=oceanus,kubernetes.io/arch=amd64
CASSANDRA_GENERATOR_WORKER_NODE_SELECTOR: cassandra.bstein.dev/node-pool=oceanus,kubernetes.io/arch=amd64
CASSANDRA_SIM_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule
CASSANDRA_GENERATOR_WORKER_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule
CASSANDRA_SIM_ACTIVE_DEADLINE_SECONDS: "7200"
CASSANDRA_GENERATOR_WORKER_ACTIVE_DEADLINE_SECONDS: "14400"
CASSANDRA_SIM_TTL_SECONDS: "3600"
CASSANDRA_GENERATOR_WORKER_TTL_SECONDS: "600"
CASSANDRA_SIM_CPU_REQUEST: 500m
CASSANDRA_SIM_CPU_LIMIT: "2"
CASSANDRA_SIM_MEMORY_REQUEST: 1Gi
CASSANDRA_SIM_MEMORY_LIMIT: 4Gi
CASSANDRA_GENERATOR_WORKER_CPU_REQUEST: 100m
CASSANDRA_GENERATOR_WORKER_CPU_LIMIT: "1"
CASSANDRA_GENERATOR_WORKER_MEMORY_REQUEST: 512Mi
CASSANDRA_GENERATOR_WORKER_MEMORY_LIMIT: 2Gi
CASSANDRA_SIM_FS_GROUP: "10001"
CASSANDRA_GENERATOR_WORKER_FS_GROUP: "10001"
CASSANDRA_MAX_ACTIVE_SIMS_PER_USER: "32"
CASSANDRA_MAX_ACTIVE_SIMS_GLOBAL: "32"
CASSANDRA_RETENTION_DAYS: "30"
@ -43,3 +57,6 @@ data:
CASSANDRA_SIM_CODEX_CLI_SECRET_PROVIDER_CLASS: cassandra-codex-cli
CASSANDRA_SIM_CODEX_CLI_MOUNT_PATH: /var/run/cassandra-codex-cli
CASSANDRA_SIM_CODEX_CLI_HOME: /home/cassandra/.codex
CASSANDRA_GENERATOR_WORKER_CODEX_CLI_SECRET_PROVIDER_CLASS: cassandra-generator-codex-cli
CASSANDRA_GENERATOR_WORKER_CODEX_CLI_MOUNT_PATH: /var/run/cassandra-codex-cli
CASSANDRA_GENERATOR_WORKER_CODEX_CLI_HOME: /home/cassandra/.codex

2
services/cassandra/frontend-deployment.yaml
View File

@ -48,7 +48,7 @@ spec:
type: RuntimeDefault
containers:
- name: frontend
image: registry.bstein.dev/cassandra/cassandra-frontend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-frontend"}
image: registry.bstein.dev/cassandra/cassandra-frontend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-frontend"}
imagePullPolicy: Always
ports:
- name: http

2
services/cassandra/kustomization.yaml
View File

@ -16,7 +16,6 @@ resources:
- postgres-statefulset.yaml
- services.yaml
- backend-deployment.yaml
- generator-deployment.yaml
- frontend-deployment.yaml
- retention-cronjob.yaml
- image.yaml
@ -24,3 +23,4 @@ resources:
- bootstrap-jobs/cassandra-secrets-ensure-job.yaml
- migration-jobs/db-copy-veles-to-cassandra-job.yaml
- migration-jobs/artifact-copy-veles-to-cassandra-job.yaml
- migration-jobs/artifact-permissions-job.yaml

26
services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml
View File

@ -3,7 +3,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-artifact-copy-from-veles-1
name: cassandra-artifact-copy-from-veles-2
namespace: cassandra
spec:
suspend: true
@ -33,6 +33,8 @@ spec:
value: /dest
- name: ALLOW_NONEMPTY_DEST
value: "0"
- name: RESET_DESTINATION
value: "1"
command: ["python", "-c"]
args:
- |
@ -42,12 +44,15 @@ spec:
import hashlib
import json
import os
import shutil
import tarfile
import time
import urllib.request
SOURCE_URL = os.environ["SOURCE_URL"].rstrip("/")
DEST = Path(os.environ.get("DEST_ROOT", "/dest")).resolve()
ALLOW_NONEMPTY = os.environ.get("ALLOW_NONEMPTY_DEST", "") in {"1", "true", "yes"}
RESET_DESTINATION = os.environ.get("RESET_DESTINATION", "") in {"1", "true", "yes"}
EXCLUDE_TOP = {"lost+found", ".cassandra-artifact-migration"}
REPORT_DIR = DEST / ".cassandra-artifact-migration"
@ -57,6 +62,13 @@ spec:
def assert_safe_destination() -> None:
DEST.mkdir(parents=True, exist_ok=True)
entries = visible_entries()
if entries and RESET_DESTINATION:
for entry in entries:
if entry.is_dir():
shutil.rmtree(entry)
else:
entry.unlink()
entries = visible_entries()
if entries and not ALLOW_NONEMPTY:
names = ", ".join(entry.name for entry in entries[:10])
raise SystemExit(f"destination is not empty; refusing to merge into existing data: {names}")
@ -66,6 +78,16 @@ spec:
with urllib.request.urlopen(f"{SOURCE_URL}/manifest.json", timeout=120) as response:
return json.loads(response.read().decode())
def wait_for_manifest() -> dict[str, object]:
last_error = ""
for _attempt in range(90):
try:
return fetch_manifest()
except Exception as exc:
last_error = str(exc)
time.sleep(5)
raise SystemExit(f"source artifact exporter did not become ready: {last_error}")
def safe_extract() -> None:
root_text = str(DEST)
with urllib.request.urlopen(f"{SOURCE_URL}/archive.tar", timeout=600) as response:
@ -116,7 +138,7 @@ spec:
}
assert_safe_destination()
source = fetch_manifest()
source = wait_for_manifest()
(REPORT_DIR / "source_manifest.json").write_text(json.dumps(source, indent=2, sort_keys=True))
safe_extract()
dest = destination_manifest()

42
services/cassandra/migration-jobs/artifact-permissions-job.yaml Normal file
View File

@ -0,0 +1,42 @@
# services/cassandra/migration-jobs/artifact-permissions-job.yaml
# One-off migration repair after copying artifacts from the Veles PVC.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-artifact-permissions-2
namespace: cassandra
spec:
suspend: true
backoffLimit: 0
ttlSecondsAfterFinished: 86400
template:
spec:
serviceAccountName: cassandra-artifact-migration
automountServiceAccountToken: false
restartPolicy: Never
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
containers:
- name: permissions
image: alpine:3.20
imagePullPolicy: IfNotPresent
command: ["/bin/sh", "-c"]
args:
- |
set -eu
chown -R 10001:10001 /data/cassandra-artifacts
find /data/cassandra-artifacts -type d -exec chmod 2775 {} +
find /data/cassandra-artifacts -type f -exec chmod 0664 {} +
printf 'artifact permission repair complete\n'
volumeMounts:
- name: artifacts
mountPath: /data/cassandra-artifacts
volumes:
- name: artifacts
persistentVolumeClaim:
claimName: cassandra-artifacts

6
services/cassandra/migration-jobs/db-copy-veles-to-cassandra-job.yaml
View File

@ -5,7 +5,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-db-copy-from-veles-1
name: cassandra-db-copy-from-veles-3
namespace: cassandra
spec:
suspend: true
@ -68,7 +68,9 @@ spec:
pg_dump --format=custom --no-owner --no-acl \
--dbname="${SOURCE_DATABASE_URL}" \
--file="${dump_path}"
pg_restore --clean --if-exists --no-owner --no-acl \
psql -X --set=ON_ERROR_STOP=1 --dbname="${TARGET_DATABASE_URL}" \
--command="DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO cassandra; GRANT ALL ON SCHEMA public TO public;"
pg_restore --no-owner --no-acl \
--dbname="${TARGET_DATABASE_URL}" \
"${dump_path}"

2
services/cassandra/retention-cronjob.yaml
View File

@ -38,7 +38,7 @@ spec:
type: RuntimeDefault
containers:
- name: retention
image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"}
image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"}
imagePullPolicy: IfNotPresent
command: ["cassandra-hygiene"]
args: ["--prune-artifacts"]

16
services/cassandra/secretproviderclass.yaml
View File

@ -73,3 +73,19 @@ spec:
secretPath: "kv/data/atlas/cassandra/codex-cli"
secretKey: "auth.json"
filePermission: 292
---
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: cassandra-generator-codex-cli
namespace: cassandra
spec:
provider: vault
parameters:
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
roleName: "cassandra"
objects: |
- objectName: "auth.json"
secretPath: "kv/data/atlas/cassandra/codex-cli"
secretKey: "auth.json"
filePermission: 292

138
services/harbor/bootstrap-jobs/cassandra-registry-ensure-job.yaml Normal file
View File

@ -0,0 +1,138 @@
# services/harbor/bootstrap-jobs/cassandra-registry-ensure-job.yaml
# One-off repair/audit job for the Cassandra Harbor migration.
# Keep suspended unless the Harbor project or robot scopes need to be recreated.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-registry-ensure-1
namespace: harbor
spec:
suspend: true
backoffLimit: 1
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "harbor"
vault.hashicorp.com/agent-inject-secret-harbor-admin.sh: "kv/data/atlas/harbor/harbor-core"
vault.hashicorp.com/agent-inject-template-harbor-admin.sh: |
{{- with secret "kv/data/atlas/harbor/harbor-core" }}
export HARBOR_ADMIN_PASSWORD="{{ .Data.data.harbor_admin_password }}"
{{- end }}
spec:
restartPolicy: Never
serviceAccountName: harbor-vault-sync
containers:
- name: ensure
image: python:3.11-alpine
command: ["python", "-c"]
args:
- |
import base64
import json
import os
import urllib.error
import urllib.request
secret_path = "/vault/secrets/harbor-admin.sh"
env = {}
with open(secret_path, "r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line.startswith("export "):
continue
key, value = line[len("export ") :].split("=", 1)
env[key] = value.strip('"')
password = env["HARBOR_ADMIN_PASSWORD"]
api = os.environ.get(
"HARBOR_API",
"http://harbor-core.harbor.svc.cluster.local/api/v2.0",
).rstrip("/")
auth = base64.b64encode(f"admin:{password}".encode("utf-8")).decode("ascii")
def request(method, path, payload=None, ok=(200,)):
data = None
headers = {"Authorization": f"Basic {auth}"}
if payload is not None:
data = json.dumps(payload).encode("utf-8")
headers["Content-Type"] = "application/json"
req = urllib.request.Request(
f"{api}{path}",
data=data,
method=method,
headers=headers,
)
try:
with urllib.request.urlopen(req, timeout=30) as response:
body = response.read()
if response.status not in ok:
raise RuntimeError(f"{method} {path} returned {response.status}")
if not body:
return None
return json.loads(body.decode("utf-8"))
except urllib.error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="replace")
if exc.code in ok:
return None
raise RuntimeError(f"{method} {path} returned {exc.code}: {detail}") from exc
projects = request("GET", "/projects?name=cassandra")
if not projects:
request(
"POST",
"/projects",
{
"project_name": "cassandra",
"metadata": {
"public": "false",
"auto_scan": "false",
"enable_content_trust": "false",
"prevent_vul": "false",
"reuse_sys_cve_allowlist": "true",
},
},
ok=(201, 409),
)
def ensure_permission(robot_name, access, description=None):
robots = request("GET", "/robots?page_size=100")
matches = [robot for robot in robots if robot.get("name") == robot_name]
if not matches:
raise RuntimeError(f"Harbor robot {robot_name} not found")
robot = request("GET", f"/robots/{matches[0]['id']}")
permissions = list(robot.get("permissions") or [])
if not any(
item.get("kind") == "project" and item.get("namespace") == "cassandra"
for item in permissions
):
permissions.append(
{"kind": "project", "namespace": "cassandra", "access": access}
)
payload = {
"name": robot["name"],
"level": robot["level"],
"disable": bool(robot.get("disable", False)),
"duration": int(robot.get("duration", -1)),
"permissions": permissions,
}
if description:
payload["description"] = description
elif robot.get("description"):
payload["description"] = robot["description"]
request("PUT", f"/robots/{robot['id']}", payload, ok=(200,))
ensure_permission(
"robot$pull-atlas",
[{"resource": "repository", "action": "pull"}],
)
ensure_permission(
"robot$jenkins-pipelines",
[
{"resource": "repository", "action": "pull"},
{"resource": "repository", "action": "push"},
],
"Jenkins image push robot for bstein, veles, and cassandra pipelines",
)
print("Cassandra Harbor project and robot scopes are present")

1
services/harbor/kustomization.yaml
View File

@ -12,6 +12,7 @@ resources:
- certificate.yaml
- helmrelease.yaml
- vault-sync-deployment.yaml
- bootstrap-jobs/cassandra-registry-ensure-job.yaml
- image.yaml
configMapGenerator:
- name: harbor-vault-entrypoint

2
services/harbor/pvc.yaml
View File

@ -8,7 +8,7 @@ spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 450Gi
storage: 600Gi
storageClassName: astreae
---
apiVersion: v1

13
services/hermes-chat/certificate.yaml Normal file
View File

@ -0,0 +1,13 @@
# services/hermes-chat/certificate.yaml
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: chat-tls
namespace: hermes-chat
spec:
secretName: chat-tls
issuerRef:
kind: ClusterIssuer
name: letsencrypt
dnsNames:
- chat.bstein.dev

153
services/hermes-chat/configmap.yaml Normal file
View File

@ -0,0 +1,153 @@
# services/hermes-chat/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: hermes-chat-config
namespace: hermes-chat
labels:
app: hermes-chat
data:
config.yaml: |
model:
provider: openai-codex
default: gpt-5.6-terra
model: gpt-5.6-terra
fallback_providers:
- provider: custom
model: gpt-oss:20b
base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
api_key: ollama
agent:
api_max_retries: 1
platform_toolsets:
cli:
- clarify
- file
- terminal
- web
api_server:
- clarify
- file
- terminal
- web
skills:
creation_nudge_interval: 15
external_dirs:
- /opt/data/workspace/skills
terminal:
backend: local
cwd: /opt/data/workspace
timeout: 180
home_mode: profile
approvals:
mode: manual
deny:
- "*kubectl apply*"
- "*kubectl create*"
- "*kubectl delete*"
- "*kubectl edit*"
- "*kubectl patch*"
- "*kubectl replace*"
- "*kubectl scale*"
- "*kubectl set*"
- "*kubectl label*"
- "*kubectl annotate*"
- "*kubectl cordon*"
- "*kubectl uncordon*"
- "*kubectl drain*"
- "*kubectl rollout restart*"
- "*kubectl rollout undo*"
- "*kubectl exec*"
- "*kubectl attach*"
- "*kubectl cp*"
- "*kubectl debug*"
- "*kubectl expose*"
- "*kubectl port-forward*"
- "*kubectl proxy*"
- "*kubectl run*"
- "*kubectl get secret*"
- "*kubectl describe secret*"
- "*flux reconcile*"
- "*flux suspend*"
- "*flux resume*"
- "*vault *"
- "*k3s *"
- "*crictl *"
- "*ctr *"
- "*169.254.169.254*"
- "*kubernetes.default*"
- "*/var/run/secrets/kubernetes.io*"
dashboard:
public_url: https://chat.bstein.dev
oauth:
provider: self-hosted
self_hosted:
issuer: https://sso.bstein.dev/realms/atlas
client_id: hermes-chat-dashboard
scopes: openid profile email groups
display:
compact: true
tool_progress: all
interim_assistant_messages: true
long_running_notifications: true
tool_loop_guardrails:
warnings_enabled: true
hard_stop_enabled: true
warn_after:
exact_failure: 2
same_tool_failure: 3
idempotent_no_progress: 2
hard_stop_after:
exact_failure: 5
same_tool_failure: 8
idempotent_no_progress: 5
updates:
pre_update_backup: quick
backup_keep: 3
non_interactive_local_changes: stash
SOUL.md: |
You are a personal AI assistant and researcher for Atlas users. Be useful,
curious, careful, and direct. Help with research, writing, planning,
learning, files, personal automation, and creative work. Use tools when
they materially improve the answer and make reusable skills when a stable
workflow is worth keeping.
Your environment is a private consumer sandbox. Work in the mounted
workspace and with public Internet services the user intentionally asks
you to use. You may inspect the Titan cluster through the dedicated
read-only Kubernetes identity to explain health and status. You do not
administer it: never attempt mutations, secret access, exec, attach,
port-forwarding, private-service access, or cloud metadata access.
AGENTS.md: |
# Personal Hermes workspace
This Hermes instance is a personal chat, research, and automation
environment. User-created files and skills belong under `/opt/data/workspace`.
You may:
- research public Internet sources and cite them
- create, read, and edit files in the workspace
- create and improve reusable skills
- configure user-owned channels and provider integrations
- run ordinary local commands needed for the user's task
- inspect non-secret Kubernetes resources and pod logs with read-only
`kubectl get`, `describe`, and `logs` commands
This container is not an infrastructure administration environment. The
Kubernetes identity is observation-only and cannot read Secrets or use pod
exec, attach, or port-forwarding. Do not attempt cluster mutations, private
service access, node LAN access, metadata services, Vault, container
runtimes, or the operator Hermes instance. If a request needs an action,
explain the evidence and say Brad must perform it from the operator
instance at `agent.bstein.dev`.

259
services/hermes-chat/deployment.yaml Normal file
View File

@ -0,0 +1,259 @@
# services/hermes-chat/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-chat
namespace: hermes-chat
labels:
app: hermes-chat
spec:
replicas: 1
revisionHistoryLimit: 2
progressDeadlineSeconds: 1800
strategy:
type: Recreate
selector:
matchLabels:
app: hermes-chat
template:
metadata:
labels:
app: hermes-chat
annotations:
ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback
ai.bstein.dev/role: personal-chat-research
ai.bstein.dev/isolation: observer-only Kubernetes RBAC, private-service egress denied
ai.bstein.dev/session-reset: "20260802-consumer-pty-recovery"
ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation
ai.bstein.dev/user-boundary: one OIDC subject with an isolated PVC subdirectory
spec:
serviceAccountName: hermes-chat
automountServiceAccountToken: true
securityContext:
fsGroup: 10000
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values:
- arm64
- key: node-role.kubernetes.io/worker
operator: In
values:
- "true"
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-18
- titan-19
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: atlas.bstein.dev/spillover
operator: DoesNotExist
- weight: 90
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
- weight: 50
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi4
initContainers:
- name: migrate-user-sessions
image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
imagePullPolicy: IfNotPresent
command:
- /opt/hermes/bin/hermes-session-migrate
env:
- name: HERMES_HOME
value: /storage/users/4794ab8284a14d421eaeea3e
- name: HERMES_MIGRATE_SOURCE_HOME
value: /storage
- name: HERMES_MIGRATE_USER_ID
value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc
- name: HERMES_MIGRATE_SESSION_IDS
value: 20260802_182723_9f53f1,20260802_173527_9dc51b
securityContext:
allowPrivilegeEscalation: false
runAsUser: 10000
runAsGroup: 10000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: home
mountPath: /storage
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 250m
memory: 256Mi
- name: init-config
image: busybox:1.37
imagePullPolicy: IfNotPresent
command:
- sh
- -c
- |
set -eu
user_home=/storage/users/4794ab8284a14d421eaeea3e
mkdir -p "${user_home}/workspace/skills" "${user_home}/home/.local/bin" "${user_home}/logs"
cp /config/config.yaml "${user_home}/config.yaml"
cp /config/SOUL.md "${user_home}/SOUL.md"
cp /config/AGENTS.md "${user_home}/workspace/AGENTS.md"
touch "${user_home}/.env"
if ! grep -q '^API_SERVER_KEY=' "${user_home}/.env"; then
api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> "${user_home}/.env"
fi
chmod 0600 "${user_home}/.env"
chown -R 10000:10000 "${user_home}"
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: home
mountPath: /storage
- name: config
mountPath: /config
readOnly: true
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
- name: install-kubectl
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- -c
- |
set -eu
cp "$(command -v kubectl)" /tools/kubectl
chmod 0755 /tools/kubectl
chown 10000:10000 /tools/kubectl
securityContext:
allowPrivilegeEscalation: false
runAsUser: 0
runAsGroup: 0
seccompProfile:
type: RuntimeDefault
volumeMounts:
- name: tools
mountPath: /tools
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
containers:
- name: hermes-chat
image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
imagePullPolicy: IfNotPresent
args:
- gateway
- run
ports:
- name: dashboard
containerPort: 9119
protocol: TCP
env:
- name: HERMES_HOME
value: /opt/data
- name: HOME
value: /opt/data/home
- name: PATH
value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
- name: HERMES_DASHBOARD
value: "1"
- name: HERMES_DASHBOARD_HOST
value: 0.0.0.0
- name: HERMES_DASHBOARD_PORT
value: "9119"
- name: HERMES_DASHBOARD_PUBLIC_URL
value: https://chat.bstein.dev
- name: HERMES_DASHBOARD_OIDC_ISSUER
value: https://sso.bstein.dev/realms/atlas
- name: HERMES_DASHBOARD_OIDC_CLIENT_ID
value: hermes-chat-dashboard
- name: HERMES_DASHBOARD_OIDC_SCOPES
value: openid profile email groups
- name: HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS
value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc
- name: API_SERVER_ENABLED
value: "true"
- name: API_SERVER_HOST
value: 0.0.0.0
- name: API_SERVER_PORT
value: "8642"
- name: API_SERVER_CORS_ORIGINS
value: https://chat.bstein.dev
volumeMounts:
- name: home
mountPath: /opt/data
subPath: users/4794ab8284a14d421eaeea3e
- name: tools
mountPath: /usr/local/bin/kubectl
subPath: kubectl
readinessProbe:
httpGet:
path: /api/status
port: dashboard
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /api/status
port: dashboard
initialDelaySeconds: 90
periodSeconds: 30
timeoutSeconds: 10
securityContext:
allowPrivilegeEscalation: false
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 100m
memory: 768Mi
ephemeral-storage: 256Mi
limits:
cpu: "1"
memory: 2Gi
ephemeral-storage: 1Gi
volumes:
- name: home
persistentVolumeClaim:
claimName: hermes-chat-home
- name: config
configMap:
name: hermes-chat-config
- name: tools
emptyDir: {}

27
services/hermes-chat/ingress.yaml Normal file
View File

@ -0,0 +1,27 @@
# services/hermes-chat/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: chat
namespace: hermes-chat
annotations:
cert-manager.io/cluster-issuer: letsencrypt
traefik.ingress.kubernetes.io/router.entrypoints: websecure
traefik.ingress.kubernetes.io/router.tls: "true"
spec:
ingressClassName: traefik
tls:
- hosts:
- chat.bstein.dev
secretName: chat-tls
rules:
- host: chat.bstein.dev
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: hermes-chat
port:
name: dashboard

15
services/hermes-chat/kustomization.yaml Normal file
View File

@ -0,0 +1,15 @@
# services/hermes-chat/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: hermes-chat
resources:
- namespace.yaml
- serviceaccount.yaml
- rbac.yaml
- configmap.yaml
- pvc.yaml
- deployment.yaml
- service.yaml
- networkpolicy.yaml
- certificate.yaml
- ingress.yaml

7
services/hermes-chat/namespace.yaml Normal file
View File

@ -0,0 +1,7 @@
# services/hermes-chat/namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: hermes-chat
labels:
app.kubernetes.io/name: hermes-chat

87
services/hermes-chat/networkpolicy.yaml Normal file
View File

@ -0,0 +1,87 @@
# services/hermes-chat/networkpolicy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-chat-isolation
namespace: hermes-chat
spec:
podSelector:
matchLabels:
app: hermes-chat
policyTypes:
- Ingress
- Egress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: traefik
podSelector:
matchLabels:
app.kubernetes.io/name: traefik
ports:
- protocol: TCP
port: 9119
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: hermes
podSelector:
matchLabels:
app: hermes-model-gate
ports:
- protocol: TCP
port: 8080
# Atlas OIDC uses the public sso.bstein.dev issuer, which resolves to the
# in-cluster Traefik load balancer for pods on the Atlas network.
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: traefik
podSelector:
matchLabels:
app: traefik
ports:
- protocol: TCP
port: 443
- to:
- ipBlock:
cidr: 10.43.0.1/32
ports:
- protocol: TCP
port: 443
# K3s applies egress policy after Service DNAT, so admit only the fixed API
# server endpoints as well as the kubernetes Service IP above.
- to:
- ipBlock:
cidr: 192.168.22.11/32
- ipBlock:
cidr: 192.168.22.12/32
- ipBlock:
cidr: 192.168.22.13/32
ports:
- protocol: TCP
port: 6443
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 10.0.0.0/8
- 100.64.0.0/10
- 127.0.0.0/8
- 169.254.0.0/16
- 172.16.0.0/12
- 192.168.0.0/16

15
services/hermes-chat/pvc.yaml Normal file
View File

@ -0,0 +1,15 @@
# services/hermes-chat/pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: hermes-chat-home
namespace: hermes-chat
labels:
app: hermes-chat
spec:
accessModes:
- ReadWriteOnce
storageClassName: astreae
resources:
requests:
storage: 2Gi

62
services/hermes-chat/rbac.yaml Normal file
View File

@ -0,0 +1,62 @@
# services/hermes-chat/rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: hermes-chat-observer
rules:
- apiGroups: [""]
resources:
- endpoints
- events
- namespaces
- nodes
- persistentvolumeclaims
- persistentvolumes
- pods
- pods/log
- replicationcontrollers
- services
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
resources:
- daemonsets
- deployments
- replicasets
- statefulsets
verbs: ["get", "list", "watch"]
- apiGroups: ["batch"]
resources:
- cronjobs
- jobs
verbs: ["get", "list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources:
- ingresses
- networkpolicies
verbs: ["get", "list", "watch"]
- apiGroups: ["helm.toolkit.fluxcd.io"]
resources:
- helmreleases
verbs: ["get", "list", "watch"]
- apiGroups: ["kustomize.toolkit.fluxcd.io"]
resources:
- kustomizations
verbs: ["get", "list", "watch"]
- apiGroups: ["source.toolkit.fluxcd.io"]
resources:
- gitrepositories
- helmrepositories
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: hermes-chat-observer
subjects:
- kind: ServiceAccount
name: hermes-chat
namespace: hermes-chat
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: hermes-chat-observer

17
services/hermes-chat/service.yaml Normal file
View File

@ -0,0 +1,17 @@
# services/hermes-chat/service.yaml
apiVersion: v1
kind: Service
metadata:
name: hermes-chat
namespace: hermes-chat
labels:
app: hermes-chat
spec:
type: ClusterIP
selector:
app: hermes-chat
ports:
- name: dashboard
port: 9119
targetPort: dashboard
protocol: TCP

7
services/hermes-chat/serviceaccount.yaml Normal file
View File

@ -0,0 +1,7 @@
# services/hermes-chat/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: hermes-chat
namespace: hermes-chat
automountServiceAccountToken: true

239
services/hermes/NOTES.md Normal file
View File

@ -0,0 +1,239 @@
# Hermes on Atlas: operator guide
This is the mental model and demonstration script for the operator instance at
`agent.bstein.dev`. Read it once, then prove each section in the live UI. The
consumer instance at `chat.bstein.dev` is intentionally separate and is not the
place to perform infrastructure triage.
## The one-sentence explanation
Hermes is the persistent agent runtime and control surface; Codex or the local
GPU model supplies reasoning, while Hermes supplies sessions, tools, skills,
approval rules, identity, and the repeatable procedure that connects the model
to Atlas evidence.
Hermes is not the model. Replacing `gpt-5.6-terra` with `gpt-oss:20b` changes the
reasoning engine, but it does not replace the Hermes sessions, tools, skills,
files, permission boundary, or workflow.
## Request and evidence path
```text
browser
-> Keycloak login
-> oauth2-proxy exact-user check
-> operator Hermes pod and persistent workspace
-> SOUL.md + AGENTS.md + selected skill
-> Codex primary model
-> read-only terminal/web/file tools
-> Ariadne deterministic evidence
-> retained Jenkins logs and artifacts
-> Gitea commits and Flux state
-> Kubernetes workloads, events, logs, and dependencies
-> Pushgateway data queried through VictoriaMetrics
-> Grafana query and alert context
-> structured finding and proposed repo-side change
-> human review and approval
-> Git/Flux delivery outside Hermes
provider failure
-> hermes-model-gate
-> Hermes owns titan-24: Ollama serves gpt-oss:20b
-> Wolf owns titan-24: gate returns 503; local inference yields
```
The operator agent pod runs on an ARM worker and does not reserve the GPU.
Ollama is the component on `titan-24` that owns GPU memory. The Lease
`hermes/titan-24-gpu-owner` controls whether the model gate admits local
inference. Normal operator conversations use independently authenticated Codex
first, so Wolf ownership does not need to block the web agent.
## Who does what
| Component | Responsibility | What it does not prove |
| --- | --- | --- |
| Hermes | Maintains the conversation, chooses tools/skills, reasons over evidence, saves reports, and enforces the approval experience | It is not the inference model and does not automatically have cluster-admin access |
| Ariadne | Collects and normalizes deterministic cluster, Jenkins, and quality evidence into a timestamped bundle; may optionally run a local diagnosis | Its diagnosis is not authoritative when stale, empty, or contradicted by the bundle |
| Jenkins | Runs the suites and retains console/artifact evidence | A final `exit 1` alone does not identify the first failed gate |
| Pushgateway | Receives suite/build quality telemetry | A cumulative counter is not the current build result |
| VictoriaMetrics | Stores and answers PromQL for quality and environment telemetry | A Grafana color is not direct proof of root cause |
| Grafana | Visualizes and alerts on VictoriaMetrics data | A red panel must be traced to its query, labels, time range, and raw series |
| Flux | Applies reviewed Git state to the cluster | A recent reconciliation is correlation until the changed path matches the failure |
| Kubernetes | Supplies workload, event, log, node, storage, and dependency evidence | An unrelated unhealthy pod is not proof that CI failed because of the cluster |
Direct Jenkins artifact requests currently require authorization and can return
HTTP 403. That does not break the workflow: Ariadne retains selected Jenkins
console tails and named artifact contents in its deterministic bundle. A report
must say `retained Ariadne evidence` when that fallback is used; it must not
pretend direct Jenkins access succeeded.
## The actual supervised triage algorithm
1. Classify the request as test/build triage, service health, or alert tuning.
2. Read the latest Ariadne diagnosis and deterministic bundle with HTTP GET.
3. Compare their timestamps. The deterministic bundle remains authoritative.
4. Select a terminal failed build. Keep running or unknown builds out of the
terminal-failure list.
5. Identify the first enforced failed gate in this order:
`style -> loc -> coverage -> tests -> gate_glue -> sonarqube -> supply_chain`.
6. Cite the smallest decisive Jenkins log/artifact evidence. State whether it
came directly from Jenkins or from Ariadne retention.
7. Query the matching build/check/test series in VictoriaMetrics. Distinguish a
current gauge from a cumulative counter and ignore zero-valued failure
series.
8. Correlate recent Gitea commits and the relevant Flux revision by timestamp
and affected path. Recency alone is not causation.
9. Check only the Kubernetes resources capable of explaining that failure.
Classify observations as direct, contributing candidate, or background.
10. Return `Finding`, `Confidence`, `Evidence`, `Likely cause`, `Blast radius`,
`Next checks`, `Repo-side fix`, and `Approval required`.
11. Stop at a proposal. A human reviews the patch or action; Git and Flux remain
the delivery path.
## What Hermes can read and what it cannot do
The live `hermes-triage` ServiceAccount can get/list/watch ordinary workload,
log, event, ingress, storage, Flux, and image-automation metadata. It cannot
read Kubernetes Secret values, create exec sessions, patch Deployments, or
patch Flux Kustomizations. The configured command deny list also blocks common
Kubernetes and Flux mutations.
The security boundary is layered:
1. Keycloak authenticates the person.
2. oauth2-proxy restricts the operator surface to Brad.
3. The operator has a separate namespace, PVC, configuration, and ServiceAccount.
4. Kubernetes RBAC is the hard API authorization boundary.
5. NetworkPolicy limits reachable paths where configured.
6. Hermes approvals and instructions provide a user-facing safety layer.
A skill is procedure and context, not a permission grant. Adding a sentence to
a skill cannot bypass Kubernetes RBAC.
## What “Hermes learns the workflow” means
It does not silently retrain model weights on cluster data. Learning here means
that a successful repeated procedure is written as a reusable skill on the
operator PVC or versioned in Git. The skill describes when it should trigger,
the evidence order, interpretation rules, output contract, and safety boundary.
The current workflow has a versioned top-level skill and persisted specialist
skills for:
- orchestration and evidence reporting;
- retained Jenkins evidence;
- quality metrics;
- Gitea/Flux correlation;
- Kubernetes failure classification;
- Grafana metric provenance;
- Soteria backup health;
- approval-required actions.
The model still evaluates fresh variable evidence on every incident. The skill
makes the process repeatable; it does not freeze the answer.
## The four operator skills to remember
- `triage-titan-test-failures`: CI, tests, builds, quality gates, and suspected
test-environment regressions.
- `triage-atlas-service-health`: active service or cluster incidents and red
high-level health panels.
- `tune-atlas-alerts`: noisy alerts, impossible metrics, bad PromQL, and
generator-owned Grafana corrections.
- `master-hermes-on-atlas`: hands-on training, assessments, architecture, and
claim audits.
Do not start by choosing raw tools. State the operational question and let the
skill route to the narrow evidence source.
## Two live, repeatable proof cases
### Proof 6: Soteria build 272
- Terminal result: failure.
- Tests: 318 passed, zero failed.
- Local coverage: 96.195 percent.
- First enforced failure: SonarQube `new_coverage=0.0` against threshold 80.
- Runtime: Soteria remained Ready.
- Correct conclusion: a Sonar new-code policy/reporting problem, not a test or
Kubernetes capacity failure.
- Correct restraint: do not weaken the policy until scanner import and baseline
evidence identifies whether the issue is configuration or legitimate new
uncovered code.
### Proof 7: Ananke build 242
- Terminal result: failure.
- Local coverage gate: 61.8 percent and failed.
- Other checks: tests, SonarQube, supply-chain, LOC, docs naming, and gate glue
were healthy.
- Sonar new-code coverage: 93.1 percent, which is a different scope.
- Correct conclusion: local coverage input/scope caused the enforced failure;
do not call the Sonar number contradictory without comparing inputs.
- Correct restraint: do not lower the threshold until the retained local gate
inputs are readable.
The full sessions and redacted Markdown exports are in Sessions and
`Files/triage-proof`.
## Five-minute demonstration
1. Open Models and show `openai-codex/gpt-5.6-terra` as primary and
`gpt-oss:20b` as local fallback.
2. Open Skills and show the four operator skills plus the persisted component
skills.
3. Open `Proof 6 - Soteria 272 Sonar new coverage triage` in Sessions.
4. Point out the timestamped bundle, exact build/artifact evidence, metric
corroboration, healthy runtime, fact/inference separation, and approval
boundary.
5. Open Proof 7 and explain why local coverage and Sonar new-code coverage can
differ without either number being fabricated.
6. End by showing that Hermes proposes a repo-side correction but cannot patch
the Deployment or Flux Kustomization with its ServiceAccount.
Use this short explanation:
> Hermes is the persistent, permissioned workflow layer around the model. In
> this cluster Ariadne collects deterministic evidence, Hermes correlates it
> with retained Jenkins artifacts, Git/Flux state, Kubernetes health, and
> quality metrics, and then returns a supervised proposal. Repeated procedures
> become explicit skills. Kubernetes RBAC prevents the agent from turning a
> diagnosis into an unreviewed infrastructure change.
## Prompts that exercise the real system
- `Use $triage-titan-test-failures. Triage the worst current terminal CI failure and link every decisive piece of evidence.`
- `Use $triage-titan-test-failures. Triage Ananke build 242. Separate local coverage from Sonar new-code coverage.`
- `Use $triage-atlas-service-health. Explain the VictoriaMetrics outage from current state and retained events; separate recovered impact from current impact.`
- `Use $tune-atlas-alerts. Trace one currently firing alert to its generated source and raw PromQL, but do not edit it.`
- `Use $master-hermes-on-atlas. Assess me on the request path and permission boundary. One question at a time.`
## Honest limits
- Hermes does not currently apply production or cluster changes autonomously.
- Direct Jenkins console/artifact access may be forbidden; Ariadne retention is
the current fallback and must be named as provenance.
- Local `gpt-oss:20b` is useful for bounded work but is slower and less reliable
for multi-source triage than Codex.
- The current Ariadne local diagnosis can return an empty model response. That
does not invalidate the deterministic bundle or prevent Codex-backed Hermes
from triaging it.
- A single successful case demonstrates capability, not mastery. Mastery means
leading different real incidents, improving the skill after failures, and
teaching the architecture without prompts.
## Your shortest path to fluency
1. Explain the request diagram without looking.
2. Explain why Ariadne and Hermes are separate components.
3. Reproduce Proof 6 from the raw bundle and metrics.
4. Reproduce Proof 7 and explain the two coverage scopes.
5. Produce the live RBAC allow/deny matrix with `kubectl auth can-i`.
6. Create one small writable skill from a repeated sub-workflow, test a trigger
and non-trigger case, then improve it.
7. Lead one new incident while Hermes coaches rather than answers for you.
8. Teach the five-minute demonstration to another person.
At that point the interview claims are demonstrable. Continue repeating real
incidents before describing yourself as fully autonomous or the system as
self-correcting.

4
services/hermes/agent-ingress.yaml
View File

@ -21,6 +21,6 @@ spec:
pathType: Prefix
backend:
service:
name: hermes
name: oauth2-proxy-hermes
port:
name: dashboard
name: http

30
services/hermes/ariadne-handoff-rbac.yaml Normal file
View File

@ -0,0 +1,30 @@
# services/hermes/ariadne-handoff-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ariadne-gpu-handoff
namespace: hermes
rules:
- apiGroups: ["coordination.k8s.io"]
resources:
- leases
resourceNames:
- titan-24-gpu-owner
verbs:
- get
- patch
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ariadne-gpu-handoff
namespace: hermes
subjects:
- kind: ServiceAccount
name: ariadne
namespace: maintenance
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ariadne-gpu-handoff

231
services/hermes/configmap.yaml
View File

@ -9,11 +9,41 @@ metadata:
data:
config.yaml: |
model:
provider: custom
default: qwen2.5:7b-instruct-q4_0
model: qwen2.5:7b-instruct-q4_0
base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1
api_key: ollama
provider: openai-codex
default: gpt-5.6-terra
model: gpt-5.6-terra
fallback_providers:
- provider: custom
model: gpt-oss:20b
base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1
api_key: ollama
agent:
api_max_retries: 1
platform_toolsets:
cli:
- clarify
- file
- session_search
- skills
- terminal
- todo
- web
api_server:
- clarify
- file
- session_search
- skills
- terminal
- todo
- web
skills:
creation_nudge_interval: 15
external_dirs:
- /opt/data/workspace/skills
terminal:
backend: local
@ -22,7 +52,7 @@ data:
home_mode: auto
approvals:
mode: manual
mode: smart
deny:
- "*kubectl apply*"
- "*kubectl delete*"
@ -80,6 +110,12 @@ data:
read-only state. Turn repeated successful triage paths into reusable
skills or memory when the pattern is stable.
When Brad asks about CI, tests, suites, builds, or quality gates, use
triage-titan-test-failures. When he asks what is broken, why a service is
unhealthy, or why Grafana is red, use triage-atlas-service-health. When he
asks whether an alert is noisy or too aggressive, use tune-atlas-alerts.
Do not make him choose a tool, page, or evidence source first.
Stay Flux-first. Do not mutate the cluster directly. Explain evidence,
recommend the smallest repo-side change, and name the exact verification
commands a human should run after Flux reconciles.
@ -89,6 +125,11 @@ data:
You are Hermes running inside the Titan Kubernetes cluster as a read-only
testing and operations triage assistant.
Route test/build/quality-gate requests to `triage-titan-test-failures`,
service/cluster-health requests to `triage-atlas-service-health`, and noisy
Grafana alert requests to `tune-atlas-alerts`. Ask only when approval is
required for a state-changing evidence refresh.
Ariadne owns deterministic evidence collection and local diagnosis. Start
every testing triage by reading:
@ -122,3 +163,181 @@ data:
Do not run mutating commands such as `kubectl apply`, `delete`, `scale`,
`patch`, `cordon`, `uncordon`, `drain`, or `rollout restart`. Do not read
Kubernetes Secret values. Draft repo changes or operator steps instead.
START-HERE.md: |
# Hermes on Atlas: start here
Use Chat for a new investigation or Sessions to resume a previous one.
The shortest useful prompts are:
- `Triage the latest test failure.`
- `What is broken in the cluster right now?`
- `Why is Grafana red, and which alerts are noise?`
- `Triage <suite> build <number> and save the report.`
Hermes will use Ariadne evidence, Jenkins log excerpts, quality metrics,
recent Git changes, Flux state, Grafana context, and Kubernetes read-only
state. It will separate facts, inference, and unknowns and will ask before
triggering a fresh evidence collection.
Hermes now has three focused operator skills:
1. `triage-titan-test-failures` for all ten custom CI suites.
2. `triage-atlas-service-health` for any live namespace or service.
3. `tune-atlas-alerts` for evidence-backed alert and dashboard corrections.
Validated CI examples are available under Sessions, including:
1. `Proof 6 - Soteria 272 Sonar new coverage triage`
2. `Proof 7 - Ananke 242 local coverage triage`
Their saved reports are in Files under `triage-proof/`. Read
`HERMES-OPERATOR-RUNBOOK.md` for the system mental model and five-minute
demonstration.
Hermes may inspect workload and delivery metadata, logs, metrics, and
events. Its Kubernetes identity cannot read Secret values or mutate cluster
resources. Apply fixes through the titan-iac GitOps workflow after review.
HERMES-CAPABILITIES.md: |
# What Hermes can do on Atlas
## Working now
- Triage all custom CI suites: Ananke, Ariadne, Atlasbot, bstein_home,
data_prepper, Lesavka, Metis, Pegasus, Soteria, and titan_iac.
- Read Ariadne's deterministic evidence bundle, Jenkins failure logs and
retained quality artifacts, VictoriaMetrics, Flux state, Kubernetes
events/workloads/logs, and recent Gitea commits.
- Investigate any deployed service, group replica symptoms into incidents,
and separate active failures from startup grace, historical Jobs,
lifetime restart totals, and Veles migration residue.
- Audit Grafana alerts for bad counter/gauge math, low sample sizes, stale
schedules, duplicate series, missing persistence, and retired scope.
- Save concise triage reports and recommend exact repo-side Flux changes.
## Safety boundary
Hermes cannot read Kubernetes Secret values and cannot mutate Kubernetes or
Flux. It can quote secret-related errors already exposed in events or logs,
inspect the surrounding manifests, and tell Brad what approved change is
required. This keeps cluster operation supervised.
## Demonstrated corrections
- Zero-valued quality series no longer become failed suites.
- Running Jenkins builds remain in-progress instead of failed.
- Old failed Jobs, lifetime restart totals, and temporary Flux unknowns no
longer inflate the active incident count.
- Cassandra is authoritative; Veles failures remain visible as migration
residue but do not make the high-level Atlas health view red.
- Worker readiness uses the actual 18 Kubernetes workers.
- Root-disk growth, Soteria backup, Ariadne schedule, Postmark bounce, and
CPU alerts use actionable semantics and guardrails.
The evidence and verification for these changes are in
`triage-proof/ATLAS-TRIAGE-PROOFS.md`.
## Good next prompts
- `Use $triage-atlas-service-health. What is broken right now?`
- `Use $triage-titan-test-failures. Triage the worst current suite.`
- `Use $tune-atlas-alerts. Audit alerts fired in the last 24 hours.`
- `Save this investigation as a reusable skill after I approve the pattern.`
ATLAS-TRIAGE-PROOFS.md: |
# Atlas triage proofs
This file is the short evidence trail for what Hermes can do today. The
operator skills remain read-only: fixes are reviewed and delivered through
Git and Flux.
## Proof 1: Cassandra secret-sync recovery
- Finding: `cassandra-vault-sync` was stuck because the expected
`VELES_BYOK_ENCRYPTION_KEY` migration input was absent from Cassandra's
Vault path.
- Action: the existing Flux-tracked, suspended one-shot migration Job was
enabled, completed successfully, and immediately returned to suspended.
- Verification: Cassandra frontend, backend, PostgreSQL, and Vault sync all
became Ready. No Kubernetes Secret value was read or committed.
- Why this matters: Hermes can correlate workload state, logs, Vault policy
manifests, and migration ownership without confusing Veles residue with
a current Veles outage.
## Proof 2: Grafana alert-noise correction
- Removed retired Veles pods and completed Jobs from high-level incident
counts while preserving them on drill-down dashboards.
- Added 15-minute persistence to Pending, CrashLoopBackOff, and image-pull
summaries.
- Replaced invalid `increase()` use on gauges, bounded CPU percentages,
required meaningful Postmark sample sizes, and scoped Soteria/Ariadne
alerts to configured current work.
- Live verification on 2026-08-03: `Problem Pods=0`,
`CrashLoop/ImagePull=0`, and `Workers Ready=18/18`.
## Proof 3: all-suite testing triage
- Ariadne build 384 passed 676 tests and the exact coverage contract:
82 source files at or above 95%.
- Deployed image: `registry.bstein.dev/bstein/ariadne:0.1.0-384`.
- The deterministic bundle covers Ananke, Ariadne, Atlasbot, bstein_home,
data_prepper, Lesavka, Metis, Pegasus, Soteria, and titan_iac.
- Zero-valued quality metrics are healthy, running Jenkins builds are
in-progress, old failed Jobs are historical, and Veles objects are
migration residue. They no longer inflate the active failure set.
- Failed builds still include direct Jenkins console and retained artifact
links so Hermes can identify the first failed gate and smallest repo fix.
## Proof 4: Cassandra generator failure classification
- Finding: the Cassandra serving path is healthy, but generation Job
`cassandra-generator-3ce198971b` failed.
- Evidence: the primary OpenAI request returned HTTP 429 because its credit
balance was exhausted. Codex CLI fallback ran, but the strict promotion
gate still rejected unresolved high-impact hint claims and blocked the
semantic scenarios.
- Impact: one generation capability/request failed; this is not a Cassandra
registry outage and not a Veles outage.
- Next action: restore provider credits or continue through Codex fallback,
then resolve the reported hint claims before retrying. This needs operator
approval because it can spend money or rerun an expensive generation.
## Proof 5: live Hermes Soteria triage
- Hermes read the fresh 2026-08-03 Ariadne bundle without an approval
timeout and chose Soteria build 270 as the worst terminal failure.
- It proved that all 318 tests passed and local coverage was 96.195%, then
isolated the enforced failure to SonarQube reporting new-code coverage
as 0.0%. It did not blame the optional supply-chain advisory.
- It kept running titan-iac, Data Prepper, and Lesavka builds out of the
terminal-failure list and requested approval before any rerun or edit.
## Proof 6: Soteria build 272 Sonar new-coverage triage
- Hermes read the fresh 2026-08-04 Ariadne bundle and selected terminal
Soteria build 272.
- It proved that all 318 tests passed and local coverage was 96.195%, then
isolated the enforced failure to SonarQube `new_coverage=0.0` against the
threshold of 80.
- It corroborated the retained Jenkins evidence with current quality
metrics and healthy Soteria/Flux state, and did not blame Kubernetes.
- It proposed no change until scanner import and baseline evidence can
distinguish configuration error from genuinely uncovered new code.
## Proof 7: Ananke build 242 local-coverage triage
- Hermes selected terminal Ananke build 242 and isolated the first failure
to the local coverage gate at 61.8%.
- It showed that tests, SonarQube, supply chain, LOC, docs naming, gate glue,
Flux, and cluster readiness were healthy.
- It distinguished the local 61.8% scope from SonarQube's 93.1% new-code
scope instead of treating the numbers as interchangeable.
- Direct Jenkins artifact reads returned HTTP 403, so it explicitly named
Ariadne's retained bundle as evidence provenance and stopped short of an
unsupported fix.
## Use Hermes next
- `What is broken in the cluster right now?`
- `Triage the worst current CI suite and link the evidence.`
- `Audit Grafana alerts from the last 24 hours and separate incidents from noise.`
- `Explain the Cassandra generator failure and give me the smallest safe next step.`

86
services/hermes/deployment.yaml
View File

@ -20,10 +20,11 @@ spec:
labels:
app: hermes
annotations:
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation
ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback
ai.bstein.dev/role: testing-triage
ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
ai.bstein.dev/config-rev: "20260721-hermes-rollout-deadline"
ai.bstein.dev/config-rev: "20260804-root-operator-docs"
spec:
serviceAccountName: hermes-triage
automountServiceAccountToken: true
@ -50,6 +51,7 @@ spec:
- titan-13
- titan-15
- titan-17
- titan-18
- titan-19
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
@ -80,10 +82,18 @@ spec:
- -c
- |
set -eu
mkdir -p /opt/data/workspace /opt/data/home/.local/bin /opt/data/logs
mkdir -p /opt/data/workspace/triage-proof /opt/data/home/.local/bin /opt/data/logs
cp /config/config.yaml /opt/data/config.yaml
cp /config/SOUL.md /opt/data/SOUL.md
cp /config/AGENTS.md /opt/data/workspace/AGENTS.md
cp /config/START-HERE.md /opt/data/workspace/START-HERE.md
cp /config/HERMES-CAPABILITIES.md /opt/data/workspace/HERMES-CAPABILITIES.md
cp /guide/OPERATOR-RUNBOOK.md /opt/data/workspace/HERMES-OPERATOR-RUNBOOK.md
cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/workspace/triage-proof/ATLAS-TRIAGE-PROOFS.md
cp /config/START-HERE.md /opt/data/START-HERE.md
cp /config/HERMES-CAPABILITIES.md /opt/data/HERMES-CAPABILITIES.md
cp /guide/OPERATOR-RUNBOOK.md /opt/data/HERMES-OPERATOR-RUNBOOK.md
cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/ATLAS-TRIAGE-PROOFS.md
touch /opt/data/.env
if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then
api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
@ -99,8 +109,8 @@ spec:
mountPath: /opt/data
- name: config
mountPath: /config
- name: tools
mountPath: /opt/data/home/.local/bin
- name: operator-guide
mountPath: /guide
resources:
requests:
cpu: 25m
@ -134,7 +144,7 @@ spec:
memory: 64Mi
containers:
- name: hermes
image: nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f
imagePullPolicy: IfNotPresent
args:
- gateway
@ -189,7 +199,20 @@ spec:
- name: home
mountPath: /opt/data
- name: tools
mountPath: /opt/data/home/.local/bin
mountPath: /usr/local/bin/kubectl
subPath: kubectl
- name: triage-skill
mountPath: /opt/data/workspace/skills/triage-titan-test-failures
readOnly: true
- name: mastery-skill
mountPath: /opt/data/workspace/skills/master-hermes-on-atlas
readOnly: true
- name: service-health-skill
mountPath: /opt/data/workspace/skills/triage-atlas-service-health
readOnly: true
- name: alert-tuning-skill
mountPath: /opt/data/workspace/skills/tune-atlas-alerts
readOnly: true
readinessProbe:
httpGet:
path: /api/status
@ -218,5 +241,54 @@ spec:
- name: config
configMap:
name: hermes-config
- name: operator-guide
configMap:
name: hermes-operator-guide
- name: tools
emptyDir: {}
- name: triage-skill
configMap:
name: hermes-triage-skill
items:
- key: SKILL.md
path: SKILL.md
- key: openai.yaml
path: agents/openai.yaml
- name: mastery-skill
configMap:
name: hermes-mastery-skill
items:
- key: SKILL.md
path: SKILL.md
- key: openai.yaml
path: agents/openai.yaml
- key: architecture.md
path: references/architecture.md
- key: curriculum.md
path: references/curriculum.md
- key: incident-drills.md
path: references/incident-drills.md
- key: mastery-rubric.md
path: references/mastery-rubric.md
- key: two-hour-proof-sprint.md
path: references/two-hour-proof-sprint.md
- name: service-health-skill
configMap:
name: hermes-service-health-skill
items:
- key: SKILL.md
path: SKILL.md
- key: openai.yaml
path: agents/openai.yaml
- key: service-map.md
path: references/service-map.md
- name: alert-tuning-skill
configMap:
name: hermes-alert-tuning-skill
items:
- key: SKILL.md
path: SKILL.md
- key: openai.yaml
path: agents/openai.yaml
- key: alert-review.md
path: references/alert-review.md

51
services/hermes/kustomization.yaml
View File

@ -4,11 +4,62 @@ kind: Kustomization
namespace: hermes
resources:
- namespace.yaml
- vault-serviceaccount.yaml
- configmap.yaml
- rbac.yaml
- pvc.yaml
- model-gate-rbac.yaml
- ariadne-handoff-rbac.yaml
- model-gate-state.yaml
- model-gate-configmap.yaml
- model-gate-deployment.yaml
- networkpolicy.yaml
- ollama-deployment.yaml
- deployment.yaml
- service.yaml
- oauth2-proxy.yaml
- agent-certificate.yaml
- agent-ingress.yaml
configMapGenerator:
- name: hermes-operator-guide
namespace: hermes
files:
- OPERATOR-RUNBOOK.md=NOTES.md
options:
disableNameSuffixHash: true
- name: hermes-triage-skill
namespace: hermes
files:
- SKILL.md=skills/triage-titan-test-failures/SKILL.md
- openai.yaml=skills/triage-titan-test-failures/agents/openai.yaml
options:
disableNameSuffixHash: true
- name: hermes-mastery-skill
namespace: hermes
files:
- SKILL.md=skills/master-hermes-on-atlas/SKILL.md
- openai.yaml=skills/master-hermes-on-atlas/agents/openai.yaml
- architecture.md=skills/master-hermes-on-atlas/references/architecture.md
- curriculum.md=skills/master-hermes-on-atlas/references/curriculum.md
- incident-drills.md=skills/master-hermes-on-atlas/references/incident-drills.md
- mastery-rubric.md=skills/master-hermes-on-atlas/references/mastery-rubric.md
- two-hour-proof-sprint.md=skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md
options:
disableNameSuffixHash: true
- name: hermes-service-health-skill
namespace: hermes
files:
- SKILL.md=skills/triage-atlas-service-health/SKILL.md
- openai.yaml=skills/triage-atlas-service-health/agents/openai.yaml
- service-map.md=skills/triage-atlas-service-health/references/service-map.md
options:
disableNameSuffixHash: true
- name: hermes-alert-tuning-skill
namespace: hermes
files:
- SKILL.md=skills/tune-atlas-alerts/SKILL.md
- openai.yaml=skills/tune-atlas-alerts/agents/openai.yaml
- alert-review.md=skills/tune-atlas-alerts/references/alert-review.md
options:
disableNameSuffixHash: true

151
services/hermes/model-gate-configmap.yaml Normal file
View File

@ -0,0 +1,151 @@
# services/hermes/model-gate-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: hermes-model-gate
namespace: hermes
data:
model_gate.py: |
#!/usr/bin/env python3
"""Fail-closed proxy that admits local inference only while Hermes owns titan-24."""
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import ssl
import threading
import time
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0")
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080"))
UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/")
LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes")
LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner")
CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1"))
API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc")
API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443")
TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token")
CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
LEASE_URL = (
f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/"
f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}"
)
_cache_lock = threading.Lock()
_cached_owner = "unavailable"
_cached_at = 0.0
def _lease_owner() -> str:
"""Return the current owner, failing closed when Kubernetes is unavailable."""
global _cached_at, _cached_owner
now = time.monotonic()
with _cache_lock:
if now - _cached_at < CACHE_TTL_SEC:
return _cached_owner
try:
token = TOKEN_PATH.read_text(encoding="utf-8").strip()
request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"})
context = ssl.create_default_context(cafile=str(CA_PATH))
with urlopen(request, timeout=3, context=context) as response:
payload = json.load(response)
owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip()
except Exception:
owner = "unavailable"
_cached_owner = owner
_cached_at = now
return owner
class Handler(BaseHTTPRequestHandler):
"""Proxy local model traffic while exposing health and ownership status."""
protocol_version = "HTTP/1.1"
def _json(self, status: int, payload: dict) -> None:
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def _local_allowed(self) -> tuple[bool, str]:
owner = _lease_owner()
return owner == "hermes", owner
def _proxy(self) -> None:
allowed, owner = self._local_allowed()
if not allowed:
self._json(
503,
{
"error": {
"message": f"local GPU inference unavailable while titan-24 owner is {owner}",
"type": "server_error",
},
"gpu_owner": owner,
"fallback_required": True,
},
)
return
length = int(self.headers.get("Content-Length", "0") or 0)
body = self.rfile.read(length) if length else None
headers = {"Content-Type": self.headers.get("Content-Type", "application/json")}
if self.headers.get("Accept"):
headers["Accept"] = self.headers["Accept"]
request = Request(f"{UPSTREAM_URL}{self.path}", data=body, headers=headers, method=self.command)
try:
response = urlopen(request, timeout=1800)
except HTTPError as exc:
response = exc
except (TimeoutError, URLError) as exc:
self._json(503, {"error": {"message": f"local model upstream unavailable: {exc}", "type": "server_error"}})
return
self.send_response(response.status)
content_type = response.headers.get("Content-Type")
if content_type:
self.send_header("Content-Type", content_type)
content_length = response.headers.get("Content-Length")
if content_length:
self.send_header("Content-Length", content_length)
else:
self.send_header("Connection", "close")
self.close_connection = True
self.send_header("Cache-Control", "no-store")
self.end_headers()
while True:
chunk = response.read(65536)
if not chunk:
break
self.wfile.write(chunk)
self.wfile.flush()
response.close()
def do_GET(self) -> None:
if self.path == "/healthz":
self._json(200, {"status": "ok"})
return
if self.path == "/gate/status":
allowed, owner = self._local_allowed()
self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed})
return
self._proxy()
def do_POST(self) -> None:
self._proxy()
def log_message(self, format_string: str, *args) -> None:
print(f"model-gate {self.address_string()} {format_string % args}", flush=True)
if __name__ == "__main__":
ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever()

125
services/hermes/model-gate-deployment.yaml Normal file
View File

@ -0,0 +1,125 @@
# services/hermes/model-gate-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: hermes-model-gate
namespace: hermes
labels:
app: hermes-model-gate
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: hermes-model-gate
template:
metadata:
labels:
app: hermes-model-gate
spec:
serviceAccountName: hermes-model-gate
securityContext:
seccompProfile:
type: RuntimeDefault
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values:
- arm64
- key: node-role.kubernetes.io/worker
operator: In
values:
- "true"
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-04
- titan-13
- titan-15
- titan-17
- titan-19
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 90
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
containers:
- name: model-gate
image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df
imagePullPolicy: IfNotPresent
command:
- python
- /opt/model-gate/model_gate.py
ports:
- name: http
containerPort: 8080
env:
- name: UPSTREAM_URL
value: http://hermes-ollama.hermes.svc.cluster.local:11434
- name: LEASE_NAMESPACE
value: hermes
- name: LEASE_NAME
value: titan-24-gpu-owner
readinessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 3
periodSeconds: 10
livenessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 10
periodSeconds: 30
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65532
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: script
mountPath: /opt/model-gate
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
- name: script
configMap:
name: hermes-model-gate
defaultMode: 0555
- name: tmp
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: hermes-model-gate
namespace: hermes
labels:
app: hermes-model-gate
spec:
type: ClusterIP
selector:
app: hermes-model-gate
ports:
- name: http
port: 11434
targetPort: http

34
services/hermes/model-gate-rbac.yaml Normal file
View File

@ -0,0 +1,34 @@
# services/hermes/model-gate-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: hermes-model-gate
namespace: hermes
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: hermes-model-gate
namespace: hermes
rules:
- apiGroups: ["coordination.k8s.io"]
resources:
- leases
resourceNames:
- titan-24-gpu-owner
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: hermes-model-gate
namespace: hermes
subjects:
- kind: ServiceAccount
name: hermes-model-gate
namespace: hermes
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: hermes-model-gate

10
services/hermes/model-gate-state.yaml Normal file
View File

@ -0,0 +1,10 @@
# services/hermes/model-gate-state.yaml
apiVersion: coordination.k8s.io/v1
kind: Lease
metadata:
name: titan-24-gpu-owner
namespace: hermes
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
holderIdentity: hermes

30
services/hermes/networkpolicy.yaml Normal file
View File

@ -0,0 +1,30 @@
# services/hermes/networkpolicy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: hermes-ollama-ingress
namespace: hermes
spec:
podSelector:
matchLabels:
app: hermes-ollama
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
app: hermes-model-gate
ports:
- protocol: TCP
port: 11434
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: maintenance
podSelector:
matchLabels:
app: ariadne
ports:
- protocol: TCP
port: 11434

161
services/hermes/oauth2-proxy.yaml Normal file
View File

@ -0,0 +1,161 @@
# services/hermes/oauth2-proxy.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: hermes-operator-allowlist
namespace: hermes
data:
allowed-emails: |
brad@bstein.dev
---
apiVersion: v1
kind: Service
metadata:
name: oauth2-proxy-hermes
namespace: hermes
labels:
app: oauth2-proxy-hermes
spec:
selector:
app: oauth2-proxy-hermes
ports:
- name: http
port: 80
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: oauth2-proxy-hermes
namespace: hermes
labels:
app: oauth2-proxy-hermes
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: oauth2-proxy-hermes
template:
metadata:
labels:
app: oauth2-proxy-hermes
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: hermes
vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/operator-oidc
vault.hashicorp.com/agent-inject-template-oidc-config: |
{{- with secret "kv/data/atlas/hermes/operator-oidc" -}}
client_id = "{{ .Data.data.client_id }}"
client_secret = "{{ .Data.data.client_secret }}"
cookie_secret = "{{ .Data.data.cookie_secret }}"
{{- end -}}
spec:
serviceAccountName: hermes-vault
automountServiceAccountToken: true
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values:
- arm64
- key: node-role.kubernetes.io/worker
operator: In
values:
- "true"
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-18
- titan-19
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 90
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
- weight: 50
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi4
containers:
- name: oauth2-proxy
image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0@sha256:dcb6ff8dd21bf3058f6a22c6fa385fa5b897a9cd3914c88a2cc2bb0a85f8065d
imagePullPolicy: IfNotPresent
args:
- --provider=oidc
- --config=/vault/secrets/oidc-config
- --redirect-url=https://agent.bstein.dev/oauth2/callback
- --oidc-issuer-url=https://sso.bstein.dev/realms/atlas
- --code-challenge-method=S256
- --scope=openid profile email
- --email-domain=*
- --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails
- --set-xauthrequest=true
- --session-cookie-minimal=true
- --cookie-secure=true
- --cookie-samesite=lax
- --cookie-refresh=0
- --cookie-expire=8h
- --upstream=http://hermes.hermes.svc.cluster.local:9119
- --http-address=0.0.0.0:4180
- --skip-provider-button=true
- --skip-jwt-bearer-tokens=true
- --cookie-domain=agent.bstein.dev
- --reverse-proxy=true
ports:
- name: http
containerPort: 4180
readinessProbe:
httpGet:
path: /ping
port: http
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /ping
port: http
initialDelaySeconds: 20
periodSeconds: 20
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 250m
memory: 256Mi
volumeMounts:
- name: allowlist
mountPath: /etc/oauth2-proxy
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
- name: allowlist
configMap:
name: hermes-operator-allowlist
- name: tmp
emptyDir:
sizeLimit: 64Mi

29
services/hermes/ollama-deployment.yaml
View File

@ -18,8 +18,8 @@ spec:
labels:
app: hermes-ollama
annotations:
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
ai.bstein.dev/gpu: accelerator MVP lane (titan-24)
ai.bstein.dev/model: gpt-oss:20b
ai.bstein.dev/gpu: titan-24 local-first lane
spec:
runtimeClassName: nvidia
affinity:
@ -33,7 +33,8 @@ spec:
- titan-24
volumes:
- name: models
emptyDir: {}
persistentVolumeClaim:
claimName: hermes-models
initContainers:
- name: warm-model
image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
@ -44,7 +45,7 @@ spec:
- name: OLLAMA_MODELS
value: /root/.ollama
- name: OLLAMA_MODEL
value: qwen2.5:7b-instruct-q4_0
value: gpt-oss:20b
- name: NVIDIA_VISIBLE_DEVICES
value: all
- name: NVIDIA_DRIVER_CAPABILITIES
@ -68,7 +69,7 @@ spec:
nvidia.com/gpu.shared: 1
limits:
cpu: "4"
memory: 10Gi
memory: 16Gi
nvidia.com/gpu.shared: 1
containers:
- name: ollama
@ -82,6 +83,16 @@ spec:
value: 0.0.0.0
- name: OLLAMA_KEEP_ALIVE
value: 6h
- name: OLLAMA_CONTEXT_LENGTH
value: "64000"
- name: OLLAMA_FLASH_ATTENTION
value: "1"
- name: OLLAMA_KV_CACHE_TYPE
value: q8_0
- name: OLLAMA_MAX_LOADED_MODELS
value: "1"
- name: OLLAMA_NUM_PARALLEL
value: "1"
- name: OLLAMA_MODELS
value: /root/.ollama
- name: NVIDIA_VISIBLE_DEVICES
@ -100,10 +111,10 @@ spec:
timeoutSeconds: 5
resources:
requests:
cpu: "2"
memory: 8Gi
cpu: "8"
memory: 24Gi
nvidia.com/gpu.shared: 1
limits:
cpu: "6"
memory: 12Gi
cpu: "16"
memory: 40Gi
nvidia.com/gpu.shared: 1

88
services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js vendored Normal file
View File

@ -0,0 +1,88 @@
(function () {
"use strict";
var pluginName = "atlas-operator-guide";
var storageKey = "atlas.operator-guide.hidden.v1";
var prompt = [
"Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint.",
"I will perform every setup and evidence step myself.",
"Start with step 1 only, require live proof, and stop at every mutation or approval boundary.",
"First confirm this session is using openai-codex/gpt-5.6-terra rather than the local fallback."
].join(" ");
function OperatorGuide() {
var sdk = window.__HERMES_PLUGIN_SDK__;
var React = sdk.React;
var state = sdk.hooks.useState(function () {
return window.localStorage.getItem(storageKey) === "1";
});
var hidden = state[0];
var setHidden = state[1];
var copiedState = sdk.hooks.useState(false);
var copied = copiedState[0];
var setCopied = copiedState[1];
if (hidden) {
return React.createElement(
"button",
{
className: "atlas-guide-restore",
onClick: function () {
window.localStorage.removeItem(storageKey);
setHidden(false);
},
type: "button"
},
"Show Atlas operator walkthrough"
);
}
function copyPrompt() {
navigator.clipboard.writeText(prompt).then(function () {
setCopied(true);
window.setTimeout(function () { setCopied(false); }, 2500);
});
}
return React.createElement(
"section",
{ className: "atlas-guide-card", role: "region", "aria-label": "Atlas operator walkthrough" },
React.createElement(
"div",
{ className: "atlas-guide-heading" },
React.createElement("div", null,
React.createElement("strong", null, "Start here: two-hour Hermes proof sprint"),
React.createElement("p", null, "You do the setup and evidence work. Hermes coaches, checks, and stops at approval boundaries.")
),
React.createElement(
"button",
{
className: "atlas-guide-hide",
onClick: function () {
window.localStorage.setItem(storageKey, "1");
setHidden(true);
},
type: "button"
},
"Hide"
)
),
React.createElement(
"ol",
{ className: "atlas-guide-steps" },
React.createElement("li", null, "Confirm Models shows openai-codex/gpt-5.6-terra for the coaching session."),
React.createElement("li", null, "Copy the sprint prompt, paste it into Chat, and press Enter."),
React.createElement("li", null, "Complete one live triage and build one writable skill yourself.")
),
React.createElement(
"div",
{ className: "atlas-guide-actions" },
React.createElement("a", { className: "atlas-guide-button", href: "/models" }, "Open Models"),
React.createElement("button", { className: "atlas-guide-button", onClick: copyPrompt, type: "button" }, copied ? "Prompt copied" : "Copy sprint prompt"),
React.createElement("a", { className: "atlas-guide-link", href: "/skills" }, "Inspect skills")
)
);
}
window.__HERMES_PLUGINS__.registerSlot(pluginName, "chat:top", OperatorGuide);
}());

64
services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css vendored Normal file
View File

@ -0,0 +1,64 @@
.atlas-guide-card {
border: 1px solid rgba(247, 201, 72, 0.7);
background: rgba(247, 201, 72, 0.08);
color: inherit;
padding: 0.75rem 1rem;
font-family: "Mondwest", monospace;
}
.atlas-guide-heading {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 1rem;
}
.atlas-guide-heading strong {
color: #f7c948;
font-family: "Rules Expanded", sans-serif;
letter-spacing: 0.04em;
text-transform: uppercase;
}
.atlas-guide-heading p {
margin: 0.25rem 0 0;
opacity: 0.8;
}
.atlas-guide-steps {
margin: 0.6rem 0;
padding-left: 1.25rem;
}
.atlas-guide-actions {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 0.5rem;
}
.atlas-guide-button,
.atlas-guide-hide,
.atlas-guide-restore {
border: 1px solid rgba(247, 201, 72, 0.7);
background: transparent;
color: #f7c948;
cursor: pointer;
padding: 0.35rem 0.6rem;
text-decoration: none;
}
.atlas-guide-button:hover,
.atlas-guide-hide:hover,
.atlas-guide-restore:hover {
background: rgba(247, 201, 72, 0.14);
}
.atlas-guide-hide,
.atlas-guide-restore {
font-size: 0.75rem;
}
.atlas-guide-link {
color: #f7c948;
}

16
services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json Normal file
View File

@ -0,0 +1,16 @@
{
"name": "atlas-operator-guide",
"label": "Atlas Operator Guide",
"description": "Hands-on onboarding for supervised Atlas triage and Hermes skill building.",
"icon": "Sparkles",
"version": "1.0.0",
"tab": {
"path": "/atlas-operator-guide",
"hidden": true
},
"slots": [
"chat:top"
],
"entry": "dist/index.js",
"css": "dist/style.css"
}

15
services/hermes/pvc.yaml
View File

@ -13,3 +13,18 @@ spec:
resources:
requests:
storage: 4Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: hermes-models
namespace: hermes
labels:
app: hermes-ollama
spec:
accessModes:
- ReadWriteOnce
storageClassName: local-path
resources:
requests:
storage: 30Gi

8
services/hermes/rbac.yaml
View File

@ -22,6 +22,7 @@ rules:
- pods
- pods/log
- replicationcontrollers
- serviceaccounts
- services
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
@ -54,6 +55,12 @@ rules:
- gitrepositories
- helmrepositories
verbs: ["get", "list", "watch"]
- apiGroups: ["image.toolkit.fluxcd.io"]
resources:
- imagepolicies
- imagerepositories
- imageupdateautomations
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
@ -67,4 +74,3 @@ subjects:
- kind: ServiceAccount
name: hermes-triage
namespace: hermes

84
services/hermes/skills/master-hermes-on-atlas/SKILL.md Normal file
View File

@ -0,0 +1,84 @@
---
name: master-hermes-on-atlas
description: Coach Brad through hands-on mastery of the live Atlas Hermes deployment, including its web and CLI interfaces, local GPU inference and Codex fallback, tools, skills, sessions, profiles, channels, security boundaries, and evidence-driven test-failure triage. Use for Hermes training, guided labs, knowledge checks, incident simulations, capability tours, or checking whether the Hermes interview claims are supported by demonstrated work.
---
# Master Hermes on Atlas
Act as a demanding, practical coach. Teach the deployed system, not a generic
Hermes installation. Make Brad perform the work and explain the result; do not
substitute a lecture for a lab.
## Select the training model
Use `openai-codex/gpt-5.6-terra` for assessments, multi-reference labs, incident
grading, and skill evaluation. If the active provider is the local
`gpt-oss:20b`, stop before reading the reference files and ask Brad to select
Codex in Models or start a Codex-backed session. Continue locally only when Brad
explicitly asks to evaluate the local model itself or requests a lightweight
single-question review.
## Start or resume training
1. Read `references/architecture.md` before teaching deployment-specific facts.
2. Read `references/curriculum.md` to select the next lab.
3. When Brad asks for the two-hour proof sprint, read
`references/two-hour-proof-sprint.md` and run it in order. Brad performs
every required UI action and evidence check; coach and verify without doing
the learning-critical steps for him.
4. Ask whether to assess, resume, or choose a lab. If no progress record exists,
default to a five-question assessment followed by the first weak area.
5. Give one bounded task at a time. State the goal, safety boundary, exact
success evidence, and at most one initial hint.
6. Wait for Brad's answer or observed command output before revealing the
explanation.
7. Grade with `references/mastery-rubric.md`. Separate demonstrated ability
from verbal familiarity.
8. Offer a concise progress update. Write it to
`/opt/data/workspace/hermes-training/progress.md` only after Brad explicitly
approves the file change.
## Run live exercises safely
- Use read-only commands by default: `kubectl get`, `describe`, `logs`, `auth
can-i`, HTTP GET, `hermes status`, and Hermes list/status commands.
- Never read Kubernetes Secret values. Never use Vault reads as a training
shortcut. Redact bearer tokens, cookies, device codes, and credentials.
- Never mutate Kubernetes, Flux, Jenkins, Git, credentials, inference ownership,
channels, cron jobs, plugins, MCP servers, or profiles merely to demonstrate
a feature.
- Draft state-changing commands and explain their effect. Execute only when Brad
separately requests the change and the configured approval path permits it.
- Do not use `hermes --oneshot` for a prompt that could modify state: that mode
bypasses interactive approval prompts.
- Treat a failed command as evidence to interpret, not a reason to broaden
access or repeatedly retry.
## Teach evidence-driven triage
For a live test-failure lab, also use `$triage-titan-test-failures`. Require the
student to distinguish:
- observed fact from inference;
- stale evidence from current state;
- application regression from environment failure;
- a read-only next check from a proposed mutation;
- a generic suggestion from the smallest Flux-tracked repo-side fix.
For offline practice, read `references/incident-drills.md` and present only the
student packet for one incident. Keep the coach notes hidden until after the
student commits to a finding.
## Validate claims honestly
Use the claim audit in `references/mastery-rubric.md`. Current configuration and
live demonstrations support current-use claims. The retained Git history and
bound OpenClaw PVC support the deployment/replacement history; require Brad to
inspect that evidence and distinguish it from proof of every past interaction.
End every session with:
- what Brad demonstrated;
- what remains unproven;
- the next smallest lab;
- any safety or operational issue discovered.

4
services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml Normal file
View File

@ -0,0 +1,4 @@
interface:
display_name: "Master Hermes on Atlas"
short_description: "Practice Hermes safely on the live Atlas stack"
default_prompt: "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint. I will perform every setup and evidence step myself. Start with step 1 only, require live proof, and stop at every mutation or approval boundary. First confirm this session is using openai-codex/gpt-5.6-terra rather than the local fallback."

117
services/hermes/skills/master-hermes-on-atlas/references/architecture.md Normal file
View File

@ -0,0 +1,117 @@
# Live Atlas Hermes architecture
Use this reference for deployment-specific facts. Re-check live state before
asserting health, placement, ownership, or current model availability.
## Two isolated agent instances
| Surface | Purpose | Identity boundary | State and permissions |
| --- | --- | --- | --- |
| `agent.bstein.dev` | Brad's operator and triage lab | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace, its own PVC and service account; read-only cluster triage plus approved internal evidence endpoints |
| `chat.bstein.dev` | Consumer chat and research | Keycloak login | `hermes-chat` namespace, separate PVC/config/auth/sessions/skills; read-only cluster observer RBAC and no access to Secrets or Kubernetes mutation APIs |
The instances do not share conversation state, credentials, profiles, skills
created on their PVCs, or Kubernetes identities. They share only the inference
service described below. Consumer users cannot reach the operator dashboard
through the chat Service or Ingress.
## Inference path
```text
operator Hermes ───> independently authenticated openai-codex/gpt-5.6-terra
consumer Hermes ───> independently authenticated openai-codex/gpt-5.6-terra
│
└─ provider error or manual selection ─> hermes-model-gate
│
├─ Hermes owns GPU ─> Ollama gpt-oss:20b
└─ Wolf owns GPU ───> HTTP 503
```
- The configured context length is 64,000 tokens.
- `hermes-model-gate` is the stable OpenAI-compatible endpoint.
- A coordination Lease stores whether Hermes or Wolf/Moonlight owns the GPU.
Ariadne has narrowly scoped permission to update that Lease during handoff.
- Wolf ownership makes the local gate return a deliberate unavailable response.
Normal interactive chat remains on its Codex primary and does not consume
titan-24 GPU resources.
- The two agent pods do not consume titan-24 GPU memory. Ollama does. Agent pods
are ARM gateway workloads with persistent state on separate PVCs.
- Codex credentials are deliberately per-instance. Never copy Brad's Codex
credential store into the consumer PVC.
Verify rather than memorize:
```sh
hermes status
hermes fallback list
kubectl -n hermes get lease titan-24-gpu-owner -o yaml
kubectl -n hermes get deploy hermes hermes-model-gate hermes-ollama -o wide
kubectl -n hermes-chat get deploy hermes-chat -o wide
```
Do not change the GPU owner as part of a lesson.
## Retained OpenClaw replacement evidence
OpenClaw is not part of the current inference path, but its deployment history
is retained:
- `1bc58e10` added the OpenClaw testing-triage workspace, workload, model,
ingress, RBAC, and storage.
- follow-up commits wired local diagnosis, OIDC, persistence, and the gateway.
- `97ecb36b` replaced OpenClaw with Hermes and moved the agent-facing resources.
- `bb5e286f` decoupled Hermes from the retired OpenClaw Flux dependency.
- the `openclaw-home` PVC remains bound as retained state while no OpenClaw
workload is running.
Have Brad verify the commits and live resource shape. This establishes the
cluster deployment and replacement sequence; Brad's direct experience remains
the evidence for how he personally used the old agent.
## Operator evidence path
The operator role is designed for supervised test-failure triage:
1. Ariadne deterministic bundle and optional diagnosis.
2. Retained Jenkins logs and artifacts.
3. Recent Git changes and Flux revision/state.
4. Kubernetes workload, node, event, storage, DNS, and dependency health.
5. Pushgateway quality metrics and VictoriaMetrics/Grafana context.
6. A concise finding, confidence, evidence, blast radius, next checks,
repo-side fix, and approval-required section.
The versioned `$triage-titan-test-failures` skill encodes this workflow. The
operator has internal URLs for Ariadne, Jenkins, and VictoriaMetrics. The
consumer instance intentionally does not.
## Security layers and their limits
Security is layered rather than delegated to a prompt:
1. Keycloak authenticates the human.
2. The operator proxy limits `agent.bstein.dev` to Brad's exact email.
3. Separate namespaces, PVCs, service accounts, configs, and ingresses isolate
operator and consumer state.
4. Kubernetes RBAC grants the consumer get/list/watch only and excludes Secrets,
exec, attach, port-forward, and mutation.
5. NetworkPolicy permits the consumer to reach DNS, the shared model gate, the
Kubernetes API, and public IPv4 while blocking arbitrary private service
access.
6. Hermes deny patterns and instructions add user-facing guardrails.
Prompt rules and shell deny patterns are not the primary security boundary.
RBAC, network policy, credential separation, and routing isolation are. The
consumer can still disclose non-secret cluster metadata it is authorized to
read; that is an intentional requirement and should be described honestly.
## CLI-to-web mental model
The pinned Hermes build exposes chat, models and fallback, auth, sessions,
profiles, skills and bundles, plugins, tools, MCP, gateway/channels, pairing,
webhooks, cron, kanban, projects, hooks, memory/journey, logs, security, backup,
and diagnostics. The web interface is a control surface over many of these
same persisted capabilities.
Use `<command> --help` and the web Documentation page for the installed version.
Do not rely on screenshots or online docs from a different release when a live
command can settle the question.

131
services/hermes/skills/master-hermes-on-atlas/references/curriculum.md Normal file
View File

@ -0,0 +1,131 @@
# Hands-on curriculum
Complete labs by evidence, not elapsed time. A focused pass can establish
operational competence in several days; mastery requires repeating real triage
and recovery work over multiple incidents.
Run the curriculum on `openai-codex/gpt-5.6-terra`. Use the local model only for a
deliberate comparison lab; it is not the default coach for multi-reference work.
## Phase 1: orientation and control
### Lab 1 — Map the system
From the web UI, identify Chat, Sessions, Files, Models, Logs, Cron, Skills,
Plugins, MCP, Channels, Webhooks, Pairing, Profiles, Config, Keys, System, and
Documentation. Explain which state belongs to the operator PVC and which
components are shared. Verify three claims with read-only CLI output.
Success evidence: a correct diagram or written request path from browser to
agent to model, including the fallback branch and the consumer boundary.
### Lab 2 — Models, context, and fallback
Inspect `hermes status`, `hermes fallback list`, deployment placement, and GPU
owner state. Explain why a 32K model was rejected, why normal Codex-backed chat
remains available when Wolf owns the GPU, and when the local fallback can run.
Success evidence: predict outcomes for local healthy, local slow, gate 503,
invalid local response, and expired Codex authorization without changing state.
### Lab 3 — Sessions, files, profiles, and logs
Create a named training session and a harmless workspace note through the UI,
then find the corresponding session/file/log surfaces. Inspect profile and
backup help without creating a profile or backup.
Success evidence: explain persistence, what survives a pod replacement, what is
instance-local, and how to recover a lost conversation without exposing auth.
## Phase 2: tools and safe autonomy
### Lab 4 — Tools versus skills versus MCP
Use `hermes tools`, `hermes skills`, `hermes plugins`, and `hermes mcp` help or
list/status views. Classify each as executable capability, procedure/context,
packaged extension, or external protocol integration. Explain why a skill is
not a security boundary.
Success evidence: choose the right mechanism for three examples: repeatable
Titan triage, a read-only external API, and a scheduled notification.
### Lab 5 — Prove the permission boundary
Use `kubectl auth can-i` as both service accounts. Inspect the relevant
ClusterRoles and NetworkPolicies. Do not attempt mutations.
Success evidence: an allow/deny matrix covering pods, logs, Secrets, exec,
deployment patch, Flux reads, Flux reconcile, the model gate, internal services,
and public web research.
### Lab 6 — Channels, pairing, webhooks, and cron
Inspect the configured state and help for gateway/channels, pairing, webhooks,
and cron. Design one safe notification workflow and one unsafe workflow. Do not
register a channel, create a webhook, or schedule a job during the lab.
Success evidence: identify the identity, secret, audience, tool policy, failure
mode, audit trail, and revocation path for the proposed integration.
## Phase 3: the interview workflow
### Lab 7 — Deterministic live triage
Invoke `$triage-titan-test-failures`. Read the current Ariadne diagnosis and
bundle, then correlate one suite with Jenkins, Flux/Git, cluster health, and
quality metrics. Keep every collection step read-only.
Success evidence: the required structured triage result with timestamps and no
invented evidence.
### Lab 8 — Incident simulations
Use one student packet from `incident-drills.md`. Diagnose it before receiving
coach notes. Repeat until three cases score at least `Independent`.
Success evidence: correct failure class, causal chain, next checks, minimal
repo-side correction, and explicit approval boundary.
### Lab 9 — Build and evaluate a reusable skill
Identify a stable repeated workflow, create a small skill in the operator's
writable skills directory, inspect it, and test it against a fresh incident.
Do not alter the Flux-mounted training or triage skills.
Success evidence: precise trigger description, concise procedure, progressive
disclosure where useful, one passing case, one adversarial case, and a stated
permission boundary.
### Lab 10 — Model outage drill
Use retained logs/current status or a separately approved maintenance window.
Do not take the live model down for training. Explain the expected trace from
model gate to fallback, recognize an auth failure, and name the restoration
checks.
Success evidence: correctly distinguish model unavailability, context rejection,
provider auth failure, and agent failure.
## Phase 4: independent operation
### Lab 11 — Supervised real incident
Lead a real failure triage from intake to an approved repo-side proposal. A
human applies any change through Git/Flux. Verify the result read-only.
### Lab 12 — Teach it back and audit the claims
Explain the complete architecture and demonstrate the workflow without hints.
Audit each sentence of the interview answer using the claim table in
`mastery-rubric.md`. Weaken or qualify anything not supported by evidence.
## Suggested pace
- Day 1: Labs 1–3.
- Days 2–3: Labs 4–6.
- Days 4–7: Labs 7–10.
- Weeks 2–4: repeat Labs 7–11 on real incidents; improve one skill from observed
failures; finish with Lab 12.
Do not advance solely because a day elapsed. Re-run any lab graded below
`Independent`.

95
services/hermes/skills/master-hermes-on-atlas/references/incident-drills.md Normal file
View File

@ -0,0 +1,95 @@
# Incident drills
Present only one `Student packet` before the student answers. Use the coach
notes afterward to grade the failure classification and reasoning. These cases
are based on retained Atlas rollout observations; treat them as training
fixtures, not proof of current live state.
## Case A — Metrics disappear after a Soteria run
### Student packet
- The Soteria test stages ran.
- The post-stage log ends with:
`Syntax error: end of file unexpected (expecting "fi")`.
- The canonical Soteria Pushgateway series did not refresh.
- Other suites continued publishing.
Ask for: finding, confidence, evidence, likely cause, blast radius, read-only
next checks, repo-side fix, and approval-required actions.
### Coach notes
The primary class is pipeline glue/shell syntax, not a Pushgateway outage and
not a product-test failure. Inspect the post block and the exact shell parsed by
`/bin/sh`; confirm the last successful metric timestamp. The smallest likely
fix is closing/correcting the conditional in the Jenkinsfile, then a normal SCM
run and read-only metric verification. Editing, pushing, triggering, or manual
metric backfill requires explicit approval.
## Case B — UI tests fail before metrics publish
### Student packet
- `bstein-dev-home` requires Playwright `1.59.1`.
- The CI test container contains Playwright `1.51.0`.
- The frontend test stage hard-fails.
- No fresh canonical quality metrics appear for the run.
### Coach notes
Separate the direct test-environment mismatch from the telemetry-control-flow
defect. The version mismatch explains the test failure; early pipeline abort
explains missing metrics. A complete proposal aligns the image version and
persists stage return codes so publishing still runs while the final gate still
fails. Do not call this a Kubernetes capacity problem without node/pod evidence.
## Case C — Promotion exits 127
### Student packet
- Titan IaC validation and tests finish.
- The Promote stage uses `python:3.12-slim`.
- The stage returns exit code `127` at its first Git command.
- The repository and remote are reachable from other jobs.
### Coach notes
Exit 127 means the invoked command is unavailable. Confirm the exact log line
and image contents; the likely issue is that the slim image lacks `git`, not bad
Git credentials. The minimal repo-side fix installs `git` and CA certificates
in the job's dependency setup or uses a suitable pinned runner image.
## Case D — Data Preppers cannot publish an image
### Student packet
- Tests and local image build complete.
- Push to `registry.bstein.dev/streaming/data-prepper:2.8.0` is denied.
- The configured Jenkins credential belongs to a different Harbor project.
- A Sonar evidence GET also returns `401` in the same run.
### Coach notes
Do not collapse independent authentication failures into a generic network
failure. Verify the Harbor repository/project and credential ID without reading
the secret value; separately verify the Sonar token injection path and API
scope. The likely image fix is selecting a streaming-scoped robot credential.
Treat the Sonar 401 as a second issue unless evidence proves a shared cause.
## Case E — Hermes local inference fails during game streaming
### Student packet
- Wolf owns titan-24 according to the GPU-owner state.
- `hermes-model-gate` is ready but returns a deliberate unavailable response.
- Both Hermes dashboards remain healthy.
- One instance answers through Codex; the other reports missing provider auth.
### Coach notes
This is expected resource arbitration plus an instance-specific fallback auth
gap. Do not restart the agent or claim the GPU is underprovisioned. Confirm the
gate response, fallback chain, and each instance's auth status without exposing
tokens. Authorize the affected instance with its intended user's account. The
separate credential stores are a security property, not configuration drift.

75
services/hermes/skills/master-hermes-on-atlas/references/mastery-rubric.md Normal file
View File

@ -0,0 +1,75 @@
# Mastery and claim rubric
## Performance levels
| Level | Evidence |
| --- | --- |
| Exposed | Recognizes terms but needs the path and commands supplied. |
| Assisted | Completes a lab with hints and can explain the result afterward. |
| Independent | Selects safe tools, gathers fresh evidence, and reaches a defensible result without hints. |
| Mastery | Handles ambiguity and failure, teaches the architecture, improves the reusable workflow, and preserves security boundaries. |
Score each domain separately:
1. Architecture and request routing.
2. Models, context, GPU ownership, and fallback.
3. Sessions, files, profiles, logs, and recovery.
4. Tools, skills, plugins, MCP, channels, webhooks, and cron.
5. Identity, RBAC, network, credentials, approvals, and auditability.
6. Jenkins/Flux/Kubernetes/metrics evidence correlation.
7. Clear triage findings and minimal repo-side proposals.
8. Skill creation, evaluation, and iteration.
Never award `Independent` from a verbal answer alone when the lab calls for live
or fixture evidence. Never award `Mastery` until Brad has led at least two
different real incidents and improved a workflow based on what failed.
## Interview claim audit
| Claim | What supports it | Minimum demonstration |
| --- | --- | --- |
| Hermes runs in the Kubernetes cluster | Flux manifests and live workloads | Trace browser, agent, model gate, Ollama, PVC, and service account. |
| Hermes fits existing automation | Operator config, internal evidence URLs, triage skill | Complete a triage using Ariadne, Jenkins, Git/Flux, cluster health, and metrics. |
| Hermes follows Brad's triage path | `$triage-titan-test-failures` procedure | Lead two evidence-backed cases with facts separated from inference. |
| Repeated workflows become reusable skills | Versioned triage/training skills plus writable user skill area | Build, test, and improve one skill from a repeated real workflow. |
| Access starts read-only and scoped | RBAC, NetworkPolicy, separate identities and state | Produce and explain the allow/deny matrix from live authorization checks. |
| Human approval protects changes | Approval config, deny patterns, Flux workflow | Identify every mutation in a proposed incident response and stop before it. |
| Local GPU gracefully yields to gaming | Owner state, model gate, fallback | Explain or observe a Wolf ownership window and verify continued fallback service. |
| OpenClaw previously ran and was replaced | `1bc58e10` deploys the OpenClaw triage stack; follow-up commits add gateway/OIDC/persistence; `97ecb36b` replaces it with Hermes; `bb5e286f` retires the dependency; the bound `openclaw-home` PVC remains without a workload | Inspect the commits and live Flux/PVC state, then explain what they prove versus what depends on Brad's personal recollection. |
## Triage answer scoring
Award one point for each:
- names the correct failure class;
- cites timestamped/build-specific evidence;
- labels inference;
- checks evidence freshness;
- identifies realistic blast radius;
- orders read-only next checks;
- proposes the smallest Flux/repo-side change or says evidence is insufficient;
- isolates approval-required actions;
- avoids invented resources, logs, metrics, or commits;
- remains concise enough for an operator to act on.
Scores below 8/10 require another incident drill. Any invented evidence or
unacknowledged mutation caps the result at `Assisted`.
## Progress record format
When Brad approves recording progress, use this compact structure:
```markdown
# Hermes training progress
- Last session: <UTC timestamp>
- Current level: <per-domain summary>
- Completed: <labs with evidence>
- Needs repetition: <specific gaps>
- Next lab: <one lab>
- Safety findings: <none or concrete issue>
- Claim status: <supported, partially supported, unproven>
```
Append evidence links or command summaries; never record tokens, cookies,
credentials, device codes, or Secret values.

86
services/hermes/skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md Normal file
View File

@ -0,0 +1,86 @@
# Atlas two-hour proof sprint
The sprint establishes usable, defensible evidence for the current Hermes
claims. It does not award mastery. Retained Git and PVC evidence establishes
the OpenClaw deployment/replacement sequence; Brad's direct experience remains
the evidence for how he personally used it. Brad performs the work; Hermes
gives one task at a time, checks the evidence, and withholds later answers until
Brad commits to a result.
## 0–15 minutes — control surface and request path
1. Have Brad start a new session and confirm it uses
`openai-codex/gpt-5.6-terra`.
2. In the web UI, have him locate Chat, Sessions, Files, Models, Logs, Skills,
Plugins, MCP, Channels, Webhooks, Cron, Profiles, Config, Keys, System, and
Documentation.
3. Have him explain browser → operator agent → Codex primary, including the
model-gate/Ollama fallback branch and the separate consumer instance.
Evidence: Brad can identify persistent, shared, and instance-local state and
can name the GPU owner check without changing it.
## 15–35 minutes — permissions and safety
1. Have Brad inspect the operator ServiceAccount, ClusterRole, NetworkPolicy,
and approval deny list using read-only commands.
2. Have him produce an allow/deny matrix for pod/log reads, Secret reads,
exec, deployment mutation, Flux reads, Flux reconcile, and internal
evidence endpoints.
3. Require him to explain why prompt instructions and skills are not the
primary security boundary.
Evidence: correct identity/RBAC/network/approval layering with no Secret values
read and no mutation attempted.
## 35–75 minutes — perform the claimed triage path
Invoke `$triage-titan-test-failures`. Brad must:
1. check Ariadne diagnosis and bundle freshness;
2. identify one current or retained failed suite/build;
3. correlate Jenkins log/artifact evidence;
4. correlate Git/Flux revision and Kubernetes environment health;
5. query the relevant Pushgateway/VictoriaMetrics quality signal and identify
the matching Grafana context;
6. produce Finding, Confidence, Evidence, Likely cause, Blast radius, Next
checks, Repo-side fix, and Approval required.
Hermes may suggest a read-only command after Brad explains why it is needed.
Do not let the built-in diagnosis replace Brad's own evidence correlation.
Evidence: an 8/10 or better result under the triage rubric with no invented
facts and no unacknowledged mutation.
## 75–105 minutes — build the reusable workflow
1. Have Brad inspect the mounted `triage-titan-test-failures` skill and explain
its trigger, fixed procedure, variable evidence, and security boundary.
2. Have Brad choose one genuinely repeated sub-workflow from the triage.
3. Guide him to create a new skill in the operator's writable skill area. Do
not edit the mounted Flux skill and do not write the skill for him.
4. Test one expected trigger and one adversarial/non-trigger case.
5. Have Brad make one improvement based on the test results.
Evidence: Brad can find the skill in the Skills UI, explain every instruction,
and show a passing and adversarial evaluation.
## 105–120 minutes — supervised proposal and claim audit
1. Have Brad turn the triage into the smallest repo-side proposal without
applying it.
2. Have him identify every step requiring approval: file edit, commit/push,
pipeline trigger, Flux reconcile, credential/environment change, or manual
backfill.
3. Audit the interview answer sentence by sentence using `mastery-rubric.md`.
Include the retained OpenClaw commits and live PVC/no-workload state.
4. Offer to record progress only after Brad approves the write.
Minimum honest outcome:
- current Hermes deployment and supervised triage use are demonstrated;
- the established Atlas automation path and read-only boundary are explained;
- one reusable writable skill is created, tested, and improved by Brad;
- autonomous production mutation is not claimed;
- historical OpenClaw deployment and replacement are supported by retained Git
and PVC evidence, with personal-use details identified as Brad's testimony.

66
services/hermes/skills/triage-atlas-service-health/SKILL.md Normal file
View File

@ -0,0 +1,66 @@
---
name: triage-atlas-service-health
description: Diagnose active Atlas service and infrastructure problems from Grafana, Ariadne, Flux, Kubernetes events, workload state, and logs. Use when asked what is broken, why a Grafana health panel is red, whether an outage is real, or how to fix a service through titan-iac. Covers every deployed namespace while distinguishing current services from migration residue and historical noise.
---
# Triage Atlas Service Health
Investigate read-only, group symptoms into incidents, and recommend the smallest Flux-tracked fix.
## Establish the current scope
1. Read `references/service-map.md` for custom-service ownership and migrations.
2. Discover the live inventory instead of relying only on the map:
```sh
kubectl get namespaces
kubectl get deploy,statefulset,daemonset -A
kubectl get ingress -A
kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io
```
3. Cassandra is authoritative for functionality migrated from Veles. Veles is
retired migration residue: its failed pods, old Jobs, and legacy Service or
Ingress objects are not current user impact. Escalate Veles only when direct
evidence proves Cassandra still depends on it or the Veles-to-Cassandra
redirect itself is failing.
## Decide whether the signal is actionable
Use all of these checks before calling something an incident:
- Current state: prefer an active condition over lifetime restart totals or old failed Jobs.
- Persistence: ignore normal startup states under 15 minutes unless users are already affected.
- Ownership: group replica pods by Deployment, StatefulSet, DaemonSet, Job, or CronJob.
- Delivery: a Flux object with `Ready=null` may be reconciling; `Ready=false` or a suspended required object is actionable.
- Authority: separate current workloads from retired, migrated, or deliberately suspended resources. Never describe Veles as unavailable or degraded user-facing service merely because its retained pods are unhealthy.
- Correlation: confirm a red Grafana panel with the underlying metric and at least one independent source such as events, logs, readiness, or Flux state.
Run the narrowest relevant checks:
```sh
kubectl get pods -A --field-selector status.phase!=Running,status.phase!=Succeeded -o wide
kubectl get events -A --sort-by=.lastTimestamp
kubectl -n <namespace> get deploy,statefulset,daemonset,job,cronjob
kubectl -n <namespace> describe pod <pod>
kubectl -n <namespace> logs <pod> --all-containers --tail=200
kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io
curl -G -fsS --data-urlencode 'query=<promql>' "$VICTORIA_METRICS_URL/api/v1/query"
```
Never read Kubernetes Secret values. Secret-key errors quoted by a pod event or log are valid evidence; inspect the SecretProviderClass and Vault policy manifests, not the secret contents.
## Produce one incident per cause
Deduplicate replica pods and repeated alerts with a signature of service, workload, failing condition, and likely dependency. Return:
- `Finding`: current incident, degraded-but-serving, migration residue, historical noise, or healthy.
- `Impact`: the user-facing service and affected capability.
- `Evidence`: timestamps, workload condition, event/log excerpt, metric, and Flux revision.
- `Noise removed`: signals inspected but not counted, with the reason.
- `Likely cause`: causal chain, marking inference explicitly.
- `Next checks`: ordered read-only commands.
- `Repo-side fix`: exact titan-iac or application path and smallest proposed change.
- `Approval required`: every write, reconcile, credential, or external-system action.
Do not claim a proposed change is applied. Do not mutate the cluster; Atlas is Flux-owned.

4
services/hermes/skills/triage-atlas-service-health/agents/openai.yaml Normal file
View File

@ -0,0 +1,4 @@
interface:
display_name: "Triage Atlas Service Health"
short_description: "Separate real Atlas incidents from dashboard noise"
default_prompt: "Use $triage-atlas-service-health to identify the active service incidents in Atlas, group duplicate symptoms, and recommend read-only checks and the smallest Flux-tracked fixes."

22
services/hermes/skills/triage-atlas-service-health/references/service-map.md Normal file
View File

@ -0,0 +1,22 @@
# Atlas custom-service map
Use this map for ownership and canonical names. Discover third-party services live from Flux and Kubernetes.
| Capability | Canonical workload/repository | Runtime location | Notes |
| --- | --- | --- | --- |
| Cluster automation and state analysis | Ariadne / `bstein/Ariadne` | `maintenance/ariadne` | Provides deterministic triage bundles. |
| Quality policy and CI telemetry | titan-iac / `bstein/titan-iac` | Jenkins plus Flux; no single app Deployment | Owns Jenkins configuration, Grafana generators, alert provisioning, and the Data Prepper pipeline. |
| Simulation registry and execution | Cassandra / Cassandra application repository | `cassandra/cassandra-frontend`, `cassandra-backend`, `cassandra-postgres`, `cassandra-vault-sync` | Cassandra is authoritative after the Veles migration. |
| Legacy simulation stack | Veles | `veles` namespace | Retired migration residue. Pod failures are not current impact; only a proven Cassandra dependency or broken redirect is actionable. |
| Backup and maintenance policy | Soteria / `bstein/soteria` | `maintenance/soteria` | Correlate CI failures separately from runtime backup health. |
| Media client | Pegasus / `bstein/pegasus` | `jellyfin/pegasus` | Correlate with Jellyfin and OIDC dependencies. |
| Cluster sentinel | Metis / `bstein/metis` | `maintenance/metis` plus sentinel DaemonSets | Distinguish controller health from per-node sentinel evidence. |
| User and service automation | Ananke / `bstein/ananke` | No current dedicated in-cluster Deployment | Treat its Jenkins suite as the authoritative runtime evidence unless discovery proves otherwise. |
| Chat automation | Atlasbot / `bstein/atlasbot` | `comms/atlasbot` | Correlate with comms and OIDC dependencies. |
| Public site | bstein_home / `bstein/bstein-dev-home` | `bstein-dev-home` frontend, backend, and Vault sync | Canonical metric suite uses underscore form. |
| Data Prepper integration | data_prepper / `bstein/titan-iac` | `logging/data-prepper` | Pipeline is under `services/logging`. |
| Desktop test application | Lesavka / `bstein/lesavka` | Desktop test host, not a normal Kubernetes Deployment | Test suite is in scope; runtime evidence may come from titan-jh. |
| Hermes operator | `hermes` namespace / `services/hermes` | `hermes/hermes`, model gate, and Ollama | Read-only cluster operator with Codex primary and local fallback. |
| Hermes consumer chat | `hermes-chat` namespace / `services/hermes-chat` | `hermes-chat/hermes-chat` | Isolated from cluster operation; do not use it for infrastructure triage. |
Canonical CI suites are `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `lesavka`, `metis`, `pegasus`, `soteria`, and `titan_iac`.

120
services/hermes/skills/triage-titan-test-failures/SKILL.md Normal file
View File

@ -0,0 +1,120 @@
---
name: triage-titan-test-failures
description: Diagnose Titan CI test failures and environment regressions from Ariadne evidence, Jenkins artifacts, Flux state, Kubernetes health, Pushgateway quality metrics, and Grafana context. Use for failed or flaky in-scope suites, suspected cluster-caused test failures, release-quality questions, or requests for a supervised read-only triage summary and repo-side next steps.
---
# Triage Titan Test Failures
Follow the established Titan evidence path. Keep the investigation read-only and distinguish observed facts from inference.
## Collect the canonical evidence
1. Read the latest Ariadne diagnosis:
```sh
curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"
```
2. Always read the deterministic bundle, even when the diagnosis looks complete:
```sh
curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"
```
3. Ask for human approval before triggering a fresh collection or diagnosis. Use these only after approval:
```sh
curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/collect"
curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"
```
Record the timestamps and status from both responses. Treat the diagnosis as stale when it predates the deterministic bundle, reports `unavailable`, or lacks concrete evidence. In any of those cases, continue the investigation from the deterministic bundle; do not stop at model availability.
Treat Ariadne's bundle as the evidence source of truth. A local-model diagnosis may be unavailable while Wolf owns titan-24; continue from the stored bundle using the active Hermes fallback model.
## Atlas facts and guardrails
- Ariadne runs as the `ariadne` Deployment and Service in the `maintenance` namespace.
- Hermes local inference runs through `hermes-model-gate` in the `hermes` namespace. Ollama is the `hermes-ollama` Deployment and Service in that namespace.
- OpenClaw is not part of this inference path. Never suggest an `openclaw` resource unless a read-only query first proves one exists.
- Custom runtime namespaces do not always match suite names. In particular,
Soteria, Metis, and Ariadne run in `maintenance`; Pegasus runs in `jellyfin`;
Atlasbot runs in `comms`; Data Prepper runs in `logging`; and Lesavka is a
desktop-hosted application. Read the service-health skill's
`references/service-map.md` before claiming that a suite has no runtime.
- A connection error recorded inside an older diagnosis proves only that the model request failed at that timestamp. It does not prove the service is currently down.
- Never invent a namespace, workload, container, port, URL, log line, metric, commit, or pod condition. Verify a target with a read-only query before presenting an exact follow-up command; otherwise state what must be discovered first.
- HTTP POST collection and diagnosis endpoints are state-changing operations. Put them only under `Approval required`; never describe them as read-only or include them in the read-only command list.
## Narrow the failure
Work in this order:
1. Confirm the failed suite and build are in the canonical scope: `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `lesavka`, `metis`, `pegasus`, `soteria`, or `titan_iac`.
2. Identify the first failed gate in the enforced order: `style`, `loc`, `coverage`, `tests`, `gate_glue`, `sonarqube`, `supply_chain`.
3. Correlate the build timestamp with retained Jenkins logs/artifacts, recent Git commits, and Flux revisions.
4. Check whether Kubernetes health, node pressure, image pulls, storage, DNS, or a shared dependency explains the failure better than a repo regression.
5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry. A zero-valued series is not a failure.
6. Keep running Jenkins builds in an in-progress collection. Do not count them as failed until Jenkins reports a terminal failure state.
7. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause.
For recent Git context, map the canonical suite to its Gitea repository and
read the latest commits. The current mappings are:
- `ananke` -> `bstein/ananke`
- `ariadne` -> `bstein/Ariadne`
- `atlasbot` -> `bstein/atlasbot`
- `bstein_home` -> `bstein/bstein-dev-home`
- `data_prepper` -> `bstein/titan-iac`
- `lesavka` -> `bstein/lesavka`
- `metis` -> `bstein/metis`
- `pegasus` -> `bstein/pegasus`
- `soteria` -> `bstein/soteria`
- `titan_iac` -> `bstein/titan-iac`
Use the read-only Gitea API and compare commit timestamps to the failing build:
```sh
curl -fsS "$GITEA_BASE_URL/api/v1/repos/<owner>/<repo>/commits?limit=10"
```
For quality telemetry, query VictoriaMetrics directly when the bundle is stale
or a specific label needs confirmation. URL-encode PromQL rather than manually
escaping it:
```sh
curl -G -fsS --data-urlencode \
'query=platform_quality_gate_runs_total{suite="<suite>"}' \
"$VICTORIA_METRICS_URL/api/v1/query"
curl -G -fsS --data-urlencode \
'query=<suite>_quality_gate_checks_total' \
"$VICTORIA_METRICS_URL/api/v1/query"
```
For image-pull failures, inspect the pod event, its ServiceAccount metadata,
and Flux image objects without reading referenced Secret values:
```sh
kubectl -n <namespace> describe pod <pod>
kubectl -n <namespace> get serviceaccount <name> -o yaml
kubectl get imagerepositories,imagepolicies,imageupdateautomations -A
```
If the latest diagnosis is unavailable, the minimum acceptable result still analyzes the deterministic bundle's failed suites, build/check evidence, freshness, and environment observations. Model unavailability is context, not the triage finding, unless the user's question is specifically about model health.
Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands.
## Produce the triage result
Return these sections:
- `Finding`: one sentence naming the most likely failure class.
- `Confidence`: low, medium, or high, with the reason.
- `Evidence`: the smallest set of concrete timestamps, build IDs, artifact paths, commits, metrics, pods, nodes, or Flux revisions that support the finding.
- `Likely cause`: explain the causal chain and label inference as inference.
- `Blast radius`: affected suites, services, branches, or environments.
- `Next checks`: ordered read-only checks with exact commands or URLs.
- `Repo-side fix`: the smallest Flux/IaC or application change, or `none yet` when evidence is insufficient.
- `Approval required`: call out every step that would modify files, infrastructure, credentials, test environments, or external systems.
Never present a proposed fix as applied. Prefer a concise evidence-backed answer over a general log summary.

4
services/hermes/skills/triage-titan-test-failures/agents/openai.yaml Normal file
View File

@ -0,0 +1,4 @@
interface:
display_name: "Triage Titan Test Failures"
short_description: "Analyze evidence for supervised test-failure triage"
default_prompt: "Use $triage-titan-test-failures to diagnose the latest Titan test failure from Ariadne evidence."

39
services/hermes/skills/tune-atlas-alerts/SKILL.md Normal file
View File

@ -0,0 +1,39 @@
---
name: tune-atlas-alerts
description: Audit and tune noisy Atlas Grafana alerts using live VictoriaMetrics evidence, Grafana notification history, persistence, minimum sample sizes, and generator ownership. Use when alerts are over-aggressive, a panel is red without user impact, PromQL behaves impossibly, or a Flux-tracked alert/dashboard correction is needed.
---
# Tune Atlas Alerts
Reduce false positives without hiding real failures. Read `references/alert-review.md` before recommending a rule change.
## Audit the signal
1. Find the provisioned rule in `services/monitoring/grafana-alerting-config.yaml`.
2. If the issue is a dashboard panel, find its Python source in `scripts/dashboards_render_atlas.py`; never hand-edit generated JSON or ConfigMaps.
3. Query the exact PromQL against VictoriaMetrics and inspect the labels and raw inputs.
4. Compare current value, recent history, notification frequency, and a second source such as Kubernetes events or service metrics.
5. Classify the problem as bad math, lifetime-versus-rate confusion, missing persistence, low sample size, duplicate series, retired scope, rollout noise, real backlog, or real incident.
Useful read-only commands:
```sh
curl -G -fsS --data-urlencode 'query=<promql>' "$VICTORIA_METRICS_URL/api/v1/query"
kubectl -n monitoring logs deploy/grafana --since=24h
kubectl -n monitoring get configmap grafana-alerting-config -o yaml
kubectl -n <namespace> get events --sort-by=.lastTimestamp
```
## Design the correction
- Use `rate`, `increase`, or `delta` according to counter/gauge semantics; clamp percentages to 0..100.
- Require persistence for startup, rollout, and scheduling conditions.
- Add a minimum denominator and absolute count to percentage alerts with small samples.
- Alert on configured resources that regressed; keep unenrolled inventory visible as backlog instead of paging continuously.
- Exclude retired or migrated resources from high-level health, while retaining them on detailed dashboards.
- Deduplicate replicas and scrape targets using stable service/workload labels.
- Preserve a drill-down path to the raw evidence.
Validate changed PromQL live, run the dashboard generator, test the generator, render the monitoring kustomization, and use a client-side dry-run. Report before/after values and what failure will still trigger the rule.
Hermes is read-only in the cluster. Draft or explain repo changes and mark reconcile or deployment actions as approval-required.

4
services/hermes/skills/tune-atlas-alerts/agents/openai.yaml Normal file
View File

@ -0,0 +1,4 @@
interface:
display_name: "Tune Atlas Alerts"
short_description: "Make Grafana alerts actionable without hiding faults"
default_prompt: "Use $tune-atlas-alerts to audit the currently noisy Atlas alerts, prove which signals are false positives, and propose the smallest generator- or alert-rule corrections."

16
services/hermes/skills/tune-atlas-alerts/references/alert-review.md Normal file
View File

@ -0,0 +1,16 @@
# Alert review checklist
For every alert, record:
1. User impact or operator action the alert demands.
2. Metric type: counter, gauge, timestamp, state marker, or recording rule.
3. Scope and deduplication labels.
4. Persistence window and startup grace.
5. Minimum sample size for ratios.
6. No-data and query-error behavior.
7. Current value and a recent range query.
8. Notification frequency in Grafana logs.
9. Current, migrated, suspended, or historical resource status.
10. Exact Flux-tracked source, generated artifacts, tests, and rollback condition.
A rule is useful only when its firing state implies a specific human action. Backlogs and unenrolled resources belong on dashboards unless they have crossed an explicitly accepted operational deadline.

6
services/hermes/vault-serviceaccount.yaml Normal file
View File

@ -0,0 +1,6 @@
# services/hermes/vault-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: hermes-vault
namespace: hermes

6
services/jenkins/configmap-jcasc.yaml
View File

@ -429,15 +429,15 @@ data:
}
}
}
pipelineJob('veles') {
pipelineJob('cassandra') {
disabled(false)
description('Staged Veles alpha image pipeline. Backend/frontend should build linux/amd64 and linux/arm64; sim-worker may begin amd64-only if Forge dependencies require it.')
description('Staged Cassandra alpha image pipeline. Backend/frontend should build linux/amd64 and linux/arm64; sim-worker may begin amd64-only if Forge dependencies require it.')
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/bstein/veles.git')
url('https://scm.bstein.dev/bstein/cassandra.git')
credentials('gitea-pat')
}
branches('*/main')

401
services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml
View File

@ -1,401 +0,0 @@
# services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml
# One-off job for sso/cassandra-realm-ensure-1.
# Purpose: create the Cassandra realm, groups, OIDC client, SMTP settings, and Vault client secret.
# Keep suspended until Cassandra Vault paths/policies have reconciled, then unsuspend once.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-realm-ensure-1
namespace: sso
spec:
suspend: true
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "sso-secrets"
vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
{{ with secret "kv/data/atlas/shared/keycloak-admin" }}
export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
{{ end }}
{{ with secret "kv/data/atlas/shared/postmark-relay" }}
export KEYCLOAK_SMTP_USER="{{ index .Data.data "apikey" }}"
export KEYCLOAK_SMTP_PASSWORD="{{ index .Data.data "apikey" }}"
{{ end }}
spec:
serviceAccountName: mas-secrets-ensure
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/worker
operator: Exists
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: kubernetes.io/arch
operator: In
values: ["arm64"]
containers:
- name: configure
image: python:3.11-alpine
env:
- name: KEYCLOAK_SERVER
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: cassandra
- name: KEYCLOAK_CLIENT_ID
value: cassandra-web
- name: KEYCLOAK_PUBLIC_ISSUER
value: https://sso.bstein.dev/realms/cassandra
- name: CASSANDRA_BASE_URL
value: https://cassandra.bstein.dev
- name: CASSANDRA_TESTER_GROUP
value: cassandra-tester
- name: KEYCLOAK_SMTP_HOST
value: mail.bstein.dev
- name: KEYCLOAK_SMTP_PORT
value: "587"
- name: KEYCLOAK_SMTP_FROM
value: no-reply-cassandra@bstein.dev
- name: KEYCLOAK_SMTP_FROM_NAME
value: Cassandra
- name: CASSANDRA_PUBLIC_REPLAY_USERS
value: cassandra-dev,daniel-test,viktor-test
command: ["/bin/sh", "-c"]
args:
- |
set -eu
. /vault/secrets/keycloak-admin-env.sh
python - <<'PY'
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/")
realm = os.environ["KEYCLOAK_REALM"]
client_id = os.environ["KEYCLOAK_CLIENT_ID"]
issuer = os.environ["KEYCLOAK_PUBLIC_ISSUER"]
app_base_url = os.environ["CASSANDRA_BASE_URL"].rstrip("/")
admin_user = os.environ["KEYCLOAK_ADMIN_USER"]
admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"]
def request(method, url, token=None, payload=None, headers=None, timeout=30):
data = None
req_headers = headers.copy() if headers else {}
if token:
req_headers["Authorization"] = f"Bearer {token}"
if payload is not None:
data = json.dumps(payload).encode()
req_headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, headers=req_headers, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
body = resp.read()
if not body:
return resp.status, None
return resp.status, json.loads(body.decode())
except urllib.error.HTTPError as exc:
raw = exc.read()
if not raw:
return exc.code, None
try:
return exc.code, json.loads(raw.decode())
except Exception:
return exc.code, {"raw": raw.decode(errors="replace")}
token_body = None
form = urllib.parse.urlencode(
{
"grant_type": "password",
"client_id": "admin-cli",
"username": admin_user,
"password": admin_password,
}
).encode()
for attempt in range(1, 11):
req = urllib.request.Request(
f"{base_url}/realms/master/protocol/openid-connect/token",
data=form,
headers={"Content-Type": "application/x-www-form-urlencoded"},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
token_body = json.loads(resp.read().decode())
break
except urllib.error.URLError as exc:
if attempt == 10:
raise SystemExit(f"Keycloak token request failed after retries: {exc}")
time.sleep(attempt * 2)
token = token_body["access_token"]
smtp = {
"host": os.environ["KEYCLOAK_SMTP_HOST"],
"port": os.environ["KEYCLOAK_SMTP_PORT"],
"from": os.environ["KEYCLOAK_SMTP_FROM"],
"fromDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"],
"replyTo": os.environ["KEYCLOAK_SMTP_FROM"],
"replyToDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"],
"user": os.environ["KEYCLOAK_SMTP_USER"],
"password": os.environ["KEYCLOAK_SMTP_PASSWORD"],
"auth": "true",
"starttls": "true",
"ssl": "false",
}
status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token)
if status == 404:
create_payload = {
"realm": realm,
"enabled": True,
"registrationAllowed": False,
"resetPasswordAllowed": True,
"verifyEmail": True,
"loginWithEmailAllowed": True,
"duplicateEmailsAllowed": False,
"smtpServer": smtp,
}
status, body = request("POST", f"{base_url}/admin/realms", token, create_payload)
if status not in (201, 204, 409):
raise SystemExit(f"Realm create failed: status={status} body={body}")
status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token)
if status != 200 or not isinstance(realm_rep, dict):
raise SystemExit(f"Realm fetch failed: status={status}")
realm_rep.update(
{
"enabled": True,
"registrationAllowed": False,
"resetPasswordAllowed": True,
"verifyEmail": True,
"loginWithEmailAllowed": True,
"duplicateEmailsAllowed": False,
"smtpServer": smtp,
}
)
status, body = request("PUT", f"{base_url}/admin/realms/{realm}", token, realm_rep)
if status not in (200, 204):
raise SystemExit(f"Realm update failed: status={status} body={body}")
def find_group(name):
status, groups = request(
"GET",
f"{base_url}/admin/realms/{realm}/groups?search={urllib.parse.quote(name)}",
token,
)
if status != 200:
raise SystemExit(f"Group search failed for {name}: status={status}")
for group in groups or []:
if group.get("name") == name:
return group
return None
def ensure_group(name):
group = find_group(name)
if group:
return group
status, body = request("POST", f"{base_url}/admin/realms/{realm}/groups", token, {"name": name})
if status not in (201, 204, 409):
raise SystemExit(f"Group create failed for {name}: status={status} body={body}")
group = find_group(name)
if not group:
raise SystemExit(f"Group missing after create for {name}")
return group
def find_user(username):
encoded = urllib.parse.quote(username)
for query in (f"username={encoded}&exact=true", f"email={encoded}&exact=true", f"search={encoded}"):
status, users = request(
"GET",
f"{base_url}/admin/realms/{realm}/users?{query}",
token,
)
if status != 200:
raise SystemExit(f"User search failed for {username}: status={status}")
for user in users or []:
candidates = {
str(user.get("username") or "").casefold(),
str(user.get("email") or "").casefold(),
}
if username.casefold() in candidates:
return user
return None
def ensure_user_group(username, group):
user = find_user(username)
if not user:
print(f"WARNING: Cassandra public replay user {username!r} not found; skipping membership")
return
user_id = user.get("id")
group_id = group.get("id")
if not user_id or not group_id:
raise SystemExit(f"Missing id for user/group membership: user={username}")
status, memberships = request(
"GET",
f"{base_url}/admin/realms/{realm}/users/{user_id}/groups",
token,
)
if status != 200:
raise SystemExit(f"User group lookup failed for {username}: status={status}")
if any(item.get("id") == group_id for item in memberships or []):
return
status, body = request(
"PUT",
f"{base_url}/admin/realms/{realm}/users/{user_id}/groups/{group_id}",
token,
)
if status not in (200, 204):
raise SystemExit(f"User group update failed for {username}: status={status} body={body}")
ensure_group("alpha")
ensure_group("admin")
ensure_group(os.environ["CASSANDRA_TESTER_GROUP"])
public_replay_group = ensure_group("public-replay")
for username in os.environ.get("CASSANDRA_PUBLIC_REPLAY_USERS", "").split(","):
username = username.strip()
if username:
ensure_user_group(username, public_replay_group)
status, clients = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}",
token,
)
if status != 200:
raise SystemExit(f"Client lookup failed: status={status}")
client_uuid = clients[0]["id"] if clients else None
client_payload = {
"clientId": client_id,
"enabled": True,
"protocol": "openid-connect",
"publicClient": False,
"standardFlowEnabled": True,
"implicitFlowEnabled": False,
"directAccessGrantsEnabled": False,
"serviceAccountsEnabled": False,
"redirectUris": [f"{app_base_url}/*"],
"webOrigins": [app_base_url],
"rootUrl": app_base_url,
"baseUrl": "/",
"attributes": {
"pkce.code.challenge.method": "S256",
"post.logout.redirect.uris": f"{app_base_url}/*",
},
}
if not client_uuid:
status, body = request("POST", f"{base_url}/admin/realms/{realm}/clients", token, client_payload)
if status not in (201, 204, 409):
raise SystemExit(f"Client create failed: status={status} body={body}")
status, clients = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}",
token,
)
client_uuid = clients[0]["id"] if clients else None
if not client_uuid:
raise SystemExit("Client cassandra-web not found after create")
status, body = request(
"PUT",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}",
token,
client_payload,
)
if status not in (200, 204):
raise SystemExit(f"Client update failed: status={status} body={body}")
mapper_payload = {
"name": "groups",
"protocol": "openid-connect",
"protocolMapper": "oidc-group-membership-mapper",
"consentRequired": False,
"config": {
"full.path": "false",
"id.token.claim": "true",
"access.token.claim": "true",
"userinfo.token.claim": "true",
"claim.name": "groups",
"jsonType.label": "String",
},
}
status, mappers = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models",
token,
)
if status != 200:
raise SystemExit(f"Mapper lookup failed: status={status}")
mapper_id = next((mapper.get("id") for mapper in mappers or [] if mapper.get("name") == "groups"), None)
if mapper_id:
mapper_update_payload = dict(mapper_payload)
mapper_update_payload["id"] = mapper_id
status, body = request(
"PUT",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models/{mapper_id}",
token,
mapper_update_payload,
)
else:
status, body = request(
"POST",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models",
token,
mapper_payload,
)
if status not in (200, 201, 204):
raise SystemExit(f"Mapper ensure failed: status={status} body={body}")
status, secret = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/client-secret",
token,
)
client_secret = (secret or {}).get("value")
if status != 200 or not client_secret:
raise SystemExit(f"Client secret fetch failed: status={status}")
vault_addr = os.environ.get("VAULT_ADDR", "http://vault.vault.svc.cluster.local:8200")
jwt = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip()
login_payload = json.dumps({"jwt": jwt, "role": os.environ.get("VAULT_ROLE", "sso-secrets")}).encode()
req = urllib.request.Request(
f"{vault_addr}/v1/auth/kubernetes/login",
data=login_payload,
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=20) as resp:
vault_token = json.loads(resp.read().decode())["auth"]["client_token"]
payload = {
"data": {
"client_id": client_id,
"client_secret": client_secret,
"issuer": issuer,
"realm": realm,
"required_groups": "alpha,admin",
}
}
req = urllib.request.Request(
f"{vault_addr}/v1/kv/data/atlas/cassandra/cassandra-oidc",
data=json.dumps(payload).encode(),
headers={"X-Vault-Token": vault_token, "Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=20) as resp:
if resp.status not in (200, 204):
raise SystemExit(f"Vault write returned {resp.status}")
print("Cassandra Keycloak realm/client ready")
PY

47
services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml → services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml
View File

@ -1,25 +1,19 @@
# services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
# One-off job for sso/cassandra-gitea-oidc-secret-ensure-1.
# Purpose: create/update the Keycloak cassandra realm Gitea OIDC client, ensure the
# cassandra-tester group exists, and write the client secret to Vault for Gitea.
# Run by setting spec.suspend to false, reconcile, then set it back to true.
# Safe to delete the finished Job/pod; it should not run continuously.
# services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml
# Purpose: create chat OIDC and Brad-only operator proxy clients.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-gitea-oidc-secret-ensure-1
name: hermes-access-oidc-client-ensure-3
namespace: sso
spec:
suspend: true
backoffLimit: 0
ttlSecondsAfterFinished: 3600
backoffLimit: 3
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "sso-secrets"
vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
vault.hashicorp.com/role: sso-secrets
vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: kv/data/atlas/shared/keycloak-admin
vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
{{ with secret "kv/data/atlas/shared/keycloak-admin" }}
export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
@ -29,11 +23,6 @@ spec:
spec:
serviceAccountName: mas-secrets-ensure
restartPolicy: Never
volumes:
- name: cassandra-gitea-oidc-secret-ensure-script
configMap:
name: cassandra-gitea-oidc-secret-ensure-script
defaultMode: 0555
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
@ -41,14 +30,32 @@ spec:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values: ["arm64"]
values:
- arm64
- key: node-role.kubernetes.io/worker
operator: Exists
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-18
containers:
- name: apply
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
command: ["/scripts/cassandra_gitea_oidc_secret_ensure.sh"]
command:
- /scripts/hermes_access_oidc_ensure.sh
volumeMounts:
- name: cassandra-gitea-oidc-secret-ensure-script
- name: script
mountPath: /scripts
readOnly: true
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
volumes:
- name: script
configMap:
name: hermes-access-oidc-script
defaultMode: 0555

7
services/keycloak/kustomization.yaml
View File

@ -27,8 +27,7 @@ resources:
- bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml
- bootstrap-jobs/quality-oidc-secret-ensure-job.yaml
- bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml
- bootstrap-jobs/cassandra-realm-ensure-job.yaml
- bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
- bootstrap-jobs/hermes-access-oidc-client-job.yaml
- bootstrap-jobs/veles-realm-ensure-job.yaml
- bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml
- bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml
@ -57,9 +56,9 @@ configMapGenerator:
- name: hermes-dashboard-oidc-client-script
files:
- hermes_dashboard_oidc_client_ensure.sh=scripts/hermes_dashboard_oidc_client_ensure.sh
- name: cassandra-gitea-oidc-secret-ensure-script
- name: hermes-access-oidc-script
files:
- cassandra_gitea_oidc_secret_ensure.sh=scripts/cassandra_gitea_oidc_secret_ensure.sh
- hermes_access_oidc_ensure.sh=scripts/hermes_access_oidc_ensure.sh
- name: veles-gitea-oidc-secret-ensure-script
files:
- veles_gitea_oidc_secret_ensure.sh=scripts/veles_gitea_oidc_secret_ensure.sh

157
services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh
View File

@ -1,157 +0,0 @@
#!/usr/bin/env sh
set -euo pipefail
. /vault/secrets/keycloak-admin-env.sh
KC_URL="${KEYCLOAK_SERVER:-http://keycloak.sso.svc.cluster.local}"
REALM="${KEYCLOAK_REALM:-cassandra}"
CLIENT_ID="${KEYCLOAK_CLIENT_ID:-gitea}"
PUBLIC_BASE_URL="${GITEA_PUBLIC_BASE_URL:-https://scm.bstein.dev}"
AUTH_SOURCE_NAME="${GITEA_AUTH_SOURCE_NAME:-cassandra}"
TESTER_GROUP="${CASSANDRA_GITEA_TESTER_GROUP:-cassandra-tester}"
VAULT_SECRET_PATH="${VAULT_SECRET_PATH:-kv/data/atlas/gitea/gitea-cassandra-oidc}"
ACCESS_TOKEN=""
for attempt in 1 2 3 4 5 6 7 8 9 10; do
if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then
break
fi
echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2
sleep $((attempt * 2))
done
for attempt in 1 2 3 4 5; do
TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \
-H 'Content-Type: application/x-www-form-urlencoded' \
-d "grant_type=password" \
-d "client_id=admin-cli" \
-d "username=${KEYCLOAK_ADMIN}" \
-d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)"
if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then
break
fi
echo "Keycloak token request failed (attempt ${attempt})" >&2
sleep $((attempt * 2))
done
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
echo "Failed to fetch Keycloak admin token" >&2
exit 1
fi
ensure_group() {
group_name="$1"
groups="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/groups?search=$(printf '%s' "${group_name}" | jq -sRr @uri)" || true)"
if echo "$groups" | jq -e --arg name "$group_name" '.[]? | select(.name == $name)' >/dev/null 2>&1; then
return
fi
status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "$(jq -nc --arg name "$group_name" '{name:$name}')" \
"${KC_URL}/admin/realms/${REALM}/groups")"
if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then
echo "Keycloak group create failed for ${group_name} (status ${status})" >&2
exit 1
fi
}
ensure_group "${TESTER_GROUP}"
CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)"
CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
client_payload="$(jq -nc \
--arg client_id "${CLIENT_ID}" \
--arg root_url "${PUBLIC_BASE_URL}" \
--arg callback "${PUBLIC_BASE_URL}/user/oauth2/${AUTH_SOURCE_NAME}/callback" \
'{clientId:$client_id,enabled:true,protocol:"openid-connect",publicClient:false,standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false,serviceAccountsEnabled:false,redirectUris:[$callback],webOrigins:[$root_url],rootUrl:$root_url,baseUrl:"/",attributes:{"pkce.code.challenge.method":"S256","post.logout.redirect.uris":($root_url + "/*")}}')"
if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then
status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${client_payload}" \
"${KC_URL}/admin/realms/${REALM}/clients")"
if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then
echo "Keycloak client create failed for ${CLIENT_ID} (status ${status})" >&2
exit 1
fi
CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)"
CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
fi
if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then
echo "Keycloak client ${CLIENT_ID} not found after create" >&2
exit 1
fi
status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${client_payload}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}")"
if [ "$status" != "200" ] && [ "$status" != "204" ]; then
echo "Keycloak client update failed for ${CLIENT_ID} (status ${status})" >&2
exit 1
fi
mapper_payload="$(jq -nc \
'{name:"groups",protocol:"openid-connect",protocolMapper:"oidc-group-membership-mapper",consentRequired:false,config:{"full.path":"false","id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","claim.name":"groups","jsonType.label":"String"}}')"
mappers="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models" || true)"
MAPPER_ID="$(echo "$mappers" | jq -r '.[]? | select(.name == "groups") | .id' | head -n1 || true)"
if [ -n "$MAPPER_ID" ] && [ "$MAPPER_ID" != "null" ]; then
status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${mapper_payload}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models/${MAPPER_ID}")"
else
status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${mapper_payload}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models")"
fi
if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then
echo "Keycloak groups mapper ensure failed (status ${status})" >&2
exit 1
fi
CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/client-secret" | jq -r '.value' 2>/dev/null || true)"
if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
echo "Keycloak client secret not found for ${CLIENT_ID}" >&2
exit 1
fi
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
payload="$(jq -nc \
--arg client_id "${CLIENT_ID}" \
--arg client_secret "${CLIENT_SECRET}" \
--arg discovery_url "${PUBLIC_BASE_URL}" \
--arg issuer "https://sso.bstein.dev/realms/${REALM}" \
--arg auto_discovery_url "https://sso.bstein.dev/realms/${REALM}/.well-known/openid-configuration" \
--arg auth_source_name "${AUTH_SOURCE_NAME}" \
--arg tester_group "${TESTER_GROUP}" \
--arg group_team_map "{\"${TESTER_GROUP}\":{\"cassandra-alpha\":[\"testers\"]}}" \
'{data:{client_id:$client_id,client_secret:$client_secret,issuer:$issuer,openid_auto_discovery_url:$auto_discovery_url,auth_source_name:$auth_source_name,required_claim_name:"groups",required_claim_value:$tester_group,group_claim_name:"groups",restricted_group:$tester_group,group_team_map:$group_team_map,gitea_base_url:$discovery_url}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-H 'Content-Type: application/json' \
-d "${payload}" "${vault_addr}/v1/${VAULT_SECRET_PATH}" >/dev/null
echo "Cassandra realm Gitea OIDC client ready"

229
services/keycloak/scripts/hermes_access_oidc_ensure.sh Executable file
View File

@ -0,0 +1,229 @@
#!/usr/bin/env sh
set -eu
. /vault/secrets/keycloak-admin-env.sh
KC_URL="http://keycloak.sso.svc.cluster.local"
CHAT_CLIENT="hermes-chat-dashboard"
CHAT_URL="https://chat.bstein.dev"
OPERATOR_CLIENT="hermes-operator-proxy"
OPERATOR_URL="https://agent.bstein.dev"
ACCESS_TOKEN=""
for attempt in 1 2 3 4 5 6 7 8 9 10; do
if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then
break
fi
echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2
sleep $((attempt * 2))
done
for attempt in 1 2 3 4 5; do
token_json="$(curl -sS -X POST "${KC_URL}/realms/master/protocol/openid-connect/token" \
-H 'Content-Type: application/x-www-form-urlencoded' \
-d "grant_type=password" \
-d "client_id=admin-cli" \
-d "username=${KEYCLOAK_ADMIN}" \
-d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
ACCESS_TOKEN="$(printf '%s' "${token_json}" | jq -r '.access_token' 2>/dev/null || true)"
if [ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ]; then
break
fi
echo "Keycloak token request failed (attempt ${attempt})" >&2
sleep $((attempt * 2))
done
if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then
echo "Failed to fetch Keycloak admin token" >&2
exit 1
fi
chat_payload="$(jq -nc \
--arg client_id "${CHAT_CLIENT}" \
--arg redirect_uri "${CHAT_URL}/auth/callback" \
--arg web_origin "${CHAT_URL}" \
'{
clientId:$client_id,
enabled:true,
protocol:"openid-connect",
publicClient:true,
standardFlowEnabled:true,
implicitFlowEnabled:false,
directAccessGrantsEnabled:false,
serviceAccountsEnabled:false,
redirectUris:[$redirect_uri],
webOrigins:[$web_origin],
rootUrl:$web_origin,
baseUrl:"/",
attributes:{
"pkce.code.challenge.method":"S256",
"post.logout.redirect.uris":$web_origin,
"access.token.lifespan":"1200"
}
}')"
chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)"
chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)"
if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then
status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${chat_payload}" \
"${KC_URL}/admin/realms/atlas/clients")"
if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then
echo "Keycloak chat client create failed (status ${status})" >&2
exit 1
fi
chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)"
chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)"
fi
if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then
echo "Keycloak chat client not found after create" >&2
exit 1
fi
status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${chat_payload}" \
"${KC_URL}/admin/realms/atlas/clients/${chat_id}")"
if [ "${status}" != "204" ]; then
echo "Keycloak chat client update failed (status ${status})" >&2
exit 1
fi
scope_id="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/client-scopes?search=groups" \
| jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)"
if [ -z "${scope_id}" ] || [ "${scope_id}" = "null" ]; then
echo "Keycloak groups client scope not found" >&2
exit 1
fi
default_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients/${chat_id}/default-client-scopes" || true)"
optional_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes" || true)"
if ! printf '%s' "${default_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1 \
&& ! printf '%s' "${optional_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1; then
status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")"
if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then
status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")"
if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then
echo "Failed to attach groups scope to chat client (status ${status})" >&2
exit 1
fi
fi
fi
operator_payload="$(jq -nc \
--arg client_id "${OPERATOR_CLIENT}" \
--arg redirect_uri "${OPERATOR_URL}/oauth2/callback" \
--arg web_origin "${OPERATOR_URL}" \
'{
clientId:$client_id,
enabled:true,
protocol:"openid-connect",
publicClient:false,
standardFlowEnabled:true,
implicitFlowEnabled:false,
directAccessGrantsEnabled:false,
serviceAccountsEnabled:false,
redirectUris:[$redirect_uri],
webOrigins:[$web_origin],
rootUrl:$web_origin,
baseUrl:"/",
attributes:{"post.logout.redirect.uris":$web_origin}
}')"
operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)"
operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)"
if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then
status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${operator_payload}" \
"${KC_URL}/admin/realms/atlas/clients")"
if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then
echo "Keycloak operator proxy client create failed (status ${status})" >&2
exit 1
fi
operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)"
operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)"
fi
if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then
echo "Keycloak operator proxy client not found after create" >&2
exit 1
fi
status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${operator_payload}" \
"${KC_URL}/admin/realms/atlas/clients/${operator_id}")"
if [ "${status}" != "204" ]; then
echo "Keycloak operator proxy client update failed (status ${status})" >&2
exit 1
fi
client_secret="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/atlas/clients/${operator_id}/client-secret" \
| jq -r '.value' 2>/dev/null || true)"
if [ -z "${client_secret}" ] || [ "${client_secret}" = "null" ]; then
echo "Keycloak operator proxy client secret not found" >&2
exit 1
fi
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt,role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "Vault login failed" >&2
exit 1
fi
read_status="$(curl -sS -o /tmp/hermes-operator-oidc-read.json -w '%{http_code}' \
-H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc" || true)"
cookie_secret=""
if [ "${read_status}" = "200" ]; then
cookie_secret="$(jq -r '.data.data.cookie_secret // empty' /tmp/hermes-operator-oidc-read.json)"
elif [ "${read_status}" != "404" ]; then
echo "Vault operator OIDC read failed (status ${read_status})" >&2
exit 1
fi
if [ -n "${cookie_secret}" ]; then
cookie_length="$(printf '%s' "${cookie_secret}" | wc -c | tr -d ' ')"
if [ "${cookie_length}" != "16" ] && [ "${cookie_length}" != "24" ] && [ "${cookie_length}" != "32" ]; then
cookie_secret=""
fi
fi
if [ -z "${cookie_secret}" ]; then
cookie_secret="$(openssl rand -hex 16 | tr -d '\n')"
fi
vault_payload="$(jq -nc \
--arg client_id "${OPERATOR_CLIENT}" \
--arg client_secret "${client_secret}" \
--arg cookie_secret "${cookie_secret}" \
'{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')"
write_status="$(curl -sS -o /tmp/hermes-operator-oidc-write.json -w '%{http_code}' -X POST \
-H "X-Vault-Token: ${vault_token}" \
-H 'Content-Type: application/json' \
-d "${vault_payload}" \
"${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc")"
if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then
echo "Vault operator OIDC write failed (status ${write_status})" >&2
exit 1
fi
echo "Hermes chat and operator OIDC clients are ready"

25
services/maintenance/apps/ariadne-deployment.yaml
View File

@ -285,8 +285,21 @@ spec:
- name: GAME_MODE_NODE_NAME
value: titan-24
- name: GAME_MODE_DISPLACE_WORKLOADS
value: >-
[{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}]
value: "[]"
- name: GAME_MODE_LEASE_NAMESPACE
value: hermes
- name: GAME_MODE_LEASE_NAME
value: titan-24-gpu-owner
- name: GAME_MODE_OLLAMA_URL
value: http://hermes-ollama.hermes.svc.cluster.local:11434
- name: GAME_MODE_OLLAMA_MODEL
value: gpt-oss:20b
- name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC
value: "900"
- name: GAME_MODE_TRANSITION_TIMEOUT_SEC
value: "900"
- name: GAME_MODE_POLL_INTERVAL_SEC
value: "1"
- name: WOLF_OIDC_CLIENT_ID
value: wolf
- name: WOLF_OIDC_BASE_URL
@ -418,7 +431,7 @@ spec:
- name: ARIADNE_VM_URL
value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
- name: ARIADNE_CLUSTER_STATE_VM_TIMEOUT_SEC
value: "5"
value: "30"
- name: ARIADNE_ALERTMANAGER_URL
value: http://alertmanager.monitoring.svc.cluster.local
- name: OPENSEARCH_URL
@ -444,11 +457,11 @@ spec:
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
value: "*/15 * * * *"
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
value: http://hermes-ollama.hermes.svc.cluster.local:11434
value: http://hermes-model-gate.hermes.svc.cluster.local:11434
- name: ARIADNE_TESTING_TRIAGE_MODEL
value: qwen2.5:7b-instruct-q4_0
value: gpt-oss:20b
- name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC
value: "180"
value: "900"
- name: JENKINS_WORKSPACE_NAMESPACE
value: jenkins
- name: JENKINS_WORKSPACE_PVC_PREFIX

1
services/maintenance/apps/ariadne-rbac.yaml
View File

@ -78,7 +78,6 @@ rules:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding

4
services/maintenance/apps/metis-configmap.yaml
View File

@ -15,8 +15,8 @@ data:
METIS_MAX_DEVICE_BYTES: "1000000000000"
METIS_NAMESPACE: maintenance
METIS_REMOTE_POD_TIMEOUT_SEC: "14400"
METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-243-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-270-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
METIS_HARBOR_REGISTRY: registry.bstein.dev
METIS_HARBOR_PROJECT: metis
METIS_HARBOR_API_BASE: https://registry.bstein.dev/api/v2.0

4
services/maintenance/kustomization.yaml
View File

@ -11,8 +11,8 @@ resources:
- networking
images:
- name: registry.bstein.dev/bstein/ariadne
newTag: 0.1.0-356 # {"$imagepolicy": "maintenance:ariadne:tag"}
newTag: 0.1.0-389 # {"$imagepolicy": "maintenance:ariadne:tag"}
- name: registry.bstein.dev/bstein/metis
newTag: 0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
newTag: 0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
- name: registry.bstein.dev/bstein/soteria
newTag: 0.1.0-120 # {"$imagepolicy": "maintenance:soteria:tag"}

2
services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml
View File

@ -32,7 +32,7 @@ spec:
kubernetes.io/arch: amd64
containers:
- name: metis-sentinel
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-270-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
imagePullPolicy: Always
envFrom:
- configMapRef:

2
services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml
View File

@ -32,7 +32,7 @@ spec:
kubernetes.io/arch: arm64
containers:
- name: metis-sentinel
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"}
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"}
imagePullPolicy: Always
envFrom:
- configMapRef:

8
services/maintenance/node-ops/node-image-sweeper-daemonset.yaml
View File

@ -25,6 +25,14 @@ spec:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
- key: cassandra.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
nodeSelector:
kubernetes.io/os: linux
containers:

92
services/monitoring/availability-backfill-v4-job.yaml Normal file
View File

@ -0,0 +1,92 @@
# services/monitoring/availability-backfill-v4-job.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: atlas-availability-request-v4-backfill-rules
namespace: monitoring
data:
atlas-request-history.yaml: |
groups:
- name: atlas.availability.request.backfill
interval: 1h
rules:
- record: atlas:availability:requests_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"[1-5].."
}[1h]
))
labels:
definition: request-v4
scope: atlas
rollup: hourly
- record: atlas:availability:failures_1h
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"5.."
}[1h]
))
labels:
definition: request-v4
scope: atlas
rollup: hourly
---
apiVersion: batch/v1
kind: Job
metadata:
name: atlas-availability-request-v4-backfill
namespace: monitoring
spec:
backoffLimit: 2
template:
metadata:
labels:
app: atlas-availability-request-v4-backfill
spec:
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-22
- titan-24
containers:
- name: vmalert-replay
image: victoriametrics/vmalert:v1.113.0
args:
- -datasource.url=http://victoria-metrics-single-server:8428
- -remoteWrite.url=http://victoria-metrics-single-server:8428
- -remoteWrite.flushInterval=1s
- -rule=/etc/vmalert/backfill/*.yaml
- -replay.timeFrom=2026-05-01T00:00:00Z
- -replay.timeTo=2026-08-04T23:00:00Z
- -replay.maxDatapointsPerQuery=48
- -replay.rulesDelay=2s
- -replay.disableProgressBar
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: "1"
memory: 512Mi
volumeMounts:
- name: rules
mountPath: /etc/vmalert/backfill
readOnly: true
volumes:
- name: rules
configMap:
name: atlas-availability-request-v4-backfill-rules

92
services/monitoring/availability-daily-backfill-v4-job.yaml Normal file
View File

@ -0,0 +1,92 @@
# services/monitoring/availability-daily-backfill-v4-job.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: atlas-availability-request-v4-daily-backfill-rules
namespace: monitoring
data:
atlas-request-daily-history.yaml: |
groups:
- name: atlas.availability.request.daily.backfill
interval: 1d
rules:
- record: atlas:availability:requests_1d
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"[1-5].."
}[1d]
))
labels:
definition: request-v4
scope: atlas
rollup: daily
- record: atlas:availability:failures_1d
expr: |
sum(increase(
traefik_entrypoint_requests_total{
entrypoint="websecure",
protocol="http",
code=~"5.."
}[1d]
))
labels:
definition: request-v4
scope: atlas
rollup: daily
---
apiVersion: batch/v1
kind: Job
metadata:
name: atlas-availability-request-v4-daily-backfill
namespace: monitoring
spec:
backoffLimit: 2
template:
metadata:
labels:
app: atlas-availability-request-v4-daily-backfill
spec:
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-22
- titan-24
containers:
- name: vmalert-replay
image: victoriametrics/vmalert:v1.113.0
args:
- -datasource.url=http://victoria-metrics-single-server:8428
- -remoteWrite.url=http://victoria-metrics-single-server:8428
- -remoteWrite.flushInterval=1s
- -rule=/etc/vmalert/backfill/*.yaml
- -replay.timeFrom=2026-05-01T00:00:00Z
- -replay.timeTo=2026-08-04T00:00:00Z
- -replay.maxDatapointsPerQuery=4
- -replay.rulesDelay=2s
- -replay.disableProgressBar
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: "1"
memory: 512Mi
volumeMounts:
- name: rules
mountPath: /etc/vmalert/backfill
readOnly: true
volumes:
- name: rules
configMap:
name: atlas-availability-request-v4-daily-backfill-rules

47
services/monitoring/availability-legacy-cleanup-job.yaml Normal file
View File

@ -0,0 +1,47 @@
# services/monitoring/availability-legacy-cleanup-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: atlas-availability-legacy-series-cleanup-v1
namespace: monitoring
spec:
activeDeadlineSeconds: 300
backoffLimit: 2
template:
metadata:
labels:
app: atlas-availability-legacy-series-cleanup
spec:
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-22
- titan-24
containers:
- name: cleanup
image: python:3.12-alpine
command: ["python", "/scripts/availability_cleanup.py"]
env:
- name: VM_URL
value: http://victoria-metrics-single-server:8428
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: script
mountPath: /scripts
readOnly: true
volumes:
- name: script
configMap:
name: atlas-availability-cleanup-script

103
services/monitoring/availability-rollup-cronjob.yaml Normal file
View File

@ -0,0 +1,103 @@
# services/monitoring/availability-rollup-cronjob.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: atlas-availability-request-v4-rollup-initial
namespace: monitoring
spec:
activeDeadlineSeconds: 900
backoffLimit: 2
template:
metadata:
labels:
app: atlas-availability-request-v4-rollup
spec:
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-22
- titan-24
containers:
- name: rollup
image: python:3.12-alpine
command: ["python", "/scripts/availability_rollup.py"]
env:
- name: VM_URL
value: http://victoria-metrics-single-server:8428
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: script
mountPath: /scripts
readOnly: true
volumes:
- name: script
configMap:
name: atlas-availability-rollup-script
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: atlas-availability-request-v4-rollup
namespace: monitoring
spec:
schedule: "10 0 * * *"
timeZone: Etc/UTC
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
jobTemplate:
spec:
activeDeadlineSeconds: 900
backoffLimit: 2
template:
metadata:
labels:
app: atlas-availability-request-v4-rollup
spec:
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-22
- titan-24
containers:
- name: rollup
image: python:3.12-alpine
command: ["python", "/scripts/availability_rollup.py"]
env:
- name: VM_URL
value: http://victoria-metrics-single-server:8428
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: script
mountPath: /scripts
readOnly: true
volumes:
- name: script
configMap:
name: atlas-availability-rollup-script

12
services/monitoring/dashboards/atlas-gpu.json
View File

File diff suppressed because one or more lines are too long

2
services/monitoring/dashboards/atlas-network.json
View File

@ -476,7 +476,7 @@
},
"targets": [
{
"expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"refId": "A",
"legendFormat": "{{node}}"
}

16
services/monitoring/dashboards/atlas-nodes.json
View File

@ -20,7 +20,7 @@
},
"targets": [
{
"expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})",
"expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})",
"refId": "A"
}
],
@ -46,7 +46,7 @@
"unit": "none",
"custom": {
"displayMode": "auto",
"valueSuffix": "/21"
"valueSuffix": "/18"
}
},
"overrides": []
@ -409,7 +409,7 @@
},
"targets": [
{
"expr": "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"expr": "avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"refId": "A",
"legendFormat": "{{node}}"
}
@ -449,7 +449,7 @@
},
"targets": [
{
"expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"refId": "A",
"legendFormat": "{{node}}"
}
@ -489,7 +489,7 @@
},
"targets": [
{
"expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
"expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
"refId": "A",
"legendFormat": "{{node}}"
}
@ -526,7 +526,7 @@
},
"targets": [
{
"expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
"expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
"refId": "A",
"legendFormat": "{{node}}"
}
@ -563,7 +563,7 @@
},
"targets": [
{
"expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"refId": "A",
"legendFormat": "{{node}}"
}
@ -601,7 +601,7 @@
},
"targets": [
{
"expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
"refId": "A",
"legendFormat": "{{node}}"
}

48
services/monitoring/dashboards/atlas-overview.json
View File

File diff suppressed because one or more lines are too long

6
services/monitoring/dashboards/atlas-pods.json
View File

@ -20,7 +20,7 @@
},
"targets": [
{
"expr": "sum(max by (namespace,pod) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})) or on() vector(0)",
"expr": "((sum(max by(namespace,pod) ((kube_pod_status_phase{phase=\"Pending\",namespace!~\"veles\"} == 1) and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)) + (sum(max by(namespace,pod) ((kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace!~\"veles\"} == 1) unless on(namespace,pod) kube_pod_owner{owner_kind=\"Job\"})) or on() vector(0)))",
"refId": "A"
}
],
@ -80,7 +80,7 @@
},
"targets": [
{
"expr": "sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})) or on() vector(0)",
"expr": "sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason{namespace!~\"veles\",reason=~\"CrashLoopBackOff|ImagePullBackOff\"} and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)",
"refId": "A"
}
],
@ -523,7 +523,7 @@
},
"targets": [
{
"expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)))))",
"expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)))))",
"refId": "A",
"instant": true,
"format": "table"

4
services/monitoring/dashboards/atlas-storage.json
View File

@ -276,7 +276,7 @@
},
"targets": [
{
"expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
"expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
"refId": "A",
"legendFormat": "{{node}}"
}
@ -314,7 +314,7 @@
},
"targets": [
{
"expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
"expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")",
"refId": "A",
"legendFormat": "{{node}}"
}

20
services/monitoring/dcgm-exporter.yaml
View File

@ -53,27 +53,15 @@ spec:
ports:
- name: metrics
containerPort: 9400
env:
- name: DCGM_EXPORTER_KUBERNETES
value: "true"
- name: KUBERNETES_VIRTUAL_GPUS
value: "true"
- name: NVIDIA_RESOURCE_NAMES
value: nvidia.com/gpu.shared
securityContext:
privileged: true
resources:
requests:
cpu: 50m
memory: 64Mi
volumeMounts:
- name: pod-resources
mountPath: /var/lib/kubelet/pod-resources
volumes:
- name: pod-resources
hostPath:
path: /var/lib/kubelet/pod-resources
type: Directory
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
---
apiVersion: v1
kind: Service

Some files were not shown because too many files have changed in this diff Show More