diff --git a/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml new file mode 100644 index 00000000..38b6d761 --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml @@ -0,0 +1,28 @@ +# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-chat + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-chat + targetNamespace: hermes-chat + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 30m + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-chat + namespace: hermes-chat + dependsOn: + - name: cert-manager + - name: core + - name: hermes + - name: keycloak + - name: longhorn diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 6a16b1b5..47cca710 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -18,6 +18,10 @@ spec: wait: true timeout: 45m healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-model-gate + namespace: hermes - apiVersion: apps/v1 kind: Deployment name: hermes-ollama @@ -26,6 +30,10 @@ spec: kind: Deployment name: hermes namespace: hermes + - apiVersion: apps/v1 + kind: Deployment + name: oauth2-proxy-hermes + namespace: hermes dependsOn: - name: cert-manager - name: core diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index b4b3a034..671f019b 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -28,9 +28,11 @@ resources: - ai-llm/kustomization.yaml - openclaw/kustomization.yaml - hermes/kustomization.yaml + - hermes-chat/kustomization.yaml - game-stream/kustomization.yaml - cassandra-auth/kustomization.yaml - cassandra/kustomization.yaml + - cassandra/image-automation.yaml - veles/kustomization.yaml - typhon/kustomization.yaml - nextcloud/kustomization.yaml diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent new file mode 100644 index 00000000..611bcbb5 --- /dev/null +++ b/dockerfiles/Dockerfile.hermes-agent @@ -0,0 +1,122 @@ +# syntax=docker/dockerfile:1 +# dockerfiles/Dockerfile.hermes-agent +FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 + +USER root + +# Keep dashboard chat sockets tied to the intended React mount and conversation. +# A resumed conversation needs a different PTY attachment key from a fresh chat; +# reconnects to that same conversation must keep using the same key. +RUN node <<'NODE' +const fs = require("node:fs"); +const path = "/opt/hermes/web/src/pages/ChatPage.tsx"; +let source = fs.readFileSync(path, "utf8"); +const socketBefore = [ + ' const url = await api.buildWsUrl("/api/pty", params);', + ' const ws = new WebSocket(url);', +].join("\n"); +const socketAfter = [ + ' const url = await api.buildWsUrl("/api/pty", params);', + ' if (unmounting) return;', + ' const ws = new WebSocket(url);', +].join("\n"); +const attachBefore = ' params.attach = ptyAttachToken(forceFresh);'; +const attachAfter = [ + ' const attachScope = resumeParam', + ' ? `resume:${resumeParam}:${scopedProfile ?? ""}`', + ' : `fresh:${scopedProfile ?? ""}`;', + ' params.attach = `${ptyAttachToken(forceFresh)}:${attachScope}`;', +].join("\n"); + +if (!source.includes(socketBefore)) { + throw new Error("Hermes ChatPage WebSocket patch context changed"); +} +if (!source.includes(attachBefore)) { + throw new Error("Hermes ChatPage PTY attachment patch context changed"); +} +source = source.replace(socketBefore, socketAfter); +source = source.replace(attachBefore, attachAfter); +fs.writeFileSync(path, source); +NODE + +# The upstream OIDC gate authenticates users but deliberately treats the +# dashboard as one shared workstation. Allow a deployment to narrow that +# workstation to explicit OIDC subjects. Enforce this after normal provider +# verification so a denied account is a 403, not a misleading provider 503. +RUN python - <<'PY' +from pathlib import Path + +path = Path("/opt/hermes/hermes_cli/dashboard_auth/middleware.py") +source = path.read_text() +helper_before = '''def _client_ip(request: Request) -> str: + fwd = request.headers.get("x-forwarded-for", "") + if fwd: + return fwd.split(",")[0].strip() + return request.client.host if request.client else "" + + +''' +helper_after = helper_before + '''def _dashboard_user_allowed(session) -> bool: + """Apply an optional deployment-level OIDC-subject allowlist.""" + import os + + allowed = { + value.strip() + for value in os.environ.get( + "HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS", "" + ).split(",") + if value.strip() + } + return not allowed or session.user_id in allowed + + +def _user_forbidden_response() -> Response: + """Return an authorization failure without exposing identities.""" + return JSONResponse( + { + "error": "forbidden", + "detail": "This Atlas account is not authorized for this dashboard.", + }, + status_code=403, + ) + + +''' +refresh_before = ''' new_session, refreshing_provider = refreshed + request.state.session = new_session + response = await call_next(request) +''' +refresh_after = ''' new_session, refreshing_provider = refreshed + if not _dashboard_user_allowed(new_session): + return _user_forbidden_response() + request.state.session = new_session + response = await call_next(request) +''' +final_before = ''' request.state.session = session + return await call_next(request) +''' +final_after = ''' if not _dashboard_user_allowed(session): + return _user_forbidden_response() + request.state.session = session + return await call_next(request) +''' +for before, after, label in ( + (helper_before, helper_after, "allowlist helper"), + (refresh_before, refresh_after, "refreshed session"), + (final_before, final_after, "verified session"), +): + if before not in source: + raise SystemExit(f"Hermes dashboard auth {label} patch context changed") + source = source.replace(before, after, 1) +path.write_text(source) +PY + +COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate + +RUN cd /opt/hermes/web \ + && npm run build \ + && grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \ + && grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \ + && grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \ + /opt/hermes/hermes_cli/dashboard_auth/middleware.py \ + && chmod 0755 /opt/hermes/bin/hermes-session-migrate diff --git a/dockerfiles/hermes-session-migrate.py b/dockerfiles/hermes-session-migrate.py new file mode 100644 index 00000000..2ba69b32 --- /dev/null +++ b/dockerfiles/hermes-session-migrate.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Copy selected legacy Hermes sessions into an isolated user home.""" + +from __future__ import annotations + +import os +from pathlib import Path +import shutil +import sqlite3 + +from hermes_state import SessionDB + + +def _copy_if_missing(source: Path, target: Path) -> None: + """Copy one credential/config file without overwriting user state.""" + if source.is_file() and not target.exists(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def _copy_session( + source: sqlite3.Connection, + target: sqlite3.Connection, + session_id: str, + user_id: str, +) -> bool: + """Copy a session and its messages, preserving original timestamps.""" + source.row_factory = sqlite3.Row + session = source.execute( + "SELECT * FROM sessions WHERE id = ?", (session_id,) + ).fetchone() + if session is None: + return False + + columns = list(session.keys()) + values = [user_id if column == "user_id" else session[column] for column in columns] + placeholders = ", ".join("?" for _ in columns) + target.execute( + f"INSERT OR IGNORE INTO sessions ({', '.join(columns)}) " + f"VALUES ({placeholders})", + values, + ) + + message_columns = [ + row[1] for row in target.execute("PRAGMA table_info(messages)").fetchall() + ] + selected_columns = [ + column + for column in message_columns + if column in { + row[1] + for row in source.execute("PRAGMA table_info(messages)").fetchall() + } + ] + column_sql = ", ".join(selected_columns) + target.execute( + f"INSERT OR IGNORE INTO messages ({column_sql}) " + f"SELECT {column_sql} FROM source_db.messages WHERE session_id = ?", + (session_id,), + ) + return True + + +def main() -> None: + """Create the isolated home and perform the idempotent session copy.""" + source_home = Path(os.environ["HERMES_MIGRATE_SOURCE_HOME"]) + target_home = Path(os.environ["HERMES_HOME"]) + user_id = os.environ["HERMES_MIGRATE_USER_ID"].strip() + session_ids = [ + value.strip() + for value in os.environ.get("HERMES_MIGRATE_SESSION_IDS", "").split(",") + if value.strip() + ] + + target_home.mkdir(parents=True, exist_ok=True) + (target_home / "home" / ".local" / "bin").mkdir(parents=True, exist_ok=True) + (target_home / "workspace" / "skills").mkdir(parents=True, exist_ok=True) + (target_home / "logs").mkdir(parents=True, exist_ok=True) + for filename in (".env", "auth.json"): + _copy_if_missing(source_home / filename, target_home / filename) + + source_db = source_home / "state.db" + target_db = target_home / "state.db" + SessionDB(db_path=target_db).close() + copied = 0 + if source_db.is_file() and session_ids: + source = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True) + target = sqlite3.connect(target_db) + target.execute("ATTACH DATABASE ? AS source_db", (str(source_db),)) + try: + with target: + for session_id in session_ids: + copied += int(_copy_session(source, target, session_id, user_id)) + finally: + target.close() + source.close() + print(f"isolated Hermes home ready; migrated_sessions={copied}") + + +if __name__ == "__main__": + main() diff --git a/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml b/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml index 0df5a6ad..4d64d9a0 100644 --- a/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml +++ b/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: longhorn-disk-tags-ensure-3 + name: longhorn-disk-tags-ensure-4 namespace: longhorn-system spec: backoffLimit: 0 diff --git a/infrastructure/traefik/deployment.yaml b/infrastructure/traefik/deployment.yaml index 19d42465..07637544 100644 --- a/infrastructure/traefik/deployment.yaml +++ b/infrastructure/traefik/deployment.yaml @@ -4,8 +4,6 @@ items: - apiVersion: apps/v1 kind: Deployment metadata: - annotations: - deployment.kubernetes.io/revision: "4" name: traefik namespace: traefik spec: @@ -73,6 +71,14 @@ items: node-role.kubernetes.io/worker: "true" affinity: nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - rpi4 preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: @@ -103,6 +109,12 @@ items: operator: In values: - rpi4 + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: traefik + topologyKey: kubernetes.io/hostname restartPolicy: Always schedulerName: default-scheduler serviceAccount: atlas-traefik-ingress-controller diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index 67c6f7f0..4be4f6b0 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -2,8 +2,8 @@ """Generate Atlas Grafana dashboards and render them into ConfigMaps. Usage: - scripts/render/dashboards_render_atlas.py --build # rebuild JSON + ConfigMaps - scripts/render/dashboards_render_atlas.py # re-render ConfigMaps from JSON + scripts/dashboards_render_atlas.py --build # rebuild JSON + ConfigMaps + scripts/dashboards_render_atlas.py # re-render ConfigMaps from JSON """ import argparse @@ -16,14 +16,7 @@ from pathlib import Path # Paths, folders, and shared metadata # --------------------------------------------------------------------------- -def repo_root() -> Path: - for path in Path(__file__).resolve().parents: - if (path / "clusters").is_dir() and (path / "services").is_dir(): - return path - raise RuntimeError("could not locate repository root") # pragma: no cover - - -ROOT = repo_root() +ROOT = Path(__file__).resolve().parents[1] DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards" CONFIG_TEMPLATE = textwrap.dedent( """# {relative_path} @@ -117,8 +110,6 @@ WORKER_NODES = [ "titan-06", "titan-07", "titan-08", - "titan-09", - "titan-10", "titan-11", "titan-20", "titan-21", @@ -126,7 +117,6 @@ WORKER_NODES = [ "titan-13", "titan-14", "titan-15", - "titan-16", "titan-17", "titan-18", "titan-19", @@ -191,7 +181,8 @@ def scoped_node_expr(base, scope=""): def node_cpu_expr(scope=""): idle = 'avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))' - base = f"(1 - {idle}) * 100" + # Scrape stalls can briefly report an impossible idle rate after recovery. + base = f"clamp_max(clamp_min((1 - {idle}) * 100, 0), 100)" return scoped_node_expr(base, scope) @@ -291,8 +282,14 @@ def dcgm_gpu_util_by_node(): ) +def nvidia_gpu_util_by_node(): + return "max by (node) (nvidia_gpu_device_utilization_percent)" + + def gpu_util_by_node(): - return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}" + process_exporter = nvidia_gpu_util_by_node() + dcgm_fallback = f"({dcgm_gpu_util_by_node()}) unless on(node) ({process_exporter})" + return f"{process_exporter} or {dcgm_fallback} or {jetson_gpu_util_by_node()}" def gpu_util_by_hostname(): @@ -302,16 +299,11 @@ def gpu_util_by_hostname(): GPU_RESOURCE_REGEX = "nvidia(_com_|[.]com/)gpu.*" -def gpu_node_labels(): - return f'max by (node) (kube_node_status_allocatable{{resource=~"{GPU_RESOURCE_REGEX}"}} > bool 0)' - - def gpu_requests_by_namespace_node(scope_var): return ( "sum by (namespace,node) (" f'kube_pod_container_resource_requests{{resource=~"{GPU_RESOURCE_REGEX}",{scope_var}}} ' "* on(namespace,pod) group_left(node) kube_pod_info " - f"* on(node) group_left() ({gpu_node_labels()})" ")" ) @@ -333,7 +325,8 @@ def gpu_usage_by_namespace(scope_var): def jetson_gpu_usage_by_namespace(scope_var): requests_by_ns = gpu_requests_by_namespace_node(scope_var) - total_by_node = f"sum by (node) ({requests_by_ns})" + all_requests = gpu_requests_by_namespace_node('namespace=~".*"') + total_by_node = f"sum by (node) ({all_requests})" return ( "sum by (namespace) (" f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) " @@ -342,6 +335,12 @@ def jetson_gpu_usage_by_namespace(scope_var): ) +def jetson_gpu_requested_nodes(): + all_requests = gpu_requests_by_namespace_node('namespace=~".*"') + requested = f"(sum by (node) ({all_requests}) > 0)" + return f"({requested}) and on(node) ({jetson_gpu_util_by_node()})" + + def namespace_share_expr(resource_expr): total = f"clamp_min(sum( {resource_expr} ), 1)" return f"100 * ( {resource_expr} ) / {total}" @@ -391,14 +390,29 @@ def gpu_total_devices_expr(): def unattributed_gpu_usage(): - legacy_total = f"(sum({legacy_gpu_util_without_process_exporter()}) or on() vector(0))" + unresolved = ( + f"({legacy_gpu_util_without_process_exporter()}) " + f"unless on(node) ({jetson_gpu_requested_nodes()})" + ) + legacy_total = f"(sum({unresolved}) or on() vector(0))" return ( f'label_replace(({legacy_total} > 0), "namespace", "unattributed", "", "")' ) def gpu_utilization_raw(scope_var): - return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})" + nvidia = ( + 'label_replace(' + f'{nvidia_process_gpu_usage_by_namespace(scope_var)}, ' + '"gpu_source", "nvidia", "", "")' + ) + jetson = ( + 'label_replace(' + f'(({jetson_gpu_usage_by_namespace(scope_var)}) > 0), ' + '"gpu_source", "jetson", "", "")' + ) + attributed = f"sum by (namespace) (({nvidia}) or ({jetson}))" + return f"({attributed}) or ({unattributed_gpu_usage()})" def gpu_pool_used_expr(scope_var): @@ -418,13 +432,20 @@ def namespace_gpu_share_expr(scope_var): PROBLEM_PODS_EXPR = ( - 'sum(max by (namespace,pod) (kube_pod_status_phase{phase!~"Running|Succeeded"})) ' - "or on() vector(0)" + '((sum(max by(namespace,pod) (' + '(kube_pod_status_phase{phase="Pending",namespace!~"veles"} == 1) ' + 'and on(namespace,pod) ((time() - kube_pod_created{namespace!~"veles"}) > 900)' + ')) or on() vector(0)) + (sum(max by(namespace,pod) (' + '(kube_pod_status_phase{phase=~"Failed|Unknown",namespace!~"veles"} == 1) ' + 'unless on(namespace,pod) kube_pod_owner{owner_kind="Job"}' + ')) or on() vector(0)))' ) CRASHLOOP_EXPR = ( - 'sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason' - '{reason=~"CrashLoopBackOff|ImagePullBackOff"})) ' - "or on() vector(0)" + 'sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason' + '{namespace!~"veles",reason=~"CrashLoopBackOff|ImagePullBackOff"} ' + 'and on(namespace,pod) ' + '((time() - kube_pod_created{namespace!~"veles"}) > 900))) ' + 'or on() vector(0)' ) STUCK_TERMINATING_EXPR = ( 'sum(max by (namespace,pod) (' @@ -436,23 +457,22 @@ STUCK_TERMINATING_EXPR = ( UPTIME_WINDOW = "365d" # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series. UPTIME_RECORDING_METRIC = ( - f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}' + f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="request-v4"}}' ) -TRAEFIK_READY_EXPR = ( - "(" - 'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})' - " > bool 0)" +AVAILABILITY_REQUESTS_1H_EXPR = ( + 'sum(increase(traefik_entrypoint_requests_total{' + 'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))' ) -CONTROL_READY_FRACTION_EXPR = ( - f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})" - " >= bool 2)" +AVAILABILITY_FAILURES_1H_EXPR = ( + 'sum(increase(traefik_entrypoint_requests_total{' + 'entrypoint="websecure",protocol="http",code=~"5.."}[1h]))' ) -UPTIME_AVAIL_EXPR = ( - f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))" +UPTIME_LIVE_FALLBACK_EXPR = ( + f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / " + f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))" ) -UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])" UPTIME_RECORDING_EXPR = ( - f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) " + f"(last_over_time({UPTIME_RECORDING_METRIC}[48h]) " f"or on() {UPTIME_LIVE_FALLBACK_EXPR})" ) @@ -631,7 +651,7 @@ PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX = ( "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube" ) PLATFORM_TEST_STANDARD_CHECK_REGEX = ( - "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|semgrep|supply_chain" + "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|supply_chain" ) PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_SUITES = [ "ariadne", @@ -1881,9 +1901,9 @@ OVERVIEW_PANEL_DESCRIPTIONS = { "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.", "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.", "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.", - "Atlas Serving Availability": "Observed availability with control-plane quorum and at least one Traefik replica serving; partial replica capacity remains available.", - "Problem Pods": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.", - "CrashLoop / ImagePull": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.", + "Atlas Availability (365d)": "Request-weighted Atlas ingress availability; every server-side 5xx response counts as a failed request.", + "Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.", + "CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.", "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.", "Hottest node: CPU": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.", "Hottest node: RAM": "Highest worker memory use right now; lower is safer, high values risk evictions.", @@ -1921,7 +1941,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = { "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.", "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.", "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.", - "Namespace GPU Utilization": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.", + "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.", "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.", "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.", "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.", @@ -1963,7 +1983,6 @@ TESTING_PANEL_DESCRIPTIONS = { "Style Failure Rate": "Percent of style checks currently failing; blue means style/docs gates pass.", "Gate Glue Failure Rate": "Percent of metric-contract checks failing; blue means dashboard telemetry is trustworthy.", "SonarQube Failure Rate": "Percent of Sonar checks failing; blue means Sonar quality gates pass.", - "Semgrep Failure Rate": "Percent of Semgrep checks failing; blue means SAST rule scans pass.", "Supply Chain Failure Rate": "Percent of supply-chain checks failing; blue means artifact/image checks pass.", "Check Healthy Rates By Suite": "Healthy percent by check family; blue means all selected checks are good.", "Tests Healthy Rate": "Percent of test checks passing or not applicable; higher is better.", @@ -1972,7 +1991,6 @@ TESTING_PANEL_DESCRIPTIONS = { "Style Healthy Rate": "Percent of style checks passing or not applicable; higher is better.", "Gate Glue Healthy Rate": "Percent of telemetry-contract checks passing; higher means cleaner reporting.", "SonarQube Healthy Rate": "Percent of Sonar checks passing or not applicable; higher is better.", - "Semgrep Healthy Rate": "Percent of Semgrep checks passing or not applicable; higher is better.", "Supply Chain Healthy Rate": "Percent of supply-chain checks passing or not applicable; higher is better.", "Test Drilldowns And Problem Tests": "Test-case detail for finding which tests are hurting reliability.", "Problematic Tests Over Time (Top failures)": "Current outlier tests by rolling 24h failures; tests need repeat failures to stay visible.", @@ -1989,13 +2007,6 @@ TESTING_PANEL_DESCRIPTIONS = { "Recent Branch Evidence by Suite (7d)": "Branches with recent CI evidence; unexpected branches can mean drift or stale work.", "Primary Branch Clean by Suite (7d)": "Percent clean of non-primary branch evidence; 100% means only main/master is reporting.", "SonarQube Project Health": "SonarQube availability, projects, fetch errors, and gate status.", - "Public ZAP Baseline": "OWASP ZAP passive baseline results for externally addressed Atlas domains.", - "ZAP Targets Scanned": "Number of public hostnames with a latest OWASP ZAP baseline scan sample.", - "ZAP High/Medium Alerts": "Current high and medium ZAP alert instance count across public targets.", - "ZAP Scan Errors": "Targets whose latest ZAP baseline failed or did not produce a usable report.", - "ZAP Alerts by Risk": "Current ZAP alert distribution by risk level.", - "ZAP Target Health": "Latest ZAP health per hostname from passive baseline scans.", - "ZAP Alerts by Host": "Current high, medium, and low ZAP alert counts by public hostname.", "SonarQube API Up": "Whether the SonarQube exporter can reach SonarQube; 1 is good.", "Sonar Projects (Selected)": "Selected SonarQube project count; zero means Sonar is not tracking that suite.", "Sonar Gate Fetch Errors": "Sonar exporter fetch errors; zero is good because stale Sonar data misleads.", @@ -2123,7 +2134,7 @@ def build_overview(): }, { "id": 27, - "title": "Atlas Serving Availability", + "title": "Atlas Availability (365d)", "expr": UPTIME_PERCENT_EXPR, "kind": "stat", "thresholds": UPTIME_PERCENT_THRESHOLDS, @@ -2131,7 +2142,7 @@ def build_overview(): "decimals": 4, "text_mode": "value", "instant": True, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.", + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists.", }, { "id": 4, @@ -4183,7 +4194,6 @@ def build_jobs_dashboard(): check_regex_style = "docs|naming|hygiene|lint|docs_naming|style" check_regex_gate_glue = "gate|glue|gate_glue" check_regex_sonarqube = "sonarqube|sonar" - check_regex_semgrep = "semgrep|sast" check_regex_supply_chain = "ironbank|supply_chain|image_compliance|artifact_security" def _check_state_percent_series(regex: str, failed: bool) -> str: @@ -4227,30 +4237,6 @@ def build_jobs_dashboard(): f"topk by (suite) (1, ({rollup_failed_tests_history}))" ) worst_test_per_suite = worst_test_per_suite_core - zap_target_health = ( - "(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) " - "or on() vector(0))" - ) - zap_targets_scanned = ( - '(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job="platform-security-zap"})) ' - "or on() vector(0))" - ) - zap_high_medium_alerts = ( - '(sum(platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk=~"high|medium"}) ' - "or on() vector(0))" - ) - zap_scan_errors = ( - '(sum(platform_zap_baseline_scan_status{exported_job="platform-security-zap",status=~"fail|error"}) ' - "or on() vector(0))" - ) - zap_alerts_by_risk = ( - 'sum by (risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk!=""}) ' - "or on() vector(0)" - ) - zap_alerts_by_host_risk = ( - 'sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",' - 'risk=~"high|medium|low"}) or on() vector(0)' - ) def _selected_status_volume(status: str) -> str: return ( @@ -4596,7 +4582,6 @@ def build_jobs_dashboard(): ("Style", check_regex_style), ("Gate Glue", check_regex_gate_glue), ("SonarQube", check_regex_sonarqube), - ("Semgrep", check_regex_semgrep), ("Supply Chain", check_regex_supply_chain), ] @@ -4620,12 +4605,12 @@ def build_jobs_dashboard(): ) panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW panels.append(panel) - for index, (label, regex) in enumerate(check_dimensions[4:8]): + for index, (label, regex) in enumerate(check_dimensions[4:]): panel = state_timeline_panel( start_id + 4 + index, f"{label} {title_prefix}", _check_state_percent_series(regex, failed), - {"h": 7, "w": 6, "x": index * 6, "y": y + 7}, + {"h": 7, "w": 8, "x": index * 8, "y": y + 7}, thresholds=trend_thresholds, description=trend_description, ) @@ -4633,7 +4618,7 @@ def build_jobs_dashboard(): panels.append(panel) _append_check_trends(130, "Failure Rate", True, 29) - _append_check_trends(160, "Healthy Rate", False, 43) + _append_check_trends(138, "Healthy Rate", False, 43) panels.append( state_timeline_panel( 145, @@ -4889,79 +4874,6 @@ def build_jobs_dashboard(): ) sonar_gate_project_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW panels.append(sonar_gate_project_panel) - panels.append( - stat_panel( - 168, - "ZAP Targets Scanned", - zap_targets_scanned, - {"h": 6, "w": 4, "x": 0, "y": 117}, - unit="none", - instant=True, - thresholds=missing_thresholds, - ) - ) - panels.append( - stat_panel( - 169, - "ZAP High/Medium Alerts", - zap_high_medium_alerts, - {"h": 6, "w": 4, "x": 4, "y": 117}, - unit="none", - instant=True, - thresholds=failures_thresholds, - ) - ) - panels.append( - stat_panel( - 170, - "ZAP Scan Errors", - zap_scan_errors, - {"h": 6, "w": 4, "x": 8, "y": 117}, - unit="none", - instant=True, - thresholds=failures_thresholds, - ) - ) - zap_risk_panel = pie_panel( - 171, - "ZAP Alerts by Risk", - zap_alerts_by_risk, - {"h": 6, "w": 4, "x": 12, "y": 117}, - ) - zap_risk_panel["targets"][0]["legendFormat"] = "{{risk}}" - panels.append(zap_risk_panel) - zap_target_panel = state_timeline_panel( - 172, - "ZAP Target Health", - zap_target_health, - {"h": 6, "w": 8, "x": 16, "y": 117}, - thresholds=success_thresholds, - unit="percent", - min_value=0, - max_value=100, - legend="{{host}}", - description=( - "Latest OWASP ZAP passive baseline health per public hostname. " - "High alerts and scan errors drop a target to zero; medium/low alerts mark warning health." - ), - ) - zap_target_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW - panels.append(zap_target_panel) - panels.append( - bargauge_panel( - 173, - "ZAP Alerts by Host", - zap_alerts_by_host_risk, - {"h": 7, "w": 24, "x": 0, "y": 123}, - unit="none", - instant=True, - legend="{{host}} ยท {{risk}}", - sort_order="desc", - thresholds=failures_thresholds, - decimals=0, - limit=20, - ) - ) panels.append( bargauge_panel( 148, @@ -5071,12 +4983,6 @@ def build_jobs_dashboard(): 33: {"h": 6, "w": 4, "x": 8, "y": 111}, 34: {"h": 6, "w": 4, "x": 12, "y": 111}, 35: {"h": 6, "w": 8, "x": 16, "y": 111}, - 168: {"h": 6, "w": 4, "x": 0, "y": 117}, - 169: {"h": 6, "w": 4, "x": 4, "y": 117}, - 170: {"h": 6, "w": 4, "x": 8, "y": 117}, - 171: {"h": 6, "w": 4, "x": 12, "y": 117}, - 172: {"h": 6, "w": 8, "x": 16, "y": 117}, - 173: {"h": 7, "w": 24, "x": 0, "y": 123}, } for panel_id, grid in row_layout.items(): panel_by_id[panel_id]["gridPos"] = grid @@ -5088,13 +4994,13 @@ def build_jobs_dashboard(): 501, "Check Failure Rates By Suite", 12, - panels=children([130, 131, 132, 133, 134, 135, 136, 137]), + panels=children([130, 131, 132, 133, 134, 135, 136]), ), row_panel( 502, "Check Healthy Rates By Suite", 13, - panels=children([160, 161, 162, 163, 164, 165, 166, 167]), + panels=children([138, 139, 140, 141, 142, 143, 144]), ), row_panel( 503, @@ -5114,12 +5020,6 @@ def build_jobs_dashboard(): 16, panels=children([31, 32, 33, 34, 35]), ), - row_panel( - 506, - "Public ZAP Baseline", - 17, - panels=children([168, 169, 170, 171, 172, 173]), - ), ] ) panels = compact_panels @@ -5621,12 +5521,13 @@ def build_gpu_dashboard(): panels.append( table_panel( 4, - "GPU Pods Reporting Device Util", - 'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))', + "GPU Processes by Pod", + 'topk(10, sum by (namespace,pod,node,process) ' + '(nvidia_process_gpu_sm_util_percent{pod!="host"}) > 0)', {"h": 8, "w": 12, "x": 12, "y": 8}, unit="percent", transformations=[{"id": "labelsToFields", "options": {}}], - description="DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.", + description="NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization.", ) ) return { diff --git a/scripts/tests/test_availability_cleanup.py b/scripts/tests/test_availability_cleanup.py new file mode 100644 index 00000000..c907b60f --- /dev/null +++ b/scripts/tests/test_availability_cleanup.py @@ -0,0 +1,28 @@ +"""Safety tests for the legacy Atlas availability cleanup.""" + +import importlib.util +from pathlib import Path + + +def load_module(): + """Load the service-owned cleanup module without packaging it.""" + path = ( + Path(__file__).resolve().parents[2] + / "services/monitoring/scripts/availability_cleanup.py" + ) + spec = importlib.util.spec_from_file_location("availability_cleanup", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_cleanup_selector_cannot_match_request_v4() -> None: + """Constrain deletion to obsolete annual Atlas definitions.""" + mod = load_module() + + assert '__name__="atlas:availability:ratio_365d"' in mod.LEGACY_SELECTOR + assert 'scope="atlas"' in mod.LEGACY_SELECTOR + assert 'definition!="request-v4"' in mod.LEGACY_SELECTOR + assert 'definition="request-v4"' in mod.PROTECTED_SELECTOR + assert "ratio_1h" not in mod.LEGACY_SELECTOR diff --git a/scripts/tests/test_availability_rollup.py b/scripts/tests/test_availability_rollup.py new file mode 100644 index 00000000..917196ce --- /dev/null +++ b/scripts/tests/test_availability_rollup.py @@ -0,0 +1,52 @@ +"""Unit tests for the Atlas availability publisher.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + + +def load_module(): + """Load the service-owned rollup module without packaging it.""" + path = ( + Path(__file__).resolve().parents[2] + / "services/monitoring/scripts/availability_rollup.py" + ) + spec = importlib.util.spec_from_file_location("availability_rollup", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_parse_export_deduplicates_replay_boundaries() -> None: + """Keep only the final value when replay chunks share a timestamp.""" + mod = load_module() + lines = [ + (json.dumps({"timestamps": [1000, 2000], "values": [2, 3]}) + "\n").encode(), + (json.dumps({"timestamps": [2000, 3000], "values": [3, 5]}) + "\n").encode(), + ] + + assert mod.parse_export(lines) == {1000: 2.0, 2000: 3.0, 3000: 5.0} + + +def test_calculate_availability_uses_all_server_failures() -> None: + """Calculate one bounded successful-request ratio.""" + mod = load_module() + + assert mod.calculate_availability(1000, 2) == pytest.approx(0.998) + with pytest.raises(ValueError): + mod.calculate_availability(0, 0) + with pytest.raises(ValueError): + mod.calculate_availability(100, -1) + + +def test_render_metric_publishes_only_the_request_v4_series() -> None: + """Render the single series selected by the Grafana panel.""" + mod = load_module() + + assert mod.render_metric(0.9995, 1234) == ( + 'atlas:availability:ratio_365d{definition="request-v4",scope="atlas",' + 'rollup="yearly"} 0.999500000000 1234\n' + ) diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 1e3b09c1..287c5e41 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -3,10 +3,8 @@ import pathlib def load_module(): - path = pathlib.Path(__file__).resolve().parents[1] / "render/dashboards_render_atlas.py" - spec = importlib.util.spec_from_file_location( - "scripts.render.dashboards_render_atlas", path - ) + path = pathlib.Path(__file__).resolve().parents[1] / "render" / "dashboards_render_atlas.py" + spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path) module = importlib.util.module_from_spec(spec) assert spec.loader is not None spec.loader.exec_module(module) @@ -49,6 +47,8 @@ def test_node_filter_and_expr_helpers(): cpu_expr = mod.node_cpu_expr("titan-.*") mem_expr = mod.node_mem_expr("titan-.*") assert "node_cpu_seconds_total" in cpu_expr + assert "clamp_max(clamp_min(" in cpu_expr + assert "* 100, 0), 100)" in cpu_expr assert "node_memory_MemAvailable_bytes" in mem_expr @@ -57,20 +57,24 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): dashboard = mod.build_overview() panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27) - assert panel["title"] == "Atlas Serving Availability" + assert panel["title"] == "Atlas Availability (365d)" availability_expr = panel["targets"][0]["expr"] assert ( - 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])' + 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[48h])' in availability_expr ) - assert ">= bool 2" in availability_expr - assert "> bool 0" in availability_expr - assert "[1h:5m]" in availability_expr + assert 'code=~"5.."' in availability_expr + assert 'code=~"[1-5].."' in availability_expr + assert "atlas:availability:failures_1d" not in availability_expr + assert "atlas:availability:requests_1d" not in availability_expr + assert "sum_over_time" not in availability_expr + assert "kube_node_status_condition" not in availability_expr + assert "kube_deployment_status_replicas_available" not in availability_expr assert panel["targets"][0]["instant"] is True - assert "at least one Traefik replica serving" in panel["description"] - assert "partial-capacity samples remain available" in panel["description"] - assert "monitoring gaps are excluded" in panel["description"] - assert "falls back to the live binary serving state" in panel["description"] + assert "Every server-side 5xx" in panel["description"] + assert "Replica counts, Grafana health" in panel["description"] + assert "daily rollup job publishes one annual sample" in panel["description"] + assert "keeps it for up to 48 hours" in panel["description"] def test_overview_uses_readable_quality_power_and_gitops_panels(): @@ -169,14 +173,41 @@ def test_overview_uses_readable_quality_power_and_gitops_panels(): gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"] assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr assert "nvidia_gpu_device_utilization_percent" in gpu_expr + assert "sum_over_time" not in gpu_expr + assert "count_over_time" not in gpu_expr + assert "avg_over_time" not in gpu_expr + assert "$__range" not in gpu_expr assert "sum by (namespace)" in gpu_expr assert 'namespace", "shared"' not in gpu_expr - assert "kube_node_labels" not in gpu_expr + assert "kube_pod_container_resource_requests" in gpu_expr + assert mod.GPU_RESOURCE_REGEX in gpu_expr + assert '"gpu_source", "nvidia"' in gpu_expr + assert '"gpu_source", "jetson"' in gpu_expr assert "100 *" in gpu_expr assert "100 -" not in gpu_expr assert 'namespace", "unattributed"' in gpu_expr assert 'namespace", "idle"' in gpu_expr assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True + assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"] + assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"] + assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"] + + +def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families(): + mod = load_module() + dashboard = mod.build_gpu_dashboard() + panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])} + + node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"] + assert "nvidia_gpu_device_utilization_percent" in node_expr + assert "DCGM_FI_DEV_GPU_UTIL" in node_expr + assert "jetson_gr3d_freq_percent" in node_expr + assert "unless on(node)" in node_expr + assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"] + + process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"] + assert "nvidia_process_gpu_sm_util_percent" in process_expr + assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr def test_overview_and_testing_panels_all_have_concise_descriptions(): @@ -396,7 +427,7 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): for child in row.get("panels", []) } - assert len(panels) == 19 + assert len(panels) == 18 assert len(visible_query_panels) == 12 assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 assert all( @@ -410,18 +441,15 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): "Test Drilldowns And Problem Tests", "Telemetry Completeness And Branches", "SonarQube Project Health", - "Public ZAP Baseline", ] assert all(row["collapsed"] for row in rows) assert "Coverage Failure Rate" in nested_panels_by_title - assert "Semgrep Failure Rate" in nested_panels_by_title assert "Supply Chain Healthy Rate" in nested_panels_by_title assert "Test Category Health History" in nested_panels_by_title assert "Selected Test Pass Rate History" in nested_panels_by_title assert "Coverage Metrics Present by Suite" in nested_panels_by_title assert "SonarQube API Up" in nested_panels_by_title - assert "ZAP Target Health" in nested_panels_by_title failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] assert failure_rate_panel["type"] == "state-timeline" @@ -472,15 +500,6 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): assert sonar_health_panel["type"] == "state-timeline" assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"] assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"] - semgrep_panel = nested_panels_by_title["Semgrep Healthy Rate"] - assert semgrep_panel["type"] == "state-timeline" - assert 'check=~"semgrep|sast"' in semgrep_panel["targets"][0]["expr"] - - zap_health_panel = nested_panels_by_title["ZAP Target Health"] - assert zap_health_panel["type"] == "state-timeline" - assert "platform_zap_baseline_target_health_percent" in zap_health_panel["targets"][0]["expr"] - zap_alerts_panel = nested_panels_by_title["ZAP Alerts by Host"] - assert "platform_zap_baseline_alerts_total" in zap_alerts_panel["targets"][0]["expr"] branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] diff --git a/scripts/tests/test_monitoring_query_capacity.py b/scripts/tests/test_monitoring_query_capacity.py index 71467991..3f812684 100644 --- a/scripts/tests/test_monitoring_query_capacity.py +++ b/scripts/tests/test_monitoring_query_capacity.py @@ -44,43 +44,113 @@ def test_victoria_metrics_has_dashboard_burst_headroom() -> None: assert {"titan-14", "titan-18"} <= set(hostname_rule["values"]) -def test_yearly_availability_reuses_the_hourly_rollup() -> None: - """Prevent the yearly rule from rescanning raw cluster metrics for 365 days.""" +def test_yearly_availability_is_published_outside_the_query_pool() -> None: + """Keep all long-range availability work out of Grafana and MetricsQL.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] - rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0] - yearly = next( - rule - for rule in rules["rules"] - if rule["record"] == "atlas:availability:ratio_365d" + groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] + rules = [rule for group in groups for rule in group["rules"]] + manifests = _documents( + REPO_ROOT / "services/monitoring/availability-rollup-cronjob.yaml" ) + cronjob = next(manifest for manifest in manifests if manifest["kind"] == "CronJob") + source = ( + REPO_ROOT / "services/monitoring/scripts/availability_rollup.py" + ).read_text() - assert "atlas:availability:ratio_1h" in yearly["expr"] - assert 'definition="serving-v2"' in yearly["expr"] - assert 'definition=""' in yearly["expr"] - assert "share_gt_over_time" in yearly["expr"] - assert "[365d:15m]" in yearly["expr"] - assert "kube_node_status_condition" not in yearly["expr"] - assert "[365d:1h]" not in yearly["expr"] + assert all(rule["record"] != "atlas:availability:ratio_365d" for rule in rules) + assert "atlas:availability:requests_1d" in source + assert "atlas:availability:failures_1d" in source + assert "/api/v1/export" in source + assert "/api/v1/import/prometheus" in source + assert cronjob["spec"]["schedule"] == "10 0 * * *" + assert cronjob["spec"]["concurrencyPolicy"] == "Forbid" -def test_availability_counts_serving_state_instead_of_replica_capacity() -> None: - """Treat quorum and any serving ingress replica as available binary states.""" +def test_daily_availability_rollups_use_the_same_request_sli() -> None: + """Keep annual availability cheap without changing its request definition.""" manifest = _documents( REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" )[0] - rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0] - hourly = next( + groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] + daily_group = next( + group for group in groups if group["name"] == "atlas.availability.rollup" + ) + daily = {rule["record"]: rule for rule in daily_group["rules"]} + + assert daily_group["interval"] == "1d" + assert 'code=~"[1-5].."' in daily["atlas:availability:requests_1d"]["expr"] + assert 'code=~"5.."' in daily["atlas:availability:failures_1d"]["expr"] + assert all(rule["labels"]["definition"] == "request-v4" for rule in daily.values()) + + +def test_availability_uses_request_failures_instead_of_replica_capacity() -> None: + """Measure HTTP outcomes without treating redundant replica loss as downtime.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + )[0] + groups = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"] + rules = [rule for group in groups for rule in group["rules"]] + requests = next( rule - for rule in rules["rules"] - if rule["record"] == "atlas:availability:ratio_1h" + for rule in rules + if rule["record"] == "atlas:availability:requests_1h" + ) + failures = next( + rule + for rule in rules + if rule["record"] == "atlas:availability:failures_1h" ) - assert ">= bool 2" in hourly["expr"] - assert "> bool 0" in hourly["expr"] - assert "/ 3" not in hourly["expr"] - assert hourly["labels"]["definition"] == "serving-v2" + assert 'code=~"[1-5].."' in requests["expr"] + assert 'code=~"5.."' in failures["expr"] + assert "traefik_entrypoint_requests_total" in requests["expr"] + assert "traefik_entrypoint_requests_total" in failures["expr"] + assert "kube_node_status_condition" not in repr(rules) + assert "kube_deployment_status_replicas_available" not in repr(rules) + assert requests["labels"]["definition"] == "request-v4" + assert failures["labels"]["definition"] == "request-v4" + + +def test_availability_backfill_replays_the_same_request_sli() -> None: + """Backfill retained history with the exact rules used for future samples.""" + manifests = _documents( + REPO_ROOT / "services/monitoring/availability-backfill-v4-job.yaml" + ) + config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap") + job = next(manifest for manifest in manifests if manifest["kind"] == "Job") + backfill = yaml.safe_load(config["data"]["atlas-request-history.yaml"]) + expressions = { + rule["record"]: rule["expr"] + for group in backfill["groups"] + for rule in group["rules"] + } + + assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1h"] + assert 'code=~"5.."' in expressions["atlas:availability:failures_1h"] + args = job["spec"]["template"]["spec"]["containers"][0]["args"] + assert "-replay.timeFrom=2026-05-01T00:00:00Z" in args + assert "-replay.timeTo=2026-08-04T23:00:00Z" in args + assert "-replay.maxDatapointsPerQuery=48" in args + + +def test_daily_availability_backfill_stays_below_query_timeout() -> None: + """Replay daily buckets in small chunks so raw history never starves Grafana.""" + manifests = _documents( + REPO_ROOT / "services/monitoring/availability-daily-backfill-v4-job.yaml" + ) + config = next(manifest for manifest in manifests if manifest["kind"] == "ConfigMap") + job = next(manifest for manifest in manifests if manifest["kind"] == "Job") + backfill = yaml.safe_load(config["data"]["atlas-request-daily-history.yaml"]) + group = backfill["groups"][0] + expressions = {rule["record"]: rule["expr"] for rule in group["rules"]} + + assert group["interval"] == "1d" + assert 'code=~"[1-5].."' in expressions["atlas:availability:requests_1d"] + assert 'code=~"5.."' in expressions["atlas:availability:failures_1d"] + args = job["spec"]["template"]["spec"]["containers"][0]["args"] + assert "-replay.maxDatapointsPerQuery=4" in args def test_quality_rollups_do_not_run_every_minute() -> None: diff --git a/scripts/tests/test_nvidia_process_exporter.py b/scripts/tests/test_nvidia_process_exporter.py new file mode 100644 index 00000000..3a01ee91 --- /dev/null +++ b/scripts/tests/test_nvidia_process_exporter.py @@ -0,0 +1,99 @@ +import importlib.util +from pathlib import Path +import sys +import types + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +EXPORTER_PATH = ROOT / "services" / "monitoring" / "scripts" / "nvidia_process_exporter.py" + + +def load_exporter(monkeypatch): + """Load the exporter without requiring an NVIDIA driver on the test host.""" + + pynvml = types.ModuleType("pynvml") + + class NVMLError(Exception): + pass + + class NVMLErrorNotFound(NVMLError): + pass + + class NVMLErrorNotSupported(NVMLError): + pass + + pynvml.NVMLError = NVMLError + pynvml.NVMLError_NotFound = NVMLErrorNotFound + pynvml.NVMLError_NotSupported = NVMLErrorNotSupported + for name in ( + "nvmlDeviceGetComputeRunningProcesses_v3", + "nvmlDeviceGetCount", + "nvmlDeviceGetGraphicsRunningProcesses_v3", + "nvmlDeviceGetHandleByIndex", + "nvmlDeviceGetName", + "nvmlDeviceGetProcessUtilization", + "nvmlDeviceGetUUID", + "nvmlDeviceGetUtilizationRates", + "nvmlInit", + ): + setattr(pynvml, name, lambda *args, **kwargs: None) + + monkeypatch.setitem(sys.modules, "pynvml", pynvml) + spec = importlib.util.spec_from_file_location("nvidia_process_exporter_test", EXPORTER_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_process_sample_window_uses_nvml_microseconds(monkeypatch): + exporter = load_exporter(monkeypatch) + observed = [] + monkeypatch.setattr(exporter.time, "time", lambda: 1_700_000_000.0) + monkeypatch.setattr( + exporter, + "nvmlDeviceGetProcessUtilization", + lambda handle, since: observed.append(since) or [], + ) + + samples, supported = exporter.process_utilization_samples(object()) + + assert samples == {} + assert supported == 1 + assert observed == [1_700_000_000_000_000 - 30_000_000] + + +def test_namespace_attribution_scales_to_current_device_total(monkeypatch): + exporter = load_exporter(monkeypatch) + + result = exporter.reconcile_namespace_utilization( + {"game-stream": 20, "hermes": 10}, + device_util=3, + ) + + assert sum(result.values()) == pytest.approx(3) + assert result["game-stream"] == pytest.approx(2) + assert result["hermes"] == pytest.approx(1) + + +def test_namespace_attribution_assigns_unexplained_compute_to_host(monkeypatch): + exporter = load_exporter(monkeypatch) + + result = exporter.reconcile_namespace_utilization( + {"hermes": 1}, + device_util=3, + ) + + assert result == {"hermes": 1, "host": 2} + + +def test_zero_device_utilization_clears_stale_process_samples(monkeypatch): + exporter = load_exporter(monkeypatch) + + result = exporter.reconcile_namespace_utilization( + {"hermes": 40, "game-stream": 5}, + device_util=0, + ) + + assert result == {"hermes": 0, "game-stream": 0} diff --git a/services/bstein-dev-home/backend-deployment.yaml b/services/bstein-dev-home/backend-deployment.yaml index fbaa425e..2a39b781 100644 --- a/services/bstein-dev-home/backend-deployment.yaml +++ b/services/bstein-dev-home/backend-deployment.yaml @@ -68,7 +68,8 @@ spec: args: - >- . /vault/secrets/portal-env.sh - && exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 600 app:app + && export SMTP_HOST="" + && exec gunicorn -b 0.0.0.0:8080 --workers 2 --timeout 1200 app:app env: - name: AI_CHAT_API value: http://ollama.ai.svc.cluster.local:11434 @@ -119,6 +120,8 @@ spec: value: http://ariadne.maintenance.svc.cluster.local - name: ARIADNE_TIMEOUT_SEC value: "10" + - name: ARIADNE_GAME_MODE_TIMEOUT_SEC + value: "900" - name: ACCOUNT_ALLOWED_GROUPS value: "" - name: HTTP_CHECK_TIMEOUT_SEC @@ -139,6 +142,8 @@ spec: value: "10" - name: PORTAL_RUN_MIGRATIONS value: "false" + - name: ACCESS_REQUEST_ENABLED + value: "false" - name: ACCESS_REQUEST_SUBMIT_RATE_LIMIT value: "30" - name: ACCESS_REQUEST_SUBMIT_RATE_WINDOW_SEC diff --git a/services/bstein-dev-home/kustomization.yaml b/services/bstein-dev-home/kustomization.yaml index a765a5ff..c03ef3f8 100644 --- a/services/bstein-dev-home/kustomization.yaml +++ b/services/bstein-dev-home/kustomization.yaml @@ -20,9 +20,9 @@ resources: - ingress.yaml images: - name: registry.bstein.dev/bstein/bstein-dev-home-frontend - newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"} + newTag: 0.1.1-441 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"} - name: registry.bstein.dev/bstein/bstein-dev-home-backend - newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"} + newTag: 0.1.1-441 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"} configMapGenerator: - name: chat-ai-gateway namespace: bstein-dev-home diff --git a/services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml b/services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml new file mode 100644 index 00000000..c2896d7a --- /dev/null +++ b/services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml @@ -0,0 +1,397 @@ +# services/cassandra-auth/bootstrap-jobs/cassandra-ldap-federation-job.yaml +# One-off job for sso/cassandra-ldap-federation-3. +# Purpose: attach the Cassandra Keycloak realm to the shared OpenLDAP directory. +# Run unsuspended for migration, then suspend/replace with a new job name for future runs. +apiVersion: batch/v1 +kind: Job +metadata: + name: cassandra-ldap-federation-3 + namespace: sso +spec: + ttlSecondsAfterFinished: 3600 + suspend: true + backoffLimit: 2 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: "sso" + vault.hashicorp.com/agent-inject-secret-keycloak-env.sh: "kv/data/atlas/shared/keycloak-admin" + vault.hashicorp.com/agent-inject-template-keycloak-env.sh: | + {{ with secret "kv/data/atlas/shared/keycloak-admin" }} + export KEYCLOAK_ADMIN="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}" + {{ end }} + {{ with secret "kv/data/atlas/sso/openldap-admin" }} + export LDAP_ADMIN_PASSWORD="{{ .Data.data.LDAP_ADMIN_PASSWORD }}" + export LDAP_CONFIG_PASSWORD="{{ .Data.data.LDAP_CONFIG_PASSWORD }}" + export LDAP_BIND_PASSWORD="${LDAP_ADMIN_PASSWORD}" + {{ end }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: hardware + operator: In + values: ["rpi5","rpi4"] + - key: node-role.kubernetes.io/worker + operator: Exists + restartPolicy: OnFailure + serviceAccountName: sso-vault + containers: + - name: configure + image: python:3.11-alpine + imagePullPolicy: IfNotPresent + env: + - name: KEYCLOAK_SERVER + value: http://keycloak.sso.svc.cluster.local + - name: KEYCLOAK_REALM + value: cassandra + - name: LDAP_URL + value: ldap://openldap.sso.svc.cluster.local:389 + - name: LDAP_BIND_DN + value: cn=admin,dc=bstein,dc=dev + - name: LDAP_USERS_DN + value: ou=users,dc=bstein,dc=dev + - name: LDAP_GROUPS_DN + value: ou=groups,dc=bstein,dc=dev + - name: VERIFY_USERS + value: veles-dev,cassandra-dev,daniel-test,viktor-test + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + . /vault/secrets/keycloak-env.sh + python - <<'PY' + import json + import os + import time + import urllib.error + import urllib.parse + import urllib.request + + base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/") + realm = os.environ["KEYCLOAK_REALM"] + admin_user = os.environ["KEYCLOAK_ADMIN_USER"] + admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"] + ldap_url = os.environ["LDAP_URL"] + ldap_bind_dn = os.environ["LDAP_BIND_DN"] + ldap_bind_password = os.environ["LDAP_BIND_PASSWORD"] + ldap_users_dn = os.environ["LDAP_USERS_DN"] + ldap_groups_dn = os.environ["LDAP_GROUPS_DN"] + + def request(method, url, token=None, payload=None, timeout=30): + data = None + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + if payload is not None: + data = json.dumps(payload).encode() + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read() + if not body: + return resp.status, None, dict(resp.headers) + return resp.status, json.loads(body.decode()), dict(resp.headers) + except urllib.error.HTTPError as exc: + raw = exc.read() + body = None + if raw: + try: + body = json.loads(raw.decode()) + except Exception: + body = {"raw": raw.decode(errors="replace")} + return exc.code, body, dict(exc.headers) + + def get_token(): + data = urllib.parse.urlencode( + { + "grant_type": "password", + "client_id": "admin-cli", + "username": admin_user, + "password": admin_password, + } + ).encode() + req = urllib.request.Request( + f"{base_url}/realms/master/protocol/openid-connect/token", + data=data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode())["access_token"] + + token = None + for attempt in range(1, 31): + try: + token = get_token() + break + except Exception as exc: + if attempt == 30: + raise SystemExit(f"Keycloak token request failed: {exc}") + time.sleep(2) + assert token + + status, realm_rep, _ = request("GET", f"{base_url}/admin/realms/{realm}", token) + if status != 200 or not isinstance(realm_rep, dict) or not realm_rep.get("id"): + raise SystemExit(f"Unable to resolve realm id for {realm}: status={status}") + realm_id = realm_rep["id"] + + status, components, _ = request( + "GET", + f"{base_url}/admin/realms/{realm}/components", + token, + ) + if status != 200: + raise SystemExit(f"Unable to list realm components: status={status}") + components = components or [] + + for component in components: + if component.get("providerId") != "ldap": + continue + if component.get("providerType") != "org.keycloak.storage.UserStorageProvider": + continue + if component.get("parentId") == realm_id: + continue + cid = component.get("id") + if not cid: + continue + status, payload, _ = request( + "GET", + f"{base_url}/admin/realms/{realm}/components/{cid}", + token, + ) + if status != 200 or not isinstance(payload, dict): + raise SystemExit(f"Unable to fetch LDAP component {cid}: status={status}") + payload["parentId"] = realm_id + status, _, _ = request( + "PUT", + f"{base_url}/admin/realms/{realm}/components/{cid}", + token, + payload, + ) + if status not in (200, 204): + raise SystemExit(f"Unable to repair LDAP component {cid}: status={status}") + + status, storage_components, _ = request( + "GET", + f"{base_url}/admin/realms/{realm}/components?type=org.keycloak.storage.UserStorageProvider", + token, + ) + if status != 200: + raise SystemExit(f"Unable to list user-storage components: status={status}") + storage_components = storage_components or [] + + ldap_components = [ + c + for c in storage_components + if c.get("providerId") == "ldap" and c.get("id") + ] + candidates = [] + for component in ldap_components: + config = component.get("config") or {} + if component.get("name") in ("openldap", "ldap") and (config.get("connectionUrl") or [None])[0] == ldap_url: + candidates.append(component) + if not candidates: + candidates = [c for c in ldap_components if c.get("name") in ("openldap", "ldap")] + candidates.sort(key=lambda item: item.get("id", "")) + ldap_component = candidates[0] if candidates else None + ldap_component_id = ldap_component.get("id") if ldap_component else None + + desired = { + "name": "openldap", + "providerId": "ldap", + "providerType": "org.keycloak.storage.UserStorageProvider", + "parentId": realm_id, + "config": { + "enabled": ["true"], + "priority": ["0"], + "importEnabled": ["true"], + "editMode": ["WRITABLE"], + "syncRegistrations": ["true"], + "vendor": ["other"], + "connectionUrl": [ldap_url], + "bindDn": [ldap_bind_dn], + "bindCredential": [ldap_bind_password], + "authType": ["simple"], + "usersDn": [ldap_users_dn], + "searchScope": ["1"], + "pagination": ["true"], + "usernameLDAPAttribute": ["uid"], + "rdnLDAPAttribute": ["uid"], + "uuidLDAPAttribute": ["entryUUID"], + "userObjectClasses": ["inetOrgPerson, organizationalPerson, person, top"], + "trustEmail": ["true"], + "useTruststoreSpi": ["never"], + "connectionPooling": ["true"], + "cachePolicy": ["DEFAULT"], + "useKerberosForPasswordAuthentication": ["false"], + "allowKerberosAuthentication": ["false"], + }, + } + + if ldap_component_id: + desired["id"] = ldap_component_id + print(f"Updating Cassandra LDAP federation provider: {ldap_component_id}") + status, _, _ = request( + "PUT", + f"{base_url}/admin/realms/{realm}/components/{ldap_component_id}", + token, + desired, + ) + if status not in (200, 204): + raise SystemExit(f"LDAP provider update failed: status={status}") + else: + print("Creating Cassandra LDAP federation provider") + status, _, headers = request( + "POST", + f"{base_url}/admin/realms/{realm}/components", + token, + desired, + ) + if status not in (201, 204): + raise SystemExit(f"LDAP provider create failed: status={status}") + location = headers.get("Location", "") + ldap_component_id = location.rstrip("/").split("/")[-1] if location else None + + if not ldap_component_id: + raise SystemExit("Unable to determine Cassandra LDAP provider id") + + status, mapper_components, _ = request( + "GET", + f"{base_url}/admin/realms/{realm}/components?type=org.keycloak.storage.ldap.mappers.LDAPStorageMapper", + token, + ) + if status != 200: + raise SystemExit(f"Unable to list LDAP mappers: status={status}") + mapper_components = mapper_components or [] + + def upsert_mapper(name, provider_id, config): + existing = next( + ( + c + for c in mapper_components + if c.get("name") == name and c.get("parentId") == ldap_component_id + ), + None, + ) + payload = { + "name": name, + "providerId": provider_id, + "providerType": "org.keycloak.storage.ldap.mappers.LDAPStorageMapper", + "parentId": ldap_component_id, + "config": config, + } + if existing: + payload["id"] = existing["id"] + status, _, _ = request( + "PUT", + f"{base_url}/admin/realms/{realm}/components/{existing['id']}", + token, + payload, + ) + action = "Updated" + else: + status, _, _ = request( + "POST", + f"{base_url}/admin/realms/{realm}/components", + token, + payload, + ) + action = "Created" + if status not in (200, 201, 204): + raise SystemExit(f"Mapper {name} upsert failed: status={status}") + print(f"{action} Cassandra LDAP mapper: {name}") + + upsert_mapper( + "openldap-groups", + "group-ldap-mapper", + { + "groups.dn": [ldap_groups_dn], + "group.name.ldap.attribute": ["cn"], + "group.object.classes": ["groupOfNames"], + "membership.ldap.attribute": ["member"], + "membership.attribute.type": ["DN"], + "mode": ["LDAP_ONLY"], + "user.roles.retrieve.strategy": ["LOAD_GROUPS_BY_MEMBER_ATTRIBUTE"], + "preserve.group.inheritance": ["true"], + }, + ) + for name, ldap_attr, user_attr in ( + ("openldap-email", "mail", "email"), + ("openldap-first-name", "givenName", "firstName"), + ("openldap-last-name", "sn", "lastName"), + ): + upsert_mapper( + name, + "user-attribute-ldap-mapper", + { + "ldap.attribute": [ldap_attr], + "user.model.attribute": [user_attr], + "read.only": ["false"], + "always.read.value.from.ldap": ["false"], + "is.mandatory.in.ldap": ["false"], + }, + ) + + status, sync_body, _ = request( + "POST", + f"{base_url}/admin/realms/{realm}/user-storage/{ldap_component_id}/sync?action=triggerFullSync", + token, + timeout=120, + ) + if status in (200, 201, 204): + print(f"Cassandra LDAP full sync requested: {sync_body}") + else: + print(f"WARNING: Cassandra LDAP full sync returned status={status} body={sync_body}") + + for username in (item.strip() for item in os.environ.get("VERIFY_USERS", "").split(",")): + if not username: + continue + encoded = urllib.parse.quote(username) + status, users, _ = request( + "GET", + f"{base_url}/admin/realms/{realm}/users?username={encoded}&exact=true", + token, + ) + found = status == 200 and any( + str(user.get("username") or "").casefold() == username.casefold() + for user in users or [] + ) + print(f"Cassandra LDAP user check {username}: {'found' if found else 'missing'}") + + for audit_realm in ("veles", realm): + status, count, _ = request( + "GET", + f"{base_url}/admin/realms/{audit_realm}/users/count", + token, + ) + if status == 200: + print(f"SSO audit realm {audit_realm}: user_count={count}") + for term in (item.strip() for item in os.environ.get("VERIFY_USERS", "").split(",")): + if not term: + continue + encoded = urllib.parse.quote(term) + status, users, _ = request( + "GET", + f"{base_url}/admin/realms/{audit_realm}/users?search={encoded}", + token, + ) + users = users if status == 200 and isinstance(users, list) else [] + matches = [] + for user in users: + username = str(user.get("username") or "") + email = str(user.get("email") or "") + if term.casefold() in username.casefold() or term.casefold() in email.casefold(): + matches.append(username if not email else f"{username}<{email}>") + joined = ", ".join(matches[:8]) if matches else "none" + print(f"SSO audit realm {audit_realm} search {term}: {joined}") + + print("Cassandra LDAP federation ready") + PY diff --git a/services/cassandra-auth/bootstrap-jobs/cassandra-realm-ensure-job.yaml b/services/cassandra-auth/bootstrap-jobs/cassandra-realm-ensure-job.yaml index 3b3d10b3..d359c434 100644 --- a/services/cassandra-auth/bootstrap-jobs/cassandra-realm-ensure-job.yaml +++ b/services/cassandra-auth/bootstrap-jobs/cassandra-realm-ensure-job.yaml @@ -1,11 +1,11 @@ # services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml -# One-off job for sso/cassandra-realm-ensure-1. +# One-off job for sso/cassandra-realm-ensure-4. # Purpose: create the Cassandra realm, groups, OIDC client, SMTP settings, and Vault client secret. # Keep suspended until Cassandra Vault paths/policies have reconciled, then unsuspend once. apiVersion: batch/v1 kind: Job metadata: - name: cassandra-realm-ensure-1 + name: cassandra-realm-ensure-4 namespace: sso spec: suspend: true @@ -70,7 +70,7 @@ spec: - name: KEYCLOAK_SMTP_FROM_NAME value: Cassandra - name: CASSANDRA_PUBLIC_REPLAY_USERS - value: cassandra-dev,daniel-test,viktor-test + value: veles-dev,daniel-test,viktor-test command: ["/bin/sh", "-c"] args: - | @@ -260,15 +260,35 @@ spec: if status not in (200, 204): raise SystemExit(f"User group update failed for {username}: status={status} body={body}") - ensure_group("alpha") + alpha_group = ensure_group("alpha") ensure_group("admin") - ensure_group(os.environ["CASSANDRA_TESTER_GROUP"]) + tester_group = ensure_group(os.environ["CASSANDRA_TESTER_GROUP"]) public_replay_group = ensure_group("public-replay") + pilot_groups = { + "pilot-skill-proficient": ensure_group("pilot-skill-proficient"), + "pilot-skill-adept": ensure_group("pilot-skill-adept"), + "pilot-skill-veteran": ensure_group("pilot-skill-veteran"), + "pilot-skill-expert": ensure_group("pilot-skill-expert"), + "pilot-skill-master": ensure_group("pilot-skill-master"), + "all-pilot-skill-level": ensure_group("all-pilot-skill-level"), + "codex-cli-fallback": ensure_group("codex-cli-fallback"), + } for username in os.environ.get("CASSANDRA_PUBLIC_REPLAY_USERS", "").split(","): username = username.strip() if username: ensure_user_group(username, public_replay_group) + ensure_user_group("veles-dev", tester_group) + ensure_user_group("veles-dev", pilot_groups["all-pilot-skill-level"]) + ensure_user_group("veles-dev", pilot_groups["codex-cli-fallback"]) + for username in ("daniel-test", "viktor-test"): + ensure_user_group(username, alpha_group) + ensure_user_group(username, tester_group) + ensure_user_group(username, pilot_groups["pilot-skill-proficient"]) + ensure_user_group(username, pilot_groups["pilot-skill-adept"]) + ensure_user_group(username, pilot_groups["pilot-skill-veteran"]) + ensure_user_group(username, pilot_groups["codex-cli-fallback"]) + status, clients = request( "GET", f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}", diff --git a/services/cassandra-auth/kustomization.yaml b/services/cassandra-auth/kustomization.yaml index a84908f3..dd064b80 100644 --- a/services/cassandra-auth/kustomization.yaml +++ b/services/cassandra-auth/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: sso resources: + - bootstrap-jobs/cassandra-ldap-federation-job.yaml - bootstrap-jobs/cassandra-realm-ensure-job.yaml - bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml configMapGenerator: diff --git a/services/cassandra/backend-deployment.yaml b/services/cassandra/backend-deployment.yaml index 2a5c504b..69308aa2 100644 --- a/services/cassandra/backend-deployment.yaml +++ b/services/cassandra/backend-deployment.yaml @@ -22,7 +22,7 @@ spec: labels: app: cassandra-backend annotations: - cassandra.bstein.dev/redeploy: "2026-07-28-0.8.51-jobs-compact" + cassandra.bstein.dev/redeploy: "2026-08-04-0.8.150-lane-semantics" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-init-first: "true" vault.hashicorp.com/agent-pre-populate-only: "true" @@ -66,7 +66,7 @@ spec: type: RuntimeDefault initContainers: - name: database-init - image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"} + image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"} imagePullPolicy: Always command: ["/bin/sh", "-c"] args: @@ -88,7 +88,7 @@ spec: memory: 512Mi containers: - name: backend - image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"} + image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"} imagePullPolicy: Always command: ["/bin/sh", "-c"] args: @@ -102,6 +102,11 @@ spec: - name: http containerPort: 8796 protocol: TCP + env: + - name: CASSANDRA_SIM_IMAGE + value: registry.bstein.dev/cassandra/cassandra-sim-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-sim-worker"} + - name: CASSANDRA_GENERATOR_WORKER_IMAGE + value: registry.bstein.dev/cassandra/cassandra-generator-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-generator-worker"} envFrom: - configMapRef: name: cassandra-config diff --git a/services/cassandra/bootstrap-jobs/cassandra-secrets-ensure-job.yaml b/services/cassandra/bootstrap-jobs/cassandra-secrets-ensure-job.yaml index 29f52a73..7c4b3f06 100644 --- a/services/cassandra/bootstrap-jobs/cassandra-secrets-ensure-job.yaml +++ b/services/cassandra/bootstrap-jobs/cassandra-secrets-ensure-job.yaml @@ -5,7 +5,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: cassandra-secrets-ensure-2 + name: cassandra-secrets-ensure-3 namespace: cassandra spec: suspend: true diff --git a/services/cassandra/configmap.yaml b/services/cassandra/configmap.yaml index b27413a7..ce514817 100644 --- a/services/cassandra/configmap.yaml +++ b/services/cassandra/configmap.yaml @@ -18,20 +18,34 @@ data: CASSANDRA_OIDC_REQUIRED_GROUPS: alpha,cassandra-tester,admin CASSANDRA_OIDC_ADMIN_GROUPS: admin CASSANDRA_NAMESPACE: cassandra - CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.8.51 # {"$imagepolicy": "cassandra:cassandra-sim-worker"} - CASSANDRA_SIM_WORKER_VERSION: "0.8.51" - CASSANDRA_GENERATOR_WORKER_VERSION: "0.8.51" + CASSANDRA_OWNER_USER_ID_ALIASES: veles-dev=d357688e-e3e4-4f18-9491-a4ed5622deb6;a21a286c-6c82-4180-9243-7c916442ecf0=d357688e-e3e4-4f18-9491-a4ed5622deb6 + CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-sim-worker"} + CASSANDRA_GENERATOR_WORKER_IMAGE: registry.bstein.dev/cassandra/cassandra-generator-worker:0.9.19 # {"$imagepolicy": "cassandra:cassandra-generator-worker"} + CASSANDRA_SIM_WORKER_VERSION: "0.8.150" + CASSANDRA_GENERATOR_WORKER_VERSION: "0.8.150" + CASSANDRA_STRATEGY_SPEC_HINT_INTERPRETATION_TIMEOUT: "420" CASSANDRA_SIM_SERVICE_ACCOUNT: cassandra-sim + CASSANDRA_GENERATOR_WORKER_SERVICE_ACCOUNT: cassandra-generator CASSANDRA_SIM_PRIORITY_CLASS: cassandra-sim + CASSANDRA_GENERATOR_WORKER_PRIORITY_CLASS: cassandra-core CASSANDRA_SIM_NODE_SELECTOR: cassandra.bstein.dev/node-pool=oceanus,kubernetes.io/arch=amd64 + CASSANDRA_GENERATOR_WORKER_NODE_SELECTOR: cassandra.bstein.dev/node-pool=oceanus,kubernetes.io/arch=amd64 CASSANDRA_SIM_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule + CASSANDRA_GENERATOR_WORKER_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule CASSANDRA_SIM_ACTIVE_DEADLINE_SECONDS: "7200" + CASSANDRA_GENERATOR_WORKER_ACTIVE_DEADLINE_SECONDS: "14400" CASSANDRA_SIM_TTL_SECONDS: "3600" + CASSANDRA_GENERATOR_WORKER_TTL_SECONDS: "600" CASSANDRA_SIM_CPU_REQUEST: 500m CASSANDRA_SIM_CPU_LIMIT: "2" CASSANDRA_SIM_MEMORY_REQUEST: 1Gi CASSANDRA_SIM_MEMORY_LIMIT: 4Gi + CASSANDRA_GENERATOR_WORKER_CPU_REQUEST: 100m + CASSANDRA_GENERATOR_WORKER_CPU_LIMIT: "1" + CASSANDRA_GENERATOR_WORKER_MEMORY_REQUEST: 512Mi + CASSANDRA_GENERATOR_WORKER_MEMORY_LIMIT: 2Gi CASSANDRA_SIM_FS_GROUP: "10001" + CASSANDRA_GENERATOR_WORKER_FS_GROUP: "10001" CASSANDRA_MAX_ACTIVE_SIMS_PER_USER: "32" CASSANDRA_MAX_ACTIVE_SIMS_GLOBAL: "32" CASSANDRA_RETENTION_DAYS: "30" @@ -43,3 +57,6 @@ data: CASSANDRA_SIM_CODEX_CLI_SECRET_PROVIDER_CLASS: cassandra-codex-cli CASSANDRA_SIM_CODEX_CLI_MOUNT_PATH: /var/run/cassandra-codex-cli CASSANDRA_SIM_CODEX_CLI_HOME: /home/cassandra/.codex + CASSANDRA_GENERATOR_WORKER_CODEX_CLI_SECRET_PROVIDER_CLASS: cassandra-generator-codex-cli + CASSANDRA_GENERATOR_WORKER_CODEX_CLI_MOUNT_PATH: /var/run/cassandra-codex-cli + CASSANDRA_GENERATOR_WORKER_CODEX_CLI_HOME: /home/cassandra/.codex diff --git a/services/cassandra/frontend-deployment.yaml b/services/cassandra/frontend-deployment.yaml index 84dad64f..1b7f7bd8 100644 --- a/services/cassandra/frontend-deployment.yaml +++ b/services/cassandra/frontend-deployment.yaml @@ -48,7 +48,7 @@ spec: type: RuntimeDefault containers: - name: frontend - image: registry.bstein.dev/cassandra/cassandra-frontend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-frontend"} + image: registry.bstein.dev/cassandra/cassandra-frontend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-frontend"} imagePullPolicy: Always ports: - name: http diff --git a/services/cassandra/kustomization.yaml b/services/cassandra/kustomization.yaml index 3679f1a1..61c33ff8 100644 --- a/services/cassandra/kustomization.yaml +++ b/services/cassandra/kustomization.yaml @@ -16,7 +16,6 @@ resources: - postgres-statefulset.yaml - services.yaml - backend-deployment.yaml - - generator-deployment.yaml - frontend-deployment.yaml - retention-cronjob.yaml - image.yaml @@ -24,3 +23,4 @@ resources: - bootstrap-jobs/cassandra-secrets-ensure-job.yaml - migration-jobs/db-copy-veles-to-cassandra-job.yaml - migration-jobs/artifact-copy-veles-to-cassandra-job.yaml + - migration-jobs/artifact-permissions-job.yaml diff --git a/services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml b/services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml index 65f43438..b6194e62 100644 --- a/services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml +++ b/services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: cassandra-artifact-copy-from-veles-1 + name: cassandra-artifact-copy-from-veles-2 namespace: cassandra spec: suspend: true @@ -33,6 +33,8 @@ spec: value: /dest - name: ALLOW_NONEMPTY_DEST value: "0" + - name: RESET_DESTINATION + value: "1" command: ["python", "-c"] args: - | @@ -42,12 +44,15 @@ spec: import hashlib import json import os + import shutil import tarfile + import time import urllib.request SOURCE_URL = os.environ["SOURCE_URL"].rstrip("/") DEST = Path(os.environ.get("DEST_ROOT", "/dest")).resolve() ALLOW_NONEMPTY = os.environ.get("ALLOW_NONEMPTY_DEST", "") in {"1", "true", "yes"} + RESET_DESTINATION = os.environ.get("RESET_DESTINATION", "") in {"1", "true", "yes"} EXCLUDE_TOP = {"lost+found", ".cassandra-artifact-migration"} REPORT_DIR = DEST / ".cassandra-artifact-migration" @@ -57,6 +62,13 @@ spec: def assert_safe_destination() -> None: DEST.mkdir(parents=True, exist_ok=True) entries = visible_entries() + if entries and RESET_DESTINATION: + for entry in entries: + if entry.is_dir(): + shutil.rmtree(entry) + else: + entry.unlink() + entries = visible_entries() if entries and not ALLOW_NONEMPTY: names = ", ".join(entry.name for entry in entries[:10]) raise SystemExit(f"destination is not empty; refusing to merge into existing data: {names}") @@ -66,6 +78,16 @@ spec: with urllib.request.urlopen(f"{SOURCE_URL}/manifest.json", timeout=120) as response: return json.loads(response.read().decode()) + def wait_for_manifest() -> dict[str, object]: + last_error = "" + for _attempt in range(90): + try: + return fetch_manifest() + except Exception as exc: + last_error = str(exc) + time.sleep(5) + raise SystemExit(f"source artifact exporter did not become ready: {last_error}") + def safe_extract() -> None: root_text = str(DEST) with urllib.request.urlopen(f"{SOURCE_URL}/archive.tar", timeout=600) as response: @@ -116,7 +138,7 @@ spec: } assert_safe_destination() - source = fetch_manifest() + source = wait_for_manifest() (REPORT_DIR / "source_manifest.json").write_text(json.dumps(source, indent=2, sort_keys=True)) safe_extract() dest = destination_manifest() diff --git a/services/cassandra/migration-jobs/artifact-permissions-job.yaml b/services/cassandra/migration-jobs/artifact-permissions-job.yaml new file mode 100644 index 00000000..6da4f715 --- /dev/null +++ b/services/cassandra/migration-jobs/artifact-permissions-job.yaml @@ -0,0 +1,42 @@ +# services/cassandra/migration-jobs/artifact-permissions-job.yaml +# One-off migration repair after copying artifacts from the Veles PVC. +apiVersion: batch/v1 +kind: Job +metadata: + name: cassandra-artifact-permissions-2 + namespace: cassandra +spec: + suspend: true + backoffLimit: 0 + ttlSecondsAfterFinished: 86400 + template: + spec: + serviceAccountName: cassandra-artifact-migration + automountServiceAccountToken: false + restartPolicy: Never + nodeSelector: + cassandra.bstein.dev/node-pool: oceanus + tolerations: + - key: veles.bstein.dev/simulation + operator: Equal + value: "true" + effect: NoSchedule + containers: + - name: permissions + image: alpine:3.20 + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - | + set -eu + chown -R 10001:10001 /data/cassandra-artifacts + find /data/cassandra-artifacts -type d -exec chmod 2775 {} + + find /data/cassandra-artifacts -type f -exec chmod 0664 {} + + printf 'artifact permission repair complete\n' + volumeMounts: + - name: artifacts + mountPath: /data/cassandra-artifacts + volumes: + - name: artifacts + persistentVolumeClaim: + claimName: cassandra-artifacts diff --git a/services/cassandra/migration-jobs/db-copy-veles-to-cassandra-job.yaml b/services/cassandra/migration-jobs/db-copy-veles-to-cassandra-job.yaml index c9ff432b..a9c3f4c5 100644 --- a/services/cassandra/migration-jobs/db-copy-veles-to-cassandra-job.yaml +++ b/services/cassandra/migration-jobs/db-copy-veles-to-cassandra-job.yaml @@ -5,7 +5,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: cassandra-db-copy-from-veles-1 + name: cassandra-db-copy-from-veles-3 namespace: cassandra spec: suspend: true @@ -68,7 +68,9 @@ spec: pg_dump --format=custom --no-owner --no-acl \ --dbname="${SOURCE_DATABASE_URL}" \ --file="${dump_path}" - pg_restore --clean --if-exists --no-owner --no-acl \ + psql -X --set=ON_ERROR_STOP=1 --dbname="${TARGET_DATABASE_URL}" \ + --command="DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO cassandra; GRANT ALL ON SCHEMA public TO public;" + pg_restore --no-owner --no-acl \ --dbname="${TARGET_DATABASE_URL}" \ "${dump_path}" diff --git a/services/cassandra/retention-cronjob.yaml b/services/cassandra/retention-cronjob.yaml index 4a20c596..53fe3ccb 100644 --- a/services/cassandra/retention-cronjob.yaml +++ b/services/cassandra/retention-cronjob.yaml @@ -38,7 +38,7 @@ spec: type: RuntimeDefault containers: - name: retention - image: registry.bstein.dev/cassandra/cassandra-backend:0.8.51 # {"$imagepolicy": "cassandra:cassandra-backend"} + image: registry.bstein.dev/cassandra/cassandra-backend:0.9.19 # {"$imagepolicy": "cassandra:cassandra-backend"} imagePullPolicy: IfNotPresent command: ["cassandra-hygiene"] args: ["--prune-artifacts"] diff --git a/services/cassandra/secretproviderclass.yaml b/services/cassandra/secretproviderclass.yaml index a1a0f483..853f3dea 100644 --- a/services/cassandra/secretproviderclass.yaml +++ b/services/cassandra/secretproviderclass.yaml @@ -73,3 +73,19 @@ spec: secretPath: "kv/data/atlas/cassandra/codex-cli" secretKey: "auth.json" filePermission: 292 +--- +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: cassandra-generator-codex-cli + namespace: cassandra +spec: + provider: vault + parameters: + vaultAddress: "http://vault.vault.svc.cluster.local:8200" + roleName: "cassandra" + objects: | + - objectName: "auth.json" + secretPath: "kv/data/atlas/cassandra/codex-cli" + secretKey: "auth.json" + filePermission: 292 diff --git a/services/harbor/bootstrap-jobs/cassandra-registry-ensure-job.yaml b/services/harbor/bootstrap-jobs/cassandra-registry-ensure-job.yaml new file mode 100644 index 00000000..c5db9c53 --- /dev/null +++ b/services/harbor/bootstrap-jobs/cassandra-registry-ensure-job.yaml @@ -0,0 +1,138 @@ +# services/harbor/bootstrap-jobs/cassandra-registry-ensure-job.yaml +# One-off repair/audit job for the Cassandra Harbor migration. +# Keep suspended unless the Harbor project or robot scopes need to be recreated. +apiVersion: batch/v1 +kind: Job +metadata: + name: cassandra-registry-ensure-1 + namespace: harbor +spec: + suspend: true + backoffLimit: 1 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: "harbor" + vault.hashicorp.com/agent-inject-secret-harbor-admin.sh: "kv/data/atlas/harbor/harbor-core" + vault.hashicorp.com/agent-inject-template-harbor-admin.sh: | + {{- with secret "kv/data/atlas/harbor/harbor-core" }} + export HARBOR_ADMIN_PASSWORD="{{ .Data.data.harbor_admin_password }}" + {{- end }} + spec: + restartPolicy: Never + serviceAccountName: harbor-vault-sync + containers: + - name: ensure + image: python:3.11-alpine + command: ["python", "-c"] + args: + - | + import base64 + import json + import os + import urllib.error + import urllib.request + + secret_path = "/vault/secrets/harbor-admin.sh" + env = {} + with open(secret_path, "r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line.startswith("export "): + continue + key, value = line[len("export ") :].split("=", 1) + env[key] = value.strip('"') + + password = env["HARBOR_ADMIN_PASSWORD"] + api = os.environ.get( + "HARBOR_API", + "http://harbor-core.harbor.svc.cluster.local/api/v2.0", + ).rstrip("/") + auth = base64.b64encode(f"admin:{password}".encode("utf-8")).decode("ascii") + + def request(method, path, payload=None, ok=(200,)): + data = None + headers = {"Authorization": f"Basic {auth}"} + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request( + f"{api}{path}", + data=data, + method=method, + headers=headers, + ) + try: + with urllib.request.urlopen(req, timeout=30) as response: + body = response.read() + if response.status not in ok: + raise RuntimeError(f"{method} {path} returned {response.status}") + if not body: + return None + return json.loads(body.decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + if exc.code in ok: + return None + raise RuntimeError(f"{method} {path} returned {exc.code}: {detail}") from exc + + projects = request("GET", "/projects?name=cassandra") + if not projects: + request( + "POST", + "/projects", + { + "project_name": "cassandra", + "metadata": { + "public": "false", + "auto_scan": "false", + "enable_content_trust": "false", + "prevent_vul": "false", + "reuse_sys_cve_allowlist": "true", + }, + }, + ok=(201, 409), + ) + + def ensure_permission(robot_name, access, description=None): + robots = request("GET", "/robots?page_size=100") + matches = [robot for robot in robots if robot.get("name") == robot_name] + if not matches: + raise RuntimeError(f"Harbor robot {robot_name} not found") + robot = request("GET", f"/robots/{matches[0]['id']}") + permissions = list(robot.get("permissions") or []) + if not any( + item.get("kind") == "project" and item.get("namespace") == "cassandra" + for item in permissions + ): + permissions.append( + {"kind": "project", "namespace": "cassandra", "access": access} + ) + payload = { + "name": robot["name"], + "level": robot["level"], + "disable": bool(robot.get("disable", False)), + "duration": int(robot.get("duration", -1)), + "permissions": permissions, + } + if description: + payload["description"] = description + elif robot.get("description"): + payload["description"] = robot["description"] + request("PUT", f"/robots/{robot['id']}", payload, ok=(200,)) + + ensure_permission( + "robot$pull-atlas", + [{"resource": "repository", "action": "pull"}], + ) + ensure_permission( + "robot$jenkins-pipelines", + [ + {"resource": "repository", "action": "pull"}, + {"resource": "repository", "action": "push"}, + ], + "Jenkins image push robot for bstein, veles, and cassandra pipelines", + ) + print("Cassandra Harbor project and robot scopes are present") diff --git a/services/harbor/kustomization.yaml b/services/harbor/kustomization.yaml index 3018b3ca..6a4ce002 100644 --- a/services/harbor/kustomization.yaml +++ b/services/harbor/kustomization.yaml @@ -12,6 +12,7 @@ resources: - certificate.yaml - helmrelease.yaml - vault-sync-deployment.yaml + - bootstrap-jobs/cassandra-registry-ensure-job.yaml - image.yaml configMapGenerator: - name: harbor-vault-entrypoint diff --git a/services/harbor/pvc.yaml b/services/harbor/pvc.yaml index e70cb81a..a34ec463 100644 --- a/services/harbor/pvc.yaml +++ b/services/harbor/pvc.yaml @@ -8,7 +8,7 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 450Gi + storage: 600Gi storageClassName: astreae --- apiVersion: v1 diff --git a/services/hermes-chat/certificate.yaml b/services/hermes-chat/certificate.yaml new file mode 100644 index 00000000..3bed14a1 --- /dev/null +++ b/services/hermes-chat/certificate.yaml @@ -0,0 +1,13 @@ +# services/hermes-chat/certificate.yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: chat-tls + namespace: hermes-chat +spec: + secretName: chat-tls + issuerRef: + kind: ClusterIssuer + name: letsencrypt + dnsNames: + - chat.bstein.dev diff --git a/services/hermes-chat/configmap.yaml b/services/hermes-chat/configmap.yaml new file mode 100644 index 00000000..84d6ab63 --- /dev/null +++ b/services/hermes-chat/configmap.yaml @@ -0,0 +1,153 @@ +# services/hermes-chat/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-chat-config + namespace: hermes-chat + labels: + app: hermes-chat +data: + config.yaml: | + model: + provider: openai-codex + default: gpt-5.6-terra + model: gpt-5.6-terra + + fallback_providers: + - provider: custom + model: gpt-oss:20b + base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 + api_key: ollama + + agent: + api_max_retries: 1 + + platform_toolsets: + cli: + - clarify + - file + - terminal + - web + api_server: + - clarify + - file + - terminal + - web + + skills: + creation_nudge_interval: 15 + external_dirs: + - /opt/data/workspace/skills + + terminal: + backend: local + cwd: /opt/data/workspace + timeout: 180 + home_mode: profile + + approvals: + mode: manual + deny: + - "*kubectl apply*" + - "*kubectl create*" + - "*kubectl delete*" + - "*kubectl edit*" + - "*kubectl patch*" + - "*kubectl replace*" + - "*kubectl scale*" + - "*kubectl set*" + - "*kubectl label*" + - "*kubectl annotate*" + - "*kubectl cordon*" + - "*kubectl uncordon*" + - "*kubectl drain*" + - "*kubectl rollout restart*" + - "*kubectl rollout undo*" + - "*kubectl exec*" + - "*kubectl attach*" + - "*kubectl cp*" + - "*kubectl debug*" + - "*kubectl expose*" + - "*kubectl port-forward*" + - "*kubectl proxy*" + - "*kubectl run*" + - "*kubectl get secret*" + - "*kubectl describe secret*" + - "*flux reconcile*" + - "*flux suspend*" + - "*flux resume*" + - "*vault *" + - "*k3s *" + - "*crictl *" + - "*ctr *" + - "*169.254.169.254*" + - "*kubernetes.default*" + - "*/var/run/secrets/kubernetes.io*" + + dashboard: + public_url: https://chat.bstein.dev + oauth: + provider: self-hosted + self_hosted: + issuer: https://sso.bstein.dev/realms/atlas + client_id: hermes-chat-dashboard + scopes: openid profile email groups + + display: + compact: true + tool_progress: all + interim_assistant_messages: true + long_running_notifications: true + + tool_loop_guardrails: + warnings_enabled: true + hard_stop_enabled: true + warn_after: + exact_failure: 2 + same_tool_failure: 3 + idempotent_no_progress: 2 + hard_stop_after: + exact_failure: 5 + same_tool_failure: 8 + idempotent_no_progress: 5 + + updates: + pre_update_backup: quick + backup_keep: 3 + non_interactive_local_changes: stash + SOUL.md: | + You are a personal AI assistant and researcher for Atlas users. Be useful, + curious, careful, and direct. Help with research, writing, planning, + learning, files, personal automation, and creative work. Use tools when + they materially improve the answer and make reusable skills when a stable + workflow is worth keeping. + + Your environment is a private consumer sandbox. Work in the mounted + workspace and with public Internet services the user intentionally asks + you to use. You may inspect the Titan cluster through the dedicated + read-only Kubernetes identity to explain health and status. You do not + administer it: never attempt mutations, secret access, exec, attach, + port-forwarding, private-service access, or cloud metadata access. + AGENTS.md: | + # Personal Hermes workspace + + This Hermes instance is a personal chat, research, and automation + environment. User-created files and skills belong under `/opt/data/workspace`. + + You may: + + - research public Internet sources and cite them + - create, read, and edit files in the workspace + - create and improve reusable skills + - configure user-owned channels and provider integrations + - run ordinary local commands needed for the user's task + - inspect non-secret Kubernetes resources and pod logs with read-only + `kubectl get`, `describe`, and `logs` commands + + This container is not an infrastructure administration environment. The + Kubernetes identity is observation-only and cannot read Secrets or use pod + exec, attach, or port-forwarding. Do not attempt cluster mutations, private + service access, node LAN access, metadata services, Vault, container + runtimes, or the operator Hermes instance. If a request needs an action, + explain the evidence and say Brad must perform it from the operator + instance at `agent.bstein.dev`. diff --git a/services/hermes-chat/deployment.yaml b/services/hermes-chat/deployment.yaml new file mode 100644 index 00000000..ac97e8bb --- /dev/null +++ b/services/hermes-chat/deployment.yaml @@ -0,0 +1,259 @@ +# services/hermes-chat/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-chat + namespace: hermes-chat + labels: + app: hermes-chat +spec: + replicas: 1 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 1800 + strategy: + type: Recreate + selector: + matchLabels: + app: hermes-chat + template: + metadata: + labels: + app: hermes-chat + annotations: + ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback + ai.bstein.dev/role: personal-chat-research + ai.bstein.dev/isolation: observer-only Kubernetes RBAC, private-service egress denied + ai.bstein.dev/session-reset: "20260802-consumer-pty-recovery" + ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation + ai.bstein.dev/user-boundary: one OIDC subject with an isolated PVC subdirectory + spec: + serviceAccountName: hermes-chat + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-13 + - titan-15 + - titan-17 + - titan-18 + - titan-19 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: atlas.bstein.dev/spillover + operator: DoesNotExist + - weight: 90 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi4 + initContainers: + - name: migrate-user-sessions + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/bin/hermes-session-migrate + env: + - name: HERMES_HOME + value: /storage/users/4794ab8284a14d421eaeea3e + - name: HERMES_MIGRATE_SOURCE_HOME + value: /storage + - name: HERMES_MIGRATE_USER_ID + value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc + - name: HERMES_MIGRATE_SESSION_IDS + value: 20260802_182723_9f53f1,20260802_173527_9dc51b + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: home + mountPath: /storage + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + - name: init-config + image: busybox:1.37 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -eu + user_home=/storage/users/4794ab8284a14d421eaeea3e + mkdir -p "${user_home}/workspace/skills" "${user_home}/home/.local/bin" "${user_home}/logs" + cp /config/config.yaml "${user_home}/config.yaml" + cp /config/SOUL.md "${user_home}/SOUL.md" + cp /config/AGENTS.md "${user_home}/workspace/AGENTS.md" + touch "${user_home}/.env" + if ! grep -q '^API_SERVER_KEY=' "${user_home}/.env"; then + api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" + printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> "${user_home}/.env" + fi + chmod 0600 "${user_home}/.env" + chown -R 10000:10000 "${user_home}" + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: home + mountPath: /storage + - name: config + mountPath: /config + readOnly: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + - name: install-kubectl + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + set -eu + cp "$(command -v kubectl)" /tools/kubectl + chmod 0755 /tools/kubectl + chown 10000:10000 /tools/kubectl + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: tools + mountPath: /tools + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + containers: + - name: hermes-chat + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + args: + - gateway + - run + ports: + - name: dashboard + containerPort: 9119 + protocol: TCP + env: + - name: HERMES_HOME + value: /opt/data + - name: HOME + value: /opt/data/home + - name: PATH + value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: HERMES_DASHBOARD + value: "1" + - name: HERMES_DASHBOARD_HOST + value: 0.0.0.0 + - name: HERMES_DASHBOARD_PORT + value: "9119" + - name: HERMES_DASHBOARD_PUBLIC_URL + value: https://chat.bstein.dev + - name: HERMES_DASHBOARD_OIDC_ISSUER + value: https://sso.bstein.dev/realms/atlas + - name: HERMES_DASHBOARD_OIDC_CLIENT_ID + value: hermes-chat-dashboard + - name: HERMES_DASHBOARD_OIDC_SCOPES + value: openid profile email groups + - name: HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS + value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc + - name: API_SERVER_ENABLED + value: "true" + - name: API_SERVER_HOST + value: 0.0.0.0 + - name: API_SERVER_PORT + value: "8642" + - name: API_SERVER_CORS_ORIGINS + value: https://chat.bstein.dev + volumeMounts: + - name: home + mountPath: /opt/data + subPath: users/4794ab8284a14d421eaeea3e + - name: tools + mountPath: /usr/local/bin/kubectl + subPath: kubectl + readinessProbe: + httpGet: + path: /api/status + port: dashboard + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /api/status + port: dashboard + initialDelaySeconds: 90 + periodSeconds: 30 + timeoutSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 768Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 1Gi + volumes: + - name: home + persistentVolumeClaim: + claimName: hermes-chat-home + - name: config + configMap: + name: hermes-chat-config + - name: tools + emptyDir: {} diff --git a/services/hermes-chat/ingress.yaml b/services/hermes-chat/ingress.yaml new file mode 100644 index 00000000..e176231f --- /dev/null +++ b/services/hermes-chat/ingress.yaml @@ -0,0 +1,27 @@ +# services/hermes-chat/ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: chat + namespace: hermes-chat + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + tls: + - hosts: + - chat.bstein.dev + secretName: chat-tls + rules: + - host: chat.bstein.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hermes-chat + port: + name: dashboard diff --git a/services/hermes-chat/kustomization.yaml b/services/hermes-chat/kustomization.yaml new file mode 100644 index 00000000..4d3529d7 --- /dev/null +++ b/services/hermes-chat/kustomization.yaml @@ -0,0 +1,15 @@ +# services/hermes-chat/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hermes-chat +resources: + - namespace.yaml + - serviceaccount.yaml + - rbac.yaml + - configmap.yaml + - pvc.yaml + - deployment.yaml + - service.yaml + - networkpolicy.yaml + - certificate.yaml + - ingress.yaml diff --git a/services/hermes-chat/namespace.yaml b/services/hermes-chat/namespace.yaml new file mode 100644 index 00000000..8b607b8b --- /dev/null +++ b/services/hermes-chat/namespace.yaml @@ -0,0 +1,7 @@ +# services/hermes-chat/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: hermes-chat + labels: + app.kubernetes.io/name: hermes-chat diff --git a/services/hermes-chat/networkpolicy.yaml b/services/hermes-chat/networkpolicy.yaml new file mode 100644 index 00000000..1c645120 --- /dev/null +++ b/services/hermes-chat/networkpolicy.yaml @@ -0,0 +1,87 @@ +# services/hermes-chat/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-chat-isolation + namespace: hermes-chat +spec: + podSelector: + matchLabels: + app: hermes-chat + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ports: + - protocol: TCP + port: 9119 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-model-gate + ports: + - protocol: TCP + port: 8080 + # Atlas OIDC uses the public sso.bstein.dev issuer, which resolves to the + # in-cluster Traefik load balancer for pods on the Atlas network. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app: traefik + ports: + - protocol: TCP + port: 443 + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - protocol: TCP + port: 443 + # K3s applies egress policy after Service DNAT, so admit only the fixed API + # server endpoints as well as the kubernetes Service IP above. + - to: + - ipBlock: + cidr: 192.168.22.11/32 + - ipBlock: + cidr: 192.168.22.12/32 + - ipBlock: + cidr: 192.168.22.13/32 + ports: + - protocol: TCP + port: 6443 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 diff --git a/services/hermes-chat/pvc.yaml b/services/hermes-chat/pvc.yaml new file mode 100644 index 00000000..f2588f79 --- /dev/null +++ b/services/hermes-chat/pvc.yaml @@ -0,0 +1,15 @@ +# services/hermes-chat/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hermes-chat-home + namespace: hermes-chat + labels: + app: hermes-chat +spec: + accessModes: + - ReadWriteOnce + storageClassName: astreae + resources: + requests: + storage: 2Gi diff --git a/services/hermes-chat/rbac.yaml b/services/hermes-chat/rbac.yaml new file mode 100644 index 00000000..4144da68 --- /dev/null +++ b/services/hermes-chat/rbac.yaml @@ -0,0 +1,62 @@ +# services/hermes-chat/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-chat-observer +rules: + - apiGroups: [""] + resources: + - endpoints + - events + - namespaces + - nodes + - persistentvolumeclaims + - persistentvolumes + - pods + - pods/log + - replicationcontrollers + - services + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: + - ingresses + - networkpolicies + verbs: ["get", "list", "watch"] + - apiGroups: ["helm.toolkit.fluxcd.io"] + resources: + - helmreleases + verbs: ["get", "list", "watch"] + - apiGroups: ["kustomize.toolkit.fluxcd.io"] + resources: + - kustomizations + verbs: ["get", "list", "watch"] + - apiGroups: ["source.toolkit.fluxcd.io"] + resources: + - gitrepositories + - helmrepositories + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hermes-chat-observer +subjects: + - kind: ServiceAccount + name: hermes-chat + namespace: hermes-chat +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hermes-chat-observer diff --git a/services/hermes-chat/service.yaml b/services/hermes-chat/service.yaml new file mode 100644 index 00000000..8b74822e --- /dev/null +++ b/services/hermes-chat/service.yaml @@ -0,0 +1,17 @@ +# services/hermes-chat/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hermes-chat + namespace: hermes-chat + labels: + app: hermes-chat +spec: + type: ClusterIP + selector: + app: hermes-chat + ports: + - name: dashboard + port: 9119 + targetPort: dashboard + protocol: TCP diff --git a/services/hermes-chat/serviceaccount.yaml b/services/hermes-chat/serviceaccount.yaml new file mode 100644 index 00000000..88eca224 --- /dev/null +++ b/services/hermes-chat/serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/hermes-chat/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-chat + namespace: hermes-chat +automountServiceAccountToken: true diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md new file mode 100644 index 00000000..2c107c0f --- /dev/null +++ b/services/hermes/NOTES.md @@ -0,0 +1,239 @@ +# Hermes on Atlas: operator guide + +This is the mental model and demonstration script for the operator instance at +`agent.bstein.dev`. Read it once, then prove each section in the live UI. The +consumer instance at `chat.bstein.dev` is intentionally separate and is not the +place to perform infrastructure triage. + +## The one-sentence explanation + +Hermes is the persistent agent runtime and control surface; Codex or the local +GPU model supplies reasoning, while Hermes supplies sessions, tools, skills, +approval rules, identity, and the repeatable procedure that connects the model +to Atlas evidence. + +Hermes is not the model. Replacing `gpt-5.6-terra` with `gpt-oss:20b` changes the +reasoning engine, but it does not replace the Hermes sessions, tools, skills, +files, permission boundary, or workflow. + +## Request and evidence path + +```text +browser + -> Keycloak login + -> oauth2-proxy exact-user check + -> operator Hermes pod and persistent workspace + -> SOUL.md + AGENTS.md + selected skill + -> Codex primary model + -> read-only terminal/web/file tools + -> Ariadne deterministic evidence + -> retained Jenkins logs and artifacts + -> Gitea commits and Flux state + -> Kubernetes workloads, events, logs, and dependencies + -> Pushgateway data queried through VictoriaMetrics + -> Grafana query and alert context + -> structured finding and proposed repo-side change + -> human review and approval + -> Git/Flux delivery outside Hermes + +provider failure + -> hermes-model-gate + -> Hermes owns titan-24: Ollama serves gpt-oss:20b + -> Wolf owns titan-24: gate returns 503; local inference yields +``` + +The operator agent pod runs on an ARM worker and does not reserve the GPU. +Ollama is the component on `titan-24` that owns GPU memory. The Lease +`hermes/titan-24-gpu-owner` controls whether the model gate admits local +inference. Normal operator conversations use independently authenticated Codex +first, so Wolf ownership does not need to block the web agent. + +## Who does what + +| Component | Responsibility | What it does not prove | +| --- | --- | --- | +| Hermes | Maintains the conversation, chooses tools/skills, reasons over evidence, saves reports, and enforces the approval experience | It is not the inference model and does not automatically have cluster-admin access | +| Ariadne | Collects and normalizes deterministic cluster, Jenkins, and quality evidence into a timestamped bundle; may optionally run a local diagnosis | Its diagnosis is not authoritative when stale, empty, or contradicted by the bundle | +| Jenkins | Runs the suites and retains console/artifact evidence | A final `exit 1` alone does not identify the first failed gate | +| Pushgateway | Receives suite/build quality telemetry | A cumulative counter is not the current build result | +| VictoriaMetrics | Stores and answers PromQL for quality and environment telemetry | A Grafana color is not direct proof of root cause | +| Grafana | Visualizes and alerts on VictoriaMetrics data | A red panel must be traced to its query, labels, time range, and raw series | +| Flux | Applies reviewed Git state to the cluster | A recent reconciliation is correlation until the changed path matches the failure | +| Kubernetes | Supplies workload, event, log, node, storage, and dependency evidence | An unrelated unhealthy pod is not proof that CI failed because of the cluster | + +Direct Jenkins artifact requests currently require authorization and can return +HTTP 403. That does not break the workflow: Ariadne retains selected Jenkins +console tails and named artifact contents in its deterministic bundle. A report +must say `retained Ariadne evidence` when that fallback is used; it must not +pretend direct Jenkins access succeeded. + +## The actual supervised triage algorithm + +1. Classify the request as test/build triage, service health, or alert tuning. +2. Read the latest Ariadne diagnosis and deterministic bundle with HTTP GET. +3. Compare their timestamps. The deterministic bundle remains authoritative. +4. Select a terminal failed build. Keep running or unknown builds out of the + terminal-failure list. +5. Identify the first enforced failed gate in this order: + `style -> loc -> coverage -> tests -> gate_glue -> sonarqube -> supply_chain`. +6. Cite the smallest decisive Jenkins log/artifact evidence. State whether it + came directly from Jenkins or from Ariadne retention. +7. Query the matching build/check/test series in VictoriaMetrics. Distinguish a + current gauge from a cumulative counter and ignore zero-valued failure + series. +8. Correlate recent Gitea commits and the relevant Flux revision by timestamp + and affected path. Recency alone is not causation. +9. Check only the Kubernetes resources capable of explaining that failure. + Classify observations as direct, contributing candidate, or background. +10. Return `Finding`, `Confidence`, `Evidence`, `Likely cause`, `Blast radius`, + `Next checks`, `Repo-side fix`, and `Approval required`. +11. Stop at a proposal. A human reviews the patch or action; Git and Flux remain + the delivery path. + +## What Hermes can read and what it cannot do + +The live `hermes-triage` ServiceAccount can get/list/watch ordinary workload, +log, event, ingress, storage, Flux, and image-automation metadata. It cannot +read Kubernetes Secret values, create exec sessions, patch Deployments, or +patch Flux Kustomizations. The configured command deny list also blocks common +Kubernetes and Flux mutations. + +The security boundary is layered: + +1. Keycloak authenticates the person. +2. oauth2-proxy restricts the operator surface to Brad. +3. The operator has a separate namespace, PVC, configuration, and ServiceAccount. +4. Kubernetes RBAC is the hard API authorization boundary. +5. NetworkPolicy limits reachable paths where configured. +6. Hermes approvals and instructions provide a user-facing safety layer. + +A skill is procedure and context, not a permission grant. Adding a sentence to +a skill cannot bypass Kubernetes RBAC. + +## What โ€œHermes learns the workflowโ€ means + +It does not silently retrain model weights on cluster data. Learning here means +that a successful repeated procedure is written as a reusable skill on the +operator PVC or versioned in Git. The skill describes when it should trigger, +the evidence order, interpretation rules, output contract, and safety boundary. + +The current workflow has a versioned top-level skill and persisted specialist +skills for: + +- orchestration and evidence reporting; +- retained Jenkins evidence; +- quality metrics; +- Gitea/Flux correlation; +- Kubernetes failure classification; +- Grafana metric provenance; +- Soteria backup health; +- approval-required actions. + +The model still evaluates fresh variable evidence on every incident. The skill +makes the process repeatable; it does not freeze the answer. + +## The four operator skills to remember + +- `triage-titan-test-failures`: CI, tests, builds, quality gates, and suspected + test-environment regressions. +- `triage-atlas-service-health`: active service or cluster incidents and red + high-level health panels. +- `tune-atlas-alerts`: noisy alerts, impossible metrics, bad PromQL, and + generator-owned Grafana corrections. +- `master-hermes-on-atlas`: hands-on training, assessments, architecture, and + claim audits. + +Do not start by choosing raw tools. State the operational question and let the +skill route to the narrow evidence source. + +## Two live, repeatable proof cases + +### Proof 6: Soteria build 272 + +- Terminal result: failure. +- Tests: 318 passed, zero failed. +- Local coverage: 96.195 percent. +- First enforced failure: SonarQube `new_coverage=0.0` against threshold 80. +- Runtime: Soteria remained Ready. +- Correct conclusion: a Sonar new-code policy/reporting problem, not a test or + Kubernetes capacity failure. +- Correct restraint: do not weaken the policy until scanner import and baseline + evidence identifies whether the issue is configuration or legitimate new + uncovered code. + +### Proof 7: Ananke build 242 + +- Terminal result: failure. +- Local coverage gate: 61.8 percent and failed. +- Other checks: tests, SonarQube, supply-chain, LOC, docs naming, and gate glue + were healthy. +- Sonar new-code coverage: 93.1 percent, which is a different scope. +- Correct conclusion: local coverage input/scope caused the enforced failure; + do not call the Sonar number contradictory without comparing inputs. +- Correct restraint: do not lower the threshold until the retained local gate + inputs are readable. + +The full sessions and redacted Markdown exports are in Sessions and +`Files/triage-proof`. + +## Five-minute demonstration + +1. Open Models and show `openai-codex/gpt-5.6-terra` as primary and + `gpt-oss:20b` as local fallback. +2. Open Skills and show the four operator skills plus the persisted component + skills. +3. Open `Proof 6 - Soteria 272 Sonar new coverage triage` in Sessions. +4. Point out the timestamped bundle, exact build/artifact evidence, metric + corroboration, healthy runtime, fact/inference separation, and approval + boundary. +5. Open Proof 7 and explain why local coverage and Sonar new-code coverage can + differ without either number being fabricated. +6. End by showing that Hermes proposes a repo-side correction but cannot patch + the Deployment or Flux Kustomization with its ServiceAccount. + +Use this short explanation: + +> Hermes is the persistent, permissioned workflow layer around the model. In +> this cluster Ariadne collects deterministic evidence, Hermes correlates it +> with retained Jenkins artifacts, Git/Flux state, Kubernetes health, and +> quality metrics, and then returns a supervised proposal. Repeated procedures +> become explicit skills. Kubernetes RBAC prevents the agent from turning a +> diagnosis into an unreviewed infrastructure change. + +## Prompts that exercise the real system + +- `Use $triage-titan-test-failures. Triage the worst current terminal CI failure and link every decisive piece of evidence.` +- `Use $triage-titan-test-failures. Triage Ananke build 242. Separate local coverage from Sonar new-code coverage.` +- `Use $triage-atlas-service-health. Explain the VictoriaMetrics outage from current state and retained events; separate recovered impact from current impact.` +- `Use $tune-atlas-alerts. Trace one currently firing alert to its generated source and raw PromQL, but do not edit it.` +- `Use $master-hermes-on-atlas. Assess me on the request path and permission boundary. One question at a time.` + +## Honest limits + +- Hermes does not currently apply production or cluster changes autonomously. +- Direct Jenkins console/artifact access may be forbidden; Ariadne retention is + the current fallback and must be named as provenance. +- Local `gpt-oss:20b` is useful for bounded work but is slower and less reliable + for multi-source triage than Codex. +- The current Ariadne local diagnosis can return an empty model response. That + does not invalidate the deterministic bundle or prevent Codex-backed Hermes + from triaging it. +- A single successful case demonstrates capability, not mastery. Mastery means + leading different real incidents, improving the skill after failures, and + teaching the architecture without prompts. + +## Your shortest path to fluency + +1. Explain the request diagram without looking. +2. Explain why Ariadne and Hermes are separate components. +3. Reproduce Proof 6 from the raw bundle and metrics. +4. Reproduce Proof 7 and explain the two coverage scopes. +5. Produce the live RBAC allow/deny matrix with `kubectl auth can-i`. +6. Create one small writable skill from a repeated sub-workflow, test a trigger + and non-trigger case, then improve it. +7. Lead one new incident while Hermes coaches rather than answers for you. +8. Teach the five-minute demonstration to another person. + +At that point the interview claims are demonstrable. Continue repeating real +incidents before describing yourself as fully autonomous or the system as +self-correcting. diff --git a/services/hermes/agent-ingress.yaml b/services/hermes/agent-ingress.yaml index 4ef033cd..9581a95b 100644 --- a/services/hermes/agent-ingress.yaml +++ b/services/hermes/agent-ingress.yaml @@ -21,6 +21,6 @@ spec: pathType: Prefix backend: service: - name: hermes + name: oauth2-proxy-hermes port: - name: dashboard + name: http diff --git a/services/hermes/ariadne-handoff-rbac.yaml b/services/hermes/ariadne-handoff-rbac.yaml new file mode 100644 index 00000000..a8da16cc --- /dev/null +++ b/services/hermes/ariadne-handoff-rbac.yaml @@ -0,0 +1,30 @@ +# services/hermes/ariadne-handoff-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ariadne-gpu-handoff + namespace: hermes +rules: + - apiGroups: ["coordination.k8s.io"] + resources: + - leases + resourceNames: + - titan-24-gpu-owner + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ariadne-gpu-handoff + namespace: hermes +subjects: + - kind: ServiceAccount + name: ariadne + namespace: maintenance +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ariadne-gpu-handoff diff --git a/services/hermes/configmap.yaml b/services/hermes/configmap.yaml index e751f45a..becb0da1 100644 --- a/services/hermes/configmap.yaml +++ b/services/hermes/configmap.yaml @@ -9,11 +9,41 @@ metadata: data: config.yaml: | model: - provider: custom - default: qwen2.5:7b-instruct-q4_0 - model: qwen2.5:7b-instruct-q4_0 - base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1 - api_key: ollama + provider: openai-codex + default: gpt-5.6-terra + model: gpt-5.6-terra + + fallback_providers: + - provider: custom + model: gpt-oss:20b + base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 + api_key: ollama + + agent: + api_max_retries: 1 + + platform_toolsets: + cli: + - clarify + - file + - session_search + - skills + - terminal + - todo + - web + api_server: + - clarify + - file + - session_search + - skills + - terminal + - todo + - web + + skills: + creation_nudge_interval: 15 + external_dirs: + - /opt/data/workspace/skills terminal: backend: local @@ -22,7 +52,7 @@ data: home_mode: auto approvals: - mode: manual + mode: smart deny: - "*kubectl apply*" - "*kubectl delete*" @@ -80,6 +110,12 @@ data: read-only state. Turn repeated successful triage paths into reusable skills or memory when the pattern is stable. + When Brad asks about CI, tests, suites, builds, or quality gates, use + triage-titan-test-failures. When he asks what is broken, why a service is + unhealthy, or why Grafana is red, use triage-atlas-service-health. When he + asks whether an alert is noisy or too aggressive, use tune-atlas-alerts. + Do not make him choose a tool, page, or evidence source first. + Stay Flux-first. Do not mutate the cluster directly. Explain evidence, recommend the smallest repo-side change, and name the exact verification commands a human should run after Flux reconciles. @@ -89,6 +125,11 @@ data: You are Hermes running inside the Titan Kubernetes cluster as a read-only testing and operations triage assistant. + Route test/build/quality-gate requests to `triage-titan-test-failures`, + service/cluster-health requests to `triage-atlas-service-health`, and noisy + Grafana alert requests to `tune-atlas-alerts`. Ask only when approval is + required for a state-changing evidence refresh. + Ariadne owns deterministic evidence collection and local diagnosis. Start every testing triage by reading: @@ -122,3 +163,181 @@ data: Do not run mutating commands such as `kubectl apply`, `delete`, `scale`, `patch`, `cordon`, `uncordon`, `drain`, or `rollout restart`. Do not read Kubernetes Secret values. Draft repo changes or operator steps instead. + START-HERE.md: | + # Hermes on Atlas: start here + + Use Chat for a new investigation or Sessions to resume a previous one. + The shortest useful prompts are: + + - `Triage the latest test failure.` + - `What is broken in the cluster right now?` + - `Why is Grafana red, and which alerts are noise?` + - `Triage build and save the report.` + + Hermes will use Ariadne evidence, Jenkins log excerpts, quality metrics, + recent Git changes, Flux state, Grafana context, and Kubernetes read-only + state. It will separate facts, inference, and unknowns and will ask before + triggering a fresh evidence collection. + + Hermes now has three focused operator skills: + + 1. `triage-titan-test-failures` for all ten custom CI suites. + 2. `triage-atlas-service-health` for any live namespace or service. + 3. `tune-atlas-alerts` for evidence-backed alert and dashboard corrections. + + Validated CI examples are available under Sessions, including: + + 1. `Proof 6 - Soteria 272 Sonar new coverage triage` + 2. `Proof 7 - Ananke 242 local coverage triage` + + Their saved reports are in Files under `triage-proof/`. Read + `HERMES-OPERATOR-RUNBOOK.md` for the system mental model and five-minute + demonstration. + + Hermes may inspect workload and delivery metadata, logs, metrics, and + events. Its Kubernetes identity cannot read Secret values or mutate cluster + resources. Apply fixes through the titan-iac GitOps workflow after review. + HERMES-CAPABILITIES.md: | + # What Hermes can do on Atlas + + ## Working now + + - Triage all custom CI suites: Ananke, Ariadne, Atlasbot, bstein_home, + data_prepper, Lesavka, Metis, Pegasus, Soteria, and titan_iac. + - Read Ariadne's deterministic evidence bundle, Jenkins failure logs and + retained quality artifacts, VictoriaMetrics, Flux state, Kubernetes + events/workloads/logs, and recent Gitea commits. + - Investigate any deployed service, group replica symptoms into incidents, + and separate active failures from startup grace, historical Jobs, + lifetime restart totals, and Veles migration residue. + - Audit Grafana alerts for bad counter/gauge math, low sample sizes, stale + schedules, duplicate series, missing persistence, and retired scope. + - Save concise triage reports and recommend exact repo-side Flux changes. + + ## Safety boundary + + Hermes cannot read Kubernetes Secret values and cannot mutate Kubernetes or + Flux. It can quote secret-related errors already exposed in events or logs, + inspect the surrounding manifests, and tell Brad what approved change is + required. This keeps cluster operation supervised. + + ## Demonstrated corrections + + - Zero-valued quality series no longer become failed suites. + - Running Jenkins builds remain in-progress instead of failed. + - Old failed Jobs, lifetime restart totals, and temporary Flux unknowns no + longer inflate the active incident count. + - Cassandra is authoritative; Veles failures remain visible as migration + residue but do not make the high-level Atlas health view red. + - Worker readiness uses the actual 18 Kubernetes workers. + - Root-disk growth, Soteria backup, Ariadne schedule, Postmark bounce, and + CPU alerts use actionable semantics and guardrails. + + The evidence and verification for these changes are in + `triage-proof/ATLAS-TRIAGE-PROOFS.md`. + + ## Good next prompts + + - `Use $triage-atlas-service-health. What is broken right now?` + - `Use $triage-titan-test-failures. Triage the worst current suite.` + - `Use $tune-atlas-alerts. Audit alerts fired in the last 24 hours.` + - `Save this investigation as a reusable skill after I approve the pattern.` + ATLAS-TRIAGE-PROOFS.md: | + # Atlas triage proofs + + This file is the short evidence trail for what Hermes can do today. The + operator skills remain read-only: fixes are reviewed and delivered through + Git and Flux. + + ## Proof 1: Cassandra secret-sync recovery + + - Finding: `cassandra-vault-sync` was stuck because the expected + `VELES_BYOK_ENCRYPTION_KEY` migration input was absent from Cassandra's + Vault path. + - Action: the existing Flux-tracked, suspended one-shot migration Job was + enabled, completed successfully, and immediately returned to suspended. + - Verification: Cassandra frontend, backend, PostgreSQL, and Vault sync all + became Ready. No Kubernetes Secret value was read or committed. + - Why this matters: Hermes can correlate workload state, logs, Vault policy + manifests, and migration ownership without confusing Veles residue with + a current Veles outage. + + ## Proof 2: Grafana alert-noise correction + + - Removed retired Veles pods and completed Jobs from high-level incident + counts while preserving them on drill-down dashboards. + - Added 15-minute persistence to Pending, CrashLoopBackOff, and image-pull + summaries. + - Replaced invalid `increase()` use on gauges, bounded CPU percentages, + required meaningful Postmark sample sizes, and scoped Soteria/Ariadne + alerts to configured current work. + - Live verification on 2026-08-03: `Problem Pods=0`, + `CrashLoop/ImagePull=0`, and `Workers Ready=18/18`. + + ## Proof 3: all-suite testing triage + + - Ariadne build 384 passed 676 tests and the exact coverage contract: + 82 source files at or above 95%. + - Deployed image: `registry.bstein.dev/bstein/ariadne:0.1.0-384`. + - The deterministic bundle covers Ananke, Ariadne, Atlasbot, bstein_home, + data_prepper, Lesavka, Metis, Pegasus, Soteria, and titan_iac. + - Zero-valued quality metrics are healthy, running Jenkins builds are + in-progress, old failed Jobs are historical, and Veles objects are + migration residue. They no longer inflate the active failure set. + - Failed builds still include direct Jenkins console and retained artifact + links so Hermes can identify the first failed gate and smallest repo fix. + + ## Proof 4: Cassandra generator failure classification + + - Finding: the Cassandra serving path is healthy, but generation Job + `cassandra-generator-3ce198971b` failed. + - Evidence: the primary OpenAI request returned HTTP 429 because its credit + balance was exhausted. Codex CLI fallback ran, but the strict promotion + gate still rejected unresolved high-impact hint claims and blocked the + semantic scenarios. + - Impact: one generation capability/request failed; this is not a Cassandra + registry outage and not a Veles outage. + - Next action: restore provider credits or continue through Codex fallback, + then resolve the reported hint claims before retrying. This needs operator + approval because it can spend money or rerun an expensive generation. + + ## Proof 5: live Hermes Soteria triage + + - Hermes read the fresh 2026-08-03 Ariadne bundle without an approval + timeout and chose Soteria build 270 as the worst terminal failure. + - It proved that all 318 tests passed and local coverage was 96.195%, then + isolated the enforced failure to SonarQube reporting new-code coverage + as 0.0%. It did not blame the optional supply-chain advisory. + - It kept running titan-iac, Data Prepper, and Lesavka builds out of the + terminal-failure list and requested approval before any rerun or edit. + + ## Proof 6: Soteria build 272 Sonar new-coverage triage + + - Hermes read the fresh 2026-08-04 Ariadne bundle and selected terminal + Soteria build 272. + - It proved that all 318 tests passed and local coverage was 96.195%, then + isolated the enforced failure to SonarQube `new_coverage=0.0` against the + threshold of 80. + - It corroborated the retained Jenkins evidence with current quality + metrics and healthy Soteria/Flux state, and did not blame Kubernetes. + - It proposed no change until scanner import and baseline evidence can + distinguish configuration error from genuinely uncovered new code. + + ## Proof 7: Ananke build 242 local-coverage triage + + - Hermes selected terminal Ananke build 242 and isolated the first failure + to the local coverage gate at 61.8%. + - It showed that tests, SonarQube, supply chain, LOC, docs naming, gate glue, + Flux, and cluster readiness were healthy. + - It distinguished the local 61.8% scope from SonarQube's 93.1% new-code + scope instead of treating the numbers as interchangeable. + - Direct Jenkins artifact reads returned HTTP 403, so it explicitly named + Ariadne's retained bundle as evidence provenance and stopped short of an + unsupported fix. + + ## Use Hermes next + + - `What is broken in the cluster right now?` + - `Triage the worst current CI suite and link the evidence.` + - `Audit Grafana alerts from the last 24 hours and separate incidents from noise.` + - `Explain the Cassandra generator failure and give me the smallest safe next step.` diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 75437fe6..247d947c 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -20,10 +20,11 @@ spec: labels: app: hermes annotations: - ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0 + ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation + ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback ai.bstein.dev/role: testing-triage ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred) - ai.bstein.dev/config-rev: "20260721-hermes-rollout-deadline" + ai.bstein.dev/config-rev: "20260804-root-operator-docs" spec: serviceAccountName: hermes-triage automountServiceAccountToken: true @@ -50,6 +51,7 @@ spec: - titan-13 - titan-15 - titan-17 + - titan-18 - titan-19 preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -80,10 +82,18 @@ spec: - -c - | set -eu - mkdir -p /opt/data/workspace /opt/data/home/.local/bin /opt/data/logs + mkdir -p /opt/data/workspace/triage-proof /opt/data/home/.local/bin /opt/data/logs cp /config/config.yaml /opt/data/config.yaml cp /config/SOUL.md /opt/data/SOUL.md cp /config/AGENTS.md /opt/data/workspace/AGENTS.md + cp /config/START-HERE.md /opt/data/workspace/START-HERE.md + cp /config/HERMES-CAPABILITIES.md /opt/data/workspace/HERMES-CAPABILITIES.md + cp /guide/OPERATOR-RUNBOOK.md /opt/data/workspace/HERMES-OPERATOR-RUNBOOK.md + cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/workspace/triage-proof/ATLAS-TRIAGE-PROOFS.md + cp /config/START-HERE.md /opt/data/START-HERE.md + cp /config/HERMES-CAPABILITIES.md /opt/data/HERMES-CAPABILITIES.md + cp /guide/OPERATOR-RUNBOOK.md /opt/data/HERMES-OPERATOR-RUNBOOK.md + cp /config/ATLAS-TRIAGE-PROOFS.md /opt/data/ATLAS-TRIAGE-PROOFS.md touch /opt/data/.env if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" @@ -99,8 +109,8 @@ spec: mountPath: /opt/data - name: config mountPath: /config - - name: tools - mountPath: /opt/data/home/.local/bin + - name: operator-guide + mountPath: /guide resources: requests: cpu: 25m @@ -134,7 +144,7 @@ spec: memory: 64Mi containers: - name: hermes - image: nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f imagePullPolicy: IfNotPresent args: - gateway @@ -189,7 +199,20 @@ spec: - name: home mountPath: /opt/data - name: tools - mountPath: /opt/data/home/.local/bin + mountPath: /usr/local/bin/kubectl + subPath: kubectl + - name: triage-skill + mountPath: /opt/data/workspace/skills/triage-titan-test-failures + readOnly: true + - name: mastery-skill + mountPath: /opt/data/workspace/skills/master-hermes-on-atlas + readOnly: true + - name: service-health-skill + mountPath: /opt/data/workspace/skills/triage-atlas-service-health + readOnly: true + - name: alert-tuning-skill + mountPath: /opt/data/workspace/skills/tune-atlas-alerts + readOnly: true readinessProbe: httpGet: path: /api/status @@ -218,5 +241,54 @@ spec: - name: config configMap: name: hermes-config + - name: operator-guide + configMap: + name: hermes-operator-guide - name: tools emptyDir: {} + - name: triage-skill + configMap: + name: hermes-triage-skill + items: + - key: SKILL.md + path: SKILL.md + - key: openai.yaml + path: agents/openai.yaml + - name: mastery-skill + configMap: + name: hermes-mastery-skill + items: + - key: SKILL.md + path: SKILL.md + - key: openai.yaml + path: agents/openai.yaml + - key: architecture.md + path: references/architecture.md + - key: curriculum.md + path: references/curriculum.md + - key: incident-drills.md + path: references/incident-drills.md + - key: mastery-rubric.md + path: references/mastery-rubric.md + - key: two-hour-proof-sprint.md + path: references/two-hour-proof-sprint.md + - name: service-health-skill + configMap: + name: hermes-service-health-skill + items: + - key: SKILL.md + path: SKILL.md + - key: openai.yaml + path: agents/openai.yaml + - key: service-map.md + path: references/service-map.md + - name: alert-tuning-skill + configMap: + name: hermes-alert-tuning-skill + items: + - key: SKILL.md + path: SKILL.md + - key: openai.yaml + path: agents/openai.yaml + - key: alert-review.md + path: references/alert-review.md diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 96e642e4..6033f3e5 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -4,11 +4,62 @@ kind: Kustomization namespace: hermes resources: - namespace.yaml + - vault-serviceaccount.yaml - configmap.yaml - rbac.yaml - pvc.yaml + - model-gate-rbac.yaml + - ariadne-handoff-rbac.yaml + - model-gate-state.yaml + - model-gate-configmap.yaml + - model-gate-deployment.yaml + - networkpolicy.yaml - ollama-deployment.yaml - deployment.yaml - service.yaml + - oauth2-proxy.yaml - agent-certificate.yaml - agent-ingress.yaml + +configMapGenerator: + - name: hermes-operator-guide + namespace: hermes + files: + - OPERATOR-RUNBOOK.md=NOTES.md + options: + disableNameSuffixHash: true + - name: hermes-triage-skill + namespace: hermes + files: + - SKILL.md=skills/triage-titan-test-failures/SKILL.md + - openai.yaml=skills/triage-titan-test-failures/agents/openai.yaml + options: + disableNameSuffixHash: true + - name: hermes-mastery-skill + namespace: hermes + files: + - SKILL.md=skills/master-hermes-on-atlas/SKILL.md + - openai.yaml=skills/master-hermes-on-atlas/agents/openai.yaml + - architecture.md=skills/master-hermes-on-atlas/references/architecture.md + - curriculum.md=skills/master-hermes-on-atlas/references/curriculum.md + - incident-drills.md=skills/master-hermes-on-atlas/references/incident-drills.md + - mastery-rubric.md=skills/master-hermes-on-atlas/references/mastery-rubric.md + - two-hour-proof-sprint.md=skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md + options: + disableNameSuffixHash: true + - name: hermes-service-health-skill + namespace: hermes + files: + - SKILL.md=skills/triage-atlas-service-health/SKILL.md + - openai.yaml=skills/triage-atlas-service-health/agents/openai.yaml + - service-map.md=skills/triage-atlas-service-health/references/service-map.md + options: + disableNameSuffixHash: true + - name: hermes-alert-tuning-skill + namespace: hermes + files: + - SKILL.md=skills/tune-atlas-alerts/SKILL.md + - openai.yaml=skills/tune-atlas-alerts/agents/openai.yaml + - alert-review.md=skills/tune-atlas-alerts/references/alert-review.md + options: + disableNameSuffixHash: true diff --git a/services/hermes/model-gate-configmap.yaml b/services/hermes/model-gate-configmap.yaml new file mode 100644 index 00000000..cf179d84 --- /dev/null +++ b/services/hermes/model-gate-configmap.yaml @@ -0,0 +1,151 @@ +# services/hermes/model-gate-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-model-gate + namespace: hermes +data: + model_gate.py: | + #!/usr/bin/env python3 + """Fail-closed proxy that admits local inference only while Hermes owns titan-24.""" + + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + import json + import os + from pathlib import Path + import ssl + import threading + import time + from urllib.error import HTTPError, URLError + from urllib.request import Request, urlopen + + + LISTEN_HOST = os.environ.get("LISTEN_HOST", "0.0.0.0") + LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8080")) + UPSTREAM_URL = os.environ.get("UPSTREAM_URL", "http://hermes-ollama.hermes.svc.cluster.local:11434").rstrip("/") + LEASE_NAMESPACE = os.environ.get("LEASE_NAMESPACE", "hermes") + LEASE_NAME = os.environ.get("LEASE_NAME", "titan-24-gpu-owner") + CACHE_TTL_SEC = float(os.environ.get("LEASE_CACHE_TTL_SEC", "1")) + API_HOST = os.environ.get("KUBERNETES_SERVICE_HOST", "kubernetes.default.svc") + API_PORT = os.environ.get("KUBERNETES_SERVICE_PORT_HTTPS", "443") + TOKEN_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/token") + CA_PATH = Path("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + LEASE_URL = ( + f"https://{API_HOST}:{API_PORT}/apis/coordination.k8s.io/v1/" + f"namespaces/{LEASE_NAMESPACE}/leases/{LEASE_NAME}" + ) + + _cache_lock = threading.Lock() + _cached_owner = "unavailable" + _cached_at = 0.0 + + + def _lease_owner() -> str: + """Return the current owner, failing closed when Kubernetes is unavailable.""" + + global _cached_at, _cached_owner + now = time.monotonic() + with _cache_lock: + if now - _cached_at < CACHE_TTL_SEC: + return _cached_owner + try: + token = TOKEN_PATH.read_text(encoding="utf-8").strip() + request = Request(LEASE_URL, headers={"Authorization": f"Bearer {token}"}) + context = ssl.create_default_context(cafile=str(CA_PATH)) + with urlopen(request, timeout=3, context=context) as response: + payload = json.load(response) + owner = str((payload.get("spec") or {}).get("holderIdentity") or "unavailable").strip() + except Exception: + owner = "unavailable" + _cached_owner = owner + _cached_at = now + return owner + + + class Handler(BaseHTTPRequestHandler): + """Proxy local model traffic while exposing health and ownership status.""" + + protocol_version = "HTTP/1.1" + + def _json(self, status: int, payload: dict) -> None: + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _local_allowed(self) -> tuple[bool, str]: + owner = _lease_owner() + return owner == "hermes", owner + + def _proxy(self) -> None: + allowed, owner = self._local_allowed() + if not allowed: + self._json( + 503, + { + "error": { + "message": f"local GPU inference unavailable while titan-24 owner is {owner}", + "type": "server_error", + }, + "gpu_owner": owner, + "fallback_required": True, + }, + ) + return + + length = int(self.headers.get("Content-Length", "0") or 0) + body = self.rfile.read(length) if length else None + headers = {"Content-Type": self.headers.get("Content-Type", "application/json")} + if self.headers.get("Accept"): + headers["Accept"] = self.headers["Accept"] + request = Request(f"{UPSTREAM_URL}{self.path}", data=body, headers=headers, method=self.command) + try: + response = urlopen(request, timeout=1800) + except HTTPError as exc: + response = exc + except (TimeoutError, URLError) as exc: + self._json(503, {"error": {"message": f"local model upstream unavailable: {exc}", "type": "server_error"}}) + return + + self.send_response(response.status) + content_type = response.headers.get("Content-Type") + if content_type: + self.send_header("Content-Type", content_type) + content_length = response.headers.get("Content-Length") + if content_length: + self.send_header("Content-Length", content_length) + else: + self.send_header("Connection", "close") + self.close_connection = True + self.send_header("Cache-Control", "no-store") + self.end_headers() + while True: + chunk = response.read(65536) + if not chunk: + break + self.wfile.write(chunk) + self.wfile.flush() + response.close() + + def do_GET(self) -> None: + if self.path == "/healthz": + self._json(200, {"status": "ok"}) + return + if self.path == "/gate/status": + allowed, owner = self._local_allowed() + self._json(200, {"gpu_owner": owner, "local_inference_allowed": allowed}) + return + self._proxy() + + def do_POST(self) -> None: + self._proxy() + + def log_message(self, format_string: str, *args) -> None: + print(f"model-gate {self.address_string()} {format_string % args}", flush=True) + + + if __name__ == "__main__": + ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler).serve_forever() diff --git a/services/hermes/model-gate-deployment.yaml b/services/hermes/model-gate-deployment.yaml new file mode 100644 index 00000000..a8c9511c --- /dev/null +++ b/services/hermes/model-gate-deployment.yaml @@ -0,0 +1,125 @@ +# services/hermes/model-gate-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-model-gate + namespace: hermes + labels: + app: hermes-model-gate +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: hermes-model-gate + template: + metadata: + labels: + app: hermes-model-gate + spec: + serviceAccountName: hermes-model-gate + securityContext: + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-04 + - titan-13 + - titan-15 + - titan-17 + - titan-19 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 90 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + containers: + - name: model-gate + image: python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df + imagePullPolicy: IfNotPresent + command: + - python + - /opt/model-gate/model_gate.py + ports: + - name: http + containerPort: 8080 + env: + - name: UPSTREAM_URL + value: http://hermes-ollama.hermes.svc.cluster.local:11434 + - name: LEASE_NAMESPACE + value: hermes + - name: LEASE_NAME + value: titan-24-gpu-owner + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + volumeMounts: + - name: script + mountPath: /opt/model-gate + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: script + configMap: + name: hermes-model-gate + defaultMode: 0555 + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-model-gate + namespace: hermes + labels: + app: hermes-model-gate +spec: + type: ClusterIP + selector: + app: hermes-model-gate + ports: + - name: http + port: 11434 + targetPort: http diff --git a/services/hermes/model-gate-rbac.yaml b/services/hermes/model-gate-rbac.yaml new file mode 100644 index 00000000..1c56556a --- /dev/null +++ b/services/hermes/model-gate-rbac.yaml @@ -0,0 +1,34 @@ +# services/hermes/model-gate-rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-model-gate + namespace: hermes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: hermes-model-gate + namespace: hermes +rules: + - apiGroups: ["coordination.k8s.io"] + resources: + - leases + resourceNames: + - titan-24-gpu-owner + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hermes-model-gate + namespace: hermes +subjects: + - kind: ServiceAccount + name: hermes-model-gate + namespace: hermes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hermes-model-gate diff --git a/services/hermes/model-gate-state.yaml b/services/hermes/model-gate-state.yaml new file mode 100644 index 00000000..75659118 --- /dev/null +++ b/services/hermes/model-gate-state.yaml @@ -0,0 +1,10 @@ +# services/hermes/model-gate-state.yaml +apiVersion: coordination.k8s.io/v1 +kind: Lease +metadata: + name: titan-24-gpu-owner + namespace: hermes + annotations: + kustomize.toolkit.fluxcd.io/ssa: IfNotPresent +spec: + holderIdentity: hermes diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml new file mode 100644 index 00000000..57bfa577 --- /dev/null +++ b/services/hermes/networkpolicy.yaml @@ -0,0 +1,30 @@ +# services/hermes/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-ollama-ingress + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-ollama + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-model-gate + ports: + - protocol: TCP + port: 11434 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: maintenance + podSelector: + matchLabels: + app: ariadne + ports: + - protocol: TCP + port: 11434 diff --git a/services/hermes/oauth2-proxy.yaml b/services/hermes/oauth2-proxy.yaml new file mode 100644 index 00000000..d9971f0d --- /dev/null +++ b/services/hermes/oauth2-proxy.yaml @@ -0,0 +1,161 @@ +# services/hermes/oauth2-proxy.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-operator-allowlist + namespace: hermes +data: + allowed-emails: | + brad@bstein.dev +--- +apiVersion: v1 +kind: Service +metadata: + name: oauth2-proxy-hermes + namespace: hermes + labels: + app: oauth2-proxy-hermes +spec: + selector: + app: oauth2-proxy-hermes + ports: + - name: http + port: 80 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth2-proxy-hermes + namespace: hermes + labels: + app: oauth2-proxy-hermes +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: oauth2-proxy-hermes + template: + metadata: + labels: + app: oauth2-proxy-hermes + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: hermes + vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/operator-oidc + vault.hashicorp.com/agent-inject-template-oidc-config: | + {{- with secret "kv/data/atlas/hermes/operator-oidc" -}} + client_id = "{{ .Data.data.client_id }}" + client_secret = "{{ .Data.data.client_secret }}" + cookie_secret = "{{ .Data.data.cookie_secret }}" + {{- end -}} + spec: + serviceAccountName: hermes-vault + automountServiceAccountToken: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-13 + - titan-15 + - titan-17 + - titan-18 + - titan-19 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 90 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi4 + containers: + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0@sha256:dcb6ff8dd21bf3058f6a22c6fa385fa5b897a9cd3914c88a2cc2bb0a85f8065d + imagePullPolicy: IfNotPresent + args: + - --provider=oidc + - --config=/vault/secrets/oidc-config + - --redirect-url=https://agent.bstein.dev/oauth2/callback + - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas + - --code-challenge-method=S256 + - --scope=openid profile email + - --email-domain=* + - --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails + - --set-xauthrequest=true + - --session-cookie-minimal=true + - --cookie-secure=true + - --cookie-samesite=lax + - --cookie-refresh=0 + - --cookie-expire=8h + - --upstream=http://hermes.hermes.svc.cluster.local:9119 + - --http-address=0.0.0.0:4180 + - --skip-provider-button=true + - --skip-jwt-bearer-tokens=true + - --cookie-domain=agent.bstein.dev + - --reverse-proxy=true + ports: + - name: http + containerPort: 4180 + readinessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 20 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: allowlist + mountPath: /etc/oauth2-proxy + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: allowlist + configMap: + name: hermes-operator-allowlist + - name: tmp + emptyDir: + sizeLimit: 64Mi diff --git a/services/hermes/ollama-deployment.yaml b/services/hermes/ollama-deployment.yaml index 946aa7b1..64a351f5 100644 --- a/services/hermes/ollama-deployment.yaml +++ b/services/hermes/ollama-deployment.yaml @@ -18,8 +18,8 @@ spec: labels: app: hermes-ollama annotations: - ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0 - ai.bstein.dev/gpu: accelerator MVP lane (titan-24) + ai.bstein.dev/model: gpt-oss:20b + ai.bstein.dev/gpu: titan-24 local-first lane spec: runtimeClassName: nvidia affinity: @@ -33,7 +33,8 @@ spec: - titan-24 volumes: - name: models - emptyDir: {} + persistentVolumeClaim: + claimName: hermes-models initContainers: - name: warm-model image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d @@ -44,7 +45,7 @@ spec: - name: OLLAMA_MODELS value: /root/.ollama - name: OLLAMA_MODEL - value: qwen2.5:7b-instruct-q4_0 + value: gpt-oss:20b - name: NVIDIA_VISIBLE_DEVICES value: all - name: NVIDIA_DRIVER_CAPABILITIES @@ -68,7 +69,7 @@ spec: nvidia.com/gpu.shared: 1 limits: cpu: "4" - memory: 10Gi + memory: 16Gi nvidia.com/gpu.shared: 1 containers: - name: ollama @@ -82,6 +83,16 @@ spec: value: 0.0.0.0 - name: OLLAMA_KEEP_ALIVE value: 6h + - name: OLLAMA_CONTEXT_LENGTH + value: "64000" + - name: OLLAMA_FLASH_ATTENTION + value: "1" + - name: OLLAMA_KV_CACHE_TYPE + value: q8_0 + - name: OLLAMA_MAX_LOADED_MODELS + value: "1" + - name: OLLAMA_NUM_PARALLEL + value: "1" - name: OLLAMA_MODELS value: /root/.ollama - name: NVIDIA_VISIBLE_DEVICES @@ -100,10 +111,10 @@ spec: timeoutSeconds: 5 resources: requests: - cpu: "2" - memory: 8Gi + cpu: "8" + memory: 24Gi nvidia.com/gpu.shared: 1 limits: - cpu: "6" - memory: 12Gi + cpu: "16" + memory: 40Gi nvidia.com/gpu.shared: 1 diff --git a/services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js b/services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js new file mode 100644 index 00000000..f927802f --- /dev/null +++ b/services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js @@ -0,0 +1,88 @@ +(function () { + "use strict"; + + var pluginName = "atlas-operator-guide"; + var storageKey = "atlas.operator-guide.hidden.v1"; + var prompt = [ + "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint.", + "I will perform every setup and evidence step myself.", + "Start with step 1 only, require live proof, and stop at every mutation or approval boundary.", + "First confirm this session is using openai-codex/gpt-5.6-terra rather than the local fallback." + ].join(" "); + + function OperatorGuide() { + var sdk = window.__HERMES_PLUGIN_SDK__; + var React = sdk.React; + var state = sdk.hooks.useState(function () { + return window.localStorage.getItem(storageKey) === "1"; + }); + var hidden = state[0]; + var setHidden = state[1]; + var copiedState = sdk.hooks.useState(false); + var copied = copiedState[0]; + var setCopied = copiedState[1]; + + if (hidden) { + return React.createElement( + "button", + { + className: "atlas-guide-restore", + onClick: function () { + window.localStorage.removeItem(storageKey); + setHidden(false); + }, + type: "button" + }, + "Show Atlas operator walkthrough" + ); + } + + function copyPrompt() { + navigator.clipboard.writeText(prompt).then(function () { + setCopied(true); + window.setTimeout(function () { setCopied(false); }, 2500); + }); + } + + return React.createElement( + "section", + { className: "atlas-guide-card", role: "region", "aria-label": "Atlas operator walkthrough" }, + React.createElement( + "div", + { className: "atlas-guide-heading" }, + React.createElement("div", null, + React.createElement("strong", null, "Start here: two-hour Hermes proof sprint"), + React.createElement("p", null, "You do the setup and evidence work. Hermes coaches, checks, and stops at approval boundaries.") + ), + React.createElement( + "button", + { + className: "atlas-guide-hide", + onClick: function () { + window.localStorage.setItem(storageKey, "1"); + setHidden(true); + }, + type: "button" + }, + "Hide" + ) + ), + React.createElement( + "ol", + { className: "atlas-guide-steps" }, + React.createElement("li", null, "Confirm Models shows openai-codex/gpt-5.6-terra for the coaching session."), + React.createElement("li", null, "Copy the sprint prompt, paste it into Chat, and press Enter."), + React.createElement("li", null, "Complete one live triage and build one writable skill yourself.") + ), + React.createElement( + "div", + { className: "atlas-guide-actions" }, + React.createElement("a", { className: "atlas-guide-button", href: "/models" }, "Open Models"), + React.createElement("button", { className: "atlas-guide-button", onClick: copyPrompt, type: "button" }, copied ? "Prompt copied" : "Copy sprint prompt"), + React.createElement("a", { className: "atlas-guide-link", href: "/skills" }, "Inspect skills") + ) + ); + } + + window.__HERMES_PLUGINS__.registerSlot(pluginName, "chat:top", OperatorGuide); +}()); diff --git a/services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css b/services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css new file mode 100644 index 00000000..ffecc67c --- /dev/null +++ b/services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css @@ -0,0 +1,64 @@ +.atlas-guide-card { + border: 1px solid rgba(247, 201, 72, 0.7); + background: rgba(247, 201, 72, 0.08); + color: inherit; + padding: 0.75rem 1rem; + font-family: "Mondwest", monospace; +} + +.atlas-guide-heading { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; +} + +.atlas-guide-heading strong { + color: #f7c948; + font-family: "Rules Expanded", sans-serif; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.atlas-guide-heading p { + margin: 0.25rem 0 0; + opacity: 0.8; +} + +.atlas-guide-steps { + margin: 0.6rem 0; + padding-left: 1.25rem; +} + +.atlas-guide-actions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.5rem; +} + +.atlas-guide-button, +.atlas-guide-hide, +.atlas-guide-restore { + border: 1px solid rgba(247, 201, 72, 0.7); + background: transparent; + color: #f7c948; + cursor: pointer; + padding: 0.35rem 0.6rem; + text-decoration: none; +} + +.atlas-guide-button:hover, +.atlas-guide-hide:hover, +.atlas-guide-restore:hover { + background: rgba(247, 201, 72, 0.14); +} + +.atlas-guide-hide, +.atlas-guide-restore { + font-size: 0.75rem; +} + +.atlas-guide-link { + color: #f7c948; +} diff --git a/services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json b/services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json new file mode 100644 index 00000000..440a16d0 --- /dev/null +++ b/services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json @@ -0,0 +1,16 @@ +{ + "name": "atlas-operator-guide", + "label": "Atlas Operator Guide", + "description": "Hands-on onboarding for supervised Atlas triage and Hermes skill building.", + "icon": "Sparkles", + "version": "1.0.0", + "tab": { + "path": "/atlas-operator-guide", + "hidden": true + }, + "slots": [ + "chat:top" + ], + "entry": "dist/index.js", + "css": "dist/style.css" +} diff --git a/services/hermes/pvc.yaml b/services/hermes/pvc.yaml index 85812f5e..9ff1830f 100644 --- a/services/hermes/pvc.yaml +++ b/services/hermes/pvc.yaml @@ -13,3 +13,18 @@ spec: resources: requests: storage: 4Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hermes-models + namespace: hermes + labels: + app: hermes-ollama +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 30Gi diff --git a/services/hermes/rbac.yaml b/services/hermes/rbac.yaml index ae44ff15..bad9a4d2 100644 --- a/services/hermes/rbac.yaml +++ b/services/hermes/rbac.yaml @@ -22,6 +22,7 @@ rules: - pods - pods/log - replicationcontrollers + - serviceaccounts - services verbs: ["get", "list", "watch"] - apiGroups: ["apps"] @@ -54,6 +55,12 @@ rules: - gitrepositories - helmrepositories verbs: ["get", "list", "watch"] + - apiGroups: ["image.toolkit.fluxcd.io"] + resources: + - imagepolicies + - imagerepositories + - imageupdateautomations + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -67,4 +74,3 @@ subjects: - kind: ServiceAccount name: hermes-triage namespace: hermes - diff --git a/services/hermes/skills/master-hermes-on-atlas/SKILL.md b/services/hermes/skills/master-hermes-on-atlas/SKILL.md new file mode 100644 index 00000000..e33e487a --- /dev/null +++ b/services/hermes/skills/master-hermes-on-atlas/SKILL.md @@ -0,0 +1,84 @@ +--- +name: master-hermes-on-atlas +description: Coach Brad through hands-on mastery of the live Atlas Hermes deployment, including its web and CLI interfaces, local GPU inference and Codex fallback, tools, skills, sessions, profiles, channels, security boundaries, and evidence-driven test-failure triage. Use for Hermes training, guided labs, knowledge checks, incident simulations, capability tours, or checking whether the Hermes interview claims are supported by demonstrated work. +--- + +# Master Hermes on Atlas + +Act as a demanding, practical coach. Teach the deployed system, not a generic +Hermes installation. Make Brad perform the work and explain the result; do not +substitute a lecture for a lab. + +## Select the training model + +Use `openai-codex/gpt-5.6-terra` for assessments, multi-reference labs, incident +grading, and skill evaluation. If the active provider is the local +`gpt-oss:20b`, stop before reading the reference files and ask Brad to select +Codex in Models or start a Codex-backed session. Continue locally only when Brad +explicitly asks to evaluate the local model itself or requests a lightweight +single-question review. + +## Start or resume training + +1. Read `references/architecture.md` before teaching deployment-specific facts. +2. Read `references/curriculum.md` to select the next lab. +3. When Brad asks for the two-hour proof sprint, read + `references/two-hour-proof-sprint.md` and run it in order. Brad performs + every required UI action and evidence check; coach and verify without doing + the learning-critical steps for him. +4. Ask whether to assess, resume, or choose a lab. If no progress record exists, + default to a five-question assessment followed by the first weak area. +5. Give one bounded task at a time. State the goal, safety boundary, exact + success evidence, and at most one initial hint. +6. Wait for Brad's answer or observed command output before revealing the + explanation. +7. Grade with `references/mastery-rubric.md`. Separate demonstrated ability + from verbal familiarity. +8. Offer a concise progress update. Write it to + `/opt/data/workspace/hermes-training/progress.md` only after Brad explicitly + approves the file change. + +## Run live exercises safely + +- Use read-only commands by default: `kubectl get`, `describe`, `logs`, `auth + can-i`, HTTP GET, `hermes status`, and Hermes list/status commands. +- Never read Kubernetes Secret values. Never use Vault reads as a training + shortcut. Redact bearer tokens, cookies, device codes, and credentials. +- Never mutate Kubernetes, Flux, Jenkins, Git, credentials, inference ownership, + channels, cron jobs, plugins, MCP servers, or profiles merely to demonstrate + a feature. +- Draft state-changing commands and explain their effect. Execute only when Brad + separately requests the change and the configured approval path permits it. +- Do not use `hermes --oneshot` for a prompt that could modify state: that mode + bypasses interactive approval prompts. +- Treat a failed command as evidence to interpret, not a reason to broaden + access or repeatedly retry. + +## Teach evidence-driven triage + +For a live test-failure lab, also use `$triage-titan-test-failures`. Require the +student to distinguish: + +- observed fact from inference; +- stale evidence from current state; +- application regression from environment failure; +- a read-only next check from a proposed mutation; +- a generic suggestion from the smallest Flux-tracked repo-side fix. + +For offline practice, read `references/incident-drills.md` and present only the +student packet for one incident. Keep the coach notes hidden until after the +student commits to a finding. + +## Validate claims honestly + +Use the claim audit in `references/mastery-rubric.md`. Current configuration and +live demonstrations support current-use claims. The retained Git history and +bound OpenClaw PVC support the deployment/replacement history; require Brad to +inspect that evidence and distinguish it from proof of every past interaction. + +End every session with: + +- what Brad demonstrated; +- what remains unproven; +- the next smallest lab; +- any safety or operational issue discovered. diff --git a/services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml b/services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml new file mode 100644 index 00000000..416c3a9e --- /dev/null +++ b/services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Master Hermes on Atlas" + short_description: "Practice Hermes safely on the live Atlas stack" + default_prompt: "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint. I will perform every setup and evidence step myself. Start with step 1 only, require live proof, and stop at every mutation or approval boundary. First confirm this session is using openai-codex/gpt-5.6-terra rather than the local fallback." diff --git a/services/hermes/skills/master-hermes-on-atlas/references/architecture.md b/services/hermes/skills/master-hermes-on-atlas/references/architecture.md new file mode 100644 index 00000000..c35188d6 --- /dev/null +++ b/services/hermes/skills/master-hermes-on-atlas/references/architecture.md @@ -0,0 +1,117 @@ +# Live Atlas Hermes architecture + +Use this reference for deployment-specific facts. Re-check live state before +asserting health, placement, ownership, or current model availability. + +## Two isolated agent instances + +| Surface | Purpose | Identity boundary | State and permissions | +| --- | --- | --- | --- | +| `agent.bstein.dev` | Brad's operator and triage lab | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace, its own PVC and service account; read-only cluster triage plus approved internal evidence endpoints | +| `chat.bstein.dev` | Consumer chat and research | Keycloak login | `hermes-chat` namespace, separate PVC/config/auth/sessions/skills; read-only cluster observer RBAC and no access to Secrets or Kubernetes mutation APIs | + +The instances do not share conversation state, credentials, profiles, skills +created on their PVCs, or Kubernetes identities. They share only the inference +service described below. Consumer users cannot reach the operator dashboard +through the chat Service or Ingress. + +## Inference path + +```text +operator Hermes โ”€โ”€โ”€> independently authenticated openai-codex/gpt-5.6-terra +consumer Hermes โ”€โ”€โ”€> independently authenticated openai-codex/gpt-5.6-terra + โ”‚ + โ””โ”€ provider error or manual selection โ”€> hermes-model-gate + โ”‚ + โ”œโ”€ Hermes owns GPU โ”€> Ollama gpt-oss:20b + โ””โ”€ Wolf owns GPU โ”€โ”€โ”€> HTTP 503 +``` + +- The configured context length is 64,000 tokens. +- `hermes-model-gate` is the stable OpenAI-compatible endpoint. +- A coordination Lease stores whether Hermes or Wolf/Moonlight owns the GPU. + Ariadne has narrowly scoped permission to update that Lease during handoff. +- Wolf ownership makes the local gate return a deliberate unavailable response. + Normal interactive chat remains on its Codex primary and does not consume + titan-24 GPU resources. +- The two agent pods do not consume titan-24 GPU memory. Ollama does. Agent pods + are ARM gateway workloads with persistent state on separate PVCs. +- Codex credentials are deliberately per-instance. Never copy Brad's Codex + credential store into the consumer PVC. + +Verify rather than memorize: + +```sh +hermes status +hermes fallback list +kubectl -n hermes get lease titan-24-gpu-owner -o yaml +kubectl -n hermes get deploy hermes hermes-model-gate hermes-ollama -o wide +kubectl -n hermes-chat get deploy hermes-chat -o wide +``` + +Do not change the GPU owner as part of a lesson. + +## Retained OpenClaw replacement evidence + +OpenClaw is not part of the current inference path, but its deployment history +is retained: + +- `1bc58e10` added the OpenClaw testing-triage workspace, workload, model, + ingress, RBAC, and storage. +- follow-up commits wired local diagnosis, OIDC, persistence, and the gateway. +- `97ecb36b` replaced OpenClaw with Hermes and moved the agent-facing resources. +- `bb5e286f` decoupled Hermes from the retired OpenClaw Flux dependency. +- the `openclaw-home` PVC remains bound as retained state while no OpenClaw + workload is running. + +Have Brad verify the commits and live resource shape. This establishes the +cluster deployment and replacement sequence; Brad's direct experience remains +the evidence for how he personally used the old agent. + +## Operator evidence path + +The operator role is designed for supervised test-failure triage: + +1. Ariadne deterministic bundle and optional diagnosis. +2. Retained Jenkins logs and artifacts. +3. Recent Git changes and Flux revision/state. +4. Kubernetes workload, node, event, storage, DNS, and dependency health. +5. Pushgateway quality metrics and VictoriaMetrics/Grafana context. +6. A concise finding, confidence, evidence, blast radius, next checks, + repo-side fix, and approval-required section. + +The versioned `$triage-titan-test-failures` skill encodes this workflow. The +operator has internal URLs for Ariadne, Jenkins, and VictoriaMetrics. The +consumer instance intentionally does not. + +## Security layers and their limits + +Security is layered rather than delegated to a prompt: + +1. Keycloak authenticates the human. +2. The operator proxy limits `agent.bstein.dev` to Brad's exact email. +3. Separate namespaces, PVCs, service accounts, configs, and ingresses isolate + operator and consumer state. +4. Kubernetes RBAC grants the consumer get/list/watch only and excludes Secrets, + exec, attach, port-forward, and mutation. +5. NetworkPolicy permits the consumer to reach DNS, the shared model gate, the + Kubernetes API, and public IPv4 while blocking arbitrary private service + access. +6. Hermes deny patterns and instructions add user-facing guardrails. + +Prompt rules and shell deny patterns are not the primary security boundary. +RBAC, network policy, credential separation, and routing isolation are. The +consumer can still disclose non-secret cluster metadata it is authorized to +read; that is an intentional requirement and should be described honestly. + +## CLI-to-web mental model + +The pinned Hermes build exposes chat, models and fallback, auth, sessions, +profiles, skills and bundles, plugins, tools, MCP, gateway/channels, pairing, +webhooks, cron, kanban, projects, hooks, memory/journey, logs, security, backup, +and diagnostics. The web interface is a control surface over many of these +same persisted capabilities. + +Use ` --help` and the web Documentation page for the installed version. +Do not rely on screenshots or online docs from a different release when a live +command can settle the question. diff --git a/services/hermes/skills/master-hermes-on-atlas/references/curriculum.md b/services/hermes/skills/master-hermes-on-atlas/references/curriculum.md new file mode 100644 index 00000000..3c3364bd --- /dev/null +++ b/services/hermes/skills/master-hermes-on-atlas/references/curriculum.md @@ -0,0 +1,131 @@ +# Hands-on curriculum + +Complete labs by evidence, not elapsed time. A focused pass can establish +operational competence in several days; mastery requires repeating real triage +and recovery work over multiple incidents. + +Run the curriculum on `openai-codex/gpt-5.6-terra`. Use the local model only for a +deliberate comparison lab; it is not the default coach for multi-reference work. + +## Phase 1: orientation and control + +### Lab 1 โ€” Map the system + +From the web UI, identify Chat, Sessions, Files, Models, Logs, Cron, Skills, +Plugins, MCP, Channels, Webhooks, Pairing, Profiles, Config, Keys, System, and +Documentation. Explain which state belongs to the operator PVC and which +components are shared. Verify three claims with read-only CLI output. + +Success evidence: a correct diagram or written request path from browser to +agent to model, including the fallback branch and the consumer boundary. + +### Lab 2 โ€” Models, context, and fallback + +Inspect `hermes status`, `hermes fallback list`, deployment placement, and GPU +owner state. Explain why a 32K model was rejected, why normal Codex-backed chat +remains available when Wolf owns the GPU, and when the local fallback can run. + +Success evidence: predict outcomes for local healthy, local slow, gate 503, +invalid local response, and expired Codex authorization without changing state. + +### Lab 3 โ€” Sessions, files, profiles, and logs + +Create a named training session and a harmless workspace note through the UI, +then find the corresponding session/file/log surfaces. Inspect profile and +backup help without creating a profile or backup. + +Success evidence: explain persistence, what survives a pod replacement, what is +instance-local, and how to recover a lost conversation without exposing auth. + +## Phase 2: tools and safe autonomy + +### Lab 4 โ€” Tools versus skills versus MCP + +Use `hermes tools`, `hermes skills`, `hermes plugins`, and `hermes mcp` help or +list/status views. Classify each as executable capability, procedure/context, +packaged extension, or external protocol integration. Explain why a skill is +not a security boundary. + +Success evidence: choose the right mechanism for three examples: repeatable +Titan triage, a read-only external API, and a scheduled notification. + +### Lab 5 โ€” Prove the permission boundary + +Use `kubectl auth can-i` as both service accounts. Inspect the relevant +ClusterRoles and NetworkPolicies. Do not attempt mutations. + +Success evidence: an allow/deny matrix covering pods, logs, Secrets, exec, +deployment patch, Flux reads, Flux reconcile, the model gate, internal services, +and public web research. + +### Lab 6 โ€” Channels, pairing, webhooks, and cron + +Inspect the configured state and help for gateway/channels, pairing, webhooks, +and cron. Design one safe notification workflow and one unsafe workflow. Do not +register a channel, create a webhook, or schedule a job during the lab. + +Success evidence: identify the identity, secret, audience, tool policy, failure +mode, audit trail, and revocation path for the proposed integration. + +## Phase 3: the interview workflow + +### Lab 7 โ€” Deterministic live triage + +Invoke `$triage-titan-test-failures`. Read the current Ariadne diagnosis and +bundle, then correlate one suite with Jenkins, Flux/Git, cluster health, and +quality metrics. Keep every collection step read-only. + +Success evidence: the required structured triage result with timestamps and no +invented evidence. + +### Lab 8 โ€” Incident simulations + +Use one student packet from `incident-drills.md`. Diagnose it before receiving +coach notes. Repeat until three cases score at least `Independent`. + +Success evidence: correct failure class, causal chain, next checks, minimal +repo-side correction, and explicit approval boundary. + +### Lab 9 โ€” Build and evaluate a reusable skill + +Identify a stable repeated workflow, create a small skill in the operator's +writable skills directory, inspect it, and test it against a fresh incident. +Do not alter the Flux-mounted training or triage skills. + +Success evidence: precise trigger description, concise procedure, progressive +disclosure where useful, one passing case, one adversarial case, and a stated +permission boundary. + +### Lab 10 โ€” Model outage drill + +Use retained logs/current status or a separately approved maintenance window. +Do not take the live model down for training. Explain the expected trace from +model gate to fallback, recognize an auth failure, and name the restoration +checks. + +Success evidence: correctly distinguish model unavailability, context rejection, +provider auth failure, and agent failure. + +## Phase 4: independent operation + +### Lab 11 โ€” Supervised real incident + +Lead a real failure triage from intake to an approved repo-side proposal. A +human applies any change through Git/Flux. Verify the result read-only. + +### Lab 12 โ€” Teach it back and audit the claims + +Explain the complete architecture and demonstrate the workflow without hints. +Audit each sentence of the interview answer using the claim table in +`mastery-rubric.md`. Weaken or qualify anything not supported by evidence. + +## Suggested pace + +- Day 1: Labs 1โ€“3. +- Days 2โ€“3: Labs 4โ€“6. +- Days 4โ€“7: Labs 7โ€“10. +- Weeks 2โ€“4: repeat Labs 7โ€“11 on real incidents; improve one skill from observed + failures; finish with Lab 12. + +Do not advance solely because a day elapsed. Re-run any lab graded below +`Independent`. diff --git a/services/hermes/skills/master-hermes-on-atlas/references/incident-drills.md b/services/hermes/skills/master-hermes-on-atlas/references/incident-drills.md new file mode 100644 index 00000000..dc974283 --- /dev/null +++ b/services/hermes/skills/master-hermes-on-atlas/references/incident-drills.md @@ -0,0 +1,95 @@ +# Incident drills + +Present only one `Student packet` before the student answers. Use the coach +notes afterward to grade the failure classification and reasoning. These cases +are based on retained Atlas rollout observations; treat them as training +fixtures, not proof of current live state. + +## Case A โ€” Metrics disappear after a Soteria run + +### Student packet + +- The Soteria test stages ran. +- The post-stage log ends with: + `Syntax error: end of file unexpected (expecting "fi")`. +- The canonical Soteria Pushgateway series did not refresh. +- Other suites continued publishing. + +Ask for: finding, confidence, evidence, likely cause, blast radius, read-only +next checks, repo-side fix, and approval-required actions. + +### Coach notes + +The primary class is pipeline glue/shell syntax, not a Pushgateway outage and +not a product-test failure. Inspect the post block and the exact shell parsed by +`/bin/sh`; confirm the last successful metric timestamp. The smallest likely +fix is closing/correcting the conditional in the Jenkinsfile, then a normal SCM +run and read-only metric verification. Editing, pushing, triggering, or manual +metric backfill requires explicit approval. + +## Case B โ€” UI tests fail before metrics publish + +### Student packet + +- `bstein-dev-home` requires Playwright `1.59.1`. +- The CI test container contains Playwright `1.51.0`. +- The frontend test stage hard-fails. +- No fresh canonical quality metrics appear for the run. + +### Coach notes + +Separate the direct test-environment mismatch from the telemetry-control-flow +defect. The version mismatch explains the test failure; early pipeline abort +explains missing metrics. A complete proposal aligns the image version and +persists stage return codes so publishing still runs while the final gate still +fails. Do not call this a Kubernetes capacity problem without node/pod evidence. + +## Case C โ€” Promotion exits 127 + +### Student packet + +- Titan IaC validation and tests finish. +- The Promote stage uses `python:3.12-slim`. +- The stage returns exit code `127` at its first Git command. +- The repository and remote are reachable from other jobs. + +### Coach notes + +Exit 127 means the invoked command is unavailable. Confirm the exact log line +and image contents; the likely issue is that the slim image lacks `git`, not bad +Git credentials. The minimal repo-side fix installs `git` and CA certificates +in the job's dependency setup or uses a suitable pinned runner image. + +## Case D โ€” Data Preppers cannot publish an image + +### Student packet + +- Tests and local image build complete. +- Push to `registry.bstein.dev/streaming/data-prepper:2.8.0` is denied. +- The configured Jenkins credential belongs to a different Harbor project. +- A Sonar evidence GET also returns `401` in the same run. + +### Coach notes + +Do not collapse independent authentication failures into a generic network +failure. Verify the Harbor repository/project and credential ID without reading +the secret value; separately verify the Sonar token injection path and API +scope. The likely image fix is selecting a streaming-scoped robot credential. +Treat the Sonar 401 as a second issue unless evidence proves a shared cause. + +## Case E โ€” Hermes local inference fails during game streaming + +### Student packet + +- Wolf owns titan-24 according to the GPU-owner state. +- `hermes-model-gate` is ready but returns a deliberate unavailable response. +- Both Hermes dashboards remain healthy. +- One instance answers through Codex; the other reports missing provider auth. + +### Coach notes + +This is expected resource arbitration plus an instance-specific fallback auth +gap. Do not restart the agent or claim the GPU is underprovisioned. Confirm the +gate response, fallback chain, and each instance's auth status without exposing +tokens. Authorize the affected instance with its intended user's account. The +separate credential stores are a security property, not configuration drift. diff --git a/services/hermes/skills/master-hermes-on-atlas/references/mastery-rubric.md b/services/hermes/skills/master-hermes-on-atlas/references/mastery-rubric.md new file mode 100644 index 00000000..c9372410 --- /dev/null +++ b/services/hermes/skills/master-hermes-on-atlas/references/mastery-rubric.md @@ -0,0 +1,75 @@ +# Mastery and claim rubric + +## Performance levels + +| Level | Evidence | +| --- | --- | +| Exposed | Recognizes terms but needs the path and commands supplied. | +| Assisted | Completes a lab with hints and can explain the result afterward. | +| Independent | Selects safe tools, gathers fresh evidence, and reaches a defensible result without hints. | +| Mastery | Handles ambiguity and failure, teaches the architecture, improves the reusable workflow, and preserves security boundaries. | + +Score each domain separately: + +1. Architecture and request routing. +2. Models, context, GPU ownership, and fallback. +3. Sessions, files, profiles, logs, and recovery. +4. Tools, skills, plugins, MCP, channels, webhooks, and cron. +5. Identity, RBAC, network, credentials, approvals, and auditability. +6. Jenkins/Flux/Kubernetes/metrics evidence correlation. +7. Clear triage findings and minimal repo-side proposals. +8. Skill creation, evaluation, and iteration. + +Never award `Independent` from a verbal answer alone when the lab calls for live +or fixture evidence. Never award `Mastery` until Brad has led at least two +different real incidents and improved a workflow based on what failed. + +## Interview claim audit + +| Claim | What supports it | Minimum demonstration | +| --- | --- | --- | +| Hermes runs in the Kubernetes cluster | Flux manifests and live workloads | Trace browser, agent, model gate, Ollama, PVC, and service account. | +| Hermes fits existing automation | Operator config, internal evidence URLs, triage skill | Complete a triage using Ariadne, Jenkins, Git/Flux, cluster health, and metrics. | +| Hermes follows Brad's triage path | `$triage-titan-test-failures` procedure | Lead two evidence-backed cases with facts separated from inference. | +| Repeated workflows become reusable skills | Versioned triage/training skills plus writable user skill area | Build, test, and improve one skill from a repeated real workflow. | +| Access starts read-only and scoped | RBAC, NetworkPolicy, separate identities and state | Produce and explain the allow/deny matrix from live authorization checks. | +| Human approval protects changes | Approval config, deny patterns, Flux workflow | Identify every mutation in a proposed incident response and stop before it. | +| Local GPU gracefully yields to gaming | Owner state, model gate, fallback | Explain or observe a Wolf ownership window and verify continued fallback service. | +| OpenClaw previously ran and was replaced | `1bc58e10` deploys the OpenClaw triage stack; follow-up commits add gateway/OIDC/persistence; `97ecb36b` replaces it with Hermes; `bb5e286f` retires the dependency; the bound `openclaw-home` PVC remains without a workload | Inspect the commits and live Flux/PVC state, then explain what they prove versus what depends on Brad's personal recollection. | + +## Triage answer scoring + +Award one point for each: + +- names the correct failure class; +- cites timestamped/build-specific evidence; +- labels inference; +- checks evidence freshness; +- identifies realistic blast radius; +- orders read-only next checks; +- proposes the smallest Flux/repo-side change or says evidence is insufficient; +- isolates approval-required actions; +- avoids invented resources, logs, metrics, or commits; +- remains concise enough for an operator to act on. + +Scores below 8/10 require another incident drill. Any invented evidence or +unacknowledged mutation caps the result at `Assisted`. + +## Progress record format + +When Brad approves recording progress, use this compact structure: + +```markdown +# Hermes training progress + +- Last session: +- Current level: +- Completed: +- Needs repetition: +- Next lab: +- Safety findings: +- Claim status: +``` + +Append evidence links or command summaries; never record tokens, cookies, +credentials, device codes, or Secret values. diff --git a/services/hermes/skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md b/services/hermes/skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md new file mode 100644 index 00000000..ace25f92 --- /dev/null +++ b/services/hermes/skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md @@ -0,0 +1,86 @@ +# Atlas two-hour proof sprint + +The sprint establishes usable, defensible evidence for the current Hermes +claims. It does not award mastery. Retained Git and PVC evidence establishes +the OpenClaw deployment/replacement sequence; Brad's direct experience remains +the evidence for how he personally used it. Brad performs the work; Hermes +gives one task at a time, checks the evidence, and withholds later answers until +Brad commits to a result. + +## 0โ€“15 minutes โ€” control surface and request path + +1. Have Brad start a new session and confirm it uses + `openai-codex/gpt-5.6-terra`. +2. In the web UI, have him locate Chat, Sessions, Files, Models, Logs, Skills, + Plugins, MCP, Channels, Webhooks, Cron, Profiles, Config, Keys, System, and + Documentation. +3. Have him explain browser โ†’ operator agent โ†’ Codex primary, including the + model-gate/Ollama fallback branch and the separate consumer instance. + +Evidence: Brad can identify persistent, shared, and instance-local state and +can name the GPU owner check without changing it. + +## 15โ€“35 minutes โ€” permissions and safety + +1. Have Brad inspect the operator ServiceAccount, ClusterRole, NetworkPolicy, + and approval deny list using read-only commands. +2. Have him produce an allow/deny matrix for pod/log reads, Secret reads, + exec, deployment mutation, Flux reads, Flux reconcile, and internal + evidence endpoints. +3. Require him to explain why prompt instructions and skills are not the + primary security boundary. + +Evidence: correct identity/RBAC/network/approval layering with no Secret values +read and no mutation attempted. + +## 35โ€“75 minutes โ€” perform the claimed triage path + +Invoke `$triage-titan-test-failures`. Brad must: + +1. check Ariadne diagnosis and bundle freshness; +2. identify one current or retained failed suite/build; +3. correlate Jenkins log/artifact evidence; +4. correlate Git/Flux revision and Kubernetes environment health; +5. query the relevant Pushgateway/VictoriaMetrics quality signal and identify + the matching Grafana context; +6. produce Finding, Confidence, Evidence, Likely cause, Blast radius, Next + checks, Repo-side fix, and Approval required. + +Hermes may suggest a read-only command after Brad explains why it is needed. +Do not let the built-in diagnosis replace Brad's own evidence correlation. + +Evidence: an 8/10 or better result under the triage rubric with no invented +facts and no unacknowledged mutation. + +## 75โ€“105 minutes โ€” build the reusable workflow + +1. Have Brad inspect the mounted `triage-titan-test-failures` skill and explain + its trigger, fixed procedure, variable evidence, and security boundary. +2. Have Brad choose one genuinely repeated sub-workflow from the triage. +3. Guide him to create a new skill in the operator's writable skill area. Do + not edit the mounted Flux skill and do not write the skill for him. +4. Test one expected trigger and one adversarial/non-trigger case. +5. Have Brad make one improvement based on the test results. + +Evidence: Brad can find the skill in the Skills UI, explain every instruction, +and show a passing and adversarial evaluation. + +## 105โ€“120 minutes โ€” supervised proposal and claim audit + +1. Have Brad turn the triage into the smallest repo-side proposal without + applying it. +2. Have him identify every step requiring approval: file edit, commit/push, + pipeline trigger, Flux reconcile, credential/environment change, or manual + backfill. +3. Audit the interview answer sentence by sentence using `mastery-rubric.md`. + Include the retained OpenClaw commits and live PVC/no-workload state. +4. Offer to record progress only after Brad approves the write. + +Minimum honest outcome: + +- current Hermes deployment and supervised triage use are demonstrated; +- the established Atlas automation path and read-only boundary are explained; +- one reusable writable skill is created, tested, and improved by Brad; +- autonomous production mutation is not claimed; +- historical OpenClaw deployment and replacement are supported by retained Git + and PVC evidence, with personal-use details identified as Brad's testimony. diff --git a/services/hermes/skills/triage-atlas-service-health/SKILL.md b/services/hermes/skills/triage-atlas-service-health/SKILL.md new file mode 100644 index 00000000..f3db0ff4 --- /dev/null +++ b/services/hermes/skills/triage-atlas-service-health/SKILL.md @@ -0,0 +1,66 @@ +--- +name: triage-atlas-service-health +description: Diagnose active Atlas service and infrastructure problems from Grafana, Ariadne, Flux, Kubernetes events, workload state, and logs. Use when asked what is broken, why a Grafana health panel is red, whether an outage is real, or how to fix a service through titan-iac. Covers every deployed namespace while distinguishing current services from migration residue and historical noise. +--- + +# Triage Atlas Service Health + +Investigate read-only, group symptoms into incidents, and recommend the smallest Flux-tracked fix. + +## Establish the current scope + +1. Read `references/service-map.md` for custom-service ownership and migrations. +2. Discover the live inventory instead of relying only on the map: + + ```sh + kubectl get namespaces + kubectl get deploy,statefulset,daemonset -A + kubectl get ingress -A + kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io + ``` + +3. Cassandra is authoritative for functionality migrated from Veles. Veles is + retired migration residue: its failed pods, old Jobs, and legacy Service or + Ingress objects are not current user impact. Escalate Veles only when direct + evidence proves Cassandra still depends on it or the Veles-to-Cassandra + redirect itself is failing. + +## Decide whether the signal is actionable + +Use all of these checks before calling something an incident: + +- Current state: prefer an active condition over lifetime restart totals or old failed Jobs. +- Persistence: ignore normal startup states under 15 minutes unless users are already affected. +- Ownership: group replica pods by Deployment, StatefulSet, DaemonSet, Job, or CronJob. +- Delivery: a Flux object with `Ready=null` may be reconciling; `Ready=false` or a suspended required object is actionable. +- Authority: separate current workloads from retired, migrated, or deliberately suspended resources. Never describe Veles as unavailable or degraded user-facing service merely because its retained pods are unhealthy. +- Correlation: confirm a red Grafana panel with the underlying metric and at least one independent source such as events, logs, readiness, or Flux state. + +Run the narrowest relevant checks: + +```sh +kubectl get pods -A --field-selector status.phase!=Running,status.phase!=Succeeded -o wide +kubectl get events -A --sort-by=.lastTimestamp +kubectl -n get deploy,statefulset,daemonset,job,cronjob +kubectl -n describe pod +kubectl -n logs --all-containers --tail=200 +kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io +curl -G -fsS --data-urlencode 'query=' "$VICTORIA_METRICS_URL/api/v1/query" +``` + +Never read Kubernetes Secret values. Secret-key errors quoted by a pod event or log are valid evidence; inspect the SecretProviderClass and Vault policy manifests, not the secret contents. + +## Produce one incident per cause + +Deduplicate replica pods and repeated alerts with a signature of service, workload, failing condition, and likely dependency. Return: + +- `Finding`: current incident, degraded-but-serving, migration residue, historical noise, or healthy. +- `Impact`: the user-facing service and affected capability. +- `Evidence`: timestamps, workload condition, event/log excerpt, metric, and Flux revision. +- `Noise removed`: signals inspected but not counted, with the reason. +- `Likely cause`: causal chain, marking inference explicitly. +- `Next checks`: ordered read-only commands. +- `Repo-side fix`: exact titan-iac or application path and smallest proposed change. +- `Approval required`: every write, reconcile, credential, or external-system action. + +Do not claim a proposed change is applied. Do not mutate the cluster; Atlas is Flux-owned. diff --git a/services/hermes/skills/triage-atlas-service-health/agents/openai.yaml b/services/hermes/skills/triage-atlas-service-health/agents/openai.yaml new file mode 100644 index 00000000..c6b4ecc3 --- /dev/null +++ b/services/hermes/skills/triage-atlas-service-health/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Triage Atlas Service Health" + short_description: "Separate real Atlas incidents from dashboard noise" + default_prompt: "Use $triage-atlas-service-health to identify the active service incidents in Atlas, group duplicate symptoms, and recommend read-only checks and the smallest Flux-tracked fixes." diff --git a/services/hermes/skills/triage-atlas-service-health/references/service-map.md b/services/hermes/skills/triage-atlas-service-health/references/service-map.md new file mode 100644 index 00000000..9d0105f1 --- /dev/null +++ b/services/hermes/skills/triage-atlas-service-health/references/service-map.md @@ -0,0 +1,22 @@ +# Atlas custom-service map + +Use this map for ownership and canonical names. Discover third-party services live from Flux and Kubernetes. + +| Capability | Canonical workload/repository | Runtime location | Notes | +| --- | --- | --- | --- | +| Cluster automation and state analysis | Ariadne / `bstein/Ariadne` | `maintenance/ariadne` | Provides deterministic triage bundles. | +| Quality policy and CI telemetry | titan-iac / `bstein/titan-iac` | Jenkins plus Flux; no single app Deployment | Owns Jenkins configuration, Grafana generators, alert provisioning, and the Data Prepper pipeline. | +| Simulation registry and execution | Cassandra / Cassandra application repository | `cassandra/cassandra-frontend`, `cassandra-backend`, `cassandra-postgres`, `cassandra-vault-sync` | Cassandra is authoritative after the Veles migration. | +| Legacy simulation stack | Veles | `veles` namespace | Retired migration residue. Pod failures are not current impact; only a proven Cassandra dependency or broken redirect is actionable. | +| Backup and maintenance policy | Soteria / `bstein/soteria` | `maintenance/soteria` | Correlate CI failures separately from runtime backup health. | +| Media client | Pegasus / `bstein/pegasus` | `jellyfin/pegasus` | Correlate with Jellyfin and OIDC dependencies. | +| Cluster sentinel | Metis / `bstein/metis` | `maintenance/metis` plus sentinel DaemonSets | Distinguish controller health from per-node sentinel evidence. | +| User and service automation | Ananke / `bstein/ananke` | No current dedicated in-cluster Deployment | Treat its Jenkins suite as the authoritative runtime evidence unless discovery proves otherwise. | +| Chat automation | Atlasbot / `bstein/atlasbot` | `comms/atlasbot` | Correlate with comms and OIDC dependencies. | +| Public site | bstein_home / `bstein/bstein-dev-home` | `bstein-dev-home` frontend, backend, and Vault sync | Canonical metric suite uses underscore form. | +| Data Prepper integration | data_prepper / `bstein/titan-iac` | `logging/data-prepper` | Pipeline is under `services/logging`. | +| Desktop test application | Lesavka / `bstein/lesavka` | Desktop test host, not a normal Kubernetes Deployment | Test suite is in scope; runtime evidence may come from titan-jh. | +| Hermes operator | `hermes` namespace / `services/hermes` | `hermes/hermes`, model gate, and Ollama | Read-only cluster operator with Codex primary and local fallback. | +| Hermes consumer chat | `hermes-chat` namespace / `services/hermes-chat` | `hermes-chat/hermes-chat` | Isolated from cluster operation; do not use it for infrastructure triage. | + +Canonical CI suites are `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `lesavka`, `metis`, `pegasus`, `soteria`, and `titan_iac`. diff --git a/services/hermes/skills/triage-titan-test-failures/SKILL.md b/services/hermes/skills/triage-titan-test-failures/SKILL.md new file mode 100644 index 00000000..013ede38 --- /dev/null +++ b/services/hermes/skills/triage-titan-test-failures/SKILL.md @@ -0,0 +1,120 @@ +--- +name: triage-titan-test-failures +description: Diagnose Titan CI test failures and environment regressions from Ariadne evidence, Jenkins artifacts, Flux state, Kubernetes health, Pushgateway quality metrics, and Grafana context. Use for failed or flaky in-scope suites, suspected cluster-caused test failures, release-quality questions, or requests for a supervised read-only triage summary and repo-side next steps. +--- + +# Triage Titan Test Failures + +Follow the established Titan evidence path. Keep the investigation read-only and distinguish observed facts from inference. + +## Collect the canonical evidence + +1. Read the latest Ariadne diagnosis: + + ```sh + curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest" + ``` + +2. Always read the deterministic bundle, even when the diagnosis looks complete: + + ```sh + curl -fsS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest" + ``` + +3. Ask for human approval before triggering a fresh collection or diagnosis. Use these only after approval: + + ```sh + curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/collect" + curl -fsS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run" + ``` + +Record the timestamps and status from both responses. Treat the diagnosis as stale when it predates the deterministic bundle, reports `unavailable`, or lacks concrete evidence. In any of those cases, continue the investigation from the deterministic bundle; do not stop at model availability. + +Treat Ariadne's bundle as the evidence source of truth. A local-model diagnosis may be unavailable while Wolf owns titan-24; continue from the stored bundle using the active Hermes fallback model. + +## Atlas facts and guardrails + +- Ariadne runs as the `ariadne` Deployment and Service in the `maintenance` namespace. +- Hermes local inference runs through `hermes-model-gate` in the `hermes` namespace. Ollama is the `hermes-ollama` Deployment and Service in that namespace. +- OpenClaw is not part of this inference path. Never suggest an `openclaw` resource unless a read-only query first proves one exists. +- Custom runtime namespaces do not always match suite names. In particular, + Soteria, Metis, and Ariadne run in `maintenance`; Pegasus runs in `jellyfin`; + Atlasbot runs in `comms`; Data Prepper runs in `logging`; and Lesavka is a + desktop-hosted application. Read the service-health skill's + `references/service-map.md` before claiming that a suite has no runtime. +- A connection error recorded inside an older diagnosis proves only that the model request failed at that timestamp. It does not prove the service is currently down. +- Never invent a namespace, workload, container, port, URL, log line, metric, commit, or pod condition. Verify a target with a read-only query before presenting an exact follow-up command; otherwise state what must be discovered first. +- HTTP POST collection and diagnosis endpoints are state-changing operations. Put them only under `Approval required`; never describe them as read-only or include them in the read-only command list. + +## Narrow the failure + +Work in this order: + +1. Confirm the failed suite and build are in the canonical scope: `ananke`, `ariadne`, `atlasbot`, `bstein_home`, `data_prepper`, `lesavka`, `metis`, `pegasus`, `soteria`, or `titan_iac`. +2. Identify the first failed gate in the enforced order: `style`, `loc`, `coverage`, `tests`, `gate_glue`, `sonarqube`, `supply_chain`. +3. Correlate the build timestamp with retained Jenkins logs/artifacts, recent Git commits, and Flux revisions. +4. Check whether Kubernetes health, node pressure, image pulls, storage, DNS, or a shared dependency explains the failure better than a repo regression. +5. Check Pushgateway and Grafana evidence for branch gaps, stale metrics, aliases, or missing zero-state telemetry. A zero-valued series is not a failure. +6. Keep running Jenkins builds in an in-progress collection. Do not count them as failed until Jenkins reports a terminal failure state. +7. State unknowns explicitly. Never invent a log line, metric, commit, pod condition, or root cause. + +For recent Git context, map the canonical suite to its Gitea repository and +read the latest commits. The current mappings are: + +- `ananke` -> `bstein/ananke` +- `ariadne` -> `bstein/Ariadne` +- `atlasbot` -> `bstein/atlasbot` +- `bstein_home` -> `bstein/bstein-dev-home` +- `data_prepper` -> `bstein/titan-iac` +- `lesavka` -> `bstein/lesavka` +- `metis` -> `bstein/metis` +- `pegasus` -> `bstein/pegasus` +- `soteria` -> `bstein/soteria` +- `titan_iac` -> `bstein/titan-iac` + +Use the read-only Gitea API and compare commit timestamps to the failing build: + +```sh +curl -fsS "$GITEA_BASE_URL/api/v1/repos///commits?limit=10" +``` + +For quality telemetry, query VictoriaMetrics directly when the bundle is stale +or a specific label needs confirmation. URL-encode PromQL rather than manually +escaping it: + +```sh +curl -G -fsS --data-urlencode \ + 'query=platform_quality_gate_runs_total{suite=""}' \ + "$VICTORIA_METRICS_URL/api/v1/query" +curl -G -fsS --data-urlencode \ + 'query=_quality_gate_checks_total' \ + "$VICTORIA_METRICS_URL/api/v1/query" +``` + +For image-pull failures, inspect the pod event, its ServiceAccount metadata, +and Flux image objects without reading referenced Secret values: + +```sh +kubectl -n describe pod +kubectl -n get serviceaccount -o yaml +kubectl get imagerepositories,imagepolicies,imageupdateautomations -A +``` + +If the latest diagnosis is unavailable, the minimum acceptable result still analyzes the deterministic bundle's failed suites, build/check evidence, freshness, and environment observations. Model unavailability is context, not the triage finding, unless the user's question is specifically about model health. + +Use read-only commands such as `kubectl get`, `kubectl describe`, `kubectl logs`, and HTTP GET requests. Do not read Secret values or run mutating Kubernetes, Flux, Vault, Jenkins, or Git commands. + +## Produce the triage result + +Return these sections: + +- `Finding`: one sentence naming the most likely failure class. +- `Confidence`: low, medium, or high, with the reason. +- `Evidence`: the smallest set of concrete timestamps, build IDs, artifact paths, commits, metrics, pods, nodes, or Flux revisions that support the finding. +- `Likely cause`: explain the causal chain and label inference as inference. +- `Blast radius`: affected suites, services, branches, or environments. +- `Next checks`: ordered read-only checks with exact commands or URLs. +- `Repo-side fix`: the smallest Flux/IaC or application change, or `none yet` when evidence is insufficient. +- `Approval required`: call out every step that would modify files, infrastructure, credentials, test environments, or external systems. + +Never present a proposed fix as applied. Prefer a concise evidence-backed answer over a general log summary. diff --git a/services/hermes/skills/triage-titan-test-failures/agents/openai.yaml b/services/hermes/skills/triage-titan-test-failures/agents/openai.yaml new file mode 100644 index 00000000..f29a6d9b --- /dev/null +++ b/services/hermes/skills/triage-titan-test-failures/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Triage Titan Test Failures" + short_description: "Analyze evidence for supervised test-failure triage" + default_prompt: "Use $triage-titan-test-failures to diagnose the latest Titan test failure from Ariadne evidence." diff --git a/services/hermes/skills/tune-atlas-alerts/SKILL.md b/services/hermes/skills/tune-atlas-alerts/SKILL.md new file mode 100644 index 00000000..74a72866 --- /dev/null +++ b/services/hermes/skills/tune-atlas-alerts/SKILL.md @@ -0,0 +1,39 @@ +--- +name: tune-atlas-alerts +description: Audit and tune noisy Atlas Grafana alerts using live VictoriaMetrics evidence, Grafana notification history, persistence, minimum sample sizes, and generator ownership. Use when alerts are over-aggressive, a panel is red without user impact, PromQL behaves impossibly, or a Flux-tracked alert/dashboard correction is needed. +--- + +# Tune Atlas Alerts + +Reduce false positives without hiding real failures. Read `references/alert-review.md` before recommending a rule change. + +## Audit the signal + +1. Find the provisioned rule in `services/monitoring/grafana-alerting-config.yaml`. +2. If the issue is a dashboard panel, find its Python source in `scripts/dashboards_render_atlas.py`; never hand-edit generated JSON or ConfigMaps. +3. Query the exact PromQL against VictoriaMetrics and inspect the labels and raw inputs. +4. Compare current value, recent history, notification frequency, and a second source such as Kubernetes events or service metrics. +5. Classify the problem as bad math, lifetime-versus-rate confusion, missing persistence, low sample size, duplicate series, retired scope, rollout noise, real backlog, or real incident. + +Useful read-only commands: + +```sh +curl -G -fsS --data-urlencode 'query=' "$VICTORIA_METRICS_URL/api/v1/query" +kubectl -n monitoring logs deploy/grafana --since=24h +kubectl -n monitoring get configmap grafana-alerting-config -o yaml +kubectl -n get events --sort-by=.lastTimestamp +``` + +## Design the correction + +- Use `rate`, `increase`, or `delta` according to counter/gauge semantics; clamp percentages to 0..100. +- Require persistence for startup, rollout, and scheduling conditions. +- Add a minimum denominator and absolute count to percentage alerts with small samples. +- Alert on configured resources that regressed; keep unenrolled inventory visible as backlog instead of paging continuously. +- Exclude retired or migrated resources from high-level health, while retaining them on detailed dashboards. +- Deduplicate replicas and scrape targets using stable service/workload labels. +- Preserve a drill-down path to the raw evidence. + +Validate changed PromQL live, run the dashboard generator, test the generator, render the monitoring kustomization, and use a client-side dry-run. Report before/after values and what failure will still trigger the rule. + +Hermes is read-only in the cluster. Draft or explain repo changes and mark reconcile or deployment actions as approval-required. diff --git a/services/hermes/skills/tune-atlas-alerts/agents/openai.yaml b/services/hermes/skills/tune-atlas-alerts/agents/openai.yaml new file mode 100644 index 00000000..fad5a07e --- /dev/null +++ b/services/hermes/skills/tune-atlas-alerts/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Tune Atlas Alerts" + short_description: "Make Grafana alerts actionable without hiding faults" + default_prompt: "Use $tune-atlas-alerts to audit the currently noisy Atlas alerts, prove which signals are false positives, and propose the smallest generator- or alert-rule corrections." diff --git a/services/hermes/skills/tune-atlas-alerts/references/alert-review.md b/services/hermes/skills/tune-atlas-alerts/references/alert-review.md new file mode 100644 index 00000000..7468ffa6 --- /dev/null +++ b/services/hermes/skills/tune-atlas-alerts/references/alert-review.md @@ -0,0 +1,16 @@ +# Alert review checklist + +For every alert, record: + +1. User impact or operator action the alert demands. +2. Metric type: counter, gauge, timestamp, state marker, or recording rule. +3. Scope and deduplication labels. +4. Persistence window and startup grace. +5. Minimum sample size for ratios. +6. No-data and query-error behavior. +7. Current value and a recent range query. +8. Notification frequency in Grafana logs. +9. Current, migrated, suspended, or historical resource status. +10. Exact Flux-tracked source, generated artifacts, tests, and rollback condition. + +A rule is useful only when its firing state implies a specific human action. Backlogs and unenrolled resources belong on dashboards unless they have crossed an explicitly accepted operational deadline. diff --git a/services/hermes/vault-serviceaccount.yaml b/services/hermes/vault-serviceaccount.yaml new file mode 100644 index 00000000..91106486 --- /dev/null +++ b/services/hermes/vault-serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/hermes/vault-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-vault + namespace: hermes diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index c137b7db..0bee610d 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -429,15 +429,15 @@ data: } } } - pipelineJob('veles') { + pipelineJob('cassandra') { disabled(false) - description('Staged Veles alpha image pipeline. Backend/frontend should build linux/amd64 and linux/arm64; sim-worker may begin amd64-only if Forge dependencies require it.') + description('Staged Cassandra alpha image pipeline. Backend/frontend should build linux/amd64 and linux/arm64; sim-worker may begin amd64-only if Forge dependencies require it.') definition { cpsScm { scm { git { remote { - url('https://scm.bstein.dev/bstein/veles.git') + url('https://scm.bstein.dev/bstein/cassandra.git') credentials('gitea-pat') } branches('*/main') diff --git a/services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml b/services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml deleted file mode 100644 index 3b3d10b3..00000000 --- a/services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml +++ /dev/null @@ -1,401 +0,0 @@ -# services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml -# One-off job for sso/cassandra-realm-ensure-1. -# Purpose: create the Cassandra realm, groups, OIDC client, SMTP settings, and Vault client secret. -# Keep suspended until Cassandra Vault paths/policies have reconciled, then unsuspend once. -apiVersion: batch/v1 -kind: Job -metadata: - name: cassandra-realm-ensure-1 - namespace: sso -spec: - suspend: true - backoffLimit: 0 - ttlSecondsAfterFinished: 3600 - template: - metadata: - annotations: - vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/agent-pre-populate-only: "true" - vault.hashicorp.com/role: "sso-secrets" - vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin" - vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: | - {{ with secret "kv/data/atlas/shared/keycloak-admin" }} - export KEYCLOAK_ADMIN="{{ .Data.data.username }}" - export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}" - export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}" - {{ end }} - {{ with secret "kv/data/atlas/shared/postmark-relay" }} - export KEYCLOAK_SMTP_USER="{{ index .Data.data "apikey" }}" - export KEYCLOAK_SMTP_PASSWORD="{{ index .Data.data "apikey" }}" - {{ end }} - spec: - serviceAccountName: mas-secrets-ensure - restartPolicy: Never - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/worker - operator: Exists - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: ["arm64"] - containers: - - name: configure - image: python:3.11-alpine - env: - - name: KEYCLOAK_SERVER - value: http://keycloak.sso.svc.cluster.local - - name: KEYCLOAK_REALM - value: cassandra - - name: KEYCLOAK_CLIENT_ID - value: cassandra-web - - name: KEYCLOAK_PUBLIC_ISSUER - value: https://sso.bstein.dev/realms/cassandra - - name: CASSANDRA_BASE_URL - value: https://cassandra.bstein.dev - - name: CASSANDRA_TESTER_GROUP - value: cassandra-tester - - name: KEYCLOAK_SMTP_HOST - value: mail.bstein.dev - - name: KEYCLOAK_SMTP_PORT - value: "587" - - name: KEYCLOAK_SMTP_FROM - value: no-reply-cassandra@bstein.dev - - name: KEYCLOAK_SMTP_FROM_NAME - value: Cassandra - - name: CASSANDRA_PUBLIC_REPLAY_USERS - value: cassandra-dev,daniel-test,viktor-test - command: ["/bin/sh", "-c"] - args: - - | - set -eu - . /vault/secrets/keycloak-admin-env.sh - python - <<'PY' - import json - import os - import time - import urllib.error - import urllib.parse - import urllib.request - - base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/") - realm = os.environ["KEYCLOAK_REALM"] - client_id = os.environ["KEYCLOAK_CLIENT_ID"] - issuer = os.environ["KEYCLOAK_PUBLIC_ISSUER"] - app_base_url = os.environ["CASSANDRA_BASE_URL"].rstrip("/") - admin_user = os.environ["KEYCLOAK_ADMIN_USER"] - admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"] - - def request(method, url, token=None, payload=None, headers=None, timeout=30): - data = None - req_headers = headers.copy() if headers else {} - if token: - req_headers["Authorization"] = f"Bearer {token}" - if payload is not None: - data = json.dumps(payload).encode() - req_headers["Content-Type"] = "application/json" - req = urllib.request.Request(url, data=data, headers=req_headers, method=method) - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - body = resp.read() - if not body: - return resp.status, None - return resp.status, json.loads(body.decode()) - except urllib.error.HTTPError as exc: - raw = exc.read() - if not raw: - return exc.code, None - try: - return exc.code, json.loads(raw.decode()) - except Exception: - return exc.code, {"raw": raw.decode(errors="replace")} - - token_body = None - form = urllib.parse.urlencode( - { - "grant_type": "password", - "client_id": "admin-cli", - "username": admin_user, - "password": admin_password, - } - ).encode() - for attempt in range(1, 11): - req = urllib.request.Request( - f"{base_url}/realms/master/protocol/openid-connect/token", - data=form, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - method="POST", - ) - try: - with urllib.request.urlopen(req, timeout=10) as resp: - token_body = json.loads(resp.read().decode()) - break - except urllib.error.URLError as exc: - if attempt == 10: - raise SystemExit(f"Keycloak token request failed after retries: {exc}") - time.sleep(attempt * 2) - token = token_body["access_token"] - - smtp = { - "host": os.environ["KEYCLOAK_SMTP_HOST"], - "port": os.environ["KEYCLOAK_SMTP_PORT"], - "from": os.environ["KEYCLOAK_SMTP_FROM"], - "fromDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"], - "replyTo": os.environ["KEYCLOAK_SMTP_FROM"], - "replyToDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"], - "user": os.environ["KEYCLOAK_SMTP_USER"], - "password": os.environ["KEYCLOAK_SMTP_PASSWORD"], - "auth": "true", - "starttls": "true", - "ssl": "false", - } - - status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token) - if status == 404: - create_payload = { - "realm": realm, - "enabled": True, - "registrationAllowed": False, - "resetPasswordAllowed": True, - "verifyEmail": True, - "loginWithEmailAllowed": True, - "duplicateEmailsAllowed": False, - "smtpServer": smtp, - } - status, body = request("POST", f"{base_url}/admin/realms", token, create_payload) - if status not in (201, 204, 409): - raise SystemExit(f"Realm create failed: status={status} body={body}") - status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token) - if status != 200 or not isinstance(realm_rep, dict): - raise SystemExit(f"Realm fetch failed: status={status}") - - realm_rep.update( - { - "enabled": True, - "registrationAllowed": False, - "resetPasswordAllowed": True, - "verifyEmail": True, - "loginWithEmailAllowed": True, - "duplicateEmailsAllowed": False, - "smtpServer": smtp, - } - ) - status, body = request("PUT", f"{base_url}/admin/realms/{realm}", token, realm_rep) - if status not in (200, 204): - raise SystemExit(f"Realm update failed: status={status} body={body}") - - def find_group(name): - status, groups = request( - "GET", - f"{base_url}/admin/realms/{realm}/groups?search={urllib.parse.quote(name)}", - token, - ) - if status != 200: - raise SystemExit(f"Group search failed for {name}: status={status}") - for group in groups or []: - if group.get("name") == name: - return group - return None - - def ensure_group(name): - group = find_group(name) - if group: - return group - status, body = request("POST", f"{base_url}/admin/realms/{realm}/groups", token, {"name": name}) - if status not in (201, 204, 409): - raise SystemExit(f"Group create failed for {name}: status={status} body={body}") - group = find_group(name) - if not group: - raise SystemExit(f"Group missing after create for {name}") - return group - - def find_user(username): - encoded = urllib.parse.quote(username) - for query in (f"username={encoded}&exact=true", f"email={encoded}&exact=true", f"search={encoded}"): - status, users = request( - "GET", - f"{base_url}/admin/realms/{realm}/users?{query}", - token, - ) - if status != 200: - raise SystemExit(f"User search failed for {username}: status={status}") - for user in users or []: - candidates = { - str(user.get("username") or "").casefold(), - str(user.get("email") or "").casefold(), - } - if username.casefold() in candidates: - return user - return None - - def ensure_user_group(username, group): - user = find_user(username) - if not user: - print(f"WARNING: Cassandra public replay user {username!r} not found; skipping membership") - return - user_id = user.get("id") - group_id = group.get("id") - if not user_id or not group_id: - raise SystemExit(f"Missing id for user/group membership: user={username}") - status, memberships = request( - "GET", - f"{base_url}/admin/realms/{realm}/users/{user_id}/groups", - token, - ) - if status != 200: - raise SystemExit(f"User group lookup failed for {username}: status={status}") - if any(item.get("id") == group_id for item in memberships or []): - return - status, body = request( - "PUT", - f"{base_url}/admin/realms/{realm}/users/{user_id}/groups/{group_id}", - token, - ) - if status not in (200, 204): - raise SystemExit(f"User group update failed for {username}: status={status} body={body}") - - ensure_group("alpha") - ensure_group("admin") - ensure_group(os.environ["CASSANDRA_TESTER_GROUP"]) - public_replay_group = ensure_group("public-replay") - for username in os.environ.get("CASSANDRA_PUBLIC_REPLAY_USERS", "").split(","): - username = username.strip() - if username: - ensure_user_group(username, public_replay_group) - - status, clients = request( - "GET", - f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}", - token, - ) - if status != 200: - raise SystemExit(f"Client lookup failed: status={status}") - client_uuid = clients[0]["id"] if clients else None - client_payload = { - "clientId": client_id, - "enabled": True, - "protocol": "openid-connect", - "publicClient": False, - "standardFlowEnabled": True, - "implicitFlowEnabled": False, - "directAccessGrantsEnabled": False, - "serviceAccountsEnabled": False, - "redirectUris": [f"{app_base_url}/*"], - "webOrigins": [app_base_url], - "rootUrl": app_base_url, - "baseUrl": "/", - "attributes": { - "pkce.code.challenge.method": "S256", - "post.logout.redirect.uris": f"{app_base_url}/*", - }, - } - if not client_uuid: - status, body = request("POST", f"{base_url}/admin/realms/{realm}/clients", token, client_payload) - if status not in (201, 204, 409): - raise SystemExit(f"Client create failed: status={status} body={body}") - status, clients = request( - "GET", - f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}", - token, - ) - client_uuid = clients[0]["id"] if clients else None - if not client_uuid: - raise SystemExit("Client cassandra-web not found after create") - status, body = request( - "PUT", - f"{base_url}/admin/realms/{realm}/clients/{client_uuid}", - token, - client_payload, - ) - if status not in (200, 204): - raise SystemExit(f"Client update failed: status={status} body={body}") - - mapper_payload = { - "name": "groups", - "protocol": "openid-connect", - "protocolMapper": "oidc-group-membership-mapper", - "consentRequired": False, - "config": { - "full.path": "false", - "id.token.claim": "true", - "access.token.claim": "true", - "userinfo.token.claim": "true", - "claim.name": "groups", - "jsonType.label": "String", - }, - } - status, mappers = request( - "GET", - f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models", - token, - ) - if status != 200: - raise SystemExit(f"Mapper lookup failed: status={status}") - mapper_id = next((mapper.get("id") for mapper in mappers or [] if mapper.get("name") == "groups"), None) - if mapper_id: - mapper_update_payload = dict(mapper_payload) - mapper_update_payload["id"] = mapper_id - status, body = request( - "PUT", - f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models/{mapper_id}", - token, - mapper_update_payload, - ) - else: - status, body = request( - "POST", - f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models", - token, - mapper_payload, - ) - if status not in (200, 201, 204): - raise SystemExit(f"Mapper ensure failed: status={status} body={body}") - - status, secret = request( - "GET", - f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/client-secret", - token, - ) - client_secret = (secret or {}).get("value") - if status != 200 or not client_secret: - raise SystemExit(f"Client secret fetch failed: status={status}") - - vault_addr = os.environ.get("VAULT_ADDR", "http://vault.vault.svc.cluster.local:8200") - jwt = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip() - login_payload = json.dumps({"jwt": jwt, "role": os.environ.get("VAULT_ROLE", "sso-secrets")}).encode() - req = urllib.request.Request( - f"{vault_addr}/v1/auth/kubernetes/login", - data=login_payload, - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=20) as resp: - vault_token = json.loads(resp.read().decode())["auth"]["client_token"] - - payload = { - "data": { - "client_id": client_id, - "client_secret": client_secret, - "issuer": issuer, - "realm": realm, - "required_groups": "alpha,admin", - } - } - req = urllib.request.Request( - f"{vault_addr}/v1/kv/data/atlas/cassandra/cassandra-oidc", - data=json.dumps(payload).encode(), - headers={"X-Vault-Token": vault_token, "Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=20) as resp: - if resp.status not in (200, 204): - raise SystemExit(f"Vault write returned {resp.status}") - - print("Cassandra Keycloak realm/client ready") - PY diff --git a/services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml similarity index 58% rename from services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml index 38039196..7c5daa78 100644 --- a/services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml @@ -1,25 +1,19 @@ -# services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml -# One-off job for sso/cassandra-gitea-oidc-secret-ensure-1. -# Purpose: create/update the Keycloak cassandra realm Gitea OIDC client, ensure the -# cassandra-tester group exists, and write the client secret to Vault for Gitea. -# Run by setting spec.suspend to false, reconcile, then set it back to true. -# Safe to delete the finished Job/pod; it should not run continuously. +# services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml +# Purpose: create chat OIDC and Brad-only operator proxy clients. apiVersion: batch/v1 kind: Job metadata: - name: cassandra-gitea-oidc-secret-ensure-1 + name: hermes-access-oidc-client-ensure-3 namespace: sso spec: - suspend: true - backoffLimit: 0 - ttlSecondsAfterFinished: 3600 + backoffLimit: 3 template: metadata: annotations: vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-pre-populate-only: "true" - vault.hashicorp.com/role: "sso-secrets" - vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin" + vault.hashicorp.com/role: sso-secrets + vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: kv/data/atlas/shared/keycloak-admin vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: | {{ with secret "kv/data/atlas/shared/keycloak-admin" }} export KEYCLOAK_ADMIN="{{ .Data.data.username }}" @@ -29,11 +23,6 @@ spec: spec: serviceAccountName: mas-secrets-ensure restartPolicy: Never - volumes: - - name: cassandra-gitea-oidc-secret-ensure-script - configMap: - name: cassandra-gitea-oidc-secret-ensure-script - defaultMode: 0555 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -41,14 +30,32 @@ spec: - matchExpressions: - key: kubernetes.io/arch operator: In - values: ["arm64"] + values: + - arm64 - key: node-role.kubernetes.io/worker operator: Exists + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-18 containers: - name: apply image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 - command: ["/scripts/cassandra_gitea_oidc_secret_ensure.sh"] + command: + - /scripts/hermes_access_oidc_ensure.sh volumeMounts: - - name: cassandra-gitea-oidc-secret-ensure-script + - name: script mountPath: /scripts readOnly: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + volumes: + - name: script + configMap: + name: hermes-access-oidc-script + defaultMode: 0555 diff --git a/services/keycloak/kustomization.yaml b/services/keycloak/kustomization.yaml index 45288712..ff30ca99 100644 --- a/services/keycloak/kustomization.yaml +++ b/services/keycloak/kustomization.yaml @@ -27,8 +27,7 @@ resources: - bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml - bootstrap-jobs/quality-oidc-secret-ensure-job.yaml - bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml - - bootstrap-jobs/cassandra-realm-ensure-job.yaml - - bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml + - bootstrap-jobs/hermes-access-oidc-client-job.yaml - bootstrap-jobs/veles-realm-ensure-job.yaml - bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml - bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml @@ -57,9 +56,9 @@ configMapGenerator: - name: hermes-dashboard-oidc-client-script files: - hermes_dashboard_oidc_client_ensure.sh=scripts/hermes_dashboard_oidc_client_ensure.sh - - name: cassandra-gitea-oidc-secret-ensure-script + - name: hermes-access-oidc-script files: - - cassandra_gitea_oidc_secret_ensure.sh=scripts/cassandra_gitea_oidc_secret_ensure.sh + - hermes_access_oidc_ensure.sh=scripts/hermes_access_oidc_ensure.sh - name: veles-gitea-oidc-secret-ensure-script files: - veles_gitea_oidc_secret_ensure.sh=scripts/veles_gitea_oidc_secret_ensure.sh diff --git a/services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh b/services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh deleted file mode 100644 index fc1e2ae5..00000000 --- a/services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env sh -set -euo pipefail - -. /vault/secrets/keycloak-admin-env.sh - -KC_URL="${KEYCLOAK_SERVER:-http://keycloak.sso.svc.cluster.local}" -REALM="${KEYCLOAK_REALM:-cassandra}" -CLIENT_ID="${KEYCLOAK_CLIENT_ID:-gitea}" -PUBLIC_BASE_URL="${GITEA_PUBLIC_BASE_URL:-https://scm.bstein.dev}" -AUTH_SOURCE_NAME="${GITEA_AUTH_SOURCE_NAME:-cassandra}" -TESTER_GROUP="${CASSANDRA_GITEA_TESTER_GROUP:-cassandra-tester}" -VAULT_SECRET_PATH="${VAULT_SECRET_PATH:-kv/data/atlas/gitea/gitea-cassandra-oidc}" - -ACCESS_TOKEN="" -for attempt in 1 2 3 4 5 6 7 8 9 10; do - if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then - break - fi - echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2 - sleep $((attempt * 2)) -done -for attempt in 1 2 3 4 5; do - TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \ - -H 'Content-Type: application/x-www-form-urlencoded' \ - -d "grant_type=password" \ - -d "client_id=admin-cli" \ - -d "username=${KEYCLOAK_ADMIN}" \ - -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)" - ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)" - if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then - break - fi - echo "Keycloak token request failed (attempt ${attempt})" >&2 - sleep $((attempt * 2)) -done -if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then - echo "Failed to fetch Keycloak admin token" >&2 - exit 1 -fi - -ensure_group() { - group_name="$1" - groups="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/${REALM}/groups?search=$(printf '%s' "${group_name}" | jq -sRr @uri)" || true)" - if echo "$groups" | jq -e --arg name "$group_name" '.[]? | select(.name == $name)' >/dev/null 2>&1; then - return - fi - status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "$(jq -nc --arg name "$group_name" '{name:$name}')" \ - "${KC_URL}/admin/realms/${REALM}/groups")" - if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then - echo "Keycloak group create failed for ${group_name} (status ${status})" >&2 - exit 1 - fi -} - -ensure_group "${TESTER_GROUP}" - -CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)" -CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)" - -client_payload="$(jq -nc \ - --arg client_id "${CLIENT_ID}" \ - --arg root_url "${PUBLIC_BASE_URL}" \ - --arg callback "${PUBLIC_BASE_URL}/user/oauth2/${AUTH_SOURCE_NAME}/callback" \ - '{clientId:$client_id,enabled:true,protocol:"openid-connect",publicClient:false,standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false,serviceAccountsEnabled:false,redirectUris:[$callback],webOrigins:[$root_url],rootUrl:$root_url,baseUrl:"/",attributes:{"pkce.code.challenge.method":"S256","post.logout.redirect.uris":($root_url + "/*")}}')" - -if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then - status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${client_payload}" \ - "${KC_URL}/admin/realms/${REALM}/clients")" - if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then - echo "Keycloak client create failed for ${CLIENT_ID} (status ${status})" >&2 - exit 1 - fi - CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)" - CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)" -fi - -if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then - echo "Keycloak client ${CLIENT_ID} not found after create" >&2 - exit 1 -fi - -status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${client_payload}" \ - "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}")" -if [ "$status" != "200" ] && [ "$status" != "204" ]; then - echo "Keycloak client update failed for ${CLIENT_ID} (status ${status})" >&2 - exit 1 -fi - -mapper_payload="$(jq -nc \ - '{name:"groups",protocol:"openid-connect",protocolMapper:"oidc-group-membership-mapper",consentRequired:false,config:{"full.path":"false","id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","claim.name":"groups","jsonType.label":"String"}}')" -mappers="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models" || true)" -MAPPER_ID="$(echo "$mappers" | jq -r '.[]? | select(.name == "groups") | .id' | head -n1 || true)" -if [ -n "$MAPPER_ID" ] && [ "$MAPPER_ID" != "null" ]; then - status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${mapper_payload}" \ - "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models/${MAPPER_ID}")" -else - status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${mapper_payload}" \ - "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models")" -fi -if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then - echo "Keycloak groups mapper ensure failed (status ${status})" >&2 - exit 1 -fi - -CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/client-secret" | jq -r '.value' 2>/dev/null || true)" -if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then - echo "Keycloak client secret not found for ${CLIENT_ID}" >&2 - exit 1 -fi - -vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" -vault_role="${VAULT_ROLE:-sso-secrets}" -jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" -vault_token="$(curl -sS --request POST --data "${login_payload}" \ - "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" -if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then - echo "vault login failed" >&2 - exit 1 -fi - -payload="$(jq -nc \ - --arg client_id "${CLIENT_ID}" \ - --arg client_secret "${CLIENT_SECRET}" \ - --arg discovery_url "${PUBLIC_BASE_URL}" \ - --arg issuer "https://sso.bstein.dev/realms/${REALM}" \ - --arg auto_discovery_url "https://sso.bstein.dev/realms/${REALM}/.well-known/openid-configuration" \ - --arg auth_source_name "${AUTH_SOURCE_NAME}" \ - --arg tester_group "${TESTER_GROUP}" \ - --arg group_team_map "{\"${TESTER_GROUP}\":{\"cassandra-alpha\":[\"testers\"]}}" \ - '{data:{client_id:$client_id,client_secret:$client_secret,issuer:$issuer,openid_auto_discovery_url:$auto_discovery_url,auth_source_name:$auth_source_name,required_claim_name:"groups",required_claim_value:$tester_group,group_claim_name:"groups",restricted_group:$tester_group,group_team_map:$group_team_map,gitea_base_url:$discovery_url}}')" - -curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ - -H 'Content-Type: application/json' \ - -d "${payload}" "${vault_addr}/v1/${VAULT_SECRET_PATH}" >/dev/null - -echo "Cassandra realm Gitea OIDC client ready" diff --git a/services/keycloak/scripts/hermes_access_oidc_ensure.sh b/services/keycloak/scripts/hermes_access_oidc_ensure.sh new file mode 100755 index 00000000..5f390cf1 --- /dev/null +++ b/services/keycloak/scripts/hermes_access_oidc_ensure.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env sh +set -eu + +. /vault/secrets/keycloak-admin-env.sh + +KC_URL="http://keycloak.sso.svc.cluster.local" +CHAT_CLIENT="hermes-chat-dashboard" +CHAT_URL="https://chat.bstein.dev" +OPERATOR_CLIENT="hermes-operator-proxy" +OPERATOR_URL="https://agent.bstein.dev" + +ACCESS_TOKEN="" +for attempt in 1 2 3 4 5 6 7 8 9 10; do + if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then + break + fi + echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2 + sleep $((attempt * 2)) +done + +for attempt in 1 2 3 4 5; do + token_json="$(curl -sS -X POST "${KC_URL}/realms/master/protocol/openid-connect/token" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d "grant_type=password" \ + -d "client_id=admin-cli" \ + -d "username=${KEYCLOAK_ADMIN}" \ + -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)" + ACCESS_TOKEN="$(printf '%s' "${token_json}" | jq -r '.access_token' 2>/dev/null || true)" + if [ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ]; then + break + fi + echo "Keycloak token request failed (attempt ${attempt})" >&2 + sleep $((attempt * 2)) +done +if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then + echo "Failed to fetch Keycloak admin token" >&2 + exit 1 +fi + +chat_payload="$(jq -nc \ + --arg client_id "${CHAT_CLIENT}" \ + --arg redirect_uri "${CHAT_URL}/auth/callback" \ + --arg web_origin "${CHAT_URL}" \ + '{ + clientId:$client_id, + enabled:true, + protocol:"openid-connect", + publicClient:true, + standardFlowEnabled:true, + implicitFlowEnabled:false, + directAccessGrantsEnabled:false, + serviceAccountsEnabled:false, + redirectUris:[$redirect_uri], + webOrigins:[$web_origin], + rootUrl:$web_origin, + baseUrl:"/", + attributes:{ + "pkce.code.challenge.method":"S256", + "post.logout.redirect.uris":$web_origin, + "access.token.lifespan":"1200" + } + }')" + +chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)" +chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)" +if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${chat_payload}" \ + "${KC_URL}/admin/realms/atlas/clients")" + if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then + echo "Keycloak chat client create failed (status ${status})" >&2 + exit 1 + fi + chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)" + chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)" +fi +if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then + echo "Keycloak chat client not found after create" >&2 + exit 1 +fi + +status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${chat_payload}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}")" +if [ "${status}" != "204" ]; then + echo "Keycloak chat client update failed (status ${status})" >&2 + exit 1 +fi + +scope_id="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/client-scopes?search=groups" \ + | jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)" +if [ -z "${scope_id}" ] || [ "${scope_id}" = "null" ]; then + echo "Keycloak groups client scope not found" >&2 + exit 1 +fi +default_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}/default-client-scopes" || true)" +optional_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes" || true)" +if ! printf '%s' "${default_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1 \ + && ! printf '%s' "${optional_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")" + if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")" + if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then + echo "Failed to attach groups scope to chat client (status ${status})" >&2 + exit 1 + fi + fi +fi + +operator_payload="$(jq -nc \ + --arg client_id "${OPERATOR_CLIENT}" \ + --arg redirect_uri "${OPERATOR_URL}/oauth2/callback" \ + --arg web_origin "${OPERATOR_URL}" \ + '{ + clientId:$client_id, + enabled:true, + protocol:"openid-connect", + publicClient:false, + standardFlowEnabled:true, + implicitFlowEnabled:false, + directAccessGrantsEnabled:false, + serviceAccountsEnabled:false, + redirectUris:[$redirect_uri], + webOrigins:[$web_origin], + rootUrl:$web_origin, + baseUrl:"/", + attributes:{"post.logout.redirect.uris":$web_origin} + }')" + +operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)" +operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)" +if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${operator_payload}" \ + "${KC_URL}/admin/realms/atlas/clients")" + if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then + echo "Keycloak operator proxy client create failed (status ${status})" >&2 + exit 1 + fi + operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)" + operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)" +fi +if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then + echo "Keycloak operator proxy client not found after create" >&2 + exit 1 +fi + +status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${operator_payload}" \ + "${KC_URL}/admin/realms/atlas/clients/${operator_id}")" +if [ "${status}" != "204" ]; then + echo "Keycloak operator proxy client update failed (status ${status})" >&2 + exit 1 +fi + +client_secret="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${operator_id}/client-secret" \ + | jq -r '.value' 2>/dev/null || true)" +if [ -z "${client_secret}" ] || [ "${client_secret}" = "null" ]; then + echo "Keycloak operator proxy client secret not found" >&2 + exit 1 +fi + +vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" +vault_role="${VAULT_ROLE:-sso-secrets}" +jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" +login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt,role:$role}')" +vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" +if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "Vault login failed" >&2 + exit 1 +fi + +read_status="$(curl -sS -o /tmp/hermes-operator-oidc-read.json -w '%{http_code}' \ + -H "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc" || true)" +cookie_secret="" +if [ "${read_status}" = "200" ]; then + cookie_secret="$(jq -r '.data.data.cookie_secret // empty' /tmp/hermes-operator-oidc-read.json)" +elif [ "${read_status}" != "404" ]; then + echo "Vault operator OIDC read failed (status ${read_status})" >&2 + exit 1 +fi +if [ -n "${cookie_secret}" ]; then + cookie_length="$(printf '%s' "${cookie_secret}" | wc -c | tr -d ' ')" + if [ "${cookie_length}" != "16" ] && [ "${cookie_length}" != "24" ] && [ "${cookie_length}" != "32" ]; then + cookie_secret="" + fi +fi +if [ -z "${cookie_secret}" ]; then + cookie_secret="$(openssl rand -hex 16 | tr -d '\n')" +fi + +vault_payload="$(jq -nc \ + --arg client_id "${OPERATOR_CLIENT}" \ + --arg client_secret "${client_secret}" \ + --arg cookie_secret "${cookie_secret}" \ + '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')" +write_status="$(curl -sS -o /tmp/hermes-operator-oidc-write.json -w '%{http_code}' -X POST \ + -H "X-Vault-Token: ${vault_token}" \ + -H 'Content-Type: application/json' \ + -d "${vault_payload}" \ + "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc")" +if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then + echo "Vault operator OIDC write failed (status ${write_status})" >&2 + exit 1 +fi + +echo "Hermes chat and operator OIDC clients are ready" diff --git a/services/maintenance/apps/ariadne-deployment.yaml b/services/maintenance/apps/ariadne-deployment.yaml index 42d9a836..bdaeadf3 100644 --- a/services/maintenance/apps/ariadne-deployment.yaml +++ b/services/maintenance/apps/ariadne-deployment.yaml @@ -285,8 +285,21 @@ spec: - name: GAME_MODE_NODE_NAME value: titan-24 - name: GAME_MODE_DISPLACE_WORKLOADS - value: >- - [{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}] + value: "[]" + - name: GAME_MODE_LEASE_NAMESPACE + value: hermes + - name: GAME_MODE_LEASE_NAME + value: titan-24-gpu-owner + - name: GAME_MODE_OLLAMA_URL + value: http://hermes-ollama.hermes.svc.cluster.local:11434 + - name: GAME_MODE_OLLAMA_MODEL + value: gpt-oss:20b + - name: GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC + value: "900" + - name: GAME_MODE_TRANSITION_TIMEOUT_SEC + value: "900" + - name: GAME_MODE_POLL_INTERVAL_SEC + value: "1" - name: WOLF_OIDC_CLIENT_ID value: wolf - name: WOLF_OIDC_BASE_URL @@ -418,7 +431,7 @@ spec: - name: ARIADNE_VM_URL value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 - name: ARIADNE_CLUSTER_STATE_VM_TIMEOUT_SEC - value: "5" + value: "30" - name: ARIADNE_ALERTMANAGER_URL value: http://alertmanager.monitoring.svc.cluster.local - name: OPENSEARCH_URL @@ -444,11 +457,11 @@ spec: - name: ARIADNE_SCHEDULE_TESTING_TRIAGE value: "*/15 * * * *" - name: ARIADNE_TESTING_TRIAGE_MODEL_URL - value: http://hermes-ollama.hermes.svc.cluster.local:11434 + value: http://hermes-model-gate.hermes.svc.cluster.local:11434 - name: ARIADNE_TESTING_TRIAGE_MODEL - value: qwen2.5:7b-instruct-q4_0 + value: gpt-oss:20b - name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC - value: "180" + value: "900" - name: JENKINS_WORKSPACE_NAMESPACE value: jenkins - name: JENKINS_WORKSPACE_PVC_PREFIX diff --git a/services/maintenance/apps/ariadne-rbac.yaml b/services/maintenance/apps/ariadne-rbac.yaml index 941942bc..61ab9dec 100644 --- a/services/maintenance/apps/ariadne-rbac.yaml +++ b/services/maintenance/apps/ariadne-rbac.yaml @@ -78,7 +78,6 @@ rules: - get - list - watch - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/services/maintenance/apps/metis-configmap.yaml b/services/maintenance/apps/metis-configmap.yaml index 393993f1..90b88af1 100644 --- a/services/maintenance/apps/metis-configmap.yaml +++ b/services/maintenance/apps/metis-configmap.yaml @@ -15,8 +15,8 @@ data: METIS_MAX_DEVICE_BYTES: "1000000000000" METIS_NAMESPACE: maintenance METIS_REMOTE_POD_TIMEOUT_SEC: "14400" - METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-243-amd64 # {"$imagepolicy": "maintenance:metis-amd64"} - METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-arm64"} + METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-270-amd64 # {"$imagepolicy": "maintenance:metis-amd64"} + METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-arm64"} METIS_HARBOR_REGISTRY: registry.bstein.dev METIS_HARBOR_PROJECT: metis METIS_HARBOR_API_BASE: https://registry.bstein.dev/api/v2.0 diff --git a/services/maintenance/kustomization.yaml b/services/maintenance/kustomization.yaml index 27a4540a..62889e7a 100644 --- a/services/maintenance/kustomization.yaml +++ b/services/maintenance/kustomization.yaml @@ -11,8 +11,8 @@ resources: - networking images: - name: registry.bstein.dev/bstein/ariadne - newTag: 0.1.0-356 # {"$imagepolicy": "maintenance:ariadne:tag"} + newTag: 0.1.0-389 # {"$imagepolicy": "maintenance:ariadne:tag"} - name: registry.bstein.dev/bstein/metis - newTag: 0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"} + newTag: 0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"} - name: registry.bstein.dev/bstein/soteria newTag: 0.1.0-120 # {"$imagepolicy": "maintenance:soteria:tag"} diff --git a/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml b/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml index 3a881009..e1295701 100644 --- a/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml +++ b/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml @@ -32,7 +32,7 @@ spec: kubernetes.io/arch: amd64 containers: - name: metis-sentinel - image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"} + image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-270-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"} imagePullPolicy: Always envFrom: - configMapRef: diff --git a/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml b/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml index ccdbaed1..be9bc2de 100644 --- a/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml +++ b/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml @@ -32,7 +32,7 @@ spec: kubernetes.io/arch: arm64 containers: - name: metis-sentinel - image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"} + image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-270-arm64 # {"$imagepolicy": "maintenance:metis-sentinel-arm64"} imagePullPolicy: Always envFrom: - configMapRef: diff --git a/services/maintenance/node-ops/node-image-sweeper-daemonset.yaml b/services/maintenance/node-ops/node-image-sweeper-daemonset.yaml index 41a04d8b..05b4d71a 100644 --- a/services/maintenance/node-ops/node-image-sweeper-daemonset.yaml +++ b/services/maintenance/node-ops/node-image-sweeper-daemonset.yaml @@ -25,6 +25,14 @@ spec: - key: node-role.kubernetes.io/master operator: Exists effect: NoSchedule + - key: veles.bstein.dev/simulation + operator: Equal + value: "true" + effect: NoSchedule + - key: cassandra.bstein.dev/simulation + operator: Equal + value: "true" + effect: NoSchedule nodeSelector: kubernetes.io/os: linux containers: diff --git a/services/monitoring/availability-backfill-v4-job.yaml b/services/monitoring/availability-backfill-v4-job.yaml new file mode 100644 index 00000000..3aedd0c1 --- /dev/null +++ b/services/monitoring/availability-backfill-v4-job.yaml @@ -0,0 +1,92 @@ +# services/monitoring/availability-backfill-v4-job.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlas-availability-request-v4-backfill-rules + namespace: monitoring +data: + atlas-request-history.yaml: | + groups: + - name: atlas.availability.request.backfill + interval: 1h + rules: + - record: atlas:availability:requests_1h + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"[1-5].." + }[1h] + )) + labels: + definition: request-v4 + scope: atlas + rollup: hourly + - record: atlas:availability:failures_1h + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"5.." + }[1h] + )) + labels: + definition: request-v4 + scope: atlas + rollup: hourly + +--- + +apiVersion: batch/v1 +kind: Job +metadata: + name: atlas-availability-request-v4-backfill + namespace: monitoring +spec: + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-request-v4-backfill + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: vmalert-replay + image: victoriametrics/vmalert:v1.113.0 + args: + - -datasource.url=http://victoria-metrics-single-server:8428 + - -remoteWrite.url=http://victoria-metrics-single-server:8428 + - -remoteWrite.flushInterval=1s + - -rule=/etc/vmalert/backfill/*.yaml + - -replay.timeFrom=2026-05-01T00:00:00Z + - -replay.timeTo=2026-08-04T23:00:00Z + - -replay.maxDatapointsPerQuery=48 + - -replay.rulesDelay=2s + - -replay.disableProgressBar + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi + volumeMounts: + - name: rules + mountPath: /etc/vmalert/backfill + readOnly: true + volumes: + - name: rules + configMap: + name: atlas-availability-request-v4-backfill-rules diff --git a/services/monitoring/availability-daily-backfill-v4-job.yaml b/services/monitoring/availability-daily-backfill-v4-job.yaml new file mode 100644 index 00000000..dbd04bd7 --- /dev/null +++ b/services/monitoring/availability-daily-backfill-v4-job.yaml @@ -0,0 +1,92 @@ +# services/monitoring/availability-daily-backfill-v4-job.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlas-availability-request-v4-daily-backfill-rules + namespace: monitoring +data: + atlas-request-daily-history.yaml: | + groups: + - name: atlas.availability.request.daily.backfill + interval: 1d + rules: + - record: atlas:availability:requests_1d + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"[1-5].." + }[1d] + )) + labels: + definition: request-v4 + scope: atlas + rollup: daily + - record: atlas:availability:failures_1d + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"5.." + }[1d] + )) + labels: + definition: request-v4 + scope: atlas + rollup: daily + +--- + +apiVersion: batch/v1 +kind: Job +metadata: + name: atlas-availability-request-v4-daily-backfill + namespace: monitoring +spec: + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-request-v4-daily-backfill + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: vmalert-replay + image: victoriametrics/vmalert:v1.113.0 + args: + - -datasource.url=http://victoria-metrics-single-server:8428 + - -remoteWrite.url=http://victoria-metrics-single-server:8428 + - -remoteWrite.flushInterval=1s + - -rule=/etc/vmalert/backfill/*.yaml + - -replay.timeFrom=2026-05-01T00:00:00Z + - -replay.timeTo=2026-08-04T00:00:00Z + - -replay.maxDatapointsPerQuery=4 + - -replay.rulesDelay=2s + - -replay.disableProgressBar + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi + volumeMounts: + - name: rules + mountPath: /etc/vmalert/backfill + readOnly: true + volumes: + - name: rules + configMap: + name: atlas-availability-request-v4-daily-backfill-rules diff --git a/services/monitoring/availability-legacy-cleanup-job.yaml b/services/monitoring/availability-legacy-cleanup-job.yaml new file mode 100644 index 00000000..70f30e07 --- /dev/null +++ b/services/monitoring/availability-legacy-cleanup-job.yaml @@ -0,0 +1,47 @@ +# services/monitoring/availability-legacy-cleanup-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: atlas-availability-legacy-series-cleanup-v1 + namespace: monitoring +spec: + activeDeadlineSeconds: 300 + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-legacy-series-cleanup + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: cleanup + image: python:3.12-alpine + command: ["python", "/scripts/availability_cleanup.py"] + env: + - name: VM_URL + value: http://victoria-metrics-single-server:8428 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: script + configMap: + name: atlas-availability-cleanup-script diff --git a/services/monitoring/availability-rollup-cronjob.yaml b/services/monitoring/availability-rollup-cronjob.yaml new file mode 100644 index 00000000..c42ed33a --- /dev/null +++ b/services/monitoring/availability-rollup-cronjob.yaml @@ -0,0 +1,103 @@ +# services/monitoring/availability-rollup-cronjob.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: atlas-availability-request-v4-rollup-initial + namespace: monitoring +spec: + activeDeadlineSeconds: 900 + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-request-v4-rollup + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: rollup + image: python:3.12-alpine + command: ["python", "/scripts/availability_rollup.py"] + env: + - name: VM_URL + value: http://victoria-metrics-single-server:8428 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: script + configMap: + name: atlas-availability-rollup-script + +--- + +apiVersion: batch/v1 +kind: CronJob +metadata: + name: atlas-availability-request-v4-rollup + namespace: monitoring +spec: + schedule: "10 0 * * *" + timeZone: Etc/UTC + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + activeDeadlineSeconds: 900 + backoffLimit: 2 + template: + metadata: + labels: + app: atlas-availability-request-v4-rollup + spec: + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-22 + - titan-24 + containers: + - name: rollup + image: python:3.12-alpine + command: ["python", "/scripts/availability_rollup.py"] + env: + - name: VM_URL + value: http://victoria-metrics-single-server:8428 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: script + configMap: + name: atlas-availability-rollup-script diff --git a/services/monitoring/dashboards/atlas-gpu.json b/services/monitoring/dashboards/atlas-gpu.json index 6bc000fd..beb788fb 100644 --- a/services/monitoring/dashboards/atlas-gpu.json +++ b/services/monitoring/dashboards/atlas-gpu.json @@ -20,7 +20,7 @@ }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -72,7 +72,7 @@ "targetBlank": false } ], - "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero." + "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero." }, { "id": 2, @@ -128,7 +128,7 @@ }, "targets": [ { - "expr": "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{Hostname}}" } @@ -152,7 +152,7 @@ { "id": 4, "type": "table", - "title": "GPU Pods Reporting Device Util", + "title": "GPU Processes by Pod", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -165,7 +165,7 @@ }, "targets": [ { - "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))", + "expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)", "refId": "A" } ], @@ -188,7 +188,7 @@ "options": {} } ], - "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value." + "description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization." } ], "time": { diff --git a/services/monitoring/dashboards/atlas-network.json b/services/monitoring/dashboards/atlas-network.json index 5c057135..90ccaa6c 100644 --- a/services/monitoring/dashboards/atlas-network.json +++ b/services/monitoring/dashboards/atlas-network.json @@ -476,7 +476,7 @@ }, "targets": [ { - "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } diff --git a/services/monitoring/dashboards/atlas-nodes.json b/services/monitoring/dashboards/atlas-nodes.json index ea2248e8..b9320520 100644 --- a/services/monitoring/dashboards/atlas-nodes.json +++ b/services/monitoring/dashboards/atlas-nodes.json @@ -20,7 +20,7 @@ }, "targets": [ { - "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})", + "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})", "refId": "A" } ], @@ -46,7 +46,7 @@ "unit": "none", "custom": { "displayMode": "auto", - "valueSuffix": "/21" + "valueSuffix": "/18" } }, "overrides": [] @@ -409,7 +409,7 @@ }, "targets": [ { - "expr": "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } @@ -449,7 +449,7 @@ }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } @@ -489,7 +489,7 @@ }, "targets": [ { - "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -526,7 +526,7 @@ }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -563,7 +563,7 @@ }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } @@ -601,7 +601,7 @@ }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index 7b27ab88..37ea4017 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -216,7 +216,7 @@ { "id": 27, "type": "stat", - "title": "Atlas Serving Availability", + "title": "Atlas Availability (365d)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[48h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -286,7 +286,7 @@ }, "textMode": "value" }, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists." }, { "id": 4, @@ -304,7 +304,7 @@ }, "targets": [ { - "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})) or on() vector(0)", + "expr": "((sum(max by(namespace,pod) ((kube_pod_status_phase{phase=\"Pending\",namespace!~\"veles\"} == 1) and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)) + (sum(max by(namespace,pod) ((kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace!~\"veles\"} == 1) unless on(namespace,pod) kube_pod_owner{owner_kind=\"Job\"})) or on() vector(0)))", "refId": "A" } ], @@ -362,7 +362,7 @@ "targetBlank": true } ], - "description": "Pods in unhealthy phases; zero is good, any count means a workload needs attention." + "description": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here." }, { "id": 6, @@ -380,7 +380,7 @@ }, "targets": [ { - "expr": "sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})) or on() vector(0)", + "expr": "sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason{namespace!~\"veles\",reason=~\"CrashLoopBackOff|ImagePullBackOff\"} and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)", "refId": "A" } ], @@ -438,7 +438,7 @@ "targetBlank": true } ], - "description": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service." + "description": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard." }, { "id": 1, @@ -456,14 +456,14 @@ }, "targets": [ { - "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})", + "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})", "refId": "A" } ], "fieldConfig": { "defaults": { "min": 0, - "max": 21, + "max": 18, "thresholds": { "mode": "absolute", "steps": [ @@ -473,15 +473,15 @@ }, { "color": "dark-orange", - "value": 19 + "value": 16 }, { "color": "dark-yellow", - "value": 20 + "value": 17 }, { "color": "dark-green", - "value": 21 + "value": 18 } ] } @@ -518,7 +518,7 @@ }, "targets": [ { - "expr": "label_replace(topk(1, avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(topk(1, avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}", "instant": true @@ -596,7 +596,7 @@ }, "targets": [ { - "expr": "label_replace(topk(1, avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(topk(1, avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}", "instant": true @@ -674,7 +674,7 @@ }, "targets": [ { - "expr": "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]) + rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]) + rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}", "instant": true @@ -744,7 +744,7 @@ }, "targets": [ { - "expr": "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_disk_read_bytes_total[5m]) + rate(node_disk_written_bytes_total[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_disk_read_bytes_total[5m]) + rate(node_disk_written_bytes_total[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}", "instant": true @@ -3728,7 +3728,7 @@ }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -3780,7 +3780,7 @@ "targetBlank": false } ], - "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero." + "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero." }, { "id": 13, @@ -3867,7 +3867,7 @@ }, "targets": [ { - "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -3915,7 +3915,7 @@ }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -3963,7 +3963,7 @@ }, "targets": [ { - "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -4001,7 +4001,7 @@ }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -4308,7 +4308,7 @@ }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } @@ -4357,7 +4357,7 @@ }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } diff --git a/services/monitoring/dashboards/atlas-pods.json b/services/monitoring/dashboards/atlas-pods.json index 648873c8..e3f06b0f 100644 --- a/services/monitoring/dashboards/atlas-pods.json +++ b/services/monitoring/dashboards/atlas-pods.json @@ -20,7 +20,7 @@ }, "targets": [ { - "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})) or on() vector(0)", + "expr": "((sum(max by(namespace,pod) ((kube_pod_status_phase{phase=\"Pending\",namespace!~\"veles\"} == 1) and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)) + (sum(max by(namespace,pod) ((kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace!~\"veles\"} == 1) unless on(namespace,pod) kube_pod_owner{owner_kind=\"Job\"})) or on() vector(0)))", "refId": "A" } ], @@ -80,7 +80,7 @@ }, "targets": [ { - "expr": "sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})) or on() vector(0)", + "expr": "sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason{namespace!~\"veles\",reason=~\"CrashLoopBackOff|ImagePullBackOff\"} and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)", "refId": "A" } ], @@ -523,7 +523,7 @@ }, "targets": [ { - "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)))))", + "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)))))", "refId": "A", "instant": true, "format": "table" diff --git a/services/monitoring/dashboards/atlas-storage.json b/services/monitoring/dashboards/atlas-storage.json index 9f989234..aa5cedb0 100644 --- a/services/monitoring/dashboards/atlas-storage.json +++ b/services/monitoring/dashboards/atlas-storage.json @@ -276,7 +276,7 @@ }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -314,7 +314,7 @@ }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } diff --git a/services/monitoring/dcgm-exporter.yaml b/services/monitoring/dcgm-exporter.yaml index e8c3b03d..c57af067 100644 --- a/services/monitoring/dcgm-exporter.yaml +++ b/services/monitoring/dcgm-exporter.yaml @@ -53,27 +53,15 @@ spec: ports: - name: metrics containerPort: 9400 - env: - - name: DCGM_EXPORTER_KUBERNETES - value: "true" - - name: KUBERNETES_VIRTUAL_GPUS - value: "true" - - name: NVIDIA_RESOURCE_NAMES - value: nvidia.com/gpu.shared securityContext: privileged: true resources: requests: cpu: 50m - memory: 64Mi - volumeMounts: - - name: pod-resources - mountPath: /var/lib/kubelet/pod-resources - volumes: - - name: pod-resources - hostPath: - path: /var/lib/kubelet/pod-resources - type: Directory + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi --- apiVersion: v1 kind: Service diff --git a/services/monitoring/grafana-alerting-config.yaml b/services/monitoring/grafana-alerting-config.yaml index a9024843..6e408752 100644 --- a/services/monitoring/grafana-alerting-config.yaml +++ b/services/monitoring/grafana-alerting-config.yaml @@ -109,7 +109,7 @@ data: model: intervalMs: 60000 maxDataPoints: 43200 - expr: max by (instance, node) ((increase((node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} - node_filesystem_free_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})[1h]) / 1024 / 1024 / 1024) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)")) + expr: max by (instance, node) ((clamp_min(delta((node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})[1h:1m]), 0) / 1024 / 1024 / 1024) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)")) legendFormat: '{{node}}' datasource: type: prometheus @@ -162,7 +162,7 @@ data: model: intervalMs: 60000 maxDataPoints: 43200 - expr: ((1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)") + expr: clamp_max(clamp_min(((1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100), 0), 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)") legendFormat: '{{node}}' datasource: type: prometheus @@ -496,7 +496,7 @@ data: labels: severity: warning - uid: maint-soteria-backup-unhealthy - title: "Soteria reports unhealthy PVC backups" + title: "Soteria reports configured PVC backups unhealthy" condition: C for: "10m" data: @@ -506,7 +506,7 @@ data: to: 0 datasourceUid: atlas-vm model: - expr: sum((1 - pvc_backup_health) > bool 0) or on() vector(0) + expr: sum((pvc_backup_health == 0) and on(namespace,pvc,volume,driver) (pvc_backup_count > 0)) or on() vector(0) intervalMs: 60000 maxDataPoints: 43200 legendFormat: unhealthy-pvcs @@ -540,7 +540,7 @@ data: noDataState: OK execErrState: Alerting annotations: - summary: "One or more PVCs are stale, missing, or failed per Soteria backup health" + summary: "A previously configured PVC backup is stale or failed per Soteria" labels: severity: warning - uid: maint-soteria-b2-scan-unhealthy @@ -650,7 +650,7 @@ data: to: 0 datasourceUid: atlas-vm model: - expr: sum(increase(kube_job_created{namespace="maintenance",job_name=~"soteria-backup-.*"}[10m])) or on() vector(0) + expr: count(kube_job_created{namespace="maintenance",job_name=~"soteria-backup-.*"} > time() - 600) or on() vector(0) intervalMs: 60000 maxDataPoints: 43200 legendFormat: soteria-backup-jobs-created-10m @@ -703,7 +703,7 @@ data: to: 0 datasourceUid: atlas-vm model: - expr: max by (task) (((time() - ariadne_schedule_last_success_timestamp_seconds{task=~"schedule\\..+"}) * on(task) group_left() (1 - ariadne_schedule_last_status{task=~"schedule\\..+"}))) + expr: max by (task) ((((time() - ariadne_schedule_last_error_timestamp_seconds{task=~"schedule\\..+"}) * on(task) group_left() (1 - ariadne_schedule_last_status{task=~"schedule\\..+"})) and on(task) (ariadne_schedule_next_run_timestamp_seconds{task=~"schedule\\..+"} < time() + 3600))) intervalMs: 60000 maxDataPoints: 43200 legendFormat: '{{task}}' @@ -806,7 +806,7 @@ data: model: intervalMs: 60000 maxDataPoints: 43200 - expr: max(postmark_outbound_bounce_rate{window="1d"}) or on() vector(0) + expr: (max(postmark_outbound_bounce_rate{window="1d"}) and on() (max(postmark_outbound_sent{window="1d"}) >= 50) and on() (max(postmark_outbound_bounced{window="1d"}) >= 3)) or on() vector(0) legendFormat: bounce 1d datasource: type: prometheus @@ -838,7 +838,7 @@ data: noDataState: OK execErrState: Error annotations: - summary: "Postmark 1d bounce rate >5%" + summary: "Postmark 1d bounce rate >5% with at least 50 sent and 3 bounced" labels: severity: warning - uid: postmark-api-down diff --git a/services/monitoring/grafana-dashboard-cassandra.yaml b/services/monitoring/grafana-dashboard-cassandra.yaml new file mode 100644 index 00000000..e8f59a43 --- /dev/null +++ b/services/monitoring/grafana-dashboard-cassandra.yaml @@ -0,0 +1,611 @@ +# services/monitoring/grafana-dashboard-cassandra.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-cassandra + labels: + grafana_dashboard: "1" +data: + cassandra-rollup.json: | + { + "uid": "cassandra-rollup", + "title": "Cassandra", + "folderUid": "atlas-internal", + "editable": true, + "tags": [ + "cassandra", + "veles", + "migration" + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "refresh": "30s", + "schemaVersion": 39, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Cassandra Ready Containers", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "expr": "sum(kube_pod_container_status_ready{namespace=\"cassandra\"}) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-green", + "value": 4 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "id": 2, + "type": "stat", + "title": "Legacy Veles Ready Containers", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "expr": "sum(kube_pod_container_status_ready{namespace=\"veles\"}) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-green", + "value": 4 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "id": 3, + "type": "stat", + "title": "Cassandra + Veles Problem Pods", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{namespace=~\"cassandra|veles\",phase!~\"Running|Succeeded\"})) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "id": 4, + "type": "stat", + "title": "Cassandra + Veles Restarts 1h", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"cassandra|veles\"}[1h])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Ready Containers By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "expr": "sum by (namespace) (kube_pod_container_status_ready{namespace=~\"cassandra|veles\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Problem Pods By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "expr": "sum by (namespace) (max by (namespace,pod) (kube_pod_status_phase{namespace=~\"cassandra|veles\",phase!~\"Running|Succeeded\"}))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "CPU By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"cassandra|veles\",container!=\"\",pod!=\"\"}[5m]))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Memory By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"cassandra|veles\",container!=\"\",pod!=\"\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 9, + "type": "bargauge", + "title": "PVC Used By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "targets": [ + { + "expr": "sum by (namespace) (kubelet_volume_stats_used_bytes{namespace=~\"cassandra|veles\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + } + }, + { + "id": 10, + "type": "timeseries", + "title": "PVC Usage Percent", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "targets": [ + { + "expr": "100 * sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"cassandra|veles\"}) / sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"cassandra|veles\"})", + "legendFormat": "{{namespace}} / {{persistentvolumeclaim}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Migration Jobs", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "targets": [ + { + "expr": "sum by (namespace,job_name) (kube_job_status_failed{namespace=~\"cassandra|veles\",job_name=~\"cassandra-.*|veles-.*\"})", + "legendFormat": "failed {{namespace}} / {{job_name}}", + "refId": "A" + }, + { + "expr": "sum by (namespace,job_name) (kube_job_status_succeeded{namespace=~\"cassandra|veles\",job_name=~\"cassandra-.*|veles-.*\"})", + "legendFormat": "succeeded {{namespace}} / {{job_name}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Titan-23 Root Filesystem", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "targets": [ + { + "expr": "100 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} * 100) * on(instance) group_left(nodename) node_uname_info{nodename=\"titan-23\"}", + "legendFormat": "titan-23 /", + "refId": "A" + }, + { + "expr": "100 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/veles\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/veles\",fstype!~\"tmpfs|overlay\"} * 100) * on(instance) group_left(nodename) node_uname_info{nodename=\"titan-23\"}", + "legendFormat": "titan-23 /mnt/veles", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + } + ] + } diff --git a/services/monitoring/grafana-dashboard-gpu.yaml b/services/monitoring/grafana-dashboard-gpu.yaml index 40692ea7..c96c615f 100644 --- a/services/monitoring/grafana-dashboard-gpu.yaml +++ b/services/monitoring/grafana-dashboard-gpu.yaml @@ -29,7 +29,7 @@ data: }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -81,7 +81,7 @@ data: "targetBlank": false } ], - "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero." + "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero." }, { "id": 2, @@ -137,7 +137,7 @@ data: }, "targets": [ { - "expr": "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{Hostname}}" } @@ -161,7 +161,7 @@ data: { "id": 4, "type": "table", - "title": "GPU Pods Reporting Device Util", + "title": "GPU Processes by Pod", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -174,7 +174,7 @@ data: }, "targets": [ { - "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))", + "expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)", "refId": "A" } ], @@ -197,7 +197,7 @@ data: "options": {} } ], - "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value." + "description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization." } ], "time": { diff --git a/services/monitoring/grafana-dashboard-network.yaml b/services/monitoring/grafana-dashboard-network.yaml index 343da7d0..c3faad4b 100644 --- a/services/monitoring/grafana-dashboard-network.yaml +++ b/services/monitoring/grafana-dashboard-network.yaml @@ -485,7 +485,7 @@ data: }, "targets": [ { - "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } diff --git a/services/monitoring/grafana-dashboard-nodes.yaml b/services/monitoring/grafana-dashboard-nodes.yaml index 4cd65e0b..480bbf68 100644 --- a/services/monitoring/grafana-dashboard-nodes.yaml +++ b/services/monitoring/grafana-dashboard-nodes.yaml @@ -29,7 +29,7 @@ data: }, "targets": [ { - "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})", + "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})", "refId": "A" } ], @@ -55,7 +55,7 @@ data: "unit": "none", "custom": { "displayMode": "auto", - "valueSuffix": "/21" + "valueSuffix": "/18" } }, "overrides": [] @@ -418,7 +418,7 @@ data: }, "targets": [ { - "expr": "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } @@ -458,7 +458,7 @@ data: }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } @@ -498,7 +498,7 @@ data: }, "targets": [ { - "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -535,7 +535,7 @@ data: }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -572,7 +572,7 @@ data: }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } @@ -610,7 +610,7 @@ data: }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index 3f6ef564..00ae1435 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -225,7 +225,7 @@ data: { "id": 27, "type": "stat", - "title": "Atlas Serving Availability", + "title": "Atlas Availability (365d)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"request-v4\"}[48h]) or on() (1 - ((sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"5..\"}[1h])) or on() vector(0)) / clamp_min(sum(increase(traefik_entrypoint_requests_total{entrypoint=\"websecure\",protocol=\"http\",code=~\"[1-5]..\"}[1h])), 1))))", "refId": "A", "instant": true } @@ -295,7 +295,7 @@ data: }, "textMode": "value" }, - "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available." + "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists." }, { "id": 4, @@ -313,7 +313,7 @@ data: }, "targets": [ { - "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})) or on() vector(0)", + "expr": "((sum(max by(namespace,pod) ((kube_pod_status_phase{phase=\"Pending\",namespace!~\"veles\"} == 1) and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)) + (sum(max by(namespace,pod) ((kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace!~\"veles\"} == 1) unless on(namespace,pod) kube_pod_owner{owner_kind=\"Job\"})) or on() vector(0)))", "refId": "A" } ], @@ -371,7 +371,7 @@ data: "targetBlank": true } ], - "description": "Pods in unhealthy phases; zero is good, any count means a workload needs attention." + "description": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here." }, { "id": 6, @@ -389,7 +389,7 @@ data: }, "targets": [ { - "expr": "sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})) or on() vector(0)", + "expr": "sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason{namespace!~\"veles\",reason=~\"CrashLoopBackOff|ImagePullBackOff\"} and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)", "refId": "A" } ], @@ -447,7 +447,7 @@ data: "targetBlank": true } ], - "description": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service." + "description": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard." }, { "id": 1, @@ -465,14 +465,14 @@ data: }, "targets": [ { - "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})", + "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})", "refId": "A" } ], "fieldConfig": { "defaults": { "min": 0, - "max": 21, + "max": 18, "thresholds": { "mode": "absolute", "steps": [ @@ -482,15 +482,15 @@ data: }, { "color": "dark-orange", - "value": 19 + "value": 16 }, { "color": "dark-yellow", - "value": 20 + "value": 17 }, { "color": "dark-green", - "value": 21 + "value": 18 } ] } @@ -527,7 +527,7 @@ data: }, "targets": [ { - "expr": "label_replace(topk(1, avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(topk(1, avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}", "instant": true @@ -605,7 +605,7 @@ data: }, "targets": [ { - "expr": "label_replace(topk(1, avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(topk(1, avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}", "instant": true @@ -683,7 +683,7 @@ data: }, "targets": [ { - "expr": "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]) + rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]) + rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}", "instant": true @@ -753,7 +753,7 @@ data: }, "targets": [ { - "expr": "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_disk_read_bytes_total[5m]) + rate(node_disk_written_bytes_total[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_disk_read_bytes_total[5m]) + rate(node_disk_written_bytes_total[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}", "instant": true @@ -3737,7 +3737,7 @@ data: }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -3789,7 +3789,7 @@ data: "targetBlank": false } ], - "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero." + "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero." }, { "id": 13, @@ -3876,7 +3876,7 @@ data: }, "targets": [ { - "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -3924,7 +3924,7 @@ data: }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -3972,7 +3972,7 @@ data: }, "targets": [ { - "expr": "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((clamp_max(clamp_min((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100, 0), 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -4010,7 +4010,7 @@ data: }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -4317,7 +4317,7 @@ data: }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } @@ -4366,7 +4366,7 @@ data: }, "targets": [ { - "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", + "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))", "refId": "A", "legendFormat": "{{node}}" } diff --git a/services/monitoring/grafana-dashboard-pods.yaml b/services/monitoring/grafana-dashboard-pods.yaml index 51bb6c8b..9a73f02c 100644 --- a/services/monitoring/grafana-dashboard-pods.yaml +++ b/services/monitoring/grafana-dashboard-pods.yaml @@ -29,7 +29,7 @@ data: }, "targets": [ { - "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})) or on() vector(0)", + "expr": "((sum(max by(namespace,pod) ((kube_pod_status_phase{phase=\"Pending\",namespace!~\"veles\"} == 1) and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)) + (sum(max by(namespace,pod) ((kube_pod_status_phase{phase=~\"Failed|Unknown\",namespace!~\"veles\"} == 1) unless on(namespace,pod) kube_pod_owner{owner_kind=\"Job\"})) or on() vector(0)))", "refId": "A" } ], @@ -89,7 +89,7 @@ data: }, "targets": [ { - "expr": "sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})) or on() vector(0)", + "expr": "sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason{namespace!~\"veles\",reason=~\"CrashLoopBackOff|ImagePullBackOff\"} and on(namespace,pod) ((time() - kube_pod_created{namespace!~\"veles\"}) > 900))) or on() vector(0)", "refId": "A" } ], @@ -532,7 +532,7 @@ data: }, "targets": [ { - "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)))))", + "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.023)))))", "refId": "A", "instant": true, "format": "table" diff --git a/services/monitoring/grafana-dashboard-storage.yaml b/services/monitoring/grafana-dashboard-storage.yaml index 506cee4a..d89f46b0 100644 --- a/services/monitoring/grafana-dashboard-storage.yaml +++ b/services/monitoring/grafana-dashboard-storage.yaml @@ -285,7 +285,7 @@ data: }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } @@ -323,7 +323,7 @@ data: }, "targets": [ { - "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", + "expr": "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")", "refId": "A", "legendFormat": "{{node}}" } diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index 418db0e4..7242b6c1 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -370,7 +370,7 @@ spec: vault.hashicorp.com/agent-requests-mem: "64Mi" vault.hashicorp.com/agent-limits-cpu: "250m" vault.hashicorp.com/agent-limits-mem: "128Mi" - monitoring.bstein.dev/restart-rev: "12" + monitoring.bstein.dev/restart-rev: "13" vault.hashicorp.com/agent-inject-secret-grafana-env.sh: "kv/data/atlas/monitoring/grafana-admin" vault.hashicorp.com/agent-inject-template-grafana-env.sh: | {{ with secret "kv/data/atlas/monitoring/grafana-admin" }} @@ -488,6 +488,8 @@ spec: hide_version: true users: default_theme: dark + date_formats: + default_timezone: America/Chicago ingress: enabled: true ingressClassName: traefik @@ -627,6 +629,15 @@ spec: updateIntervalSeconds: 10 options: path: /var/lib/grafana/dashboards/power + - name: cassandra + orgId: 1 + folder: Atlas Internal + type: file + disableDeletion: false + editable: true + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards/cassandra dashboardsConfigMaps: overview: grafana-dashboard-overview overview-public: grafana-dashboard-overview @@ -639,6 +650,7 @@ spec: testing: grafana-dashboard-testing testing-public: grafana-dashboard-testing power: grafana-dashboard-power + cassandra: grafana-dashboard-cassandra extraConfigmapMounts: - name: grafana-folders mountPath: /etc/grafana/provisioning/folders diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index 8e2f9cd2..1c8f537f 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -14,10 +14,15 @@ resources: - grafana-dashboard-network.yaml - grafana-dashboard-gpu.yaml - grafana-dashboard-gitops.yaml + - grafana-dashboard-cassandra.yaml - grafana-dashboard-power.yaml - grafana-dashboard-mail.yaml - grafana-dashboard-testing.yaml - vmalert-atlas-availability.yaml + - availability-backfill-v4-job.yaml + - availability-daily-backfill-v4-job.yaml + - availability-rollup-cronjob.yaml + - availability-legacy-cleanup-job.yaml - dcgm-exporter.yaml - nvidia-process-exporter.yaml - jetson-tegrastats-exporter.yaml @@ -65,3 +70,15 @@ configMapGenerator: - platform_quality_suite_probe.sh=scripts/platform_quality_suite_probe.sh options: disableNameSuffixHash: true + - name: atlas-availability-rollup-script + namespace: monitoring + files: + - availability_rollup.py=scripts/availability_rollup.py + options: + disableNameSuffixHash: true + - name: atlas-availability-cleanup-script + namespace: monitoring + files: + - availability_cleanup.py=scripts/availability_cleanup.py + options: + disableNameSuffixHash: true diff --git a/services/monitoring/nvidia-process-exporter.yaml b/services/monitoring/nvidia-process-exporter.yaml index 408e4c42..325a0379 100644 --- a/services/monitoring/nvidia-process-exporter.yaml +++ b/services/monitoring/nvidia-process-exporter.yaml @@ -49,6 +49,7 @@ spec: annotations: prometheus.io/scrape: "true" prometheus.io/port: "9401" + monitoring.bstein.dev/restart-rev: "20260802-current-attribution" spec: serviceAccountName: nvidia-process-exporter imagePullSecrets: diff --git a/services/monitoring/scripts/availability_cleanup.py b/services/monitoring/scripts/availability_cleanup.py new file mode 100644 index 00000000..16cd9149 --- /dev/null +++ b/services/monitoring/scripts/availability_cleanup.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Remove obsolete Atlas annual-availability series after the request-v4 migration.""" + +from __future__ import annotations + +import json +import os +import time +from datetime import datetime, timedelta, timezone +from urllib.parse import urlencode +from urllib.request import Request, urlopen + + +VM_URL = os.environ.get( + "VM_URL", "http://victoria-metrics-single-server:8428" +).rstrip("/") +LEGACY_SELECTOR = ( + '{__name__="atlas:availability:ratio_365d",scope="atlas",' + 'definition!="request-v4"}' +) +PROTECTED_SELECTOR = ( + '{__name__="atlas:availability:ratio_365d",scope="atlas",' + 'definition="request-v4"}' +) + + +def list_series(selector: str) -> list[dict[str, str]]: + """List recently visible series matching an exact safety selector.""" + end = datetime.now(timezone.utc) + start = end - timedelta(hours=48) + query = urlencode( + { + "match[]": selector, + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + } + ) + with urlopen(f"{VM_URL}/api/v1/series?{query}", timeout=60) as response: + payload = json.load(response) + if payload.get("status") != "success": + raise RuntimeError(f"VictoriaMetrics series lookup failed: {payload}") + return payload["data"] + + +def delete_legacy_series() -> None: + """Delete only annual Atlas series outside the protected request-v4 label set.""" + query = urlencode({"match[]": LEGACY_SELECTOR}) + request = Request( + f"{VM_URL}/api/v1/admin/tsdb/delete_series?{query}", + data=b"", + method="POST", + ) + with urlopen(request, timeout=60) as response: + if response.status not in {200, 204}: + raise RuntimeError(f"VictoriaMetrics deletion returned HTTP {response.status}") + + +def main() -> None: + """Guard the new series, delete retired definitions, and verify convergence.""" + protected_before = list_series(PROTECTED_SELECTOR) + if len(protected_before) != 1: + raise RuntimeError( + f"expected exactly one protected request-v4 series, found {len(protected_before)}" + ) + legacy_before = list_series(LEGACY_SELECTOR) + if legacy_before: + delete_legacy_series() + + legacy_after = legacy_before + for _ in range(10): + legacy_after = list_series(LEGACY_SELECTOR) + if not legacy_after: + break + time.sleep(1) + if legacy_after: + raise RuntimeError(f"legacy availability series remain: {legacy_after}") + if len(list_series(PROTECTED_SELECTOR)) != 1: + raise RuntimeError("protected request-v4 series was not preserved") + + print( + json.dumps( + { + "deleted_legacy_series": len(legacy_before), + "protected_series": len(protected_before), + "selector": LEGACY_SELECTOR, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/services/monitoring/scripts/availability_rollup.py b/services/monitoring/scripts/availability_rollup.py new file mode 100644 index 00000000..c8a73892 --- /dev/null +++ b/services/monitoring/scripts/availability_rollup.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Publish Atlas request availability from deduplicated daily rollups.""" + +from __future__ import annotations + +import json +import os +import time +from datetime import datetime, timedelta, timezone +from typing import Iterable +from urllib.parse import urlencode +from urllib.request import Request, urlopen + + +VM_URL = os.environ.get( + "VM_URL", "http://victoria-metrics-single-server:8428" +).rstrip("/") +SCOPE = "atlas" +DEFINITION = "request-v4" +REQUESTS_METRIC = "atlas:availability:requests_1d" +FAILURES_METRIC = "atlas:availability:failures_1d" +OUTPUT_METRIC = "atlas:availability:ratio_365d" +WINDOW_DAYS = 365 + + +def parse_export(lines: Iterable[bytes]) -> dict[int, float]: + """Return the last exported value for each timestamp.""" + points: dict[int, float] = {} + for raw_line in lines: + if not raw_line.strip(): + continue + series = json.loads(raw_line) + for timestamp, value in zip(series["timestamps"], series["values"], strict=True): + points[int(timestamp)] = float(value) + return points + + +def fetch_rollup(metric: str, start: datetime, end: datetime) -> dict[int, float]: + """Stream one compact rollup series from VictoriaMetrics.""" + matcher = ( + f'{{__name__="{metric}",scope="{SCOPE}",definition="{DEFINITION}"}}' + ) + query = urlencode( + { + "match[]": matcher, + "start": start.isoformat().replace("+00:00", "Z"), + "end": end.isoformat().replace("+00:00", "Z"), + } + ) + with urlopen(f"{VM_URL}/api/v1/export?{query}", timeout=600) as response: + return parse_export(response) + + +def calculate_availability(requests: float, failures: float) -> float: + """Calculate the bounded successful-request ratio.""" + if requests <= 0: + raise ValueError("availability requires at least one observed request") + if failures < 0: + raise ValueError("failed request count cannot be negative") + return max(0.0, min(1.0, 1.0 - (failures / requests))) + + +def render_metric(value: float, timestamp_ms: int) -> str: + """Render one VictoriaMetrics Prometheus-import sample.""" + return ( + f'{OUTPUT_METRIC}{{definition="{DEFINITION}",scope="{SCOPE}",' + f'rollup="yearly"}} {value:.12f} {timestamp_ms}\n' + ) + + +def publish(value: float, timestamp_ms: int) -> None: + """Write the calculated annual ratio to VictoriaMetrics.""" + request = Request( + f"{VM_URL}/api/v1/import/prometheus", + data=render_metric(value, timestamp_ms).encode(), + headers={"Content-Type": "text/plain"}, + method="POST", + ) + with urlopen(request, timeout=30) as response: + if response.status not in {200, 204}: + raise RuntimeError(f"VictoriaMetrics import returned HTTP {response.status}") + + +def main() -> None: + """Rebuild and publish the rolling request-availability sample.""" + end = datetime.now(timezone.utc) + start = end - timedelta(days=WINDOW_DAYS) + requests = sum(fetch_rollup(REQUESTS_METRIC, start, end).values()) + failures = sum(fetch_rollup(FAILURES_METRIC, start, end).values()) + availability = calculate_availability(requests, failures) + timestamp_ms = time.time_ns() // 1_000_000 + publish(availability, timestamp_ms) + print( + json.dumps( + { + "requests": requests, + "failures": failures, + "availability_percent": availability * 100, + "timestamp_ms": timestamp_ms, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/services/monitoring/scripts/nvidia_process_exporter.py b/services/monitoring/scripts/nvidia_process_exporter.py index a43b2b45..136a199c 100644 --- a/services/monitoring/scripts/nvidia_process_exporter.py +++ b/services/monitoring/scripts/nvidia_process_exporter.py @@ -134,7 +134,8 @@ def running_process_memory(handle): def process_utilization_samples(handle): try: - since = int(time.time() * 1000) - SAMPLE_WINDOW_MS + # NVML process sample timestamps are microseconds since the epoch. + since = int(time.time() * 1_000_000) - (SAMPLE_WINDOW_MS * 1000) samples = nvmlDeviceGetProcessUtilization(handle, since) except NVMLError_NotFound: return {}, 1 @@ -156,6 +157,32 @@ def process_utilization_samples(handle): return by_pid, 1 +def reconcile_namespace_utilization(namespace_sm, device_util): + """Make namespace attribution add up to the device's current utilization.""" + + current = max(float(device_util), 0.0) + reconciled = { + namespace: max(float(value), 0.0) + for namespace, value in namespace_sm.items() + } + attributed = sum(reconciled.values()) + + if current == 0: + return {namespace: 0.0 for namespace in reconciled} + + # Per-process NVML samples and the device gauge are collected on slightly + # different intervals. Scale a stale/overlapping process sum down to the + # device's current total while preserving the namespace proportions. + if attributed > current and attributed > 0: + scale = current / attributed + return {namespace: value * scale for namespace, value in reconciled.items()} + + residual = current - attributed + if residual > 0.1: + reconciled["host"] = reconciled.get("host", 0.0) + residual + return reconciled + + def collect_metrics(): nvmlInit() pods = load_pods() @@ -204,10 +231,7 @@ def collect_metrics(): lines.append(metric_line("nvidia_process_gpu_sm_util_percent", labels, sm_util)) lines.append(metric_line("nvidia_process_gpu_memory_used_bytes", labels, int(proc_info["memory"]))) - attributed = sum(namespace_sm.values()) - residual = max(device_util - attributed, 0.0) - if residual > 0.1: - namespace_sm["host"] = namespace_sm.get("host", 0.0) + residual + namespace_sm = reconcile_namespace_utilization(namespace_sm, device_util) for namespace, value in sorted(namespace_sm.items()): labels = {**base, "namespace": namespace, "pod": "__namespace_total__"} diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index 1bfcc1e4..c2e1f78e 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -7,40 +7,66 @@ metadata: data: atlas-availability.yaml: | groups: - - name: atlas.availability - interval: 15m - eval_offset: 14m + - name: atlas.availability.gateway + interval: 1h + eval_offset: 59m rules: - - record: atlas:availability:ratio_1h + - record: atlas:availability:requests_1h expr: | - avg_over_time(( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - >= bool 2 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - > bool 0 - ) - ) - )[1h:5m]) + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"[1-5].." + }[1h] + )) labels: - definition: serving-v2 + definition: request-v4 scope: atlas rollup: hourly - - record: atlas:availability:ratio_365d + - record: atlas:availability:failures_1h expr: | - share_gt_over_time(( - max by (scope, rollup) ( - atlas:availability:ratio_1h{scope="atlas",definition="serving-v2"} - or atlas:availability:ratio_1h{scope="atlas",definition=""} - ) - )[365d:15m], 0) + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"5.." + }[1h] + )) labels: - definition: serving-v2 + definition: request-v4 scope: atlas - rollup: yearly + rollup: hourly + - name: atlas.availability.rollup + interval: 1d + eval_offset: 23h59m + rules: + - record: atlas:availability:requests_1d + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"[1-5].." + }[1d] + )) + labels: + definition: request-v4 + scope: atlas + rollup: daily + - record: atlas:availability:failures_1d + expr: | + sum(increase( + traefik_entrypoint_requests_total{ + entrypoint="websecure", + protocol="http", + code=~"5.." + }[1d] + )) + labels: + definition: request-v4 + scope: atlas + rollup: daily platform-quality.yaml: | groups: - name: platform.quality diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml new file mode 100644 index 00000000..2d794291 --- /dev/null +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -0,0 +1,54 @@ +# services/vault/hermes-auth-role-bootstrap-job.yaml +# Purpose: apply the Vault read/write boundaries needed by Hermes operator OIDC. +apiVersion: batch/v1 +kind: Job +metadata: + name: vault-k8s-auth-hermes-1 + namespace: vault +spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: vault-admin + restartPolicy: Never + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + containers: + - name: configure-k8s-auth + image: hashicorp/vault:1.21.4 + imagePullPolicy: IfNotPresent + command: + - sh + - /scripts/vault_k8s_auth_configure.sh + env: + - name: VAULT_ADDR + value: http://10.43.57.249:8200 + - name: VAULT_K8S_ROLE + value: vault-admin + - name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE + value: /var/run/secrets/vault-token-reviewer/token + - name: VAULT_K8S_ROLE_TTL + value: 1h + volumeMounts: + - name: k8s-auth-config-script + mountPath: /scripts + readOnly: true + - name: token-reviewer + mountPath: /var/run/secrets/vault-token-reviewer + readOnly: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + volumes: + - name: k8s-auth-config-script + configMap: + name: vault-k8s-auth-config-script + defaultMode: 0555 + - name: token-reviewer + secret: + secretName: vault-admin-token-reviewer diff --git a/services/vault/kustomization.yaml b/services/vault/kustomization.yaml index 98b47e44..d370373c 100644 --- a/services/vault/kustomization.yaml +++ b/services/vault/kustomization.yaml @@ -11,6 +11,7 @@ resources: - configmap.yaml - statefulset.yaml - k8s-auth-config-cronjob.yaml + - hermes-auth-role-bootstrap-job.yaml - oidc-config-cronjob.yaml - service.yaml - certificate.yaml diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index d0876b6e..cc5903b3 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -253,6 +253,8 @@ write_policy_and_role "health" "health" "health-vault-sync" \ "health/*" "" write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ "game-stream/*" "" +write_policy_and_role "hermes" "hermes" "hermes-vault" \ + "hermes/operator-oidc" "" write_policy_and_role "veles" "veles" "veles-backend,veles-generator,veles-postgres,veles-vault-sync" \ "veles/* shared/harbor-pull shared/postmark-relay" "" write_policy_and_role "veles-sim" "veles" "veles-sim" \ @@ -296,7 +298,7 @@ write_policy_and_role "vault" "vault" "vault" \ write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \ "shared/keycloak-admin shared/postmark-relay maintenance/metis-ssh-keys" \ - "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc" \ + "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc hermes/operator-oidc" \ ' path "kv/data/atlas/nodes/*" { capabilities = ["create", "update", "read"] diff --git a/services/vaultwarden/deployment.yaml b/services/vaultwarden/deployment.yaml index 4cf8f332..5d1c5446 100644 --- a/services/vaultwarden/deployment.yaml +++ b/services/vaultwarden/deployment.yaml @@ -61,7 +61,7 @@ spec: - titan-11 containers: - name: vaultwarden - image: vaultwarden/server:1.35.2 + image: vaultwarden/server:1.37.0@sha256:e6443e3d5ed8fcee2204b89ec778d7f24d0173bcc42d1ea34f990304f5f63f51 command: ["/bin/sh", "-c"] args: - >- diff --git a/services/veles/ingress.yaml b/services/veles/ingress.yaml index 8a2e569c..2cfa2ab6 100644 --- a/services/veles/ingress.yaml +++ b/services/veles/ingress.yaml @@ -7,7 +7,7 @@ metadata: annotations: cert-manager.io/cluster-issuer: letsencrypt traefik.ingress.kubernetes.io/router.entrypoints: websecure - traefik.ingress.kubernetes.io/router.middlewares: veles-redirect-to-cassandra@kubernetescrd + traefik.ingress.kubernetes.io/router.middlewares: veles-veles-to-cassandra-redirect@kubernetescrd traefik.ingress.kubernetes.io/router.tls: "true" spec: ingressClassName: traefik diff --git a/services/veles/redirect-middleware.yaml b/services/veles/redirect-middleware.yaml index 4e60e9cc..454ff0e4 100644 --- a/services/veles/redirect-middleware.yaml +++ b/services/veles/redirect-middleware.yaml @@ -2,10 +2,10 @@ apiVersion: traefik.io/v1alpha1 kind: Middleware metadata: - name: redirect-to-cassandra + name: veles-to-cassandra-redirect namespace: veles spec: redirectRegex: - regex: ^https?://veles\.bstein\.dev/?(.*) - replacement: https://cassandra.bstein.dev/${1} + regex: ^https?://veles\.bstein\.dev(.*) + replacement: https://cassandra.bstein.dev${1} permanent: true