feat(ariadne): declare the fix categories, and show them beside the actions

The three categories now appear in the deployment next to the action
allowlist, and the monitor prints both at the policy gate so the difference is
visible during a demo rather than asserted: two ids Ariadne may execute on its
own authority, three categories it may only ask Hermes to propose a patch for.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.gitignore

@@ -5,6 +5,7 @@
 __pycache__/
 *.py[cod]
 .pytest_cache
+.ruff_cache/
 .coverage
 build/
 test-results/
@@ -12,6 +13,7 @@ artifacts/
 .venv
 .venv-ci
 tmp/
+.mainfix/
 .terraform/
 **/.terraform/
 *.tfvars
@@ -19,3 +21,6 @@ tmp/
 *.tfstate.*
 crash.log
 terraform/atlas/generated/
+
+# Local demo credentials (never commit)
+scripts/ops/hermes_triage_demo.env

Jenkinsfile

@@ -56,6 +56,11 @@ spec:
       command:
         - cat
       tty: true
+    - name: semgrep
+      image: semgrep/semgrep:1.171.0
+      command:
+        - cat
+      tty: true
 """
     }
   }
@@ -70,6 +75,8 @@ spec:
     VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
     QUALITY_GATE_SONARQUBE_ENFORCE = '0'
     QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
+    QUALITY_GATE_SEMGREP_ENFORCE = '0'
+    QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
     QUALITY_GATE_IRONBANK_ENFORCE = '1'
     QUALITY_GATE_IRONBANK_REQUIRED = '0'
     QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
@@ -110,6 +117,26 @@ spec:
         '''
       }
     }
+    stage('Collect Semgrep evidence') {
+      steps {
+        container('semgrep') {
+          sh '''#!/bin/sh
+            set -eu
+            mkdir -p build
+            set +e
+            semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
+            semgrep_rc=$?
+            set -e
+            printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
+          '''
+        }
+        sh '''
+          set -eu
+          semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
+          python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
+        '''
+      }
+    }
     stage('Collect SonarQube evidence') {
       steps {
         container('quality-tools') {
@@ -126,6 +153,7 @@ spec:
               "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
             )
             [ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
+            [ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
             set +e
             sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
             rc=${PIPESTATUS[0]}
@@ -326,6 +354,33 @@ PY
             esac
           fi
 
+          if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
+            semgrep_status="$(python3 - <<'PY'
+import json
+from pathlib import Path
+
+path = Path("build/semgrep-report.json")
+if not path.exists():
+    print("missing")
+    raise SystemExit(0)
+try:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+except Exception:  # noqa: BLE001
+    print("error")
+    raise SystemExit(0)
+status = str(payload.get("status") or "").strip().lower()
+print(status or "missing")
+PY
+)"
+            case "${semgrep_status}" in
+              ok|pass|passed|success) ;;
+              *)
+                echo "semgrep gate failed: ${semgrep_status}" >&2
+                fail=1
+                ;;
+            esac
+          fi
+
           ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
           if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
             ironbank_required=1

Makefile

@@ -0,0 +1,105 @@
+# Makefile
+SHELL := /usr/bin/env bash
+
+KUSTOMIZE ?= kustomize
+PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3)
+TERRAFORM ?= terraform
+TRIVY ?= trivy
+BUILD_DIR ?= build
+ATLAS_FLUX_ROOT := clusters/atlas/flux-system
+AETHER_FLUX_ROOT := clusters/aether/flux-system
+AETHER_TF_DIR := terraform/aether
+FLUX_KUSTOMIZATION ?= flux-system
+FLUX_NAMESPACE ?= flux-system
+FLUX_PATH ?= $(ATLAS_FLUX_ROOT)
+TRIVY_CACHE_DIR ?= $(BUILD_DIR)/trivy-cache
+TRIVY_DB_FLAGS ?= --skip-db-update
+
+.PHONY: help render render-atlas render-aether render-services render-infrastructure validate dashboards knowledge test flux-diff flux-inventory trivy-scan security-report security aether-fmt aether-init aether-validate aether-plan
+
+help:
+	@printf '%s\n' \
+		'Targets:' \
+		'  render                Render Atlas, Aether, services, and infrastructure.' \
+		'  render-atlas          Render the Atlas Flux root.' \
+		'  render-aether         Render the Aether Flux root.' \
+		'  render-services       Render every service kustomization.' \
+		'  render-infrastructure Render infrastructure kustomizations.' \
+		'  validate              Run render checks plus the local quality gate.' \
+		'  dashboards            Regenerate Grafana dashboard artifacts.' \
+		'  knowledge             Regenerate Atlas knowledge artifacts and comms mirror.' \
+		'  test                  Run the local quality gate.' \
+		'  flux-diff             Diff a Flux Kustomization; override FLUX_KUSTOMIZATION/FLUX_PATH.' \
+		'  flux-inventory        Print Flux Kustomization inventory.' \
+		'  trivy-scan            Run the local Trivy filesystem scan.' \
+		'  security-report       Build IronBank report from build/trivy-fs.json.' \
+		'  security              Run Trivy scan and build the IronBank report.' \
+		'  aether-fmt            Check Terraform formatting for Aether.' \
+		'  aether-init           Initialize Aether Terraform providers.' \
+		'  aether-validate       Validate Aether Terraform after init.' \
+		'  aether-plan           Plan Aether Terraform from local variables/env.'
+
+render: render-atlas render-aether render-services render-infrastructure
+
+render-atlas:
+	$(KUSTOMIZE) build $(ATLAS_FLUX_ROOT) >/tmp/titan-iac-atlas-flux-system.yaml
+
+render-aether:
+	$(KUSTOMIZE) build $(AETHER_FLUX_ROOT) >/tmp/titan-iac-aether-flux-system.yaml
+
+render-services:
+	@set -euo pipefail; \
+	while IFS= read -r k; do \
+		d="$${k%/kustomization.yaml}"; \
+		printf 'render %s\n' "$$d"; \
+		$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
+	done < <(find services -name kustomization.yaml | sort)
+
+render-infrastructure:
+	@set -euo pipefail; \
+	while IFS= read -r k; do \
+		d="$${k%/kustomization.yaml}"; \
+		printf 'render %s\n' "$$d"; \
+		$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
+	done < <(find infrastructure -name kustomization.yaml | sort)
+
+validate: render test
+
+dashboards:
+	$(PYTHON) scripts/render/dashboards_render_atlas.py --build
+	$(PYTHON) scripts/render/dashboards_render_logs.py --build
+	$(PYTHON) scripts/render/logging_render_observability.py --build
+
+knowledge:
+	$(PYTHON) scripts/render/knowledge_render_atlas.py --write --sync-comms
+
+test:
+	$(PYTHON) -m testing.quality_gate --profile local --build-dir $(BUILD_DIR)
+
+flux-diff:
+	flux diff kustomization $(FLUX_KUSTOMIZATION) --namespace $(FLUX_NAMESPACE) --path $(FLUX_PATH)
+
+flux-inventory:
+	$(PYTHON) scripts/render/flux_inventory.py $(ATLAS_FLUX_ROOT)
+
+trivy-scan:
+	mkdir -p $(BUILD_DIR)
+	$(TRIVY) fs --cache-dir "$(TRIVY_CACHE_DIR)" $(TRIVY_DB_FLAGS) --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output $(BUILD_DIR)/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
+
+security-report:
+	@test -s $(BUILD_DIR)/trivy-fs.json || { printf '%s\n' 'missing build/trivy-fs.json; run make trivy-scan first'; exit 1; }
+	$(PYTHON) ci/scripts/supply_chain_report.py --trivy-json $(BUILD_DIR)/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output $(BUILD_DIR)/ironbank-compliance.json
+
+security: trivy-scan security-report
+
+aether-fmt:
+	$(TERRAFORM) -chdir=$(AETHER_TF_DIR) fmt -check
+
+aether-init:
+	$(TERRAFORM) -chdir=$(AETHER_TF_DIR) init
+
+aether-validate:
+	$(TERRAFORM) -chdir=$(AETHER_TF_DIR) validate
+
+aether-plan:
+	$(TERRAFORM) -chdir=$(AETHER_TF_DIR) plan

ci/Jenkinsfile.titan-iac

@@ -55,6 +55,11 @@ spec:
       command:
         - cat
       tty: true
+    - name: semgrep
+      image: semgrep/semgrep:1.171.0
+      command:
+        - cat
+      tty: true
 """
     }
   }
@@ -69,6 +74,8 @@ spec:
     VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
     QUALITY_GATE_SONARQUBE_ENFORCE = '0'
     QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
+    QUALITY_GATE_SEMGREP_ENFORCE = '0'
+    QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
     QUALITY_GATE_IRONBANK_ENFORCE = '1'
     QUALITY_GATE_IRONBANK_REQUIRED = '0'
     QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
@@ -109,6 +116,26 @@ spec:
         '''
       }
     }
+    stage('Collect Semgrep evidence') {
+      steps {
+        container('semgrep') {
+          sh '''#!/bin/sh
+            set -eu
+            mkdir -p build
+            set +e
+            semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
+            semgrep_rc=$?
+            set -e
+            printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
+          '''
+        }
+        sh '''
+          set -eu
+          semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
+          python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
+        '''
+      }
+    }
     stage('Collect SonarQube evidence') {
       steps {
         container('quality-tools') {
@@ -125,6 +152,7 @@ spec:
               "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
             )
             [ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
+            [ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
             set +e
             sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
             rc=${PIPESTATUS[0]}
@@ -325,6 +353,33 @@ PY
             esac
           fi
 
+          if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
+            semgrep_status="$(python3 - <<'PY'
+import json
+from pathlib import Path
+
+path = Path("build/semgrep-report.json")
+if not path.exists():
+    print("missing")
+    raise SystemExit(0)
+try:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+except Exception:  # noqa: BLE001
+    print("error")
+    raise SystemExit(0)
+status = str(payload.get("status") or "").strip().lower()
+print(status or "missing")
+PY
+)"
+            case "${semgrep_status}" in
+              ok|pass|passed|success) ;;
+              *)
+                echo "semgrep gate failed: ${semgrep_status}" >&2
+                fail=1
+                ;;
+            esac
+          fi
+
           ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
           if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
             ironbank_required=1

ci/scripts/publish_test_metrics.py

@@ -20,6 +20,7 @@ CANONICAL_CHECKS = _quality_helpers.CANONICAL_CHECKS
 _build_check_statuses = _quality_helpers._build_check_statuses
 _combine_statuses = _quality_helpers._combine_statuses
 _infer_sonarqube_status = _quality_helpers._infer_sonarqube_status
+_infer_semgrep_status = _quality_helpers._infer_semgrep_status
 _infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500
 _infer_supply_chain_status = _quality_helpers._infer_supply_chain_status
 _infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent
@@ -286,6 +287,7 @@ def main() -> int:
     if source_lines_over_500 <= 0:
         source_lines_over_500 = _infer_source_lines_over_500(summary)
     sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json"))
+    semgrep_report = _load_optional_json(os.getenv("QUALITY_GATE_SEMGREP_REPORT", "build/semgrep-report.json"))
     supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json"))
     truthy = {"1", "true", "yes", "on"}
     supply_chain_required = (
@@ -298,6 +300,7 @@ def main() -> int:
         workspace_line_coverage_percent=workspace_line_coverage_percent,
         source_lines_over_500=source_lines_over_500,
         sonarqube_report=sonarqube_report,
+        semgrep_report=semgrep_report,
         supply_chain_report=supply_chain_report,
         supply_chain_required=supply_chain_required,
     )

ci/scripts/publish_test_metrics_quality.py

@@ -18,6 +18,7 @@ CANONICAL_CHECKS = [
     "docs_naming",
     "gate_glue",
     "sonarqube",
+    "semgrep",
     "supply_chain",
 ]
 
@@ -135,12 +136,26 @@ def _infer_supply_chain_status(report: dict, required: bool) -> str:
     return normalized
 
 
+def _infer_semgrep_status(report: dict) -> str:
+    """Infer canonical Semgrep check status from its JSON report payload."""
+    if not report:
+        return "not_applicable"
+    status = report.get("status")
+    if status is None:
+        blocking_findings = report.get("blocking_findings")
+        errors_total = report.get("errors_total")
+        if isinstance(blocking_findings, int) and isinstance(errors_total, int):
+            return "failed" if blocking_findings > 0 or errors_total > 0 else "ok"
+    return _normalize_result_status(str(status) if status is not None else None, default="failed")
+
+
 def _build_check_statuses(
     summary: dict | None,
     tests: dict[str, int],
     workspace_line_coverage_percent: float,
     source_lines_over_500: int,
     sonarqube_report: dict,
+    semgrep_report: dict,
     supply_chain_report: dict,
     supply_chain_required: bool,
 ) -> dict[str, str]:
@@ -188,6 +203,7 @@ def _build_check_statuses(
         gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable"
 
     sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report)
+    semgrep_status = status_by_name.get("semgrep") or _infer_semgrep_status(semgrep_report)
     supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status(
         supply_chain_report,
         required=supply_chain_required,
@@ -200,5 +216,6 @@ def _build_check_statuses(
         "docs_naming": docs_naming_status,
         "gate_glue": gate_glue_status,
         "sonarqube": sonarqube_status,
+        "semgrep": semgrep_status,
         "supply_chain": supply_chain_status,
     }

ci/scripts/semgrep_report.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""Build a compact Semgrep gate report from Semgrep JSON output."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+BLOCKING_SEVERITIES = {"ERROR"}
+SEVERITIES = ("ERROR", "WARNING", "INFO", "UNKNOWN")
+SONAR_SEVERITY_BY_SEMGREP = {
+    "ERROR": "CRITICAL",
+    "WARNING": "MAJOR",
+    "INFO": "MINOR",
+    "UNKNOWN": "INFO",
+}
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    """Read a JSON object from disk, returning an error-shaped payload on failure."""
+    if not path.exists():
+        return {"errors": [{"message": f"report missing: {path}"}], "results": []}
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []}
+    if not isinstance(payload, dict):
+        return {"errors": [{"message": "report payload is not an object"}], "results": []}
+    return payload
+
+
+def _finding_severity(finding: dict[str, Any]) -> str:
+    """Return a normalized Semgrep finding severity."""
+    extra = finding.get("extra")
+    severity = extra.get("severity") if isinstance(extra, dict) else None
+    normalized = str(severity or "UNKNOWN").strip().upper()
+    return normalized if normalized in SEVERITIES else "UNKNOWN"
+
+
+def build_report(
+    semgrep_payload: dict[str, Any],
+    *,
+    semgrep_exit_code: int,
+    blocking_severities: set[str] | None = None,
+) -> dict[str, Any]:
+    """Summarize Semgrep evidence into the quality-gate report contract."""
+    blocking = blocking_severities or BLOCKING_SEVERITIES
+    raw_results = semgrep_payload.get("results", [])
+    raw_errors = semgrep_payload.get("errors", [])
+    results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
+    errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
+
+    severity_counts = {severity: 0 for severity in SEVERITIES}
+    blocking_findings = 0
+    for finding in results:
+        severity = _finding_severity(finding)
+        severity_counts[severity] += 1
+        if severity in blocking:
+            blocking_findings += 1
+
+    engine_error = semgrep_exit_code not in {0, 1} or bool(errors)
+    status = "failed" if engine_error or blocking_findings else "ok"
+    return {
+        "status": status,
+        "scanner": "semgrep",
+        "semgrep_rc": semgrep_exit_code,
+        "findings_total": len(results),
+        "blocking_findings": blocking_findings,
+        "errors_total": len(errors),
+        "severity_counts": severity_counts,
+        "blocking_severities": sorted(blocking),
+    }
+
+
+def _line_number(value: Any, default: int = 1) -> int:
+    """Return a Sonar-compatible one-indexed line number."""
+    try:
+        line = int(value)
+    except (TypeError, ValueError):
+        return default
+    return max(line, default)
+
+
+def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
+    """Convert one Semgrep finding into SonarQube generic issue format."""
+    path = str(finding.get("path") or "").strip()
+    if not path:
+        return None
+    extra = finding.get("extra") if isinstance(finding.get("extra"), dict) else {}
+    severity = _finding_severity(finding)
+    start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
+    end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
+    start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
+    end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
+    if end_line < start_line:
+        end_line = start_line
+    return {
+        "engineId": "semgrep",
+        "ruleId": str(finding.get("check_id") or "semgrep.unknown"),
+        "type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL",
+        "severity": SONAR_SEVERITY_BY_SEMGREP[severity],
+        "primaryLocation": {
+            "message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"),
+            "filePath": path,
+            "textRange": {
+                "startLine": start_line,
+                "endLine": end_line,
+            },
+        },
+    }
+
+
+def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
+    """Build SonarQube generic external issues from Semgrep JSON output."""
+    raw_results = semgrep_payload.get("results", [])
+    results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
+    issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))]
+    return {"issues": issues}
+
+
+def main(argv: list[str] | None = None) -> int:
+    """CLI entrypoint used by Jenkins after Semgrep finishes."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--semgrep-json", required=True)
+    parser.add_argument("--exit-code", type=int, default=0)
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--sonar-issues-output")
+    parser.add_argument(
+        "--blocking-severity",
+        action="append",
+        default=[],
+        help="Severity that should mark the report failed. Defaults to ERROR.",
+    )
+    args = parser.parse_args(argv)
+
+    blocking = {item.strip().upper() for item in args.blocking_severity if item.strip()}
+    payload = _read_json(Path(args.semgrep_json))
+    report = build_report(
+        payload,
+        semgrep_exit_code=args.exit_code,
+        blocking_severities=blocking or BLOCKING_SEVERITIES,
+    )
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    if args.sonar_issues_output:
+        sonar_output = Path(args.sonar_issues_output)
+        sonar_output.parent.mkdir(parents=True, exist_ok=True)
+        sonar_output.write_text(
+            json.dumps(build_sonar_issues(payload), indent=2, sort_keys=True) + "\n",
+            encoding="utf-8",
+        )
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

ci/scripts/supply_chain_report.py

@@ -131,7 +131,11 @@ def build_report(
     critical = _count_vulnerabilities(trivy_payload, "CRITICAL")
     high = _count_vulnerabilities(trivy_payload, "HIGH")
     secrets = _count_secrets(trivy_payload)
-    status = "ok" if critical == 0 and secrets == 0 and not open_misconfigs else "failed"
+    status = (
+        "ok"
+        if critical == 0 and secrets == 0 and not open_misconfigs and expired_waivers == 0
+        else "failed"
+    )
 
     return {
         "status": status,

ci/titan-iac-trivy-waivers.json

@@ -1,7 +1,7 @@
 {
   "version": 1,
   "generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan",
-  "default_expires_at": "2026-05-22",
+  "default_expires_at": "2026-09-30",
   "ticket": "atlas-quality-wave-k8s-hardening",
   "default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.",
   "misconfigurations": [
@@ -15,15 +15,15 @@
       "id": "KSV-0009",
       "targets": [
         "services/mailu/vip-controller.yaml",
-        "services/maintenance/k3s-agent-restart-daemonset.yaml"
+        "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml"
       ]
     },
     {
       "id": "KSV-0010",
       "targets": [
-        "services/maintenance/k3s-agent-restart-daemonset.yaml",
-        "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
-        "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
+        "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
+        "services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
+        "services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
         "services/monitoring/jetson-tegrastats-exporter.yaml"
       ]
     },
@@ -47,8 +47,8 @@
         "services/bstein-dev-home/backend-deployment.yaml",
         "services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
         "services/bstein-dev-home/frontend-deployment.yaml",
-        "services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml",
-        "services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml",
+        "services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
+        "services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
         "services/bstein-dev-home/vault-sync-deployment.yaml",
         "services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
         "services/comms/atlasbot-deployment.yaml",
@@ -59,16 +59,16 @@
         "services/comms/livekit-token-deployment.yaml",
         "services/comms/livekit.yaml",
         "services/comms/mas-deployment.yaml",
-        "services/comms/oneoffs/bstein-force-leave-job.yaml",
-        "services/comms/oneoffs/comms-secrets-ensure-job.yaml",
-        "services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml",
-        "services/comms/oneoffs/mas-db-ensure-job.yaml",
-        "services/comms/oneoffs/mas-local-users-ensure-job.yaml",
-        "services/comms/oneoffs/othrys-kick-numeric-job.yaml",
-        "services/comms/oneoffs/synapse-admin-ensure-job.yaml",
-        "services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml",
-        "services/comms/oneoffs/synapse-signingkey-ensure-job.yaml",
-        "services/comms/oneoffs/synapse-user-seed-job.yaml",
+        "services/comms/repair-jobs/bstein-force-leave-job.yaml",
+        "services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
+        "services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
+        "services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
         "services/comms/pin-othrys-job.yaml",
         "services/comms/reset-othrys-room-job.yaml",
         "services/comms/seed-othrys-room.yaml",
@@ -83,7 +83,7 @@
         "services/finance/firefly-cronjob.yaml",
         "services/finance/firefly-deployment.yaml",
         "services/finance/firefly-user-sync-cronjob.yaml",
-        "services/finance/oneoffs/finance-secrets-ensure-job.yaml",
+        "services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
         "services/gitea/deployment.yaml",
         "services/harbor/vault-sync-deployment.yaml",
         "services/health/wger-admin-ensure-cronjob.yaml",
@@ -94,63 +94,62 @@
         "services/jenkins/deployment.yaml",
         "services/jenkins/vault-sync-deployment.yaml",
         "services/keycloak/deployment.yaml",
-        "services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/ldap-federation-job.yaml",
-        "services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/mas-secrets-ensure-job.yaml",
-        "services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-client-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-target-client-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml",
-        "services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/realm-settings-job.yaml",
-        "services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/user-overrides-job.yaml",
-        "services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
+        "services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
+        "services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
+        "services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
+        "services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
         "services/keycloak/vault-sync-deployment.yaml",
         "services/logging/node-image-gc-rpi4-daemonset.yaml",
         "services/logging/node-image-prune-rpi5-daemonset.yaml",
         "services/logging/node-log-rotation-daemonset.yaml",
         "services/logging/oauth2-proxy.yaml",
-        "services/logging/oneoffs/opensearch-dashboards-setup-job.yaml",
-        "services/logging/oneoffs/opensearch-ism-job.yaml",
-        "services/logging/oneoffs/opensearch-observability-setup-job.yaml",
+        "services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
+        "services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
+        "services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
         "services/logging/opensearch-prune-cronjob.yaml",
         "services/logging/vault-sync-deployment.yaml",
         "services/mailu/mailu-sync-cronjob.yaml",
         "services/mailu/mailu-sync-listener.yaml",
-        "services/mailu/oneoffs/mailu-sync-job.yaml",
         "services/mailu/vault-sync-deployment.yaml",
         "services/mailu/vip-controller.yaml",
-        "services/maintenance/ariadne-deployment.yaml",
-        "services/maintenance/disable-k3s-traefik-daemonset.yaml",
-        "services/maintenance/image-sweeper-cronjob.yaml",
-        "services/maintenance/k3s-agent-restart-daemonset.yaml",
-        "services/maintenance/metis-deployment.yaml",
-        "services/maintenance/metis-k3s-token-sync-cronjob.yaml",
-        "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
-        "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
-        "services/maintenance/node-image-sweeper-daemonset.yaml",
-        "services/maintenance/node-nofile-daemonset.yaml",
-        "services/maintenance/oauth2-proxy-metis.yaml",
-        "services/maintenance/oauth2-proxy-soteria.yaml",
-        "services/maintenance/oneoffs/ariadne-migrate-job.yaml",
-        "services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml",
-        "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
-        "services/maintenance/pod-cleaner-cronjob.yaml",
-        "services/maintenance/soteria-deployment.yaml",
-        "services/maintenance/vault-sync-deployment.yaml",
+        "services/maintenance/apps/ariadne-deployment.yaml",
+        "services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
+        "services/maintenance/node-ops/image-sweeper-cronjob.yaml",
+        "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
+        "services/maintenance/apps/metis-deployment.yaml",
+        "services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
+        "services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
+        "services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
+        "services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
+        "services/maintenance/node-ops/node-nofile-daemonset.yaml",
+        "services/maintenance/networking/oauth2-proxy-metis.yaml",
+        "services/maintenance/networking/oauth2-proxy-soteria.yaml",
+        "services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
+        "services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
+        "services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
+        "services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
+        "services/maintenance/apps/soteria-deployment.yaml",
+        "services/maintenance/bootstrap/vault-sync-deployment.yaml",
         "services/monitoring/dcgm-exporter.yaml",
         "services/monitoring/jetson-tegrastats-exporter.yaml",
-        "services/monitoring/oneoffs/grafana-org-bootstrap.yaml",
-        "services/monitoring/oneoffs/grafana-user-dedupe-job.yaml",
+        "services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
+        "services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
         "services/monitoring/platform-quality-gateway-deployment.yaml",
         "services/monitoring/platform-quality-suite-probe-cronjob.yaml",
         "services/monitoring/postmark-exporter-deployment.yaml",
@@ -188,15 +187,15 @@
         "services/logging/node-image-gc-rpi4-daemonset.yaml",
         "services/logging/node-image-prune-rpi5-daemonset.yaml",
         "services/logging/node-log-rotation-daemonset.yaml",
-        "services/maintenance/disable-k3s-traefik-daemonset.yaml",
-        "services/maintenance/image-sweeper-cronjob.yaml",
-        "services/maintenance/k3s-agent-restart-daemonset.yaml",
-        "services/maintenance/metis-deployment.yaml",
-        "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
-        "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
-        "services/maintenance/node-image-sweeper-daemonset.yaml",
-        "services/maintenance/node-nofile-daemonset.yaml",
-        "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
+        "services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
+        "services/maintenance/node-ops/image-sweeper-cronjob.yaml",
+        "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
+        "services/maintenance/apps/metis-deployment.yaml",
+        "services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
+        "services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
+        "services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
+        "services/maintenance/node-ops/node-nofile-daemonset.yaml",
+        "services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
         "services/monitoring/dcgm-exporter.yaml",
         "services/monitoring/jetson-tegrastats-exporter.yaml"
       ]
@@ -211,7 +210,7 @@
         "services/comms/comms-secrets-ensure-rbac.yaml",
         "services/comms/mas-db-ensure-rbac.yaml",
         "services/comms/mas-secrets-ensure-rbac.yaml",
-        "services/maintenance/soteria-rbac.yaml"
+        "services/maintenance/apps/soteria-rbac.yaml"
       ]
     },
     {
@@ -226,7 +225,7 @@
         "services/comms/comms-secrets-ensure-rbac.yaml",
         "services/comms/mas-db-ensure-rbac.yaml",
         "services/jenkins/serviceaccount.yaml",
-        "services/maintenance/ariadne-rbac.yaml"
+        "services/maintenance/apps/ariadne-rbac.yaml"
       ]
     },
     {
@@ -235,8 +234,8 @@
         "infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
         "infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
         "services/jenkins/serviceaccount.yaml",
-        "services/maintenance/disable-k3s-traefik-rbac.yaml",
-        "services/maintenance/k3s-traefik-cleanup-rbac.yaml"
+        "services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml",
+        "services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml"
       ]
     },
     {
@@ -266,8 +265,8 @@
         "services/bstein-dev-home/backend-deployment.yaml",
         "services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
         "services/bstein-dev-home/frontend-deployment.yaml",
-        "services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml",
-        "services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml",
+        "services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
+        "services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
         "services/bstein-dev-home/vault-sync-deployment.yaml",
         "services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
         "services/comms/atlasbot-deployment.yaml",
@@ -277,16 +276,16 @@
         "services/comms/livekit-token-deployment.yaml",
         "services/comms/livekit.yaml",
         "services/comms/mas-deployment.yaml",
-        "services/comms/oneoffs/bstein-force-leave-job.yaml",
-        "services/comms/oneoffs/comms-secrets-ensure-job.yaml",
-        "services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml",
-        "services/comms/oneoffs/mas-db-ensure-job.yaml",
-        "services/comms/oneoffs/mas-local-users-ensure-job.yaml",
-        "services/comms/oneoffs/othrys-kick-numeric-job.yaml",
-        "services/comms/oneoffs/synapse-admin-ensure-job.yaml",
-        "services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml",
-        "services/comms/oneoffs/synapse-signingkey-ensure-job.yaml",
-        "services/comms/oneoffs/synapse-user-seed-job.yaml",
+        "services/comms/repair-jobs/bstein-force-leave-job.yaml",
+        "services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
+        "services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
+        "services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
+        "services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
         "services/comms/pin-othrys-job.yaml",
         "services/comms/reset-othrys-room-job.yaml",
         "services/comms/seed-othrys-room.yaml",
@@ -300,7 +299,7 @@
         "services/finance/firefly-cronjob.yaml",
         "services/finance/firefly-deployment.yaml",
         "services/finance/firefly-user-sync-cronjob.yaml",
-        "services/finance/oneoffs/finance-secrets-ensure-job.yaml",
+        "services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
         "services/gitea/deployment.yaml",
         "services/harbor/vault-sync-deployment.yaml",
         "services/health/wger-admin-ensure-cronjob.yaml",
@@ -309,63 +308,62 @@
         "services/jellyfin/loader.yaml",
         "services/jenkins/deployment.yaml",
         "services/jenkins/vault-sync-deployment.yaml",
-        "services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/ldap-federation-job.yaml",
-        "services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/mas-secrets-ensure-job.yaml",
-        "services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-client-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-target-client-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml",
-        "services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml",
-        "services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/realm-settings-job.yaml",
-        "services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml",
-        "services/keycloak/oneoffs/user-overrides-job.yaml",
-        "services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
+        "services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
+        "services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
+        "services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
+        "services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
+        "services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
+        "services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
         "services/keycloak/vault-sync-deployment.yaml",
         "services/logging/node-image-gc-rpi4-daemonset.yaml",
         "services/logging/node-image-prune-rpi5-daemonset.yaml",
         "services/logging/node-log-rotation-daemonset.yaml",
         "services/logging/oauth2-proxy.yaml",
-        "services/logging/oneoffs/opensearch-dashboards-setup-job.yaml",
-        "services/logging/oneoffs/opensearch-ism-job.yaml",
-        "services/logging/oneoffs/opensearch-observability-setup-job.yaml",
+        "services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
+        "services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
+        "services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
         "services/logging/opensearch-prune-cronjob.yaml",
         "services/logging/vault-sync-deployment.yaml",
         "services/mailu/mailu-sync-cronjob.yaml",
         "services/mailu/mailu-sync-listener.yaml",
-        "services/mailu/oneoffs/mailu-sync-job.yaml",
         "services/mailu/vault-sync-deployment.yaml",
         "services/mailu/vip-controller.yaml",
-        "services/maintenance/ariadne-deployment.yaml",
-        "services/maintenance/disable-k3s-traefik-daemonset.yaml",
-        "services/maintenance/image-sweeper-cronjob.yaml",
-        "services/maintenance/k3s-agent-restart-daemonset.yaml",
-        "services/maintenance/metis-deployment.yaml",
-        "services/maintenance/metis-k3s-token-sync-cronjob.yaml",
-        "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
-        "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
-        "services/maintenance/node-image-sweeper-daemonset.yaml",
-        "services/maintenance/node-nofile-daemonset.yaml",
-        "services/maintenance/oauth2-proxy-metis.yaml",
-        "services/maintenance/oauth2-proxy-soteria.yaml",
-        "services/maintenance/oneoffs/ariadne-migrate-job.yaml",
-        "services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml",
-        "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
-        "services/maintenance/pod-cleaner-cronjob.yaml",
-        "services/maintenance/soteria-deployment.yaml",
-        "services/maintenance/vault-sync-deployment.yaml",
+        "services/maintenance/apps/ariadne-deployment.yaml",
+        "services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
+        "services/maintenance/node-ops/image-sweeper-cronjob.yaml",
+        "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
+        "services/maintenance/apps/metis-deployment.yaml",
+        "services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
+        "services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
+        "services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
+        "services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
+        "services/maintenance/node-ops/node-nofile-daemonset.yaml",
+        "services/maintenance/networking/oauth2-proxy-metis.yaml",
+        "services/maintenance/networking/oauth2-proxy-soteria.yaml",
+        "services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
+        "services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
+        "services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
+        "services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
+        "services/maintenance/apps/soteria-deployment.yaml",
+        "services/maintenance/bootstrap/vault-sync-deployment.yaml",
         "services/monitoring/dcgm-exporter.yaml",
         "services/monitoring/jetson-tegrastats-exporter.yaml",
-        "services/monitoring/oneoffs/grafana-org-bootstrap.yaml",
-        "services/monitoring/oneoffs/grafana-user-dedupe-job.yaml",
+        "services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
+        "services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
         "services/monitoring/platform-quality-gateway-deployment.yaml",
         "services/monitoring/platform-quality-suite-probe-cronjob.yaml",
         "services/monitoring/postmark-exporter-deployment.yaml",
@@ -395,12 +393,12 @@
         "services/logging/node-image-gc-rpi4-daemonset.yaml",
         "services/logging/node-image-prune-rpi5-daemonset.yaml",
         "services/logging/node-log-rotation-daemonset.yaml",
-        "services/maintenance/disable-k3s-traefik-daemonset.yaml",
-        "services/maintenance/image-sweeper-cronjob.yaml",
-        "services/maintenance/metis-deployment.yaml",
-        "services/maintenance/node-image-sweeper-daemonset.yaml",
-        "services/maintenance/node-nofile-daemonset.yaml",
-        "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml"
+        "services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
+        "services/maintenance/node-ops/image-sweeper-cronjob.yaml",
+        "services/maintenance/apps/metis-deployment.yaml",
+        "services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
+        "services/maintenance/node-ops/node-nofile-daemonset.yaml",
+        "services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml"
       ]
     }
   ]

clusters/aether/flux-system/applications/kustomization.yaml

@@ -1,4 +1,4 @@
-# infrastructure/modules/profiles/oceanus-validator/kustomization.yaml
+# clusters/aether/flux-system/applications/kustomization.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources: []

clusters/aether/flux-system/kustomization.yaml

@@ -0,0 +1,6 @@
+# clusters/aether/flux-system/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - platform
+  - applications

clusters/aether/flux-system/platform/kustomization.yaml

@@ -1,4 +1,4 @@
-# clusters/oceanus/applications/kustomization.yaml
+# clusters/aether/flux-system/platform/kustomization.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources: []

clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml

@@ -6,9 +6,10 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Migration jobs run only during portal schema changes."
 spec:
   interval: 10m
-  path: ./services/bstein-dev-home/oneoffs/migrations
+  path: ./services/bstein-dev-home/migration-jobs
   prune: true
   force: true
   sourceRef:

clusters/atlas/flux-system/applications/finance/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Finance apps are parked until the next storage and SSO pass."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/game-stream/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Game streaming is optional and resumes only for planned use."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/health/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Health stack is staged pending account and mail flows."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml

@@ -0,0 +1,28 @@
+# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-chat
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes-chat
+  targetNamespace: hermes-chat
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 30m
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: hermes-chat
+      namespace: hermes-chat
+  dependsOn:
+    - name: cert-manager
+    - name: core
+    - name: hermes
+    - name: keycloak
+    - name: longhorn

clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: hermes-triage-demo
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/hermes-triage-demo
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: false
+  timeout: 5m

clusters/atlas/flux-system/applications/hermes/kustomization.yaml

@@ -18,6 +18,10 @@ spec:
   wait: true
   timeout: 45m
   healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: hermes-model-gate
+      namespace: hermes
     - apiVersion: apps/v1
       kind: Deployment
       name: hermes-ollama
@@ -26,6 +30,10 @@ spec:
       kind: Deployment
       name: hermes
       namespace: hermes
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: oauth2-proxy-hermes
+      namespace: hermes
   dependsOn:
     - name: cert-manager
     - name: core

clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Media stack stays paused while auth and storage changes are staged."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/jenkins/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -28,6 +28,8 @@ resources:
   - ai-llm/kustomization.yaml
   - openclaw/kustomization.yaml
   - hermes/kustomization.yaml
+  - hermes-chat/kustomization.yaml
+  - hermes-triage-demo/kustomization.yaml
   - game-stream/kustomization.yaml
   - cassandra-auth/kustomization.yaml
   - cassandra/kustomization.yaml

clusters/atlas/flux-system/applications/mailu/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Mail stack is staged and resumes only during mail rollout work."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Mail sync resumes after Nextcloud and Mailu are active."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Nextcloud is staged pending storage, SSO, and mail validation."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/outline/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Outline is staged until shared auth and mail are ready."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/planka/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Planka is staged until shared auth and mail are ready."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/quality/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Quality stack changes resume only during quality-gate rollout work."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/typhon/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Climate automation is staged until sensor and control loops are verified."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Vaultwarden stays separate from SSO rollout and resumes by hand."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Temporary wallet RPC stack is opt-in for maintenance windows."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Mining workloads are disabled unless explicitly enabled for a short run."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/platform/descheduler/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Descheduler is paused during node recovery and placement stabilization."
 spec:
   interval: 30m
   suspend: true

clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "GitOps UI is optional and resumes only for operator access work."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Longhorn UI ingress is optional and opened only for storage work."
 spec:
   interval: 10m
   suspend: true

clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: flux-system
   annotations:
     kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+    atlas.bstein.dev/suspend-reason: "Guardrail rollout is paused until service resource requests are normalized."
 spec:
   interval: 10m
   suspend: true

clusters/oceanus/flux-system/kustomization.yaml

@@ -1,9 +0,0 @@
-# clusters/oceanus/flux-system/kustomization.yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  # Populate when oceanus cluster is bootstrapped with Flux.
-  # - gotk-components.yaml
-  # - gotk-sync.yaml
-  - ../platform
-  - ../applications

clusters/oceanus/platform/kustomization.yaml

@@ -1,6 +0,0 @@
-# clusters/oceanus/platform/kustomization.yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ../../infrastructure/modules/base
-  - ../../infrastructure/modules/profiles/oceanus-validator

dockerfiles/Dockerfile.hermes-agent

@@ -0,0 +1,122 @@
+# syntax=docker/dockerfile:1
+# dockerfiles/Dockerfile.hermes-agent
+FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
+
+USER root
+
+# Keep dashboard chat sockets tied to the intended React mount and conversation.
+# A resumed conversation needs a different PTY attachment key from a fresh chat;
+# reconnects to that same conversation must keep using the same key.
+RUN node <<'NODE'
+const fs = require("node:fs");
+const path = "/opt/hermes/web/src/pages/ChatPage.tsx";
+let source = fs.readFileSync(path, "utf8");
+const socketBefore = [
+  '      const url = await api.buildWsUrl("/api/pty", params);',
+  '      const ws = new WebSocket(url);',
+].join("\n");
+const socketAfter = [
+  '      const url = await api.buildWsUrl("/api/pty", params);',
+  '      if (unmounting) return;',
+  '      const ws = new WebSocket(url);',
+].join("\n");
+const attachBefore = '      params.attach = ptyAttachToken(forceFresh);';
+const attachAfter = [
+  '      const attachScope = resumeParam',
+  '        ? `resume:${resumeParam}:${scopedProfile ?? ""}`',
+  '        : `fresh:${scopedProfile ?? ""}`;',
+  '      params.attach = `${ptyAttachToken(forceFresh)}:${attachScope}`;',
+].join("\n");
+
+if (!source.includes(socketBefore)) {
+  throw new Error("Hermes ChatPage WebSocket patch context changed");
+}
+if (!source.includes(attachBefore)) {
+  throw new Error("Hermes ChatPage PTY attachment patch context changed");
+}
+source = source.replace(socketBefore, socketAfter);
+source = source.replace(attachBefore, attachAfter);
+fs.writeFileSync(path, source);
+NODE
+
+# The upstream OIDC gate authenticates users but deliberately treats the
+# dashboard as one shared workstation. Allow a deployment to narrow that
+# workstation to explicit OIDC subjects. Enforce this after normal provider
+# verification so a denied account is a 403, not a misleading provider 503.
+RUN python - <<'PY'
+from pathlib import Path
+
+path = Path("/opt/hermes/hermes_cli/dashboard_auth/middleware.py")
+source = path.read_text()
+helper_before = '''def _client_ip(request: Request) -> str:
+    fwd = request.headers.get("x-forwarded-for", "")
+    if fwd:
+        return fwd.split(",")[0].strip()
+    return request.client.host if request.client else ""
+
+
+'''
+helper_after = helper_before + '''def _dashboard_user_allowed(session) -> bool:
+    """Apply an optional deployment-level OIDC-subject allowlist."""
+    import os
+
+    allowed = {
+        value.strip()
+        for value in os.environ.get(
+            "HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS", ""
+        ).split(",")
+        if value.strip()
+    }
+    return not allowed or session.user_id in allowed
+
+
+def _user_forbidden_response() -> Response:
+    """Return an authorization failure without exposing identities."""
+    return JSONResponse(
+        {
+            "error": "forbidden",
+            "detail": "This Atlas account is not authorized for this dashboard.",
+        },
+        status_code=403,
+    )
+
+
+'''
+refresh_before = '''            new_session, refreshing_provider = refreshed
+            request.state.session = new_session
+            response = await call_next(request)
+'''
+refresh_after = '''            new_session, refreshing_provider = refreshed
+            if not _dashboard_user_allowed(new_session):
+                return _user_forbidden_response()
+            request.state.session = new_session
+            response = await call_next(request)
+'''
+final_before = '''    request.state.session = session
+    return await call_next(request)
+'''
+final_after = '''    if not _dashboard_user_allowed(session):
+        return _user_forbidden_response()
+    request.state.session = session
+    return await call_next(request)
+'''
+for before, after, label in (
+    (helper_before, helper_after, "allowlist helper"),
+    (refresh_before, refresh_after, "refreshed session"),
+    (final_before, final_after, "verified session"),
+):
+    if before not in source:
+        raise SystemExit(f"Hermes dashboard auth {label} patch context changed")
+    source = source.replace(before, after, 1)
+path.write_text(source)
+PY
+
+COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
+
+RUN cd /opt/hermes/web \
+    && npm run build \
+    && grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \
+    && grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \
+    && grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \
+        /opt/hermes/hermes_cli/dashboard_auth/middleware.py \
+    && chmod 0755 /opt/hermes/bin/hermes-session-migrate

dockerfiles/hermes-session-migrate.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Copy selected legacy Hermes sessions into an isolated user home."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import shutil
+import sqlite3
+
+from hermes_state import SessionDB
+
+
+def _copy_if_missing(source: Path, target: Path) -> None:
+    """Copy one credential/config file without overwriting user state."""
+    if source.is_file() and not target.exists():
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, target)
+
+
+def _copy_session(
+    source: sqlite3.Connection,
+    target: sqlite3.Connection,
+    session_id: str,
+    user_id: str,
+) -> bool:
+    """Copy a session and its messages, preserving original timestamps."""
+    source.row_factory = sqlite3.Row
+    session = source.execute(
+        "SELECT * FROM sessions WHERE id = ?", (session_id,)
+    ).fetchone()
+    if session is None:
+        return False
+
+    columns = list(session.keys())
+    values = [user_id if column == "user_id" else session[column] for column in columns]
+    placeholders = ", ".join("?" for _ in columns)
+    target.execute(
+        f"INSERT OR IGNORE INTO sessions ({', '.join(columns)}) "
+        f"VALUES ({placeholders})",
+        values,
+    )
+
+    message_columns = [
+        row[1] for row in target.execute("PRAGMA table_info(messages)").fetchall()
+    ]
+    selected_columns = [
+        column
+        for column in message_columns
+        if column in {
+            row[1]
+            for row in source.execute("PRAGMA table_info(messages)").fetchall()
+        }
+    ]
+    column_sql = ", ".join(selected_columns)
+    target.execute(
+        f"INSERT OR IGNORE INTO messages ({column_sql}) "
+        f"SELECT {column_sql} FROM source_db.messages WHERE session_id = ?",
+        (session_id,),
+    )
+    return True
+
+
+def main() -> None:
+    """Create the isolated home and perform the idempotent session copy."""
+    source_home = Path(os.environ["HERMES_MIGRATE_SOURCE_HOME"])
+    target_home = Path(os.environ["HERMES_HOME"])
+    user_id = os.environ["HERMES_MIGRATE_USER_ID"].strip()
+    session_ids = [
+        value.strip()
+        for value in os.environ.get("HERMES_MIGRATE_SESSION_IDS", "").split(",")
+        if value.strip()
+    ]
+
+    target_home.mkdir(parents=True, exist_ok=True)
+    (target_home / "home" / ".local" / "bin").mkdir(parents=True, exist_ok=True)
+    (target_home / "workspace" / "skills").mkdir(parents=True, exist_ok=True)
+    (target_home / "logs").mkdir(parents=True, exist_ok=True)
+    for filename in (".env", "auth.json"):
+        _copy_if_missing(source_home / filename, target_home / filename)
+
+    source_db = source_home / "state.db"
+    target_db = target_home / "state.db"
+    SessionDB(db_path=target_db).close()
+    copied = 0
+    if source_db.is_file() and session_ids:
+        source = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)
+        target = sqlite3.connect(target_db)
+        target.execute("ATTACH DATABASE ? AS source_db", (str(source_db),))
+        try:
+            with target:
+                for session_id in session_ids:
+                    copied += int(_copy_session(source, target, session_id, user_id))
+        finally:
+            target.close()
+            source.close()
+    print(f"isolated Hermes home ready; migrated_sessions={copied}")
+
+
+if __name__ == "__main__":
+    main()

hosts/inventory/lab.yaml

@@ -12,7 +12,7 @@ all:
           ansible_host: REPLACE_ME
           ansible_user: debian
           roleset: minipc_gpu
-    baremetal:
+    dedicated_hosts:
       hosts:
         titan-db:
           ansible_host: REPLACE_ME
@@ -22,7 +22,3 @@ all:
           ansible_host: REPLACE_ME
           ansible_user: jump
           roleset: jumphost
-        oceanus:
-          ansible_host: REPLACE_ME
-          ansible_user: validator
-          roleset: validator

hosts/playbooks/site.yaml

@@ -14,13 +14,6 @@
     - common
     - titan_jh
 
-- name: Configure oceanus validator host
-  hosts: oceanus
-  gather_facts: true
-  roles:
-    - common
-    - oceanus_base
-
 - name: Prepare hybrid tethys node
   hosts: titan-24
   gather_facts: true

hosts/roles/oceanus_base/tasks/main.yaml

@@ -1,6 +0,0 @@
-# hosts/roles/oceanus_base/tasks/main.yaml
----
-- name: Placeholder for oceanus base configuration
-  ansible.builtin.debug:
-    msg: "Install validator prerequisites and monitoring exporters here."
-  tags: ['oceanus']

infrastructure/core/cert-manager/letsencrypt-prod.yaml

@@ -1,4 +1,4 @@
-# infrastructure/sources/cert-manager/letsencrypt-prod.yaml
+# infrastructure/core/cert-manager/letsencrypt-prod.yaml
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:

infrastructure/core/cert-manager/letsencrypt.yaml

@@ -1,4 +1,4 @@
-# infrastructure/sources/cert-manager/letsencrypt.yaml
+# infrastructure/core/cert-manager/letsencrypt.yaml
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:

infrastructure/core/kustomization.yaml

@@ -11,5 +11,5 @@ resources:
   - coredns-deployment.yaml
   - ntp-sync-daemonset.yaml
   - workload-profiles.yaml
-  - ../sources/cert-manager/letsencrypt.yaml
-  - ../sources/cert-manager/letsencrypt-prod.yaml
+  - cert-manager/letsencrypt.yaml
+  - cert-manager/letsencrypt-prod.yaml

infrastructure/flux-system/kustomization.yaml

@@ -2,4 +2,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ../clusters/atlas/flux-system
+  - ../../clusters/atlas/flux-system

infrastructure/traefik/deployment.yaml

@@ -4,8 +4,6 @@ items:
 - apiVersion: apps/v1
   kind: Deployment
   metadata:
-    annotations:
-      deployment.kubernetes.io/revision: "4"
     name: traefik
     namespace: traefik
   spec:
@@ -73,6 +71,14 @@ items:
           node-role.kubernetes.io/worker: "true"
         affinity:
           nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                  - rpi5
+                  - rpi4
             preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
               preference:
@@ -103,6 +109,12 @@ items:
                   operator: In
                   values:
                   - rpi4
+          podAntiAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: traefik
+              topologyKey: kubernetes.io/hostname
         restartPolicy: Always
         schedulerName: default-scheduler
         serviceAccount: atlas-traefik-ingress-controller

infrastructure/vault-injector/helmrelease.yaml

@@ -33,7 +33,12 @@ spec:
       enabled: false
     injector:
       enabled: true
-      replicas: 1
+      # Two replicas because the webhook is failurePolicy: Ignore. With a
+      # single replica, any pod created while the injector restarts is
+      # admitted unmutated: it comes up without its Vault agent sidecar,
+      # never finds /vault/secrets, and crash-loops indefinitely with no
+      # indication that injection was skipped. Observed twice on ariadne.
+      replicas: 2
       agentImage:
         repository: hashicorp/vault
         tag: "1.17.6"

knowledge/INDEX.md

@@ -12,7 +12,7 @@ Layout
 
 Regeneration
 - Update manifests/docs, then regenerate generated artifacts:
-  - `python scripts/knowledge_render_atlas.py --write`
+  - `python scripts/render/knowledge_render_atlas.py --write`
 
 Authoring rules
 - Never include secret values. Prefer `secretRef` names or Vault paths like `kv/atlas/...`.

knowledge/catalog/atlas-summary.json

@@ -1,8 +1,8 @@
 {
   "counts": {
-    "helmrelease_host_hints": 19,
-    "http_endpoints": 45,
-    "services": 47,
-    "workloads": 74
+    "helmrelease_host_hints": 22,
+    "http_endpoints": 54,
+    "services": 70,
+    "workloads": 100
   }
 }

knowledge/diagrams/atlas-http.mmd

@@ -1,9 +1,19 @@
 flowchart LR
+  host_agent_bstein_dev["agent.bstein.dev"]
+  svc_hermes_hermes["hermes/hermes (Service)"]
+  host_agent_bstein_dev --> svc_hermes_hermes
+  wl_hermes_hermes["hermes/hermes (Deployment)"]
+  svc_hermes_hermes --> wl_hermes_hermes
   host_auth_bstein_dev["auth.bstein.dev"]
   svc_sso_oauth2_proxy["sso/oauth2-proxy (Service)"]
   host_auth_bstein_dev --> svc_sso_oauth2_proxy
   wl_sso_oauth2_proxy["sso/oauth2-proxy (Deployment)"]
   svc_sso_oauth2_proxy --> wl_sso_oauth2_proxy
+  host_backup_bstein_dev["backup.bstein.dev"]
+  svc_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Service)"]
+  host_backup_bstein_dev --> svc_maintenance_oauth2_proxy_soteria
+  wl_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Deployment)"]
+  svc_maintenance_oauth2_proxy_soteria --> wl_maintenance_oauth2_proxy_soteria
   host_bstein_dev["bstein.dev"]
   svc_bstein_dev_home_bstein_dev_home_frontend["bstein-dev-home/bstein-dev-home-frontend (Service)"]
   host_bstein_dev --> svc_bstein_dev_home_bstein_dev_home_frontend
@@ -111,6 +121,16 @@ flowchart LR
   host_pegasus_bstein_dev --> svc_jellyfin_pegasus
   wl_jellyfin_pegasus["jellyfin/pegasus (Deployment)"]
   svc_jellyfin_pegasus --> wl_jellyfin_pegasus
+  host_quality_bstein_dev["quality.bstein.dev"]
+  svc_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Service)"]
+  host_quality_bstein_dev --> svc_quality_oauth2_proxy_sonarqube
+  wl_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Deployment)"]
+  svc_quality_oauth2_proxy_sonarqube --> wl_quality_oauth2_proxy_sonarqube
+  host_recovery_bstein_dev["recovery.bstein.dev"]
+  svc_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Service)"]
+  host_recovery_bstein_dev --> svc_maintenance_oauth2_proxy_metis
+  wl_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Deployment)"]
+  svc_maintenance_oauth2_proxy_metis --> wl_maintenance_oauth2_proxy_metis
   host_scm_bstein_dev["scm.bstein.dev"]
   svc_gitea_gitea["gitea/gitea (Service)"]
   host_scm_bstein_dev --> svc_gitea_gitea
@@ -141,6 +161,20 @@ flowchart LR
   host_vault_bstein_dev --> svc_vaultwarden_vaultwarden_service
   wl_vaultwarden_vaultwarden["vaultwarden/vaultwarden (Deployment)"]
   svc_vaultwarden_vaultwarden_service --> wl_vaultwarden_vaultwarden
+  host_veles_bstein_dev["veles.bstein.dev"]
+  svc_veles_veles_frontend["veles/veles-frontend (Service)"]
+  host_veles_bstein_dev --> svc_veles_veles_frontend
+  wl_veles_veles_frontend["veles/veles-frontend (Deployment)"]
+  svc_veles_veles_frontend --> wl_veles_veles_frontend
+  svc_veles_veles_backend["veles/veles-backend (Service)"]
+  host_veles_bstein_dev --> svc_veles_veles_backend
+  wl_veles_veles_backend["veles/veles-backend (Deployment)"]
+  svc_veles_veles_backend --> wl_veles_veles_backend
+  host_wolf_bstein_dev["wolf.bstein.dev"]
+  svc_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Service)"]
+  host_wolf_bstein_dev --> svc_game_stream_oauth2_proxy_wolf
+  wl_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Deployment)"]
+  svc_game_stream_oauth2_proxy_wolf --> wl_game_stream_oauth2_proxy_wolf
 
   subgraph bstein_dev_home[bstein-dev-home]
     svc_bstein_dev_home_bstein_dev_home_frontend
@@ -175,6 +209,10 @@ flowchart LR
     svc_finance_firefly
     wl_finance_firefly
   end
+  subgraph game_stream[game-stream]
+    svc_game_stream_oauth2_proxy_wolf
+    wl_game_stream_oauth2_proxy_wolf
+  end
   subgraph gitea[gitea]
     svc_gitea_gitea
     wl_gitea_gitea
@@ -183,6 +221,10 @@ flowchart LR
     svc_health_wger
     wl_health_wger
   end
+  subgraph hermes[hermes]
+    svc_hermes_hermes
+    wl_hermes_hermes
+  end
   subgraph jellyfin[jellyfin]
     svc_jellyfin_pegasus
     wl_jellyfin_pegasus
@@ -204,6 +246,12 @@ flowchart LR
   subgraph mailu_mailserver[mailu-mailserver]
     svc_mailu_mailserver_mailu_front
   end
+  subgraph maintenance[maintenance]
+    svc_maintenance_oauth2_proxy_soteria
+    wl_maintenance_oauth2_proxy_soteria
+    svc_maintenance_oauth2_proxy_metis
+    wl_maintenance_oauth2_proxy_metis
+  end
   subgraph nextcloud[nextcloud]
     svc_nextcloud_nextcloud
     wl_nextcloud_nextcloud
@@ -218,6 +266,10 @@ flowchart LR
     svc_planka_planka
     wl_planka_planka
   end
+  subgraph quality[quality]
+    svc_quality_oauth2_proxy_sonarqube
+    wl_quality_oauth2_proxy_sonarqube
+  end
   subgraph sso[sso]
     svc_sso_oauth2_proxy
     wl_sso_oauth2_proxy
@@ -232,3 +284,9 @@ flowchart LR
     svc_vaultwarden_vaultwarden_service
     wl_vaultwarden_vaultwarden
   end
+  subgraph veles[veles]
+    svc_veles_veles_frontend
+    wl_veles_veles_frontend
+    svc_veles_veles_backend
+    wl_veles_veles_backend
+  end

knowledge/hermes-investigative-triage-demo.md

@@ -0,0 +1,74 @@
+# Hermes Investigative Triage Demonstration
+
+- Date: 2026-08-05
+- Run ID: run_d2f61902c00044948a3912a94e895ab9
+- Duration: 128 seconds, status completed
+- Usage: {"input_tokens": 200961, "output_tokens": 4370, "total_tokens": 205331}
+- Mode: supervised, read-only; approvals `smart` with mutation deny-list; zero mutations, zero denied approvals
+- Tool audit (from the run's SSE event stream): 8 skill loads (triage-titan-test-failures pack: orchestrator, jenkins-retained-evidence, platform-quality-metrics, kubernetes-readonly-failure-classifier, flux-git-change-correlation, ...), 4 read-only terminal batches (Ariadne internal API sweep, kubectl logs/jobs/configmap in hermes-triage-demo, VictoriaMetrics queries incl. last_over_time fallback, Flux/Gitea revision correlation)
+
+Unlike the automated loop (where Ariadne supplies a sanitized bundle), this run
+was given ONLY an incident ID and gathered all evidence itself — the operator's
+triage path executed by the agent. Notable behaviors: honest dead-end reporting
+(Jenkins 403 -> pivot to Kubernetes-retained logs; Ariadne audit endpoints 404),
+instant-query lookback miss -> switched to last_over_time, causal chain
+reconstruction across five sources, and an unprompted, legitimate hardening
+suggestion (add a build label to ariadne_hermes_triage_action_total).
+
+## Prompt
+
+```text
+Use $triage-titan-test-failures.
+Investigate Jenkins incident hermes-triage-demo/10. No evidence bundle is attached this time: gather the evidence yourself with your read-only tools, following the operator's standard triage path:
+1. Ariadne is the source of truth: query its internal API (base URL in ARIADNE_BASE_URL, e.g. GET /api/internal/testing/triage/latest and any internal audit endpoints you can reach) for records about this incident.
+2. Quality metrics: query VictoriaMetrics (VICTORIA_METRICS_URL) for the series ariadne_hermes_triage_incident{jenkins_job="hermes-triage-demo",build="10"} and ariadne_hermes_triage_action_total, and note what they show.
+3. Service health: use read-only kubectl to inspect the hermes-triage-demo namespace (jobs, pods, recent events) and confirm what ran there.
+4. Change correlation: check Flux state (kubectl get kustomizations -n flux-system; the flux-system GitRepository revision) for recent changes relevant to this incident.
+Then return a concise operator-style report with EXACTLY these sections:
+FACTS (each with source and reference), INFERENCES, LIKELY CAUSE, WHAT HAPPENED (was it repaired, by what mechanism, final state), NEXT CHECKS (what a human should verify next), and REPO-SIDE FIXES (if any are warranted).
+Read-only throughout. Do not mutate anything. Do not use kubectl apply/delete/patch/scale.
+```
+
+## Full agent report
+
+```text
+FACTS
+Ariadne diagnosis/latest, generated 2026-08-05T21:31:06Z, references only the broad testing bundle and does not identify hermes-triage-demo/10. Ariadne’s incident/audit candidate endpoints returned HTTP 404.
+Kubernetes logs, hermes-triage-demo/hermes-demo-test-10-dgvpj: build 10 emitted incident_id="hermes-triage-demo/10", classification_hint="demo_fixture_unhealthy", and “expected fixture state healthy; found unhealthy.”
+Kubernetes Job status: hermes-demo-test-10 was created 2026-08-05T20:59:47Z and failed; its pod ran on titan-06 and had no restarts or image-pull events.
+Kubernetes logs, hermes-demo-repair-10-nkrzz: repair job patched hermes-triage-demo-fixture and reported “fixture state reset to healthy”; the Job completed successfully at 2026-08-05T21:01:06Z.
+Kubernetes logs/status, hermes-demo-test-11-hv5mh: follow-up test completed at 2026-08-05T21:01:35Z and reported “fixture state healthy.”
+Kubernetes ConfigMap hermes-triage-demo-fixture: current data.state is healthy.
+VictoriaMetrics, last_over_time(ariadne_hermes_triage_incident{jenkins_job="hermes-triage-demo",build="10"}[24h]): sample at 2026-08-05T21:41:01Z has resolved=1 and every other lifecycle state, including failed and human_required, at 0. The instantaneous selector returned no result because its samples were outside the default instant-query lookback.
+VictoriaMetrics, ariadne_hermes_triage_action_total over 24h: repair_demo_fixture has requested=1, accepted=1, success=1 for instance 10.42.19.143; no build label exists on this counter, so it is correlated by the incident instance and timeline rather than proven per-build.
+Flux Kustomization hermes-triage-demo: Ready=True at revision main@sha1:aa36b06b2bc43eda21b36e54a7d312e55d27c370. Flux source GitRepository is Ready=True at that same revision.
+Gitea commit 49aab8da3298ebfc1b9edab5ce2e6938ad033a23, “feat(hermes-triage): enable autoremediation for the demo loop,” predates this incident and modifies services/maintenance/apps/ariadne-deployment.yaml. The current revision aa36b06 is an unrelated maintenance image update.
+Direct Jenkins GET for /job/hermes-triage-demo/10 API metadata and console returned HTTP 403. The Kubernetes Job logs are therefore the retained direct execution evidence used here.
+
+INFERENCES
+The failure was an intentional demo-fixture health assertion failure, not an image, scheduling, node, or storage failure. The failed test, repair, and successful follow-up all ran in sequence; this is a strong causal chain.
+The repair was automatically accepted under the previously deployed allowlisted autoremediation path. The repair Job itself performed the ConfigMap patch; Flux did not repair the fixture during this incident.
+The unrelated cluster concerns in the Ariadne bundle, including Longhorn probe failures and pending Veles pods, have no evidenced link to this ConfigMap-based test path.
+
+LIKELY CAUSE
+The fixture was deliberately or otherwise pre-seeded unhealthy before Jenkins incident hermes-triage-demo/10. The test correctly failed on that condition. Confidence: high.
+
+WHAT HAPPENED
+Build 10 failed at 20:59:47Z because the fixture state was unhealthy. The deployed autoremediation mechanism accepted repair_demo_fixture, ran hermes-demo-repair-10, and patched only hermes-triage-demo-fixture to healthy. Test 11 then passed at 21:01:35Z. Final state is repaired/resolved: ConfigMap state=healthy, incident metric resolved=1, repair success counter=1, and the demo Flux Kustomization is Ready at aa36b06.
+
+NEXT CHECKS
+Human should verify the current deployed revision and readiness after the next Flux interval:
+kubectl -n flux-system get kustomization hermes-triage-demo
+kubectl -n flux-system describe gitrepository flux-system
+
+Human should retain the build-10 Jenkins evidence if access is available:
+curl -fsS https://ci.bstein.dev/job/hermes-triage-demo/10/api/json
+curl -fsS https://ci.bstein.dev/job/hermes-triage-demo/10/consoleText
+These currently require Jenkins authentication (HTTP 403).
+
+Human should confirm that the resolved gauge remains the only asserted lifecycle state:
+curl -G -fsS --data-urlencode 'query=last_over_time(ariadne_hermes_triage_incident{jenkins_job="hermes-triage-demo",build="10"}[24h])' "$VICTORIA_METRICS_URL/api/v1/query"
+
+REPO-SIDE FIXES
+None warranted for this incident. The observed behavior matches the committed demo design: an isolated ConfigMap fixture, one allowlisted repair action, a scoped repair ServiceAccount, and one repair per incident. A future hardening change, if desired, would be to add a build label to ariadne_hermes_triage_action_total so repair counters can be directly attributed to a specific incident rather than correlated by instance and time.
+```

knowledge/hermes-triage-diagram-corrections.md

@@ -0,0 +1,134 @@
+# Corrections for `mermaid/TestAutomationV2.mmd`
+
+Written 2026-08-06 against the diagram as it stood that morning. The diagram
+is not edited here on purpose: it is being maintained in parallel, so this is
+the change list rather than a patch.
+
+The structure holds. The trust boundary, the gate chain, the action registry
+and the output taxonomy are all still accurate. What follows is what tonight's
+work invalidated.
+
+## Nodes that are now factually wrong
+
+**`collector`** says *"Keeps the head and tail"* and *"Keeps the earliest
+useful failure regions"*. Earliest-first selection was the defect: on a long
+pipeline the byte budget was consumed entirely by successful tool output, and
+the enforced failure never reached Hermes. Replace with:
+
+```
+Ariadne evidence collector
+Reads the full console up to 2 MB
+Ranks failure regions by evidence strength
+A definite failure outranks a tool that merely ran
+Ignores passing test lines that quote failures
+Merges repeats and overlapping context
+```
+
+**`repair_fixture`** says *"Creates one scoped Job"*. It no longer does. The
+repair is an in-process ConfigMap patch; no pod is created.
+
+**`candidate_files`** says *"Ranks the earliest hints first"*. It now also
+reads the failing test and follows its imports to the module under test, which
+is what made a pull request on a real service repository possible at all:
+
+```
+Candidate file selection
+Uses path and line hints from failure regions
+Reads the failing test as context
+Follows its imports to the module under test
+Reading is wider than writing: a test is readable, never patchable
+```
+
+**`action_scope`** says *"No real service classification maps to a mutation"*.
+No longer true. `retry_transient_infra` is allowlisted for every allowlisted
+job, so a real service can now receive an action - a Jenkins rebuild.
+
+**`timings`** - the fixture loop is not four minutes. Measure from the red
+build, not from arming, because arming depends on the agent pool:
+
+```
+Red -> fixture patched healthy: 25s
+Red -> rebuild green and incident resolved: 1m04s
+```
+
+**`examples`** - supersede with the stronger 2026-08-06 results, including
+`ariadne/409` opening a real pull request with the correct one-line fix.
+
+## Structurally misleading
+
+**The `route` decision reads as three exclusive branches.** It is not. An
+escalating incident attempts a code proposal *and* files an issue *and* stays
+`human_required`. A pull request rides along on the escalation; it never
+replaces the issue. As drawn, a reader concludes one substitutes for the
+other.
+
+**`branch_build` overstates coverage.** `hermes-code-demo-branches` only
+builds branches in the demo repository. A pull request opened against a real
+service repository gets no automatic branch build. Mark it `limited`.
+
+**`alert_output` is stale.** Alerting no longer fires on every escalation.
+Two narrow rules remain: a repair that ran and failed, and an escalation
+untouched for six hours. The issue is the durable artifact; email is the
+exception.
+
+## Missing, and worth adding
+
+- The demo classification and `repair_demo_fixture` are now forbidden on any
+  job other than `hermes-triage-demo`. Hermes misapplied that label to real
+  services twice at 0.96 and 0.99 confidence; the gates refused it both times,
+  and it can no longer be produced at all.
+- **A build that never finishes** is escalated once it passes the time cap,
+  with no model call, because its console is still being written. Before this
+  existed such a build produced nothing anywhere while holding an agent slot.
+- **A failure superseded by a newer build can be skipped entirely.** Detection
+  reads `lastBuild`, so a red build replaced quickly is never triaged.
+
+## Second pass, after the remaining work landed
+
+Everything previously marked conditional is now done, which changes three
+things structurally rather than cosmetically.
+
+**Not every incident reaches Hermes any more.** The diagram's main flow runs
+intake -> hermes_plane -> controls -> response, which reads as though every
+incident is diagnosed by the model. A build that overruns its time cap is now
+escalated by Ariadne alone, with no model call at all, because its console is
+still being written and any root cause would be invented. That needs its own
+edge from the detector straight to the human path, bypassing `hermes_plane`
+and `controls` entirely. It is also the honest picture of the trust boundary:
+some conclusions are Ariadne's own observations, and the issue body now says
+so rather than crediting a diagnosis that never happened.
+
+**The Jenkins evidence limitation is gone, not narrowed.** Delete
+`jenkins_limit` and the "Per-test evidence waits on Jenkins plugins" line from
+`current_limits`. `junit` and `pipeline-stage-view` are installed, `testReport`
+and `wfapi` return 200, and the bundle now carries `failed_tests` with the
+failing test name, class and assertion, plus `first_failed_stage`. The
+collector always fetched both and had simply been receiving 404s. The `bundle`
+node should list them as populated fields, because they are what the diagnosis
+now rests on rather than scraped console text.
+
+**The detector watches branches, not just jobs.** A multibranch project is a
+folder with no `lastBuild`; it was returned as skipped on every tick. The
+detector now expands a folder into its branch jobs, capped at five, so
+`detector` should say it watches configured jobs *and the branches inside
+multibranch folders*.
+
+Smaller corrections to the same nodes:
+
+- `evidence_sources` may include a service's own namespace, not only the demo
+  namespace and `jenkins`, where the job is mapped to one.
+- The human path files an issue even for an incident triaged before issue
+  filing existed, reusing the classification the original diagnosis recorded
+  so one open issue per job and classification still holds.
+- Hermes's credential comes from Vault, not a manually created Secret. If the
+  diagram ever shows credential provenance, that is now the accurate source.
+
+## Conditional on work that was blocked
+
+`model_gate` should now read `anthropic/claude-opus-5` primary, with
+`openai-codex/gpt-5.6-terra` first fallback and local `gpt-oss:20b` second.
+
+`jenkins_limit` and *"Per-test evidence waits on Jenkins plugins"* come out
+once the `junit` plugin is live. Note the plugin was never blocked on a core
+upgrade as previously recorded: `junit 1369.v15da_00283f06` runs on core
+2.528.3, only the latest release requires 2.533.

knowledge/metis.md

@@ -1,12 +1,14 @@
 # Metis (node recovery)
 
 ## Node classes (current map)
-- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent)
+- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent)
 - rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)
 - rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)
 - rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)
+- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent)
 - amd64 agents: titan-22/24 (Debian 13, k3s agent)
-- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, future titan-20/21 (when added), plus any newcomers.
+- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels)
+- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers.
 
 ## Longhorn disk UUIDs (critical nodes)
 - titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4)
@@ -17,10 +19,9 @@
 ## Metis repo (~/Development/metis)
 - CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).
 - `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).
-- `AGENTS.md` in repo is untracked and holds raw notes.
 
 ## Next implementation steps
 - Add per-class golden image refs and checksums (Harbor or file://) when ready.
 - Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.
 - Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.
-- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited.
+- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited.

knowledge/runbooks/ci-gitea-jenkins.md

@@ -3,7 +3,7 @@ title: "CI: Gitea → Jenkins pipeline"
 tags: ["atlas", "ci", "gitea", "jenkins"]
 owners: ["brad"]
 entrypoints: ["scm.bstein.dev", "ci.bstein.dev"]
-source_paths: ["services/gitea", "services/jenkins", "scripts/jenkins_cred_sync.sh", "scripts/gitea_cred_sync.sh"]
+source_paths: ["services/gitea", "services/jenkins", "scripts/sync/jenkins_cred_sync.sh", "scripts/sync/gitea_cred_sync.sh"]
 ---
 
 # CI: Gitea → Jenkins pipeline
@@ -14,7 +14,7 @@ Atlas uses Gitea for source control and Jenkins for CI. Authentication is via Ke
 ## Where it is configured
 - Gitea manifests: `services/gitea/`
 - Jenkins manifests: `services/jenkins/`
-- Credential sync helpers: `scripts/gitea_cred_sync.sh`, `scripts/jenkins_cred_sync.sh`
+- Credential sync helpers: `scripts/sync/gitea_cred_sync.sh`, `scripts/sync/jenkins_cred_sync.sh`
 
 ## What users do (typical flow)
 - Create a repo in Gitea.

knowledge/runbooks/cluster-power-recovery.md

@@ -14,8 +14,8 @@ Bootstrapping risk to remember
 - Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.
 
 Script
-- `scripts/cluster_power_recovery.sh`
-- `scripts/cluster_power_console.sh`
+- `scripts/ops/cluster_power_recovery.sh`
+- `scripts/ops/cluster_power_console.sh`
 - Modes:
   - `prepare`
   - `shutdown`
@@ -26,21 +26,21 @@ Script
 
 Dry-run examples
 - Shutdown preview:
-  - `scripts/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`
+  - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`
 - Startup preview:
-  - `scripts/cluster_power_recovery.sh startup`
+  - `scripts/ops/cluster_power_recovery.sh startup`
 - Harbor seed preview:
-  - `scripts/cluster_power_recovery.sh harbor-seed`
+  - `scripts/ops/cluster_power_recovery.sh harbor-seed`
 
 Execute examples
 - Prepare helper image on every node:
-  - `scripts/cluster_power_recovery.sh prepare --execute`
+  - `scripts/ops/cluster_power_recovery.sh prepare --execute`
 - Seed Harbor runtime images onto `titan-05` from the control-host bundle:
-  - `scripts/cluster_power_recovery.sh harbor-seed --execute`
+  - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute`
 - Planned shutdown:
-  - `scripts/cluster_power_recovery.sh shutdown --execute`
+  - `scripts/ops/cluster_power_recovery.sh shutdown --execute`
 - Planned startup (canonical branch):
-  - `scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main`
+  - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main`
 
 Manual remote console examples
 - Canonical operator hosts:
@@ -129,7 +129,7 @@ Operational notes
 - Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.
 - Harbor is reconciled after the first critical stateful services.
 - Harbor bootstrap is now designed around a control-host bundle:
-  - Build the Harbor bundle locally with `scripts/build_harbor_bootstrap_bundle.sh`.
+  - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`.
   - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.
   - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.
 - The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.

knowledge/runbooks/hermes-triage-demo.md

@@ -0,0 +1,145 @@
+# Hermes Automated Triage Demo — Runbook
+
+How to arm, run, watch, and roll back the automated failure-to-repair demo.
+Architecture background: `knowledge/hermes-automated-triage-24h-plan.md`.
+
+## What the loop does
+
+```text
+You arm the failure (SEED_FAILURE=true)
+  -> Jenkins job hermes-triage-demo fails (fixture reads "unhealthy")
+  -> test-runner pod also writes the incident JSON to stdout
+     (Fluent Bit ships it to OpenSearch kube-*)
+  -> Ariadne polls the job every minute, opens incident <job>/<build>
+  -> Ariadne bundles Jenkins evidence + bounded OpenSearch excerpts
+  -> Ariadne calls the Hermes Agent API (/v1/runs) with
+     $triage-titan-test-failures
+  -> Hermes returns schema-valid diagnosis + requested_action
+  -> Ariadne authorizes (twelve gates) and, if remediation is enabled,
+     patches the fixture ConfigMap back to "healthy" in process
+  -> Ariadne triggers ONE rebuild with SEED_FAILURE=false
+  -> rebuild passes -> incident resolved
+Anything else -> issue in the service repo + human_required metric
+```
+
+The model behind Hermes is `anthropic/claude-opus-5` as of 2026-08-06, with
+`openai-codex/gpt-5.6-terra` as first fallback and a local `gpt-oss:20b`
+behind that, so an expired Anthropic credential degrades rather than stops.
+
+Two behaviours worth knowing before you demo, because both look like nothing
+happening:
+
+- **A build that never finishes** is escalated once it passes
+  `ARIADNE_HERMES_HUNG_BUILD_MINUTES` (default 45). No model is consulted -
+  the console is still being written - so the issue says only that the build
+  overran and is holding an agent slot.
+- **Alerting no longer fires on every escalation.** The issue in the service
+  repository is the durable artifact. Email now means either a repair ran and
+  failed, or an escalation has sat untouched for six hours.
+
+## Arming the demo
+
+Jenkins UI: `https://ci.bstein.dev/job/hermes-triage-demo/` → *Build with
+Parameters* → check `SEED_FAILURE` → Build.
+
+CLI (any Jenkins user API token):
+
+```bash
+curl -u <user>:<token> -X POST \
+  "https://ci.bstein.dev/job/hermes-triage-demo/buildWithParameters?SEED_FAILURE=true"
+```
+
+That is the only manual step. Everything after the red build is automatic.
+
+Run `scripts/ops/hermes_triage_demo.sh preflight` first. The check that most
+often decides whether a rehearsal holds its timings is the agent pool: the
+Kubernetes cloud caps concurrent agent pods at `containerCapStr: "5"`, and when
+real CI has taken all five the demo build sits in the queue reporting *"All
+nodes of label ... are offline"* with no other symptom. Observed 2026-08-06: a
+run armed at 02:13:42 did not start seeding until 02:19:37, close to six
+minutes of dead air, purely because five other builds held the pool. Wait for a
+free slot before starting, or quiesce CI.
+
+## Expected timings (measured live 2026-08-06, Ariadne 0.1.0-402)
+
+Time the demo from the moment the build goes **red**, not from arming. Arming
+only queues a Jenkins build, and that leg is at the mercy of the agent pool.
+
+- Red → fixture patched back to healthy: **25s** (≤60 s detection tick, ~16 s
+  Hermes diagnosis, then a single in-process Kubernetes API call).
+- Red → rebuild triggered: **38s**.
+- Red → rebuild green and incident resolved: **1m04s**.
+
+The whole automated leg is just over a minute, and there is no silent phase
+longer than the detection tick. Budgeted Hermes timeout is 420 s; observed
+diagnosis runs are 15–21 s.
+
+Arming → red was **6m21s** on this run, but 5m35s of that was queue wait
+behind a saturated agent pool. On an idle pool expect roughly 1m45s.
+
+The earlier 2026-08-05 figures (1m15s red→repaired, 4m00s total) were measured
+when the repair spawned its own Kubernetes Job. Converting the repair to an
+in-process call removed a whole pod launch from the critical path.
+
+## Watching it live
+
+- Jenkins: `https://ci.bstein.dev/job/hermes-triage-demo/` (red build N,
+  then green build N+1 with `SEED_FAILURE=false`).
+- Ariadne incident state:
+  `GET http://ariadne.maintenance/api/internal/audit/events` (in-cluster) or
+  `/api/admin/audit/events` (Keycloak JWT) — event types
+  `hermes_autotriage_incident`, `hermes_autotriage_diagnosis`,
+  `hermes_autotriage_action`. Status flow:
+  `detected → diagnosed → repairing → awaiting_rebuild → resolved`.
+- Metrics (VictoriaMetrics / Grafana Explore):
+  `ariadne_hermes_triage_incident{jenkins_job="hermes-triage-demo"}`,
+  `ariadne_hermes_triage_action_total`,
+  `ariadne_hermes_triage_duration_seconds`.
+- Repair evidence: `kubectl -n hermes-triage-demo get jobs` shows
+  `hermes-demo-test-<N>` (failed) and `hermes-demo-repair-<N>` (succeeded);
+  both TTL-clean after 1 h.
+- Hermes side: the run appears in the dashboard at
+  `https://agent.bstein.dev` (session/run history).
+- Escalation path: alert `HermesTriageHumanRequired` in vmalert
+  (`vmalert-atlas-availability` deployment, 1 m interval, `for: 2m`)
+  fires to Alertmanager for any `human_required` incident. Gauges are
+  republished every tick from stored incident state (restart-safe), and
+  the alert self-clears once a newer build of the same job is green.
+  Note: Alertmanager's default receiver is currently null — the alert is
+  visible in vmalert/Alertmanager/Grafana but pushes no notification.
+
+## Demonstrating safe escalation (second path)
+
+Any failure that does not match the demo-fixture signature — or any
+invalid/low-confidence/unknown-action Hermes response — ends as
+`status="human_required"` with **no mutation**. The simplest live demo:
+temporarily set `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` (see below)
+and arm the failure; Ariadne diagnoses fully but executes nothing, and the
+alert fires instead.
+
+## Kill switch and rollback
+
+- Instant behavioral off-switch (Flux-managed, in
+  `services/maintenance/apps/ariadne-deployment.yaml`):
+  `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` → diagnose-only.
+  `ARIADNE_HERMES_AUTOTRIAGE_ENABLED=false` → fully off.
+- The automatic loop can only ever: create Jobs named
+  `hermes-demo-repair-*` in namespace `hermes-triage-demo`, and trigger
+  rebuilds of allowlisted jobs (`ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST`,
+  currently `hermes-triage-demo` only). One action + one rebuild per
+  incident, ever (storage-backed idempotency).
+- Full teardown: remove `hermes-triage-demo` from
+  `clusters/atlas/flux-system/applications/kustomization.yaml` (Flux prunes
+  the namespace) and delete the `pipelineJob('hermes-triage-demo')` block
+  from `services/jenkins/configmap-jcasc.yaml`.
+
+## Credentials
+
+- Ariadne → Hermes: `Authorization: Bearer` key shared via the
+  `hermes-api-server-key` Secret present in both `hermes` and `maintenance`
+  namespaces (Hermes's init container seeds it into the persistent `.env`).
+  NOTE: currently manually created (Vault migration pending — see plan
+  handoff); rotating = write new value to both Secrets, restart hermes
+  deployment and ariadne deployment.
+- Ariadne → Jenkins: existing `JENKINS_API_USER/TOKEN` from Vault
+  (`atlas/maintenance/ariadne-db`).

knowledge/runbooks/kb-authoring.md

@@ -3,7 +3,7 @@ title: "KB authoring: what to write (and what not to)"
 tags: ["atlas", "kb", "runbooks"]
 owners: ["brad"]
 entrypoints: []
-source_paths: ["knowledge/runbooks", "scripts/knowledge_render_atlas.py"]
+source_paths: ["knowledge/runbooks", "scripts/render/knowledge_render_atlas.py"]
 ---
 
 # KB authoring: what to write (and what not to)

knowledge/software/metis.md

@@ -1,12 +1,14 @@
 # Metis (node recovery)
 
 ## Node classes (current map)
-- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent)
+- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent)
 - rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)
 - rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)
 - rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)
+- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent)
 - amd64 agents: titan-22/24 (Debian 13, k3s agent)
-- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, plus any newcomers.
+- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels)
+- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers.
 
 ### Jetson nodes (titan-20/21)
 - Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64.
@@ -22,13 +24,12 @@
 ## Metis repo (~/Development/metis)
 - CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).
 - `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).
-- `AGENTS.md` in repo is untracked and holds raw notes.
 
 ## Next implementation steps
 - Add per-class golden image refs and checksums (Harbor or file://) when ready.
 - Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.
 - Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.
-- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited.
+- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited.
 
 ## Node OS/Kernel/CRI snapshot (Jan 2026)
 - titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64
@@ -55,10 +56,10 @@
 - titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64
 
 
-### External hosts
+### Dedicated and special-purpose hosts
 - titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled.
 - titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q).
-- titan-23/oceanus: TODO audit (future).
+- titan-23: Atlas worker carrying the `oceanus` node-pool labels for Veles storage/simulation placement.
 
 
 ### Control plane Pis (titan-0a/0b/0c)

mermaid/HardwareArchitecture.mmd

@@ -0,0 +1,503 @@
+%% Titan Lab physical hardware architecture.
+%% Solid links are physical network, storage, power, or sensor paths.
+%% Dashed links are administration, telemetry, or recovery control.
+%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 30, "rankSpacing": 65, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "NETWORK_SIMPLEX", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000"}, "themeCSS": "& { background-color: #000000 !important; }"}}%%
+flowchart TB
+    internet["Internet"]:::external
+    ac_cloud["AC Infinity account"]:::external
+
+    subgraph lab["Titan Lab"]
+        direction TB
+
+        subgraph link_legend["Link colors"]
+            direction LR
+
+            subgraph legend_network["Network"]
+                direction TB
+                legend_1g["1 Gbps"]:::legend1g
+                legend_25g["2.5 Gbps"]:::legend25g
+                legend_external["External network"]:::legendExternal
+            end
+
+            subgraph legend_resources["Resources"]
+                direction TB
+                legend_local["Local attachment"]:::legendLocal
+                legend_shared["Shared storage"]:::legendShared
+                legend_gpu["GPU"]:::legendGpu
+            end
+
+            subgraph legend_operations["Operations"]
+                direction TB
+                legend_power["Power and NUT"]:::legendPower
+                legend_telemetry["Telemetry and environment"]:::legendTelemetry
+                legend_control["Administration and recovery"]:::legendControl
+                legend_safety["Protective shutdown"]:::legendSafety
+            end
+        end
+
+        subgraph network["Network fabric"]
+            direction LR
+
+            router["ASUS RT-AX88U<br/>Router"]:::network
+            cisco["Cisco CBS110-24T<br/>1 Gbps switch"]:::network
+            trendnet["TRENDnet 9-port<br/>2.5 Gbps switch"]:::network
+
+            router ==> cisco
+            router ==> trendnet
+        end
+
+        subgraph atlas["Atlas Kubernetes cluster"]
+            direction TB
+
+            subgraph control_plane["High-availability control plane, ARM64"]
+                direction LR
+
+                subgraph titan_0a["titan-0a"]
+                    direction TB
+                    t0a_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
+                    t0a_storage["Local<br/>500 GiB boot SSD"]:::localStorage
+                    t0a_host --- t0a_storage
+                end
+
+                subgraph titan_0b["titan-0b"]
+                    direction TB
+                    t0b_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
+                    t0b_storage["Local<br/>500 GiB boot SSD"]:::localStorage
+                    t0b_host --- t0b_storage
+                end
+
+                subgraph titan_0c["titan-0c"]
+                    direction TB
+                    t0c_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
+                    t0c_storage["Local<br/>500 GiB boot SSD"]:::localStorage
+                    t0c_host --- t0c_storage
+                end
+
+                t0a_host --- t0b_host
+                t0b_host --- t0c_host
+            end
+
+            subgraph workers["Workers"]
+                direction TB
+
+                subgraph pi5_workers["Newer workers, ARM64 Raspberry Pi 5"]
+                    direction TB
+
+                    subgraph pi5_row_a[" "]
+                        direction LR
+
+                        subgraph titan_04["titan-04"]
+                            direction TB
+                            t04_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
+                            t04_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
+                            t04_host --- t04_storage
+                        end
+
+                        subgraph titan_05["titan-05"]
+                            direction TB
+                            t05_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
+                            t05_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
+                            t05_host --- t05_storage
+                        end
+
+                        subgraph titan_06["titan-06"]
+                            direction TB
+                            t06_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
+                            t06_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
+                            t06_host --- t06_storage
+                        end
+                    end
+
+                    subgraph pi5_row_b[" "]
+                        direction LR
+
+                        subgraph titan_07["titan-07"]
+                            direction TB
+                            t07_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
+                            t07_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
+                            t07_host --- t07_storage
+                        end
+
+                        subgraph titan_08["titan-08"]
+                            direction TB
+                            t08_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
+                            t08_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
+                            t08_host --- t08_storage
+                        end
+
+                        subgraph titan_11["titan-11"]
+                            direction TB
+                            t11_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
+                            t11_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
+                            t11_host --- t11_storage
+                        end
+                    end
+                end
+
+                subgraph pi4_workers["Older workers, ARM64 Raspberry Pi 4"]
+                    direction TB
+
+                    subgraph pi4_row_a[" "]
+                        direction LR
+
+                        subgraph titan_12["titan-12"]
+                            direction TB
+                            t12_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
+                            t12_root["Local<br/>SD root"]:::localStorage
+                            t12_host --- t12_root
+                        end
+
+                        subgraph titan_13["titan-13"]
+                            direction TB
+                            t13_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
+                            t13_root["Local<br/>SD root"]:::localStorage
+                            t13_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
+                            t13_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
+                            t13_host --- t13_root
+                            t13_host --- t13_astreae
+                            t13_host --- t13_asteria
+                        end
+
+                        subgraph titan_14["titan-14"]
+                            direction TB
+                            t14_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
+                            t14_root["Local<br/>SD root"]:::localStorage
+                            t14_host --- t14_root
+                        end
+
+                        subgraph titan_15["titan-15"]
+                            direction TB
+                            t15_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
+                            t15_root["Local<br/>SD root"]:::localStorage
+                            t15_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
+                            t15_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
+                            t15_host --- t15_root
+                            t15_host --- t15_astreae
+                            t15_host --- t15_asteria
+                        end
+                    end
+
+                    subgraph pi4_row_b[" "]
+                        direction LR
+
+                        subgraph titan_17["titan-17"]
+                            direction TB
+                            t17_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
+                            t17_root["Local<br/>SD root"]:::localStorage
+                            t17_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
+                            t17_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
+                            t17_host --- t17_root
+                            t17_host --- t17_astreae
+                            t17_host --- t17_asteria
+                        end
+
+                        subgraph titan_18["titan-18"]
+                            direction TB
+                            t18_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
+                            t18_root["Local<br/>SD root"]:::localStorage
+                            t18_host --- t18_root
+                        end
+
+                        subgraph titan_19["titan-19"]
+                            direction TB
+                            t19_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
+                            t19_root["Local<br/>SD root"]:::localStorage
+                            t19_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
+                            t19_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
+                            t19_host --- t19_root
+                            t19_host --- t19_astreae
+                            t19_host --- t19_asteria
+                        end
+                    end
+                end
+
+                subgraph accelerator_workers["Accelerators and high-capacity workers"]
+                    direction TB
+
+                    subgraph jetson_workers["ARM64 NVIDIA Jetson"]
+                        direction LR
+
+                        subgraph titan_20["titan-20"]
+                            direction TB
+                            t20_host["Jetson Xavier NX<br/>16 GiB RAM<br/>Ubuntu 20.04<br/>Accelerator"]:::accelerator
+                            t20_gpu["GPU<br/>NVIDIA Xavier"]:::gpu
+                            t20_storage["Local<br/>256 GiB disk"]:::localStorage
+                            t20_host --- t20_gpu
+                            t20_host --- t20_storage
+                        end
+
+                        subgraph titan_21["titan-21"]
+                            direction TB
+                            t21_host["Jetson Xavier NX<br/>16 GiB RAM<br/>Ubuntu 20.04<br/>Accelerator"]:::accelerator
+                            t21_gpu["GPU<br/>NVIDIA Xavier"]:::gpu
+                            t21_storage["Local<br/>256 GiB disk"]:::localStorage
+                            t21_host --- t21_gpu
+                            t21_host --- t21_storage
+                        end
+                    end
+
+                    subgraph x86_workers["x86_64 systems"]
+                        direction LR
+
+                        subgraph titan_22["titan-22"]
+                            direction TB
+                            t22_host["X830 mini PC<br/>Core i9-12900H<br/>32 GiB RAM<br/>Debian 13"]:::accelerator
+                            t22_gpu["GPU<br/>NVIDIA RTX 3050 Ti"]:::gpu
+                            t22_igpu["Intel Iris Xe<br/>Not pooled"]:::localGpu
+                            t22_storage["Local<br/>1 TiB disk"]:::localStorage
+                            t22_host --- t22_gpu
+                            t22_host --- t22_igpu
+                            t22_host --- t22_storage
+                        end
+
+                        subgraph titan_23["titan-23, Oceanus"]
+                            direction TB
+                            t23_host["EPYC 74F3<br/>24 cores, 48 threads<br/>256 GiB RAM<br/>Debian 13"]:::server
+                            t23_gpu["ASPEED graphics<br/>Not pooled"]:::localGpu
+                            t23_root["Local<br/>1 TiB disk"]:::localStorage
+                            t23_data["Local<br/>4 TiB Veles<br/>4 TiB SUI"]:::localStorage
+                            t23_host --- t23_gpu
+                            t23_host --- t23_root
+                            t23_host --- t23_data
+                        end
+
+                        subgraph titan_24["titan-24, Tethys"]
+                            direction TB
+                            t24_host["Ryzen 9 3900X<br/>24 threads, 64 GiB RAM<br/>Debian 13"]:::accelerator
+                            t24_gpu["GPU<br/>NVIDIA RTX 3080"]:::gpu
+                            t24_root["Local<br/>500 GiB disk"]:::localStorage
+                            t24_data["Local<br/>500 GiB home<br/>500 GiB temporary"]:::localStorage
+                            t24_host --- t24_gpu
+                            t24_host --- t24_root
+                            t24_host --- t24_data
+                        end
+                    end
+                end
+
+                pi5_workers ~~~ pi4_workers
+                pi4_workers ~~~ accelerator_workers
+            end
+
+            subgraph virtual_resources["Virtual resource pools"]
+                direction LR
+
+                subgraph cluster_storage["Longhorn storage classes"]
+                    direction LR
+
+                    astreae["Astreae<br/>4 x 8 TiB<br/>32 TiB"]:::sharedPool
+                    asteria["Asteria<br/>4 x 12 TiB<br/>48 TiB"]:::sharedPool
+                end
+
+                subgraph cluster_gpus["Kubernetes GPU pool"]
+                    direction LR
+
+                    arm_gpu_pool["NVIDIA ARM64<br/>titan-20 and titan-21"]:::gpuPool
+                    x86_gpu_pool["NVIDIA x86_64<br/>titan-22 and titan-24"]:::gpuPool
+                end
+            end
+
+            subgraph cluster_telemetry["Cluster telemetry"]
+                direction LR
+
+                typhon["Typhon<br/>Kubernetes workload"]:::clusterService
+                victoria["VictoriaMetrics"]:::telemetry
+                grafana["Grafana"]:::telemetry
+
+                typhon ==>|environment metrics| victoria
+                victoria --> grafana
+            end
+        end
+
+        subgraph outside_cluster["Lab systems outside the Atlas cluster"]
+            direction TB
+
+            subgraph support_hosts["Support and host-level services"]
+                direction LR
+
+                subgraph titan_db["titan-db"]
+                    direction TB
+
+                    tdb_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::support
+                    tdb_storage["Local<br/>500 GiB boot SSD"]:::localStorage
+                    tdb_postgres["PostgreSQL<br/>HA control-plane database"]:::hostSoftware
+                    ananke_db["NUT and Ananke<br/>Host services<br/>Automatic recovery"]:::hostService
+
+                    tdb_host --- tdb_storage
+                    tdb_host --- tdb_postgres
+                    tdb_host --- ananke_db
+                end
+
+                subgraph titan_jh["titan-jh"]
+                    direction TB
+
+                    tjh_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Arch Linux ARM<br/>Jump host"]:::support
+                    tjh_storage["Local<br/>256 GiB boot SSD"]:::localStorage
+                    lesavka["Lesavka<br/>Remote control for Tethys"]:::hostService
+
+                    tjh_host --- tjh_storage
+                    tjh_host --- lesavka
+                end
+
+                ananke_t24["NUT and Ananke on titan-24<br/>Host services outside Kubernetes<br/>UPS telemetry and safe shutdown"]:::hostService
+            end
+
+            subgraph failed_hardware["Boards awaiting replacement"]
+                direction LR
+
+                titan_09["titan-09<br/>Raspberry Pi 5"]:::failed
+                titan_10["titan-10<br/>Raspberry Pi 5"]:::failed
+                titan_16["titan-16<br/>Raspberry Pi 4"]:::failed
+            end
+        end
+
+        subgraph facilities["Power and enclosure"]
+            direction LR
+
+            subgraph protected_power["UPS protection"]
+                direction LR
+
+                subgraph pyrphoros_power["Pyrphoros"]
+                    direction TB
+
+                    pyrphoros["CyberPower<br/>850 VA UPS"]:::power
+                    pyrphoros_loads["Protected loads<br/>Router and Cisco switch<br/>Control plane and Pi workers<br/>Jetsons, titan-22, titan-db<br/>Cooling equipment"]:::powerLoad
+                    pyrphoros --> pyrphoros_loads
+                end
+
+                subgraph statera_power["Statera"]
+                    direction TB
+
+                    statera["CyberPower<br/>1500PFCLCD"]:::power
+                    statera_loads["Protected loads<br/>TRENDnet switch<br/>titan-23, titan-24, titan-jh"]:::powerLoad
+                    statera --> statera_loads
+                end
+            end
+
+            subgraph cooling["Enclosure cooling"]
+                direction TB
+
+                enclosure["Filtered equipment tent<br/>AC-cooled intake<br/>Controlled exhaust"]:::environment
+                ac_controller["AC Infinity controller<br/>Temperature<br/>Humidity<br/>Pressure"]:::environment
+                fans["Controlled intake<br/>and exhaust fans"]:::environment
+
+                enclosure --- ac_controller
+                ac_controller --> fans
+            end
+        end
+
+        cisco ==> control_plane
+        cisco ==> pi5_workers
+        cisco ==> pi4_workers
+        cisco ==> jetson_workers
+        cisco ==> titan_db
+        trendnet ==> titan_jh
+
+        trendnet ==> x86_workers
+
+        t0b_host -.->|HA PostgreSQL| tdb_postgres
+        tjh_host -.->|administration| control_plane
+        tjh_host -.->|administration| workers
+        lesavka -.->|remote control| t24_host
+
+        t13_astreae ==> astreae
+        t15_astreae ==> astreae
+        t17_astreae ==> astreae
+        t19_astreae ==> astreae
+        t13_asteria ==> asteria
+        t15_asteria ==> asteria
+        t17_asteria ==> asteria
+        t19_asteria ==> asteria
+
+        t20_gpu ==> arm_gpu_pool
+        t21_gpu ==> arm_gpu_pool
+        t22_gpu ==> x86_gpu_pool
+        t24_gpu ==> x86_gpu_pool
+
+        t24_host -.->|same physical host| ananke_t24
+        pyrphoros -->|NUT| ananke_db
+        statera -->|NUT| ananke_t24
+        ananke_db -.->|power metrics| victoria
+        ananke_t24 -.->|power metrics| victoria
+        ananke_db -.->|database and cluster recovery| t0a_host
+        ananke_t24 -.->|low-battery shutdown| control_plane
+
+        ac_controller --> ac_cloud
+        ac_cloud -.->|environment data| typhon
+    end
+
+    internet --> router
+
+    classDef external fill:#292d33,stroke:#a0a8b3,color:#ffffff,stroke-width:2px
+    classDef network fill:#17324a,stroke:#65b5e8,color:#ffffff,stroke-width:2px
+    classDef control fill:#163e48,stroke:#65d2df,color:#ffffff,stroke-width:2px
+    classDef pi5 fill:#173d2a,stroke:#68d391,color:#ffffff,stroke-width:2px
+    classDef pi4 fill:#293b23,stroke:#9acb70,color:#ffffff,stroke-width:2px
+    classDef accelerator fill:#382b50,stroke:#b49aef,color:#ffffff,stroke-width:2px
+    classDef server fill:#3c3046,stroke:#c4a4dc,color:#ffffff,stroke-width:2px
+    classDef support fill:#293845,stroke:#9fc5dc,color:#ffffff,stroke-width:2px
+    classDef localStorage fill:#1d3040,stroke:#6f9fbd,color:#ffffff
+    classDef sharedStorage fill:#493b18,stroke:#efc65b,color:#ffffff,stroke-width:2px
+    classDef sharedPool fill:#594817,stroke:#ffd166,color:#ffffff,stroke-width:3px
+    classDef gpu fill:#482c5f,stroke:#d0a6ff,color:#ffffff,stroke-width:2px
+    classDef localGpu fill:#292d35,stroke:#9ba4b2,color:#ffffff
+    classDef gpuPool fill:#573476,stroke:#d8b4fe,color:#ffffff,stroke-width:3px
+    classDef clusterService fill:#153c38,stroke:#5fc9bc,color:#ffffff,stroke-width:2px
+    classDef telemetry fill:#203651,stroke:#67a9e9,color:#ffffff,stroke-width:2px
+    classDef hostSoftware fill:#243a45,stroke:#73b8d0,color:#ffffff,stroke-width:2px
+    classDef hostService fill:#48351c,stroke:#e3ae59,color:#ffffff,stroke-width:2px
+    classDef failed fill:#3d1f25,stroke:#f06b78,color:#ffffff,stroke-width:2px,stroke-dasharray:8 6
+    classDef power fill:#49301c,stroke:#ec9f55,color:#ffffff,stroke-width:2px
+    classDef powerLoad fill:#3b2d20,stroke:#c98e55,color:#ffffff
+    classDef environment fill:#153d3a,stroke:#58c9bd,color:#ffffff,stroke-width:2px
+    classDef legend1g fill:#0b1726,stroke:#4ea1ff,color:#ffffff,stroke-width:3px
+    classDef legend25g fill:#082027,stroke:#22d3ee,color:#ffffff,stroke-width:3px
+    classDef legendExternal fill:#17162c,stroke:#818cf8,color:#ffffff,stroke-width:3px
+    classDef legendLocal fill:#171b20,stroke:#a8b3c2,color:#ffffff,stroke-width:3px
+    classDef legendShared fill:#2f270d,stroke:#f6c453,color:#ffffff,stroke-width:3px
+    classDef legendGpu fill:#271536,stroke:#c084fc,color:#ffffff,stroke-width:3px
+    classDef legendPower fill:#301b0c,stroke:#fb923c,color:#ffffff,stroke-width:3px
+    classDef legendTelemetry fill:#0b2917,stroke:#4ade80,color:#ffffff,stroke-width:3px
+    classDef legendControl fill:#32132a,stroke:#f472b6,color:#ffffff,stroke-width:3px
+    classDef legendSafety fill:#321217,stroke:#fb7185,color:#ffffff,stroke-width:3px
+
+    style lab fill:#020202,stroke:#89919c,stroke-width:4px
+    style link_legend fill:#050505,stroke:#89919c,stroke-width:2px
+    style legend_network fill:#03070b,stroke:#4ea1ff,stroke-width:1px
+    style legend_resources fill:#080706,stroke:#f6c453,stroke-width:1px
+    style legend_operations fill:#080506,stroke:#f472b6,stroke-width:1px
+    style network fill:#03070b,stroke:#65b5e8,stroke-width:3px
+    style atlas fill:#020608,stroke:#65d2df,stroke-width:4px
+    style control_plane fill:#061014,stroke:#65d2df,stroke-width:3px
+    style workers fill:#030603,stroke:#72b879,stroke-width:3px
+    style pi5_workers fill:#050d08,stroke:#68d391,stroke-width:2px
+    style pi5_row_a fill:transparent,stroke:transparent
+    style pi5_row_b fill:transparent,stroke:transparent
+    style pi4_workers fill:#080d05,stroke:#9acb70,stroke-width:2px
+    style pi4_row_a fill:transparent,stroke:transparent
+    style pi4_row_b fill:transparent,stroke:transparent
+    style accelerator_workers fill:#09060d,stroke:#b49aef,stroke-width:2px
+    style jetson_workers fill:#0b0710,stroke:#b49aef,stroke-width:2px
+    style x86_workers fill:#0b0710,stroke:#c4a4dc,stroke-width:2px
+    style virtual_resources fill:#08080b,stroke:#a8a2b3,stroke-width:3px
+    style cluster_storage fill:#100d05,stroke:#ffd166,stroke-width:2px
+    style cluster_gpus fill:#0d0813,stroke:#d8b4fe,stroke-width:2px
+    style cluster_telemetry fill:#04100f,stroke:#5fc9bc,stroke-width:2px
+    style outside_cluster fill:#07090b,stroke:#9fc5dc,stroke-width:3px
+    style support_hosts fill:#07090b,stroke:#9fc5dc,stroke-width:2px
+    style failed_hardware fill:#0d0708,stroke:#f06b78,stroke-width:2px,stroke-dasharray:8 6
+    style facilities fill:#070604,stroke:#ec9f55,stroke-width:3px
+    style protected_power fill:#0e0905,stroke:#ec9f55,stroke-width:2px
+    style pyrphoros_power fill:#100b06,stroke:#d99a58,stroke-width:2px
+    style statera_power fill:#100b06,stroke:#d99a58,stroke-width:2px
+    style cooling fill:#04100f,stroke:#58c9bd,stroke-width:2px
+
+    linkStyle 0,54,55,56,57,58,59 stroke:#4ea1ff,color:#4ea1ff,stroke-width:3px
+    linkStyle 1,60 stroke:#22d3ee,color:#22d3ee,stroke-width:4px
+    linkStyle 86 stroke:#818cf8,color:#818cf8,stroke-width:3px
+    linkStyle 2,3,4,7,8,9,10,11,12,13,14,17,18,21,24,25,29,31,34,36,37,39,40,45,48 stroke:#a8b3c2,color:#a8b3c2,stroke-width:2px
+    linkStyle 15,16,19,20,22,23,26,27,65,66,67,68,69,70,71,72 stroke:#f6c453,color:#f6c453,stroke-width:3px
+    linkStyle 28,30,32,33,35,38,73,74,75,76 stroke:#c084fc,color:#c084fc,stroke-width:3px
+    linkStyle 50,51,78,79 stroke:#fb923c,color:#fb923c,stroke-width:3px
+    linkStyle 43,44,52,53,80,81,84,85 stroke:#4ade80,color:#4ade80,stroke-width:3px
+    linkStyle 5,6,46,47,49,61,62,63,64,77,82 stroke:#f472b6,color:#f472b6,stroke-width:2px
+    linkStyle 83 stroke:#fb7185,color:#fb7185,stroke-width:4px

mermaid/ServiceAccessArchitecture.mmd

@@ -0,0 +1,705 @@
+%% Atlas service access architecture.
+%% Domains, identity profiles, entry services, and selected internal dependencies.
+%% Node placement and short labels carry the detail.
+%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 30, "rankSpacing": 65, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "LINEAR_SEGMENTS", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000", "lineColor": "#8b95a5", "primaryTextColor": "#ffffff", "clusterBkg": "#03040d", "clusterBorder": "#6d7485"}, "themeCSS": "& { background-color: #000000 !important; }"}}%%
+flowchart TB
+    subgraph system["Atlas Service Access Architecture"]
+        direction TB
+
+        subgraph legend["Legend"]
+            direction LR
+
+            subgraph node_key["Nodes"]
+                direction LR
+                key_domain["DNS name"]:::domain
+                key_access["OIDC access"]:::access
+                key_oauth["Access proxy"]:::oauth
+                key_directory["LDAP or synced account"]:::directory
+                key_entry["Entry service"]:::user
+                key_internal["Internal service"]:::support
+                key_data["Data service"]:::data
+                key_job["Job or worker"]:::ephemeral
+                key_external["Outside Kubernetes"]:::external
+            end
+
+            subgraph link_key["Links"]
+                direction LR
+                key_ingress["Blue<br/>Domain to identity"]:::linkIngress
+                key_service["Green<br/>Identity to service"]:::linkService
+                key_direct["Gold<br/>Direct access"]:::linkDirect
+                key_auth["Purple<br/>Identity control"]:::linkAuth
+                key_data_link["Violet<br/>Data flow"]:::linkData
+                key_telemetry["Teal<br/>Telemetry"]:::linkTelemetry
+                key_control["Orange<br/>Delivery and control"]:::linkControl
+                key_internal_link["Gray<br/>Internal flow"]:::linkInternal
+            end
+        end
+
+        subgraph application_band["Applications and Communications"]
+            direction LR
+
+            subgraph application_domains["Domains"]
+                direction TB
+                portal_domain["bstein.dev"]:::domain
+                portal_chat_domain["chat.ai.bstein.dev"]:::domain
+                cassandra_domain["cassandra.bstein.dev"]:::domain
+                element_domain["live.bstein.dev"]:::domain
+                matrix_domain["matrix.live.bstein.dev"]:::domain
+                call_domain["call.live.bstein.dev"]:::domain
+                livekit_domain["kit.live.bstein.dev"]:::domain
+                turn_domain["turn.live.bstein.dev"]:::domain
+            end
+
+            subgraph application_access["Keycloak Access"]
+                direction LR
+
+                subgraph application_atlas_access["atlas realm"]
+                    direction TB
+                    portal_oidc_access["Portal API<br/>OIDC token"]:::access
+                    portal_chat_oidc_access["Portal Chat<br/>OIDC"]:::access
+                    element_oidc_access["Element Web<br/>OIDC through MAS"]:::access
+                    matrix_oidc_access["Matrix<br/>OIDC through MAS"]:::access
+                end
+
+                subgraph application_cassandra_access["cassandra realm"]
+                    direction TB
+                    cassandra_oidc_access["Cassandra<br/>OIDC"]:::access
+                end
+            end
+
+            subgraph application_services["Cluster Services"]
+                direction TB
+
+                application_delivery["Flux delivery<br/>Application workloads"]:::control
+                application_state_clients["Shared PostgreSQL<br/>Portal and Matrix"]:::control
+
+                subgraph website_ns["bstein-dev-home"]
+                    direction LR
+                    website_frontend["Portal"]:::user
+                    website_chat_gateway["Chat gateway"]:::user
+                    website_backend["Backend"]:::support
+
+                    website_frontend --> website_backend
+                    website_chat_gateway --> website_backend
+                end
+
+                subgraph cassandra_ns["cassandra"]
+                    direction LR
+                    cassandra_frontend["Cassandra UI"]:::user
+                    cassandra_backend["API"]:::support
+                    cassandra_postgres["PostgreSQL"]:::data
+                    cassandra_generator["Generator"]:::ephemeral
+                    cassandra_simulation["Simulation jobs"]:::ephemeral
+                    cassandra_retention["Retention job"]:::ephemeral
+                    cassandra_artifacts["Artifacts"]:::data
+
+                    cassandra_frontend --> cassandra_backend
+                    cassandra_backend --> cassandra_postgres
+                    cassandra_generator --> cassandra_backend
+                    cassandra_simulation --> cassandra_backend
+                    cassandra_simulation --> cassandra_artifacts
+                    cassandra_retention --> cassandra_artifacts
+                end
+
+                subgraph comms_ns["comms"]
+                    direction LR
+                    comms_element_web["Element Web"]:::user
+                    comms_element_call["Element Call"]:::user
+                    comms_wellknown["Matrix discovery"]:::support
+                    comms_guest["Guest registration"]:::support
+                    comms_mas["Matrix auth service"]:::support
+                    comms_synapse["Synapse"]:::support
+                    comms_replication["Synapse replication"]:::support
+                    comms_redis["Redis"]:::data
+                    comms_livekit_token["LiveKit token service"]:::support
+                    comms_livekit["LiveKit"]:::support
+                    comms_coturn["Coturn"]:::support
+                    comms_atlasbot["Atlasbot"]:::support
+
+                    comms_element_web --> comms_mas
+                    comms_element_web --> comms_synapse
+                    comms_element_call --> comms_livekit_token
+                    comms_livekit_token --> comms_livekit
+                    comms_mas --> comms_synapse
+                    comms_synapse --> comms_replication
+                    comms_synapse --> comms_redis
+                    comms_guest --> comms_synapse
+                    comms_atlasbot --> comms_synapse
+                end
+            end
+
+            portal_domain portal_identity@-->|HTTPS| portal_oidc_access
+            portal_oidc_access portal_entry@--> website_frontend
+            portal_chat_domain portal_chat_identity@-->|HTTPS| portal_chat_oidc_access
+            portal_chat_oidc_access portal_chat_entry@--> website_chat_gateway
+            cassandra_domain cassandra_identity@-->|HTTPS| cassandra_oidc_access
+            cassandra_oidc_access cassandra_entry@--> cassandra_frontend
+            element_domain element_identity@-->|HTTPS| element_oidc_access
+            element_oidc_access element_entry@--> comms_element_web
+            matrix_domain matrix_identity@-->|HTTPS| matrix_oidc_access
+            matrix_oidc_access matrix_entry@--> comms_synapse
+            matrix_domain matrix_api@-->|Matrix API| comms_synapse
+            call_domain call_entry@-->|HTTPS| comms_element_call
+            livekit_domain livekit_entry@-->|HTTPS and WebRTC| comms_livekit
+            turn_domain turn_entry@-->|TURN| comms_coturn
+        end
+
+        subgraph personal_band["Personal Services"]
+            direction LR
+
+            subgraph personal_domains["Domains"]
+                direction LR
+
+                subgraph personal_domains_a["Accounts and Health"]
+                    direction TB
+                    budget_domain["budget.bstein.dev"]:::domain
+                    firefly_domain["money.bstein.dev"]:::domain
+                    wolf_domain["wolf.bstein.dev"]:::domain
+                    moonlight_domain["moonlight.bstein.dev"]:::domain
+                    wger_domain["health.bstein.dev"]:::domain
+                    chat_domain["chat.bstein.dev"]:::domain
+                end
+
+                subgraph personal_domains_b["Media and Productivity"]
+                    direction TB
+                    jellyfin_domain["stream.bstein.dev"]:::domain
+                    pegasus_domain["pegasus.bstein.dev"]:::domain
+                    mailu_domain["mail.bstein.dev"]:::domain
+                    cloud_domain["cloud.bstein.dev"]:::domain
+                    office_domain["office.bstein.dev"]:::domain
+                    outline_domain["notes.bstein.dev"]:::domain
+                    planka_domain["tasks.bstein.dev"]:::domain
+                    vaultwarden_domain["vault.bstein.dev"]:::domain
+                    monero_domain["monero.bstein.dev"]:::domain
+                end
+            end
+
+            subgraph personal_access["atlas realm and directory access"]
+                direction LR
+
+                subgraph personal_access_a["Accounts"]
+                    direction TB
+                    budget_oidc_access["Actual Budget<br/>OpenID"]:::access
+                    firefly_synced_access["Firefly<br/>Synced account"]:::directory
+                    wolf_oauth_access["Wolf<br/>Access proxy"]:::oauth
+                    wger_synced_access["Wger<br/>Synced account"]:::directory
+                    chat_oidc_access["Hermes Chat<br/>OIDC"]:::access
+                end
+
+                subgraph personal_access_b["Media and Productivity"]
+                    direction TB
+                    jellyfin_ldap_access["Jellyfin<br/>LDAP"]:::directory
+                    cloud_oidc_access["Nextcloud<br/>OIDC"]:::access
+                    outline_oidc_access["Outline<br/>OIDC"]:::access
+                    planka_oidc_access["Planka<br/>OIDC"]:::access
+                end
+            end
+
+            subgraph personal_services["Cluster Services"]
+                direction LR
+
+                personal_delivery["Flux delivery<br/>Personal workloads"]:::control
+                personal_state_clients["Shared PostgreSQL<br/>Personal services"]:::control
+
+                subgraph personal_services_a["Accounts, Health, and Media"]
+                    direction TB
+
+                    subgraph finance_ns["finance"]
+                        direction LR
+                        finance_budget["Actual Budget"]:::user
+                        finance_firefly["Firefly"]:::user
+                        finance_sync["Account sync"]:::ephemeral
+                        finance_import["Import job"]:::ephemeral
+                        finance_sync --> finance_firefly
+                        finance_import --> finance_firefly
+                    end
+
+                    subgraph game_ns["game-stream"]
+                        direction LR
+                        game_proxy["Wolf access proxy"]:::oauth
+                        game_manager["Wolf manager"]:::support
+                        game_api["Wolf API"]:::support
+                        game_wolf["Wolf"]:::support
+                        game_gatekeeper["Gatekeeper"]:::support
+                        game_moonlight["Moonlight"]:::user
+                        game_proxy --> game_manager
+                        game_manager --> game_api
+                        game_api --> game_wolf
+                        game_gatekeeper --> game_wolf
+                        game_moonlight --> game_wolf
+                    end
+
+                    subgraph health_ns["health"]
+                        direction LR
+                        health_wger["Wger"]:::user
+                        health_sync["Account sync"]:::ephemeral
+                        health_admin["Admin sync"]:::ephemeral
+                        health_sync --> health_wger
+                        health_admin --> health_wger
+                    end
+
+                    subgraph hermes_chat_ns["hermes-chat"]
+                        direction LR
+                        hermes_chat["Hermes Chat"]:::user
+                    end
+
+                    subgraph jellyfin_ns["jellyfin"]
+                        direction LR
+                        streaming_pegasus["Pegasus"]:::user
+                        streaming_jellyfin["Jellyfin"]:::user
+                        streaming_pegasus --> streaming_jellyfin
+                    end
+                end
+
+                subgraph personal_services_b["Mail and Productivity"]
+                    direction TB
+
+                    subgraph mail_ns["mailu-mailserver"]
+                        direction LR
+                        mail_front["Mail web"]:::user
+                        mail_protocols["Mail protocols"]:::user
+                        mail_admin["Admin"]:::support
+                        mail_dovecot["Dovecot"]:::support
+                        mail_postfix["Postfix"]:::support
+                        mail_rspamd["Rspamd"]:::support
+                        mail_clamav["ClamAV"]:::support
+                        mail_extractors["Tika and oletools"]:::support
+                        mail_redis["Redis"]:::data
+                        mail_front --> mail_admin
+                        mail_protocols --> mail_dovecot
+                        mail_protocols --> mail_postfix
+                        mail_postfix --> mail_rspamd
+                        mail_rspamd --> mail_clamav
+                        mail_rspamd --> mail_extractors
+                        mail_admin --> mail_redis
+                    end
+
+                    subgraph cloud_ns["nextcloud"]
+                        direction LR
+                        cloud_nextcloud["Nextcloud"]:::user
+                        cloud_collabora["Collabora"]:::support
+                        cloud_mail_sync["Mail sync"]:::ephemeral
+                        cloud_maintenance["Maintenance job"]:::ephemeral
+                        cloud_nextcloud --> cloud_collabora
+                        cloud_mail_sync --> cloud_nextcloud
+                        cloud_maintenance --> cloud_nextcloud
+                    end
+
+                    subgraph notes_tasks_ns["Notes and Tasks"]
+                        direction LR
+                        outline_service["Outline"]:::user
+                        outline_redis["Redis"]:::data
+                        planka_service["Planka"]:::user
+                        outline_service --> outline_redis
+                    end
+
+                    subgraph vaultwarden_ns["vaultwarden"]
+                        direction LR
+                        vaultwarden_service["Vaultwarden"]:::user
+                    end
+
+                    subgraph crypto_ns["crypto"]
+                        direction LR
+                        crypto_wallet["Wallet RPC"]:::support
+                        crypto_p2pool["P2Pool"]:::support
+                        crypto_monerod["Monerod"]:::user
+                        crypto_wallet --> crypto_monerod
+                        crypto_p2pool --> crypto_monerod
+                    end
+                end
+            end
+
+            budget_domain budget_identity@-->|HTTPS| budget_oidc_access
+            budget_oidc_access budget_entry@--> finance_budget
+            firefly_domain firefly_identity@-->|HTTPS| firefly_synced_access
+            firefly_synced_access firefly_entry@--> finance_firefly
+            wolf_domain wolf_identity@-->|HTTPS| wolf_oauth_access
+            wolf_oauth_access wolf_entry@--> game_manager
+            moonlight_domain moonlight_entry@-->|Streaming| game_moonlight
+            wger_domain wger_identity@-->|HTTPS| wger_synced_access
+            wger_synced_access wger_entry@--> health_wger
+            chat_domain chat_identity@-->|HTTPS| chat_oidc_access
+            chat_oidc_access chat_entry@--> hermes_chat
+            jellyfin_domain jellyfin_identity@-->|HTTPS| jellyfin_ldap_access
+            jellyfin_ldap_access jellyfin_entry@--> streaming_jellyfin
+            pegasus_domain pegasus_entry@-->|HTTPS| streaming_pegasus
+            mailu_domain mail_web_entry@-->|HTTPS| mail_front
+            mailu_domain mail_protocol_entry@-->|Mail protocols| mail_protocols
+            cloud_domain cloud_identity@-->|HTTPS| cloud_oidc_access
+            cloud_oidc_access cloud_entry@--> cloud_nextcloud
+            office_domain office_entry@-->|WOPI| cloud_collabora
+            outline_domain outline_identity@-->|HTTPS| outline_oidc_access
+            outline_oidc_access outline_entry@--> outline_service
+            planka_domain planka_identity@-->|HTTPS| planka_oidc_access
+            planka_oidc_access planka_entry@--> planka_service
+            vaultwarden_domain vaultwarden_entry@-->|HTTPS and local auth| vaultwarden_service
+            monero_domain monero_entry@-->|RPC| crypto_monerod
+        end
+
+        subgraph platform_band["Platform Services"]
+            direction LR
+
+            subgraph platform_domains["Domains"]
+                direction LR
+
+                subgraph platform_domains_a["Delivery"]
+                    direction TB
+                    gitops_domain["cd.bstein.dev"]:::domain
+                    gitea_domain["scm.bstein.dev"]:::domain
+                    harbor_domain["registry.bstein.dev"]:::domain
+                    hermes_domain["agent.bstein.dev"]:::domain
+                    jenkins_domain["ci.bstein.dev"]:::domain
+                end
+
+                subgraph platform_domains_b["Operations"]
+                    direction TB
+                    logs_domain["logs.bstein.dev"]:::domain
+                    longhorn_domain["longhorn.bstein.dev"]:::domain
+                    grafana_domain["metrics.bstein.dev"]:::domain
+                    alerts_domain["alerts.bstein.dev"]:::domain
+                    quality_domain["quality.bstein.dev"]:::domain
+                    keycloak_domain["sso.bstein.dev"]:::domain
+                    vault_domain["secret.bstein.dev"]:::domain
+                end
+            end
+
+            subgraph platform_access["Keycloak Access"]
+                direction LR
+
+                subgraph platform_access_a["atlas realm delivery"]
+                    direction TB
+                    gitops_oidc_access["Weave GitOps<br/>OIDC"]:::access
+                    gitea_atlas_oidc_access["Gitea<br/>atlas realm"]:::access
+                    harbor_oidc_access["Harbor UI<br/>OIDC"]:::access
+                    hermes_oidc_access["Hermes Agent<br/>OIDC"]:::access
+                    jenkins_oidc_access["Jenkins<br/>OIDC"]:::access
+                end
+
+                subgraph platform_access_b["atlas realm operations"]
+                    direction TB
+                    logs_oauth_access["OpenSearch Dashboards<br/>Access proxy"]:::oauth
+                    longhorn_oauth_access["Longhorn<br/>Admin proxy"]:::oauth
+                    grafana_oidc_access["Grafana<br/>OIDC"]:::access
+                    quality_oauth_access["SonarQube<br/>Admin and developer proxy"]:::oauth
+                    keycloak_admin_access["Keycloak console<br/>Realm admin"]:::access
+                    vault_oidc_access["Vault<br/>Admin OIDC"]:::access
+                end
+
+                subgraph platform_access_c["cassandra realm client"]
+                    direction TB
+                    gitea_cassandra_oidc_access["Gitea<br/>OIDC"]:::access
+                end
+            end
+
+            subgraph platform_services["Cluster Services"]
+                direction LR
+
+                platform_delivery["Flux delivery<br/>Platform workloads"]:::control
+                platform_state_clients["Shared PostgreSQL<br/>Platform services"]:::control
+
+                subgraph delivery_services["Delivery and Automation"]
+                    direction TB
+
+                    subgraph flux_ns["flux-system"]
+                        direction LR
+                        flux_source["Source controller"]:::support
+                        flux_kustomize["Kustomize controller"]:::support
+                        flux_helm["Helm controller"]:::support
+                        flux_image_reflector["Image reflector"]:::support
+                        flux_image_automation["Image automation"]:::support
+                        flux_notification["Notification controller"]:::support
+                        flux_webhook["Webhook receiver"]:::support
+                        flux_weave["Weave GitOps"]:::user
+                        flux_source --> flux_kustomize
+                        flux_source --> flux_helm
+                        flux_image_reflector --> flux_image_automation
+                        flux_webhook --> flux_notification
+                        flux_weave --> flux_source
+                    end
+
+                    subgraph gitea_ns["gitea"]
+                        direction LR
+                        gitea_http["Gitea web"]:::user
+                        gitea_ssh["Gitea SSH"]:::support
+                    end
+
+                    subgraph harbor_ns["harbor"]
+                        direction LR
+                        harbor_portal["Harbor portal"]:::user
+                        harbor_core["Harbor core"]:::support
+                        harbor_jobservice["Job service"]:::support
+                        harbor_registry["OCI registry"]:::data
+                        harbor_redis["Redis"]:::data
+                        harbor_portal --> harbor_core
+                        harbor_jobservice --> harbor_core
+                        harbor_core --> harbor_registry
+                        harbor_core --> harbor_redis
+                    end
+
+                    subgraph hermes_ns["hermes"]
+                        direction LR
+                        hermes_agent["Hermes Agent"]:::user
+                        hermes_model_gate["Model gate"]:::support
+                        hermes_gpt_oss["gpt-oss:20b<br/>Fallback model"]:::support
+                        hermes_agent --> hermes_model_gate
+                        hermes_model_gate --> hermes_gpt_oss
+                    end
+
+                    subgraph jenkins_ns["jenkins"]
+                        direction LR
+                        jenkins_service["Jenkins"]:::user
+                        jenkins_agents["Build agents"]:::ephemeral
+                        jenkins_service --> jenkins_agents
+                    end
+                end
+
+                subgraph operations_services["Operations"]
+                    direction TB
+
+                    subgraph logging_ns["logging"]
+                        direction LR
+                        logging_oauth_proxy["Dashboard proxy"]:::oauth
+                        logging_dashboards["OpenSearch Dashboards"]:::user
+                        logging_opensearch["OpenSearch"]:::data
+                        logging_fluent_bit["Fluent Bit"]:::support
+                        logging_otel["OpenTelemetry"]:::support
+                        logging_data_prepper["Data Prepper"]:::support
+                        logging_oauth_proxy --> logging_dashboards
+                        logging_dashboards --> logging_opensearch
+                        logging_fluent_bit --> logging_opensearch
+                        logging_otel --> logging_data_prepper
+                        logging_data_prepper --> logging_opensearch
+                    end
+
+                    subgraph longhorn_ns["longhorn-system"]
+                        direction LR
+                        longhorn_oauth_proxy["Longhorn proxy"]:::oauth
+                        longhorn_frontend["Longhorn UI"]:::user
+                        longhorn_backend["Longhorn manager"]:::data
+                        longhorn_webhooks["Admission and recovery"]:::support
+                        longhorn_oauth_proxy --> longhorn_frontend
+                        longhorn_frontend --> longhorn_backend
+                        longhorn_webhooks --> longhorn_backend
+                    end
+
+                    subgraph monitoring_ns["monitoring"]
+                        direction LR
+                        monitoring_grafana["Grafana"]:::user
+                        monitoring_victoria["VictoriaMetrics"]:::data
+                        monitoring_exporters["Cluster and node exporters"]:::support
+                        monitoring_gpu_exporters["GPU exporters"]:::support
+                        monitoring_quality_gateway["Quality gateway"]:::support
+                        monitoring_postmark["Postmark exporter"]:::support
+                        monitoring_vmalert["vmalert"]:::support
+                        monitoring_alertmanager["Alertmanager"]:::user
+                        monitoring_exporters --> monitoring_victoria
+                        monitoring_gpu_exporters --> monitoring_victoria
+                        monitoring_quality_gateway --> monitoring_victoria
+                        monitoring_postmark --> monitoring_victoria
+                        monitoring_victoria --> monitoring_grafana
+                        monitoring_vmalert --> monitoring_alertmanager
+                    end
+                end
+
+                subgraph security_services["Identity and Security"]
+                    direction TB
+
+                    subgraph quality_ns["quality"]
+                        direction LR
+                        quality_oauth_proxy["SonarQube proxy"]:::oauth
+                        quality_sonarqube["SonarQube"]:::user
+                        quality_exporter["Quality exporter"]:::support
+                        quality_oauth_proxy --> quality_sonarqube
+                        quality_exporter --> quality_sonarqube
+                    end
+
+                    subgraph sso_ns["sso"]
+                        direction LR
+                        sso_oauth_proxy["Shared access proxy"]:::oauth
+                        sso_keycloak["Keycloak"]:::user
+                        atlas_realm["atlas realm"]:::control
+                        cassandra_realm["cassandra realm"]:::control
+                        sso_openldap["OpenLDAP"]:::directory
+                        sso_jobs["Realm bootstrap and checks"]:::ephemeral
+                        sso_keycloak --> atlas_realm
+                        sso_keycloak --> cassandra_realm
+                        sso_jobs --> sso_keycloak
+                    end
+
+                    subgraph vault_ns["vault"]
+                        direction LR
+                        vault_service["Vault"]:::user
+                        vault_internal["Vault cluster service"]:::support
+                        vault_injector["Secret injector"]:::support
+                        vault_jobs["Configuration jobs"]:::ephemeral
+                        vault_service --> vault_internal
+                        vault_injector --> vault_service
+                        vault_jobs --> vault_service
+                    end
+                end
+            end
+
+            gitops_domain gitops_identity@-->|HTTPS| gitops_oidc_access
+            gitops_oidc_access gitops_entry@--> flux_weave
+            gitea_domain gitea_atlas_identity@-->|HTTPS| gitea_atlas_oidc_access
+            gitea_atlas_oidc_access gitea_atlas_entry@--> gitea_http
+            gitea_domain gitea_cassandra_identity@-->|HTTPS| gitea_cassandra_oidc_access
+            gitea_cassandra_oidc_access gitea_cassandra_entry@--> gitea_http
+            gitea_domain gitea_ssh_entry@-->|SSH| gitea_ssh
+            harbor_domain harbor_identity@-->|HTTPS| harbor_oidc_access
+            harbor_oidc_access harbor_entry@--> harbor_portal
+            harbor_domain harbor_registry_entry@-->|OCI| harbor_registry
+            hermes_domain hermes_identity@-->|HTTPS| hermes_oidc_access
+            hermes_oidc_access hermes_entry@--> hermes_agent
+            jenkins_domain jenkins_identity@-->|HTTPS| jenkins_oidc_access
+            jenkins_oidc_access jenkins_entry@--> jenkins_service
+            logs_domain logs_identity@-->|HTTPS| logs_oauth_access
+            logs_oauth_access logs_entry@--> logging_oauth_proxy
+            longhorn_domain longhorn_identity@-->|HTTPS| longhorn_oauth_access
+            longhorn_oauth_access longhorn_entry@--> longhorn_oauth_proxy
+            grafana_domain grafana_identity@-->|HTTPS| grafana_oidc_access
+            grafana_oidc_access grafana_entry@--> monitoring_grafana
+            alerts_domain alerts_entry@-->|HTTPS| monitoring_alertmanager
+            quality_domain quality_identity@-->|HTTPS| quality_oauth_access
+            quality_oauth_access quality_entry@--> quality_oauth_proxy
+            keycloak_domain keycloak_identity@-->|HTTPS| keycloak_admin_access
+            keycloak_admin_access keycloak_entry@--> sso_keycloak
+            vault_domain vault_identity@-->|HTTPS| vault_oidc_access
+            vault_oidc_access vault_entry@--> vault_service
+        end
+
+        subgraph shared_plane["Shared Cluster Services"]
+            direction LR
+
+            subgraph ingress_plane["Ingress and Networking"]
+                direction TB
+                metallb_control["MetalLB<br/>Service addresses"]:::control
+                traefik_service["Traefik<br/>Load balancer"]:::support
+                traefik_controller["Traefik<br/>TLS and routing"]:::support
+                cert_manager["cert-manager"]:::support
+                cert_webhooks["Certificate webhooks"]:::support
+                metallb_control --> traefik_service
+                traefik_service --> traefik_controller
+                cert_webhooks --> cert_manager
+                cert_manager --> traefik_controller
+            end
+
+            subgraph shared_state["Shared State and Models"]
+                direction TB
+                postgres_service["Shared PostgreSQL<br/>Application and platform databases"]:::data
+                ai_ollama["Ollama<br/>Local model service"]:::support
+                default_zot_proxy["Zot access proxy"]:::oauth
+            end
+
+            subgraph maintenance_ns["maintenance"]
+                direction TB
+                maintenance_ariadne["Ariadne"]:::support
+                maintenance_metis_proxy["Metis proxy"]:::oauth
+                maintenance_metis["Metis"]:::support
+                maintenance_soteria_proxy["Soteria proxy"]:::oauth
+                maintenance_soteria["Soteria"]:::support
+                maintenance_node_ops["Node operations"]:::ephemeral
+                maintenance_jobs["Repair and migration jobs"]:::ephemeral
+                maintenance_metis_proxy --> maintenance_metis
+                maintenance_soteria_proxy --> maintenance_soteria
+                maintenance_ariadne --> maintenance_node_ops
+                maintenance_ariadne --> maintenance_jobs
+            end
+
+            subgraph telemetry_services["Additional Telemetry"]
+                direction TB
+                climate_typhon["Typhon<br/>Tent climate"]:::support
+                sui_metrics["SUI metrics"]:::support
+            end
+        end
+
+        subgraph external_systems["Host and Physical Systems"]
+            direction LR
+            ananke_hosts["Ananke<br/>titan-db and titan-24"]:::external
+            pyrphoros["Pyrphoros UPS"]:::external
+            statera["Statera UPS"]:::external
+            tent_system["AC Infinity<br/>Controller and cloud"]:::external
+            ananke_hosts --> pyrphoros
+            ananke_hosts --> statera
+        end
+
+        firefly_synced_access firefly_directory@-.-> sso_openldap
+        wger_synced_access wger_directory@-.-> sso_openldap
+        jellyfin_ldap_access jellyfin_directory@-.-> sso_openldap
+
+        website_backend website_model@-.-> ai_ollama
+        website_backend website_automation@-.-> maintenance_ariadne
+        hermes_chat hermes_chat_model@-.-> hermes_model_gate
+        hermes_agent hermes_jenkins@-.-> jenkins_service
+        hermes_agent hermes_metrics@-.-> monitoring_victoria
+        hermes_agent hermes_ariadne@-.-> maintenance_ariadne
+
+        gitea_http gitea_jenkins@-.->|SCM webhook| jenkins_service
+        gitea_http gitea_flux@-.->|Flux webhook| flux_webhook
+        jenkins_agents jenkins_gitea@-.->|Clone and fetch| gitea_http
+        jenkins_agents jenkins_quality@-.->|Quality scan| quality_sonarqube
+        jenkins_agents jenkins_harbor@-.->|Images and artifacts| harbor_registry
+        flux_source flux_gitea@-.->|Desired state| gitea_ssh
+        flux_image_reflector flux_harbor@-.->|Image metadata| harbor_registry
+        flux_image_automation flux_commit@-.->|Manifest update| gitea_ssh
+        flux_kustomize flux_apps@-.-> application_delivery
+        flux_kustomize flux_personal@-.-> personal_delivery
+        flux_kustomize flux_platform@-.-> platform_delivery
+
+        application_state_clients app_postgres@-.-> postgres_service
+        personal_state_clients personal_postgres@-.-> postgres_service
+        platform_state_clients platform_postgres@-.-> postgres_service
+        cloud_nextcloud nextcloud_mail@-.->|Mail delivery| mail_protocols
+
+        quality_exporter quality_metrics@-.-> monitoring_victoria
+        sui_metrics sui_telemetry@-.-> monitoring_victoria
+        monitoring_victoria ananke_telemetry@-.-> ananke_hosts
+        climate_typhon climate_source@--> tent_system
+        climate_typhon climate_telemetry@-.-> monitoring_victoria
+
+        class portal_identity,portal_chat_identity,cassandra_identity,element_identity,matrix_identity,budget_identity,firefly_identity,wolf_identity,wger_identity,chat_identity,jellyfin_identity,cloud_identity,outline_identity,planka_identity,gitops_identity,gitea_atlas_identity,gitea_cassandra_identity,harbor_identity,hermes_identity,jenkins_identity,logs_identity,longhorn_identity,grafana_identity,quality_identity,keycloak_identity,vault_identity routeIngress
+        class portal_entry,portal_chat_entry,cassandra_entry,element_entry,matrix_entry,budget_entry,firefly_entry,wolf_entry,wger_entry,chat_entry,jellyfin_entry,cloud_entry,outline_entry,planka_entry,gitops_entry,gitea_atlas_entry,gitea_cassandra_entry,harbor_entry,hermes_entry,jenkins_entry,logs_entry,longhorn_entry,grafana_entry,quality_entry,keycloak_entry,vault_entry routeService
+        class matrix_api,call_entry,livekit_entry,turn_entry,moonlight_entry,pegasus_entry,mail_web_entry,mail_protocol_entry,office_entry,vaultwarden_entry,monero_entry,gitea_ssh_entry,harbor_registry_entry,alerts_entry routeDirect
+        class firefly_directory,wger_directory,jellyfin_directory routeAuth
+        class app_postgres,personal_postgres,platform_postgres,nextcloud_mail,climate_source routeData
+        class quality_metrics,sui_telemetry,ananke_telemetry,climate_telemetry routeTelemetry
+        class website_automation,hermes_ariadne,gitea_jenkins,gitea_flux,jenkins_gitea,jenkins_quality,jenkins_harbor,flux_gitea,flux_harbor,flux_commit,flux_apps,flux_personal,flux_platform routeControl
+        class website_model,hermes_chat_model,hermes_jenkins,hermes_metrics routeInternal
+    end
+
+    classDef domain fill:#16324f,stroke:#6cb6ff,color:#ffffff
+    classDef user fill:#173f2b,stroke:#61d095,color:#ffffff
+    classDef support fill:#30343b,stroke:#aab2bf,color:#ffffff
+    classDef data fill:#3b2f52,stroke:#b79cff,color:#ffffff
+    classDef ephemeral fill:#4a301e,stroke:#f0a35b,color:#ffffff,stroke-dasharray:6 3
+    classDef control fill:#3b354e,stroke:#c1a7ff,color:#ffffff
+    classDef access fill:#213a56,stroke:#6cb6ff,color:#ffffff
+    classDef oauth fill:#3b2f52,stroke:#c1a7ff,color:#ffffff
+    classDef directory fill:#344a2d,stroke:#8ed081,color:#ffffff
+    classDef external fill:#44301f,stroke:#ffb86c,color:#ffffff
+
+    classDef linkIngress fill:#101820,stroke:#4ea1ff,color:#ffffff
+    classDef linkService fill:#101820,stroke:#61d095,color:#ffffff
+    classDef linkDirect fill:#101820,stroke:#e4bd55,color:#ffffff
+    classDef linkAuth fill:#101820,stroke:#c1a7ff,color:#ffffff
+    classDef linkData fill:#101820,stroke:#b79cff,color:#ffffff
+    classDef linkTelemetry fill:#101820,stroke:#55c8d3,color:#ffffff
+    classDef linkControl fill:#101820,stroke:#f0a35b,color:#ffffff
+    classDef linkInternal fill:#101820,stroke:#aab2bf,color:#ffffff
+
+    classDef routeIngress stroke:#4ea1ff,stroke-width:2.5px,color:#4ea1ff
+    classDef routeService stroke:#61d095,stroke-width:2.5px,color:#61d095
+    classDef routeDirect stroke:#e4bd55,stroke-width:2.5px,color:#e4bd55
+    classDef routeAuth stroke:#c1a7ff,stroke-width:2.2px,color:#c1a7ff,stroke-dasharray:6 4
+    classDef routeData stroke:#b79cff,stroke-width:2.2px,color:#b79cff,stroke-dasharray:6 4
+    classDef routeTelemetry stroke:#55c8d3,stroke-width:2.2px,color:#55c8d3,stroke-dasharray:6 4
+    classDef routeControl stroke:#f0a35b,stroke-width:2.2px,color:#f0a35b,stroke-dasharray:6 4
+    classDef routeInternal stroke:#aab2bf,stroke-width:2px,color:#aab2bf,stroke-dasharray:5 4
+
+    style system fill:#000000,stroke:#8b95a5,stroke-width:2px
+    style legend fill:#050505,stroke:#6d7485,stroke-width:1.5px
+    style application_band fill:#02030d,stroke:#55c8d3,stroke-width:2px
+    style personal_band fill:#06030d,stroke:#8ed081,stroke-width:2px
+    style platform_band fill:#07040c,stroke:#c1a7ff,stroke-width:2px
+    style shared_plane fill:#080604,stroke:#e4bd55,stroke-width:2px
+    style external_systems fill:#080402,stroke:#ffb86c,stroke-width:2px

mermaid/TestAutomation.mmd

@@ -0,0 +1,213 @@
+%% Automatic test failure triage and response.
+%% Hermes reads and recommends. Ariadne validates and performs every write.
+%% Relationships carry the detail; node text stays concise.
+%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 32, "rankSpacing": 48, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "LINEAR_SEGMENTS", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000"}, "themeCSS": "& { background-color: #000000 !important; }"}}%%
+flowchart TD
+
+    subgraph system["Hermes Test Failure Triage"]
+        direction TB
+
+        subgraph main_row["Main flow"]
+            direction TB
+
+            subgraph upper_flow["Evidence and analysis"]
+                direction LR
+
+                subgraph reference["Legend"]
+                    direction TB
+
+                    key_build["Blue<br/>Build and validation"]:::build
+                    key_evidence["Green<br/>Evidence"]:::evidence
+                    key_ariadne["Teal<br/>Ariadne orchestration"]:::ariadne
+                    key_hermes["Purple<br/>Hermes"]:::hermes
+                    key_policy["Gold<br/>Policy and action"]:::policy
+                    key_human["Red<br/>Human required"]:::human
+                    key_limit["Dashed gray<br/>Current boundary"]:::limited
+                end
+
+                subgraph intake["Detect and gather"]
+                    direction TB
+
+                    detector["Jenkins detector<br/>Allowlisted jobs<br/>Terminal failures only"]:::ariadne
+                    evidence_sources["Evidence sources<br/>Jenkins and OpenSearch<br/>Metrics, Kubernetes, Flux<br/>Gitea and Grafana"]:::evidence
+                    collector["Console reader<br/>Full console<br/>2 MB cap"]:::ariadne
+                    failure_ranker["Failure ranker<br/>Strong evidence first<br/>Tool output alone is neutral"]:::ariadne
+                    context_filter["Context filter<br/>Passing quotes removed<br/>Repeats and overlaps merged"]:::ariadne
+                    bundle["Incident bundle<br/>Failures, logs, metrics<br/>Health, revisions, links"]:::evidence
+
+                    detector --> evidence_sources
+                    evidence_sources --> collector
+                    collector --> failure_ranker
+                    failure_ranker --> context_filter
+                    context_filter --> bundle
+                end
+
+                subgraph hermes_plane["Hermes analysis"]
+                    direction TB
+
+                    hermes_access["Read-only access<br/>Jenkins, metrics, Kubernetes<br/>Flux, Gitea, Grafana<br/>Investigation by incident ID"]:::hermes
+                    model_gate["Model gate<br/>openai-codex/gpt-5.6-terra<br/>local gpt-oss:20b fallback"]:::model
+                    skills["Triage skills<br/>Jenkins evidence<br/>Quality and cluster health<br/>Flux and Git correlation"]:::skill
+                    recommendation["Structured recommendation<br/>Classification and confidence<br/>Sourced facts<br/>Action request or patch data"]:::hermes
+                    tool_stream["Agent event stream<br/>Evidence and tool calls"]:::audit
+
+                    hermes_access --> skills
+                    model_gate -.-> skills
+                    skills --> recommendation
+                    hermes_access -.-> tool_stream
+                end
+            end
+
+            subgraph lower_flow["Policy and response"]
+                direction LR
+
+                subgraph controls["Ariadne policy gates"]
+                    direction TB
+
+                response_check["Response check<br/>Schema, incident, sources<br/>Confidence and action"]:::policy
+                repair_guard["Scoped repair guard<br/>Classification and action<br/>must match the owning job"]:::policy
+                marker_check["Transient marker check<br/>DNS, connection, TLS<br/>image pull, upstream 5xx<br/>Disk full is excluded"]:::policy
+                authorizer["Action authorizer<br/>Allowlist, evidence, confidence<br/>One action, kill switch"]:::policy
+
+                response_check -->|scoped repair| repair_guard
+                repair_guard --> authorizer
+                response_check -->|transient claim| marker_check
+                marker_check -->|marker confirmed| authorizer
+                response_check -->|other result| authorizer
+                end
+
+                subgraph response["Ariadne response"]
+                    direction LR
+
+                    subgraph response_paths["Response paths"]
+                        direction TB
+
+                        route{"Policy result"}:::decision
+
+                        subgraph immediate_paths["Action or escalation"]
+                            direction LR
+
+                            subgraph operational_path["Allowlisted action"]
+                                direction TB
+
+                                registry["Action registry"]:::policy
+                                repair_fixture["Scoped ConfigMap repair<br/>In-process patch<br/>No pod created"]:::action
+                                retry_infra["Transient retry<br/>One Jenkins rebuild<br/>No cluster write"]:::action
+                                operational_result["Action result<br/>One attempt<br/>One validation build"]:::action
+
+                                registry --> repair_fixture
+                                registry --> retry_infra
+                                repair_fixture --> operational_result
+                                retry_infra --> operational_result
+                            end
+
+                            subgraph escalation_path["Human-required response"]
+                                direction TB
+
+                                human_required["Diagnosis and next checks<br/>Incident stays human required"]:::human
+                                gitea_issue["Ariadne opens an issue<br/>Evidence and suggested fix"]:::human
+
+                                human_required --> gitea_issue
+                            end
+                        end
+
+                        route -->|authorized action| registry
+                        route -->|human required| human_required
+                        operational_result -->|action failed| human_required
+                    end
+
+                    subgraph code_path["Optional source proposal"]
+                        direction TB
+
+                        subgraph proposal_stage["Prepare and check"]
+                            direction TB
+
+                            duplicate_guard["Open proposal check<br/>Existing repair branch"]:::policy
+                            candidate_files["Candidate files<br/>Path and line hints<br/>Read failing test"]:::evidence
+                            source_target["Patch target<br/>Follow test imports<br/>Tests remain read only"]:::policy
+                            patch_proposal["Hermes patch data<br/>Path, anchor, replacement<br/>Reason"]:::hermes
+                            patch_validator["Patch validator<br/>Allowed path and branch<br/>20 lines, 4 KB<br/>Exact anchor"]:::policy
+
+                            duplicate_guard --> candidate_files
+                            candidate_files --> source_target
+                            source_target --> patch_proposal
+                            patch_proposal --> patch_validator
+                        end
+
+                        subgraph pr_stage["Deliver for review"]
+                            direction TB
+
+                            gitea_pr["Ariadne opens a pull request<br/>hermes-repair branch<br/>Ariadne holds the token"]:::git
+                            branch_build["Branch validation<br/>Configured repositories only"]:::limited
+                            human_merge["Human review and merge<br/>No automatic merge"]:::human
+                            flux_delivery["Flux delivery<br/>After human merge"]:::git
+
+                            gitea_pr -.->|where configured| branch_build
+                            branch_build --> human_merge
+                            gitea_pr -->|otherwise| human_merge
+                            human_merge --> flux_delivery
+                        end
+
+                        patch_validator -->|validated patch| gitea_pr
+                    end
+
+                    response_paths -.->|eligible source defect| code_path
+                end
+
+                subgraph outputs["Inspectable outputs"]
+                    direction TB
+
+                output_entry["Ariadne records every outcome"]:::audit
+                audit_events["Audit events<br/>Incident, diagnosis, action<br/>Patch proposal and agent IDs"]:::audit
+                metrics_output["Triage metrics<br/>VictoriaMetrics and Grafana"]:::metrics
+                alert_output["Narrow alerts<br/>Failed repair<br/>Escalation untouched for six hours"]:::metrics
+                issue_output["Gitea issue<br/>Human-required incident<br/>Written by Ariadne"]:::git
+                pr_output["Gitea pull request<br/>Source proposal<br/>Written by Ariadne"]:::git
+
+                output_entry --> audit_events
+                output_entry --> metrics_output
+                metrics_output --> alert_output
+                end
+            end
+
+            reference ~~~ intake
+            intake ==>|incident bundle| hermes_plane
+            controls ==>|policy result| response
+            response ==>|records and artifacts| outputs
+            upper_flow ==>|recommendation| lower_flow
+        end
+
+    end
+
+    classDef build fill:#16324f,stroke:#6cb6ff,color:#ffffff
+    classDef evidence fill:#173f2b,stroke:#61d095,color:#ffffff
+    classDef ariadne fill:#24414a,stroke:#69c5d1,color:#ffffff
+    classDef hermes fill:#3c3155,stroke:#c1a7ff,color:#ffffff
+    classDef skill fill:#42345b,stroke:#c1a7ff,color:#ffffff
+    classDef model fill:#342b4f,stroke:#a78bfa,color:#ffffff
+    classDef policy fill:#3d3525,stroke:#e4bd55,color:#ffffff
+    classDef decision fill:#47391c,stroke:#e4bd55,color:#ffffff,stroke-width:3px
+    classDef action fill:#4a3b16,stroke:#e4bd55,color:#ffffff
+    classDef human fill:#4a2428,stroke:#ff6b78,color:#ffffff
+    classDef git fill:#253f35,stroke:#78c89a,color:#ffffff
+    classDef metrics fill:#243750,stroke:#65a9e8,color:#ffffff
+    classDef audit fill:#353125,stroke:#d9b55b,color:#ffffff
+    classDef limited fill:#1f2227,stroke:#9ba4b1,color:#c4c9d1,stroke-width:2px,stroke-dasharray:8 6
+
+    style system fill:#030303,stroke:#6b7280,stroke-width:3px
+    style main_row fill:#050607,stroke:#6b7280,stroke-width:2px
+    style upper_flow fill:#050607,stroke:#61d095,stroke-width:2px
+    style lower_flow fill:#070605,stroke:#e4bd55,stroke-width:2px
+    style reference fill:#07090c,stroke:#9ba4b1,stroke-width:2px
+    style intake fill:#070d09,stroke:#61d095,stroke-width:2px
+    style hermes_plane fill:#0c0912,stroke:#c1a7ff,stroke-width:3px
+    style controls fill:#0d0b07,stroke:#e4bd55,stroke-width:2px
+    style response fill:#090806,stroke:#e4bd55,stroke-width:3px
+    style response_paths fill:#070707,stroke:#6b7280,stroke-width:1px
+    style immediate_paths fill:#080706,stroke:#8b8170,stroke-width:1px
+    style operational_path fill:#0d0b07,stroke:#e4bd55,stroke-width:2px
+    style escalation_path fill:#0d090a,stroke:#ff6b78,stroke-width:2px
+    style code_path fill:#0b090d,stroke:#c1a7ff,stroke-width:2px
+    style proposal_stage fill:#09080b,stroke:#9f86da,stroke-width:1px
+    style pr_stage fill:#07100a,stroke:#78c89a,stroke-width:1px
+    style outputs fill:#07100d,stroke:#78c89a,stroke-width:2px

scripts/comms_sync_kb.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-python scripts/knowledge_render_atlas.py --write
-python scripts/knowledge_render_atlas.py --write --out services/comms/knowledge

scripts/manual-tests/test_user_cleanup.sh

@@ -1,13 +1,13 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Manual-only helper to run `scripts/test_user_cleanup.py` inside the portal backend container.
+# Manual-only helper to run `scripts/manual-tests/test_user_cleanup.py` inside the portal backend container.
 #
 # Usage (dry-run):
-#   scripts/test_user_cleanup.sh --prefix test-
+#   scripts/manual-tests/test_user_cleanup.sh --prefix test-
 #
 # Usage (apply):
-#   scripts/test_user_cleanup.sh --prefix test- --apply --confirm test-
+#   scripts/manual-tests/test_user_cleanup.sh --prefix test- --apply --confirm test-
 
 NS="${PORTAL_NAMESPACE:-bstein-dev-home}"
 TARGET="${PORTAL_BACKEND_EXEC_TARGET:-deploy/bstein-dev-home-backend}"

scripts/manual-tests/test_vaultwarden_user_cleanup.py

@@ -11,10 +11,10 @@ conservative:
 - Supports a protected email allowlist to prevent catastrophic mistakes.
 
 Example (dry-run):
-  scripts/test_vaultwarden_user_cleanup.py --prefix e2e-
+  scripts/manual-tests/test_vaultwarden_user_cleanup.py --prefix e2e-
 
 Example (apply):
-  scripts/test_vaultwarden_user_cleanup.py --prefix e2e- --apply --confirm e2e-
+  scripts/manual-tests/test_vaultwarden_user_cleanup.py --prefix e2e- --apply --confirm e2e-
 """
 
 from __future__ import annotations

scripts/manual-tests/test_vaultwarden_user_cleanup.sh

@@ -4,10 +4,10 @@ set -euo pipefail
 # Manual-only helper to clean Vaultwarden test users and invites from Postgres.
 #
 # Usage (dry-run):
-#   scripts/test_vaultwarden_user_cleanup.sh --prefix e2e-
+#   scripts/manual-tests/test_vaultwarden_user_cleanup.sh --prefix e2e-
 #
 # Usage (apply):
-#   scripts/test_vaultwarden_user_cleanup.sh --prefix e2e- --apply --confirm e2e-
+#   scripts/manual-tests/test_vaultwarden_user_cleanup.sh --prefix e2e- --apply --confirm e2e-
 
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 

scripts/ops/build_ananke_node_helper.sh

@@ -26,7 +26,7 @@ while [[ $# -gt 0 ]]; do
       ;;
     -h|--help)
       cat <<USAGE
-Usage: scripts/build_ananke_node_helper.sh [--image <image>] [--docker-config <path>] [--platforms <csv>] [--builder <name>]
+Usage: scripts/ops/build_ananke_node_helper.sh [--image <image>] [--docker-config <path>] [--platforms <csv>] [--builder <name>]
 USAGE
       exit 0
       ;;

scripts/ops/build_harbor_bootstrap_bundle.sh

@@ -31,7 +31,7 @@ while [[ $# -gt 0 ]]; do
       ;;
     -h|--help)
       cat <<USAGE
-Usage: scripts/build_harbor_bootstrap_bundle.sh [--images-file <path>] [--bundle-file <path>] [--docker-config <path>] [--platform <linux/arm64>] [--zstd-level <level>]
+Usage: scripts/ops/build_harbor_bootstrap_bundle.sh [--images-file <path>] [--bundle-file <path>] [--docker-config <path>] [--platform <linux/arm64>] [--zstd-level <level>]
 USAGE
       exit 0
       ;;

scripts/ops/cluster_power_console.sh

@@ -6,7 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 usage() {
   cat <<'USAGE'
 Usage:
-  scripts/cluster_power_console.sh [--repo-dir <path>] [--delegate-host <host>] <shutdown|startup> [recovery-script-options...]
+  scripts/ops/cluster_power_console.sh [--repo-dir <path>] [--delegate-host <host>] <shutdown|startup> [recovery-script-options...]
 
 Purpose:
   Friendly manual entrypoint for running Ananke from a remote console.
@@ -17,9 +17,9 @@ Defaults:
   --delegate-host  titan-db
 
 Examples:
-  scripts/cluster_power_console.sh shutdown --execute
-  scripts/cluster_power_console.sh startup --execute --force-flux-branch main
-  scripts/cluster_power_console.sh --delegate-host titan-24 shutdown --execute
+  scripts/ops/cluster_power_console.sh shutdown --execute
+  scripts/ops/cluster_power_console.sh startup --execute --force-flux-branch main
+  scripts/ops/cluster_power_console.sh --delegate-host titan-24 shutdown --execute
 USAGE
 }
 
@@ -57,7 +57,7 @@ if [[ $# -lt 1 ]]; then
 fi
 
 SIBLING_SCRIPT="${SCRIPT_DIR}/cluster_power_recovery.sh"
-REPO_SCRIPT="${REPO_DIR}/scripts/cluster_power_recovery.sh"
+REPO_SCRIPT="${REPO_DIR}/scripts/ops/cluster_power_recovery.sh"
 LOCAL_SCRIPT=""
 
 if [[ -x "${SIBLING_SCRIPT}" ]]; then
@@ -82,6 +82,6 @@ remote_cmd=""
 if [[ -n "${REMOTE_REPO_DIR}" ]]; then
   remote_cmd+="ANANKE_REPO_DIR=$(printf '%q' "${REMOTE_REPO_DIR}") "
 fi
-remote_cmd+="if [ -x ~/ananke-tools/cluster_power_recovery.sh ]; then ~/ananke-tools/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ${quoted_args}; else echo 'cluster-power-console: remote recovery script not found' >&2; exit 1; fi"
+remote_cmd+="if [ -x ~/ananke-tools/cluster_power_recovery.sh ]; then ~/ananke-tools/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/ops/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/ops/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ${quoted_args}; else echo 'cluster-power-console: remote recovery script not found' >&2; exit 1; fi"
 
 exec ssh -o BatchMode=yes -o ConnectTimeout=8 "${DELEGATE_HOST}" "${remote_cmd}"

scripts/ops/cluster_power_recovery.sh

@@ -16,7 +16,7 @@ fi
 usage() {
   cat <<USAGE
 Usage:
-  scripts/cluster_power_recovery.sh <prepare|status|bootstrap-seed|harbor-seed|longhorn-seed|longhorn-unlock|flux-hold|shutdown|startup> [options]
+  scripts/ops/cluster_power_recovery.sh <prepare|status|bootstrap-seed|harbor-seed|longhorn-seed|longhorn-unlock|flux-hold|shutdown|startup> [options]
 
 Options:
   --execute                     Actually run commands (default is dry-run)
@@ -77,14 +77,14 @@ Options:
   -h, --help                    Show help
 
 Examples:
-  scripts/cluster_power_recovery.sh prepare --execute
-  scripts/cluster_power_recovery.sh bootstrap-seed --execute
-  scripts/cluster_power_recovery.sh harbor-seed --execute
-  scripts/cluster_power_recovery.sh longhorn-unlock --execute
-  scripts/cluster_power_recovery.sh flux-hold --execute
-  scripts/cluster_power_recovery.sh status
-  scripts/cluster_power_recovery.sh shutdown --execute
-  scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main
+  scripts/ops/cluster_power_recovery.sh prepare --execute
+  scripts/ops/cluster_power_recovery.sh bootstrap-seed --execute
+  scripts/ops/cluster_power_recovery.sh harbor-seed --execute
+  scripts/ops/cluster_power_recovery.sh longhorn-unlock --execute
+  scripts/ops/cluster_power_recovery.sh flux-hold --execute
+  scripts/ops/cluster_power_recovery.sh status
+  scripts/ops/cluster_power_recovery.sh shutdown --execute
+  scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main
 USAGE
 }
 

scripts/ops/crypto_wallet_monero_setup.fish

@@ -1,3 +1,5 @@
+#!/usr/bin/env fish
+
 ### ------- helpers ---------------------------------------------------------
 
 function _need --description "ensure a command exists"

scripts/ops/crypto_wallet_sui_setup.fish

@@ -1,3 +1,5 @@
+#!/usr/bin/env fish
+
 ### --------- helpers ----------
 function _need --description "ensure a command exists"
   for c in $argv

scripts/ops/hermes_triage_demo.env.example

@@ -0,0 +1,15 @@
+# Copy to hermes_triage_demo.env (same directory) and fill in.
+# That filename is git-ignored so it can hold real tokens.
+
+# Jenkins API token: https://ci.bstein.dev -> your user -> Configure -> API Token
+export JENKINS_USER="your-jenkins-user"
+export JENKINS_TOKEN="your-jenkins-api-token"
+
+# Gitea token, used only to report open pull requests during preflight.
+# Preflight still works without it; that one line will read "?".
+export GITEA_TOKEN="your-gitea-token"
+
+# Override only if you are not pointing at the usual lab.
+# export JENKINS_URL="https://ci.bstein.dev"
+# export GITEA_URL="https://scm.bstein.dev"
+# export CODE_REPO_DIR="$HOME/Development/hermes-code-demo"

scripts/ops/hermes_triage_demo.sh

@@ -0,0 +1,294 @@
+#!/usr/bin/env bash
+# Drive and narrate the Hermes automated-triage demos.
+#
+#   hermes_triage_demo.sh fixture   # autonomous loop: fail -> repair -> green
+#   hermes_triage_demo.sh code      # proposal loop: fail -> Hermes patch -> PR
+#   hermes_triage_demo.sh status    # current incident/alert state, no changes
+#   hermes_triage_demo.sh preflight # confirm the lab is ready to demo
+#   hermes_triage_demo.sh reset     # restore the demo to its pre-run state
+#   hermes_triage_demo.sh monitor [code]  # stream the flow chart stages live
+#
+# FIRST RUN: copy hermes_triage_demo.env.example to hermes_triage_demo.env in
+# this directory and fill it in. That file is git-ignored precisely so it can
+# hold real tokens; this script sources it automatically, so nothing needs to
+# be exported by hand.
+#
+# Needs kubectl access to the cluster as well. Nothing here mutates the cluster
+# directly: the fixture demo only asks Jenkins to run a parameterized build,
+# and the code demo only pushes a seeded defect to the demo repository.
+set -euo pipefail
+
+# Local, git-ignored credentials. Sourced before anything else so every value
+# below can be overridden from it.
+_DEMO_ENV="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_demo.env"
+# shellcheck disable=SC1090
+[ -r "$_DEMO_ENV" ] && . "$_DEMO_ENV"
+
+JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}"
+FIXTURE_JOB="hermes-triage-demo"
+CODE_JOB="hermes-code-demo"
+DEMO_NS="hermes-triage-demo"
+CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}"
+
+say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; }
+note() { printf '        %s\n' "$*"; }
+
+require_jenkins() {
+  if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then
+    echo "Missing Jenkins credentials." >&2
+    echo "Create $_DEMO_ENV from hermes_triage_demo.env.example and fill it in." >&2
+    exit 1
+  fi
+}
+
+jenkins_get() { curl -sk -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; }
+jenkins_post() { curl -sk -o /dev/null -w '%{http_code}' -u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"; }
+
+last_build_number() {
+  jenkins_get "/job/$1/api/json?tree=lastBuild[number]" |
+    python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])'
+}
+
+wait_for_build() { # job number -> prints result
+  local job="$1" num="$2" tries="${3:-120}"
+  for _ in $(seq 1 "$tries"); do
+    sleep 10
+    local body result building
+    body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)"
+    building="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)"
+    if [ "$building" = "False" ]; then
+      result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')"
+      printf '%s' "$result"
+      return 0
+    fi
+  done
+  printf 'TIMEOUT'
+}
+
+ariadne_ticks() { # tail the autotriage decisions in human-readable form
+  kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null |
+    grep 'hermes autotriage tick' |
+    python3 -c '
+import sys, json
+for line in sys.stdin:
+    try:
+        d = json.loads(line)
+    except ValueError:
+        continue
+    print("       ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
+}
+
+# `monitor` follows the fixture job; `monitor code` follows the code-proposal
+# job, which takes the source-proposal branch of the chart instead.
+cmd_monitor() {
+  local which="${1:-fixture}"
+  [ "$which" = "code" ] && export MONITOR_JOB="$CODE_JOB"
+  exec python3 "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_triage_monitor.py"
+}
+
+# Deletes the demo repositories' issues, pull requests and repair branches so a
+# rerun starts from nothing. DEMO_REPOS is the whole blast radius and is
+# deliberately explicit: a real service's issues are genuine triage records,
+# and clearing them to tidy a demo would destroy the evidence the system
+# exists to produce.
+DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}"
+
+cmd_reset() {
+  require_jenkins
+  say "Reset — restoring the demo to its pre-run state"
+
+  note "fixture -> healthy"
+  if kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \
+      --type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1; then
+    note "  fixture: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
+  else
+    note "  fixture patch failed (is the demo namespace present?)"
+  fi
+
+  local gitea="${GITEA_URL:-https://scm.bstein.dev}"
+  if [ -z "${GITEA_TOKEN:-}" ]; then
+    note "GITEA_TOKEN unset; skipping repository cleanup"
+  else
+    for repo in $DEMO_REPOS; do
+      note "clearing bstein/$repo (demo repository)"
+      local items
+      items="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \
+        "$gitea/api/v1/repos/bstein/$repo/issues?state=all&limit=100" |
+        python3 -c 'import json,sys
+for i in json.load(sys.stdin):
+    print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)"
+      if [ -z "$items" ]; then
+        note "  no issues or pull requests"
+      else
+        while read -r num kind; do
+          [ -z "$num" ] && continue
+          curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
+            "$gitea/api/v1/repos/bstein/$repo/issues/$num"
+          note "  deleted $kind #$num"
+        done <<< "$items"
+      fi
+
+      local branches
+      branches="$(curl -s --max-time 25 -H "Authorization: token $GITEA_TOKEN" \
+        "$gitea/api/v1/repos/bstein/$repo/branches" |
+        python3 -c 'import json,sys,urllib.parse
+for b in json.load(sys.stdin):
+    if b["name"].startswith("hermes-repair/"):
+        print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)"
+      if [ -z "$branches" ]; then
+        note "  no repair branches"
+      else
+        for ref in $branches; do
+          curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
+            "$gitea/api/v1/repos/bstein/$repo/branches/$ref"
+          note "  deleted branch $(printf '%b' "${ref//%/\\x}")"
+        done
+      fi
+    done
+  fi
+
+  if [ -d "$CODE_REPO_DIR/.git" ]; then
+    note "restoring the demo repository working state"
+    ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q --ff-only 2>/dev/null || true )
+    if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
+      note "  src/discount.py is correct; demo is armable"
+    else
+      note "  src/discount.py still carries the seeded defect — revert it before demoing"
+    fi
+  else
+    note "demo repository not cloned at $CODE_REPO_DIR; skipping"
+  fi
+
+  say "Ready"
+  note "real service repositories were not touched"
+  note "run 'preflight' next, then arm the fixture build"
+}
+
+cmd_preflight() {
+  require_jenkins
+  say "Preflight"
+  note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)"
+  note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')"
+  note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)"
+  # Printed as a list rather than the raw comma-separated setting: this is the
+  # outermost safety boundary, so it is worth being able to read at a glance.
+  local allowlist count
+  allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')"
+  count=0
+  for _job in $allowlist; do count=$((count + 1)); done
+  note "jobs Ariadne may triage ($count):"
+  for _job in $allowlist; do note "  - $_job"; done
+  note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')"
+  local queued
+  queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')"
+  note "jenkins queue depth: $queued (demo is fastest when this is 0)"
+  # The Kubernetes cloud caps concurrent agent pods at containerCapStr. When
+  # real CI saturates that cap the demo build sits in the queue reporting
+  # "all nodes are offline" and the timings in the runbook do not apply.
+  local agents cap
+  cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null |
+    grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)"
+  agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)"
+  note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)"
+  local open_prs
+  open_prs="$(curl -s --max-time 25 -H "Authorization: token ${GITEA_TOKEN:-}" \
+    "${GITEA_URL:-https://scm.bstein.dev}/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null |
+    python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')"
+  note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)"
+}
+
+cmd_status() {
+  say "Incident state (last ticks)"
+  ariadne_ticks 600 8
+  say "Firing alerts"
+  kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null |
+    python3 -c '
+import json,sys
+alerts = json.load(sys.stdin).get("data", {}).get("alerts", [])
+print("        none" if not alerts else "")
+for a in alerts:
+    print("       ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null ||
+    note "(query vmalert directly if this fails)"
+  say "Demo namespace"
+  kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/        /'
+}
+
+cmd_fixture() {
+  require_jenkins
+  local start_num next_num
+  start_num="$(last_build_number "$FIXTURE_JOB")"
+  next_num=$((start_num + 1))
+  say "Arming the demo failure (SEED_FAILURE=true) -> build #$next_num"
+  note "HTTP $(jenkins_post "/job/$FIXTURE_JOB/buildWithParameters?SEED_FAILURE=true")"
+  note "Only manual step. Everything after this is automatic."
+
+  say "Waiting for the seeded build to fail"
+  note "result: $(wait_for_build "$FIXTURE_JOB" "$next_num")"
+
+  say "Ariadne detects, gathers evidence, asks Hermes, authorizes, repairs"
+  note "the repair is a single in-process ConfigMap patch, so watch the fixture"
+  for _ in $(seq 1 40); do
+    sleep 10
+    if [ "$(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null)" = "healthy" ]; then
+      note "fixture patched back to healthy"
+      break
+    fi
+  done
+  ariadne_ticks 400 4
+
+  say "Ariadne triggers one rebuild with seeding disabled"
+  note "result: $(wait_for_build "$FIXTURE_JOB" $((next_num + 1)))"
+
+  say "Resolution"
+  sleep 45
+  ariadne_ticks 200 3
+  note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
+}
+
+cmd_code() {
+  require_jenkins
+  [ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; }
+  local start_num next_num
+  start_num="$(last_build_number "$CODE_JOB")"
+  next_num=$((start_num + 1))
+
+  say "Seeding a one-line defect in src/discount.py"
+  ( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q &&
+    python3 - <<'PY'
+import pathlib
+p = pathlib.Path("src/discount.py")
+s = p.read_text()
+old, new = "percent / 100", "percent / 10"
+if old not in s:
+    raise SystemExit("defect already present or file changed; reset master first")
+p.write_text(s.replace(old, new))
+PY
+    git commit -qam "refactor: simplify discount percentage math" && git push -q origin master )
+  note "pushed: a plausible-looking change that breaks three regression tests"
+
+  say "Running the test gate -> build #$next_num"
+  note "HTTP $(jenkins_post "/job/$CODE_JOB/build")"
+  note "result: $(wait_for_build "$CODE_JOB" "$next_num")"
+
+  say "Ariadne collects evidence and asks Hermes for a minimal patch"
+  note "Hermes returns an anchored patch as data; Ariadne validates path, size,"
+  note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num"
+  for _ in $(seq 1 40); do
+    sleep 15
+    ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break
+  done
+  ariadne_ticks 400 3
+
+  say "Pull request awaiting human review (nothing merges automatically)"
+  note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
+}
+
+case "${1:-}" in
+  fixture) cmd_fixture ;;
+  code) cmd_code ;;
+  status) cmd_status ;;
+  preflight) cmd_preflight ;;
+  reset) cmd_reset ;;
+  monitor) shift; cmd_monitor "$@" ;;
+  *) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
+esac

scripts/ops/hermes_triage_monitor.py

@@ -0,0 +1,463 @@
+#!/usr/bin/env python3
+"""Narrate the triage flow live, showing every command it runs.
+
+Run in a second terminal beside the demo. When a stage of
+`mermaid/TestAutomation.mmd` is reached this prints, in order:
+
+    the stage banner, naming the chart subgraph
+    the service UI to look at, if one changes at that stage
+    each command, echoed before it runs, then its output
+
+Echoing the commands is the point. An audience watching a dashboard has to
+take the result on trust; watching `kubectl` run against the cluster and
+reading the raw answer is the difference between a demonstration and an
+assertion.
+
+Two honest limits. This reports at subgraph granularity, not per node: the
+chart draws the collector, response check, marker check and authorizer
+separately, and a refusal here names the gate rather than walking all twelve.
+And it follows one incident on one job, whereas the chart describes the whole
+system.
+
+Read-only. Every command below is a read; nothing here changes the cluster.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+
+JOB = os.environ.get("MONITOR_JOB", "hermes-triage-demo")
+NS_DEMO = "hermes-triage-demo"
+NS_ARIADNE = "maintenance"
+JENKINS = os.environ.get("JENKINS_URL", "https://ci.bstein.dev")
+GITEA = os.environ.get("GITEA_URL", "https://scm.bstein.dev")
+GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev")
+HERMES_UI = os.environ.get("HERMES_URL", "https://agent.bstein.dev")
+POLL_SECONDS = 6
+
+# The fixture job repairs a ConfigMap; the code job proposes a patch. Evidence
+# that suits one is false for the other, so the stages branch on it.
+IS_CODE_JOB = JOB == "hermes-code-demo"
+
+BOLD, DIM, GREEN, CYAN, YELLOW, RED, RESET = (
+    "\033[1m", "\033[2m", "\033[32m", "\033[36m", "\033[33m", "\033[31m", "\033[0m"
+)
+
+# stage key -> (chart subgraph, what it means, chart path, UI worth showing)
+STAGES: dict[str, tuple[str, str, str, str]] = {
+    "detect": (
+        "Detect and gather",
+        "A failed build becomes one incident",
+        "Jenkins detector -> Evidence sources",
+        f"{JENKINS}/job/{JOB}/ — the red build",
+    ),
+    "evidence": (
+        "Detect and gather",
+        "Ariadne assembles the bounded evidence bundle",
+        "Console reader -> Failure ranker -> Context filter -> Incident bundle",
+        "",
+    ),
+    "hermes": (
+        "Hermes analysis",
+        "Hermes returns a recommendation it cannot act on",
+        "Triage skills -> Structured recommendation",
+        f"{HERMES_UI} — the agent run appears here",
+    ),
+    "gates": (
+        "Ariadne policy gates",
+        "Ariadne decides on its own reading of the evidence",
+        "Response check -> Scoped repair guard -> Action authorizer",
+        "",
+    ),
+    "route": (
+        "Ariadne response",
+        "The policy result, and which branch it opens",
+        "Policy result -> Action registry",
+        "",
+    ),
+    "response": (
+        "Ariadne response",
+        "Ariadne executes, proposes, or escalates",
+        "Action registry -> Scoped ConfigMap repair -> Action result",
+        "",
+    ),
+    "verify": (
+        "Ariadne response",
+        "The branch build checks the proposal" if JOB == "hermes-code-demo"
+        else "One rebuild decides whether the repair held",
+        "Action result -> validation build",
+        f"{JENKINS}/job/{JOB}/ — a new build starts on its own",
+    ),
+    "outputs": (
+        "Inspectable outputs",
+        "Incident closed; the artifacts remain",
+        "Ariadne records every outcome -> Audit events, Triage metrics",
+        f"{GRAFANA}/d/atlas-testing — triage panels",
+    ),
+}
+ORDER = ["detect", "evidence", "hermes", "gates", "route", "response", "verify", "outputs"]
+
+
+def stamp() -> str:
+    return datetime.now(timezone.utc).strftime("%H:%M:%S")
+
+
+def run(cmd: list[str], *, show: bool = True, limit: int = 12) -> str:
+    """Echo a command, run it, print its output, and return stdout."""
+
+    if show:
+        print(f"  {DIM}${RESET} {CYAN}{' '.join(cmd)}{RESET}")
+    try:
+        # Ariadne emits ~200 lines per tick, so a ten-minute window is a
+        # few thousand lines; the fetch needs room to finish.
+        done = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
+        out = done.stdout.strip()
+    except Exception as exc:  # a failed read must never stop the narration
+        print(f"  {RED}(command failed: {exc}){RESET}")
+        return ""
+    if show:
+        for line in (out.split("\n")[:limit] or ["(no output)"]):
+            print(f"      {line[:150]}")
+        print()
+    return out
+
+
+def quiet(cmd: list[str]) -> str:
+    return run(cmd, show=False)
+
+
+def banner(key: str) -> None:
+    subgraph, meaning, path, ui = STAGES[key]
+    print(f"\n{GREEN}{'─' * 70}{RESET}")
+    print(f"{GREEN}{BOLD} {stamp()}  {subgraph}{RESET}{GREEN} — {meaning}{RESET}")
+    print(f"{DIM} chart: {path}{RESET}")
+    if ui:
+        print(f"{YELLOW} look at: {ui}{RESET}")
+    print(f"{GREEN}{'─' * 70}{RESET}")
+
+
+def ariadne_records(since: str = "10m") -> list[dict]:
+    """Return recent Ariadne log records that parse as JSON.
+
+    Windowed by time, not line count. Ariadne emits roughly two hundred lines
+    per tick once it is polling every job, so a --tail window wide enough to
+    be useful is impossible to guess and a narrow one silently drops ticks.
+    """
+
+    raw = quiet(
+        ["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--since={since}"]
+    )
+    records = []
+    for line in raw.split("\n"):
+        line = line.strip()
+        if line.startswith("{"):
+            try:
+                records.append(json.loads(line))
+            except ValueError:
+                continue
+    return records
+
+
+def job_states(records: list[dict]) -> list[dict]:
+    """Return every tick result for this job in the window, oldest first.
+
+    The stages are reconstructed from all of them rather than from the newest
+    alone. A tick reporting the repair is replaced by the next tick within
+    seconds, so reading only the latest state means a poll landing at the
+    wrong moment loses that step permanently.
+    """
+
+    states = []
+    for record in records:
+        if record.get("event") == "hermes_autotriage" and record.get("jobs"):
+            try:
+                state = json.loads(record["jobs"]).get(JOB)
+            except ValueError:
+                continue
+            if state:
+                states.append(state)
+    return states
+
+
+_DIAG_QUERY = (
+    ". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; python3 -c \""
+    "import json,os,psycopg;"
+    "conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);"
+    "cur=conn.cursor();"
+    "cur.execute(\\\"select detail from ariadne_events where event_type='hermes_autotriage_diagnosis'"
+    " order by id desc limit 1\\\");"
+    "r=cur.fetchone()[0];"
+    "d=r if isinstance(r,dict) else json.loads(r);"
+    "print(json.dumps({'incident':d.get('incident_id'),'authorized':d.get('authorized'),"
+    "'authorize_reason':d.get('authorize_reason'),'evidence_marker':d.get('evidence_marker'),"
+    "'run_id':(d.get('run') or {}).get('run_id'),"
+    "'run_seconds':(d.get('run') or {}).get('duration_seconds'),"
+    "'outcome':d.get('outcome')},indent=1))\""
+)
+
+
+_HISTORY_QUERY = (
+    ". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; python3 -c \""
+    "import json,os,psycopg;"
+    "conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);"
+    "cur=conn.cursor();"
+    "cur.execute(\\\"select created_at,detail from ariadne_events where"
+    " event_type='hermes_autotriage_incident' order by id desc limit 40\\\");"
+    "rows=[(t,d if isinstance(d,dict) else json.loads(d)) for t,d in cur.fetchall()];"
+    "rows=[r for r in rows if r[1].get('incident_id')=='__INCIDENT__'];"
+    "[print(str(t)[11:19], v.get('status'), json.dumps(v.get('phase') or {})[:80])"
+    " for t,v in reversed(rows)]\""
+)
+
+
+_BUNDLE_QUERY = (
+    "python3 -c \""
+    "from ariadne.services import hermes_autotriage_evidence as ev;"
+    "lb={'number':__BUILD__,'result':'FAILURE','building':False,'timestamp':0,'duration':0,'url':''};"
+    "b=ev.collect_evidence('__INCIDENT__','__JOB__',lb);"
+    "j=b['jenkins']; regions=j.get('console_failures') or []; tests=j.get('failed_tests') or [];"
+    "recs=(b.get('log_evidence') or {}).get('records') or [];"
+    "print('jenkins.console_failures : %d region(s), truncated=%s' % (len(regions), j.get('console_truncated')));"
+    "[print('   | ' + l[:110]) for l in ((regions[0].get('text') or '').strip().split(chr(10))[-3:] if regions else [])];"
+    "print('jenkins.failed_tests     : %d' % len(tests));"
+    "[print('   | %s :: %s' % (t.get('className'), t.get('name'))) for t in tests[:2]];"
+    "print('log_evidence.records     : %d from OpenSearch kube-*' % len(recs));"
+    "[print('   | [%s] %s' % (r.get('namespace'), (r.get('message') or '')[:90])) for r in recs[:2]]\""
+)
+
+
+def bundle_sample(incident: str) -> None:
+    """Show a trimmed sample of the bundle that was sent to Hermes.
+
+    Rebuilt from the same collector Ariadne used. A finished build's console
+    does not change, so this is durable rather than a live reading, and it is
+    trimmed hard on purpose: the point is to show what kind of evidence each
+    source contributes, not to reprint the bundle.
+    """
+
+    if "/" not in incident:
+        return
+    job, _, build = incident.rpartition("/")
+    query = (
+        _BUNDLE_QUERY.replace("__BUILD__", build)
+        .replace("__INCIDENT__", incident)
+        .replace("__JOB__", job)
+    )
+    run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
+         "sh", "-c", ". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; " + query], limit=14)
+
+
+def incident_history(incident: str) -> None:
+    """Show this incident's recorded state changes, oldest first.
+
+    Durable evidence on purpose. A stage describes a moment that has passed,
+    so reading live cluster state at that point misrepresents it: by the time
+    the detection stage is narrated the repair has already run, and the
+    fixture would read healthy as though it had never failed.
+    """
+
+    if not incident:
+        return
+    run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
+         "sh", "-c", _HISTORY_QUERY.replace("__INCIDENT__", incident)], limit=10)
+
+
+def diagnosis_event() -> None:
+    """Show the diagnosis Ariadne stored: what Hermes said, and the verdict."""
+
+    run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
+         "sh", "-c", _DIAG_QUERY], limit=16)
+
+
+def evidence_for(key: str, incident: str = "") -> None:
+    """Run the reads that show this stage actually happened."""
+
+    if key == "detect":
+        print(f"  {DIM}the incident's recorded state changes; durable, so it still reads"
+              f" true after the repair has run:{RESET}")
+        incident_history(incident)
+    elif key == "evidence":
+        print(f"  {DIM}a sample of what each source contributed to the bundle Hermes"
+              f" received:{RESET}")
+        bundle_sample(incident)
+    elif key == "hermes":
+        print(f"  {DIM}what Hermes actually returned, as Ariadne stored it. The run_id below"
+              f" identifies this run in the Hermes dashboard, where the prompt it was given"
+              f" and every tool call it made are visible:{RESET}")
+        diagnosis_event()
+        print(f"  {DIM}Hermes holds no Git or Kubernetes write access; this JSON is its"
+              f" entire output{RESET}\n")
+    elif key == "gates":
+        print(f"  {DIM}the authorization is a match between three separate things: an action"
+              f" Ariadne already has code to perform, an action id Hermes is permitted to"
+              f" request, and the action Hermes actually recommended. Ariadne holds the"
+              f" registry below; Hermes cannot add to it, and the recommendation only"
+              f" proceeds because it names something already in it.{RESET}")
+        run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
+             "printenv", "ARIADNE_HERMES_ALLOWED_ACTIONS"], limit=2)
+        run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
+             "printenv", "ARIADNE_HERMES_MIN_CONFIDENCE"], limit=2)
+        print(f"  {DIM}a separate setting governs what Hermes may be asked to propose a fix"
+              f" for. It is separate because these become pull requests rather than"
+              f" changes Ariadne makes itself:{RESET}")
+        run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
+             "printenv", "ARIADNE_HERMES_FIX_CATEGORIES"], limit=2)
+        print()
+    elif key == "route":
+        if IS_CODE_JOB:
+            print(f"  {DIM}a source fix is not one of the registered actions, so this takes"
+                  f" the Optional source proposal branch rather than the Action registry.{RESET}\n")
+        else:
+            print(f"  {DIM}the fixture repair is an operational action, so the Optional source"
+                  f" proposal branch is not taken for this incident{RESET}\n")
+    elif key in {"response", "outputs"}:
+        if IS_CODE_JOB:
+            print(f"  {DIM}the pull request Ariadne opened is recorded on the incident above"
+                  f" (branch, pr_number, url). Hermes produced the patch as data; Ariadne"
+                  f" validated it and pushed the branch.{RESET}")
+            print(f"  {YELLOW}pull requests: {GITEA}/bstein/hermes-code-demo/pulls{RESET}\n")
+        else:
+            run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture",
+                 "-o", "jsonpath={.data.state}"])
+            if key == "response":
+                print(f"  {DIM}that value read 'unhealthy' when the build failed - the seeded"
+                      f" fault - and reads 'healthy' above because Ariadne has just patched"
+                      f" it. That single field changing is the repair.{RESET}\n")
+        if key == "outputs":
+            print(f"  {DIM}on the chart this is the 'records and artifacts' edge out of the"
+                  f" whole Ariadne response box, not out of one branch. Every path ends"
+                  f" here: an executed action, an escalation, or a pull request all record"
+                  f" the same audit events and metrics.{RESET}")
+            print(f"  {YELLOW}issues filed by triage: {GITEA}/bstein/ariadne/issues{RESET}\n")
+    elif key == "verify":
+        if IS_CODE_JOB:
+            print(f"  {DIM}the branch build validates the proposal. Nothing merges"
+                  f" automatically: the incident stays human-required whatever the branch"
+                  f" build says, because a person decides whether the fix is right.{RESET}\n")
+        else:
+            print(f"  {DIM}exactly one rebuild is triggered; it never retries in a loop. This"
+                  f" completes the operational branch: because the action was authorized and"
+                  f" performed, neither the human-required response nor the optional source"
+                  f" proposal is entered for this incident.{RESET}\n")
+
+
+class Monitor:
+    """Track which flow-chart stage the current incident has reached."""
+
+    def __init__(self) -> None:
+        self.done: set[str] = set()
+        self.incident = ""
+        self.summarised = False
+        self.last_tick = ""
+
+    def mark(self, key: str, evidence: str) -> None:
+        if key in self.done:
+            return
+        self.done.add(key)
+        banner(key)
+        print(f"  {BOLD}what happened:{RESET} {evidence}\n")
+        evidence_for(key, self.incident)
+
+    def new_incident(self, incident: str) -> None:
+        if incident and incident != self.incident:
+            self.incident = incident
+            self.done.clear()
+            self.summarised = False
+            print(f"\n{BOLD}{YELLOW}══ incident {incident} ══{RESET}")
+
+    def checklist(self) -> None:
+        print(f"\n{BOLD}  flow chart progress{RESET}")
+        for key in ORDER:
+            tick = f"{GREEN}✓{RESET}" if key in self.done else f"{DIM}·{RESET}"
+            print(f"    {tick} {key:<9} {STAGES[key][0]:<22} {DIM}{STAGES[key][2]}{RESET}")
+        print()
+
+
+def main() -> None:
+    print(f"{BOLD}Hermes triage monitor — following mermaid/TestAutomation.mmd{RESET}")
+    print(f"{DIM}Every command is echoed before it runs. Read-only. Ctrl-C to stop.{RESET}")
+    print(f"{YELLOW}Jenkins: {JENKINS}/job/{JOB}/{RESET}")
+    print(f"{YELLOW}Gitea:   {GITEA}/bstein{RESET}")
+    print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}\n")
+    monitor = Monitor()
+    while True:
+        states = job_states(ariadne_records())
+        # Replay every tick in the window, so a step that lasted one tick is
+        # never lost to poll timing. mark() is idempotent.
+        for state in states:
+            status = str(state.get("status") or "")
+            monitor.new_incident(str(state.get("incident_id") or ""))
+            if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}:
+                monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure")
+            if status == "awaiting_rebuild":
+                monitor.mark("evidence", "bundle collected: console regions, tests, logs")
+                monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema")
+                monitor.mark("gates", "every gate passed; the predefined action was authorized")
+                monitor.mark(
+                "route",
+                "policy result: authorized -> Action registry (Optional source proposal not taken)",
+            )
+                monitor.mark(
+                "response",
+                f"Scoped ConfigMap repair: {state.get('repair', 'action')} on {state.get('target', '')}",
+            )
+                monitor.mark("verify", "one rebuild triggered with seeding disabled")
+            if status == "human_required":
+                reason = str(state.get("reason") or "")
+                proposed = reason == "code_fix_proposed"
+                monitor.mark("evidence", "bundle collected")
+                monitor.mark(
+                    "hermes",
+                    "patch proposed as data: a path, an exact anchor and a replacement"
+                    if proposed
+                    else "diagnosis returned",
+                )
+                monitor.mark(
+                    "gates",
+                    "no action was authorized; a source fix is not an action, so this takes"
+                    " the proposal branch rather than the registry"
+                    if proposed
+                    else f"Ariadne refused: {reason} - nothing ran",
+                )
+                monitor.mark(
+                    "route",
+                    "policy result: no action -> Optional source proposal -> Patch validator"
+                    if proposed
+                    else "policy result: refused -> Diagnosis and next checks -> opens an issue",
+                )
+                monitor.mark(
+                    "response",
+                    "Ariadne opens a pull request; nothing merges without a human"
+                    if proposed
+                    else "escalated; issue filed in the service repository",
+                )
+                if proposed:
+                    monitor.mark(
+                        "verify",
+                        "the branch build validates the proposal; the incident stays"
+                        " human-required either way",
+                    )
+            if status == "healthy" and state.get("resolved") and not monitor.summarised:
+                monitor.mark("verify", "rebuild finished green")
+                monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}")
+                monitor.checklist()
+                monitor.summarised = True
+        status = str(states[-1].get("status") or "") if states else ""
+
+        if status and status != monitor.last_tick:
+            monitor.last_tick = status
+            print(f"{DIM}{stamp()}  tick: {status}{RESET}")
+        sys.stdout.flush()
+        time.sleep(POLL_SECONDS)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("\nstopped")

scripts/ops/jellyfin_manual_load.fish

@@ -4,10 +4,10 @@
 # Fallback: kubectl port-forward to service (OK for small/medium files).
 #
 # Usage:
-#   scripts/jellyfin_manual_load.fish <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]
+#   scripts/ops/jellyfin_manual_load.fish <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]
 # Examples:
-#   scripts/jellyfin_manual_load.fish "$HOME/Downloads/Avatar - The Last Airbender (2005 - 2008) [1080p]" kids_tv "$JELLYFIN_API_TOKEN"
-#   scripts/jellyfin_manual_load.fish "$HOME/Movies/." movies   # copy contents-only into /media/movies
+#   scripts/ops/jellyfin_manual_load.fish "$HOME/Downloads/Avatar - The Last Airbender (2005 - 2008) [1080p]" kids_tv "$JELLYFIN_API_TOKEN"
+#   scripts/ops/jellyfin_manual_load.fish "$HOME/Movies/." movies   # copy contents-only into /media/movies
 
 function usage
   echo "Usage: "(basename (status filename))" <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]"

scripts/ops/k3s_version_update.fish

@@ -1,3 +1,5 @@
+#!/usr/bin/env fish
+
 # Pick the correct K3s asset for a remote host (arm64 vs x86_64)
 function __k3s_asset_for_host
     set -l host $argv[1]

scripts/ops/node_recover.sh

@@ -3,7 +3,7 @@ set -euo pipefail
 
 usage() {
   cat <<USAGE
-Usage: scripts/node_recover.sh <node-name> [options]
+Usage: scripts/ops/node_recover.sh <node-name> [options]
 
 Options:
   --yes             Skip confirmation prompt

scripts/render/dashboards_render_atlas.py

@@ -16,7 +16,7 @@ from pathlib import Path
 # Paths, folders, and shared metadata
 # ---------------------------------------------------------------------------
 
-ROOT = Path(__file__).resolve().parents[1]
+ROOT = Path(__file__).resolve().parents[2]
 DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards"
 CONFIG_TEMPLATE = textwrap.dedent(
     """# {relative_path}
@@ -110,8 +110,6 @@ WORKER_NODES = [
     "titan-06",
     "titan-07",
     "titan-08",
-    "titan-09",
-    "titan-10",
     "titan-11",
     "titan-20",
     "titan-21",
@@ -119,7 +117,6 @@ WORKER_NODES = [
     "titan-13",
     "titan-14",
     "titan-15",
-    "titan-16",
     "titan-17",
     "titan-18",
     "titan-19",
@@ -184,7 +181,8 @@ def scoped_node_expr(base, scope=""):
 
 def node_cpu_expr(scope=""):
     idle = 'avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))'
-    base = f"(1 - {idle}) * 100"
+    # Scrape stalls can briefly report an impossible idle rate after recovery.
+    base = f"clamp_max(clamp_min((1 - {idle}) * 100, 0), 100)"
     return scoped_node_expr(base, scope)
 
 
@@ -284,8 +282,14 @@ def dcgm_gpu_util_by_node():
     )
 
 
+def nvidia_gpu_util_by_node():
+    return "max by (node) (nvidia_gpu_device_utilization_percent)"
+
+
 def gpu_util_by_node():
-    return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}"
+    process_exporter = nvidia_gpu_util_by_node()
+    dcgm_fallback = f"({dcgm_gpu_util_by_node()}) unless on(node) ({process_exporter})"
+    return f"{process_exporter} or {dcgm_fallback} or {jetson_gpu_util_by_node()}"
 
 
 def gpu_util_by_hostname():
@@ -295,16 +299,11 @@ def gpu_util_by_hostname():
 GPU_RESOURCE_REGEX = "nvidia(_com_|[.]com/)gpu.*"
 
 
-def gpu_node_labels():
-    return f'max by (node) (kube_node_status_allocatable{{resource=~"{GPU_RESOURCE_REGEX}"}} > bool 0)'
-
-
 def gpu_requests_by_namespace_node(scope_var):
     return (
         "sum by (namespace,node) ("
         f'kube_pod_container_resource_requests{{resource=~"{GPU_RESOURCE_REGEX}",{scope_var}}} '
         "* on(namespace,pod) group_left(node) kube_pod_info "
-        f"* on(node) group_left() ({gpu_node_labels()})"
         ")"
     )
 
@@ -326,7 +325,8 @@ def gpu_usage_by_namespace(scope_var):
 
 def jetson_gpu_usage_by_namespace(scope_var):
     requests_by_ns = gpu_requests_by_namespace_node(scope_var)
-    total_by_node = f"sum by (node) ({requests_by_ns})"
+    all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
+    total_by_node = f"sum by (node) ({all_requests})"
     return (
         "sum by (namespace) ("
         f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) "
@@ -335,6 +335,12 @@ def jetson_gpu_usage_by_namespace(scope_var):
     )
 
 
+def jetson_gpu_requested_nodes():
+    all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
+    requested = f"(sum by (node) ({all_requests}) > 0)"
+    return f"({requested}) and on(node) ({jetson_gpu_util_by_node()})"
+
+
 def namespace_share_expr(resource_expr):
     total = f"clamp_min(sum( {resource_expr} ), 1)"
     return f"100 * ( {resource_expr} ) / {total}"
@@ -384,14 +390,29 @@ def gpu_total_devices_expr():
 
 
 def unattributed_gpu_usage():
-    legacy_total = f"(sum({legacy_gpu_util_without_process_exporter()}) or on() vector(0))"
+    unresolved = (
+        f"({legacy_gpu_util_without_process_exporter()}) "
+        f"unless on(node) ({jetson_gpu_requested_nodes()})"
+    )
+    legacy_total = f"(sum({unresolved}) or on() vector(0))"
     return (
         f'label_replace(({legacy_total} > 0), "namespace", "unattributed", "", "")'
     )
 
 
 def gpu_utilization_raw(scope_var):
-    return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
+    nvidia = (
+        'label_replace('
+        f'{nvidia_process_gpu_usage_by_namespace(scope_var)}, '
+        '"gpu_source", "nvidia", "", "")'
+    )
+    jetson = (
+        'label_replace('
+        f'(({jetson_gpu_usage_by_namespace(scope_var)}) > 0), '
+        '"gpu_source", "jetson", "", "")'
+    )
+    attributed = f"sum by (namespace) (({nvidia}) or ({jetson}))"
+    return f"({attributed}) or ({unattributed_gpu_usage()})"
 
 
 def gpu_pool_used_expr(scope_var):
@@ -411,13 +432,20 @@ def namespace_gpu_share_expr(scope_var):
 
 
 PROBLEM_PODS_EXPR = (
-    'sum(max by (namespace,pod) (kube_pod_status_phase{phase!~"Running|Succeeded"})) '
-    "or on() vector(0)"
+    '((sum(max by(namespace,pod) ('
+    '(kube_pod_status_phase{phase="Pending",namespace!~"veles"} == 1) '
+    'and on(namespace,pod) ((time() - kube_pod_created{namespace!~"veles"}) > 900)'
+    ')) or on() vector(0)) + (sum(max by(namespace,pod) ('
+    '(kube_pod_status_phase{phase=~"Failed|Unknown",namespace!~"veles"} == 1) '
+    'unless on(namespace,pod) kube_pod_owner{owner_kind="Job"}'
+    ')) or on() vector(0)))'
 )
 CRASHLOOP_EXPR = (
-    'sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason'
-    '{reason=~"CrashLoopBackOff|ImagePullBackOff"})) '
-    "or on() vector(0)"
+    'sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason'
+    '{namespace!~"veles",reason=~"CrashLoopBackOff|ImagePullBackOff"} '
+    'and on(namespace,pod) '
+    '((time() - kube_pod_created{namespace!~"veles"}) > 900))) '
+    'or on() vector(0)'
 )
 STUCK_TERMINATING_EXPR = (
     'sum(max by (namespace,pod) ('
@@ -428,21 +456,24 @@ STUCK_TERMINATING_EXPR = (
 )
 UPTIME_WINDOW = "365d"
 # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series.
-UPTIME_RECORDING_METRIC = f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas"}}'
-UPTIME_RECORDING_EXPR = f"last_over_time({UPTIME_RECORDING_METRIC}[24h])"
-TRAEFIK_READY_EXPR = (
-    "("
-    'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
-    " / clamp_min("
-    'sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)'
-    ")"
+UPTIME_RECORDING_METRIC = (
+    f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="request-v4"}}'
 )
-CONTROL_READY_FRACTION_EXPR = (
-    f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
-    f" / {CONTROL_TOTAL})"
+AVAILABILITY_REQUESTS_1H_EXPR = (
+    'sum(increase(traefik_entrypoint_requests_total{'
+    'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))'
 )
-UPTIME_AVAIL_EXPR = (
-    f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
+AVAILABILITY_FAILURES_1H_EXPR = (
+    'sum(increase(traefik_entrypoint_requests_total{'
+    'entrypoint="websecure",protocol="http",code=~"5.."}[1h]))'
+)
+UPTIME_LIVE_FALLBACK_EXPR = (
+    f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / "
+    f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
+)
+UPTIME_RECORDING_EXPR = (
+    f"(last_over_time({UPTIME_RECORDING_METRIC}[48h]) "
+    f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
 )
 
 # Tie-breaker to deterministically pick one node per namespace when shares tie.
@@ -1870,9 +1901,9 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.",
     "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.",
     "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.",
-    "Atlas Availability (365d)": "Rolling one-year Atlas availability; higher is better, below target means users saw downtime.",
-    "Problem Pods": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.",
-    "CrashLoop / ImagePull": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.",
+    "Atlas Availability (365d)": "Request-weighted Atlas ingress availability; every server-side 5xx response counts as a failed request.",
+    "Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.",
+    "CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.",
     "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.",
     "Hottest node: CPU": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.",
     "Hottest node: RAM": "Highest worker memory use right now; lower is safer, high values risk evictions.",
@@ -1910,7 +1941,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
     "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
     "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
-    "Namespace GPU Utilization": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.",
+    "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
     "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
     "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
     "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",
@@ -2111,7 +2142,7 @@ def build_overview():
             "decimals": 4,
             "text_mode": "value",
             "instant": True,
-            "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data.",
+            "description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists.",
         },
         {
             "id": 4,
@@ -2335,10 +2366,10 @@ def build_overview():
     }
     overview_avg_coverage = f"(avg(({QUALITY_GATE_COVERAGE_BY_SUITE})) or on() vector(0))"
     overview_category_health = (
-        f'avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{'
+        f'(avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{'
         f'suite=~"{PLATFORM_TEST_SUITE_CANONICAL_MATCHER}",branch!="",branch=~"main|master|origin/main|origin/master",'
         f'category=~"{PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"'
-        "})"
+        '})) or label_set(vector(0), "category", "none")'
     )
     for panel_id, title, draw_expr, runtime_expr, y_pos in [
         (40, "Pyrphoros UPS Current", ANANKE_UPS_DRAW_WATTS_DB, ANANKE_UPS_RUNTIME_DB, 7),
@@ -5017,11 +5048,100 @@ def build_jobs_dashboard():
     }
 
 
+HERMES_TRIAGE_OPEN_EXPR = (
+    'sum(ariadne_hermes_triage_incident{status="human_required"})'
+)
+HERMES_TRIAGE_RESOLVED_EXPR = (
+    'sum(increase(ariadne_hermes_triage_action_total{result="success"}[24h]))'
+)
+HERMES_TRIAGE_DIAGNOSIS_EXPR = (
+    'ariadne_hermes_triage_duration_seconds{phase="diagnosis"}'
+)
+HERMES_TRIAGE_ACTIONS_EXPR = (
+    "sum by (action, result) "
+    "(increase(ariadne_hermes_triage_action_total[1h]))"
+)
+HERMES_TRIAGE_INCIDENTS_EXPR = (
+    "sum by (jenkins_job, status) (ariadne_hermes_triage_incident) > 0"
+)
+
+
+def _hermes_triage_panels():
+    """Return the Hermes automated-triage panels for the testing dashboard.
+
+    Inputs: none. Outputs: a list of Grafana panel definitions covering open
+    escalations, recent automated actions, diagnosis latency and per-job
+    incident state, sourced from Ariadne's bounded triage metrics.
+    """
+
+    return [
+        stat_panel(
+            600,
+            "Triage Escalations Awaiting a Human",
+            HERMES_TRIAGE_OPEN_EXPR,
+            {"h": 4, "w": 6, "x": 0, "y": 100},
+            instant=True,
+            description=(
+                "Incidents Hermes diagnosed where Ariadne refused to act "
+                "automatically. Each one has a Gitea issue when its job is "
+                "mapped, and fires HermesTriageHumanRequired."
+            ),
+        ),
+        stat_panel(
+            601,
+            "Automated Actions Succeeded (24h)",
+            HERMES_TRIAGE_RESOLVED_EXPR,
+            {"h": 4, "w": 6, "x": 6, "y": 100},
+            instant=True,
+            description=(
+                "Allowlisted actions Ariadne executed and completed: fixture "
+                "repair, transient-infra retry, or a pushed patch proposal."
+            ),
+        ),
+        stat_panel(
+            602,
+            "Hermes Diagnosis Time (s)",
+            HERMES_TRIAGE_DIAGNOSIS_EXPR,
+            {"h": 4, "w": 6, "x": 12, "y": 100},
+            unit="s",
+            decimals=1,
+            instant=True,
+            description=(
+                "Wall-clock time of the most recent Hermes Agent run. This is "
+                "the pause between a red build and a diagnosis."
+            ),
+        ),
+        timeseries_panel(
+            603,
+            "Triage Actions by Result (1h rate)",
+            HERMES_TRIAGE_ACTIONS_EXPR,
+            {"h": 8, "w": 12, "x": 0, "y": 104},
+            legend="{{action}} · {{result}}",
+            description=(
+                "requested/accepted/rejected/success/failed per action id. A "
+                "rejected action means an authorization gate refused it."
+            ),
+        ),
+        timeseries_panel(
+            604,
+            "Incident State by Job",
+            HERMES_TRIAGE_INCIDENTS_EXPR,
+            {"h": 8, "w": 12, "x": 12, "y": 104},
+            legend="{{jenkins_job}} · {{status}}",
+            description=(
+                "Lifecycle of each incident: detected, diagnosed, repairing, "
+                "awaiting_rebuild, resolved, human_required or failed."
+            ),
+        ),
+    ]
+
+
 def build_testing_dashboard():
     dashboard = build_jobs_dashboard()
     dashboard["uid"] = "atlas-testing"
     dashboard["folderUid"] = PUBLIC_DASHBOARD_FOLDER
     dashboard["editable"] = False
+    dashboard["panels"] = list(dashboard["panels"]) + _hermes_triage_panels()
     return dashboard
 
 
@@ -5490,12 +5610,13 @@ def build_gpu_dashboard():
     panels.append(
         table_panel(
             4,
-            "GPU Pods Reporting Device Util",
-            'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))',
+            "GPU Processes by Pod",
+            'topk(10, sum by (namespace,pod,node,process) '
+            '(nvidia_process_gpu_sm_util_percent{pod!="host"}) > 0)',
             {"h": 8, "w": 12, "x": 12, "y": 8},
             unit="percent",
             transformations=[{"id": "labelsToFields", "options": {}}],
-            description="DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.",
+            description="NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization.",
         )
     )
     return {

scripts/render/dashboards_render_logs.py

@@ -2,8 +2,8 @@
 """Generate OpenSearch Dashboards saved objects and render them into ConfigMaps.
 
 Usage:
-  scripts/dashboards_render_logs.py --build   # rebuild NDJSON + ConfigMap
-  scripts/dashboards_render_logs.py           # re-render ConfigMap from NDJSON
+  scripts/render/dashboards_render_logs.py --build   # rebuild NDJSON + ConfigMap
+  scripts/render/dashboards_render_logs.py           # re-render ConfigMap from NDJSON
 """
 
 from __future__ import annotations
@@ -14,14 +14,21 @@ import textwrap
 from dataclasses import dataclass
 from pathlib import Path
 
-ROOT = Path(__file__).resolve().parents[1]
+def repo_root() -> Path:
+    for path in Path(__file__).resolve().parents:
+        if (path / "clusters").is_dir() and (path / "services").is_dir():
+            return path
+    raise RuntimeError("could not locate repository root")  # pragma: no cover
+
+
+ROOT = repo_root()
 DASHBOARD_DIR = ROOT / "services" / "logging" / "dashboards"
 NDJSON_PATH = DASHBOARD_DIR / "logs.ndjson"
 CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-dashboards-objects.yaml"
 
 CONFIG_TEMPLATE = textwrap.dedent(
     """# {relative_path}
-# Generated by scripts/dashboards_render_logs.py --build
+# Generated by scripts/render/dashboards_render_logs.py --build
 apiVersion: v1
 kind: ConfigMap
 metadata:

scripts/render/flux_inventory.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Print a compact inventory of Flux Kustomization resources."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _iter_docs(path: Path) -> list[dict[str, Any]]:
+    docs: list[dict[str, Any]] = []
+    for raw in yaml.safe_load_all(path.read_text(encoding="utf-8")):
+        if isinstance(raw, dict):
+            docs.append(raw)
+    return docs
+
+
+def _is_flux_kustomization(doc: dict[str, Any]) -> bool:
+    return (
+        doc.get("kind") == "Kustomization"
+        and str(doc.get("apiVersion") or "").startswith("kustomize.toolkit.fluxcd.io/")
+    )
+
+
+def _depends_on(spec: dict[str, Any]) -> str:
+    names = []
+    for dep in spec.get("dependsOn") or []:
+        if isinstance(dep, dict) and dep.get("name"):
+            names.append(str(dep["name"]))
+    return ",".join(names)
+
+
+def _row(doc: dict[str, Any], path: Path) -> dict[str, str]:
+    metadata = doc.get("metadata") or {}
+    spec = doc.get("spec") or {}
+    annotations = metadata.get("annotations") or {}
+    return {
+        "name": str(metadata.get("name") or ""),
+        "path": str(spec.get("path") or ""),
+        "namespace": str(spec.get("targetNamespace") or ""),
+        "depends": _depends_on(spec),
+        "prune": str(spec.get("prune", "")),
+        "wait": str(spec.get("wait", "")),
+        "suspend": str(spec.get("suspend", False)),
+        "reason": str(annotations.get("atlas.bstein.dev/suspend-reason") or ""),
+        "file": str(path),
+    }
+
+
+def build_inventory(root: Path) -> list[dict[str, str]]:
+    rows: list[dict[str, str]] = []
+    for path in sorted(root.rglob("*.yaml")):
+        for doc in _iter_docs(path):
+            if _is_flux_kustomization(doc):
+                rows.append(_row(doc, path))
+    return sorted(rows, key=lambda item: item["name"])
+
+
+def _print_table(rows: list[dict[str, str]]) -> None:
+    columns = ["name", "path", "namespace", "depends", "prune", "wait", "suspend", "reason"]
+    widths = {
+        column: max([len(column), *[len(row[column]) for row in rows]]) for column in columns
+    }
+    header = "  ".join(column.ljust(widths[column]) for column in columns)
+    print(header)
+    print("  ".join("-" * widths[column] for column in columns))
+    for row in rows:
+        print("  ".join(row[column].ljust(widths[column]) for column in columns))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("root", nargs="?", default="clusters/atlas/flux-system")
+    args = parser.parse_args()
+
+    rows = build_inventory(Path(args.root))
+    _print_table(rows)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/render/knowledge_render_atlas.py

@@ -25,7 +25,14 @@ from typing import Any, Iterable
 
 import yaml
 
-REPO_ROOT = Path(__file__).resolve().parents[1]
+def repo_root() -> Path:
+    for path in Path(__file__).resolve().parents:
+        if (path / "clusters").is_dir() and (path / "services").is_dir():
+            return path
+    raise RuntimeError("could not locate repository root")  # pragma: no cover
+
+
+REPO_ROOT = repo_root()
 DASHBOARD_DIR = REPO_ROOT / "services" / "monitoring" / "dashboards"
 
 CLUSTER_SCOPED_KINDS = {
@@ -580,7 +587,7 @@ def main() -> int:
     catalog_rel = catalog_path.relative_to(REPO_ROOT).as_posix()
     catalog_path.write_text(
         f"# {catalog_rel}\n"
-        "# Generated by scripts/knowledge_render_atlas.py (do not edit by hand)\n"
+        "# Generated by scripts/render/knowledge_render_atlas.py (do not edit by hand)\n"
         + yaml.safe_dump(catalog, sort_keys=False),
         encoding="utf-8",
     )

scripts/render/logging_render_observability.py

@@ -2,8 +2,8 @@
 """Generate OpenSearch Observability seed objects and render them into ConfigMaps.
 
 Usage:
-  scripts/logging_render_observability.py --build   # rebuild JSON + ConfigMap
-  scripts/logging_render_observability.py           # re-render ConfigMap from JSON
+  scripts/render/logging_render_observability.py --build   # rebuild JSON + ConfigMap
+  scripts/render/logging_render_observability.py           # re-render ConfigMap from JSON
 """
 
 from __future__ import annotations
@@ -14,7 +14,14 @@ import textwrap
 from dataclasses import dataclass
 from pathlib import Path
 
-ROOT = Path(__file__).resolve().parents[1]
+def repo_root() -> Path:
+    for path in Path(__file__).resolve().parents:
+        if (path / "clusters").is_dir() and (path / "services").is_dir():
+            return path
+    raise RuntimeError("could not locate repository root")  # pragma: no cover
+
+
+ROOT = repo_root()
 OBS_DIR = ROOT / "services" / "logging" / "observability"
 APPS_PATH = OBS_DIR / "applications.json"
 QUERIES_PATH = OBS_DIR / "saved_queries.json"
@@ -23,7 +30,7 @@ CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-observability-objects.
 
 CONFIG_TEMPLATE = textwrap.dedent(
     """# {relative_path}
-# Generated by scripts/logging_render_observability.py --build
+# Generated by scripts/render/logging_render_observability.py --build
 apiVersion: v1
 kind: ConfigMap
 metadata: