agent: replace OpenClaw with Hermes

This commit is contained in:
jenkins 2026-07-21 21:02:06 -03:00
parent e90ed5752c
commit 97ecb36b56
26 changed files with 399 additions and 550 deletions

View File

@ -0,0 +1,35 @@
# clusters/atlas/flux-system/applications/hermes/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/hermes
targetNamespace: hermes
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 30m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-ollama
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes
namespace: hermes
dependsOn:
- name: cert-manager
- name: core
- name: keycloak
- name: longhorn
- name: openclaw
- name: traefik

View File

@ -27,6 +27,7 @@ resources:
- jenkins/kustomization.yaml
- ai-llm/kustomization.yaml
- openclaw/kustomization.yaml
- hermes/kustomization.yaml
- game-stream/kustomization.yaml
- veles/kustomization.yaml
- typhon/kustomization.yaml

View File

@ -16,19 +16,8 @@ spec:
name: flux-system
namespace: flux-system
wait: true
timeout: 30m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: openclaw-ollama
namespace: openclaw
- apiVersion: apps/v1
kind: Deployment
name: openclaw
namespace: openclaw
timeout: 5m
dependsOn:
- name: cert-manager
- name: core
- name: longhorn
- name: traefik

View File

@ -70,6 +70,12 @@ items:
name: atlas-default-compute
namespace: health
spec: *defaultCompute
- apiVersion: v1
kind: LimitRange
metadata:
name: atlas-default-compute
namespace: hermes
spec: *defaultCompute
- apiVersion: v1
kind: LimitRange
metadata:

View File

@ -138,7 +138,7 @@ STARTUP_IGNORE_WORKLOADS_REGEX="${STARTUP_IGNORE_WORKLOADS_REGEX:-}"
STARTUP_WORKLOAD_NAMESPACE_EXCLUDES_REGEX="${STARTUP_WORKLOAD_NAMESPACE_EXCLUDES_REGEX:-^(kube-system|kube-public|kube-node-lease|flux-system)$}"
STARTUP_OPTIONAL_KUSTOMIZATIONS="${STARTUP_OPTIONAL_KUSTOMIZATIONS:-}"
RECOVERY_FLUX_OPTIONAL_KUSTOMIZATIONS="${RECOVERY_FLUX_OPTIONAL_KUSTOMIZATIONS:-ai-llm,bstein-dev-home-migrations,descheduler,finance,game-stream,gitops-ui,health,jellyfin,jenkins,longhorn-ui,mailu,nextcloud,nextcloud-mail-sync,outline,planka,quality,resource-guardrails,typhon,vaultwarden,veles,wallet-monero-temp,xmr-miner}"
RECOVERY_FLUX_CRITICAL_KUSTOMIZATIONS="${RECOVERY_FLUX_CRITICAL_KUSTOMIZATIONS:-core,helm,cert-manager,longhorn-adopt,longhorn,metallb,traefik,vault-csi,vault-injector,vault,postgres,harbor,gitea,keycloak,oauth2-proxy,openldap,openclaw,monitoring,bstein-dev-home,comms,crypto,logging,maintenance,monerod,sui-metrics}"
RECOVERY_FLUX_CRITICAL_KUSTOMIZATIONS="${RECOVERY_FLUX_CRITICAL_KUSTOMIZATIONS:-core,helm,cert-manager,longhorn-adopt,longhorn,metallb,traefik,vault-csi,vault-injector,vault,postgres,harbor,gitea,keycloak,oauth2-proxy,openldap,hermes,monitoring,bstein-dev-home,comms,crypto,logging,maintenance,monerod,sui-metrics}"
RECOVERY_FLUX_CRITICAL_HELMRELEASES="${RECOVERY_FLUX_CRITICAL_HELMRELEASES:-cert-manager/cert-manager,comms/othrys-element,comms/othrys-synapse,harbor/harbor,kube-system/secrets-store-csi-driver,logging/data-prepper,logging/fluent-bit,logging/opensearch,logging/opensearch-dashboards,logging/otel-collector,longhorn-system/longhorn,metallb-system/metallb,monitoring/alertmanager,monitoring/grafana,monitoring/kube-state-metrics,monitoring/node-exporter,monitoring/victoria-metrics-single,vault/vault-injector}"
RECOVERY_FLUX_OPTIONAL_SNAPSHOT_FILE="${RECOVERY_FLUX_OPTIONAL_SNAPSHOT_FILE:-${HOME}/${STATE_SUBDIR:-.local/share/ananke}/longhorn_unlock_optional_flux.tsv}"
RECOVERY_FLUX_RESTART_KUSTOMIZE_CONTROLLER="${RECOVERY_FLUX_RESTART_KUSTOMIZE_CONTROLLER:-1}"
@ -1428,7 +1428,7 @@ logging deployment oauth2-proxy-logs
longhorn-system deployment oauth2-proxy-longhorn
maintenance deployment oauth2-proxy-metis
maintenance deployment oauth2-proxy-soteria
openclaw deployment oauth2-proxy-agent
hermes deployment hermes
quality deployment oauth2-proxy-sonarqube
quality deployment sonarqube-exporter
sso deployment oauth2-proxy

View File

@ -1,9 +1,9 @@
# services/openclaw/agent-certificate.yaml
# services/hermes/agent-certificate.yaml
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: agent-tls
namespace: openclaw
namespace: hermes
spec:
secretName: agent-tls
issuerRef:

View File

@ -1,9 +1,9 @@
# services/openclaw/agent-ingress.yaml
# services/hermes/agent-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: agent
namespace: openclaw
namespace: hermes
annotations:
cert-manager.io/cluster-issuer: letsencrypt
traefik.ingress.kubernetes.io/router.entrypoints: websecure
@ -21,6 +21,6 @@ spec:
pathType: Prefix
backend:
service:
name: oauth2-proxy-agent
name: hermes
port:
number: 80
name: dashboard

View File

@ -0,0 +1,124 @@
# services/hermes/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: hermes-config
namespace: hermes
labels:
app: hermes
data:
config.yaml: |
model:
provider: custom
default: qwen2.5:7b-instruct-q4_0
model: qwen2.5:7b-instruct-q4_0
base_url: http://hermes-ollama.hermes.svc.cluster.local:11434/v1
api_key: ollama
terminal:
backend: local
cwd: /opt/data/workspace
timeout: 180
home_mode: auto
approvals:
mode: manual
deny:
- "*kubectl apply*"
- "*kubectl delete*"
- "*kubectl patch*"
- "*kubectl scale*"
- "*kubectl cordon*"
- "*kubectl uncordon*"
- "*kubectl drain*"
- "*kubectl rollout restart*"
- "*flux suspend*"
- "*flux resume*"
- "*flux reconcile*"
- "*vault kv get*"
- "*kubectl get secret*"
- "*kubectl describe secret*"
dashboard:
public_url: https://agent.bstein.dev
oauth:
provider: self-hosted
self_hosted:
issuer: https://sso.bstein.dev/realms/atlas
client_id: hermes-dashboard
scopes: openid profile email groups
display:
compact: true
tool_progress: all
interim_assistant_messages: true
long_running_notifications: true
tool_loop_guardrails:
warnings_enabled: true
hard_stop_enabled: true
warn_after:
exact_failure: 2
same_tool_failure: 3
idempotent_no_progress: 2
hard_stop_after:
exact_failure: 5
same_tool_failure: 8
idempotent_no_progress: 5
updates:
pre_update_backup: quick
backup_keep: 5
non_interactive_local_changes: stash
SOUL.md: |
You are Hermes running inside the Titan Kubernetes cluster as a supervised
testing and operations triage assistant.
Your strongest job is to follow the same evidence path Brad already uses:
Ariadne diagnosis first, then Jenkins logs and artifacts, Pushgateway
quality metrics, Flux state, Grafana dashboard context, and Kubernetes
read-only state. Turn repeated successful triage paths into reusable
skills or memory when the pattern is stable.
Stay Flux-first. Do not mutate the cluster directly. Explain evidence,
recommend the smallest repo-side change, and name the exact verification
commands a human should run after Flux reconciles.
AGENTS.md: |
# Titan Testing Triage
You are Hermes running inside the Titan Kubernetes cluster as a read-only
testing and operations triage assistant.
Ariadne owns deterministic evidence collection and local diagnosis. Start
every testing triage by reading:
- `curl -sS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"`
- if that is missing or stale, ask a human to run:
`curl -sS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"`
- if the diagnosis is unavailable, fall back to:
`curl -sS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"`
Treat the Ariadne diagnosis and evidence bundle as the source of truth.
Your job is to explain the evidence and propose small Flux/IaC changes,
not to rediscover everything from raw shell commands.
Useful read-only commands:
- `curl -sS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"`
- `curl -sS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"`
- `kubectl get nodes -o wide`
- `kubectl get pods -A -o wide`
- `kubectl get pods -A --field-selector status.phase!=Running,status.phase!=Succeeded -o wide`
- `kubectl get events -A --sort-by=.lastTimestamp`
- `kubectl -n <namespace> describe pod <pod>`
- `kubectl -n <namespace> logs <pod> --all-containers --tail=200`
- `kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io`
- `curl -sS "$VICTORIA_METRICS_URL/api/v1/query?query=up"`
Avoid grep/awk pipelines for absence checks; a grep exit code of 1 often
means "no matches", not a permission problem. Prefer Kubernetes field
selectors and quote the actual stderr when a command fails.
Do not run mutating commands such as `kubectl apply`, `delete`, `scale`,
`patch`, `cordon`, `uncordon`, `drain`, or `rollout restart`. Do not read
Kubernetes Secret values. Draft repo changes or operator steps instead.

View File

@ -1,11 +1,11 @@
# services/openclaw/deployment.yaml
# services/hermes/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: openclaw
namespace: openclaw
name: hermes
namespace: hermes
labels:
app: openclaw
app: hermes
spec:
replicas: 1
revisionHistoryLimit: 2
@ -13,22 +13,21 @@ spec:
type: Recreate
selector:
matchLabels:
app: openclaw
app: hermes
template:
metadata:
labels:
app: openclaw
app: hermes
annotations:
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
ai.bstein.dev/instructions: kubectl-field-selectors
ai.bstein.dev/role: testing-triage
ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
ai.bstein.dev/config-rev: "20260521-oauth-device-bypass"
ai.bstein.dev/config-rev: "20260721-hermes-replacement"
spec:
serviceAccountName: openclaw-triage
serviceAccountName: hermes-triage
automountServiceAccountToken: true
securityContext:
fsGroup: 1000
fsGroup: 10000
seccompProfile:
type: RuntimeDefault
affinity:
@ -72,32 +71,6 @@ spec:
values:
- rpi4
initContainers:
- name: init-permissions
image: busybox:1.37
imagePullPolicy: IfNotPresent
command:
- sh
- -c
- |
set -e
chown -R 1000:1000 /home/node/.openclaw /home/node/.local/bin /tmp
securityContext:
runAsUser: 0
runAsGroup: 0
volumeMounts:
- name: home
mountPath: /home/node/.openclaw
- name: tmp
mountPath: /tmp
- name: tools
mountPath: /home/node/.local/bin
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
- name: init-config
image: busybox:1.37
imagePullPolicy: IfNotPresent
@ -105,18 +78,28 @@ spec:
- sh
- -c
- |
set -e
cp /config/openclaw.json /home/node/.openclaw/openclaw.json
mkdir -p /home/node/.openclaw/workspace
cp /config/AGENTS.md /home/node/.openclaw/workspace/AGENTS.md
set -eu
mkdir -p /opt/data/workspace /opt/data/home/.local/bin /opt/data/logs
cp /config/config.yaml /opt/data/config.yaml
cp /config/SOUL.md /opt/data/SOUL.md
cp /config/AGENTS.md /opt/data/workspace/AGENTS.md
touch /opt/data/.env
if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then
api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')"
printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> /opt/data/.env
fi
chmod 0600 /opt/data/.env
chown -R 10000:10000 /opt/data
securityContext:
runAsUser: 1000
runAsGroup: 1000
runAsUser: 0
runAsGroup: 0
volumeMounts:
- name: home
mountPath: /home/node/.openclaw
mountPath: /opt/data
- name: config
mountPath: /config
- name: tools
mountPath: /opt/data/home/.local/bin
resources:
requests:
cpu: 25m
@ -134,6 +117,7 @@ spec:
set -e
cp "$(command -v kubectl)" /tools/kubectl
chmod 0755 /tools/kubectl
chown 10000:10000 /tools/kubectl
volumeMounts:
- name: tools
mountPath: /tools
@ -145,27 +129,48 @@ spec:
cpu: 100m
memory: 64Mi
containers:
- name: gateway
image: ghcr.io/openclaw/openclaw:slim@sha256:ca134367c291b982ec301038af4602f397c7f90cc6c6d263c12a78a5fe1bde57
- name: hermes
image: nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
imagePullPolicy: IfNotPresent
command:
- node
- /app/dist/index.js
args:
- gateway
- run
ports:
- name: gateway
containerPort: 18789
- name: api
containerPort: 8642
protocol: TCP
- name: dashboard
containerPort: 9119
protocol: TCP
env:
- name: HERMES_HOME
value: /opt/data
- name: HOME
value: /home/node
- name: OPENCLAW_CONFIG_DIR
value: /home/node/.openclaw
- name: NODE_ENV
value: production
value: /opt/data/home
- name: PATH
value: /home/node/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
- name: HERMES_DASHBOARD
value: "1"
- name: HERMES_DASHBOARD_HOST
value: 0.0.0.0
- name: HERMES_DASHBOARD_PORT
value: "9119"
- name: HERMES_DASHBOARD_PUBLIC_URL
value: https://agent.bstein.dev
- name: HERMES_DASHBOARD_OIDC_ISSUER
value: https://sso.bstein.dev/realms/atlas
- name: HERMES_DASHBOARD_OIDC_CLIENT_ID
value: hermes-dashboard
- name: HERMES_DASHBOARD_OIDC_SCOPES
value: openid profile email groups
- name: API_SERVER_ENABLED
value: "true"
- name: API_SERVER_HOST
value: 0.0.0.0
- name: API_SERVER_PORT
value: "8642"
- name: API_SERVER_CORS_ORIGINS
value: https://agent.bstein.dev
- name: VICTORIA_METRICS_URL
value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
- name: ARIADNE_BASE_URL
@ -178,53 +183,36 @@ spec:
value: https://metrics.bstein.dev
volumeMounts:
- name: home
mountPath: /home/node/.openclaw
- name: tmp
mountPath: /tmp
mountPath: /opt/data
- name: tools
mountPath: /home/node/.local/bin
mountPath: /opt/data/home/.local/bin
readinessProbe:
exec:
command:
- node
- -e
- "require('http').get('http://127.0.0.1:18789/readyz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"
initialDelaySeconds: 20
httpGet:
path: /api/status
port: dashboard
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
exec:
command:
- node
- -e
- "require('http').get('http://127.0.0.1:18789/healthz', r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1))"
initialDelaySeconds: 60
httpGet:
path: /api/status
port: dashboard
initialDelaySeconds: 90
periodSeconds: 30
timeoutSeconds: 10
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
resources:
requests:
cpu: 250m
memory: 512Mi
cpu: 500m
memory: 1Gi
limits:
cpu: "1"
memory: 2Gi
cpu: "2"
memory: 4Gi
volumes:
- name: home
persistentVolumeClaim:
claimName: openclaw-home
claimName: hermes-home
- name: config
configMap:
name: openclaw-config
- name: tmp
emptyDir: {}
name: hermes-config
- name: tools
emptyDir: {}

View File

@ -0,0 +1,14 @@
# services/hermes/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: hermes
resources:
- namespace.yaml
- configmap.yaml
- rbac.yaml
- pvc.yaml
- ollama-deployment.yaml
- deployment.yaml
- service.yaml
- agent-certificate.yaml
- agent-ingress.yaml

View File

@ -0,0 +1,7 @@
# services/hermes/namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: hermes
labels:
app.kubernetes.io/name: hermes

View File

@ -1,22 +1,22 @@
# services/openclaw/ollama-deployment.yaml
# services/hermes/ollama-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: openclaw-ollama
namespace: openclaw
name: hermes-ollama
namespace: hermes
labels:
app: openclaw-ollama
app: hermes-ollama
spec:
revisionHistoryLimit: 2
strategy:
type: Recreate
selector:
matchLabels:
app: openclaw-ollama
app: hermes-ollama
template:
metadata:
labels:
app: openclaw-ollama
app: hermes-ollama
annotations:
ai.bstein.dev/model: qwen2.5:7b-instruct-q4_0
ai.bstein.dev/gpu: accelerator MVP lane (titan-24)

15
services/hermes/pvc.yaml Normal file
View File

@ -0,0 +1,15 @@
# services/hermes/pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: hermes-home
namespace: hermes
labels:
app: hermes
spec:
accessModes:
- ReadWriteOnce
storageClassName: astreae
resources:
requests:
storage: 4Gi

View File

@ -1,14 +1,14 @@
# services/openclaw/rbac.yaml
# services/hermes/rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: openclaw-triage
namespace: openclaw
name: hermes-triage
namespace: hermes
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: openclaw-triage-readonly
name: hermes-triage-readonly
rules:
- apiGroups: [""]
resources:
@ -58,13 +58,13 @@ rules:
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: openclaw-triage-readonly
name: hermes-triage-readonly
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: openclaw-triage-readonly
name: hermes-triage-readonly
subjects:
- kind: ServiceAccount
name: openclaw-triage
namespace: openclaw
name: hermes-triage
namespace: hermes

View File

@ -0,0 +1,38 @@
# services/hermes/service.yaml
apiVersion: v1
kind: Service
metadata:
name: hermes
namespace: hermes
labels:
app: hermes
spec:
type: ClusterIP
selector:
app: hermes
ports:
- name: api
port: 8642
targetPort: api
protocol: TCP
- name: dashboard
port: 9119
targetPort: dashboard
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: hermes-ollama
namespace: hermes
labels:
app: hermes-ollama
spec:
type: ClusterIP
selector:
app: hermes-ollama
ports:
- name: http
port: 11434
targetPort: http
protocol: TCP

View File

@ -26,7 +26,7 @@ resources:
- oneoffs/metis-oidc-secret-ensure-job.yaml
- oneoffs/soteria-oidc-secret-ensure-job.yaml
- oneoffs/quality-oidc-secret-ensure-job.yaml
- oneoffs/agent-oidc-secret-ensure-job.yaml
- oneoffs/hermes-dashboard-oidc-client-job.yaml
- oneoffs/veles-realm-ensure-job.yaml
- oneoffs/veles-gitea-oidc-secret-ensure-job.yaml
- oneoffs/metis-ssh-keys-secret-ensure-job.yaml
@ -52,9 +52,9 @@ configMapGenerator:
- name: actual-oidc-secret-ensure-script
files:
- actual_oidc_secret_ensure.sh=scripts/actual_oidc_secret_ensure.sh
- name: agent-oidc-secret-ensure-script
- name: hermes-dashboard-oidc-client-script
files:
- agent_oidc_secret_ensure.sh=scripts/agent_oidc_secret_ensure.sh
- hermes_dashboard_oidc_client_ensure.sh=scripts/hermes_dashboard_oidc_client_ensure.sh
- name: veles-gitea-oidc-secret-ensure-script
files:
- veles_gitea_oidc_secret_ensure.sh=scripts/veles_gitea_oidc_secret_ensure.sh

View File

@ -1,16 +1,13 @@
# services/keycloak/oneoffs/agent-oidc-secret-ensure-job.yaml
# One-off job for sso/agent-oidc-secret-ensure-1.
# Purpose: ensure the agent UI oauth2-proxy OIDC client and Vault secret exist.
# Keep this manifest around; set suspend false and bump the suffix to rerun.
# services/keycloak/oneoffs/hermes-dashboard-oidc-client-job.yaml
# Purpose: ensure the Hermes dashboard public OIDC client exists in Keycloak.
# Bump the suffix if the immutable Job needs to be rerun after it completes.
apiVersion: batch/v1
kind: Job
metadata:
name: agent-oidc-secret-ensure-1
name: hermes-dashboard-oidc-client-ensure-1
namespace: sso
spec:
ttlSecondsAfterFinished: 3600
suspend: true
backoffLimit: 0
backoffLimit: 3
template:
metadata:
annotations:
@ -28,9 +25,9 @@ spec:
serviceAccountName: mas-secrets-ensure
restartPolicy: Never
volumes:
- name: agent-oidc-secret-ensure-script
- name: hermes-dashboard-oidc-client-script
configMap:
name: agent-oidc-secret-ensure-script
name: hermes-dashboard-oidc-client-script
defaultMode: 0555
affinity:
nodeAffinity:
@ -45,8 +42,8 @@ spec:
containers:
- name: apply
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
command: ["/scripts/agent_oidc_secret_ensure.sh"]
command: ["/scripts/hermes_dashboard_oidc_client_ensure.sh"]
volumeMounts:
- name: agent-oidc-secret-ensure-script
- name: hermes-dashboard-oidc-client-script
mountPath: /scripts
readOnly: true

View File

@ -1,11 +1,10 @@
#!/usr/bin/env sh
set -euo pipefail
set -eu
. /vault/secrets/keycloak-admin-env.sh
CLIENT_NAME="agent"
CLIENT_NAME="hermes-dashboard"
PUBLIC_URL="https://agent.bstein.dev"
VAULT_SECRET_PATH="openclaw/agent-oidc"
KC_URL="http://keycloak.sso.svc.cluster.local"
ACCESS_TOKEN=""
@ -16,6 +15,7 @@ for attempt in 1 2 3 4 5 6 7 8 9 10; do
echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2
sleep $((attempt * 2))
done
for attempt in 1 2 3 4 5; do
TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \
-H 'Content-Type: application/x-www-form-urlencoded' \
@ -35,14 +35,32 @@ if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
exit 1
fi
client_payload="$(jq -nc \
--arg client_id "${CLIENT_NAME}" \
--arg redirect_uri "${PUBLIC_URL}/auth/callback" \
--arg web_origin "${PUBLIC_URL}" \
'{
clientId:$client_id,
enabled:true,
protocol:"openid-connect",
publicClient:true,
standardFlowEnabled:true,
implicitFlowEnabled:false,
directAccessGrantsEnabled:false,
serviceAccountsEnabled:false,
redirectUris:[$redirect_uri],
webOrigins:[$web_origin],
rootUrl:$web_origin,
baseUrl:"/",
attributes:{
"pkce.code.challenge.method":"S256",
"post.logout.redirect.uris":$web_origin
}
}')"
CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"$KC_URL/admin/realms/atlas/clients?clientId=${CLIENT_NAME}" || true)"
CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
client_payload="$(jq -nc \
--arg client_id "${CLIENT_NAME}" \
--arg redirect_uri "${PUBLIC_URL}/oauth2/callback" \
--arg web_origin "${PUBLIC_URL}" \
'{clientId:$client_id,enabled:true,protocol:"openid-connect",publicClient:false,standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false,serviceAccountsEnabled:false,redirectUris:[$redirect_uri],webOrigins:[$web_origin],rootUrl:$web_origin,baseUrl:"/"}')"
if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
@ -64,6 +82,16 @@ if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
exit 1
fi
status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${client_payload}" \
"$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}")"
if [ "$status" != "204" ]; then
echo "Keycloak client update failed (status ${status})" >&2
exit 1
fi
SCOPE_ID="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"$KC_URL/admin/realms/atlas/client-scopes?search=groups" | jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)"
if [ -z "$SCOPE_ID" ] || [ "$SCOPE_ID" = "null" ]; then
@ -92,77 +120,4 @@ if ! echo "$DEFAULT_SCOPES" | jq -e '.[] | select(.name=="groups")' >/dev/null 2
fi
fi
status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${client_payload}" \
"$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}")"
if [ "$status" != "204" ]; then
echo "Keycloak client update failed (status ${status})" >&2
exit 1
fi
CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/client-secret" | jq -r '.value' 2>/dev/null || true)"
if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
echo "Keycloak client secret not found" >&2
exit 1
fi
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
read_status="$(curl -sS -o /tmp/agent-oidc-read.json -w "%{http_code}" \
-H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/${VAULT_SECRET_PATH}" || true)"
COOKIE_SECRET=""
if [ "${read_status}" = "200" ]; then
COOKIE_SECRET="$(jq -r '.data.data.cookie_secret // empty' /tmp/agent-oidc-read.json)"
elif [ "${read_status}" != "404" ]; then
echo "Vault read failed (status ${read_status})" >&2
cat /tmp/agent-oidc-read.json >&2 || true
exit 1
fi
if [ -n "${COOKIE_SECRET}" ]; then
length="$(printf '%s' "${COOKIE_SECRET}" | wc -c | tr -d ' ')"
if [ "${length}" != "16" ] && [ "${length}" != "24" ] && [ "${length}" != "32" ]; then
COOKIE_SECRET=""
fi
fi
if [ -z "${COOKIE_SECRET}" ]; then
COOKIE_SECRET="$(openssl rand -hex 16 | tr -d '\n')"
fi
payload="$(jq -nc \
--arg client_id "${CLIENT_NAME}" \
--arg client_secret "${CLIENT_SECRET}" \
--arg cookie_secret "${COOKIE_SECRET}" \
'{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')"
write_status="$(curl -sS -o /tmp/agent-oidc-write.json -w "%{http_code}" -X POST \
-H "X-Vault-Token: ${vault_token}" \
-H 'Content-Type: application/json' \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/${VAULT_SECRET_PATH}")"
if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then
echo "Vault write failed (status ${write_status})" >&2
cat /tmp/agent-oidc-write.json >&2 || true
exit 1
fi
verify_status="$(curl -sS -o /tmp/agent-oidc-verify.json -w "%{http_code}" \
-H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/${VAULT_SECRET_PATH}" || true)"
if [ "${verify_status}" != "200" ]; then
echo "Vault verify failed (status ${verify_status})" >&2
cat /tmp/agent-oidc-verify.json >&2 || true
exit 1
fi
echo "Agent OIDC secret ready in Vault"
echo "Hermes dashboard OIDC client ready"

View File

@ -286,7 +286,7 @@ spec:
value: titan-24
- name: GAME_MODE_DISPLACE_WORKLOADS
value: >-
[{"kind":"Deployment","namespace":"openclaw","name":"openclaw-ollama","restoreReplicas":1}]
[{"kind":"Deployment","namespace":"hermes","name":"hermes-ollama","restoreReplicas":1}]
- name: WOLF_OIDC_CLIENT_ID
value: wolf
- name: WOLF_OIDC_BASE_URL
@ -444,7 +444,7 @@ spec:
- name: ARIADNE_SCHEDULE_TESTING_TRIAGE
value: "*/15 * * * *"
- name: ARIADNE_TESTING_TRIAGE_MODEL_URL
value: http://openclaw-ollama.openclaw.svc.cluster.local:11434
value: http://hermes-ollama.hermes.svc.cluster.local:11434
- name: ARIADNE_TESTING_TRIAGE_MODEL
value: qwen2.5:7b-instruct-q4_0
- name: ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC

View File

@ -0,0 +1,7 @@
# services/openclaw/NOTES.md
OpenClaw is retired in favor of Hermes. This kustomization intentionally keeps
only the namespace and `openclaw-home` PVC so the old agent state is available
for rollback or manual migration review.
Do not add new workloads here. New agent work belongs under `services/hermes/`.

View File

@ -1,6 +0,0 @@
# services/openclaw/agent-vault-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: agent-vault
namespace: openclaw

View File

@ -1,136 +0,0 @@
# services/openclaw/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: openclaw-config
namespace: openclaw
labels:
app: openclaw
data:
openclaw.json: |
{
"agents": {
"defaults": {
"workspace": "/home/node/.openclaw/workspace",
"model": {
"primary": "ollama-cluster/qwen2.5:7b-instruct-q4_0"
},
"models": {
"ollama-cluster/qwen2.5:7b-instruct-q4_0": {}
}
},
"list": [
{
"id": "testing-triage",
"name": "Titan Testing Triage",
"workspace": "/home/node/.openclaw/workspace"
}
]
},
"gateway": {
"mode": "local",
"auth": {
"mode": "trusted-proxy",
"trustedProxy": {
"userHeader": "x-forwarded-email"
}
},
"port": 18789,
"bind": "lan",
"trustedProxies": ["10.42.0.0/16"],
"controlUi": {
"enabled": true,
"dangerouslyDisableDeviceAuth": true,
"allowedOrigins": [
"http://localhost:18789",
"http://127.0.0.1:18789",
"https://agent.bstein.dev"
]
},
"tailscale": {
"mode": "off",
"resetOnExit": false
}
},
"session": {
"dmScope": "per-channel-peer"
},
"tools": {
"profile": "coding"
},
"models": {
"mode": "merge",
"providers": {
"ollama-cluster": {
"baseUrl": "http://openclaw-ollama.openclaw.svc.cluster.local:11434/v1",
"api": "openai-completions",
"apiKey": "ollama",
"models": [
{
"id": "qwen2.5:7b-instruct-q4_0",
"name": "qwen2.5:7b-instruct-q4_0 (Titan local)",
"contextWindow": 32768,
"maxTokens": 4096,
"input": ["text"],
"cost": {
"input": 0,
"output": 0,
"cacheRead": 0,
"cacheWrite": 0
},
"reasoning": false
}
]
}
}
}
}
AGENTS.md: |
# Titan Testing Triage
You are OpenClaw running inside the Titan Kubernetes cluster as a read-only
testing and operations triage assistant.
Ariadne owns deterministic evidence collection and local diagnosis. Start
every testing triage by reading:
- `curl -sS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"`
- if that is missing or stale, ask a human to run:
`curl -sS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"`
- if the diagnosis is unavailable, fall back to:
`curl -sS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"`
Treat the Ariadne diagnosis and evidence bundle as the source of truth. Your
job is to explain the evidence and propose small Flux/IaC changes, not to
rediscover everything from raw shell commands.
Your job is to explain failing or suspicious test runs without mutating the
cluster. Prefer concise incident summaries with:
- affected suite, namespace, pod, build, or node
- likely root cause
- exact evidence gathered
- the smallest suggested Flux/IaC change
- commands a human can run to verify the conclusion
Useful read-only commands:
- `curl -sS "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/latest"`
- `curl -sS -X POST "$ARIADNE_BASE_URL/api/internal/testing/triage/diagnosis/run"`
- `curl -sS "$ARIADNE_BASE_URL/api/internal/testing/triage/latest"`
- `kubectl get nodes -o wide`
- `kubectl get pods -A -o wide`
- `kubectl get pods -A --field-selector status.phase!=Running,status.phase!=Succeeded -o wide`
- `kubectl get events -A --sort-by=.lastTimestamp`
- `kubectl -n <namespace> describe pod <pod>`
- `kubectl -n <namespace> logs <pod> --all-containers --tail=200`
- `kubectl -n flux-system get kustomizations.kustomize.toolkit.fluxcd.io`
- `curl -sS "$VICTORIA_METRICS_URL/api/v1/query?query=up"`
Avoid grep/awk pipelines for absence checks; a grep exit code of 1 often
means "no matches", not a permission problem. Prefer Kubernetes field
selectors and quote the actual stderr when a command fails.
Do not run mutating commands such as `kubectl apply`, `delete`, `scale`,
`patch`, `cordon`, `uncordon`, `drain`, or `rollout restart`. Do not read
Kubernetes Secret values. Draft repo changes or operator steps instead.

View File

@ -4,13 +4,4 @@ kind: Kustomization
namespace: openclaw
resources:
- namespace.yaml
- configmap.yaml
- rbac.yaml
- agent-vault-serviceaccount.yaml
- pvc.yaml
- ollama-deployment.yaml
- deployment.yaml
- service.yaml
- oauth2-proxy-agent.yaml
- agent-certificate.yaml
- agent-ingress.yaml

View File

@ -1,139 +0,0 @@
# services/openclaw/oauth2-proxy-agent.yaml
apiVersion: v1
kind: Service
metadata:
name: oauth2-proxy-agent
namespace: openclaw
labels:
app: oauth2-proxy-agent
spec:
ports:
- name: http
port: 80
targetPort: 4180
selector:
app: oauth2-proxy-agent
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: oauth2-proxy-agent
namespace: openclaw
labels:
app: oauth2-proxy-agent
spec:
replicas: 2
selector:
matchLabels:
app: oauth2-proxy-agent
template:
metadata:
labels:
app: oauth2-proxy-agent
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/role: "openclaw"
vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/openclaw/agent-oidc"
vault.hashicorp.com/agent-inject-template-oidc-config: |
{{- with secret "kv/data/atlas/openclaw/agent-oidc" -}}
client_id = "{{ .Data.data.client_id }}"
client_secret = "{{ .Data.data.client_secret }}"
cookie_secret = "{{ .Data.data.cookie_secret }}"
{{- end -}}
spec:
serviceAccountName: agent-vault
nodeSelector:
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values: ["arm64"]
- key: kubernetes.io/hostname
operator: NotIn
values: ["titan-13", "titan-15", "titan-17", "titan-19"]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: atlas.bstein.dev/spillover
operator: DoesNotExist
- weight: 90
preference:
matchExpressions:
- key: hardware
operator: In
values: ["rpi5"]
- weight: 50
preference:
matchExpressions:
- key: hardware
operator: In
values: ["rpi4"]
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: oauth2-proxy-agent
containers:
- name: oauth2-proxy
image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0
imagePullPolicy: IfNotPresent
args:
- --provider=oidc
- --config=/vault/secrets/oidc-config
- --redirect-url=https://agent.bstein.dev/oauth2/callback
- --oidc-issuer-url=https://sso.bstein.dev/realms/atlas
- --scope=openid profile email groups
- --email-domain=*
- --allowed-group=admin
- --allowed-group=/admin
- --allowed-group=dev
- --allowed-group=/dev
- --set-xauthrequest=true
- --pass-access-token=true
- --set-authorization-header=true
- --reverse-proxy=true
- --cookie-name=_oauth2_proxy_agent
- --cookie-secure=true
- --cookie-samesite=lax
- --cookie-csrf-per-request=true
- --cookie-refresh=20m
- --cookie-expire=168h
- --insecure-oidc-allow-unverified-email=true
- --upstream=http://openclaw.openclaw.svc.cluster.local:18789
- --http-address=0.0.0.0:4180
- --skip-provider-button=true
- --approval-prompt=auto
- --skip-jwt-bearer-tokens=true
- --oidc-groups-claim=groups
- --cookie-domain=agent.bstein.dev
ports:
- containerPort: 4180
name: http
readinessProbe:
httpGet:
path: /ping
port: 4180
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /ping
port: 4180
initialDelaySeconds: 20
periodSeconds: 20
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 250m
memory: 256Mi

View File

@ -1,35 +0,0 @@
# services/openclaw/service.yaml
apiVersion: v1
kind: Service
metadata:
name: openclaw
namespace: openclaw
labels:
app: openclaw
spec:
type: ClusterIP
selector:
app: openclaw
ports:
- name: gateway
port: 18789
targetPort: gateway
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: openclaw-ollama
namespace: openclaw
labels:
app: openclaw-ollama
spec:
type: ClusterIP
selector:
app: openclaw-ollama
ports:
- name: http
port: 11434
targetPort: http
protocol: TCP

View File

@ -253,8 +253,6 @@ write_policy_and_role "health" "health" "health-vault-sync" \
"health/*" ""
write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \
"game-stream/*" ""
write_policy_and_role "openclaw" "openclaw" "agent-vault" \
"openclaw/*" ""
write_policy_and_role "veles" "veles" "veles-backend,veles-postgres,veles-vault-sync" \
"veles/* shared/harbor-pull shared/postmark-relay" ""
write_policy_and_role "veles-sim" "veles" "veles-sim" \
@ -289,7 +287,7 @@ write_policy_and_role "vault" "vault" "vault" \
write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
"shared/keycloak-admin shared/postmark-relay maintenance/metis-ssh-keys" \
"harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys openclaw/agent-oidc veles/veles-oidc gitea/gitea-veles-oidc" \
"harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc gitea/gitea-veles-oidc" \
'
path "kv/data/atlas/nodes/*" {
capabilities = ["create", "update", "read"]