diff --git a/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml new file mode 100644 index 000000000..38b6d761e --- /dev/null +++ b/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml @@ -0,0 +1,28 @@ +# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: hermes-chat + namespace: flux-system +spec: + interval: 10m + path: ./services/hermes-chat + targetNamespace: hermes-chat + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + wait: true + timeout: 30m + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: hermes-chat + namespace: hermes-chat + dependsOn: + - name: cert-manager + - name: core + - name: hermes + - name: keycloak + - name: longhorn diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index ec31a55c0..47cca710b 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -30,6 +30,10 @@ spec: kind: Deployment name: hermes namespace: hermes + - apiVersion: apps/v1 + kind: Deployment + name: oauth2-proxy-hermes + namespace: hermes dependsOn: - name: cert-manager - name: core diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index b5ca23784..671f019b8 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -28,6 +28,7 @@ resources: - ai-llm/kustomization.yaml - openclaw/kustomization.yaml - hermes/kustomization.yaml + - hermes-chat/kustomization.yaml - game-stream/kustomization.yaml - cassandra-auth/kustomization.yaml - cassandra/kustomization.yaml diff --git a/services/hermes-chat/certificate.yaml b/services/hermes-chat/certificate.yaml new file mode 100644 index 000000000..3bed14a1f --- /dev/null +++ b/services/hermes-chat/certificate.yaml @@ -0,0 +1,13 @@ +# services/hermes-chat/certificate.yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: chat-tls + namespace: hermes-chat +spec: + secretName: chat-tls + issuerRef: + kind: ClusterIssuer + name: letsencrypt + dnsNames: + - chat.bstein.dev diff --git a/services/hermes-chat/configmap.yaml b/services/hermes-chat/configmap.yaml new file mode 100644 index 000000000..f395ffba9 --- /dev/null +++ b/services/hermes-chat/configmap.yaml @@ -0,0 +1,142 @@ +# services/hermes-chat/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-chat-config + namespace: hermes-chat + labels: + app: hermes-chat +data: + config.yaml: | + model: + provider: custom + default: gpt-oss:20b + model: gpt-oss:20b + context_length: 64000 + base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 + api_key: ollama + + fallback_providers: + - provider: openai-codex + model: gpt-5.4 + + agent: + api_max_retries: 1 + + skills: + creation_nudge_interval: 15 + external_dirs: + - /opt/data/workspace/skills + + terminal: + backend: local + cwd: /opt/data/workspace + timeout: 180 + home_mode: profile + + approvals: + mode: manual + deny: + - "*kubectl apply*" + - "*kubectl create*" + - "*kubectl delete*" + - "*kubectl edit*" + - "*kubectl patch*" + - "*kubectl replace*" + - "*kubectl scale*" + - "*kubectl set*" + - "*kubectl label*" + - "*kubectl annotate*" + - "*kubectl cordon*" + - "*kubectl uncordon*" + - "*kubectl drain*" + - "*kubectl rollout restart*" + - "*kubectl rollout undo*" + - "*kubectl exec*" + - "*kubectl attach*" + - "*kubectl cp*" + - "*kubectl debug*" + - "*kubectl expose*" + - "*kubectl port-forward*" + - "*kubectl proxy*" + - "*kubectl run*" + - "*kubectl get secret*" + - "*kubectl describe secret*" + - "*flux reconcile*" + - "*flux suspend*" + - "*flux resume*" + - "*vault *" + - "*k3s *" + - "*crictl *" + - "*ctr *" + - "*169.254.169.254*" + - "*kubernetes.default*" + - "*/var/run/secrets/kubernetes.io*" + + dashboard: + public_url: https://chat.bstein.dev + oauth: + provider: self-hosted + self_hosted: + issuer: https://sso.bstein.dev/realms/atlas + client_id: hermes-chat-dashboard + scopes: openid profile email groups + + display: + compact: true + tool_progress: all + interim_assistant_messages: true + long_running_notifications: true + + tool_loop_guardrails: + warnings_enabled: true + hard_stop_enabled: true + warn_after: + exact_failure: 2 + same_tool_failure: 3 + idempotent_no_progress: 2 + hard_stop_after: + exact_failure: 5 + same_tool_failure: 8 + idempotent_no_progress: 5 + + updates: + pre_update_backup: quick + backup_keep: 3 + non_interactive_local_changes: stash + SOUL.md: | + You are a personal AI assistant and researcher for Atlas users. Be useful, + curious, careful, and direct. Help with research, writing, planning, + learning, files, personal automation, and creative work. Use tools when + they materially improve the answer and make reusable skills when a stable + workflow is worth keeping. + + Your environment is a private consumer sandbox. Work in the mounted + workspace and with public Internet services the user intentionally asks + you to use. You may inspect the Titan cluster through the dedicated + read-only Kubernetes identity to explain health and status. You do not + administer it: never attempt mutations, secret access, exec, attach, + port-forwarding, private-service access, or cloud metadata access. + AGENTS.md: | + # Personal Hermes workspace + + This Hermes instance is a personal chat, research, and automation + environment. User-created files and skills belong under `/opt/data/workspace`. + + You may: + + - research public Internet sources and cite them + - create, read, and edit files in the workspace + - create and improve reusable skills + - configure user-owned channels and provider integrations + - run ordinary local commands needed for the user's task + - inspect non-secret Kubernetes resources and pod logs with read-only + `kubectl get`, `describe`, and `logs` commands + + This container is not an infrastructure administration environment. The + Kubernetes identity is observation-only and cannot read Secrets or use pod + exec, attach, or port-forwarding. Do not attempt cluster mutations, private + service access, node LAN access, metadata services, Vault, container + runtimes, or the operator Hermes instance. If a request needs an action, + explain the evidence and say Brad must perform it from the operator + instance at `agent.bstein.dev`. diff --git a/services/hermes-chat/deployment.yaml b/services/hermes-chat/deployment.yaml new file mode 100644 index 000000000..ef3888c3d --- /dev/null +++ b/services/hermes-chat/deployment.yaml @@ -0,0 +1,220 @@ +# services/hermes-chat/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-chat + namespace: hermes-chat + labels: + app: hermes-chat +spec: + replicas: 1 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 1800 + strategy: + type: Recreate + selector: + matchLabels: + app: hermes-chat + template: + metadata: + labels: + app: hermes-chat + annotations: + ai.bstein.dev/model: gpt-oss:20b with independent openai-codex fallback + ai.bstein.dev/role: personal-chat-research + ai.bstein.dev/isolation: observer-only Kubernetes RBAC, private-service egress denied + spec: + serviceAccountName: hermes-chat + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-13 + - titan-15 + - titan-17 + - titan-19 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: atlas.bstein.dev/spillover + operator: DoesNotExist + - weight: 90 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi4 + initContainers: + - name: init-config + image: busybox:1.37 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -eu + mkdir -p /opt/data/workspace/skills /opt/data/home/.local/bin /opt/data/logs + cp /config/config.yaml /opt/data/config.yaml + cp /config/SOUL.md /opt/data/SOUL.md + cp /config/AGENTS.md /opt/data/workspace/AGENTS.md + touch /opt/data/.env + if ! grep -q '^API_SERVER_KEY=' /opt/data/.env; then + api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" + printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> /opt/data/.env + fi + chmod 0600 /opt/data/.env + chown -R 10000:10000 /opt/data + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: home + mountPath: /opt/data + - name: config + mountPath: /config + readOnly: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + - name: install-kubectl + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + set -eu + cp "$(command -v kubectl)" /tools/kubectl + chmod 0755 /tools/kubectl + chown 10000:10000 /tools/kubectl + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: tools + mountPath: /tools + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + containers: + - name: hermes-chat + image: nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 + imagePullPolicy: IfNotPresent + args: + - gateway + - run + ports: + - name: dashboard + containerPort: 9119 + protocol: TCP + env: + - name: HERMES_HOME + value: /opt/data + - name: HOME + value: /opt/data/home + - name: PATH + value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: HERMES_DASHBOARD + value: "1" + - name: HERMES_DASHBOARD_HOST + value: 0.0.0.0 + - name: HERMES_DASHBOARD_PORT + value: "9119" + - name: HERMES_DASHBOARD_PUBLIC_URL + value: https://chat.bstein.dev + - name: HERMES_DASHBOARD_OIDC_ISSUER + value: https://sso.bstein.dev/realms/atlas + - name: HERMES_DASHBOARD_OIDC_CLIENT_ID + value: hermes-chat-dashboard + - name: HERMES_DASHBOARD_OIDC_SCOPES + value: openid profile email groups + - name: API_SERVER_ENABLED + value: "true" + - name: API_SERVER_HOST + value: 0.0.0.0 + - name: API_SERVER_PORT + value: "8642" + - name: API_SERVER_CORS_ORIGINS + value: https://chat.bstein.dev + volumeMounts: + - name: home + mountPath: /opt/data + - name: tools + mountPath: /opt/data/home/.local/bin + readinessProbe: + httpGet: + path: /api/status + port: dashboard + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /api/status + port: dashboard + initialDelaySeconds: 90 + periodSeconds: 30 + timeoutSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 768Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 1Gi + volumes: + - name: home + persistentVolumeClaim: + claimName: hermes-chat-home + - name: config + configMap: + name: hermes-chat-config + - name: tools + emptyDir: {} diff --git a/services/hermes-chat/ingress.yaml b/services/hermes-chat/ingress.yaml new file mode 100644 index 000000000..e176231f5 --- /dev/null +++ b/services/hermes-chat/ingress.yaml @@ -0,0 +1,27 @@ +# services/hermes-chat/ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: chat + namespace: hermes-chat + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" +spec: + ingressClassName: traefik + tls: + - hosts: + - chat.bstein.dev + secretName: chat-tls + rules: + - host: chat.bstein.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hermes-chat + port: + name: dashboard diff --git a/services/hermes-chat/kustomization.yaml b/services/hermes-chat/kustomization.yaml new file mode 100644 index 000000000..4d3529d77 --- /dev/null +++ b/services/hermes-chat/kustomization.yaml @@ -0,0 +1,15 @@ +# services/hermes-chat/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hermes-chat +resources: + - namespace.yaml + - serviceaccount.yaml + - rbac.yaml + - configmap.yaml + - pvc.yaml + - deployment.yaml + - service.yaml + - networkpolicy.yaml + - certificate.yaml + - ingress.yaml diff --git a/services/hermes-chat/namespace.yaml b/services/hermes-chat/namespace.yaml new file mode 100644 index 000000000..8b607b8b9 --- /dev/null +++ b/services/hermes-chat/namespace.yaml @@ -0,0 +1,7 @@ +# services/hermes-chat/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: hermes-chat + labels: + app.kubernetes.io/name: hermes-chat diff --git a/services/hermes-chat/networkpolicy.yaml b/services/hermes-chat/networkpolicy.yaml new file mode 100644 index 000000000..c6cf5718d --- /dev/null +++ b/services/hermes-chat/networkpolicy.yaml @@ -0,0 +1,63 @@ +# services/hermes-chat/networkpolicy.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-chat-isolation + namespace: hermes-chat +spec: + podSelector: + matchLabels: + app: hermes-chat + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ports: + - protocol: TCP + port: 9119 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: hermes + podSelector: + matchLabels: + app: hermes-model-gate + ports: + - protocol: TCP + port: 8080 + - to: + - ipBlock: + cidr: 10.43.0.1/32 + ports: + - protocol: TCP + port: 443 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 diff --git a/services/hermes-chat/pvc.yaml b/services/hermes-chat/pvc.yaml new file mode 100644 index 000000000..f2588f791 --- /dev/null +++ b/services/hermes-chat/pvc.yaml @@ -0,0 +1,15 @@ +# services/hermes-chat/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hermes-chat-home + namespace: hermes-chat + labels: + app: hermes-chat +spec: + accessModes: + - ReadWriteOnce + storageClassName: astreae + resources: + requests: + storage: 2Gi diff --git a/services/hermes-chat/rbac.yaml b/services/hermes-chat/rbac.yaml new file mode 100644 index 000000000..4144da689 --- /dev/null +++ b/services/hermes-chat/rbac.yaml @@ -0,0 +1,62 @@ +# services/hermes-chat/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hermes-chat-observer +rules: + - apiGroups: [""] + resources: + - endpoints + - events + - namespaces + - nodes + - persistentvolumeclaims + - persistentvolumes + - pods + - pods/log + - replicationcontrollers + - services + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: + - ingresses + - networkpolicies + verbs: ["get", "list", "watch"] + - apiGroups: ["helm.toolkit.fluxcd.io"] + resources: + - helmreleases + verbs: ["get", "list", "watch"] + - apiGroups: ["kustomize.toolkit.fluxcd.io"] + resources: + - kustomizations + verbs: ["get", "list", "watch"] + - apiGroups: ["source.toolkit.fluxcd.io"] + resources: + - gitrepositories + - helmrepositories + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hermes-chat-observer +subjects: + - kind: ServiceAccount + name: hermes-chat + namespace: hermes-chat +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hermes-chat-observer diff --git a/services/hermes-chat/service.yaml b/services/hermes-chat/service.yaml new file mode 100644 index 000000000..8b74822ea --- /dev/null +++ b/services/hermes-chat/service.yaml @@ -0,0 +1,17 @@ +# services/hermes-chat/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hermes-chat + namespace: hermes-chat + labels: + app: hermes-chat +spec: + type: ClusterIP + selector: + app: hermes-chat + ports: + - name: dashboard + port: 9119 + targetPort: dashboard + protocol: TCP diff --git a/services/hermes-chat/serviceaccount.yaml b/services/hermes-chat/serviceaccount.yaml new file mode 100644 index 000000000..88eca224c --- /dev/null +++ b/services/hermes-chat/serviceaccount.yaml @@ -0,0 +1,7 @@ +# services/hermes-chat/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-chat + namespace: hermes-chat +automountServiceAccountToken: true diff --git a/services/hermes/agent-ingress.yaml b/services/hermes/agent-ingress.yaml index 4ef033cd0..9581a95b4 100644 --- a/services/hermes/agent-ingress.yaml +++ b/services/hermes/agent-ingress.yaml @@ -21,6 +21,6 @@ spec: pathType: Prefix backend: service: - name: hermes + name: oauth2-proxy-hermes port: - name: dashboard + name: http diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 7287255d9..3c7172a97 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization namespace: hermes resources: - namespace.yaml + - vault-serviceaccount.yaml - configmap.yaml - rbac.yaml - pvc.yaml @@ -16,6 +17,7 @@ resources: - ollama-deployment.yaml - deployment.yaml - service.yaml + - oauth2-proxy.yaml - agent-certificate.yaml - agent-ingress.yaml diff --git a/services/hermes/oauth2-proxy.yaml b/services/hermes/oauth2-proxy.yaml new file mode 100644 index 000000000..3ed20ce85 --- /dev/null +++ b/services/hermes/oauth2-proxy.yaml @@ -0,0 +1,160 @@ +# services/hermes/oauth2-proxy.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-operator-allowlist + namespace: hermes +data: + allowed-emails: | + brad@bstein.dev +--- +apiVersion: v1 +kind: Service +metadata: + name: oauth2-proxy-hermes + namespace: hermes + labels: + app: oauth2-proxy-hermes +spec: + selector: + app: oauth2-proxy-hermes + ports: + - name: http + port: 80 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth2-proxy-hermes + namespace: hermes + labels: + app: oauth2-proxy-hermes +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: oauth2-proxy-hermes + template: + metadata: + labels: + app: oauth2-proxy-hermes + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: hermes + vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/operator-oidc + vault.hashicorp.com/agent-inject-template-oidc-config: | + {{- with secret "kv/data/atlas/hermes/operator-oidc" -}} + client_id = "{{ .Data.data.client_id }}" + client_secret = "{{ .Data.data.client_secret }}" + cookie_secret = "{{ .Data.data.cookie_secret }}" + {{- end -}} + spec: + serviceAccountName: hermes-vault + automountServiceAccountToken: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + - key: kubernetes.io/hostname + operator: NotIn + values: + - titan-13 + - titan-15 + - titan-17 + - titan-19 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 90 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi4 + containers: + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0@sha256:dcb6ff8dd21bf3058f6a22c6fa385fa5b897a9cd3914c88a2cc2bb0a85f8065d + imagePullPolicy: IfNotPresent + args: + - --provider=oidc + - --config=/vault/secrets/oidc-config + - --redirect-url=https://agent.bstein.dev/oauth2/callback + - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas + - --scope=openid profile email + - --email-domain=* + - --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails + - --set-xauthrequest=true + - --pass-access-token=true + - --set-authorization-header=true + - --cookie-secure=true + - --cookie-samesite=lax + - --cookie-refresh=20m + - --cookie-expire=24h + - --upstream=http://hermes.hermes.svc.cluster.local:9119 + - --http-address=0.0.0.0:4180 + - --skip-provider-button=true + - --skip-jwt-bearer-tokens=true + - --cookie-domain=agent.bstein.dev + - --reverse-proxy=true + ports: + - name: http + containerPort: 4180 + readinessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ping + port: http + initialDelaySeconds: 20 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: allowlist + mountPath: /etc/oauth2-proxy + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: allowlist + configMap: + name: hermes-operator-allowlist + - name: tmp + emptyDir: + sizeLimit: 64Mi diff --git a/services/hermes/vault-serviceaccount.yaml b/services/hermes/vault-serviceaccount.yaml new file mode 100644 index 000000000..911064861 --- /dev/null +++ b/services/hermes/vault-serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/hermes/vault-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-vault + namespace: hermes diff --git a/services/keycloak/kustomization.yaml b/services/keycloak/kustomization.yaml index 56677b8b5..ff6ae911f 100644 --- a/services/keycloak/kustomization.yaml +++ b/services/keycloak/kustomization.yaml @@ -27,6 +27,7 @@ resources: - oneoffs/soteria-oidc-secret-ensure-job.yaml - oneoffs/quality-oidc-secret-ensure-job.yaml - oneoffs/hermes-dashboard-oidc-client-job.yaml + - oneoffs/hermes-access-oidc-client-job.yaml - oneoffs/veles-realm-ensure-job.yaml - oneoffs/veles-gitea-oidc-secret-ensure-job.yaml - oneoffs/metis-ssh-keys-secret-ensure-job.yaml @@ -55,6 +56,9 @@ configMapGenerator: - name: hermes-dashboard-oidc-client-script files: - hermes_dashboard_oidc_client_ensure.sh=scripts/hermes_dashboard_oidc_client_ensure.sh + - name: hermes-access-oidc-script + files: + - hermes_access_oidc_ensure.sh=scripts/hermes_access_oidc_ensure.sh - name: veles-gitea-oidc-secret-ensure-script files: - veles_gitea_oidc_secret_ensure.sh=scripts/veles_gitea_oidc_secret_ensure.sh diff --git a/services/keycloak/oneoffs/hermes-access-oidc-client-job.yaml b/services/keycloak/oneoffs/hermes-access-oidc-client-job.yaml new file mode 100644 index 000000000..21a42e9e9 --- /dev/null +++ b/services/keycloak/oneoffs/hermes-access-oidc-client-job.yaml @@ -0,0 +1,57 @@ +# services/keycloak/oneoffs/hermes-access-oidc-client-job.yaml +# Purpose: create chat OIDC and Brad-only operator proxy clients. +apiVersion: batch/v1 +kind: Job +metadata: + name: hermes-access-oidc-client-ensure-1 + namespace: sso +spec: + backoffLimit: 3 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: sso-secrets + vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: kv/data/atlas/shared/keycloak-admin + vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: | + {{ with secret "kv/data/atlas/shared/keycloak-admin" }} + export KEYCLOAK_ADMIN="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}" + {{ end }} + spec: + serviceAccountName: mas-secrets-ensure + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: node-role.kubernetes.io/worker + operator: Exists + containers: + - name: apply + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + command: + - /scripts/hermes_access_oidc_ensure.sh + volumeMounts: + - name: script + mountPath: /scripts + readOnly: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + volumes: + - name: script + configMap: + name: hermes-access-oidc-script + defaultMode: 0555 diff --git a/services/keycloak/scripts/hermes_access_oidc_ensure.sh b/services/keycloak/scripts/hermes_access_oidc_ensure.sh new file mode 100755 index 000000000..ea68d2dc9 --- /dev/null +++ b/services/keycloak/scripts/hermes_access_oidc_ensure.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env sh +set -eu + +. /vault/secrets/keycloak-admin-env.sh + +KC_URL="http://keycloak.sso.svc.cluster.local" +CHAT_CLIENT="hermes-chat-dashboard" +CHAT_URL="https://chat.bstein.dev" +OPERATOR_CLIENT="hermes-operator-proxy" +OPERATOR_URL="https://agent.bstein.dev" + +ACCESS_TOKEN="" +for attempt in 1 2 3 4 5 6 7 8 9 10; do + if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then + break + fi + echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2 + sleep $((attempt * 2)) +done + +for attempt in 1 2 3 4 5; do + token_json="$(curl -sS -X POST "${KC_URL}/realms/master/protocol/openid-connect/token" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d "grant_type=password" \ + -d "client_id=admin-cli" \ + -d "username=${KEYCLOAK_ADMIN}" \ + -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)" + ACCESS_TOKEN="$(printf '%s' "${token_json}" | jq -r '.access_token' 2>/dev/null || true)" + if [ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ]; then + break + fi + echo "Keycloak token request failed (attempt ${attempt})" >&2 + sleep $((attempt * 2)) +done +if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then + echo "Failed to fetch Keycloak admin token" >&2 + exit 1 +fi + +chat_payload="$(jq -nc \ + --arg client_id "${CHAT_CLIENT}" \ + --arg redirect_uri "${CHAT_URL}/auth/callback" \ + --arg web_origin "${CHAT_URL}" \ + '{ + clientId:$client_id, + enabled:true, + protocol:"openid-connect", + publicClient:true, + standardFlowEnabled:true, + implicitFlowEnabled:false, + directAccessGrantsEnabled:false, + serviceAccountsEnabled:false, + redirectUris:[$redirect_uri], + webOrigins:[$web_origin], + rootUrl:$web_origin, + baseUrl:"/", + attributes:{ + "pkce.code.challenge.method":"S256", + "post.logout.redirect.uris":$web_origin + } + }')" + +chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)" +chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)" +if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${chat_payload}" \ + "${KC_URL}/admin/realms/atlas/clients")" + if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then + echo "Keycloak chat client create failed (status ${status})" >&2 + exit 1 + fi + chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)" + chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)" +fi +if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then + echo "Keycloak chat client not found after create" >&2 + exit 1 +fi + +status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${chat_payload}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}")" +if [ "${status}" != "204" ]; then + echo "Keycloak chat client update failed (status ${status})" >&2 + exit 1 +fi + +scope_id="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/client-scopes?search=groups" \ + | jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)" +if [ -z "${scope_id}" ] || [ "${scope_id}" = "null" ]; then + echo "Keycloak groups client scope not found" >&2 + exit 1 +fi +default_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}/default-client-scopes" || true)" +optional_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes" || true)" +if ! printf '%s' "${default_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1 \ + && ! printf '%s' "${optional_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")" + if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")" + if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then + echo "Failed to attach groups scope to chat client (status ${status})" >&2 + exit 1 + fi + fi +fi + +operator_payload="$(jq -nc \ + --arg client_id "${OPERATOR_CLIENT}" \ + --arg redirect_uri "${OPERATOR_URL}/oauth2/callback" \ + --arg web_origin "${OPERATOR_URL}" \ + '{ + clientId:$client_id, + enabled:true, + protocol:"openid-connect", + publicClient:false, + standardFlowEnabled:true, + implicitFlowEnabled:false, + directAccessGrantsEnabled:false, + serviceAccountsEnabled:false, + redirectUris:[$redirect_uri], + webOrigins:[$web_origin], + rootUrl:$web_origin, + baseUrl:"/", + attributes:{"post.logout.redirect.uris":$web_origin} + }')" + +operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)" +operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)" +if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${operator_payload}" \ + "${KC_URL}/admin/realms/atlas/clients")" + if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then + echo "Keycloak operator proxy client create failed (status ${status})" >&2 + exit 1 + fi + operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)" + operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)" +fi +if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then + echo "Keycloak operator proxy client not found after create" >&2 + exit 1 +fi + +status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${operator_payload}" \ + "${KC_URL}/admin/realms/atlas/clients/${operator_id}")" +if [ "${status}" != "204" ]; then + echo "Keycloak operator proxy client update failed (status ${status})" >&2 + exit 1 +fi + +client_secret="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${operator_id}/client-secret" \ + | jq -r '.value' 2>/dev/null || true)" +if [ -z "${client_secret}" ] || [ "${client_secret}" = "null" ]; then + echo "Keycloak operator proxy client secret not found" >&2 + exit 1 +fi + +vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" +vault_role="${VAULT_ROLE:-sso-secrets}" +jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" +login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt,role:$role}')" +vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" +if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "Vault login failed" >&2 + exit 1 +fi + +read_status="$(curl -sS -o /tmp/hermes-operator-oidc-read.json -w '%{http_code}' \ + -H "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc" || true)" +cookie_secret="" +if [ "${read_status}" = "200" ]; then + cookie_secret="$(jq -r '.data.data.cookie_secret // empty' /tmp/hermes-operator-oidc-read.json)" +elif [ "${read_status}" != "404" ]; then + echo "Vault operator OIDC read failed (status ${read_status})" >&2 + exit 1 +fi +if [ -n "${cookie_secret}" ]; then + cookie_length="$(printf '%s' "${cookie_secret}" | wc -c | tr -d ' ')" + if [ "${cookie_length}" != "16" ] && [ "${cookie_length}" != "24" ] && [ "${cookie_length}" != "32" ]; then + cookie_secret="" + fi +fi +if [ -z "${cookie_secret}" ]; then + cookie_secret="$(openssl rand -hex 16 | tr -d '\n')" +fi + +vault_payload="$(jq -nc \ + --arg client_id "${OPERATOR_CLIENT}" \ + --arg client_secret "${client_secret}" \ + --arg cookie_secret "${cookie_secret}" \ + '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')" +write_status="$(curl -sS -o /tmp/hermes-operator-oidc-write.json -w '%{http_code}' -X POST \ + -H "X-Vault-Token: ${vault_token}" \ + -H 'Content-Type: application/json' \ + -d "${vault_payload}" \ + "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc")" +if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then + echo "Vault operator OIDC write failed (status ${write_status})" >&2 + exit 1 +fi + +echo "Hermes chat and operator OIDC clients are ready" diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml new file mode 100644 index 000000000..2d794291a --- /dev/null +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -0,0 +1,54 @@ +# services/vault/hermes-auth-role-bootstrap-job.yaml +# Purpose: apply the Vault read/write boundaries needed by Hermes operator OIDC. +apiVersion: batch/v1 +kind: Job +metadata: + name: vault-k8s-auth-hermes-1 + namespace: vault +spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: vault-admin + restartPolicy: Never + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + containers: + - name: configure-k8s-auth + image: hashicorp/vault:1.21.4 + imagePullPolicy: IfNotPresent + command: + - sh + - /scripts/vault_k8s_auth_configure.sh + env: + - name: VAULT_ADDR + value: http://10.43.57.249:8200 + - name: VAULT_K8S_ROLE + value: vault-admin + - name: VAULT_K8S_TOKEN_REVIEWER_JWT_FILE + value: /var/run/secrets/vault-token-reviewer/token + - name: VAULT_K8S_ROLE_TTL + value: 1h + volumeMounts: + - name: k8s-auth-config-script + mountPath: /scripts + readOnly: true + - name: token-reviewer + mountPath: /var/run/secrets/vault-token-reviewer + readOnly: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 128Mi + volumes: + - name: k8s-auth-config-script + configMap: + name: vault-k8s-auth-config-script + defaultMode: 0555 + - name: token-reviewer + secret: + secretName: vault-admin-token-reviewer diff --git a/services/vault/kustomization.yaml b/services/vault/kustomization.yaml index 98b47e441..d370373c9 100644 --- a/services/vault/kustomization.yaml +++ b/services/vault/kustomization.yaml @@ -11,6 +11,7 @@ resources: - configmap.yaml - statefulset.yaml - k8s-auth-config-cronjob.yaml + - hermes-auth-role-bootstrap-job.yaml - oidc-config-cronjob.yaml - service.yaml - certificate.yaml diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index d0876b6ed..cc5903b34 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -253,6 +253,8 @@ write_policy_and_role "health" "health" "health-vault-sync" \ "health/*" "" write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ "game-stream/*" "" +write_policy_and_role "hermes" "hermes" "hermes-vault" \ + "hermes/operator-oidc" "" write_policy_and_role "veles" "veles" "veles-backend,veles-generator,veles-postgres,veles-vault-sync" \ "veles/* shared/harbor-pull shared/postmark-relay" "" write_policy_and_role "veles-sim" "veles" "veles-sim" \ @@ -296,7 +298,7 @@ write_policy_and_role "vault" "vault" "vault" \ write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \ "shared/keycloak-admin shared/postmark-relay maintenance/metis-ssh-keys" \ - "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc" \ + "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc hermes/operator-oidc" \ ' path "kv/data/atlas/nodes/*" { capabilities = ["create", "update", "read"]