diff --git a/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml index 38b6d761e..64047c84e 100644 --- a/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml @@ -15,14 +15,7 @@ spec: namespace: flux-system wait: true timeout: 30m - healthChecks: - - apiVersion: apps/v1 - kind: Deployment - name: hermes-chat - namespace: hermes-chat dependsOn: - - name: cert-manager - name: core - name: hermes - - name: keycloak - name: longhorn diff --git a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml index 47cca710b..155718554 100644 --- a/clusters/atlas/flux-system/applications/hermes/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/hermes/kustomization.yaml @@ -32,10 +32,31 @@ spec: namespace: hermes - apiVersion: apps/v1 kind: Deployment - name: oauth2-proxy-hermes + name: hermes-agent + namespace: hermes + - apiVersion: apps/v1 + kind: StatefulSet + name: hermes-chat-tenant + namespace: hermes + - apiVersion: apps/v1 + kind: Deployment + name: hermes-chat-router + namespace: hermes + - apiVersion: apps/v1 + kind: Deployment + name: oauth2-proxy-hermes-agent + namespace: hermes + - apiVersion: apps/v1 + kind: Deployment + name: oauth2-proxy-hermes-chat + namespace: hermes + - apiVersion: apps/v1 + kind: Deployment + name: oauth2-proxy-hermes-triage namespace: hermes dependsOn: - name: cert-manager - name: core - name: keycloak - name: longhorn + - name: vault diff --git a/dockerfiles/Dockerfile.hermes-webui b/dockerfiles/Dockerfile.hermes-webui new file mode 100644 index 000000000..338ed40de --- /dev/null +++ b/dockerfiles/Dockerfile.hermes-webui @@ -0,0 +1,75 @@ +# syntax=docker/dockerfile:1 +# dockerfiles/Dockerfile.hermes-webui +FROM ghcr.io/nesquena/hermes-webui@sha256:a83a3893111dcb250e7aa7aa657d3d6f4570b0e2fd00d9b7569246fc5e7339b2 AS webui + +FROM registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + +USER root + +# Keep WebUI and Hermes pinned together. The WebUI imports Hermes internals, +# while the gateway remains the only process that owns an agent conversation. +COPY --from=webui /apptoo /opt/hermes-webui + +# The account policy caps user-selected reasoning at xhigh even when a provider +# advertises a newer, more expensive level. +RUN /opt/hermes/.venv/bin/python - <<'PY' +from pathlib import Path + +config = Path("/opt/hermes-webui/api/config.py") +source = config.read_text(encoding="utf-8") +before = 'VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh", "max")' +after = 'VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh")' +if before not in source: + raise SystemExit("Hermes WebUI reasoning-effort patch context changed") +config.write_text(source.replace(before, after, 1), encoding="utf-8") + +index = Path("/opt/hermes-webui/static/index.html") +source = index.read_text(encoding="utf-8") +before = '
Max
\n' +if before not in source: + raise SystemExit("Hermes WebUI xhigh UI patch context changed") +index.write_text(source.replace(before, "", 1), encoding="utf-8") +PY + +RUN /opt/hermes/.venv/bin/python -c 'import cryptography, yaml' \ + && grep -Fq 'VALID_REASONING_EFFORTS = ("minimal", "low", "medium", "high", "xhigh")' \ + /opt/hermes-webui/api/config.py \ + && ! grep -Fq 'data-effort="max"' /opt/hermes-webui/static/index.html + +# Exercise the real server process in the target architecture before publish. +RUN set -eu; \ + mkdir -p /tmp/hermes-webui-smoke/home /tmp/hermes-webui-smoke/state /tmp/hermes-webui-smoke/workspace; \ + HERMES_HOME=/tmp/hermes-webui-smoke/home \ + HOME=/tmp/hermes-webui-smoke/home \ + HERMES_WEBUI_STATE_DIR=/tmp/hermes-webui-smoke/state \ + HERMES_WEBUI_DEFAULT_WORKSPACE=/tmp/hermes-webui-smoke/workspace \ + HERMES_WEBUI_HOST=127.0.0.1 \ + HERMES_WEBUI_PORT=18787 \ + HERMES_WEBUI_SKIP_ONBOARDING=1 \ + /opt/hermes/.venv/bin/python /opt/hermes-webui/server.py >/tmp/hermes-webui-smoke.log 2>&1 & \ + server_pid=$!; \ + ready=0; \ + for attempt in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do \ + if /opt/hermes/.venv/bin/python -c 'from urllib.request import urlopen; urlopen("http://127.0.0.1:18787/health", timeout=2).read()' >/dev/null 2>&1; then ready=1; break; fi; \ + sleep 1; \ + done; \ + kill "${server_pid}" 2>/dev/null || true; \ + wait "${server_pid}" 2>/dev/null || true; \ + if [ "${ready}" != "1" ]; then cat /tmp/hermes-webui-smoke.log; exit 1; fi; \ + rm -rf /tmp/hermes-webui-smoke /tmp/hermes-webui-smoke.log + +ENV HERMES_WEBUI_AGENT_DIR=/opt/hermes \ + HERMES_WEBUI_HOST=0.0.0.0 \ + HERMES_WEBUI_PORT=8787 \ + HERMES_WEBUI_CHAT_BACKEND=gateway \ + HERMES_WEBUI_GATEWAY_BASE_URL=http://127.0.0.1:8642 \ + HERMES_WEBUI_GATEWAY_USE_RUNS_API=true \ + HERMES_WEBUI_SKIP_ONBOARDING=1 \ + HERMES_WEBUI_SECURE=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /opt/hermes-webui +USER 10000:10000 +EXPOSE 8787 +ENTRYPOINT ["/opt/hermes/.venv/bin/python", "/opt/hermes-webui/server.py"] diff --git a/infrastructure/core/coredns-custom.yaml b/infrastructure/core/coredns-custom.yaml index 3011dc4ae..beda2f43d 100644 --- a/infrastructure/core/coredns-custom.yaml +++ b/infrastructure/core/coredns-custom.yaml @@ -10,7 +10,7 @@ data: errors cache 30 hosts { - 192.168.22.9 agent.bstein.dev + 192.168.22.9 agent.hermes.bstein.dev 192.168.22.9 alerts.bstein.dev 192.168.22.9 auth.bstein.dev 192.168.22.9 bstein.dev @@ -18,6 +18,7 @@ data: 192.168.22.9 call.live.bstein.dev 192.168.22.9 cd.bstein.dev 192.168.22.9 chat.ai.bstein.dev + 192.168.22.9 chat.hermes.bstein.dev 192.168.22.9 ci.bstein.dev 192.168.22.9 cloud.bstein.dev 192.168.22.9 health.bstein.dev @@ -44,6 +45,7 @@ data: 192.168.22.9 stream.bstein.dev 192.168.22.9 wolf.bstein.dev 192.168.22.9 tasks.bstein.dev + 192.168.22.9 triage.hermes.bstein.dev 192.168.22.9 vault.bstein.dev fallthrough } diff --git a/services/hermes-chat/certificate.yaml b/services/hermes-chat/certificate.yaml deleted file mode 100644 index 3bed14a1f..000000000 --- a/services/hermes-chat/certificate.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# services/hermes-chat/certificate.yaml -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: chat-tls - namespace: hermes-chat -spec: - secretName: chat-tls - issuerRef: - kind: ClusterIssuer - name: letsencrypt - dnsNames: - - chat.bstein.dev diff --git a/services/hermes-chat/configmap.yaml b/services/hermes-chat/configmap.yaml deleted file mode 100644 index 84d6ab63c..000000000 --- a/services/hermes-chat/configmap.yaml +++ /dev/null @@ -1,153 +0,0 @@ -# services/hermes-chat/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: hermes-chat-config - namespace: hermes-chat - labels: - app: hermes-chat -data: - config.yaml: | - model: - provider: openai-codex - default: gpt-5.6-terra - model: gpt-5.6-terra - - fallback_providers: - - provider: custom - model: gpt-oss:20b - base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 - api_key: ollama - - agent: - api_max_retries: 1 - - platform_toolsets: - cli: - - clarify - - file - - terminal - - web - api_server: - - clarify - - file - - terminal - - web - - skills: - creation_nudge_interval: 15 - external_dirs: - - /opt/data/workspace/skills - - terminal: - backend: local - cwd: /opt/data/workspace - timeout: 180 - home_mode: profile - - approvals: - mode: manual - deny: - - "*kubectl apply*" - - "*kubectl create*" - - "*kubectl delete*" - - "*kubectl edit*" - - "*kubectl patch*" - - "*kubectl replace*" - - "*kubectl scale*" - - "*kubectl set*" - - "*kubectl label*" - - "*kubectl annotate*" - - "*kubectl cordon*" - - "*kubectl uncordon*" - - "*kubectl drain*" - - "*kubectl rollout restart*" - - "*kubectl rollout undo*" - - "*kubectl exec*" - - "*kubectl attach*" - - "*kubectl cp*" - - "*kubectl debug*" - - "*kubectl expose*" - - "*kubectl port-forward*" - - "*kubectl proxy*" - - "*kubectl run*" - - "*kubectl get secret*" - - "*kubectl describe secret*" - - "*flux reconcile*" - - "*flux suspend*" - - "*flux resume*" - - "*vault *" - - "*k3s *" - - "*crictl *" - - "*ctr *" - - "*169.254.169.254*" - - "*kubernetes.default*" - - "*/var/run/secrets/kubernetes.io*" - - dashboard: - public_url: https://chat.bstein.dev - oauth: - provider: self-hosted - self_hosted: - issuer: https://sso.bstein.dev/realms/atlas - client_id: hermes-chat-dashboard - scopes: openid profile email groups - - display: - compact: true - tool_progress: all - interim_assistant_messages: true - long_running_notifications: true - - tool_loop_guardrails: - warnings_enabled: true - hard_stop_enabled: true - warn_after: - exact_failure: 2 - same_tool_failure: 3 - idempotent_no_progress: 2 - hard_stop_after: - exact_failure: 5 - same_tool_failure: 8 - idempotent_no_progress: 5 - - updates: - pre_update_backup: quick - backup_keep: 3 - non_interactive_local_changes: stash - SOUL.md: | - You are a personal AI assistant and researcher for Atlas users. Be useful, - curious, careful, and direct. Help with research, writing, planning, - learning, files, personal automation, and creative work. Use tools when - they materially improve the answer and make reusable skills when a stable - workflow is worth keeping. - - Your environment is a private consumer sandbox. Work in the mounted - workspace and with public Internet services the user intentionally asks - you to use. You may inspect the Titan cluster through the dedicated - read-only Kubernetes identity to explain health and status. You do not - administer it: never attempt mutations, secret access, exec, attach, - port-forwarding, private-service access, or cloud metadata access. - AGENTS.md: | - # Personal Hermes workspace - - This Hermes instance is a personal chat, research, and automation - environment. User-created files and skills belong under `/opt/data/workspace`. - - You may: - - - research public Internet sources and cite them - - create, read, and edit files in the workspace - - create and improve reusable skills - - configure user-owned channels and provider integrations - - run ordinary local commands needed for the user's task - - inspect non-secret Kubernetes resources and pod logs with read-only - `kubectl get`, `describe`, and `logs` commands - - This container is not an infrastructure administration environment. The - Kubernetes identity is observation-only and cannot read Secrets or use pod - exec, attach, or port-forwarding. Do not attempt cluster mutations, private - service access, node LAN access, metadata services, Vault, container - runtimes, or the operator Hermes instance. If a request needs an action, - explain the evidence and say Brad must perform it from the operator - instance at `agent.bstein.dev`. diff --git a/services/hermes-chat/deployment.yaml b/services/hermes-chat/deployment.yaml deleted file mode 100644 index ac97e8bb4..000000000 --- a/services/hermes-chat/deployment.yaml +++ /dev/null @@ -1,259 +0,0 @@ -# services/hermes-chat/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hermes-chat - namespace: hermes-chat - labels: - app: hermes-chat -spec: - replicas: 1 - revisionHistoryLimit: 2 - progressDeadlineSeconds: 1800 - strategy: - type: Recreate - selector: - matchLabels: - app: hermes-chat - template: - metadata: - labels: - app: hermes-chat - annotations: - ai.bstein.dev/model: openai-codex/gpt-5.6-terra with local gpt-oss:20b fallback - ai.bstein.dev/role: personal-chat-research - ai.bstein.dev/isolation: observer-only Kubernetes RBAC, private-service egress denied - ai.bstein.dev/session-reset: "20260802-consumer-pty-recovery" - ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation - ai.bstein.dev/user-boundary: one OIDC subject with an isolated PVC subdirectory - spec: - serviceAccountName: hermes-chat - automountServiceAccountToken: true - securityContext: - fsGroup: 10000 - fsGroupChangePolicy: OnRootMismatch - seccompProfile: - type: RuntimeDefault - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - arm64 - - key: node-role.kubernetes.io/worker - operator: In - values: - - "true" - - key: kubernetes.io/hostname - operator: NotIn - values: - - titan-13 - - titan-15 - - titan-17 - - titan-18 - - titan-19 - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: atlas.bstein.dev/spillover - operator: DoesNotExist - - weight: 90 - preference: - matchExpressions: - - key: hardware - operator: In - values: - - rpi5 - - weight: 50 - preference: - matchExpressions: - - key: hardware - operator: In - values: - - rpi4 - initContainers: - - name: migrate-user-sessions - image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f - imagePullPolicy: IfNotPresent - command: - - /opt/hermes/bin/hermes-session-migrate - env: - - name: HERMES_HOME - value: /storage/users/4794ab8284a14d421eaeea3e - - name: HERMES_MIGRATE_SOURCE_HOME - value: /storage - - name: HERMES_MIGRATE_USER_ID - value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc - - name: HERMES_MIGRATE_SESSION_IDS - value: 20260802_182723_9f53f1,20260802_173527_9dc51b - securityContext: - allowPrivilegeEscalation: false - runAsUser: 10000 - runAsGroup: 10000 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - name: home - mountPath: /storage - resources: - requests: - cpu: 25m - memory: 64Mi - limits: - cpu: 250m - memory: 256Mi - - name: init-config - image: busybox:1.37 - imagePullPolicy: IfNotPresent - command: - - sh - - -c - - | - set -eu - user_home=/storage/users/4794ab8284a14d421eaeea3e - mkdir -p "${user_home}/workspace/skills" "${user_home}/home/.local/bin" "${user_home}/logs" - cp /config/config.yaml "${user_home}/config.yaml" - cp /config/SOUL.md "${user_home}/SOUL.md" - cp /config/AGENTS.md "${user_home}/workspace/AGENTS.md" - touch "${user_home}/.env" - if ! grep -q '^API_SERVER_KEY=' "${user_home}/.env"; then - api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" - printf '\nAPI_SERVER_KEY=%s\n' "${api_key}" >> "${user_home}/.env" - fi - chmod 0600 "${user_home}/.env" - chown -R 10000:10000 "${user_home}" - securityContext: - allowPrivilegeEscalation: false - runAsUser: 0 - runAsGroup: 0 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - name: home - mountPath: /storage - - name: config - mountPath: /config - readOnly: true - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - - name: install-kubectl - image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -c - - | - set -eu - cp "$(command -v kubectl)" /tools/kubectl - chmod 0755 /tools/kubectl - chown 10000:10000 /tools/kubectl - securityContext: - allowPrivilegeEscalation: false - runAsUser: 0 - runAsGroup: 0 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - name: tools - mountPath: /tools - resources: - requests: - cpu: 25m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - containers: - - name: hermes-chat - image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f - imagePullPolicy: IfNotPresent - args: - - gateway - - run - ports: - - name: dashboard - containerPort: 9119 - protocol: TCP - env: - - name: HERMES_HOME - value: /opt/data - - name: HOME - value: /opt/data/home - - name: PATH - value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - - name: HERMES_DASHBOARD - value: "1" - - name: HERMES_DASHBOARD_HOST - value: 0.0.0.0 - - name: HERMES_DASHBOARD_PORT - value: "9119" - - name: HERMES_DASHBOARD_PUBLIC_URL - value: https://chat.bstein.dev - - name: HERMES_DASHBOARD_OIDC_ISSUER - value: https://sso.bstein.dev/realms/atlas - - name: HERMES_DASHBOARD_OIDC_CLIENT_ID - value: hermes-chat-dashboard - - name: HERMES_DASHBOARD_OIDC_SCOPES - value: openid profile email groups - - name: HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS - value: 26b56113-0ec5-40b0-be7d-c2cd862f9bbc - - name: API_SERVER_ENABLED - value: "true" - - name: API_SERVER_HOST - value: 0.0.0.0 - - name: API_SERVER_PORT - value: "8642" - - name: API_SERVER_CORS_ORIGINS - value: https://chat.bstein.dev - volumeMounts: - - name: home - mountPath: /opt/data - subPath: users/4794ab8284a14d421eaeea3e - - name: tools - mountPath: /usr/local/bin/kubectl - subPath: kubectl - readinessProbe: - httpGet: - path: /api/status - port: dashboard - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - livenessProbe: - httpGet: - path: /api/status - port: dashboard - initialDelaySeconds: 90 - periodSeconds: 30 - timeoutSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - resources: - requests: - cpu: 100m - memory: 768Mi - ephemeral-storage: 256Mi - limits: - cpu: "1" - memory: 2Gi - ephemeral-storage: 1Gi - volumes: - - name: home - persistentVolumeClaim: - claimName: hermes-chat-home - - name: config - configMap: - name: hermes-chat-config - - name: tools - emptyDir: {} diff --git a/services/hermes-chat/ingress.yaml b/services/hermes-chat/ingress.yaml deleted file mode 100644 index e176231f5..000000000 --- a/services/hermes-chat/ingress.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# services/hermes-chat/ingress.yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: chat - namespace: hermes-chat - annotations: - cert-manager.io/cluster-issuer: letsencrypt - traefik.ingress.kubernetes.io/router.entrypoints: websecure - traefik.ingress.kubernetes.io/router.tls: "true" -spec: - ingressClassName: traefik - tls: - - hosts: - - chat.bstein.dev - secretName: chat-tls - rules: - - host: chat.bstein.dev - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: hermes-chat - port: - name: dashboard diff --git a/services/hermes-chat/kustomization.yaml b/services/hermes-chat/kustomization.yaml index 4d3529d77..3e509f1a2 100644 --- a/services/hermes-chat/kustomization.yaml +++ b/services/hermes-chat/kustomization.yaml @@ -4,12 +4,4 @@ kind: Kustomization namespace: hermes-chat resources: - namespace.yaml - - serviceaccount.yaml - - rbac.yaml - - configmap.yaml - pvc.yaml - - deployment.yaml - - service.yaml - - networkpolicy.yaml - - certificate.yaml - - ingress.yaml diff --git a/services/hermes-chat/networkpolicy.yaml b/services/hermes-chat/networkpolicy.yaml deleted file mode 100644 index 1c6451204..000000000 --- a/services/hermes-chat/networkpolicy.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# services/hermes-chat/networkpolicy.yaml -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: hermes-chat-isolation - namespace: hermes-chat -spec: - podSelector: - matchLabels: - app: hermes-chat - policyTypes: - - Ingress - - Egress - ingress: - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: traefik - podSelector: - matchLabels: - app.kubernetes.io/name: traefik - ports: - - protocol: TCP - port: 9119 - egress: - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: kube-system - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: hermes - podSelector: - matchLabels: - app: hermes-model-gate - ports: - - protocol: TCP - port: 8080 - # Atlas OIDC uses the public sso.bstein.dev issuer, which resolves to the - # in-cluster Traefik load balancer for pods on the Atlas network. - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: traefik - podSelector: - matchLabels: - app: traefik - ports: - - protocol: TCP - port: 443 - - to: - - ipBlock: - cidr: 10.43.0.1/32 - ports: - - protocol: TCP - port: 443 - # K3s applies egress policy after Service DNAT, so admit only the fixed API - # server endpoints as well as the kubernetes Service IP above. - - to: - - ipBlock: - cidr: 192.168.22.11/32 - - ipBlock: - cidr: 192.168.22.12/32 - - ipBlock: - cidr: 192.168.22.13/32 - ports: - - protocol: TCP - port: 6443 - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 10.0.0.0/8 - - 100.64.0.0/10 - - 127.0.0.0/8 - - 169.254.0.0/16 - - 172.16.0.0/12 - - 192.168.0.0/16 diff --git a/services/hermes-chat/rbac.yaml b/services/hermes-chat/rbac.yaml deleted file mode 100644 index 4144da689..000000000 --- a/services/hermes-chat/rbac.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# services/hermes-chat/rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: hermes-chat-observer -rules: - - apiGroups: [""] - resources: - - endpoints - - events - - namespaces - - nodes - - persistentvolumeclaims - - persistentvolumes - - pods - - pods/log - - replicationcontrollers - - services - verbs: ["get", "list", "watch"] - - apiGroups: ["apps"] - resources: - - daemonsets - - deployments - - replicasets - - statefulsets - verbs: ["get", "list", "watch"] - - apiGroups: ["batch"] - resources: - - cronjobs - - jobs - verbs: ["get", "list", "watch"] - - apiGroups: ["networking.k8s.io"] - resources: - - ingresses - - networkpolicies - verbs: ["get", "list", "watch"] - - apiGroups: ["helm.toolkit.fluxcd.io"] - resources: - - helmreleases - verbs: ["get", "list", "watch"] - - apiGroups: ["kustomize.toolkit.fluxcd.io"] - resources: - - kustomizations - verbs: ["get", "list", "watch"] - - apiGroups: ["source.toolkit.fluxcd.io"] - resources: - - gitrepositories - - helmrepositories - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hermes-chat-observer -subjects: - - kind: ServiceAccount - name: hermes-chat - namespace: hermes-chat -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: hermes-chat-observer diff --git a/services/hermes-chat/service.yaml b/services/hermes-chat/service.yaml deleted file mode 100644 index 8b74822ea..000000000 --- a/services/hermes-chat/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# services/hermes-chat/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: hermes-chat - namespace: hermes-chat - labels: - app: hermes-chat -spec: - type: ClusterIP - selector: - app: hermes-chat - ports: - - name: dashboard - port: 9119 - targetPort: dashboard - protocol: TCP diff --git a/services/hermes-chat/serviceaccount.yaml b/services/hermes-chat/serviceaccount.yaml deleted file mode 100644 index 88eca224c..000000000 --- a/services/hermes-chat/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# services/hermes-chat/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: hermes-chat - namespace: hermes-chat -automountServiceAccountToken: true diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index 2c107c0f6..f86b1cbc8 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -1,10 +1,38 @@ # Hermes on Atlas: operator guide This is the mental model and demonstration script for the operator instance at -`agent.bstein.dev`. Read it once, then prove each section in the live UI. The -consumer instance at `chat.bstein.dev` is intentionally separate and is not the +`triage.hermes.bstein.dev`. Read it once, then prove each section in the live UI. The +consumer instance at `chat.hermes.bstein.dev` is intentionally separate and is not the place to perform infrastructure triage. +## Consumer chat and Telegram + +`chat.hermes.bstein.dev` uses the pinned Hermes WebUI rather than the operator +dashboard. Keycloak still authenticates every browser request, and the tenant +router permanently assigns each Keycloak subject to one Hermes process and one +PVC. The four slots are an isolation pool, not a provider round robin: every +user starts with the same automatic provider/fallback policy and may change the +model or reasoning effort for their own conversation. The WebUI and API reject +reasoning levels above `xhigh`. + +Telegram is optional. The Keycloak bootstrap creates +`kv/atlas/hermes/chat-telegram` with a generated relay key and an empty +`bot_token`. After creating the shared bot with BotFather, set only that field: + +```sh +vault kv patch kv/atlas/hermes/chat-telegram bot_token='' +``` + +Restart or reconcile `hermes-chat-router` after changing the token. A user then +signs in to the WebUI, selects `Telegram`, creates a ten-minute code, and sends +the displayed `/link` command to the bot. The router accepts only direct chats, +stores hashed Keycloak and Telegram identities, and forwards the message to +that user's tenant API with the shared relay key. `/unlink` works from Telegram +or the WebUI. Browser chat remains available when `bot_token` is empty. + +The bot token and relay key must never be added to Git or a Kubernetes Secret. +The router does not log prompt bodies, raw Telegram IDs, link codes, or tokens. + ## The one-sentence explanation Hermes is the persistent agent runtime and control surface; Codex or the local diff --git a/services/hermes/agent-certificate.yaml b/services/hermes/agent-certificate.yaml index f5b52ae1b..486bab8ba 100644 --- a/services/hermes/agent-certificate.yaml +++ b/services/hermes/agent-certificate.yaml @@ -2,12 +2,14 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: agent-tls + name: hermes-sites-tls namespace: hermes spec: - secretName: agent-tls + secretName: hermes-sites-tls issuerRef: kind: ClusterIssuer name: letsencrypt dnsNames: - - agent.bstein.dev + - agent.hermes.bstein.dev + - chat.hermes.bstein.dev + - triage.hermes.bstein.dev diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml new file mode 100644 index 000000000..adaf8cd28 --- /dev/null +++ b/services/hermes/agent-configmap.yaml @@ -0,0 +1,193 @@ +# services/hermes/agent-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-agent-config + namespace: hermes + labels: + app: hermes-agent +data: + config.yaml: | + model: + provider: openai-codex + default: gpt-5.6-terra + model: gpt-5.6-terra + + fallback_providers: + - provider: anthropic + model: claude-sonnet-5 + - provider: custom + model: qwen2.5:14b-instruct-q4_0 + base_url: http://ollama.ai.svc.cluster.local:11434/v1 + api_key: ollama + - provider: custom + model: gpt-oss:20b + base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 + api_key: ollama + + agent: + api_max_retries: 1 + reasoning_effort: medium + + toolsets: + - kanban + + platform_toolsets: + cli: + - clarify + - file + - session_search + - skills + - terminal + - todo + - web + api_server: + - clarify + - file + - session_search + - skills + - terminal + - todo + - web + + gateway: + api_server: + max_concurrent_runs: 4 + + kanban: + # The board is authoritative state; Herdr, invoked by the coordinator, + # owns worker execution so a task cannot launch twice. + dispatch_in_gateway: false + dispatch_interval_seconds: 15 + failure_limit: 2 + orchestrator_profile: default + default_assignee: codex-medium + max_in_progress_per_profile: 1 + auto_decompose: true + auto_decompose_per_tick: 2 + dispatch_stale_timeout_seconds: 14400 + + model_catalog: + enabled: true + ttl_hours: 1 + + skills: + creation_nudge_interval: 15 + external_dirs: + - /opt/data/workspace/skills + + terminal: + backend: local + cwd: /opt/data/workspace + timeout: 300 + home_mode: auto + + approvals: + mode: smart + deny: + - "*kubectl apply*" + - "*kubectl delete*" + - "*kubectl patch*" + - "*kubectl scale*" + - "*kubectl exec*" + - "*kubectl port-forward*" + - "*flux reconcile*" + - "*flux suspend*" + - "*flux resume*" + - "*vault kv*" + + dashboard: + public_url: https://agent.hermes.bstein.dev + + display: + compact: true + tool_progress: all + interim_assistant_messages: true + long_running_notifications: true + + tool_loop_guardrails: + warnings_enabled: true + hard_stop_enabled: true + warn_after: + exact_failure: 2 + same_tool_failure: 3 + idempotent_no_progress: 2 + hard_stop_after: + exact_failure: 5 + same_tool_failure: 8 + idempotent_no_progress: 5 + + updates: + pre_update_backup: quick + backup_keep: 5 + non_interactive_local_changes: stash + SOUL.md: | + You are Brad's private Hermes coordinator at agent.hermes.bstein.dev. Turn + objectives into organized, reviewable delivery without making Brad manage + model names, terminals, or provider capacity. + + Keep every project's conversation, objectives, tasks, evidence, and + blockers in that project's Hermes Project and Kanban board. Cassandra is + the initial project. Use Herdr as the execution fabric for persistent Codex + and Claude Code workers; you remain responsible for planning, routing, + fallback, review, and the final synthesized answer. + + Prefer Codex for implementation, debugging, test loops, and focused repo + changes. Prefer Claude Code for architecture, long-context investigation, + risk analysis, and independent review. Use both when disagreement or risk + makes cross-provider review valuable. Never exceed xhigh effort. + + Local Jetson inference is the first provider-independent fallback. Use it + for bounded classification, summaries, and continuity when hosted capacity + is constrained. Do not silently treat a local fallback as equivalent to a + high-risk xhigh review; disclose the downgrade and preserve the task. + AGENTS.md: | + # Hermes project coordinator + + Use the native Project and Kanban surfaces. Cassandra uses project and board + slug `cassandra` with workspace `/opt/data/workspace/projects/cassandra`. + Put objectives needing decomposition in Triage. Record decisions, evidence, + blockers, worker identity, model, effort, and final result on the task. + + ## Difficulty routing + + - `low`: simple questions, lookup, formatting, or a tiny reversible edit. + - `medium`: normal bounded implementation or analysis with clear tests. + - `high`: multi-component work, difficult debugging, or material ambiguity. + - `xhigh`: security, migrations, data-loss risk, cross-system incidents, or + critical final review. `xhigh` is the hard maximum; never request max or + ultracode. + + Read `/opt/data/workspace/coordinator/model-routing.json` before naming a + model. The hourly steward discovers the models currently available to both + accounts and preserves the last working route during catalog outages. + Profiles are `codex-{low,medium,high,xhigh}` and + `claude-{low,medium,high,xhigh}`, plus `synthesis-xhigh`. + + For persistent coding work, plan or launch a worker with: + + `herdr-dispatch --shape --effort [--provider codex|claude]` + + Add `--start --project --task --prompt ` to + create a Herdr workspace and launch the selected CLI. Use `herdr agent list`, + `herdr agent wait`, `herdr agent read`, and `herdr agent prompt` to supervise + it. If Codex reports its first-use login requirement, run + `codex login --device-auth` once and ask Brad to complete the displayed code. + + A hosted capacity failure should fall across providers at the same effort + before dropping to local inference. Do not duplicate a task that is still + running. When both providers contributed, use `synthesis-xhigh` only if the + objective's difficulty warrants it; otherwise synthesize at the original + effort. + + This pod has no Kubernetes RBAC. Do not use it for automated test intake or + cluster mutation. Triage belongs at triage.hermes.bstein.dev and changes to + Atlas are delivered through the titan-iac Git/Flux workflow. + START-HERE.md: | + # Agent Hermes + + Select the Cassandra project and state the outcome you want. Hermes will + classify its difficulty, choose Codex or Claude Code, preserve the task on + the Cassandra board, supervise the worker through Herdr, and synthesize the + evidence. The first native Codex worker requires one device-code login; + subsequent sessions persist on the agent volume. diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml new file mode 100644 index 000000000..a5990288e --- /dev/null +++ b/services/hermes/agent-deployment.yaml @@ -0,0 +1,359 @@ +# services/hermes/agent-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-agent + namespace: hermes + labels: + app: hermes-agent +spec: + replicas: 1 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 2700 + strategy: + type: Recreate + selector: + matchLabels: + app: hermes-agent + template: + metadata: + labels: + app: hermes-agent + annotations: + ai.bstein.dev/role: project-coordinator + ai.bstein.dev/execution: Herdr-supervised Codex and Claude Code + ai.bstein.dev/model-policy: difficulty-aware low through xhigh, cross-provider fallback + ai.bstein.dev/placement: titan-20 preferred, Jetson preferred, arm64 fallback + ai.bstein.dev/config-rev: "20260808-herdr-coordinator" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-agent + vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-template-anthropic-token: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.anthropic_oauth_token }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-gitea-token: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-template-gitea-token: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{- with index .Data.data "gitea_token" -}} + {{ . }} + {{- end -}} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 32Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-agent + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-13, titan-15, titan-17, titan-18, titan-19] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [titan-20] + - weight: 90 + preference: + matchExpressions: + - key: jetson + operator: In + values: ["true"] + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + initContainers: + - name: init-config + image: busybox:1.37 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -eu + env_file=/opt/data/.env + mkdir -p \ + /opt/data/home/.claude \ + /opt/data/home/.codex \ + /opt/data/home/.config/herdr \ + /opt/data/herdr \ + /opt/data/logs \ + /opt/data/tools/bin \ + /opt/data/workspace/coordinator \ + /opt/data/workspace/projects \ + /opt/data/workspace/skills + cp /config/config.yaml /opt/data/config.yaml + cp /config/SOUL.md /opt/data/SOUL.md + cp /config/AGENTS.md /opt/data/workspace/AGENTS.md + cp /config/START-HERE.md /opt/data/workspace/START-HERE.md + touch "${env_file}" + upsert_env() { + key="$1" + value="$2" + { grep -v "^${key}=" "${env_file}" || true; printf '%s=%s\n' "${key}" "${value}"; } > "${env_file}.tmp" + mv "${env_file}.tmp" "${env_file}" + } + if ! grep -q '^API_SERVER_KEY=' "${env_file}"; then + api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" + upsert_env API_SERVER_KEY "${api_key}" + fi + if [ -s /vault/secrets/anthropic-token ]; then + token="$(tr -d '\r\n' < /vault/secrets/anthropic-token)" + [ -z "${token}" ] || upsert_env CLAUDE_CODE_OAUTH_TOKEN "${token}" + fi + if [ -s /vault/secrets/gitea-token ]; then + token="$(tr -d '\r\n' < /vault/secrets/gitea-token)" + case "${token}" in ""|""|"") ;; *) upsert_env GITEA_TOKEN "${token}" ;; esac + fi + upsert_env GITEA_USERNAME bstein + upsert_env GIT_ASKPASS /opt/coordinator/gitea_askpass.sh + upsert_env GIT_TERMINAL_PROMPT 0 + chmod 0600 "${env_file}" + chown -R 10000:10000 /opt/data + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: home + mountPath: /opt/data + - name: config + mountPath: /config + readOnly: true + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 100m, memory: 64Mi} + - name: install-agent-tools + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -eu + tools=/opt/data/tools + mkdir -p "${tools}/bin" + herdr_version="$("${tools}/bin/herdr" --version 2>/dev/null || true)" + case "${herdr_version}" in *0.8.0*) herdr_ready=1 ;; *) herdr_ready=0 ;; esac + if [ "${herdr_ready}" != "1" ]; then + curl -fsSL -o "${tools}/bin/herdr.tmp" https://github.com/herdrdev/herdr/releases/download/v0.8.0/herdr-linux-aarch64 + printf '%s %s\n' f647ac66468d9efbc642fe534fb284468f0aea60641606fc008dfc0d82a3ca87 "${tools}/bin/herdr.tmp" | sha256sum -c - + chmod 0755 "${tools}/bin/herdr.tmp" + mv "${tools}/bin/herdr.tmp" "${tools}/bin/herdr" + fi + if [ ! -f "${tools}/.cli-versions-0.147.0-2.1.226" ]; then + npm install --global --omit=dev --no-audit --no-fund --prefix "${tools}" \ + @openai/codex@0.147.0 \ + @anthropic-ai/claude-code@2.1.226 + touch "${tools}/.cli-versions-0.147.0-2.1.226" + fi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: home + mountPath: /opt/data + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: "1", memory: 1Gi} + - name: patch-auth + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/coordinator/patch_hermes_auth.py + - /opt/hermes/hermes_cli/auth.py + - /patched/auth.py + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: coordinator + mountPath: /opt/coordinator + readOnly: true + - name: auth-patch + mountPath: /patched + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 100m, memory: 128Mi} + - name: bootstrap-coordinator + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/coordinator/hermes_coordinator.py + - --once + env: + - {name: HERMES_HOME, value: /opt/data} + - {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json} + - {name: HOME, value: /opt/data/home} + - {name: PYTHONPATH, value: /opt/hermes} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: home, mountPath: /opt/data} + - {name: provider-auth, mountPath: /shared-auth} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {cpu: 500m, memory: 512Mi} + containers: + - name: hermes + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + args: [gateway, run] + ports: + - {name: dashboard, containerPort: 9119, protocol: TCP} + env: + - {name: HERMES_HOME, value: /opt/data} + - {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json} + - {name: HOME, value: /opt/data/home} + - {name: CODEX_HOME, value: /opt/data/home/.codex} + - {name: CLAUDE_CONFIG_DIR, value: /opt/data/home/.claude} + - {name: HERDR_CONFIG_PATH, value: /opt/data/home/.config/herdr/config.toml} + - {name: HERDR_SOCKET_PATH, value: /opt/data/herdr/herdr.sock} + - {name: PATH, value: /opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin} + - {name: HERMES_DASHBOARD, value: "1"} + - {name: HERMES_DASHBOARD_HOST, value: 0.0.0.0} + - {name: HERMES_DASHBOARD_PORT, value: "9119"} + - {name: HERMES_DASHBOARD_PUBLIC_URL, value: https://agent.hermes.bstein.dev} + volumeMounts: + - {name: home, mountPath: /opt/data} + - {name: provider-auth, mountPath: /shared-auth} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} + - {name: coordinator, mountPath: /opt/data/home/.local/bin/herdr-dispatch, subPath: herdr_dispatch.py, readOnly: true} + readinessProbe: + httpGet: {path: /api/status, port: dashboard} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: {path: /api/status, port: dashboard} + initialDelaySeconds: 90 + periodSeconds: 30 + timeoutSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 500m, memory: 1Gi} + limits: {cpu: "2", memory: 4Gi} + - name: herdr-server + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -eu + set -a + . /opt/data/.env + set +a + herdr server & + server_pid=$! + trap 'kill "${server_pid}" 2>/dev/null || true' TERM INT + for attempt in $(seq 1 60); do + if herdr status server >/dev/null 2>&1; then break; fi + sleep 1 + done + herdr integration install codex || true + herdr integration install claude || true + wait "${server_pid}" + env: + - {name: HOME, value: /opt/data/home} + - {name: CODEX_HOME, value: /opt/data/home/.codex} + - {name: CLAUDE_CONFIG_DIR, value: /opt/data/home/.claude} + - {name: HERDR_CONFIG_PATH, value: /opt/data/home/.config/herdr/config.toml} + - {name: HERDR_SOCKET_PATH, value: /opt/data/herdr/herdr.sock} + - {name: PATH, value: /opt/data/tools/bin:/usr/local/bin:/usr/bin:/bin} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: home, mountPath: /opt/data} + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: "1", memory: 2Gi} + - name: model-steward + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + command: [/opt/hermes/.venv/bin/python, /opt/coordinator/hermes_coordinator.py, --loop, --interval, "3600"] + env: + - {name: HERMES_HOME, value: /opt/data} + - {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json} + - {name: HOME, value: /opt/data/home} + - {name: PYTHONPATH, value: /opt/hermes} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: home, mountPath: /opt/data} + - {name: provider-auth, mountPath: /shared-auth} + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} + resources: + requests: {cpu: 25m, memory: 128Mi} + limits: {cpu: 250m, memory: 512Mi} + volumes: + - name: home + persistentVolumeClaim: + claimName: hermes-agent-home + - name: provider-auth + persistentVolumeClaim: + claimName: hermes-provider-auth + - name: config + configMap: + name: hermes-agent-config + - name: coordinator + configMap: + name: hermes-coordinator + defaultMode: 0555 + - name: auth-patch + emptyDir: {} diff --git a/services/hermes/agent-ingress.yaml b/services/hermes/agent-ingress.yaml index 9581a95b4..1288f81ad 100644 --- a/services/hermes/agent-ingress.yaml +++ b/services/hermes/agent-ingress.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: agent + name: hermes-sites namespace: hermes annotations: cert-manager.io/cluster-issuer: letsencrypt @@ -11,16 +11,39 @@ metadata: spec: ingressClassName: traefik tls: - - hosts: ["agent.bstein.dev"] - secretName: agent-tls + - hosts: + - agent.hermes.bstein.dev + - chat.hermes.bstein.dev + - triage.hermes.bstein.dev + secretName: hermes-sites-tls rules: - - host: agent.bstein.dev + - host: agent.hermes.bstein.dev http: paths: - path: / pathType: Prefix backend: service: - name: oauth2-proxy-hermes + name: oauth2-proxy-hermes-agent + port: + name: http + - host: chat.hermes.bstein.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: oauth2-proxy-hermes-chat + port: + name: http + - host: triage.hermes.bstein.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: oauth2-proxy-hermes-triage port: name: http diff --git a/services/hermes/chat-configmap.yaml b/services/hermes/chat-configmap.yaml new file mode 100644 index 000000000..9ee5cf044 --- /dev/null +++ b/services/hermes/chat-configmap.yaml @@ -0,0 +1,57 @@ +# services/hermes/chat-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hermes-chat-config + namespace: hermes + labels: + app: hermes-chat-tenant +data: + config.yaml: | + model: + provider: openai-codex + default: gpt-5.6-terra + model: gpt-5.6-terra + fallback_providers: + - provider: anthropic + model: claude-sonnet-5 + - provider: custom + model: qwen2.5:14b-instruct-q4_0 + base_url: http://ollama.ai.svc.cluster.local:11434/v1 + api_key: ollama + - provider: custom + model: gpt-oss:20b + base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 + api_key: ollama + agent: + api_max_retries: 1 + model_catalog: + enabled: true + ttl_hours: 1 + platform_toolsets: + cli: [clarify, session_search, web] + api_server: [clarify, session_search, web] + dashboard: + public_url: https://chat.hermes.bstein.dev + display: + compact: true + tool_progress: all + interim_assistant_messages: true + long_running_notifications: true + SOUL.md: | + You are a high-quality private AI chat assistant. Help the current person + with questions, writing, research, planning, and learning. Be direct, + thoughtful, and careful. Use public web research when freshness matters. + + This is a personal sandbox. Never attempt cluster administration, private + service access, repository modification, terminal execution, credentials, + or coordination of Brad's project agents. Those capabilities are not part + of this chat product. The user's conversations and files must never be + mixed with another Keycloak user's state. + AGENTS.md: | + # Private Hermes chat + + This runtime belongs to one authenticated Keycloak identity and one private + persistent volume. Provide conversational help with clarify, session search, + and public web tools only. Do not claim access to Kubernetes, Vault, Gitea, + Brad's projects, other users, the agent coordinator, or automated triage. diff --git a/services/hermes/chat-router.yaml b/services/hermes/chat-router.yaml new file mode 100644 index 000000000..d4ab57e69 --- /dev/null +++ b/services/hermes/chat-router.yaml @@ -0,0 +1,135 @@ +# services/hermes/chat-router.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-chat-router + namespace: hermes + labels: + app: hermes-chat-router +spec: + replicas: 1 + revisionHistoryLimit: 2 + strategy: + type: Recreate + selector: + matchLabels: + app: hermes-chat-router + template: + metadata: + labels: + app: hermes-chat-router + annotations: + ai.bstein.dev/role: privacy-preserving-chat-tenant-router + ai.bstein.dev/config-rev: "20260808-webui-telegram" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/role: hermes-chat + vault.hashicorp.com/agent-inject-secret-telegram-config: kv/data/atlas/hermes/chat-telegram + vault.hashicorp.com/agent-inject-template-telegram-config: | + {{- with secret "kv/data/atlas/hermes/chat-telegram" -}} + bot_token={{ .Data.data.bot_token }} + relay_key={{ .Data.data.relay_key }} + {{- end }} + spec: + serviceAccountName: hermes-chat + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-13, titan-15, titan-17, titan-18, titan-19] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + initContainers: + - name: build-router + image: golang:1.24-alpine@sha256:8bee1901f1e530bfb4a7850aa7a479d17ae3a18beb6e09064ed54cfd245b7191 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -eu + cd /src + GOCACHE=/tmp/go-cache GO111MODULE=off CGO_ENABLED=0 go test . + GOCACHE=/tmp/go-cache GO111MODULE=off CGO_ENABLED=0 go build -trimpath -o /tools/chat-router . + chmod 0755 /tools/chat-router + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: source, mountPath: /src, readOnly: true} + - {name: tools, mountPath: /tools} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 100m, memory: 128Mi} + limits: {cpu: "1", memory: 512Mi} + containers: + - name: router + image: busybox:1.37 + imagePullPolicy: IfNotPresent + command: [/tools/chat-router] + ports: + - {name: http, containerPort: 8080, protocol: TCP} + env: + - {name: TENANT_SLOTS, value: "4"} + - {name: TENANT_STATE_PATH, value: /state/tenants.json} + - {name: TELEGRAM_CONFIG_PATH, value: /vault/secrets/telegram-config} + readinessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 2 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: http} + initialDelaySeconds: 10 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: tools, mountPath: /tools, readOnly: true} + - {name: state, mountPath: /state} + - {name: tmp, mountPath: /tmp} + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + volumes: + - name: source + configMap: + name: hermes-chat-router-source + - name: tools + emptyDir: {} + - name: state + persistentVolumeClaim: + claimName: hermes-chat-router-state + - name: tmp + emptyDir: + sizeLimit: 128Mi diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml new file mode 100644 index 000000000..339a86bfa --- /dev/null +++ b/services/hermes/chat-statefulset.yaml @@ -0,0 +1,275 @@ +# services/hermes/chat-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hermes-chat-tenant + namespace: hermes + labels: + app: hermes-chat-tenant +spec: + serviceName: hermes-chat-tenant + replicas: 4 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: hermes-chat-tenant + template: + metadata: + labels: + app: hermes-chat-tenant + annotations: + ai.bstein.dev/role: isolated-user-chat + ai.bstein.dev/isolation: one Hermes process and PVC per Keycloak subject + ai.bstein.dev/model-policy: uniform automatic policy with per-user overrides + ai.bstein.dev/config-rev: "20260808-webui-telegram" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: hermes-chat + vault.hashicorp.com/agent-inject-secret-anthropic-token: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-template-anthropic-token: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.anthropic_oauth_token }} + {{- end }} + vault.hashicorp.com/agent-inject-secret-chat-relay-key: kv/data/atlas/hermes/chat-telegram + vault.hashicorp.com/agent-inject-template-chat-relay-key: | + {{- with secret "kv/data/atlas/hermes/chat-telegram" -}} + {{ .Data.data.relay_key }} + {{- end }} + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 32Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi + spec: + serviceAccountName: hermes-chat + automountServiceAccountToken: true + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [arm64] + - key: node-role.kubernetes.io/worker + operator: In + values: ["true"] + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-13, titan-15, titan-17, titan-18, titan-19] + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi5] + - weight: 40 + preference: + matchExpressions: + - key: hardware + operator: In + values: [rpi4] + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: hermes-chat-tenant + topologyKey: kubernetes.io/hostname + initContainers: + - name: init-config + image: busybox:1.37 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -eu + mkdir -p /opt/data/home/.local/bin /opt/data/logs /opt/data/workspace + cp /config/config.yaml /opt/data/config.yaml + cp /config/SOUL.md /opt/data/SOUL.md + cp /config/AGENTS.md /opt/data/workspace/AGENTS.md + touch /opt/data/.env + relay_key="" + if [ -s /vault/secrets/chat-relay-key ]; then + relay_key="$(tr -d '\r\n' < /vault/secrets/chat-relay-key)" + fi + if [ -z "${relay_key}" ]; then + api_key="$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n')" + relay_key="${api_key}" + fi + { grep -v '^API_SERVER_KEY=' /opt/data/.env || true; printf 'API_SERVER_KEY=%s\n' "${relay_key}"; } > /opt/data/.env.tmp + mv /opt/data/.env.tmp /opt/data/.env + if [ -s /vault/secrets/anthropic-token ]; then + token="$(tr -d '\r\n' < /vault/secrets/anthropic-token)" + if [ -n "${token}" ]; then + { grep -v '^CLAUDE_CODE_OAUTH_TOKEN=' /opt/data/.env || true; printf 'CLAUDE_CODE_OAUTH_TOKEN=%s\n' "${token}"; } > /opt/data/.env.tmp + mv /opt/data/.env.tmp /opt/data/.env + fi + fi + chmod 0600 /opt/data/.env + chown -R 10000:10000 /opt/data + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + runAsGroup: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: home, mountPath: /opt/data} + - {name: config, mountPath: /config, readOnly: true} + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 100m, memory: 64Mi} + - name: patch-auth + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/coordinator/patch_hermes_auth.py + - /opt/hermes/hermes_cli/auth.py + - /patched/auth.py + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: auth-patch, mountPath: /patched} + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 100m, memory: 128Mi} + containers: + - name: hermes + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + args: [gateway, run] + ports: + - {name: api, containerPort: 8642, protocol: TCP} + env: + - {name: HERMES_HOME, value: /opt/data} + - {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json} + - {name: HOME, value: /opt/data/home} + - {name: PATH, value: /opt/data/home/.local/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin} + - {name: HERMES_DASHBOARD, value: "0"} + - {name: API_SERVER_ENABLED, value: "true"} + - {name: API_SERVER_HOST, value: 0.0.0.0} + - {name: API_SERVER_PORT, value: "8642"} + - {name: API_SERVER_CORS_ORIGINS, value: https://chat.hermes.bstein.dev} + volumeMounts: + - {name: home, mountPath: /opt/data} + - {name: provider-auth, mountPath: /shared-auth} + - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} + readinessProbe: + tcpSocket: {port: api} + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + tcpSocket: {port: api} + initialDelaySeconds: 90 + periodSeconds: 30 + timeoutSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 250m, memory: 512Mi} + limits: {cpu: "1", memory: 2Gi} + - name: webui + image: registry.bstein.dev/bstein/hermes-webui@sha256:a771858bd668d25e19c74864baea5425101c8cd5215d1ba3a312f3312ce6c5e1 + imagePullPolicy: IfNotPresent + command: [/bin/sh, -ec] + args: + - | + api_key="$(sed -n 's/^API_SERVER_KEY=//p' /opt/data/.env | tail -n 1)" + test -n "${api_key}" + export API_SERVER_KEY="${api_key}" + export HERMES_WEBUI_GATEWAY_API_KEY="${api_key}" + exec /opt/hermes/.venv/bin/python /opt/hermes-webui/server.py + ports: + - {name: webui, containerPort: 8787, protocol: TCP} + env: + - {name: HERMES_HOME, value: /opt/data} + - {name: HERMES_AUTH_FILE, value: /shared-auth/auth.json} + - {name: HOME, value: /opt/data/home} + - {name: HERMES_WEBUI_AGENT_DIR, value: /opt/hermes} + - {name: HERMES_WEBUI_HOST, value: 0.0.0.0} + - {name: HERMES_WEBUI_PORT, value: "8787"} + - {name: HERMES_WEBUI_STATE_DIR, value: /opt/data/webui} + - {name: HERMES_WEBUI_DEFAULT_WORKSPACE, value: /opt/data/workspace} + - {name: HERMES_WEBUI_CHAT_BACKEND, value: gateway} + - {name: HERMES_WEBUI_GATEWAY_BASE_URL, value: http://127.0.0.1:8642} + - {name: HERMES_WEBUI_GATEWAY_USE_RUNS_API, value: "true"} + - {name: HERMES_WEBUI_SKIP_ONBOARDING, value: "1"} + - {name: HERMES_WEBUI_SECURE, value: "1"} + - {name: HERMES_WEBUI_ALLOWED_ORIGINS, value: https://chat.hermes.bstein.dev} + - {name: HERMES_WEBUI_TRUST_FORWARDED_HOST, value: "1"} + - {name: HERMES_WEBUI_TRUST_FORWARDED_PROTO, value: "1"} + volumeMounts: + - {name: home, mountPath: /opt/data} + - {name: provider-auth, mountPath: /shared-auth, readOnly: true} + - {name: tmp, mountPath: /tmp} + readinessProbe: + httpGet: {path: /health, port: webui} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + httpGet: {path: /health, port: webui} + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: 750m, memory: 1Gi} + volumes: + - name: provider-auth + persistentVolumeClaim: + claimName: hermes-provider-auth + - name: config + configMap: + name: hermes-chat-config + - name: coordinator + configMap: + name: hermes-coordinator + defaultMode: 0555 + - name: auth-patch + emptyDir: {} + - name: tmp + emptyDir: + sizeLimit: 256Mi + volumeClaimTemplates: + - metadata: + name: home + labels: + app: hermes-chat-tenant + spec: + accessModes: [ReadWriteOnce] + storageClassName: astreae + resources: + requests: + storage: 2Gi diff --git a/services/hermes/configmap.yaml b/services/hermes/configmap.yaml index 359049db9..1dc290b51 100644 --- a/services/hermes/configmap.yaml +++ b/services/hermes/configmap.yaml @@ -16,6 +16,10 @@ data: fallback_providers: - provider: openai-codex model: gpt-5.6-terra + - provider: custom + model: qwen2.5:14b-instruct-q4_0 + base_url: http://ollama.ai.svc.cluster.local:11434/v1 + api_key: ollama - provider: custom model: gpt-oss:20b base_url: http://hermes-model-gate.hermes.svc.cluster.local:11434/v1 @@ -23,6 +27,11 @@ data: agent: api_max_retries: 1 + reasoning_effort: medium + + model_catalog: + enabled: true + ttl_hours: 1 platform_toolsets: cli: @@ -72,13 +81,7 @@ data: - "*kubectl describe secret*" dashboard: - public_url: https://agent.bstein.dev - oauth: - provider: self-hosted - self_hosted: - issuer: https://sso.bstein.dev/realms/atlas - client_id: hermes-dashboard - scopes: openid profile email groups + public_url: https://triage.hermes.bstein.dev display: compact: true @@ -106,6 +109,11 @@ data: You are Hermes running inside the Titan Kubernetes cluster as a supervised testing and operations triage assistant. + This is the dedicated triage appliance at triage.hermes.bstein.dev. Keep + automated Ariadne intake and testing conversations here. Project delivery + and coding orchestration belong to agent.hermes.bstein.dev; general user + chat belongs to chat.hermes.bstein.dev. + Your strongest job is to follow the same evidence path Brad already uses: Ariadne diagnosis first, then Jenkins logs and artifacts, Pushgateway quality metrics, Flux state, Grafana dashboard context, and Kubernetes diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 978beacb1..ea777ab04 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -23,8 +23,8 @@ spec: ai.bstein.dev/frontend-fix: scope PTY attachment by selected conversation ai.bstein.dev/model: anthropic/claude-opus-5, falling back to openai-codex/gpt-5.6-terra then local gpt-oss:20b ai.bstein.dev/role: testing-triage - ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred) - ai.bstein.dev/config-rev: "20260804-root-operator-docs" + ai.bstein.dev/placement: titan-21 preferred, Jetson preferred, arm64 fallback + ai.bstein.dev/config-rev: "20260808-dedicated-triage" # The Anthropic credential comes from Vault rather than a manually # created Secret. The role is declared in # services/vault/scripts/vault_k8s_auth_configure.sh and bound to @@ -76,11 +76,25 @@ spec: - titan-19 preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - titan-21 + - weight: 90 + preference: + matchExpressions: + - key: jetson + operator: In + values: + - "true" + - weight: 80 preference: matchExpressions: - key: atlas.bstein.dev/spillover operator: DoesNotExist - - weight: 90 + - weight: 60 preference: matchExpressions: - key: hardware @@ -158,7 +172,13 @@ spec: mv /opt/data/.env.tmp /opt/data/.env fi chmod 0600 /opt/data/.env + mkdir -p /shared-auth + if [ ! -s /shared-auth/auth.json ] && [ -s /opt/data/auth.json ]; then + cp /opt/data/auth.json /shared-auth/auth.json + chmod 0600 /shared-auth/auth.json + fi chown -R 10000:10000 /opt/data + chown -R 10000:10000 /shared-auth securityContext: runAsUser: 0 runAsGroup: 0 @@ -169,6 +189,8 @@ spec: mountPath: /config - name: operator-guide mountPath: /guide + - name: provider-auth + mountPath: /shared-auth resources: requests: cpu: 25m @@ -176,6 +198,33 @@ spec: limits: cpu: 100m memory: 64Mi + - name: patch-auth + image: registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f + imagePullPolicy: IfNotPresent + command: + - /opt/hermes/.venv/bin/python + - /opt/coordinator/patch_hermes_auth.py + - /opt/hermes/hermes_cli/auth.py + - /patched/auth.py + securityContext: + allowPrivilegeEscalation: false + runAsUser: 10000 + runAsGroup: 10000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: coordinator + mountPath: /opt/coordinator + readOnly: true + - name: auth-patch + mountPath: /patched + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi - name: install-kubectl image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 imagePullPolicy: IfNotPresent @@ -217,6 +266,8 @@ spec: env: - name: HERMES_HOME value: /opt/data + - name: HERMES_AUTH_FILE + value: /shared-auth/auth.json - name: HOME value: /opt/data/home - name: PATH @@ -228,13 +279,7 @@ spec: - name: HERMES_DASHBOARD_PORT value: "9119" - name: HERMES_DASHBOARD_PUBLIC_URL - value: https://agent.bstein.dev - - name: HERMES_DASHBOARD_OIDC_ISSUER - value: https://sso.bstein.dev/realms/atlas - - name: HERMES_DASHBOARD_OIDC_CLIENT_ID - value: hermes-dashboard - - name: HERMES_DASHBOARD_OIDC_SCOPES - value: openid profile email groups + value: https://triage.hermes.bstein.dev - name: API_SERVER_ENABLED value: "true" - name: API_SERVER_HOST @@ -242,7 +287,7 @@ spec: - name: API_SERVER_PORT value: "8642" - name: API_SERVER_CORS_ORIGINS - value: https://agent.bstein.dev + value: https://triage.hermes.bstein.dev - name: VICTORIA_METRICS_URL value: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 - name: ARIADNE_BASE_URL @@ -264,6 +309,11 @@ spec: volumeMounts: - name: home mountPath: /opt/data + - name: provider-auth + mountPath: /shared-auth + - name: auth-patch + mountPath: /opt/hermes/hermes_cli/auth.py + subPath: auth.py - name: tools mountPath: /usr/local/bin/kubectl subPath: kubectl @@ -304,6 +354,9 @@ spec: - name: home persistentVolumeClaim: claimName: hermes-home + - name: provider-auth + persistentVolumeClaim: + claimName: hermes-provider-auth - name: config configMap: name: hermes-config @@ -312,6 +365,12 @@ spec: name: hermes-operator-guide - name: tools emptyDir: {} + - name: coordinator + configMap: + name: hermes-coordinator + defaultMode: 0555 + - name: auth-patch + emptyDir: {} - name: triage-skill configMap: name: hermes-triage-skill diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index 6033f3e5a..ea2258a1c 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -6,6 +6,8 @@ resources: - namespace.yaml - vault-serviceaccount.yaml - configmap.yaml + - agent-configmap.yaml + - chat-configmap.yaml - rbac.yaml - pvc.yaml - model-gate-rbac.yaml @@ -16,6 +18,9 @@ resources: - networkpolicy.yaml - ollama-deployment.yaml - deployment.yaml + - agent-deployment.yaml + - chat-statefulset.yaml + - chat-router.yaml - service.yaml - oauth2-proxy.yaml - agent-certificate.yaml @@ -28,6 +33,26 @@ configMapGenerator: - OPERATOR-RUNBOOK.md=NOTES.md options: disableNameSuffixHash: true + - name: hermes-coordinator + namespace: hermes + files: + - gitea_askpass.sh=scripts/gitea_askpass.sh + - herdr_dispatch.py=scripts/herdr_dispatch.py + - hermes_coordinator.py=scripts/hermes_coordinator.py + - hermes_model_routing.py=scripts/hermes_model_routing.py + - patch_hermes_auth.py=scripts/patch_hermes_auth.py + options: + disableNameSuffixHash: true + - name: hermes-chat-router-source + namespace: hermes + files: + - main.go=router/main.go + - main_test.go=router/main_test.go + - telegram.go=router/telegram.go + - telegram_test.go=router/telegram_test.go + - web.go=router/web.go + options: + disableNameSuffixHash: true - name: hermes-triage-skill namespace: hermes files: diff --git a/services/hermes/networkpolicy.yaml b/services/hermes/networkpolicy.yaml index 57bfa577f..6f70516f4 100644 --- a/services/hermes/networkpolicy.yaml +++ b/services/hermes/networkpolicy.yaml @@ -28,3 +28,287 @@ spec: ports: - protocol: TCP port: 11434 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-triage-ingress + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: oauth2-proxy-hermes-triage + ports: + - {protocol: TCP, port: 9119} + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: maintenance + podSelector: + matchLabels: + app: ariadne + ports: + - {protocol: TCP, port: 8642} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-agent-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-agent + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: + matchLabels: + app: oauth2-proxy-hermes-agent + ports: + - {protocol: TCP, port: 9119} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ai + podSelector: + matchLabels: + app: ollama + ports: + - {protocol: TCP, port: 11434} + - to: + - podSelector: + matchLabels: + app: hermes-model-gate + ports: + - {protocol: TCP, port: 8080} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app: traefik + ports: + - {protocol: TCP, port: 443} + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-chat-tenant-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-chat-tenant + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: + matchLabels: + app: hermes-chat-router + ports: + - {protocol: TCP, port: 8787} + - {protocol: TCP, port: 8642} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ai + podSelector: + matchLabels: + app: ollama + ports: + - {protocol: TCP, port: 11434} + - to: + - podSelector: + matchLabels: + app: hermes-model-gate + ports: + - {protocol: TCP, port: 8080} + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-chat-router-isolation + namespace: hermes +spec: + podSelector: + matchLabels: + app: hermes-chat-router + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: + matchLabels: + app: oauth2-proxy-hermes-chat + ports: + - {protocol: TCP, port: 8080} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - podSelector: + matchLabels: + app: hermes-chat-tenant + ports: + - {protocol: TCP, port: 8787} + - {protocol: TCP, port: 8642} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - {protocol: TCP, port: 443} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hermes-oauth2-proxies + namespace: hermes +spec: + podSelector: + matchExpressions: + - key: app + operator: In + values: + - oauth2-proxy-hermes-agent + - oauth2-proxy-hermes-chat + - oauth2-proxy-hermes-triage + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app: traefik + ports: + - {protocol: TCP, port: 4180} + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + podSelector: + matchLabels: + app: vault + ports: + - {protocol: TCP, port: 8200} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: traefik + podSelector: + matchLabels: + app: traefik + ports: + - {protocol: TCP, port: 443} + - to: + - podSelector: + matchExpressions: + - key: app + operator: In + values: [hermes, hermes-agent, hermes-chat-router] + ports: + - {protocol: TCP, port: 9119} + - {protocol: TCP, port: 8080} diff --git a/services/hermes/oauth2-proxy.yaml b/services/hermes/oauth2-proxy.yaml index d9971f0d6..4bfaee925 100644 --- a/services/hermes/oauth2-proxy.yaml +++ b/services/hermes/oauth2-proxy.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: hermes-operator-allowlist + name: hermes-owner-allowlist namespace: hermes data: allowed-emails: | @@ -11,85 +11,67 @@ data: apiVersion: v1 kind: Service metadata: - name: oauth2-proxy-hermes + name: oauth2-proxy-hermes-agent namespace: hermes - labels: - app: oauth2-proxy-hermes spec: selector: - app: oauth2-proxy-hermes + app: oauth2-proxy-hermes-agent ports: - - name: http - port: 80 - targetPort: http + - {name: http, port: 80, targetPort: http} +--- +apiVersion: v1 +kind: Service +metadata: + name: oauth2-proxy-hermes-triage + namespace: hermes +spec: + selector: + app: oauth2-proxy-hermes-triage + ports: + - {name: http, port: 80, targetPort: http} +--- +apiVersion: v1 +kind: Service +metadata: + name: oauth2-proxy-hermes-chat + namespace: hermes +spec: + selector: + app: oauth2-proxy-hermes-chat + ports: + - {name: http, port: 80, targetPort: http} --- apiVersion: apps/v1 kind: Deployment metadata: - name: oauth2-proxy-hermes + name: oauth2-proxy-hermes-agent namespace: hermes labels: - app: oauth2-proxy-hermes + app: oauth2-proxy-hermes-agent spec: replicas: 1 revisionHistoryLimit: 2 selector: matchLabels: - app: oauth2-proxy-hermes + app: oauth2-proxy-hermes-agent template: metadata: labels: - app: oauth2-proxy-hermes + app: oauth2-proxy-hermes-agent annotations: vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-pre-populate-only: "true" - vault.hashicorp.com/role: hermes - vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/operator-oidc + vault.hashicorp.com/role: hermes-agent + vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/agent-oidc vault.hashicorp.com/agent-inject-template-oidc-config: | - {{- with secret "kv/data/atlas/hermes/operator-oidc" -}} + {{- with secret "kv/data/atlas/hermes/agent-oidc" -}} client_id = "{{ .Data.data.client_id }}" client_secret = "{{ .Data.data.client_secret }}" cookie_secret = "{{ .Data.data.cookie_secret }}" {{- end -}} spec: - serviceAccountName: hermes-vault + serviceAccountName: hermes-agent automountServiceAccountToken: true - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - arm64 - - key: node-role.kubernetes.io/worker - operator: In - values: - - "true" - - key: kubernetes.io/hostname - operator: NotIn - values: - - titan-13 - - titan-15 - - titan-17 - - titan-18 - - titan-19 - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 90 - preference: - matchExpressions: - - key: hardware - operator: In - values: - - rpi5 - - weight: 50 - preference: - matchExpressions: - - key: hardware - operator: In - values: - - rpi4 containers: - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0@sha256:dcb6ff8dd21bf3058f6a22c6fa385fa5b897a9cd3914c88a2cc2bb0a85f8065d @@ -97,65 +79,235 @@ spec: args: - --provider=oidc - --config=/vault/secrets/oidc-config - - --redirect-url=https://agent.bstein.dev/oauth2/callback + - --redirect-url=https://agent.hermes.bstein.dev/oauth2/callback - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas + - --user-id-claim=sub - --code-challenge-method=S256 - - --scope=openid profile email + - --scope=openid profile email groups - --email-domain=* - --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails - --set-xauthrequest=true + - --pass-user-headers=true + - --pass-basic-auth=false + - --proxy-websockets=true - --session-cookie-minimal=true + - --cookie-name=__Host-hermes_agent + - --cookie-path=/ - --cookie-secure=true - --cookie-samesite=lax - - --cookie-refresh=0 + - --cookie-refresh=1h - --cookie-expire=8h - - --upstream=http://hermes.hermes.svc.cluster.local:9119 + - --upstream=http://hermes-agent.hermes.svc.cluster.local:9119 - --http-address=0.0.0.0:4180 - --skip-provider-button=true - - --skip-jwt-bearer-tokens=true - - --cookie-domain=agent.bstein.dev - --reverse-proxy=true + - --trusted-proxy-ip=10.42.0.0/16 ports: - - name: http - containerPort: 4180 + - {name: http, containerPort: 4180} readinessProbe: - httpGet: - path: /ping - port: http + httpGet: {path: /ping, port: http} initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: - httpGet: - path: /ping - port: http + httpGet: {path: /ping, port: http} initialDelaySeconds: 20 periodSeconds: 20 securityContext: allowPrivilegeEscalation: false capabilities: - drop: - - ALL + drop: [ALL] readOnlyRootFilesystem: true runAsNonRoot: true seccompProfile: type: RuntimeDefault resources: - requests: - cpu: 25m - memory: 64Mi - limits: - cpu: 250m - memory: 256Mi + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} volumeMounts: - - name: allowlist - mountPath: /etc/oauth2-proxy - readOnly: true - - name: tmp - mountPath: /tmp + - {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true} + - {name: tmp, mountPath: /tmp} volumes: - name: allowlist configMap: - name: hermes-operator-allowlist + name: hermes-owner-allowlist - name: tmp - emptyDir: - sizeLimit: 64Mi + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth2-proxy-hermes-triage + namespace: hermes + labels: + app: oauth2-proxy-hermes-triage +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: oauth2-proxy-hermes-triage + template: + metadata: + labels: + app: oauth2-proxy-hermes-triage + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: hermes + vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/triage-oidc + vault.hashicorp.com/agent-inject-template-oidc-config: | + {{- with secret "kv/data/atlas/hermes/triage-oidc" -}} + client_id = "{{ .Data.data.client_id }}" + client_secret = "{{ .Data.data.client_secret }}" + cookie_secret = "{{ .Data.data.cookie_secret }}" + {{- end -}} + spec: + serviceAccountName: hermes-vault + automountServiceAccountToken: true + containers: + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0@sha256:dcb6ff8dd21bf3058f6a22c6fa385fa5b897a9cd3914c88a2cc2bb0a85f8065d + imagePullPolicy: IfNotPresent + args: + - --provider=oidc + - --config=/vault/secrets/oidc-config + - --redirect-url=https://triage.hermes.bstein.dev/oauth2/callback + - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas + - --user-id-claim=sub + - --code-challenge-method=S256 + - --scope=openid profile email groups + - --email-domain=* + - --authenticated-emails-file=/etc/oauth2-proxy/allowed-emails + - --set-xauthrequest=true + - --pass-user-headers=true + - --pass-basic-auth=false + - --proxy-websockets=true + - --session-cookie-minimal=true + - --cookie-name=__Host-hermes_triage + - --cookie-path=/ + - --cookie-secure=true + - --cookie-samesite=lax + - --cookie-refresh=1h + - --cookie-expire=8h + - --upstream=http://hermes-triage.hermes.svc.cluster.local:9119 + - --http-address=0.0.0.0:4180 + - --skip-provider-button=true + - --reverse-proxy=true + - --trusted-proxy-ip=10.42.0.0/16 + ports: + - {name: http, containerPort: 4180} + readinessProbe: + httpGet: {path: /ping, port: http} + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /ping, port: http} + initialDelaySeconds: 20 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumeMounts: + - {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: allowlist + configMap: + name: hermes-owner-allowlist + - name: tmp + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth2-proxy-hermes-chat + namespace: hermes + labels: + app: oauth2-proxy-hermes-chat +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: oauth2-proxy-hermes-chat + template: + metadata: + labels: + app: oauth2-proxy-hermes-chat + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: hermes-chat + vault.hashicorp.com/agent-inject-secret-oidc-config: kv/data/atlas/hermes/chat-oidc + vault.hashicorp.com/agent-inject-template-oidc-config: | + {{- with secret "kv/data/atlas/hermes/chat-oidc" -}} + client_id = "{{ .Data.data.client_id }}" + client_secret = "{{ .Data.data.client_secret }}" + cookie_secret = "{{ .Data.data.cookie_secret }}" + {{- end -}} + spec: + serviceAccountName: hermes-chat + automountServiceAccountToken: true + containers: + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0@sha256:dcb6ff8dd21bf3058f6a22c6fa385fa5b897a9cd3914c88a2cc2bb0a85f8065d + imagePullPolicy: IfNotPresent + args: + - --provider=oidc + - --config=/vault/secrets/oidc-config + - --redirect-url=https://chat.hermes.bstein.dev/oauth2/callback + - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas + - --user-id-claim=sub + - --code-challenge-method=S256 + - --scope=openid profile email groups + - --email-domain=* + - --set-xauthrequest=true + - --pass-user-headers=true + - --pass-basic-auth=false + - --proxy-websockets=true + - --session-cookie-minimal=true + - --cookie-name=__Host-hermes_chat + - --cookie-path=/ + - --cookie-secure=true + - --cookie-samesite=lax + - --cookie-refresh=1h + - --cookie-expire=8h + - --upstream=http://hermes-chat-router.hermes.svc.cluster.local:8080 + - --http-address=0.0.0.0:4180 + - --skip-provider-button=true + - --reverse-proxy=true + - --trusted-proxy-ip=10.42.0.0/16 + ports: + - {name: http, containerPort: 4180} + readinessProbe: + httpGet: {path: /ping, port: http} + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /ping, port: http} + initialDelaySeconds: 20 + periodSeconds: 20 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {cpu: 250m, memory: 256Mi} + volumeMounts: + - {name: tmp, mountPath: /tmp} + volumes: + - name: tmp + emptyDir: {sizeLimit: 64Mi} diff --git a/services/hermes/pvc.yaml b/services/hermes/pvc.yaml index 9ff1830fb..3a02cbfb7 100644 --- a/services/hermes/pvc.yaml +++ b/services/hermes/pvc.yaml @@ -16,6 +16,51 @@ spec: --- apiVersion: v1 kind: PersistentVolumeClaim +metadata: + name: hermes-agent-home + namespace: hermes + labels: + app: hermes-agent +spec: + accessModes: + - ReadWriteOnce + storageClassName: astreae + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hermes-provider-auth + namespace: hermes + labels: + app.kubernetes.io/part-of: hermes +spec: + accessModes: + - ReadWriteMany + storageClassName: astreae + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hermes-chat-router-state + namespace: hermes + labels: + app: hermes-chat-router +spec: + accessModes: + - ReadWriteOnce + storageClassName: astreae + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim metadata: name: hermes-models namespace: hermes diff --git a/services/hermes/router/main.go b/services/hermes/router/main.go new file mode 100644 index 000000000..3a49f331d --- /dev/null +++ b/services/hermes/router/main.go @@ -0,0 +1,342 @@ +// Hermes chat tenant router assigns each Keycloak subject to one isolated +// Hermes pod and exposes only the user-facing parts of Hermes WebUI. +package main + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base32" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "net/http" + "net/http/httputil" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type linkRecord struct { + Slot int `json:"slot"` + ExpiresAt int64 `json:"expires_at"` +} + +type tenantState struct { + Salt string `json:"salt"` + Assignments map[string]int `json:"assignments"` + Telegram map[string]int `json:"telegram,omitempty"` + LinkCodes map[string]linkRecord `json:"link_codes,omitempty"` + TelegramOffset int64 `json:"telegram_offset,omitempty"` +} + +type tenantRouter struct { + mu sync.Mutex + state tenantState + statePath string + slots int + backendURL func(int) string + backendAPIURL func(int) string + now func() time.Time + telegram *telegramBot +} + +var deniedPrefixes = []string{ + "/api/admin", "/api/commands/exec", "/api/config", "/api/console", "/api/credentials", + "/api/cron", "/api/curator", "/api/dashboard/config", "/api/env", + "/api/escape", "/api/extensions", "/api/file", "/api/folder", + "/api/gateway", "/api/git", "/api/git-info", "/api/health/restart", "/api/kanban", "/api/logs", "/api/mcp", + "/api/memory", "/api/messaging", "/api/notes", "/api/ops", + "/api/oauth", "/api/onboarding/oauth", "/api/pairing", "/api/plugins", "/api/profiles", "/api/providers", + "/api/rollback", "/api/shutdown", "/api/skills", "/api/terminal", + "/api/tools", "/api/updates", "/api/webhooks", "/api/wiki", + "/api/workspace", "/api/workspaces", +} + +func newTenantRouter(statePath string, slots int, backendURL func(int) string) (*tenantRouter, error) { + if slots < 1 { + return nil, fmt.Errorf("tenant slots must be positive") + } + router := &tenantRouter{ + statePath: statePath, + slots: slots, + backendURL: backendURL, + now: time.Now, + state: tenantState{ + Assignments: map[string]int{}, + Telegram: map[string]int{}, + LinkCodes: map[string]linkRecord{}, + }, + } + content, err := os.ReadFile(statePath) + if err == nil { + if err := json.Unmarshal(content, &router.state); err != nil { + return nil, fmt.Errorf("decode tenant state: %w", err) + } + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("read tenant state: %w", err) + } + if router.state.Assignments == nil { + router.state.Assignments = map[string]int{} + } + if router.state.Telegram == nil { + router.state.Telegram = map[string]int{} + } + if router.state.LinkCodes == nil { + router.state.LinkCodes = map[string]linkRecord{} + } + if router.state.Salt == "" { + salt := make([]byte, 32) + if _, err := rand.Read(salt); err != nil { + return nil, fmt.Errorf("create tenant salt: %w", err) + } + router.state.Salt = hex.EncodeToString(salt) + if err := router.saveLocked(); err != nil { + return nil, err + } + } + return router, nil +} + +func (router *tenantRouter) saveLocked() error { + content, err := json.MarshalIndent(router.state, "", " ") + if err != nil { + return fmt.Errorf("encode tenant state: %w", err) + } + if err := os.MkdirAll(filepath.Dir(router.statePath), 0700); err != nil { + return fmt.Errorf("create tenant state directory: %w", err) + } + temporary := router.statePath + ".tmp" + if err := os.WriteFile(temporary, append(content, '\n'), 0600); err != nil { + return fmt.Errorf("write tenant state: %w", err) + } + if err := os.Rename(temporary, router.statePath); err != nil { + return fmt.Errorf("replace tenant state: %w", err) + } + return nil +} + +func (router *tenantRouter) identityHash(kind, value string) string { + digest := sha256.Sum256([]byte(router.state.Salt + "\x00" + kind + "\x00" + value)) + return hex.EncodeToString(digest[:]) +} + +func (router *tenantRouter) slotFor(subject string) (int, error) { + router.mu.Lock() + defer router.mu.Unlock() + identity := router.identityHash("keycloak", subject) + if slot, ok := router.state.Assignments[identity]; ok { + return slot, nil + } + used := make(map[int]bool, len(router.state.Assignments)) + for _, slot := range router.state.Assignments { + used[slot] = true + } + for slot := 0; slot < router.slots; slot++ { + if used[slot] { + continue + } + router.state.Assignments[identity] = slot + if err := router.saveLocked(); err != nil { + delete(router.state.Assignments, identity) + return 0, err + } + return slot, nil + } + return 0, fmt.Errorf("all isolated chat slots are assigned") +} + +func (router *tenantRouter) createLink(subject string) (string, time.Time, error) { + slot, err := router.slotFor(subject) + if err != nil { + return "", time.Time{}, err + } + raw := make([]byte, 5) + if _, err := rand.Read(raw); err != nil { + return "", time.Time{}, fmt.Errorf("create link code: %w", err) + } + code := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw) + expires := router.now().Add(10 * time.Minute) + router.mu.Lock() + defer router.mu.Unlock() + for digest, record := range router.state.LinkCodes { + if record.Slot == slot || record.ExpiresAt <= router.now().Unix() { + delete(router.state.LinkCodes, digest) + } + } + digest := router.identityHash("link", strings.ToUpper(code)) + router.state.LinkCodes[digest] = linkRecord{Slot: slot, ExpiresAt: expires.Unix()} + if err := router.saveLocked(); err != nil { + delete(router.state.LinkCodes, digest) + return "", time.Time{}, err + } + return code, expires, nil +} + +func (router *tenantRouter) consumeLink(telegramUser, code string) (int, error) { + router.mu.Lock() + defer router.mu.Unlock() + digest := router.identityHash("link", strings.ToUpper(strings.TrimSpace(code))) + record, ok := router.state.LinkCodes[digest] + if !ok || record.ExpiresAt <= router.now().Unix() { + delete(router.state.LinkCodes, digest) + return 0, fmt.Errorf("link code is invalid or expired") + } + delete(router.state.LinkCodes, digest) + router.state.Telegram[router.identityHash("telegram", telegramUser)] = record.Slot + if err := router.saveLocked(); err != nil { + return 0, err + } + return record.Slot, nil +} + +func (router *tenantRouter) telegramSlot(telegramUser string) (int, bool) { + router.mu.Lock() + defer router.mu.Unlock() + slot, ok := router.state.Telegram[router.identityHash("telegram", telegramUser)] + return slot, ok +} + +func (router *tenantRouter) telegramLinked(subject string) (bool, error) { + slot, err := router.slotFor(subject) + if err != nil { + return false, err + } + router.mu.Lock() + defer router.mu.Unlock() + for _, linkedSlot := range router.state.Telegram { + if linkedSlot == slot { + return true, nil + } + } + return false, nil +} + +func (router *tenantRouter) unlinkTelegram(subject string) error { + slot, err := router.slotFor(subject) + if err != nil { + return err + } + router.mu.Lock() + defer router.mu.Unlock() + for identity, linkedSlot := range router.state.Telegram { + if linkedSlot == slot { + delete(router.state.Telegram, identity) + } + } + for digest, record := range router.state.LinkCodes { + if record.Slot == slot { + delete(router.state.LinkCodes, digest) + } + } + return router.saveLocked() +} + +func authenticatedSubject(request *http.Request) string { + for _, header := range []string{"X-Forwarded-User", "X-Auth-Request-User"} { + if subject := strings.TrimSpace(request.Header.Get(header)); subject != "" { + return subject + } + } + return "" +} + +func pathDenied(path string) bool { + for _, prefix := range deniedPrefixes { + if path == prefix || strings.HasPrefix(path, prefix+"/") { + return true + } + } + return false +} + +func (router *tenantRouter) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/healthz" { + writer.Header().Set("Content-Type", "text/plain") + _, _ = writer.Write([]byte("ok\n")) + return + } + subject := authenticatedSubject(request) + if subject == "" { + http.Error(writer, "authenticated identity required", http.StatusUnauthorized) + return + } + if router.serveTelegramWeb(writer, request, subject) { + return + } + if pathDenied(request.URL.Path) { + http.Error(writer, "chat administration is disabled", http.StatusForbidden) + return + } + slot, err := router.slotFor(subject) + if err != nil { + http.Error(writer, err.Error(), http.StatusServiceUnavailable) + return + } + target, err := url.Parse(router.backendURL(slot)) + if err != nil { + http.Error(writer, "invalid tenant backend", http.StatusInternalServerError) + return + } + proxy := httputil.NewSingleHostReverseProxy(target) + proxy.ModifyResponse = injectChatBridge + proxy.ErrorHandler = func(writer http.ResponseWriter, _ *http.Request, proxyErr error) { + log.Printf("tenant slot %d unavailable", slot) + http.Error(writer, "private chat runtime is starting", http.StatusBadGateway) + } + originalDirector := proxy.Director + proxy.Director = func(outbound *http.Request) { + originalDirector(outbound) + for _, header := range []string{ + "Authorization", "Cookie", "X-Auth-Request-Access-Token", + "X-Auth-Request-Email", "X-Auth-Request-Groups", "X-Auth-Request-User", + "X-Forwarded-Access-Token", "X-Forwarded-Email", "X-Forwarded-Groups", + "X-Forwarded-Preferred-Username", "X-Forwarded-User", + } { + outbound.Header.Del(header) + } + outbound.Header.Del("Accept-Encoding") + } + proxy.ServeHTTP(writer, request) +} + +func main() { + slots, err := strconv.Atoi(os.Getenv("TENANT_SLOTS")) + if err != nil || slots < 1 { + log.Fatal("TENANT_SLOTS must be a positive integer") + } + statePath := os.Getenv("TENANT_STATE_PATH") + if statePath == "" { + statePath = "/state/tenants.json" + } + router, err := newTenantRouter(statePath, slots, func(slot int) string { + return fmt.Sprintf("http://hermes-chat-tenant-%d.hermes-chat-tenant.hermes.svc.cluster.local:8787", slot) + }) + if err != nil { + log.Fatal(err) + } + router.backendAPIURL = func(slot int) string { + return fmt.Sprintf("http://hermes-chat-tenant-%d.hermes-chat-tenant.hermes.svc.cluster.local:8642", slot) + } + telegramConfig, err := readTelegramConfig(os.Getenv("TELEGRAM_CONFIG_PATH")) + if err != nil { + log.Printf("Telegram is disabled: configuration is unavailable") + } else if telegramConfig.BotToken != "" && telegramConfig.RelayKey != "" { + router.telegram = newTelegramBot(telegramConfig, router) + go router.telegram.run() + log.Printf("Telegram transport enabled") + } else { + log.Printf("Telegram is prepared but disabled until bot_token is set in Vault") + } + server := &http.Server{ + Addr: ":8080", + Handler: router, + ReadHeaderTimeout: 10 * time.Second, + } + log.Printf("Hermes chat tenant router ready with %d isolated slots", slots) + log.Fatal(server.ListenAndServe()) +} diff --git a/services/hermes/router/main_test.go b/services/hermes/router/main_test.go new file mode 100644 index 000000000..5c13ed0e9 --- /dev/null +++ b/services/hermes/router/main_test.go @@ -0,0 +1,182 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestSubjectsReceiveStableIsolatedSlots(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "tenants.json") + router, err := newTenantRouter(statePath, 2, func(slot int) string { return "" }) + if err != nil { + t.Fatal(err) + } + first, err := router.slotFor("keycloak-subject-a") + if err != nil { + t.Fatal(err) + } + again, _ := router.slotFor("keycloak-subject-a") + second, err := router.slotFor("keycloak-subject-b") + if err != nil { + t.Fatal(err) + } + if first != again || first == second { + t.Fatalf("unexpected slots: first=%d again=%d second=%d", first, again, second) + } + if _, err := router.slotFor("keycloak-subject-c"); err == nil { + t.Fatal("expected the fixed private tenant pool to report capacity") + } + reloaded, err := newTenantRouter(statePath, 2, func(slot int) string { return "" }) + if err != nil { + t.Fatal(err) + } + persisted, _ := reloaded.slotFor("keycloak-subject-a") + if persisted != first { + t.Fatalf("assignment changed after reload: %d != %d", persisted, first) + } +} + +func TestRouterBlocksAdministrationAndRequiresIdentity(t *testing.T) { + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "http://127.0.0.1" }) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + method string + path string + user string + want int + }{ + {http.MethodGet, "/", "", http.StatusUnauthorized}, + {http.MethodPost, "/api/providers", "subject", http.StatusForbidden}, + {http.MethodPost, "/api/commands/exec", "subject", http.StatusForbidden}, + {http.MethodGet, "/api/dashboard/config", "subject", http.StatusForbidden}, + {http.MethodGet, "/api/env", "subject", http.StatusForbidden}, + } { + request := httptest.NewRequest(test.method, test.path, nil) + request.Header.Set("X-Forwarded-User", test.user) + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != test.want { + t.Fatalf("%s %s: got %d, want %d", test.method, test.path, response.Code, test.want) + } + } +} + +func TestRouterProxiesWebUIAndAddsTelegramShortcut(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Header.Get("X-Forwarded-User") != "" || request.Header.Get("Cookie") != "" { + t.Fatal("identity or session cookie leaked to tenant backend") + } + writer.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = io.WriteString(writer, "Hermes WebUI") + })) + defer backend.Close() + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return backend.URL }) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "/", nil) + request.Header.Set("X-Forwarded-User", "subject") + request.Header.Set("Cookie", "oauth-cookie") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("got status %d", response.Code) + } + if !strings.Contains(response.Body.String(), "hermes-chat-bridge.js") || !strings.Contains(response.Body.String(), "hermes-chat-bridge.css") { + t.Fatal("Telegram shortcut assets were not injected") + } +} + +func TestWebUIModelAndReasoningOverridesAreProxied(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + t.Fatalf("got method %s", request.Method) + } + writer.WriteHeader(http.StatusNoContent) + })) + defer backend.Close() + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return backend.URL }) + if err != nil { + t.Fatal(err) + } + for _, path := range []string{"/api/model/set", "/api/reasoning"} { + request := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{}`)) + request.Header.Set("X-Forwarded-User", "subject") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != http.StatusNoContent { + t.Fatalf("POST %s: got %d", path, response.Code) + } + } +} + +func TestTelegramLinkUsesHashedStateAndExpires(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "state.json") + router, err := newTenantRouter(statePath, 1, func(slot int) string { return "" }) + if err != nil { + t.Fatal(err) + } + now := time.Date(2026, 8, 8, 12, 0, 0, 0, time.UTC) + router.now = func() time.Time { return now } + code, expires, err := router.createLink("raw-keycloak-subject") + if err != nil { + t.Fatal(err) + } + if expires.Sub(now) != 10*time.Minute { + t.Fatalf("unexpected expiry: %s", expires) + } + if _, err := router.consumeLink("123456789", code); err != nil { + t.Fatal(err) + } + if _, linked := router.telegramSlot("123456789"); !linked { + t.Fatal("Telegram identity was not linked") + } + content, err := os.ReadFile(statePath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), "raw-keycloak-subject") || strings.Contains(string(content), "123456789") || strings.Contains(string(content), code) { + t.Fatal("raw identity or one-time code was persisted") + } + secondCode, _, err := router.createLink("raw-keycloak-subject") + if err != nil { + t.Fatal(err) + } + router.now = func() time.Time { return now.Add(11 * time.Minute) } + if _, err := router.consumeLink("987654321", secondCode); err == nil { + t.Fatal("expected expired link code to be rejected") + } +} + +func TestTelegramWebActionsRequireExplicitSameOriginHeader(t *testing.T) { + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "" }) + if err != nil { + t.Fatal(err) + } + router.telegram = &telegramBot{} + request := httptest.NewRequest(http.MethodPost, "/api/telegram/link", strings.NewReader(`{}`)) + request.Header.Set("X-Forwarded-User", "subject") + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != http.StatusForbidden { + t.Fatalf("got %d", response.Code) + } + request = httptest.NewRequest(http.MethodPost, "/api/telegram/link", strings.NewReader(`{}`)) + request.Header.Set("X-Forwarded-User", "subject") + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Hermes-Action", "telegram-link") + response = httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("got %d: %s", response.Code, response.Body.String()) + } +} diff --git a/services/hermes/router/telegram.go b/services/hermes/router/telegram.go new file mode 100644 index 000000000..e58fcde6f --- /dev/null +++ b/services/hermes/router/telegram.go @@ -0,0 +1,392 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + "unicode/utf8" +) + +type telegramConfig struct { + BotToken string + RelayKey string +} + +type telegramBot struct { + config telegramConfig + router *tenantRouter + apiBase string + client *http.Client + agentClient *http.Client + mu sync.RWMutex + botUsername string + workLimit chan struct{} + slotLocks map[int]*sync.Mutex + slotLocksMux sync.Mutex +} + +type telegramUpdate struct { + UpdateID int64 `json:"update_id"` + Message *telegramMessage `json:"message"` +} + +type telegramMessage struct { + MessageID int64 `json:"message_id"` + From *telegramUser `json:"from"` + Chat telegramChat `json:"chat"` + Text string `json:"text"` +} + +type telegramUser struct { + ID int64 `json:"id"` +} + +type telegramChat struct { + ID int64 `json:"id"` + Type string `json:"type"` +} + +func readTelegramConfig(path string) (telegramConfig, error) { + if path == "" { + path = "/vault/secrets/telegram-config" + } + content, err := os.ReadFile(path) + if err != nil { + return telegramConfig{}, err + } + config := telegramConfig{} + for _, line := range strings.Split(string(content), "\n") { + key, value, found := strings.Cut(strings.TrimSpace(line), "=") + if !found { + continue + } + switch strings.TrimSpace(key) { + case "bot_token": + config.BotToken = strings.TrimSpace(value) + case "relay_key": + config.RelayKey = strings.TrimSpace(value) + } + } + return config, nil +} + +func newTelegramBot(config telegramConfig, router *tenantRouter) *telegramBot { + return &telegramBot{ + config: config, + router: router, + apiBase: "https://api.telegram.org/bot" + config.BotToken, + client: &http.Client{Timeout: 70 * time.Second}, + agentClient: &http.Client{Timeout: 15 * time.Minute}, + workLimit: make(chan struct{}, 4), + slotLocks: map[int]*sync.Mutex{}, + } +} + +func (bot *telegramBot) username() string { + bot.mu.RLock() + defer bot.mu.RUnlock() + return bot.botUsername +} + +func (bot *telegramBot) setUsername(username string) { + bot.mu.Lock() + defer bot.mu.Unlock() + bot.botUsername = strings.TrimPrefix(strings.TrimSpace(username), "@") +} + +func (bot *telegramBot) slotLock(slot int) *sync.Mutex { + bot.slotLocksMux.Lock() + defer bot.slotLocksMux.Unlock() + if bot.slotLocks[slot] == nil { + bot.slotLocks[slot] = &sync.Mutex{} + } + return bot.slotLocks[slot] +} + +func (bot *telegramBot) call(ctx context.Context, method string, values url.Values, result any) error { + request, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + bot.apiBase+"/"+method, + strings.NewReader(values.Encode()), + ) + if err != nil { + return errors.New("create Telegram request") + } + request.Header.Set("Content-Type", "application/x-www-form-urlencoded") + response, err := bot.client.Do(request) + if err != nil { + return errors.New("Telegram API unavailable") + } + defer response.Body.Close() + body, err := io.ReadAll(io.LimitReader(response.Body, 2<<20)) + if err != nil { + return errors.New("read Telegram response") + } + var envelope struct { + OK bool `json:"ok"` + Result json.RawMessage `json:"result"` + } + if response.StatusCode != http.StatusOK || json.Unmarshal(body, &envelope) != nil || !envelope.OK { + return fmt.Errorf("Telegram API returned status %d", response.StatusCode) + } + if result != nil && len(envelope.Result) > 0 { + if err := json.Unmarshal(envelope.Result, result); err != nil { + return errors.New("decode Telegram response") + } + } + return nil +} + +func (bot *telegramBot) run() { + for { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + var me struct { + Username string `json:"username"` + } + err := bot.call(ctx, "getMe", url.Values{}, &me) + cancel() + if err == nil { + bot.setUsername(me.Username) + break + } + time.Sleep(10 * time.Second) + } + for { + offset := bot.router.telegramOffset() + ctx, cancel := context.WithTimeout(context.Background(), 65*time.Second) + values := url.Values{ + "offset": {strconv.FormatInt(offset, 10)}, + "timeout": {"50"}, + "allowed_updates": {`["message"]`}, + } + var updates []telegramUpdate + err := bot.call(ctx, "getUpdates", values, &updates) + cancel() + if err != nil { + time.Sleep(3 * time.Second) + continue + } + for _, update := range updates { + _ = bot.router.setTelegramOffset(update.UpdateID + 1) + bot.workLimit <- struct{}{} + go func(update telegramUpdate) { + defer func() { <-bot.workLimit }() + bot.handleUpdate(update) + }(update) + } + } +} + +func commandParts(text string) (string, []string) { + fields := strings.Fields(strings.TrimSpace(text)) + if len(fields) == 0 || !strings.HasPrefix(fields[0], "/") { + return "", nil + } + command := strings.TrimPrefix(strings.ToLower(fields[0]), "/") + command, _, _ = strings.Cut(command, "@") + return command, fields[1:] +} + +func (bot *telegramBot) handleUpdate(update telegramUpdate) { + message := update.Message + if message == nil || message.From == nil || message.Chat.Type != "private" || message.Chat.ID != message.From.ID { + return + } + userID := strconv.FormatInt(message.From.ID, 10) + command, args := commandParts(message.Text) + if command == "start" || command == "link" { + if len(args) == 0 { + _ = bot.sendText(message.Chat.ID, "Sign in to chat.hermes.bstein.dev, open Telegram, and create a one-time link code.") + return + } + if _, err := bot.router.consumeLink(userID, args[0]); err != nil { + _ = bot.sendText(message.Chat.ID, "That link code is invalid or expired. Create a new one in Hermes WebUI.") + return + } + _ = bot.sendText(message.Chat.ID, "Telegram is linked to your private Hermes account. Send a message whenever you are ready.") + return + } + if command == "unlink" { + if err := bot.router.unlinkTelegramUser(userID); err != nil { + _ = bot.sendText(message.Chat.ID, "I could not unlink Telegram right now. Try again shortly.") + return + } + _ = bot.sendText(message.Chat.ID, "Telegram has been unlinked from Hermes.") + return + } + if command == "help" { + _ = bot.sendText(message.Chat.ID, "Send any text to chat with Hermes. Use /unlink to disconnect this Telegram account. Model and intensity controls are available in Hermes WebUI.") + return + } + slot, linked := bot.router.telegramSlot(userID) + if !linked { + _ = bot.sendText(message.Chat.ID, "This Telegram account is not linked. Sign in to chat.hermes.bstein.dev and open Telegram to connect it.") + return + } + text := strings.TrimSpace(message.Text) + if text == "" { + _ = bot.sendText(message.Chat.ID, "Text messages are supported now; attachment support will be added separately.") + return + } + if utf8.RuneCountInString(text) > 12000 { + _ = bot.sendText(message.Chat.ID, "That message is too long. Please split it into smaller parts.") + return + } + lock := bot.slotLock(slot) + lock.Lock() + defer lock.Unlock() + _ = bot.sendAction(message.Chat.ID, "typing") + reply, err := bot.askTenant(slot, text, update.UpdateID) + if err != nil { + _ = bot.sendText(message.Chat.ID, "Hermes could not answer right now. Please try again shortly.") + return + } + _ = bot.sendText(message.Chat.ID, reply) +} + +func (bot *telegramBot) askTenant(slot int, text string, updateID int64) (string, error) { + if bot.router.backendAPIURL == nil { + return "", errors.New("tenant API unavailable") + } + payload, _ := json.Marshal(map[string]any{ + "input": text, + "conversation": "telegram", + "store": true, + }) + ctx, cancel := context.WithTimeout(context.Background(), 14*time.Minute) + defer cancel() + request, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + bot.router.backendAPIURL(slot)+"/v1/responses", + bytes.NewReader(payload), + ) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Bearer "+bot.config.RelayKey) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Hermes-Session-Key", "telegram") + request.Header.Set("Idempotency-Key", "telegram-"+strconv.FormatInt(updateID, 10)) + response, err := bot.agentClient.Do(request) + if err != nil { + return "", errors.New("tenant request failed") + } + defer response.Body.Close() + body, err := io.ReadAll(io.LimitReader(response.Body, 8<<20)) + if err != nil || response.StatusCode != http.StatusOK { + return "", fmt.Errorf("tenant returned status %d", response.StatusCode) + } + var parsed struct { + Output []struct { + Type string `json:"type"` + Role string `json:"role"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } `json:"output"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return "", errors.New("decode tenant response") + } + var parts []string + for _, item := range parsed.Output { + if item.Type != "message" || item.Role != "assistant" { + continue + } + for _, content := range item.Content { + if (content.Type == "output_text" || content.Type == "text") && strings.TrimSpace(content.Text) != "" { + parts = append(parts, strings.TrimSpace(content.Text)) + } + } + } + answer := strings.Join(parts, "\n\n") + if answer == "" { + return "", errors.New("tenant returned no assistant text") + } + return answer, nil +} + +func (bot *telegramBot) sendAction(chatID int64, action string) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return bot.call(ctx, "sendChatAction", url.Values{ + "chat_id": {strconv.FormatInt(chatID, 10)}, + "action": {action}, + }, nil) +} + +func splitTelegramText(text string) []string { + runes := []rune(strings.TrimSpace(text)) + if len(runes) == 0 { + return []string{"Hermes completed the request without a text response."} + } + const limit = 3900 + var chunks []string + for len(runes) > limit { + cut := limit + for index := limit; index > limit-500; index-- { + if runes[index-1] == '\n' || runes[index-1] == ' ' { + cut = index + break + } + } + chunks = append(chunks, strings.TrimSpace(string(runes[:cut]))) + runes = runes[cut:] + } + if tail := strings.TrimSpace(string(runes)); tail != "" { + chunks = append(chunks, tail) + } + return chunks +} + +func (bot *telegramBot) sendText(chatID int64, text string) error { + for _, chunk := range splitTelegramText(text) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + err := bot.call(ctx, "sendMessage", url.Values{ + "chat_id": {strconv.FormatInt(chatID, 10)}, + "text": {chunk}, + "disable_web_page_preview": {"true"}, + }, nil) + cancel() + if err != nil { + return err + } + } + return nil +} + +func (router *tenantRouter) telegramOffset() int64 { + router.mu.Lock() + defer router.mu.Unlock() + return router.state.TelegramOffset +} + +func (router *tenantRouter) setTelegramOffset(offset int64) error { + router.mu.Lock() + defer router.mu.Unlock() + if offset <= router.state.TelegramOffset { + return nil + } + router.state.TelegramOffset = offset + return router.saveLocked() +} + +func (router *tenantRouter) unlinkTelegramUser(userID string) error { + router.mu.Lock() + defer router.mu.Unlock() + delete(router.state.Telegram, router.identityHash("telegram", userID)) + return router.saveLocked() +} diff --git a/services/hermes/router/telegram_test.go b/services/hermes/router/telegram_test.go new file mode 100644 index 000000000..6762ab33d --- /dev/null +++ b/services/hermes/router/telegram_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReadTelegramConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "telegram-config") + if err := os.WriteFile(path, []byte("bot_token=123:abc\nrelay_key=relay-secret\n"), 0600); err != nil { + t.Fatal(err) + } + config, err := readTelegramConfig(path) + if err != nil { + t.Fatal(err) + } + if config.BotToken != "123:abc" || config.RelayKey != "relay-secret" { + t.Fatalf("unexpected config: %#v", config) + } +} + +func TestAskTenantUsesAuthenticatedNamedConversation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/v1/responses" { + t.Fatalf("unexpected path %s", request.URL.Path) + } + if request.Header.Get("Authorization") != "Bearer relay-secret" { + t.Fatal("relay authentication was not set") + } + if request.Header.Get("X-Hermes-Session-Key") != "telegram" { + t.Fatal("Telegram session scope was not set") + } + var payload map[string]any + if err := json.NewDecoder(request.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + if payload["conversation"] != "telegram" || payload["input"] != "hello" { + t.Fatalf("unexpected payload: %#v", payload) + } + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Hello from Hermes"}]}]}`)) + })) + defer server.Close() + router, err := newTenantRouter(filepath.Join(t.TempDir(), "state.json"), 1, func(slot int) string { return "" }) + if err != nil { + t.Fatal(err) + } + router.backendAPIURL = func(slot int) string { return server.URL } + bot := newTelegramBot(telegramConfig{RelayKey: "relay-secret"}, router) + answer, err := bot.askTenant(0, "hello", 42) + if err != nil { + t.Fatal(err) + } + if answer != "Hello from Hermes" { + t.Fatalf("unexpected answer %q", answer) + } +} + +func TestSplitTelegramTextStaysUnderTelegramLimit(t *testing.T) { + chunks := splitTelegramText(strings.Repeat("word ", 2000)) + if len(chunks) < 2 { + t.Fatal("expected a long response to be split") + } + for _, chunk := range chunks { + if len([]rune(chunk)) > 3900 { + t.Fatalf("chunk exceeds limit: %d", len([]rune(chunk))) + } + } +} diff --git a/services/hermes/router/web.go b/services/hermes/router/web.go new file mode 100644 index 000000000..ea571ca7f --- /dev/null +++ b/services/hermes/router/web.go @@ -0,0 +1,232 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +const telegramPage = ` + + + + + Hermes on Telegram + + + +
+ ← Back to Hermes +

Hermes on Telegram

+

Link this Keycloak account to a private Telegram chat. Messages will use the same isolated Hermes tenant as the WebUI.

+

Checking Telegram…

+ + +

Codes expire after 10 minutes. Only direct messages are accepted; group messages are ignored.

+
+ + +` + +const bridgeCSS = ` +#hermes-telegram-shortcut{position:fixed;right:18px;bottom:18px;z-index:9999;padding:10px 14px;border-radius:999px;background:#229ed9;color:#fff;text-decoration:none;font:600 14px system-ui,sans-serif;box-shadow:0 5px 20px #0005} +.hermes-link-page{margin:0;min-height:100vh;display:grid;place-items:center;background:#0f172a;color:#e2e8f0;font:16px/1.5 system-ui,sans-serif} +.hermes-link-card{width:min(620px,calc(100% - 40px));box-sizing:border-box;padding:32px;border:1px solid #334155;border-radius:18px;background:#111827;box-shadow:0 20px 60px #0006} +.hermes-link-card h1{margin:.6rem 0}.hermes-back{color:#7dd3fc}.hermes-link-actions{display:flex;gap:12px;flex-wrap:wrap;margin:24px 0} +.hermes-link-card button{border:0;border-radius:10px;padding:11px 16px;background:#229ed9;color:#fff;font-weight:700;cursor:pointer}.hermes-link-card button.secondary{background:#334155} +#telegram-result{padding:16px;border-radius:10px;background:#1e293b;overflow-wrap:anywhere}#telegram-result a{color:#7dd3fc}.hermes-fine-print{color:#94a3b8;font-size:13px} +` + +const bridgeJS = `(() => { + const page = document.querySelector('[data-telegram-page]'); + if (!page) { + if (!document.getElementById('hermes-telegram-shortcut')) { + const link = document.createElement('a'); + link.id = 'hermes-telegram-shortcut'; + link.href = '/telegram'; + link.textContent = 'Telegram'; + link.setAttribute('aria-label', 'Connect Hermes to Telegram'); + document.body.appendChild(link); + } + return; + } + const status = document.getElementById('telegram-status'); + const result = document.getElementById('telegram-result'); + const linkButton = document.getElementById('telegram-link'); + const unlinkButton = document.getElementById('telegram-unlink'); + const action = async (path) => { + const response = await fetch(path, {method:'POST',headers:{'Content-Type':'application/json','X-Hermes-Action':'telegram-link'},body:'{}'}); + const payload = await response.json(); + if (!response.ok) throw new Error(payload.error || 'Request failed'); + return payload; + }; + const refresh = async () => { + try { + const response = await fetch('/api/telegram/status', {cache:'no-store'}); + const payload = await response.json(); + if (!payload.configured) { + status.textContent = 'Telegram is prepared, but the bot token has not been added by the operator yet.'; + linkButton.disabled = true; + unlinkButton.hidden = true; + return; + } + status.textContent = payload.linked ? 'Telegram is linked to this private account.' : 'Telegram is ready to link.'; + unlinkButton.hidden = !payload.linked; + } catch (_) { status.textContent = 'Telegram status is temporarily unavailable.'; } + }; + linkButton.addEventListener('click', async () => { + try { + const payload = await action('/api/telegram/link'); + result.hidden = false; + result.replaceChildren(); + const text = document.createElement('p'); + text.textContent = 'Send /link ' + payload.code + ' to the Hermes bot. This code expires at ' + new Date(payload.expires_at).toLocaleTimeString() + '.'; + result.appendChild(text); + if (payload.deep_link) { + const anchor = document.createElement('a'); + anchor.href = payload.deep_link; + anchor.rel = 'noopener noreferrer'; + anchor.textContent = 'Open Telegram and link now'; + result.appendChild(anchor); + } + } catch (error) { status.textContent = error.message; } + }); + unlinkButton.addEventListener('click', async () => { + try { await action('/api/telegram/unlink'); result.hidden = true; await refresh(); } + catch (error) { status.textContent = error.message; } + }); + refresh(); +})();` + +func writeJSON(writer http.ResponseWriter, status int, value any) { + writer.Header().Set("Content-Type", "application/json") + writer.Header().Set("Cache-Control", "no-store") + writer.WriteHeader(status) + _ = json.NewEncoder(writer).Encode(value) +} + +func validTelegramAction(request *http.Request) bool { + return request.Header.Get("X-Hermes-Action") == "telegram-link" && + strings.HasPrefix(request.Header.Get("Content-Type"), "application/json") +} + +func (router *tenantRouter) serveTelegramWeb(writer http.ResponseWriter, request *http.Request, subject string) bool { + switch request.URL.Path { + case "/hermes-chat-bridge.css": + if request.Method != http.MethodGet { + http.Error(writer, "method not allowed", http.StatusMethodNotAllowed) + return true + } + writer.Header().Set("Content-Type", "text/css; charset=utf-8") + writer.Header().Set("Cache-Control", "public, max-age=3600") + _, _ = io.WriteString(writer, bridgeCSS) + return true + case "/hermes-chat-bridge.js": + if request.Method != http.MethodGet { + http.Error(writer, "method not allowed", http.StatusMethodNotAllowed) + return true + } + writer.Header().Set("Content-Type", "application/javascript; charset=utf-8") + writer.Header().Set("Cache-Control", "public, max-age=3600") + _, _ = io.WriteString(writer, bridgeJS) + return true + case "/telegram": + if request.Method != http.MethodGet { + http.Error(writer, "method not allowed", http.StatusMethodNotAllowed) + return true + } + writer.Header().Set("Content-Type", "text/html; charset=utf-8") + writer.Header().Set("Cache-Control", "no-store") + writer.Header().Set("Content-Security-Policy", "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; base-uri 'none'; form-action 'self'") + _, _ = io.WriteString(writer, telegramPage) + return true + case "/api/telegram/status": + if request.Method != http.MethodGet { + writeJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"}) + return true + } + linked, err := router.telegramLinked(subject) + if err != nil { + writeJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": err.Error()}) + return true + } + username := "" + if router.telegram != nil { + username = router.telegram.username() + } + writeJSON(writer, http.StatusOK, map[string]any{ + "configured": router.telegram != nil, + "linked": linked, + "bot_username": username, + }) + return true + case "/api/telegram/link": + if request.Method != http.MethodPost || !validTelegramAction(request) { + writeJSON(writer, http.StatusForbidden, map[string]string{"error": "same-origin action required"}) + return true + } + if router.telegram == nil { + writeJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "Telegram bot token is not configured"}) + return true + } + code, expires, err := router.createLink(subject) + if err != nil { + writeJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": err.Error()}) + return true + } + username := router.telegram.username() + deepLink := "" + if username != "" { + deepLink = fmt.Sprintf("https://t.me/%s?start=%s", url.PathEscape(username), url.QueryEscape(code)) + } + writeJSON(writer, http.StatusOK, map[string]any{ + "code": code, + "expires_at": expires.Format(time.RFC3339), + "deep_link": deepLink, + }) + return true + case "/api/telegram/unlink": + if request.Method != http.MethodPost || !validTelegramAction(request) { + writeJSON(writer, http.StatusForbidden, map[string]string{"error": "same-origin action required"}) + return true + } + if err := router.unlinkTelegram(subject); err != nil { + writeJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": err.Error()}) + return true + } + writeJSON(writer, http.StatusOK, map[string]bool{"unlinked": true}) + return true + default: + return false + } +} + +func injectChatBridge(response *http.Response) error { + if !strings.Contains(response.Header.Get("Content-Type"), "text/html") { + return nil + } + body, err := io.ReadAll(response.Body) + if err != nil { + return err + } + _ = response.Body.Close() + content := string(body) + if !strings.Contains(content, "hermes-chat-bridge.js") { + content = strings.Replace(content, "", ``, 1) + content = strings.Replace(content, "", ``, 1) + } + response.Body = io.NopCloser(strings.NewReader(content)) + response.ContentLength = int64(len(content)) + response.Header.Set("Content-Length", strconv.Itoa(len(content))) + response.Header.Set("Cache-Control", "no-store") + response.Header.Del("ETag") + return nil +} diff --git a/services/hermes/scripts/gitea_askpass.sh b/services/hermes/scripts/gitea_askpass.sh new file mode 100644 index 000000000..a52635c62 --- /dev/null +++ b/services/hermes/scripts/gitea_askpass.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh +set -eu + +case "${1:-}" in + *Username*) printf '%s\n' "${GITEA_USERNAME:-bstein}" ;; + *Password*) printf '%s\n' "${GITEA_TOKEN:-}" ;; + *) exit 1 ;; +esac diff --git a/services/hermes/scripts/herdr_dispatch.py b/services/hermes/scripts/herdr_dispatch.py new file mode 100644 index 000000000..4822e8db0 --- /dev/null +++ b/services/hermes/scripts/herdr_dispatch.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Plan and launch difficulty-aware Codex or Claude Code workers through Herdr.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +from pathlib import Path +from typing import Any + + +ALLOWED_EFFORTS = ("low", "medium", "high", "xhigh") +ROUTING_PATH = Path("/opt/data/workspace/coordinator/model-routing.json") +HERDR_BIN = Path("/opt/data/tools/bin/herdr") +CODEX_AUTH = Path("/opt/data/home/.codex/auth.json") + + +def _load_routes(path: Path) -> dict[str, Any]: + """Load the non-secret route status generated by the model steward.""" + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"routing status unavailable: {path}") from error + if not isinstance(value, dict) or not isinstance(value.get("routes"), dict): + raise RuntimeError(f"routing status is malformed: {path}") + return value + + +def _split_route(route: str) -> tuple[str, str]: + """Split a provider/model route without corrupting model punctuation.""" + provider, separator, model = route.partition("/") + if not separator or not provider or not model: + raise RuntimeError(f"invalid route: {route}") + return provider, model + + +def select_plan( + status: dict[str, Any], shape: str, effort: str, provider: str | None = None +) -> dict[str, Any]: + """Select a capped route and retain the complete capacity fallback chain.""" + if effort not in ALLOWED_EFFORTS: + raise ValueError(f"effort must be one of: {', '.join(ALLOWED_EFFORTS)}") + default_provider = "codex" if shape == "implementation" else "claude" + selected_provider = provider or default_provider + if selected_provider not in {"codex", "claude"}: + raise ValueError("provider must be codex or claude") + profile = f"{selected_provider}-{effort}" + chain = status["routes"].get(profile) + if not isinstance(chain, list) or not chain: + raise RuntimeError(f"route profile unavailable: {profile}") + primary_provider, model = _split_route(str(chain[0])) + return { + "shape": shape, + "effort": effort, + "profile": profile, + "worker": selected_provider, + "provider": primary_provider, + "model": model, + "fallback_chain": [str(route) for route in chain[1:]], + } + + +def _slug(value: str, limit: int = 32) -> str: + """Return a Herdr-safe stable label.""" + cleaned = re.sub(r"[^a-z0-9_-]+", "-", value.lower()).strip("-") + if not cleaned or not cleaned[0].isalpha(): + cleaned = f"task-{cleaned}" + return cleaned[:limit].rstrip("-") + + +def _run(command: list[str], env: dict[str, str]) -> dict[str, Any]: + """Run one Herdr JSON command and surface a concise error.""" + completed = subprocess.run( + command, + env=env, + text=True, + capture_output=True, + timeout=320, + check=False, + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise RuntimeError(detail or f"command failed with {completed.returncode}") + try: + value = json.loads(completed.stdout) + except json.JSONDecodeError as error: + raise RuntimeError("Herdr returned non-JSON output") from error + if not isinstance(value, dict): + raise RuntimeError("Herdr returned an unexpected response") + return value + + +def launch_worker( + plan: dict[str, Any], project: Path, task: str, prompt: str | None +) -> dict[str, Any]: + """Create an isolated Herdr workspace and launch the selected worker.""" + if not HERDR_BIN.is_file(): + raise RuntimeError("Herdr is not installed in the agent tools volume") + if not project.is_dir(): + raise RuntimeError(f"project workspace does not exist: {project}") + if plan["worker"] == "codex" and not CODEX_AUTH.is_file(): + raise RuntimeError( + "Codex CLI needs its one-time device login. Ask Hermes to run " + "`codex login --device-auth`, complete the code, then retry." + ) + + env = os.environ.copy() + env.update( + { + "HOME": "/opt/data/home", + "CODEX_HOME": "/opt/data/home/.codex", + "CLAUDE_CONFIG_DIR": "/opt/data/home/.claude", + "HERDR_CONFIG_PATH": "/opt/data/home/.config/herdr/config.toml", + "HERDR_SOCKET_PATH": "/opt/data/herdr/herdr.sock", + "PATH": "/opt/data/tools/bin:" + env.get("PATH", ""), + } + ) + label = _slug(task) + created = _run( + [ + str(HERDR_BIN), + "workspace", + "create", + "--cwd", + str(project), + "--label", + label, + "--no-focus", + ], + env, + ) + try: + pane = str(created["result"]["root_pane"]["pane_id"]) + except (KeyError, TypeError) as error: + raise RuntimeError("Herdr workspace response omitted the root pane") from error + + agent_name = _slug(f"{plan['worker']}-{task}") + command = [ + str(HERDR_BIN), + "agent", + "start", + agent_name, + "--kind", + plan["worker"], + "--pane", + pane, + "--timeout", + "120000", + "--", + ] + if plan["worker"] == "codex": + command.extend( + [ + "-m", + plan["model"], + "-c", + f'model_reasoning_effort="{plan["effort"]}"', + "-c", + 'approval_policy="on-request"', + "-c", + 'sandbox_mode="workspace-write"', + ] + ) + else: + command.extend( + [ + "--model", + plan["model"], + "--effort", + plan["effort"], + "--permission-mode", + "acceptEdits", + ] + ) + started = _run(command, env) + result = {**plan, "agent": agent_name, "pane": pane, "started": started} + if prompt: + result["prompted"] = _run( + [str(HERDR_BIN), "agent", "prompt", agent_name, prompt], env + ) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--shape", choices=("implementation", "architecture", "review"), required=True) + parser.add_argument("--effort", choices=ALLOWED_EFFORTS, required=True) + parser.add_argument("--provider", choices=("codex", "claude")) + parser.add_argument("--routes", type=Path, default=ROUTING_PATH) + parser.add_argument("--start", action="store_true") + parser.add_argument("--project", type=Path) + parser.add_argument("--task", default="objective") + parser.add_argument("--prompt") + args = parser.parse_args() + + plan = select_plan(_load_routes(args.routes), args.shape, args.effort, args.provider) + if args.start: + if args.project is None: + parser.error("--project is required with --start") + plan = launch_worker(plan, args.project.resolve(), args.task, args.prompt) + print(json.dumps(plan, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scripts/hermes_coordinator.py b/services/hermes/scripts/hermes_coordinator.py new file mode 100644 index 000000000..433d5e3ec --- /dev/null +++ b/services/hermes/scripts/hermes_coordinator.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Keep Hermes agent project coordination and model routes current.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import time +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from hermes_model_routing import ( + _atomic_write, + _read_env, + configure_routes, + discover_claude_models, + discover_codex_models, +) + + +CASSANDRA_PATH = Path("/opt/data/workspace/projects/cassandra") +CASSANDRA_REMOTE = "https://scm.bstein.dev/bstein/cassandra.git" + + +def bootstrap_cassandra(root: Path) -> None: + """Create the initial isolated board and project without resetting user state.""" + os.environ["HERMES_HOME"] = str(root) + from hermes_cli import kanban_db as kb + from hermes_cli import projects_db as pdb + + first_create = not kb.board_exists("cassandra") + kb.create_board( + "cassandra", + name="Cassandra", + description="Objectives, implementation tasks, reviews, and evidence for Cassandra.", + default_workdir=str(CASSANDRA_PATH), + ) + if first_create: + kb.set_current_board("cassandra") + + CASSANDRA_PATH.parent.mkdir(parents=True, exist_ok=True) + with pdb.connect_closing() as connection: + project = pdb.get_project(connection, "cassandra") + if project is None: + project_id = pdb.create_project( + connection, + name="Cassandra", + slug="cassandra", + folders=[str(CASSANDRA_PATH)], + primary_path=str(CASSANDRA_PATH), + description="Cassandra project objectives and coordinated delivery.", + board_slug="cassandra", + ) + if pdb.get_active_id(connection) is None: + pdb.set_active(connection, project_id) + else: + pdb.update_project( + connection, + project.id, + name="Cassandra", + description="Cassandra project objectives and coordinated delivery.", + board_slug="cassandra", + ) + pdb.add_folder(connection, project.id, str(CASSANDRA_PATH), is_primary=True) + + +def sync_cassandra_repo(env_values: dict[str, str]) -> str: + """Clone or fetch Cassandra when the optional Gitea token is available.""" + token = env_values.get("GITEA_TOKEN", "").strip() + if shutil.which("git") is None: + return "git-unavailable" + if (CASSANDRA_PATH / ".git").is_dir(): + if not token: + return "ready; fetch skipped until Gitea token is configured" + command = [ + "git", + "-C", + str(CASSANDRA_PATH), + "fetch", + "--quiet", + "--prune", + "origin", + ] + else: + CASSANDRA_PATH.mkdir(parents=True, exist_ok=True) + if any(CASSANDRA_PATH.iterdir()): + return "unmanaged-nonempty-directory" + if not token: + return "awaiting-gitea-token" + command = ["git", "clone", "--quiet", CASSANDRA_REMOTE, str(CASSANDRA_PATH)] + child_env = os.environ.copy() + child_env.update(env_values) + try: + completed = subprocess.run( + command, + env=child_env, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=180, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return "sync-failed" + return ( + "ready" if completed.returncode == 0 else f"sync-failed-{completed.returncode}" + ) + + +def refresh_once(root: Path) -> dict[str, Any]: + """Refresh provider catalogs and all managed coordinator state once.""" + env_values = _read_env(root / ".env") + if env_values.get("CLAUDE_CODE_OAUTH_TOKEN"): + os.environ["CLAUDE_CODE_OAUTH_TOKEN"] = env_values["CLAUDE_CODE_OAUTH_TOKEN"] + codex = discover_codex_models() + claude = discover_claude_models() + routes = configure_routes(root, codex, claude) + bootstrap_cassandra(root) + repo_state = sync_cassandra_repo(env_values) + status = { + "updated_at": datetime.now(timezone.utc).isoformat(), + "refresh_interval_hours": 1, + "providers": { + codex.provider: asdict(codex), + claude.provider: asdict(claude), + }, + "routes": routes, + "projects": { + "cassandra": { + "board": "cassandra", + "workspace": str(CASSANDRA_PATH), + "repository": CASSANDRA_REMOTE, + "state": repo_state, + } + }, + } + _atomic_write( + root / "workspace" / "coordinator" / "model-routing.json", + json.dumps(status, indent=2, sort_keys=True) + "\n", + ) + return status + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--once", action="store_true", help="refresh once and exit") + parser.add_argument("--loop", action="store_true", help="refresh until stopped") + parser.add_argument( + "--interval", type=int, default=3600, help="loop interval in seconds" + ) + args = parser.parse_args() + root = Path(os.environ.get("HERMES_HOME", "/opt/data")).resolve() + while True: + try: + status = refresh_once(root) + states = ", ".join( + f"{name}={details['state']}" + for name, details in status["providers"].items() + ) + print(f"Hermes coordinator refresh complete: {states}", flush=True) + except Exception as error: + print( + f"Hermes coordinator refresh failed: {type(error).__name__}: {error}", + flush=True, + ) + if args.once or not args.loop: + return 1 + if args.once or not args.loop: + return 0 + time.sleep(max(300, args.interval)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/scripts/hermes_model_routing.py b/services/hermes/scripts/hermes_model_routing.py new file mode 100644 index 000000000..b685a60f3 --- /dev/null +++ b/services/hermes/scripts/hermes_model_routing.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Keep Hermes agent provider catalogs and managed profiles current.""" + +from __future__ import annotations + +import copy +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +import yaml + + +CODEX_BASELINE = "gpt-5.6-terra" +CLAUDE_BASELINE = "claude-opus-5" +EFFORTS = ("low", "medium", "high", "xhigh") +JETSON_FALLBACK = { + "provider": "custom", + "model": "qwen2.5:14b-instruct-q4_0", + "base_url": "http://ollama.ai.svc.cluster.local:11434/v1", + "api_key": "ollama", +} +ATLAS_FALLBACK = { + "provider": "custom", + "model": "gpt-oss:20b", + "base_url": "http://hermes-model-gate.hermes.svc.cluster.local:11434/v1", + "api_key": "ollama", +} +# Backwards-compatible name used by the focused unit tests and status tooling. +LOCAL_FALLBACK = ATLAS_FALLBACK +MANAGED_ENV_KEYS = { + "CLAUDE_CODE_OAUTH_TOKEN", + "GITEA_TOKEN", + "GITEA_USERNAME", + "GIT_ASKPASS", + "GIT_TERMINAL_PROMPT", +} + + +@dataclass(frozen=True) +class Catalog: + """Non-secret provider discovery result.""" + + provider: str + models: list[str] + live: bool + connected: bool + state: str + + +def _unique_models(models: Iterable[str]) -> list[str]: + """Return normalized model IDs without changing provider order.""" + seen: set[str] = set() + result: list[str] = [] + for model in models: + value = str(model or "").strip() + if value and value.lower() not in seen: + result.append(value) + seen.add(value.lower()) + return result + + +def model_version(model: str) -> tuple[int, ...]: + """Extract a sortable model version while ignoring dated aliases.""" + value = re.sub(r"-\d{8}$", "", model.lower()) + groups = re.findall(r"\d+(?:\.\d+)*", value) + parts: list[int] = [] + for group in groups: + parts.extend(int(piece) for piece in group.split(".")) + return tuple(parts) or (0,) + + +def _codex_quality(model: str) -> int: + """Rank known Codex capability tiers for delegated implementation work.""" + value = model.lower() + if "sol" in value: + return 60 + if "terra" in value: + return 50 + if "codex" in value and "spark" not in value: + return 45 + if re.fullmatch(r"gpt-\d+(?:\.\d+)*", value): + return 40 + if "luna" in value: + return 20 + if "mini" in value or "nano" in value: + return 10 + if "spark" in value: + return 5 + return 0 + + +def choose_codex_model( + models: Iterable[str], current: str = CODEX_BASELINE, *, balanced: bool = False +) -> str: + """Choose a current full Codex model, retaining current when unknown.""" + candidates = [m for m in _unique_models(models) if m.lower().startswith("gpt-")] + full = [m for m in candidates if _codex_quality(m) >= 40] + pool = full or candidates + if not pool: + return current + newest = max(model_version(model) for model in pool) + latest = [model for model in pool if model_version(model) == newest] + if balanced: + terra = [model for model in latest if "terra" in model.lower()] + if terra: + return max(terra, key=lambda model: (_codex_quality(model), model)) + return max(latest, key=lambda model: (_codex_quality(model), model)) + + +def _choose_by_hints( + models: Iterable[str], hints: tuple[str, ...], current: str, prefix: str +) -> str: + """Pick the newest model in the first available capability class.""" + candidates = [ + model + for model in _unique_models(models) + if model.lower().startswith(prefix) + ] + if not candidates: + return current + for hint in hints: + tier = [model for model in candidates if hint in model.lower()] + if tier: + return max(tier, key=lambda model: (model_version(model), model)) + return max(candidates, key=lambda model: (model_version(model), model)) + + +def choose_codex_for_effort( + models: Iterable[str], effort: str, current: str = CODEX_BASELINE +) -> str: + """Choose the account-visible Codex tier for an allowed effort level.""" + hints = { + "low": ("luna", "mini", "spark", "terra", "sol", "codex"), + "medium": ("terra", "codex", "sol", "luna", "mini", "spark"), + "high": ("sol", "codex", "terra", "luna", "mini", "spark"), + "xhigh": ("sol", "codex", "terra", "luna", "mini", "spark"), + } + if effort not in EFFORTS: + raise ValueError(f"unsupported effort: {effort}") + return _choose_by_hints(models, hints[effort], current, "gpt-") + + +def _claude_quality(model: str) -> int: + """Rank known Claude capability tiers for architecture and review work.""" + value = model.lower() + if "opus" in value: + return 40 + if "sonnet" in value: + return 30 + if "fable" in value: + return 20 + if "haiku" in value: + return 10 + return 0 + + +def choose_claude_model(models: Iterable[str], current: str = CLAUDE_BASELINE) -> str: + """Choose the newest full Claude reasoning model visible to the account.""" + candidates = [m for m in _unique_models(models) if m.lower().startswith("claude-")] + full = [m for m in candidates if _claude_quality(m) >= 30] + pool = full or candidates + if not pool: + return current + return max( + pool, + key=lambda model: (model_version(model), _claude_quality(model), model), + ) + + +def choose_claude_for_effort( + models: Iterable[str], effort: str, current: str = CLAUDE_BASELINE +) -> str: + """Choose the account-visible Claude tier for an allowed effort level.""" + hints = { + "low": ("haiku", "fable", "sonnet", "opus"), + "medium": ("sonnet", "fable", "opus", "haiku"), + "high": ("opus", "sonnet", "fable", "haiku"), + "xhigh": ("opus", "sonnet", "fable", "haiku"), + } + if effort not in EFFORTS: + raise ValueError(f"unsupported effort: {effort}") + return _choose_by_hints(models, hints[effort], current, "claude-") + + +def _read_yaml(path: Path) -> dict[str, Any]: + """Read a mapping from YAML, returning an empty mapping when unavailable.""" + if not path.is_file(): + return {} + try: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return {} + return value if isinstance(value, dict) else {} + + +def _atomic_write(path: Path, content: str, mode: int | None = None) -> bool: + """Replace a file only when its content changes.""" + path.parent.mkdir(parents=True, exist_ok=True) + try: + if path.read_text(encoding="utf-8") == content: + if mode is not None: + path.chmod(mode) + return False + except OSError: + pass + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(content, encoding="utf-8") + if mode is not None: + temporary.chmod(mode) + os.replace(temporary, path) + return True + + +def _write_yaml(path: Path, value: dict[str, Any], mode: int | None = None) -> bool: + """Serialize a mapping and atomically update the target YAML file.""" + return _atomic_write(path, yaml.safe_dump(value, sort_keys=False), mode) + + +def _read_env(path: Path) -> dict[str, str]: + """Read the small dotenv subset used by Hermes provider credentials.""" + values: dict[str, str] = {} + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + return values + for line in lines: + value = line.strip() + if not value or value.startswith("#") or "=" not in value: + continue + key, raw = value.removeprefix("export ").split("=", 1) + raw = raw.strip() + if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in "\"'": + raw = raw[1:-1] + values[key.strip()] = raw + return values + + +def _update_profile_env(path: Path, source: dict[str, str]) -> None: + """Refresh managed credentials while preserving user-owned environment keys.""" + try: + old_lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + old_lines = [] + kept = [ + line + for line in old_lines + if not any(line.lstrip().startswith(f"{key}=") for key in MANAGED_ENV_KEYS) + ] + kept.extend( + f"{key}={source[key]}" for key in sorted(MANAGED_ENV_KEYS) if source.get(key) + ) + _atomic_write(path, "\n".join(kept).rstrip() + "\n", 0o600) + + +def _existing_model(config: dict[str, Any], provider: str, default: str) -> str: + """Return the last configured model for a provider or a safe baseline.""" + model_cfg = config.get("model", {}) + if isinstance(model_cfg, dict) and model_cfg.get("provider") == provider: + return str(model_cfg.get("model") or model_cfg.get("default") or default) + fallbacks = config.get("fallback_providers", []) + if isinstance(fallbacks, list): + for fallback in fallbacks: + if isinstance(fallback, dict) and fallback.get("provider") == provider: + return str(fallback.get("model") or default) + return default + + +def discover_codex_models() -> Catalog: + """Use the authenticated Codex endpoint; catalogs are status-only fallback.""" + token = "" + try: + from hermes_cli.auth import resolve_codex_runtime_credentials + + credentials = resolve_codex_runtime_credentials(refresh_if_expiring=True) or {} + token = str(credentials.get("api_key") or "").strip() + except Exception: + token = "" + live: list[str] = [] + if token: + try: + from hermes_cli.codex_models import _fetch_models_from_api + + live = _unique_models(_fetch_models_from_api(token)) + except Exception: + live = [] + if live: + return Catalog("openai-codex", live, True, True, "connected") + try: + from hermes_cli.models import provider_model_ids + + known = _unique_models(provider_model_ids("openai-codex", force_refresh=True)) + except Exception: + known = [] + state = "degraded" if token else "not-configured" + return Catalog("openai-codex", known, False, bool(token), state) + + +def discover_claude_models() -> Catalog: + """Use Anthropic's authenticated model endpoint when configured.""" + token = str(os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") or "").strip() + live: list[str] = [] + if token: + try: + from hermes_cli.models import _fetch_anthropic_models + + live = _unique_models(_fetch_anthropic_models(timeout=10.0) or []) + except Exception: + live = [] + if live: + return Catalog("anthropic", live, True, True, "connected") + try: + from hermes_cli.models import provider_model_ids + + known = _unique_models(provider_model_ids("anthropic", force_refresh=True)) + except Exception: + known = [] + state = "degraded" if token else "not-configured" + return Catalog("anthropic", known, False, bool(token), state) + + +def _profile_config( + base: dict[str, Any], + primary_provider: str, + primary_model: str, + fallback: dict[str, str], + effort: str, +) -> dict[str, Any]: + """Derive a worker configuration with an explicit cross-provider fallback.""" + config = copy.deepcopy(base) + config["model"] = { + "provider": primary_provider, + "default": primary_model, + "model": primary_model, + } + config["fallback_providers"] = [ + fallback, + copy.deepcopy(JETSON_FALLBACK), + copy.deepcopy(ATLAS_FALLBACK), + ] + agent = config.setdefault("agent", {}) + if isinstance(agent, dict): + agent["reasoning_effort"] = effort + config["toolsets"] = [] + return config + + +def _write_profile( + root: Path, + name: str, + description: str, + soul: str, + config: dict[str, Any], + env_values: dict[str, str], +) -> None: + """Create or refresh a managed Hermes worker profile.""" + profile = root / "profiles" / name + for directory in ("logs", "sessions", "skills", "workspace", "home"): + (profile / directory).mkdir(parents=True, exist_ok=True) + _write_yaml(profile / "config.yaml", config) + _write_yaml( + profile / "profile.yaml", + {"description": description, "description_auto": False}, + ) + _atomic_write(profile / "SOUL.md", soul.rstrip() + "\n") + _update_profile_env(profile / ".env", env_values) + + +def configure_routes(root: Path, codex: Catalog, claude: Catalog) -> dict[str, Any]: + """Update the coordinator and managed worker profiles.""" + config_path = root / "config.yaml" + base = _read_yaml(config_path) + old_coordinator = _existing_model(base, "openai-codex", CODEX_BASELINE) + old_claude_root = _existing_model(base, "anthropic", CLAUDE_BASELINE) + codex_models: dict[str, str] = {} + claude_models: dict[str, str] = {} + for effort in EFFORTS: + old_codex = _existing_model( + _read_yaml(root / "profiles" / f"codex-{effort}" / "config.yaml"), + "openai-codex", + old_coordinator, + ) + old_claude = _existing_model( + _read_yaml(root / "profiles" / f"claude-{effort}" / "config.yaml"), + "anthropic", + old_claude_root, + ) + codex_models[effort] = ( + choose_codex_for_effort(codex.models, effort, old_codex) + if codex.live + else old_codex + ) + claude_models[effort] = ( + choose_claude_for_effort(claude.models, effort, old_claude) + if claude.live + else old_claude + ) + + codex_coordinator = codex_models["medium"] + claude_coordinator = claude_models["medium"] + + base["model"] = { + "provider": "openai-codex", + "default": codex_coordinator, + "model": codex_coordinator, + } + base["fallback_providers"] = [ + {"provider": "anthropic", "model": claude_coordinator}, + copy.deepcopy(JETSON_FALLBACK), + copy.deepcopy(ATLAS_FALLBACK), + ] + base["model_catalog"] = {"enabled": True, "ttl_hours": 1} + _write_yaml(config_path, base) + + env_values = _read_env(root / ".env") + routes: dict[str, list[str]] = {} + for effort in EFFORTS: + codex_model = codex_models[effort] + claude_model = claude_models[effort] + codex_name = f"codex-{effort}" + claude_name = f"claude-{effort}" + _write_profile( + root, + codex_name, + f"Codex implementation worker at {effort} effort, with Claude and local fallback.", + "You are an implementation worker. Make focused, tested changes for the assigned task, preserve unrelated work, and report evidence and blockers to the coordinator.", + _profile_config( + base, + "openai-codex", + codex_model, + {"provider": "anthropic", "model": claude_model}, + effort, + ), + env_values, + ) + _write_profile( + root, + claude_name, + f"Claude architecture and review worker at {effort} effort, with Codex and local fallback.", + "You are an architecture and review worker. Analyze the assigned task deeply, change files only when asked, and return concise conclusions, evidence, and risks to the coordinator.", + _profile_config( + base, + "anthropic", + claude_model, + {"provider": "openai-codex", "model": codex_model}, + effort, + ), + env_values, + ) + local = [ + "custom/qwen2.5:14b-instruct-q4_0", + "custom/gpt-oss:20b", + ] + routes[codex_name] = [ + f"openai-codex/{codex_model}", + f"anthropic/{claude_model}", + *local, + ] + routes[claude_name] = [ + f"anthropic/{claude_model}", + f"openai-codex/{codex_model}", + *local, + ] + + _write_profile( + root, + "synthesis-xhigh", + "Cross-provider synthesis and critical review, capped at xhigh effort.", + "Synthesize the worker evidence into one answer. Resolve disagreements explicitly, verify high-risk claims, and never claim completion without cited validation.", + _profile_config( + base, + "anthropic", + claude_models["xhigh"], + {"provider": "openai-codex", "model": codex_models["xhigh"]}, + "xhigh", + ), + env_values, + ) + routes["synthesis-xhigh"] = [ + f"anthropic/{claude_models['xhigh']}", + f"openai-codex/{codex_models['xhigh']}", + "custom/qwen2.5:14b-instruct-q4_0", + "custom/gpt-oss:20b", + ] + _write_yaml( + root / "profile.yaml", + { + "description": "Coordinator for project objectives, delegating implementation to Codex and architecture or review to Claude through isolated Kanban boards.", + "description_auto": False, + }, + ) + routes["coordinator"] = [ + f"openai-codex/{codex_coordinator}", + f"anthropic/{claude_coordinator}", + "custom/qwen2.5:14b-instruct-q4_0", + "custom/gpt-oss:20b", + ] + return routes diff --git a/services/hermes/scripts/patch_hermes_auth.py b/services/hermes/scripts/patch_hermes_auth.py new file mode 100644 index 000000000..4fb7b4d2b --- /dev/null +++ b/services/hermes/scripts/patch_hermes_auth.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Patch Hermes to use one explicitly mounted, lock-protected auth store.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + + +BEFORE = '''def _auth_file_path() -> Path: + path = get_hermes_home() / "auth.json" +''' +AFTER = '''def _auth_file_path() -> Path: + configured = os.environ.get("HERMES_AUTH_FILE", "").strip() + path = Path(configured) if configured else get_hermes_home() / "auth.json" +''' + + +def patch(source: Path, destination: Path) -> None: + """Apply the narrow environment override and fail on upstream drift.""" + content = source.read_text(encoding="utf-8") + if BEFORE not in content: + raise RuntimeError("Hermes auth patch context changed") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content.replace(BEFORE, AFTER, 1), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("destination", type=Path) + args = parser.parse_args() + patch(args.source, args.destination) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/hermes/service.yaml b/services/hermes/service.yaml index 473823f6c..5b31ca7b3 100644 --- a/services/hermes/service.yaml +++ b/services/hermes/service.yaml @@ -22,6 +22,77 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: hermes-triage + namespace: hermes + labels: + app: hermes +spec: + type: ClusterIP + selector: + app: hermes + ports: + - name: api + port: 8642 + targetPort: api + - name: dashboard + port: 9119 + targetPort: dashboard +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-agent + namespace: hermes + labels: + app: hermes-agent +spec: + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: dashboard + port: 9119 + targetPort: dashboard +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-chat-tenant + namespace: hermes + labels: + app: hermes-chat-tenant +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app: hermes-chat-tenant + ports: + - name: webui + port: 8787 + targetPort: webui + - name: api + port: 8642 + targetPort: api +--- +apiVersion: v1 +kind: Service +metadata: + name: hermes-chat-router + namespace: hermes + labels: + app: hermes-chat-router +spec: + type: ClusterIP + selector: + app: hermes-chat-router + ports: + - name: http + port: 8080 + targetPort: http +--- +apiVersion: v1 +kind: Service metadata: name: hermes-ollama namespace: hermes diff --git a/services/hermes/skills/master-hermes-on-atlas/references/architecture.md b/services/hermes/skills/master-hermes-on-atlas/references/architecture.md index c35188d6e..37d41caa9 100644 --- a/services/hermes/skills/master-hermes-on-atlas/references/architecture.md +++ b/services/hermes/skills/master-hermes-on-atlas/references/architecture.md @@ -7,8 +7,9 @@ asserting health, placement, ownership, or current model availability. | Surface | Purpose | Identity boundary | State and permissions | | --- | --- | --- | --- | -| `agent.bstein.dev` | Brad's operator and triage lab | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace, its own PVC and service account; read-only cluster triage plus approved internal evidence endpoints | -| `chat.bstein.dev` | Consumer chat and research | Keycloak login | `hermes-chat` namespace, separate PVC/config/auth/sessions/skills; read-only cluster observer RBAC and no access to Secrets or Kubernetes mutation APIs | +| `triage.hermes.bstein.dev` | Brad's automated testing triage | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace, its own PVC and service account; read-only cluster triage plus approved internal evidence endpoints | +| `agent.hermes.bstein.dev` | Brad's project coordinator | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace, separate PVC and no Kubernetes RBAC; Herdr supervises Codex and Claude Code workers | +| `chat.hermes.bstein.dev` | Private consumer chat and research through Hermes WebUI or a linked Telegram DM | Keycloak login plus one-time Telegram account link | One Hermes process and PVC per assigned Keycloak subject; no Kubernetes RBAC, terminal, or private-service access | The instances do not share conversation state, credentials, profiles, skills created on their PVCs, or Kubernetes identities. They share only the inference diff --git a/services/hermes/vault-serviceaccount.yaml b/services/hermes/vault-serviceaccount.yaml index 911064861..037951cab 100644 --- a/services/hermes/vault-serviceaccount.yaml +++ b/services/hermes/vault-serviceaccount.yaml @@ -4,3 +4,15 @@ kind: ServiceAccount metadata: name: hermes-vault namespace: hermes +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-agent + namespace: hermes +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hermes-chat + namespace: hermes diff --git a/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml b/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml index 7c5daa78a..643a3d7a8 100644 --- a/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml +++ b/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml @@ -1,9 +1,9 @@ # services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml -# Purpose: create chat OIDC and Brad-only operator proxy clients. +# Purpose: create isolated chat, Brad-only agent, and Brad-only triage clients. apiVersion: batch/v1 kind: Job metadata: - name: hermes-access-oidc-client-ensure-3 + name: hermes-access-oidc-client-ensure-5 namespace: sso spec: backoffLimit: 3 diff --git a/services/keycloak/bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml b/services/keycloak/bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml deleted file mode 100644 index 8a5232890..000000000 --- a/services/keycloak/bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# services/keycloak/bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml -# Purpose: ensure the Hermes dashboard public OIDC client exists in Keycloak. -# Bump the suffix if the immutable Job needs to be rerun after it completes. -apiVersion: batch/v1 -kind: Job -metadata: - name: hermes-dashboard-oidc-client-ensure-1 - namespace: sso -spec: - backoffLimit: 3 - template: - metadata: - annotations: - vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/agent-pre-populate-only: "true" - vault.hashicorp.com/role: "sso-secrets" - vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin" - vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: | - {{ with secret "kv/data/atlas/shared/keycloak-admin" }} - export KEYCLOAK_ADMIN="{{ .Data.data.username }}" - export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}" - export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}" - {{ end }} - spec: - serviceAccountName: mas-secrets-ensure - restartPolicy: Never - volumes: - - name: hermes-dashboard-oidc-client-script - configMap: - name: hermes-dashboard-oidc-client-script - defaultMode: 0555 - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: ["arm64"] - - key: node-role.kubernetes.io/worker - operator: Exists - containers: - - name: apply - image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 - command: ["/scripts/hermes_dashboard_oidc_client_ensure.sh"] - volumeMounts: - - name: hermes-dashboard-oidc-client-script - mountPath: /scripts - readOnly: true diff --git a/services/keycloak/kustomization.yaml b/services/keycloak/kustomization.yaml index ff30ca997..40c573882 100644 --- a/services/keycloak/kustomization.yaml +++ b/services/keycloak/kustomization.yaml @@ -26,7 +26,6 @@ resources: - bootstrap-jobs/metis-oidc-secret-ensure-job.yaml - bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml - bootstrap-jobs/quality-oidc-secret-ensure-job.yaml - - bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml - bootstrap-jobs/hermes-access-oidc-client-job.yaml - bootstrap-jobs/veles-realm-ensure-job.yaml - bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml @@ -53,9 +52,6 @@ configMapGenerator: - name: actual-oidc-secret-ensure-script files: - actual_oidc_secret_ensure.sh=scripts/actual_oidc_secret_ensure.sh - - name: hermes-dashboard-oidc-client-script - files: - - hermes_dashboard_oidc_client_ensure.sh=scripts/hermes_dashboard_oidc_client_ensure.sh - name: hermes-access-oidc-script files: - hermes_access_oidc_ensure.sh=scripts/hermes_access_oidc_ensure.sh diff --git a/services/keycloak/scripts/hermes_access_oidc_ensure.sh b/services/keycloak/scripts/hermes_access_oidc_ensure.sh index 5f390cf16..90ed5723a 100755 --- a/services/keycloak/scripts/hermes_access_oidc_ensure.sh +++ b/services/keycloak/scripts/hermes_access_oidc_ensure.sh @@ -4,10 +4,8 @@ set -eu . /vault/secrets/keycloak-admin-env.sh KC_URL="http://keycloak.sso.svc.cluster.local" -CHAT_CLIENT="hermes-chat-dashboard" -CHAT_URL="https://chat.bstein.dev" -OPERATOR_CLIENT="hermes-operator-proxy" -OPERATOR_URL="https://agent.bstein.dev" +VAULT_ADDR="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" +VAULT_ROLE="${VAULT_ROLE:-sso-secrets}" ACCESS_TOKEN="" for attempt in 1 2 3 4 5 6 7 8 9 10; do @@ -29,7 +27,6 @@ for attempt in 1 2 3 4 5; do if [ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ]; then break fi - echo "Keycloak token request failed (attempt ${attempt})" >&2 sleep $((attempt * 2)) done if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then @@ -37,193 +34,151 @@ if [ -z "${ACCESS_TOKEN}" ] || [ "${ACCESS_TOKEN}" = "null" ]; then exit 1 fi -chat_payload="$(jq -nc \ - --arg client_id "${CHAT_CLIENT}" \ - --arg redirect_uri "${CHAT_URL}/auth/callback" \ - --arg web_origin "${CHAT_URL}" \ - '{ - clientId:$client_id, - enabled:true, - protocol:"openid-connect", - publicClient:true, - standardFlowEnabled:true, - implicitFlowEnabled:false, - directAccessGrantsEnabled:false, - serviceAccountsEnabled:false, - redirectUris:[$redirect_uri], - webOrigins:[$web_origin], - rootUrl:$web_origin, - baseUrl:"/", - attributes:{ - "pkce.code.challenge.method":"S256", - "post.logout.redirect.uris":$web_origin, - "access.token.lifespan":"1200" - } - }')" - -chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)" -chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)" -if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then - status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${chat_payload}" \ - "${KC_URL}/admin/realms/atlas/clients")" - if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then - echo "Keycloak chat client create failed (status ${status})" >&2 - exit 1 - fi - chat_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients?clientId=${CHAT_CLIENT}" || true)" - chat_id="$(printf '%s' "${chat_query}" | jq -r '.[0].id' 2>/dev/null || true)" -fi -if [ -z "${chat_id}" ] || [ "${chat_id}" = "null" ]; then - echo "Keycloak chat client not found after create" >&2 - exit 1 -fi - -status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${chat_payload}" \ - "${KC_URL}/admin/realms/atlas/clients/${chat_id}")" -if [ "${status}" != "204" ]; then - echo "Keycloak chat client update failed (status ${status})" >&2 - exit 1 -fi - -scope_id="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/client-scopes?search=groups" \ - | jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)" -if [ -z "${scope_id}" ] || [ "${scope_id}" = "null" ]; then - echo "Keycloak groups client scope not found" >&2 - exit 1 -fi -default_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients/${chat_id}/default-client-scopes" || true)" -optional_scopes="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes" || true)" -if ! printf '%s' "${default_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1 \ - && ! printf '%s' "${optional_scopes}" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1; then - status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")" - if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then - status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients/${chat_id}/optional-client-scopes/${scope_id}")" - if [ "${status}" != "200" ] && [ "${status}" != "201" ] && [ "${status}" != "204" ]; then - echo "Failed to attach groups scope to chat client (status ${status})" >&2 - exit 1 - fi - fi -fi - -operator_payload="$(jq -nc \ - --arg client_id "${OPERATOR_CLIENT}" \ - --arg redirect_uri "${OPERATOR_URL}/oauth2/callback" \ - --arg web_origin "${OPERATOR_URL}" \ - '{ - clientId:$client_id, - enabled:true, - protocol:"openid-connect", - publicClient:false, - standardFlowEnabled:true, - implicitFlowEnabled:false, - directAccessGrantsEnabled:false, - serviceAccountsEnabled:false, - redirectUris:[$redirect_uri], - webOrigins:[$web_origin], - rootUrl:$web_origin, - baseUrl:"/", - attributes:{"post.logout.redirect.uris":$web_origin} - }')" - -operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)" -operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)" -if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then - status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${operator_payload}" \ - "${KC_URL}/admin/realms/atlas/clients")" - if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then - echo "Keycloak operator proxy client create failed (status ${status})" >&2 - exit 1 - fi - operator_query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients?clientId=${OPERATOR_CLIENT}" || true)" - operator_id="$(printf '%s' "${operator_query}" | jq -r '.[0].id' 2>/dev/null || true)" -fi -if [ -z "${operator_id}" ] || [ "${operator_id}" = "null" ]; then - echo "Keycloak operator proxy client not found after create" >&2 - exit 1 -fi - -status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${operator_payload}" \ - "${KC_URL}/admin/realms/atlas/clients/${operator_id}")" -if [ "${status}" != "204" ]; then - echo "Keycloak operator proxy client update failed (status ${status})" >&2 - exit 1 -fi - -client_secret="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "${KC_URL}/admin/realms/atlas/clients/${operator_id}/client-secret" \ - | jq -r '.value' 2>/dev/null || true)" -if [ -z "${client_secret}" ] || [ "${client_secret}" = "null" ]; then - echo "Keycloak operator proxy client secret not found" >&2 - exit 1 -fi - -vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" -vault_role="${VAULT_ROLE:-sso-secrets}" jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" -login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt,role:$role}')" +login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${VAULT_ROLE}" '{jwt:$jwt,role:$role}')" vault_token="$(curl -sS --request POST --data "${login_payload}" \ - "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" + "${VAULT_ADDR}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then echo "Vault login failed" >&2 exit 1 fi -read_status="$(curl -sS -o /tmp/hermes-operator-oidc-read.json -w '%{http_code}' \ - -H "X-Vault-Token: ${vault_token}" \ - "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc" || true)" -cookie_secret="" -if [ "${read_status}" = "200" ]; then - cookie_secret="$(jq -r '.data.data.cookie_secret // empty' /tmp/hermes-operator-oidc-read.json)" -elif [ "${read_status}" != "404" ]; then - echo "Vault operator OIDC read failed (status ${read_status})" >&2 - exit 1 -fi -if [ -n "${cookie_secret}" ]; then - cookie_length="$(printf '%s' "${cookie_secret}" | wc -c | tr -d ' ')" - if [ "${cookie_length}" != "16" ] && [ "${cookie_length}" != "24" ] && [ "${cookie_length}" != "32" ]; then - cookie_secret="" +ensure_proxy_client() { + client_id="$1" + public_url="$2" + vault_path="$3" + payload="$(jq -nc \ + --arg client_id "${client_id}" \ + --arg redirect_uri "${public_url}/oauth2/callback" \ + --arg web_origin "${public_url}" \ + '{ + clientId:$client_id, + name:$client_id, + enabled:true, + protocol:"openid-connect", + publicClient:false, + standardFlowEnabled:true, + implicitFlowEnabled:false, + directAccessGrantsEnabled:false, + serviceAccountsEnabled:false, + redirectUris:[$redirect_uri], + webOrigins:[$web_origin], + rootUrl:$web_origin, + baseUrl:"/", + attributes:{ + "pkce.code.challenge.method":"S256", + "post.logout.redirect.uris":$web_origin, + "access.token.lifespan":"1200" + } + }')" + + query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${client_id}" || true)" + internal_id="$(printf '%s' "${query}" | jq -r '.[0].id' 2>/dev/null || true)" + if [ -z "${internal_id}" ] || [ "${internal_id}" = "null" ]; then + status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${payload}" \ + "${KC_URL}/admin/realms/atlas/clients")" + if [ "${status}" != "201" ] && [ "${status}" != "204" ] && [ "${status}" != "409" ]; then + echo "Keycloak client ${client_id} create failed (status ${status})" >&2 + exit 1 + fi + query="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients?clientId=${client_id}" || true)" + internal_id="$(printf '%s' "${query}" | jq -r '.[0].id' 2>/dev/null || true)" + fi + if [ -z "${internal_id}" ] || [ "${internal_id}" = "null" ]; then + echo "Keycloak client ${client_id} was not found after create" >&2 + exit 1 fi -fi -if [ -z "${cookie_secret}" ]; then - cookie_secret="$(openssl rand -hex 16 | tr -d '\n')" -fi -vault_payload="$(jq -nc \ - --arg client_id "${OPERATOR_CLIENT}" \ - --arg client_secret "${client_secret}" \ - --arg cookie_secret "${cookie_secret}" \ - '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')" -write_status="$(curl -sS -o /tmp/hermes-operator-oidc-write.json -w '%{http_code}' -X POST \ - -H "X-Vault-Token: ${vault_token}" \ - -H 'Content-Type: application/json' \ - -d "${vault_payload}" \ - "${vault_addr}/v1/kv/data/atlas/hermes/operator-oidc")" -if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then - echo "Vault operator OIDC write failed (status ${write_status})" >&2 - exit 1 -fi + status="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${payload}" \ + "${KC_URL}/admin/realms/atlas/clients/${internal_id}")" + if [ "${status}" != "204" ]; then + echo "Keycloak client ${client_id} update failed (status ${status})" >&2 + exit 1 + fi -echo "Hermes chat and operator OIDC clients are ready" + client_secret="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/atlas/clients/${internal_id}/client-secret" \ + | jq -r '.value' 2>/dev/null || true)" + if [ -z "${client_secret}" ] || [ "${client_secret}" = "null" ]; then + echo "Keycloak client ${client_id} secret was not returned" >&2 + exit 1 + fi + + state_file="/tmp/hermes-$(printf '%s' "${client_id}" | tr -c 'a-zA-Z0-9' '-').json" + read_status="$(curl -sS -o "${state_file}" -w '%{http_code}' \ + -H "X-Vault-Token: ${vault_token}" \ + "${VAULT_ADDR}/v1/kv/data/atlas/${vault_path}" || true)" + cookie_secret="" + if [ "${read_status}" = "200" ]; then + cookie_secret="$(jq -r '.data.data.cookie_secret // empty' "${state_file}")" + elif [ "${read_status}" != "404" ]; then + echo "Vault ${vault_path} read failed (status ${read_status})" >&2 + exit 1 + fi + cookie_length="$(printf '%s' "${cookie_secret}" | wc -c | tr -d ' ')" + case "${cookie_length}" in 16|24|32) ;; *) cookie_secret="$(openssl rand -hex 16 | tr -d '\n')" ;; esac + + vault_payload="$(jq -nc \ + --arg client_id "${client_id}" \ + --arg client_secret "${client_secret}" \ + --arg cookie_secret "${cookie_secret}" \ + '{data:{client_id:$client_id,client_secret:$client_secret,cookie_secret:$cookie_secret}}')" + write_status="$(curl -sS -o "${state_file}.write" -w '%{http_code}' -X POST \ + -H "X-Vault-Token: ${vault_token}" \ + -H 'Content-Type: application/json' \ + -d "${vault_payload}" \ + "${VAULT_ADDR}/v1/kv/data/atlas/${vault_path}")" + if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then + echo "Vault ${vault_path} write failed (status ${write_status})" >&2 + exit 1 + fi + echo "Hermes OIDC client ${client_id} is ready" +} + +ensure_telegram_config() { + vault_path="hermes/chat-telegram" + state_file="/tmp/hermes-chat-telegram.json" + read_status="$(curl -sS -o "${state_file}" -w '%{http_code}' \ + -H "X-Vault-Token: ${vault_token}" \ + "${VAULT_ADDR}/v1/kv/data/atlas/${vault_path}" || true)" + bot_token="" + relay_key="" + if [ "${read_status}" = "200" ]; then + bot_token="$(jq -r '.data.data.bot_token // empty' "${state_file}")" + relay_key="$(jq -r '.data.data.relay_key // empty' "${state_file}")" + elif [ "${read_status}" != "404" ]; then + echo "Vault ${vault_path} read failed (status ${read_status})" >&2 + exit 1 + fi + relay_length="$(printf '%s' "${relay_key}" | wc -c | tr -d ' ')" + if [ "${relay_length}" -lt 64 ]; then + relay_key="$(openssl rand -hex 32 | tr -d '\n')" + fi + vault_payload="$(jq -nc \ + --arg bot_token "${bot_token}" \ + --arg relay_key "${relay_key}" \ + '{data:{bot_token:$bot_token,relay_key:$relay_key}}')" + write_status="$(curl -sS -o "${state_file}.write" -w '%{http_code}' -X POST \ + -H "X-Vault-Token: ${vault_token}" \ + -H 'Content-Type: application/json' \ + -d "${vault_payload}" \ + "${VAULT_ADDR}/v1/kv/data/atlas/${vault_path}")" + if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then + echo "Vault ${vault_path} write failed (status ${write_status})" >&2 + exit 1 + fi + echo "Hermes Telegram transport secret is ready" +} + +ensure_proxy_client "hermes-chat-proxy" "https://chat.hermes.bstein.dev" "hermes/chat-oidc" +ensure_proxy_client "hermes-agent-proxy" "https://agent.hermes.bstein.dev" "hermes/agent-oidc" +ensure_proxy_client "hermes-triage-proxy" "https://triage.hermes.bstein.dev" "hermes/triage-oidc" +ensure_telegram_config diff --git a/services/keycloak/scripts/hermes_dashboard_oidc_client_ensure.sh b/services/keycloak/scripts/hermes_dashboard_oidc_client_ensure.sh deleted file mode 100755 index 973bc2c64..000000000 --- a/services/keycloak/scripts/hermes_dashboard_oidc_client_ensure.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env sh -set -eu - -. /vault/secrets/keycloak-admin-env.sh - -CLIENT_NAME="hermes-dashboard" -PUBLIC_URL="https://agent.bstein.dev" -KC_URL="http://keycloak.sso.svc.cluster.local" - -ACCESS_TOKEN="" -for attempt in 1 2 3 4 5 6 7 8 9 10; do - if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then - break - fi - echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2 - sleep $((attempt * 2)) -done - -for attempt in 1 2 3 4 5; do - TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \ - -H 'Content-Type: application/x-www-form-urlencoded' \ - -d "grant_type=password" \ - -d "client_id=admin-cli" \ - -d "username=${KEYCLOAK_ADMIN}" \ - -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)" - ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)" - if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then - break - fi - echo "Keycloak token request failed (attempt ${attempt})" >&2 - sleep $((attempt * 2)) -done -if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then - echo "Failed to fetch Keycloak admin token" >&2 - exit 1 -fi - -client_payload="$(jq -nc \ - --arg client_id "${CLIENT_NAME}" \ - --arg redirect_uri "${PUBLIC_URL}/auth/callback" \ - --arg web_origin "${PUBLIC_URL}" \ - '{ - clientId:$client_id, - enabled:true, - protocol:"openid-connect", - publicClient:true, - standardFlowEnabled:true, - implicitFlowEnabled:false, - directAccessGrantsEnabled:false, - serviceAccountsEnabled:false, - redirectUris:[$redirect_uri], - webOrigins:[$web_origin], - rootUrl:$web_origin, - baseUrl:"/", - attributes:{ - "pkce.code.challenge.method":"S256", - "post.logout.redirect.uris":$web_origin - } - }')" - -CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "$KC_URL/admin/realms/atlas/clients?clientId=${CLIENT_NAME}" || true)" -CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)" - -if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then - status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${client_payload}" \ - "$KC_URL/admin/realms/atlas/clients")" - if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then - echo "Keycloak client create failed (status ${status})" >&2 - exit 1 - fi - CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "$KC_URL/admin/realms/atlas/clients?clientId=${CLIENT_NAME}" || true)" - CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)" -fi - -if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then - echo "Keycloak client ${CLIENT_NAME} not found" >&2 - exit 1 -fi - -status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'Content-Type: application/json' \ - -d "${client_payload}" \ - "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}")" -if [ "$status" != "204" ]; then - echo "Keycloak client update failed (status ${status})" >&2 - exit 1 -fi - -SCOPE_ID="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "$KC_URL/admin/realms/atlas/client-scopes?search=groups" | jq -r '.[] | select(.name=="groups") | .id' 2>/dev/null | head -n1 || true)" -if [ -z "$SCOPE_ID" ] || [ "$SCOPE_ID" = "null" ]; then - echo "Keycloak client scope groups not found" >&2 - exit 1 -fi - -DEFAULT_SCOPES="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/default-client-scopes" || true)" -OPTIONAL_SCOPES="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/optional-client-scopes" || true)" - -if ! echo "$DEFAULT_SCOPES" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1 \ - && ! echo "$OPTIONAL_SCOPES" | jq -e '.[] | select(.name=="groups")' >/dev/null 2>&1; then - status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/optional-client-scopes/${SCOPE_ID}")" - if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then - status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/optional-client-scopes/${SCOPE_ID}")" - if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then - echo "Failed to attach groups client scope to ${CLIENT_NAME} (status ${status})" >&2 - exit 1 - fi - fi -fi - -echo "Hermes dashboard OIDC client ready" diff --git a/services/maintenance/apps/ariadne-deployment.yaml b/services/maintenance/apps/ariadne-deployment.yaml index d5ca509fe..8f56325de 100644 --- a/services/maintenance/apps/ariadne-deployment.yaml +++ b/services/maintenance/apps/ariadne-deployment.yaml @@ -555,7 +555,7 @@ spec: # open the run that wrote it rather than taking "Proposed by # Hermes" on trust. - name: ARIADNE_HERMES_UI_URL - value: https://agent.bstein.dev + value: https://triage.hermes.bstein.dev - name: ARIADNE_HERMES_SONAR_ENABLED value: "true" - name: ARIADNE_HERMES_SONAR_URL diff --git a/services/quality/zap-baseline-configmap.yaml b/services/quality/zap-baseline-configmap.yaml index a22747448..06bb82bb7 100644 --- a/services/quality/zap-baseline-configmap.yaml +++ b/services/quality/zap-baseline-configmap.yaml @@ -26,7 +26,9 @@ data: https://budget.bstein.dev https://money.bstein.dev https://health.bstein.dev - https://agent.bstein.dev + https://agent.hermes.bstein.dev + https://chat.hermes.bstein.dev + https://triage.hermes.bstein.dev https://cassandra.bstein.dev https://veles.bstein.dev https://matrix.live.bstein.dev diff --git a/services/vault/hermes-auth-role-bootstrap-job.yaml b/services/vault/hermes-auth-role-bootstrap-job.yaml index 2d794291a..dc930330a 100644 --- a/services/vault/hermes-auth-role-bootstrap-job.yaml +++ b/services/vault/hermes-auth-role-bootstrap-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-hermes-1 + name: vault-k8s-auth-hermes-4 namespace: vault spec: backoffLimit: 2 diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 191acaa6c..084de6f46 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -254,7 +254,11 @@ write_policy_and_role "health" "health" "health-vault-sync" \ write_policy_and_role "game-stream" "game-stream" "game-stream-vault" \ "game-stream/*" "" write_policy_and_role "hermes" "hermes" "hermes-vault,hermes-triage" \ - "hermes/operator-oidc hermes/agent-tokens" "" + "hermes/triage-oidc hermes/agent-tokens" "" +write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \ + "hermes/agent-oidc hermes/agent-tokens" "" +write_policy_and_role "hermes-chat" "hermes" "hermes-chat" \ + "hermes/chat-oidc hermes/chat-telegram hermes/agent-tokens" "" write_policy_and_role "veles" "veles" "veles-backend,veles-generator,veles-postgres,veles-vault-sync" \ "veles/* shared/harbor-pull shared/postmark-relay" "" write_policy_and_role "veles-sim" "veles" "veles-sim" \ @@ -298,7 +302,7 @@ write_policy_and_role "vault" "vault" "vault" \ write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \ "shared/keycloak-admin shared/postmark-relay maintenance/metis-ssh-keys" \ - "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc hermes/operator-oidc" \ + "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc hermes/chat-oidc hermes/chat-telegram hermes/agent-oidc hermes/triage-oidc" \ ' path "kv/data/atlas/nodes/*" { capabilities = ["create", "update", "read"] diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py new file mode 100644 index 000000000..bd7d6e4b9 --- /dev/null +++ b/testing/tests/test_hermes_coordinator.py @@ -0,0 +1,173 @@ +"""Unit tests for Hermes coordinator model routing and profile generation.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import yaml + + +SCRIPT = ( + Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" +) +sys.path.insert(0, str(SCRIPT.parent)) +routing = importlib.import_module("hermes_model_routing") +SPEC = importlib.util.spec_from_file_location("hermes_coordinator", SCRIPT) +assert SPEC and SPEC.loader +coordinator = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = coordinator +SPEC.loader.exec_module(coordinator) + + +def _base_config() -> dict: + """Return the minimal coordinator configuration used by routing tests.""" + return { + "model": { + "provider": "openai-codex", + "default": "gpt-5.6-terra", + "model": "gpt-5.6-terra", + }, + "fallback_providers": [ + {"provider": "anthropic", "model": "claude-opus-5"}, + dict(routing.LOCAL_FALLBACK), + ], + "toolsets": ["kanban"], + "terminal": {"cwd": "/opt/data/workspace"}, + } + + +def test_model_version_and_quality_selection_handle_new_and_small_models(): + assert routing.model_version("claude-3-5-sonnet-20241022") == (3, 5) + assert routing.model_version("gpt-5.7-terra") == (5, 7) + + codex = ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.7-luna", "gpt-5.3-codex-spark"] + assert routing.choose_codex_model(codex) == "gpt-5.6-sol" + assert routing.choose_codex_model(codex, balanced=True) == "gpt-5.6-terra" + assert routing.choose_codex_model(codex + ["gpt-5.7-terra"]) == "gpt-5.7-terra" + + claude = ["claude-opus-4.8", "claude-haiku-5", "claude-sonnet-5"] + assert routing.choose_claude_model(claude) == "claude-sonnet-5" + + assert routing.choose_codex_for_effort(codex, "low") == "gpt-5.7-luna" + assert routing.choose_codex_for_effort(codex, "medium") == "gpt-5.6-terra" + assert routing.choose_codex_for_effort(codex, "xhigh") == "gpt-5.6-sol" + assert routing.choose_claude_for_effort(claude, "low") == "claude-haiku-5" + assert routing.choose_claude_for_effort(claude, "medium") == "claude-sonnet-5" + assert routing.choose_claude_for_effort(claude, "xhigh") == "claude-opus-4.8" + + +def test_empty_catalog_retains_current_models(): + assert routing.choose_codex_model([], "gpt-5.6-terra") == "gpt-5.6-terra" + assert routing.choose_claude_model([], "claude-opus-5") == "claude-opus-5" + + +def test_configure_routes_builds_cross_provider_fallback_profiles(tmp_path: Path): + (tmp_path / "config.yaml").write_text( + yaml.safe_dump(_base_config()), encoding="utf-8" + ) + (tmp_path / ".env").write_text( + "API_SERVER_KEY=keep-root-only\n" + "CLAUDE_CODE_OAUTH_TOKEN=claude-secret\n" + "GITEA_TOKEN=gitea-secret\n" + "GIT_ASKPASS=/opt/coordinator/gitea_askpass.sh\n", + encoding="utf-8", + ) + codex = routing.Catalog( + "openai-codex", ["gpt-5.6-sol", "gpt-5.6-terra"], True, True, "connected" + ) + claude = routing.Catalog( + "anthropic", ["claude-sonnet-5", "claude-opus-5"], True, True, "connected" + ) + + routes = routing.configure_routes(tmp_path, codex, claude) + + root = yaml.safe_load((tmp_path / "config.yaml").read_text(encoding="utf-8")) + codex_profile = yaml.safe_load( + (tmp_path / "profiles/codex-high/config.yaml").read_text(encoding="utf-8") + ) + claude_profile = yaml.safe_load( + (tmp_path / "profiles/claude-high/config.yaml").read_text(encoding="utf-8") + ) + assert root["model"]["model"] == "gpt-5.6-terra" + assert codex_profile["model"]["model"] == "gpt-5.6-sol" + assert codex_profile["fallback_providers"][0] == { + "provider": "anthropic", + "model": "claude-opus-5", + } + assert claude_profile["model"]["model"] == "claude-opus-5" + assert claude_profile["fallback_providers"][0]["provider"] == "openai-codex" + assert codex_profile["toolsets"] == [] + assert codex_profile["agent"]["reasoning_effort"] == "high" + assert codex_profile["fallback_providers"][1] == routing.JETSON_FALLBACK + assert routes["coordinator"][0] == "openai-codex/gpt-5.6-terra" + assert "max" not in json.dumps(routes) + + profile_env = (tmp_path / "profiles/codex-high/.env").read_text(encoding="utf-8") + assert "CLAUDE_CODE_OAUTH_TOKEN=claude-secret" in profile_env + assert "GITEA_TOKEN=gitea-secret" in profile_env + assert "API_SERVER_KEY" not in profile_env + assert "claude-secret" not in json.dumps(routes) + assert (tmp_path / "profiles/codex-high/.env").stat().st_mode & 0o777 == 0o600 + + +def test_degraded_catalog_does_not_replace_known_working_route(tmp_path: Path): + base = _base_config() + base["model"]["model"] = base["model"]["default"] = "gpt-5.6-sol" + (tmp_path / "config.yaml").write_text(yaml.safe_dump(base), encoding="utf-8") + (tmp_path / ".env").write_text("", encoding="utf-8") + codex = routing.Catalog("openai-codex", ["gpt-5.4"], False, True, "degraded") + claude = routing.Catalog("anthropic", ["claude-haiku-4.5"], False, True, "degraded") + + routing.configure_routes(tmp_path, codex, claude) + + current = yaml.safe_load((tmp_path / "config.yaml").read_text(encoding="utf-8")) + assert current["model"]["model"] == "gpt-5.6-sol" + assert current["fallback_providers"][0]["model"] == "claude-opus-5" + + +def test_degraded_refresh_preserves_last_worker_specific_model(tmp_path: Path): + """A catalog outage must not collapse the Codex worker onto coordinator tier.""" + (tmp_path / "config.yaml").write_text( + yaml.safe_dump(_base_config()), encoding="utf-8" + ) + (tmp_path / ".env").write_text("", encoding="utf-8") + live_codex = routing.Catalog( + "openai-codex", ["gpt-5.6-sol", "gpt-5.6-terra"], True, True, "connected" + ) + live_claude = routing.Catalog( + "anthropic", ["claude-opus-5"], True, True, "connected" + ) + routing.configure_routes(tmp_path, live_codex, live_claude) + + degraded_codex = routing.Catalog( + "openai-codex", ["gpt-5.4"], False, True, "degraded" + ) + routing.configure_routes(tmp_path, degraded_codex, live_claude) + + worker = yaml.safe_load( + (tmp_path / "profiles/codex-high/config.yaml").read_text(encoding="utf-8") + ) + assert worker["model"]["model"] == "gpt-5.6-sol" + + +def test_refresh_writes_non_secret_routing_status(tmp_path: Path, monkeypatch): + (tmp_path / "config.yaml").write_text( + yaml.safe_dump(_base_config()), encoding="utf-8" + ) + (tmp_path / ".env").write_text("GITEA_TOKEN=do-not-report\n", encoding="utf-8") + codex = routing.Catalog("openai-codex", ["gpt-5.6-terra"], True, True, "connected") + claude = routing.Catalog("anthropic", ["claude-opus-5"], True, True, "connected") + monkeypatch.setattr(coordinator, "discover_codex_models", lambda: codex) + monkeypatch.setattr(coordinator, "discover_claude_models", lambda: claude) + monkeypatch.setattr(coordinator, "bootstrap_cassandra", lambda root: None) + monkeypatch.setattr(coordinator, "sync_cassandra_repo", lambda env: "ready") + + status = coordinator.refresh_once(tmp_path) + + status_path = tmp_path / "workspace/coordinator/model-routing.json" + assert status_path.is_file() + assert status["projects"]["cassandra"]["state"] == "ready" + assert "do-not-report" not in status_path.read_text(encoding="utf-8") diff --git a/testing/tests/test_hermes_herdr.py b/testing/tests/test_hermes_herdr.py new file mode 100644 index 000000000..91d7f2441 --- /dev/null +++ b/testing/tests/test_hermes_herdr.py @@ -0,0 +1,72 @@ +"""Focused tests for Hermes-to-Herdr routing and the shared auth patch.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +SCRIPTS = Path(__file__).parents[2] / "services/hermes/scripts" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +dispatch = _load("herdr_dispatch") +auth_patch = _load("patch_hermes_auth") + + +def test_herdr_plan_chooses_task_shape_and_caps_effort(): + status = { + "routes": { + "codex-high": [ + "openai-codex/gpt-5.6-sol", + "anthropic/claude-opus-5", + "custom/qwen2.5:14b-instruct-q4_0", + ], + "claude-medium": [ + "anthropic/claude-sonnet-5", + "openai-codex/gpt-5.6-terra", + ], + } + } + implementation = dispatch.select_plan(status, "implementation", "high") + architecture = dispatch.select_plan(status, "architecture", "medium") + assert implementation["worker"] == "codex" + assert implementation["model"] == "gpt-5.6-sol" + assert architecture["worker"] == "claude" + assert architecture["model"] == "claude-sonnet-5" + with pytest.raises(ValueError, match="effort"): + dispatch.select_plan(status, "review", "max") + + +def test_auth_patch_honors_explicit_shared_store(tmp_path: Path): + source = tmp_path / "auth.py" + destination = tmp_path / "patched" / "auth.py" + source.write_text( + "from pathlib import Path\nimport os\n\n" + "def _auth_file_path() -> Path:\n" + " path = get_hermes_home() / \"auth.json\"\n" + " return path\n", + encoding="utf-8", + ) + auth_patch.patch(source, destination) + content = destination.read_text(encoding="utf-8") + assert 'os.environ.get("HERMES_AUTH_FILE"' in content + assert "Path(configured)" in content + + +def test_auth_patch_fails_closed_on_upstream_drift(tmp_path: Path): + source = tmp_path / "auth.py" + source.write_text("def changed():\n pass\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="context changed"): + auth_patch.patch(source, tmp_path / "patched.py")