vault: provision Hermes LAN model gateway credentials

services/vault/hermes-auth-role-bootstrap-job.yaml

@@ -3,7 +3,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: vault-k8s-auth-hermes-11
+  name: vault-k8s-auth-hermes-12
   namespace: vault
 spec:
   backoffLimit: 2

services/vault/hermes-model-gate-lan-token-seed-job.yaml

@@ -0,0 +1,79 @@
+# services/vault/hermes-model-gate-lan-token-seed-job.yaml
+# Purpose: create-only seed of the model-gate LAN API bearer token at
+# kv/atlas/hermes/model-gate-lan-api (field: token). KV-v2 CAS create
+# semantics -- an existing token is never overwritten. The script waits for
+# the vault-k8s-auth-config CronJob (every 15 minutes) to write the seeder
+# auth role, so a fresh merge does not fail the Job on the role race.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hermes-model-gate-lan-token-seed
+  namespace: vault
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: vault-hermes-model-gate-lan-token-seed-1
+  namespace: vault
+spec:
+  backoffLimit: 2
+  template:
+    spec:
+      serviceAccountName: hermes-model-gate-lan-token-seed
+      enableServiceLinks: false
+      restartPolicy: Never
+      nodeSelector:
+        hardware: rpi5
+        kubernetes.io/arch: arm64
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-04, titan-14, titan-18, titan-19, titan-24]
+      securityContext:
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: seed
+          image: docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569
+          imagePullPolicy: IfNotPresent
+          command: [sh, /scripts/vault_hermes_model_gate_lan_token_ensure.sh]
+          env:
+            - name: HOME
+              value: /tmp
+            - name: VAULT_ADDR
+              value: http://vault.vault.svc.cluster.local:8200
+            - name: VAULT_K8S_ROLE
+              value: hermes-model-gate-lan-token-seed
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+            runAsGroup: 1000
+            runAsNonRoot: true
+            runAsUser: 100
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {cpu: 250m, memory: 128Mi}
+      volumes:
+        - name: scripts
+          configMap:
+            name: vault-hermes-model-gate-lan-token-seed-script
+            defaultMode: 0555
+        - name: tmp
+          emptyDir: {}

services/vault/kustomization.yaml

@@ -12,6 +12,7 @@ resources:
   - statefulset.yaml
   - k8s-auth-config-cronjob.yaml
   - hermes-auth-role-bootstrap-job.yaml
+  - hermes-model-gate-lan-token-seed-job.yaml
   - oidc-config-cronjob.yaml
   - service.yaml
   - certificate.yaml
@@ -27,6 +28,9 @@ configMapGenerator:
   - name: vault-k8s-auth-config-script
     files:
       - vault_k8s_auth_configure.sh=scripts/vault_k8s_auth_configure.sh
+  - name: vault-hermes-model-gate-lan-token-seed-script
+    files:
+      - vault_hermes_model_gate_lan_token_ensure.sh=scripts/vault_hermes_model_gate_lan_token_ensure.sh
   - name: vault-entrypoint
     files:
       - vault-entrypoint.sh=scripts/vault-entrypoint.sh

services/vault/scripts/vault_hermes_model_gate_lan_token_ensure.sh

@@ -0,0 +1,177 @@
+#!/usr/bin/env sh
+set -eu
+
+secret_path="kv/atlas/hermes/model-gate-lan-api"
+secret_api_path="kv/data/atlas/hermes/model-gate-lan-api"
+jwt_file="${VAULT_K8S_JWT_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}"
+vault_role="${VAULT_K8S_ROLE:-hermes-model-gate-lan-token-seed}"
+payload_file="${TMPDIR:-/tmp}/hermes-model-gate-lan-token.json"
+
+log() { printf '[model-gate-lan-token] %s\n' "$*" >&2; }
+cleanup() { rm -f "${payload_file}"; }
+trap cleanup EXIT HUP INT TERM
+
+retry_command() {
+  attempt=1
+  while [ "${attempt}" -le 5 ]; do
+    set +e
+    command_output="$("$@" 2>&1)"
+    command_status=$?
+    set -e
+    if [ "${command_status}" -eq 0 ]; then
+      printf '%s' "${command_output}"
+      return 0
+    fi
+    if [ "${attempt}" -lt 5 ]; then
+      sleep "${attempt}"
+    fi
+    attempt=$((attempt + 1))
+  done
+  return "${command_status}"
+}
+
+ensure_token() {
+  if [ -n "${VAULT_TOKEN:-}" ]; then
+    return
+  fi
+  # The seeder auth role is written by the vault-k8s-auth-config CronJob
+  # (every 15 minutes), so a fresh merge can reach this Job before the role
+  # exists. Wait out that window instead of failing the Job.
+  attempt=1
+  while [ "${attempt}" -le 40 ]; do
+    jwt="$(cat "${jwt_file}")"
+    set +e
+    VAULT_TOKEN="$(vault write -field=token \
+      auth/kubernetes/login role="${vault_role}" jwt="${jwt}" 2>&1)"
+    login_status=$?
+    set -e
+    unset jwt
+    if [ "${login_status}" -eq 0 ] && [ -n "${VAULT_TOKEN}" ]; then
+      export VAULT_TOKEN
+      return
+    fi
+    log "Vault Kubernetes login not ready (attempt ${attempt}/40); retrying"
+    sleep 30
+    attempt=$((attempt + 1))
+  done
+  log "Vault Kubernetes login failed after waiting for the auth role"
+  exit 1
+}
+
+# Return 0 for the complete field, 10 when absent, 11 when present but
+# incomplete, and 12 for a persistent authorization/transport failure.
+read_lan_token() {
+  attempt=1
+  while [ "${attempt}" -le 5 ]; do
+    set +e
+    secret_json="$(vault read -format=json "${secret_api_path}" 2>&1)"
+    secret_status=$?
+    set -e
+    if [ "${secret_status}" -eq 0 ]; then
+      unset secret_json
+      set +e
+      lan_token="$(vault kv get -field=token "${secret_path}" 2>&1)"
+      field_status=$?
+      set -e
+      if [ "${field_status}" -eq 0 ]; then
+        case "${lan_token}" in
+          ''|*[!0-9a-f]*) unset lan_token; return 11 ;;
+        esac
+        if [ "${#lan_token}" -eq 64 ]; then
+          unset lan_token
+          return 0
+        fi
+        unset lan_token
+        return 11
+      fi
+      if printf '%s' "${lan_token}" | grep -q 'No value found'; then
+        unset lan_token
+        return 11
+      fi
+      unset lan_token
+    elif printf '%s' "${secret_json}" | grep -Eq 'Code: 404|No value found at'; then
+      unset secret_json
+      return 10
+    fi
+    if [ "${attempt}" -lt 5 ]; then
+      sleep "${attempt}"
+    fi
+    attempt=$((attempt + 1))
+  done
+  unset secret_json
+  return 12
+}
+
+ensure_token
+
+if read_lan_token; then
+  read_status=0
+else
+  read_status=$?
+fi
+case "${read_status}" in
+  0)
+    log "LAN API token already present; no write performed"
+    exit 0
+    ;;
+  11)
+    log "secret exists without a valid token; refusing to overwrite existing fields"
+    exit 1
+    ;;
+  12)
+    log "secret read failed after retries; refusing to seed"
+    exit 1
+    ;;
+  10) ;;
+  *)
+    log "unexpected secret read state ${read_status}"
+    exit 1
+    ;;
+esac
+
+new_token="$(retry_command vault write -field=random_bytes \
+  sys/tools/random/32 format=hex)" || {
+    log "Vault random token generation failed after retries"
+    exit 1
+  }
+case "${new_token}" in
+  ''|*[!0-9a-f]*)
+    unset new_token
+    log "Vault returned an invalid random token"
+    exit 1
+    ;;
+esac
+if [ "${#new_token}" -ne 64 ]; then
+  unset new_token
+  log "Vault returned an invalid random token"
+  exit 1
+fi
+
+umask 077
+printf '{"options":{"cas":0},"data":{"token":"%s"}}\n' \
+  "${new_token}" > "${payload_file}"
+unset new_token
+
+set +e
+create_error="$(vault write "${secret_api_path}" @"${payload_file}" 2>&1 >/dev/null)"
+create_status=$?
+set -e
+cleanup
+if [ "${create_status}" -eq 0 ]; then
+  log "LAN API token created with KV-v2 CAS create-only semantics"
+  exit 0
+fi
+
+if printf '%s' "${create_error}" | grep -qi 'check-and-set'; then
+  unset create_error
+  if read_lan_token; then
+    log "another seeder won CAS; existing token preserved"
+    exit 0
+  fi
+  log "CAS race did not produce a readable token; refusing to continue"
+  exit 1
+fi
+
+unset create_error
+log "KV-v2 CAS create failed; refusing to retry a write"
+exit 1

services/vault/scripts/vault_k8s_auth_configure.sh

@@ -278,6 +278,23 @@ write_policy_and_role "hermes-agent" "hermes" "hermes-agent" \
   "hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram hermes/developer-keycloak hermes/developer-harbor hermes/developer-jenkins hermes/developer-ssh" ""
 write_policy_and_role "hermes-switchyard" "hermes" "hermes-switchyard" \
   "hermes/chat-telegram" ""
+write_policy_and_role "hermes-model-gate" "hermes" "hermes-model-gate" \
+  "hermes/model-gate-lan-api" ""
+hermes_model_gate_lan_token_seed_policy='
+path "kv/data/atlas/hermes/model-gate-lan-api" {
+  capabilities = ["create", "read"]
+}
+path "sys/tools/random/32" {
+  capabilities = ["update"]
+}
+'
+write_raw_policy "hermes-model-gate-lan-token-seed" "${hermes_model_gate_lan_token_seed_policy}"
+log "writing role hermes-model-gate-lan-token-seed"
+vault_cmd write "auth/kubernetes/role/hermes-model-gate-lan-token-seed" \
+  bound_service_account_names="hermes-model-gate-lan-token-seed" \
+  bound_service_account_namespaces="vault" \
+  policies="hermes-model-gate-lan-token-seed" \
+  ttl="${role_ttl}"
 hermes_jenkins_token_seed_policy='
 path "kv/data/atlas/hermes/developer-jenkins" {
   capabilities = ["create", "read"]