vault: retry vault cli operations

services/vault/scripts/vault_k8s_auth_configure.sh

@@ -3,29 +3,35 @@ set -eu
 
 log() { echo "[vault-k8s-auth] $*"; }
 
+vault_cmd() {
+  for attempt in 1 2 3 4 5 6; do
+    output="$(vault "$@" 2>&1)"
+    status=$?
+    if [ "${status}" -eq 0 ]; then
+      printf '%s' "${output}"
+      return 0
+    fi
+    log "vault command failed; retrying (${attempt}/6)"
+    sleep $((attempt * 2))
+  done
+  log "vault command failed; giving up"
+  return 1
+}
+
 ensure_token() {
   if [ -n "${VAULT_TOKEN:-}" ]; then
     return
   fi
   role="${VAULT_K8S_ROLE:-vault}"
   jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-  if ! VAULT_TOKEN="$(vault write -field=token auth/kubernetes/login role="${role}" jwt="${jwt}")"; then
+  if ! VAULT_TOKEN="$(vault_cmd write -field=token auth/kubernetes/login role="${role}" jwt="${jwt}")"; then
     log "kubernetes auth login failed; set VAULT_TOKEN or fix role ${role}"
     exit 1
   fi
   export VAULT_TOKEN
 }
 
-status_json=""
+if ! status_json="$(vault_cmd status -format=json)"; then
-for attempt in 1 2 3 4 5 6; do
-  status_json="$(vault status -format=json 2>/dev/null || true)"
-  if [ -n "${status_json}" ]; then
-    break
-  fi
-  log "vault status failed; retrying (${attempt}/6)"
-  sleep $((attempt * 2))
-done
-if [ -z "${status_json}" ]; then
   log "vault status failed; check VAULT_ADDR and VAULT_TOKEN"
   exit 1
 fi
@@ -55,13 +61,13 @@ if [ -z "${token_reviewer_jwt}" ]; then
   token_reviewer_jwt="${k8s_token}"
 fi
 
-if ! vault auth list -format=json | grep -q '"kubernetes/"'; then
+if ! vault_cmd auth list -format=json | grep -q '"kubernetes/"'; then
   log "enabling kubernetes auth"
-  vault auth enable kubernetes
+  vault_cmd auth enable kubernetes
 fi
 
 log "configuring kubernetes auth"
-vault write auth/kubernetes/config \
+vault_cmd write auth/kubernetes/config \
   token_reviewer_jwt="${token_reviewer_jwt}" \
   kubernetes_host="${k8s_host}" \
   kubernetes_ca_cert="${k8s_ca}"
@@ -70,7 +76,7 @@ write_raw_policy() {
   name="$1"
   body="$2"
   log "writing policy ${name}"
-  printf '%s\n' "${body}" | vault policy write "${name}" -
+  printf '%s\n' "${body}" | vault_cmd policy write "${name}" -
 }
 
 write_policy_and_role() {
@@ -103,10 +109,10 @@ path \"kv/metadata/atlas/${path}\" {
   done
 
   log "writing policy ${role}"
-  printf '%s\n' "${policy_body}" | vault policy write "${role}" -
+  printf '%s\n' "${policy_body}" | vault_cmd policy write "${role}" -
 
   log "writing role ${role}"
-  vault write "auth/kubernetes/role/${role}" \
+  vault_cmd write "auth/kubernetes/role/${role}" \
     bound_service_account_names="${service_accounts}" \
     bound_service_account_namespaces="${namespace}" \
     policies="${role}" \
@@ -184,7 +190,7 @@ path "kv/data/atlas/shared/*" {
 '
 write_raw_policy "dev-kv" "${dev_kv_policy}"
 log "writing role vault-admin"
-vault write "auth/kubernetes/role/vault-admin" \
+vault_cmd write "auth/kubernetes/role/vault-admin" \
   bound_service_account_names="vault-admin" \
   bound_service_account_namespaces="vault" \
   policies="vault-admin" \

services/vault/scripts/vault_oidc_configure.sh

@@ -3,29 +3,35 @@ set -eu
 
 log() { echo "[vault-oidc] $*"; }
 
+vault_cmd() {
+  for attempt in 1 2 3 4 5 6; do
+    output="$(vault "$@" 2>&1)"
+    status=$?
+    if [ "${status}" -eq 0 ]; then
+      printf '%s' "${output}"
+      return 0
+    fi
+    log "vault command failed; retrying (${attempt}/6)"
+    sleep $((attempt * 2))
+  done
+  log "vault command failed; giving up"
+  return 1
+}
+
 ensure_token() {
   if [ -n "${VAULT_TOKEN:-}" ]; then
     return
   fi
   role="${VAULT_K8S_ROLE:-vault}"
   jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-  if ! VAULT_TOKEN="$(vault write -field=token auth/kubernetes/login role="${role}" jwt="${jwt}")"; then
+  if ! VAULT_TOKEN="$(vault_cmd write -field=token auth/kubernetes/login role="${role}" jwt="${jwt}")"; then
     log "kubernetes auth login failed; set VAULT_TOKEN or fix role ${role}"
     exit 1
   fi
   export VAULT_TOKEN
 }
 
-status_json=""
+if ! status_json="$(vault_cmd status -format=json)"; then
-for attempt in 1 2 3 4 5 6; do
-  status_json="$(vault status -format=json 2>/dev/null || true)"
-  if [ -n "${status_json}" ]; then
-    break
-  fi
-  log "vault status failed; retrying (${attempt}/6)"
-  sleep $((attempt * 2))
-done
-if [ -z "${status_json}" ]; then
   log "vault status failed; check VAULT_ADDR and VAULT_TOKEN"
   exit 1
 fi
@@ -65,19 +71,19 @@ dev_policies="${VAULT_OIDC_DEV_POLICIES:-default,dev-kv}"
 user_group="${VAULT_OIDC_USER_GROUP:-${dev_group}}"
 user_policies="${VAULT_OIDC_USER_POLICIES:-${VAULT_OIDC_TOKEN_POLICIES:-${dev_policies}}}"
 
-if ! vault auth list -format=json | grep -q '"oidc/"'; then
+if ! vault_cmd auth list -format=json | grep -q '"oidc/"'; then
   log "enabling oidc auth method"
-  vault auth enable oidc
+  vault_cmd auth enable oidc
 fi
 
 log "configuring oidc auth"
-vault write auth/oidc/config \
+vault_cmd write auth/oidc/config \
   oidc_discovery_url="${VAULT_OIDC_DISCOVERY_URL}" \
   oidc_client_id="${VAULT_OIDC_CLIENT_ID}" \
   oidc_client_secret="${VAULT_OIDC_CLIENT_SECRET}" \
   default_role="${default_role}"
 
-vault auth tune -listing-visibility=unauth oidc >/dev/null
+vault_cmd auth tune -listing-visibility=unauth oidc >/dev/null
 
 build_bound_claims() {
   claim="$1"
@@ -149,7 +155,7 @@ configure_role() {
 }
 EOF
   log "configuring oidc role ${role_name}"
-  vault write "auth/oidc/role/${role_name}" @"${payload_file}"
+  vault_cmd write "auth/oidc/role/${role_name}" @"${payload_file}"
   rm -f "${payload_file}"
 }