From a9c2d3c5e85e8ad6544e0910583a57df1c2b5e21 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Sat, 17 Jan 2026 03:00:25 -0300 Subject: [PATCH] vault: retry vault cli operations --- .../vault/scripts/vault_k8s_auth_configure.sh | 42 +++++++++++-------- .../vault/scripts/vault_oidc_configure.sh | 38 ++++++++++------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 48dfe78..bbb5e3a 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -3,29 +3,35 @@ set -eu log() { echo "[vault-k8s-auth] $*"; } +vault_cmd() { + for attempt in 1 2 3 4 5 6; do + output="$(vault "$@" 2>&1)" + status=$? + if [ "${status}" -eq 0 ]; then + printf '%s' "${output}" + return 0 + fi + log "vault command failed; retrying (${attempt}/6)" + sleep $((attempt * 2)) + done + log "vault command failed; giving up" + return 1 +} + ensure_token() { if [ -n "${VAULT_TOKEN:-}" ]; then return fi role="${VAULT_K8S_ROLE:-vault}" jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" - if ! VAULT_TOKEN="$(vault write -field=token auth/kubernetes/login role="${role}" jwt="${jwt}")"; then + if ! VAULT_TOKEN="$(vault_cmd write -field=token auth/kubernetes/login role="${role}" jwt="${jwt}")"; then log "kubernetes auth login failed; set VAULT_TOKEN or fix role ${role}" exit 1 fi export VAULT_TOKEN } -status_json="" -for attempt in 1 2 3 4 5 6; do - status_json="$(vault status -format=json 2>/dev/null || true)" - if [ -n "${status_json}" ]; then - break - fi - log "vault status failed; retrying (${attempt}/6)" - sleep $((attempt * 2)) -done -if [ -z "${status_json}" ]; then +if ! status_json="$(vault_cmd status -format=json)"; then log "vault status failed; check VAULT_ADDR and VAULT_TOKEN" exit 1 fi @@ -55,13 +61,13 @@ if [ -z "${token_reviewer_jwt}" ]; then token_reviewer_jwt="${k8s_token}" fi -if ! vault auth list -format=json | grep -q '"kubernetes/"'; then +if ! vault_cmd auth list -format=json | grep -q '"kubernetes/"'; then log "enabling kubernetes auth" - vault auth enable kubernetes + vault_cmd auth enable kubernetes fi log "configuring kubernetes auth" -vault write auth/kubernetes/config \ +vault_cmd write auth/kubernetes/config \ token_reviewer_jwt="${token_reviewer_jwt}" \ kubernetes_host="${k8s_host}" \ kubernetes_ca_cert="${k8s_ca}" @@ -70,7 +76,7 @@ write_raw_policy() { name="$1" body="$2" log "writing policy ${name}" - printf '%s\n' "${body}" | vault policy write "${name}" - + printf '%s\n' "${body}" | vault_cmd policy write "${name}" - } write_policy_and_role() { @@ -103,10 +109,10 @@ path \"kv/metadata/atlas/${path}\" { done log "writing policy ${role}" - printf '%s\n' "${policy_body}" | vault policy write "${role}" - + printf '%s\n' "${policy_body}" | vault_cmd policy write "${role}" - log "writing role ${role}" - vault write "auth/kubernetes/role/${role}" \ + vault_cmd write "auth/kubernetes/role/${role}" \ bound_service_account_names="${service_accounts}" \ bound_service_account_namespaces="${namespace}" \ policies="${role}" \ @@ -184,7 +190,7 @@ path "kv/data/atlas/shared/*" { ' write_raw_policy "dev-kv" "${dev_kv_policy}" log "writing role vault-admin" -vault write "auth/kubernetes/role/vault-admin" \ +vault_cmd write "auth/kubernetes/role/vault-admin" \ bound_service_account_names="vault-admin" \ bound_service_account_namespaces="vault" \ policies="vault-admin" \ diff --git a/services/vault/scripts/vault_oidc_configure.sh b/services/vault/scripts/vault_oidc_configure.sh index 0f569e8..d703ed5 100644 --- a/services/vault/scripts/vault_oidc_configure.sh +++ b/services/vault/scripts/vault_oidc_configure.sh @@ -3,29 +3,35 @@ set -eu log() { echo "[vault-oidc] $*"; } +vault_cmd() { + for attempt in 1 2 3 4 5 6; do + output="$(vault "$@" 2>&1)" + status=$? + if [ "${status}" -eq 0 ]; then + printf '%s' "${output}" + return 0 + fi + log "vault command failed; retrying (${attempt}/6)" + sleep $((attempt * 2)) + done + log "vault command failed; giving up" + return 1 +} + ensure_token() { if [ -n "${VAULT_TOKEN:-}" ]; then return fi role="${VAULT_K8S_ROLE:-vault}" jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" - if ! VAULT_TOKEN="$(vault write -field=token auth/kubernetes/login role="${role}" jwt="${jwt}")"; then + if ! VAULT_TOKEN="$(vault_cmd write -field=token auth/kubernetes/login role="${role}" jwt="${jwt}")"; then log "kubernetes auth login failed; set VAULT_TOKEN or fix role ${role}" exit 1 fi export VAULT_TOKEN } -status_json="" -for attempt in 1 2 3 4 5 6; do - status_json="$(vault status -format=json 2>/dev/null || true)" - if [ -n "${status_json}" ]; then - break - fi - log "vault status failed; retrying (${attempt}/6)" - sleep $((attempt * 2)) -done -if [ -z "${status_json}" ]; then +if ! status_json="$(vault_cmd status -format=json)"; then log "vault status failed; check VAULT_ADDR and VAULT_TOKEN" exit 1 fi @@ -65,19 +71,19 @@ dev_policies="${VAULT_OIDC_DEV_POLICIES:-default,dev-kv}" user_group="${VAULT_OIDC_USER_GROUP:-${dev_group}}" user_policies="${VAULT_OIDC_USER_POLICIES:-${VAULT_OIDC_TOKEN_POLICIES:-${dev_policies}}}" -if ! vault auth list -format=json | grep -q '"oidc/"'; then +if ! vault_cmd auth list -format=json | grep -q '"oidc/"'; then log "enabling oidc auth method" - vault auth enable oidc + vault_cmd auth enable oidc fi log "configuring oidc auth" -vault write auth/oidc/config \ +vault_cmd write auth/oidc/config \ oidc_discovery_url="${VAULT_OIDC_DISCOVERY_URL}" \ oidc_client_id="${VAULT_OIDC_CLIENT_ID}" \ oidc_client_secret="${VAULT_OIDC_CLIENT_SECRET}" \ default_role="${default_role}" -vault auth tune -listing-visibility=unauth oidc >/dev/null +vault_cmd auth tune -listing-visibility=unauth oidc >/dev/null build_bound_claims() { claim="$1" @@ -149,7 +155,7 @@ configure_role() { } EOF log "configuring oidc role ${role_name}" - vault write "auth/oidc/role/${role_name}" @"${payload_file}" + vault_cmd write "auth/oidc/role/${role_name}" @"${payload_file}" rm -f "${payload_file}" }