recovery: keep storage nodes as spillover only

This commit is contained in:
jenkins 2026-05-15 11:52:03 -03:00
parent 67253315f0
commit c79489d0b8
6 changed files with 91 additions and 1 deletions

View File

@ -4,6 +4,9 @@ kind: Kustomization
resources:
- ../modules/base
- ../modules/profiles/atlas-ha
- node-prefer-noschedule-serviceaccount.yaml
- node-prefer-noschedule-rbac.yaml
- node-prefer-noschedule-cronjob.yaml
- coredns-custom.yaml
- coredns-deployment.yaml
- ntp-sync-daemonset.yaml

View File

@ -0,0 +1,35 @@
# infrastructure/core/node-prefer-noschedule-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
name: node-prefer-noschedule
namespace: kube-system
spec:
schedule: "*/20 * * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 1
template:
spec:
serviceAccountName: node-prefer-noschedule
restartPolicy: OnFailure
containers:
- name: taint
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
command:
- /usr/bin/env
- bash
- -ceu
- |
for node in titan-13 titan-15 titan-17 titan-19; do
if kubectl get node "${node}" >/dev/null 2>&1; then
kubectl label node "${node}" atlas.bstein.dev/spillover=true --overwrite=true
kubectl taint node "${node}" longhorn=true:PreferNoSchedule --overwrite=true
kubectl taint node "${node}" atlas.bstein.dev/spillover=true:PreferNoSchedule --overwrite=true
else
echo "skipping missing node ${node}"
fi
done

View File

@ -0,0 +1,22 @@
# infrastructure/core/node-prefer-noschedule-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: node-prefer-noschedule
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: node-prefer-noschedule
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: node-prefer-noschedule
subjects:
- kind: ServiceAccount
name: node-prefer-noschedule
namespace: kube-system

View File

@ -0,0 +1,6 @@
# infrastructure/core/node-prefer-noschedule-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: node-prefer-noschedule
namespace: kube-system

View File

@ -107,17 +107,22 @@ payload="$(jq -nc \
--arg client_id "vault-oidc" \
--arg client_secret "${CLIENT_SECRET}" \
--arg default_role "admin" \
--arg token_policies "default" \
--arg scopes "openid profile email groups" \
--arg user_claim "preferred_username" \
--arg groups_claim "groups" \
--arg redirect_uris "https://secret.bstein.dev/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \
--arg bound_audiences "vault-oidc" \
--arg bound_claims_type "string" \
--arg admin_group "admin" \
--arg admin_policies "default,vault-admin" \
--arg admin_bound_claims '{"groups":"admin"}' \
--arg dev_group "dev" \
--arg dev_policies "default,dev-kv" \
--arg user_group "dev" \
--arg user_policies "default,dev-kv" \
'{data:{discovery_url:$discovery_url,client_id:$client_id,client_secret:$client_secret,default_role:$default_role,scopes:$scopes,user_claim:$user_claim,groups_claim:$groups_claim,redirect_uris:$redirect_uris,bound_audiences:$bound_audiences,admin_group:$admin_group,admin_policies:$admin_policies,dev_group:$dev_group,dev_policies:$dev_policies,user_group:$user_group,user_policies:$user_policies}}')"
--arg ui_default_auth_method "oidc" \
--arg ui_default_auth_path "oidc" \
'{data:{discovery_url:$discovery_url,client_id:$client_id,client_secret:$client_secret,default_role:$default_role,token_policies:$token_policies,scopes:$scopes,user_claim:$user_claim,groups_claim:$groups_claim,redirect_uris:$redirect_uris,bound_audiences:$bound_audiences,bound_claims_type:$bound_claims_type,admin_group:$admin_group,admin_policies:$admin_policies,admin_bound_claims:$admin_bound_claims,dev_group:$dev_group,dev_policies:$dev_policies,user_group:$user_group,user_policies:$user_policies,ui_default_auth_method:$ui_default_auth_method,ui_default_auth_path:$ui_default_auth_path}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-d "${payload}" "${vault_addr}/v1/kv/data/atlas/vault/vault-oidc-config" >/dev/null

View File

@ -86,15 +86,34 @@ spec:
export VAULT_OIDC_SCOPES="{{ .Data.data.scopes }}"
export VAULT_OIDC_USER_CLAIM="{{ .Data.data.user_claim }}"
export VAULT_OIDC_GROUPS_CLAIM="{{ .Data.data.groups_claim }}"
{{- if .Data.data.token_policies }}
export VAULT_OIDC_TOKEN_POLICIES="{{ .Data.data.token_policies }}"
{{- else }}
export VAULT_OIDC_TOKEN_POLICIES="default"
{{- end }}
export VAULT_OIDC_ADMIN_GROUP="{{ .Data.data.admin_group }}"
export VAULT_OIDC_ADMIN_POLICIES="{{ .Data.data.admin_policies }}"
{{- if .Data.data.admin_bound_claims }}
export VAULT_OIDC_ADMIN_BOUND_CLAIMS="{{ .Data.data.admin_bound_claims }}"
{{- else }}
export VAULT_OIDC_ADMIN_BOUND_CLAIMS="{\"groups\":\"admin\"}"
{{- end }}
export VAULT_OIDC_DEV_GROUP="{{ .Data.data.dev_group }}"
export VAULT_OIDC_DEV_POLICIES="{{ .Data.data.dev_policies }}"
export VAULT_OIDC_USER_GROUP="{{ .Data.data.user_group }}"
export VAULT_OIDC_USER_POLICIES="{{ .Data.data.user_policies }}"
export VAULT_OIDC_REDIRECT_URIS="{{ .Data.data.redirect_uris }}"
export VAULT_OIDC_BOUND_AUDIENCES="{{ .Data.data.bound_audiences }}"
{{- if .Data.data.ui_default_auth_method }}
export VAULT_UI_DEFAULT_AUTH_METHOD="{{ .Data.data.ui_default_auth_method }}"
{{- else }}
export VAULT_UI_DEFAULT_AUTH_METHOD="oidc"
{{- end }}
{{- if .Data.data.ui_default_auth_path }}
export VAULT_UI_DEFAULT_AUTH_PATH="{{ .Data.data.ui_default_auth_path }}"
{{- else }}
export VAULT_UI_DEFAULT_AUTH_PATH="oidc"
{{- end }}
{{- if .Data.data.bound_claims_type }}
export VAULT_OIDC_BOUND_CLAIMS_TYPE="{{ .Data.data.bound_claims_type }}"
{{- else }}