From c79489d0b8293ad25c133fd6afca5eecd01ea66a Mon Sep 17 00:00:00 2001 From: jenkins Date: Fri, 15 May 2026 11:52:03 -0300 Subject: [PATCH] recovery: keep storage nodes as spillover only --- infrastructure/core/kustomization.yaml | 3 ++ .../core/node-prefer-noschedule-cronjob.yaml | 35 +++++++++++++++++++ .../core/node-prefer-noschedule-rbac.yaml | 22 ++++++++++++ ...node-prefer-noschedule-serviceaccount.yaml | 6 ++++ .../scripts/vault_oidc_secret_ensure.sh | 7 +++- services/maintenance/ariadne-deployment.yaml | 19 ++++++++++ 6 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 infrastructure/core/node-prefer-noschedule-cronjob.yaml create mode 100644 infrastructure/core/node-prefer-noschedule-rbac.yaml create mode 100644 infrastructure/core/node-prefer-noschedule-serviceaccount.yaml diff --git a/infrastructure/core/kustomization.yaml b/infrastructure/core/kustomization.yaml index 257e1f06..d34e023c 100644 --- a/infrastructure/core/kustomization.yaml +++ b/infrastructure/core/kustomization.yaml @@ -4,6 +4,9 @@ kind: Kustomization resources: - ../modules/base - ../modules/profiles/atlas-ha + - node-prefer-noschedule-serviceaccount.yaml + - node-prefer-noschedule-rbac.yaml + - node-prefer-noschedule-cronjob.yaml - coredns-custom.yaml - coredns-deployment.yaml - ntp-sync-daemonset.yaml diff --git a/infrastructure/core/node-prefer-noschedule-cronjob.yaml b/infrastructure/core/node-prefer-noschedule-cronjob.yaml new file mode 100644 index 00000000..dd4257df --- /dev/null +++ b/infrastructure/core/node-prefer-noschedule-cronjob.yaml @@ -0,0 +1,35 @@ +# infrastructure/core/node-prefer-noschedule-cronjob.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: node-prefer-noschedule + namespace: kube-system +spec: + schedule: "*/20 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 1 + template: + spec: + serviceAccountName: node-prefer-noschedule + restartPolicy: OnFailure + containers: + - name: taint + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + command: + - /usr/bin/env + - bash + - -ceu + - | + for node in titan-13 titan-15 titan-17 titan-19; do + if kubectl get node "${node}" >/dev/null 2>&1; then + kubectl label node "${node}" atlas.bstein.dev/spillover=true --overwrite=true + kubectl taint node "${node}" longhorn=true:PreferNoSchedule --overwrite=true + kubectl taint node "${node}" atlas.bstein.dev/spillover=true:PreferNoSchedule --overwrite=true + else + echo "skipping missing node ${node}" + fi + done diff --git a/infrastructure/core/node-prefer-noschedule-rbac.yaml b/infrastructure/core/node-prefer-noschedule-rbac.yaml new file mode 100644 index 00000000..73d69525 --- /dev/null +++ b/infrastructure/core/node-prefer-noschedule-rbac.yaml @@ -0,0 +1,22 @@ +# infrastructure/core/node-prefer-noschedule-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: node-prefer-noschedule +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: node-prefer-noschedule +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node-prefer-noschedule +subjects: + - kind: ServiceAccount + name: node-prefer-noschedule + namespace: kube-system diff --git a/infrastructure/core/node-prefer-noschedule-serviceaccount.yaml b/infrastructure/core/node-prefer-noschedule-serviceaccount.yaml new file mode 100644 index 00000000..4a5137b5 --- /dev/null +++ b/infrastructure/core/node-prefer-noschedule-serviceaccount.yaml @@ -0,0 +1,6 @@ +# infrastructure/core/node-prefer-noschedule-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-prefer-noschedule + namespace: kube-system diff --git a/services/keycloak/scripts/vault_oidc_secret_ensure.sh b/services/keycloak/scripts/vault_oidc_secret_ensure.sh index a951cfa7..6fc5483a 100755 --- a/services/keycloak/scripts/vault_oidc_secret_ensure.sh +++ b/services/keycloak/scripts/vault_oidc_secret_ensure.sh @@ -107,17 +107,22 @@ payload="$(jq -nc \ --arg client_id "vault-oidc" \ --arg client_secret "${CLIENT_SECRET}" \ --arg default_role "admin" \ + --arg token_policies "default" \ --arg scopes "openid profile email groups" \ --arg user_claim "preferred_username" \ --arg groups_claim "groups" \ --arg redirect_uris "https://secret.bstein.dev/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \ --arg bound_audiences "vault-oidc" \ + --arg bound_claims_type "string" \ --arg admin_group "admin" \ --arg admin_policies "default,vault-admin" \ + --arg admin_bound_claims '{"groups":"admin"}' \ --arg dev_group "dev" \ --arg dev_policies "default,dev-kv" \ --arg user_group "dev" \ --arg user_policies "default,dev-kv" \ - '{data:{discovery_url:$discovery_url,client_id:$client_id,client_secret:$client_secret,default_role:$default_role,scopes:$scopes,user_claim:$user_claim,groups_claim:$groups_claim,redirect_uris:$redirect_uris,bound_audiences:$bound_audiences,admin_group:$admin_group,admin_policies:$admin_policies,dev_group:$dev_group,dev_policies:$dev_policies,user_group:$user_group,user_policies:$user_policies}}')" + --arg ui_default_auth_method "oidc" \ + --arg ui_default_auth_path "oidc" \ + '{data:{discovery_url:$discovery_url,client_id:$client_id,client_secret:$client_secret,default_role:$default_role,token_policies:$token_policies,scopes:$scopes,user_claim:$user_claim,groups_claim:$groups_claim,redirect_uris:$redirect_uris,bound_audiences:$bound_audiences,bound_claims_type:$bound_claims_type,admin_group:$admin_group,admin_policies:$admin_policies,admin_bound_claims:$admin_bound_claims,dev_group:$dev_group,dev_policies:$dev_policies,user_group:$user_group,user_policies:$user_policies,ui_default_auth_method:$ui_default_auth_method,ui_default_auth_path:$ui_default_auth_path}}')" curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ -d "${payload}" "${vault_addr}/v1/kv/data/atlas/vault/vault-oidc-config" >/dev/null diff --git a/services/maintenance/ariadne-deployment.yaml b/services/maintenance/ariadne-deployment.yaml index 7c464f69..e3d643f5 100644 --- a/services/maintenance/ariadne-deployment.yaml +++ b/services/maintenance/ariadne-deployment.yaml @@ -86,15 +86,34 @@ spec: export VAULT_OIDC_SCOPES="{{ .Data.data.scopes }}" export VAULT_OIDC_USER_CLAIM="{{ .Data.data.user_claim }}" export VAULT_OIDC_GROUPS_CLAIM="{{ .Data.data.groups_claim }}" + {{- if .Data.data.token_policies }} export VAULT_OIDC_TOKEN_POLICIES="{{ .Data.data.token_policies }}" + {{- else }} + export VAULT_OIDC_TOKEN_POLICIES="default" + {{- end }} export VAULT_OIDC_ADMIN_GROUP="{{ .Data.data.admin_group }}" export VAULT_OIDC_ADMIN_POLICIES="{{ .Data.data.admin_policies }}" + {{- if .Data.data.admin_bound_claims }} + export VAULT_OIDC_ADMIN_BOUND_CLAIMS="{{ .Data.data.admin_bound_claims }}" + {{- else }} + export VAULT_OIDC_ADMIN_BOUND_CLAIMS="{\"groups\":\"admin\"}" + {{- end }} export VAULT_OIDC_DEV_GROUP="{{ .Data.data.dev_group }}" export VAULT_OIDC_DEV_POLICIES="{{ .Data.data.dev_policies }}" export VAULT_OIDC_USER_GROUP="{{ .Data.data.user_group }}" export VAULT_OIDC_USER_POLICIES="{{ .Data.data.user_policies }}" export VAULT_OIDC_REDIRECT_URIS="{{ .Data.data.redirect_uris }}" export VAULT_OIDC_BOUND_AUDIENCES="{{ .Data.data.bound_audiences }}" + {{- if .Data.data.ui_default_auth_method }} + export VAULT_UI_DEFAULT_AUTH_METHOD="{{ .Data.data.ui_default_auth_method }}" + {{- else }} + export VAULT_UI_DEFAULT_AUTH_METHOD="oidc" + {{- end }} + {{- if .Data.data.ui_default_auth_path }} + export VAULT_UI_DEFAULT_AUTH_PATH="{{ .Data.data.ui_default_auth_path }}" + {{- else }} + export VAULT_UI_DEFAULT_AUTH_PATH="oidc" + {{- end }} {{- if .Data.data.bound_claims_type }} export VAULT_OIDC_BOUND_CLAIMS_TYPE="{{ .Data.data.bound_claims_type }}" {{- else }}