2bfdee6169
titan-24 is an accelerator node (co-hosts the out-of-cluster Sui validator), not a general worker. The amd64 build leg was requiring node-role worker=true, which forced labeling titan-24 as a worker and opened it to unrelated cluster scheduling. It already pins by hostname+arch, so drop the worker requirement and remove the titan-24 worker-join from the node-prefer CronJob entirely. The build targets titan-24 specifically (hostname) and tolerates its taint; nothing else in the cluster gets scheduled there. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
111 lines
4.9 KiB
YAML
111 lines
4.9 KiB
YAML
# infrastructure/core/node-prefer-noschedule-cronjob.yaml
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: node-prefer-noschedule
|
|
namespace: kube-system
|
|
spec:
|
|
schedule: "* * * * *"
|
|
concurrencyPolicy: Replace
|
|
successfulJobsHistoryLimit: 1
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
serviceAccountName: node-prefer-noschedule
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: taint
|
|
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
|
|
command:
|
|
- /usr/bin/env
|
|
- bash
|
|
- -ceu
|
|
- |
|
|
KUBE_TOKEN_PATH="/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
KUBE_CA_PATH="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
KUBE_SERVER="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS:-443}"
|
|
|
|
k() {
|
|
kubectl \
|
|
--server="${KUBE_SERVER}" \
|
|
--certificate-authority="${KUBE_CA_PATH}" \
|
|
--token="$(cat "${KUBE_TOKEN_PATH}")" \
|
|
--request-timeout=10s \
|
|
"$@"
|
|
}
|
|
|
|
clear_worker() {
|
|
local node="${1}"
|
|
local hardware="${2}"
|
|
if k get node "${node}" >/dev/null 2>&1; then
|
|
k label node "${node}" node-role.kubernetes.io/worker=true "hardware=${hardware}" --overwrite=true || true
|
|
k label node "${node}" node-role.kubernetes.io/storage-backbone- || true
|
|
k label node "${node}" atlas.bstein.dev/spillover- || true
|
|
# Recovery cordons are owned by Ananke, not this role reconciler.
|
|
else
|
|
echo "skipping missing node ${node}"
|
|
fi
|
|
}
|
|
|
|
clear_worker titan-04 rpi5
|
|
clear_worker titan-05 rpi5
|
|
clear_worker titan-07 rpi5
|
|
clear_worker titan-08 rpi5
|
|
clear_worker titan-11 rpi5
|
|
clear_worker titan-12 rpi4
|
|
clear_worker titan-14 rpi4
|
|
clear_worker titan-18 rpi4
|
|
clear_worker titan-22 amd64
|
|
|
|
if k get node titan-22 >/dev/null 2>&1; then
|
|
k label node titan-22 \
|
|
atlas.bstein.dev/general-compute=last-resort \
|
|
atlas.bstein.dev/media-primary=true \
|
|
--overwrite=true || true
|
|
# Keep spare capacity available without letting routine pods displace media service.
|
|
k taint node titan-22 atlas.bstein.dev/media-primary=true:PreferNoSchedule --overwrite=true || true
|
|
fi
|
|
|
|
|
|
if k get node titan-23 >/dev/null 2>&1; then
|
|
k label node titan-23 \
|
|
veles.bstein.dev/simulation=true \
|
|
veles.bstein.dev/node-pool=oceanus \
|
|
cassandra.bstein.dev/simulation=true \
|
|
cassandra.bstein.dev/node-pool=oceanus \
|
|
node-role.kubernetes.io/veles-sim=true \
|
|
node-role.kubernetes.io/cassandra-sim=true \
|
|
longhorn-host=true \
|
|
hardware=oceanus \
|
|
--overwrite=true || true
|
|
k label node titan-23 node-role.kubernetes.io/worker- || true
|
|
k taint node titan-23 veles.bstein.dev/simulation=true:NoSchedule --overwrite=true || true
|
|
else
|
|
echo "skipping missing node titan-23"
|
|
fi
|
|
|
|
if k get node titan-21 >/dev/null 2>&1; then
|
|
# The Tegra iSCSI initiator cannot host Longhorn engines.
|
|
k label node titan-21 longhorn-host- || true
|
|
else
|
|
echo "skipping missing node titan-21"
|
|
fi
|
|
|
|
for node in titan-13 titan-15 titan-17 titan-19; do
|
|
if k get node "${node}" >/dev/null 2>&1; then
|
|
k label node "${node}" \
|
|
atlas.bstein.dev/spillover=true \
|
|
longhorn-host=true \
|
|
node-role.kubernetes.io/worker=true \
|
|
node-role.kubernetes.io/storage-backbone=true \
|
|
--overwrite=true || true
|
|
k taint node "${node}" longhorn=true:PreferNoSchedule --overwrite=true || true
|
|
k taint node "${node}" atlas.bstein.dev/spillover=true:PreferNoSchedule --overwrite=true || true
|
|
else
|
|
echo "skipping missing node ${node}"
|
|
fi
|
|
done
|