titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

node(titan-22): return worker to service

Browse Source
This commit is contained in:
jenkins 2026-08-22 21:21:03 -03:00
parent 091a787fc7
commit 21264a71c8
6 changed files with 114 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
infrastructure/core/kustomization.yaml
View File

@ -7,6 +7,7 @@ resources:
- node-prefer-noschedule-serviceaccount.yaml
- node-prefer-noschedule-rbac.yaml
- node-prefer-noschedule-cronjob.yaml
- titan-22-return-to-service-job.yaml
- coredns-custom.yaml
- coredns-deployment.yaml
- ntp-sync-daemonset.yaml

27
infrastructure/core/titan-22-return-to-service-job.yaml Normal file
View File

@ -0,0 +1,27 @@
# infrastructure/core/titan-22-return-to-service-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: titan-22-return-to-service-20260822
namespace: kube-system
spec:
backoffLimit: 2
template:
spec:
serviceAccountName: node-prefer-noschedule
restartPolicy: Never
containers:
- name: uncordon
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
command:
- /usr/bin/env
- bash
- -ceu
- |
kubectl uncordon titan-22
kubectl label node titan-22 \
atlas.bstein.dev/general-compute=last-resort \
--overwrite=true
kubectl annotate node titan-22 \
atlas.bstein.dev/returned-to-service-at=2026-08-22T19:20:00-05:00 \
--overwrite=true

2
scripts/ops/cluster_power_recovery.sh
View File

@ -182,7 +182,7 @@ LONGHORN_MANAGER_IMAGE="${LONGHORN_MANAGER_IMAGE:-registry.bstein.dev/infra/long
LONGHORN_MANAGER_CACHE_BUNDLE_FILE="${LONGHORN_MANAGER_CACHE_BUNDLE_FILE:-${STATE_ROOT}/bundles/longhorn-manager-v1.8.2-${BOOTSTRAP_BUNDLE_ARCH:-arm64}.tar}"
LONGHORN_UNLOCK_SSH_KNOWN_HOSTS="${LONGHORN_UNLOCK_SSH_KNOWN_HOSTS:-/tmp/ananke_longhorn_unlock_known_hosts}"
BOOTSTRAP_BUNDLE_ARCH="${BOOTSTRAP_BUNDLE_ARCH:-arm64}"
RECOVERY_UNCORDON_DENYLIST="${RECOVERY_UNCORDON_DENYLIST:-titan-18,titan-22,titan-24}"
RECOVERY_UNCORDON_DENYLIST="${RECOVERY_UNCORDON_DENYLIST:-titan-18,titan-24}"
STALE_TERMINATING_POD_SECONDS="${STALE_TERMINATING_POD_SECONDS:-300}"
RECOVERY_NODE_RUNTIME_RESTART_ENABLED="${RECOVERY_NODE_RUNTIME_RESTART_ENABLED:-1}"
RECOVERY_NODE_RUNTIME_RESTART_DENYLIST="${RECOVERY_NODE_RUNTIME_RESTART_DENYLIST:-${RECOVERY_UNCORDON_DENYLIST}}"

1
services/maintenance/node-ops/kustomization.yaml
View File

@ -17,6 +17,7 @@ resources:
- k3s-agent-restart-daemonset.yaml
- titan-24-docker-daemonset.yaml
- titan-22-link-keeper-daemonset.yaml
- titan-22-gpu-canary-job.yaml
- node-image-sweeper-serviceaccount.yaml
- node-image-sweeper-daemonset.yaml
- image-sweeper-cronjob.yaml

75
services/maintenance/node-ops/titan-22-gpu-canary-job.yaml Normal file
View File

@ -0,0 +1,75 @@
# services/maintenance/node-ops/titan-22-gpu-canary-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: titan-22-gpu-canary-20260822
namespace: maintenance
spec:
backoffLimit: 1
template:
spec:
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: titan-22
containers:
- name: cuda-canary
image: docker.io/nvidia/cuda@sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -ceu
- |
cat >/tmp/titan_22_gpu_canary.cu <<'CUDA'
#include <cuda_runtime.h>
#include <stdio.h>
__global__ void fill(float *values, int count) {
int index = blockIdx.x * blockDim.x + threadIdx.x;
if (index < count) {
values[index] = 22.0f;
}
}
int main(void) {
int devices = 0;
cudaError_t result = cudaGetDeviceCount(&devices);
if (result != cudaSuccess || devices < 1) {
fprintf(stderr, "cudaGetDeviceCount failed: %s\n", cudaGetErrorString(result));
return 1;
}
const int count = 16 * 1024 * 1024;
float *values = NULL;
result = cudaMalloc(&values, count * sizeof(float));
if (result != cudaSuccess) {
fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(result));
return 1;
}
fill<<<(count + 255) / 256, 256>>>(values, count);
result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
fprintf(stderr, "GPU kernel failed: %s\n", cudaGetErrorString(result));
cudaFree(values);
return 1;
}
float last = 0.0f;
result = cudaMemcpy(&last, values + count - 1, sizeof(float), cudaMemcpyDeviceToHost);
cudaFree(values);
if (result != cudaSuccess || last != 22.0f) {
fprintf(stderr, "GPU verification failed: %s value=%f\n", cudaGetErrorString(result), last);
return 1;
}
printf("CUDA canary passed: devices=%d value=%.1f\n", devices, last);
return 0;
}
CUDA
nvcc -O2 /tmp/titan_22_gpu_canary.cu -o /tmp/titan_22_gpu_canary
/tmp/titan_22_gpu_canary
resources:
limits:
nvidia.com/gpu.shared: "1"
requests:
nvidia.com/gpu.shared: "1"

9
services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml
View File

@ -84,6 +84,15 @@ spec:
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends firmware-realtek
fi
stale_swap_uuid="ccab2cd7-43fe-480d-b0a6-e567e5d7310a"
if [ -f /swapfile ] && grep -q -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab; then
echo "removing stale swap UUID ${stale_swap_uuid} from /etc/fstab"
fstab_tmp="$(mktemp)"
grep -v -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab >"${fstab_tmp}"
install -m 0644 "${fstab_tmp}" /etc/fstab
rm -f "${fstab_tmp}"
fi
if nmcli connection show "Wired connection 1" >/dev/null 2>&1; then
nmcli connection modify "Wired connection 1" connection.autoconnect no || true
fi