From 21264a71c881ce8da503dfb413f3427ca9639a04 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sat, 22 Aug 2026 21:21:03 -0300 Subject: [PATCH] node(titan-22): return worker to service --- infrastructure/core/kustomization.yaml | 1 + .../core/titan-22-return-to-service-job.yaml | 27 +++++++ scripts/ops/cluster_power_recovery.sh | 2 +- .../maintenance/node-ops/kustomization.yaml | 1 + .../node-ops/titan-22-gpu-canary-job.yaml | 75 +++++++++++++++++++ .../titan-22-link-keeper-daemonset.yaml | 9 +++ 6 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 infrastructure/core/titan-22-return-to-service-job.yaml create mode 100644 services/maintenance/node-ops/titan-22-gpu-canary-job.yaml diff --git a/infrastructure/core/kustomization.yaml b/infrastructure/core/kustomization.yaml index 68972357..e79fa738 100644 --- a/infrastructure/core/kustomization.yaml +++ b/infrastructure/core/kustomization.yaml @@ -7,6 +7,7 @@ resources: - node-prefer-noschedule-serviceaccount.yaml - node-prefer-noschedule-rbac.yaml - node-prefer-noschedule-cronjob.yaml + - titan-22-return-to-service-job.yaml - coredns-custom.yaml - coredns-deployment.yaml - ntp-sync-daemonset.yaml diff --git a/infrastructure/core/titan-22-return-to-service-job.yaml b/infrastructure/core/titan-22-return-to-service-job.yaml new file mode 100644 index 00000000..171773d4 --- /dev/null +++ b/infrastructure/core/titan-22-return-to-service-job.yaml @@ -0,0 +1,27 @@ +# infrastructure/core/titan-22-return-to-service-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: titan-22-return-to-service-20260822 + namespace: kube-system +spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: node-prefer-noschedule + restartPolicy: Never + containers: + - name: uncordon + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + command: + - /usr/bin/env + - bash + - -ceu + - | + kubectl uncordon titan-22 + kubectl label node titan-22 \ + atlas.bstein.dev/general-compute=last-resort \ + --overwrite=true + kubectl annotate node titan-22 \ + atlas.bstein.dev/returned-to-service-at=2026-08-22T19:20:00-05:00 \ + --overwrite=true diff --git a/scripts/ops/cluster_power_recovery.sh b/scripts/ops/cluster_power_recovery.sh index 8bcf8874..19fd489b 100755 --- a/scripts/ops/cluster_power_recovery.sh +++ b/scripts/ops/cluster_power_recovery.sh @@ -182,7 +182,7 @@ LONGHORN_MANAGER_IMAGE="${LONGHORN_MANAGER_IMAGE:-registry.bstein.dev/infra/long LONGHORN_MANAGER_CACHE_BUNDLE_FILE="${LONGHORN_MANAGER_CACHE_BUNDLE_FILE:-${STATE_ROOT}/bundles/longhorn-manager-v1.8.2-${BOOTSTRAP_BUNDLE_ARCH:-arm64}.tar}" LONGHORN_UNLOCK_SSH_KNOWN_HOSTS="${LONGHORN_UNLOCK_SSH_KNOWN_HOSTS:-/tmp/ananke_longhorn_unlock_known_hosts}" BOOTSTRAP_BUNDLE_ARCH="${BOOTSTRAP_BUNDLE_ARCH:-arm64}" -RECOVERY_UNCORDON_DENYLIST="${RECOVERY_UNCORDON_DENYLIST:-titan-18,titan-22,titan-24}" +RECOVERY_UNCORDON_DENYLIST="${RECOVERY_UNCORDON_DENYLIST:-titan-18,titan-24}" STALE_TERMINATING_POD_SECONDS="${STALE_TERMINATING_POD_SECONDS:-300}" RECOVERY_NODE_RUNTIME_RESTART_ENABLED="${RECOVERY_NODE_RUNTIME_RESTART_ENABLED:-1}" RECOVERY_NODE_RUNTIME_RESTART_DENYLIST="${RECOVERY_NODE_RUNTIME_RESTART_DENYLIST:-${RECOVERY_UNCORDON_DENYLIST}}" diff --git a/services/maintenance/node-ops/kustomization.yaml b/services/maintenance/node-ops/kustomization.yaml index 0ac0581f..2596721d 100644 --- a/services/maintenance/node-ops/kustomization.yaml +++ b/services/maintenance/node-ops/kustomization.yaml @@ -17,6 +17,7 @@ resources: - k3s-agent-restart-daemonset.yaml - titan-24-docker-daemonset.yaml - titan-22-link-keeper-daemonset.yaml + - titan-22-gpu-canary-job.yaml - node-image-sweeper-serviceaccount.yaml - node-image-sweeper-daemonset.yaml - image-sweeper-cronjob.yaml diff --git a/services/maintenance/node-ops/titan-22-gpu-canary-job.yaml b/services/maintenance/node-ops/titan-22-gpu-canary-job.yaml new file mode 100644 index 00000000..9b690dc8 --- /dev/null +++ b/services/maintenance/node-ops/titan-22-gpu-canary-job.yaml @@ -0,0 +1,75 @@ +# services/maintenance/node-ops/titan-22-gpu-canary-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: titan-22-gpu-canary-20260822 + namespace: maintenance +spec: + backoffLimit: 1 + template: + spec: + restartPolicy: Never + nodeSelector: + kubernetes.io/hostname: titan-22 + containers: + - name: cuda-canary + image: docker.io/nvidia/cuda@sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5 + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -ceu + - | + cat >/tmp/titan_22_gpu_canary.cu <<'CUDA' + #include + #include + + __global__ void fill(float *values, int count) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) { + values[index] = 22.0f; + } + } + + int main(void) { + int devices = 0; + cudaError_t result = cudaGetDeviceCount(&devices); + if (result != cudaSuccess || devices < 1) { + fprintf(stderr, "cudaGetDeviceCount failed: %s\n", cudaGetErrorString(result)); + return 1; + } + + const int count = 16 * 1024 * 1024; + float *values = NULL; + result = cudaMalloc(&values, count * sizeof(float)); + if (result != cudaSuccess) { + fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(result)); + return 1; + } + + fill<<<(count + 255) / 256, 256>>>(values, count); + result = cudaDeviceSynchronize(); + if (result != cudaSuccess) { + fprintf(stderr, "GPU kernel failed: %s\n", cudaGetErrorString(result)); + cudaFree(values); + return 1; + } + + float last = 0.0f; + result = cudaMemcpy(&last, values + count - 1, sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(values); + if (result != cudaSuccess || last != 22.0f) { + fprintf(stderr, "GPU verification failed: %s value=%f\n", cudaGetErrorString(result), last); + return 1; + } + + printf("CUDA canary passed: devices=%d value=%.1f\n", devices, last); + return 0; + } + CUDA + nvcc -O2 /tmp/titan_22_gpu_canary.cu -o /tmp/titan_22_gpu_canary + /tmp/titan_22_gpu_canary + resources: + limits: + nvidia.com/gpu.shared: "1" + requests: + nvidia.com/gpu.shared: "1" diff --git a/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml b/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml index 6a3b2dba..e956e7df 100644 --- a/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml +++ b/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml @@ -84,6 +84,15 @@ spec: DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends firmware-realtek fi + stale_swap_uuid="ccab2cd7-43fe-480d-b0a6-e567e5d7310a" + if [ -f /swapfile ] && grep -q -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab; then + echo "removing stale swap UUID ${stale_swap_uuid} from /etc/fstab" + fstab_tmp="$(mktemp)" + grep -v -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab >"${fstab_tmp}" + install -m 0644 "${fstab_tmp}" /etc/fstab + rm -f "${fstab_tmp}" + fi + if nmcli connection show "Wired connection 1" >/dev/null 2>&1; then nmcli connection modify "Wired connection 1" connection.autoconnect no || true fi