node(titan-22): return worker to service

infrastructure/core/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
   - node-prefer-noschedule-serviceaccount.yaml
   - node-prefer-noschedule-rbac.yaml
   - node-prefer-noschedule-cronjob.yaml
+  - titan-22-return-to-service-job.yaml
   - coredns-custom.yaml
   - coredns-deployment.yaml
   - ntp-sync-daemonset.yaml

infrastructure/core/titan-22-return-to-service-job.yaml

@@ -0,0 +1,27 @@
+# infrastructure/core/titan-22-return-to-service-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: titan-22-return-to-service-20260822
+  namespace: kube-system
+spec:
+  backoffLimit: 2
+  template:
+    spec:
+      serviceAccountName: node-prefer-noschedule
+      restartPolicy: Never
+      containers:
+        - name: uncordon
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command:
+            - /usr/bin/env
+            - bash
+            - -ceu
+            - |
+              kubectl uncordon titan-22
+              kubectl label node titan-22 \
+                atlas.bstein.dev/general-compute=last-resort \
+                --overwrite=true
+              kubectl annotate node titan-22 \
+                atlas.bstein.dev/returned-to-service-at=2026-08-22T19:20:00-05:00 \
+                --overwrite=true

scripts/ops/cluster_power_recovery.sh

@@ -182,7 +182,7 @@ LONGHORN_MANAGER_IMAGE="${LONGHORN_MANAGER_IMAGE:-registry.bstein.dev/infra/long
 LONGHORN_MANAGER_CACHE_BUNDLE_FILE="${LONGHORN_MANAGER_CACHE_BUNDLE_FILE:-${STATE_ROOT}/bundles/longhorn-manager-v1.8.2-${BOOTSTRAP_BUNDLE_ARCH:-arm64}.tar}"
 LONGHORN_UNLOCK_SSH_KNOWN_HOSTS="${LONGHORN_UNLOCK_SSH_KNOWN_HOSTS:-/tmp/ananke_longhorn_unlock_known_hosts}"
 BOOTSTRAP_BUNDLE_ARCH="${BOOTSTRAP_BUNDLE_ARCH:-arm64}"
-RECOVERY_UNCORDON_DENYLIST="${RECOVERY_UNCORDON_DENYLIST:-titan-18,titan-22,titan-24}"
+RECOVERY_UNCORDON_DENYLIST="${RECOVERY_UNCORDON_DENYLIST:-titan-18,titan-24}"
 STALE_TERMINATING_POD_SECONDS="${STALE_TERMINATING_POD_SECONDS:-300}"
 RECOVERY_NODE_RUNTIME_RESTART_ENABLED="${RECOVERY_NODE_RUNTIME_RESTART_ENABLED:-1}"
 RECOVERY_NODE_RUNTIME_RESTART_DENYLIST="${RECOVERY_NODE_RUNTIME_RESTART_DENYLIST:-${RECOVERY_UNCORDON_DENYLIST}}"

services/maintenance/node-ops/kustomization.yaml

@@ -17,6 +17,7 @@ resources:
   - k3s-agent-restart-daemonset.yaml
   - titan-24-docker-daemonset.yaml
   - titan-22-link-keeper-daemonset.yaml
+  - titan-22-gpu-canary-job.yaml
   - node-image-sweeper-serviceaccount.yaml
   - node-image-sweeper-daemonset.yaml
   - image-sweeper-cronjob.yaml

services/maintenance/node-ops/titan-22-gpu-canary-job.yaml

@@ -0,0 +1,75 @@
+# services/maintenance/node-ops/titan-22-gpu-canary-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: titan-22-gpu-canary-20260822
+  namespace: maintenance
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: titan-22
+      containers:
+        - name: cuda-canary
+          image: docker.io/nvidia/cuda@sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5
+          imagePullPolicy: IfNotPresent
+          command:
+            - /bin/bash
+            - -ceu
+            - |
+              cat >/tmp/titan_22_gpu_canary.cu <<'CUDA'
+              #include <cuda_runtime.h>
+              #include <stdio.h>
+
+              __global__ void fill(float *values, int count) {
+                int index = blockIdx.x * blockDim.x + threadIdx.x;
+                if (index < count) {
+                  values[index] = 22.0f;
+                }
+              }
+
+              int main(void) {
+                int devices = 0;
+                cudaError_t result = cudaGetDeviceCount(&devices);
+                if (result != cudaSuccess || devices < 1) {
+                  fprintf(stderr, "cudaGetDeviceCount failed: %s\n", cudaGetErrorString(result));
+                  return 1;
+                }
+
+                const int count = 16 * 1024 * 1024;
+                float *values = NULL;
+                result = cudaMalloc(&values, count * sizeof(float));
+                if (result != cudaSuccess) {
+                  fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(result));
+                  return 1;
+                }
+
+                fill<<<(count + 255) / 256, 256>>>(values, count);
+                result = cudaDeviceSynchronize();
+                if (result != cudaSuccess) {
+                  fprintf(stderr, "GPU kernel failed: %s\n", cudaGetErrorString(result));
+                  cudaFree(values);
+                  return 1;
+                }
+
+                float last = 0.0f;
+                result = cudaMemcpy(&last, values + count - 1, sizeof(float), cudaMemcpyDeviceToHost);
+                cudaFree(values);
+                if (result != cudaSuccess || last != 22.0f) {
+                  fprintf(stderr, "GPU verification failed: %s value=%f\n", cudaGetErrorString(result), last);
+                  return 1;
+                }
+
+                printf("CUDA canary passed: devices=%d value=%.1f\n", devices, last);
+                return 0;
+              }
+              CUDA
+              nvcc -O2 /tmp/titan_22_gpu_canary.cu -o /tmp/titan_22_gpu_canary
+              /tmp/titan_22_gpu_canary
+          resources:
+            limits:
+              nvidia.com/gpu.shared: "1"
+            requests:
+              nvidia.com/gpu.shared: "1"

services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml

@@ -84,6 +84,15 @@ spec:
                     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends firmware-realtek
                   fi
 
+                  stale_swap_uuid="ccab2cd7-43fe-480d-b0a6-e567e5d7310a"
+                  if [ -f /swapfile ] && grep -q -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab; then
+                    echo "removing stale swap UUID ${stale_swap_uuid} from /etc/fstab"
+                    fstab_tmp="$(mktemp)"
+                    grep -v -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab >"${fstab_tmp}"
+                    install -m 0644 "${fstab_tmp}" /etc/fstab
+                    rm -f "${fstab_tmp}"
+                  fi
+
                   if nmcli connection show "Wired connection 1" >/dev/null 2>&1; then
                     nmcli connection modify "Wired connection 1" connection.autoconnect no || true
                   fi