diff --git a/infrastructure/core/kustomization.yaml b/infrastructure/core/kustomization.yaml index e79fa738..68972357 100644 --- a/infrastructure/core/kustomization.yaml +++ b/infrastructure/core/kustomization.yaml @@ -7,7 +7,6 @@ resources: - node-prefer-noschedule-serviceaccount.yaml - node-prefer-noschedule-rbac.yaml - node-prefer-noschedule-cronjob.yaml - - titan-22-return-to-service-job.yaml - coredns-custom.yaml - coredns-deployment.yaml - ntp-sync-daemonset.yaml diff --git a/infrastructure/core/titan-22-return-to-service-job.yaml b/infrastructure/core/titan-22-return-to-service-job.yaml deleted file mode 100644 index 171773d4..00000000 --- a/infrastructure/core/titan-22-return-to-service-job.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# infrastructure/core/titan-22-return-to-service-job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: titan-22-return-to-service-20260822 - namespace: kube-system -spec: - backoffLimit: 2 - template: - spec: - serviceAccountName: node-prefer-noschedule - restartPolicy: Never - containers: - - name: uncordon - image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 - command: - - /usr/bin/env - - bash - - -ceu - - | - kubectl uncordon titan-22 - kubectl label node titan-22 \ - atlas.bstein.dev/general-compute=last-resort \ - --overwrite=true - kubectl annotate node titan-22 \ - atlas.bstein.dev/returned-to-service-at=2026-08-22T19:20:00-05:00 \ - --overwrite=true diff --git a/services/maintenance/node-ops/kustomization.yaml b/services/maintenance/node-ops/kustomization.yaml index 2596721d..0ac0581f 100644 --- a/services/maintenance/node-ops/kustomization.yaml +++ b/services/maintenance/node-ops/kustomization.yaml @@ -17,7 +17,6 @@ resources: - k3s-agent-restart-daemonset.yaml - titan-24-docker-daemonset.yaml - titan-22-link-keeper-daemonset.yaml - - titan-22-gpu-canary-job.yaml - node-image-sweeper-serviceaccount.yaml - node-image-sweeper-daemonset.yaml - image-sweeper-cronjob.yaml diff --git a/services/maintenance/node-ops/titan-22-gpu-canary-job.yaml b/services/maintenance/node-ops/titan-22-gpu-canary-job.yaml deleted file mode 100644 index 9b690dc8..00000000 --- a/services/maintenance/node-ops/titan-22-gpu-canary-job.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# services/maintenance/node-ops/titan-22-gpu-canary-job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: titan-22-gpu-canary-20260822 - namespace: maintenance -spec: - backoffLimit: 1 - template: - spec: - restartPolicy: Never - nodeSelector: - kubernetes.io/hostname: titan-22 - containers: - - name: cuda-canary - image: docker.io/nvidia/cuda@sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5 - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - -ceu - - | - cat >/tmp/titan_22_gpu_canary.cu <<'CUDA' - #include - #include - - __global__ void fill(float *values, int count) { - int index = blockIdx.x * blockDim.x + threadIdx.x; - if (index < count) { - values[index] = 22.0f; - } - } - - int main(void) { - int devices = 0; - cudaError_t result = cudaGetDeviceCount(&devices); - if (result != cudaSuccess || devices < 1) { - fprintf(stderr, "cudaGetDeviceCount failed: %s\n", cudaGetErrorString(result)); - return 1; - } - - const int count = 16 * 1024 * 1024; - float *values = NULL; - result = cudaMalloc(&values, count * sizeof(float)); - if (result != cudaSuccess) { - fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(result)); - return 1; - } - - fill<<<(count + 255) / 256, 256>>>(values, count); - result = cudaDeviceSynchronize(); - if (result != cudaSuccess) { - fprintf(stderr, "GPU kernel failed: %s\n", cudaGetErrorString(result)); - cudaFree(values); - return 1; - } - - float last = 0.0f; - result = cudaMemcpy(&last, values + count - 1, sizeof(float), cudaMemcpyDeviceToHost); - cudaFree(values); - if (result != cudaSuccess || last != 22.0f) { - fprintf(stderr, "GPU verification failed: %s value=%f\n", cudaGetErrorString(result), last); - return 1; - } - - printf("CUDA canary passed: devices=%d value=%.1f\n", devices, last); - return 0; - } - CUDA - nvcc -O2 /tmp/titan_22_gpu_canary.cu -o /tmp/titan_22_gpu_canary - /tmp/titan_22_gpu_canary - resources: - limits: - nvidia.com/gpu.shared: "1" - requests: - nvidia.com/gpu.shared: "1"