node(titan-22): remove completed canaries
This commit is contained in:
parent
21264a71c8
commit
76fe550871
@ -7,7 +7,6 @@ resources:
|
|||||||
- node-prefer-noschedule-serviceaccount.yaml
|
- node-prefer-noschedule-serviceaccount.yaml
|
||||||
- node-prefer-noschedule-rbac.yaml
|
- node-prefer-noschedule-rbac.yaml
|
||||||
- node-prefer-noschedule-cronjob.yaml
|
- node-prefer-noschedule-cronjob.yaml
|
||||||
- titan-22-return-to-service-job.yaml
|
|
||||||
- coredns-custom.yaml
|
- coredns-custom.yaml
|
||||||
- coredns-deployment.yaml
|
- coredns-deployment.yaml
|
||||||
- ntp-sync-daemonset.yaml
|
- ntp-sync-daemonset.yaml
|
||||||
|
|||||||
@ -1,27 +0,0 @@
|
|||||||
# infrastructure/core/titan-22-return-to-service-job.yaml
|
|
||||||
apiVersion: batch/v1
|
|
||||||
kind: Job
|
|
||||||
metadata:
|
|
||||||
name: titan-22-return-to-service-20260822
|
|
||||||
namespace: kube-system
|
|
||||||
spec:
|
|
||||||
backoffLimit: 2
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
serviceAccountName: node-prefer-noschedule
|
|
||||||
restartPolicy: Never
|
|
||||||
containers:
|
|
||||||
- name: uncordon
|
|
||||||
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
|
|
||||||
command:
|
|
||||||
- /usr/bin/env
|
|
||||||
- bash
|
|
||||||
- -ceu
|
|
||||||
- |
|
|
||||||
kubectl uncordon titan-22
|
|
||||||
kubectl label node titan-22 \
|
|
||||||
atlas.bstein.dev/general-compute=last-resort \
|
|
||||||
--overwrite=true
|
|
||||||
kubectl annotate node titan-22 \
|
|
||||||
atlas.bstein.dev/returned-to-service-at=2026-08-22T19:20:00-05:00 \
|
|
||||||
--overwrite=true
|
|
||||||
@ -17,7 +17,6 @@ resources:
|
|||||||
- k3s-agent-restart-daemonset.yaml
|
- k3s-agent-restart-daemonset.yaml
|
||||||
- titan-24-docker-daemonset.yaml
|
- titan-24-docker-daemonset.yaml
|
||||||
- titan-22-link-keeper-daemonset.yaml
|
- titan-22-link-keeper-daemonset.yaml
|
||||||
- titan-22-gpu-canary-job.yaml
|
|
||||||
- node-image-sweeper-serviceaccount.yaml
|
- node-image-sweeper-serviceaccount.yaml
|
||||||
- node-image-sweeper-daemonset.yaml
|
- node-image-sweeper-daemonset.yaml
|
||||||
- image-sweeper-cronjob.yaml
|
- image-sweeper-cronjob.yaml
|
||||||
|
|||||||
@ -1,75 +0,0 @@
|
|||||||
# services/maintenance/node-ops/titan-22-gpu-canary-job.yaml
|
|
||||||
apiVersion: batch/v1
|
|
||||||
kind: Job
|
|
||||||
metadata:
|
|
||||||
name: titan-22-gpu-canary-20260822
|
|
||||||
namespace: maintenance
|
|
||||||
spec:
|
|
||||||
backoffLimit: 1
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
restartPolicy: Never
|
|
||||||
nodeSelector:
|
|
||||||
kubernetes.io/hostname: titan-22
|
|
||||||
containers:
|
|
||||||
- name: cuda-canary
|
|
||||||
image: docker.io/nvidia/cuda@sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5
|
|
||||||
imagePullPolicy: IfNotPresent
|
|
||||||
command:
|
|
||||||
- /bin/bash
|
|
||||||
- -ceu
|
|
||||||
- |
|
|
||||||
cat >/tmp/titan_22_gpu_canary.cu <<'CUDA'
|
|
||||||
#include <cuda_runtime.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
|
|
||||||
__global__ void fill(float *values, int count) {
|
|
||||||
int index = blockIdx.x * blockDim.x + threadIdx.x;
|
|
||||||
if (index < count) {
|
|
||||||
values[index] = 22.0f;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(void) {
|
|
||||||
int devices = 0;
|
|
||||||
cudaError_t result = cudaGetDeviceCount(&devices);
|
|
||||||
if (result != cudaSuccess || devices < 1) {
|
|
||||||
fprintf(stderr, "cudaGetDeviceCount failed: %s\n", cudaGetErrorString(result));
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const int count = 16 * 1024 * 1024;
|
|
||||||
float *values = NULL;
|
|
||||||
result = cudaMalloc(&values, count * sizeof(float));
|
|
||||||
if (result != cudaSuccess) {
|
|
||||||
fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(result));
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
fill<<<(count + 255) / 256, 256>>>(values, count);
|
|
||||||
result = cudaDeviceSynchronize();
|
|
||||||
if (result != cudaSuccess) {
|
|
||||||
fprintf(stderr, "GPU kernel failed: %s\n", cudaGetErrorString(result));
|
|
||||||
cudaFree(values);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
float last = 0.0f;
|
|
||||||
result = cudaMemcpy(&last, values + count - 1, sizeof(float), cudaMemcpyDeviceToHost);
|
|
||||||
cudaFree(values);
|
|
||||||
if (result != cudaSuccess || last != 22.0f) {
|
|
||||||
fprintf(stderr, "GPU verification failed: %s value=%f\n", cudaGetErrorString(result), last);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
printf("CUDA canary passed: devices=%d value=%.1f\n", devices, last);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
CUDA
|
|
||||||
nvcc -O2 /tmp/titan_22_gpu_canary.cu -o /tmp/titan_22_gpu_canary
|
|
||||||
/tmp/titan_22_gpu_canary
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
nvidia.com/gpu.shared: "1"
|
|
||||||
requests:
|
|
||||||
nvidia.com/gpu.shared: "1"
|
|
||||||
Loading…
x
Reference in New Issue
Block a user