titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

node(titan-22): remove completed canaries

Browse Source
This commit is contained in:
jenkins 2026-08-22 21:22:33 -03:00
parent 21264a71c8
commit 76fe550871
4 changed files with 0 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
infrastructure/core/kustomization.yaml
View File

@ -7,7 +7,6 @@ resources:
- node-prefer-noschedule-serviceaccount.yaml
- node-prefer-noschedule-rbac.yaml
- node-prefer-noschedule-cronjob.yaml
- titan-22-return-to-service-job.yaml
- coredns-custom.yaml
- coredns-deployment.yaml
- ntp-sync-daemonset.yaml

27
infrastructure/core/titan-22-return-to-service-job.yaml
View File

@ -1,27 +0,0 @@
# infrastructure/core/titan-22-return-to-service-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: titan-22-return-to-service-20260822
namespace: kube-system
spec:
backoffLimit: 2
template:
spec:
serviceAccountName: node-prefer-noschedule
restartPolicy: Never
containers:
- name: uncordon
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
command:
- /usr/bin/env
- bash
- -ceu
- |
kubectl uncordon titan-22
kubectl label node titan-22 \
atlas.bstein.dev/general-compute=last-resort \
--overwrite=true
kubectl annotate node titan-22 \
atlas.bstein.dev/returned-to-service-at=2026-08-22T19:20:00-05:00 \
--overwrite=true

1
services/maintenance/node-ops/kustomization.yaml
View File

@ -17,7 +17,6 @@ resources:
- k3s-agent-restart-daemonset.yaml
- titan-24-docker-daemonset.yaml
- titan-22-link-keeper-daemonset.yaml
- titan-22-gpu-canary-job.yaml
- node-image-sweeper-serviceaccount.yaml
- node-image-sweeper-daemonset.yaml
- image-sweeper-cronjob.yaml

75
services/maintenance/node-ops/titan-22-gpu-canary-job.yaml
View File

@ -1,75 +0,0 @@
# services/maintenance/node-ops/titan-22-gpu-canary-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: titan-22-gpu-canary-20260822
namespace: maintenance
spec:
backoffLimit: 1
template:
spec:
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: titan-22
containers:
- name: cuda-canary
image: docker.io/nvidia/cuda@sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -ceu
- |
cat >/tmp/titan_22_gpu_canary.cu <<'CUDA'
#include <cuda_runtime.h>
#include <stdio.h>
__global__ void fill(float *values, int count) {
int index = blockIdx.x * blockDim.x + threadIdx.x;
if (index < count) {
values[index] = 22.0f;
}
}
int main(void) {
int devices = 0;
cudaError_t result = cudaGetDeviceCount(&devices);
if (result != cudaSuccess || devices < 1) {
fprintf(stderr, "cudaGetDeviceCount failed: %s\n", cudaGetErrorString(result));
return 1;
}
const int count = 16 * 1024 * 1024;
float *values = NULL;
result = cudaMalloc(&values, count * sizeof(float));
if (result != cudaSuccess) {
fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(result));
return 1;
}
fill<<<(count + 255) / 256, 256>>>(values, count);
result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
fprintf(stderr, "GPU kernel failed: %s\n", cudaGetErrorString(result));
cudaFree(values);
return 1;
}
float last = 0.0f;
result = cudaMemcpy(&last, values + count - 1, sizeof(float), cudaMemcpyDeviceToHost);
cudaFree(values);
if (result != cudaSuccess || last != 22.0f) {
fprintf(stderr, "GPU verification failed: %s value=%f\n", cudaGetErrorString(result), last);
return 1;
}
printf("CUDA canary passed: devices=%d value=%.1f\n", devices, last);
return 0;
}
CUDA
nvcc -O2 /tmp/titan_22_gpu_canary.cu -o /tmp/titan_22_gpu_canary
/tmp/titan_22_gpu_canary
resources:
limits:
nvidia.com/gpu.shared: "1"
requests:
nvidia.com/gpu.shared: "1"