diff --git a/docs/CLUSTER_OPERATIONS.md b/docs/CLUSTER_OPERATIONS.md index 66d2eb53..6c76cc49 100644 --- a/docs/CLUSTER_OPERATIONS.md +++ b/docs/CLUSTER_OPERATIONS.md @@ -164,6 +164,25 @@ requests before deciding that Jenkins is hung. Do not increase concurrency to solve an unschedulable agent. The tracked JCasC revision triggers a controller rollout, so quiet the queue and arrange active-build completion before editing. +Lesavka's Android runtime work uses a separate, single-allocation KVM resource on +Titan-22. `services/jenkins/lesavka-kvm-device-plugin.yaml` pins the +[upstream generic device plugin](https://github.com/squat/generic-device-plugin) +by AMD64 image digest and advertises only `lesavka.dev/kvm: 1`. Its only host +mounts are the kubelet plugin socket directory and read-only `/dev/kvm` discovery; +it has no API token, added capabilities, privileged mode or host network/PID. +Socket registration runs as UID 0; this is a narrow host integration, not an +unprivileged workload. Metrics bind to pod loopback. No SSH user/group membership, +Docker permissions, USB or other device access is changed. + +Workers must request this resource explicitly and separately bound memory, swap, +CPU, task count and temporary storage. The node's device group is GID 993 as +observed November 9; verify effective worker device access rather than assuming +registration proves an Android VM can run. The plugin alone starts no emulator. +Observe DaemonSet readiness, the node's allocatable resource and a bounded actual +KVM API probe before qualifying a worker. To retire the facility, first stop and +reap its owned workers, then remove this manifest from the Jenkins kustomization +and let Flux prune it. Leave unrelated plugins and node device permissions intact. + Custom pipelines explicitly use `workspaceVolume dynamicPVC(...)` with the `ci-scratch` StorageClass in `infrastructure/core/storageclass-ci-scratch.yaml`. Workspaces and compiler/package caches use that volume. Docker-in-Docker stores diff --git a/services/jenkins/kustomization.yaml b/services/jenkins/kustomization.yaml index 4e00efcb..9533ef79 100644 --- a/services/jenkins/kustomization.yaml +++ b/services/jenkins/kustomization.yaml @@ -19,6 +19,7 @@ resources: - service.yaml - ingress.yaml - lesavka-userns-probe.yaml + - lesavka-kvm-device-plugin.yaml configMapGenerator: - name: jenkins-jcasc-revision diff --git a/services/jenkins/lesavka-kvm-device-plugin.yaml b/services/jenkins/lesavka-kvm-device-plugin.yaml new file mode 100644 index 00000000..be4af736 --- /dev/null +++ b/services/jenkins/lesavka-kvm-device-plugin.yaml @@ -0,0 +1,60 @@ +# Explicit single-device allocation for the remote Android qualification worker. +# Upstream source: squat/generic-device-plugin f7c3291ebca3c00434ed1e624192f61fd21a59c9. +# The plugin only discovers /dev/kvm; workers must request lesavka.dev/kvm: 1. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: lesavka-kvm-device-plugin + namespace: jenkins + labels: + app.kubernetes.io/name: lesavka-kvm-device-plugin +spec: + selector: + matchLabels: + app.kubernetes.io/name: lesavka-kvm-device-plugin + template: + metadata: + labels: + app.kubernetes.io/name: lesavka-kvm-device-plugin + spec: + automountServiceAccountToken: false + enableServiceLinks: false + terminationGracePeriodSeconds: 15 + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/hostname: titan-22 + containers: + - name: device-plugin + image: docker.io/squat/generic-device-plugin@sha256:5f6c12bcd803ac7c60a0a63b8948bd78c6f42da20ad302573518c8b71d4a02f3 + args: + - --domain=lesavka.dev + - --listen=127.0.0.1:8080 + - --device={"name":"kvm","groups":[{"count":1,"paths":[{"path":"/dev/kvm"}]}]} + securityContext: + # Own only the kubelet plugin socket; no host root filesystem or API token. + runAsUser: 0 + runAsGroup: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + resources: + requests: {cpu: 25m, memory: 16Mi} + limits: {cpu: 100m, memory: 64Mi} + volumeMounts: + - name: plugin-sockets + mountPath: /var/lib/kubelet/device-plugins + - name: kvm + mountPath: /dev/kvm + readOnly: true + volumes: + - name: plugin-sockets + hostPath: + path: /var/lib/kubelet/device-plugins + type: Directory + - name: kvm + hostPath: + path: /dev/kvm + type: CharDevice