atlas-iac/services/jenkins/configmap-jcasc.yaml
Hermes Agent f997171b55 jenkins: rank the controller above its own build agents
The Jenkins controller and the ephemeral build agents it schedules both ran
at priority 0. Because the controller Deployment uses the Recreate strategy,
any JCasC change tears the controller down before its replacement is
scheduled, and while the eligible node pool is saturated that freed slot can
be taken by one of the controller's own pending agents.

That is what happened on 2026-08-23. Merging #48 changed
services/jenkins/configmap-jcasc.yaml, which rolled the hashed
jenkins-jcasc-revision ConfigMap and so rolled the Deployment. At 12:19:57Z
the old controller pod was deleted; at 12:19:59Z the build agent
typhon-133-1pmgb, pending since 11:42:15Z, took the freed capacity on
titan-11, and the new controller pod has been Pending ever since. The
scheduler cannot resolve it because every candidate victim is also priority
0: "No preemption victims found for incoming pod". The agent cannot exit on
its own either, since its jnlp container just loops on "Connection refused"
against the controller that cannot start. It is a self-deadlock, not a
transient capacity dip.

Give the controller a jenkins-core class (500) so it outranks its agents and
can preempt one when it has nowhere else to go, and mark the JCasC default
agent template jenkins-agent (50, preemptionPolicy: Never) so agents never
preempt anything themselves. Values follow the existing *-core/*-sim classes.

The controller-side class is what breaks the deadlock: agents declared inline
by a Jenkinsfile podTemplate, like typhon-133, do not inherit the JCasC
default template, so only the controller's own priority covers every agent.

PriorityClasses ship in infrastructure/modules/base via the core
Kustomization, so jenkins now dependsOn core, matching cassandra and veles.
Verified no dependency cycle: core itself has no dependsOn.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-23 13:12:40 +00:00

862 lines
32 KiB
YAML

# services/jenkins/configmap-jcasc.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: jenkins-jcasc
namespace: jenkins
data:
securityrealm.yaml: |
jenkins:
securityRealm:
oic:
clientId: "${OIDC_CLIENT_ID}"
clientSecret: "${OIDC_CLIENT_SECRET}"
serverConfiguration:
wellKnown:
wellKnownOpenIDConfigurationUrl: "${OIDC_ISSUER}/.well-known/openid-configuration"
scopesOverride: "openid profile email"
logoutFromOpenIdProvider: true
postLogoutRedirectUrl: "https://ci.bstein.dev"
sendScopesInTokenRequest: true
rootURLFromRequest: false
userNameField: "preferred_username"
fullNameFieldName: "name"
emailFieldName: "email"
groupsFieldName: "groups"
authorization.yaml: |
jenkins:
authorizationStrategy:
loggedInUsersCanDoAnything:
allowAnonymousRead: false
creds.yaml: |
credentials:
system:
domainCredentials:
- credentials:
- usernamePassword:
scope: GLOBAL
id: gitea-pat
username: "${GITEA_PAT_USERNAME}"
password: "${GITEA_PAT_TOKEN}"
description: "Gitea PAT for pipelines"
- usernamePassword:
scope: GLOBAL
id: harbor-robot
username: "${HARBOR_ROBOT_USERNAME}"
password: "${HARBOR_ROBOT_PASSWORD}"
description: "Harbor robot for pipelines"
- usernamePassword:
scope: GLOBAL
id: harbor-robot-streaming
username: "${HARBOR_STREAMING_ROBOT_USERNAME}"
password: "${HARBOR_STREAMING_ROBOT_PASSWORD}"
description: "Harbor robot for streaming pushes"
- string:
scope: GLOBAL
id: sonarqube-token
secret: "${SONARQUBE_TOKEN}"
description: "SonarQube token for quality-gate evidence collection"
jobs.yaml: |
jobs:
- script: |
pipelineJob('harbor-arm-build') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/bstein/harbor-arm-build.git')
credentials('gitea-pat')
}
branches('*/master')
}
}
}
}
}
pipelineJob('bstein-dev-home') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/2 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/bstein/bstein-dev-home.git')
credentials('gitea-pat')
}
branches('*/master')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('ariadne') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/2 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/ariadne.git')
credentials('gitea-pat')
}
branches('*/master')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('metis') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/metis.git')
credentials('gitea-pat')
}
branches('*/master')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('ananke') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/ananke.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('lesavka') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/bstein/lesavka.git')
credentials('gitea-pat')
}
branches('*/master')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('arcanagon') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/bstein/arcanagon.git')
credentials('gitea-pat')
}
branches('*/master')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('pegasus') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/pegasus.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('atlasbot') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/bstein/atlasbot.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('soteria') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/soteria.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('data-prepper') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/titan-iac.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('services/logging/Jenkinsfile.data-prepper')
}
}
}
pipelineJob('titan-iac') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H/12 * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/titan-iac.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('typhon') {
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
cron {
spec('H H * * *')
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/typhon.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('cassandra') {
disabled(false)
description('Staged Cassandra alpha image pipeline. Backend/frontend should build linux/amd64 and linux/arm64; sim-worker may begin amd64-only if Forge dependencies require it.')
properties {
pipelineTriggers {
triggers {
scmTrigger {
scmpoll_spec('H/5 * * * *')
ignorePostCommitHooks(false)
}
}
}
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/cassandra.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('Jenkinsfile')
}
}
}
multibranchPipelineJob('hermes-code-demo-branches') {
description('Branch validation for the Hermes code-repair demo. Builds hermes-repair/* proposal branches so a pull request carries a green test gate before a human merges it.')
branchSources {
branchSource {
source {
git {
id('hermes-code-demo-branches')
remote('https://scm.bstein.dev/atlas/hermes-code-demo.git')
credentialsId('gitea-pat')
traits {
gitBranchDiscovery()
}
}
}
}
}
factory {
workflowBranchProjectFactory {
scriptPath('Jenkinsfile')
}
}
orphanedItemStrategy {
discardOldItems {
numToKeep(20)
}
}
triggers {
periodicFolderTrigger {
interval('1m')
}
}
}
pipelineJob('hermes-code-demo') {
disabled(false)
description('Hermes code-repair demo: pytest gate over a small pricing helper. A seeded one-line defect fails the gate; Hermes proposes a minimal patch and Ariadne opens a pull request for human review.')
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/hermes-code-demo.git')
credentials('gitea-pat')
}
branches('*/master')
}
}
scriptPath('Jenkinsfile')
}
}
}
pipelineJob('hermes-triage-demo') {
disabled(false)
description('Hermes automated-triage demo: runs a deterministic fixture check as a Kubernetes Job in the hermes-triage-demo namespace. Build with SEED_FAILURE=true to arm the demo failure; the automated repair loop rebuilds with SEED_FAILURE=false.')
parameters {
booleanParam('SEED_FAILURE', false, 'Write "unhealthy" into the demo fixture before the check (arms the demo failure).')
}
definition {
cps {
sandbox(true)
script('''
podTemplate(
cloud: 'kubernetes',
yaml: """
apiVersion: v1
kind: Pod
spec:
serviceAccountName: jenkins
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
fsGroupChangePolicy: "OnRootMismatch"
nodeSelector:
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
containers:
- name: kubectl
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
command: ["sleep"]
args: ["3600"]
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 250m
memory: 128Mi
"""
) {
node(POD_LABEL) {
container('kubectl') {
stage('Fixture check') {
def buildNum = '' + env.BUILD_NUMBER
def incidentId = 'hermes-triage-demo/' + buildNum
def jobName = 'hermes-demo-test-' + buildNum
sh 'kubectl -n hermes-triage-demo get configmap hermes-triage-demo-fixture || kubectl -n hermes-triage-demo create configmap hermes-triage-demo-fixture --from-literal=state=healthy'
if (params.SEED_FAILURE) {
writeFile file: 'seed.json', text: '{"data":{"state":"unhealthy"}}'
sh 'kubectl -n hermes-triage-demo patch configmap hermes-triage-demo-fixture --type merge --patch-file seed.json'
echo 'Demo failure armed: fixture state set to unhealthy'
}
def manifest = [
'apiVersion: batch/v1',
'kind: Job',
'metadata:',
' name: ' + jobName,
' namespace: hermes-triage-demo',
' labels:',
' app.kubernetes.io/part-of: hermes-triage-demo',
' atlas.bstein.dev/role: demo-test-runner',
'spec:',
' backoffLimit: 0',
' ttlSecondsAfterFinished: 3600',
' template:',
' metadata:',
' labels:',
' app.kubernetes.io/part-of: hermes-triage-demo',
' atlas.bstein.dev/role: demo-test-runner',
' spec:',
' restartPolicy: Never',
' nodeSelector:',
' node-role.kubernetes.io/worker: "true"',
' containers:',
' - name: test-runner',
' image: busybox:1.37',
' env:',
' - name: INCIDENT_ID',
' value: "' + incidentId + '"',
' command:',
' - sh',
' - -c',
' - |',
' STATE=$(cat /fixture/state)',
' if [ "$STATE" = "healthy" ]; then',
' echo "{\\\\"event\\\\":\\\\"hermes_demo_test_pass\\\\",\\\\"incident_id\\\\":\\\\"$INCIDENT_ID\\\\",\\\\"message\\\\":\\\\"fixture state healthy\\\\"}"',
' exit 0',
' fi',
' echo "{\\\\"event\\\\":\\\\"hermes_demo_test_failure\\\\",\\\\"incident_id\\\\":\\\\"$INCIDENT_ID\\\\",\\\\"classification_hint\\\\":\\\\"demo_fixture_unhealthy\\\\",\\\\"message\\\\":\\\\"expected fixture state healthy; found $STATE\\\\"}"',
' exit 1',
' volumeMounts:',
' - name: fixture',
' mountPath: /fixture',
' readOnly: true',
' resources:',
' requests:',
' cpu: 25m',
' memory: 16Mi',
' limits:',
' cpu: 100m',
' memory: 32Mi',
' volumes:',
' - name: fixture',
' configMap:',
' name: hermes-triage-demo-fixture'
]
writeFile file: 'test-runner-job.yaml', text: manifest.join('\\n') + '\\n'
sh 'kubectl -n hermes-triage-demo create -f test-runner-job.yaml'
def verdict = 'unknown'
// Read the container's exit code, not the Job status.
// The kubelet records the exit code the moment the
// container stops; the Job controller can take minutes
// to reconcile .status.failed, and has been observed
// taking over half an hour on this cluster. Waiting on
// the Job made the build look hung long after the test
// had actually finished.
timeout(time: 3, unit: 'MINUTES') {
waitUntil {
def code = sh(script: 'kubectl -n hermes-triage-demo get pods -l job-name=' + jobName + ' -o jsonpath="{.items[0].status.containerStatuses[0].state.terminated.exitCode}" 2>/dev/null || true', returnStdout: true).trim()
if (code == '0') { verdict = 'pass'; return true }
if (code != '') { verdict = 'fail'; return true }
return false
}
}
def runnerLog = sh(script: 'kubectl -n hermes-triage-demo logs job/' + jobName + ' --tail=20 || true', returnStdout: true).trim()
echo 'test-runner output: ' + runnerLog
def junitBody
if (verdict == 'pass') {
junitBody = '<testsuite name="hermes-triage-demo" tests="1" failures="0"><testcase classname="hermes-triage-demo" name="fixture-state-check"/></testsuite>'
} else {
def safeLog = runnerLog.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
junitBody = '<testsuite name="hermes-triage-demo" tests="1" failures="1"><testcase classname="hermes-triage-demo" name="fixture-state-check"><failure message="demo fixture unhealthy">' + safeLog + '</failure></testcase></testsuite>'
}
writeFile file: 'demo-junit.xml', text: junitBody
archiveArtifacts artifacts: 'demo-junit.xml', fingerprint: false
if (verdict != 'pass') {
error('Demo fixture check failed for incident ' + incidentId)
}
}
}
}
}
''')
}
}
}
pipelineJob('hermes-agent-image') {
disabled(false)
description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.')
authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/titan-iac.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('ci/Jenkinsfile.hermes-agent-image')
}
}
}
pipelineJob('hermes-webui-image') {
disabled(false)
description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes an immutable Hermes WebUI image and archives a narrow two-workload Flux digest patch; it never mutates Git or deploys.')
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/titan-iac.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('ci/Jenkinsfile.hermes-webui-image')
}
}
}
multibranchPipelineJob('titan-iac-quality-gate') {
branchSources {
branchSource {
source {
git {
id('titan-iac-quality-gate')
remote('https://scm.bstein.dev/atlas/titan-iac.git')
credentialsId('gitea-pat')
traits {
gitBranchDiscovery()
}
}
}
}
}
factory {
workflowBranchProjectFactory {
scriptPath('ci/Jenkinsfile.titan-iac')
}
}
orphanedItemStrategy {
discardOldItems {
numToKeep(30)
}
}
triggers {
periodicFolderTrigger {
interval('12h')
}
}
configure { node ->
def webhookToken = System.getenv('TITAN_IAC_WEBHOOK_TOKEN') ?: ''
def triggers = node / 'triggers'
def webhook = triggers.appendNode('com.igalg.jenkins.plugins.mswt.trigger.ComputedFolderWebHookTrigger')
webhook.appendNode('token', webhookToken)
}
}
base.yaml: |
jenkins:
disableRememberMe: false
mode: NORMAL
numExecutors: 0
labelString: ""
projectNamingStrategy: "standard"
markupFormatter:
plainText
globalNodeProperties:
- envVars:
env:
- key: "GIT_CONFIG_COUNT"
value: "1"
- key: "GIT_CONFIG_KEY_0"
value: "safe.directory"
- key: "GIT_CONFIG_VALUE_0"
value: "*"
clouds:
- kubernetes:
containerCapStr: "5"
connectTimeout: "20"
readTimeout: "90"
jenkinsUrl: "http://jenkins.jenkins.svc.cluster.local:8080"
jenkinsTunnel: "jenkins.jenkins.svc.cluster.local:50000"
skipTlsVerify: false
maxRequestsPerHostStr: "32"
retentionTimeout: "5"
waitForPodSec: "600"
name: "kubernetes"
namespace: "jenkins"
restrictedPssSecurityContext: false
serverUrl: "https://kubernetes.default"
credentialsId: ""
podLabels:
- key: "jenkins/jenkins-jenkins-agent"
value: "true"
templates:
- name: "default"
namespace: "jenkins"
workspaceVolume:
dynamicPVC:
accessModes: "ReadWriteOnce"
requestsSize: "20Gi"
storageClassName: "astreae"
containers:
- name: "jnlp"
args: "^${computer.jnlpmac} ^${computer.name}"
envVars:
- envVar:
key: "JENKINS_URL"
value: "http://jenkins.jenkins.svc.cluster.local:8080/"
image: "jenkins/inbound-agent:3355.v388858a_47b_33-3"
privileged: "false"
resourceLimitCpu: 512m
resourceLimitMemory: 512Mi
resourceRequestCpu: 512m
resourceRequestMemory: 512Mi
ttyEnabled: false
workingDir: /home/jenkins/agent
idleMinutes: 0
instanceCap: 2147483647
label: "jenkins-jenkins-agent "
nodeUsageMode: "NORMAL"
podRetention: Never
serviceAccount: "jenkins"
slaveConnectTimeoutStr: "100"
yaml: |
spec:
priorityClassName: jenkins-agent
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
fsGroupChangePolicy: "OnRootMismatch"
nodeSelector:
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-06
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: atlas.bstein.dev/spillover
operator: DoesNotExist
- weight: 95
preference:
matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-19
- weight: 85
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
- weight: 45
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi4
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
jenkins/jenkins-jenkins-agent: "true"
yamlMergeStrategy: override
inheritYamlMergeStrategy: false
slaveAgentPort: 50000
crumbIssuer:
standard:
excludeClientIPFromCrumb: true
unclassified:
location:
url: "https://ci.bstein.dev/"