atlas-iac/ci/Jenkinsfile.hermes-agent-image

pipeline {
  agent {
    kubernetes {
      defaultContainer 'python'
      yaml """
apiVersion: v1
kind: Pod
metadata:
  labels:
    atlas.bstein.dev/workload: hermes-agent-image-builder
spec:
  serviceAccountName: hermes-image-builder
  automountServiceAccountToken: false
  enableServiceLinks: false
  restartPolicy: Never
  securityContext:
    fsGroup: 1000
    fsGroupChangePolicy: OnRootMismatch
  nodeSelector:
    kubernetes.io/arch: arm64
    node-role.kubernetes.io/worker: "true"
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: kubernetes.io/hostname
                operator: NotIn
                values:
                  - titan-04
                  - titan-08
                  - titan-14
                  - titan-18
                  - titan-19
                  - titan-22
                  - titan-24
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          preference:
            matchExpressions:
              - key: hardware
                operator: In
                values:
                  - rpi5
  imagePullSecrets:
    - name: harbor-bstein-robot
  containers:
    - name: jnlp
      image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 25m
          memory: 256Mi
        limits:
          cpu: 500m
          memory: 512Mi
    - name: python
      image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
      command: ["sleep"]
      args: ["99d"]
      tty: true
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 25m
          memory: 64Mi
        limits:
          cpu: 250m
          memory: 256Mi
    - name: kaniko
      image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
      command: ["/busybox/sh", "-c"]
      args: ["/busybox/sleep 99d"]
      tty: true
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
        privileged: false
        runAsUser: 0
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 250m
          memory: 1Gi
          # The agent image build expands the SQLite and web build layers in
          # its workspace. Reserve enough node-local space to keep it off the
          # 25 GiB workers, whose unrequested Kaniko workspace was evicted.
          ephemeral-storage: 32Gi
        limits:
          cpu: "2"
          memory: 4Gi
          ephemeral-storage: 40Gi
"""
    }
  }
  parameters {
    booleanParam(
      name: 'PUBLISH_IMAGE',
      defaultValue: false,
      description: 'Publish the reviewed main revision to Harbor.'
    )
    string(
      name: 'EXPECTED_SOURCE_REVISION',
      defaultValue: '',
      description: 'Full reviewed commit already contained by titan/atlas-iac main.'
    )
    string(
      name: 'CONFIRM_PUBLISH',
      defaultValue: '',
      description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
    )
  }
  environment {
    HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
  }
  options {
    disableConcurrentBuilds()
    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
    skipDefaultCheckout(true)
    timeout(time: 150, unit: 'MINUTES')
  }
  stages {
    stage('Checkout reviewed source') {
      steps {
        checkout scm
      }
    }
    stage('Enforce release boundary') {
      steps {
        container('jnlp') {
        sh '''
          set -eu
          mkdir -p build
          test "${PUBLISH_IMAGE}" = "true"
          test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
          case "${EXPECTED_SOURCE_REVISION}" in
            *[!0-9a-f]*|'')
              echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
              exit 2
              ;;
          esac
          test "${#EXPECTED_SOURCE_REVISION}" -eq 40
          main_revision="$(git rev-parse origin/main)"
          git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
          git checkout --detach "${EXPECTED_SOURCE_REVISION}"
          actual_revision="$(git rev-parse HEAD)"
          test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
          test -z "$(git status --porcelain)"
          test -f dockerfiles/Dockerfile.hermes-agent
          case "${BUILD_NUMBER}" in
            ''|0*|*[!0-9]*)
              echo "BUILD_NUMBER must be a positive decimal integer" >&2
              exit 2
              ;;
          esac
          printf '%s\n' \
            "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
            > build/hermes-agent.destination
          printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
        '''
        }
      }
    }
    stage('Validate reviewed release source') {
      steps {
        container('python') {
          sh '''
            set -eu
            # Install the pinned test deps fully offline from the reviewed,
            # in-repo wheelhouse (ci/vendor/hermes-agent-test-wheels). --no-index
            # forbids any network index, so this stage never resolves
            # pypi.org/files.pythonhosted.org and cannot fail on public-internet
            # DNS. The wheels match this pipeline's arm64 python:3.12 container
            # (PyYAML is the cp312 manylinux aarch64 build; the rest are
            # py3-none-any). Bump the wheelhouse when these pins change.
            python3 -m pip install --disable-pip-version-check --no-cache-dir \
              --no-index --find-links="${WORKSPACE}/ci/vendor/hermes-agent-test-wheels" \
              --target=/tmp/hermes-agent-release-test-deps \
              pytest==8.3.4 PyYAML==6.0.2
            PYTHONPATH=/tmp/hermes-agent-release-test-deps \
              python3 -m pytest -q \
              testing/tests/test_hermes_image_builder.py \
              testing/tests/test_hermes_image_builder_adversarial.py \
              testing/tests/test_hermes_image_builder_coverage.py \
              testing/tests/test_hermes_image_builder_fresh_review.py \
              testing/tests/test_hermes_oci_promote.py \
              testing/tests/test_hermes_multiarch_combine.py \
              testing/tests/test_hermes_image_automation.py
          '''
        }
      }
    }
    stage('Reject replay before publish') {
      steps {
        withCredentials([usernamePassword(
          credentialsId: 'harbor-robot',
          usernameVariable: 'HARBOR_USER',
          passwordVariable: 'HARBOR_PASSWORD'
        )]) {
          sh '''
            set -eu
            set +x
            destination="$(cat build/hermes-agent.destination)"
            source_revision="$(cat build/hermes-agent.source-revision)"
            python3 ci/scripts/hermes_image_release.py assert-absent \
              --source-revision "${source_revision}" \
              --build-number "${BUILD_NUMBER}" \
              --destination "${destination}"
          '''
        }
      }
    }
    stage('Build arm64 leg without a daemon') {
      steps {
        container('kaniko') {
          sh '''#!/busybox/sh
            set -eu
            minimum_available_kib=16777216
            available_kib="$(/busybox/df -Pk / | /busybox/awk 'NR == 2 { print $4 }')"
            case "${available_kib}" in
              ''|*[!0-9]*)
                echo "cannot determine Kaniko ephemeral-storage availability" >&2
                exit 2
                ;;
            esac
            if [ "${available_kib}" -lt "${minimum_available_kib}" ]; then
              echo "Kaniko requires at least 16 GiB of free ephemeral storage" >&2
              exit 1
            fi
          '''
          withCredentials([usernamePassword(
            credentialsId: 'harbor-robot',
            usernameVariable: 'HARBOR_USER',
            passwordVariable: 'HARBOR_PASSWORD'
          )]) {
            sh '''#!/busybox/sh
              set -eu
              set +x
              config_path=/kaniko/.docker/config.json
              destination="$(cat build/hermes-agent.destination)-arm64"
              source_revision="$(cat build/hermes-agent.source-revision)"
              umask 077
              auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
              /busybox/mkdir -p /kaniko/.docker
              /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
              unset HARBOR_USER HARBOR_PASSWORD auth
              trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
              umask 022
              /kaniko/executor \
                --registry-mirror=harbor-core.harbor.svc.cluster.local \
                --insecure-registry=harbor-core.harbor.svc.cluster.local \
                --context="dir://${WORKSPACE}" \
                --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
                --destination="${destination}" \
                --digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \
                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \
                --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
                --label="org.opencontainers.image.revision=${source_revision}" \
                --cleanup \
                --push-retry=3
              /busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image
            '''
          }
        }
      }
    }
    stage('Build amd64 leg without a daemon') {
      agent {
        kubernetes {
          yaml """
apiVersion: v1
kind: Pod
metadata:
  labels:
    atlas.bstein.dev/workload: hermes-agent-image-builder-amd64
spec:
  serviceAccountName: hermes-image-builder
  automountServiceAccountToken: false
  enableServiceLinks: false
  restartPolicy: Never
  securityContext:
    fsGroup: 1000
    fsGroupChangePolicy: OnRootMismatch
  # titan-24 is an accelerator node (not a general worker) that co-hosts the
  # out-of-cluster Sui validator. Pin the disposable amd64 build to it by
  # hostname + arch ONLY — do NOT require node-role worker, so titan-24 is never
  # opened to general cluster scheduling. The toleration + tight caps below keep
  # this off the validator's back.
  nodeSelector:
    kubernetes.io/arch: amd64
    kubernetes.io/hostname: titan-24
  tolerations:
    # titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
    # PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
    # build lands, and rely on the tight resource caps below (not scheduling
    # priority) to keep the disposable build from starving the validator.
    - operator: Exists
  imagePullSecrets:
    - name: harbor-bstein-robot
  containers:
    - name: jnlp
      image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 25m
          memory: 128Mi
        limits:
          cpu: 250m
          memory: 384Mi
    - name: kaniko
      image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
      command: ["/busybox/sh", "-c"]
      args: ["/busybox/sleep 99d"]
      tty: true
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
        privileged: false
        runAsUser: 0
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 100m
          memory: 512Mi
        limits:
          cpu: "1500m"
          memory: 3Gi
"""
        }
      }
      steps {
        // This amd64 leg runs on its own fresh pod (titan-24), so it must check
        // out the SCM itself before the reviewed-revision git boundary check —
        // otherwise `git rev-parse origin/main` fails with "not a git repository".
        checkout scm
        container('jnlp') {
          sh '''
            set -eu
            mkdir -p build
            test "${PUBLISH_IMAGE}" = "true"
            test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
            case "${EXPECTED_SOURCE_REVISION}" in
              *[!0-9a-f]*|'')
                echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
                exit 2
                ;;
            esac
            test "${#EXPECTED_SOURCE_REVISION}" -eq 40
            main_revision="$(git rev-parse origin/main)"
            git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
            git checkout --detach "${EXPECTED_SOURCE_REVISION}"
            actual_revision="$(git rev-parse HEAD)"
            test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
            test -z "$(git status --porcelain)"
            test -f dockerfiles/Dockerfile.hermes-agent
            case "${BUILD_NUMBER}" in
              ''|0*|*[!0-9]*)
                echo "BUILD_NUMBER must be a positive decimal integer" >&2
                exit 2
                ;;
            esac
            printf '%s\n' \
              "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
              > build/hermes-agent.destination
            printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
          '''
        }
        container('kaniko') {
          withCredentials([usernamePassword(
            credentialsId: 'harbor-robot',
            usernameVariable: 'HARBOR_USER',
            passwordVariable: 'HARBOR_PASSWORD'
          )]) {
            sh '''#!/busybox/sh
              set -eu
              set +x
              config_path=/kaniko/.docker/config.json
              destination="$(cat build/hermes-agent.destination)-amd64"
              source_revision="$(cat build/hermes-agent.source-revision)"
              umask 077
              auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
              /busybox/mkdir -p /kaniko/.docker
              /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
              unset HARBOR_USER HARBOR_PASSWORD auth
              trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
              umask 022
              /kaniko/executor \
                --registry-mirror=harbor-core.harbor.svc.cluster.local \
                --insecure-registry=harbor-core.harbor.svc.cluster.local \
                --context="dir://${WORKSPACE}" \
                --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
                --destination="${destination}" \
                --digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \
                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \
                --build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
                --label="org.opencontainers.image.revision=${source_revision}" \
                --cleanup \
                --push-retry=3
              /busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image
            '''
          }
        }
        stash(
          name: 'hermes-agent-amd64-evidence',
          includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image'
        )
      }
    }
    stage('Combine multi-arch index') {
      steps {
        unstash 'hermes-agent-amd64-evidence'
        withCredentials([usernamePassword(
          credentialsId: 'harbor-robot',
          usernameVariable: 'HARBOR_USER',
          passwordVariable: 'HARBOR_PASSWORD'
        )]) {
          sh '''
            set -eu
            set +x
            destination="$(cat build/hermes-agent.destination)"
            source_revision="$(cat build/hermes-agent.source-revision)"
            python3 ci/scripts/hermes_multiarch_combine.py \
              --destination "${destination}" \
              --source-revision "${source_revision}" \
              --build-number "${BUILD_NUMBER}" \
              --arm64-digest-file build/hermes-agent-arm64.digest \
              --arm64-image-file build/hermes-agent-arm64.image \
              --amd64-digest-file build/hermes-agent-amd64.digest \
              --amd64-image-file build/hermes-agent-amd64.image \
              --digest-file build/hermes-agent.digest \
              --image-file build/hermes-agent.image
            test -s build/hermes-agent.digest
            test -s build/hermes-agent.image
          '''
        }
      }
    }
    stage('Render reviewed Flux handoff') {
      steps {
        withCredentials([usernamePassword(
          credentialsId: 'harbor-robot',
          usernameVariable: 'HARBOR_USER',
          passwordVariable: 'HARBOR_PASSWORD'
        )]) {
          sh '''
            set -eu
            set +x
            destination="$(cat build/hermes-agent.destination)"
            source_revision="$(cat build/hermes-agent.source-revision)"
            python3 ci/scripts/hermes_image_release.py render \
              --digest-file build/hermes-agent.digest \
              --image-file build/hermes-agent.image \
              --source-revision "${source_revision}" \
              --build-number "${BUILD_NUMBER}" \
              --destination "${destination}" \
              --kustomization services/hermes/kustomization.yaml \
              --output-dir build/hermes-agent-release
            test -s build/hermes-agent-release/hermes-image-update.patch
            test -s build/hermes-agent-release/hermes-agent-image.json
          '''
        }
      }
    }
    stage('Verify and archive release evidence') {
      steps {
      sh '''
        set -eu
        expected_files="$(printf '%s\n' \
          build/hermes-agent.destination \
          build/hermes-agent.digest \
          build/hermes-agent.image \
          build/hermes-agent.source-revision \
          build/hermes-agent-arm64.digest \
          build/hermes-agent-arm64.image \
          build/hermes-agent-amd64.digest \
          build/hermes-agent-amd64.image \
          build/hermes-agent-release/hermes-agent-image.json \
          build/hermes-agent-release/hermes-image-update.patch \
          build/hermes-agent-release/hermes-kustomization.yaml \
          | LC_ALL=C sort)"
        actual_files="$(find build -type f -print | LC_ALL=C sort)"
        test "${actual_files}" = "${expected_files}"
        destination="$(cat build/hermes-agent.destination)"
        source_revision="$(cat build/hermes-agent.source-revision)"
        python3 ci/scripts/hermes_image_release.py verify-evidence \
          --digest-file build/hermes-agent.digest \
          --image-file build/hermes-agent.image \
          --source-revision "${source_revision}" \
          --build-number "${BUILD_NUMBER}" \
          --destination "${destination}" \
          --kustomization services/hermes/kustomization.yaml \
          --output-dir build/hermes-agent-release
      '''
      archiveArtifacts(
        artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
        allowEmptyArchive: false,
        fingerprint: true
      )
      }
    }
    stage('Publish Flux release tag') {
      steps {
        withCredentials([usernamePassword(
          credentialsId: 'harbor-robot',
          usernameVariable: 'HARBOR_USER',
          passwordVariable: 'HARBOR_PASSWORD'
        )]) {
          sh '''
            set -eu
            set +x
            destination="$(cat build/hermes-agent.destination)"
            source_revision="$(cat build/hermes-agent.source-revision)"
            python3 ci/scripts/hermes_oci_promote.py \
              --destination "${destination}" \
              --digest-file build/hermes-agent.digest \
              --source-revision "${source_revision}" \
              --build-number "${BUILD_NUMBER}"
          '''
        }
      }
    }
  }
}