atlas-iac/ci/Jenkinsfile.hermes-webui-image

pipeline {
  agent {
    kubernetes {
      defaultContainer 'python'
      yaml """
apiVersion: v1
kind: Pod
metadata:
  labels:
    atlas.bstein.dev/workload: hermes-webui-image-builder
spec:
  serviceAccountName: hermes-image-builder
  automountServiceAccountToken: false
  enableServiceLinks: false
  restartPolicy: Never
  securityContext:
    fsGroup: 1000
    fsGroupChangePolicy: OnRootMismatch
  nodeSelector:
    kubernetes.io/arch: arm64
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: kubernetes.io/hostname
                operator: In
                # Keep disposable Kaniko expansion off the astreae Longhorn
                # replica nodes. titan-20 is the roomy ARM image builder and
                # has local NVMe for this transient I/O.
                values:
                  - titan-20
  imagePullSecrets:
    - name: harbor-bstein-robot
  containers:
    - name: jnlp
      image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 25m
          memory: 256Mi
        limits:
          cpu: 500m
          memory: 512Mi
    - name: python
      image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
      command: ["sleep"]
      args: ["99d"]
      tty: true
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
        runAsNonRoot: false
        runAsUser: 0
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 25m
          memory: 64Mi
        limits:
          cpu: 250m
          memory: 256Mi
    - name: kaniko
      image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
      command: ["/busybox/sh", "-c"]
      args: ["/busybox/sleep 99d"]
      tty: true
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
        privileged: false
        runAsUser: 0
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 250m
          memory: 1Gi
          ephemeral-storage: 10Gi
        limits:
          cpu: "2"
          memory: 4Gi
          ephemeral-storage: 20Gi
"""
    }
  }
  parameters {
    booleanParam(
      name: 'PUBLISH_IMAGE',
      defaultValue: false,
      description: 'Publish the reviewed main revision to Harbor.'
    )
    string(
      name: 'EXPECTED_SOURCE_REVISION',
      defaultValue: '',
      description: 'Full reviewed commit that must be contained by titan/atlas-iac main.'
    )
    string(
      name: 'CONFIRM_PUBLISH',
      defaultValue: '',
      description: 'Enter PUBLISH HERMES WEBUI to confirm the release.'
    )
  }
  environment {
    HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-webui'
  }
  options {
    disableConcurrentBuilds()
    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
    skipDefaultCheckout(true)
    timeout(time: 150, unit: 'MINUTES')
  }
  stages {
    stage('Checkout reviewed source') {
      steps {
        checkout scm
      }
    }
    stage('Enforce release boundary') {
      steps {
        container('jnlp') {
        sh '''
          set -eu
          mkdir -p build
          test "${PUBLISH_IMAGE}" = "true"
          test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"
          case "${EXPECTED_SOURCE_REVISION}" in
            *[!0-9a-f]*|'')
              echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
              exit 2
              ;;
          esac
          test "${#EXPECTED_SOURCE_REVISION}" -eq 40
          main_revision="$(git rev-parse HEAD)"
          test "${main_revision}" = "$(git rev-parse origin/main)"
          git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
          git checkout --detach "${EXPECTED_SOURCE_REVISION}"
          actual_revision="$(git rev-parse HEAD)"
          test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
          test -z "$(git status --porcelain)"
          test -f dockerfiles/Dockerfile.hermes-webui
          case "${BUILD_NUMBER}" in
            ''|0*|*[!0-9]*)
              echo "BUILD_NUMBER must be a positive decimal integer" >&2
              exit 2
              ;;
          esac
          printf '%s\n' \
            "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
            > build/hermes-webui.destination
          printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision
        '''
        }
      }
    }
    stage('Validate reviewed WebUI source') {
      steps {
        container('python') {
          sh '''
            set -eu
            export DEBIAN_FRONTEND=noninteractive
            apt-get update
            apt-get install -y --no-install-recommends ffmpeg nodejs
            rm -rf /var/lib/apt/lists/*
            command -v ffmpeg >/dev/null
            command -v node >/dev/null
            python3 -m pip install --disable-pip-version-check --no-cache-dir \
              --target=/tmp/hermes-webui-release-test-deps \
              pytest==8.3.4 PyYAML==6.0.2
            PYTHONPATH=/tmp/hermes-webui-release-test-deps \
              python3 -m pytest -q \
              testing/tests/test_hermes_chat_quality.py \
              testing/tests/test_hermes_handsfree_stt.py \
              testing/tests/test_hermes_voice_instrument.py \
              testing/tests/test_hermes_voice_full_duplex.py \
              testing/tests/test_hermes_voice_route_preflight.py \
              testing/tests/test_hermes_voice_preflight_delivery.py \
              testing/tests/test_hermes_thinking_voice_cues.py \
              testing/tests/test_hermes_voice_language_routing.py \
              testing/tests/test_hermes_webui_brand.py \
              testing/tests/test_hermes_webui_release.py \
              testing/tests/test_hermes_webui_hux_bff.py \
              testing/tests/test_hermes_webui_hux_context.py \
              testing/tests/test_hermes_webui_hux_integration.py \
              testing/tests/test_hermes_webui_hux_backend_e2e.py \
              testing/tests/test_hermes_hux_ui_runtime_wave_a.py \
              testing/tests/test_hermes_hux_runtime_autonomy_privacy.py \
              testing/tests/test_hermes_hux_runtime_stop.py \
              testing/tests/test_hermes_hux_ui_runtime_wave_b.py \
              testing/tests/test_hermes_hux_runtime_wave_c.py \
              testing/tests/test_hermes_hux_runtime_plugin.py \
              testing/tests/test_hermes_hux_runtime_vendor_parity.py \
              testing/tests/test_hermes_hux_delivery.py \
              testing/tests/test_hermes_oci_promote.py \
              testing/tests/test_hermes_multiarch_combine.py \
              testing/tests/test_hermes_image_automation.py
            HUX_BACKEND_TESTS="$(find testing/tests -maxdepth 1 -type f \
              -name 'test_hermes_hux_*.py' \
              ! -name '*_ui_*' \
              ! -name '*runtime*' \
              ! -name '*delivery*' \
              | sort)"
            test -n "${HUX_BACKEND_TESTS}"
            PYTHONPATH=/tmp/hermes-webui-release-test-deps \
              python3 -m pytest -q ${HUX_BACKEND_TESTS}
          '''
        }
      }
    }
    stage('Reject replay before publish') {
      steps {
        withCredentials([usernamePassword(
          credentialsId: 'harbor-robot',
          usernameVariable: 'HARBOR_USER',
          passwordVariable: 'HARBOR_PASSWORD'
        )]) {
          sh '''
            set -eu
            set +x
            destination="$(cat build/hermes-webui.destination)"
            source_revision="$(cat build/hermes-webui.source-revision)"
            python3 ci/scripts/hermes_webui_release.py assert-absent \
              --source-revision "${source_revision}" \
              --build-number "${BUILD_NUMBER}" \
              --destination "${destination}"
          '''
        }
      }
    }
    stage('Build arm64 leg without a daemon') {
      steps {
        container('kaniko') {
          withCredentials([usernamePassword(
            credentialsId: 'harbor-robot',
            usernameVariable: 'HARBOR_USER',
            passwordVariable: 'HARBOR_PASSWORD'
          )]) {
            sh '''#!/busybox/sh
              set -eu
              set +x
              config_path=/kaniko/.docker/config.json
              destination="$(cat build/hermes-webui.destination)-arm64"
              source_revision="$(cat build/hermes-webui.source-revision)"
              umask 077
              auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
              /busybox/mkdir -p /kaniko/.docker
              /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
              unset HARBOR_USER HARBOR_PASSWORD auth
              trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
              umask 022
              /kaniko/executor \
                --registry-mirror=harbor-core.harbor.svc.cluster.local \
                --insecure-registry=harbor-core.harbor.svc.cluster.local \
                --context="dir://${WORKSPACE}" \
                --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-webui" \
                --destination="${destination}" \
                --build-arg="HERMES_WEBUI_RELEASE_ID=git-${source_revision}-build-${BUILD_NUMBER}" \
                --digest-file="${WORKSPACE}/build/hermes-webui-arm64.digest" \
                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui-arm64.image" \
                --label="org.opencontainers.image.revision=${source_revision}" \
                --label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
                --label="org.opencontainers.image.title=hermes-webui" \
                --cleanup \
                --push-retry=3
              /busybox/chmod 644 build/hermes-webui-arm64.digest build/hermes-webui-arm64.image
            '''
          }
        }
      }
    }
    stage('Build amd64 leg without a daemon') {
      agent {
        kubernetes {
          yaml """
apiVersion: v1
kind: Pod
metadata:
  labels:
    atlas.bstein.dev/workload: hermes-webui-image-builder-amd64
spec:
  serviceAccountName: hermes-image-builder
  automountServiceAccountToken: false
  enableServiceLinks: false
  restartPolicy: Never
  securityContext:
    fsGroup: 1000
    fsGroupChangePolicy: OnRootMismatch
  # titan-24 is an accelerator node (not a general worker) that co-hosts the
  # out-of-cluster Sui validator. Pin the disposable amd64 build to it by
  # hostname + arch ONLY — do NOT require node-role worker, so titan-24 is never
  # opened to general cluster scheduling. The toleration + tight caps below keep
  # this off the validator's back.
  nodeSelector:
    kubernetes.io/arch: amd64
    kubernetes.io/hostname: titan-24
  tolerations:
    # titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
    # PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
    # build lands, and rely on the tight resource caps below (not scheduling
    # priority) to keep the disposable build from starving the validator.
    - operator: Exists
  imagePullSecrets:
    - name: harbor-bstein-robot
  containers:
    - name: jnlp
      image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 25m
          memory: 128Mi
        limits:
          cpu: 250m
          memory: 384Mi
    - name: kaniko
      image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
      command: ["/busybox/sh", "-c"]
      args: ["/busybox/sleep 99d"]
      tty: true
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
          add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
        privileged: false
        runAsUser: 0
        seccompProfile:
          type: RuntimeDefault
      resources:
        requests:
          cpu: 100m
          memory: 512Mi
          ephemeral-storage: 10Gi
        limits:
          cpu: "1500m"
          memory: 3Gi
          ephemeral-storage: 20Gi
"""
        }
      }
      steps {
        // This amd64 leg runs on its own fresh pod (titan-24), so it must check
        // out the SCM itself before the reviewed-revision git boundary check —
        // otherwise `git rev-parse origin/main` fails with "not a git repository".
        checkout scm
        container('jnlp') {
          sh '''
            set -eu
            mkdir -p build
            test "${PUBLISH_IMAGE}" = "true"
            test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"
            case "${EXPECTED_SOURCE_REVISION}" in
              *[!0-9a-f]*|'')
                echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
                exit 2
                ;;
            esac
            test "${#EXPECTED_SOURCE_REVISION}" -eq 40
            main_revision="$(git rev-parse HEAD)"
            test "${main_revision}" = "$(git rev-parse origin/main)"
            git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
            git checkout --detach "${EXPECTED_SOURCE_REVISION}"
            actual_revision="$(git rev-parse HEAD)"
            test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
            test -z "$(git status --porcelain)"
            test -f dockerfiles/Dockerfile.hermes-webui
            case "${BUILD_NUMBER}" in
              ''|0*|*[!0-9]*)
                echo "BUILD_NUMBER must be a positive decimal integer" >&2
                exit 2
                ;;
            esac
            printf '%s\n' \
              "${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
              > build/hermes-webui.destination
            printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision
          '''
        }
        container('kaniko') {
          withCredentials([usernamePassword(
            credentialsId: 'harbor-robot',
            usernameVariable: 'HARBOR_USER',
            passwordVariable: 'HARBOR_PASSWORD'
          )]) {
            sh '''#!/busybox/sh
              set -eu
              set +x
              config_path=/kaniko/.docker/config.json
              destination="$(cat build/hermes-webui.destination)-amd64"
              source_revision="$(cat build/hermes-webui.source-revision)"
              umask 077
              auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
              /busybox/mkdir -p /kaniko/.docker
              /busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
              unset HARBOR_USER HARBOR_PASSWORD auth
              trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
              umask 022
              /kaniko/executor \
                --registry-mirror=harbor-core.harbor.svc.cluster.local \
                --insecure-registry=harbor-core.harbor.svc.cluster.local \
                --context="dir://${WORKSPACE}" \
                --dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-webui" \
                --destination="${destination}" \
                --build-arg="HERMES_WEBUI_RELEASE_ID=git-${source_revision}-build-${BUILD_NUMBER}" \
                --digest-file="${WORKSPACE}/build/hermes-webui-amd64.digest" \
                --image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui-amd64.image" \
                --label="org.opencontainers.image.revision=${source_revision}" \
                --label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
                --label="org.opencontainers.image.title=hermes-webui" \
                --cleanup \
                --push-retry=3
              /busybox/chmod 644 build/hermes-webui-amd64.digest build/hermes-webui-amd64.image
            '''
          }
        }
        stash(
          name: 'hermes-webui-amd64-evidence',
          includes: 'build/hermes-webui-amd64.digest,build/hermes-webui-amd64.image'
        )
      }
    }
    stage('Combine multi-arch index') {
      steps {
        unstash 'hermes-webui-amd64-evidence'
        withCredentials([usernamePassword(
          credentialsId: 'harbor-robot',
          usernameVariable: 'HARBOR_USER',
          passwordVariable: 'HARBOR_PASSWORD'
        )]) {
          sh '''
            set -eu
            set +x
            destination="$(cat build/hermes-webui.destination)"
            source_revision="$(cat build/hermes-webui.source-revision)"
            python3 ci/scripts/hermes_multiarch_combine.py \
              --destination "${destination}" \
              --source-revision "${source_revision}" \
              --build-number "${BUILD_NUMBER}" \
              --arm64-digest-file build/hermes-webui-arm64.digest \
              --arm64-image-file build/hermes-webui-arm64.image \
              --amd64-digest-file build/hermes-webui-amd64.digest \
              --amd64-image-file build/hermes-webui-amd64.image \
              --digest-file build/hermes-webui.digest \
              --image-file build/hermes-webui.image
            test -s build/hermes-webui.digest
            test -s build/hermes-webui.image
          '''
        }
      }
    }
    stage('Render reviewed Flux handoff') {
      steps {
        withCredentials([usernamePassword(
          credentialsId: 'harbor-robot',
          usernameVariable: 'HARBOR_USER',
          passwordVariable: 'HARBOR_PASSWORD'
        )]) {
          sh '''
            set -eu
            set +x
            destination="$(cat build/hermes-webui.destination)"
            source_revision="$(cat build/hermes-webui.source-revision)"
            python3 ci/scripts/hermes_webui_release.py render \
              --digest-file build/hermes-webui.digest \
              --image-file build/hermes-webui.image \
              --source-revision "${source_revision}" \
              --build-number "${BUILD_NUMBER}" \
              --destination "${destination}" \
              --chat-manifest services/hermes/chat-statefulset.yaml \
              --dashboard-manifest services/hermes/deployment.yaml \
              --output-dir build/hermes-webui-release
            test -s build/hermes-webui-release/hermes-webui-image-update.patch
            test -s build/hermes-webui-release/hermes-webui-image.json
          '''
        }
      }
    }
    stage('Verify and archive release evidence') {
      steps {
      sh '''
        set -eu
        expected_files="$(printf '%s\n' \
          build/hermes-webui.destination \
          build/hermes-webui.digest \
          build/hermes-webui.image \
          build/hermes-webui.source-revision \
          build/hermes-webui-arm64.digest \
          build/hermes-webui-arm64.image \
          build/hermes-webui-amd64.digest \
          build/hermes-webui-amd64.image \
          build/hermes-webui-release/hermes-chat-statefulset.yaml \
          build/hermes-webui-release/hermes-dashboard-deployment.yaml \
          build/hermes-webui-release/hermes-webui-image.json \
          build/hermes-webui-release/hermes-webui-image-update.patch \
          | LC_ALL=C sort)"
        actual_files="$(find build -type f -print | LC_ALL=C sort)"
        test "${actual_files}" = "${expected_files}"
        destination="$(cat build/hermes-webui.destination)"
        source_revision="$(cat build/hermes-webui.source-revision)"
        python3 ci/scripts/hermes_webui_release.py verify-evidence \
          --digest-file build/hermes-webui.digest \
          --image-file build/hermes-webui.image \
          --source-revision "${source_revision}" \
          --build-number "${BUILD_NUMBER}" \
          --destination "${destination}" \
          --chat-manifest services/hermes/chat-statefulset.yaml \
          --dashboard-manifest services/hermes/deployment.yaml \
          --output-dir build/hermes-webui-release
      '''
      archiveArtifacts(
        artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui.source-revision,build/hermes-webui-arm64.digest,build/hermes-webui-arm64.image,build/hermes-webui-amd64.digest,build/hermes-webui-amd64.image,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch',
        allowEmptyArchive: false,
        fingerprint: true
      )
      }
    }
    stage('Publish Flux release tag') {
      steps {
        withCredentials([usernamePassword(
          credentialsId: 'harbor-robot',
          usernameVariable: 'HARBOR_USER',
          passwordVariable: 'HARBOR_PASSWORD'
        )]) {
          sh '''
            set -eu
            set +x
            destination="$(cat build/hermes-webui.destination)"
            source_revision="$(cat build/hermes-webui.source-revision)"
            python3 ci/scripts/hermes_oci_promote.py \
              --destination "${destination}" \
              --digest-file build/hermes-webui.digest \
              --source-revision "${source_revision}" \
              --build-number "${BUILD_NUMBER}"
          '''
        }
      }
    }
  }
}