532 lines
20 KiB
Plaintext
532 lines
20 KiB
Plaintext
pipeline {
|
|
agent {
|
|
kubernetes {
|
|
defaultContainer 'python'
|
|
yaml """
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
labels:
|
|
atlas.bstein.dev/workload: hermes-agent-image-builder
|
|
spec:
|
|
serviceAccountName: hermes-image-builder
|
|
automountServiceAccountToken: false
|
|
enableServiceLinks: false
|
|
restartPolicy: Never
|
|
securityContext:
|
|
fsGroup: 1000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
nodeSelector:
|
|
kubernetes.io/arch: arm64
|
|
node-role.kubernetes.io/worker: "true"
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: kubernetes.io/hostname
|
|
operator: NotIn
|
|
values:
|
|
- titan-04
|
|
- titan-14
|
|
- titan-18
|
|
- titan-19
|
|
- titan-22
|
|
- titan-24
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
preference:
|
|
matchExpressions:
|
|
- key: hardware
|
|
operator: In
|
|
values:
|
|
- rpi5
|
|
imagePullSecrets:
|
|
- name: harbor-bstein-robot
|
|
containers:
|
|
- name: jnlp
|
|
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 512Mi
|
|
- name: python
|
|
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
|
command: ["sleep"]
|
|
args: ["99d"]
|
|
tty: true
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 250m
|
|
memory: 256Mi
|
|
- name: kaniko
|
|
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
|
command: ["/busybox/sh", "-c"]
|
|
args: ["/busybox/sleep 99d"]
|
|
tty: true
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
|
privileged: false
|
|
runAsUser: 0
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 250m
|
|
memory: 1Gi
|
|
limits:
|
|
cpu: "2"
|
|
memory: 4Gi
|
|
"""
|
|
}
|
|
}
|
|
parameters {
|
|
booleanParam(
|
|
name: 'PUBLISH_IMAGE',
|
|
defaultValue: false,
|
|
description: 'Publish the reviewed main revision to Harbor.'
|
|
)
|
|
string(
|
|
name: 'EXPECTED_SOURCE_REVISION',
|
|
defaultValue: '',
|
|
description: 'Full reviewed commit already contained by titan/atlas-iac main.'
|
|
)
|
|
string(
|
|
name: 'CONFIRM_PUBLISH',
|
|
defaultValue: '',
|
|
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
|
|
)
|
|
}
|
|
environment {
|
|
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
|
|
}
|
|
options {
|
|
disableConcurrentBuilds()
|
|
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
|
skipDefaultCheckout(true)
|
|
timeout(time: 150, unit: 'MINUTES')
|
|
}
|
|
stages {
|
|
stage('Checkout reviewed source') {
|
|
steps {
|
|
checkout scm
|
|
}
|
|
}
|
|
stage('Enforce release boundary') {
|
|
steps {
|
|
container('jnlp') {
|
|
sh '''
|
|
set -eu
|
|
mkdir -p build
|
|
test "${PUBLISH_IMAGE}" = "true"
|
|
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
|
case "${EXPECTED_SOURCE_REVISION}" in
|
|
*[!0-9a-f]*|'')
|
|
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
|
main_revision="$(git rev-parse origin/main)"
|
|
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
|
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
|
actual_revision="$(git rev-parse HEAD)"
|
|
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
|
test -z "$(git status --porcelain)"
|
|
test -f dockerfiles/Dockerfile.hermes-agent
|
|
case "${BUILD_NUMBER}" in
|
|
''|0*|*[!0-9]*)
|
|
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
printf '%s\n' \
|
|
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
|
> build/hermes-agent.destination
|
|
printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Validate reviewed release source') {
|
|
steps {
|
|
container('python') {
|
|
sh '''
|
|
set -eu
|
|
# Install the pinned test deps fully offline from the reviewed,
|
|
# in-repo wheelhouse (ci/vendor/hermes-agent-test-wheels). --no-index
|
|
# forbids any network index, so this stage never resolves
|
|
# pypi.org/files.pythonhosted.org and cannot fail on public-internet
|
|
# DNS. The wheels match this pipeline's arm64 python:3.12 container
|
|
# (PyYAML is the cp312 manylinux aarch64 build; the rest are
|
|
# py3-none-any). Bump the wheelhouse when these pins change.
|
|
python3 -m pip install --disable-pip-version-check --no-cache-dir \
|
|
--no-index --find-links="${WORKSPACE}/ci/vendor/hermes-agent-test-wheels" \
|
|
--target=/tmp/hermes-agent-release-test-deps \
|
|
pytest==8.3.4 PyYAML==6.0.2
|
|
PYTHONPATH=/tmp/hermes-agent-release-test-deps \
|
|
python3 -m pytest -q \
|
|
testing/tests/test_hermes_image_builder.py \
|
|
testing/tests/test_hermes_image_builder_adversarial.py \
|
|
testing/tests/test_hermes_image_builder_coverage.py \
|
|
testing/tests/test_hermes_image_builder_fresh_review.py \
|
|
testing/tests/test_hermes_oci_promote.py \
|
|
testing/tests/test_hermes_multiarch_combine.py \
|
|
testing/tests/test_hermes_image_automation.py
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Reject replay before publish') {
|
|
steps {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''
|
|
set -eu
|
|
set +x
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_image_release.py assert-absent \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}" \
|
|
--destination "${destination}"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Build arm64 leg without a daemon') {
|
|
steps {
|
|
container('kaniko') {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''#!/busybox/sh
|
|
set -eu
|
|
set +x
|
|
config_path=/kaniko/.docker/config.json
|
|
destination="$(cat build/hermes-agent.destination)-arm64"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
umask 077
|
|
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
|
/busybox/mkdir -p /kaniko/.docker
|
|
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
|
unset HARBOR_USER HARBOR_PASSWORD auth
|
|
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
|
umask 022
|
|
/kaniko/executor \
|
|
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
|
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
|
--context="dir://${WORKSPACE}" \
|
|
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
|
--destination="${destination}" \
|
|
--digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \
|
|
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \
|
|
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
|
|
--label="org.opencontainers.image.revision=${source_revision}" \
|
|
--cleanup \
|
|
--push-retry=3
|
|
/busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
}
|
|
stage('Build amd64 leg without a daemon') {
|
|
agent {
|
|
kubernetes {
|
|
yaml """
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
labels:
|
|
atlas.bstein.dev/workload: hermes-agent-image-builder-amd64
|
|
spec:
|
|
serviceAccountName: hermes-image-builder
|
|
automountServiceAccountToken: false
|
|
enableServiceLinks: false
|
|
restartPolicy: Never
|
|
securityContext:
|
|
fsGroup: 1000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
# titan-24 is an accelerator node (not a general worker) that co-hosts the
|
|
# out-of-cluster Sui validator. Pin the disposable amd64 build to it by
|
|
# hostname + arch ONLY — do NOT require node-role worker, so titan-24 is never
|
|
# opened to general cluster scheduling. The toleration + tight caps below keep
|
|
# this off the validator's back.
|
|
nodeSelector:
|
|
kubernetes.io/arch: amd64
|
|
kubernetes.io/hostname: titan-24
|
|
tolerations:
|
|
# titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
|
|
# PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
|
|
# build lands, and rely on the tight resource caps below (not scheduling
|
|
# priority) to keep the disposable build from starving the validator.
|
|
- operator: Exists
|
|
imagePullSecrets:
|
|
- name: harbor-bstein-robot
|
|
containers:
|
|
- name: jnlp
|
|
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 250m
|
|
memory: 384Mi
|
|
- name: kaniko
|
|
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
|
command: ["/busybox/sh", "-c"]
|
|
args: ["/busybox/sleep 99d"]
|
|
tty: true
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
|
privileged: false
|
|
runAsUser: 0
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: "1500m"
|
|
memory: 3Gi
|
|
"""
|
|
}
|
|
}
|
|
steps {
|
|
// This amd64 leg runs on its own fresh pod (titan-24), so it must check
|
|
// out the SCM itself before the reviewed-revision git boundary check —
|
|
// otherwise `git rev-parse origin/main` fails with "not a git repository".
|
|
checkout scm
|
|
container('jnlp') {
|
|
sh '''
|
|
set -eu
|
|
mkdir -p build
|
|
test "${PUBLISH_IMAGE}" = "true"
|
|
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
|
case "${EXPECTED_SOURCE_REVISION}" in
|
|
*[!0-9a-f]*|'')
|
|
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
|
main_revision="$(git rev-parse origin/main)"
|
|
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
|
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
|
actual_revision="$(git rev-parse HEAD)"
|
|
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
|
test -z "$(git status --porcelain)"
|
|
test -f dockerfiles/Dockerfile.hermes-agent
|
|
case "${BUILD_NUMBER}" in
|
|
''|0*|*[!0-9]*)
|
|
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
printf '%s\n' \
|
|
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
|
> build/hermes-agent.destination
|
|
printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
|
|
'''
|
|
}
|
|
container('kaniko') {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''#!/busybox/sh
|
|
set -eu
|
|
set +x
|
|
config_path=/kaniko/.docker/config.json
|
|
destination="$(cat build/hermes-agent.destination)-amd64"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
umask 077
|
|
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
|
/busybox/mkdir -p /kaniko/.docker
|
|
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
|
unset HARBOR_USER HARBOR_PASSWORD auth
|
|
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
|
umask 022
|
|
/kaniko/executor \
|
|
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
|
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
|
--context="dir://${WORKSPACE}" \
|
|
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
|
--destination="${destination}" \
|
|
--digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \
|
|
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \
|
|
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
|
|
--label="org.opencontainers.image.revision=${source_revision}" \
|
|
--cleanup \
|
|
--push-retry=3
|
|
/busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image
|
|
'''
|
|
}
|
|
}
|
|
stash(
|
|
name: 'hermes-agent-amd64-evidence',
|
|
includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image'
|
|
)
|
|
}
|
|
}
|
|
stage('Combine multi-arch index') {
|
|
steps {
|
|
unstash 'hermes-agent-amd64-evidence'
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''
|
|
set -eu
|
|
set +x
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_multiarch_combine.py \
|
|
--destination "${destination}" \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}" \
|
|
--arm64-digest-file build/hermes-agent-arm64.digest \
|
|
--arm64-image-file build/hermes-agent-arm64.image \
|
|
--amd64-digest-file build/hermes-agent-amd64.digest \
|
|
--amd64-image-file build/hermes-agent-amd64.image \
|
|
--digest-file build/hermes-agent.digest \
|
|
--image-file build/hermes-agent.image
|
|
test -s build/hermes-agent.digest
|
|
test -s build/hermes-agent.image
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Render reviewed Flux handoff') {
|
|
steps {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''
|
|
set -eu
|
|
set +x
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_image_release.py render \
|
|
--digest-file build/hermes-agent.digest \
|
|
--image-file build/hermes-agent.image \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}" \
|
|
--destination "${destination}" \
|
|
--kustomization services/hermes/kustomization.yaml \
|
|
--output-dir build/hermes-agent-release
|
|
test -s build/hermes-agent-release/hermes-image-update.patch
|
|
test -s build/hermes-agent-release/hermes-agent-image.json
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Verify and archive release evidence') {
|
|
steps {
|
|
sh '''
|
|
set -eu
|
|
expected_files="$(printf '%s\n' \
|
|
build/hermes-agent.destination \
|
|
build/hermes-agent.digest \
|
|
build/hermes-agent.image \
|
|
build/hermes-agent.source-revision \
|
|
build/hermes-agent-arm64.digest \
|
|
build/hermes-agent-arm64.image \
|
|
build/hermes-agent-amd64.digest \
|
|
build/hermes-agent-amd64.image \
|
|
build/hermes-agent-release/hermes-agent-image.json \
|
|
build/hermes-agent-release/hermes-image-update.patch \
|
|
build/hermes-agent-release/hermes-kustomization.yaml \
|
|
| LC_ALL=C sort)"
|
|
actual_files="$(find build -type f -print | LC_ALL=C sort)"
|
|
test "${actual_files}" = "${expected_files}"
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_image_release.py verify-evidence \
|
|
--digest-file build/hermes-agent.digest \
|
|
--image-file build/hermes-agent.image \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}" \
|
|
--destination "${destination}" \
|
|
--kustomization services/hermes/kustomization.yaml \
|
|
--output-dir build/hermes-agent-release
|
|
'''
|
|
archiveArtifacts(
|
|
artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
|
|
allowEmptyArchive: false,
|
|
fingerprint: true
|
|
)
|
|
}
|
|
}
|
|
stage('Publish Flux release tag') {
|
|
steps {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''
|
|
set -eu
|
|
set +x
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_oci_promote.py \
|
|
--destination "${destination}" \
|
|
--digest-file build/hermes-agent.digest \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|