ff2c003b4b
Repoint the hermes-agent base FROM at the upstream multi-arch OCI INDEX digest (tag v2026.7.7.2, revision 9de9c25f) whose arm64 leaf is byte-for-byte the previously pinned single-arch base, so the arm64 build is unchanged while the same reviewed version now also resolves an amd64 leaf. Kaniko selects the matching leaf per build platform. Rework the release pipeline to build both arches natively and promote a multi-arch image without switching off kaniko or weakening any existing security assertion: - Keep the arm64 kaniko leg on the unchanged rpi5 coordinating pod; it now pushes an arch-suffixed candidate tag (...-build-<N>-arm64). - Add a second native amd64 kaniko leg on a titan-24-pinned, tolerating, resource-capped pod (ceiling strictly below the arm64 leg) that independently re-verifies the reviewed revision and stashes its leaf evidence (...-build-<N>-amd64). - Add ci/scripts/hermes_multiarch_combine.py: a pure-python, fail-closed combiner that re-reads each per-arch leaf from the registry, proves its digest AND its config architecture, assembles a Docker manifest LIST (already inside the promote allow-list), refuses to overwrite an existing final tag, publishes the arch-less ...-build-<N> tag, and re-verifies the registry resolved the exact index referencing exactly the two leaves. It emits the index digest in the SAME digest-file/image-file format the single-arch step produced, so render/verify-evidence/hermes_oci_promote.py promote the INDEX with no change to those scripts. Tests: add test_hermes_multiarch_combine.py (full hash/verification chain); strengthen the image-builder suites for the two-arch topology (both kaniko legs carry the reviewed heredoc-compat build-arg; amd64 leg pinned+capped+ boundary-checked; combine stage wiring; expanded evidence archive) without weakening the arm64-leg assertions. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BvMSXH8VH2tMWXanb8SJdf
516 lines
19 KiB
Plaintext
516 lines
19 KiB
Plaintext
pipeline {
|
|
agent {
|
|
kubernetes {
|
|
defaultContainer 'python'
|
|
yaml """
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
labels:
|
|
atlas.bstein.dev/workload: hermes-agent-image-builder
|
|
spec:
|
|
serviceAccountName: hermes-image-builder
|
|
automountServiceAccountToken: false
|
|
enableServiceLinks: false
|
|
restartPolicy: Never
|
|
securityContext:
|
|
fsGroup: 1000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
nodeSelector:
|
|
kubernetes.io/arch: arm64
|
|
node-role.kubernetes.io/worker: "true"
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: kubernetes.io/hostname
|
|
operator: NotIn
|
|
values:
|
|
- titan-04
|
|
- titan-14
|
|
- titan-18
|
|
- titan-19
|
|
- titan-22
|
|
- titan-24
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
preference:
|
|
matchExpressions:
|
|
- key: hardware
|
|
operator: In
|
|
values:
|
|
- rpi5
|
|
imagePullSecrets:
|
|
- name: harbor-bstein-robot
|
|
containers:
|
|
- name: jnlp
|
|
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 512Mi
|
|
- name: python
|
|
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
|
command: ["sleep"]
|
|
args: ["99d"]
|
|
tty: true
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 250m
|
|
memory: 256Mi
|
|
- name: kaniko
|
|
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
|
command: ["/busybox/sh", "-c"]
|
|
args: ["/busybox/sleep 99d"]
|
|
tty: true
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
|
privileged: false
|
|
runAsUser: 0
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 250m
|
|
memory: 1Gi
|
|
limits:
|
|
cpu: "2"
|
|
memory: 4Gi
|
|
"""
|
|
}
|
|
}
|
|
parameters {
|
|
booleanParam(
|
|
name: 'PUBLISH_IMAGE',
|
|
defaultValue: false,
|
|
description: 'Publish the reviewed main revision to Harbor.'
|
|
)
|
|
string(
|
|
name: 'EXPECTED_SOURCE_REVISION',
|
|
defaultValue: '',
|
|
description: 'Full reviewed commit already contained by atlas/titan-iac main.'
|
|
)
|
|
string(
|
|
name: 'CONFIRM_PUBLISH',
|
|
defaultValue: '',
|
|
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
|
|
)
|
|
}
|
|
environment {
|
|
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
|
|
}
|
|
options {
|
|
disableConcurrentBuilds()
|
|
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
|
skipDefaultCheckout(true)
|
|
timeout(time: 150, unit: 'MINUTES')
|
|
}
|
|
stages {
|
|
stage('Checkout reviewed source') {
|
|
steps {
|
|
checkout scm
|
|
}
|
|
}
|
|
stage('Enforce release boundary') {
|
|
steps {
|
|
container('jnlp') {
|
|
sh '''
|
|
set -eu
|
|
mkdir -p build
|
|
test "${PUBLISH_IMAGE}" = "true"
|
|
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
|
case "${EXPECTED_SOURCE_REVISION}" in
|
|
*[!0-9a-f]*|'')
|
|
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
|
main_revision="$(git rev-parse origin/main)"
|
|
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
|
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
|
actual_revision="$(git rev-parse HEAD)"
|
|
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
|
test -z "$(git status --porcelain)"
|
|
test -f dockerfiles/Dockerfile.hermes-agent
|
|
case "${BUILD_NUMBER}" in
|
|
''|0*|*[!0-9]*)
|
|
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
printf '%s\n' \
|
|
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
|
> build/hermes-agent.destination
|
|
printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Validate reviewed release source') {
|
|
steps {
|
|
container('python') {
|
|
sh '''
|
|
set -eu
|
|
python3 -m pip install --disable-pip-version-check --no-cache-dir \
|
|
--target=/tmp/hermes-agent-release-test-deps \
|
|
pytest==8.3.4 PyYAML==6.0.2
|
|
PYTHONPATH=/tmp/hermes-agent-release-test-deps \
|
|
python3 -m pytest -q \
|
|
testing/tests/test_hermes_image_builder.py \
|
|
testing/tests/test_hermes_image_builder_adversarial.py \
|
|
testing/tests/test_hermes_image_builder_coverage.py \
|
|
testing/tests/test_hermes_image_builder_fresh_review.py \
|
|
testing/tests/test_hermes_oci_promote.py \
|
|
testing/tests/test_hermes_multiarch_combine.py \
|
|
testing/tests/test_hermes_image_automation.py
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Reject replay before publish') {
|
|
steps {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''
|
|
set -eu
|
|
set +x
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_image_release.py assert-absent \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}" \
|
|
--destination "${destination}"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Build arm64 leg without a daemon') {
|
|
steps {
|
|
container('kaniko') {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''#!/busybox/sh
|
|
set -eu
|
|
set +x
|
|
config_path=/kaniko/.docker/config.json
|
|
destination="$(cat build/hermes-agent.destination)-arm64"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
umask 077
|
|
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
|
/busybox/mkdir -p /kaniko/.docker
|
|
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
|
unset HARBOR_USER HARBOR_PASSWORD auth
|
|
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
|
umask 022
|
|
/kaniko/executor \
|
|
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
|
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
|
--context="dir://${WORKSPACE}" \
|
|
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
|
--destination="${destination}" \
|
|
--digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \
|
|
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \
|
|
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
|
|
--label="org.opencontainers.image.revision=${source_revision}" \
|
|
--cleanup \
|
|
--push-retry=3
|
|
/busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
}
|
|
stage('Build amd64 leg without a daemon') {
|
|
agent {
|
|
kubernetes {
|
|
yaml """
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
labels:
|
|
atlas.bstein.dev/workload: hermes-agent-image-builder-amd64
|
|
spec:
|
|
serviceAccountName: hermes-image-builder
|
|
automountServiceAccountToken: false
|
|
enableServiceLinks: false
|
|
restartPolicy: Never
|
|
securityContext:
|
|
fsGroup: 1000
|
|
fsGroupChangePolicy: OnRootMismatch
|
|
nodeSelector:
|
|
kubernetes.io/arch: amd64
|
|
kubernetes.io/hostname: titan-24
|
|
node-role.kubernetes.io/worker: "true"
|
|
tolerations:
|
|
# titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
|
|
# PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
|
|
# build lands, and rely on the tight resource caps below (not scheduling
|
|
# priority) to keep the disposable build from starving the validator.
|
|
- operator: Exists
|
|
imagePullSecrets:
|
|
- name: harbor-bstein-robot
|
|
containers:
|
|
- name: jnlp
|
|
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 250m
|
|
memory: 384Mi
|
|
- name: kaniko
|
|
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
|
command: ["/busybox/sh", "-c"]
|
|
args: ["/busybox/sleep 99d"]
|
|
tty: true
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
|
privileged: false
|
|
runAsUser: 0
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: "1500m"
|
|
memory: 3Gi
|
|
"""
|
|
}
|
|
}
|
|
steps {
|
|
container('jnlp') {
|
|
sh '''
|
|
set -eu
|
|
mkdir -p build
|
|
test "${PUBLISH_IMAGE}" = "true"
|
|
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
|
case "${EXPECTED_SOURCE_REVISION}" in
|
|
*[!0-9a-f]*|'')
|
|
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
|
main_revision="$(git rev-parse origin/main)"
|
|
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
|
|
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
|
|
actual_revision="$(git rev-parse HEAD)"
|
|
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
|
test -z "$(git status --porcelain)"
|
|
test -f dockerfiles/Dockerfile.hermes-agent
|
|
case "${BUILD_NUMBER}" in
|
|
''|0*|*[!0-9]*)
|
|
echo "BUILD_NUMBER must be a positive decimal integer" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
printf '%s\n' \
|
|
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
|
|
> build/hermes-agent.destination
|
|
printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
|
|
'''
|
|
}
|
|
container('kaniko') {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''#!/busybox/sh
|
|
set -eu
|
|
set +x
|
|
config_path=/kaniko/.docker/config.json
|
|
destination="$(cat build/hermes-agent.destination)-amd64"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
umask 077
|
|
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
|
/busybox/mkdir -p /kaniko/.docker
|
|
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
|
unset HARBOR_USER HARBOR_PASSWORD auth
|
|
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
|
umask 022
|
|
/kaniko/executor \
|
|
--registry-mirror=harbor-core.harbor.svc.cluster.local \
|
|
--insecure-registry=harbor-core.harbor.svc.cluster.local \
|
|
--context="dir://${WORKSPACE}" \
|
|
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
|
--destination="${destination}" \
|
|
--digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \
|
|
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \
|
|
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
|
|
--label="org.opencontainers.image.revision=${source_revision}" \
|
|
--cleanup \
|
|
--push-retry=3
|
|
/busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image
|
|
'''
|
|
}
|
|
}
|
|
stash(
|
|
name: 'hermes-agent-amd64-evidence',
|
|
includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image'
|
|
)
|
|
}
|
|
}
|
|
stage('Combine multi-arch index') {
|
|
steps {
|
|
unstash 'hermes-agent-amd64-evidence'
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''
|
|
set -eu
|
|
set +x
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_multiarch_combine.py \
|
|
--destination "${destination}" \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}" \
|
|
--arm64-digest-file build/hermes-agent-arm64.digest \
|
|
--arm64-image-file build/hermes-agent-arm64.image \
|
|
--amd64-digest-file build/hermes-agent-amd64.digest \
|
|
--amd64-image-file build/hermes-agent-amd64.image \
|
|
--digest-file build/hermes-agent.digest \
|
|
--image-file build/hermes-agent.image
|
|
test -s build/hermes-agent.digest
|
|
test -s build/hermes-agent.image
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Render reviewed Flux handoff') {
|
|
steps {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''
|
|
set -eu
|
|
set +x
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_image_release.py render \
|
|
--digest-file build/hermes-agent.digest \
|
|
--image-file build/hermes-agent.image \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}" \
|
|
--destination "${destination}" \
|
|
--kustomization services/hermes/kustomization.yaml \
|
|
--output-dir build/hermes-agent-release
|
|
test -s build/hermes-agent-release/hermes-image-update.patch
|
|
test -s build/hermes-agent-release/hermes-agent-image.json
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Verify and archive release evidence') {
|
|
steps {
|
|
sh '''
|
|
set -eu
|
|
expected_files="$(printf '%s\n' \
|
|
build/hermes-agent.destination \
|
|
build/hermes-agent.digest \
|
|
build/hermes-agent.image \
|
|
build/hermes-agent.source-revision \
|
|
build/hermes-agent-arm64.digest \
|
|
build/hermes-agent-arm64.image \
|
|
build/hermes-agent-amd64.digest \
|
|
build/hermes-agent-amd64.image \
|
|
build/hermes-agent-release/hermes-agent-image.json \
|
|
build/hermes-agent-release/hermes-image-update.patch \
|
|
build/hermes-agent-release/hermes-kustomization.yaml \
|
|
| LC_ALL=C sort)"
|
|
actual_files="$(find build -type f -print | LC_ALL=C sort)"
|
|
test "${actual_files}" = "${expected_files}"
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_image_release.py verify-evidence \
|
|
--digest-file build/hermes-agent.digest \
|
|
--image-file build/hermes-agent.image \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}" \
|
|
--destination "${destination}" \
|
|
--kustomization services/hermes/kustomization.yaml \
|
|
--output-dir build/hermes-agent-release
|
|
'''
|
|
archiveArtifacts(
|
|
artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
|
|
allowEmptyArchive: false,
|
|
fingerprint: true
|
|
)
|
|
}
|
|
}
|
|
stage('Publish Flux release tag') {
|
|
steps {
|
|
withCredentials([usernamePassword(
|
|
credentialsId: 'harbor-robot',
|
|
usernameVariable: 'HARBOR_USER',
|
|
passwordVariable: 'HARBOR_PASSWORD'
|
|
)]) {
|
|
sh '''
|
|
set -eu
|
|
set +x
|
|
destination="$(cat build/hermes-agent.destination)"
|
|
source_revision="$(cat build/hermes-agent.source-revision)"
|
|
python3 ci/scripts/hermes_oci_promote.py \
|
|
--destination "${destination}" \
|
|
--digest-file build/hermes-agent.digest \
|
|
--source-revision "${source_revision}" \
|
|
--build-number "${BUILD_NUMBER}"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|