diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image index b034de5e..10acae62 100644 --- a/ci/Jenkinsfile.hermes-agent-image +++ b/ci/Jenkinsfile.hermes-agent-image @@ -100,9 +100,14 @@ spec: requests: cpu: 250m memory: 1Gi + # The agent image build expands the SQLite and web build layers in + # its workspace. Reserve enough node-local space to keep it off the + # 25 GiB workers, whose unrequested Kaniko workspace was evicted. + ephemeral-storage: 32Gi limits: cpu: "2" memory: 4Gi + ephemeral-storage: 40Gi """ } } @@ -226,6 +231,21 @@ spec: stage('Build arm64 leg without a daemon') { steps { container('kaniko') { + sh '''#!/busybox/sh + set -eu + minimum_available_kib=16777216 + available_kib="$(/busybox/df -Pk / | /busybox/awk 'NR == 2 { print $4 }')" + case "${available_kib}" in + ''|*[!0-9]*) + echo "cannot determine Kaniko ephemeral-storage availability" >&2 + exit 2 + ;; + esac + if [ "${available_kib}" -lt "${minimum_available_kib}" ]; then + echo "Kaniko requires at least 16 GiB of free ephemeral storage" >&2 + exit 1 + fi + ''' withCredentials([usernamePassword( credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USER', diff --git a/testing/tests/test_hermes_image_builder_fresh_review.py b/testing/tests/test_hermes_image_builder_fresh_review.py index 6df1a15c..e731482b 100644 --- a/testing/tests/test_hermes_image_builder_fresh_review.py +++ b/testing/tests/test_hermes_image_builder_fresh_review.py @@ -77,6 +77,25 @@ def test_builder_prefers_rpi5_with_healthy_arm64_worker_fallback() -> None: } +def test_builder_reserves_storage_before_kaniko_expands_the_workspace() -> None: + """The ARM build cannot be scheduled onto the 25 GiB worker disks.""" + containers = {item["name"]: item for item in _pod_spec()["containers"]} + resources = containers["kaniko"]["resources"] + assert resources["requests"]["ephemeral-storage"] == "32Gi" + assert resources["limits"]["ephemeral-storage"] == "40Gi" + source = PIPELINE.read_text(encoding="utf-8") + arm_build = source.split("stage('Build arm64 leg without a daemon')", 1)[1].split( + "stage('Build amd64 leg without a daemon')", 1 + )[0] + assert 'minimum_available_kib=16777216' in arm_build + assert '/busybox/df -Pk /' in arm_build + assert 'cannot determine Kaniko ephemeral-storage availability' in arm_build + assert 'Kaniko requires at least 16 GiB of free ephemeral storage' in arm_build + assert arm_build.index("minimum_available_kib=16777216") < arm_build.index( + "withCredentials([usernamePassword(" + ) + + @pytest.mark.parametrize( "unsupported", [