ci: move IaC and Data Prepper scratch onto disposable volumes

Jenkinsfile

@@ -2,11 +2,16 @@
 pipeline {
   agent {
     kubernetes {
+      // Keep build I/O off the node's runtime USB drive.
+      workspaceVolume dynamicPVC(accessModes: 'ReadWriteOnce', requestsSize: '20Gi', storageClassName: 'ci-scratch')
       defaultContainer 'python'
       yaml """
 apiVersion: v1
 kind: Pod
 spec:
+  securityContext:
+    fsGroup: 1000
+    fsGroupChangePolicy: OnRootMismatch
   nodeSelector:
     kubernetes.io/arch: arm64
     node-role.kubernetes.io/worker: "true"
@@ -62,6 +67,10 @@ spec:
     }
   }
   environment {
+    PIP_CACHE_DIR = '/home/jenkins/agent/.cache/pip'
+    NPM_CONFIG_CACHE = '/home/jenkins/agent/.cache/npm'
+    SONAR_USER_HOME = '/home/jenkins/agent/.cache/sonar'
+    TMPDIR = '/home/jenkins/agent/.cache/tmp'
     PIP_DISABLE_PIP_VERSION_CHECK = '1'
     PYTHONUNBUFFERED = '1'
     SUITE_NAME = 'titan_iac'
@@ -83,6 +92,11 @@ spec:
     buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
   }
   stages {
+    stage('Prepare build scratch') {
+      steps {
+        sh 'mkdir -p /home/jenkins/agent/.cache/tmp /home/jenkins/agent/.cache/go-tmp'
+      }
+    }
     stage('Checkout') {
       steps {
         checkout scm

ci/Jenkinsfile.titan-iac

@@ -1,11 +1,16 @@
 pipeline {
   agent {
     kubernetes {
+      // Keep build I/O off the node's runtime USB drive.
+      workspaceVolume dynamicPVC(accessModes: 'ReadWriteOnce', requestsSize: '20Gi', storageClassName: 'ci-scratch')
       defaultContainer 'python'
       yaml """
 apiVersion: v1
 kind: Pod
 spec:
+  securityContext:
+    fsGroup: 1000
+    fsGroupChangePolicy: OnRootMismatch
   nodeSelector:
     kubernetes.io/arch: arm64
     node-role.kubernetes.io/worker: "true"
@@ -61,6 +66,10 @@ spec:
     }
   }
   environment {
+    PIP_CACHE_DIR = '/home/jenkins/agent/.cache/pip'
+    NPM_CONFIG_CACHE = '/home/jenkins/agent/.cache/npm'
+    SONAR_USER_HOME = '/home/jenkins/agent/.cache/sonar'
+    TMPDIR = '/home/jenkins/agent/.cache/tmp'
     PIP_DISABLE_PIP_VERSION_CHECK = '1'
     PYTHONUNBUFFERED = '1'
     SUITE_NAME = 'titan_iac'
@@ -82,6 +91,11 @@ spec:
     buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
   }
   stages {
+    stage('Prepare build scratch') {
+      steps {
+        sh 'mkdir -p /home/jenkins/agent/.cache/tmp /home/jenkins/agent/.cache/go-tmp'
+      }
+    }
     stage('Checkout') {
       steps {
         checkout scm

services/logging/Jenkinsfile.data-prepper

@@ -1,10 +1,15 @@
 pipeline {
   agent {
     kubernetes {
+      // Keep build I/O off the node's runtime USB drive.
+      workspaceVolume dynamicPVC(accessModes: 'ReadWriteOnce', requestsSize: '20Gi', storageClassName: 'ci-scratch')
       yaml """
 apiVersion: v1
 kind: Pod
 spec:
+  securityContext:
+    fsGroup: 1000
+    fsGroupChangePolicy: OnRootMismatch
   restartPolicy: Never
   serviceAccountName: jenkins
   nodeSelector:
@@ -83,6 +88,10 @@ spec:
     }
   }
   environment {
+    PIP_CACHE_DIR = '/home/jenkins/agent/.cache/pip'
+    NPM_CONFIG_CACHE = '/home/jenkins/agent/.cache/npm'
+    SONAR_USER_HOME = '/home/jenkins/agent/.cache/sonar'
+    TMPDIR = '/home/jenkins/agent/.cache/tmp'
     SUITE_NAME = 'data_prepper'
     PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
     SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
@@ -106,6 +115,11 @@ spec:
     buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
   }
   stages {
+    stage('Prepare build scratch') {
+      steps {
+        sh 'mkdir -p /home/jenkins/agent/.cache/tmp /home/jenkins/agent/.cache/go-tmp'
+      }
+    }
     stage('Checkout') {
       steps {
         container('git') {