hermes: add daemonless agent image release lane
This commit is contained in:
jenkins
parent
9bf1ab8a9c
commit
3cfde3a5a7
220
ci/Jenkinsfile.hermes-agent-image
Normal file
220
ci/Jenkinsfile.hermes-agent-image
Normal file
@ -0,0 +1,220 @@
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
defaultContainer 'python'
|
||||
yaml """
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
labels:
|
||||
atlas.bstein.dev/workload: hermes-agent-image-builder
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: arm64
|
||||
node-role.kubernetes.io/worker: "true"
|
||||
hardware: rpi5
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: NotIn
|
||||
values:
|
||||
- titan-04
|
||||
- titan-14
|
||||
- titan-18
|
||||
- titan-19
|
||||
- titan-22
|
||||
- titan-24
|
||||
imagePullSecrets:
|
||||
- name: harbor-bstein-robot
|
||||
containers:
|
||||
- name: jnlp
|
||||
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
- name: python
|
||||
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
|
||||
command: ["sleep"]
|
||||
args: ["99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
|
||||
command: ["/busybox/sh", "-c"]
|
||||
args: ["/busybox/sleep 99d"]
|
||||
tty: true
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
privileged: false
|
||||
runAsUser: 0
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 1Gi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
"""
|
||||
}
|
||||
}
|
||||
parameters {
|
||||
booleanParam(
|
||||
name: 'PUBLISH_IMAGE',
|
||||
defaultValue: false,
|
||||
description: 'Publish the reviewed main revision to Harbor.'
|
||||
)
|
||||
string(
|
||||
name: 'EXPECTED_SOURCE_REVISION',
|
||||
defaultValue: '',
|
||||
description: 'Exact 40-character commit on atlas/titan-iac main.'
|
||||
)
|
||||
string(
|
||||
name: 'CONFIRM_PUBLISH',
|
||||
defaultValue: '',
|
||||
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
|
||||
)
|
||||
}
|
||||
environment {
|
||||
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 90, unit: 'MINUTES')
|
||||
}
|
||||
stages {
|
||||
stage('Checkout reviewed source') {
|
||||
steps {
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Enforce release boundary') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
mkdir -p build
|
||||
test "${PUBLISH_IMAGE}" = "true"
|
||||
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
|
||||
case "${EXPECTED_SOURCE_REVISION}" in
|
||||
*[!0-9a-f]*|'')
|
||||
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
|
||||
actual_revision="$(git rev-parse HEAD)"
|
||||
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
|
||||
test "${actual_revision}" = "$(git rev-parse origin/main)"
|
||||
test -z "$(git status --porcelain)"
|
||||
test -f dockerfiles/Dockerfile.hermes-agent
|
||||
printf '%s\n' "${HERMES_IMAGE}:git-${actual_revision}" > build/hermes-agent.destination
|
||||
'''
|
||||
}
|
||||
}
|
||||
stage('Build and publish without a daemon') {
|
||||
steps {
|
||||
container('kaniko') {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'harbor-robot',
|
||||
usernameVariable: 'HARBOR_USER',
|
||||
passwordVariable: 'HARBOR_PASSWORD'
|
||||
)]) {
|
||||
sh '''#!/busybox/sh
|
||||
set -eu
|
||||
set +x
|
||||
config_path=/kaniko/.docker/config.json
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
umask 077
|
||||
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
|
||||
/busybox/mkdir -p /kaniko/.docker
|
||||
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
|
||||
unset HARBOR_USER HARBOR_PASSWORD auth
|
||||
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
|
||||
/kaniko/executor \
|
||||
--context="dir://${WORKSPACE}" \
|
||||
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
|
||||
--destination="${destination}" \
|
||||
--digest-file="${WORKSPACE}/build/hermes-agent.digest" \
|
||||
--image-name-with-digest-file="${WORKSPACE}/build/hermes-agent.image" \
|
||||
--label="org.opencontainers.image.revision=${EXPECTED_SOURCE_REVISION}" \
|
||||
--cleanup \
|
||||
--push-retry=3
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Render reviewed Flux handoff') {
|
||||
steps {
|
||||
sh '''
|
||||
set -eu
|
||||
destination="$(cat build/hermes-agent.destination)"
|
||||
python3 ci/scripts/hermes_image_release.py \
|
||||
--digest-file build/hermes-agent.digest \
|
||||
--source-revision "${EXPECTED_SOURCE_REVISION}" \
|
||||
--destination "${destination}" \
|
||||
--kustomization services/hermes/kustomization.yaml \
|
||||
--output-dir build/hermes-agent-release
|
||||
test -s build/hermes-agent-release/hermes-image-update.patch
|
||||
test -s build/hermes-agent-release/hermes-agent-image.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
archiveArtifacts(
|
||||
artifacts: 'build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.destination,build/hermes-agent-release/**',
|
||||
allowEmptyArchive: true,
|
||||
fingerprint: true
|
||||
)
|
||||
}
|
||||
cleanup {
|
||||
container('kaniko') {
|
||||
sh '''#!/busybox/sh
|
||||
/busybox/rm -f /kaniko/.docker/config.json
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
126
ci/scripts/hermes_image_release.py
Executable file
126
ci/scripts/hermes_image_release.py
Executable file
@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render reviewable Flux artifacts for a published Hermes agent image."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
|
||||
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
|
||||
|
||||
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
|
||||
"""Return a normalized value when it matches the release contract."""
|
||||
normalized = value.strip()
|
||||
if not pattern.fullmatch(normalized):
|
||||
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
|
||||
return normalized
|
||||
|
||||
|
||||
def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
|
||||
"""Replace exactly one matching Kustomize image digest without reformatting."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
lines = source.splitlines(keepends=True)
|
||||
matches: list[int] = []
|
||||
|
||||
for index, line in enumerate(lines):
|
||||
if line.strip() != f"- name: {image}":
|
||||
continue
|
||||
name_indent = len(line) - len(line.lstrip())
|
||||
for candidate_index in range(index + 1, len(lines)):
|
||||
candidate = lines[candidate_index]
|
||||
stripped = candidate.strip()
|
||||
candidate_indent = len(candidate) - len(candidate.lstrip())
|
||||
if stripped.startswith("- name:") and candidate_indent == name_indent:
|
||||
break
|
||||
if stripped.startswith("digest:") and candidate_indent > name_indent:
|
||||
matches.append(candidate_index)
|
||||
break
|
||||
|
||||
if len(matches) != 1:
|
||||
raise ValueError(
|
||||
f"expected exactly one digest for image {image!r}; found {len(matches)}"
|
||||
)
|
||||
|
||||
index = matches[0]
|
||||
newline = "\n" if lines[index].endswith("\n") else ""
|
||||
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
|
||||
lines[index] = f"{prefix}digest: {digest}{newline}"
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def write_release_artifacts(
|
||||
*,
|
||||
digest: str,
|
||||
source_revision: str,
|
||||
destination: str,
|
||||
kustomization: Path,
|
||||
output_dir: Path,
|
||||
) -> dict[str, str]:
|
||||
"""Write a rendered manifest, patch, and credential-free release metadata."""
|
||||
digest = _validated(digest, DIGEST_PATTERN, "image digest")
|
||||
source_revision = _validated(
|
||||
source_revision, REVISION_PATTERN, "source revision"
|
||||
)
|
||||
if destination != f"{DEFAULT_IMAGE}:git-{source_revision}":
|
||||
raise ValueError("destination must be the immutable git-<revision> tag")
|
||||
|
||||
source = kustomization.read_text(encoding="utf-8")
|
||||
rendered = render_kustomization(source, digest)
|
||||
relative_name = kustomization.name
|
||||
patch = "".join(
|
||||
difflib.unified_diff(
|
||||
source.splitlines(keepends=True),
|
||||
rendered.splitlines(keepends=True),
|
||||
fromfile=f"a/services/hermes/{relative_name}",
|
||||
tofile=f"b/services/hermes/{relative_name}",
|
||||
)
|
||||
)
|
||||
if not patch:
|
||||
raise ValueError("published digest already matches the Flux manifest")
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "hermes-kustomization.yaml").write_text(
|
||||
rendered, encoding="utf-8"
|
||||
)
|
||||
(output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
|
||||
metadata = {
|
||||
"digest": digest,
|
||||
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
|
||||
"image": DEFAULT_IMAGE,
|
||||
"published_tag": destination,
|
||||
"source_revision": source_revision,
|
||||
}
|
||||
(output_dir / "hermes-agent-image.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Validate Kaniko output and render artifacts for the reviewed Flux PR."""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--digest-file", required=True, type=Path)
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--destination", required=True)
|
||||
parser.add_argument("--kustomization", required=True, type=Path)
|
||||
parser.add_argument("--output-dir", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
write_release_artifacts(
|
||||
digest=args.digest_file.read_text(encoding="utf-8"),
|
||||
source_revision=args.source_revision,
|
||||
destination=args.destination,
|
||||
kustomization=args.kustomization,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -102,6 +102,27 @@ console tails and named artifact contents in its deterministic bundle. A report
|
||||
must say `retained Ariadne evidence` when that fallback is used; it must not
|
||||
pretend direct Jenkins access succeeded.
|
||||
|
||||
## Publishing an agent image after review
|
||||
|
||||
The `hermes-agent-image` Jenkins job is the only supported agent image builder.
|
||||
It runs daemonless Kaniko without a service-account token, host socket,
|
||||
privileged container, or writable Git credential. It accepts only an exact
|
||||
40-character revision that is both the checked-out commit and current
|
||||
`atlas/titan-iac` `main`, so a human must merge the source PR first.
|
||||
|
||||
From agent.hermes, trigger that one fixed job with:
|
||||
|
||||
```sh
|
||||
jenkins_image_build_trigger.py '<reviewed full main commit>'
|
||||
```
|
||||
|
||||
The helper has no general Jenkins credential or caller-selectable job name. Its
|
||||
Vault-projected token is bound by Jenkins only to `hermes-agent-image`. The job
|
||||
also requires its fixed publish confirmation, publishes a source-derived tag,
|
||||
records the immutable registry digest, and archives a JSON record plus a Flux
|
||||
digest patch. Apply that patch on a new branch and submit it for human review;
|
||||
the build never changes Git, reconciles Flux, or deploys by itself.
|
||||
|
||||
## The actual supervised triage algorithm
|
||||
|
||||
1. Classify the request as test/build triage, service health, or alert tuning.
|
||||
|
||||
@ -56,6 +56,11 @@ spec:
|
||||
{{- with secret "kv/data/atlas/hermes/developer-gitea" -}}
|
||||
{{ .Data.data.username }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-jenkins-image-build-token: kv/data/atlas/hermes/developer-jenkins
|
||||
vault.hashicorp.com/agent-inject-template-jenkins-image-build-token: |
|
||||
{{- with secret "kv/data/atlas/hermes/developer-jenkins" -}}
|
||||
{{ .Data.data.build_token }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-node-ssh-private-key: kv/data/atlas/hermes/developer-ssh
|
||||
vault.hashicorp.com/agent-inject-template-node-ssh-private-key: |
|
||||
{{- with secret "kv/data/atlas/hermes/developer-ssh" -}}
|
||||
|
||||
@ -77,6 +77,7 @@ configMapGenerator:
|
||||
- image_broker.py=scripts/image_broker.py
|
||||
- install_agent_tools.sh=scripts/install_agent_tools.sh
|
||||
- jenkins_build_evidence.py=scripts/jenkins_build_evidence.py
|
||||
- jenkins_image_build_trigger.py=scripts/jenkins_image_build_trigger.py
|
||||
- kanban_status_recovery.py=scripts/kanban_status_recovery.py
|
||||
- migrate_herdr_state.py=scripts/migrate_herdr_state.py
|
||||
- migrate_api_session_lineage.py=scripts/migrate_api_session_lineage.py
|
||||
|
||||
104
services/hermes/scripts/jenkins_image_build_trigger.py
Executable file
104
services/hermes/scripts/jenkins_image_build_trigger.py
Executable file
@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Trigger only the reviewed-main Hermes agent image release job."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
JENKINS_BUILD_URL = (
|
||||
"http://jenkins.jenkins.svc.cluster.local:8080/"
|
||||
"buildByToken/buildWithParameters"
|
||||
)
|
||||
JOB_NAME = "hermes-agent-image"
|
||||
TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token")
|
||||
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
|
||||
QUEUE_PATH_PATTERN = re.compile(r"^/queue/item/[0-9]+/?$")
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Keep a queued-build redirect from becoming an unauthorized job read."""
|
||||
|
||||
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
|
||||
return None
|
||||
|
||||
|
||||
def _open_without_redirect(request: urllib.request.Request, timeout: int):
|
||||
"""Return the Build Token Root response, including its expected HTTP 303."""
|
||||
opener = urllib.request.build_opener(_NoRedirect())
|
||||
try:
|
||||
return opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 303:
|
||||
return exc
|
||||
raise
|
||||
|
||||
|
||||
def trigger_build(
|
||||
revision: str,
|
||||
*,
|
||||
token_file: Path = TOKEN_FILE,
|
||||
opener=_open_without_redirect,
|
||||
) -> dict[str, str | int]:
|
||||
"""Post the fixed job parameters using its job-scoped build token."""
|
||||
revision = revision.strip()
|
||||
if not REVISION_PATTERN.fullmatch(revision):
|
||||
raise ValueError("revision must be a lowercase full 40-character commit")
|
||||
token = token_file.read_text(encoding="utf-8").strip()
|
||||
if not token:
|
||||
raise RuntimeError("Jenkins image-build token is empty")
|
||||
payload = urllib.parse.urlencode(
|
||||
{
|
||||
"job": JOB_NAME,
|
||||
"token": token,
|
||||
"PUBLISH_IMAGE": "true",
|
||||
"EXPECTED_SOURCE_REVISION": revision,
|
||||
"CONFIRM_PUBLISH": "PUBLISH HERMES AGENT",
|
||||
}
|
||||
).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
JENKINS_BUILD_URL,
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
method="POST",
|
||||
)
|
||||
with opener(request, timeout=20) as response:
|
||||
status = int(response.status)
|
||||
location = response.headers.get("Location", "")
|
||||
if status not in {200, 201, 202, 303}:
|
||||
raise RuntimeError(f"Jenkins trigger returned HTTP {status}")
|
||||
# Never return the submitted URL or response query: either may contain the
|
||||
# job token. The queue's numeric path is the only useful safe field.
|
||||
queue_path = urllib.parse.urlsplit(location).path
|
||||
if not QUEUE_PATH_PATTERN.fullmatch(queue_path):
|
||||
queue_path = ""
|
||||
return {
|
||||
"job": JOB_NAME,
|
||||
"queue_path": queue_path,
|
||||
"source_revision": revision,
|
||||
"status": status,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Validate one revision, trigger the bounded job, and print safe metadata."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("revision", help="reviewed full commit currently on main")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = trigger_build(args.revision)
|
||||
except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
|
||||
print(json.dumps({"error": str(exc)}, sort_keys=True))
|
||||
return 1
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - exercised through main()
|
||||
raise SystemExit(main())
|
||||
@ -95,6 +95,7 @@ def stage_agent() -> None:
|
||||
"chat-relay-key",
|
||||
"gitea-token",
|
||||
"gitea-username",
|
||||
"jenkins-image-build-token",
|
||||
"node-ssh-private-key",
|
||||
"node-ssh-config",
|
||||
"node-ssh-known-hosts",
|
||||
|
||||
@ -652,6 +652,25 @@ data:
|
||||
}
|
||||
}
|
||||
}
|
||||
pipelineJob('hermes-agent-image') {
|
||||
disabled(false)
|
||||
description('Human-gated, daemonless Kaniko build for the reviewed atlas/titan-iac main revision. Publishes a content-addressed Hermes agent image and archives a Flux digest patch; it never mutates Git or deploys.')
|
||||
authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))
|
||||
definition {
|
||||
cpsScm {
|
||||
scm {
|
||||
git {
|
||||
remote {
|
||||
url('https://scm.bstein.dev/atlas/titan-iac.git')
|
||||
credentials('gitea-pat')
|
||||
}
|
||||
branches('*/main')
|
||||
}
|
||||
}
|
||||
scriptPath('ci/Jenkinsfile.hermes-agent-image')
|
||||
}
|
||||
}
|
||||
}
|
||||
multibranchPipelineJob('titan-iac-quality-gate') {
|
||||
branchSources {
|
||||
branchSource {
|
||||
|
||||
@ -20,6 +20,7 @@ data:
|
||||
gitea:268.v75e47974c01d
|
||||
gitea-checks:603.621.vc708da_fb_371d
|
||||
multibranch-scan-webhook-trigger:1.0.11
|
||||
build-token-root:365.v717f8685a_09e
|
||||
# Structured test evidence. Without junit the `junit` step throws
|
||||
# NoSuchMethodError, jenkins.failed_tests is always empty, and triage
|
||||
# has only raw console text to work from. Pinned to the newest release
|
||||
|
||||
@ -65,6 +65,9 @@ spec:
|
||||
ARIADNE_JENKINS_API_USER={{ .Data.data.username }}
|
||||
ARIADNE_JENKINS_API_TOKEN={{ .Data.data.token }}
|
||||
{{ end }}
|
||||
{{ with secret "kv/data/atlas/hermes/developer-jenkins" }}
|
||||
HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}
|
||||
{{ end }}
|
||||
bstein.dev/restarted-at: "2026-05-20T09:40:31Z"
|
||||
spec:
|
||||
serviceAccountName: jenkins
|
||||
|
||||
@ -304,6 +304,29 @@ write_policy_and_role "postgres" "postgres" "postgres-vault" \
|
||||
write_policy_and_role "vault" "vault" "vault" \
|
||||
"vault/*" ""
|
||||
|
||||
# The Build Token Root plugin binds this value to one fixed Jenkins job. Both
|
||||
# Jenkins and agent.hermes receive it from Vault at pod start; it is never
|
||||
# rendered into Git, a ConfigMap, a Kubernetes Secret, or a build log.
|
||||
if existing_build_token="$(vault kv get -field=build_token kv/atlas/hermes/developer-jenkins 2>/dev/null)" \
|
||||
&& [ -n "${existing_build_token}" ]; then
|
||||
log "Hermes Jenkins image-build token already present"
|
||||
else
|
||||
if vault kv get kv/atlas/hermes/developer-jenkins >/dev/null 2>&1; then
|
||||
log "Hermes Jenkins credential exists without build_token; refusing to overwrite it"
|
||||
exit 1
|
||||
fi
|
||||
build_token="$(vault_cmd write -field=random_bytes sys/tools/random/32 format=hex)"
|
||||
if [ -z "${build_token}" ]; then
|
||||
log "Vault returned an empty Hermes Jenkins image-build token"
|
||||
exit 1
|
||||
fi
|
||||
vault_cmd kv put kv/atlas/hermes/developer-jenkins \
|
||||
build_token="${build_token}" >/dev/null
|
||||
unset build_token
|
||||
log "Hermes Jenkins image-build token created in Vault"
|
||||
fi
|
||||
unset existing_build_token
|
||||
|
||||
write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
|
||||
"shared/keycloak-admin shared/postmark-relay maintenance/metis-ssh-keys" \
|
||||
"harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc hermes/chat-oidc hermes/chat-telegram hermes/agent-oidc hermes/triage-oidc hermes/developer-keycloak" \
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
}
|
||||
],
|
||||
"managed_modules": [
|
||||
"ci/scripts/hermes_image_release.py",
|
||||
"ci/scripts/publish_test_metrics.py",
|
||||
"ci/scripts/publish_test_metrics_quality.py",
|
||||
"ci/scripts/semgrep_report.py",
|
||||
@ -35,6 +36,7 @@
|
||||
"testing/tests/test_quality_gate.py"
|
||||
],
|
||||
"lint_paths": [
|
||||
"ci/scripts/hermes_image_release.py",
|
||||
"ci/scripts/publish_test_metrics.py",
|
||||
"ci/scripts/publish_test_metrics_quality.py",
|
||||
"ci/scripts/semgrep_report.py",
|
||||
@ -168,6 +170,7 @@
|
||||
"coverage": {
|
||||
"minimum_percent": 95.0,
|
||||
"tracked_files": [
|
||||
"ci/scripts/hermes_image_release.py",
|
||||
"ci/scripts/publish_test_metrics.py",
|
||||
"ci/scripts/publish_test_metrics_quality.py",
|
||||
"ci/scripts/semgrep_report.py",
|
||||
|
||||
362
testing/tests/test_hermes_image_builder.py
Normal file
362
testing/tests/test_hermes_image_builder.py
Normal file
@ -0,0 +1,362 @@
|
||||
"""Safety and artifact contracts for the Hermes agent image release lane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PIPELINE_PATH = REPO_ROOT / "ci/Jenkinsfile.hermes-agent-image"
|
||||
RELEASE_SCRIPT = REPO_ROOT / "ci/scripts/hermes_image_release.py"
|
||||
TRIGGER_SCRIPT = (
|
||||
REPO_ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py"
|
||||
)
|
||||
|
||||
|
||||
def _load_release_module():
|
||||
spec = importlib.util.spec_from_file_location("hermes_image_release", RELEASE_SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _load_trigger_module():
|
||||
spec = importlib.util.spec_from_file_location("jenkins_image_build_trigger", TRIGGER_SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _pod_spec() -> dict:
|
||||
source = PIPELINE_PATH.read_text(encoding="utf-8")
|
||||
pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
|
||||
return yaml.safe_load(pod_yaml)["spec"]
|
||||
|
||||
|
||||
def test_builder_pod_is_daemonless_and_kernel_unprivileged() -> None:
|
||||
"""The builder must not gain host, daemon, service-token, or Linux privileges."""
|
||||
source = PIPELINE_PATH.read_text(encoding="utf-8")
|
||||
spec = _pod_spec()
|
||||
assert spec["automountServiceAccountToken"] is False
|
||||
assert spec["enableServiceLinks"] is False
|
||||
assert "hostPath" not in source
|
||||
assert "docker.sock" not in source
|
||||
assert "tcp://" not in source
|
||||
assert "buildkitd" not in source.lower()
|
||||
assert "dind" not in source.lower()
|
||||
|
||||
containers = {item["name"]: item for item in spec["containers"]}
|
||||
kaniko = containers["kaniko"]
|
||||
assert kaniko["image"] == (
|
||||
"gcr.io/kaniko-project/executor@sha256:"
|
||||
"c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e"
|
||||
)
|
||||
for container in containers.values():
|
||||
security = container["securityContext"]
|
||||
assert security["allowPrivilegeEscalation"] is False
|
||||
assert security["capabilities"]["drop"] == ["ALL"]
|
||||
assert security["seccompProfile"]["type"] == "RuntimeDefault"
|
||||
assert security.get("privileged", False) is False
|
||||
|
||||
|
||||
def test_builder_is_restricted_to_healthy_rpi5_capacity() -> None:
|
||||
"""Disposable builds must stay off unhealthy and reserved Atlas nodes."""
|
||||
spec = _pod_spec()
|
||||
assert spec["nodeSelector"]["hardware"] == "rpi5"
|
||||
expressions = spec["affinity"]["nodeAffinity"][
|
||||
"requiredDuringSchedulingIgnoredDuringExecution"
|
||||
]["nodeSelectorTerms"][0]["matchExpressions"]
|
||||
host_rule = next(rule for rule in expressions if rule["key"] == "kubernetes.io/hostname")
|
||||
assert host_rule["operator"] == "NotIn"
|
||||
assert set(host_rule["values"]) >= {
|
||||
"titan-04",
|
||||
"titan-14",
|
||||
"titan-18",
|
||||
"titan-19",
|
||||
"titan-22",
|
||||
"titan-24",
|
||||
}
|
||||
|
||||
|
||||
def test_pipeline_requires_reviewed_main_and_runtime_credentials() -> None:
|
||||
"""Publishing requires explicit confirmation and a reviewed main commit."""
|
||||
source = PIPELINE_PATH.read_text(encoding="utf-8")
|
||||
assert 'test "${PUBLISH_IMAGE}" = "true"' in source
|
||||
assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"' in source
|
||||
assert 'git rev-parse origin/main' in source
|
||||
assert "credentialsId: 'harbor-robot'" in source
|
||||
assert "set +x" in source
|
||||
assert "umask 077" in source
|
||||
assert "unset HARBOR_USER HARBOR_PASSWORD auth" in source
|
||||
assert "/busybox/rm -f /kaniko/.docker/config.json" in source
|
||||
assert "--digest-file=" in source
|
||||
assert "--image-name-with-digest-file=" in source
|
||||
assert "--destination=" in source
|
||||
|
||||
|
||||
def test_jenkins_job_is_manual_and_reads_pipeline_from_main() -> None:
|
||||
"""JCasC must not publish unreviewed branch contents or poll automatically."""
|
||||
config = yaml.safe_load(
|
||||
(REPO_ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
jobs = config["data"]["jobs.yaml"]
|
||||
block = jobs.split("pipelineJob('hermes-agent-image')", 1)[1].split(
|
||||
"pipelineJob(", 1
|
||||
)[0]
|
||||
assert "branches('*/main')" in block
|
||||
assert "scriptPath('ci/Jenkinsfile.hermes-agent-image')" in block
|
||||
assert "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block
|
||||
assert "pipelineTriggers" not in block
|
||||
assert "scmTrigger" not in block
|
||||
|
||||
|
||||
def test_agent_trigger_is_limited_to_the_image_job(tmp_path: Path) -> None:
|
||||
"""Agent Hermes gets one job token, fixed parameters, and no Jenkins admin API."""
|
||||
module = _load_trigger_module()
|
||||
revision = "a" * 40
|
||||
token_path = tmp_path / "token"
|
||||
token_path.write_text("private-job-token\n", encoding="utf-8")
|
||||
captured = {}
|
||||
|
||||
class Response(io.BytesIO):
|
||||
status = 201
|
||||
headers = {"Location": "http://jenkins/queue/item/42/"}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
self.close()
|
||||
|
||||
def opener(request, timeout):
|
||||
captured["request"] = request
|
||||
captured["timeout"] = timeout
|
||||
return Response(b"")
|
||||
|
||||
result = module.trigger_build(revision, token_file=token_path, opener=opener)
|
||||
request = captured["request"]
|
||||
fields = urllib.parse.parse_qs(request.data.decode("utf-8"))
|
||||
assert request.full_url == module.JENKINS_BUILD_URL
|
||||
assert fields == {
|
||||
"CONFIRM_PUBLISH": ["PUBLISH HERMES AGENT"],
|
||||
"EXPECTED_SOURCE_REVISION": [revision],
|
||||
"PUBLISH_IMAGE": ["true"],
|
||||
"job": ["hermes-agent-image"],
|
||||
"token": ["private-job-token"],
|
||||
}
|
||||
assert captured["timeout"] == 20
|
||||
assert "private-job-token" not in json.dumps(result)
|
||||
assert result["source_revision"] == revision
|
||||
|
||||
|
||||
def test_agent_trigger_rejects_unsafe_revision_and_empty_token(tmp_path: Path) -> None:
|
||||
"""No user-controlled job, URL, or abbreviated revision reaches Jenkins."""
|
||||
module = _load_trigger_module()
|
||||
token_path = tmp_path / "token"
|
||||
token_path.write_text("token\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
module.trigger_build("main", token_file=token_path)
|
||||
token_path.write_text("\n", encoding="utf-8")
|
||||
with pytest.raises(RuntimeError, match="empty"):
|
||||
module.trigger_build("a" * 40, token_file=token_path)
|
||||
|
||||
|
||||
def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None:
|
||||
"""HTTP 303 means the exact release is already queued, not a trigger failure."""
|
||||
module = _load_trigger_module()
|
||||
token_path = tmp_path / "token"
|
||||
token_path.write_text("token\n", encoding="utf-8")
|
||||
|
||||
class Response(io.BytesIO):
|
||||
status = 303
|
||||
headers = {"Location": "http://jenkins/queue/item/7/"}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
self.close()
|
||||
|
||||
result = module.trigger_build(
|
||||
"e" * 40, token_file=token_path, opener=lambda *_args, **_kwargs: Response()
|
||||
)
|
||||
assert result["status"] == 303
|
||||
assert result["queue_path"] == "/queue/item/7/"
|
||||
|
||||
|
||||
def test_job_token_is_generated_and_injected_only_at_runtime() -> None:
|
||||
"""The fixed-job credential stays in Vault and pod-lifetime memory."""
|
||||
plugins = (REPO_ROOT / "services/jenkins/configmap-plugins.yaml").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
vault = (
|
||||
REPO_ROOT / "services/vault/scripts/vault_k8s_auth_configure.sh"
|
||||
).read_text(encoding="utf-8")
|
||||
jenkins = (REPO_ROOT / "services/jenkins/deployment.yaml").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
agent = (REPO_ROOT / "services/hermes/agent-deployment.yaml").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
stage = (
|
||||
REPO_ROOT / "services/hermes/scripts/stage_runtime_access.py"
|
||||
).read_text(encoding="utf-8")
|
||||
assert "build-token-root:365.v717f8685a_09e" in plugins
|
||||
assert "sys/tools/random/32 format=hex" in vault
|
||||
assert "kv/atlas/hermes/developer-jenkins" in vault
|
||||
assert "HERMES_AGENT_IMAGE_BUILD_TOKEN={{ .Data.data.build_token }}" in jenkins
|
||||
assert "agent-inject-secret-jenkins-image-build-token" in agent
|
||||
assert '"jenkins-image-build-token"' in stage
|
||||
|
||||
|
||||
def test_release_renderer_preserves_manifest_and_emits_safe_artifacts(tmp_path: Path) -> None:
|
||||
"""The release artifact is exact, reviewable, and contains no credentials."""
|
||||
module = _load_release_module()
|
||||
old_digest = "sha256:" + "1" * 64
|
||||
new_digest = "sha256:" + "2" * 64
|
||||
revision = "a" * 40
|
||||
manifest = tmp_path / "kustomization.yaml"
|
||||
manifest.write_text(
|
||||
"apiVersion: kustomize.config.k8s.io/v1beta1\n"
|
||||
"kind: Kustomization\n"
|
||||
"images:\n"
|
||||
f" - name: {module.DEFAULT_IMAGE}\n"
|
||||
f" digest: {old_digest}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
output = tmp_path / "out"
|
||||
metadata = module.write_release_artifacts(
|
||||
digest=f"{new_digest}\n",
|
||||
source_revision=revision,
|
||||
destination=f"{module.DEFAULT_IMAGE}:git-{revision}",
|
||||
kustomization=manifest,
|
||||
output_dir=output,
|
||||
)
|
||||
|
||||
assert manifest.read_text(encoding="utf-8").endswith(f"{old_digest}\n")
|
||||
assert (output / "hermes-kustomization.yaml").read_text(
|
||||
encoding="utf-8"
|
||||
).endswith(f"{new_digest}\n")
|
||||
patch = (output / "hermes-image-update.patch").read_text(encoding="utf-8")
|
||||
assert f"- digest: {old_digest}" in patch
|
||||
assert f"+ digest: {new_digest}" in patch
|
||||
assert json.loads(
|
||||
(output / "hermes-agent-image.json").read_text(encoding="utf-8")
|
||||
) == metadata
|
||||
assert set(metadata) == {
|
||||
"digest",
|
||||
"flux_image",
|
||||
"image",
|
||||
"published_tag",
|
||||
"source_revision",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("digest", "revision", "destination"),
|
||||
[
|
||||
("latest", "a" * 40, "registry.bstein.dev/bstein/hermes-agent:git-" + "a" * 40),
|
||||
("sha256:" + "1" * 64, "short", "registry.bstein.dev/bstein/hermes-agent:git-short"),
|
||||
("sha256:" + "1" * 64, "a" * 40, "registry.bstein.dev/bstein/hermes-agent:latest"),
|
||||
],
|
||||
)
|
||||
def test_release_renderer_rejects_unpinned_inputs(
|
||||
tmp_path: Path, digest: str, revision: str, destination: str
|
||||
) -> None:
|
||||
"""Only exact digests and immutable source-derived tags are accepted."""
|
||||
module = _load_release_module()
|
||||
manifest = tmp_path / "kustomization.yaml"
|
||||
manifest.write_text(
|
||||
"images:\n"
|
||||
f" - name: {module.DEFAULT_IMAGE}\n"
|
||||
" digest: sha256:" + "0" * 64 + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
module.write_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=revision,
|
||||
destination=destination,
|
||||
kustomization=manifest,
|
||||
output_dir=tmp_path / "out",
|
||||
)
|
||||
|
||||
|
||||
def test_release_renderer_fails_closed_on_manifest_drift(tmp_path: Path) -> None:
|
||||
"""Missing, duplicate, or already-current image entries require human review."""
|
||||
module = _load_release_module()
|
||||
digest = "sha256:" + "f" * 64
|
||||
missing = "images:\n - name: example.invalid/other\n digest: " + digest + "\n"
|
||||
with pytest.raises(ValueError, match="found 0"):
|
||||
module.render_kustomization(missing, digest)
|
||||
|
||||
duplicate = (
|
||||
"images:\n"
|
||||
f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n"
|
||||
f" - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n"
|
||||
)
|
||||
with pytest.raises(ValueError, match="found 2"):
|
||||
module.render_kustomization(duplicate, digest)
|
||||
|
||||
manifest = tmp_path / "kustomization.yaml"
|
||||
manifest.write_text(
|
||||
f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: {digest}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
revision = "b" * 40
|
||||
with pytest.raises(ValueError, match="already matches"):
|
||||
module.write_release_artifacts(
|
||||
digest=digest,
|
||||
source_revision=revision,
|
||||
destination=f"{module.DEFAULT_IMAGE}:git-{revision}",
|
||||
kustomization=manifest,
|
||||
output_dir=tmp_path / "out",
|
||||
)
|
||||
|
||||
|
||||
def test_release_cli_reads_digest_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The pipeline CLI uses the Kaniko digest file as its sole digest input."""
|
||||
module = _load_release_module()
|
||||
digest = "sha256:" + "c" * 64
|
||||
revision = "d" * 40
|
||||
manifest = tmp_path / "kustomization.yaml"
|
||||
manifest.write_text(
|
||||
f"images:\n - name: {module.DEFAULT_IMAGE}\n digest: sha256:"
|
||||
+ "0" * 64
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
digest_file = tmp_path / "digest"
|
||||
digest_file.write_text(digest + "\n", encoding="utf-8")
|
||||
output = tmp_path / "out"
|
||||
monkeypatch.setattr(
|
||||
"sys.argv",
|
||||
[
|
||||
"hermes_image_release.py",
|
||||
"--digest-file",
|
||||
str(digest_file),
|
||||
"--source-revision",
|
||||
revision,
|
||||
"--destination",
|
||||
f"{module.DEFAULT_IMAGE}:git-{revision}",
|
||||
"--kustomization",
|
||||
str(manifest),
|
||||
"--output-dir",
|
||||
str(output),
|
||||
],
|
||||
)
|
||||
assert module.main() == 0
|
||||
assert json.loads(
|
||||
(output / "hermes-agent-image.json").read_text(encoding="utf-8")
|
||||
)["digest"] == digest
|
||||
@ -177,6 +177,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
|
||||
"chat-relay-key": "relay-key",
|
||||
"gitea-token": "gitea-key",
|
||||
"gitea-username": "hermes-automation",
|
||||
"jenkins-image-build-token": "job-scoped-token",
|
||||
"node-ssh-private-key": "private-key",
|
||||
"node-ssh-config": "host-config",
|
||||
"node-ssh-known-hosts": "known-hosts",
|
||||
@ -196,6 +197,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
|
||||
|
||||
assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600
|
||||
assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600
|
||||
assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600
|
||||
assert (runtime / "claude/settings.json").is_symlink()
|
||||
assert (runtime / "codex/skills").is_symlink()
|
||||
assert not (runtime / "claude/backups").exists()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user