release(hermes): automate private voice images

This commit is contained in:
jenkins 2026-08-23 15:54:56 -03:00
parent df18acbfa7
commit 79369c2357
12 changed files with 422 additions and 21 deletions

View File

@ -0,0 +1,212 @@
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-voice-image-builder
spec:
serviceAccountName: hermes-image-builder
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
nodeSelector:
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-04, titan-14, titan-18, titan-19, titan-22, titan-24]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: hardware
operator: In
values: [rpi5]
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: ["ALL"]}
runAsNonRoot: true
runAsUser: 1000
seccompProfile: {type: RuntimeDefault}
resources:
requests: {cpu: 25m, memory: 256Mi}
limits: {cpu: 500m, memory: 512Mi}
- name: python
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
command: ["sleep"]
args: ["99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: ["ALL"]}
runAsNonRoot: true
runAsUser: 1000
seccompProfile: {type: RuntimeDefault}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 500m, memory: 512Mi}
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
runAsUser: 0
seccompProfile: {type: RuntimeDefault}
resources:
requests: {cpu: 250m, memory: 1Gi}
limits: {cpu: "2", memory: 4Gi}
"""
}
}
parameters {
booleanParam(name: 'PUBLISH_IMAGE', defaultValue: false, description: 'Publish the reviewed voice image to Harbor.')
choice(name: 'IMAGE_COMPONENT', choices: ['stt', 'tts'], description: 'Private voice component to build.')
string(name: 'EXPECTED_SOURCE_REVISION', defaultValue: '', description: 'Full reviewed commit contained by main.')
string(name: 'CONFIRM_PUBLISH', defaultValue: '', description: 'Exact component-specific confirmation.')
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
skipDefaultCheckout(true)
timeout(time: 150, unit: 'MINUTES')
}
stages {
stage('Checkout reviewed source') {
steps { checkout scm }
}
stage('Enforce release boundary') {
steps {
container('jnlp') {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
case "${IMAGE_COMPONENT}" in
stt) expected_confirmation='PUBLISH HERMES STT' ;;
tts) expected_confirmation='PUBLISH HERMES TTS' ;;
*) echo 'IMAGE_COMPONENT must be stt or tts' >&2; exit 2 ;;
esac
test "${CONFIRM_PUBLISH}" = "${expected_confirmation}"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'') echo 'EXPECTED_SOURCE_REVISION must be a lowercase full commit' >&2; exit 2 ;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "$(git rev-parse origin/main)"
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${actual_revision}"
test -z "$(git status --porcelain)"
case "${BUILD_NUMBER}" in ''|0*|*[!0-9]*) exit 2 ;; esac
image="registry.bstein.dev/bstein/hermes-jetson-${IMAGE_COMPONENT}"
printf '%s\n' "${image}:git-${actual_revision}-build-${BUILD_NUMBER}" > build/hermes-voice.destination
printf '%s\n' "${actual_revision}" > build/hermes-voice.source-revision
printf '%s\n' "${IMAGE_COMPONENT}" > build/hermes-voice.component
test -f "dockerfiles/Dockerfile.hermes-jetson-${IMAGE_COMPONENT}"
'''
}
}
}
stage('Validate reviewed voice source') {
steps {
container('python') {
sh '''
set -eu
python3 -m pip install --disable-pip-version-check --no-cache-dir \
--target=/tmp/hermes-voice-test-deps pytest==8.3.4 PyYAML==6.0.2
PYTHONPATH=/tmp/hermes-voice-test-deps python3 -m pytest -q \
testing/tests/test_hermes_tts_language_routing.py \
testing/tests/test_hermes_voice_language_routing.py \
testing/tests/test_hermes_oci_promote.py \
testing/tests/test_hermes_image_automation.py \
testing/tests/test_hermes_voice_release.py
'''
}
}
}
stage('Build and publish without a daemon') {
steps {
container('kaniko') {
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USER', passwordVariable: 'HARBOR_PASSWORD')]) {
sh '''#!/busybox/sh
set -eu
set +x
component="$(cat build/hermes-voice.component)"
destination="$(cat build/hermes-voice.destination)"
source_revision="$(cat build/hermes-voice.source-revision)"
config_path=/kaniko/.docker/config.json
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
umask 022
/kaniko/executor \
--registry-mirror=harbor-core.harbor.svc.cluster.local \
--insecure-registry=harbor-core.harbor.svc.cluster.local \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-jetson-${component}" \
--destination="${destination}" \
--digest-file="${WORKSPACE}/build/hermes-voice.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-voice.image" \
--label="org.opencontainers.image.revision=${source_revision}" \
--label="org.opencontainers.image.source=https://scm.bstein.dev/atlas/titan-iac" \
--label="org.opencontainers.image.title=hermes-jetson-${component}" \
--cleanup --push-retry=3
/busybox/chmod 644 build/hermes-voice.digest build/hermes-voice.image
'''
}
}
}
}
stage('Verify, archive, and publish Flux release') {
steps {
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USER', passwordVariable: 'HARBOR_PASSWORD')]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-voice.destination)"
source_revision="$(cat build/hermes-voice.source-revision)"
component="$(cat build/hermes-voice.component)"
digest="$(cat build/hermes-voice.digest)"
image="$(cat build/hermes-voice.image)"
test "${image}" = "${destination}@${digest}"
python3 ci/scripts/hermes_oci_promote.py \
--destination "${destination}" \
--digest-file build/hermes-voice.digest \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
> build/hermes-voice.promotion.json
python3 -c 'import json; data=json.load(open("build/hermes-voice.promotion.json")); assert data["result"] in {"published", "already-present"}'
'''
archiveArtifacts(
artifacts: 'build/hermes-voice.component,build/hermes-voice.destination,build/hermes-voice.digest,build/hermes-voice.image,build/hermes-voice.source-revision,build/hermes-voice.promotion.json',
allowEmptyArchive: false,
fingerprint: true
)
}
}
}
}
}

View File

@ -17,7 +17,8 @@ from typing import Any, Callable
REGISTRY_ORIGIN = "https://registry.bstein.dev"
DESTINATION_PATTERN = re.compile(
r"^registry\.bstein\.dev/bstein/(?P<component>hermes-(?:agent|webui)):"
r"^registry\.bstein\.dev/bstein/"
r"(?P<component>hermes-(?:agent|webui|jetson-(?:stt|tts))):"
r"git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)$"
)
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")

View File

@ -371,8 +371,9 @@ data:
For an explicitly requested Hermes runtime release, trigger only the
reviewed image lanes with `/opt/coordinator/jenkins_image_build_trigger.py`.
Use `--component agent` for backend/runtime changes and `--component webui`
for chat UI changes, passing a full commit already contained by `main`.
Use `--component agent` for backend/runtime changes, `--component webui`
for chat UI changes, and `--component stt` or `--component tts` for the
private voice services, passing a full commit already contained by `main`.
Jenkins builds the newest main containing that commit, publishes a final
immutable release tag only after its evidence passes, and Flux applies the
resulting digest. Candidate tags and failed builds never deploy. Follow the

View File

@ -50,3 +50,55 @@ spec:
order: asc
digestReflectionPolicy: Always
interval: 1m0s
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageRepository
metadata:
name: hermes-stt-release
namespace: hermes
spec:
image: registry.bstein.dev/bstein/hermes-jetson-stt
interval: 1m0s
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImagePolicy
metadata:
name: hermes-stt-release
namespace: hermes
spec:
imageRepositoryRef:
name: hermes-stt-release
filterTags:
pattern: '^git-[0-9a-f]{40}-build-(?P<build>[1-9][0-9]*)-release$'
extract: '$build'
policy:
numerical:
order: asc
digestReflectionPolicy: Always
interval: 1m0s
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageRepository
metadata:
name: hermes-tts-release
namespace: hermes
spec:
image: registry.bstein.dev/bstein/hermes-jetson-tts
interval: 1m0s
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImagePolicy
metadata:
name: hermes-tts-release
namespace: hermes
spec:
imageRepositoryRef:
name: hermes-tts-release
filterTags:
pattern: '^git-[0-9a-f]{40}-build-(?P<build>[1-9][0-9]*)-release$'
extract: '$build'
policy:
numerical:
order: asc
digestReflectionPolicy: Always
interval: 1m0s

View File

@ -23,6 +23,16 @@ JOBS = {
"job": "hermes-webui-image",
"confirmation": "PUBLISH HERMES WEBUI",
},
"stt": {
"job": "hermes-voice-image",
"confirmation": "PUBLISH HERMES STT",
"parameters": {"IMAGE_COMPONENT": "stt"},
},
"tts": {
"job": "hermes-voice-image",
"confirmation": "PUBLISH HERMES TTS",
"parameters": {"IMAGE_COMPONENT": "tts"},
},
}
JOB_NAME = JOBS["agent"]["job"]
TOKEN_FILE = Path("/runtime-access/jenkins-image-build-token")
@ -61,19 +71,19 @@ def trigger_build(
raise ValueError("revision must be a lowercase full 40-character commit")
job = JOBS.get(component)
if job is None:
raise ValueError("component must be agent or webui")
raise ValueError("component must be agent, webui, stt, or tts")
token = token_file.read_text(encoding="utf-8").strip()
if not token:
raise RuntimeError("Jenkins image-build token is empty")
payload = urllib.parse.urlencode(
{
"job": job["job"],
"token": token,
"PUBLISH_IMAGE": "true",
"EXPECTED_SOURCE_REVISION": revision,
"CONFIRM_PUBLISH": job["confirmation"],
}
).encode("utf-8")
fields = {
"job": job["job"],
"token": token,
"PUBLISH_IMAGE": "true",
"EXPECTED_SOURCE_REVISION": revision,
"CONFIRM_PUBLISH": job["confirmation"],
}
fields.update(job.get("parameters", {}))
payload = urllib.parse.urlencode(fields).encode("utf-8")
request = urllib.request.Request(
JENKINS_BUILD_URL,
data=payload,

View File

@ -31,7 +31,7 @@ spec:
kubernetes.io/hostname: titan-21
containers:
- name: stt
image: registry.bstein.dev/bstein/hermes-jetson-stt@sha256:dfb0b0788dcf63747d8c761c6d3ea6a7459bcd8d0a5c92bcd4f6f434a760249f
image: registry.bstein.dev/bstein/hermes-jetson-stt@sha256:dfb0b0788dcf63747d8c761c6d3ea6a7459bcd8d0a5c92bcd4f6f434a760249f # {"$imagepolicy": "hermes:hermes-stt-release"}
imagePullPolicy: IfNotPresent
ports:
- {name: http, containerPort: 9000, protocol: TCP}
@ -114,7 +114,7 @@ spec:
app: hermes-tts
annotations:
ai.bstein.dev/role: private-chat-text-to-speech
ai.bstein.dev/model: piper-en-us-lessac-medium
ai.bstein.dev/model: piper-multilingual-en-ru-es
ai.bstein.dev/gpu: CPU-only beside Whisper on the voice node
spec:
automountServiceAccountToken: false
@ -123,7 +123,7 @@ spec:
kubernetes.io/hostname: titan-21
containers:
- name: tts
image: registry.bstein.dev/bstein/hermes-jetson-tts@sha256:5cb9e57faab46365bff606c559af57b9505892be2909aea1ac523568d478b2cc
image: registry.bstein.dev/bstein/hermes-jetson-tts@sha256:5cb9e57faab46365bff606c559af57b9505892be2909aea1ac523568d478b2cc # {"$imagepolicy": "hermes:hermes-tts-release"}
imagePullPolicy: IfNotPresent
ports:
- {name: http, containerPort: 9001, protocol: TCP}
@ -131,7 +131,6 @@ spec:
- {name: HOME, value: /tmp}
- {name: XDG_CACHE_HOME, value: /tmp/cache}
- {name: HERMES_TTS_PORT, value: "9001"}
- {name: HERMES_TTS_VOICE, value: en_US-lessac-medium}
- {name: HERMES_TTS_CACHE, value: /opt/models/piper}
- {name: HERMES_TTS_ONNX_THREADS, value: "2"}
startupProbe:

View File

@ -700,6 +700,31 @@ data:
}
}
}
pipelineJob('hermes-voice-image') {
disabled(false)
description('Bounded daemonless Kaniko release for reviewed Hermes STT or TTS source. A validated immutable release tag is consumed by Flux.')
authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))
parameters {
booleanParam('PUBLISH_IMAGE', false, 'Publish the reviewed Hermes voice image.')
stringParam('IMAGE_COMPONENT', '', 'Exact component: stt or tts.')
stringParam('EXPECTED_SOURCE_REVISION', '', 'Full reviewed commit that must be contained by atlas/titan-iac main.')
stringParam('CONFIRM_PUBLISH', '', 'Exact component-specific confirmation.')
}
definition {
cpsScm {
scm {
git {
remote {
url('https://scm.bstein.dev/atlas/titan-iac.git')
credentials('gitea-pat')
}
branches('*/main')
}
}
scriptPath('ci/Jenkinsfile.hermes-voice-image')
}
}
}
multibranchPipelineJob('titan-iac-quality-gate') {
branchSources {
branchSource {

View File

@ -463,7 +463,9 @@ def test_voice_workloads_have_deliberate_xavier_placement():
tts_env = {
item["name"]: item["value"] for item in tts["containers"][0]["env"]
}
assert tts_env["HERMES_TTS_VOICE"] == "en_US-lessac-medium"
# Keep the deployment neutral so the old and new immutable images can
# cross one Flux rollout safely; each image owns its compatible default.
assert "HERMES_TTS_VOICE" not in tts_env
assert tts_env["HERMES_TTS_ONNX_THREADS"] == "2"
assert tts["containers"][0]["resources"]["limits"]["cpu"] == "4"
assert all("hostPath" not in volume for volume in stt["volumes"])

View File

@ -27,7 +27,12 @@ def test_image_policies_observe_only_validated_release_tags() -> None:
for item in documents
if item["kind"] == "ImagePolicy"
}
assert set(repositories) == {"hermes-agent-release", "hermes-webui-release"}
assert set(repositories) == {
"hermes-agent-release",
"hermes-webui-release",
"hermes-stt-release",
"hermes-tts-release",
}
assert set(policies) == set(repositories)
for name, policy in policies.items():
assert policy["metadata"]["namespace"] == "hermes"
@ -40,7 +45,7 @@ def test_image_policies_observe_only_validated_release_tags() -> None:
assert policy["spec"]["digestReflectionPolicy"] == "Always"
def test_flux_updates_only_the_three_hermes_image_digests() -> None:
def test_flux_updates_only_the_reviewed_hermes_image_digests() -> None:
"""Flux persists selected digests to Git and rolls all matching workloads."""
service_kustomization = (SERVICE / "kustomization.yaml").read_text(encoding="utf-8")
applications_kustomization = (APPLICATIONS / "kustomization.yaml").read_text(
@ -52,6 +57,7 @@ def test_flux_updates_only_the_three_hermes_image_digests() -> None:
agent = (SERVICE / "kustomization.yaml").read_text(encoding="utf-8")
chat = (SERVICE / "chat-statefulset.yaml").read_text(encoding="utf-8")
dashboard = (SERVICE / "deployment.yaml").read_text(encoding="utf-8")
voice = (SERVICE / "voice-deployment.yaml").read_text(encoding="utf-8")
assert " - image.yaml" in service_kustomization
assert " - hermes/image-automation.yaml" in applications_kustomization
@ -73,3 +79,8 @@ def test_flux_updates_only_the_three_hermes_image_digests() -> None:
)
assert "registry.bstein.dev/bstein/hermes-webui:" in marked_line
assert "@sha256:" in marked_line
for component in ("stt", "tts"):
marker = f'"$imagepolicy": "hermes:hermes-{component}-release"'
assert voice.count(marker) == 1
marked_line = next(line for line in voice.splitlines() if marker in line)
assert f"registry.bstein.dev/bstein/hermes-jetson-{component}@sha256:" in marked_line

View File

@ -271,12 +271,50 @@ def test_agent_trigger_can_select_only_the_bounded_webui_job(tmp_path: Path) ->
assert fields["job"] == ["hermes-webui-image"]
assert fields["CONFIRM_PUBLISH"] == ["PUBLISH HERMES WEBUI"]
assert result["component"] == "webui"
with pytest.raises(ValueError, match="agent or webui"):
with pytest.raises(ValueError, match="agent, webui, stt, or tts"):
module.trigger_build(
"b" * 40, component="other", token_file=token_path, opener=opener
)
@pytest.mark.parametrize(
("component", "confirmation"),
[("stt", "PUBLISH HERMES STT"), ("tts", "PUBLISH HERMES TTS")],
)
def test_agent_trigger_binds_private_voice_component(
tmp_path: Path, component: str, confirmation: str
) -> None:
"""Voice releases select one fixed Jenkins job and component parameter."""
module = _load_trigger_module()
token_path = tmp_path / "token"
token_path.write_text("private-job-token\n", encoding="utf-8")
captured = {}
class Response(io.BytesIO):
status = 201
headers = {"Location": "https://ci.bstein.dev/queue/item/85/"}
def __enter__(self):
return self
def __exit__(self, *_args):
self.close()
def opener(request, timeout):
assert timeout == 20
captured["request"] = request
return Response(b"")
result = module.trigger_build(
"c" * 40, component=component, token_file=token_path, opener=opener
)
fields = urllib.parse.parse_qs(captured["request"].data.decode("utf-8"))
assert fields["job"] == ["hermes-voice-image"]
assert fields["IMAGE_COMPONENT"] == [component]
assert fields["CONFIRM_PUBLISH"] == [confirmation]
assert result["component"] == component
def test_agent_trigger_accepts_existing_queue_redirect(tmp_path: Path) -> None:
"""HTTP 303 means the exact release is already queued, not a trigger failure."""
module = _load_trigger_module()

View File

@ -105,6 +105,13 @@ def test_default_voice_name_matches_the_dockerfile_env_default(tts):
assert tts.DEFAULT_VOICE_NAME == AMY
def test_deployment_does_not_override_the_image_default() -> None:
"""An old single-voice image and the new image can cross the Flux transition."""
manifest = (ROOT / "services/hermes/voice-deployment.yaml").read_text()
assert "HERMES_TTS_VOICE" not in manifest
assert "piper-multilingual-en-ru-es" in manifest
def test_resolved_voice_is_always_one_of_the_three_baked_names(tts):
assert frozenset({AMY, IRINA, CLAUDE}) == tts.BAKED_VOICE_NAMES
fuzz_inputs = [

View File

@ -0,0 +1,43 @@
"""Contracts for the bounded Hermes private-voice image release lane."""
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
PIPELINE = ROOT / "ci/Jenkinsfile.hermes-voice-image"
JOBS = ROOT / "services/jenkins/configmap-jcasc.yaml"
TRIGGER = ROOT / "services/hermes/scripts/jenkins_image_build_trigger.py"
def test_voice_pipeline_binds_component_source_digest_and_release() -> None:
pipeline = PIPELINE.read_text(encoding="utf-8")
assert "IMAGE_COMPONENT must be stt or tts" in pipeline
assert "git merge-base --is-ancestor" in pipeline
assert "git status --porcelain" in pipeline
assert "Dockerfile.hermes-jetson-${component}" in pipeline
assert "--digest-file=" in pipeline
assert "hermes_oci_promote.py" in pipeline
assert "PUBLISH HERMES STT" in pipeline
assert "PUBLISH HERMES TTS" in pipeline
assert "archiveArtifacts(" in pipeline
def test_jenkins_declares_one_token_guarded_voice_job() -> None:
jobs = JOBS.read_text(encoding="utf-8")
assert jobs.count("pipelineJob('hermes-voice-image')") == 1
block = jobs.split("pipelineJob('hermes-voice-image')", 1)[1].split(
"multibranchPipelineJob('titan-iac-quality-gate')", 1
)[0]
assert "authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block
assert "scriptPath('ci/Jenkinsfile.hermes-voice-image')" in block
def test_runtime_trigger_exposes_both_voice_components() -> None:
trigger = TRIGGER.read_text(encoding="utf-8")
for component, confirmation in (
("stt", "PUBLISH HERMES STT"),
("tts", "PUBLISH HERMES TTS"),
):
assert f'"{component}": {{' in trigger
assert f'"confirmation": "{confirmation}"' in trigger
assert f'"parameters": {{"IMAGE_COMPONENT": "{component}"}}' in trigger