Compare commits

..

No commits in common. "main" and "feature/sso" have entirely different histories.

1727 changed files with 7144 additions and 323727 deletions

28
.gitignore vendored
View File

@ -1,27 +1 @@
*.md
!README.md
!knowledge/**/*.md
!services/comms/knowledge/**/*.md
__pycache__/
*.py[cod]
.pytest_cache
.ruff_cache/
.coverage
build/
test-results/
artifacts/
.venv
.venv-ci
tmp/
.mainfix/
.terraform/
**/.terraform/
*.tfvars
*.tfstate
*.tfstate.*
crash.log
terraform/atlas/generated/
# Local demo credentials (never commit)
scripts/ops/hermes_demo.env
scripts/ops/hermes_triage_demo.env
AGENTS.md

512
Jenkinsfile vendored
View File

@ -1,512 +0,0 @@
// Mirror of ci/Jenkinsfile.titan-iac for multibranch discovery.
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
spec:
nodeSelector:
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-04
- titan-06
- titan-11
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-19
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
jenkins/jenkins-jenkins-agent: "true"
containers:
- name: jnlp
image: jenkins/inbound-agent:3355.v388858a_47b_33-2-jdk21
resources:
requests:
cpu: "25m"
memory: "256Mi"
- name: python
image: registry.bstein.dev/bstein/python:3.12-slim
command:
- cat
tty: true
- name: quality-tools
image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
command:
- cat
tty: true
- name: semgrep
image: semgrep/semgrep:1.171.0
command:
- cat
tty: true
"""
}
}
environment {
PIP_DISABLE_PIP_VERSION_CHECK = '1'
PYTHONUNBUFFERED = '1'
SUITE_NAME = 'titan_iac'
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
SONARQUBE_PROJECT_KEY = 'titan_iac'
SONARQUBE_TOKEN = credentials('sonarqube-token')
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
QUALITY_GATE_SEMGREP_ENFORCE = '0'
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
QUALITY_GATE_IRONBANK_ENFORCE = '1'
QUALITY_GATE_IRONBANK_REQUIRED = '0'
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
}
stages {
stage('Checkout') {
steps {
checkout scm
}
}
stage('Install deps') {
steps {
sh '''
set -eu
if ! command -v git >/dev/null 2>&1; then
apt-get update
apt-get install -y --no-install-recommends git ca-certificates
rm -rf /var/lib/apt/lists/*
fi
pip install --no-cache-dir -r ci/requirements.txt
'''
}
}
stage('Prepare local quality evidence') {
steps {
sh '''
set -eu
mkdir -p build
set +e
python3 -m testing.quality_gate --profile local --build-dir build
local_quality_rc=$?
set -e
printf '%s\n' "${local_quality_rc}" > build/local-quality-gate.rc
'''
}
}
stage('Collect Semgrep evidence') {
steps {
container('semgrep') {
sh '''#!/bin/sh
set -eu
mkdir -p build
set +e
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
semgrep_rc=$?
set -e
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
'''
}
sh '''
set -eu
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
'''
}
}
stage('Collect SonarQube evidence') {
steps {
container('quality-tools') {
sh '''#!/usr/bin/env bash
set -euo pipefail
mkdir -p build
args=(
"-Dsonar.host.url=${SONARQUBE_HOST_URL}"
"-Dsonar.login=${SONARQUBE_TOKEN}"
"-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.sources=."
"-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**,services/monitoring/dashboards/**,services/monitoring/grafana-dashboard-*.yaml,services/game-stream/**"
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
)
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
set +e
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
rc=${PIPESTATUS[0]}
set -e
printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
'''
}
sh '''
set -eu
mkdir -p build
python3 - <<'PY'
import base64
import json
import os
import time
import urllib.parse
import urllib.request
from pathlib import Path
host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
token = os.getenv('SONARQUBE_TOKEN', '').strip()
report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
payload = {
"status": "ERROR",
"note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY",
}
if host and project_key:
task_file = Path('.scannerwork/report-task.txt')
task_id = ''
if task_file.exists():
for line in task_file.read_text(encoding='utf-8').splitlines():
key, _, value = line.partition('=')
if key == 'ceTaskId':
task_id = value.strip()
break
if task_id:
ce_query = urllib.parse.urlencode({"id": task_id})
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
ce_request = urllib.request.Request(f"{host}/api/ce/task?{ce_query}", method="GET")
if token:
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
ce_request.add_header("Authorization", f"Basic {encoded}")
try:
with urllib.request.urlopen(ce_request, timeout=12) as response:
ce_payload = json.loads(response.read().decode("utf-8"))
except Exception:
time.sleep(3)
continue
status = str(ce_payload.get("task", {}).get("status", "")).upper()
if status in {"SUCCESS", "FAILED", "CANCELED"}:
break
time.sleep(3)
query = urllib.parse.urlencode({"projectKey": project_key})
request = urllib.request.Request(
f"{host}/api/qualitygates/project_status?{query}",
method="GET",
)
if token:
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
request.add_header("Authorization", f"Basic {encoded}")
try:
with urllib.request.urlopen(request, timeout=12) as response:
payload = json.loads(response.read().decode("utf-8"))
except Exception as exc: # noqa: BLE001
payload = {"status": "ERROR", "error": str(exc)}
with open(report_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2, sort_keys=True)
handle.write("\\n")
PY
'''
}
}
stage('Collect IronBank evidence') {
steps {
container('quality-tools') {
sh '''#!/usr/bin/env bash
set -euo pipefail
mkdir -p build
set +e
trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --skip-check-update --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
trivy_rc=$?
set -e
if [ "${trivy_rc}" -ne 0 ]; then
rm -f build/trivy-fs.json
cat > build/ironbank-compliance.json <<EOF
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy scan failed","trivy_rc":${trivy_rc}}
EOF
exit 0
fi
if [ ! -s build/trivy-fs.json ]; then
cat > build/ironbank-compliance.json <<EOF
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
EOF
exit 0
fi
'''
}
sh '''
set -eu
mkdir -p build
if [ -s build/trivy-fs.json ]; then
python3 ci/scripts/supply_chain_report.py --trivy-json build/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output build/ironbank-compliance.json
exit 0
fi
python3 - <<'PY'
import json
import os
from pathlib import Path
report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
if report_path.exists():
raise SystemExit(0)
status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
payload = {
"status": status or "unknown",
"compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
}
payload = {k: v for k, v in payload.items() if v is not None}
if "status" not in payload:
payload["status"] = "unknown"
payload["note"] = (
"Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
"or write build/ironbank-compliance.json in image-building repos."
)
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
PY
'''
}
}
stage('Run quality gate') {
steps {
sh '''
set -eu
mkdir -p build
set +e
python3 -m testing.quality_gate --profile jenkins --build-dir build
quality_gate_rc=$?
set -e
printf '%s\n' "${quality_gate_rc}" > build/quality-gate.rc
'''
}
}
stage('Publish test metrics') {
steps {
sh '''
set -eu
export JUNIT_GLOB='build/junit-*.xml'
export QUALITY_GATE_EXIT_CODE_PATH='build/quality-gate.rc'
export QUALITY_GATE_SUMMARY_PATH='build/quality-gate-summary.json'
python3 ci/scripts/publish_test_metrics.py
'''
}
}
stage('Enforce quality gate') {
steps {
sh '''
set -euo pipefail
gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)"
fail=0
if [ "${gate_rc}" -ne 0 ]; then
echo "quality gate failed with rc=${gate_rc}" >&2
fail=1
fi
enabled() {
case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
1|true|yes|on) return 0 ;;
*) return 1 ;;
esac
}
if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
sonar_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/sonarqube-quality-gate.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${sonar_status}" in
ok|pass|passed|success) ;;
*)
echo "sonarqube gate failed: ${sonar_status}" >&2
fail=1
;;
esac
fi
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
semgrep_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/semgrep-report.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = str(payload.get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${semgrep_status}" in
ok|pass|passed|success) ;;
*)
echo "semgrep gate failed: ${semgrep_status}" >&2
fail=1
;;
esac
fi
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
ironbank_required=1
fi
if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
supply_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/ironbank-compliance.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
compliant = payload.get("compliant")
if compliant is True:
print("ok")
elif compliant is False:
print("failed")
else:
status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
print(status or "missing")
PY
)"
case "${supply_status}" in
ok|pass|passed|success|compliant) ;;
not_applicable|na|n/a)
if enabled "${ironbank_required}"; then
echo "supply chain gate required but status=${supply_status}" >&2
fail=1
fi
;;
*)
if enabled "${ironbank_required}"; then
echo "supply chain gate failed: ${supply_status}" >&2
fail=1
else
echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
fi
;;
esac
fi
exit "${fail}"
'''
}
}
stage('Resolve Flux branch') {
steps {
script {
env.FLUX_BRANCH = sh(
returnStdout: true,
script: "grep -m1 '^\\s*branch:' clusters/atlas/flux-system/gotk-sync.yaml | sed 's/^\\s*branch:\\s*//'"
).trim()
if (!env.FLUX_BRANCH) {
error('Flux branch not found in gotk-sync.yaml')
}
echo "Flux branch: ${env.FLUX_BRANCH}"
}
}
}
stage('Promote') {
when {
expression {
def branch = env.BRANCH_NAME ?: (env.GIT_BRANCH ?: '').replaceFirst('origin/', '')
return env.FLUX_BRANCH && branch == env.FLUX_BRANCH
}
}
steps {
withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
sh '''
set -euo pipefail
if ! command -v git >/dev/null 2>&1; then
if command -v apk >/dev/null 2>&1; then
apk add --no-cache git >/dev/null
elif command -v apt-get >/dev/null 2>&1; then
apt-get update >/dev/null
apt-get install -y git >/dev/null
fi
fi
cd "${WORKSPACE:-$PWD}"
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "workspace is not a git checkout; skipping promote"
exit 0
fi
set +x
git config user.email "jenkins@bstein.dev"
git config user.name "jenkins"
git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/titan/atlas-iac.git
git push origin HEAD:${FLUX_BRANCH}
'''
}
}
}
}
post {
always {
script {
try {
if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
try {
junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
} catch (Throwable err) {
echo "junit step unavailable: ${err.class.simpleName}"
}
}
archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
} catch (Throwable err) {
if (err.class.simpleName == 'MissingContextVariableException') {
echo 'workspace unavailable; skipping post-build artifact collection'
} else {
throw err
}
}
}
}
}
}

105
Makefile
View File

@ -1,105 +0,0 @@
# Makefile
SHELL := /usr/bin/env bash
KUSTOMIZE ?= kustomize
PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3)
TERRAFORM ?= terraform
TRIVY ?= trivy
BUILD_DIR ?= build
ATLAS_FLUX_ROOT := clusters/atlas/flux-system
AETHER_FLUX_ROOT := clusters/aether/flux-system
AETHER_TF_DIR := terraform/aether
FLUX_KUSTOMIZATION ?= flux-system
FLUX_NAMESPACE ?= flux-system
FLUX_PATH ?= $(ATLAS_FLUX_ROOT)
TRIVY_CACHE_DIR ?= $(BUILD_DIR)/trivy-cache
TRIVY_DB_FLAGS ?= --skip-db-update
.PHONY: help render render-atlas render-aether render-services render-infrastructure validate dashboards knowledge test flux-diff flux-inventory trivy-scan security-report security aether-fmt aether-init aether-validate aether-plan
help:
@printf '%s\n' \
'Targets:' \
' render Render Atlas, Aether, services, and infrastructure.' \
' render-atlas Render the Atlas Flux root.' \
' render-aether Render the Aether Flux root.' \
' render-services Render every service kustomization.' \
' render-infrastructure Render infrastructure kustomizations.' \
' validate Run render checks plus the local quality gate.' \
' dashboards Regenerate Grafana dashboard artifacts.' \
' knowledge Regenerate Atlas knowledge artifacts and comms mirror.' \
' test Run the local quality gate.' \
' flux-diff Diff a Flux Kustomization; override FLUX_KUSTOMIZATION/FLUX_PATH.' \
' flux-inventory Print Flux Kustomization inventory.' \
' trivy-scan Run the local Trivy filesystem scan.' \
' security-report Build IronBank report from build/trivy-fs.json.' \
' security Run Trivy scan and build the IronBank report.' \
' aether-fmt Check Terraform formatting for Aether.' \
' aether-init Initialize Aether Terraform providers.' \
' aether-validate Validate Aether Terraform after init.' \
' aether-plan Plan Aether Terraform from local variables/env.'
render: render-atlas render-aether render-services render-infrastructure
render-atlas:
$(KUSTOMIZE) build $(ATLAS_FLUX_ROOT) >/tmp/titan-iac-atlas-flux-system.yaml
render-aether:
$(KUSTOMIZE) build $(AETHER_FLUX_ROOT) >/tmp/titan-iac-aether-flux-system.yaml
render-services:
@set -euo pipefail; \
while IFS= read -r k; do \
d="$${k%/kustomization.yaml}"; \
printf 'render %s\n' "$$d"; \
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
done < <(find services -name kustomization.yaml | sort)
render-infrastructure:
@set -euo pipefail; \
while IFS= read -r k; do \
d="$${k%/kustomization.yaml}"; \
printf 'render %s\n' "$$d"; \
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
done < <(find infrastructure -name kustomization.yaml | sort)
validate: render test
dashboards:
$(PYTHON) scripts/render/dashboards_render_atlas.py --build
$(PYTHON) scripts/render/dashboards_render_logs.py --build
$(PYTHON) scripts/render/logging_render_observability.py --build
knowledge:
$(PYTHON) scripts/render/knowledge_render_atlas.py --write --sync-comms
test:
$(PYTHON) -m testing.quality_gate --profile local --build-dir $(BUILD_DIR)
flux-diff:
flux diff kustomization $(FLUX_KUSTOMIZATION) --namespace $(FLUX_NAMESPACE) --path $(FLUX_PATH)
flux-inventory:
$(PYTHON) scripts/render/flux_inventory.py $(ATLAS_FLUX_ROOT)
trivy-scan:
mkdir -p $(BUILD_DIR)
$(TRIVY) fs --cache-dir "$(TRIVY_CACHE_DIR)" $(TRIVY_DB_FLAGS) --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output $(BUILD_DIR)/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
security-report:
@test -s $(BUILD_DIR)/trivy-fs.json || { printf '%s\n' 'missing build/trivy-fs.json; run make trivy-scan first'; exit 1; }
$(PYTHON) ci/scripts/supply_chain_report.py --trivy-json $(BUILD_DIR)/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output $(BUILD_DIR)/ironbank-compliance.json
security: trivy-scan security-report
aether-fmt:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) fmt -check
aether-init:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) init
aether-validate:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) validate
aether-plan:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) plan

View File

@ -1,17 +0,0 @@
# atlas-iac
Flux-managed Kubernetes desired-state config for `bstein.dev`.
Canonical source URL:
- `ssh://git@scm.bstein.dev:2242/titan/atlas-iac.git`
## Scope
This repo contains cluster configuration consumed by Flux:
- platform/infrastructure manifests
- service manifests and kustomizations
- operational scripts for render/reconcile workflows
## Apply model
I use Git + Flux as the source of truth and avoid manual in-cluster edits for durable changes.

View File

@ -1,552 +0,0 @@
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-agent-image-builder
spec:
serviceAccountName: hermes-image-builder
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
nodeSelector:
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-04
- titan-08
- titan-14
- titan-18
- titan-19
- titan-22
- titan-24
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
- name: python
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
command: ["sleep"]
args: ["99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 250m
memory: 256Mi
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
privileged: false
runAsUser: 0
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 250m
memory: 1Gi
# The agent image build expands the SQLite and web build layers in
# its workspace. Reserve enough node-local space to keep it off the
# 25 GiB workers, whose unrequested Kaniko workspace was evicted.
ephemeral-storage: 32Gi
limits:
cpu: "2"
memory: 4Gi
ephemeral-storage: 40Gi
"""
}
}
parameters {
booleanParam(
name: 'PUBLISH_IMAGE',
defaultValue: false,
description: 'Publish the reviewed main revision to Harbor.'
)
string(
name: 'EXPECTED_SOURCE_REVISION',
defaultValue: '',
description: 'Full reviewed commit already contained by titan/atlas-iac main.'
)
string(
name: 'CONFIRM_PUBLISH',
defaultValue: '',
description: 'Enter PUBLISH HERMES AGENT to confirm the release.'
)
}
environment {
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-agent'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
skipDefaultCheckout(true)
timeout(time: 150, unit: 'MINUTES')
}
stages {
stage('Checkout reviewed source') {
steps {
checkout scm
}
}
stage('Enforce release boundary') {
steps {
container('jnlp') {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'')
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
exit 2
;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
main_revision="$(git rev-parse origin/main)"
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
test -z "$(git status --porcelain)"
test -f dockerfiles/Dockerfile.hermes-agent
case "${BUILD_NUMBER}" in
''|0*|*[!0-9]*)
echo "BUILD_NUMBER must be a positive decimal integer" >&2
exit 2
;;
esac
printf '%s\n' \
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
> build/hermes-agent.destination
printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
'''
}
}
}
stage('Validate reviewed release source') {
steps {
container('python') {
sh '''
set -eu
# Install the pinned test deps fully offline from the reviewed,
# in-repo wheelhouse (ci/vendor/hermes-agent-test-wheels). --no-index
# forbids any network index, so this stage never resolves
# pypi.org/files.pythonhosted.org and cannot fail on public-internet
# DNS. The wheels match this pipeline's arm64 python:3.12 container
# (PyYAML is the cp312 manylinux aarch64 build; the rest are
# py3-none-any). Bump the wheelhouse when these pins change.
python3 -m pip install --disable-pip-version-check --no-cache-dir \
--no-index --find-links="${WORKSPACE}/ci/vendor/hermes-agent-test-wheels" \
--target=/tmp/hermes-agent-release-test-deps \
pytest==8.3.4 PyYAML==6.0.2
PYTHONPATH=/tmp/hermes-agent-release-test-deps \
python3 -m pytest -q \
testing/tests/test_hermes_image_builder.py \
testing/tests/test_hermes_image_builder_adversarial.py \
testing/tests/test_hermes_image_builder_coverage.py \
testing/tests/test_hermes_image_builder_fresh_review.py \
testing/tests/test_hermes_oci_promote.py \
testing/tests/test_hermes_multiarch_combine.py \
testing/tests/test_hermes_image_automation.py
'''
}
}
}
stage('Reject replay before publish') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-agent.destination)"
source_revision="$(cat build/hermes-agent.source-revision)"
python3 ci/scripts/hermes_image_release.py assert-absent \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}"
'''
}
}
}
stage('Build arm64 leg without a daemon') {
steps {
container('kaniko') {
sh '''#!/busybox/sh
set -eu
minimum_available_kib=16777216
available_kib="$(/busybox/df -Pk / | /busybox/awk 'NR == 2 { print $4 }')"
case "${available_kib}" in
''|*[!0-9]*)
echo "cannot determine Kaniko ephemeral-storage availability" >&2
exit 2
;;
esac
if [ "${available_kib}" -lt "${minimum_available_kib}" ]; then
echo "Kaniko requires at least 16 GiB of free ephemeral storage" >&2
exit 1
fi
'''
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''#!/busybox/sh
set -eu
set +x
config_path=/kaniko/.docker/config.json
destination="$(cat build/hermes-agent.destination)-arm64"
source_revision="$(cat build/hermes-agent.source-revision)"
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
umask 022
/kaniko/executor \
--registry-mirror=harbor-core.harbor.svc.cluster.local \
--insecure-registry=harbor-core.harbor.svc.cluster.local \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
--destination="${destination}" \
--digest-file="${WORKSPACE}/build/hermes-agent-arm64.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-arm64.image" \
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
--label="org.opencontainers.image.revision=${source_revision}" \
--cleanup \
--push-retry=3
/busybox/chmod 644 build/hermes-agent-arm64.digest build/hermes-agent-arm64.image
'''
}
}
}
}
stage('Build amd64 leg without a daemon') {
agent {
kubernetes {
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-agent-image-builder-amd64
spec:
serviceAccountName: hermes-image-builder
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
# titan-24 is an accelerator node (not a general worker) that co-hosts the
# out-of-cluster Sui validator. Pin the disposable amd64 build to it by
# hostname + arch ONLY — do NOT require node-role worker, so titan-24 is never
# opened to general cluster scheduling. The toleration + tight caps below keep
# this off the validator's back.
nodeSelector:
kubernetes.io/arch: amd64
kubernetes.io/hostname: titan-24
tolerations:
# titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
# PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
# build lands, and rely on the tight resource caps below (not scheduling
# priority) to keep the disposable build from starving the validator.
- operator: Exists
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 128Mi
limits:
cpu: 250m
memory: 384Mi
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
privileged: false
runAsUser: 0
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: "1500m"
memory: 3Gi
"""
}
}
steps {
// This amd64 leg runs on its own fresh pod (titan-24), so it must check
// out the SCM itself before the reviewed-revision git boundary check —
// otherwise `git rev-parse origin/main` fails with "not a git repository".
checkout scm
container('jnlp') {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES AGENT"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'')
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
exit 2
;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
main_revision="$(git rev-parse origin/main)"
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
test -z "$(git status --porcelain)"
test -f dockerfiles/Dockerfile.hermes-agent
case "${BUILD_NUMBER}" in
''|0*|*[!0-9]*)
echo "BUILD_NUMBER must be a positive decimal integer" >&2
exit 2
;;
esac
printf '%s\n' \
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
> build/hermes-agent.destination
printf '%s\n' "${actual_revision}" > build/hermes-agent.source-revision
'''
}
container('kaniko') {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''#!/busybox/sh
set -eu
set +x
config_path=/kaniko/.docker/config.json
destination="$(cat build/hermes-agent.destination)-amd64"
source_revision="$(cat build/hermes-agent.source-revision)"
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
umask 022
/kaniko/executor \
--registry-mirror=harbor-core.harbor.svc.cluster.local \
--insecure-registry=harbor-core.harbor.svc.cluster.local \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-agent" \
--destination="${destination}" \
--digest-file="${WORKSPACE}/build/hermes-agent-amd64.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-agent-amd64.image" \
--build-arg=HERMES_KANIKO_HEREDOC_COMPAT=1 \
--label="org.opencontainers.image.revision=${source_revision}" \
--cleanup \
--push-retry=3
/busybox/chmod 644 build/hermes-agent-amd64.digest build/hermes-agent-amd64.image
'''
}
}
stash(
name: 'hermes-agent-amd64-evidence',
includes: 'build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image'
)
}
}
stage('Combine multi-arch index') {
steps {
unstash 'hermes-agent-amd64-evidence'
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-agent.destination)"
source_revision="$(cat build/hermes-agent.source-revision)"
python3 ci/scripts/hermes_multiarch_combine.py \
--destination "${destination}" \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--arm64-digest-file build/hermes-agent-arm64.digest \
--arm64-image-file build/hermes-agent-arm64.image \
--amd64-digest-file build/hermes-agent-amd64.digest \
--amd64-image-file build/hermes-agent-amd64.image \
--digest-file build/hermes-agent.digest \
--image-file build/hermes-agent.image
test -s build/hermes-agent.digest
test -s build/hermes-agent.image
'''
}
}
}
stage('Render reviewed Flux handoff') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-agent.destination)"
source_revision="$(cat build/hermes-agent.source-revision)"
python3 ci/scripts/hermes_image_release.py render \
--digest-file build/hermes-agent.digest \
--image-file build/hermes-agent.image \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--kustomization services/hermes/kustomization.yaml \
--output-dir build/hermes-agent-release
test -s build/hermes-agent-release/hermes-image-update.patch
test -s build/hermes-agent-release/hermes-agent-image.json
'''
}
}
}
stage('Verify and archive release evidence') {
steps {
sh '''
set -eu
expected_files="$(printf '%s\n' \
build/hermes-agent.destination \
build/hermes-agent.digest \
build/hermes-agent.image \
build/hermes-agent.source-revision \
build/hermes-agent-arm64.digest \
build/hermes-agent-arm64.image \
build/hermes-agent-amd64.digest \
build/hermes-agent-amd64.image \
build/hermes-agent-release/hermes-agent-image.json \
build/hermes-agent-release/hermes-image-update.patch \
build/hermes-agent-release/hermes-kustomization.yaml \
| LC_ALL=C sort)"
actual_files="$(find build -type f -print | LC_ALL=C sort)"
test "${actual_files}" = "${expected_files}"
destination="$(cat build/hermes-agent.destination)"
source_revision="$(cat build/hermes-agent.source-revision)"
python3 ci/scripts/hermes_image_release.py verify-evidence \
--digest-file build/hermes-agent.digest \
--image-file build/hermes-agent.image \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--kustomization services/hermes/kustomization.yaml \
--output-dir build/hermes-agent-release
'''
archiveArtifacts(
artifacts: 'build/hermes-agent.destination,build/hermes-agent.digest,build/hermes-agent.image,build/hermes-agent.source-revision,build/hermes-agent-arm64.digest,build/hermes-agent-arm64.image,build/hermes-agent-amd64.digest,build/hermes-agent-amd64.image,build/hermes-agent-release/hermes-agent-image.json,build/hermes-agent-release/hermes-image-update.patch,build/hermes-agent-release/hermes-kustomization.yaml',
allowEmptyArchive: false,
fingerprint: true
)
}
}
stage('Publish Flux release tag') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-agent.destination)"
source_revision="$(cat build/hermes-agent.source-revision)"
python3 ci/scripts/hermes_oci_promote.py \
--destination "${destination}" \
--digest-file build/hermes-agent.digest \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}"
'''
}
}
}
}
}

View File

@ -1,339 +0,0 @@
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-chat-router-image-builder
spec:
serviceAccountName: hermes-image-builder
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
nodeSelector:
kubernetes.io/arch: arm64
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: [titan-20]
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests: {cpu: 25m, memory: 128Mi}
limits: {cpu: 500m, memory: 512Mi}
- name: python
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
command: ["sleep"]
args: ["99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 250m, memory: 256Mi}
- name: golang
image: golang:1.24-alpine@sha256:8bee1901f1e530bfb4a7850aa7a479d17ae3a18beb6e09064ed54cfd245b7191
command: ["sleep"]
args: ["99d"]
tty: true
env:
- {name: GOCACHE, value: /tmp/go-cache}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests: {cpu: 100m, memory: 128Mi}
limits: {cpu: "1", memory: 1Gi}
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
privileged: false
runAsUser: 0
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 100m
memory: 256Mi
ephemeral-storage: 2Gi
limits:
cpu: "1"
memory: 2Gi
ephemeral-storage: 5Gi
"""
}
}
parameters {
booleanParam(
name: 'PUBLISH_IMAGE',
defaultValue: false,
description: 'Publish the reviewed main revision to Harbor.'
)
string(
name: 'EXPECTED_SOURCE_REVISION',
defaultValue: '',
description: 'Full reviewed commit that must be contained by titan/atlas-iac main.'
)
string(
name: 'CONFIRM_PUBLISH',
defaultValue: '',
description: 'Enter PUBLISH HERMES CHAT ROUTER to confirm the release.'
)
}
environment {
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-chat-router'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
skipDefaultCheckout(true)
timeout(time: 45, unit: 'MINUTES')
}
stages {
stage('Checkout reviewed source') {
steps {
checkout scm
}
}
stage('Enforce release boundary') {
steps {
container('jnlp') {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES CHAT ROUTER"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'')
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
exit 2
;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
main_revision="$(git rev-parse HEAD)"
test "${main_revision}" = "$(git rev-parse origin/main)"
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
test -z "$(git status --porcelain)"
test -f dockerfiles/Dockerfile.hermes-chat-router
case "${BUILD_NUMBER}" in
''|0*|*[!0-9]*)
echo "BUILD_NUMBER must be a positive decimal integer" >&2
exit 2
;;
esac
printf '%s\n' \
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
> build/hermes-chat-router.destination
printf '%s\n' "${actual_revision}" \
> build/hermes-chat-router.source-revision
'''
}
}
}
stage('Validate reviewed router source') {
steps {
container('python') {
sh '''
set -eu
python3 -m pip install --disable-pip-version-check --no-cache-dir \
--target=/tmp/hermes-chat-router-test-deps \
pytest==8.3.4 PyYAML==6.0.2
PYTHONPATH=/tmp/hermes-chat-router-test-deps \
python3 -m pytest -q \
testing/tests/test_hermes_chat_router_release.py \
testing/tests/test_hermes_oci_promote.py \
testing/tests/test_hermes_image_automation.py
'''
}
container('golang') {
sh '''
set -eu
cd services/hermes/router
GO111MODULE=off CGO_ENABLED=0 go test ./...
'''
}
}
}
stage('Reject replay before publish') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-chat-router.destination)"
source_revision="$(cat build/hermes-chat-router.source-revision)"
python3 ci/scripts/hermes_chat_router_release.py assert-absent \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}"
'''
}
}
}
stage('Build and publish without a daemon') {
steps {
container('kaniko') {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''#!/busybox/sh
set -eu
set +x
config_path=/kaniko/.docker/config.json
destination="$(cat build/hermes-chat-router.destination)"
source_revision="$(cat build/hermes-chat-router.source-revision)"
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
umask 022
/kaniko/executor \
--registry-mirror=harbor-core.harbor.svc.cluster.local \
--insecure-registry=harbor-core.harbor.svc.cluster.local \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-chat-router" \
--destination="${destination}" \
--digest-file="${WORKSPACE}/build/hermes-chat-router.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-chat-router.image" \
--label="org.opencontainers.image.revision=${source_revision}" \
--label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
--label="org.opencontainers.image.title=hermes-chat-router" \
--cleanup \
--push-retry=3
/busybox/chmod 644 \
build/hermes-chat-router.digest build/hermes-chat-router.image
'''
}
}
}
}
stage('Render reviewed Flux handoff') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-chat-router.destination)"
source_revision="$(cat build/hermes-chat-router.source-revision)"
python3 ci/scripts/hermes_chat_router_release.py render \
--digest-file build/hermes-chat-router.digest \
--image-file build/hermes-chat-router.image \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--manifest services/hermes/chat-router.yaml \
--output-dir build/hermes-chat-router-release
'''
}
}
}
stage('Verify and archive release evidence') {
steps {
sh '''
set -eu
expected_files="$(printf '%s\n' \
build/hermes-chat-router.destination \
build/hermes-chat-router.digest \
build/hermes-chat-router.image \
build/hermes-chat-router.source-revision \
build/hermes-chat-router-release/hermes-chat-router-deployment.yaml \
build/hermes-chat-router-release/hermes-chat-router-image-update.patch \
build/hermes-chat-router-release/hermes-chat-router-image.json \
| LC_ALL=C sort)"
actual_files="$(find build -type f -print | LC_ALL=C sort)"
test "${actual_files}" = "${expected_files}"
destination="$(cat build/hermes-chat-router.destination)"
source_revision="$(cat build/hermes-chat-router.source-revision)"
python3 ci/scripts/hermes_chat_router_release.py verify-evidence \
--digest-file build/hermes-chat-router.digest \
--image-file build/hermes-chat-router.image \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--manifest services/hermes/chat-router.yaml \
--output-dir build/hermes-chat-router-release
'''
archiveArtifacts(
artifacts: 'build/hermes-chat-router.destination,build/hermes-chat-router.digest,build/hermes-chat-router.image,build/hermes-chat-router.source-revision,build/hermes-chat-router-release/hermes-chat-router-deployment.yaml,build/hermes-chat-router-release/hermes-chat-router-image-update.patch,build/hermes-chat-router-release/hermes-chat-router-image.json',
allowEmptyArchive: false,
fingerprint: true
)
}
}
stage('Publish Flux release tag') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-chat-router.destination)"
source_revision="$(cat build/hermes-chat-router.source-revision)"
python3 ci/scripts/hermes_oci_promote.py \
--destination "${destination}" \
--digest-file build/hermes-chat-router.digest \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}"
'''
}
}
}
}
}

View File

@ -1,236 +0,0 @@
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-voice-image-builder
spec:
serviceAccountName: hermes-image-builder
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
nodeSelector:
kubernetes.io/arch: arm64
tolerations:
# Jetson kubelet/network maintenance can briefly outlast Kubernetes' five
# minute default. Keep the disposable build workspace intact long enough
# for the node to recover instead of restarting a large image expansion.
- key: node.kubernetes.io/not-ready
operator: Exists
effect: NoExecute
tolerationSeconds: 600
- key: node.kubernetes.io/unreachable
operator: Exists
effect: NoExecute
tolerationSeconds: 600
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
# Voice images expand the large Jetson base and two Whisper
# models. Keep that transient I/O on the roomy idle
# accelerator, away from Longhorn and the live speech node.
values: [titan-20]
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: ["ALL"]}
runAsNonRoot: true
runAsUser: 1000
seccompProfile: {type: RuntimeDefault}
resources:
requests: {cpu: 25m, memory: 256Mi}
limits: {cpu: 500m, memory: 512Mi}
- name: python
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
command: ["sleep"]
args: ["99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities: {drop: ["ALL"]}
runAsNonRoot: true
runAsUser: 1000
seccompProfile: {type: RuntimeDefault}
resources:
requests: {cpu: 25m, memory: 64Mi}
limits: {cpu: 500m, memory: 512Mi}
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
# The pinned Jetson Whisper base contains gst-ptp-helper with a
# security.capability xattr. Kaniko needs SETFCAP only while
# unpacking that reviewed base; the pod remains non-privileged.
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID", "SETFCAP"]
runAsUser: 0
seccompProfile: {type: RuntimeDefault}
resources:
# The STT image snapshots two checksum-pinned Whisper models. A 1 GiB
# request let the builder overpack titan-20 and a 4 GiB cgroup killed
# Kaniko while it emitted the large model layer.
requests: {cpu: 250m, memory: 2Gi, ephemeral-storage: 10Gi}
limits: {cpu: "2", memory: 6Gi, ephemeral-storage: 20Gi}
"""
}
}
parameters {
booleanParam(name: 'PUBLISH_IMAGE', defaultValue: false, description: 'Publish the reviewed voice image to Harbor.')
choice(name: 'IMAGE_COMPONENT', choices: ['stt', 'tts'], description: 'Private voice component to build.')
string(name: 'EXPECTED_SOURCE_REVISION', defaultValue: '', description: 'Full reviewed commit contained by main.')
string(name: 'CONFIRM_PUBLISH', defaultValue: '', description: 'Exact component-specific confirmation.')
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
skipDefaultCheckout(true)
timeout(time: 150, unit: 'MINUTES')
}
stages {
stage('Checkout reviewed source') {
steps { checkout scm }
}
stage('Enforce release boundary') {
steps {
container('jnlp') {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
case "${IMAGE_COMPONENT}" in
stt) expected_confirmation='PUBLISH HERMES STT' ;;
tts) expected_confirmation='PUBLISH HERMES TTS' ;;
*) echo 'IMAGE_COMPONENT must be stt or tts' >&2; exit 2 ;;
esac
test "${CONFIRM_PUBLISH}" = "${expected_confirmation}"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'') echo 'EXPECTED_SOURCE_REVISION must be a lowercase full commit' >&2; exit 2 ;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
main_revision="$(git rev-parse HEAD)"
test "${main_revision}" = "$(git rev-parse origin/main)"
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
test -z "$(git status --porcelain)"
case "${BUILD_NUMBER}" in ''|0*|*[!0-9]*) exit 2 ;; esac
image="registry.bstein.dev/bstein/hermes-jetson-${IMAGE_COMPONENT}"
printf '%s\n' "${image}:git-${actual_revision}-build-${BUILD_NUMBER}" > build/hermes-voice.destination
printf '%s\n' "${actual_revision}" > build/hermes-voice.source-revision
printf '%s\n' "${IMAGE_COMPONENT}" > build/hermes-voice.component
test -f "dockerfiles/Dockerfile.hermes-jetson-${IMAGE_COMPONENT}"
'''
}
}
}
stage('Validate reviewed voice source') {
steps {
container('python') {
sh '''
set -eu
python3 -m pip install --disable-pip-version-check --no-cache-dir \
--target=/tmp/hermes-voice-test-deps pytest==8.3.4 PyYAML==6.0.2
python3 -m py_compile \
dockerfiles/hermes-jetson-stt-server.py \
dockerfiles/hermes-jetson-tts-server.py \
dockerfiles/hermes_jetson_tts_cues.py
PYTHONPATH=/tmp/hermes-voice-test-deps python3 -m pytest -q \
testing/tests/test_hermes_stt_streaming.py \
testing/tests/test_hermes_stt_rolling_model.py \
testing/tests/test_hermes_tts_language_routing.py \
testing/tests/test_hermes_voice_language_routing.py \
testing/tests/test_hermes_oci_promote.py \
testing/tests/test_hermes_image_automation.py \
testing/tests/test_hermes_voice_release.py
'''
}
}
}
stage('Build and publish without a daemon') {
steps {
container('kaniko') {
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USER', passwordVariable: 'HARBOR_PASSWORD')]) {
sh '''#!/busybox/sh
set -eu
set +x
component="$(cat build/hermes-voice.component)"
destination="$(cat build/hermes-voice.destination)"
source_revision="$(cat build/hermes-voice.source-revision)"
config_path=/kaniko/.docker/config.json
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
umask 022
/kaniko/executor \
--registry-mirror=harbor-core.harbor.svc.cluster.local \
--insecure-registry=harbor-core.harbor.svc.cluster.local \
--compressed-caching=false \
--snapshot-mode=redo \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-jetson-${component}" \
--destination="${destination}" \
--digest-file="${WORKSPACE}/build/hermes-voice.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-voice.image" \
--label="org.opencontainers.image.revision=${source_revision}" \
--label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
--label="org.opencontainers.image.title=hermes-jetson-${component}" \
--cleanup --push-retry=3
/busybox/chmod 644 build/hermes-voice.digest build/hermes-voice.image
'''
}
}
}
}
stage('Verify, archive, and publish Flux release') {
steps {
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USER', passwordVariable: 'HARBOR_PASSWORD')]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-voice.destination)"
source_revision="$(cat build/hermes-voice.source-revision)"
component="$(cat build/hermes-voice.component)"
digest="$(cat build/hermes-voice.digest)"
image="$(cat build/hermes-voice.image)"
test "${image}" = "${destination}@${digest}"
python3 ci/scripts/hermes_oci_promote.py \
--destination "${destination}" \
--digest-file build/hermes-voice.digest \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
> build/hermes-voice.promotion.json
python3 -c 'import json; data=json.load(open("build/hermes-voice.promotion.json")); assert data["result"] in {"published", "already-present"}'
'''
archiveArtifacts(
artifacts: 'build/hermes-voice.component,build/hermes-voice.destination,build/hermes-voice.digest,build/hermes-voice.image,build/hermes-voice.source-revision,build/hermes-voice.promotion.json',
allowEmptyArchive: false,
fingerprint: true
)
}
}
}
}
}

View File

@ -1,559 +0,0 @@
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-webui-image-builder
spec:
serviceAccountName: hermes-image-builder
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
nodeSelector:
kubernetes.io/arch: arm64
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
# Keep disposable Kaniko expansion off the astreae Longhorn
# replica nodes. titan-20 is the roomy ARM image builder and
# has local NVMe for this transient I/O.
values:
- titan-20
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
- name: python
image: registry.bstein.dev/bstein/python@sha256:269541d3387baae008df4608ead893dba2b5cdaad1a5a380731a88992d34b808
command: ["sleep"]
args: ["99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
runAsNonRoot: false
runAsUser: 0
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 64Mi
limits:
cpu: 250m
memory: 256Mi
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
privileged: false
runAsUser: 0
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 250m
memory: 1Gi
ephemeral-storage: 10Gi
limits:
cpu: "2"
memory: 4Gi
ephemeral-storage: 20Gi
"""
}
}
parameters {
booleanParam(
name: 'PUBLISH_IMAGE',
defaultValue: false,
description: 'Publish the reviewed main revision to Harbor.'
)
string(
name: 'EXPECTED_SOURCE_REVISION',
defaultValue: '',
description: 'Full reviewed commit that must be contained by titan/atlas-iac main.'
)
string(
name: 'CONFIRM_PUBLISH',
defaultValue: '',
description: 'Enter PUBLISH HERMES WEBUI to confirm the release.'
)
}
environment {
HERMES_IMAGE = 'registry.bstein.dev/bstein/hermes-webui'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '100', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '100'))
skipDefaultCheckout(true)
timeout(time: 150, unit: 'MINUTES')
}
stages {
stage('Checkout reviewed source') {
steps {
checkout scm
}
}
stage('Enforce release boundary') {
steps {
container('jnlp') {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'')
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
exit 2
;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
main_revision="$(git rev-parse HEAD)"
test "${main_revision}" = "$(git rev-parse origin/main)"
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
test -z "$(git status --porcelain)"
test -f dockerfiles/Dockerfile.hermes-webui
case "${BUILD_NUMBER}" in
''|0*|*[!0-9]*)
echo "BUILD_NUMBER must be a positive decimal integer" >&2
exit 2
;;
esac
printf '%s\n' \
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
> build/hermes-webui.destination
printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision
'''
}
}
}
stage('Validate reviewed WebUI source') {
steps {
container('python') {
sh '''
set -eu
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y --no-install-recommends ffmpeg nodejs
rm -rf /var/lib/apt/lists/*
command -v ffmpeg >/dev/null
command -v node >/dev/null
python3 -m pip install --disable-pip-version-check --no-cache-dir \
--target=/tmp/hermes-webui-release-test-deps \
pytest==8.3.4 PyYAML==6.0.2
PYTHONPATH=/tmp/hermes-webui-release-test-deps \
python3 -m pytest -q \
testing/tests/test_hermes_chat_quality.py \
testing/tests/test_hermes_handsfree_stt.py \
testing/tests/test_hermes_voice_instrument.py \
testing/tests/test_hermes_voice_full_duplex.py \
testing/tests/test_hermes_voice_route_preflight.py \
testing/tests/test_hermes_voice_preflight_delivery.py \
testing/tests/test_hermes_thinking_voice_cues.py \
testing/tests/test_hermes_voice_language_routing.py \
testing/tests/test_hermes_webui_brand.py \
testing/tests/test_hermes_webui_release.py \
testing/tests/test_hermes_webui_hux_bff.py \
testing/tests/test_hermes_webui_hux_context.py \
testing/tests/test_hermes_webui_hux_integration.py \
testing/tests/test_hermes_webui_hux_backend_e2e.py \
testing/tests/test_hermes_hux_ui_runtime_wave_a.py \
testing/tests/test_hermes_hux_runtime_autonomy_privacy.py \
testing/tests/test_hermes_hux_runtime_stop.py \
testing/tests/test_hermes_hux_ui_runtime_wave_b.py \
testing/tests/test_hermes_hux_runtime_wave_c.py \
testing/tests/test_hermes_hux_runtime_plugin.py \
testing/tests/test_hermes_hux_runtime_vendor_parity.py \
testing/tests/test_hermes_hux_delivery.py \
testing/tests/test_hermes_oci_promote.py \
testing/tests/test_hermes_multiarch_combine.py \
testing/tests/test_hermes_image_automation.py
HUX_BACKEND_TESTS="$(find testing/tests -maxdepth 1 -type f \
-name 'test_hermes_hux_*.py' \
! -name '*_ui_*' \
! -name '*runtime*' \
! -name '*delivery*' \
| sort)"
test -n "${HUX_BACKEND_TESTS}"
PYTHONPATH=/tmp/hermes-webui-release-test-deps \
python3 -m pytest -q ${HUX_BACKEND_TESTS}
'''
}
}
}
stage('Reject replay before publish') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-webui.destination)"
source_revision="$(cat build/hermes-webui.source-revision)"
python3 ci/scripts/hermes_webui_release.py assert-absent \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}"
'''
}
}
}
stage('Build arm64 leg without a daemon') {
steps {
container('kaniko') {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''#!/busybox/sh
set -eu
set +x
config_path=/kaniko/.docker/config.json
destination="$(cat build/hermes-webui.destination)-arm64"
source_revision="$(cat build/hermes-webui.source-revision)"
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
umask 022
/kaniko/executor \
--registry-mirror=harbor-core.harbor.svc.cluster.local \
--insecure-registry=harbor-core.harbor.svc.cluster.local \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-webui" \
--destination="${destination}" \
--build-arg="HERMES_WEBUI_RELEASE_ID=git-${source_revision}-build-${BUILD_NUMBER}" \
--digest-file="${WORKSPACE}/build/hermes-webui-arm64.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui-arm64.image" \
--label="org.opencontainers.image.revision=${source_revision}" \
--label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
--label="org.opencontainers.image.title=hermes-webui" \
--cleanup \
--push-retry=3
/busybox/chmod 644 build/hermes-webui-arm64.digest build/hermes-webui-arm64.image
'''
}
}
}
}
stage('Build amd64 leg without a daemon') {
agent {
kubernetes {
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
atlas.bstein.dev/workload: hermes-webui-image-builder-amd64
spec:
serviceAccountName: hermes-image-builder
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
securityContext:
fsGroup: 1000
fsGroupChangePolicy: OnRootMismatch
# titan-24 is an accelerator node (not a general worker) that co-hosts the
# out-of-cluster Sui validator. Pin the disposable amd64 build to it by
# hostname + arch ONLY — do NOT require node-role worker, so titan-24 is never
# opened to general cluster scheduling. The toleration + tight caps below keep
# this off the validator's back.
nodeSelector:
kubernetes.io/arch: amd64
kubernetes.io/hostname: titan-24
tolerations:
# titan-24 co-hosts the out-of-cluster Sui validator; tolerate whatever
# PreferNoSchedule/NoSchedule guard taint the node carries so the pinned
# build lands, and rely on the tight resource caps below (not scheduling
# priority) to keep the disposable build from starving the validator.
- operator: Exists
imagePullSecrets:
- name: harbor-bstein-robot
containers:
- name: jnlp
image: jenkins/inbound-agent@sha256:8eda4fe2a66bcf6a5e43436d9918fc14c306204dc8fcd75f4e15e0e6e5dc759a
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 25m
memory: 128Mi
limits:
cpu: 250m
memory: 384Mi
- name: kaniko
image: gcr.io/kaniko-project/executor@sha256:c3109d5926a997b100c4343944e06c6b30a6804b2f9abe0994d3de6ef92b028e
command: ["/busybox/sh", "-c"]
args: ["/busybox/sleep 99d"]
tty: true
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
add: ["CHOWN", "FOWNER", "DAC_OVERRIDE", "SETGID", "SETUID"]
privileged: false
runAsUser: 0
seccompProfile:
type: RuntimeDefault
resources:
requests:
cpu: 100m
memory: 512Mi
ephemeral-storage: 10Gi
limits:
cpu: "1500m"
memory: 3Gi
ephemeral-storage: 20Gi
"""
}
}
steps {
// This amd64 leg runs on its own fresh pod (titan-24), so it must check
// out the SCM itself before the reviewed-revision git boundary check —
// otherwise `git rev-parse origin/main` fails with "not a git repository".
checkout scm
container('jnlp') {
sh '''
set -eu
mkdir -p build
test "${PUBLISH_IMAGE}" = "true"
test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"
case "${EXPECTED_SOURCE_REVISION}" in
*[!0-9a-f]*|'')
echo "EXPECTED_SOURCE_REVISION must be a lowercase full commit" >&2
exit 2
;;
esac
test "${#EXPECTED_SOURCE_REVISION}" -eq 40
main_revision="$(git rev-parse HEAD)"
test "${main_revision}" = "$(git rev-parse origin/main)"
git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}" "${main_revision}"
git checkout --detach "${EXPECTED_SOURCE_REVISION}"
actual_revision="$(git rev-parse HEAD)"
test "${actual_revision}" = "${EXPECTED_SOURCE_REVISION}"
test -z "$(git status --porcelain)"
test -f dockerfiles/Dockerfile.hermes-webui
case "${BUILD_NUMBER}" in
''|0*|*[!0-9]*)
echo "BUILD_NUMBER must be a positive decimal integer" >&2
exit 2
;;
esac
printf '%s\n' \
"${HERMES_IMAGE}:git-${actual_revision}-build-${BUILD_NUMBER}" \
> build/hermes-webui.destination
printf '%s\n' "${actual_revision}" > build/hermes-webui.source-revision
'''
}
container('kaniko') {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''#!/busybox/sh
set -eu
set +x
config_path=/kaniko/.docker/config.json
destination="$(cat build/hermes-webui.destination)-amd64"
source_revision="$(cat build/hermes-webui.source-revision)"
umask 077
auth="$(printf '%s:%s' "${HARBOR_USER}" "${HARBOR_PASSWORD}" | /busybox/base64 | /busybox/tr -d '\n')"
/busybox/mkdir -p /kaniko/.docker
/busybox/printf '{"auths":{"registry.bstein.dev":{"auth":"%s"}}}\n' "${auth}" > "${config_path}"
unset HARBOR_USER HARBOR_PASSWORD auth
trap '/busybox/rm -f "${config_path}"' EXIT HUP INT TERM
umask 022
/kaniko/executor \
--registry-mirror=harbor-core.harbor.svc.cluster.local \
--insecure-registry=harbor-core.harbor.svc.cluster.local \
--context="dir://${WORKSPACE}" \
--dockerfile="${WORKSPACE}/dockerfiles/Dockerfile.hermes-webui" \
--destination="${destination}" \
--build-arg="HERMES_WEBUI_RELEASE_ID=git-${source_revision}-build-${BUILD_NUMBER}" \
--digest-file="${WORKSPACE}/build/hermes-webui-amd64.digest" \
--image-name-tag-with-digest-file="${WORKSPACE}/build/hermes-webui-amd64.image" \
--label="org.opencontainers.image.revision=${source_revision}" \
--label="org.opencontainers.image.source=https://scm.bstein.dev/titan/atlas-iac" \
--label="org.opencontainers.image.title=hermes-webui" \
--cleanup \
--push-retry=3
/busybox/chmod 644 build/hermes-webui-amd64.digest build/hermes-webui-amd64.image
'''
}
}
stash(
name: 'hermes-webui-amd64-evidence',
includes: 'build/hermes-webui-amd64.digest,build/hermes-webui-amd64.image'
)
}
}
stage('Combine multi-arch index') {
steps {
unstash 'hermes-webui-amd64-evidence'
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-webui.destination)"
source_revision="$(cat build/hermes-webui.source-revision)"
python3 ci/scripts/hermes_multiarch_combine.py \
--destination "${destination}" \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--arm64-digest-file build/hermes-webui-arm64.digest \
--arm64-image-file build/hermes-webui-arm64.image \
--amd64-digest-file build/hermes-webui-amd64.digest \
--amd64-image-file build/hermes-webui-amd64.image \
--digest-file build/hermes-webui.digest \
--image-file build/hermes-webui.image
test -s build/hermes-webui.digest
test -s build/hermes-webui.image
'''
}
}
}
stage('Render reviewed Flux handoff') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-webui.destination)"
source_revision="$(cat build/hermes-webui.source-revision)"
python3 ci/scripts/hermes_webui_release.py render \
--digest-file build/hermes-webui.digest \
--image-file build/hermes-webui.image \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--chat-manifest services/hermes/chat-statefulset.yaml \
--dashboard-manifest services/hermes/deployment.yaml \
--output-dir build/hermes-webui-release
test -s build/hermes-webui-release/hermes-webui-image-update.patch
test -s build/hermes-webui-release/hermes-webui-image.json
'''
}
}
}
stage('Verify and archive release evidence') {
steps {
sh '''
set -eu
expected_files="$(printf '%s\n' \
build/hermes-webui.destination \
build/hermes-webui.digest \
build/hermes-webui.image \
build/hermes-webui.source-revision \
build/hermes-webui-arm64.digest \
build/hermes-webui-arm64.image \
build/hermes-webui-amd64.digest \
build/hermes-webui-amd64.image \
build/hermes-webui-release/hermes-chat-statefulset.yaml \
build/hermes-webui-release/hermes-dashboard-deployment.yaml \
build/hermes-webui-release/hermes-webui-image.json \
build/hermes-webui-release/hermes-webui-image-update.patch \
| LC_ALL=C sort)"
actual_files="$(find build -type f -print | LC_ALL=C sort)"
test "${actual_files}" = "${expected_files}"
destination="$(cat build/hermes-webui.destination)"
source_revision="$(cat build/hermes-webui.source-revision)"
python3 ci/scripts/hermes_webui_release.py verify-evidence \
--digest-file build/hermes-webui.digest \
--image-file build/hermes-webui.image \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}" \
--destination "${destination}" \
--chat-manifest services/hermes/chat-statefulset.yaml \
--dashboard-manifest services/hermes/deployment.yaml \
--output-dir build/hermes-webui-release
'''
archiveArtifacts(
artifacts: 'build/hermes-webui.destination,build/hermes-webui.digest,build/hermes-webui.image,build/hermes-webui.source-revision,build/hermes-webui-arm64.digest,build/hermes-webui-arm64.image,build/hermes-webui-amd64.digest,build/hermes-webui-amd64.image,build/hermes-webui-release/hermes-chat-statefulset.yaml,build/hermes-webui-release/hermes-dashboard-deployment.yaml,build/hermes-webui-release/hermes-webui-image.json,build/hermes-webui-release/hermes-webui-image-update.patch',
allowEmptyArchive: false,
fingerprint: true
)
}
}
stage('Publish Flux release tag') {
steps {
withCredentials([usernamePassword(
credentialsId: 'harbor-robot',
usernameVariable: 'HARBOR_USER',
passwordVariable: 'HARBOR_PASSWORD'
)]) {
sh '''
set -eu
set +x
destination="$(cat build/hermes-webui.destination)"
source_revision="$(cat build/hermes-webui.source-revision)"
python3 ci/scripts/hermes_oci_promote.py \
--destination "${destination}" \
--digest-file build/hermes-webui.digest \
--source-revision "${source_revision}" \
--build-number "${BUILD_NUMBER}"
'''
}
}
}
}
}

View File

@ -1,511 +0,0 @@
pipeline {
agent {
kubernetes {
defaultContainer 'python'
yaml """
apiVersion: v1
kind: Pod
spec:
nodeSelector:
kubernetes.io/arch: arm64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-04
- titan-06
- titan-11
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-19
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
jenkins/jenkins-jenkins-agent: "true"
containers:
- name: jnlp
image: jenkins/inbound-agent:3355.v388858a_47b_33-2-jdk21
resources:
requests:
cpu: "25m"
memory: "256Mi"
- name: python
image: registry.bstein.dev/bstein/python:3.12-slim
command:
- cat
tty: true
- name: quality-tools
image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
command:
- cat
tty: true
- name: semgrep
image: semgrep/semgrep:1.171.0
command:
- cat
tty: true
"""
}
}
environment {
PIP_DISABLE_PIP_VERSION_CHECK = '1'
PYTHONUNBUFFERED = '1'
SUITE_NAME = 'titan_iac'
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
SONARQUBE_PROJECT_KEY = 'titan_iac'
SONARQUBE_TOKEN = credentials('sonarqube-token')
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
QUALITY_GATE_SEMGREP_ENFORCE = '0'
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
QUALITY_GATE_IRONBANK_ENFORCE = '1'
QUALITY_GATE_IRONBANK_REQUIRED = '0'
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
}
options {
disableConcurrentBuilds()
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
}
stages {
stage('Checkout') {
steps {
checkout scm
}
}
stage('Install deps') {
steps {
sh '''
set -eu
if ! command -v git >/dev/null 2>&1; then
apt-get update
apt-get install -y --no-install-recommends git ca-certificates
rm -rf /var/lib/apt/lists/*
fi
pip install --no-cache-dir -r ci/requirements.txt
'''
}
}
stage('Prepare local quality evidence') {
steps {
sh '''
set -eu
mkdir -p build
set +e
python3 -m testing.quality_gate --profile local --build-dir build
local_quality_rc=$?
set -e
printf '%s\n' "${local_quality_rc}" > build/local-quality-gate.rc
'''
}
}
stage('Collect Semgrep evidence') {
steps {
container('semgrep') {
sh '''#!/bin/sh
set -eu
mkdir -p build
set +e
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
semgrep_rc=$?
set -e
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
'''
}
sh '''
set -eu
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
'''
}
}
stage('Collect SonarQube evidence') {
steps {
container('quality-tools') {
sh '''#!/usr/bin/env bash
set -euo pipefail
mkdir -p build
args=(
"-Dsonar.host.url=${SONARQUBE_HOST_URL}"
"-Dsonar.login=${SONARQUBE_TOKEN}"
"-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
"-Dsonar.sources=."
"-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**,services/monitoring/dashboards/**,services/monitoring/grafana-dashboard-*.yaml,services/game-stream/**"
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
)
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
set +e
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
rc=${PIPESTATUS[0]}
set -e
printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
'''
}
sh '''
set -eu
mkdir -p build
python3 - <<'PY'
import base64
import json
import os
import time
import urllib.parse
import urllib.request
from pathlib import Path
host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
token = os.getenv('SONARQUBE_TOKEN', '').strip()
report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
payload = {
"status": "ERROR",
"note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY",
}
if host and project_key:
task_file = Path('.scannerwork/report-task.txt')
task_id = ''
if task_file.exists():
for line in task_file.read_text(encoding='utf-8').splitlines():
key, _, value = line.partition('=')
if key == 'ceTaskId':
task_id = value.strip()
break
if task_id:
ce_query = urllib.parse.urlencode({"id": task_id})
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
ce_request = urllib.request.Request(f"{host}/api/ce/task?{ce_query}", method="GET")
if token:
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
ce_request.add_header("Authorization", f"Basic {encoded}")
try:
with urllib.request.urlopen(ce_request, timeout=12) as response:
ce_payload = json.loads(response.read().decode("utf-8"))
except Exception:
time.sleep(3)
continue
status = str(ce_payload.get("task", {}).get("status", "")).upper()
if status in {"SUCCESS", "FAILED", "CANCELED"}:
break
time.sleep(3)
query = urllib.parse.urlencode({"projectKey": project_key})
request = urllib.request.Request(
f"{host}/api/qualitygates/project_status?{query}",
method="GET",
)
if token:
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
request.add_header("Authorization", f"Basic {encoded}")
try:
with urllib.request.urlopen(request, timeout=12) as response:
payload = json.loads(response.read().decode("utf-8"))
except Exception as exc: # noqa: BLE001
payload = {"status": "ERROR", "error": str(exc)}
with open(report_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2, sort_keys=True)
handle.write("\\n")
PY
'''
}
}
stage('Collect IronBank evidence') {
steps {
container('quality-tools') {
sh '''#!/usr/bin/env bash
set -euo pipefail
mkdir -p build
set +e
trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --skip-check-update --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
trivy_rc=$?
set -e
if [ "${trivy_rc}" -ne 0 ]; then
rm -f build/trivy-fs.json
cat > build/ironbank-compliance.json <<EOF
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy scan failed","trivy_rc":${trivy_rc}}
EOF
exit 0
fi
if [ ! -s build/trivy-fs.json ]; then
cat > build/ironbank-compliance.json <<EOF
{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
EOF
exit 0
fi
'''
}
sh '''
set -eu
mkdir -p build
if [ -s build/trivy-fs.json ]; then
python3 ci/scripts/supply_chain_report.py --trivy-json build/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output build/ironbank-compliance.json
exit 0
fi
python3 - <<'PY'
import json
import os
from pathlib import Path
report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
if report_path.exists():
raise SystemExit(0)
status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
payload = {
"status": status or "unknown",
"compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
}
payload = {k: v for k, v in payload.items() if v is not None}
if "status" not in payload:
payload["status"] = "unknown"
payload["note"] = (
"Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
"or write build/ironbank-compliance.json in image-building repos."
)
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
PY
'''
}
}
stage('Run quality gate') {
steps {
sh '''
set -eu
mkdir -p build
set +e
python3 -m testing.quality_gate --profile jenkins --build-dir build
quality_gate_rc=$?
set -e
printf '%s\n' "${quality_gate_rc}" > build/quality-gate.rc
'''
}
}
stage('Publish test metrics') {
steps {
sh '''
set -eu
export JUNIT_GLOB='build/junit-*.xml'
export QUALITY_GATE_EXIT_CODE_PATH='build/quality-gate.rc'
export QUALITY_GATE_SUMMARY_PATH='build/quality-gate-summary.json'
python3 ci/scripts/publish_test_metrics.py
'''
}
}
stage('Enforce quality gate') {
steps {
sh '''
set -euo pipefail
gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)"
fail=0
if [ "${gate_rc}" -ne 0 ]; then
echo "quality gate failed with rc=${gate_rc}" >&2
fail=1
fi
enabled() {
case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
1|true|yes|on) return 0 ;;
*) return 1 ;;
esac
}
if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
sonar_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/sonarqube-quality-gate.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${sonar_status}" in
ok|pass|passed|success) ;;
*)
echo "sonarqube gate failed: ${sonar_status}" >&2
fail=1
;;
esac
fi
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
semgrep_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/semgrep-report.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = str(payload.get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${semgrep_status}" in
ok|pass|passed|success) ;;
*)
echo "semgrep gate failed: ${semgrep_status}" >&2
fail=1
;;
esac
fi
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
ironbank_required=1
fi
if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
supply_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/ironbank-compliance.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
compliant = payload.get("compliant")
if compliant is True:
print("ok")
elif compliant is False:
print("failed")
else:
status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
print(status or "missing")
PY
)"
case "${supply_status}" in
ok|pass|passed|success|compliant) ;;
not_applicable|na|n/a)
if enabled "${ironbank_required}"; then
echo "supply chain gate required but status=${supply_status}" >&2
fail=1
fi
;;
*)
if enabled "${ironbank_required}"; then
echo "supply chain gate failed: ${supply_status}" >&2
fail=1
else
echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
fi
;;
esac
fi
exit "${fail}"
'''
}
}
stage('Resolve Flux branch') {
steps {
script {
env.FLUX_BRANCH = sh(
returnStdout: true,
script: "grep -m1 '^\\s*branch:' clusters/atlas/flux-system/gotk-sync.yaml | sed 's/^\\s*branch:\\s*//'"
).trim()
if (!env.FLUX_BRANCH) {
error('Flux branch not found in gotk-sync.yaml')
}
echo "Flux branch: ${env.FLUX_BRANCH}"
}
}
}
stage('Promote') {
when {
expression {
def branch = env.BRANCH_NAME ?: (env.GIT_BRANCH ?: '').replaceFirst('origin/', '')
return env.FLUX_BRANCH && branch == env.FLUX_BRANCH
}
}
steps {
withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
sh '''
set -euo pipefail
if ! command -v git >/dev/null 2>&1; then
if command -v apk >/dev/null 2>&1; then
apk add --no-cache git >/dev/null
elif command -v apt-get >/dev/null 2>&1; then
apt-get update >/dev/null
apt-get install -y git >/dev/null
fi
fi
cd "${WORKSPACE:-$PWD}"
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "workspace is not a git checkout; skipping promote"
exit 0
fi
set +x
git config user.email "jenkins@bstein.dev"
git config user.name "jenkins"
git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/titan/atlas-iac.git
git push origin HEAD:${FLUX_BRANCH}
'''
}
}
}
}
post {
always {
script {
try {
if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
try {
junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
} catch (Throwable err) {
echo "junit step unavailable: ${err.class.simpleName}"
}
}
archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
} catch (Throwable err) {
if (err.class.simpleName == 'MissingContextVariableException') {
echo 'workspace unavailable; skipping post-build artifact collection'
} else {
throw err
}
}
}
}
}
}

View File

@ -1,7 +0,0 @@
pytest==8.3.4
pytest-cov==6.0.0
coverage==7.6.10
kubernetes==30.1.0
PyYAML==6.0.2
requests==2.32.3
ruff==0.8.4

View File

@ -1,479 +0,0 @@
#!/usr/bin/env python3
"""Verify and render a reviewable Hermes chat-router image release."""
from __future__ import annotations
import argparse
import base64
import difflib
import json
import os
import re
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Callable
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-chat-router"
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
HARBOR_PROJECT = "bstein"
HARBOR_REPOSITORY = "hermes-chat-router"
IMMUTABLE_REPOSITORY_PATTERN = "hermes-chat-router"
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
DESTINATION_PATTERN = re.compile(
r"^registry\.bstein\.dev/bstein/hermes-chat-router:"
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
)
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Never forward registry credentials to a redirect target."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
"""Return a normalized value only when it matches the release contract."""
normalized = value.strip()
if not pattern.fullmatch(normalized):
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
return normalized
def validate_destination(
destination: str, source_revision: str, build_number: str
) -> tuple[str, str]:
"""Bind the unique build tag to one reviewed source and Jenkins build."""
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
build = _validated(build_number, BUILD_PATTERN, "build number")
match = DESTINATION_PATTERN.fullmatch(destination.strip())
if not match or match.groups() != (revision, build):
raise ValueError("destination does not match the reviewed revision and build")
return revision, build
def validate_kaniko_evidence(
*, digest_text: str, image_text: str, destination: str
) -> str:
"""Cross-check Kaniko's two independent output files."""
digests = digest_text.splitlines()
images = image_text.splitlines()
if len(digests) != 1 or len(images) != 1:
raise ValueError("Kaniko evidence must contain exactly one line per file")
digest = _validated(digests[0], DIGEST_PATTERN, "image digest")
if images[0].strip() != f"{destination}@{digest}":
raise ValueError("Kaniko image evidence does not match destination and digest")
return digest
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
"""Return same-origin registry responses without following redirects."""
opener = urllib.request.build_opener(_NoRedirect())
try:
return opener.open(request, timeout=timeout)
except urllib.error.HTTPError as exc:
return exc
def _authorization(username: str, password: str) -> str:
"""Build a Basic header without placing credentials in a URL."""
if not username or not password:
raise RuntimeError("Harbor credentials are unavailable")
token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
return f"Basic {token}"
def _artifact_response(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes]:
"""Read one exact Harbor artifact by candidate tag."""
if not DESTINATION_PATTERN.fullmatch(destination):
raise ValueError("invalid destination")
tag = urllib.parse.quote(destination.rsplit(":", 1)[1], safe="")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
f"{HARBOR_REPOSITORY}/artifacts/{tag}?with_immutable_status=true",
headers={
"Accept": "application/json",
"Authorization": _authorization(username, password),
},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor artifact response exceeded the size limit")
return int(response.status), body
def _rules_response(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes, dict[str, str]]:
"""Read the complete bounded Harbor immutable-rule page."""
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
"?page=1&page_size=100",
headers={
"Accept": "application/json",
"Authorization": _authorization(username, password),
},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
return int(response.status), body, dict(response.headers)
def _normalized_rule(rule: dict[str, Any]) -> dict[str, Any]:
"""Select only immutable-policy fields used by this lane."""
return {
"disabled": bool(rule.get("disabled", False)),
"action": rule.get("action"),
"template": rule.get("template"),
"tag_selectors": [
{key: item.get(key) for key in ("kind", "decoration", "pattern")}
for item in rule.get("tag_selectors") or []
if isinstance(item, dict)
],
"scope_selectors": {
"repository": [
{key: item.get(key) for key in ("kind", "decoration", "pattern")}
for item in (rule.get("scope_selectors") or {}).get("repository", [])
if isinstance(item, dict)
]
},
}
def verify_immutable_policy(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Fail closed unless one exact active router immutability rule exists."""
status, body, headers = _rules_response(
username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
try:
rules = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
raise RuntimeError("Harbor immutable rule list has an invalid shape")
total = next(
(value for key, value in headers.items() if key.lower() == "x-total-count"),
None,
)
if total is None or not str(total).isdecimal() or int(total) != len(rules):
raise RuntimeError("Harbor immutable rule page is incomplete")
expected = {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": IMMUTABLE_TAG_PATTERN,
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
}
]
},
}
matches = [
value
for value in map(_normalized_rule, rules)
if value["tag_selectors"] == expected["tag_selectors"]
and value["scope_selectors"] == expected["scope_selectors"]
]
if matches != [expected]:
raise RuntimeError("Harbor router immutable build-tag policy is not exact")
def assert_tag_absent(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Reject replay before Kaniko can target an already-used build tag."""
status, _ = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status == 404:
return
if status == 200:
raise RuntimeError("Harbor destination tag already exists")
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
def verify_registry_digest(
destination: str,
digest: str,
source_revision: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Verify Harbor's digest, immutable tag, and persisted source label."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
status, body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
try:
artifact = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
tag = destination.rsplit(":", 1)[1]
matching = [
item
for item in artifact.get("tags") or []
if isinstance(item, dict) and item.get("name") == tag
]
labels = ((artifact.get("extra_attrs") or {}).get("config") or {}).get("Labels")
if artifact.get("digest") != digest:
raise RuntimeError("Harbor digest does not match Kaniko evidence")
if len(matching) != 1 or matching[0].get("immutable") is not True:
raise RuntimeError("Harbor did not enforce the candidate tag as immutable")
if not isinstance(labels, dict) or labels.get(
"org.opencontainers.image.revision"
) != revision:
raise RuntimeError("Harbor OCI source-revision label does not match")
def render_workload(source: str, digest: str) -> str:
"""Replace the single exact router image while preserving the Flux marker."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
identity = "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes-chat-router\n"
if not source.startswith("# services/hermes/chat-router.yaml\n" + identity):
raise ValueError("Flux target identity changed")
lines = source.splitlines(keepends=True)
matches: list[int] = []
for index, line in enumerate(lines):
stripped = line.strip()
if not stripped.startswith("image: "):
continue
value = stripped.removeprefix("image: ").split(" #", 1)[0]
image, separator, current_digest = value.rpartition("@")
if separator and re.fullmatch(
rf"{re.escape(DEFAULT_IMAGE)}(?::[A-Za-z0-9_][A-Za-z0-9_.-]{{0,127}})?",
image,
):
_validated(current_digest, DIGEST_PATTERN, "current Flux image digest")
matches.append(index)
if len(matches) != 1:
raise ValueError(f"expected exactly one router image; found {len(matches)}")
index = matches[0]
indent = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
comment = ""
if " #" in lines[index]:
comment = " #" + lines[index].split(" #", 1)[1].rstrip("\n")
newline = "\n" if lines[index].endswith("\n") else ""
lines[index] = f"{indent}image: {DEFAULT_IMAGE}@{digest}{comment}{newline}"
return "".join(lines)
def _metadata(
digest: str, revision: str, build: str, destination: str
) -> dict[str, Any]:
return {
"build_number": build,
"digest": digest,
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
"flux_targets": ["apps/Deployment/hermes/hermes-chat-router"],
"image": DEFAULT_IMAGE,
"published_tag": destination,
"source_revision": revision,
}
def _expected_artifacts(
*,
digest: str,
source_revision: str,
build_number: str,
destination: str,
manifest: Path,
) -> dict[str, str]:
source = manifest.read_text(encoding="utf-8")
rendered = render_workload(source, digest)
patch = "".join(
difflib.unified_diff(
source.splitlines(keepends=True),
rendered.splitlines(keepends=True),
fromfile="a/services/hermes/chat-router.yaml",
tofile="b/services/hermes/chat-router.yaml",
)
)
if not patch:
raise ValueError("published digest already matches the Flux target")
metadata = _metadata(digest, source_revision, build_number, destination)
return {
"hermes-chat-router-deployment.yaml": rendered,
"hermes-chat-router-image-update.patch": patch,
"hermes-chat-router-image.json": json.dumps(
metadata, indent=2, sort_keys=True
)
+ "\n",
}
def write_release_artifacts(
*,
digest: str,
source_revision: str,
build_number: str,
destination: str,
manifest: Path,
output_dir: Path,
) -> None:
"""Write deterministic, credential-free Flux handoff evidence."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
revision, build = validate_destination(destination, source_revision, build_number)
expected = _expected_artifacts(
digest=digest,
source_revision=revision,
build_number=build,
destination=destination,
manifest=manifest,
)
output_dir.mkdir(parents=True, exist_ok=True)
for name, content in expected.items():
(output_dir / name).write_text(content, encoding="utf-8")
def validate_release_artifacts(
*,
digest: str,
source_revision: str,
build_number: str,
destination: str,
manifest: Path,
output_dir: Path,
) -> None:
"""Recompute and compare every archived handoff byte."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
revision, build = validate_destination(destination, source_revision, build_number)
expected = _expected_artifacts(
digest=digest,
source_revision=revision,
build_number=build,
destination=destination,
manifest=manifest,
)
entries = list(output_dir.iterdir())
if {entry.name for entry in entries} != set(expected) or not all(
entry.is_file() and not entry.is_symlink() for entry in entries
):
raise ValueError("release output must contain exactly three evidence files")
for name, content in expected.items():
if (output_dir / name).read_text(encoding="utf-8") != content:
raise ValueError(f"release evidence is incomplete or mismatched: {name}")
def _credentials() -> tuple[str, str]:
username = os.environ.get("HARBOR_USER", "")
password = os.environ.get("HARBOR_PASSWORD", "")
if not username or not password:
raise RuntimeError("Harbor credentials are unavailable")
return username, password
def _add_common(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--source-revision", required=True)
parser.add_argument("--build-number", required=True)
parser.add_argument("--destination", required=True)
def main() -> int:
"""Run one fail-closed candidate or evidence operation."""
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
absent = commands.add_parser("assert-absent")
_add_common(absent)
for name in ("render", "verify-evidence"):
command = commands.add_parser(name)
_add_common(command)
command.add_argument("--digest-file", required=True, type=Path)
command.add_argument("--image-file", required=True, type=Path)
command.add_argument("--manifest", required=True, type=Path)
command.add_argument("--output-dir", required=True, type=Path)
args = parser.parse_args()
validate_destination(args.destination, args.source_revision, args.build_number)
if args.command == "verify-evidence":
digest = validate_kaniko_evidence(
digest_text=args.digest_file.read_text(encoding="utf-8"),
image_text=args.image_file.read_text(encoding="utf-8"),
destination=args.destination,
)
validate_release_artifacts(
digest=digest,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
manifest=args.manifest,
output_dir=args.output_dir,
)
return 0
username, password = _credentials()
if args.command == "assert-absent":
verify_immutable_policy(username=username, password=password)
assert_tag_absent(
args.destination, username=username, password=password
)
return 0
digest = validate_kaniko_evidence(
digest_text=args.digest_file.read_text(encoding="utf-8"),
image_text=args.image_file.read_text(encoding="utf-8"),
destination=args.destination,
)
verify_registry_digest(
args.destination,
digest,
args.source_revision,
username=username,
password=password,
)
write_release_artifacts(
digest=digest,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
manifest=args.manifest,
output_dir=args.output_dir,
)
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

View File

@ -1,502 +0,0 @@
#!/usr/bin/env python3
"""Verify and render a reviewable Hermes agent image release."""
from __future__ import annotations
import argparse
import base64
import difflib
import json
import os
import re
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Callable
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-agent"
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
HARBOR_PROJECT = "bstein"
HARBOR_REPOSITORY = "hermes-agent"
IMMUTABLE_REPOSITORY_PATTERN = "hermes-agent"
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
DESTINATION_PATTERN = re.compile(
r"^registry\.bstein\.dev/bstein/hermes-agent:"
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
)
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Never send registry credentials to a redirect target."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
def _validated(value: str, pattern: re.Pattern[str], label: str) -> str:
"""Return a normalized value when it matches the release contract."""
normalized = value.strip()
if not pattern.fullmatch(normalized):
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
return normalized
def validate_destination(
destination: str, source_revision: str, build_number: str
) -> tuple[str, str]:
"""Bind one unique build tag to the reviewed revision and Jenkins build."""
revision = _validated(source_revision, REVISION_PATTERN, "source revision")
build = _validated(build_number, BUILD_PATTERN, "build number")
match = DESTINATION_PATTERN.fullmatch(destination.strip())
if not match or match.groups() != (revision, build):
raise ValueError(
"destination must bind the reviewed revision and unique Jenkins build"
)
return revision, build
def validate_kaniko_evidence(
*, digest_text: str, image_text: str, destination: str
) -> str:
"""Cross-check both independent Kaniko output files against the destination."""
digest_lines = digest_text.splitlines()
image_lines = image_text.splitlines()
if len(digest_lines) != 1:
raise ValueError("Kaniko digest evidence must contain exactly one line")
if len(image_lines) != 1:
raise ValueError("Kaniko image evidence must contain exactly one line")
digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
if image_lines[0].strip() != f"{destination}@{digest}":
raise ValueError("Kaniko image evidence does not match destination and digest")
return digest
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
"""Make a registry request without following redirects."""
opener = urllib.request.build_opener(_NoRedirect())
try:
return opener.open(request, timeout=timeout)
except urllib.error.HTTPError as exc:
return exc
def _artifact_response(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes]:
"""Read one exact Harbor artifact by tag with bounded response size."""
match = DESTINATION_PATTERN.fullmatch(destination)
if not match:
raise ValueError("invalid destination")
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
tag = destination.rsplit(":", 1)[1]
encoded_tag = urllib.parse.quote(tag, safe="")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
"?with_immutable_status=true",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor artifact response exceeded the size limit")
return int(response.status), body
def _immutable_rules_response(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes, dict[str, str]]:
"""Read the project policy with the same least-privilege publish identity."""
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
"?page=1&page_size=100",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
return int(response.status), body, dict(response.headers)
def _require_complete_rule_page(
rules: list[dict[str, Any]], headers: dict[str, str]
) -> None:
"""Require proof that the bounded first page contains every rule."""
raw_total = next(
(value for key, value in headers.items() if key.lower() == "x-total-count"),
None,
)
if raw_total is None or not str(raw_total).isdecimal():
raise RuntimeError("Harbor immutable rule list omitted a valid total count")
if int(raw_total) != len(rules):
raise RuntimeError("Harbor immutable rule list was truncated")
def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
"""Select only fields that bind the server-side build-tag policy."""
return {
"disabled": bool(rule.get("disabled", False)),
"action": rule.get("action"),
"template": rule.get("template"),
"tag_selectors": [
{
"kind": item.get("kind"),
"decoration": item.get("decoration"),
"pattern": item.get("pattern"),
}
for item in rule.get("tag_selectors") or []
if isinstance(item, dict)
],
"scope_selectors": {
"repository": [
{
"kind": item.get("kind"),
"decoration": item.get("decoration"),
"pattern": item.get("pattern"),
}
for item in (rule.get("scope_selectors") or {}).get("repository", [])
if isinstance(item, dict)
]
},
}
def verify_immutable_policy(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Fail closed before build unless the exact Harbor rule is active."""
status, body, headers = _immutable_rules_response(
username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
try:
rules = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
raise RuntimeError("Harbor immutable rule list has an invalid shape")
_require_complete_rule_page(rules, headers)
expected = {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": IMMUTABLE_TAG_PATTERN,
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
}
]
},
}
matches = [
_normalized_immutable_rule(item)
for item in rules
if _normalized_immutable_rule(item)["tag_selectors"]
== expected["tag_selectors"]
and _normalized_immutable_rule(item)["scope_selectors"]
== expected["scope_selectors"]
]
if matches != [expected]:
raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
def assert_tag_absent(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Reject replay before Kaniko can push an already-used immutable identity."""
status, _body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status == 404:
return
if status == 200:
raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
def verify_registry_digest(
destination: str,
digest: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Verify Harbor independently resolves the pushed tag to Kaniko's digest."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
status, body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
try:
artifact = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
harbor_digest = str(artifact.get("digest") or "").strip()
if not DIGEST_PATTERN.fullmatch(harbor_digest):
raise RuntimeError("Harbor response omitted a valid artifact digest")
if harbor_digest != digest:
raise RuntimeError("Harbor digest does not match Kaniko evidence")
expected_tag = destination.rsplit(":", 1)[1]
matching_tags = [
item
for item in artifact.get("tags") or []
if isinstance(item, dict) and item.get("name") == expected_tag
]
if len(matching_tags) != 1:
raise RuntimeError("Harbor artifact does not contain the expected tag")
if matching_tags[0].get("immutable") is not True:
raise RuntimeError("Harbor did not enforce the expected tag as immutable")
def render_kustomization(source: str, digest: str, image: str = DEFAULT_IMAGE) -> str:
"""Replace exactly one matching Kustomize image digest without reformatting."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
lines = source.splitlines(keepends=True)
matches: list[int] = []
for index, line in enumerate(lines):
if line.strip() != f"- name: {image}":
continue
name_indent = len(line) - len(line.lstrip())
for candidate_index in range(index + 1, len(lines)):
candidate = lines[candidate_index]
stripped = candidate.strip()
candidate_indent = len(candidate) - len(candidate.lstrip())
if stripped.startswith("- name:") and candidate_indent == name_indent:
break
if stripped.startswith("digest:") and candidate_indent > name_indent:
matches.append(candidate_index)
break
if len(matches) != 1:
raise ValueError(
f"expected exactly one digest for image {image!r}; found {len(matches)}"
)
index = matches[0]
newline = "\n" if lines[index].endswith("\n") else ""
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
value = lines[index].strip().removeprefix("digest:").strip()
_current_digest, separator, comment = value.partition(" #")
suffix = f" #{comment}" if separator else ""
lines[index] = f"{prefix}digest: {digest}{suffix}{newline}"
return "".join(lines)
def write_release_artifacts(
*,
digest: str,
source_revision: str,
build_number: str,
destination: str,
kustomization: Path,
output_dir: Path,
) -> dict[str, str]:
"""Write a rendered manifest, patch, and credential-free release metadata."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
source_revision, build_number = validate_destination(
destination, source_revision, build_number
)
source = kustomization.read_text(encoding="utf-8")
rendered = render_kustomization(source, digest)
relative_name = kustomization.name
patch = "".join(
difflib.unified_diff(
source.splitlines(keepends=True),
rendered.splitlines(keepends=True),
fromfile=f"a/services/hermes/{relative_name}",
tofile=f"b/services/hermes/{relative_name}",
)
)
if not patch:
raise ValueError("published digest already matches the Flux manifest")
output_dir.mkdir(parents=True, exist_ok=True)
(output_dir / "hermes-kustomization.yaml").write_text(rendered, encoding="utf-8")
(output_dir / "hermes-image-update.patch").write_text(patch, encoding="utf-8")
metadata = {
"build_number": build_number,
"digest": digest,
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
"image": DEFAULT_IMAGE,
"published_tag": destination,
"source_revision": source_revision,
}
(output_dir / "hermes-agent-image.json").write_text(
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
return metadata
def validate_release_artifacts(
*,
digest_file: Path,
image_file: Path,
source_revision: str,
build_number: str,
destination: str,
kustomization: Path,
output_dir: Path,
) -> None:
"""Revalidate the exact successful-build evidence without rewriting it."""
digest = validate_kaniko_evidence(
digest_text=digest_file.read_text(encoding="utf-8"),
image_text=image_file.read_text(encoding="utf-8"),
destination=destination,
)
source_revision, build_number = validate_destination(
destination, source_revision, build_number
)
expected_names = {
"hermes-agent-image.json",
"hermes-image-update.patch",
"hermes-kustomization.yaml",
}
entries = list(output_dir.iterdir())
if {entry.name for entry in entries} != expected_names or not all(
entry.is_file() and not entry.is_symlink() for entry in entries
):
raise ValueError("release output must contain exactly three evidence files")
source = kustomization.read_text(encoding="utf-8")
rendered = render_kustomization(source, digest)
relative_name = kustomization.name
patch = "".join(
difflib.unified_diff(
source.splitlines(keepends=True),
rendered.splitlines(keepends=True),
fromfile=f"a/services/hermes/{relative_name}",
tofile=f"b/services/hermes/{relative_name}",
)
)
metadata = {
"build_number": build_number,
"digest": digest,
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
"image": DEFAULT_IMAGE,
"published_tag": destination,
"source_revision": source_revision,
}
expected = {
"hermes-agent-image.json": json.dumps(metadata, indent=2, sort_keys=True)
+ "\n",
"hermes-image-update.patch": patch,
"hermes-kustomization.yaml": rendered,
}
for name, expected_text in expected.items():
if (output_dir / name).read_text(encoding="utf-8") != expected_text:
raise ValueError(f"release evidence is incomplete or mismatched: {name}")
def _credentials() -> tuple[str, str]:
"""Read the masked, runtime-only Jenkins credential environment."""
username = os.environ.get("HARBOR_USER", "")
password = os.environ.get("HARBOR_PASSWORD", "")
if not username or not password:
raise RuntimeError("Harbor credentials are unavailable")
return username, password
def _common_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--source-revision", required=True)
parser.add_argument("--build-number", required=True)
parser.add_argument("--destination", required=True)
def main() -> int:
"""Fail closed around the unique tag, then verify and render after push."""
parser = argparse.ArgumentParser()
commands = parser.add_subparsers(dest="command", required=True)
absent = commands.add_parser("assert-absent")
_common_arguments(absent)
render = commands.add_parser("render")
_common_arguments(render)
render.add_argument("--digest-file", required=True, type=Path)
render.add_argument("--image-file", required=True, type=Path)
render.add_argument("--kustomization", required=True, type=Path)
render.add_argument("--output-dir", required=True, type=Path)
verify = commands.add_parser("verify-evidence")
_common_arguments(verify)
verify.add_argument("--digest-file", required=True, type=Path)
verify.add_argument("--image-file", required=True, type=Path)
verify.add_argument("--kustomization", required=True, type=Path)
verify.add_argument("--output-dir", required=True, type=Path)
args = parser.parse_args()
validate_destination(args.destination, args.source_revision, args.build_number)
if args.command == "verify-evidence":
validate_release_artifacts(
digest_file=args.digest_file,
image_file=args.image_file,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
kustomization=args.kustomization,
output_dir=args.output_dir,
)
return 0
username, password = _credentials()
if args.command == "assert-absent":
verify_immutable_policy(username=username, password=password)
assert_tag_absent(args.destination, username=username, password=password)
return 0
digest = validate_kaniko_evidence(
digest_text=args.digest_file.read_text(encoding="utf-8"),
image_text=args.image_file.read_text(encoding="utf-8"),
destination=args.destination,
)
verify_registry_digest(
args.destination, digest, username=username, password=password
)
write_release_artifacts(
digest=digest,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
kustomization=args.kustomization,
output_dir=args.output_dir,
)
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

View File

@ -1,393 +0,0 @@
#!/usr/bin/env python3
"""Assemble a fail-closed multi-arch manifest list from two per-arch leaves.
Kaniko builds one native image per architecture (arm64 on an rpi5 pod, amd64 on
titan-24) and pushes each under an arch-suffixed candidate tag
``...-build-<N>-<arch>``. Kaniko cannot combine, so this step:
1. Independently re-reads each per-arch candidate manifest from the registry and
binds it to the exact Kaniko digest evidence.
2. Proves each leaf really is the architecture it claims by reading its image
config (a swapped or cross-built leaf fails closed here).
3. Builds a Docker manifest *list* (not an OCI index) from the two verified
leaves -- Docker manifest lists are already inside the promotion allow-list,
so this keeps the security surface of ``hermes_oci_promote.py`` unchanged.
4. Refuses to overwrite an existing final tag, PUTs the list to the final
``...-build-<N>`` tag, and re-reads it to confirm the registry resolved the
exact index digest referencing exactly the two expected leaves.
The output ``--digest-file``/``--image-file`` deliberately use the SAME format
the single-arch Kaniko step produced (``<digest>`` and ``<destination>@<digest>``
for the final, arch-less tag). The whole downstream evidence chain --
``hermes_image_release.py`` render/verify-evidence and ``hermes_oci_promote.py``
-- therefore promotes the multi-arch INDEX with no further change.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import os
import re
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Callable
REGISTRY_ORIGIN = "https://registry.bstein.dev"
# The final (arch-less) Flux-visible tag; identical contract to the promoter.
# Both Hermes images that the multi-arch pipelines publish share the exact same
# arch-less final-tag contract; the repository name is the only difference and is
# captured here so the combiner stays fail-closed to just these two components.
DESTINATION_PATTERN = re.compile(
r"^registry\.bstein\.dev/bstein/(?P<component>hermes-agent|hermes-webui):"
r"git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)$"
)
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
DOCKER_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
# A per-arch leaf must be a single-image manifest, never itself a list/index.
LEAF_MANIFEST_TYPES = {
"application/vnd.docker.distribution.manifest.v2+json",
"application/vnd.oci.image.manifest.v1+json",
}
IMAGE_CONFIG_TYPES = {
"application/vnd.docker.container.image.v1+json",
"application/vnd.oci.image.config.v1+json",
}
# Deterministic architecture order -> deterministic manifest-list bytes/digest.
ARCHITECTURES = ("amd64", "arm64")
MAX_MANIFEST_BYTES = 4 * 1024 * 1024
MAX_CONFIG_BYTES = 1024 * 1024
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Never forward registry credentials to another origin."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
"""Return normal and HTTP error responses without following redirects."""
opener = urllib.request.build_opener(_NoRedirect())
try:
return opener.open(request, timeout=timeout)
except urllib.error.HTTPError as exc:
return exc
def _status(response: Any) -> int:
"""Normalize urllib response and HTTPError status fields."""
return int(getattr(response, "status", getattr(response, "code", 0)))
def _authorization(username: str, password: str) -> str:
"""Build a Basic authorization value without placing it in a URL."""
if not username or not password:
raise RuntimeError("Harbor credentials are unavailable")
encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
return f"Basic {encoded}"
def _manifest_url(component: str, reference: str) -> str:
"""Return one same-origin, path-escaped Docker Registry manifest URL."""
encoded = urllib.parse.quote(reference, safe="")
return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded}"
def _blob_url(component: str, digest: str) -> str:
"""Return one same-origin, path-escaped Docker Registry blob URL."""
encoded = urllib.parse.quote(digest, safe="")
return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/blobs/{encoded}"
def _read_evidence_pair(
*, digest_text: str, image_text: str, per_arch_tag_ref: str
) -> str:
"""Cross-check both Kaniko output files for one arch against its tag."""
digest_lines = digest_text.splitlines()
image_lines = image_text.splitlines()
if len(digest_lines) != 1:
raise ValueError("Kaniko digest evidence must contain exactly one line")
if len(image_lines) != 1:
raise ValueError("Kaniko image evidence must contain exactly one line")
digest = digest_lines[0].strip()
if not DIGEST_PATTERN.fullmatch(digest):
raise ValueError("invalid per-arch image digest")
if image_lines[0].strip() != f"{per_arch_tag_ref}@{digest}":
raise ValueError("per-arch image evidence does not match tag and digest")
return digest
def _verified_leaf(
*,
component: str,
per_arch_tag: str,
architecture: str,
expected_digest: str,
authorization: str,
opener: Callable[[urllib.request.Request, int], Any],
) -> dict[str, Any]:
"""Re-read one per-arch leaf and prove its digest, type, and architecture."""
accept = ", ".join(sorted(LEAF_MANIFEST_TYPES))
request = urllib.request.Request(
_manifest_url(component, per_arch_tag),
headers={"Accept": accept, "Authorization": authorization},
method="GET",
)
with opener(request, 30) as response:
if _status(response) != 200:
raise RuntimeError(
f"{architecture} leaf manifest returned HTTP {_status(response)}"
)
body = response.read(MAX_MANIFEST_BYTES + 1)
if len(body) > MAX_MANIFEST_BYTES:
raise RuntimeError(f"{architecture} leaf manifest exceeded the size limit")
observed_digest = response.headers.get("Docker-Content-Digest", "")
content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
if observed_digest != expected_digest:
raise RuntimeError(f"{architecture} leaf digest does not match build evidence")
# Defence in depth: the digest header is registry-asserted; recompute it too.
if f"sha256:{hashlib.sha256(body).hexdigest()}" != expected_digest:
raise RuntimeError(f"{architecture} leaf bytes do not hash to its digest")
if content_type not in LEAF_MANIFEST_TYPES:
raise RuntimeError(f"{architecture} leaf is not a single-image manifest")
try:
manifest = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError(f"{architecture} leaf manifest is not valid JSON") from exc
config = manifest.get("config")
if not isinstance(config, dict):
raise RuntimeError(f"{architecture} leaf manifest omits its config descriptor")
config_digest = str(config.get("digest") or "")
config_type = str(config.get("mediaType") or "")
if not DIGEST_PATTERN.fullmatch(config_digest):
raise RuntimeError(f"{architecture} leaf config digest is invalid")
if config_type not in IMAGE_CONFIG_TYPES:
raise RuntimeError(f"{architecture} leaf config media type is unsupported")
config_request = urllib.request.Request(
_blob_url(component, config_digest),
headers={"Accept": config_type, "Authorization": authorization},
method="GET",
)
with opener(config_request, 30) as response:
if _status(response) != 200:
raise RuntimeError(
f"{architecture} leaf config returned HTTP {_status(response)}"
)
config_body = response.read(MAX_CONFIG_BYTES + 1)
if len(config_body) > MAX_CONFIG_BYTES:
raise RuntimeError(f"{architecture} leaf config exceeded the size limit")
if f"sha256:{hashlib.sha256(config_body).hexdigest()}" != config_digest:
raise RuntimeError(f"{architecture} leaf config bytes do not hash to its digest")
try:
config_json = json.loads(config_body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError(f"{architecture} leaf config is not valid JSON") from exc
if config_json.get("architecture") != architecture:
raise RuntimeError(
f"{architecture} leaf config reports architecture "
f"{config_json.get('architecture')!r}"
)
if config_json.get("os") != "linux":
raise RuntimeError(f"{architecture} leaf config reports a non-linux os")
return {
"mediaType": content_type,
"size": len(body),
"digest": expected_digest,
"platform": {"architecture": architecture, "os": "linux"},
}
def _manifest_list_bytes(descriptors: list[dict[str, Any]]) -> bytes:
"""Serialize the manifest list deterministically for a stable index digest."""
document = {
"schemaVersion": 2,
"mediaType": DOCKER_MANIFEST_LIST,
"manifests": descriptors,
}
return json.dumps(document, sort_keys=True, separators=(",", ":")).encode("utf-8")
def combine_multiarch_index(
*,
destination: str,
arch_digests: dict[str, str],
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> dict[str, str]:
"""Verify both leaves, publish, and re-verify one multi-arch index tag."""
match = DESTINATION_PATTERN.fullmatch(destination.strip())
if not match:
raise ValueError("invalid multi-arch destination")
if set(arch_digests) != set(ARCHITECTURES):
raise ValueError("expected exactly the arm64 and amd64 per-arch digests")
component = match.group("component")
index_tag = destination.rsplit(":", 1)[1]
authorization = _authorization(username, password)
descriptors = [
_verified_leaf(
component=component,
per_arch_tag=f"{index_tag}-{architecture}",
architecture=architecture,
expected_digest=arch_digests[architecture],
authorization=authorization,
opener=opener,
)
for architecture in ARCHITECTURES
]
manifest_list = _manifest_list_bytes(descriptors)
if len(manifest_list) > MAX_MANIFEST_BYTES:
raise RuntimeError("assembled manifest list exceeded the size limit")
index_digest = f"sha256:{hashlib.sha256(manifest_list).hexdigest()}"
index_url = _manifest_url(component, index_tag)
head_request = urllib.request.Request(
index_url,
headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization},
method="HEAD",
)
with opener(head_request, 20) as response:
head_status = _status(response)
existing_digest = response.headers.get("Docker-Content-Digest", "")
if head_status == 200:
if existing_digest != index_digest:
raise RuntimeError("final tag already exists with another index digest")
result = "already-present"
elif head_status == 404:
put_request = urllib.request.Request(
index_url,
data=manifest_list,
headers={
"Authorization": authorization,
"Content-Type": DOCKER_MANIFEST_LIST,
},
method="PUT",
)
with opener(put_request, 30) as response:
put_status = _status(response)
put_digest = response.headers.get("Docker-Content-Digest", "")
if put_status not in {201, 202}:
raise RuntimeError(f"index manifest returned HTTP {put_status}")
if put_digest and put_digest != index_digest:
raise RuntimeError("index manifest digest changed during publish")
result = "published"
else:
raise RuntimeError(f"final tag preflight returned HTTP {head_status}")
verify_request = urllib.request.Request(
index_url,
headers={"Accept": DOCKER_MANIFEST_LIST, "Authorization": authorization},
method="GET",
)
with opener(verify_request, 30) as response:
if _status(response) != 200:
raise RuntimeError(f"index verification returned HTTP {_status(response)}")
verify_body = response.read(MAX_MANIFEST_BYTES + 1)
if len(verify_body) > MAX_MANIFEST_BYTES:
raise RuntimeError("index verification exceeded the size limit")
verify_digest = response.headers.get("Docker-Content-Digest", "")
verify_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
if verify_digest != index_digest:
raise RuntimeError("registry resolved the final tag to another index digest")
if verify_type != DOCKER_MANIFEST_LIST:
raise RuntimeError("registry did not store a Docker manifest list")
try:
published = json.loads(verify_body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("registry returned invalid index JSON") from exc
published_leaves = {
(
str((item or {}).get("platform", {}).get("architecture")),
str((item or {}).get("digest")),
)
for item in published.get("manifests") or []
}
expected_leaves = {
(architecture, arch_digests[architecture]) for architecture in ARCHITECTURES
}
if published_leaves != expected_leaves:
raise RuntimeError("published index does not reference the exact two leaves")
return {
"component": component,
"index_digest": index_digest,
"index_tag": index_tag,
"result": result,
**{f"{architecture}_digest": arch_digests[architecture] for architecture in ARCHITECTURES},
}
def _load_arch_digest(
*, destination: str, architecture: str, digest_file: Path, image_file: Path
) -> str:
"""Bind one arch's two Kaniko evidence files to its arch-suffixed tag."""
per_arch_tag_ref = f"{destination}-{architecture}"
return _read_evidence_pair(
digest_text=digest_file.read_text(encoding="utf-8"),
image_text=image_file.read_text(encoding="utf-8"),
per_arch_tag_ref=per_arch_tag_ref,
)
def main() -> int:
"""Combine two verified per-arch leaves and emit index digest evidence."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--destination", required=True)
parser.add_argument("--source-revision", required=True)
parser.add_argument("--build-number", required=True)
parser.add_argument("--arm64-digest-file", required=True, type=Path)
parser.add_argument("--arm64-image-file", required=True, type=Path)
parser.add_argument("--amd64-digest-file", required=True, type=Path)
parser.add_argument("--amd64-image-file", required=True, type=Path)
parser.add_argument("--digest-file", required=True, type=Path)
parser.add_argument("--image-file", required=True, type=Path)
args = parser.parse_args()
try:
match = DESTINATION_PATTERN.fullmatch(args.destination.strip())
if not match:
raise ValueError("invalid multi-arch destination")
if match.group("revision") != args.source_revision.strip():
raise ValueError("destination revision does not match evidence")
if match.group("build") != args.build_number.strip():
raise ValueError("destination build number does not match evidence")
destination = args.destination.strip()
arch_digests = {
"arm64": _load_arch_digest(
destination=destination,
architecture="arm64",
digest_file=args.arm64_digest_file,
image_file=args.arm64_image_file,
),
"amd64": _load_arch_digest(
destination=destination,
architecture="amd64",
digest_file=args.amd64_digest_file,
image_file=args.amd64_image_file,
),
}
result = combine_multiarch_index(
destination=destination,
arch_digests=arch_digests,
username=os.environ.get("HARBOR_USER", ""),
password=os.environ.get("HARBOR_PASSWORD", ""),
)
args.digest_file.write_text(result["index_digest"] + "\n", encoding="utf-8")
args.image_file.write_text(
f"{destination}@{result['index_digest']}\n", encoding="utf-8"
)
except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
print(json.dumps({"error": str(exc)}, sort_keys=True))
return 1
print(json.dumps(result, indent=2, sort_keys=True))
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

View File

@ -1,193 +0,0 @@
#!/usr/bin/env python3
"""Publish a validated Hermes candidate manifest under its Flux release tag."""
from __future__ import annotations
import argparse
import base64
import json
import os
import re
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Callable
REGISTRY_ORIGIN = "https://registry.bstein.dev"
DESTINATION_PATTERN = re.compile(
r"^registry\.bstein\.dev/bstein/"
r"(?P<component>hermes-(?:agent|webui|chat-router|jetson-(?:stt|tts))):"
r"git-(?P<revision>[0-9a-f]{40})-build-(?P<build>[1-9][0-9]*)$"
)
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
MANIFEST_TYPES = {
"application/vnd.docker.distribution.manifest.v2+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.oci.image.index.v1+json",
}
MAX_MANIFEST_BYTES = 4 * 1024 * 1024
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Never forward registry credentials to another origin."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
"""Return normal and HTTP error responses without following redirects."""
opener = urllib.request.build_opener(_NoRedirect())
try:
return opener.open(request, timeout=timeout)
except urllib.error.HTTPError as exc:
return exc
def _status(response: Any) -> int:
"""Normalize urllib response and HTTPError status fields."""
return int(getattr(response, "status", getattr(response, "code", 0)))
def _authorization(username: str, password: str) -> str:
"""Build a Basic authorization value without placing it in a URL."""
if not username or not password:
raise RuntimeError("Harbor credentials are unavailable")
encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
return f"Basic {encoded}"
def _validated_release(
destination: str,
digest: str,
source_revision: str,
build_number: str,
) -> tuple[str, str, str]:
"""Bind the candidate and release tag to one reviewed source and build."""
match = DESTINATION_PATTERN.fullmatch(destination.strip())
if not match:
raise ValueError("invalid Hermes candidate destination")
if match.group("revision") != source_revision.strip():
raise ValueError("candidate source revision does not match evidence")
if match.group("build") != build_number.strip():
raise ValueError("candidate build number does not match evidence")
normalized_digest = digest.strip()
if not DIGEST_PATTERN.fullmatch(normalized_digest):
raise ValueError("invalid candidate digest")
candidate_tag = destination.rsplit(":", 1)[1]
return match.group("component"), candidate_tag, normalized_digest
def _manifest_url(component: str, tag: str) -> str:
"""Return one same-origin, path-escaped Docker Registry manifest URL."""
encoded_tag = urllib.parse.quote(tag, safe="")
return f"{REGISTRY_ORIGIN}/v2/bstein/{component}/manifests/{encoded_tag}"
def promote_candidate(
*,
destination: str,
digest: str,
source_revision: str,
build_number: str,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> dict[str, str]:
"""Copy an exact candidate manifest to the immutable ``-release`` tag."""
component, candidate_tag, normalized_digest = _validated_release(
destination, digest, source_revision, build_number
)
release_tag = f"{candidate_tag}-release"
authorization = _authorization(username, password)
accept = ", ".join(sorted(MANIFEST_TYPES))
candidate_request = urllib.request.Request(
_manifest_url(component, candidate_tag),
headers={"Accept": accept, "Authorization": authorization},
method="GET",
)
with opener(candidate_request, 30) as response:
if _status(response) != 200:
raise RuntimeError(f"candidate manifest returned HTTP {_status(response)}")
manifest = response.read(MAX_MANIFEST_BYTES + 1)
if len(manifest) > MAX_MANIFEST_BYTES:
raise RuntimeError("candidate manifest exceeded the size limit")
observed_digest = response.headers.get("Docker-Content-Digest", "")
content_type = response.headers.get("Content-Type", "").split(";", 1)[0]
if observed_digest != normalized_digest:
raise RuntimeError("candidate manifest digest does not match build evidence")
if content_type not in MANIFEST_TYPES:
raise RuntimeError("candidate manifest returned an unsupported content type")
release_url = _manifest_url(component, release_tag)
head_request = urllib.request.Request(
release_url,
headers={"Accept": accept, "Authorization": authorization},
method="HEAD",
)
with opener(head_request, 20) as response:
head_status = _status(response)
existing_digest = response.headers.get("Docker-Content-Digest", "")
if head_status == 200:
if existing_digest != normalized_digest:
raise RuntimeError("release tag already exists with another digest")
result = "already-present"
elif head_status == 404:
put_request = urllib.request.Request(
release_url,
data=manifest,
headers={
"Authorization": authorization,
"Content-Type": content_type,
},
method="PUT",
)
with opener(put_request, 30) as response:
put_status = _status(response)
promoted_digest = response.headers.get("Docker-Content-Digest", "")
if put_status not in {201, 202}:
raise RuntimeError(f"release manifest returned HTTP {put_status}")
if promoted_digest and promoted_digest != normalized_digest:
raise RuntimeError("release manifest digest changed during promotion")
result = "published"
else:
raise RuntimeError(f"release tag preflight returned HTTP {head_status}")
return {
"component": component,
"digest": normalized_digest,
"release_tag": release_tag,
"result": result,
"source_revision": source_revision,
}
def main() -> int:
"""Promote one evidence-bound candidate and print credential-free metadata."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--destination", required=True)
parser.add_argument("--digest-file", required=True, type=Path)
parser.add_argument("--source-revision", required=True)
parser.add_argument("--build-number", required=True)
args = parser.parse_args()
try:
result = promote_candidate(
destination=args.destination,
digest=args.digest_file.read_text(encoding="utf-8"),
source_revision=args.source_revision,
build_number=args.build_number,
username=os.environ.get("HARBOR_USER", ""),
password=os.environ.get("HARBOR_PASSWORD", ""),
)
except (OSError, ValueError, RuntimeError, urllib.error.URLError) as exc:
print(json.dumps({"error": str(exc)}, sort_keys=True))
return 1
print(json.dumps(result, indent=2, sort_keys=True))
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

View File

@ -1,273 +0,0 @@
#!/usr/bin/env python3
"""Render and revalidate the two-workload Hermes WebUI Flux handoff."""
from __future__ import annotations
import difflib
import json
import re
from pathlib import Path
from typing import Any
DEFAULT_IMAGE = "registry.bstein.dev/bstein/hermes-webui"
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
REVISION_PATTERN = re.compile(r"^[0-9a-f]{40}$")
BUILD_PATTERN = re.compile(r"^[1-9][0-9]*$")
DESTINATION_PATTERN = re.compile(
r"^registry\.bstein\.dev/bstein/hermes-webui:"
r"git-([0-9a-f]{40})-build-([1-9][0-9]*)$"
)
def validated(value: str, pattern: re.Pattern[str], label: str) -> str:
"""Return a normalized value when it matches the release contract."""
normalized = value.strip()
if not pattern.fullmatch(normalized):
raise ValueError(f"invalid {label}: expected {pattern.pattern}")
return normalized
def validate_destination(
destination: str, source_revision: str, build_number: str
) -> tuple[str, str]:
"""Bind one unique build tag to the reviewed revision and Jenkins build."""
revision = validated(source_revision, REVISION_PATTERN, "source revision")
build = validated(build_number, BUILD_PATTERN, "build number")
match = DESTINATION_PATTERN.fullmatch(destination.strip())
if not match or match.groups() != (revision, build):
raise ValueError(
"destination must bind the reviewed revision and unique Jenkins build"
)
return revision, build
def render_workload(
source: str,
digest: str,
*,
kind: str,
name: str,
image: str = DEFAULT_IMAGE,
expected_images: int | tuple[int, ...] = 1,
) -> str:
"""Replace every expected WebUI consumer in one exact Flux workload."""
digest = validated(digest, DIGEST_PATTERN, "image digest")
identity = re.compile(
rf"\A(?:#[^\n]*\n)*apiVersion: apps/v1\nkind: {re.escape(kind)}\n"
rf"metadata:\n name: {re.escape(name)}\n"
)
if not identity.search(source):
raise ValueError(f"Flux target identity changed: expected {kind}/{name}")
lines = source.splitlines(keepends=True)
matches: list[int] = []
suffixes: dict[int, str] = {}
for index, line in enumerate(lines):
stripped = line.strip()
if not stripped.startswith("image: "):
continue
value, separator, comment = stripped.removeprefix("image: ").partition(" #")
current_image, at, current_digest = value.rpartition("@")
if not at or not re.fullmatch(
rf"{re.escape(image)}(?::[A-Za-z0-9_][A-Za-z0-9_.-]{{0,127}})?",
current_image,
):
continue
validated(current_digest, DIGEST_PATTERN, "current Flux image digest")
matches.append(index)
suffixes[index] = f" #{comment}" if separator else ""
allowed = (expected_images,) if isinstance(expected_images, int) else expected_images
if not allowed or any(count < 1 for count in allowed) or len(matches) not in allowed:
raise ValueError(
f"expected {allowed} {image!r} image(s) in {kind}/{name}; "
f"found {len(matches)}"
)
for index in matches:
newline = "\n" if lines[index].endswith("\n") else ""
prefix = lines[index][: len(lines[index]) - len(lines[index].lstrip())]
lines[index] = f"{prefix}image: {image}@{digest}{suffixes[index]}{newline}"
return "".join(lines)
def render_hux_build_metadata(
source: str, digest: str, source_revision: str, build_number: str
) -> str:
"""Bind HUX metadata when the activated sidecar fields are present."""
digest = validated(digest, DIGEST_PATTERN, "image digest")
revision = validated(source_revision, REVISION_PATTERN, "source revision")
build = validated(build_number, BUILD_PATTERN, "build number")
replacements = {
"HUX_IMAGE_TAG": f"git-{revision}-build-{build}-release",
"HUX_IMAGE_DIGEST": digest,
}
rendered = source
# Block style only: Flux setters cannot attach to values inside flow
# mappings, so the manifest keeps these as two-line entries with the
# marker comment on the value scalar. The renderer stays a belt on top
# of the Flux :tag/:digest setters and binds the same values.
present = {
name: rendered.count(f"- name: {name}\n") for name in replacements
}
if set(present.values()) == {0}:
return rendered
if any(count != 1 for count in present.values()):
raise ValueError(f"incomplete HUX build binding fields: {present}")
for name, value in replacements.items():
pattern = re.compile(
rf"^(?P<head>(?P<indent>\s*)- name: {name}\n(?P=indent) value: )"
rf"[^#\n]+(?P<suffix> #[^\n]*)?$",
re.MULTILINE,
)
rendered, count = pattern.subn(
lambda match: (
f"{match.group('head')}{value}{match.group('suffix') or ''}"
),
rendered,
)
if count != 1:
raise ValueError(f"expected exactly one {name} HUX build binding; found {count}")
return rendered
def _targets(chat_manifest: Path, dashboard_manifest: Path):
return (
(
chat_manifest,
"StatefulSet",
"hermes-chat-tenant",
"hermes-chat-statefulset.yaml",
(1, 2, 3),
),
(
dashboard_manifest,
"Deployment",
"hermes",
"hermes-dashboard-deployment.yaml",
1,
),
)
def _metadata(
digest: str, source_revision: str, build_number: str, destination: str
) -> dict[str, Any]:
return {
"build_number": build_number,
"digest": digest,
"flux_image": f"{DEFAULT_IMAGE}@{digest}",
"flux_targets": [
"apps/StatefulSet/hermes/hermes-chat-tenant",
"apps/Deployment/hermes/hermes",
],
"image": DEFAULT_IMAGE,
"published_tag": destination,
"source_revision": source_revision,
}
def _rendered_and_patch(
digest: str,
source_revision: str,
build_number: str,
chat_manifest: Path,
dashboard_manifest: Path,
) -> tuple[dict[str, str], str]:
rendered_targets: dict[str, str] = {}
patch_parts: list[str] = []
for path, kind, name, artifact_name, expected_images in _targets(chat_manifest, dashboard_manifest):
source = path.read_text(encoding="utf-8")
rendered = render_workload(
source,
digest,
kind=kind,
name=name,
expected_images=expected_images,
)
if name == "hermes-chat-tenant":
rendered = render_hux_build_metadata(
rendered, digest, source_revision, build_number
)
rendered_targets[artifact_name] = rendered
patch_parts.append(
"".join(
difflib.unified_diff(
source.splitlines(keepends=True),
rendered.splitlines(keepends=True),
fromfile=f"a/services/hermes/{path.name}",
tofile=f"b/services/hermes/{path.name}",
)
)
)
return rendered_targets, "".join(patch_parts)
def write_release_artifacts(
*,
digest: str,
source_revision: str,
build_number: str,
destination: str,
chat_manifest: Path,
dashboard_manifest: Path,
output_dir: Path,
) -> dict[str, Any]:
"""Write two rendered Flux targets, one patch, and credential-free evidence."""
digest = validated(digest, DIGEST_PATTERN, "image digest")
source_revision, build_number = validate_destination(
destination, source_revision, build_number
)
rendered_targets, patch = _rendered_and_patch(
digest, source_revision, build_number, chat_manifest, dashboard_manifest
)
if not patch:
raise ValueError("published digest already matches every Flux target")
output_dir.mkdir(parents=True, exist_ok=True)
for artifact_name, rendered in rendered_targets.items():
(output_dir / artifact_name).write_text(rendered, encoding="utf-8")
(output_dir / "hermes-webui-image-update.patch").write_text(patch, encoding="utf-8")
metadata = _metadata(digest, source_revision, build_number, destination)
(output_dir / "hermes-webui-image.json").write_text(
json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
return metadata
def validate_release_artifacts(
*,
digest: str,
source_revision: str,
build_number: str,
destination: str,
chat_manifest: Path,
dashboard_manifest: Path,
output_dir: Path,
) -> None:
"""Revalidate the exact successful-build evidence without rewriting it."""
digest = validated(digest, DIGEST_PATTERN, "image digest")
source_revision, build_number = validate_destination(
destination, source_revision, build_number
)
expected_names = {
"hermes-webui-image.json",
"hermes-webui-image-update.patch",
"hermes-chat-statefulset.yaml",
"hermes-dashboard-deployment.yaml",
}
entries = list(output_dir.iterdir())
if {entry.name for entry in entries} != expected_names or not all(
entry.is_file() and not entry.is_symlink() for entry in entries
):
raise ValueError("release output must contain exactly four evidence files")
rendered_targets, patch = _rendered_and_patch(
digest, source_revision, build_number, chat_manifest, dashboard_manifest
)
metadata = _metadata(digest, source_revision, build_number, destination)
expected = {
"hermes-webui-image.json": json.dumps(metadata, indent=2, sort_keys=True)
+ "\n",
"hermes-webui-image-update.patch": patch,
**rendered_targets,
}
for name, expected_text in expected.items():
if (output_dir / name).read_text(encoding="utf-8") != expected_text:
raise ValueError(f"release evidence is incomplete or mismatched: {name}")

View File

@ -1,469 +0,0 @@
#!/usr/bin/env python3
"""Verify and render a reviewable Hermes WebUI image release."""
from __future__ import annotations
import argparse
import base64
import json
import os
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Callable
from hermes_webui_flux_release import (
DEFAULT_IMAGE as DEFAULT_IMAGE,
DESTINATION_PATTERN,
DIGEST_PATTERN,
REVISION_PATTERN,
render_hux_build_metadata as render_hux_build_metadata,
render_workload as render_workload,
validate_destination,
validate_release_artifacts as validate_flux_release_artifacts,
validated as _validated,
write_release_artifacts,
)
HARBOR_API_ORIGIN = "https://registry.bstein.dev/api/v2.0"
HARBOR_PROJECT = "bstein"
HARBOR_REPOSITORY = "hermes-webui"
IMMUTABLE_REPOSITORY_PATTERN = "hermes-webui"
IMMUTABLE_TAG_PATTERN = "git-*-build-*"
class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Never send registry credentials to a redirect target."""
def redirect_request(self, _request, _file, _code, _message, _headers, _url):
return None
def validate_kaniko_evidence(
*, digest_text: str, image_text: str, destination: str
) -> str:
"""Cross-check both independent Kaniko output files against the destination."""
digest_lines = digest_text.splitlines()
image_lines = image_text.splitlines()
if len(digest_lines) != 1:
raise ValueError("Kaniko digest evidence must contain exactly one line")
if len(image_lines) != 1:
raise ValueError("Kaniko image evidence must contain exactly one line")
digest = _validated(digest_lines[0], DIGEST_PATTERN, "image digest")
if image_lines[0].strip() != f"{destination}@{digest}":
raise ValueError("Kaniko image evidence does not match destination and digest")
return digest
def _registry_request(request: urllib.request.Request, timeout: int) -> Any:
"""Make a registry request without following redirects."""
opener = urllib.request.build_opener(_NoRedirect())
try:
return opener.open(request, timeout=timeout)
except urllib.error.HTTPError as exc:
return exc
def _artifact_response(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes]:
"""Read one exact Harbor artifact by tag with bounded response size."""
match = DESTINATION_PATTERN.fullmatch(destination)
if not match:
raise ValueError("invalid destination")
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
tag = destination.rsplit(":", 1)[1]
encoded_tag = urllib.parse.quote(tag, safe="")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
f"{HARBOR_REPOSITORY}/artifacts/{encoded_tag}"
"?with_immutable_status=true",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor artifact response exceeded the size limit")
return int(response.status), body
def _immutable_rules_response(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes, dict[str, str]]:
"""Read the project policy with the same least-privilege publish identity."""
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/immutabletagrules"
"?page=1&page_size=100",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor immutable rule response exceeded the size limit")
return int(response.status), body, dict(response.headers)
def _require_complete_rule_page(
rules: list[dict[str, Any]], headers: dict[str, str]
) -> None:
"""Require proof that the bounded first page contains every rule."""
raw_total = next(
(value for key, value in headers.items() if key.lower() == "x-total-count"),
None,
)
if raw_total is None or not str(raw_total).isdecimal():
raise RuntimeError("Harbor immutable rule list omitted a valid total count")
if int(raw_total) != len(rules):
raise RuntimeError("Harbor immutable rule list was truncated")
def _normalized_immutable_rule(rule: dict[str, Any]) -> dict[str, Any]:
"""Select only fields that bind the server-side build-tag policy."""
return {
"disabled": bool(rule.get("disabled", False)),
"action": rule.get("action"),
"template": rule.get("template"),
"tag_selectors": [
{
"kind": item.get("kind"),
"decoration": item.get("decoration"),
"pattern": item.get("pattern"),
}
for item in rule.get("tag_selectors") or []
if isinstance(item, dict)
],
"scope_selectors": {
"repository": [
{
"kind": item.get("kind"),
"decoration": item.get("decoration"),
"pattern": item.get("pattern"),
}
for item in (rule.get("scope_selectors") or {}).get("repository", [])
if isinstance(item, dict)
]
},
}
def verify_immutable_policy(
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Fail closed before build unless the exact Harbor rule is active."""
status, body, headers = _immutable_rules_response(
username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor immutable policy preflight returned HTTP {status}")
try:
rules = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid immutable rule JSON") from exc
if not isinstance(rules, list) or not all(isinstance(item, dict) for item in rules):
raise RuntimeError("Harbor immutable rule list has an invalid shape")
_require_complete_rule_page(rules, headers)
expected = {
"disabled": False,
"action": "immutable",
"template": "immutable_template",
"tag_selectors": [
{
"kind": "doublestar",
"decoration": "matches",
"pattern": IMMUTABLE_TAG_PATTERN,
}
],
"scope_selectors": {
"repository": [
{
"kind": "doublestar",
"decoration": "repoMatches",
"pattern": IMMUTABLE_REPOSITORY_PATTERN,
}
]
},
}
matches = [
_normalized_immutable_rule(item)
for item in rules
if _normalized_immutable_rule(item)["tag_selectors"]
== expected["tag_selectors"]
and _normalized_immutable_rule(item)["scope_selectors"]
== expected["scope_selectors"]
]
if matches != [expected]:
raise RuntimeError("Harbor immutable build-tag policy is absent or not exact")
def assert_tag_absent(
destination: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Reject replay before Kaniko can push an already-used immutable identity."""
status, _body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status == 404:
return
if status == 200:
raise RuntimeError("Harbor destination tag already exists; refusing overwrite")
raise RuntimeError(f"Harbor destination preflight returned HTTP {status}")
OCI_REVISION_LABEL = "org.opencontainers.image.revision"
def _child_artifact_response(
child_digest: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> tuple[int, bytes]:
"""Read one per-arch child artifact of a multi-arch index by its digest."""
if not username or not password:
raise RuntimeError("Harbor credentials are empty")
encoded = urllib.parse.quote(child_digest, safe="")
auth = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
request = urllib.request.Request(
f"{HARBOR_API_ORIGIN}/projects/{HARBOR_PROJECT}/repositories/"
f"{HARBOR_REPOSITORY}/artifacts/{encoded}"
"?with_immutable_status=true",
headers={"Accept": "application/json", "Authorization": f"Basic {auth}"},
method="GET",
)
with opener(request, 20) as response:
body = response.read(1_048_577)
if len(body) > 1_048_576:
raise RuntimeError("Harbor child artifact response exceeded the size limit")
return int(response.status), body
def _config_labels(artifact: dict[str, Any]) -> Any:
"""Extract the OCI config labels Harbor reports for one artifact, if any."""
return ((artifact.get("extra_attrs") or {}).get("config") or {}).get("Labels")
def _verify_source_revision_label(
artifact: dict[str, Any],
source_revision: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any],
) -> None:
"""Assert the published image carries the reviewed source revision label.
A single-arch image exposes ``org.opencontainers.image.revision`` on its own
config. A multi-arch manifest list has no top-level config, so Harbor reports
the label on each per-arch child instead; verify every child in that case.
"""
labels = _config_labels(artifact)
if isinstance(labels, dict):
if labels.get(OCI_REVISION_LABEL) != source_revision:
raise RuntimeError("Harbor OCI source-revision label does not match")
return
references = artifact.get("references")
if not isinstance(references, list) or not references:
raise RuntimeError("Harbor artifact omitted OCI image labels")
for reference in references:
child_digest = str((reference or {}).get("child_digest") or "").strip()
if not DIGEST_PATTERN.fullmatch(child_digest):
raise RuntimeError("Harbor index reference omitted a valid child digest")
status, body = _child_artifact_response(
child_digest, username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(
f"Harbor child artifact verification returned HTTP {status}"
)
try:
child = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid child artifact JSON") from exc
child_labels = _config_labels(child)
if not isinstance(child_labels, dict):
raise RuntimeError("Harbor artifact omitted OCI image labels")
if child_labels.get(OCI_REVISION_LABEL) != source_revision:
raise RuntimeError("Harbor OCI source-revision label does not match")
def verify_registry_digest(
destination: str,
digest: str,
source_revision: str,
*,
username: str,
password: str,
opener: Callable[[urllib.request.Request, int], Any] = _registry_request,
) -> None:
"""Verify Harbor resolves the tag, digest, and persisted source revision."""
digest = _validated(digest, DIGEST_PATTERN, "image digest")
source_revision = _validated(
source_revision, REVISION_PATTERN, "source revision"
)
status, body = _artifact_response(
destination, username=username, password=password, opener=opener
)
if status != 200:
raise RuntimeError(f"Harbor manifest verification returned HTTP {status}")
try:
artifact = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Harbor returned invalid artifact JSON") from exc
harbor_digest = str(artifact.get("digest") or "").strip()
if not DIGEST_PATTERN.fullmatch(harbor_digest):
raise RuntimeError("Harbor response omitted a valid artifact digest")
if harbor_digest != digest:
raise RuntimeError("Harbor digest does not match Kaniko evidence")
expected_tag = destination.rsplit(":", 1)[1]
matching_tags = [
item
for item in artifact.get("tags") or []
if isinstance(item, dict) and item.get("name") == expected_tag
]
if len(matching_tags) != 1:
raise RuntimeError("Harbor artifact does not contain the expected tag")
if matching_tags[0].get("immutable") is not True:
raise RuntimeError("Harbor did not enforce the expected tag as immutable")
_verify_source_revision_label(
artifact,
source_revision,
username=username,
password=password,
opener=opener,
)
def validate_release_artifacts(
*,
digest_file: Path,
image_file: Path,
source_revision: str,
build_number: str,
destination: str,
chat_manifest: Path,
dashboard_manifest: Path,
output_dir: Path,
) -> None:
"""Revalidate the exact successful-build evidence without rewriting it."""
digest = validate_kaniko_evidence(
digest_text=digest_file.read_text(encoding="utf-8"),
image_text=image_file.read_text(encoding="utf-8"),
destination=destination,
)
validate_flux_release_artifacts(
digest=digest,
source_revision=source_revision,
build_number=build_number,
destination=destination,
chat_manifest=chat_manifest,
dashboard_manifest=dashboard_manifest,
output_dir=output_dir,
)
def _credentials() -> tuple[str, str]:
"""Read the masked, runtime-only Jenkins credential environment."""
username = os.environ.get("HARBOR_USER", "")
password = os.environ.get("HARBOR_PASSWORD", "")
if not username or not password:
raise RuntimeError("Harbor credentials are unavailable")
return username, password
def _common_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--source-revision", required=True)
parser.add_argument("--build-number", required=True)
parser.add_argument("--destination", required=True)
def main() -> int:
"""Fail closed around the unique tag, then verify and render after push."""
parser = argparse.ArgumentParser()
commands = parser.add_subparsers(dest="command", required=True)
absent = commands.add_parser("assert-absent")
_common_arguments(absent)
render = commands.add_parser("render")
_common_arguments(render)
render.add_argument("--digest-file", required=True, type=Path)
render.add_argument("--image-file", required=True, type=Path)
render.add_argument("--chat-manifest", required=True, type=Path)
render.add_argument("--dashboard-manifest", required=True, type=Path)
render.add_argument("--output-dir", required=True, type=Path)
verify = commands.add_parser("verify-evidence")
_common_arguments(verify)
verify.add_argument("--digest-file", required=True, type=Path)
verify.add_argument("--image-file", required=True, type=Path)
verify.add_argument("--chat-manifest", required=True, type=Path)
verify.add_argument("--dashboard-manifest", required=True, type=Path)
verify.add_argument("--output-dir", required=True, type=Path)
args = parser.parse_args()
validate_destination(args.destination, args.source_revision, args.build_number)
if args.command == "verify-evidence":
validate_release_artifacts(
digest_file=args.digest_file,
image_file=args.image_file,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
chat_manifest=args.chat_manifest,
dashboard_manifest=args.dashboard_manifest,
output_dir=args.output_dir,
)
return 0
username, password = _credentials()
if args.command == "assert-absent":
verify_immutable_policy(username=username, password=password)
assert_tag_absent(args.destination, username=username, password=password)
return 0
digest = validate_kaniko_evidence(
digest_text=args.digest_file.read_text(encoding="utf-8"),
image_text=args.image_file.read_text(encoding="utf-8"),
destination=args.destination,
)
verify_registry_digest(
args.destination,
digest,
args.source_revision,
username=username,
password=password,
)
write_release_artifacts(
digest=digest,
source_revision=args.source_revision,
build_number=args.build_number,
destination=args.destination,
chat_manifest=args.chat_manifest,
dashboard_manifest=args.dashboard_manifest,
output_dir=args.output_dir,
)
return 0
if __name__ == "__main__": # pragma: no cover - exercised through main()
raise SystemExit(main())

View File

@ -1,379 +0,0 @@
#!/usr/bin/env python3
"""Publish titan-iac quality-gate results to Pushgateway."""
from __future__ import annotations
import hashlib
import json
import os
from glob import glob
from pathlib import Path
import sys
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from ci.scripts import publish_test_metrics_quality as _quality_helpers
CANONICAL_CHECKS = _quality_helpers.CANONICAL_CHECKS
_build_check_statuses = _quality_helpers._build_check_statuses
_combine_statuses = _quality_helpers._combine_statuses
_infer_sonarqube_status = _quality_helpers._infer_sonarqube_status
_infer_semgrep_status = _quality_helpers._infer_semgrep_status
_infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500
_infer_supply_chain_status = _quality_helpers._infer_supply_chain_status
_infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent
_load_optional_json = _quality_helpers._load_optional_json
_normalize_result_status = _quality_helpers._normalize_result_status
TEST_CASE_LABEL_MAX_BYTES = 240
def _escape_label(value: str) -> str:
"""Escape a Prometheus label value without changing its content."""
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
def _label_str(labels: dict[str, str]) -> str:
"""Render a stable Prometheus label set from a mapping."""
parts = [f'{key}="{_escape_label(val)}"' for key, val in labels.items() if val]
return "{" + ",".join(parts) + "}" if parts else ""
def _bounded_test_name(value: str) -> str:
"""Keep test labels readable, unique, and safely below scraper limits."""
encoded = value.encode("utf-8")
if len(encoded) <= TEST_CASE_LABEL_MAX_BYTES:
return value
digest = hashlib.sha256(encoded).hexdigest()[:16]
suffix = f"...[sha256:{digest}]"
prefix = encoded[: TEST_CASE_LABEL_MAX_BYTES - len(suffix)]
return prefix.decode("utf-8", errors="ignore") + suffix
def _read_text(url: str) -> str:
"""Fetch a plain-text response body from the given URL."""
with urllib.request.urlopen(url, timeout=10) as response:
return response.read().decode("utf-8")
def _post_text(url: str, payload: str) -> None:
"""PUT a plain-text payload and fail on any 4xx/5xx response."""
request = urllib.request.Request(
url,
data=payload.encode("utf-8"),
method="PUT",
headers={"Content-Type": "text/plain"},
)
with urllib.request.urlopen(request, timeout=10) as response:
if response.status >= 400:
raise RuntimeError(f"push failed with status={response.status}")
def _parse_junit(path: str) -> dict[str, int]:
"""Parse a JUnit XML file into aggregate test counters."""
if not os.path.exists(path):
return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
tree = ET.parse(path)
root = tree.getroot()
totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
suites: list[ET.Element]
if root.tag == "testsuite":
suites = [root]
elif root.tag == "testsuites":
suites = [elem for elem in root if elem.tag == "testsuite"]
else:
suites = []
for suite in suites:
for key in totals:
raw_value = suite.attrib.get(key, "0")
try:
totals[key] += int(float(raw_value))
except ValueError:
totals[key] += 0
return totals
def _collect_junit_totals(pattern: str) -> dict[str, int]:
"""Sum JUnit counters across every XML file matching the pattern."""
totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
for path in sorted(glob(pattern)):
parsed = _parse_junit(path)
for key in totals:
totals[key] += parsed[key]
return totals
def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]:
"""Collect individual JUnit test-case statuses for flaky-test trend panels."""
cases: list[tuple[str, str]] = []
for path in sorted(glob(pattern)):
if not os.path.exists(path):
continue
root = ET.parse(path).getroot()
suites: list[ET.Element]
if root.tag == "testsuite":
suites = [root]
elif root.tag == "testsuites":
suites = [elem for elem in root if elem.tag == "testsuite"]
else:
suites = []
for suite in suites:
for test_case in suite.findall("testcase"):
case_name = test_case.attrib.get("name", "").strip()
class_name = test_case.attrib.get("classname", "").strip()
if not case_name:
continue
full_name = f"{class_name}.{case_name}" if class_name else case_name
status = "passed"
if test_case.find("failure") is not None or test_case.find("error") is not None:
status = "failed"
elif test_case.find("skipped") is not None:
status = "skipped"
cases.append((_bounded_test_name(full_name), status))
return cases
def _read_exit_code(path: str) -> int:
"""Read the quality-gate exit code, defaulting to failure if missing."""
try:
with open(path, "r", encoding="utf-8") as handle:
return int(handle.read().strip())
except (FileNotFoundError, ValueError):
return 1
def _load_summary(path: str) -> dict:
"""Load the JSON quality-gate summary, returning an empty mapping on error."""
try:
with open(path, "r", encoding="utf-8") as handle:
return json.load(handle)
except (FileNotFoundError, json.JSONDecodeError):
return {}
def _summary_float(summary: dict, key: str) -> float:
"""Extract a float-like value from the summary, defaulting to 0.0."""
value = summary.get(key)
if isinstance(value, (int, float)):
return float(value)
return 0.0
def _summary_int(summary: dict, key: str) -> int:
"""Extract an int-like value from the summary, defaulting to 0."""
value = summary.get(key)
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
return 0
def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str, str]) -> float:
"""Return the current counter value for a labeled metric if present."""
text = _read_text(f"{pushgateway_url.rstrip('/')}/metrics")
for line in text.splitlines():
if not line.startswith(metric + "{"):
continue
if any(f'{key}="{value}"' not in line for key, value in labels.items()):
continue
parts = line.split()
if len(parts) < 2:
continue
try:
return float(parts[1])
except ValueError:
return 0.0
return 0.0
def _build_payload(
suite: str,
status: str,
tests: dict[str, int],
test_cases: list[tuple[str, str]],
ok_count: int,
failed_count: int,
branch: str,
build_number: str,
jenkins_job: str,
summary: dict | None = None,
workspace_line_coverage_percent: float = 0.0,
source_files_total: int = 0,
source_lines_over_500: int = 0,
check_statuses: dict[str, str] | None = None,
) -> str:
"""Build the Pushgateway payload for the current suite run."""
passed = max(tests["tests"] - tests["failures"] - tests["errors"] - tests["skipped"], 0)
build_labels = _label_str(
{
"suite": suite,
"branch": branch or "unknown",
"build_number": build_number or "unknown",
"jenkins_job": jenkins_job or suite,
}
)
test_case_base_labels = {
"suite": suite,
"branch": branch or "unknown",
"build_number": build_number or "unknown",
"jenkins_job": jenkins_job or suite,
}
lines = [
"# TYPE platform_quality_gate_runs_total counter",
f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}',
f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count}',
"# TYPE titan_iac_quality_gate_tests_total gauge",
f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="passed"}} {passed}',
f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="failed"}} {tests["failures"]}',
f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="error"}} {tests["errors"]}',
f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {tests["skipped"]}',
"# TYPE titan_iac_quality_gate_run_status gauge",
f'titan_iac_quality_gate_run_status{{suite="{suite}",status="ok"}} {1 if status == "ok" else 0}',
f'titan_iac_quality_gate_run_status{{suite="{suite}",status="failed"}} {1 if status == "failed" else 0}',
"# TYPE platform_quality_gate_build_info gauge",
f"platform_quality_gate_build_info{build_labels} 1",
"# TYPE titan_iac_quality_gate_build_info gauge",
f"titan_iac_quality_gate_build_info{build_labels} 1",
"# TYPE platform_quality_gate_workspace_line_coverage_percent gauge",
f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {workspace_line_coverage_percent:.3f}',
"# TYPE platform_quality_gate_source_files_total gauge",
f'platform_quality_gate_source_files_total{{suite="{suite}"}} {source_files_total}',
"# TYPE platform_quality_gate_source_lines_over_500_total gauge",
f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}',
]
if check_statuses:
lines.append("# TYPE titan_iac_quality_gate_checks_total gauge")
for check_name in CANONICAL_CHECKS:
check_status = check_statuses.get(check_name, "not_applicable")
lines.append(
f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(check_name)}",result="{_escape_label(check_status)}"}} 1'
)
lines.append("# TYPE platform_quality_gate_test_case_result gauge")
if test_cases:
for test_name, test_status in test_cases:
labels = {
**test_case_base_labels,
"test": _bounded_test_name(test_name),
"status": test_status,
}
lines.append(
f"platform_quality_gate_test_case_result{_label_str(labels)} 1"
)
else:
labels = {**test_case_base_labels, "test": "__no_test_cases__", "status": "skipped"}
lines.append(
f"platform_quality_gate_test_case_result{_label_str(labels)} 1"
)
return "\n".join(lines) + "\n"
def main() -> int:
"""Publish the quality-gate metrics and print a compact run summary."""
suite = os.getenv("SUITE_NAME", "titan_iac")
pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091")
job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci")
junit_glob = os.getenv("JUNIT_GLOB", os.getenv("JUNIT_PATH", "build/junit-*.xml"))
exit_code_path = os.getenv("QUALITY_GATE_EXIT_CODE_PATH", os.getenv("GLUE_EXIT_CODE_PATH", "build/quality-gate.rc"))
summary_path = os.getenv("QUALITY_GATE_SUMMARY_PATH", "build/quality-gate-summary.json")
branch = os.getenv("BRANCH_NAME") or os.getenv("GIT_BRANCH") or "unknown"
if branch.startswith("origin/"):
branch = branch[len("origin/") :]
build_number = os.getenv("BUILD_NUMBER", "")
jenkins_job = os.getenv("JOB_NAME", "titan-iac")
tests = _collect_junit_totals(junit_glob)
test_cases = _collect_junit_cases(junit_glob)
exit_code = _read_exit_code(exit_code_path)
status = "ok" if exit_code == 0 else "failed"
summary = _load_summary(summary_path)
workspace_line_coverage_percent = _summary_float(summary, "workspace_line_coverage_percent")
if workspace_line_coverage_percent <= 0:
workspace_line_coverage_percent = _infer_workspace_coverage_percent(summary, "build/coverage-unit.xml")
source_files_total = _summary_int(summary, "source_files_total")
source_lines_over_500 = _summary_int(summary, "source_lines_over_500")
if source_lines_over_500 <= 0:
source_lines_over_500 = _infer_source_lines_over_500(summary)
sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json"))
semgrep_report = _load_optional_json(os.getenv("QUALITY_GATE_SEMGREP_REPORT", "build/semgrep-report.json"))
supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json"))
truthy = {"1", "true", "yes", "on"}
supply_chain_required = (
os.getenv("QUALITY_GATE_IRONBANK_REQUIRED", "0").strip().lower() in truthy
or os.getenv("PUBLISH_IMAGES", "false").strip().lower() in truthy
)
check_statuses = _build_check_statuses(
summary=summary,
tests=tests,
workspace_line_coverage_percent=workspace_line_coverage_percent,
source_lines_over_500=source_lines_over_500,
sonarqube_report=sonarqube_report,
semgrep_report=semgrep_report,
supply_chain_report=supply_chain_report,
supply_chain_required=supply_chain_required,
)
ok_count = int(
_fetch_existing_counter(
pushgateway_url,
"platform_quality_gate_runs_total",
{"job": job_name, "suite": suite, "status": "ok"},
)
)
failed_count = int(
_fetch_existing_counter(
pushgateway_url,
"platform_quality_gate_runs_total",
{"job": job_name, "suite": suite, "status": "failed"},
)
)
if status == "ok":
ok_count += 1
else:
failed_count += 1
payload = _build_payload(
suite=suite,
status=status,
tests=tests,
test_cases=test_cases,
ok_count=ok_count,
failed_count=failed_count,
branch=branch,
build_number=build_number,
jenkins_job=jenkins_job,
summary=summary,
workspace_line_coverage_percent=workspace_line_coverage_percent,
source_files_total=source_files_total,
source_lines_over_500=source_lines_over_500,
check_statuses=check_statuses,
)
push_url = f"{pushgateway_url.rstrip('/')}/metrics/job/{job_name}/suite/{suite}"
_post_text(push_url, payload)
summary = {
"suite": suite,
"status": status,
"tests_total": tests["tests"],
"tests_failed": tests["failures"],
"tests_error": tests["errors"],
"tests_skipped": tests["skipped"],
"ok_count": ok_count,
"failed_count": failed_count,
"checks_recorded": len(check_statuses),
"workspace_line_coverage_percent": workspace_line_coverage_percent,
"source_files_total": source_files_total,
"source_lines_over_500": source_lines_over_500,
}
print(json.dumps(summary, sort_keys=True))
return 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())

View File

@ -1,221 +0,0 @@
#!/usr/bin/env python3
"""Quality/status helpers for publish_test_metrics."""
from __future__ import annotations
import json
from pathlib import Path
import xml.etree.ElementTree as ET
SUCCESS_STATUSES = {"ok", "pass", "passed", "success", "compliant"}
NOT_APPLICABLE_STATUSES = {"not_applicable", "n/a", "na", "none", "skipped"}
FAILED_STATUSES = {"failed", "fail", "error", "errors", "warn", "warning", "red"}
CANONICAL_CHECKS = [
"tests",
"coverage",
"loc",
"docs_naming",
"gate_glue",
"sonarqube",
"semgrep",
"supply_chain",
]
def _infer_workspace_coverage_percent(summary: dict, default_xml: str) -> float:
"""Infer workspace line coverage from quality summary coverage XML metadata."""
results = summary.get("results", []) if isinstance(summary, dict) else []
coverage_xml = default_xml
for result in results:
if not isinstance(result, dict):
continue
if str(result.get("name") or "").strip().lower() != "coverage":
continue
candidate = str(result.get("coverage_xml") or "").strip()
if candidate:
coverage_xml = candidate
break
xml_path = Path(coverage_xml)
if not xml_path.exists():
return 0.0
try:
root = ET.parse(xml_path).getroot()
line_rate = root.attrib.get("line-rate")
if line_rate is None:
return 0.0
return float(line_rate) * 100.0
except (ET.ParseError, OSError, ValueError):
return 0.0
def _infer_source_lines_over_500(summary: dict) -> int:
"""Infer over-limit source file count from hygiene issue payloads."""
results = summary.get("results", []) if isinstance(summary, dict) else []
for result in results:
if not isinstance(result, dict):
continue
if str(result.get("name") or "").strip().lower() not in {"hygiene", "loc", "smell"}:
continue
issues = result.get("issues")
if not isinstance(issues, list):
continue
return sum(1 for item in issues if isinstance(item, str) and item.startswith("file exceeds"))
return 0
def _normalize_result_status(value: str | None, default: str = "failed") -> str:
"""Map arbitrary check status text into canonical check result buckets."""
if not value:
return default
normalized = value.strip().lower()
if normalized in SUCCESS_STATUSES:
return "ok"
if normalized in NOT_APPLICABLE_STATUSES:
return "not_applicable"
if normalized in FAILED_STATUSES:
return "failed"
return default
def _load_optional_json(path: str | None) -> dict:
"""Load an optional JSON report file, returning an empty object when absent."""
if not path:
return {}
candidate = Path(path)
if not candidate.exists():
return {}
try:
return json.loads(candidate.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
def _combine_statuses(statuses: list[str]) -> str:
"""Roll up many check statuses into one canonical result."""
if not statuses:
return "not_applicable"
if any(status == "failed" for status in statuses):
return "failed"
if all(status == "not_applicable" for status in statuses):
return "not_applicable"
if all(status in {"ok", "not_applicable"} for status in statuses):
return "ok"
return "failed"
def _infer_sonarqube_status(report: dict) -> str:
"""Infer canonical SonarQube check status from its JSON report payload."""
if not report:
return "not_applicable"
status = (
report.get("projectStatus", {}).get("status")
or report.get("qualityGate", {}).get("status")
or report.get("status")
)
return _normalize_result_status(str(status) if status is not None else None, default="failed")
def _infer_supply_chain_status(report: dict, required: bool) -> str:
"""Infer canonical supply-chain status from IronBank/artifact report payload."""
if not report:
return "failed" if required else "not_applicable"
compliant = report.get("compliant")
if isinstance(compliant, bool):
if compliant:
return "ok"
return "failed" if required else "not_applicable"
status = report.get("status")
if status is None:
return "failed" if required else "not_applicable"
normalized = _normalize_result_status(str(status), default="failed")
if normalized == "failed" and not required:
return "not_applicable"
if normalized == "not_applicable" and required:
return "failed"
return normalized
def _infer_semgrep_status(report: dict) -> str:
"""Infer canonical Semgrep check status from its JSON report payload."""
if not report:
return "not_applicable"
status = report.get("status")
if status is None:
blocking_findings = report.get("blocking_findings")
errors_total = report.get("errors_total")
if isinstance(blocking_findings, int) and isinstance(errors_total, int):
return "failed" if blocking_findings > 0 or errors_total > 0 else "ok"
return _normalize_result_status(str(status) if status is not None else None, default="failed")
def _build_check_statuses(
summary: dict | None,
tests: dict[str, int],
workspace_line_coverage_percent: float,
source_lines_over_500: int,
sonarqube_report: dict,
semgrep_report: dict,
supply_chain_report: dict,
supply_chain_required: bool,
) -> dict[str, str]:
"""Generate the canonical quality-check status map for dashboarding."""
raw_results = summary.get("results", []) if isinstance(summary, dict) else []
status_by_name: dict[str, str] = {}
for result in raw_results:
if not isinstance(result, dict):
continue
check_name = str(result.get("name") or "").strip().lower()
if not check_name:
continue
status_by_name[check_name] = _normalize_result_status(result.get("status"), default="failed")
tests_status = status_by_name.get("tests")
if not tests_status:
candidate_keys = ["unit", "integration", "e2e", "pytest", "test", "tests"]
candidates = [status_by_name[key] for key in candidate_keys if key in status_by_name]
if candidates:
tests_status = _combine_statuses(candidates)
elif tests["tests"] > 0:
tests_status = "ok" if (tests["failures"] + tests["errors"]) == 0 else "failed"
else:
tests_status = "not_applicable"
coverage_status = status_by_name.get("coverage")
if not coverage_status:
if workspace_line_coverage_percent > 0:
coverage_status = "ok" if workspace_line_coverage_percent >= 95.0 else "failed"
else:
coverage_status = "not_applicable"
loc_status = status_by_name.get("loc")
if not loc_status:
loc_status = "ok" if source_lines_over_500 == 0 else "failed"
docs_naming_status = status_by_name.get("docs_naming")
if not docs_naming_status:
candidates = [status_by_name[key] for key in ["docs", "hygiene", "smell", "lint", "naming"] if key in status_by_name]
docs_naming_status = _combine_statuses(candidates) if candidates else "not_applicable"
gate_glue_status = status_by_name.get("gate_glue")
if not gate_glue_status:
candidates = [status_by_name[key] for key in ["gate_glue", "glue", "gate"] if key in status_by_name]
gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable"
sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report)
semgrep_status = status_by_name.get("semgrep") or _infer_semgrep_status(semgrep_report)
supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status(
supply_chain_report,
required=supply_chain_required,
)
return {
"tests": tests_status,
"coverage": coverage_status,
"loc": loc_status,
"docs_naming": docs_naming_status,
"gate_glue": gate_glue_status,
"sonarqube": sonarqube_status,
"semgrep": semgrep_status,
"supply_chain": supply_chain_status,
}

View File

@ -1,158 +0,0 @@
#!/usr/bin/env python3
"""Build a compact Semgrep gate report from Semgrep JSON output."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
BLOCKING_SEVERITIES = {"ERROR"}
SEVERITIES = ("ERROR", "WARNING", "INFO", "UNKNOWN")
SONAR_SEVERITY_BY_SEMGREP = {
"ERROR": "CRITICAL",
"WARNING": "MAJOR",
"INFO": "MINOR",
"UNKNOWN": "INFO",
}
def _read_json(path: Path) -> dict[str, Any]:
"""Read a JSON object from disk, returning an error-shaped payload on failure."""
if not path.exists():
return {"errors": [{"message": f"report missing: {path}"}], "results": []}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []}
if not isinstance(payload, dict):
return {"errors": [{"message": "report payload is not an object"}], "results": []}
return payload
def _finding_severity(finding: dict[str, Any]) -> str:
"""Return a normalized Semgrep finding severity."""
extra = finding.get("extra")
severity = extra.get("severity") if isinstance(extra, dict) else None
normalized = str(severity or "UNKNOWN").strip().upper()
return normalized if normalized in SEVERITIES else "UNKNOWN"
def build_report(
semgrep_payload: dict[str, Any],
*,
semgrep_exit_code: int,
blocking_severities: set[str] | None = None,
) -> dict[str, Any]:
"""Summarize Semgrep evidence into the quality-gate report contract."""
blocking = blocking_severities or BLOCKING_SEVERITIES
raw_results = semgrep_payload.get("results", [])
raw_errors = semgrep_payload.get("errors", [])
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
severity_counts = dict.fromkeys(SEVERITIES, 0)
blocking_findings = 0
for finding in results:
severity = _finding_severity(finding)
severity_counts[severity] += 1
if severity in blocking:
blocking_findings += 1
engine_error = semgrep_exit_code not in {0, 1} or bool(errors)
status = "failed" if engine_error or blocking_findings else "ok"
return {
"status": status,
"scanner": "semgrep",
"semgrep_rc": semgrep_exit_code,
"findings_total": len(results),
"blocking_findings": blocking_findings,
"errors_total": len(errors),
"severity_counts": severity_counts,
"blocking_severities": sorted(blocking),
}
def _line_number(value: Any, default: int = 1) -> int:
"""Return a Sonar-compatible one-indexed line number."""
try:
line = int(value)
except (TypeError, ValueError):
return default
return max(line, default)
def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
"""Convert one Semgrep finding into SonarQube generic issue format."""
path = str(finding.get("path") or "").strip()
if not path:
return None
extra = finding.get("extra") if isinstance(finding.get("extra"), dict) else {}
severity = _finding_severity(finding)
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
# _line_number floors the end line at start_line, so the range never inverts.
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
return {
"engineId": "semgrep",
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
"type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL",
"severity": SONAR_SEVERITY_BY_SEMGREP[severity],
"primaryLocation": {
"message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"),
"filePath": path,
"textRange": {
"startLine": start_line,
"endLine": end_line,
},
},
}
def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
"""Build SonarQube generic external issues from Semgrep JSON output."""
raw_results = semgrep_payload.get("results", [])
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))]
return {"issues": issues}
def main(argv: list[str] | None = None) -> int:
"""CLI entrypoint used by Jenkins after Semgrep finishes."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--semgrep-json", required=True)
parser.add_argument("--exit-code", type=int, default=0)
parser.add_argument("--output", required=True)
parser.add_argument("--sonar-issues-output")
parser.add_argument(
"--blocking-severity",
action="append",
default=[],
help="Severity that should mark the report failed. Defaults to ERROR.",
)
args = parser.parse_args(argv)
blocking = {item.strip().upper() for item in args.blocking_severity if item.strip()}
payload = _read_json(Path(args.semgrep_json))
report = build_report(
payload,
semgrep_exit_code=args.exit_code,
blocking_severities=blocking or BLOCKING_SEVERITIES,
)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if args.sonar_issues_output:
sonar_output = Path(args.sonar_issues_output)
sonar_output.parent.mkdir(parents=True, exist_ok=True)
sonar_output.write_text(
json.dumps(build_sonar_issues(payload), indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())

View File

@ -1,177 +0,0 @@
"""Build a titan-iac supply-chain compliance report from Trivy evidence."""
from __future__ import annotations
import argparse
import datetime as dt
import json
from pathlib import Path
from typing import Any
FAIL_SEVERITIES = {"HIGH", "CRITICAL"}
def _read_json(path: Path) -> dict[str, Any]:
"""Read a JSON object from disk for use as pipeline evidence."""
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError(f"{path} must contain a JSON object")
return payload
def _parse_day(raw: str | None) -> dt.date | None:
"""Parse an ISO day while letting optional waiver dates stay optional."""
if not raw:
return None
return dt.date.fromisoformat(raw)
def _today(override: str | None = None) -> dt.date:
"""Return the policy day so tests can pin expiry behavior."""
return _parse_day(override) or dt.date.today()
def _load_waiver_pairs(path: Path | None, policy_day: dt.date) -> tuple[set[tuple[str, str]], int]:
"""Return active ``(misconfiguration id, target)`` waivers and expired count."""
if path is None or not path.exists():
return set(), 0
payload = _read_json(path)
default_expires_at = payload.get("default_expires_at")
active: set[tuple[str, str]] = set()
expired = 0
for entry in payload.get("misconfigurations", []):
if not isinstance(entry, dict):
continue
misconfiguration_id = str(entry.get("id") or "").strip()
if not misconfiguration_id:
continue
expires_at = _parse_day(str(entry.get("expires_at") or default_expires_at or ""))
targets = entry.get("targets", [])
if not isinstance(targets, list):
continue
if expires_at and expires_at < policy_day:
expired += len(targets)
continue
# Waivers are target-specific so a new unsafe manifest fails until it is
# either fixed or deliberately accepted with a fresh expiration.
for target in targets:
if isinstance(target, str) and target:
active.add((misconfiguration_id, target))
return active, expired
def _iter_failed_misconfigurations(payload: dict[str, Any]):
"""Yield failed high/critical Trivy misconfiguration records."""
for result in payload.get("Results", []):
if not isinstance(result, dict):
continue
target = str(result.get("Target") or "")
for item in result.get("Misconfigurations") or []:
if not isinstance(item, dict):
continue
if item.get("Status") != "FAIL":
continue
if str(item.get("Severity") or "").upper() not in FAIL_SEVERITIES:
continue
yield target, item
def _count_vulnerabilities(payload: dict[str, Any], severity: str) -> int:
"""Count Trivy vulnerabilities at a specific severity."""
count = 0
for result in payload.get("Results", []):
if not isinstance(result, dict):
continue
for item in result.get("Vulnerabilities") or []:
if isinstance(item, dict) and str(item.get("Severity") or "").upper() == severity:
count += 1
return count
def _count_secrets(payload: dict[str, Any]) -> int:
"""Count detected secrets in the Trivy filesystem report."""
count = 0
for result in payload.get("Results", []):
if isinstance(result, dict):
count += len(result.get("Secrets") or [])
return count
def build_report(
trivy_payload: dict[str, Any],
waiver_path: Path | None = None,
today_override: str | None = None,
) -> dict[str, Any]:
"""Build the compliance summary consumed by the quality gate."""
policy_day = _today(today_override)
active_waivers, expired_waivers = _load_waiver_pairs(waiver_path, policy_day)
open_misconfigs: list[dict[str, str]] = []
waived_misconfigs = 0
for target, item in _iter_failed_misconfigurations(trivy_payload):
misconfiguration_id = str(item.get("ID") or "")
if (misconfiguration_id, target) in active_waivers:
waived_misconfigs += 1
continue
open_misconfigs.append(
{
"id": misconfiguration_id,
"target": target,
"severity": str(item.get("Severity") or ""),
"title": str(item.get("Title") or ""),
}
)
critical = _count_vulnerabilities(trivy_payload, "CRITICAL")
high = _count_vulnerabilities(trivy_payload, "HIGH")
secrets = _count_secrets(trivy_payload)
status = (
"ok"
if critical == 0 and secrets == 0 and not open_misconfigs and expired_waivers == 0
else "failed"
)
return {
"status": status,
"compliant": status == "ok",
"category": "artifact_security",
"scan_type": "filesystem",
"scanner": "trivy",
"critical_vulnerabilities": critical,
"high_vulnerabilities": high,
"high_vulnerability_policy": "observe",
"secrets": secrets,
"high_or_critical_misconfigurations": len(open_misconfigs),
"waived_misconfigurations": waived_misconfigs,
"expired_waivers": expired_waivers,
"waiver_file": str(waiver_path) if waiver_path else "",
"open_misconfiguration_examples": open_misconfigs[:20],
}
def main(argv: list[str] | None = None) -> int:
"""CLI entrypoint used by Jenkins after the Trivy scan completes."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--trivy-json", required=True)
parser.add_argument("--waivers")
parser.add_argument("--output", required=True)
parser.add_argument("--today")
args = parser.parse_args(argv)
trivy_payload = _read_json(Path(args.trivy_json))
waiver_path = Path(args.waivers) if args.waivers else None
report = build_report(trivy_payload, waiver_path=waiver_path, today_override=args.today)
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
return 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())

View File

@ -1,18 +0,0 @@
max_success_age_hours: 48
allow_suspended:
- bstein-dev-home/vaultwarden-cred-sync
- comms/guest-name-randomizer
- comms/othrys-room-reset
- comms/pin-othrys-invite
- comms/seed-othrys-room
- finance/firefly-user-sync
- health/wger-admin-ensure
- health/wger-user-sync
- mailu-mailserver/mailu-sync-nightly
- nextcloud/nextcloud-mail-sync
- vault/vault-oidc-config
ariadne_schedule_tasks:
- schedule.mailu_sync
- schedule.nextcloud_sync
- schedule.vaultwarden_sync
- schedule.wger_admin

View File

@ -1,34 +0,0 @@
"""Skip live glue checks when the VictoriaMetrics endpoint is unavailable."""
from __future__ import annotations
import os
import socket
from urllib.parse import urlparse
import pytest
def _endpoint_available() -> bool:
"""Return whether the configured VictoriaMetrics host resolves locally."""
vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
parsed = urlparse(vm_url)
host = parsed.hostname
port = parsed.port or (443 if parsed.scheme == "https" else 80)
if not host:
return False
try:
socket.getaddrinfo(host, port)
except socket.gaierror:
return False
return True
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
"""Mark glue tests skipped when the VictoriaMetrics host cannot resolve."""
del config
if _endpoint_available():
return
skip_marker = pytest.mark.skip(reason="VictoriaMetrics endpoint is not reachable in this workspace")
for item in items:
item.add_marker(skip_marker)

View File

@ -1,108 +0,0 @@
"""Glue checks for Ariadne schedules exported to VictoriaMetrics."""
from __future__ import annotations
import os
from datetime import datetime, timezone
from pathlib import Path
import requests
import yaml
CONFIG_PATH = Path(__file__).with_name("config.yaml")
def _load_config() -> dict:
with CONFIG_PATH.open("r", encoding="utf-8") as handle:
return yaml.safe_load(handle) or {}
def _query(promql: str) -> list[dict]:
vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
response = requests.get(f"{vm_url}/api/v1/query", params={"query": promql}, timeout=10)
response.raise_for_status()
payload = response.json()
return payload.get("data", {}).get("result", [])
def _expected_tasks() -> list[dict]:
cfg = _load_config()
tasks = [
_normalize_task(item, cfg)
for item in cfg.get("ariadne_schedule_tasks", [])
]
assert tasks, "No Ariadne schedule tasks configured"
return tasks
def _normalize_task(item: object, cfg: dict) -> dict:
if isinstance(item, str):
return {
"task": item,
"check_last_success": True,
"max_success_age_hours": cfg.get("max_success_age_hours", 48),
}
if isinstance(item, dict):
normalized = dict(item)
normalized.setdefault("check_last_success", True)
normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
return normalized
raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
def _tracked_tasks(tasks: list[dict]) -> list[dict]:
tracked = [item for item in tasks if item.get("check_last_success")]
assert tracked, "No Ariadne schedule tasks are marked for success tracking"
return tracked
def _task_regex(tasks: list[dict]) -> str:
return "|".join(item["task"] for item in tasks)
def test_ariadne_schedule_series_exist():
tasks = _expected_tasks()
selector = _task_regex(tasks)
series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}')
seen = {item.get("metric", {}).get("task") for item in series}
missing = [item["task"] for item in tasks if item["task"] not in seen]
assert not missing, f"Missing next-run metrics for: {', '.join(missing)}"
def test_ariadne_schedule_recent_success():
tasks = _tracked_tasks(_expected_tasks())
selector = _task_regex(tasks)
series = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
seen = {item.get("metric", {}).get("task") for item in series}
missing = [item["task"] for item in tasks if item["task"] not in seen]
assert not missing, f"Missing last-success metrics for: {', '.join(missing)}"
now = datetime.now(timezone.utc)
age_by_task = {
item.get("metric", {}).get("task"): (now - datetime.fromtimestamp(float(item["value"][1]), tz=timezone.utc)).total_seconds() / 3600
for item in series
}
too_old = [
f"{task} ({age_by_task[task]:.1f}h > {item['max_success_age_hours']}h)"
for item in tasks
if (task := item["task"]) in age_by_task and age_by_task[task] > float(item["max_success_age_hours"])
]
assert not too_old, "Ariadne schedules are stale: " + ", ".join(too_old)
def test_ariadne_schedule_last_status_present_and_boolean():
tasks = _tracked_tasks(_expected_tasks())
selector = _task_regex(tasks)
series = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
seen = {item.get("metric", {}).get("task") for item in series}
missing = [item["task"] for item in tasks if item["task"] not in seen]
assert not missing, f"Missing last-status metrics for: {', '.join(missing)}"
invalid = []
for item in series:
task = item.get("metric", {}).get("task")
value = float(item["value"][1])
if value not in (0.0, 1.0):
invalid.append(f"{task}={value}")
assert not invalid, f"Unexpected Ariadne last-status values: {', '.join(invalid)}"

View File

@ -1,46 +0,0 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
import yaml
from kubernetes import client, config
CONFIG_PATH = Path(__file__).with_name("config.yaml")
def _load_config() -> dict:
with CONFIG_PATH.open("r", encoding="utf-8") as handle:
return yaml.safe_load(handle) or {}
def _load_kube():
try:
config.load_incluster_config()
except config.ConfigException:
config.load_kube_config()
def test_glue_cronjobs_recent_success():
cfg = _load_config()
max_age_hours = int(cfg.get("max_success_age_hours", 48))
allow_suspended = set(cfg.get("allow_suspended", []))
_load_kube()
batch = client.BatchV1Api()
cronjobs = batch.list_cron_job_for_all_namespaces(label_selector="atlas.bstein.dev/glue=true").items
assert cronjobs, "No glue cronjobs found with atlas.bstein.dev/glue=true"
now = datetime.now(timezone.utc)
for cronjob in cronjobs:
name = f"{cronjob.metadata.namespace}/{cronjob.metadata.name}"
if cronjob.spec.suspend:
assert name in allow_suspended, f"{name} is suspended but not in allow_suspended"
continue
last_success = cronjob.status.last_successful_time
assert last_success is not None, f"{name} has no lastSuccessfulTime"
age_hours = (now - last_success).total_seconds() / 3600
assert age_hours <= max_age_hours, f"{name} last success {age_hours:.1f}h ago"

View File

@ -1,87 +0,0 @@
"""Glue checks for the metrics the quality-gate publishes."""
from __future__ import annotations
import os
from pathlib import Path
import requests
import yaml
VM_URL = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
CONFIG_PATH = Path(__file__).with_name("config.yaml")
def _load_config() -> dict:
with CONFIG_PATH.open("r", encoding="utf-8") as handle:
return yaml.safe_load(handle) or {}
def _query(promql: str) -> list[dict]:
response = requests.get(f"{VM_URL}/api/v1/query", params={"query": promql}, timeout=10)
response.raise_for_status()
payload = response.json()
return payload.get("data", {}).get("result", [])
def _expected_tasks() -> list[dict]:
cfg = _load_config()
tasks = [
_normalize_task(item, cfg)
for item in cfg.get("ariadne_schedule_tasks", [])
]
assert tasks, "No Ariadne schedule tasks configured"
return tasks
def _normalize_task(item: object, cfg: dict) -> dict:
if isinstance(item, str):
return {
"task": item,
"check_last_success": True,
"max_success_age_hours": cfg.get("max_success_age_hours", 48),
}
if isinstance(item, dict):
normalized = dict(item)
normalized.setdefault("check_last_success", True)
normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
return normalized
raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
def _tracked_tasks(tasks: list[dict]) -> list[dict]:
tracked = [item for item in tasks if item.get("check_last_success")]
assert tracked, "No Ariadne schedule tasks are marked for success tracking"
return tracked
def _task_regex(tasks: list[dict]) -> str:
return "|".join(item["task"] for item in tasks)
def test_ariadne_schedule_metrics_present():
tasks = _expected_tasks()
selector = _task_regex(tasks)
series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}')
seen = {item.get("metric", {}).get("task") for item in series}
missing = [item["task"] for item in tasks if item["task"] not in seen]
assert not missing, f"Missing Ariadne schedule metrics for: {', '.join(missing)}"
def test_ariadne_schedule_success_and_status_metrics_present():
tasks = _tracked_tasks(_expected_tasks())
selector = _task_regex(tasks)
success = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
status = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
success_tasks = {item.get("metric", {}).get("task") for item in success}
status_tasks = {item.get("metric", {}).get("task") for item in status}
expected = {item["task"] for item in tasks}
missing_success = sorted(expected - success_tasks)
missing_status = sorted(expected - status_tasks)
assert not missing_success, f"Missing Ariadne success metrics for: {', '.join(missing_success)}"
assert not missing_status, f"Missing Ariadne status metrics for: {', '.join(missing_status)}"

View File

@ -1,405 +0,0 @@
{
"version": 1,
"generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan",
"default_expires_at": "2026-09-30",
"ticket": "atlas-quality-wave-k8s-hardening",
"default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.",
"misconfigurations": [
{
"id": "DS-0002",
"targets": [
"dockerfiles/Dockerfile.ananke-node-helper"
]
},
{
"id": "KSV-0009",
"targets": [
"services/mailu/vip-controller.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml"
]
},
{
"id": "KSV-0010",
"targets": [
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml"
]
},
{
"id": "KSV-0014",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml",
"infrastructure/core/node-prefer-noschedule-cronjob.yaml",
"infrastructure/core/ntp-sync-daemonset.yaml",
"infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml",
"infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml",
"infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml",
"infrastructure/longhorn/core/vault-sync-deployment.yaml",
"infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml",
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
"infrastructure/postgres/statefulset.yaml",
"infrastructure/vault-csi/vault-csi-provider.yaml",
"services/ai-llm/deployment.yaml",
"services/bstein-dev-home/backend-deployment.yaml",
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
"services/bstein-dev-home/frontend-deployment.yaml",
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/vault-sync-deployment.yaml",
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
"services/comms/atlasbot-deployment.yaml",
"services/comms/coturn.yaml",
"services/comms/element-call-deployment.yaml",
"services/comms/guest-name-job.yaml",
"services/comms/guest-register-deployment.yaml",
"services/comms/livekit-token-deployment.yaml",
"services/comms/livekit.yaml",
"services/comms/mas-deployment.yaml",
"services/comms/repair-jobs/bstein-force-leave-job.yaml",
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
"services/comms/pin-othrys-job.yaml",
"services/comms/reset-othrys-room-job.yaml",
"services/comms/seed-othrys-room.yaml",
"services/comms/vault-sync-deployment.yaml",
"services/comms/wellknown.yaml",
"services/crypto/monerod/deployment.yaml",
"services/crypto/wallet-monero-temp/deployment.yaml",
"services/crypto/xmr-miner/deployment.yaml",
"services/crypto/xmr-miner/vault-sync-deployment.yaml",
"services/crypto/xmr-miner/xmrig-daemonset.yaml",
"services/finance/actual-budget-deployment.yaml",
"services/finance/firefly-cronjob.yaml",
"services/finance/firefly-deployment.yaml",
"services/finance/firefly-user-sync-cronjob.yaml",
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
"services/gitea/deployment.yaml",
"services/harbor/vault-sync-deployment.yaml",
"services/health/wger-admin-ensure-cronjob.yaml",
"services/health/wger-deployment.yaml",
"services/health/wger-user-sync-cronjob.yaml",
"services/jellyfin/deployment.yaml",
"services/jellyfin/loader.yaml",
"services/jenkins/deployment.yaml",
"services/jenkins/vault-sync-deployment.yaml",
"services/keycloak/deployment.yaml",
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/vault-sync-deployment.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/logging/oauth2-proxy.yaml",
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
"services/logging/opensearch-prune-cronjob.yaml",
"services/logging/vault-sync-deployment.yaml",
"services/mailu/mailu-sync-cronjob.yaml",
"services/mailu/mailu-sync-listener.yaml",
"services/mailu/vault-sync-deployment.yaml",
"services/mailu/vip-controller.yaml",
"services/maintenance/apps/ariadne-deployment.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/networking/oauth2-proxy-metis.yaml",
"services/maintenance/networking/oauth2-proxy-soteria.yaml",
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
"services/maintenance/apps/soteria-deployment.yaml",
"services/maintenance/bootstrap/vault-sync-deployment.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml",
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
"services/monitoring/platform-quality-gateway-deployment.yaml",
"services/monitoring/platform-quality-suite-probe-cronjob.yaml",
"services/monitoring/postmark-exporter-deployment.yaml",
"services/monitoring/vmalert-atlas-availability.yaml",
"services/monitoring/vault-sync-deployment.yaml",
"services/nextcloud-mail-sync/cronjob.yaml",
"services/nextcloud/collabora.yaml",
"services/nextcloud/cronjob.yaml",
"services/nextcloud/deployment.yaml",
"services/nextcloud/maintenance-cronjob.yaml",
"services/oauth2-proxy/deployment.yaml",
"services/openldap/statefulset.yaml",
"services/outline/deployment.yaml",
"services/outline/redis-deployment.yaml",
"services/pegasus/deployment.yaml",
"services/pegasus/vault-sync-deployment.yaml",
"services/planka/deployment.yaml",
"services/quality/oauth2-proxy-sonarqube.yaml",
"services/quality/sonarqube-deployment.yaml",
"services/quality/sonarqube-exporter-deployment.yaml",
"services/sui-metrics/base/deployment.yaml",
"services/typhon/vault-sync-deployment.yaml",
"services/vault/k8s-auth-config-cronjob.yaml",
"services/vault/oidc-config-cronjob.yaml",
"services/vault/statefulset.yaml",
"services/vaultwarden/deployment.yaml"
]
},
{
"id": "KSV-0017",
"targets": [
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml"
]
},
{
"id": "KSV-0041",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
"infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
"infrastructure/traefik/clusterrole.yaml",
"services/bstein-dev-home/rbac.yaml",
"services/comms/comms-secrets-ensure-rbac.yaml",
"services/comms/mas-db-ensure-rbac.yaml",
"services/comms/mas-secrets-ensure-rbac.yaml",
"services/maintenance/apps/soteria-rbac.yaml"
]
},
{
"id": "KSV-0047",
"targets": [
"services/monitoring/rbac.yaml"
]
},
{
"id": "KSV-0053",
"targets": [
"services/comms/comms-secrets-ensure-rbac.yaml",
"services/comms/mas-db-ensure-rbac.yaml",
"services/jenkins/serviceaccount.yaml",
"services/maintenance/apps/ariadne-rbac.yaml"
]
},
{
"id": "KSV-0056",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
"infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
"services/jenkins/serviceaccount.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml",
"services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml"
]
},
{
"id": "KSV-0114",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml"
]
},
{
"id": "KSV-0118",
"targets": [
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml",
"infrastructure/core/coredns-deployment.yaml",
"infrastructure/core/node-prefer-noschedule-cronjob.yaml",
"infrastructure/core/ntp-sync-daemonset.yaml",
"infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml",
"infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml",
"infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml",
"infrastructure/longhorn/core/vault-sync-deployment.yaml",
"infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml",
"infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
"infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
"infrastructure/postgres/statefulset.yaml",
"infrastructure/vault-csi/vault-csi-provider.yaml",
"services/ai-llm/deployment.yaml",
"services/bstein-dev-home/backend-deployment.yaml",
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
"services/bstein-dev-home/frontend-deployment.yaml",
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/vault-sync-deployment.yaml",
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
"services/comms/atlasbot-deployment.yaml",
"services/comms/coturn.yaml",
"services/comms/element-call-deployment.yaml",
"services/comms/guest-name-job.yaml",
"services/comms/livekit-token-deployment.yaml",
"services/comms/livekit.yaml",
"services/comms/mas-deployment.yaml",
"services/comms/repair-jobs/bstein-force-leave-job.yaml",
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
"services/comms/pin-othrys-job.yaml",
"services/comms/reset-othrys-room-job.yaml",
"services/comms/seed-othrys-room.yaml",
"services/comms/vault-sync-deployment.yaml",
"services/comms/wellknown.yaml",
"services/crypto/monerod/deployment.yaml",
"services/crypto/wallet-monero-temp/deployment.yaml",
"services/crypto/xmr-miner/deployment.yaml",
"services/crypto/xmr-miner/vault-sync-deployment.yaml",
"services/crypto/xmr-miner/xmrig-daemonset.yaml",
"services/finance/firefly-cronjob.yaml",
"services/finance/firefly-deployment.yaml",
"services/finance/firefly-user-sync-cronjob.yaml",
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
"services/gitea/deployment.yaml",
"services/harbor/vault-sync-deployment.yaml",
"services/health/wger-admin-ensure-cronjob.yaml",
"services/health/wger-deployment.yaml",
"services/health/wger-user-sync-cronjob.yaml",
"services/jellyfin/loader.yaml",
"services/jenkins/deployment.yaml",
"services/jenkins/vault-sync-deployment.yaml",
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/vault-sync-deployment.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/logging/oauth2-proxy.yaml",
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
"services/logging/opensearch-prune-cronjob.yaml",
"services/logging/vault-sync-deployment.yaml",
"services/mailu/mailu-sync-cronjob.yaml",
"services/mailu/mailu-sync-listener.yaml",
"services/mailu/vault-sync-deployment.yaml",
"services/mailu/vip-controller.yaml",
"services/maintenance/apps/ariadne-deployment.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/networking/oauth2-proxy-metis.yaml",
"services/maintenance/networking/oauth2-proxy-soteria.yaml",
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
"services/maintenance/apps/soteria-deployment.yaml",
"services/maintenance/bootstrap/vault-sync-deployment.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml",
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
"services/monitoring/platform-quality-gateway-deployment.yaml",
"services/monitoring/platform-quality-suite-probe-cronjob.yaml",
"services/monitoring/postmark-exporter-deployment.yaml",
"services/monitoring/vmalert-atlas-availability.yaml",
"services/monitoring/vault-sync-deployment.yaml",
"services/nextcloud/collabora.yaml",
"services/oauth2-proxy/deployment.yaml",
"services/openldap/statefulset.yaml",
"services/outline/deployment.yaml",
"services/outline/redis-deployment.yaml",
"services/pegasus/vault-sync-deployment.yaml",
"services/quality/oauth2-proxy-sonarqube.yaml",
"services/quality/sonarqube-deployment.yaml",
"services/quality/sonarqube-exporter-deployment.yaml",
"services/sui-metrics/base/deployment.yaml",
"services/sui-metrics/overlays/atlas/patch-node-selector.yaml",
"services/typhon/deployment.yaml",
"services/typhon/vault-sync-deployment.yaml",
"services/vault/k8s-auth-config-cronjob.yaml",
"services/vault/oidc-config-cronjob.yaml",
"services/vaultwarden/deployment.yaml"
]
},
{
"id": "KSV-0121",
"targets": [
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml"
]
}
]
}

View File

@ -1,6 +0,0 @@
# clusters/aether/flux-system/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- platform
- applications

View File

@ -0,0 +1,12 @@
# clusters/atlas/applications/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../services/crypto
- ../../services/gitea
- ../../services/jellyfin
- ../../services/jitsi
- ../../services/monitoring
- ../../services/pegasus
- ../../services/vault
- ../../services/zot

View File

@ -1,27 +0,0 @@
# clusters/atlas/flux-system/applications/ai-llm/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: ai-llm
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: Merge
spec:
interval: 10m
suspend: false
timeout: 30m
path: ./services/ai-llm
targetNamespace: ai
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: ollama
namespace: ai
dependsOn:
- name: core

View File

@ -1,20 +0,0 @@
# clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: bstein-dev-home-migrations
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Migration jobs run only during portal schema changes."
spec:
interval: 10m
path: ./services/bstein-dev-home/migration-jobs
prune: true
force: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: bstein-dev-home
wait: false
suspend: true

View File

@ -1,26 +0,0 @@
# clusters/atlas/flux-system/applications/bstein-dev-home/image-automation.yaml
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: bstein-dev-home
namespace: bstein-dev-home
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: main
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(bstein-dev-home): automated image update"
push:
branch: main
update:
strategy: Setters
path: services/bstein-dev-home

View File

@ -1,17 +0,0 @@
# clusters/atlas/flux-system/applications/bstein-dev-home/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: bstein-dev-home
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/bstein-dev-home
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: bstein-dev-home
wait: false

View File

@ -1,23 +0,0 @@
# clusters/atlas/flux-system/applications/cassandra-auth/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cassandra-auth
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/cassandra-auth
targetNamespace: sso
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: keycloak
- name: vault
- name: vault-injector
wait: false
timeout: 10m

View File

@ -1,29 +0,0 @@
# clusters/atlas/flux-system/applications/cassandra/image-automation.yaml
# Staged for the first Cassandra image rollout. Add this file to the parent
# applications kustomization after the namespace exists and the Harbor repos
# have initial tags.
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: cassandra
namespace: cassandra
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: main
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(cassandra): automated image update"
push:
branch: main
update:
strategy: Setters
path: services/cassandra

View File

@ -1,28 +0,0 @@
# clusters/atlas/flux-system/applications/cassandra/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cassandra
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/cassandra
targetNamespace: cassandra
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: cert-manager
- name: core
- name: keycloak
- name: longhorn
- name: traefik
- name: vault
- name: vault-csi
- name: vault-injector
wait: false
timeout: 20m

View File

@ -1,17 +0,0 @@
# clusters/atlas/flux-system/applications/comms/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: comms
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
prune: true
sourceRef:
kind: GitRepository
name: flux-system
path: ./services/comms
targetNamespace: comms
timeout: 2m

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: crypto
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/crypto

View File

@ -1,28 +0,0 @@
# clusters/atlas/flux-system/applications/finance/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: finance
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Finance apps are parked until the next storage and SSO pass."
spec:
interval: 10m
suspend: true
path: ./services/finance
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: finance
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: actual-budget
namespace: finance
- apiVersion: apps/v1
kind: Deployment
name: firefly
namespace: finance
wait: false

View File

@ -1,31 +0,0 @@
# clusters/atlas/flux-system/applications/game-stream/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: game-stream
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Game streaming is optional and resumes only for planned use."
spec:
interval: 10m
suspend: true
path: ./services/game-stream
targetNamespace: game-stream
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: cert-manager
- name: keycloak
- name: traefik
- name: vault
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-wolf
namespace: game-stream
wait: false
timeout: 10m

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: gitea
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/gitea
@ -15,8 +13,4 @@ spec:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: longhorn
- name: vault
- name: postgres
wait: true

View File

@ -1,27 +0,0 @@
# clusters/atlas/flux-system/applications/harbor/image-automation.yaml
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: harbor
namespace: harbor
spec:
suspend: true
interval: 5m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: feature/ci-gitops
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(harbor): apply image updates"
push:
branch: feature/ci-gitops
update:
strategy: Setters
path: ./services/harbor

View File

@ -1,37 +0,0 @@
# clusters/atlas/flux-system/applications/harbor/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: harbor
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/harbor
targetNamespace: harbor
prune: false
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 10m
healthChecks:
- apiVersion: batch/v1
kind: Job
name: harbor-hermes-agent-immutability-ensure-1
namespace: harbor
- apiVersion: batch/v1
kind: Job
name: harbor-hermes-webui-immutability-ensure-1
namespace: harbor
- apiVersion: batch/v1
kind: Job
name: harbor-hermes-chat-router-immutability-ensure-1
namespace: harbor
dependsOn:
- name: core
- name: longhorn
- name: vault
- name: postgres

View File

@ -1,28 +0,0 @@
# clusters/atlas/flux-system/applications/health/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: health
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Health stack is staged pending account and mail flows."
spec:
interval: 10m
suspend: true
path: ./services/health
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: health
dependsOn:
- name: keycloak
- name: postgres
- name: vault
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: wger
namespace: health
wait: false

View File

@ -1,21 +0,0 @@
# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-chat
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-chat
targetNamespace: hermes-chat
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 30m
dependsOn:
- name: core
- name: hermes
- name: longhorn

View File

@ -1,19 +0,0 @@
# clusters/atlas/flux-system/applications/hermes-observer-bindings/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-observer-bindings
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-observer-bindings
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 5m
dependsOn:
- name: hermes
- name: hermes-observer-rbac

View File

@ -1,26 +0,0 @@
# clusters/atlas/flux-system/applications/hermes-scm-broker/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-scm-broker
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-scm-broker
targetNamespace: hermes-scm
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 10m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-scm-broker
namespace: hermes-scm
dependsOn:
- name: vault
- name: gitea
- name: hermes-scm-namespace

View File

@ -1,16 +0,0 @@
# clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-triage-demo
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-triage-demo
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: false
timeout: 5m

View File

@ -1,26 +0,0 @@
# clusters/atlas/flux-system/applications/hermes/image-automation.yaml
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: hermes
namespace: hermes
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: main
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(hermes): promote validated image release"
push:
branch: main
update:
strategy: Setters
path: services/hermes

View File

@ -1,69 +0,0 @@
# clusters/atlas/flux-system/applications/hermes/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: Merge
spec:
interval: 10m
path: ./services/hermes
targetNamespace: hermes
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: false
timeout: 10m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-model-gate
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-switchyard
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-agent
namespace: hermes
# hermes-execution-worker and its mediators are deliberately absent. They run
# at scavenger priority on a best-effort pool whose first start installs the
# provider CLIs, so gating this Kustomization's 10m health window on them
# would stall hermes-chat and hermes-observer-bindings, which dependsOn
# hermes. The pool reports its own health through the worker readiness probe,
# the per-ordinal mediator /ready endpoint, and the coordinator on :9007.
# The node SSH hardener is deliberately absent. It reconciles every node,
# including offline and maintenance hosts, so its availability must not
# turn a node-local account issue into a blocked owner-service rollout.
- apiVersion: apps/v1
kind: Deployment
name: hermes
namespace: hermes
- apiVersion: apps/v1
kind: StatefulSet
name: hermes-chat-tenant
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: hermes-chat-router
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-hermes-chat
namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-hermes-triage
namespace: hermes
dependsOn:
- name: cert-manager
- name: core
- name: keycloak
- name: longhorn
- name: vault
- name: jenkins
- name: hermes-observer-rbac

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata:
name: jellyfin
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Media stack stays paused while auth and storage changes are staged."
spec:
interval: 10m
suspend: true
path: ./services/jellyfin
targetNamespace: jellyfin
prune: true
@ -19,6 +15,5 @@ spec:
namespace: flux-system
dependsOn:
- name: core
- name: openldap
wait: true
timeout: 5m

View File

@ -1,32 +0,0 @@
# clusters/atlas/flux-system/applications/jenkins/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: jenkins
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: Merge
spec:
interval: 10m
suspend: false
path: ./services/jenkins
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: jenkins
dependsOn:
- name: helm
- name: harbor
- name: vault-hermes-jenkins-token-seed
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: jenkins
namespace: jenkins
- apiVersion: v1
kind: Service
name: jenkins
namespace: jenkins
wait: false
timeout: 5m

View File

@ -1,16 +1,19 @@
# clusters/atlas/flux-system/applications/hermes-scm-namespace/kustomization.yaml
# clusters/atlas/flux-system/applications/jitsi/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-scm-namespace
name: jitsi
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-scm-namespace
path: ./services/jitsi
targetNamespace: jitsi
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: core
wait: true
timeout: 5m

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: keycloak
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
prune: true
@ -14,8 +12,4 @@ spec:
name: flux-system
path: ./services/keycloak
targetNamespace: sso
dependsOn:
- name: longhorn
- name: vault
- name: postgres
timeout: 2m

View File

@ -2,48 +2,16 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- zot/kustomization.yaml
- gitea/kustomization.yaml
- vault/kustomization.yaml
- vault-hermes-jenkins-token-seed/kustomization.yaml
- vaultwarden/kustomization.yaml
- comms/kustomization.yaml
- jitsi/kustomization.yaml
- crypto/kustomization.yaml
- monerod/kustomization.yaml
- pegasus/kustomization.yaml
- pegasus/image-automation.yaml
- bstein-dev-home/kustomization.yaml
- bstein-dev-home/image-automation.yaml
- bstein-dev-home-migrations/kustomization.yaml
- harbor/kustomization.yaml
- harbor/image-automation.yaml
- jellyfin/kustomization.yaml
- xmr-miner/kustomization.yaml
- wallet-monero-temp/kustomization.yaml
- sui-metrics/kustomization.yaml
- openldap/kustomization.yaml
- keycloak/kustomization.yaml
- quality/kustomization.yaml
- oauth2-proxy/kustomization.yaml
- mailu/kustomization.yaml
- jenkins/kustomization.yaml
- ai-llm/kustomization.yaml
- openclaw/kustomization.yaml
- hermes-scm-namespace/kustomization.yaml
- hermes/kustomization.yaml
- hermes/image-automation.yaml
- hermes-observer-rbac/kustomization.yaml
- hermes-observer-bindings/kustomization.yaml
- hermes-scm-broker/kustomization.yaml
- hermes-chat/kustomization.yaml
- hermes-triage-demo/kustomization.yaml
- game-stream/kustomization.yaml
- cassandra-auth/kustomization.yaml
- cassandra/kustomization.yaml
- cassandra/image-automation.yaml
- typhon/kustomization.yaml
- nextcloud/kustomization.yaml
- nextcloud-mail-sync/kustomization.yaml
- outline/kustomization.yaml
- planka/kustomization.yaml
- finance/kustomization.yaml
- health/kustomization.yaml

View File

@ -1,22 +0,0 @@
# clusters/atlas/flux-system/applications/mailu/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: mailu
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mail stack is staged and resumes only during mail rollout work."
spec:
interval: 10m
suspend: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
path: ./services/mailu
targetNamespace: mailu-mailserver
prune: true
wait: true
dependsOn:
- name: helm

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: monerod
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/crypto/monerod
@ -18,4 +16,4 @@ spec:
dependsOn:
- name: crypto
wait: true
timeout: 15m
timeout: 5m

View File

@ -1,21 +0,0 @@
# clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: nextcloud-mail-sync
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mail sync resumes after Nextcloud and Mailu are active."
spec:
interval: 10m
suspend: true
prune: true
sourceRef:
kind: GitRepository
name: flux-system
path: ./services/nextcloud-mail-sync
targetNamespace: nextcloud
timeout: 2m
dependsOn:
- name: keycloak

View File

@ -1,20 +0,0 @@
# clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
kind: Kustomization
metadata:
name: nextcloud
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Nextcloud is staged pending storage, SSO, and mail validation."
spec:
interval: 10m
suspend: true
path: ./services/nextcloud
targetNamespace: nextcloud
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: oauth2-proxy
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
prune: true

View File

@ -1,22 +0,0 @@
# clusters/atlas/flux-system/applications/openclaw/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: openclaw
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: Merge
spec:
interval: 10m
path: ./services/openclaw
targetNamespace: openclaw
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 5m
dependsOn:
- name: core
- name: longhorn

View File

@ -1,21 +0,0 @@
# clusters/atlas/flux-system/applications/openldap/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: openldap
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
path: ./services/openldap
targetNamespace: sso
dependsOn:
- name: core
wait: true
timeout: 5m

View File

@ -1,31 +0,0 @@
# clusters/atlas/flux-system/applications/outline/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: outline
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Outline is staged until shared auth and mail are ready."
spec:
interval: 10m
suspend: true
path: ./services/outline
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: outline
dependsOn:
- name: keycloak
- name: mailu
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: outline
namespace: outline
- apiVersion: v1
kind: Service
name: outline
namespace: outline
wait: false

View File

@ -1,26 +1,20 @@
# clusters/atlas/flux-system/applications/pegasus/image-automation.yaml
apiVersion: image.toolkit.fluxcd.io/v1
apiVersion: image.toolkit.fluxcd.io/v1beta1
kind: ImageUpdateAutomation
metadata:
name: pegasus
namespace: jellyfin
namespace: flux-system
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: feature/ci-gitops
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(pegasus): apply image updates"
push:
branch: feature/ci-gitops
messageTemplate: "chore(pegasus): update image to {{range .Updated.Images}}{{.}}{{end}}"
update:
strategy: Setters
path: services/pegasus
path: ./services/pegasus

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: pegasus
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/pegasus

View File

@ -1,31 +0,0 @@
# clusters/atlas/flux-system/applications/planka/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: planka
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Planka is staged until shared auth and mail are ready."
spec:
interval: 10m
suspend: true
path: ./services/planka
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: planka
dependsOn:
- name: keycloak
- name: mailu
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: planka
namespace: planka
- apiVersion: v1
kind: Service
name: planka
namespace: planka
wait: false

View File

@ -1,38 +0,0 @@
# clusters/atlas/flux-system/applications/quality/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: quality
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Quality stack changes resume only during quality-gate rollout work."
spec:
interval: 10m
suspend: true
path: ./services/quality
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: quality
dependsOn:
- name: cert-manager
- name: keycloak
- name: vault
- name: postgres
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: sonarqube
namespace: quality
- apiVersion: apps/v1
kind: Deployment
name: sonarqube-exporter
namespace: quality
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-sonarqube
namespace: quality
wait: false
timeout: 20m

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: sui-metrics
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/sui-metrics/overlays/atlas

View File

@ -1,33 +0,0 @@
# clusters/atlas/flux-system/applications/typhon/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: typhon
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Climate automation is staged until sensor and control loops are verified."
spec:
interval: 10m
suspend: true
path: ./services/typhon
prune: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: climate
dependsOn:
- name: vault
- name: vault-csi
- name: monitoring
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: typhon
namespace: climate
- apiVersion: v1
kind: Service
name: typhon
namespace: climate
wait: false
timeout: 20m

View File

@ -1,26 +0,0 @@
# clusters/atlas/flux-system/applications/vault-hermes-jenkins-token-seed/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: vault-hermes-jenkins-token-seed
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
path: ./services/vault-hermes-jenkins-token-seed
targetNamespace: vault
prune: true
wait: true
timeout: 5m
dependsOn:
- name: vault
healthChecks:
- apiVersion: batch/v1
kind: Job
name: vault-hermes-jenkins-build-token-seed-2
namespace: vault

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: vault
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
sourceRef:
@ -16,11 +14,5 @@ spec:
targetNamespace: vault
prune: true
wait: true
healthChecks:
- apiVersion: batch/v1
kind: Job
name: vault-k8s-auth-hermes-11
namespace: vault
dependsOn:
- name: longhorn
- name: helm

View File

@ -1,22 +0,0 @@
# clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: vaultwarden
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Vaultwarden stays separate from SSO rollout and resumes by hand."
spec:
interval: 10m
suspend: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
path: ./services/vaultwarden
targetNamespace: vaultwarden
prune: true
wait: true
dependsOn:
- name: helm

View File

@ -1,29 +0,0 @@
# clusters/atlas/flux-system/applications/veles/image-automation.yaml
# Staged for the first Veles image rollout. Add this file to the parent
# applications kustomization after the namespace exists and the Harbor repos
# have initial tags.
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: veles
namespace: veles
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: main
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(veles): automated image update"
push:
branch: main
update:
strategy: Setters
path: services/veles

View File

@ -1,28 +0,0 @@
# clusters/atlas/flux-system/applications/veles/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: veles
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/veles
targetNamespace: veles
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: cert-manager
- name: core
- name: keycloak
- name: longhorn
- name: traefik
- name: vault
- name: vault-csi
- name: vault-injector
wait: false
timeout: 20m

View File

@ -1,23 +0,0 @@
# clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: wallet-monero-temp
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Temporary wallet RPC stack is opt-in for maintenance windows."
spec:
interval: 10m
suspend: true
path: ./services/crypto/wallet-monero-temp
targetNamespace: crypto
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: crypto
- name: xmr-miner
wait: true

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata:
name: xmr-miner
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mining workloads are disabled unless explicitly enabled for a short run."
spec:
interval: 10m
suspend: true
path: ./services/crypto/xmr-miner
targetNamespace: crypto
prune: true
@ -21,4 +17,3 @@ spec:
- name: crypto
- name: monerod
wait: true
timeout: 30m

View File

@ -1,16 +1,18 @@
# clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml
# clusters/atlas/flux-system/applications/zot/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-observer-rbac
name: zot
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-observer-rbac
prune: true
path: ./services/zot
targetNamespace: zot
prune: false
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 5m
dependsOn:
- name: core

File diff suppressed because it is too large Load Diff

View File

@ -1,4 +1,3 @@
# clusters/atlas/flux-system/gotk-sync.yaml
# This manifest was generated by flux. DO NOT EDIT.
---
apiVersion: source.toolkit.fluxcd.io/v1
@ -7,12 +6,12 @@ metadata:
name: flux-system
namespace: flux-system
spec:
interval: 15m0s
interval: 1m0s
ref:
branch: main
branch: feature/sso
secretRef:
name: flux-system-gitea
url: ssh://git@scm.bstein.dev:2242/titan/atlas-iac.git
url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
@ -20,7 +19,7 @@ metadata:
name: flux-system
namespace: flux-system
spec:
interval: 1h0m0s
interval: 10m0s
path: ./clusters/atlas/flux-system
prune: true
sourceRef:

View File

@ -1,19 +0,0 @@
# clusters/atlas/flux-system/platform/cert-manager-cleanup/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cert-manager-cleanup
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 30m
path: ./infrastructure/cert-manager/cleanup
prune: true
force: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
targetNamespace: cert-manager
wait: true

View File

@ -1,21 +0,0 @@
# clusters/atlas/flux-system/platform/cert-manager/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cert-manager
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 30m
path: ./infrastructure/cert-manager
prune: true
force: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
targetNamespace: cert-manager
dependsOn:
- name: helm
wait: true

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: core
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./infrastructure/core

View File

@ -1,23 +0,0 @@
# clusters/atlas/flux-system/platform/descheduler/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: descheduler
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Descheduler is paused during node recovery and placement stabilization."
spec:
interval: 30m
suspend: true
path: ./infrastructure/descheduler
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
targetNamespace: kube-system
dependsOn:
- name: helm
- name: core
wait: true

View File

@ -1,23 +0,0 @@
# clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: gitops-ui
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "GitOps UI is optional and resumes only for operator access work."
spec:
interval: 10m
suspend: true
timeout: 10m
path: ./services/gitops-ui
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
targetNamespace: flux-system
dependsOn:
- name: helm
wait: true

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: helm
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 30m
sourceRef:

View File

@ -4,19 +4,6 @@ kind: Kustomization
resources:
- core/kustomization.yaml
- helm/kustomization.yaml
- descheduler/kustomization.yaml
- resource-guardrails/kustomization.yaml
- cert-manager/kustomization.yaml
- metallb/kustomization.yaml
- traefik/kustomization.yaml
- gitops-ui/kustomization.yaml
- monitoring/kustomization.yaml
- logging/kustomization.yaml
- maintenance/kustomization.yaml
- maintenance/image-automation.yaml
- longhorn-adopt/kustomization.yaml
- longhorn/kustomization.yaml
- longhorn-ui/kustomization.yaml
- postgres/kustomization.yaml
- ../platform/vault-csi/kustomization.yaml
- ../platform/vault-injector/kustomization.yaml

View File

@ -1,16 +0,0 @@
# clusters/atlas/flux-system/platform/logging/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: logging
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/logging
prune: true
sourceRef:
kind: GitRepository
name: flux-system
wait: false

View File

@ -1,19 +0,0 @@
# clusters/atlas/flux-system/platform/longhorn-adopt/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: longhorn-adopt
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 30m
path: ./infrastructure/longhorn/adopt
prune: true
force: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
targetNamespace: longhorn-system
wait: true

View File

@ -4,12 +4,8 @@ kind: Kustomization
metadata:
name: longhorn-ui
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Longhorn UI ingress is optional and opened only for storage work."
spec:
interval: 10m
suspend: true
path: ./infrastructure/longhorn/ui-ingress
targetNamespace: longhorn-system
prune: true
@ -19,5 +15,4 @@ spec:
namespace: flux-system
dependsOn:
- name: core
- name: longhorn
wait: true

View File

@ -1,22 +0,0 @@
# clusters/atlas/flux-system/platform/longhorn/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: longhorn
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 30m
path: ./infrastructure/longhorn/core
prune: true
force: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
targetNamespace: longhorn-system
dependsOn:
- name: helm
- name: longhorn-adopt
wait: false

View File

@ -1,26 +0,0 @@
# clusters/atlas/flux-system/platform/maintenance/image-automation.yaml
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: maintenance
namespace: maintenance
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: main
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(maintenance): automated image update"
push:
branch: main
update:
strategy: Setters
path: services/maintenance

View File

@ -1,17 +0,0 @@
# clusters/atlas/flux-system/platform/maintenance/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: maintenance
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/maintenance
prune: true
force: true
sourceRef:
kind: GitRepository
name: flux-system
wait: false

View File

@ -1,18 +0,0 @@
# clusters/atlas/flux-system/platform/metallb/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: metallb
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 30m
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
path: ./infrastructure/metallb
prune: true
wait: true
targetNamespace: metallb-system

View File

@ -4,8 +4,6 @@ kind: Kustomization
metadata:
name: monitoring
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/monitoring

View File

@ -1,27 +0,0 @@
# clusters/atlas/flux-system/platform/postgres/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: postgres
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./infrastructure/postgres
prune: true
force: true
sourceRef:
kind: GitRepository
name: flux-system
targetNamespace: postgres
dependsOn:
- name: longhorn
- name: vault
- name: vault-csi
healthChecks:
- apiVersion: apps/v1
kind: StatefulSet
name: postgres
namespace: postgres
wait: true

View File

@ -1,21 +0,0 @@
# clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: resource-guardrails
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Guardrail rollout is paused until service resource requests are normalized."
spec:
interval: 10m
suspend: true
path: ./infrastructure/resource-guardrails
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: core
wait: true

Some files were not shown because too many files have changed in this diff Show More